diff --git a/.github/ISSUE_TEMPLATE/bug-report.md b/.github/ISSUE_TEMPLATE/bug-report.md index 521d59f3d3..a24b00cdc9 100644 --- a/.github/ISSUE_TEMPLATE/bug-report.md +++ b/.github/ISSUE_TEMPLATE/bug-report.md @@ -1,9 +1,9 @@ --- name: Bug Report about: Report a bug encountered while operating bk-job -title: 'bugfix: ' -labels: kind/bug -assignees: 'nekzhang' +title: 'fix: ' +labels: kind/bug,backlog +assignees: '' --- diff --git a/.github/ISSUE_TEMPLATE/enhancement.md b/.github/ISSUE_TEMPLATE/enhancement.md index 9a33852a12..33bef9f3b7 100644 --- a/.github/ISSUE_TEMPLATE/enhancement.md +++ b/.github/ISSUE_TEMPLATE/enhancement.md @@ -2,7 +2,7 @@ name: Enhancement Request about: Suggest an enhancement to the bk-job project title: '' -labels: kind/enhancement +labels: kind/enhancement,backlog assignees: '' --- diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index ceb48806cb..56f4934e56 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -36,7 +36,7 @@ jobs: with: java-version: 1.8 - name: Cache Gradle - uses: actions/cache@v1 + uses: actions/cache@v4 with: path: ~/.gradle/caches key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }} @@ -44,32 +44,5 @@ jobs: ${{ runner.os }}-gradle- - name: Gradle Build working-directory: src/backend - run: ./gradlew clean build -DmysqlURL=127.0.0.1:3306 -DmysqlUser=root -DmysqlPasswd=root -DmavenRepoUrl="https://maven.aliyun.com/nexus/content/groups/public/" - buildEE: - # The type of runner that the job will run on - runs-on: ubuntu-latest - - # Steps represent a sequence of tasks that will be executed as part of the job - steps: - # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v2 - - name: init mysql - run: | - sudo /etc/init.d/mysql start - export MYSQL_PWD=root - for i in */*.sql;do echo $i;mysql -h localhost -uroot < $i;done - working-directory: support-files/sql - - name: Set up JDK 1.8 - uses: actions/setup-java@v1 - with: - java-version: 1.8 - - name: Cache Gradle - uses: actions/cache@v1 - with: - path: ~/.gradle/caches - key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }} - restore-keys: | - ${{ runner.os }}-gradle- - - name: Gradle Build - working-directory: src/backend - run: ./gradlew clean build -DmysqlURL=127.0.0.1:3306 -DmysqlUser=root -DmysqlPasswd=root -DmavenRepoUrl="https://maven.aliyun.com/nexus/content/groups/public/" -Pjob.edition=ee -Pjob.package.type=release + # run: ./gradlew clean build -DmysqlURL=127.0.0.1:3306 -DmysqlUser=root -DmysqlPasswd=root -DmavenRepoUrl="https://maven.aliyun.com/nexus/content/groups/public/" + run: ./gradlew clean build -DmysqlURL=127.0.0.1:3306 -DmysqlUser=root -DmysqlPasswd=root -DmavenRepoUrl="https://mirrors.cloud.tencent.com/nexus/repository/maven-public/" --info --stacktrace diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ae97f1022f..16b4912a21 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -28,7 +28,7 @@ jobs: with: java-version: 1.8 - name: Cache Gradle - uses: actions/cache@v1 + uses: actions/cache@v4 with: path: ~/.gradle/caches key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }} diff --git a/.gitignore b/.gitignore index 72e96c54e4..6e836bdc4d 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,6 @@ support-files/dependJarLists/ **/*.DS_Store .codecc build.yml + + +pre-*-bkcodeai diff --git a/CONTRIBUTING.en.md b/CONTRIBUTING.en.md index 376f3c91fb..88320bc139 100644 --- a/CONTRIBUTING.en.md +++ b/CONTRIBUTING.en.md @@ -46,17 +46,15 @@ git commit -m 'fix: the bug xxx #123' | Mark | Description | | -------- | -------------------------------------- | -| feature/feat | Develop a new feature | -| bug/fix/bugfix | Fix a bug | -| refactor/perf | Refactor the code/Optimize configurations & parameters/Optimize the logic and the functionality | -| test | Add unit testing cases | +| feat | Develop a new feature | +| fix | Fix a bug | | docs | Add documents | -| info | Add comment information | -| format | On the premise of not modifying business logic, only format the code | -| merge | Only merge and synchronize branches | -| depend | Add, delete or modify the dependencies of the project | +| style | On the premise of not modifying business logic, only format the code | +| refactor | Refactor the code | +| perf | Optimize configurations & parameters/Optimize the logic and the functionality | +| test | Add unit testing cases | | chore | Relevant code like build scripts and tasks | -| del | Destructive actions like deleting functionalities and APIs that are still in use | +| merge | Only merge and synchronize branches | ## Pull Request/Merge Request diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7dbb388b03..fd933abc4a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -47,17 +47,15 @@ git commit -m 'fix: xxx问题 #123' | 标记 | 说明 | | -------- | -------------------------------------- | -| feature/feat | 新功能开发 | -| bug/fix/bugfix | bug修复 | -| refactor/perf | 重构代码/优化配置&参数/优化逻辑及功能 | -| test | 添加单元测试用例相关 | +| feat | 新功能开发 | +| fix | bug修复 | | docs | 添加文档 | -| info | 添加注释类信息 | -| format | 不修改业务逻辑下，仅做代码规范的格式化 | -| merge | 仅做分支合并同步 | -| depend | 对工程的依赖进行增删改 | +| style | 不修改业务逻辑下，仅做代码规范的格式化 | +| refactor | 重构代码 | +| perf | 优化配置&参数/优化逻辑及功能 | +| test | 添加单元测试用例相关 | | chore | 构建脚本、任务等相关代码 | -| del | 删除可能仍然有人用到的功能、API等破坏性动作 | +| merge | 仅做分支合并同步 | ## Pull Request/Merge Request diff --git a/LICENSE.txt b/LICENSE.txt index 67e17e092c..a2d55b79a8 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,6 +1,6 @@ Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. -Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +Copyright (C) 2021 Tencent. All rights reserved. BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. diff --git a/README.en.md b/README.en.md index f846518703..d7917eb343 100644 --- a/README.en.md +++ b/README.en.md @@ -59,7 +59,7 @@ For more information, please check Blueking's official website [Platform Product ## Getting started - [Download and Compile](docs/overview/source_compile.en.md) -- [Installation Setup](docs/overview/installation.en.md) +- [Install and Operation](docs/overview/operation_en.md) ## Support 1. [GitHub Community](https://github.com/Tencent/bk-job/discussions) diff --git a/README.md b/README.md index 47d91480ca..de85510336 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ bk-job 提供了快速执行、任务编排、定时执行等核心服务，多 ## Getting started - [下载与编译](docs/overview/source_compile.md) -- [安装部署](docs/overview/installation.md) +- [部署与运维](docs/overview/operation.md) ## Support 1. [GitHub讨论区](https://github.com/Tencent/bk-job/discussions) diff --git a/UPGRADE.md b/UPGRADE.md index 1ec08e9563..6ce290d89c 100644 --- a/UPGRADE.md +++ b/UPGRADE.md @@ -1,21 +1,23 @@ -## 升级说明 +# 升级说明 [English](UPGRADE.en.md) | 简体中文 -### 版本3.3.4.x -该版本增加一个升级工具（upgrader-3.3.4.x.jar，与其他二进制包在同一目录），需要在更新Job二进制Jar文件重启进程（所有微服务进程）前后用不同参数分别执行一次，工具总共执行两次，用于迁移权限数据和账号加密数据。 +## 一、升级工具介绍与用法 +### 1.介绍 +从3.3.4版本开始，包内开始提供一个升级工具（upgrader-${version}.jar，与其他二进制包在同一目录），用于在大版本之间升级时（例如：3.2.x->3.3.x, 3.3.x->3.4.x等）对不兼容的数据进行迁移，每次进行大版本升级时，**必须查阅此文档**，查看是否有需要执行的升级任务，若有则需要按照说明执行，否则可能出现数据异常、权限丢失等问题。 -**适用范围** -适用于从3.x.x.x向3.3.4.x及以上版本升级，3.3.4.x版本内升级（如3.3.4.0-->3.3.4.3）无需执行。 +说明：同一大版本的多个rc/release版本之间升级（例如：3.7.3->3.7.4-rc.8）无需执行。 + +### 2.用法 +#### （1）二进制环境 -**工具用法** 使用命令`java -Dfile.encoding=utf8 -Djob.log.dir=path/to/log/dir -Dconfig.file=/path/to/config/file -jar upgrader-[x.x.x.x].jar [fromVersion] [toVersion] [executeTime]` 启动工具，再根据命令行提示输入与具体版本升级任务相关的特定参数，运行升级工具。 -/path/to/log/dir用于指定工具日志保存路径，通常为${BK_HOME}/logs/job -/path/to/config/file用于指定工具读取的配置文件，该配置文件由部署脚本自动渲染生成，路径为${BK_HOME}/etc/job/upgrader/upgrader.properties -fromVersion为当前作业平台版本，如3.2.7.3 -toVersion为目标作业平台版本，如3.3.4.0 -executeTime为升级任务执行的时间点，取值为BEFORE_UPDATE_JOB、AFTER_UPDATE_JOB，在更新作业平台进程前执行本工具填写BEFORE_UPDATE_JOB，更新进程后执行则填写AFTER_UPDATE_JOB +**/path/to/log/dir** 用于指定工具日志保存路径，通常为${BK_HOME}/logs/job +**/path/to/config/file** 用于指定工具读取的配置文件，该配置文件由部署脚本自动渲染生成，路径为${BK_HOME}/etc/job/upgrader/upgrader.properties +**fromVersion** 为升级前的作业平台版本，如3.2.7.3 +**toVersion** 为升级后的作业平台版本，如3.3.4.0 +**executeTime** 为升级任务执行的时间点，取值为BEFORE_UPDATE_JOB、AFTER_UPDATE_JOB，在更新作业平台进程前执行本工具填写BEFORE_UPDATE_JOB，更新进程后执行则填写AFTER_UPDATE_JOB 示例： Job进程更新前： @@ -27,8 +29,28 @@ Job进程更新后： /opt/java/bin/java -Dfile.encoding=utf8 -Djob.log.dir=/data/bkee/logs/job -Dconfig.file=/data/bkee/etc/job/upgrader/upgrader.properties -jar upgrader-3.3.4.0.jar 3.2.7.3 3.3.4.0 AFTER_UPDATE_JOB ``` -### 版本3.5.x.x -该版本对接了CMDB业务集，需要对Job原有业务集数据进行迁移，除了执行以上升级工具外，还需要额外执行以下步骤： +#### （2）容器化环境 +1. 从对应版本的二进制包内获取upgrader二进制jar包与配置模板文件； +2. 选定一台机器（可选中控机），准备Java运行环境（JDK 1.8），作为运行迁移工具的机器； +3. 查看配置模板文件，从部署Job的K8s集群中将需要访问的服务（job-manage/job-crontab等）端口使用port-forward等方式映射到有固定IP的机器（能操作K8s集群的机器，可以为中控机）端口（可自行分配）上； +4. 根据配置模板文件与环境变量对应的实际数据填写配置文件（其中job服务的机器与端口填写上一步骤中映射好的固定机器IP与端口）； +5. 准备日志目录，按照二进制环境的命令运行迁移工具； +6. 确认迁移成功后，清理迁移过程中临时使用的机器端口映射，日志文件与Java环境可按需保留。 + + +## 二、各大版本间升级需要执行的升级任务 + +### 3.0.x/3.1.x/3.2.x -> 3.3.x +需要在更新Job二进制Jar文件重启进程（所有微服务进程）前后用不同参数分别执行一次升级工具，工具总共执行两次，用于迁移权限数据和账号加密数据。 + +### 3.3.x -> 3.4.x +需要在更新Job后执行一次升级工具，用于迁移作业模板/脚本引用的标签数据。 + +### 3.4.x -> 3.5.x +1.在更新Job后执行一次升级工具，用于迁移业务集数据； + +2.除了执行以上升级工具外，还需要额外执行以下步骤： + （1）获取安装包内根目录下support-files/bk-cmdb/changeBizSetId.js脚本文件； （2）获取上一步中upgrader执行完成后在upgrader同一路径下生成的biz_set_list.json文件，使用其中的数据替换changeBizSetId.js脚本文件中占位符${biz_set_list}； （3）将替换后的脚本文件移动到一台具有MongoDB命令行且有权限访问CMDB的MongoDB数据库的机器上，执行以下命令，完成CMDB中的业务集ID更改： @@ -49,3 +71,15 @@ echo $BK_CMDB_MONGODB_补全。 /opt/java/bin/java -Dfile.encoding=utf8 -Djob.log.dir=/data/bkee/logs/job -Dconfig.file=/data/bkee/etc/job/upgrader/upgrader.properties -Dtarget.tasks=BizSetMigrationStatusUpdateTask -jar upgrader-3.5.0.0.jar 3.4.5.0 3.5.0.0 MAKE_UP true ``` 注：命令中涉及到的版本号请使用真实值，示例代码中仅为参考。 + +### 3.5.x -> 3.6.x +无需执行升级工具。 + +### 3.6.x -> 3.7.x +需要在更新Job后执行一次升级工具，用于为存量的作业模板/执行方案/定时任务/IP白名单中的主机数据添加hostId，以及迁移对所有业务生效的IP白名单数据。 + +### 3.7.x -> 3.8.x +无需执行升级工具。 + +### 3.8.x -> 3.9.x +需要在更新Job后执行一次升级工具，用于导出已配置的平台信息数据（title/footer/助手链接等），从3.9.3版本开始，作业平台改为使用基于BK-REPO的全局配置统一方案，升级任务运行后会输出平台信息全局配置base.js文件，以便于迁移之前用户已通过页面【平台管理-全局设置-平台信息】配置的数据：将该任务生成的base.js文件导入BK-REPO中（若已存在则覆盖），路径取值与chart values的bkSharedResUrl、bkSharedBaseJsPath配置项保持一致，默认为：${当前环境部署的BK-REPO根地址}/generic/blueking/bk-config/bk_job/base.js。 diff --git a/docs/apidoc/README.md b/docs/apidoc/README.md new file mode 100644 index 0000000000..6648669f1d --- /dev/null +++ b/docs/apidoc/README.md @@ -0,0 +1,67 @@ +## 作业平台 V3 API 简介 + +**作业平台（Job）是一套运维基础操作管理系统，除了支持脚本执行、文件分发、定时任务等一系列基础运维场景以外，还支持通过流程调度能力将零碎的单个任务组装成一个自动化作业流程；而每个作业都可做为一个原子节点，提供给上层或周边系统/平台使用，实现调度自动化。作业平台通过API的方式向其他系统提供原子能力，当前支持的API资源汇总如下：** + + +| 资源名称 | 资源描述 | +| -------- | ------- | +| [batch_get_job_instance_ip_log](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/batch_get_job_instance_ip_log.md) | 根据主机列表批量查询作业执行日志 | +| [callback_protocol](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/callback_protocol.md) | 此组件用于展示回调协议文档 | +| [check_script](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/check_script.md) | 高危脚本检测 | +| [create_account](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/create_account.md) | 创建账号（当前仅支持系统账号） | +| [create_credential](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/create_credential.md) | 新建凭证 | +| [create_dangerous_rule](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/create_dangerous_rule.md) | 新建高危语句检测规则 | +| [create_file_source](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/create_file_source.md) | 新建文件源 | +| [create_public_script](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/create_public_script.md) | 新建公共脚本 | +| [create_public_script_version](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/create_public_script_version.md) | 新建公共脚本版本 | +| [create_script](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/create_script.md) | 新建脚本 | +| [create_script_version](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/create_script_version.md) | 新建脚本版本 | +| [delete_account](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/delete_account.md) | 删除账号 | +| [delete_cron](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/delete_cron.md) | 删除定时任务 | +| [delete_dangerous_rule](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/delete_dangerous_rule.md) | 删除高危语句检测规则 | +| [delete_public_script](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/delete_public_script.md) | 删除公共脚本，同时会删除该脚本下的所有的脚本版本 | +| [delete_public_script_version](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/delete_public_script_version.md) | 删除公共脚本版本 | +| [delete_script](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/delete_script.md) | 删除脚本，同时会删除该脚本下的所有的脚本版本 | +| [delete_script_version](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/delete_script_version.md) | 删除脚本版本 | +| [disable_dangerous_rule](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/disable_dangerous_rule.md) | 停用高危语句检测规则 | +| [disable_public_script_version](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/disable_public_script_version.md) | 禁用公共脚本版本 | +| [disable_script_version](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/disable_script_version.md) | 禁用脚本版本 | +| [enable_dangerous_rule](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/enable_dangerous_rule.md) | 启用高危语句检测规则 | +| [execute_job_plan](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/execute_job_plan.md) | 启动作业执行方案 | +| [fast_execute_script](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/fast_execute_script.md) | 快速执行脚本 | +| [fast_execute_sql](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/fast_execute_sql.md) | 快速执行SQL脚本 | +| [fast_transfer_file](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/fast_transfer_file.md) | 快速分发文件 | +| [generate_local_file_upload_url](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/generate_local_file_upload_url.md) | 生成本地文件上传URL | +| [get_account_list](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_account_list.md) | 查询业务下用户有权限的执行账号列表 | +| [get_cron_detail](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_cron_detail.md) | 查询定时作业详情 | +| [get_cron_list](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_cron_list.md) | 查询业务下定时作业信息 | +| [get_dangerous_rule_list](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_dangerous_rule_list.md) | 查看高危语句检测规则列表 | +| [get_job_instance_global_var_value](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_job_instance_global_var_value.md) | 获取作业实例全局变量的值 | +| [get_job_instance_ip_log](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_job_instance_ip_log.md) | 根据主机查询作业执行日志 | +| [get_job_instance_list](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_job_instance_list.md) | 查询作业实例列表（执行历史) | +| [get_job_instance_status](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_job_instance_status.md) | 根据作业实例 ID 查询作业执行状态 | +| [get_job_plan_detail](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_job_plan_detail.md) | 根据作业执行方案 ID 查询作业执行方案详情 | +| [get_job_plan_list](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_job_plan_list.md) | 查询执行方案列表 | +| [get_job_template_list](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_job_template_list.md) | 查询作业模版列表 | +| [get_latest_service_version](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_latest_service_version.md) | 查询作业平台最新的服务版本号 | +| [get_public_script_list](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_public_script_list.md) | 查询公共脚本列表 | +| [get_public_script_version_detail](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_public_script_version_detail.md) | 查询公共脚本版本详情 | +| [get_public_script_version_list](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_public_script_version_list.md) | 查询公共脚本版本列表 | +| [get_script_list](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_script_list.md) | 查询业务脚本列表 | +| [get_script_version_detail](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_script_version_detail.md) | 查询业务脚本版本详情 | +| [get_script_version_list](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_script_version_list.md) | 查询业务脚本版本列表 | +| [operate_job_instance](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/operate_job_instance.md) | 用于对执行的作业实例进行操作，例如终止作业 | +| [operate_step_instance](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/operate_step_instance.md) | 用于对执行的实例的步骤进行操作，例如重试，忽略错误等 | +| [publish_public_script_version](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/publish_public_script_version.md) | 上线公共脚本版本 | +| [publish_script_version](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/publish_script_version.md) | 上线脚本版本 | +| [push_config_file](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/push_config_file.md) | 分发配置文件，此接口用于分发配置文件等小的纯文本文件 | +| [save_cron](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/save_cron.md) | 新建或保存定时任务 | +| [update_credential](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/update_credential.md) | 更新凭证 | +| [update_cron_status](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/update_cron_status.md) | 更新定时作业状态，如启动或暂停 | +| [update_dangerous_rule](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/update_dangerous_rule.md) | 修改高危语句检测规则 | +| [update_file_source](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/update_file_source.md) | 更新文件源 | +| [update_public_script_basic](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/update_public_script_basic.md) | 更新公共脚本基础信息 | +| [update_public_script_version](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/update_public_script_version.md) | 修改公共脚本版本 | +| [update_script_basic](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/update_script_basic.md) | 更新脚本基础信息 | +| [update_script_version](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/update_script_version.md) | 修改脚本版本信息 | + diff --git a/docs/apidoc/README_en.md b/docs/apidoc/README_en.md new file mode 100644 index 0000000000..43bce80e2c --- /dev/null +++ b/docs/apidoc/README_en.md @@ -0,0 +1,65 @@ +## Job V3 API Introduction + +**The Job Platform (Job) is a fundamental operations management system. In addition to supporting a series of basic operations scenarios such as script execution, file distribution, and scheduled tasks, it also supports the ability to assemble fragmented individual tasks into an automated job process through workflow scheduling capabilities. Each job can serve as an atomic node, available for use by upper-layer or surrounding systems/platforms to achieve scheduling automation. The Job Platform provides atomic capabilities to other systems via APIs. The currently supported API resources are summarized as follows:** + + +| Resource Name | Resource Description | +| -------- |-------------------------------------------------------------------------------------------------------| +| [batch_get_job_instance_ip_log](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/batch_get_job_instance_ip_log.md) | Bulk query of job execution logs by host list | +| [callback_protocol](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/callback_protocol.md) | This component is only used to present the callback protocol when job finished | +| [check_script](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/check_script.md) | High risk script detection | +| [create_credential](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/create_credential.md) | Create credential | +| [create_dangerous_rule](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/create_dangerous_rule.md) | Create dangerous rule | +| [create_file_source](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/create_file_source.md) | Create a new file source | +| [create_public_script](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/create_public_script.md) | Create public script | +| [create_public_script_version](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/create_public_script_version.md) | Create Public Script Version | +| [create_script](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/create_script.md) | Create Script | +| [create_script_version](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/create_script_version.md) | Create Script Version | +| [delete_cron](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/delete_cron.md) | Delete cron job | +| [delete_dangerous_rule](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/delete_dangerous_rule.md) | Delete dangerous rule | +| [delete_public_script](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/delete_public_script.md) | Delete public script | +| [delete_public_script_version](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/delete_public_script_version.md) | Delete script version | +| [delete_script](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/delete_script.md) | Delete script | +| [delete_script_version](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/delete_script_version.md) | Delete script version | +| [disable_dangerous_rule](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/disable_dangerous_rule.md) | Disable dangerous rule | +| [disable_public_script_version](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/disable_public_script_version.md) | Disable the public script version | +| [disable_script_version](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/disable_script_version.md) | Disable script version | +| [enable_dangerous_rule](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/enable_dangerous_rule.md) | Enable dangerous rule | +| [execute_job_plan](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/execute_job_plan.md) | Start Job Plan | +| [fast_execute_script](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/fast_execute_script.md) | Start Job Plan | +| [fast_execute_sql](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/fast_execute_sql.md) | Quick execution of SQL scripts | +| [fast_transfer_file](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/fast_transfer_file.md) | Quick File transfer | +| [generate_local_file_upload_url](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/generate_local_file_upload_url.md) | Generate the local file upload URL | +| [get_account_list](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_account_list.md) | Query the list of executive accounts to which users have access under the business | +| [get_cron_detail](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_cron_detail.md) | Query Cron job details | +| [get_cron_list](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_cron_list.md) | Query information of cron jobs under business | +| [get_dangerous_rule_list](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_dangerous_rule_list.md) | Get dangerous rule list | +| [get_job_instance_global_var_value](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_job_instance_global_var_value.md) | Gets the value of the job instance global variable | +| [get_job_instance_ip_log](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_job_instance_ip_log.md) | Query job execution log by host | +| [get_job_instance_list](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_job_instance_list.md) | Query job instance list (execution history) | +| [get_job_instance_status](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_job_instance_status.md) | Query job execution status based on job instance ID | +| [get_job_plan_detail](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_job_plan_detail.md) | Check Job Plan details by Job Plan ID | +| [get_job_plan_list](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_job_plan_list.md) | Search Job Plan list | +| [get_job_template_list](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_job_template_list.md) | Query job template list | +| [get_latest_service_version](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_latest_service_version.md) | Query the latest service version | +| [get_public_script_list](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_public_script_list.md) | Query public script list | +| [get_public_script_version_detail](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_public_script_version_detail.md) | Query public script version details | +| [get_public_script_version_list](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_public_script_version_list.md) | Query public script version list | +| [get_script_list](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_script_list.md) | Query business script list | +| [get_script_version_detail](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_script_version_detail.md) | Query business script version details | +| [get_script_version_list](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_script_version_list.md) | Query business script version list | +| [operate_job_instance](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/operate_job_instance.md) | Use to perform actions on an executed job instance, such as terminating a job | +| [operate_step_instance](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/operate_step_instance.md) | Used to perform actions on the steps of an executed instance, such as retry, ignore errors, and so on | +| [publish_public_script_version](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/publish_public_script_version.md) | Set public script version status to online | +| [publish_script_version](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/publish_script_version.md) | Set script version status to online | +| [push_config_file](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/push_config_file.md) | Distribute configuration files | +| [save_cron](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/save_cron.md) | Create or update Cron Job | +| [update_credential](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/update_credential.md) | Update credentials | +| [update_cron_status](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/update_cron_status.md) | Update Cron job status | +| [update_dangerous_rule](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/update_dangerous_rule.md) | update dangerous rule | +| [update_file_source](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/update_file_source.md) | Update the file source | +| [update_public_script_basic](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/update_public_script_basic.md) | Update basic information of public scripts | +| [update_public_script_version](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/update_public_script_version.md) | Update public script version | +| [update_script_basic](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/update_script_basic.md) | Update script basic information | +| [update_script_version](https://github.com/TencentBlueKing/bk-job/blob/3.9.x/docs/apidoc/esb/jobv3-confapis/apidocs/en/update_script_version.md) | Update Script Version Information | + diff --git a/docs/apidoc/bk-api-gateway/v3/zh/batch_get_job_instance_ip_log.md b/docs/apidoc/bk-api-gateway/v3/zh/batch_get_job_instance_ip_log.md new file mode 100644 index 0000000000..440ecd3bc0 --- /dev/null +++ b/docs/apidoc/bk-api-gateway/v3/zh/batch_get_job_instance_ip_log.md @@ -0,0 +1,201 @@ +### 功能描述 + +根据主机列表批量查询作业执行日志 + +### 请求参数 + +{{ bkapi_authorization_description }} + +#### Header参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| X-Bkapi-Authorization | string | 是 | 认证信息。详情参考[调用网关 API](https://github.com/TencentBlueKing/BKDocs/blob/master/ZH/7.0/APIGateway/apigateway/use-api/use-apigw-api.md) | +| Accept | string | 是 | 固定值。application/json| +| Content-Type | string | 是 | 固定值。application/json| + +#### Body参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| bk_biz_id | long | 是 | *已弃用*。业务ID。此字段已被弃用并由字段bk_scope_type+bk_scope_id替换 | +| job_instance_id | long | 是 | 作业实例ID | +| step_instance_id | long | 是 | 步骤实例ID | +| host_id_list | array | 否 | 主机ID列表 | +| ip_list | array | 否 | ***不推荐使用，建议使用host_id_list参数***;如果host_id_list与ip_list同时存在，将忽略ip_list参数。主机IP 列表，定义见ip | + +##### ip + +| 字段 | 类型 | 必选 | 描述 | +| ----------- | ------ | ---- | -------- | +| bk_cloud_id | int | 是 | 管控区域ID | +| ip | string | 是 | IP地址 | + +### 请求参数示例 +- POST +```json +{ + "bk_scope_type": "biz", + "bk_scope_id": "1", + "job_instance_id": 100, + "step_instance_id": 200, + "host_id_list": [ + 101,102 + ] +} +``` + +### 返回结果示例 + +#### 脚本执行步骤 +```json +{ + "result": true, + "code": 0, + "message": "", + "data": { + "log_type": 1, + "job_instance_id": 100, + "step_instance_id": 200, + "script_task_logs": [ + { + "bk_host_id": 101, + "ip": "127.0.0.1", + "bk_cloud_id": 0, + "log_content": "[2018-03-15 14:39:30][PID:56875] job_start\n" + }, + { + "bk_host_id": 102, + "ip": "127.0.0.2", + "bk_cloud_id": 0, + "log_content": "[2018-03-15 14:39:30][PID:16789] job_start\n" + } + ] + } +} +``` + +#### 文件分发步骤 + +```json +{ + "result": true, + "code": 0, + "message": "", + "data": { + "log_type": 2, + "job_instance_id": 100, + "step_instance_id": 200, + "file_task_logs": [ + { + "bk_host_id": 101, + "ip": "127.0.0.1", + "bk_cloud_id": 0, + "file_logs": [ + { + "mode": 1, + "src_ip": { + "bk_host_id": 102, + "bk_cloud_id": 0, + "ip": "127.0.0.2" + }, + "src_path": "/data/1.log", + "dest_ip": { + "bk_host_id": 101, + "bk_cloud_id": 0, + "ip": "127.0.0.1" + }, + "dest_path": "/tmp/1.log", + "status": 4, + "log_content": "[2021-06-28 11:32:16] FileName: /tmp/1.log FileSize: 9.0 Bytes State: dest agent success download file Speed: 1 KB/s Progress: 100% StatusDesc: dest agent success download file Detail: success", + "size": "1.0 Bytes", + "speed": "0 KB/s", + "process": "100%" + } + ] + }, + { + "bk_host_id": 102, + "ip": "127.0.0.2", + "bk_cloud_id": 0, + "file_logs": [ + { + "mode": 0, + "src_ip": { + "bk_host_id": 102, + "bk_cloud_id": 0, + "ip": "127.0.0.2" + }, + "src_path": "/data/1.log", + "status": 4, + "log_content": "[2021-06-28 11:32:16] FileName: /data/1.log FileSize: 9.0 Bytes State: source agent success upload file Speed: 1 KB/s Progress: 100% StatusDesc: source agent success upload file Detail: success upload", + "size": "1.0 Bytes", + "speed": "0 KB/s", + "process": "100%" + } + ] + } + ] + } +} +``` + +**文件任务返回结果说明** + +- 如果需要返回文件源的上传日志，需要在 host_id_list/ip_list添加源文件主机信息 + +### 返回结果说明 + +##### data + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| bk_host_id | long | 主机ID | +| bk_cloud_id | int | 目标服务器管控区域ID | +| ip | string | 目标服务器IP地址 | +| log_type | int | 日志类型。1-脚本执行任务日志;2-文件分发任务日志 | +| script_task_logs | array | 脚本执行任务日志。定义见script_task_log| +| file_task_logs | array | 文件分发任务日志。定义见file_task_log| + +##### script_task_log + +| 字段 | 类型 | 描述 | +|-----------|------------|--------| +| bk_host_id | long | 主机ID | +| bk_cloud_id | long | 管控区域ID | +| ip | string | 目标IP地址 | +| log_content | string | 脚本执行日志内容 | + +##### file_task_log + +| 字段 | 类型 | 描述 | +|-----------|------------|--------| +| bk_host_id | long | 主机ID | +| bk_cloud_id | long | 管控区域ID | +| ip | string | 源/目标IP地址 | +| file_logs | array | 文件分发日志内容。定义见file_log | + +##### file_log + +| 字段 | 类型 | 描述 | +|--------------|-----------|-----------| +| mode | 分发模式 | 0:上传;1:下载| +| src_ip | object |文件源主机IP。定义见ip | +| src_path | string | 源文件路径 | +| dest_ip | object | 分发目标主机IP，mode=1时有值。定义见ip | +| dest_path | string | 目标路径，mode=1时有值 | +| status | int | 任务状态。1-等待开始;2-上传中;3-下载中;4-成功;5-失败 | +| log_content | string | 文件分发日志内容 | +| size | string | 文件大小 | +| speed | string | 文件传输速率 | +| process | string | 文件传输进度 | + +##### ip + +| 字段 | 类型 | 描述 | +|-----------|------------|--------| +| bk_host_id | long | 主机ID | +| bk_cloud_id | long | 管控区域ID | +| ip | string | IP地址 | diff --git a/docs/apidoc/bk-api-gateway/v3/zh/bkci_plugin_batch_get_job_instance_execute_object_log.md b/docs/apidoc/bk-api-gateway/v3/zh/bkci_plugin_batch_get_job_instance_execute_object_log.md new file mode 100644 index 0000000000..c16300beb4 --- /dev/null +++ b/docs/apidoc/bk-api-gateway/v3/zh/bkci_plugin_batch_get_job_instance_execute_object_log.md @@ -0,0 +1,261 @@ +### 功能描述 + +根据执行对象列表批量查询执行日志(蓝盾作业执行插件专用，非正式公开 API) + +### 请求参数 + +#### Header 参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| X-Bkapi-Authorization | string | 是 | 认证信息。详情参考[调用网关 API](https://github.com/TencentBlueKing/BKDocs/blob/master/ZH/7.0/APIGateway/apigateway/use-api/use-apigw-api.md) | +| Accept | string | 是 | 固定值。application/json | +| Content-Type | string | 是 | 固定值。application/json| + + +#### Body 参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| job_instance_id | long | 是 | 作业实例ID | +| step_instance_id | long | 是 | 步骤实例ID | +| execute_object_list | array | 否 | 执行对象列表，定义见 exeucte_object | + +##### exeucte_object + +| 字段 | 类型 | 必选 | 描述 | +| ----------- | ------ | ---- | -------- | +| type | int | 是 | 执行对象类型, 1 - 主机，2 - 容器 | +| resource_id | string | 是 | 执行对象资源 ID（比如主机对应的资源 ID 为 bk_host_id) | + +### 请求参数示例 + +- Body +```json +{ + "bk_scope_type": "biz", + "bk_scope_id": "2", + "job_instance_id": 100, + "step_instance_id": 200, + "execute_object_list": [ + { + "type": 1, + "resource_id": "101" + }, + { + "type": 2, + "resource_id": "10002" + } + ] +} +``` + +### 返回结果示例 + +#### 脚本执行步骤 +```json +{ + "result": true, + "code": 0, + "data": { + "log_type": 1, + "job_instance_id": 100, + "step_instance_id": 200, + "script_execute_object_task_logs": [ + { + "execute_object": { + "type": 1, + "resource_id": "101", + "host": { + "bk_host_id": 101, + "ip": "127.0.0.1", + "bk_cloud_id": 0 + } + }, + "log_content": "[2018-03-15 14:39:30][PID:56875] job_start\n" + }, + { + "execute_object": { + "type": 2, + "resource_id": "10002", + "container": { + "id": 10002, + "container_uid": "docker://0f65a78b83d247615a696f1f8d136aa39afdc578fc8589e765ee5c3a38751858" + } + }, + "log_content": "[2018-03-15 14:39:30][PID:16789] job_start\n" + } + ] + } +} +``` + +#### 文件分发步骤 + +```json +{ + "result": true, + "code": 0, + "data": { + "log_type": 2, + "job_instance_id": 100, + "step_instance_id": 200, + "file_execute_object_task_logs": [ + { + "execute_object": { + "type": 1, + "resource_id": "101", + "host": { + "bk_host_id": 101, + "ip": "127.0.0.1", + "bk_cloud_id": 0 + } + }, + "file_atomic_task_logs": [ + { + "mode": 1, + "src_execute_object": { + "type": 2, + "resource_id": "10002", + "container": { + "id": 10002, + "container_uid": "docker://0f65a78b83d247615a696f1f8d136aa39afdc578fc8589e765ee5c3a38751858", + "name": "job_test" + } + }, + "src_path": "/data/1.log", + "dest_execute_object": { + "type": 1, + "resource_id": "101", + "host": { + "bk_host_id": 101, + "ip": "127.0.0.1", + "bk_cloud_id": 0 + } + }, + "dest_path": "/tmp/1.log", + "status": 4, + "log_content": "[2021-06-28 11:32:16] FileName: /tmp/1.log FileSize: 9.0 Bytes State: dest agent success download file Speed: 1 KB/s Progress: 100% StatusDesc: dest agent success download file Detail: success", + "size": "1.0 Bytes", + "speed": "0 KB/s", + "process": "100%" + } + ] + }, + { + "execute_object": { + "type": 2, + "resource_id": "10002", + "container": { + "id": 10002, + "container_uid": "docker://0f65a78b83d247615a696f1f8d136aa39afdc578fc8589e765ee5c3a38751858", + "name": "job_test" + } + }, + "file_atomic_task_logs": [ + { + "mode": 0, + "src_execute_object": { + "type": 2, + "resource_id": "10002", + "container": { + "id": 10002, + "container_uid": "docker://0f65a78b83d247615a696f1f8d136aa39afdc578fc8589e765ee5c3a38751858", + "name": "job_test" + } + }, + "src_path": "/data/1.log", + "status": 4, + "log_content": "[2021-06-28 11:32:16] FileName: /data/1.log FileSize: 9.0 Bytes State: source agent success upload file Speed: 1 KB/s Progress: 100% StatusDesc: source agent success upload file Detail: success upload", + "size": "1.0 Bytes", + "speed": "0 KB/s", + "process": "100%" + } + ] + } + ] + } +} +``` + + +### 返回结果说明 + +#### response + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + +##### data + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| job_instance_id | long | 是 | 作业实例ID | +| step_instance_id | long | 是 | 步骤实例ID | +| log_type | int | 日志类型。1-脚本执行任务日志;2-文件分发任务日志 | +| script_execute_object_task_logs | array | 脚本执行任务日志。定义见 script_execute_object_task_log | +| file_execute_object_task_logs | array | 文件分发任务日志。定义见 file_execute_object_task_log | + +##### script_execute_object_task_log + +| 字段 | 类型 | 描述 | +|-----------|------------|--------| +| execute_object | object | 执行对象，见 execute_object 定义 | +| log_content | string | 脚本执行日志内容 | + +##### file_execute_object_task_log + +| 字段 | 类型 | 描述 | +|-----------|------------|--------| +| execute_object | object | 执行对象，见 execute_object 定义 | +| file_atomic_task_logs | array | 文件分发原子任务，指一个文件从一个源执行对象分发到一个目标执行对象的任务。定义见 file_atomic_task_log | + + +##### file_atomic_task_log + +| 字段 | 类型 | 描述 | +|--------------|-----------|-----------| +| mode | int | 分发模式。0:上传;1:下载 | +| src_execute_object | object |源文件所在执行对象。定义见 execute_object | +| src_path | string | 源文件路径 | +| dest_execute_object | object |文件分发目标执行对象。定义见 execute_object | +| dest_path | string | 目标路径，mode=1时有值 | +| status | int | 任务状态。1-等待开始;2-上传中;3-下载中;4-成功;5-失败 | +| log_content | string | 文件分发日志内容 | +| size | string | 文件大小 | +| speed | string | 文件传输速率 | +| process | string | 文件传输进度 | + +##### execute_object + +| 字段 | 类型 | 描述 | +|-----------|------------|--------| +| type | int | 执行对象类型, 1 - 主机，2 - 容器 | +| resource_id | string | 执行对象资源 ID（比如主机对应的资源 ID 为 bk_host_id) | +| host | object | 主机详情，定义见 host | +| container | object | 容器详情, 定义见 container | + +##### host + +| 字段 | 类型 | 描述 | +|-----------|------------|--------| +| bk_host_id | long | 主机 ID (cmdb) | +| bk_cloud_id | long | 管控区域ID | +| ip | string | Ipv4 | + +##### container + +| 字段 | 类型 | 描述 | +|-----------|------------|--------| +| id | long | 容器 ID (cmdb) | +| container_uid | string | 容器在集群中的唯一 ID (UID) | +| name | string | 容器名称 | + + diff --git a/docs/apidoc/bk-api-gateway/v3/zh/bkci_plugin_fast_execute_script.md b/docs/apidoc/bk-api-gateway/v3/zh/bkci_plugin_fast_execute_script.md new file mode 100644 index 0000000000..edec4eafae --- /dev/null +++ b/docs/apidoc/bk-api-gateway/v3/zh/bkci_plugin_fast_execute_script.md @@ -0,0 +1,258 @@ +### 功能描述 + +快速执行脚本(蓝盾作业执行插件专用，非正式公开 API) + +### 请求参数 + +#### Header参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| X-Bkapi-Authorization | string | 是 | 认证信息。详情参考[调用网关 API](https://github.com/TencentBlueKing/BKDocs/blob/master/ZH/7.0/APIGateway/apigateway/use-api/use-apigw-api.md) | +| Accept | string | 是 | 固定值。application/json | +| Content-Type | string | 是 | 固定值。application/json| + + +#### Body参数 + +| 字段 | 类型 | 必选 | 描述 | +|---------------|------------|--------|--------------------------------------------------------------------------------------------------------------------------------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| script_version_id | long | 否 | 脚本版本ID。当script_version_id不为空的时候，使用script_version_id对应的脚本版本 | +| script_id | string | 否 | 脚本ID。当传入script_id，且script_version_id为空的时候，使用脚本的上线版本 | +| script_content | string | 否 | 脚本内容Base64。如果不存在script_version_id和script_id,那么使用script_content。优先级：script_version_id>script_id>script_content | +| task_name | string | 否 | 自定义作业名称 | +| script_param | string | 否 | 脚本参数Base64。脚本参数长度不得超过 5000 个字符。注意：如果有多个参数，比如"param1 param2"这种，需要对"param1 param2"整体进行base64编码，而不是对每个参数进行base64编码再拼接起来 | +| windows_interpreter | string | 否 | 自定义Windows解释器路径，当该值不为空时，在windows中优先使用该解释器运行脚本，只对Windows机器生效 | +| timeout | long | 否 | 脚本执行超时时间，秒。默认7200，取值范围1-86400 | +| account_alias | string | 否 | 执行账号别名。与account_id必须存在一个。当同时存在account_alias和account_id时，account_id优先。 | +| account_id | long | 否 | 执行账号ID。与account_alias必须存在一个。当同时存在account_alias和account_id时，account_id优先。 | +| is_param_sensitive | boolean | 否 | 敏感参数将会在执行详情页面上隐藏。默认为 false | +| script_language | int | 否 | 脚本语言：1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell。当使用script_content传入自定义脚本的时候，需要指定script_language | +| execute_target | object | 是 | 执行目标，定义见 execute_target | +| callback | object | 否 | 回调配置。当任务执行完成后，JOB会调用该URL告知任务执行结果。回调协议参考callback_protocol组件文档 ｜ +| rolling_config | object | 否 | 滚动配置，见rolling_config定义 | + +##### execute_target + +| 字段 | 类型 | 必选 | 描述 | +| ------------------ | ----- | ---- | ----------------------------------- | +| host_list | array | 否 | 主机列表，调用方可以选择使用 bk_host_id 或者 bk_cloud_id+ip 指定主机两种方式。 见 host 定义 | +| host_dynamic_group_list | array | 否 | 主机动态分组列表，定义见 dynamic_group | +| host_topo_node_list | array | 否 | 主机动态 topo 节点列表，定义见 host_topo_node | +| kube_container_filters | array | 否 | 容器过滤器，多个过滤器为 OR 关系，取容器并集。定义见 kube_container_filter | + +##### host + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------| +| bk_host_id | long | 否 | 主机 ID (cmdb) | +| bk_cloud_id | long | 否 | 管控区域ID | +| ip | string | 否 | Ipv4 ｜ + +##### dynamic_group + +| 字段 | 类型 | 必选 | 描述 | +| ---- | ------ | ---- | -------------- | +| id | string | 是 | CMDB动态分组ID | + +##### host_topo_node + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| id | long | 是 | 动态topo节点ID，对应CMDB API 中的 bk_inst_id | +| node_type | string | 是 | 动态topo节点类型，对应CMDB API 中的 bk_obj_id,比如"module","set" | + +##### kube_container_filter + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| kube_cluster_filter | object | 否 | 集群过滤器. 过滤器定义见 kube_cluster_filter | +| kube_namespace_filter | object | 否 | namespace 过滤器. 过滤器定义见 kube_namespace_filter | +| kube_workload_filter | object | 否 | workload 过滤器. 过滤器定义见 kube_workload_filter | +| kube_pod_filter | object | 否 | pod 属性过滤器，过滤器定义见 kube_pod_filter | +| kube_container_prop_filter | object | 是 | 容器属性过滤器，过滤器定义见 kube_container_prop_filter | +| is_empty_filter | boolean | 是 | 标识一个没有设置任何条件的过滤器；默认值为 false。如果为 true, 将忽略其他的条件（kube_cluster_filter/kube_namespace_filter/kube_workload_filter/kube_pod_filter/kube_container_prop_filter)，返回业务下的所有容器 | +| fetch_any_one_container | boolean | 否 | 是否从过滤结果集中选择任意一个容器作为执行对象（只有一个容器会被执行）；默认为 false | + +##### kube_cluster_filter + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| cluster_uid_list | array | 是 | 集群ID列表, 例如: "BCS-K8S-00001" | + +##### kube_namespace_filter + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| namespace_name_list | array | 是 | namespace 名称列表 | + + +##### kube_workload_filter + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| kind | string | 是 | workload类型，目前支持的workload类型有deployment、daemonSet、statefulSet、gameStatefulSet、gameDeployment、cronJob、job) | +| workload_name_list | array | 否 | workload 名称列表 | + +##### kube_pod_prop_filter + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| pod_name_list | array | 否 | pod 名称列表 | +| label_selector | array | 否 | Job 定义的label selector，由多个 Requirement 构成。多个 Requirement 之间为 AND 关系。表达式 见 requirement 定义 | +| label_selector_expr | string | 否 | k8s 官方定义的 label selector 表达式，[lable selector doc](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors) | + +##### requirement + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| key | string | 是 | label key | +| operator | string | 是 | 计算操作符, 支持 equals、not_equals、in、not_in、exists、not_exists | +| values | array | 否 | label value 列表, 当计算操作符为 in、not_in、equals、not_equals 时需要设置 | + +##### kube_container_prop_filter + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| container_name_list | array | 是 | 容器名称列表 | + + +##### rolling_config + +| 字段 | 类型 | 必选 | 描述 | +| ---- | ------ | ---- | -------------- | +| expression | string | 是 | 滚动策略表达式 | +| mode | int | 是 | 滚动机制,1-执行失败则暂停；2-忽略失败，自动滚动下一批；3-人工确认 | + +##### callback + +| 字段 | 类型 | 必选 | 描述 | +| ---- | ------ | ---- | -------------- | +| url | string | 是 | 回调URL | + +### 请求参数示例 + +- Body +```json +{ + "bk_scope_type": "biz", + "bk_scope_id": "2", + "script_version_id": 1, + "script_content": "ZWNobyAkMQ==", + "script_param": "aGVsbG8=", + "windows_interpreter": "D:\\Software\\python3\\python.exe", + "timeout": 1000, + "account_id": 1000, + "is_param_sensitive": 0, + "script_language": 1, + "execute_target": { + "host_dynamic_group_list": [ + { + "id": "blo8gojho0skft7pr5q0" + } + ], + "host_list": [ + { + "bk_host_id": 101 + }, + { + "bk_cloud_id": 0, + "ip": "127.0.0.2" + } + ], + "host_topo_node_list": [ + { + "id": 1000, + "node_type": "module" + } + ], + "kube_container_filters": [ + { + "kube_cluster_filter": { + "cluster_uid_list": [ + "BCS-K8S-00001", + "BCS-K8S-00002" + ] + }, + "kube_namespace_filter": { + "namespace_name_list": [ + "job-prod", + "job-gray" + ] + }, + "kube_workload_filter": { + "kind": "deployment", + "workload_name_list": [ + "bk-job-manage", + "bk-job-execute" + ] + }, + "kube_pod_filter": { + "pod_name_list": [ + "bk-job-execute-6fcd8cf5c7-jvctq", + "bk-job-manage-6fcd8cf5c7-abues" + ], + "label_selector": [ + { + "label_key": "application", + "operator": "in", + "label_values": [ + "job-execute", + "job-manage" + ] + }, + { + "label_key": "env", + "operator": "equals", + "label_value": "prod" + } + ] + }, + "kube_container_prop_filter": { + "container_name_list": [ + "job-execute", + "job-manage" + ] + }, + "fetch_any_one_container": true + } + ] + } +} +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "data": { + "job_instance_name": "API Quick execution script1521100521303", + "job_instance_id": 10000, + "step_instance_id": 10001 + } +} +``` + +### 返回结果参数说明 + +#### response + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + +##### data + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| job_instance_id | long | 作业实例ID | +| job_instance_name | string | 作业实例名称 | +| step_instance_id | long | 步骤实例ID | diff --git a/docs/apidoc/bk-api-gateway/v3/zh/bkci_plugin_get_job_instance_status.md b/docs/apidoc/bk-api-gateway/v3/zh/bkci_plugin_get_job_instance_status.md new file mode 100644 index 0000000000..8d618a211c --- /dev/null +++ b/docs/apidoc/bk-api-gateway/v3/zh/bkci_plugin_get_job_instance_status.md @@ -0,0 +1,180 @@ +### 功能描述 + +根据作业实例 ID 查询作业执行状态(蓝盾作业执行插件专用，非正式公开 API) + +### 请求参数说明 + +#### Header参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| X-Bkapi-Authorization | string | 是 | 认证信息。详情参考[调用网关 API](https://github.com/TencentBlueKing/BKDocs/blob/master/ZH/7.0/APIGateway/apigateway/use-api/use-apigw-api.md) | +| Accept | string | 是 | 固定值。application/json | +| Content-Type | string | 是 | 固定值。application/json| + + +#### Body 参数 + +| 字段 | 类型 | 必选 | 描述 | +|------------------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| job_instance_id | long | 是 | 作业实例ID | +| include_execute_object_task_result | boolean | 否 | 是否返回每个执行对象上的任务详情，对应返回结果中的 step_execute_object_result_list 。默认值为false。 | + +### 请求参数示例 + +- Body + +```json +{ + "bk_scope_type": "biz", + "bk_scope_id": "2", + "job_instance_id": 100, + "include_execute_object_task_result": true +} +``` + + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "data": { + "finished": true, + "job_instance": { + "job_instance_id": 100, + "bk_scope_type": "biz", + "bk_scope_id": "1", + "name": "API Quick execution script1521089795887", + "create_time": 1605064271000, + "status": 4, + "start_time": 1605064271000, + "end_time": 1605064272000, + "total_time": 1000 + }, + "step_instance_list": [ + { + "status": 4, + "total_time": 1000, + "name": "API Quick execution scriptxxx", + "step_instance_id": 75, + "execute_count": 0, + "create_time": 1605064271000, + "end_time": 1605064272000, + "type": 1, + "start_time": 1605064271000, + "execute_object_result_list": [ + { + "execute_object": { + "type": 1, + "resource_id": "101", + "host": { + "bk_host_id": 101, + "ip": "127.0.0.1", + "bk_cloud_id": 0 + } + }, + "status": 9, + "tag": "", + "exit_code": 0, + "error_code": 0, + "start_time": 1605064271000, + "end_time": 1605064272000, + "total_time": 1000 + } + ] + } + ] + } +} +``` +### 返回结果说明 + +#### response + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + +##### data + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| finished | bool | 作业是否结束 | +| job_instance | object | 作业实例基本信息。见job_instance定义 | +| step_instance_list | array | 作业步骤列表。见step_instance定义 | + +##### job_instance + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| name | string | 作业实例名称 | +| status | int | 作业状态码: 1.未执行; 2.正在执行; 3.执行成功; 4.执行失败; 5.跳过; 6.忽略错误; 7.等待用户; 8.手动结束; 9.状态异常; 10.步骤强制终止中; 11.步骤强制终止成功 | +| create_time | long | 作业创建时间，Unix时间戳，单位毫秒 | +| start_time | long | 开始执行时间，Unix时间戳，单位毫秒 | +| end_time | long | 执行结束时间，Unix时间戳，单位毫秒 | +| total_time | int | 总耗时，单位毫秒 | +| bk_scope_type | string |资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| job_instance_id | long | 作业实例ID | + +##### step_instance + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| step_instance_id | long | 作业步骤实例ID | +| type | int | 步骤类型：1.脚本步骤; 2.文件步骤; 4.SQL步骤 | +| name | string | 步骤名称 | +| status | int | 作业步骤状态码: 1.未执行; 2.正在执行; 3.执行成功; 4.执行失败; 5.跳过; 6.忽略错误; 7.等待用户; 8.手动结束; 9.状态异常; 10.步骤强制终止中; 11.步骤强制终止成功; 12.步骤强制终止失败 | +| create_time | long | 作业步骤实例创建时间，Unix时间戳，单位毫秒 | +| start_time | long | 开始执行时间，Unix时间戳，单位毫秒 | +| end_time | long | 执行结束时间，Unix时间戳，单位毫秒 | +| total_time | int | 总耗时，单位毫秒 | +| execute_count | int | 步骤重试次数 | +| execute_object_result_list | array | 每个主机的任务执行结果，定义见 execute_object_result | + + +##### execute_object_result + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| execute_object | object | 执行对象，定义见 execute_object | +| status | int | 作业执行状态:1.Agent异常; 5.等待执行; 7.正在执行; 9.执行成功; 11.执行失败; 12.任务下发失败; 403.任务强制终止成功; 404.任务强制终止失败 | +| tag | string | 用户通过job_success/job_fail函数模板自定义输出的结果。仅脚本任务存在该参数 | +| exit_code | int | 脚本任务exit code | +| error_code | int | 主机任务状态码，1.Agent异常; 3.上次已成功; 5.等待执行; 7.正在执行; 9.执行成功; 11.任务失败; 12.任务下发失败; 13.任务超时; 15.任务日志错误; 101.脚本执行失败; 102.脚本执行超时; 103.脚本执行被终止; 104.脚本返回码非零; 202.文件传输失败; 203.源文件不存在; 310.Agent异常; 311.用户名不存在; 320.文件获取失败; 321.文件超出限制; 329.文件传输错误; 399.任务执行出错 | +| start_time | long | 开始执行时间，Unix时间戳，单位毫秒 | +| end_time | long | 执行结束时间，Unix时间戳，单位毫秒 | +| total_time | int | 总耗时，单位毫秒 | + +##### execute_object + +| 字段 | 类型 | 描述 | +|-----------|------------|--------| +| type | int | 执行对象类型, 1 - 主机，2 - 容器 | +| resource_id | string | 执行对象资源 ID（比如主机对应的资源 ID 为 bk_host_id) | +| host | object | 主机详情，定义见 host | +| container | object | 容器详情, 定义见 container | + +##### host + +| 字段 | 类型 | 描述 | +|-----------|------------|--------| +| bk_host_id | long | 主机 ID (cmdb) | +| bk_cloud_id | long | 管控区域ID | +| ip | string | Ipv4 | + +##### container + +| 字段 | 类型 | 描述 | +|-----------|------------|--------| +| id | long | 容器 ID (cmdb) | +| container_uid | string | 容器在集群中的唯一 ID (UID) | +| name | string | 容器名称 | diff --git a/docs/apidoc/bk-api-gateway/v3/zh/callback_protocol.md b/docs/apidoc/bk-api-gateway/v3/zh/callback_protocol.md new file mode 100644 index 0000000000..95050cb608 --- /dev/null +++ b/docs/apidoc/bk-api-gateway/v3/zh/callback_protocol.md @@ -0,0 +1,50 @@ +### 功能描述 + +此组件仅用于展示回调协议文档。 + +对作业执行类的请求传入的回调 callback_url 地址进行回调时所传递的报文结构描述。 + +### 请求头 +Content-Type: application/json + +兼容说明： +由于老版本Job使用了Content-Type: application/x-www-form-urlencoded执行回调请求，并且部分调用方按该方式进行解析接入，当前版本Job将优先使用Content-Type: application/json执行回调，如果执行失败，将再使用Content-Type: application/x-www-form-urlencoded重试一次，来兼容已接入的调用方。新接入的调用方请按照Content-Type: application/json进行解析。 + +### 请求参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------------|------------|--------|------------| +| job_instance_id | long | 是 | 作业实例ID | +| status | int | 是 | 作业状态码: 1.等待执行; 2.正在执行; 3.执行成功; 4.执行失败; 5.跳过; 6.忽略错误; 7.等待用户; 8.强制终止; 9.状态异常; 10.强制终止中; 11.强制终止成功,13.确认终止 | +| step_instance_list | array | 是 | 步骤块中包含的各个步骤执行状态 | + +#### step_instances + +| 字段 | 类型 | 必选 | 描述 | +|-----------------|------------|--------|------------| +| step_instance_id | long | 是 | 作业步骤实例ID | +| status | int | 是 | 作业步骤状态码: 1.等待执行; 2.正在执行; 3.执行成功; 4.执行失败; 5.跳过; 6.忽略错误; 7.等待用户; 8.强制终止; 9.状态异常; 10.强制终止中; 11.强制终止成功,13.确认终止 | + +### 请求参数示例 + +```json +{ + "job_instance_id": 12345, + "status": 2, + "step_instance_list": [ + { + "step_instance_id": 16271, + "status": 3 + }, + { + "step_instance_id": 16272, + "status": 2 + } + ] +} +``` + +### 回调响应 + +回调成功以HTTP状态为准，如果成功，则状态码200，其他表示失败。 +当前版本Job将优先使用Content-Type: application/json执行回调，如果执行成功，则回调结束，如果执行失败，将再使用Content-Type: application/x-www-form-urlencoded重试一次，来兼容按照老协议进行解析的调用方。 diff --git a/docs/apidoc/bk-api-gateway/v3/zh/common.md b/docs/apidoc/bk-api-gateway/v3/zh/common.md new file mode 100644 index 0000000000..293b33e8ec --- /dev/null +++ b/docs/apidoc/bk-api-gateway/v3/zh/common.md @@ -0,0 +1,45 @@ +## 通用字段和约定 + +#### global_var + +| 字段 | 类型 | 必选 | 描述 | +|-------------|------------|--------|------------| +| id | long | 否 | 全局变量 id，唯一标识。如果 id 为空，那么使用 name 作为唯一标识 | +| name | string | 否 | 全局变量名 | +| description | string | 否 | 全局变量描述 | +| type | int | 否 | 全局变量类型 | +| required | int | 否 | 模版/执行方案中该变量是否必填 | +| value | string | 否 | 字符、密码、数组类型的全局变量的值 | +| server | object | 否 | 主机类型全局变量的值 | + +#### server +| 字段 | 类型 | 必选 | 描述 | +|-----------------------|-------|--------|------------| +| ip_list | array | 否 | 静态 IP 列表，定义见ip | +| dynamic_group_list | array | 否 | 动态分组列表，定义见dynamic_group | +| topo_node_list | array | 否 | 动态 topo 节点列表，定义见topo_node | + +#### ip +| 字段 | 类型 | 必选 | 描述 | +|-------------|---------|--------|---------| +| bk_cloud_id | int | 是 | 管控区域ID | +| ip | string | 是 | IP地址 | + +#### dynamic_group + +| 字段 | 类型 | 必选 | 描述 | +| ---- | ------ | ---- | -------------- | +| id | string | 是 | CMDB动态分组ID | + +#### topo_node + +| 字段 | 类型 | 必选 | 描述 | +|------------------|--------|--------|------------| +| id | long | 是 | 动态topo节点ID，对应CMDB API 中的 bk_inst_id | +| node_type | string | 是 | 动态topo节点类型，对应CMDB API 中的 bk_obj_id,比如"module","set"| + +#### account +| 字段 | 类型 | 必选 | 描述 | +|-------|--------|--------|------------| +| id | long | 否 | 账号 ID | +| name | string | 否 | 账号名 | diff --git a/docs/apidoc/bk-api-gateway/v3/zh/create_account.md b/docs/apidoc/bk-api-gateway/v3/zh/create_account.md new file mode 100644 index 0000000000..5919968d41 --- /dev/null +++ b/docs/apidoc/bk-api-gateway/v3/zh/create_account.md @@ -0,0 +1,100 @@ +### 功能描述 + +创建账号（当前仅支持系统账号） + +### 请求参数 + +{{ bkapi_authorization_description }} + +#### Header参数 + +| 字段 | 类型 | 必选 | 描述 | +|---------------------------|------------|---------|------------| +| X-Bkapi-Authorization | string | 是 | 认证信息。详情参考[调用网关 API](https://github.com/TencentBlueKing/BKDocs/blob/master/ZH/7.0/APIGateway/apigateway/use-api/use-apigw-api.md) | +| Accept | string | 是 | 固定值。application/json| +| Content-Type | string | 是 | 固定值。application/json| + +#### Body参数 + +| 字段 | 类型 | 必选 | 描述 | +|---------------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID，与bk_scope_type对应，表示业务ID或者业务集ID | +| account | string | 是 | 账号名称 | +| type | int | 是 | 账号类型（1：Linux，2：Windows）| +| category | int | 是 | 账号用途（1：系统账号）| +| password | string | 否 | 系统账号密码，账号用途为系统账号且账号类型为Windows时必传 | +| alias | string | 否 | 别名，不传则以账号名称作为别名 | +| description | string | 否 | 描述 | + +### 请求参数示例 + +- POST +```json +{ + "bk_scope_type": "biz", + "bk_scope_id": "1", + "account": "Admin", + "type": 2, + "category": 1, + "password": "AdminPassword", + "alias": "Admin", + "description": "An account for windows" +} +``` + +### 返回结果示例 + +```json +{ + "code": 0, + "message": null, + "result": true, + "data": { + "id": 70, + "bk_scope_type": "biz", + "bk_scope_id": "1", + "account": "Admin", + "alias": "Admin", + "category": 1, + "type": 2, + "os": "Windows", + "description": "An account for windows", + "creator": "admin", + "create_time": 1614659536108, + "last_modify_user": "admin", + "last_modify_time": 1614659536116 + }, + "job_request_id": "4e7acb216087eb96" +} +``` + +### 返回结果参数说明 + +#### response +| 字段 | 类型 | 描述 | +|--------------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + + +#### data + +| 字段 | 类型 | 描述 | +|-----------------------|-----------|-----------| +| id | long | 账号ID | +| bk_scope_type | string | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| account | string | 账号名称 | +| alias | string | 账号别名 | +| category | int | 账号用途（1：系统账号） | +| type | int | 账号类型（1：Linux，2：Windows）| +| os | string | 账号用途为系统账号时该字段生效，账号对应的OS | +| description | string | 账号描述 | +| creator | string | 创建人 | +| create_time | long | 创建时间Unix时间戳（ms） | +| last_modify_user | string | 最近一次修改人 | +| last_modify_time | long | 最近一次修改时间Unix时间戳（ms） | diff --git a/docs/apidoc/bk-api-gateway/v3/zh/delete_account.md b/docs/apidoc/bk-api-gateway/v3/zh/delete_account.md new file mode 100644 index 0000000000..d558310e09 --- /dev/null +++ b/docs/apidoc/bk-api-gateway/v3/zh/delete_account.md @@ -0,0 +1,92 @@ +### 功能描述 + +删除账号。 + +### 请求参数 + +{{ bkapi_authorization_description }} + +#### Header参数 + +| 字段 | 类型 | 必选 | 描述 | +|---------------------------|------------|---------|------------| +| X-Bkapi-Authorization | string | 是 | 认证信息。详情参考[调用网关 API](https://github.com/TencentBlueKing/BKDocs/blob/master/ZH/7.0/APIGateway/apigateway/use-api/use-apigw-api.md) | +| Accept | string | 是 | 固定值。application/json| +| Content-Type | string | 是 | 固定值。application/json| + +#### Body参数 + +| 字段 | 类型 | 必选 | 描述 | +|---------------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| id | long | 否 | 账号ID | + +### 请求参数示例 + +- POST +```json +{ + "bk_scope_type": "biz", + "bk_scope_id": "1", + "id": 70 +} +``` + +### 返回结果示例 + +```json +{ + "code": 0, + "message": null, + "result": true, + "data": { + "id": 70, + "bk_scope_type": "biz", + "bk_scope_id": "1", + "account": "Admin", + "alias": "Admin", + "category": 1, + "type": 2, + "db_system_account_id": null, + "os": "Windows", + "description": "An account for windows", + "creator": "admin", + "create_time": 1614659536108, + "last_modify_user": "admin", + "last_modify_time": 1614659536116 + }, + "job_request_id": "4e7acb216087eb96" +} +``` + +### 返回结果参数说明 + +#### response +| 字段 | 类型 | 描述 | +|--------------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + + +#### data + +| 字段 | 类型 | 描述 | +|-----------------------|-----------|-----------| +| id | long | 账号ID | +| bk_scope_type | string | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| account | string | 账号名称 | +| alias | string | 账号别名 | +| category | int | 账号用途（1：系统账号，2：数据库账号） | +| type | int | 账号类型（1：Linux，2：Windows，9：MySQL，10：Oracle，11：DB2）| +| db_system_account_id | long | 账号用途为数据库账号时该字段生效，表示数据库账号对应的系统账号ID | +| os | string | 账号用途为系统账号时该字段生效，账号对应的OS | +| description | string | 描述 | +| creator | string | 创建人 | +| create_time | long | 创建时间Unix时间戳（ms） | +| last_modify_user | string | 最近一次修改人 | +| last_modify_time | long | 最近一次修改时间Unix时间戳（ms） | diff --git a/docs/apidoc/bk-api-gateway/v3/zh/delete_cron.md b/docs/apidoc/bk-api-gateway/v3/zh/delete_cron.md new file mode 100644 index 0000000000..676f7cf123 --- /dev/null +++ b/docs/apidoc/bk-api-gateway/v3/zh/delete_cron.md @@ -0,0 +1,62 @@ +### 功能描述 + +删除定时任务 + +### 请求参数 + +{{ bkapi_authorization_description }} + +#### Header参数 + +| 字段 | 类型 | 必选 | 描述 | +|---------------------------|------------|---------|------------| +| X-Bkapi-Authorization | string | 是 | 认证信息。详情参考[调用网关 API](https://github.com/TencentBlueKing/BKDocs/blob/master/ZH/7.0/APIGateway/apigateway/use-api/use-apigw-api.md) | +| Accept | string | 是 | 固定值。application/json| +| Content-Type | string | 是 | 固定值。application/json| + +#### Body参数 + +| 字段 | 类型 | 必选 | 描述 | +|---------------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| id | long | 否 | 定时任务ID | + +### 请求参数示例 + +- POST +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type":"biz", + "bk_scope_id":"2", + "id": 1000045 +} +``` + +### 返回结果示例 + +```json +{ + "code": 0, + "result": true +} +``` + +### 返回结果参数说明 + +#### response +| 字段 | 类型 | 描述 | +|--------------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + + +#### data + +无 diff --git a/docs/apidoc/bk-api-gateway/v3/zh/execute_job_plan.md b/docs/apidoc/bk-api-gateway/v3/zh/execute_job_plan.md new file mode 100644 index 0000000000..02e8c2c59d --- /dev/null +++ b/docs/apidoc/bk-api-gateway/v3/zh/execute_job_plan.md @@ -0,0 +1,133 @@ +### 功能描述 + +启动作业执行方案 + +### 请求参数 + +{{ bkapi_authorization_description }} + +#### Header参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| X-Bkapi-Authorization | string | 是 | 认证信息。详情参考[调用网关 API](https://github.com/TencentBlueKing/BKDocs/blob/master/ZH/7.0/APIGateway/apigateway/use-api/use-apigw-api.md) | +| Accept | string | 是 | 固定值。application/json| +| Content-Type | string | 是 | 固定值。application/json| + +#### Body参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| bk_biz_id | long | 是 | *已弃用*。业务ID。此字段已被弃用并由字段bk_scope_type+bk_scope_id替换 | +| job_plan_id | long | 是 | 作业执行方案ID | +| global_var_list | array | 否 | 全局变量。对于作业执行方案中的全局变量值，如果请求参数中包含该变量，则使用传入的变量值；否则使用执行方案当前已配置的默认值。定义见global_var | +| callback_url | string | 否 | 回调URL，当任务执行完成后，JOB会调用该URL告知任务执行结果。回调协议参考callback_protocol组件文档 | + +##### global_var + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| id | long | 否 | 全局变量id，唯一标识。如果id为空，那么使用name作为唯一标识 | +| name | string | 否 | 全局变量name | +| value | string | 否 | 字符、密码、数组、命名空间类型的全局变量的值 | +| server | object | 否 | 主机类型全局变量的值，见server定义 | + +##### server + +| 字段 | 类型 | 必选 | 描述 | +| ------------------ | ----- | ---- | ----------------------------------- | +| host_id_list | array | 否 | 主机ID列表 | +| ip_list | array | 否 | 静态 IP 列表，定义见ip | +| dynamic_group_list | array | 否 | 动态分组列表，定义见dynamic_group | +| topo_node_list | array | 否 | 动态 topo 节点列表，定义见topo_node | + +##### ip + +| 字段 | 类型 | 必选 | 描述 | +| ----------- | ------ | ---- | -------- | +| bk_cloud_id | int | 是 | 管控区域ID | +| ip | string | 是 | IP地址 | + +##### dynamic_group + +| 字段 | 类型 | 必选 | 描述 | +| ---- | ------ | ---- | -------------- | +| id | string | 是 | CMDB动态分组ID | + +##### topo_node + +| 字段 | 类型 | 必选 | 描述 | +| --------- | ------ | ---- | ------------------------------------------------------------ | +| id | long | 是 | 动态topo节点ID，对应CMDB API 中的 bk_inst_id | +| node_type | string | 是 | 动态topo节点类型，对应CMDB API 中的 bk_obj_id,比如"module","set" | + +### 请求参数示例 + +- POST +```json +{ + "bk_scope_type": "biz", + "bk_scope_id": "1", + "job_plan_id": 100, + "global_var_list": [ + { + "id": 436, + "server": { + "dynamic_group_list": [ + { + "id": "blo8gojho0skft7pr5q0" + } + ], + "host_id_list": [ + 101, + 102 + ], + "topo_node_list": [ + { + "id": 1000, + "node_type": "module" + } + ] + } + }, + { + "name": "param_name", + "value": "param_value" + } + ] +} +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "job_instance_name": "Test", + "job_instance_id": 10000 + } +} +``` + +### 返回结果参数说明 + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + +##### data + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| job_instance_id | long | 作业实例ID | +| job_instance_name | long | 作业实例名称 | + diff --git a/docs/apidoc/bk-api-gateway/v3/zh/fast_execute_script.md b/docs/apidoc/bk-api-gateway/v3/zh/fast_execute_script.md new file mode 100644 index 0000000000..c93b43b917 --- /dev/null +++ b/docs/apidoc/bk-api-gateway/v3/zh/fast_execute_script.md @@ -0,0 +1,140 @@ +### 功能描述 + +快速执行脚本 + +### 请求参数 + +{{ bkapi_authorization_description }} + +#### Header参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| X-Bkapi-Authorization | string | 是 | 认证信息。详情参考[调用网关 API](https://github.com/TencentBlueKing/BKDocs/blob/master/ZH/7.0/APIGateway/apigateway/use-api/use-apigw-api.md) | +| Accept | string | 是 | 固定值。application/json| +| Content-Type | string | 是 | 固定值。application/json| + +#### Body参数 + +| 字段 | 类型 | 必选 | 描述 | +|---------------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| bk_biz_id | long | 是 | *已弃用*。业务ID。此字段已被弃用并由字段bk_scope_type+bk_scope_id替换 | +| script_version_id | long | 否 | 脚本版本ID。当script_version_id不为空的时候，使用script_version_id对应的脚本版本 | +| script_id | string | 否 | 脚本ID。当传入script_id，且script_version_id为空的时候，使用脚本的上线版本 | +| script_content | string | 否 | 脚本内容Base64。如果不存在script_version_id和script_id,那么使用script_content。优先级：script_version_id>script_id>script_content | +| task_name | string | 否 | 自定义作业名称 | +| script_param | string | 否 | 脚本参数Base64。注意：如果有多个参数，比如"param1 param2"这种，需要对"param1 param2"整体进行base64编码，而不是对每个参数进行base64编码再拼接起来 | +| windows_interpreter | string | 否 | 自定义Windows解释器路径，当该值不为空时，在windows中优先使用该解释器运行脚本，只对Windows机器生效 | +| timeout | long | 否 | 脚本执行超时时间，秒。默认7200，取值范围1-86400 | +| account_alias | string | 否 | 执行账号别名。与account_id必须存在一个。当同时存在account_alias和account_id时，account_id优先。 | +| account_id | long | 否 | 执行账号ID。与account_alias必须存在一个。当同时存在account_alias和account_id时，account_id优先。 | +| is_param_sensitive | int | 否 | 敏感参数将会在执行详情页面上隐藏, 0:不是（默认），1:是 | +| script_language | int | 否 | 脚本语言：1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell。当使用script_content传入自定义脚本的时候，需要指定script_language | +| target_server | object | 否 | 目标服务器，见server定义 | +| callback_url | string | 否 | 回调URL，当任务执行完成后，JOB会调用该URL告知任务执行结果。回调协议参考callback_protocol组件文档 | +| rolling_config | object | 否 | 滚动配置，见rolling_config定义 | + +##### server +| 字段 | 类型 | 必选 | 描述 | +| ------------------ | ----- | ---- | ----------------------------------- | +| host_id_list | array | 否 | 主机ID列表 | +| ip_list | array | 否 | ***不推荐使用，建议使用host_id_list参数***。主机IP 列表，定义见ip | +| dynamic_group_list | array | 否 | 动态分组列表，定义见dynamic_group | +| topo_node_list | array | 否 | 动态 topo 节点列表，定义见topo_node | + +##### ip + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| bk_cloud_id | long | 是 | 管控区域ID | +| ip | string | 是 | IP地址 | + +##### dynamic_group + +| 字段 | 类型 | 必选 | 描述 | +| ---- | ------ | ---- | -------------- | +| id | string | 是 | CMDB动态分组ID | + +##### topo_node_list + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| id | long | 是 | 动态topo节点ID，对应CMDB API 中的 bk_inst_id | +| node_type | string | 是 | 动态topo节点类型，对应CMDB API 中的 bk_obj_id,比如"module","set"| + +##### rolling_config + +| 字段 | 类型 | 必选 | 描述 | +| ---- | ------ | ---- | -------------- | +| expression | string | 是 | 滚动策略表达式 | +| mode | int | 是 | 滚动机制,1-执行失败则暂停；2-忽略失败，自动滚动下一批；3-人工确认 | + +### 请求参数示例 + +- POST +```json +{ + "bk_scope_type": "biz", + "bk_scope_id": "1", + "script_version_id": 1, + "script_content": "ZWNobyAkMQ==", + "script_param": "aGVsbG8=", + "windows_interpreter": "D:\\Software\\python3\\python.exe", + "timeout": 1000, + "account_id": 1000, + "is_param_sensitive": 0, + "script_language": 1, + "target_server": { + "dynamic_group_list": [ + { + "id": "blo8gojho0skft7pr5q0" + } + ], + "host_id_list": [ + 101, + 102 + ], + "topo_node_list": [ + { + "id": 1000, + "node_type": "module" + } + ] + } +} +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "job_instance_name": "API Quick execution script1521100521303", + "job_instance_id": 10000, + "step_instance_id": 10001 + } +} +``` + +### 返回结果参数说明 + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + +##### data + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| job_instance_id | long | 作业实例ID | +| job_instance_name | long | 作业实例名称 | +| step_instance_id | long | 步骤实例ID | diff --git a/docs/apidoc/bk-api-gateway/v3/zh/fast_execute_sql.md b/docs/apidoc/bk-api-gateway/v3/zh/fast_execute_sql.md new file mode 100644 index 0000000000..26b822d35a --- /dev/null +++ b/docs/apidoc/bk-api-gateway/v3/zh/fast_execute_sql.md @@ -0,0 +1,122 @@ +### 功能描述 + +快速执行SQL脚本 + +### 请求参数 + +{{ bkapi_authorization_description }} + +#### Header参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| X-Bkapi-Authorization | string | 是 | 认证信息。详情参考[调用网关 API](https://github.com/TencentBlueKing/BKDocs/blob/master/ZH/7.0/APIGateway/apigateway/use-api/use-apigw-api.md) | +| Accept | string | 是 | 固定值。application/json| +| Content-Type | string | 是 | 固定值。application/json| + +#### Body参数 + +| 字段 | 类型 | 必选 | 描述 | +|---------------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| bk_biz_id | long | 是 | *已弃用*。业务ID。此字段已被弃用并由字段bk_scope_type+bk_scope_id替换 | +| task_name | string | 否 | 自定义作业名称 | +| script_version_id | long | 否 | SQL脚本版本ID | +| script_id | string | 否 | 脚本ID。当传入script_id，且script_version_id为空的时候，使用当前脚本的上线版本 | +| script_content | string | 否 | 脚本内容Base64。如果不存在script_version_id和script_id,那么使用script_content。优先级：script_version_id>script_id>script_content | +| timeout | int | 否 | 脚本超时时间，秒。默认7200，取值范围1-86400 | +| db_account_id | long | 是 | SQL执行的db账号ID，必填, 从账号管理-DB账号处获得。 | +| target_server | object | 否 | 目标服务器，见server定义 | +| callback_url | string | 否 | 回调URL，当任务执行完成后，JOB会调用该URL告知任务执行结果。回调协议参考callback_protocol组件文档 | + +##### server +| 字段 | 类型 | 必选 | 描述 | +| ------------------ | ----- | ---- | ----------------------------------- | +| host_id_list | array | 否 | 主机ID列表 | +| ip_list | array | 否 | ***不推荐使用，建议使用host_id_list参数***。主机IP 列表，定义见ip | +| dynamic_group_list | array | 否 | 动态分组列表，定义见dynamic_group | +| topo_node_list | array | 否 | 动态 topo 节点列表，定义见topo_node | + +##### ip + +| 字段 | 类型 | 必选 | 描述 | +| ----------- | ------ | ---- | -------- | +| bk_cloud_id | long | 是 | 管控区域ID | +| ip | string | 是 | IP地址 | + +##### dynamic_group + +| 字段 | 类型 | 必选 | 描述 | +| ---- | ------ | ---- | -------------- | +| id | string | 是 | CMDB动态分组ID | + +##### topo_node_list + +| 字段 | 类型 | 必选 | 描述 | +| --------- | ------ | ---- | ------------------------------------------------------------ | +| id | long | 是 | 动态topo节点ID，对应CMDB API 中的 bk_inst_id | +| node_type | string | 是 | 动态topo节点类型，对应CMDB API 中的 bk_obj_id,比如"module","set" | + +### 请求参数示例 + +- POST +```json +{ + "bk_scope_type": "biz", + "bk_scope_id": "1", + "script_version_id": 1, + "timeout": 1000, + "db_account_id": 32, + "target_server": { + "dynamic_group_list": [ + { + "id": "blo8gojho0skft7pr5q0" + } + ], + "host_id_list": [ + 101, + 102 + ], + "topo_node_list": [ + { + "id": 1000, + "node_type": "module" + } + ] + } +} +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "job_instance_name": "API Quick SQL Execution1524454292038", + "job_instance_id": 10000, + "step_instance_id": 10001 + } +} +``` + +### 返回结果参数说明 + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + +##### data + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| job_instance_id | long | 作业实例ID | +| job_instance_name | long | 作业实例名称 | +| step_instance_id | long | 步骤实例ID | diff --git a/docs/apidoc/bk-api-gateway/v3/zh/fast_transfer_file.md b/docs/apidoc/bk-api-gateway/v3/zh/fast_transfer_file.md new file mode 100644 index 0000000000..3fad83359c --- /dev/null +++ b/docs/apidoc/bk-api-gateway/v3/zh/fast_transfer_file.md @@ -0,0 +1,187 @@ +### 功能描述 +快速分发文件 + +### 请求参数 + +{{ bkapi_authorization_description }} + +#### Header参数 + +| 字段 | 类型 | 必选 | 描述 | +| --------------------- | ------ | ---- | ------------------------------------------------------------ | +| X-Bkapi-Authorization | string | 是 | 认证信息。详情参考[调用网关 API](https://github.com/TencentBlueKing/BKDocs/blob/master/ZH/7.0/APIGateway/apigateway/use-api/use-apigw-api.md) | +| Accept | string | 是 | 固定值。application/json | +| Content-Type | string | 是 | 固定值。application/json | + +#### Body参数 +| 字段 | 类型 | 必选 | 描述 | +|------------------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| bk_biz_id | long | 是 | *已弃用*。业务ID。此字段已被弃用并由字段bk_scope_type+bk_scope_id替换 | +| task_name | string | 否 | 自定义作业名称 | +| account_alias | string | 否 | 目标执行账号别名，可从账号页面获取，推荐使用。与account_id必须存在一个。当同时存在account_alias和account_id时，account_id优先。 | +| account_id | long | 否 | 目标执行账号ID，可从get_account_list接口获取。与account_alias必须存在一个。当同时存在account_alias和account_id时，account_id优先。 | +| file_target_path | string | 是 | 文件传输目标路径 | +| file_source_list | array | 是 | 源文件对象数组，见下面file_source定义 | +| timeout | int | 否 | 任务超时时间，秒，默认值为7200。取值范围1-86400。| +| download_speed_limit| int | 否 | 下载限速，单位MB。如果未传入该参数，表示不限速| +| upload_speed_limit| int | 否 | 上传限速，单位MB。如果未传入该参数，表示不限速| +| transfer_mode | int | 否 | 传输模式。1-严谨模式，2-强制模式。默认使用强制模式 | +| target_server | object | 否 | 目标服务器，见server定义 | +| callback_url | string | 否 | 回调URL，当任务执行完成后，JOB会调用该URL告知任务执行结果。回调协议参考callback_protocol组件文档 | +| rolling_config | object | 否 | 滚动配置，见rolling_config定义 | + +##### file_source +| 字段 | 类型 | 必选 | 描述 | +|---------------|------------|--------|------------| +| file_list | array | 是 | 支持多个文件，若文件源类型为服务器文件，填写源文件的绝对路径数组；若文件源类型为第三方文件源，COS文件源填写的路径为"bucket名称/文件路径"，例如：testbucket/test.txt | +| account | object | 是 | 文件源账号，见account定义，文件源类型为服务器文件源时必填，文件源类型为第三方文件源时无需填写 | +| server | object | 否 | 源文件服务器，见server定义 | +| file_type | int | 否 | 文件源类型，1：服务器文件，3：第三方文件源文件，不传默认为1 | +| file_source_id | int | 否 | file_type为3时，file_source_id与file_source_code选择一个填写，若都填写，优先使用file_source_id，第三方文件源Id，可从get_job_detail接口返回结果中的步骤详情获取 | +| file_source_code| string | 否 | file_type为3时，file_source_id与file_source_code选择一个填写，若都填写，优先使用file_source_id，第三方文件源标识，可从作业平台的文件分发页面->选择文件源文件弹框中获取 | + +##### account + +| 字段 | 类型 | 必选 | 描述 | +| ----- | ------ | ---- | ------------------------------------------------------------ | +| id | long | 否 | 源执行账号ID，可从get_account_list接口获取。与alias必须存在一个。当同时存在alias和id时，id优先。 | +| alias | string | 否 | 源执行账号别名，可从账号页面获取，推荐使用。与alias必须存在一个。当同时存在alias和id时，id优先。 | + +##### server +| 字段 | 类型 | 必选 | 描述 | +| ------------------ | ----- | ---- | ----------------------------------- | +| host_id_list | array | 否 | 主机ID列表 | +| ip_list | array | 否 | ***不推荐使用，建议使用host_id_list参数***。主机IP 列表，定义见ip | +| dynamic_group_list | array | 否 | 动态分组列表，定义见dynamic_group | +| topo_node_list | array | 否 | 动态 topo 节点列表，定义见topo_node | + +##### ip_list +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| bk_cloud_id | long | 是 | 管控区域ID | +| ip | string | 是 | IP地址 | + +##### topo_node_list +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| id | long | 是 | 动态topo节点ID，对应CMDB API 中的 bk_inst_id | +| node_type | string | 是 | 动态topo节点类型，对应CMDB API 中的 bk_obj_id,比如"module","set"| + +##### rolling_config + +| 字段 | 类型 | 必选 | 描述 | +| ---- | ------ | ---- | -------------- | +| expression | string | 是 | 滚动策略表达式 | +| mode | int | 是 | 滚动机制,1-执行失败则暂停；2-忽略失败，自动滚动下一批；3-人工确认 | + +### 请求参数示例 + +- POST +```json +{ + "bk_scope_type": "biz", + "bk_scope_id": "1", + "file_target_path": "/tmp/", + "transfer_mode": 1, + "file_source_list": [ + { + "file_list": [ + "/tmp/REGEX:[a-z]*.txt" + ], + "account": { + "id": 100 + }, + "server": { + "dynamic_group_list": [ + { + "id": "blo8gojho0skft7pr5q0" + }, + { + "id": "blo8gojho0sabc7priuy" + } + ], + "host_id_list": [ + 101, + 102 + ], + "topo_node_list": [ + { + "id": 1000, + "node_type": "module" + } + ] + }, + "file_type": 1 + }, + { + "file_list": [ + "testbucket/test.txt" + ], + "file_type": 3, + "file_source_id": 1 + }, + { + "file_list": [ + "testbucket/test2.txt" + ], + "file_type": 3, + "file_source_code": "testInnerCOS" + } + ], + "target_server": { + "dynamic_group_list": [ + { + "id": "blo8gojho0skft7pr5q0" + }, + { + "id": "blo8gojho0sabc7priuy" + } + ], + "host_id_list": [ + 103, + 104 + ], + "topo_node_list": [ + { + "id": 1000, + "node_type": "module" + } + ] + }, + "account_id": 101 +} +``` +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "job_instance_name": "API Quick Distribution File1521101427176", + "job_instance_id": 10000, + "step_instance_id": 10001 + } +} +``` + +### 返回结果参数说明 + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + +##### data + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| job_instance_id | long | 作业实例ID | +| job_instance_name | long | 作业实例名称 | +| step_instance_id | long | 步骤实例ID | diff --git a/docs/apidoc/bk-api-gateway/v3/zh/get_account_list.md b/docs/apidoc/bk-api-gateway/v3/zh/get_account_list.md new file mode 100644 index 0000000000..598389b374 --- /dev/null +++ b/docs/apidoc/bk-api-gateway/v3/zh/get_account_list.md @@ -0,0 +1,102 @@ +### 功能描述 + +查询业务下的执行账号列表 + +### 请求参数 + +{{ bkapi_authorization_description }} + +#### Header参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| X-Bkapi-Authorization | string | 是 | 认证信息。详情参考[调用网关 API](https://github.com/TencentBlueKing/BKDocs/blob/master/ZH/7.0/APIGateway/apigateway/use-api/use-apigw-api.md) | +| Accept | string | 是 | 固定值。application/json| +| Content-Type | string | 是 | 固定值。application/json| + +#### Query参数 + +| 字段 | 类型 | 必选 | 描述 | +|----------------------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| bk_biz_id | long | 是 | *已弃用*。业务ID。此字段已被弃用并由字段bk_scope_type+bk_scope_id替换 | +| category | int | 否 | 账号用途（1：系统账号，2：DB账号），不传则不区分 | +| account | string | 否 | 账号名称 | +| alias | string | 否 | 账号别名 | +| start | int | 否 | 分页记录起始位置，不传默认为0 | +| length | int | 否 | 单次返回最大记录数，最大1000，不传默认为20 | + +### 请求参数示例 + +- GET +```json +/api/v3/get_account_list?bk_scope_type=biz&bk_scope_id=1&category=1&account=aaa&alias=aaa&start=0&length=1 +``` + +### 返回结果示例 + +```json +{ + "code": 0, + "message": null, + "result": true, + "data": { + "start": 0, + "total": 12, + "data": [ + { + "id": 70, + "account": "aaa", + "alias": "aaa", + "category": 1, + "type": 1, + "os": "Linux", + "creator": "admin", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "create_time": 1614659536108, + "last_modify_user": "admin", + "last_modify_time": 1614659536116 + } + ], + "length": 1 + } +} +``` + +### 返回结果说明 + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + +##### data + +| 字段 | 类型 | 描述 | +|----------|-----------|-----------| +| start | int | 分页记录起始位置 | +| total | int | 查询结果总量 | +| data | long | 分页数据，见data.data定义 | + +##### data.data + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| id | long | 账号ID | +| bk_scope_type | string |资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| account | string | 账号名称 | +| alias | string | 账号别名 | +| category | int | 账号用途（1：系统账号，2：DB账号） | +| type | int | 账号类型（1：Linux，2：Windows，9：MySQL，10：Oracle，11：DB2）| +| db_system_account_id | long | 账号用途为DB账号时该字段生效，表示DB账号对应的系统账号ID | +| os | string | 账号用途为系统账号时该字段生效，账号对应的OS | +| creator | string | 创建人 | +| create_time | long | 创建时间Unix时间戳（ms） | +| last_modify_user | string | 最近一次修改人 | +| last_modify_time | long | 最近一次修改时间Unix时间戳（ms） | diff --git a/docs/apidoc/bk-api-gateway/v3/zh/get_credential_detail.md b/docs/apidoc/bk-api-gateway/v3/zh/get_credential_detail.md new file mode 100644 index 0000000000..249aa62f9c --- /dev/null +++ b/docs/apidoc/bk-api-gateway/v3/zh/get_credential_detail.md @@ -0,0 +1,77 @@ +### 功能描述 + +查询凭证详情 + +### 请求参数 + +{{ bkapi_authorization_description }} + +#### Header参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| X-Bkapi-Authorization | string | 是 | 认证信息。详情参考[调用网关 API](https://github.com/TencentBlueKing/BKDocs/blob/master/ZH/7.0/APIGateway/apigateway/use-api/use-apigw-api.md) | +| Accept | string | 是 | 固定值。application/json| +| Content-Type | string | 是 | 固定值。application/json| + +#### Query参数 + +| 字段 | 类型 | 必选 | 描述 | +|------------------------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| bk_biz_id | long | 是 | *已弃用*。业务ID。此字段已被弃用并由字段bk_scope_type+bk_scope_id替换 | +| id | String | 是 | 凭证ID | + +### 请求参数示例 + +- GET +```json +/api/v3/get_credential_detail?bk_scope_type=biz&bk_scope_id=2&id=xxx +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "", + "data": { + "bk_scope_type": "biz", + "bk_scope_id": "2", + "id": "xxx", + "name": "xxx", + "type": "xxx", + "description": "xxx", + "creator": "admin", + "create_time": 1712049204719, + "last_modify_user": "admin", + "last_modify_time": 1712049204719 + } +} +``` + +### 返回结果参数说明 + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + +##### data +| 字段 | 类型 | 描述 | +| ---------------- | ------ | ------------------------------------------------------------ | +| bk_scope_type | string | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| id | string | 凭证ID | +| name | string | 凭证名称 | +| type | string | 类型。可选值：APP_ID_SECRET_KEY，PASSWORD，USERNAME_PASSWORD，SECRET_KEY| +| description | string | 描述 | +| creator | string | 作业创建人账号 | +| create_time | long | 创建时间，Unix时间戳（ms） | +| last_modify_user | string | 作业修改人账号 | +| last_modify_time | long | 最后修改时间，Unix时间戳（ms） | diff --git a/docs/apidoc/bk-api-gateway/v3/zh/get_cron_detail.md b/docs/apidoc/bk-api-gateway/v3/zh/get_cron_detail.md new file mode 100644 index 0000000000..b96e23a4b1 --- /dev/null +++ b/docs/apidoc/bk-api-gateway/v3/zh/get_cron_detail.md @@ -0,0 +1,148 @@ +### 功能描述 + +查询定时作业详情 + +### 请求参数 + +{{ bkapi_authorization_description }} + +#### Header参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| X-Bkapi-Authorization | string | 是 | 认证信息。详情参考[调用网关 API](https://github.com/TencentBlueKing/BKDocs/blob/master/ZH/7.0/APIGateway/apigateway/use-api/use-apigw-api.md) | +| Accept | string | 是 | 固定值。application/json| +| Content-Type | string | 是 | 固定值。application/json| + +#### Query参数 + +| 字段 | 类型 | 必选 | 描述 | +|------------------------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| bk_biz_id | long | 是 | *已弃用*。业务ID。此字段已被弃用并由字段bk_scope_type+bk_scope_id替换 | +| id | long | 否 | 定时任务 ID | + +### 请求参数示例 + +- GET +```json +/api/v3/get_cron_detail?bk_scope_type=biz&bk_scope_id=1&id=1 +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "", + "data": { + "bk_scope_type": "biz", + "bk_scope_id": "1", + "job_plan_id": 100, + "id": 1, + "name": "test", + "status": 1, + "expression": "0/5 * * * ?", + "global_var_list": [ + { + "id": 436, + "name": "ip", + "server": { + "dynamic_group_list": [ + { + "id": "blo8gojho0skft7pr5q0" + }, + { + "id": "blo8gojho0sabc7priuy" + } + ], + "ip_list": [ + { + "bk_host_id": 101, + "bk_cloud_id": 0, + "ip": "127.0.0.1" + }, + { + "bk_host_id": 102, + "bk_cloud_id": 0, + "ip": "127.0.0.2" + } + ], + "topo_node_list": [ + { + "id": 1000, + "node_type": "module" + } + ] + } + }, + { + "id": 437, + "name": "text", + "value": "new String value" + } + ], + "creator": "admin", + "create_time": 1546272000000, + "last_modify_user": "admin", + "last_modify_time": 1577807999999 + } +} +``` + +### 返回结果参数说明 + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + +##### data +| 字段 | 类型 | 描述 | +|------------------|-----------|-----------| +| bk_scope_type | string |资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| job_plan_id | long | 执行方案 ID | +| id | long | 定时作业 ID | +| name | string | 定时作业名称 | +| status | int | 定时作业状态：1.已启动、2.已暂停 | +| expression | string | 定时任务crontab的定时规则，新建时必填，修改时选填。各字段含义为：分 时 日 月 周，如: 0/5 * * * ? 表示每5分钟执行一次 | +| global_var_list | array | 全局变量信息 | +| creator | string | 作业创建人账号 | +| create_time | long | 创建时间，Unix 时间戳 | +| last_modify_user | string | 作业修改人账号 | +| last_modify_time | long | 最后修改时间，Unix 时间戳 | + +##### global_var + +| 字段 | 类型 | 描述 | +|-----------|-----------|------------| +| id | long | 全局变量 id，唯一标识。如果 id 为空，那么使用 name 作为唯一标识 | +| name | string | 全局变量 name | +| value | string | 字符、密码、数组类型的全局变量的值 | +| server | object | 主机类型全局变量的值 | + +##### server +| 字段 | 类型 | 描述 | +|-----------------------|-------|------------| +| ip_list | array | 静态 IP 列表 | +| dynamic_group_list | array | 动态分组 列表 | +| topo_node_list | array | 动态 topo 节点列表 | + +##### ip + +| 字段 | 类型 | 描述 | +|-------------|---------|---------| +| bk_cloud_id | int | 管控区域ID | +| ip | string | IP地址 | + +##### topo_node +| 字段 | 类型 | 描述 | +|------------------|--------|------------| +| id | long | 动态topo节点ID，对应CMDB API 中的 bk_inst_id | +| node_type | string | 动态topo节点类型，对应CMDB API 中的 bk_obj_id,比如"module","set"| diff --git a/docs/apidoc/bk-api-gateway/v3/zh/get_cron_list.md b/docs/apidoc/bk-api-gateway/v3/zh/get_cron_list.md new file mode 100644 index 0000000000..b5818c16a7 --- /dev/null +++ b/docs/apidoc/bk-api-gateway/v3/zh/get_cron_list.md @@ -0,0 +1,95 @@ +### 功能描述 + +查询业务下定时作业信息 + +### 请求参数 + +{{ bkapi_authorization_description }} + +#### Header参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| X-Bkapi-Authorization | string | 是 | 认证信息。详情参考[调用网关 API](https://github.com/TencentBlueKing/BKDocs/blob/master/ZH/7.0/APIGateway/apigateway/use-api/use-apigw-api.md) | +| Accept | string | 是 | 固定值。application/json| +| Content-Type | string | 是 | 固定值。application/json| + +#### Query参数 + +| 字段 | 类型 | 必选 | 描述 | +|------------------------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| bk_biz_id | long | 是 | *已弃用*。业务ID。此字段已被弃用并由字段bk_scope_type+bk_scope_id替换 | +| name | string | 否 | 定时作业名称 | +| id | long | 否 | 定时任务 ID，如果存在则忽略其他筛选条件，只查询这个指定的作业信息 | +| status | int | 否 | 定时作业状态：1.已启动、2.已暂停 | +| creator | string | 否 | 定时作业创建人账号 | +| create_time_start | long | 否 | 创建起始时间，Unix 时间戳(s) | +| create_time_end | long | 否 | 创建结束时间，Unix 时间戳(s) | +| last_modify_user | string | 否 | 作业修改人账号 | +| last_modify_time_start | long | 否 | 最后修改起始时间，Unix 时间戳(s) | +| last_modify_time_end | long | 否 | 最后修改结束时间，Unix 时间戳(s) | +| start | int | 否 | 默认 0 表示从第 1 条记录开始返回 | +| length | int | 否 | 单次返回最大记录数，最大1000，不传默认为20 | + +### 请求参数示例 + +- GET +```json +/api/v3/get_cron_detail?bk_scope_type=biz&bk_scope_id=1&name=test&id=1000031&status=1&status=admin&create_time_start=1601371525&create_time_end=1617285956&last_modify_user=admin&last_modify_time_start=1601371525&last_modify_time_end=1617286227&start=0&length=1 +``` + +### 返回结果示例 + +```json +{ + "code": 0, + "result": true, + "data": { + "start": 0, + "total": 8, + "data": [ + { + "id": 1000031, + "name": "tylertest动态分组ID变更", + "status": 1, + "creator": "admin", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "job_plan_id": 1000193, + "expression": "* * * * * *", + "create_time": 1617285956, + "last_modify_user": "admin", + "last_modify_time": 1617286227 + } + ], + "length": 1 + } +} +``` + +### 返回结果说明 + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + +##### data +| 字段 | 类型 | 描述 | +|------------------|-----------|-----------| +| bk_scope_type | string |资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| job_plan_id | long | 执行方案 ID | +| id | long | 定时作业 ID | +| name | string | 定时作业名称 | +| status | int | 定时作业状态：1.已启动、2.已暂停 | +| expression | string | 定时任务crontab的定时规则，新建时必填，修改时选填。各字段含义为：分 时 日 月 周，如: 0/5 * * * ? 表示每5分钟执行一次 | +| creator | string | 作业创建人账号 | +| create_time | long | 创建时间，Unix 时间戳(s) | +| last_modify_user | string | 作业修改人账号 | +| last_modify_time | long | 最后修改时间，Unix 时间戳(s) | diff --git a/docs/apidoc/bk-api-gateway/v3/zh/get_file_source_detail.md b/docs/apidoc/bk-api-gateway/v3/zh/get_file_source_detail.md new file mode 100644 index 0000000000..bedc7baffe --- /dev/null +++ b/docs/apidoc/bk-api-gateway/v3/zh/get_file_source_detail.md @@ -0,0 +1,85 @@ +### 功能描述 + +查询文件源详情 + +### 请求参数 + +{{ bkapi_authorization_description }} + +#### Header参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| X-Bkapi-Authorization | string | 是 | 认证信息。详情参考[调用网关 API](https://github.com/TencentBlueKing/BKDocs/blob/master/ZH/7.0/APIGateway/apigateway/use-api/use-apigw-api.md) | +| Accept | string | 是 | 固定值。application/json| +| Content-Type | string | 是 | 固定值。application/json| + +#### Query参数 + +| 字段 | 类型 | 必选 | 描述 | +|------------------------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| bk_biz_id | long | 是 | *已弃用*。业务ID。此字段已被弃用并由字段bk_scope_type+bk_scope_id替换 | +| code | String | 是 | 文件源code | + +### 请求参数示例 + +- GET +```json +/api/v3/get_file_source_detail?bk_scope_type=biz&bk_scope_id=2&code=bkrepo-1 +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "", + "data": { + "bk_scope_type": "biz", + "bk_scope_id": "2", + "id": 10001, + "code": "bkrepo-1", + "alias": "xxx", + "status": 0, + "file_source_type_code": "BLUEKING_ARTIFACTORY", + "is_public": false, + "credential_id": "xxx", + "enable": true, + "creator": "admin", + "create_time": 1712050614742, + "last_modify_user": "admin", + "last_modify_time": 1712050614742 + } +} +``` + +### 返回结果参数说明 + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + +##### data +| 字段 | 类型 | 描述 | +| ---------------- | ------ | ------------------------------------------------------------ | +| bk_scope_type | string | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| id | int | 文件源ID | +| code | string | 文件源code | +| alias | string | 文件源别名 | +| status | int | 状态 | +| file_source_type_code | string | 文件源类型code。可选值：BLUEKING_ARTIFACTORY | +| is_public | bool | 是否为公用文件源 | +| credential_id | string | 凭证ID | +| enable | bool | 是否启用 | +| creator | string | 作业创建人账号 | +| create_time | long | 创建时间，Unix时间戳（ms） | +| last_modify_user | string | 作业修改人账号 | +| last_modify_time | long | 最后修改时间，Unix时间戳（ms） | diff --git a/docs/apidoc/bk-api-gateway/v3/zh/get_job_instance_global_var_value.md b/docs/apidoc/bk-api-gateway/v3/zh/get_job_instance_global_var_value.md new file mode 100644 index 0000000000..ed7ac3f2dd --- /dev/null +++ b/docs/apidoc/bk-api-gateway/v3/zh/get_job_instance_global_var_value.md @@ -0,0 +1,132 @@ +### 功能描述 + +获取作业实例全局变量的值 + +### 请求参数 + +{{ bkapi_authorization_description }} + +#### Header参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| X-Bkapi-Authorization | string | 是 | 认证信息。详情参考[调用网关 API](https://github.com/TencentBlueKing/BKDocs/blob/master/ZH/7.0/APIGateway/apigateway/use-api/use-apigw-api.md) | +| Accept | string | 是 | 固定值。application/json| +| Content-Type | string | 是 | 固定值。application/json| + +#### Query参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| bk_biz_id | long | 是 | *已弃用*。业务ID。此字段已被弃用并由字段bk_scope_type+bk_scope_id替换 | +| job_instance_id | long | 是 | 作业实例ID | + +### 请求参数示例 + +- GET +```json +/api/v3/get_job_instance_global_var_value?bk_scope_type=biz&bk_scope_id=1&job_instance_id=100 +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "", + "data": { + "job_instance_id": 100, + "step_instance_var_list": [{ + "step_instance_id": 292778, + "global_var_list": [{ + "name": "aa", + "value": "AA", + "type": 1 + }, { + "name": "password", + "value": "mypassword", + "type": 4 + } + ] + }, { + "step_instance_id": 292779, + "global_var_list": [{ + "name": "aa", + "value": "AAAA", + "type": 1 + }, { + "name": "password", + "value": "mypassword", + "type": 4 + } + ] + } + ] + } +} + +``` + +### 返回结果说明 + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + +##### data + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| job_instance_id | long | 作业实例ID | +| step_instance_var_list | array | 作业步骤实例全局变量值。定义见step_instance_var | + +##### step_instance_var + +| 字段 | 类型 | 描述 | +| ---------------- | ----- | -------------------------------- | +| step_instance_id | long | 步骤实例ID | +| global_var_list | array | 全局变量值列表，定义见global_var | + +##### global_var + +| 字段 | 类型 | 必选 | 描述 | +| ------ | ------ | ---- | ---------------------------------------------------------- | +| id | long | 否 | 全局变量id，唯一标识。如果id为空，那么使用name作为唯一标识 | +| name | string | 否 | 全局变量name | +| value | string | 否 | 字符、密码、数组类型的全局变量的值 | +| server | object | 否 | 主机类型全局变量的值，见server定义 | + +##### server + +| 字段 | 类型 | 必选 | 描述 | +| ------------------ | ----- | ---- | ----------------------------------- | +| ip_list | array | 否 | 静态 IP 列表，定义见ip | +| dynamic_group_list | array | 否 | 动态分组列表，定义见dynamic_group | +| topo_node_list | array | 否 | 动态 topo 节点列表，定义见topo_node | + +##### ip + +| 字段 | 类型 | 必选 | 描述 | +| ----------- | ------ | ---- | -------- | +| bk_cloud_id | int | 是 | 管控区域ID | +| ip | string | 是 | IP地址 | + +##### dynamic_group + +| 字段 | 类型 | 必选 | 描述 | +| ---- | ------ | ---- | -------------- | +| id | string | 是 | CMDB动态分组ID | + +##### topo_node + +| 字段 | 类型 | 必选 | 描述 | +| --------- | ------ | ---- | ------------------------------------------------------------ | +| id | long | 是 | 动态topo节点ID，对应CMDB API 中的 bk_inst_id | +| node_type | string | 是 | 动态topo节点类型，对应CMDB API 中的 bk_obj_id,比如"module","set" | \ No newline at end of file diff --git a/docs/apidoc/bk-api-gateway/v3/zh/get_job_instance_ip_log.md b/docs/apidoc/bk-api-gateway/v3/zh/get_job_instance_ip_log.md new file mode 100644 index 0000000000..59cd6fc2ba --- /dev/null +++ b/docs/apidoc/bk-api-gateway/v3/zh/get_job_instance_ip_log.md @@ -0,0 +1,143 @@ +### 功能描述 + +根据主机查询作业执行日志 + +### 请求参数 + +{{ bkapi_authorization_description }} + +#### Header参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| X-Bkapi-Authorization | string | 是 | 认证信息。详情参考[调用网关 API](https://github.com/TencentBlueKing/BKDocs/blob/master/ZH/7.0/APIGateway/apigateway/use-api/use-apigw-api.md) | +| Accept | string | 是 | 固定值。application/json| +| Content-Type | string | 是 | 固定值。application/json| + +#### Query参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| bk_biz_id | long | 是 | *已弃用*。业务ID。此字段已被弃用并由字段bk_scope_type+bk_scope_id替换 | +| job_instance_id | long | 是 | 作业实例ID | +| step_instance_id | long | 是 | 步骤实例ID | +| bk_cloud_id | int | 是 | 目标服务器管控区域ID | +| ip | string | 是 | 目标服务器IP | + +### 请求参数示例 +- GET +```json +/api/v3/get_job_instance_ip_log?bk_scope_type=biz&bk_scope_id=1&job_instance_id=50&step_instance_id=100&bk_cloud_id=0&ip=127.0.0.1 +``` + +### 返回结果示例 + +#### 脚本执行步骤 +```json +{ + "result": true, + "code": 0, + "message": "", + "data": { + "log_type": 1, + "bk_host_id": 101, + "ip": "127.0.0.1", + "bk_cloud_id": 0, + "log_content": "[2018-03-15 14:39:30][PID:56875] job_start\n" + } +} +``` + +#### 文件分发步骤 + +```json +{ + "result": true, + "code": 0, + "message": "", + "data": { + "log_type": 2, + "bk_host_id": 101, + "ip": "127.0.0.1", + "bk_cloud_id": 0, + "file_logs": [ + { + "mode": 1, + "src_ip": { + "host_id": 102, + "bk_cloud_id": 0, + "ip": "127.0.0.2" + }, + "src_path": "/data/1.log", + "dest_ip": { + "bk_host_id": 101, + "bk_cloud_id": 0, + "ip": "127.0.0.1" + }, + "dest_path": "/tmp/1.log", + "status": 4, + "log_content": "[2021-06-28 11:32:16] FileName: /tmp/1.log FileSize: 9.0 Bytes State: dest agent success download file Speed: 1 KB/s Progress: 100% StatusDesc: dest agent success download file Detail: success" + }, + { + "mode": 0, + "src_ip": { + "bk_host_id": 102, + "bk_cloud_id": 0, + "ip": "127.0.0.2" + }, + "src_path": "/data/1.log", + "status": 4, + "log_content": "[2021-06-28 11:32:16] FileName: /data/1.log FileSize: 9.0 Bytes State: source agent success upload file Speed: 1 KB/s Progress: 100% StatusDesc: source agent success upload file Detail: success upload" + } + ] + } +} +``` + +**文件任务返回结果说明** + +- 文件分发日志，除了目标服务器的文件下载任务日志，也会返回源服务器的文件上传任务日志(mode=0) +- dest_ip 与请求参数的bk_cloud_id/ip对应 + +### 返回结果说明 + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + +##### data + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| bk_host_id | long | 主机ID | +| bk_cloud_id | int | 目标服务器管控区域ID | +| ip | string | 目标服务器IP地址 | +| log_type | int | 日志类型。1-脚本执行任务日志;2-文件分发任务日志 | +| log_content | string | 作业脚本输出的日志内容 | +| file_logs | array | 文件分发任务日志。定义见file_log| + +##### file_log + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| mode | 分发模式 | 0:上传;1:下载| +| src_ip | object |文件源主机IP。定义见ip | +| src_path | string | 源文件路径 | +| dest_ip | object | 分发目标主机IP，mode=1时有值。定义见ip | +| dest_path | string | 目标路径，mode=1时有值 | +| status | int | 任务状态。1-等待开始;2-上传中;3-下载中;4-成功;5-失败 | +| log_content | string | 文件分发日志内容 | + +##### ip + +| 字段 | 类型 | 描述 | +|-----------|------------|--------| +| bk_host_id | long | 主机ID | +| bk_cloud_id | long | 管控区域ID | +| ip | string | IP地址 | diff --git a/docs/apidoc/bk-api-gateway/v3/zh/get_job_instance_list.md b/docs/apidoc/bk-api-gateway/v3/zh/get_job_instance_list.md new file mode 100644 index 0000000000..b8e9f43817 --- /dev/null +++ b/docs/apidoc/bk-api-gateway/v3/zh/get_job_instance_list.md @@ -0,0 +1,104 @@ +### 功能描述 + +查询作业实例列表（执行历史) + +### 请求参数 + +{{ bkapi_authorization_description }} + +#### Header参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| X-Bkapi-Authorization | string | 是 | 认证信息。详情参考[调用网关 API](https://github.com/TencentBlueKing/BKDocs/blob/master/ZH/7.0/APIGateway/apigateway/use-api/use-apigw-api.md) | +| Accept | string | 是 | 固定值。application/json| +| Content-Type | string | 是 | 固定值。application/json| + +#### Query参数 + +| 字段 | 类型 | 必选 | 描述 | +| ----------------- | ------ | ---- | ------------------------------------------------------------ | +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| bk_biz_id | long | 是 | *已弃用*。业务ID。此字段已被弃用并由字段bk_scope_type+bk_scope_id替换 | +| create_time_start | long | 是 | 创建起始时间，Unix 时间戳，单位毫秒 | +| create_time_end | long | 是 | 创建结束时间，Unix 时间戳，单位毫秒 | +| job_instance_id | long | 否 | 任务实例ID。 如果出入job_instance_id，将忽略其他查询条件 | +| job_cron_id | long | 否 | 定时任务ID | +| operator | string | 否 | 执行人，精准匹配 | +| name | string | 否 | 任务名称，模糊匹配 | +| launch_mode | int | 否 | 执行方式。1 - 页面执行，2 - API调用，3 - 定时执行 | +| type | int | 否 | 任务类型。0 - 作业执行，1 - 脚本执行，2 - 文件分发 | +| status | int | 否 | 任务状态。1 - 等待执行，2 - 正在执行，3 - 执行成功，4 - 执行失败，7 - 等待确认，10 - 强制终止中，11 - 强制终止成功，13 - 确认终止 | +| ip | string | 否 | 执行目标服务器IP, 精准匹配 | +| start | int | 否 | 默认0表示从第1条记录开始返回 | +| length | int | 否 | 单次返回最大记录数，最大1000，不传默认为20 | + +### 请求参数示例 + +- GET +```json +/api/v3/get_job_instance_list?bk_scope_type=biz&bk_scope_id=1&type=0&launch_mode=1&status=3&operator=admin&name=test&create_time_start=1546272000000&create_time_end=1577807999999&start=0&length=20 +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "data": [ + { + "bk_scope_type": "biz", + "bk_scope_id": "1", + "id": 100, + "job_template_id": 1, + "job_plan_id": 1, + "name": "test", + "operator": "admin", + "create_time": 1546272000000, + "start_time": 1546272000000, + "end_time": 1546272001000, + "total_time": 1000, + "launch_mode": 1, + "task_status": 3, + "task_type": 0 + } + ], + "start": 0, + "length": 20, + "total": 1 + } +} +``` + +### 返回结果说明 + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + +##### data + +| 字段 | 类型 | 描述 | +| --------------- | ------ | ------------------------------------------------------------ | +| bk_scope_type | string |资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| id | long | 执行方案 ID | +| job_template_id | long | 作业模版 ID，当任务为执行方案的时候有值 | +| job_plan_id | long | 作业执行方案 ID，当任务为执行方案的时候有值 | +| name | string | 任务名称 | +| operator | string | 操作者 | +| create_time | long | 创建时间，Unix 时间戳，单位毫秒 | +| status | int | 任务状态。1 - 等待执行，2 - 正在执行，3 - 执行成功，4 - 执行失败，7 - 等待确认，10 - 强制终止中，11 - 强制终止成功，13 - 确认终止 | +| type | int | 任务类型。0 - 作业执行，1 - 脚本执行，2 - 文件分发 | +| launch_mode | int | 执行方式。1 - 页面执行，2 - API调用，3 - 定时执行 | +| start_time | long | 任务启动时间，Unix 时间戳，单位毫秒 | +| end_time | long | 任务结束时间，Unix 时间戳，单位毫秒 | +| total_time | long | 任务执行时间，Unix 时间戳，单位毫秒 \ No newline at end of file diff --git a/docs/apidoc/bk-api-gateway/v3/zh/get_job_instance_status.md b/docs/apidoc/bk-api-gateway/v3/zh/get_job_instance_status.md new file mode 100644 index 0000000000..f06010ea56 --- /dev/null +++ b/docs/apidoc/bk-api-gateway/v3/zh/get_job_instance_status.md @@ -0,0 +1,145 @@ +### 功能描述 + +根据作业实例 ID 查询作业执行状态 + +### 请求参数说明 + +{{ bkapi_authorization_description }} + +#### Header参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| X-Bkapi-Authorization | string | 是 | 认证信息。详情参考[调用网关 API](https://github.com/TencentBlueKing/BKDocs/blob/master/ZH/7.0/APIGateway/apigateway/use-api/use-apigw-api.md) | +| Accept | string | 是 | 固定值。application/json| +| Content-Type | string | 是 | 固定值。application/json| + +#### Query参数 + +| 字段 | 类型 | 必选 | 描述 | +|------------------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| bk_biz_id | long | 是 | *已弃用*。业务ID。此字段已被弃用并由字段bk_scope_type+bk_scope_id替换 | +| job_instance_id | long | 是 | 作业实例ID | +| return_ip_result | boolean | 否 | 是否返回每个ip上的任务详情，对应返回结果中的step_ip_result_list。默认值为false。 | + +### 请求参数示例 + +- GET +```json +/api/v3/get_job_instance_status?bk_scope_type=biz&bk_scope_id=1&job_instance_id=100&return_ip_result=true +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "", + "data": { + "finished": true, + "job_instance": { + "job_instance_id": 100, + "bk_scope_type": "biz", + "bk_scope_id": "1", + "name": "API Quick execution script1521089795887", + "create_time": 1605064271000, + "status": 4, + "start_time": 1605064271000, + "end_time": 1605064272000, + "total_time": 1000 + }, + "step_instance_list": [ + { + "status": 4, + "total_time": 1000, + "name": "API Quick execution scriptxxx", + "step_instance_id": 75, + "execute_count": 0, + "create_time": 1605064271000, + "end_time": 1605064272000, + "type": 1, + "start_time": 1605064271000, + "step_ip_result_list": [ + { + "bk_host_id": 101, + "ip": "127.0.0.1", + "bk_cloud_id": 0, + "status": 9, + "tag": "", + "exit_code": 0, + "error_code": 0, + "start_time": 1605064271000, + "end_time": 1605064272000, + "total_time": 1000 + } + ] + } + ] + } +} +``` +### 返回结果说明 + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + +##### data + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| finished | bool | 作业是否结束 | +| job_instance | object | 作业实例基本信息。见job_instance定义 | +| step_instance_list | array | 作业步骤列表。见step_instance定义 | + +##### job_instance + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| name | string | 作业实例名称 | +| status | int | 作业状态码: 1.未执行; 2.正在执行; 3.执行成功; 4.执行失败; 5.跳过; 6.忽略错误; 7.等待用户; 8.手动结束; 9.状态异常; 10.步骤强制终止中; 11.步骤强制终止成功 | +| create_time | long | 作业创建时间，Unix时间戳，单位毫秒 | +| start_time | long | 开始执行时间，Unix时间戳，单位毫秒 | +| end_time | long | 执行结束时间，Unix时间戳，单位毫秒 | +| total_time | int | 总耗时，单位毫秒 | +| bk_scope_type | string |资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| job_instance_id | long | 作业实例ID | + +##### step_instance + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| step_instance_id | long | 作业步骤实例ID | +| type | int | 步骤类型：1.脚本步骤; 2.文件步骤; 4.SQL步骤 | +| name | string | 步骤名称 | +| status | int | 作业步骤状态码: 1.未执行; 2.正在执行; 3.执行成功; 4.执行失败; 5.跳过; 6.忽略错误; 7.等待用户; 8.手动结束; 9.状态异常; 10.步骤强制终止中; 11.步骤强制终止成功; 12.步骤强制终止失败 | +| create_time | long | 作业步骤实例创建时间，Unix时间戳，单位毫秒 | +| start_time | long | 开始执行时间，Unix时间戳，单位毫秒 | +| end_time | long | 执行结束时间，Unix时间戳，单位毫秒 | +| total_time | int | 总耗时，单位毫秒 | +| execute_count | int | 步骤重试次数 | +| step_ip_result_list | array | 每个主机的任务执行结果，定义见step_ip_result | + + +##### step_ip_result + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| bk_host_id | long | 主机ID | +| ip | string | IP | +| bk_cloud_id | long | 管控区域ID | +| status | int | 作业执行状态:1.Agent异常; 5.等待执行; 7.正在执行; 9.执行成功; 11.执行失败; 12.任务下发失败; 403.任务强制终止成功; 404.任务强制终止失败 | +| tag | string | 用户通过job_success/job_fail函数模板自定义输出的结果。仅脚本任务存在该参数 | +| exit_code | int | 脚本任务exit code | +| error_code | int | 主机任务状态码，1.Agent异常; 3.上次已成功; 5.等待执行; 7.正在执行; 9.执行成功; 11.任务失败; 12.任务下发失败; 13.任务超时; 15.任务日志错误; 101.脚本执行失败; 102.脚本执行超时; 103.脚本执行被终止; 104.脚本返回码非零; 202.文件传输失败; 203.源文件不存在; 310.Agent异常; 311.用户名不存在; 320.文件获取失败; 321.文件超出限制; 329.文件传输错误; 399.任务执行出错 | +| start_time | long | 开始执行时间，Unix时间戳，单位毫秒 | +| end_time | long | 执行结束时间，Unix时间戳，单位毫秒 | +| total_time | int | 总耗时，单位毫秒 | diff --git a/docs/apidoc/bk-api-gateway/v3/zh/get_job_plan_detail.md b/docs/apidoc/bk-api-gateway/v3/zh/get_job_plan_detail.md new file mode 100644 index 0000000000..f5e4017123 --- /dev/null +++ b/docs/apidoc/bk-api-gateway/v3/zh/get_job_plan_detail.md @@ -0,0 +1,312 @@ +### 功能描述 + +根据作业执行方案 ID 查询作业执行方案详情 + +### 请求参数 + +{{ bkapi_authorization_description }} + +#### Header参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| X-Bkapi-Authorization | string | 是 | 认证信息。详情参考[调用网关 API](https://github.com/TencentBlueKing/BKDocs/blob/master/ZH/7.0/APIGateway/apigateway/use-api/use-apigw-api.md) | +| Accept | string | 是 | 固定值。application/json| +| Content-Type | string | 是 | 固定值。application/json| + +#### Query参数 + +| 字段 | 类型 | 必选 | 描述 | +|-------------|-------------|--------|--------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| bk_biz_id | long | 是 | *已弃用*。业务ID。此字段已被弃用并由字段bk_scope_type+bk_scope_id替换 | +| job_plan_id | long | 是 | 作业执行方案 ID | + +### 请求参数示例 + +- GET +```json +/api/v3/get_job_plan_detail?bk_scope_type=biz&bk_scope_id=1&job_plan_id=100 +``` + +### 返回结果示例 +```json +{ + "result": true, + "code": 0, + "message": "", + "data": { + "bk_scope_type": "biz", + "bk_scope_id": "1", + "job_plan_id": 100, + "name": "test", + "creator": "admin", + "create_time": 1546272000000, + "last_modify_user": "admin", + "last_modify_time": 1577807999999, + "global_var_list": [ + { + "id": 11, + "type": 1, + "name": "varName", + "value": "value is Me", + "description": "hello", + "required": 1 + }, + { + "id": 12, + "type": 3, + "name": "servers", + "description": "", + "required": 0, + "server": { + "dynamic_group_list": [ + { + "id": "blo8gojho0skft7pr5q0" + }, + { + "id": "blo8gojho0sabc7priuy" + } + ], + "ip_list": [ + { + "bk_host_id": 101, + "bk_cloud_id": 0, + "ip": "127.0.0.1" + }, + { + "bk_host_id": 102, + "bk_cloud_id": 0, + "ip": "127.0.0.2" + } + ], + "topo_node_list": [ + { + "id": 1000, + "node_type": "module" + } + ] + } + } + ], + "step_list": [ + { + "id": 1059, + "type": 1, + "name": "run local script", + "script_info": { + "script_type": 1, + "script_timeout": 1000, + "script_content": "ZWNobyAkMSAkMiAkMw==", + "script_param": "YTEgYTIgYTM=", + "is_param_sensitive": 0, + "account": "root" + } + }, + { + "id": 1060, + "type": 1, + "name": "run cite script", + "script_info": { + "script_type": 2, + "script_id": "aaaaa-bbb-ccc-ddddd", + "script_version_id": 1078, + "script_timeout": 1000, + "script_param": "YTEgYTIgYTM=", + "is_param_sensitive": 1, + "account": "root" + } + }, + { + "id": 1061, + "type": 2, + "name": "xxx", + "file_info": { + "file_source": [ + { + "file_list": [ + "/tmp/REGEX:[a-z]*.txt" + ], + "server": { + "variable": "servers" + }, + "account": { + "id": 1, + "name": "root" + }, + "file_type": 1 + }, + { + "file_list": [ + "testbucket/test.txt" + ], + "file_type": 3, + "file_source_id": 1 + } + ], + "file_destination": { + "path": "/tmp/", + "account": { + "id": 1, + "name": "root" + }, + "server": { + "variable": "", + "dynamic_group_list": [ + { + "id": "blo8gojho0skft7pr5q0" + }, + { + "id": "blo8gojho0sabc7priuy" + } + ], + "ip_list": [ + { + "bk_host_id": 103, + "bk_cloud_id": 0, + "ip": "127.0.0.3" + }, + { + "bk_host_id": 104, + "bk_cloud_id": 0, + "ip": "127.0.0.4" + } + ], + "topo_node_list": [ + { + "id": 1000, + "node_type": "module" + } + ] + } + }, + "timeout": 60, + "transfer_mode": 1, + "upload_speed_limit": 1000, + "download_speed_limit": 1000 + } + } + ] + } +} +``` + +### 返回结果说明 + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + +##### data + +| 字段 | 类型 | 描述 | +|------------------|-----------|-----------| +| bk_scope_type | string |资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| job_plan_id | long | 执行方案 ID | +| name | string | 作业名称 | +| creator | string | 作业创建人账号 | +| create_time | long | 创建时间，Unix 时间戳 | +| last_modify_user | string | 作业修改人账号 | +| last_modify_time | long | 最后修改时间，Unix 时间戳 | +| step_list | array | 步骤对象 | +| global_var_list | array | 全局变量信息 | + +##### step + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| id | long | 作业步骤ID | +| name | string | 作业步骤名称 | +| type | int | 步骤类型：1.脚本步骤; 2.文件步骤; 3.人工确认步骤 | +| script_info | object | 脚本信息。当 type=1 时才有这个字段。 | +| file_info | object | 文件传输步骤信息。当 type=2 时才有这个字段 | + +##### script_info +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| script_type | int | 脚本类型：1 本地脚本 2 引用脚本 3 公共脚本 | +| script_id | string | 脚本 ID。当 script_type=2,3 时才有这个字段。 | +| script_version_id | long | 脚本版本 ID。当 script_type=2,3 时才有这个字段。 | +| script_content | string | 脚本内容。当 type=1 时才有这个字段。 | +| script_language | int | 脚本语言：1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - SQL | +| script_param | string | 脚本参数 | +| script_timeout | int | 脚本超时时间，秒。默认3600，取值范围60-86400 | +| is_param_sensitive | int | 是否敏感参数, 0.不是（默认），1.是。| +| account | object | 执行账号名/别名 | + +##### file_info +| 字段 | 类型 | 描述 | +|----------------------|--------|-----------| +| file_source_list | array | 源文件信息 | +| file_destination | object | 目标信息 | +| timeout | int | 文件传输超时设置 | +| transfer_mode | int | 文件传输模式 | +| upload_speed_limit | int | 上传限速 | +| download_speed_limit | int | 下载限速 | + +##### global_var + +| 字段 | 类型 | 描述 | +|-------------|-----------|------------| +| id | long | 全局变量 id，唯一标识。如果 id 为空，那么使用 name 作为唯一标识 | +| name | string | 全局变量名 | +| description | string | 全局变量描述 | +| type | int | 全局变量类型 | +| required | int | 模版/执行方案中该变量是否必填 | +| value | string | 字符、密码、数组类型的全局变量的值 | +| server | object | 主机类型全局变量的值 | +| used | bool | 变量是否被执行方案引用 | + +##### server +| 字段 | 类型 | 描述 | +|-----------------------|-------|------------| +| variable | string | 引用的变量名 | +| ip_list | array | 静态 IP 列表 | +| dynamic_group_list | array | 动态分组 ID 列表 | +| topo_node_list | array | 动态 topo 节点列表 | + +##### ip +| 字段 | 类型 | 描述 | +|-------------|---------|---------| +| bk_host_id | long | 主机ID | +| bk_cloud_id | int | 管控区域ID | +| ip | string | IP地址 | + +##### dynamic_group +| 字段 | 类型 | 描述 | +|-----|---------|------------| +| id | string | 动态分组 ID | + +##### topo_node +| 字段 | 类型 | 描述 | +|------------------|--------|------------| +| id | long | 动态topo节点ID，对应CMDB API 中的 bk_inst_id | +| node_type | string | 动态topo节点类型，对应CMDB API 中的 bk_obj_id,比如"module","set"| + +##### account +| 字段 | 类型 | 必选 | 描述 | +|-------|--------|--------|------------| +| id | long | 否 | 账号 ID | +| name | string | 否 | 账号名 | + +##### file_source +| 字段 | 类型 | 描述 | +|-----------------|---------|-----------| +| file_list | array | 源文件的绝对路径数组，支持多个文件 | +| account | object | 执行账号名/别名 | +| server | object | 目标服务器 | +| file_type | int | 文件源类型，1：服务器文件，2：本地文件，3：第三方文件源文件 | +| file_source_id | int | 文件源类型为第三方文件源文件时的第三方文件源Id | + +##### file_destination +| 字段 | 类型 | 描述 | +|-----------------|---------|-----------| +| path | string | 目标文件存放的路径 | +| account | object | 执行账号名/别名| +| server | object | 目标服务器 | diff --git a/docs/apidoc/bk-api-gateway/v3/zh/get_job_plan_list.md b/docs/apidoc/bk-api-gateway/v3/zh/get_job_plan_list.md new file mode 100644 index 0000000000..115677a60e --- /dev/null +++ b/docs/apidoc/bk-api-gateway/v3/zh/get_job_plan_list.md @@ -0,0 +1,92 @@ +### 功能描述 + +查询执行方案列表 + +### 请求参数 + +{{ bkapi_authorization_description }} + +#### Header参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| X-Bkapi-Authorization | string | 是 | 认证信息。详情参考[调用网关 API](https://github.com/TencentBlueKing/BKDocs/blob/master/ZH/7.0/APIGateway/apigateway/use-api/use-apigw-api.md) | +| Accept | string | 是 | 固定值。application/json| +| Content-Type | string | 是 | 固定值。application/json| + +#### Query参数 + +| 字段 | 类型 | 必选 | 描述 | +|------------------------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| bk_biz_id | long | 是 | *已弃用*。业务ID。此字段已被弃用并由字段bk_scope_type+bk_scope_id替换 | +| job_template_id | long | 否 | 作业模版 ID | +| creator | string | 否 | 作业执行方案创建人账号 | +| name | string | 否 | 作业执行方案名称，模糊匹配 | +| create_time_start | long | 否 | 创建起始时间，Unix 时间戳 | +| create_time_end | long | 否 | 创建结束时间，Unix 时间戳 | +| last_modify_user | string | 否 | 作业执行方案修改人账号 | +| last_modify_time_start | long | 否 | 最后修改起始时间，Unix 时间戳 | +| last_modify_time_end | long | 否 | 最后修改结束时间，Unix 时间戳 | +| start | int | 否 | 默认0表示从第1条记录开始返回 | +| length | int | 否 | 单次返回最大记录数，最大1000，不传默认为20 | + +### 请求参数示例 + +-GET +```json +/api/v3/get_job_plan_list?bk_scope_type=biz&bk_scope_id=1&job_template_id=1&creator=admin&name=test&create_time_start=1546272000000&create_time_end=1577807999999&start=0&length=20 +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "data": [ + { + "bk_scope_type": "biz", + "bk_scope_id": "1", + "id": 100, + "job_template_id": 1, + "name": "test", + "creator": "admin", + "create_time": 1546272000000, + "last_modify_user": "admin", + "last_modify_time": 1546272000000 + } + ], + "start": 0, + "length": 20, + "total": 1 + } +} +``` + +### 返回结果说明 + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + +##### data + +| 字段 | 类型 | 描述 | +|------------------|-----------|-----------| +| bk_scope_type | string |资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| id | long | 执行方案 ID | +| job_template_id | long | 作业模版 ID | +| name | string | 执行方案名称 | +| creator | string | 创建人账号 | +| create_time | long | 创建时间，Unix 时间戳 | +| last_modify_user | string | 修改人账号 | +| last_modify_time | long | 最后修改时间，Unix 时间戳 | diff --git a/docs/apidoc/bk-api-gateway/v3/zh/get_job_template_list.md b/docs/apidoc/bk-api-gateway/v3/zh/get_job_template_list.md new file mode 100644 index 0000000000..e9a15d1ec4 --- /dev/null +++ b/docs/apidoc/bk-api-gateway/v3/zh/get_job_template_list.md @@ -0,0 +1,89 @@ +### 功能描述 + +查询作业模版列表 + +### 请求参数 + +{{ bkapi_authorization_description }} + +#### Header参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| X-Bkapi-Authorization | string | 是 | 认证信息。详情参考[调用网关 API](https://github.com/TencentBlueKing/BKDocs/blob/master/ZH/7.0/APIGateway/apigateway/use-api/use-apigw-api.md) | +| Accept | string | 是 | 固定值。application/json| +| Content-Type | string | 是 | 固定值。application/json| + +#### Query参数 + +| 字段 | 类型 | 必选 | 描述 | +|------------------------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| bk_biz_id | long | 是 | *已弃用*。业务ID。此字段已被弃用并由字段bk_scope_type+bk_scope_id替换 | +| creator | string | 否 | 作业执行方案创建人账号 | +| name | string | 否 | 作业执行方案名称，模糊匹配 | +| create_time_start | long | 否 | 创建起始时间，Unix 时间戳 | +| create_time_end | long | 否 | 创建结束时间，Unix 时间戳 | +| last_modify_user | string | 否 | 作业执行方案修改人账号 | +| last_modify_time_start | long | 否 | 最后修改起始时间，Unix 时间戳 | +| last_modify_time_end | long | 否 | 最后修改结束时间，Unix 时间戳 | +| start | int | 否 | 默认0表示从第1条记录开始返回 | +| length | int | 否 | 单次返回最大记录数，最大1000，不传默认为20 | + +### 请求参数示例 + +- GET +```json +/api/v3/get_job_template_list?bk_scope_type=biz&bk_scope_id=1&creator=admin&start=0&length=20&create_time_start=1546272000000&create_time_end=1577807999999 +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "data": [ + { + "bk_scope_type": "biz", + "bk_scope_id": "1", + "id": 100, + "name": "test", + "creator": "admin", + "create_time": 1546272000000, + "last_modify_user": "admin", + "last_modify_time": 1546272000000 + } + ], + "start": 0, + "length": 20, + "total": 1 + } +} +``` + +### 返回结果说明 + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + +##### data + +| 字段 | 类型 | 描述 | +|------------------|-----------|-----------| +| bk_scope_type | string |资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| id | long | 作业模版 ID | +| name | string | 作业模版名称 | +| creator | string | 创建人账号 | +| create_time | long | 创建时间，Unix 时间戳 | +| last_modify_user | string | 修改人账号 | +| last_modify_time | long | 最后修改时间，Unix 时间戳 | diff --git a/docs/apidoc/bk-api-gateway/v3/zh/get_latest_service_version.md b/docs/apidoc/bk-api-gateway/v3/zh/get_latest_service_version.md new file mode 100644 index 0000000000..4a24308b8e --- /dev/null +++ b/docs/apidoc/bk-api-gateway/v3/zh/get_latest_service_version.md @@ -0,0 +1,54 @@ +### 功能描述 + +查询作业平台最新服务版本号 + +### 请求参数 + +{{ bkapi_authorization_description }} + +#### Header参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| X-Bkapi-Authorization | string | 是 | 认证信息。详情参考[调用网关 API](https://github.com/TencentBlueKing/BKDocs/blob/master/ZH/7.0/APIGateway/apigateway/use-api/use-apigw-api.md) | +| Accept | string | 是 | 固定值。application/json| +| Content-Type | string | 是 | 固定值。application/json| + +#### Query参数 + +无 + +### 请求参数示例 + +- GET +```json +/api/v3/get_latest_service_version +``` + +### 返回结果示例 +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "version": "3.6.4" + } +} +``` + +### 返回结果参数说明 + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + +##### data + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| version | string |服务最新版本号| diff --git a/docs/apidoc/bk-api-gateway/v3/zh/get_public_script_list.md b/docs/apidoc/bk-api-gateway/v3/zh/get_public_script_list.md new file mode 100644 index 0000000000..89b983353d --- /dev/null +++ b/docs/apidoc/bk-api-gateway/v3/zh/get_public_script_list.md @@ -0,0 +1,82 @@ +### 功能描述 + +查询公共脚本列表 + +### 请求参数 + +{{ bkapi_authorization_description }} + +#### Header参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| X-Bkapi-Authorization | string | 是 | 认证信息。详情参考[调用网关 API](https://github.com/TencentBlueKing/BKDocs/blob/master/ZH/7.0/APIGateway/apigateway/use-api/use-apigw-api.md) | +| Accept | string | 是 | 固定值。application/json| +| Content-Type | string | 是 | 固定值。application/json| + +#### Query参数 + +| 字段 | 类型 | 必选 | 描述 | +|----------------------|------------|--------|------------| +| name | string | 否 | 脚本名称，支持模糊查询 | +| script_language | int | 否 | 脚本语言。1：shell，2：bat，3：perl，4：python，5：powershell，6：sql。如果不传，默认返回所有脚本语言 | +| start | int | 否 | 分页记录起始位置，不传默认为0 | +| length | int | 否 | 单次返回最大记录数，最大1000，不传默认为20 | + +### 请求参数示例 + +- GET +```json +/api/v3/get_public_script_list?name=script1&script_language=1&start=0&length=10 +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "data": [ + { + "id": "000dbdddc06c453baf1f2decddf00c69", + "name": "script1", + "script_language": 1, + "public": false, + "online_script_version_id": 100, + "creator": "admin", + "create_time": 1600746078520, + "last_modify_user": "admin", + "last_modify_time": 1600746078520 + } + ], + "start": 0, + "length": 10, + "total": 1 + } +} +``` + +### 返回结果参数说明 + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + +##### data + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| id | string | 脚本ID | +| name | string | 脚本名称 | +| script_language | int | 脚本语言。1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - SQL | +| online_script_version_id | long | 已上线脚本版本ID;如果脚本没有已上线版本，该值为空 | +| creator | string | 创建人 | +| create_time | long | 创建时间Unix时间戳（ms） | +| last_modify_user| string | 最近一次修改人 | +| last_modify_time| long | 最近一次修改时间Unix时间戳（ms） | diff --git a/docs/apidoc/bk-api-gateway/v3/zh/get_public_script_version_detail.md b/docs/apidoc/bk-api-gateway/v3/zh/get_public_script_version_detail.md new file mode 100644 index 0000000000..80eba2e5ab --- /dev/null +++ b/docs/apidoc/bk-api-gateway/v3/zh/get_public_script_version_detail.md @@ -0,0 +1,83 @@ +### 功能描述 + +查询公共脚本版本详情 + +### 请求参数 + +{{ bkapi_authorization_description }} + +#### Header参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| X-Bkapi-Authorization | string | 是 | 认证信息。详情参考[调用网关 API](https://github.com/TencentBlueKing/BKDocs/blob/master/ZH/7.0/APIGateway/apigateway/use-api/use-apigw-api.md) | +| Accept | string | 是 | 固定值。application/json| +| Content-Type | string | 是 | 固定值。application/json| + +#### Query参数 + +| 字段 | 类型 | 必选 | 描述 | +|----------------------|------------|--------|------------| +| id | long | 否 | 脚本版本ID，若传入则以此条件为准屏蔽其他条件 | +| script_id | string | 否 | 脚本ID（可与version一起传入定位某个脚本版本） | +| version | string | 否 | 脚本版本（可与script_id一起传入定位某个脚本版本） | + +### 请求参数示例 + +- GET +```json +/api/v3/get_public_script_version_detail?id=1 +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "id": 1, + "script_id": "000dbdddc06c453baf1f2decddf00c69", + "version": "V1.0", + "content": "#!/bin/bash***", + "status": 1, + "version_desc": "版本描述", + "creator": "admin", + "create_time": 1600746078520, + "last_modify_user": "admin", + "last_modify_time": 1600746078520, + "script_language": 1, + "public_script": true, + "description": "脚本描述" + } +} +``` + +### 返回结果说明 + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + +##### data + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| id | long | 脚本版本ID | +| script_id | string | 脚本版本所属的脚本ID | +| version | string | 版本号 | +| content | string | 脚本版本内容 | +| status | int | 脚本版本状态（0：未上线，1：已上线，2：已下线，3：已禁用） | +| version_desc | string | 版本描述 | +| creator | string | 创建人 | +| create_time | long | 创建时间Unix时间戳（ms） | +| last_modify_user | string | 最近一次修改人 | +| last_modify_time | long | 最近一次修改时间Unix时间戳（ms） | +| script_language | int | 脚本语言:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell | +| public_script | bool | 是否公共脚本 | +| description | string | 脚本描述 | diff --git a/docs/apidoc/bk-api-gateway/v3/zh/get_public_script_version_list.md b/docs/apidoc/bk-api-gateway/v3/zh/get_public_script_version_list.md new file mode 100644 index 0000000000..5175916c24 --- /dev/null +++ b/docs/apidoc/bk-api-gateway/v3/zh/get_public_script_version_list.md @@ -0,0 +1,91 @@ +### 功能描述 + +查询公共脚本版本列表 + +### 请求参数 + +{{ bkapi_authorization_description }} + +#### Header参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| X-Bkapi-Authorization | string | 是 | 认证信息。详情参考[调用网关 API](https://github.com/TencentBlueKing/BKDocs/blob/master/ZH/7.0/APIGateway/apigateway/use-api/use-apigw-api.md) | +| Accept | string | 是 | 固定值。application/json| +| Content-Type | string | 是 | 固定值。application/json| + +#### Query参数 + +| 字段 | 类型 | 必选 | 描述 | +|----------------------|------------|--------|------------| +| script_id | string | 是 | 脚本ID | +| return_script_content | bool | 否 | 是否需要返回脚本内容。true:返回脚本内容；false：不返回脚本内容。默认为false。 | +| start | int | 否 | 分页记录起始位置，不传默认为0 | +| length | int | 否 | 单次返回最大记录数，最大1000，不传默认为20 | + +### 请求参数示例 + +- GET +```json +/api/v3/get_public_script_version_list?script_id=000dbdddc06c453baf1f2decddf00c69&return_script_content=true&start=0&length=10 +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "data": [ + { + "id": 1, + "script_id": "000dbdddc06c453baf1f2decddf00c69", + "version": "V1.0", + "content": "#!/bin/bash***", + "status": 1, + "version_desc": "版本描述", + "creator": "admin", + "create_time": 1600746078520, + "last_modify_user": "admin", + "last_modify_time": 1600746078520, + "script_language": 1, + "public_script": true, + "description": "脚本描述" + } + ], + "start": 0, + "length": 10, + "total": 1 + } +} +``` + +### 返回结果说明 + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + +##### data + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| id | long | 脚本版本ID | +| script_id | string | 脚本版本所属的脚本ID | +| version | string | 版本号 | +| content | string | 脚本内容 | +| status | int | 脚本版本状态（0：未上线，1：已上线，2：已下线，3：已禁用） | +| version_desc | string | 版本描述 | +| creator | string | 创建人 | +| create_time | long | 创建时间Unix时间戳（ms） | +| last_modify_user | string | 最近一次修改人 | +| last_modify_time | long | 最近一次修改时间Unix时间戳（ms） | +| script_language | int | 脚本语言:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell | +| public_script | bool | 是否公共脚本 | +| description | string | 脚本描述 | diff --git a/docs/apidoc/bk-api-gateway/v3/zh/get_script_list.md b/docs/apidoc/bk-api-gateway/v3/zh/get_script_list.md new file mode 100644 index 0000000000..f34968276f --- /dev/null +++ b/docs/apidoc/bk-api-gateway/v3/zh/get_script_list.md @@ -0,0 +1,88 @@ +### 功能描述 + +查询业务脚本列表 + +### 请求参数 + +{{ bkapi_authorization_description }} + +#### Header参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| X-Bkapi-Authorization | string | 是 | 认证信息。详情参考[调用网关 API](https://github.com/TencentBlueKing/BKDocs/blob/master/ZH/7.0/APIGateway/apigateway/use-api/use-apigw-api.md) | +| Accept | string | 是 | 固定值。application/json| +| Content-Type | string | 是 | 固定值。application/json| + +#### Query参数 + +| 字段 | 类型 | 必选 | 描述 | +|----------------------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| bk_biz_id | long | 是 | *已弃用*。业务ID。此字段已被弃用并由字段bk_scope_type+bk_scope_id替换 | +| name | string | 否 | 脚本名称，支持模糊查询 | +| script_language | int | 否 | 脚本语言。0：所有脚本类型，1：shell，2：bat，3：perl，4：python，5：powershell，6：sql。默认值为0 | +| start | int | 否 | 分页记录起始位置，不传默认为0 | +| length | int | 否 | 单次返回最大记录数，最大1000，不传默认为20 | + +### 请求参数示例 + +- GET +```json +/api/v3/get_script_list?bk_scope_type=biz&bk_scope_id=1&name=script1&script_language=1&start=0&length=10 +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "data": [ + { + "id": "000dbdddc06c453baf1f2decddf00c69", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "name": "a.sh", + "script_language": 1, + "online_script_version_id": 100, + "creator": "admin", + "create_time": 1600746078520, + "last_modify_user": "admin", + "last_modify_time": 1600746078520 + } + ], + "start": 0, + "length": 10, + "total": 1 + } +} +``` + +### 返回结果参数说明 + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + +##### data + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| id | string | 脚本ID | +| bk_scope_type | string |资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| name | string | 脚本名称 | +| script_language | int | 脚本语言。1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - SQL | +| online_script_version_id | long | 已上线脚本版本ID;如果脚本没有已上线版本，该值为空 | +| creator | string | 创建人 | +| create_time | long | 创建时间Unix时间戳（ms） | +| last_modify_user| string | 最近一次修改人 | +| last_modify_time| long | 最近一次修改时间Unix时间戳（ms） | diff --git a/docs/apidoc/bk-api-gateway/v3/zh/get_script_version_detail.md b/docs/apidoc/bk-api-gateway/v3/zh/get_script_version_detail.md new file mode 100644 index 0000000000..7b850c8b81 --- /dev/null +++ b/docs/apidoc/bk-api-gateway/v3/zh/get_script_version_detail.md @@ -0,0 +1,90 @@ +### 功能描述 + +查询业务脚本版本详情 + +### 请求参数 + +{{ bkapi_authorization_description }} + +#### Header参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| X-Bkapi-Authorization | string | 是 | 认证信息。详情参考[调用网关 API](https://github.com/TencentBlueKing/BKDocs/blob/master/ZH/7.0/APIGateway/apigateway/use-api/use-apigw-api.md) | +| Accept | string | 是 | 固定值。application/json| +| Content-Type | string | 是 | 固定值。application/json| + +#### Query参数 + +| 字段 | 类型 | 必选 | 描述 | +|----------------------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| bk_biz_id | long | 是 | *已弃用*。业务ID。此字段已被弃用并由字段bk_scope_type+bk_scope_id替换 | +| id | long | 否 | 脚本版本ID，若传入则以此条件为准屏蔽其他条件 | +| script_id | string | 否 | 脚本ID（可与version一起传入定位某个脚本版本） | +| version | string | 否 | 脚本版本（可与script_id一起传入定位某个脚本版本） | + +### 请求参数示例 + +- GET +```json +/api/v3/get_script_version_detail?bk_scope_type=biz&bk_scope_id=1&id=1 +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "id": 1, + "bk_scope_type": "biz", + "bk_scope_id": "1", + "script_id": "000dbdddc06c453baf1f2decddf00c69", + "version": "V1.0", + "content": "#!/bin/bash***", + "status": 1, + "version_desc": "版本描述", + "creator": "admin", + "create_time": 1600746078520, + "last_modify_user": "admin", + "last_modify_time": 1600746078520, + "script_language": 1, + "public_script": false, + "description": "脚本描述" + } +} +``` + +### 返回结果说明 + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + +##### data + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| id | long | 脚本版本ID | +| bk_scope_type | string |资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| script_id | string | 脚本版本所属的脚本ID | +| version | string | 版本号 | +| content | string | 脚本版本内容 | +| status | int | 脚本版本状态（0：未上线，1：已上线，2：已下线，3：已禁用） | +| version_desc | string | 版本描述 | +| creator | string | 创建人 | +| create_time | long | 创建时间Unix时间戳（ms） | +| last_modify_user | string | 最近一次修改人 | +| last_modify_time | long | 最近一次修改时间Unix时间戳（ms） | +| script_language | int | 脚本语言:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell | +| public_script | bool | 是否公共脚本 | +| description | string | 脚本描述 | diff --git a/docs/apidoc/bk-api-gateway/v3/zh/get_script_version_list.md b/docs/apidoc/bk-api-gateway/v3/zh/get_script_version_list.md new file mode 100644 index 0000000000..2fb742357a --- /dev/null +++ b/docs/apidoc/bk-api-gateway/v3/zh/get_script_version_list.md @@ -0,0 +1,98 @@ +### 功能描述 + +查询业务脚本版本列表 + +### 请求参数 + +{{ bkapi_authorization_description }} + +#### Header参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| X-Bkapi-Authorization | string | 是 | 认证信息。详情参考[调用网关 API](https://github.com/TencentBlueKing/BKDocs/blob/master/ZH/7.0/APIGateway/apigateway/use-api/use-apigw-api.md) | +| Accept | string | 是 | 固定值。application/json| +| Content-Type | string | 是 | 固定值。application/json| + +#### Query参数 + +| 字段 | 类型 | 必选 | 描述 | +|----------------------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| bk_biz_id | long | 是 | *已弃用*。业务ID。此字段已被弃用并由字段bk_scope_type+bk_scope_id替换 | +| script_id | string | 是 | 脚本ID | +| return_script_content | bool | 否 | 是否需要返回脚本内容。true:返回脚本内容；false：不返回脚本内容。默认为false。 | +| start | int | 否 | 分页记录起始位置，不传默认为0 | +| length | int | 否 | 单次返回最大记录数，最大1000，不传默认为20 | + +### 请求参数示例 + +- GET +```json +/api/v3/get_script_version_list?bk_scope_type=biz&bk_scope_id=1&script_id=000dbdddc06c453baf1f2decddf00c69&return_script_content=true&start=0&length=10 +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "data": [ + { + "id": 1, + "bk_scope_type": "biz", + "bk_scope_id": "1", + "script_id": "000dbdddc06c453baf1f2decddf00c69", + "version": "V1.0", + "content": "#!/bin/bash", + "status": 1, + "version_desc": "版本描述", + "creator": "admin", + "create_time": 1600746078520, + "last_modify_user": "admin", + "last_modify_time": 1600746078520, + "script_language": 1, + "public_script": false, + "description": "脚本描述" + } + ], + "start": 0, + "length": 10, + "total": 1 + } +} +``` + +### 返回结果说明 + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + +##### data + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| id | long | 脚本版本ID | +| bk_scope_type | string |资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| script_id | string | 脚本版本所属的脚本ID | +| version | string | 版本号 | +| content | string | 脚本内容 | +| status | int | 脚本版本状态（0：未上线，1：已上线，2：已下线，3：已禁用） | +| version_desc | string | 版本描述 | +| creator | string | 创建人 | +| create_time | long | 创建时间Unix时间戳（ms） | +| last_modify_user | string | 最近一次修改人 | +| last_modify_time | long | 最近一次修改时间Unix时间戳（ms） | +| script_language | int | 脚本语言:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell | +| public_script | bool | 是否公共脚本 | +| description | string | 脚本描述 | diff --git a/docs/apidoc/bk-api-gateway/v3/zh/get_step_instance_detail.md b/docs/apidoc/bk-api-gateway/v3/zh/get_step_instance_detail.md new file mode 100644 index 0000000000..178b9a3d67 --- /dev/null +++ b/docs/apidoc/bk-api-gateway/v3/zh/get_step_instance_detail.md @@ -0,0 +1,268 @@ +### 功能描述 + +根据步骤实例 ID 查询步骤实例详情 + +### 请求参数说明 + +{{ bkapi_authorization_description }} + +#### Header参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| X-Bkapi-Authorization | string | 是 | 认证信息。详情参考[调用网关 API](https://github.com/TencentBlueKing/BKDocs/blob/master/ZH/7.0/APIGateway/apigateway/use-api/use-apigw-api.md) | +| Accept | string | 是 | 固定值。application/json| +| Content-Type | string | 是 | 固定值。application/json| + +#### Query参数 + +| 字段 | 类型 | 必选 | 描述 | +|------------------------|-----------|-------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| job_instance_id | long | 是 | 作业实例ID | +| step_instance_id | long | 是 | 步骤实例ID | + + +### 请求参数示例 + +- GET +```json +/api/v3/get_step_instance_detail?bk_scope_type=biz&bk_scope_id=1&job_instance_id=100&step_instance_id=100 +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "", + "data": { + "id": 4, + "name": "Step 1", + "type": 1, + "script_info": { + "script_type": 1, + "script_id": null, + "script_version_id": null, + "script_content": "#!/bin/bash\n\nanynowtime=\"date +'%Y-%m-%d %H:%M:%S'\"\nNOW=\"echo [\\`$anynowtime\\`][PID:$$]\"\n\n##### 可在脚本开始运行时调用，打印当时的时间戳及PID。\nfunction job_start\n{\n echo \"`eval $NOW` job_starts\"\n}\n\n##### 可在脚本执行成功的逻辑分支处调用，打印当时的时间戳及PID。 \nfunction job_success\n{\n MSG=\"$*\"\n echo \"`eval $NOW` job_success:[$MSG]\"\n exit 0\n}\n\n##### 可在脚本执行失败的逻辑分支处调用，打印当时的时间戳及PID。\nfunction job_fail\n{\n MSG=\"$*\"\n echo \"`eval $NOW` job_fail:[$MSG]\"\n exit 1\n}\n\njob_start\n\n###### 作业平台中执行脚本成功和失败的标准只取决于脚本最后一条执行语句的返回值\n###### 如果返回值为0，则认为此脚本执行成功，如果非0，则认为脚本执行失败\n###### 可在此处开始编写您的脚本逻辑代码\n\necho 1", + "script_language": 1, + "script_param": "1 2 3", + "script_timeout": 1000, + "account": { + "id": 123, + "name": "root" + }, + "server": { + "variable": null, + "ip_list": [{ + "bk_host_id": 1, + "bk_cloud_id": 0, + "bk_cloud_name": "Default Area", + "ip": "192.168.1.1", + "ipv6": null, + "bk_agent_id": "020000000052540079809816967513755991", + "alive": 1 + }], + "topo_node_list": [{ + "node_type": "set", + "id": 123 + }], + "dynamic_group_list": [{ + "id": "07f99504-7bcb-11eb-980b-5254008ed702" + }] + }, + "is_param_sensitive": 0, + "is_ignore_error": 0 + }, + "file_info": { + "file_source_list": [ + { + "file_type": 1, + "file_list": [ + "/tmp/1.txt", + "/tmp/2.txt" + ], + "server": { + "variable": null, + "server": { + "ip_list": [{ + "bk_host_id": 1, + "bk_cloud_id": 0, + "bk_cloud_name": "Default Area", + "ip": "192.168.1.1", + "ipv6": null, + "bk_agent_id": "020000000052540079809816967513755991", + "alive": 1 + }], + "topo_node_list": [{ + "node_type": "set", + "id": 123 + }], + "dynamic_group_list": [{ + "id": "07f99504-7bcb-11eb-980b-5254008ed702" + }] + } + }, + "account": { + "id": 1, + "name": "root" + }, + "file_source_id": 1 + } + ], + "file_destination": { + "path": "/tmp", + "account": { + "id": 1, + "name": "root" + }, + "server": { + "variable": null, + "ip_list": [{ + "bk_host_id": 1, + "bk_cloud_id": 0, + "bk_cloud_name": "Default Area", + "ip": "192.168.1.1", + "ipv6": null, + "bk_agent_id": "020000000052540079809816967513755991", + "alive": 1 + }], + "topo_node_list": [{ + "node_type": "set", + "id": 123 + }], + "dynamic_group_list": [{ + "id": "07f99504-7bcb-11eb-980b-5254008ed702" + }] + } + }, + "timeout": 1000, + "source_speed_limit": 10, + "destination_speed_limit": 10, + "transfer_mode": 2, + "is_ignore_error": 0 + }, + "approval_info": { + "approval_message": "Pass" + } + } +} +``` +### 返回结果说明 + +| 字段 | 类型 | 是否一定存在 | 描述 | +|--------------|--------|------------|-------------------------------| +| result | bool | 是 |请求成功与否。true:请求成功；false请求失败 | +| code | int | 是 |错误编码。 0表示success，>0表示失败错误 | +| message | string | 是 |请求失败返回的错误信息| +| data | object | 否 |请求返回的数据| +| permission | object | 否 |权限信息| + +##### data + +| 字段 | 类型 | 是否一定存在 |描述 | +|----------------------|-----------|------------|-----------| +| id | long | 是 | 作业步骤ID | +| name | string | 是 | 步骤名称 | +| type | int | 是 | 步骤类型：1-脚本，2-文件，3-人工确认 | +| script_info | object | 否 | 脚本步骤信息 | +| file_info | object | 否 | 文件步骤信息 | +| approval_info | object | 否 | 审批步骤信息 | + + +##### script_info + +| 字段 | 类型 | 是否一定存在 | 描述 | +|--------------------|-----------|------------|-----------| +| script_type | int | 是 | 脚本类型：1-本地脚本，2-引用业务脚本，3-引用公共脚本 | +| script_id | string | 否 | 脚本ID | +| script_version_id | long | 否 | 脚本版本ID | +| script_content | string | 否 | 脚本内容 | +| script_language | int | 是 | 脚本语言：1-shell，2-bat，3-perl，4-python，5-powershell，6-sql | +| script_param | string | 是 | 脚本参数 | +| script_timeout | int | 是 | 脚本超时时间，单位为秒 | +| account | object | 是 | 执行账号，详情见account对象定义 | +| server | object | 是 | 执行目标机器，详情见server对象定义 | +| is_param_sensitive | int | 是 | 参数是否为敏感参数：0-不敏感，1-敏感 | +| is_ignore_error | int | 是 | 是否忽略错误：0-不忽略，1-忽略 | + +##### account + +| 字段 | 类型 | 是否一定存在 | 描述 | +|--------------------|-----------|------------|-----------| +| id | long | 是 | 账号ID | +| name | string | 否 | 账号名称 | + +##### server + +| 字段 | 类型 | 是否一定存在 | 描述 | +|----------------------|----------------------|------------|-----------| +| variable | string | 否 | 引用的全局变量名称 | +| ip_list | list | 否 | 主机列表，元素详情见host对象定义 | +| topo_node_list | list | 否 | 拓扑节点列表，元素详情见topo_node对象定义 | +| dynamic_group_list | list | 否 | 动态分组列表，元素详情见dynamic_group对象定义 | + +##### host + +| 字段 | 类型 | 是否一定存在 | 描述 | +|---------------|-----------|------------|-----------| +| bk_host_id | long | 是 | 主机ID | +| bk_cloud_id | long | 否 | 管控区域ID | +| bk_cloud_name | string | 否 | 管控区域名称 | +| ip | string | 否 | IP | +| ipv6 | string | 否 | IPv6 | +| bk_agent_id | string | 否 | Agent ID | +| alive | int | 否 | Agent是否正常，取值为：1-正常，0-异常 | + +##### topo_node + +| 字段 | 类型 | 是否一定存在 | 描述 | +|-------------|-----------|------------|-----------| +| node_type | string | 是 | 动态topo节点类型，对应CMDB API中的 bk_obj_id，例如module、set等 | +| id | int | 是 | 动态topo节点ID，对应CMDB API中的 bk_inst_id | + +##### dynamic_group + +| 字段 | 类型 | 是否一定存在 | 描述 | +|-------------|-----------|------------|---------------| +| id | string | 是 | CMDB动态分组ID | + + +##### file_info + +| 字段 | 类型 | 是否一定存在 | 描述 | +|-------------------------|-------------------|------------|-----------| +| file_source_list | list | 是 | 源文件列表，元素详情见file_source对象定义 | +| file_destination | object | 是 | 目标信息，详情见file_destination对象定义 | +| timeout | int | 是 | 超时，单位为秒 | +| source_speed_limit | int | 否 | 上传文件限速，单位为MB/s，没有值表示不限速 | +| destination_speed_limit | int | 否 | 下载文件限速，单位为MB/s，没有值表示不限速 | +| transfer_mode | int | 是 | 传输模式： 1-严谨模式，2-强制模式，3-安全模式 | +| is_ignore_error | int | 是 | 是否忽略错误：0-不忽略，1-忽略 | + +##### file_source + +| 字段 | 类型 | 是否一定存在 | 描述 | +|-------------------|--------------|------------|-----------| +| file_type | int | 是 | 文件类型：1-服务器文件，2-本地文件，3-文件源文件 | +| file_list | list | 是 | 文件路径列表 | +| server | object | 否 | 源文件所在机器，详情见server对象定义 | +| account | object | 是 | 执行账号，详情见account对象定义 | +| file_source_id | long | 否 | 第三方文件源ID | + +##### file_destination + +| 字段 | 类型 | 是否一定存在 | 描述 | +|-----------------|-----------|------------|-----------| +| path | string | 是 | 目标路径 | +| account | object | 是 | 执行账号，详情见account对象定义 | +| server | object | 是 | 分发目标机器，详情见server对象定义 | + + +##### approval_info + +| 字段 | 类型 | 是否一定存在 | 描述 | +|-------------------|-----------|------------|-----------| +| approval_message | string | 是 | 确认消息 | diff --git a/docs/apidoc/bk-api-gateway/v3/zh/get_step_instance_status.md b/docs/apidoc/bk-api-gateway/v3/zh/get_step_instance_status.md new file mode 100644 index 0000000000..c1bc86a304 --- /dev/null +++ b/docs/apidoc/bk-api-gateway/v3/zh/get_step_instance_status.md @@ -0,0 +1,161 @@ +### 功能描述 + +根据步骤实例 ID 查询步骤执行状态 + +### 请求参数说明 + +{{ bkapi_authorization_description }} + +#### Header参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| X-Bkapi-Authorization | string | 是 | 认证信息。详情参考[调用网关 API](https://github.com/TencentBlueKing/BKDocs/blob/master/ZH/7.0/APIGateway/apigateway/use-api/use-apigw-api.md) | +| Accept | string | 是 | 固定值。application/json| +| Content-Type | string | 是 | 固定值。application/json| + +#### Query参数 + +| 字段 | 类型 | 必选 | 描述 | +|------------------------|-----------|-------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| job_instance_id | long | 是 | 作业实例ID | +| step_instance_id | long | 是 | 步骤实例ID | +| execute_count | int | 否 | 步骤重试次数，从0开始计数，不传表示获取最近一次重试的数据。 | +| batch | int | 否 | 滚动批次，从0开始计数，默认值为null表示获取所有批次的数据。 | +| max_host_num_per_group | int | 否 | 每个分组（按照status与tag进行分组）里的最大主机数量，不传则返回全量数据。 | +| keyword | string | 否 | 日志搜索关键字，只返回日志中包含该关键字的主机执行状态数据（注意：仅脚本步骤支持） | +| search_ip | string | 否 | 主机IP/IPv6搜索关键字，只返回主机IP/IPv6地址包含该关键字的主机执行状态数据 | +| status | int | 否 | 执行状态，对应返回值中的status字段，传入后则只有执行状态匹配分组下才返回主机执行状态数据，其余分组下只返回主机数量等基本分组信息。可能的取值：0-未知错误，1-Agent异常，2-无效主机，3-上次已成功，5-等待执行，7-正在执行，9-执行成功，11-执行失败，12-任务下发失败，13-任务超时，15-任务日志错误，16-GSE脚本日志超时，17-GSE文件日志超时，101-脚本执行失败，102-脚本执行超时，103-脚本执行被终止，104-脚本返回码非零，202-文件传输失败，203-源文件不存在，301-文件任务系统错误-未分类的，303-文件任务超时，310-Agent异常，311-用户名不存在，312-用户密码错误，320-文件获取失败，321-文件超出限制，329-文件传输错误，399-任务执行出错，403-任务强制终止成功，404-任务强制终止失败，500-未知状态 | +| tag | string | 否 | 结果标签，对应返回值中的tag字段 | + +### 请求参数示例 + +- GET +```json +/api/v3/get_step_instance_status?bk_scope_type=biz&bk_scope_id=1&job_instance_id=100&step_instance_id=100&execute_count=0&batch=0&max_host_num_per_group=1&keyword=aaa&search_ip=&status=9&tag= +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "", + "data": { + "step_instance_id": 75, + "execute_count": 0, + "name": "API Quick execution scriptxxx", + "type": 1, + "status": 4, + "create_time": 1605064271000, + "start_time": 1605064271000, + "end_time": 1605064272000, + "total_time": 1000, + "step_result_group_list": [ + { + "result_type": 9, + "result_type_desc": "执行成功", + "tag": "tag1", + "host_size": 2, + "host_result_list": [ + { + "bk_host_id": 101, + "ip": "127.0.0.1", + "ipv6": null, + "bk_agent_id": null, + "bk_cloud_id": 0, + "bk_cloud_name": "Default Area", + "status": 9, + "status_desc": "执行成功", + "tag": "tag1", + "exit_code": 0, + "start_time": 1605064271000, + "end_time": 1605064272000, + "total_time": 1000 + } + ] + }, + { + "result_type": 9, + "result_type_desc": "执行成功", + "tag": "tag2", + "host_size": 2, + "host_result_list": [ + { + "bk_host_id": 102, + "ip": "127.0.0.2", + "ipv6": null, + "bk_agent_id": null, + "bk_cloud_id": 0, + "bk_cloud_name": "Default Area", + "status": 9, + "status_desc": "执行成功", + "tag": "tag2", + "exit_code": 0, + "start_time": 1605064271000, + "end_time": 1605064272000, + "total_time": 1000 + } + ] + } + ] + } +} +``` +### 返回结果说明 + +| 字段 | 类型 | 是否一定存在 | 描述 | +|--------------|--------|------------|-----------| +| result | bool | 是 | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 是 | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 是 | 请求失败返回的错误信息| +| data | object | 否 | 请求返回的数据| +| permission | object | 否 | 权限信息| + +##### data + +| 字段 | 类型 | 是否一定存在 | 描述 | +|------------------------|--------------------------|------------|-----------| +| step_instance_id | long | 是 | 作业步骤实例ID | +| execute_count | int | 是 | 步骤重试次数 | +| name | string | 是 | 步骤名称 | +| type | int | 是 | 步骤类型：1-脚本步骤；2-文件步骤；4-SQL步骤 | +| status | int | 是 | 作业步骤状态码: 1-未执行，2-正在执行，3-执行成功，4-执行失败，5-跳过，6-忽略错误，7-等待用户，8-手动结束，9-状态异常，10-步骤强制终止中，11-步骤强制终止成功，12-步骤强制终止失败 | +| create_time | long | 是 | 作业步骤实例创建时间，Unix时间戳，单位毫秒 | +| start_time | long | 是 | 开始执行时间，Unix时间戳，单位毫秒 | +| end_time | long | 是 | 执行结束时间，Unix时间戳，单位毫秒 | +| total_time | int | 是 | 总耗时，单位毫秒 | +| step_result_group_list | list | 是 | 任务执行结果分组列表，元素定义见step_result_group | + + +##### step_result_group + +| 字段 | 类型 | 是否一定存在 | 描述 | +|--------------------|--------------------|------------|-------------| +| result_type | int | 是 | 分组类型 | +| result_type_desc | string | 是 | 分组类型描述 | +| tag | string | 是 | 分组标签 | +| host_size | int | 是 | 分组内主机总量 | +| host_result_list | list | 是 | 每个分组内的主机任务执行结果列表，元素定义见host_result | + + +##### host_result + +| 字段 | 类型 | 是否一定存在 | 描述 | +|--------------------|-----------|------------|-----------| +| bk_host_id | long | 是 | 主机ID | +| ip | string | 否 | IP | +| ipv6 | string | 否 | IPv6 | +| bk_agent_id | string | 否 | AgentId | +| bk_cloud_id | long | 否 | 管控区域ID | +| bk_cloud_name | string | 否 | 管控区域名称 | +| status | int | 是 | 任务状态：0-未知错误，1-Agent异常，2-无效主机，3-上次已成功，5-等待执行，7-正在执行，9-执行成功，11-执行失败，12-任务下发失败，13-任务超时，15-任务日志错误，16-GSE脚本日志超时，17-GSE文件日志超时，101-脚本执行失败，102-脚本执行超时，103-脚本执行被终止，104-脚本返回码非零，202-文件传输失败，203-源文件不存在，301-文件任务系统错误-未分类的，303-文件任务超时，310-Agent异常，311-用户名不存在，312-用户密码错误，320-文件获取失败，321-文件超出限制，329-文件传输错误，399-任务执行出错，403-任务强制终止成功，404-任务强制终止失败，500-未知状态 | +| status_desc | string | 是 | 任务状态描述 | +| tag | string | 否 | 用户通过job_success/job_fail函数模板自定义输出的结果。仅脚本任务存在该参数 | +| exit_code | int | 否 | 脚本任务exit code | +| start_time | long | 是 | 开始执行时间，Unix时间戳，单位毫秒 | +| end_time | long | 是 | 执行结束时间，Unix时间戳，单位毫秒 | +| total_time | int | 是 | 总耗时，单位毫秒 | diff --git a/docs/apidoc/bk-api-gateway/v3/zh/operate_job_instance.md b/docs/apidoc/bk-api-gateway/v3/zh/operate_job_instance.md new file mode 100644 index 0000000000..280e3f455b --- /dev/null +++ b/docs/apidoc/bk-api-gateway/v3/zh/operate_job_instance.md @@ -0,0 +1,53 @@ +### 功能描述 + +用于对执行的作业实例进行操作 + +### 请求参数 + +{{ bkapi_authorization_description }} + +#### Header参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| X-Bkapi-Authorization | string | 是 | 认证信息。详情参考[调用网关 API](https://github.com/TencentBlueKing/BKDocs/blob/master/ZH/7.0/APIGateway/apigateway/use-api/use-apigw-api.md) | +| Accept | string | 是 | 固定值。application/json| +| Content-Type | string | 是 | 固定值。application/json| + +#### Body参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| bk_biz_id | long | 是 | *已弃用*。业务ID。此字段已被弃用并由字段bk_scope_type+bk_scope_id替换 | +| job_instance_id | long | 是 | 作业实例ID | +| operation_code | int | 是 | 操作类型：1、终止作业 | + +### 请求参数示例 + +- POST +```json +{ + "bk_scope_type": "biz", + "bk_scope_id": "1", + "job_instance_id": 100, + "operation_code": 1 +} +``` + +### 返回结果示例 + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + +##### data + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| job_instance_id | long | 作业实例ID | diff --git a/docs/apidoc/bk-api-gateway/v3/zh/operate_step_instance.md b/docs/apidoc/bk-api-gateway/v3/zh/operate_step_instance.md new file mode 100644 index 0000000000..002ea106d8 --- /dev/null +++ b/docs/apidoc/bk-api-gateway/v3/zh/operate_step_instance.md @@ -0,0 +1,82 @@ +### 功能描述 + +用于对执行的实例的步骤进行操作 + +### 请求参数 + +{{ bkapi_authorization_description }} + +#### Header参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| X-Bkapi-Authorization | string | 是 | 认证信息。详情参考[调用网关 API](https://github.com/TencentBlueKing/BKDocs/blob/master/ZH/7.0/APIGateway/apigateway/use-api/use-apigw-api.md) | +| Accept | string | 是 | 固定值。application/json| +| Content-Type | string | 是 | 固定值。application/json| + +#### Body参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| bk_biz_id | long | 是 | *已弃用*。业务ID。此字段已被弃用并由字段bk_scope_type+bk_scope_id替换 | +| job_instance_id | long | 是 | 作业实例ID | +| step_instance_id | long | 是 | 步骤实例ID | +| operation_code | int | 是 | 操作类型：2、失败IP重做，3、忽略错误，4、执行，5、跳过，6、确认继续 8、全部重试，9、终止确认流程，10-重新发起确认 | + +##### operation_code 详细说明 +| operation_code | 操作类型 | 适用步骤 | 描述 | +|-----------|------------|--------|------------| +| 2 | 失败IP重做 | 脚本执行，文件分发步骤 | 对失败的IP重新下发任务 | +| 3 | 忽略错误 | 脚本执行，文件分发步骤 | 忽略错误，继续执行 | +| 6 | 确认继续 | 人工确认步骤 | 确认继续执行 | +| 8 | 全部重试 | 脚本执行，文件分发步骤 | 对所有的IP重新下发任务 | +| 9 | 终止确认流程 | 人工确认步骤 | 确认终止执行 | +| 10 | 重新发起确认 | 人工确认步骤 | 重新发起确认 | +| 11 | 进入下一步 | 脚本执行，文件分发步骤 | 当步骤状态为终止成功，用于继续执行后续步骤 | +| 12 | 强制跳过 | 脚本执行，文件分发步骤 | 当步骤状态为终止中，用于强制跳过当前步骤，执行后续步骤| + +### 请求参数示例 + +- POST +```json +{ + "bk_scope_type": "biz", + "bk_scope_id": "1", + "job_instance_id": 100, + "step_instance_id": 200, + "operation_code": 2 +} +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "success",s + "data": { + "step_instance_id": 200, + "job_instance_id": 100 + } +} +``` + +### 返回结果参数说明 + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + +##### data + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| job_instance_id | long | 作业实例ID | +| step_instance_id | long | 步骤实例ID | diff --git a/docs/apidoc/bk-api-gateway/v3/zh/push_config_file.md b/docs/apidoc/bk-api-gateway/v3/zh/push_config_file.md new file mode 100644 index 0000000000..02bcb8183f --- /dev/null +++ b/docs/apidoc/bk-api-gateway/v3/zh/push_config_file.md @@ -0,0 +1,117 @@ +### 功能描述 + +分发配置文件，此接口用于分发配置文件等小的纯文本文件 + +### 请求参数 + +{{ bkapi_authorization_description }} + +#### Header参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| X-Bkapi-Authorization | string | 是 | 认证信息。详情参考[调用网关 API](https://github.com/TencentBlueKing/BKDocs/blob/master/ZH/7.0/APIGateway/apigateway/use-api/use-apigw-api.md) | +| Accept | string | 是 | 固定值。application/json| +| Content-Type | string | 是 | 固定值。application/json| + +#### Body参数 + +| 字段 | 类型 | 必选 | 描述 | +|-------------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| bk_biz_id | long | 是 | *已弃用*。业务ID。此字段已被弃用并由字段bk_scope_type+bk_scope_id替换 | +| task_name | string | 否 | 自定义作业名称 | +| account_alias | string | 是 | 执行账号别名 | +| file_target_path | string | 是 | 文件传输目标路径 | +| file_list | array | 是 | 源文件对象数组，见下面file定义 | +| target_server | object | 是 | 目标服务器，见server定义 | + +##### file + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| file_name | string | 是 | 文件名称 | +| content | string | 是 | 文件内容Base64 | + +##### server + +| 字段 | 类型 | 必选 | 描述 | +| ------------------ | ----- | ---- | ----------------------------------- | +| host_id_list | array | 否 | 主机ID列表 | +| ip_list | array | 否 | ***不推荐使用，建议使用host_id_list参数***;如果host_id_list与ip_list同时存在，将忽略ip_list参数。主机IP 列表，定义见ip | +| dynamic_group_list | array | 否 | 动态分组列表，定义见dynamic_group | +| topo_node_list | array | 否 | 动态 topo 节点列表，定义见topo_node | + +##### ip + +| 字段 | 类型 | 必选 | 描述 | +| ----------- | ------ | ---- | -------- | +| bk_cloud_id | long | 是 | 管控区域ID | +| ip | string | 是 | IP地址 | + +##### dynamic_group + +| 字段 | 类型 | 必选 | 描述 | +| ---- | ------ | ---- | -------------- | +| id | string | 是 | CMDB动态分组ID | + +##### topo_node_list + +| 字段 | 类型 | 必选 | 描述 | +| --------- | ------ | ---- | ------------------------------------------------------------ | +| id | long | 是 | 动态topo节点ID，对应CMDB API 中的 bk_inst_id | +| node_type | string | 是 | 动态topo节点类型，对应CMDB API 中的 bk_obj_id,比如"module","set" | + +### 请求参数示例 + +- POST +```json +{ + "bk_scope_type": "biz", + "bk_scope_id": "1", + "account": "root", + "file_target_path": "/tmp/", + "file_list": [ + { + "file_name": "a.txt", + "content": "aGVsbG8gd29ybGQh" + } + ], + "target_server": { + "dynamic_group_list": [ + { + "id": "blo8gojho0skft7pr5q0" + } + ], + "host_id_list": [ + 101, + 102 + ], + "topo_node_list": [ + { + "id": 1000, + "node_type": "module" + } + ] + } +} +``` + +### 返回结果示例 + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + +##### data + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| job_instance_id | long | 作业实例ID | +| job_instance_name | long | 作业实例名称 | +| step_instance_id | long | 步骤实例ID | diff --git a/docs/apidoc/bk-api-gateway/v3/zh/query_agent_info.md b/docs/apidoc/bk-api-gateway/v3/zh/query_agent_info.md new file mode 100644 index 0000000000..a6f64272f2 --- /dev/null +++ b/docs/apidoc/bk-api-gateway/v3/zh/query_agent_info.md @@ -0,0 +1,88 @@ +### 功能描述 + +查询主机Agent在作业平台中的信息（可执行状态、版本等） + +### 请求参数说明 + +{{ bkapi_authorization_description }} + +#### Header参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| X-Bkapi-Authorization | string | 是 | 认证信息。详情参考[调用网关 API](https://github.com/TencentBlueKing/BKDocs/blob/master/ZH/7.0/APIGateway/apigateway/use-api/use-apigw-api.md) | +| Accept | string | 是 | 固定值。application/json| +| Content-Type | string | 是 | 固定值。application/json| + +#### Body参数 + +| 字段 | 类型 | 必选 | 描述 | +|------------------------|--------------|-------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID，与bk_scope_type对应, 表示业务ID或者业务集ID | +| host_id_list | list | 是 | 主机ID数组，单次查询主机数量不可超过5000 | + + +### 请求参数示例 + +- POST +```json +{ + "bk_scope_type": "biz", + "bk_scope_id": "1", + "host_id_list": [1,2,3] +} +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "", + "data": { + "agent_info_list": [ + { + "bk_host_id": 1, + "status": 0, + "version": "2.1.4" + }, + { + "bk_host_id": 2, + "status": 1, + "version": "2.1.5" + }, + { + "bk_host_id": 3, + "status": 1, + "version": "2.1.6" + } + ] + } +} +``` +### 返回结果说明 + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + +##### data + +| 字段 | 类型 | 描述 | +|-----------------------|----------------|-----------| +| agent_info_list | list | 元素为Agent信息，详情见agent_info对象定义，若传入的host_id在返回结果的列表中不存在，则表示未查询到该主机的agent信息 | + + +##### agent_info + +| 字段 | 类型 | 描述 | +|-----------------|------------------|-----------| +| bk_host_id | long | 主机ID | +| status | int | Agent状态：0-异常，1-正常 | +| version | string | Agent的版本 | diff --git a/docs/apidoc/bk-api-gateway/v3/zh/save_cron.md b/docs/apidoc/bk-api-gateway/v3/zh/save_cron.md new file mode 100644 index 0000000000..84ed8958fc --- /dev/null +++ b/docs/apidoc/bk-api-gateway/v3/zh/save_cron.md @@ -0,0 +1,382 @@ +### 功能描述 + +新建或保存定时任务；新建定时任务，定时任务状态默认为暂停，如有需要可调用update_cron_status接口开启。 + +### 请求参数 + +{{ bkapi_authorization_description }} + +#### Header参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| X-Bkapi-Authorization | string | 是 | 认证信息。详情参考[调用网关 API](https://github.com/TencentBlueKing/BKDocs/blob/master/ZH/7.0/APIGateway/apigateway/use-api/use-apigw-api.md) | +| Accept | string | 是 | 固定值。application/json| +| Content-Type | string | 是 | 固定值。application/json| + +#### Body参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| bk_biz_id | long | 是 | *已弃用*。业务ID。此字段已被弃用并由字段bk_scope_type+bk_scope_id替换 | +| job_plan_id | long | 是 | 要定时执行的作业的执行方案 ID | +| id | long | 否 | 定时任务 ID，更新定时任务时，必须传这个值 | +| name | string | 否 | 定时作业名称，新建时必填，修改时选填 | +| expression | string | 否 | 定时任务 crontab 的定时规则，新建时必填，修改时选填，各字段含义为：分 时 日 月 周，如: 0/5 * * * * 表示每5分钟执行一次，注意：不支持? | +| execute_time | long | 否 | 定时任务单次执行的执行时间，Unix时间戳，新建定时任务时不可与expression同时为空 | +| global_var_list | array | 否 | 全局变量信息，可使用 查询执行方案详情 接口查询方案可设置的变量信息 | + +##### global_var + +| 字段 | 类型 | 必选 | 描述 | +|-----------|-----------|--------|------------| +| id | long | 否 | 全局变量 id，唯一标识。如果 id 为空，那么使用 name 作为唯一标识 | +| name | string | 否 | 全局变量 name | +| value | string | 否 | 字符、密码、数组类型的全局变量的值 | +| server | object | 否 | 主机类型全局变量的值 | + +##### server +| 字段 | 类型 | 必选 | 描述 | +|-----------------------|-------|--------|------------| +| host_id_list | array | 否 | 主机ID列表 | +| ip_list | array | 否 | ***不推荐使用，建议使用host_id_list参数***;如果host_id_list与ip_list同时存在，将忽略ip_list参数。主机IP 列表，定义见ip | +| dynamic_group_list | array | 否 | 动态分组列表 | +| topo_node_list | array | 否 | 动态 topo 节点列表 | + +##### ip + +| 字段 | 类型 | 必选 | 描述 | +|-------------|---------|--------|---------| +| bk_cloud_id | int | 是 | 管控区域ID | +| ip | string | 是 | IP地址 | + +##### topo_node +| 字段 | 类型 | 必选 | 描述 | +|------------------|--------|--------|------------| +| id | long | 是 | 动态topo节点ID，对应CMDB API 中的 bk_inst_id | +| node_type | string | 是 | 动态topo节点类型，对应CMDB API 中的 bk_obj_id,比如"module","set"| + + +### 请求参数示例 +- POST +#### 1.创建定时任务 +```json +{ + "bk_scope_type": "biz", + "bk_scope_id": "1", + "execute_time": 0, + "expression": "0 0/5 * * *", + "job_plan_id": 1023060, + "name": "test API", + "global_var_list": [ + { + "name": "stringVar", + "value": "value11112" + }, + { + "name": "nsVar", + "value": "nsvalue11112" + }, + { + "name": "secretVar", + "value": "secretvalue11112" + }, + { + "name": "dictVar", + "value": "([\"var1\"]=1, [\"var2\"]=2)" + }, + { + "name": "indexArrVar", + "value": "(2 3 4)" + }, + { + "name": "hostVar", + "server": { + "host_id_list": [1,2,3] + } + } + ] +} +``` +#### 2.更新定时任务 +```json +{ + "id": 1000064, + "bk_scope_type": "biz", + "bk_scope_id": "1", + "execute_time": 0, + "expression": "0 0/5 * * *", + "job_plan_id": 1023060, + "name": "test API", + "global_var_list": [ + { + "name": "stringVar", + "value": "value111333312" + }, + { + "name": "nsVar", + "value": "nsvalue111333312" + }, + { + "name": "secretVar", + "value": "secretvalue111333312" + }, + { + "name": "dictVar", + "value": "([\"var1\"]=1, [\"var2\"]=2)" + }, + { + "name": "indexArrVar", + "value": "(22 3 4)" + }, + { + "name": "hostVar", + "server": { + "host_id_list": [2,3,4] + } + } + ] +} +``` + +### 返回结果示例 +#### 1.创建定时任务 +```json +{ + "code": 0, + "result": true, + "data": { + "id": 1000067, + "name": "test API", + "status": 2, + "creator": "admin", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "job_plan_id": 1023060, + "expression": "0 0/5 * * *", + "global_var_list": [ + { + "id": 1001101, + "name": "stringVar", + "value": "value11112", + "description": null, + "type": 1, + "required": null, + "server": null + }, + { + "id": 1001102, + "name": "nsVar", + "value": "nsvalue11112", + "description": null, + "type": 2, + "required": null, + "server": null + }, + { + "id": 1001104, + "name": "secretVar", + "value": "secretvalue11112", + "description": null, + "type": 4, + "required": null, + "server": null + }, + { + "id": 1001105, + "name": "dictVar", + "value": "([\"var1\"]=1, [\"var2\"]=2)", + "description": null, + "type": 5, + "required": null, + "server": null + }, + { + "id": 1001106, + "name": "indexArrVar", + "value": "(2 3 4)", + "description": null, + "type": 6, + "required": null, + "server": null + }, + { + "id": 1001103, + "name": "hostVar", + "value": null, + "description": null, + "type": 3, + "required": null, + "server": { + "ip_list": [ + { + "bk_host_id": 101, + "bk_cloud_id": 0, + "ip": "127.0.0.1" + } + ], + "dynamic_group_list": null, + "topo_node_list": null + } + } + ], + "create_time": 1642045370, + "last_modify_user": "admin", + "last_modify_time": 1642045370 + } +} +``` +#### 2.更新定时任务 +```json +{ + "code": 0, + "result": true, + "data": { + "id": 1000064, + "name": "test API", + "status": 2, + "creator": "admin", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "job_plan_id": 1023060, + "expression": "0 0/5 * * *", + "global_var_list": [ + { + "id": 1001101, + "name": "stringVar", + "value": "value111333312", + "description": null, + "type": 1, + "required": null, + "server": null + }, + { + "id": 1001102, + "name": "nsVar", + "value": "nsvalue111333312", + "description": null, + "type": 2, + "required": null, + "server": null + }, + { + "id": 1001104, + "name": "secretVar", + "value": "secretvalue111333312", + "description": null, + "type": 4, + "required": null, + "server": null + }, + { + "id": 1001105, + "name": "dictVar", + "value": "([\"var1\"]=1, [\"var2\"]=2)", + "description": null, + "type": 5, + "required": null, + "server": null + }, + { + "id": 1001106, + "name": "indexArrVar", + "value": "(22 3 4)", + "description": null, + "type": 6, + "required": null, + "server": null + }, + { + "id": 1001103, + "name": "hostVar", + "value": null, + "description": null, + "type": 3, + "required": null, + "server": { + "ip_list": [ + { + "bk_host_id": 101, + "bk_cloud_id": 0, + "ip": "127.0.0.1" + } + ], + "dynamic_group_list": null, + "topo_node_list": null + } + } + ], + "create_time": 1641990674, + "last_modify_user": "admin", + "last_modify_time": 1641995052 + } +} +``` + +### 返回结果参数说明 + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + +##### data + +| 字段 | 类型 | 描述 | +|-----------------|------------|-------------| +| id | long | 定时任务 ID| +| name | string | 定时作业名称| +| status | int | 定时任务状态| +| bk_scope_type | string |资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| job_plan_id | long | 要定时执行的作业的执行方案 ID | +| creator | string | 创建人 | +| create_time | long | 创建时间Unix时间戳（s） | +| last_modify_user | string | 最近一次修改人 | +| last_modify_time | long | 最近一次修改时间Unix时间戳（s） | +| expression | string | 定时任务 crontab 的定时规则，| +| execute_time | long | 定时任务单次执行的执行时间，Unix时间戳| +| global_var_list | array | 全局变量信息，定义见global_var | + +##### global_var + +| 字段 | 类型 | 描述 | +|-------------|-----------|------------| +| id | long | 全局变量 id，唯一标识。如果 id 为空，那么使用 name 作为唯一标识 | +| name | string | 全局变量名 | +| description | string | 全局变量描述 | +| type | int | 全局变量类型 | +| required | int | 模版/执行方案中该变量是否必填 | +| value | string | 字符、密码、数组类型的全局变量的值 | +| server | object | 主机类型全局变量的值 | + +##### server +| 字段 | 类型 | 描述 | +|-----------------------|-------|------------| +| variable | string | 引用的变量名 | +| ip_list | array | 静态 IP 列表 | +| host_id_list | array | 静态 主机ID 列表，元素为Long类型 | +| dynamic_group_list | array | 动态分组 ID 列表 | +| topo_node_list | array | 动态 topo 节点列表 | + +##### ip +| 字段 | 类型 | 描述 | +|-------------|---------|---------| +| bk_cloud_id | int | 管控区域ID | +| ip | string | IP地址 | + +##### dynamic_group +| 字段 | 类型 | 描述 | +|-----|---------|------------| +| id | string | 动态分组 ID | + +##### topo_node +| 字段 | 类型 | 描述 | +|------------------|--------|------------| +| id | long | 动态topo节点ID，对应CMDB API 中的 bk_inst_id | +| node_type | string | 动态topo节点类型，对应CMDB API 中的 bk_obj_id,比如"module","set"| diff --git a/docs/apidoc/bk-api-gateway/v3/zh/update_cron_status.md b/docs/apidoc/bk-api-gateway/v3/zh/update_cron_status.md new file mode 100644 index 0000000000..f782963ec3 --- /dev/null +++ b/docs/apidoc/bk-api-gateway/v3/zh/update_cron_status.md @@ -0,0 +1,64 @@ +### 功能描述 + +更新定时作业状态，如启动或暂停 + +### 请求参数 + +{{ bkapi_authorization_description }} + +#### Header参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| X-Bkapi-Authorization | string | 是 | 认证信息。详情参考[调用网关 API](https://github.com/TencentBlueKing/BKDocs/blob/master/ZH/7.0/APIGateway/apigateway/use-api/use-apigw-api.md) | +| Accept | string | 是 | 固定值。application/json| +| Content-Type | string | 是 | 固定值。application/json| + +#### Body参数 + +| 字段 | 类型 | 必选 | 描述 | +|----------- |------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| bk_biz_id | long | 是 | *已弃用*。业务ID。此字段已被弃用并由字段bk_scope_type+bk_scope_id替换 | +| id | long | 是 | 定时作业 ID | +| status | int | 是 | 定时状态，1.启动、2.暂停 | + +### 请求参数示例 + +- POST +```json +{ + "bk_scope_type": "biz", + "bk_scope_id": "1", + "id": 2, + "status": 1 +} +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": 2 +} +``` + +### 返回结果参数说明 + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + +##### data + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| data | long | 定时任务ID | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/batch_get_job_instance_ip_log.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/batch_get_job_instance_ip_log.md new file mode 100644 index 0000000000..223453c3d1 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/batch_get_job_instance_ip_log.md @@ -0,0 +1,204 @@ +### Function Description + +Bulk query of job execution logs by host list + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +|-----------|------------|--------|------------| +| bk_scope_type | string | yes | Resource scope type. Optional values: biz - Business, biz_set - Business Set | +| bk_scope_id | string | yes | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| job_instance_id | long | yes | Job instance ID | +| step_instance_id | long | yes | Step instance ID | +| host_id_list | array | no | Host ID list | +| ip_list | array | no | Host IP list. ***Deprecated, it is recommended to use the host_id_list parameter***; if host_id_list and ip_list exist at the same time, the ip_list parameter will be ignored. | + +##### ip + +| Fields | Type | Required | Description | +| ----------- | ------ | -------- | ------------- | +| bk_cloud_id | int | yes | BK-Net ID | +| ip | string | yes | IP address | + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "job_instance_id": 100, + "step_instance_id": 200, + "host_id_list": [ + 101,102 + ] +} +``` + +### Example of response + +#### Script execution steps +```json +{ + "result": true, + "code": 0, + "message": "", + "data": { + "log_type": 1, + "job_instance_id": 100, + "step_instance_id": 200, + "script_task_logs": [ + { + "bk_host_id": 101, + "ip": "127.0.0.1", + "bk_cloud_id": 0, + "log_content": "[2018-03-15 14:39:30][PID:56875] job_start\n" + }, + { + "bk_host_id": 102, + "ip": "127.0.0.2", + "bk_cloud_id": 0, + "log_content": "[2018-03-15 14:39:30][PID:16789] job_start\n" + } + ] + } +} +``` + +#### File execution steps +```json +{ + "result": true, + "code": 0, + "message": "", + "data": { + "log_type": 2, + "job_instance_id": 100, + "step_instance_id": 200, + "file_task_logs": [ + { + "bk_host_id": 101, + "ip": "127.0.0.1", + "bk_cloud_id": 0, + "file_logs": [ + { + "mode": 1, + "src_ip": { + "bk_host_id": 102, + "bk_cloud_id": 0, + "ip": "127.0.0.2" + }, + "src_path": "/data/1.log", + "dest_ip": { + "bk_host_id": 101, + "bk_cloud_id": 0, + "ip": "127.0.0.1" + }, + "dest_path": "/tmp/1.log", + "status": 4, + "log_content": "[2021-06-28 11:32:16] FileName: /tmp/1.log FileSize: 9.0 Bytes State: dest agent success download file Speed: 1 KB/s Progress: 100% StatusDesc: dest agent success download file Detail: success", + "size": "1.0 Bytes", + "speed": "0 KB/s", + "process": "100%" + } + ] + }, + { + "bk_host_id": 102, + "ip": "127.0.0.2", + "bk_cloud_id": 0, + "file_logs": [ + { + "mode": 0, + "src_ip": { + "bk_host_id": 102, + "bk_cloud_id": 0, + "ip": "127.0.0.2" + }, + "src_path": "/data/1.log", + "status": 4, + "log_content": "[2021-06-28 11:32:16] FileName: /data/1.log FileSize: 9.0 Bytes State: source agent success upload file Speed: 1 KB/s Progress: 100% StatusDesc: source agent success upload file Detail: success upload", + "size": "1.0 Bytes", + "speed": "0 KB/s", + "process": "100%" + } + ] + } + ] + } +} +``` + +**File distribution task return result description** + +- If you need to return the upload log of the file source, you need to add the source file server IP to ip_list + +### Return Result Description + +#### response +| Fields | Type | Description | +|-----------|-----------|-----------| +| result | bool | Request success or failure. true: Request successful; false: Request failed | +| code | int | Error code. 0 means SUCCESS, >0 means FAIL | +| message | string | Error message | +| data | object | Data returned by request | +| permission | object | Permission information | +| request_id | string | Request chain id | + +#### data + +| Fields | Type | Description | +|-----------|-----------|-----------| +| bk_host_id | long | Host ID | +| bk_cloud_id | int | Target server BK-Net ID | +| ip | string | Target server IP address | +| log_type | int | Log type. 1 - Script execution task log; 2 - File distribution task log | +| script_task_logs | array | Log of script execution task. See script_task_log for definition | +| file_task_logs | array | File distribution task log. See file_task_log for definition | + +#### script_task_log + +| Fields | Type | Description | +|-----------|------------|--------| +| bk_host_id | long | Host ID | +| bk_cloud_id | long | BK-Net ID | +| ip | string | Target IP address | +| log_content | string | Script execution log content | + +#### file_task_log + +| Fields | Type | Description | +|-----------|------------|--------| +| bk_host_id | long | Host ID | +| bk_cloud_id | long | BK-Net ID | +| ip | string | Source/target IP address | +| file_logs | array | File distribution log content. See file_log for definition | + +#### file_log + +| Fields | Type | Description | +|-------------|-----------|-----------| +| mode | int | Distribution mode. 0: Upload; 1: Download | +| src_ip | object |File source host IP. see ip for definition | +| src_path | string | Source file paths | +| dest_ip | object | Distribute the target host IP, with value for mode=1. See ip for definition. | +| dest_path | string | Target path, with value for mode=1 | +| status | int | Task status. 1-Waiting; 2-Uploading; 3-Downloading; 4- Success; 5- Failure | +| log_content | string | File distribution log contents | +| size | string | File size | +| speed | string | File transfer speed | +| process | string | File transfer process | + +#### ip + +| Fields | Type | Description | +|-----------|------------|--------| +| bk_host_id | long | Host ID | +| bk_cloud_id | long | BK-Net ID | +| ip | string | IP Address | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/callback_protocol.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/callback_protocol.md new file mode 100644 index 0000000000..8dad1af2de --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/callback_protocol.md @@ -0,0 +1,42 @@ +### Function Description + +This component is only used to present the callback protocol when job finished. + + +### Request Parameters + +| Fields | Type | Required | Description | +|-----------------|------------|--------|------------| +| job_instance_id | long | yes | Job instance ID | +| status | int | yes | Job status code. 1 - Pending; 2 - Running 3 - Successful; 4 - Failed; 5 - Skipped; 6 - Ignore Error; 7 - Waiting; 8 - Terminated; 9 - Abnormal; 10 - Terminating; 11 - Terminate Success; 13 - Termination Confirmed; 14 - Abandoned| +| step_instance_list | array | yes | The execution result of steps | + +#### step_instances + +| Fields | Type | Required | Description | +|-----------------|------------|--------|------------| +| step_instance_id | long | yes | Job step instance ID | +| status | int | yes | Job step status code: 1 - Pending; 2 - Running 3 - Successful; 4 - Failed; 5 - Skipped; 6 - Ignore Error; 7 - Waiting; 8 - Terminated; 9 - Abnormal; 10 - Terminating; 11 - Terminate Success; 13 - Termination Confirmed; 14 - Abandoned| + +### Example of request + +```json +{ + "job_instance_id": 12345, + "status": 2, + "step_instance_list": [ + { + "step_instance_id": 16271, + "status": 3 + }, + { + "step_instance_id": 16272, + "status": 2 + } + ] +} +``` + +### Callback response + +The success of the callback is based on HTTP status, if it succeeds, the status code 200, the other indicates failure, Job will do a retry for the failed one, if it still fails, the failure will be ignored and no more callbacks. \ No newline at end of file diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/check_script.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/check_script.md new file mode 100644 index 0000000000..f78d25922f --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/check_script.md @@ -0,0 +1,74 @@ +### Function Description + +High risk script detection + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +| --------------- | ------ | -------- | ------------------------------------------------------------ | +| script_language | int | yes | Script language:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - sql | +| content | string | yes | Script content, requiring Base64 encoding | + + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "script_language": 1, + "content": "cm0gLXJmIC8=" +} +``` + +### Example of responses + +```json +{ + "code": 0, + "result": true, + "data": [ + { + "line": 1, + "line_content": "rm /tmp", + "match_content": "rm /tmp", + "level": 1, + "description": "The first line of the script does not define a valid script type, for example: #!/bin/bash" + }, + { + "line": 1, + "line_content": "rm /tmp", + "match_content": "rm", + "level": 3, + "description": "dangerous！！！" + } + ] +} +``` + +### Response Description + +#### response + +| Fields | Type | Description | +| ---------- | ------ | ------------------------------------------------------------ | +| result | bool | Whether the request succeeded or not. True: request succeeded;False: request failed | +| code | int | Error code. 0 indicates success, >0 indicates failure | +| message | string | Error message | +| data | object | Data returned by request | +| permission | object | Permission information | + +#### data + +| Fields | Type | Description | +| ------------- | ------ | --------------------------------------------------- | +| line | int | Number of rows where the error occurred | +| line_content | string | The content of the line where the script is located | +| match_content | string | Matching Content | +| level | int | Error level: 1- Warning, 2- Error, 3- Fatal | +| description | string | Description | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/common.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/common.md new file mode 100644 index 0000000000..c106d51491 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/common.md @@ -0,0 +1,45 @@ +## Common Fields and Conventions + +#### global_var + +| Fields | Type | **Required** | **Description** | +|-------------|------------|--------|------------| +| id | long | no | Global variables ID . If id is empty, then name is used as the unique identifier | +| name | string | no | Global variables name | +| description | string | no | Global variables description | +| type | int | no | Global variables types | +| required | int | no | Is this variable required | +| value | string | no | Values of global variables of String, Ciphertext, Namespace, and Array types | +| server | object | no | Value of global variable of Host type | + +#### server +| Fields | Type | **Required** | **Description** | +|-----------------------|-------|--------|------------| +| ip_list | array | no | Static IP list, see ip for definition | +| dynamic_group_list | array | no | Dynamic grouping list, see dynamic_group for definition | +| topo_node_list | array | no | Dynamic topo node list, see topo_node for definition | + +#### ip +| Fields | Type | **Required** | **Description** | +|-------------|---------|--------|---------| +| bk_cloud_id | int | yes | BK-Net ID | +| ip | string | yes | IP Address | + +#### dynamic_group + +| Fields | Type | **Required** | **Description** | +| ------ | ------ | ------------ | --------------------- | +| id | string | yes | CMDB Dynamic Group ID | + +#### topo_node + +| Fields | Type | **Required** | **Description** | +|------------------|--------|--------|------------| +| id | long | yes | Dynamic topo node ID, corresponding to bk_inst_id in CMDB API | +| node_type | string | yes | Dynamic topo node type, corresponding to bk_obj_id in CMDB API, such as "module", "set" | + +#### account +| Fields | Type | **Required** | **Description** | +|-------|--------|--------|------------| +| id | long | no | Account ID | +| name | string | no | Account name | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/create_credential.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/create_credential.md new file mode 100644 index 0000000000..806711396a --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/create_credential.md @@ -0,0 +1,70 @@ +### Function Description + +Create credential. + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +|----------------------------|------------|--------|------------| +| bk_scope_type | string | yes | Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | yes | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| name | string | yes | Required Credential Name | +| type | string | yes | Credential type，value can be ACCESS_KEY_SECRET_KEY,PASSWORD,USERNAME_PASSWORD,SECRET_KEY | +| description | string | no | Credentials Description | +| credential_access_key | string | no | When the credential type is ACCESS_KEY_SECRET_KEY, required | +| credential_secret_key | string | no | When the credential type is ACCESS_KEY_SECRET_KEY/SECRET_KEY, required | +| credential_username | string | no | When the credential type is USERNAME_PASSWORD, required | +| credential_password | string | no | When the credential type is USERNAME_PASSWORD/PASSWORD, required | + + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "name": "testCredential", + "type": "USERNAME_PASSWORD", + "description": "This is a test credential", + "credential_username": "admin", + "credential_password": "password" +} +``` + +### Example of responses + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "id": "06644309e10e4068b3c7b32799668210" + } +} +``` + +### Response Description + +#### response +| **Fields** | **Type** | **Description** | +|-----------|-----------|-----------| +| result | bool | Success or failure of the request. true: request successful; false: request failed | +| code | int | Error code. 0 means SUCCESS, >0 means FAIL | +| message | string | Error message | +| data | object | Data returned by request | +| permission | object | Permission information | +| request_id | string | Request chain id | + +#### data + +| Fields | **Type** |Whether the field must exist | **Description** | +|-----------|-------|---------------|---------| +| id | string |yes | Credential ID | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/create_dangerous_rule.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/create_dangerous_rule.md new file mode 100644 index 0000000000..3891e75888 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/create_dangerous_rule.md @@ -0,0 +1,79 @@ +### Function Description + +Create dangerous rule + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +| -------------------- | ------ | -------- | ------------------------------------------------------------ | +| expression | string | yes | Expression | +| script_language_list | array | yes | Script language:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - sql | +| description | string | yes | Rule description | +| action | int | yes | Processing actions: 1- Scan, 2- Intercept | + + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "expression": "rm -rf", + "script_language_list": [1], + "description": "drangerous!!!", + "action": 2 +} +``` + +### Example of responses + +```json +{ + "code": 0, + "result": true, + "data": { + "id": 1, + "expression": "rm -rf", + "script_language_list": [1], + "description": "drangerous!!!", + "action": 2, + "status": 0, + "creator": "admin", + "create_time": 1695193968000, + "last_modify_user": "admin", + "last_modify_time": 1695193968000 + } +} +``` + +### Response Description + +#### response + +| Fields | Type | Description | +| ---------- | ------ | ------------------------------------------------------------ | +| result | bool | Whether the request succeeded or not. True: request succeeded;False: request failed | +| code | int | Error code. 0 indicates success, >0 indicates failure | +| message | string | Error message | +| data | object | Data returned by request | +| permission | object | Permission information | + +#### data + +| Fields | Type | Description | +| -------------------- | ------ | ------------------------------------------------------------ | +| id | long | Rule id | +| expression | string | expression | +| script_language_list | array | Script language:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - sql | +| description | string | Description | +| action | int | Processing actions: 1- Scan, 2- Intercept | +| status | int | Enabling status: 0- disabled, 1- enabled | +| creator | string | Creator | +| create_time | long | Created time, Unix timestamp | +| last_modify_user | string | Last modify user | +| last_modify_time | long | Last modified time, Unix timestamp | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/create_file_source.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/create_file_source.md new file mode 100644 index 0000000000..84b110d2d0 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/create_file_source.md @@ -0,0 +1,87 @@ +### Function Description + +Create a new file source, currently only BlueKing Repository file sources that are associated with automatically selected public access points are supported. +The created file source has the following default features: + +| Characteristic | Value | +|-----------------|------------| +|Storage Type|Object Storage| +|File Source Type|BlueKing Repository| +|Whether it is a public file source|no| +| Access Point Selection Range |Public Access Point| +|Access Point Selection Mode|Automatic| + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| **Fields** | **Type** | **Required** | **Description** | +|-----------------|------------|--------|------------| +| bk_scope_type | string | yes | Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | yes | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| code | string | yes | File source identification, beginning with English characters, 1-32 English characters, underscores, numbers, cannot be changed after creation | +| alias | string | yes | File source alias | +| type | string | yes | File Source Type, Currently only the BlueKing Repository is supported，BLUEKING_ARTIFACTORY | +| access_params | object | yes | File source access parameters, pass in different objects according to type, see subsequent instructions | +| credential_id | string | no | Credential Id used by the file source | +| file_prefix | string | no | JOB prefix for files distributed from this source, no prefix by default | + +### access_params +Type is BLUEKING_ARTIFACTORY + +| **Fields** | **Type** | **Required** | **Description** | +|-----------------|------------|--------|------------| +| base_url | string | yes | The root url of the BlueKing Repository instance, for example: https://bkrepo.com | + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "code": "sopsArtifactory", + "alias": "SOPS BK-REPO File Source ", + "type": "BLUEKING_ARTIFACTORY", + "access_params": { + "base_url": "https://bkrepo.com" + }, + "credential_id": "06644309e10e4068b3c7b32799668210", + "file_prefix": "" +} +``` + +### Example of responses + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "id": 1 + } +} +``` + +### Response Description + +#### response +| **Fields** | **Type** | **Description** | +|-----------|-----------|-----------| +| result | bool | Request success or failure. true: Request successful; false: Request failed | +| code | int | Error code. 0 means SUCCESS, >0 means FAIL | +| message | string | Error message | +| data | object | Data returned by request | +| permission | object | Permission information | +| request_id | string | Request chain id | + +#### data + +| Fields | **Type** | Whether the field must exist | **Description** | +|-----------|-------|---------------|---------| +| id | int |yes | File Source ID | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/create_public_script.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/create_public_script.md new file mode 100644 index 0000000000..c2914ab1cd --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/create_public_script.md @@ -0,0 +1,87 @@ +### Function Description + +Create public script + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +| --------------- | ------ | -------- | ------------------------------------------------------------ | +| name | string | yes | Public script name | +| description | string | no | Public script description | +| script_language | int | yes | Script language:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - sql | +| content | string | yes | Script content, requiring Base64 encoding | +| version | string | yes | Version | +| version_desc | string | no | Version Description | + + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "name": "public script test", + "description": "public script test", + "script_language": "1", + "content": "IyEvYmluL2Jhc2gKbHM=", + "version": "1.0" +} +``` + +### Example of responses + +```json +{ + "code": 0, + "result": true, + "data": { + "id": 1000018, + "script_id": "4537fb49ec0840a1b91cef4179c99f9c", + "name": "public script test", + "script_language": 1, + "content": "#!/bin/bash\nls", + "creator": "admin", + "create_time": 1691739630000, + "last_modify_user": "admin", + "last_modify_time": 1691739630000, + "version": "1.0", + "status": 0, + "description": "public script test" + } +} +``` + +### Response Description + +#### response + +| Fields | Type | Description | +| ---------- | ------ | ------------------------------------------------------------ | +| result | bool | Whether the request succeeded or not. True: request succeeded;False: request failed | +| code | int | Error code. 0 indicates success, >0 indicates failure | +| message | string | Error message | +| data | object | Data returned by request | +| permission | object | Permission information | + +#### data + +| Fields | Type | Description | +| ----------------- | ------ | ------------------------------------------------------------ | +| id | long | Script version id | +| script_id | string | Script id | +| name | string | Script name | +| script_language | int | Script language:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - sql | +| content | string | Script content | +| creator | string | Creator | +| create_time | long | Created time, Unix timestamp | +| last_modify_user | string | Last modify user | +| last_modify_time | long | Last modified time, Unix timestamp | +| version | string | Script version | +| version_desc | string | Version description | +| status | int | Script version status (0: Not online, 1: Online, 2: Offline, 3: Disabled) | +| description | string | Description | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/create_public_script_version.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/create_public_script_version.md new file mode 100644 index 0000000000..41346677a9 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/create_public_script_version.md @@ -0,0 +1,85 @@ +### Function Description + +Create Public Script Version + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +| ------------ | ------ | -------- | ----------------------------------------- | +| script_id | string | yes | Script id | +| content | string | yes | Script content, requiring Base64 encoding | +| version | string | yes | Version | +| version_desc | string | 否 | Version Description | + + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "script_id": "4537fb49ec0840a1b91cef4179c99f9c", + "content": "IyEvYmluL2Jhc2gKbHM=", + "version": "2.0", + "version_desc": "version 2" +} +``` + +### Example of responses + +```json +{ + "code": 0, + "result": true, + "data": { + "id": 1000020, + "script_id": "4537fb49ec0840a1b91cef4179c99f9c", + "name": "public script test", + "script_language": 1, + "content": "#!/bin/bash\nls", + "creator": "admin", + "create_time": 1691739630000, + "last_modify_user": "admin", + "last_modify_time": 1691739630000, + "version": "1.0", + "version_desc": "version 2", + "status": 0, + "description": "public script test" + } +} +``` + +### Response Description + +#### response + +| Fields | Type | Description | +| ---------- | ------ | ------------------------------------------------------------ | +| result | bool | Whether the request succeeded or not. True: request succeeded;False: request failed | +| code | int | Error code. 0 indicates success, >0 indicates failure | +| message | string | Error message | +| data | object | Data returned by request | +| permission | object | Permission information | + +#### data + +| Fields | Type | Description | +| ----------------- | ------ | ------------------------------------------------------------ | +| id | long | Script version id | +| script_id | string | Script id | +| name | string | Script name | +| script_language | int | Script language:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - sql | +| content | string | Script content | +| creator | string | Creator | +| create_time | long | Created time, Unix timestamp | +| last_modify_user | string | Last modify user | +| last_modify_time | long | Last modified time, Unix timestamp | +| version | string | Script version | +| version_desc | string | Version description | +| status | int | Script version status (0: Not online, 1: Online, 2: Offline, 3: Disabled) | +| description | string | Description | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/create_script.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/create_script.md new file mode 100644 index 0000000000..ec5af25dd6 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/create_script.md @@ -0,0 +1,96 @@ +### Function Description + +Create Script + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +| --------------- | ------ | -------- | ------------------------------------------------------------ | +| bk_scope_type | string | yes | Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | yes | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| name | string | yes | Script name | +| description | string | no | Script description | +| script_language | int | yes | Script language:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - sql | +| content | string | yes | Script content, requiring Base64 encoding | +| version | string | yes | Version | +| version_desc | string | no | Version Description | + + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type":"biz", + "bk_scope_id":"2", + "name": "script test", + "description": "script test", + "script_language": "1", + "content": "IyEvYmluL2Jhc2gKZGF0ZQo=", + "version": "1.0" +} +``` + +### Example of responses + +```json +{ + "code": 0, + "result": true, + "data": { + "id": 1000019, + "script_id": "4a350b0e0707450e93326f6ace921072", + "name": "script test", + "script_language": 1, + "bk_scope_type": "biz", + "bk_scope_id": "2", + "content": "#!/bin/bash\ndate\n", + "creator": "admin", + "create_time": 1691741073000, + "last_modify_user": "admin", + "last_modify_time": 1691741073000, + "version": "1.0", + "status": 0, + "description": "script test" + } +} +``` + +### Response Description + +#### response + +| Fields | Type | Description | +| ---------- | ------ | ------------------------------------------------------------ | +| result | bool | Whether the request succeeded or not. True: request succeeded;False: request failed | +| code | int | Error code. 0 indicates success, >0 indicates failure | +| message | string | Error message | +| data | object | Data returned by request | +| permission | object | Permission information | + +#### data + +| Fields | Type | Description | +| ----------------- | ------ | ------------------------------------------------------------ | +| id | long | Script version id | +| script_id | string | Script id | +| name | string | Script name | +| script_language | int | Script language:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - sql | +| bk_scope_type | string | Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| content | string | Script content | +| creator | string | Creator | +| create_time | long | Created time, Unix timestamp | +| last_modify_user | string | Last modify user | +| last_modify_time | long | Last modified time, Unix timestamp | +| version | string | Script version | +| version_desc | string | Version description | +| status | int | Script version status (0: Not online, 1: Online, 2: Offline, 3: Disabled) | +| description | string | Description | + diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/create_script_version.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/create_script_version.md new file mode 100644 index 0000000000..cc1ab0fbb3 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/create_script_version.md @@ -0,0 +1,93 @@ +### Function Description + +Create Script Version + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +| ------------- | ------ | -------- | ------------------------------------------------------------ | +| bk_scope_type | string | yes | Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | yes | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| script_id | string | yes | Script id | +| content | string | yes | Script content, requiring Base64 encoding | +| version | string | yes | Version | +| version_desc | string | no | Version Description | + + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "2", + "script_id": "4a350b0e0707450e93326f6ace921072", + "content": "IyEvYmluL2Jhc2gKZGF0ZQo=", + "version": "2.0", + "version_desc": "version 2" +} +``` + +### Example of responses + +```json +{ + "code": 0, + "result": true, + "data": { + "id": 1000021, + "script_id": "4a350b0e0707450e93326f6ace921072", + "name": "script test", + "script_language": 1, + "bk_scope_type": "biz", + "bk_scope_id": "2", + "content": "#!/bin/bash\ndate\n", + "creator": "admin", + "create_time": 1691743535000, + "last_modify_user": "admin", + "last_modify_time": 1691743535000, + "version": "2.0", + "version_desc": "version 2", + "status": 0, + "description": "script test" + } +} +``` + +### Response Description + +#### response + +| Fields | Type | Description | +| ---------- | ------ | ------------------------------------------------------------ | +| result | bool | Whether the request succeeded or not. True: request succeeded;False: request failed | +| code | int | Error code. 0 indicates success, >0 indicates failure | +| message | string | Error message | +| data | object | Data returned by request | +| permission | object | Permission information | + +#### data + +| Fields | Type | Description | +| ----------------- | ------ | ------------------------------------------------------------ | +| id | long | Script version id | +| script_id | string | Script id | +| name | string | Script name | +| script_language | int | Script language:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - sql | +| bk_scope_type | string | Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| content | string | Script content | +| creator | string | Creator | +| create_time | long | Created time, Unix timestamp | +| last_modify_user | string | Last modify user | +| last_modify_time | long | Last modified time, Unix timestamp | +| version | string | Script version | +| version_desc | string | Version description | +| status | int | Script version status (0: Not online, 1: Online, 2: Offline, 3: Disabled) | +| description | string | Description | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/delete_cron.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/delete_cron.md new file mode 100644 index 0000000000..257057bb37 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/delete_cron.md @@ -0,0 +1,53 @@ +### Function Description + +Delete cron job + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +| ------------- | ------ | -------- | ------------------------------------------------------------ | +| bk_scope_type | string | yes | Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | yes | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| id | long | yes | cron job id | + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type":"biz", + "bk_scope_id":"2", + "id": 1000045 +} +``` + +### Example of responses + +```json +{ + "code": 0, + "result": true +} +``` + +### Response Description + +#### response + +| Fields | Type | Description | +| ---------- | ------ | ------------------------------------------------------------ | +| result | bool | Whether the request succeeded or not. True: request succeeded;False: request failed | +| code | int | Error code. 0 indicates success, >0 indicates failure | +| message | string | Error message | +| data | object | Data returned by request | +| permission | object | Permission information | + +#### data + +null diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/delete_dangerous_rule.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/delete_dangerous_rule.md new file mode 100644 index 0000000000..6440acc40c --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/delete_dangerous_rule.md @@ -0,0 +1,50 @@ +### Function Description + +Delete dangerous rule + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +| ------ | ---- | -------- | ----------- | +| id | long | yes | Rule id | + + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "id": 1 +} +``` + +### Example of responses + +```json +{ + "code": 0, + "result": true +} +``` + +### Response Description + +#### response + +| Fields | Type | Description | +| ---------- | ------ | ------------------------------------------------------------ | +| result | bool | Whether the request succeeded or not. True: request succeeded;False: request failed | +| code | int | Error code. 0 indicates success, >0 indicates failure | +| message | string | Error message | +| data | object | Data returned by request | +| permission | object | Permission information | + +#### data + +null diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/delete_public_script.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/delete_public_script.md new file mode 100644 index 0000000000..3b1fd4e372 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/delete_public_script.md @@ -0,0 +1,50 @@ +### Function Description + +Delete public script will also delete all script versions under that script. + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +| --------- | ------ | -------- | ---------------- | +| script_id | string | yes | Public script id | + + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "script_id": "4537fb49ec0840a1b91cef4179c99f9c" +} +``` + +### Example of responses + +```json +{ + "code": 0, + "result": true +} +``` + +### Response Description + +#### response + +| Fields | Type | Description | +| ---------- | ------ | ------------------------------------------------------------ | +| result | bool | Whether the request succeeded or not. True: request succeeded;False: request failed | +| code | int | Error code. 0 indicates success, >0 indicates failure | +| message | string | Error message | +| data | object | Data returned by request | +| permission | object | Permission information | + +#### data + +null diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/delete_public_script_version.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/delete_public_script_version.md new file mode 100644 index 0000000000..b300ba2aac --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/delete_public_script_version.md @@ -0,0 +1,52 @@ +### Function Description + +Delete script version + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +| ----------------- | ------ | -------- | ------------------------ | +| script_id | string | yes | Public script id | +| script_version_id | long | yes | Public script version id | + + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "script_id":"4537fb49ec0840a1b91cef4179c99f9c", + "script_version_id": "1000017" +} +``` + +### Example of responses + +```json +{ + "code": 0, + "result": true +} +``` + +### Response Description + +#### response + +| Fields | Type | Description | +| ---------- | ------ | ------------------------------------------------------------ | +| result | bool | Whether the request succeeded or not. True: request succeeded;False: request failed | +| code | int | Error code. 0 indicates success, >0 indicates failure | +| message | string | Error message | +| data | object | Data returned by request | +| permission | object | Permission information | + +#### data + +null diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/delete_script.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/delete_script.md new file mode 100644 index 0000000000..7e5f0d9c8b --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/delete_script.md @@ -0,0 +1,54 @@ +### Function Description + +Delete script will also delete all script versions under that script + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +| ------------- | ------ | -------- | ------------------------------------------------------------ | +| bk_scope_type | string | yes | Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | yes | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| script_id | long | yes | Script id | + + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type":"biz", + "bk_scope_id":"2", + "script_id": "4a350b0e0707450e93326f6ace921072" +} +``` + +### Example of responses + +```json +{ + "code": 0, + "result": true +} +``` + +### Response Description + +#### response + +| Fields | Type | Description | +| ---------- | ------ | ------------------------------------------------------------ | +| result | bool | Whether the request succeeded or not. True: request succeeded;False: request failed | +| code | int | Error code. 0 indicates success, >0 indicates failure | +| message | string | Error message | +| data | object | Data returned by request | +| permission | object | Permission information | + +#### data + +null diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/delete_script_version.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/delete_script_version.md new file mode 100644 index 0000000000..421e4c3be7 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/delete_script_version.md @@ -0,0 +1,56 @@ +### Function Description + +Delete script version + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +| ----------------- | ------ | -------- | ------------------------------------------------------------ | +| bk_scope_type | string | yes | Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | yes | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| script_id | string | yes | Script id | +| script_version_id | long | yes | Script version id | + + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type":"biz", + "bk_scope_id":"2", + "script_id": "4a350b0e0707450e93326f6ace921072", + "script_version_id": 1000019 +} +``` + +### Example of responses + +```json +{ + "code": 0, + "result": true +} +``` + +### Response Description + +#### response + +| Fields | Type | Description | +| ---------- | ------ | ------------------------------------------------------------ | +| result | bool | Whether the request succeeded or not. True: request succeeded;False: request failed | +| code | int | Error code. 0 indicates success, >0 indicates failure | +| message | string | Error message | +| data | object | Data returned by request | +| permission | object | Permission information | + +#### data + +null diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/disable_dangerous_rule.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/disable_dangerous_rule.md new file mode 100644 index 0000000000..df49db3a6b --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/disable_dangerous_rule.md @@ -0,0 +1,57 @@ +### Function Description + +Disable dangerous rule + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +| ------ | ---- | -------- | ----------- | +| id | long | yes | Rule id | + + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "id": 1 +} +``` + +### Example of responses + +```json +{ + "code": 0, + "result": true, + "data": { + "id": 1, + "status": 0 + } +} +``` + +### Response Description + +#### response + +| Fields | Type | Description | +| ---------- | ------ | ------------------------------------------------------------ | +| result | bool | Whether the request succeeded or not. True: request succeeded;False: request failed | +| code | int | Error code. 0 indicates success, >0 indicates failure | +| message | string | Error message | +| data | object | Data returned by request | +| permission | object | Permission information | + +#### data + +| Fields | Type | Description | +| ------ | ---- | ---------------------------------------- | +| id | long | Rule id | +| status | int | Enabling status: 0- disabled, 1- enabled | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/disable_public_script_version.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/disable_public_script_version.md new file mode 100644 index 0000000000..3e8e594929 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/disable_public_script_version.md @@ -0,0 +1,61 @@ +### Function Description + +Disable the public script version. Once disabled successfully, it cannot be restored! And any job steps that reference this version online will not be able to be executed, please be cautious when operating. + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +| ----------------- | ------ | -------- | ------------------------ | +| script_id | string | yes | Public script id | +| script_version_id | long | yes | Public script version id | + + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "script_id":"4537fb49ec0840a1b91cef4179c99f9c", + "script_version_id": 1000018 +} +``` + +### Example of responses + +```json +{ + "code": 0, + "result": true, + "data": { + "id": 1000018, + "script_id": "4537fb49ec0840a1b91cef4179c99f9c", + "status": 3 + } +} +``` + +### Response Description + +#### response + +| Fields | Type | Description | +| ---------- | ------ | ------------------------------------------------------------ | +| result | bool | Whether the request succeeded or not. True: request succeeded;False: request failed | +| code | int | Error code. 0 indicates success, >0 indicates failure | +| message | string | Error message | +| data | object | Data returned by request | +| permission | object | Permission information | + +#### data + +| Fields | Type | Description | +| ----------------- | ------ | ------------------------------------------------------------ | +| script_id | string | Script id | +| id | long | Script version id | +| status | int | Script version status (0: Not online, 1: Online, 2: Offline, 3: Disabled) | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/disable_script_version.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/disable_script_version.md new file mode 100644 index 0000000000..32b620f446 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/disable_script_version.md @@ -0,0 +1,65 @@ +### Function Description + +Disable script version, once successfully disabled, cannot be restored! And any job steps that reference this version online will not be able to be executed, please be cautious when operating. + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +| ----------------- | ------ | -------- | ------------------------------------------------------------ | +| bk_scope_type | string | yes | Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | yes | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| script_id | string | yes | Script id | +| script_version_id | long | yes | Script version id | + + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type":"biz", + "bk_scope_id":"2", + "script_id": "4a350b0e0707450e93326f6ace921072", + "script_version_id": 1000019 +} +``` + +### Example of responses + +```json +{ + "code": 0, + "result": true, + "data": { + "id": 1000019, + "script_id": "4a350b0e0707450e93326f6ace921072", + "status": 3 + } +} +``` + +### Response Description + +#### response + +| Fields | Type | Description | +| ---------- | ------ | ------------------------------------------------------------ | +| result | bool | Whether the request succeeded or not. True: request succeeded;False: request failed | +| code | int | Error code. 0 indicates success, >0 indicates failure | +| message | string | Error message | +| data | object | Data returned by request | +| permission | object | Permission information | + +#### data + +| Fields | Type | Description | +| ----------------- | ------ | ------------------------------------------------------------ | +| script_id | string | Script id | +| id | long | Script version id | +| status | int | Script version status (0: Not online, 1: Online, 2: Offline, 3: Disabled) | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/enable_dangerous_rule.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/enable_dangerous_rule.md new file mode 100644 index 0000000000..164b2cf79a --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/enable_dangerous_rule.md @@ -0,0 +1,57 @@ +### Function Description + +Enable dangerous rule + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +| ------ | ---- | -------- | ----------- | +| id | long | yes | Rule id | + + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "id": 1 +} +``` + +### Example of responses + +```json +{ + "code": 0, + "result": true, + "data": { + "id": 1, + "status": 1 + } +} +``` + +### Response Description + +#### response + +| Fields | Type | Description | +| ---------- | ------ | ------------------------------------------------------------ | +| result | bool | Whether the request succeeded or not. True: request succeeded;False: request failed | +| code | int | Error code. 0 indicates success, >0 indicates failure | +| message | string | Error message | +| data | object | Data returned by request | +| permission | object | Permission information | + +#### data + +| Fields | Type | Description | +| ------ | ---- | ---------------------------------------- | +| id | long | Rule id | +| status | int | Enabling status: 0- disabled, 1- enabled | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/execute_job_plan.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/execute_job_plan.md new file mode 100644 index 0000000000..572a025329 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/execute_job_plan.md @@ -0,0 +1,128 @@ +### Function Description + +Start Job Plan + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +|-----------|------------|--------|------------| +| bk_scope_type | string | yes | Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | yes | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| job_plan_id | long | yes | JOB plan ID | +| global_var_list | array | no | Global variables. For global variable values in a job execution plan, the value of the incoming variable is used if it is included in the request parameters; otherwise, the default value currently configured by the JOB Plan is used. See global_var for definition | +| callback_url | string | no | Callback URL, when the task execution is completed, JOB will call this URL to inform the task execution result. The callback protocol refers to the callback_protocol component documentation | + +#### global_var + +| Fields | Type | Required | Description | +|-----------|------------|--------|------------| +| id | long | no | Global variable id, unique identifier. If id is empty, then name is used as the unique identifier | +| name | string | no | Global variable name | +| value | string | no | Values of global variables of String, Ciphertext, Array, and Namespace types | +| server | object | no | The value of the global variable of the Host type, see server definition | + +#### server + +| Fields | Type | Required | Description | +| ------------------ | ----- | -------- | ------------------------------------------------------- | +| host_id_list | array | no | Host ID list | +| ip_list | array | no | Static IP list, see ip for definition. ***Deprecated, it is recommended to use the host_id_list parameter***; if host_id_list and ip_list exist at the same time, the ip_list parameter will be ignored. | +| dynamic_group_list | array | no | Dynamic grouping list, see dynamic_group for definition | +| topo_node_list | array | no | Dynamic topo node list, see topo_node for definition | + +#### ip + +| Fields | Type | Required | Description | +| ----------- | ------ | -------- | ------------- | +| bk_cloud_id | int | yes | BK-Net ID | +| ip | string | yes | IP Address | + +#### dynamic_group + +| Fields | Type | Required | Description | +| ------ | ------ | -------- | --------------------- | +| id | string | yes | CMDB Dynamic Group ID | + +#### topo_node + +| Fields | Type | Required | Description | +| --------- | ------ | -------- | ------------------------------------------------------------ | +| id | long | yes | Dynamic topo node ID, corresponding to bk_inst_id in CMDB API | +| node_type | string | yes | Dynamic topo node type, corresponding to bk_obj_id in CMDB API, such as "module", "set" | + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "job_plan_id": 100, + "global_var_list": [ + { + "id": 436, + "server": { + "dynamic_group_list": [ + { + "id": "blo8gojho0skft7pr5q0" + } + ], + "host_id_list": [ + 101, + 102 + ], + "topo_node_list": [ + { + "id": 1000, + "node_type": "module" + } + ] + } + }, + { + "name": "param_name", + "value": "param_value" + } + ] +} +``` + +### Example of responses + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "job_instance_name": "Test", + "job_instance_id": 10000 + } +} +``` + +### Response Description + +#### response +| Fields | Type | Description | +|-----------|-----------|-----------| +| result | bool | Success or failure of the request. true: request successful; false: request failed | +| code | int | Error code. 0 means SUCCESS, >0 means FAIL | +| message | string | Error message | +| data | object | Data returned by request | +| permission | object | Permission information | +| request_id | string | Request chain id | + +#### data + +| Fields | Type | Description | +|-----------|-----------|-----------| +| job_instance_id | long | Job instance ID | +| job_instance_name | long | Job instance name | + diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/fast_execute_script.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/fast_execute_script.md new file mode 100644 index 0000000000..23aad4be69 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/fast_execute_script.md @@ -0,0 +1,125 @@ +### Function Description + +Start Job Plan + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +|---------------|------------|--------|------------| +| bk_scope_type | string | yes | Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | yes | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| script_version_id | long | no | Script version ID. When script_version_id is not empty, the script version corresponding to script_version_id is used| +| script_id | string | no |Script id. When script_id is passed in and script_version_id is empty, the online version of the script is used| +| script_content | string | no |Script content Base64. If script_version_id and script_id do not exist, script_content is used. Priority: script_version_id>script_id>script_content| +| task_name | string | no | Custom job name| +| script_param | string | no | Script parameter Base64 encoding.| +| timeout | long | no | Script execution timeout in seconds. The default value is 7200, and the value range is [1,86400]| +| account_alias | string | no | Execution account alias. The account_id takes precedence when both account_alias and account_id exist. | +| account_id | long | no |Execution account ID. The account_id takes precedence when both account_alias and account_id exist. | +| is_param_sensitive | int | no | Sensitive parameters will be hidden on the execution details page, 0: No (default), 1: Yes| +| script_language | int | no |Scripting languages: 1 - shell, 2 - bat, 3 - Perl, 4 - Python, 5 - PowerShell. Script_language needs to be specified when you pass in a custom script using script_content| +| target_server | object | no | Target server, see server definition| +| callback_url | string | no | Callback URL, when the task execution is completed, the JOB will call this URL to inform the task execution result. Callback protocol refer to the callback_protocol component documentation| + +#### server +| Fields | Type | Required | Description | +| ------------------ | ----- | -------- | ------------------------------------------------------- | +| host_id_list | array | no | Host ID list | +| ip_list | array | no | Static IP list, see ip for definition. ***Deprecated, it is recommended to use the host_id_list parameter***; if host_id_list and ip_list exist at the same time, the ip_list parameter will be ignored. | +| dynamic_group_list | array | no | Dynamic grouping list, see dynamic_group for definition | +| topo_node_list | array | no | Dynamic topo node list, see topo_node for definition | + +#### ip + +| Fields | Type | Required | Description | +|-----------|------------|--------|------------| +| bk_cloud_id | long | yes | BK-Net ID | +| ip | string | yes | IP Address | + +#### dynamic_group + +| Fields | Type | Required | Description | +| ------ | ------ | -------- | -------------- | +| id | string | yes | CMDB dynamic grouping ID| + +#### topo_node_list + +| Fields | Type | Required | Description | +|-----------|------------|--------|------------| +| id | long | yes |Dynamic topo node ID, corresponding to bk_inst_id in CMDB API| +| node_type | string | yes |Dynamic topo node type, corresponding to bk_obj_id in CMDB API, such as "module" and "set"| + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "script_version_id": 1, + "script_content": "ZWNobyAkMQ==", + "script_param": "aGVsbG8=", + "timeout": 1000, + "account_id": 1000, + "is_param_sensitive": 0, + "script_language": 1, + "target_server": { + "dynamic_group_list": [ + { + "id": "blo8gojho0skft7pr5q0" + } + ], + "host_id_list": [ + 101, + 102 + ], + "topo_node_list": [ + { + "id": 1000, + "node_type": "module" + } + ] + } +} +``` + +### Example of responses + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "job_instance_name": "API Quick execution script1521100521303", + "job_instance_id": 10000, + "step_instance_id": 10001 + } +} +``` + +### Response Description + +#### response +| Fields | Type | Description | +|-----------|-----------|-----------| +| result | bool | Whether the request was successful or not. True: request succeeded;False: request failed| +| code | int | Error code. 0 indicates success, >0 indicates failure| +| message | string |Error message| +| data | object |Data returned by request| +| permission | object |Permission information| +| request_id | string |Request chain id| + +#### data + +| Fields | Type | Description | +|-----------|-----------|-----------| +| job_instance_id | long | Job instance ID| +| job_instance_name | long | Job instance name| +| step_instance_id | long | Step instance ID| diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/fast_execute_sql.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/fast_execute_sql.md new file mode 100644 index 0000000000..82fd6575b3 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/fast_execute_sql.md @@ -0,0 +1,117 @@ +### Function Description + +Quick execution of SQL scripts + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +|---------------|------------|--------|------------| +| bk_scope_type | string | yes | Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | yes | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| script_version_id | long | no | SQL script version ID| +| script_id | string | no |Script id. When script_id is passed in and script_version_id is empty, the online version of the current script is used| +| script_content | string | no | Script content Base64. If script_version_id and script_id do not exist, script_content is used. Priority: script_version_id>script_id>script_content| +| timeout | int | no | Script timeout in seconds. Default 7200, value range 1 86400| +| db_account_id | long | yes |Db account ID for SQL execution| +| target_server | object | no | Target server, see server definition| +| callback_url | string | no | Callback URL, when the task execution is completed, the JOB will call this URL to inform the task execution result. Callback protocol refer to the callback_protocol component documentation| + +#### server + +| Fields | Type | Required | Description | +| ------------------ | ----- | -------- | ------------------------------------------------------- | +| host_id_list | array | no | Host ID list | +| ip_list | array | no | Static IP list, see ip for definition. ***Deprecated, it is recommended to use the host_id_list parameter***; if host_id_list and ip_list exist at the same time, the ip_list parameter will be ignored. | +| dynamic_group_list | array | no | Dynamic grouping list, see dynamic_group for definition | +| topo_node_list | array | no | Dynamic topo node list, see topo_node for definition | + +#### ip + +| Fields | Type | Required | Description | +| ----------- | ------ | -------- | ------------- | +| bk_cloud_id | long | yes | BK-Net ID | +| ip | string | yes | IP Address | + +#### dynamic_group + +| Fields | Type | Required | Description | +| ------ | ------ | -------- | -------------- | +| id | string | yes | CMDB dynamic grouping ID| + +#### topo_node_list + +| Fields | Type | Required | Description | +| --------- | ------ | -------- | ------------------------------------------------------------ | +| id | long | yes | Dynamic topo node ID, corresponding to bk_inst_id in CMDB API | +| node_type | string | yes | Dynamic topo node type, corresponding to bk_obj_id in CMDB API, such as "module,""set"| + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "script_version_id": 1, + "timeout": 1000, + "db_account_id": 32, + "target_server": { + "dynamic_group_list": [ + { + "id": "blo8gojho0skft7pr5q0" + } + ], + "host_id_list": [ + 101, + 102 + ], + "topo_node_list": [ + { + "id": 1000, + "node_type": "module" + } + ] + } +} +``` + +### Example of responses + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "job_instance_name": "API Quick SQL Execution1524454292038", + "job_instance_id": 10000, + "step_instance_id": 10001 + } +} +``` + +### Response Description + +#### response +| Fields | Type | Description | +|-----------|-----------|-----------| +| result | bool | Whether the request was successful or not. True: request succeeded;False: request failed| +| code | int | Error code. 0 indicates success, >0 indicates failure| +| message | string |Error message| +| data | object |Data returned by request| +| permission | object |Permission information| +| request_id | string |Request chain id| + +#### data + +| Fields | Type | Description | +|-----------|-----------|-----------| +| job_instance_id | long | Job instance ID| +| job_instance_name | long | Job instance name| +| step_instance_id | long | Step instance ID| diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/fast_transfer_file.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/fast_transfer_file.md new file mode 100644 index 0000000000..d1c6610a93 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/fast_transfer_file.md @@ -0,0 +1,173 @@ +### Function Description +Quick File transfer + +### Request Parameters +{{ common_args_desc }} + +#### Interface parameters +| Fields | Type | Required | Description | +|------------------|------------|--------|------------| +| bk_scope_type | string | yes | Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | yes | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| account_alias | string | no |Target execution account alias, available from the account page, recommended. When both account_alias and account_id exist, account_id takes precedence. | +| account_id | long | no |Target execution account ID, available from the get_account_List api. When both account_alias and account_id exist, account_id takes precedence. | +| file_target_path | string | yes |File transfer destination path| +| file_source_list | array | yes |File source object array, see file_source Definition below| +| timeout | int | no | Task timeout in seconds, default is 7200. Value range 1 86400.| +| download_speed_limit| int | no | Download speed limit in MB. If this parameter is not passed in, it means no speed limit| +| upload_speed_limit| int | no | Upload speed limit, in MB. If this parameter is not passed in, it means no speed limit| +| transfer_mode | int | no |Transmission mode. 1 - Strict mode, 2 - Forced mode. Force mode is used by default| +| target_server | object | no | Target server, see server definition| +| callback_url | string | no | Callback URL, when the task execution is completed, the JOB will call this URL to inform the task execution result. Callback protocol refer to the callback_protocol component documentation| + +#### file_source +| Fields | Type | Required | Description | +|---------------|------------|--------|------------| +| file_list | array | yes |Multiple files are supported. If the file source type is a server file, fill in the absolute path array of the source file; If the file source type is a third-party file source, the path filled in by COS file source is "bucketName/file path," for example: testbucket/test.txt| +| account | object | yes |For file source account number, see account definition. It is required when the file source type is server file source, but not required when the file source type is third-party file source| +| server | object | no | Source file server, see server definition| +| file_type | int | no | File source type, 1: server file, 3: Third-party file source file. Default value is 1 if not transferred| +| file_source_id | int | no |File source ID. When file_type is 3, select one of file_source_id and file_source_code to fill in. If both are filled in, file_source_id, the third-party file source Id, is preferred, which can be obtained from the step details in the returned result of get_job_detail api| +| file_source_code| string | no |File source code. When file_type is 3, select one of file_source_id and file_source_code to fill in. If both of them are filled in, file_source_id, the third-party file source ID, is preferred. It can be obtained from the file distribution page> Select File source file pop-up box of the operation platform| + +#### account + +| Fields | Type | Required | Description | +| ------ | ------ | -------- | ------------------------------------------------------------ | +| id | long | no | Source execution account ID. When alias and id exist at the same time, id takes precedence. | +| alias | string | no | Source execution account alias. When alias and id exist at the same time, id takes precedence. | + +#### server + +| Fields | Type | Required | Description | +| ------------------ | ----- | -------- | ------------------------------------------------------- | +| host_id_list | array | no | Host ID list | +| ip_list | array | no | Static IP list, see ip for definition. ***Deprecated, it is recommended to use the host_id_list parameter***; if host_id_list and ip_list exist at the same time, the ip_list parameter will be ignored. | +| dynamic_group_list | array | no | Dynamic grouping list, see dynamic_group for definition | +| topo_node_list | array | no | Dynamic topo node list, see topo_node for definition | + +#### ip_list + +| Fields | Type | Required | Description | +|-----------|------------|--------|------------| +| bk_cloud_id | long | yes | BK-Net ID | +| ip | string | yes | IP Address | + +#### topo_node_list + +| Fields | Type | Required | Description | +|-----------|------------|--------|------------| +| id | long | yes |Dynamic topo node ID, corresponding to bk_inst_id in CMDB API| +| node_type | string | yes |Dynamic topo node type, corresponding to bk_obj_id in CMDB API, such as "module" and "set"| + +### Example of request +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "file_target_path": "/tmp/", + "transfer_mode": 1, + "file_source_list": [ + { + "file_list": [ + "/tmp/REGEX:[a-z]*.txt" + ], + "account": { + "id": 100 + }, + "server": { + "dynamic_group_list": [ + { + "id": "blo8gojho0skft7pr5q0" + }, + { + "id": "blo8gojho0sabc7priuy" + } + ], + "host_id_list": [ + 101, + 102 + ], + "topo_node_list": [ + { + "id": 1000, + "node_type": "module" + } + ] + }, + "file_type": 1 + }, + { + "file_list": [ + "testbucket/test.txt" + ], + "file_type": 3, + "file_source_id": 1 + }, + { + "file_list": [ + "testbucket/test2.txt" + ], + "file_type": 3, + "file_source_code": "testInnerCOS" + } + ], + "target_server": { + "dynamic_group_list": [ + { + "id": "blo8gojho0skft7pr5q0" + }, + { + "id": "blo8gojho0sabc7priuy" + } + ], + "host_id_list": [ + 103, + 104 + ], + "topo_node_list": [ + { + "id": 1000, + "node_type": "module" + } + ] + }, + "account_id": 101 +} +``` +### Example of responses +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "job_instance_name": "API Quick Distribution File1521101427176", + "job_instance_id": 10000, + "step_instance_id": 10001 + } +} +``` + +### Response Description + +#### response +| Fields | Type | Description | +|-----------|-----------|-----------| +| result | bool | Whether the request succeeded or not. True: request succeeded;False: request failed| +| code | int | Error code. 0 indicates success, >0 indicates failure| +| message | string |Error message| +| data | object |Data returned by request| +| permission | object |Permission information| +| request_id | string |Request chain id| + +#### data + +| Fields | Type | Description | +|-----------|-----------|-----------| +| job_instance_id | long | Job instance ID| +| job_instance_name | long | Job instance name| +| step_instance_id | long | Step instance ID| diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/generate_local_file_upload_url.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/generate_local_file_upload_url.md new file mode 100644 index 0000000000..0e76c96b7e --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/generate_local_file_upload_url.md @@ -0,0 +1,72 @@ +### Function Description + +Generate the local file upload URL. + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +|----------------------------|------------|--------|------------| +| bk_scope_type | string | yes | Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | yes | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| file_name_list | string [] | yes |List of file names to upload| + + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "file_name_list": [ + "file1.txt", + "file2.txt" + ] +} +``` + +### Example of responses + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "url_map": { + "file1.txt": { + "upload_url": "http://bkrepo.com/generic/temporary/upload/bkjob/localupload/1/008f821f-259b-4f62-bd84-1e89d6f05f0d/admin/file1.txt?token=30adf862fdce4b02b909e6a1a1c762c6", + "path": "1/008f821f-259b-4f62-bd84-1e89d6f05f0d/admin/file1.txt" + }, + "file2.txt": { + "upload_url": "http://bkrepo.com/generic/temporary/upload/bkjob/localupload/1/008f821f-259b-4f62-bd84-1e89d6f05f0d/admin/file2.txt?token=30adf862fdce4b02b909e6a1a1c762c6", + "path": "1/008f821f-259b-4f62-bd84-1e89d6f05f0d/admin/file2.txt" + } + } + } +} +``` + +### Response Description + +#### response +| Fields | Type | Description | +|-----------|-----------|-----------| +| result | bool | Whether the request was successful or not. True: request succeeded;False: request failed| +| code | int | Error code. 0 indicates success, >0 indicates failure| +| message | string |Error message| +| data | object |Data returned by request| +| permission | object |Permission information| +| request_id | string |Request chain id| + +#### data + +| Fields | Type |Whether the field must exist | Description | +|-----------|----------|---------------|---------| +| url_map | map | yes | key: file name passed in, value: Upload_url is the file upload address with credentials, and path is the path to the file distribution interface when distributing the file| diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_account_list.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_account_list.md new file mode 100644 index 0000000000..f0e0339eb4 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_account_list.md @@ -0,0 +1,96 @@ +### Function Description + +Query the list of executive accounts to which users have access under the business + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +|----------------------|------------|--------|------------| +| bk_scope_type | string | yes | Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | yes | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| category | int | no | Account usage (1: system account, 2: DB account)| +| start | int | no | Start position of paging record, default 0| +| length | int | no |The maximum number of records returned in a single time is 1000 at most, default 20| + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "category": 1, + "start": 0, + "length": 1 +} +``` + +### Example of responses + +```json +{ + "code": 0, + "message": null, + "result": true, + "data": { + "start": 0, + "total": 12, + "data": [ + { + "id": 70, + "account": "aaa", + "alias": "aaa", + "category": 1, + "type": 1, + "os": "Linux", + "description":"xxx", + "creator": "admin", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "create_time": 1614659536108, + "last_modify_user": "admin", + "last_modify_time": 1614659536116 + } + ], + "length": 1 + }, + "request_id": "4e7acb216087eb96" +} +``` + +### Response Description + +#### response +| Fields | Type | Description | +|-----------|-----------|-----------| +| result | bool | Whether the request was successful or not. True: request succeeded;False: request failed| +| code | int | Error code. 0 indicates success, >0 indicates failure| +| message | string |Error message| +| data | object |Data returned by request| +| permission | object |Permission information| +| request_id | string |Request chain id| + +#### data.data + +| Fields | Type | Description | +|-----------|-----------|-----------| +| id | long | Account ID| +| bk_scope_type | string |Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| account | string | Account name| +| alias | string | Account alias| +| category | int | Account usage (1: system account, 2: DB account)| +| type | int | Account type (1: Linux, 2: Windows，9：MySQL，10： Oracle，11：DB2）| +| db_system_account_id | long | This field takes effect when the account usage is DB account, indicating the system account ID corresponding to DB account| +| os | string | This field takes effect when the account is used as a system account. The OS corresponding to the account| +| description | string | Account description| +| creator | string | creator| +| create_time | long | Creation time Unix timestamp (ms)| +| last_modify_user | string | Last modify user| +| last_modify_time | long | Last modified time Unix timestamp (ms)| diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_cron_detail.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_cron_detail.md new file mode 100644 index 0000000000..ec7c0cf9ab --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_cron_detail.md @@ -0,0 +1,147 @@ +### Function Description + +Query Cron job details + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +|------------------------|------------|--------|------------| +| bk_scope_type | string | yes | Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | yes | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| id | long | no | Cron Job ID | + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "id": 1 +} +``` + +### Example of responses + +```json +{ + "result": true, + "code": 0, + "message": "", + "data": { + "bk_scope_type": "biz", + "bk_scope_id": "1", + "job_plan_id": 100, + "id": 1, + "name": "test", + "status": 1, + "expression": "0/5 * * * ?", + "global_var_list": [ + { + "id": 436, + "name": "ip", + "server": { + "dynamic_group_list": [ + { + "id": "blo8gojho0skft7pr5q0" + }, + { + "id": "blo8gojho0sabc7priuy" + } + ], + "ip_list": [ + { + "bk_host_id": 101, + "bk_cloud_id": 0, + "ip": "127.0.0.1" + }, + { + "bk_host_id": 102, + "bk_cloud_id": 0, + "ip": "127.0.0.2" + } + ], + "topo_node_list": [ + { + "id": 1000, + "node_type": "module" + } + ] + } + }, + { + "id": 437, + "name": "text", + "value": "new String value" + } + ], + "creator": "admin", + "create_time": 1546272000000, + "last_modify_user": "admin", + "last_modify_time": 1577807999999 + } +} +``` + +### Response Description + +#### response +| Fields | Type | Description | +|-----------|-----------|-----------| +| result | bool | Whether the request was successful or not. True: request succeeded;False: request failed| +| code | int | Error code. 0 indicates success, >0 indicates failure| +| message | string |Error message| +| data | object |Data returned by request| +| permission | object |Permission information| +| request_id | string |Request chain id| + +#### data +| Fields | Type | Description | +|------------------|-----------|-----------| +| bk_scope_type | string |Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| job_plan_id | long | Job Plan ID | +| id | long | Cron job ID | +| name | string | Cron job name | +| status | int | Cron job status: 1. Started, 2. Paused | +| expression | string | Timing rules for Cron Job crontab, required when creating, optional when modifying. The meaning of each field is: minute hour day month week, for example: 0/5 * * * * ? means execute every 5 minutes | +| global_var_list | array | Global variable information| +| creator | string | Cron job creator| +| create_time | long | Creation time, Unix timestamp| +| last_modify_user | string | Cron job modifier| +| last_modify_time | long | Last modified time, Unix timestamp| + +#### global_var + +| Fields | Type | Description | +|-----------|-----------|------------| +| id | long | Global variable id, unique identification. If the id is empty, then name is used as the unique identification| +| name | string | Global variable name| +| value | string | Character, password, value of global variable of array type| +| server | object | Host type global variable value| + +#### server +| Fields | Type | Description | +|-----------------------|-------|------------| +| ip_list | array |Static IP list| +| dynamic_group_list | array |Dynamic grouping list| +| topo_node_list | array |Dynamic topo node list| + +#### ip + +| Fields | Type | Description | +|-------------|---------|---------| +| bk_cloud_id | int | BK-Net ID | +| ip | string | IP Address | + +#### topo_node +| Fields | Type | Description | +|------------------|--------|------------| +| id | long | Dynamic topo node ID, corresponding to bk_inst_id in CMDB API| +| node_type | string |Dynamic topo node type, corresponding to bk_obj_id in CMDB API, such as "module" and "set"| diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_cron_list.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_cron_list.md new file mode 100644 index 0000000000..dc9d199cb0 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_cron_list.md @@ -0,0 +1,105 @@ +### Function Description + +Query information of cron jobs under business + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +|------------------------|------------|--------|------------| +| bk_scope_type | string | yes | Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | yes | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| name | string | no | Cron job name | +| id | long | no | Cron Job ID. If it exists, ignore other filtering criteria and only query the specified job information | +| status | int | no | Cron job status: 1. Started, 2. Paused | +| creator | string | no | Cron job creator | +| create_time_start | long | no | Creation time start, Unix timestamp| +| create_time_end | long | no |Create time end, Unix timestamp| +| last_modify_user | string | no | Job modifier | +| last_modify_time_start | long | no | Last modification time start, Unix timestamp| +| last_modify_time_end | long | no | Last modification time end, Unix timestamp| +| start | int | no | Start position of paging record, default 0| +| length | int | no | The maximum number of records returned in a single time is 1000 at most, default 20| + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "name": "test", + "id": 1000031, + "status": 1, + "creator": "admin", + "create_time_start": 1601371525, + "create_time_end": 1617285956, + "last_modify_user": "admin", + "last_modify_time_start": 1601371525, + "last_modify_time_end": 1617286227, + "start": 0, + "length": 1 +} +``` + +### Example of responses + +```json +{ + "code": 0, + "result": true, + "data": { + "start": 0, + "total": 8, + "data": [ + { + "id": 1000031, + "name": "test", + "status": 1, + "creator": "admin", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "job_plan_id": 1000193, + "expression": "* * * * * *", + "create_time": 1617285956, + "last_modify_user": "admin", + "last_modify_time": 1617286227 + } + ], + "length": 1 + }, + "request_id": "9720d3549c49a48a" +} +``` + +### Response Description + +#### response +| Fields | Type | Description | +|-----------|-----------|-----------| +| result | bool | Whether the request succeeded or not. True: request succeeded;False: request failed| +| code | int | Error code. 0 indicates success, >0 indicates failure| +| message | string |Error message| +| data | object |Data returned by request| +| permission | object |Permission information| +| request_id | string |Request chain id| + +#### data +| Fields | Type | Description | +|------------------|-----------|-----------| +| bk_scope_type | string |Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| job_plan_id | long | Job Plan ID | +| id | long | Cron job ID | +| name | string | Cron job name | +| status | int | Cron job status: 1. Started, 2. Paused | +| expression | string | Timing rules for Cron Job crontab, required when creating, optional when modifying. The meaning of each field is: minute hour day month week, for example: 0/5 * * * ? means execute every 5 minutes | +| creator | string | Job creator| +| create_time | long | Created time, Unix timestamp| +| last_modify_user | string | Job modifier | +| last_modify_time | long | Last modified time, Unix timestamp| diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_dangerous_rule_list.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_dangerous_rule_list.md new file mode 100644 index 0000000000..d77f0ad4e7 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_dangerous_rule_list.md @@ -0,0 +1,79 @@ +### Function Description + +Get dangerous rule list + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +| -------------------- | ------ | -------- | ------------------------------------------------------------ | +| expression | string | no | Expression | +| script_language_list | array | no | Script language:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - sql | +| description | string | no | Rule description | +| action | int | no | Processing actions: 1- Scan, 2- Intercept | + + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "script_language_list": [1], + "action": 2 +} +``` + +### Example of responses + +```json +{ + "code": 0, + "result": true, + "data": [ + { + "id": 1, + "expression": "rm", + "script_language_list": [1], + "description": "drangerous!!!" + "action": 2, + "status": 0, + "creator": "admin", + "create_time": 1695193968000, + "last_modify_user": "admin", + "last_modify_time": 1695302417000 + } + ] +} +``` + +### Response Description + +#### response + +| Fields | Type | Description | +| ---------- | ------ | ------------------------------------------------------------ | +| result | bool | Whether the request succeeded or not. True: request succeeded;False: request failed | +| code | int | Error code. 0 indicates success, >0 indicates failure | +| message | string | Error message | +| data | object | Data returned by request | +| permission | object | Permission information | + +#### data + +| Fields | Type | Description | +| -------------------- | ------ | ------------------------------------------------------------ | +| id | long | Rule id | +| expression | string | expression | +| script_language_list | array | Script language:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - sql | +| description | string | Description | +| action | int | Processing actions: 1- Scan, 2- Intercept | +| status | int | Enabling status: 0- disabled, 1- enabled | +| creator | string | Creator | +| create_time | long | Created time, Unix timestamp | +| last_modify_user | string | Last modify user | +| last_modify_time | long | Last modified time, Unix timestamp | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_job_instance_global_var_value.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_job_instance_global_var_value.md new file mode 100644 index 0000000000..3fcb3170be --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_job_instance_global_var_value.md @@ -0,0 +1,136 @@ +### Function Description + +Gets the value of the job instance global variable + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +|-----------|------------|--------|------------| +| bk_scope_type | string | yes | Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | yes | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| job_instance_id | long | yes |Job instance ID| + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "job_instance_id": 100 +} +``` + +### Example of responses + +```json +{ + "result": true, + "code": 0, + "message": "", + "data": { + "job_instance_id": 100, + "step_instance_var_list": [ + { + "step_instance_id": 292778, + "global_var_list": [ + { + "name": "aa", + "value": "AA", + "type": 1 + }, + { + "name": "password", + "value": "mypassword", + "type": 4 + } + ] + }, + { + "step_instance_id": 292779, + "global_var_list": [ + { + "name": "aa", + "value": "AAAA", + "type": 1 + }, + { + "name": "password", + "value": "mypassword", + "type": 4 + } + ] + } + ] + } +} +``` + +### Response Description + +#### response +| Fields | Type | Description | +|-----------|-----------|-----------| +| result | bool | Whether the request was successful or not. True: request succeeded;False: request failed| +| code | int | Error code. 0 indicates success, >0 indicates failure| +| message | string |Error message returned by request failure| +| data | object |Data returned by request| +| permission | object |Permission information| +| request_id | string |Request chain id| + +#### data + +| Fields | Type | Description | +|-----------|-----------|-----------| +| job_instance_id | long | Job instance ID| +| step_instance_var_list | array | Job step instance global variable value. See step_instance_var for definitions| + +#### step_instance_var + +| Fields | Type | Description | +| ---------------- | ----- | -------------------------------- | +| step_instance_id | long |Step instance ID | +| global_var_list | array |For global variable value list, see global_var for definition| + +#### global_var + +| Fields | Type | Required | Description | +| ------ | ------ | ---- | ---------------------------------------------------------- | +| id | long | no |Global variable id, unique identification. If the id is empty, then name is used as the unique identification| +| name | string | no |Global variable name | +| value | string | no | Values of global variables of String, Ciphertext, Namespace, and Array types | +| server | object | no |For the value of the host type global variable, see server definition | + +#### server + +| Fields | Type | Required | Description | +| ------------------ | ----- | -------- | ----------------------------------- | +| ip_list | array | no | Static IP list, as defined in ip | +| dynamic_group_list | array | no | Dynamic grouping list, see dynamic_group for definition | +| topo_node_list | array | no | For dynamic topo node list, see topo_node for definition| + +#### ip + +| Fields | Type | Required | Description | +| ----------- | ------ | -------- | ------------- | +| bk_cloud_id | int | yes | BK-Net ID | +| ip | string | yes | IP Address | + +#### dynamic_group + +| Fields | Type | Required | Description | +| ------ | ------ | -------- | -------------- | +| id | string | yes | CMDB dynamic grouping ID| + +#### topo_node + +| Fields | Type | Required | Description | +| --------- | ------ | -------- | ------------------------------------------------------------ | +| id | long | yes | Dynamic topo node ID, corresponding to bk_inst_id in CMDB API | +| node_type | string | yes | Dynamic topo node type, corresponding to bk_obj_id in CMDB API, such as "module" and "set"| \ No newline at end of file diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_job_instance_ip_log.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_job_instance_ip_log.md new file mode 100644 index 0000000000..00a72c7686 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_job_instance_ip_log.md @@ -0,0 +1,144 @@ +### Function Description + +Query job execution log by host + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +|-----------|------------|--------|------------| +| bk_scope_type | string | yes | Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | yes | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| job_instance_id | long | yes |Job instance ID| +| step_instance_id | long | yes |Step instance ID| +| bk_cloud_id | int | no |Target server BK-Net ID| +| ip | string | no |Destination server IP. ***Deprecated, it is recommended to use the bk_host_id parameter| +| bk_host_id | string | no | Host ID| + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "job_instance_id": 50, + "step_instance_id": 100, + "bk_host_id": 101 +} +``` + +### Example of responses + +#### Script execution steps +```json +{ + "result": true, + "code": 0, + "message": "", + "data": { + "log_type": 1, + "bk_host_id": 101, + "ip": "127.0.0.1", + "bk_cloud_id": 0, + "log_content": "[2018-03-15 14:39:30][PID:56875] job_start\n" + } +} +``` + +#### File distribution steps + +```json +{ + "result": true, + "code": 0, + "message": "", + "data": { + "log_type": 2, + "bk_host_id": 101, + "ip": "127.0.0.1", + "bk_cloud_id": 0, + "file_logs": [ + { + "mode": 1, + "src_ip": { + "host_id": 102, + "bk_cloud_id": 0, + "ip": "127.0.0.2" + }, + "src_path": "/data/1.log", + "dest_ip": { + "bk_host_id": 101, + "bk_cloud_id": 0, + "ip": "127.0.0.1" + }, + "dest_path": "/tmp/1.log", + "status": 4, + "log_content": "[2021-06-28 11:32:16] FileName: /tmp/1.log FileSize: 9.0 Bytes State: dest agent success download file Speed: 1 KB/s Progress: 100% StatusDesc: dest agent success download file Detail: success" + }, + { + "mode": 0, + "src_ip": { + "bk_host_id": 102, + "bk_cloud_id": 0, + "ip": "127.0.0.2" + }, + "src_path": "/data/1.log", + "status": 4, + "log_content": "[2021-06-28 11:32:16] FileName: /data/1.log FileSize: 9.0 Bytes State: source agent success upload file Speed: 1 KB/s Progress: 100% StatusDesc: source agent success upload file Detail: success upload" + } + ] + } +} +``` + +**Return result description** + +- The file distribution log returns the file upload task log of the source server in addition to the file download task log of the target server (mode=0) +- Dest_ip corresponds to the bk_Cloud_id/IP of the request parameter + +### Response Description + +#### response +| Fields | Type | Description | +|-----------|-----------|-----------| +| result | bool | Request success or failure. true: Request successful; false: Request failed | +| code | int | Error code. 0 indicates success, >0 indicates failure| +| message | string |Error message| +| data | object |Data returned by request| +| permission | object |Permission information| +| request_id | string |Request chain id| + +#### data + +| Fields | Type | Description | +|-----------|-----------|-----------| +| bk_cloud_id | int | Target server cloud area ID | +| ip | string | Destination server IP address| +| log_type | int | Log type. 1 - Script execution task log; 2 - File distribution task log | +| log_content | string | Log content of job script output| +| file_logs | array | File distribution task log. See file_log for definition| + +#### file_log + +| Fields | Type | Description | +|-----------|-----------|-----------| +| mode | int | Distribution mode. 0: Upload; 1: Download | +| src_ip | object |File source host IP. see ip for definition | +| src_path | string | Source file paths | +| dest_ip | object | Distribute the target host IP, with value for mode=1. See ip for definition. | +| dest_path | string | Target path, with value for mode=1 | +| status | int | Task status. 1-Waiting; 2-Uploading; 3-Downloading; 4- Success; 5- Failure | +| log_content | string | File distribution log contents | + +#### ip + +| Fields | Type | Description | +|-----------|------------|--------| +| bk_cloud_id | long | BK-Net ID | +| ip | string | IP Address | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_job_instance_list.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_job_instance_list.md new file mode 100644 index 0000000000..da443c2910 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_job_instance_list.md @@ -0,0 +1,111 @@ +### Function Description + +Query job instance list (execution history) + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +| ----------------- | ------ | ---- | ------------------------------------------------------------ | +| bk_scope_type | string | yes | Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | yes | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| create_time_start | long | yes |Creation time start, Unix timestamp, in milliseconds | +| create_time_end | long | yes |Creation time end, Unix timestamp, in milliseconds | +| job_instance_id | long | no |Task instance ID. If job_instance_id is in or out, other query criteria are ignored | +| job_cron_id | long | no |Cron Job ID | +| operator | string | no |Executor, accurate match | +| name | string | no |Task name, fuzzy match | +| launch_mode | int | no |Launch mode. 1 - Web UI, 2 - API, 3 - Cron Job | +| type | int | no |Task type. 0 - Job Execution, 1 - Script Execution, 2 - File Transfer | +| status | int | no | Job status code. 1 - Pending; 2 - Running 3 - Successful; 4 - Failed; 5 - Skipped; 6 - Ignore Error; 7 - Waiting; 8 - Terminated; 9 - Abnormal; 10 - Terminating; 11 - Terminate Success; 13 - Termination Confirmed; 14 - Abandoned| +| ip | string | no |Execute target server IP for accurate matching | +| start | int | no |Start position of paging record, default 0 | +| length | int | no |The maximum number of records returned in a single time is 1000 at most, default 20 | + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "type": 0, + "launch_mode": 1, + "status": 3, + "operator": "admin", + "name": "test", + "create_time_start": 1546272000000, + "create_time_end": 1577807999999, + "start": 0, + "length": 20 +} +``` + +### Example of responses + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "data": [ + { + "bk_scope_type": "biz", + "bk_scope_id": "1", + "id": 100, + "job_template_id": 1, + "job_plan_id": 1, + "name": "test", + "operator": "admin", + "create_time": 1546272000000, + "start_time": 1546272000000, + "end_time": 1546272001000, + "total_time": 1000, + "launch_mode": 1, + "task_status": 3, + "task_type": 0 + } + ], + "start": 0, + "length": 20, + "total": 1 + } +} +``` + +### Response Description + +#### response +| Fields | Type | Description | +|-----------|-----------|-----------| +| result | bool | Whether the request succeeded or not. True: request succeeded;False: request failed| +| code | int | Error code. 0 indicates success, >0 indicates failure| +| message | string |Error message| +| data | object |Data returned by request| +| permission | object |Permission information| +| request_id | string |Request chain id| + +#### data + +| Fields | Type | Description | +| --------------- | ------ | ------------------------------------------------------------ | +| bk_scope_type | string | Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| id | long | Job Plan ID | +| job_template_id | long | Job Template ID, which has a value when the task is Job Plan | +| job_plan_id | long | Job Plan ID, which has a value when the task is a Job Plan | +| name | string |Task name | +| operator | string |Operator | +| create_time | long | Creation time, Unix timestamp, in milliseconds | +| status | int | Job status code. 1 - Pending; 2 - Running 3 - Successful; 4 - Failed; 5 - Skipped; 6 - Ignore Error; 7 - Waiting; 8 - Terminated; 9 - Abnormal; 10 - Terminating; 11 - Terminate Success; 13 - Termination Confirmed; 14 - Abandoned| +| type | int | Task type. 0 - Plan execution, 1 - Script execution, 2 - File distribution | +| launch_mode | int | Launch mode. 1 - Web UI, 2 - API, 3 - Cron Job | +| start_time | long | Task start time, Unix timestamp, in milliseconds | +| end_time | long | Task end time, Unix timestamp, in milliseconds | +| total_time | long | Task execution time, Unix timestamp, in milliseconds | \ No newline at end of file diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_job_instance_status.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_job_instance_status.md new file mode 100644 index 0000000000..ccdcd22982 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_job_instance_status.md @@ -0,0 +1,144 @@ +### Function Description + +Query job execution status based on job instance ID + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +|------------------|------------|--------|------------| +| bk_scope_type | string | yes | Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | yes | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| job_instance_id | long | yes |Job instance ID| +| return_ip_result | boolean | no |Whether to return the task details on each ip corresponds to step_ip_result_list in the return result. The default value is false. | + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "job_instance_id": 100 +} +``` + +### Example of responses + +```json +{ + "result": true, + "code": 0, + "message": "", + "data": { + "finished": true, + "job_instance": { + "job_instance_id": 100, + "bk_scope_type": "biz", + "bk_scope_id": "1", + "name": "API Quick execution script1521089795887", + "create_time": 1605064271000, + "status": 4, + "start_time": 1605064271000, + "end_time": 1605064272000, + "total_time": 1000 + }, + "step_instance_list": [ + { + "status": 4, + "total_time": 1000, + "name": "API Quick execution scriptxxx", + "step_instance_id": 75, + "execute_count": 0, + "create_time": 1605064271000, + "end_time": 1605064272000, + "type": 1, + "start_time": 1605064271000, + "step_ip_result_list": [ + { + "bk_host_id": 101, + "ip": "127.0.0.1", + "bk_cloud_id": 0, + "status": 9, + "tag": "", + "exit_code": 0, + "error_code": 0, + "start_time": 1605064271000, + "end_time": 1605064272000, + "total_time": 1000 + } + ] + } + ] + } +} +``` +### Response Description + +#### response +| Fields | Type | Description | +|-----------|-----------|-----------| +| result | bool | Whether the request was successful or not. True: request succeeded;False: request failed| +| code | int | Error code. 0 indicates success, >0 indicates failure| +| message | string |Error message| +| data | object |Data returned by request| +| permission | object |Permission information| +| request_id | string |Request chain id| + +#### data + +| Fields | Type | Description | +|-----------|-----------|-----------| +| finished | bool | End job| +| job_instance | object | Job instance basic information. See job_instance definition| +| step_instance_list | array | List of job steps. See step_instance definition| + +#### job_instance + +| Fields | Type | Description | +|-----------|-----------|-----------| +| name | string | Job instance name| +| status | int | Job status code. 1 - Pending; 2 - Running 3 - Successful; 4 - Failed; 5 - Skipped; 6 - Ignore Error; 7 - Waiting; 8 - Terminated; 9 - Abnormal; 10 - Terminating; 11 - Terminate Success; 13 - Termination Confirmed; 14 - Abandoned| +| create_time | long | Job creation time, Unix timestamp, in milliseconds| +| start_time | long | Execution start time, Unix timestamp, in milliseconds| +| end_time | long | Execution end time, Unix timestamp, in milliseconds| +| total_time | int | Total elapsed time in milliseconds| +| bk_scope_type | string |Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| job_instance_id | long | Job instance ID| + +#### step_instance + +| Fields | Type | Description | +|-----------|-----------|-----------| +| step_instance_id | long | Job step instance ID| +| type | int | Step Type: 1. Script step; 2. File step; 4. SQL step| +| name | string | Step name| +| status | int | Step status code. 1 - Pending; 2 - Running 3 - Successful; 4 - Failed; 5 - Skipped; 6 - Ignore Error; 7 - Waiting; 8 - Terminated; 9 - Abnormal; 10 - Terminating; 11 - Terminate Success; 13 - Termination Confirmed; 14 - Abandoned| +| create_time | long | Job step instance creation time, Unix timestamp, in milliseconds| +| start_time | long |Execution start time, Unix timestamp, in milliseconds| +| end_time | long |Execution end time, Unix timestamp, in milliseconds| +| total_time | int |Total elapsed time in milliseconds| +| execute_count | int | Step retry times| +| step_ip_result_list | array | See step_ip_result for the task execution result of each host| + + +#### step_ip_result + +| Fields | Type | Description | +|-----------|-----------|-----------| +| bk_host_id | long | Host ID | +| ip | string | IP | +| bk_cloud_id | long | BK-Net ID | +| status | int | Agent task execution status: 1. Agent exception; Pending implementation Successful execution Failed to distribute the task; 404. Forced termination of task failed| +| tag | string |The user customizes the output results through the job_SUCCESS/job_fail function template. This parameter exists only for script tasks| +| exit_code | int |Script task exit code| +| error_code | int |Agent task status code, 1 - Agent exception; 5 - Waiting for execution; 7 - Executing; 9 - Execution succeeded; 11 - Execution failed; 12 - Task failed to deliver; 403 - Task forced termination succeeded; 404 - Task forced termination failed| +| start_time | long |Execution start time, Unix timestamp, in milliseconds| +| end_time | long |Execution end time, Unix timestamp, in milliseconds| +| total_time | int |Total elapsed time in milliseconds| diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_job_plan_detail.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_job_plan_detail.md new file mode 100644 index 0000000000..6b5c3503b4 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_job_plan_detail.md @@ -0,0 +1,311 @@ +### Function Description + +Check Job Plan details by Job Plan ID + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +|-------------|-------------|--------|--------------| +| bk_scope_type | string | yes | Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | yes | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| job_plan_id | long | yes |Job Plan ID| + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "job_plan_id": 100 +} +``` + +### Example of responses +```json +{ + "result": true, + "code": 0, + "message": "", + "data": { + "bk_scope_type": "biz", + "bk_scope_id": "1", + "job_plan_id": 100, + "name": "test", + "creator": "admin", + "create_time": 1546272000000, + "last_modify_user": "admin", + "last_modify_time": 1577807999999, + "global_var_list": [ + { + "id": 11, + "type": 1, + "name": "varName", + "value": "value is Me", + "description": "hello", + "required": 1 + }, + { + "id": 12, + "type": 3, + "name": "servers", + "description": "", + "required": 0, + "server": { + "dynamic_group_list": [ + { + "id": "blo8gojho0skft7pr5q0" + }, + { + "id": "blo8gojho0sabc7priuy" + } + ], + "ip_list": [ + { + "bk_host_id": 101, + "bk_cloud_id": 0, + "ip": "127.0.0.1" + }, + { + "bk_host_id": 102, + "bk_cloud_id": 0, + "ip": "127.0.0.2" + } + ], + "topo_node_list": [ + { + "id": 1000, + "node_type": "module" + } + ] + } + } + ], + "step_list": [ + { + "id": 1059, + "type": 1, + "name": "run local script", + "script_info": { + "script_type": 1, + "script_timeout": 1000, + "script_content": "ZWNobyAkMSAkMiAkMw==", + "script_param": "YTEgYTIgYTM=", + "is_param_sensitive": 0, + "account": "root" + } + }, + { + "id": 1060, + "type": 1, + "name": "run cite script", + "script_info": { + "script_type": 2, + "script_id": "aaaaa-bbb-ccc-ddddd", + "script_version_id": 1078, + "script_timeout": 1000, + "script_param": "YTEgYTIgYTM=", + "is_param_sensitive": 1, + "account": "root" + } + }, + { + "id": 1061, + "type": 2, + "name": "xxx", + "file_info": { + "file_source": [ + { + "file_list": [ + "/tmp/REGEX:[a-z]*.txt" + ], + "server": { + "variable": "servers" + }, + "account": { + "id": 1, + "name": "root" + }, + "file_type": 1 + }, + { + "file_list": [ + "testbucket/test.txt" + ], + "file_type": 3, + "file_source_id": 1 + } + ], + "file_destination": { + "path": "/tmp/", + "account": { + "id": 1, + "name": "root" + }, + "server": { + "variable": "", + "dynamic_group_list": [ + { + "id": "blo8gojho0skft7pr5q0" + }, + { + "id": "blo8gojho0sabc7priuy" + } + ], + "ip_list": [ + { + "bk_host_id": 103, + "bk_cloud_id": 0, + "ip": "127.0.0.3" + }, + { + "bk_host_id": 104, + "bk_cloud_id": 0, + "ip": "127.0.0.4" + } + ], + "topo_node_list": [ + { + "id": 1000, + "node_type": "module" + } + ] + } + }, + "timeout": 60, + "transfer_mode": 1, + "upload_speed_limit": 1000, + "download_speed_limit": 1000 + } + } + ] + } +} +``` + + +### Response Description + +#### response +| Fields | Type | Description | +|-----------|-----------|-----------| +| result | bool | Whether the request succeeded or not. True: request succeeded;False: request failed| +| code | int | Error code. 0 indicates success, >0 indicates failure| +| message | string |Error message| +| data | object |Data returned by request| +| permission | object |Permission information| +| request_id | string |Request chain id| + +#### data + +| Fields | Type | Description | +|------------------|-----------|-----------| +| bk_scope_type | string |Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| job_plan_id | long | Job Plan ID | +| name | string | Job name| +| creator | string | Job creator account number| +| create_time | long | Creation time, Unix timestamp| +| last_modify_user | string | Job latest modifier | +| last_modify_time | long | Last modified time, Unix timestamp| +| step_list | array | Step object| +| global_var_list | array | Global variable information| + +#### step + +| Fields | Type | Description | +|-----------|-----------|-----------| +| id | long | Job step ID| +| name | string | Job step name| +| type | int | Step Type: 1. Script step; Manual confirmation step| +| script_info | object | Script information. This field is only available when type=1. | +| file_info | object | File transfer step information. This field is only available when type=2| + +#### script_info +| Fields | Type | Description | +|-----------|-----------|-----------| +| script_type | int | Script Type: 1 local Script 2 reference Script 3 public script| +| script_id | string | Script id. This field is only available when script_type=2,3. | +| script_version_id | long | Script version ID. This field is only available when script_type=2,3. | +| script_content | string | Script content. This field is only available when type=1. | +| script_language | int |Scripting languages: 1 byte shell, 2 byte bat, 3 byte Perl, 4 byte Python, 5 byte PowerShell, 6 byte SQL| +| script_param | string | Script parameter| +| script_timeout | int | Script timeout in seconds. Default 3600, value range 60-86400| +| is_param_sensitive | int | Sensitive parameter, 0. Not (default), 1. Yes.| +| account | object | Execution account name/alias| + +#### file_info +| Fields | Type | Description | +|----------------------|--------|-----------| +| file_source_list | array |Source file information| +| file_destination | object |Target information| +| timeout | int | File transfer timeout settings| +| transfer_mode | int | File transfer mode| +| upload_speed_limit | int | Upload speed limit| +| download_speed_limit | int | Download speed limit| + +#### global_var + +| Fields | Type | Description | +|-------------|-----------|------------| +| id | long | Global variable id. If the id is empty, then name is used as the unique identification| +| name | string | Global variable name| +| description | string | Global variable description| +| type | int | Global variable type| +| required | int | Is this variable required | +| value | string | Values of global variables of String, Ciphertext, Namespace, and Array types| +| server | object | Value of global variable of Host type| + +#### server +| Fields | Type | Description | +|-----------------------|-------|------------| +| variable | string |Referenced variable name| +| ip_list | array |Static IP list| +| dynamic_group_list | array |Dynamic group ID list| +| topo_node_list | array |Dynamic topo node list| + +#### ip +| Fields | Type | Description | +|-------------|---------|---------| +| bk_host_id | long | Host ID | +| bk_cloud_id | int | BK-Net ID | +| ip | string | IP Address | + +#### dynamic_group +| Fields | Type | Description | +|-----|---------|------------| +| id | string | Dynamic Group ID | + +#### topo_node +| Fields | Type | Description | +|------------------|--------|------------| +| id | long | Dynamic topo node ID, corresponding to bk_inst_id in CMDB API| +| node_type | string |Dynamic topo node type, corresponding to bk_obj_id in CMDB API, such as "module" and "set"| + +#### account +| Fields | Type | Required | Description | +|-------|--------|--------|------------| +| id | long | no | Account ID| +| name | string | no | Account name| + +#### file_source +| Fields | Type | Description | +|-----------------|---------|-----------| +| file_list | array | Absolute path array of source file, supporting multiple files| +| account | object |Execution account name/alias| +| server | object |Target server| +| file_type | int | File source type, 1: server file, 2: Local files, 3: third-party file source files| +| file_source_id | int | Third-party file source Id when file source type is third-party file source file| + +#### file_destination +| Fields | Type | Description | +|-----------------|---------|-----------| +| path | string |Path where the target file is stored| +| account | object |Execution account name/alias| +| server | object |Target server| diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_job_plan_list.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_job_plan_list.md new file mode 100644 index 0000000000..168fb15d46 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_job_plan_list.md @@ -0,0 +1,100 @@ +### Function Description + +Search Job Plan list + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +|------------------------|------------|--------|------------| +| bk_scope_type | string | yes | Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | yes | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| job_template_id | long | no | Job template ID| +| creator | string | no | Job Plan creator | +| name | string | no |Job Plan name, fuzzy match| +| create_time_start | long | no | Creation time start, Unix timestamp| +| create_time_end | long | no | Creation time end, Unix timestamp| +| last_modify_user | string | no | Last modify user | +| last_modify_time_start | long | no | Last modification time start, Unix timestamp| +| last_modify_time_end | long | no | Last modification time end, Unix timestamp| +| start | int | no | Start position of paging record, default 0| +| length | int | no | The maximum number of records returned in a single time is 1000 at most, default 20| + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "job_template_id": 1, + "creator": "admin", + "name": "test", + "create_time_start": 1546272000000, + "create_time_end": 1577807999999, + "last_modify_user": "admin", + "last_modify_time_start": 1546272000000, + "last_modify_time_end": 1577807999999, + "start": 0, + "length": 20 +} +``` + +### Example of responses + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "data": [ + { + "bk_scope_type": "biz", + "bk_scope_id": "1", + "id": 100, + "job_template_id": 1, + "name": "test", + "creator": "admin", + "create_time": 1546272000000, + "last_modify_user": "admin", + "last_modify_time": 1546272000000 + } + ], + "start": 0, + "length": 20, + "total": 1 + } +} +``` + +### Response Description + +#### response +| Fields | Type | Description | +|-----------|-----------|-----------| +| result | bool | Whether the request succeeded or not. True: request succeeded;False: request failed| +| code | int | Error code. 0 indicates success, >0 indicates failure| +| message | string |Error message| +| data | object |Data returned by request| +| permission | object |Permission information| +| request_id | string |Request chain id| + +#### data + +| Fields | Type | Description | +|------------------|-----------|-----------| +| bk_scope_type | string |Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| id | long | Job Plan ID | +| job_template_id | long | Job Template ID | +| name | string | Job Plan name | +| creator | string | Creator account number| +| create_time | long | Creation time, Unix timestamp| +| last_modify_user | string | Modified by acct No.| +| last_modify_time | long | Last modified time, Unix timestamp| diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_job_template_list.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_job_template_list.md new file mode 100644 index 0000000000..9a297fe77a --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_job_template_list.md @@ -0,0 +1,96 @@ +### Function Description + +Query job template list + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +|------------------------|------------|--------|------------| +| bk_scope_type | string | yes | Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | yes | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| creator | string | no | Job Plan creator account | +| name | string | no | Job Plan name, fuzzy match | +| create_time_start | long | no | Creation time start, Unix timestamp| +| create_time_end | long | no | Creation time end, Unix timestamp| +| last_modify_user | string | no | Last modify user| +| last_modify_time_start | long | no | Last modification time start, Unix timestamp| +| last_modify_time_end | long | no | Last modification time end, Unix timestamp| +| start | int | no | Start position of paging record, default 0| +| length | int | no |The maximum number of records returned in a single time is 1000 at most, default 20| + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "creator": "admin", + "name": "test", + "create_time_start": 1546272000000, + "create_time_end": 1577807999999, + "last_modify_user": "admin", + "last_modify_time_start": 1546272000000, + "last_modify_time_end": 1577807999999, + "start": 0, + "length": 20 +} +``` + +### Example of responses + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "data": [ + { + "bk_scope_type": "biz", + "bk_scope_id": "1", + "id": 100, + "name": "test", + "creator": "admin", + "create_time": 1546272000000, + "last_modify_user": "admin", + "last_modify_time": 1546272000000 + } + ], + "start": 0, + "length": 20, + "total": 1 + } +} +``` + +### Response Description + +#### response +| Fields | Type | Description | +|-----------|-----------|-----------| +| result | bool | Whether the request succeeded or not. True: request succeeded;False: request failed| +| code | int | Error code. 0 indicates success, >0 indicates failure| +| message | string |Error message| +| data | object |Data returned by request| +| permission | object |Permission information| +| request_id | string |Request chain id| + +#### data + +| Fields | Type | Description | +|------------------|-----------|-----------| +| bk_scope_type | string |Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| id | long | Job template ID| +| name | string | Job template name| +| creator | string | Creator account number| +| create_time | long | Creation time, Unix timestamp| +| last_modify_user | string | Modified by acct No.| +| last_modify_time | long | Last modified time, Unix timestamp| diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_latest_service_version.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_latest_service_version.md new file mode 100644 index 0000000000..db2eaf9ba7 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_latest_service_version.md @@ -0,0 +1,53 @@ +### Function Description + +Query the latest service version + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +null + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_username": "admin" +} +``` + +### Example of responses + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "version": "3.6.4" + } +} +``` + +### Response Description + +#### response +| Fields | Type | Description | +|-----------|-----------|-----------| +| result | bool | Whether the request succeeded or not. True: request succeeded;False: request failed| +| code | int | Error code. 0 indicates success, >0 indicates failure| +| message | string |Error message| +| data | object |Data returned by request| +| permission | object |Permission information| +| request_id | string |Request chain id| + +#### data + +| Fields | Type | Description | +|-----------|-----------|-----------| +| version | string |The latest version number of the service| diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_public_script_list.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_public_script_list.md new file mode 100644 index 0000000000..35db675109 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_public_script_list.md @@ -0,0 +1,82 @@ +### Function Description + +Query public script list + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +|----------------------|------------|--------|------------| +| name | string | no | Script name, supports fuzzy query| +| script_language | int | no | Scripting language. 1：shell，2：bat，3：perl，4：python，5：powershell，6：sql. | +| start | int | no | Start position of paging record, default 0| +| length | int | no | The maximum number of records returned in a single time is 1000 at most, default 20| + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "name": "a.sh", + "script_language": 1, + "start": 0, + "length": 10 +} +``` + +### Example of responses + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "data": [ + { + "id": "000dbdddc06c453baf1f2decddf00c69", + "name": "a.sh", + "script_language": 1, + "online_script_version_id": 100, + "creator": "admin", + "create_time": 1600746078520, + "last_modify_user": "admin", + "last_modify_time": 1600746078520 + } + ], + "start": 0, + "length": 10, + "total": 1 + } +} +``` + +### Response Description + +#### response +| Fields | Type | Description | +|-----------|-----------|-----------| +| result | bool | Whether the request succeeded or not. True: request succeeded;False: request failed| +| code | int | Error code. 0 indicates success, >0 indicates failure| +| message | string |Error message| +| data | object |Data returned by request| +| permission | object |Permission information| +| request_id | string |Request chain id| + +#### data + +| Fields | Type | Description | +|-----------|-----------|-----------| +| id | string | Script ID| +| name | string | Script name| +| script_language | int | Scripting language. 1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - SQL | +| online_script_version_id | long | Online script version ID; null if the script does not have an online version| +| creator | string | Creator| +| create_time | long | Creation time Unix timestamp (ms)| +| last_modify_user| string | Last modify user| +| last_modify_time| long | Last modified time Unix timestamp (ms)| diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_public_script_version_detail.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_public_script_version_detail.md new file mode 100644 index 0000000000..b99b7eee37 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_public_script_version_detail.md @@ -0,0 +1,79 @@ +### Function Description + +Query public script version details + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +|----------------------|------------|--------|------------| +| id | long | yes |Script version ID. If passed in, other conditions will be masked based on this condition| +| script_id | string | no | Script ID (can be passed in with version to locate a script version)| +| version | string | no | Script version (can be passed in with script_id to locate a script version)| + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "id": 1 +} +``` + +### Example of responses + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "id": 1, + "script_id": "000dbdddc06c453baf1f2decddf00c69", + "version": "V1.0", + "content": "#!/bin/bash***", + "status": 1, + "version_desc": "version description", + "creator": "admin", + "create_time": 1600746078520, + "last_modify_user": "admin", + "last_modify_time": 1600746078520, + "script_language": 1, + "description": "script description" + } +} +``` + +### Response Description + +#### response +| Fields | Type | Description | +|-----------|-----------|-----------| +| result | bool | Whether the request was successful or not. True: request succeeded;False: request failed| +| code | int | Error code. 0 indicates success, >0 indicates failure| +| message | string |Error message| +| data | object |Data returned by request| +| permission | object |Permission information| +| request_id | string |Request chain id| + +#### data + +| Fields | Type | Description | +|-----------|-----------|-----------| +| id | long | Script version ID| +| script_id | string | Script ID to which the script version belongs| +| version | string | Version| +| content | string | Script version content| +| status | int | Script version status (0: not online, 1: Online, 2: offline, 3: Disabled)| +| version_desc | string | Version description| +| creator | string | Creator| +| create_time | long | Creation time Unix timestamp (ms)| +| last_modify_user | string | Last modify user| +| last_modify_time | long | Last modified time Unix timestamp (ms)| +| script_language | int | Script language:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - sql | +| description | string | Description | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_public_script_version_list.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_public_script_version_list.md new file mode 100644 index 0000000000..1c25b5bf00 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_public_script_version_list.md @@ -0,0 +1,90 @@ +### Function Description + +Query public script version list + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +|----------------------|------------|--------|------------| +| script_id | string | yes |Script ID| +| return_script_content | bool | no | Whether the script content needs to be returned. True: returns script content; False: does not return script content. The default is false. | +| start | int | no | Start position of paging record, default 0| +| length | int | no | The maximum number of records returned in a single time is 1000 at most, default 20| + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "script_id": "000dbdddc06c453baf1f2decddf00c69", + "return_script_content": true, + "start": 0, + "length": 10 +} +``` + +### Example of responses + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "data": [ + { + "id": 1, + "script_id": "000dbdddc06c453baf1f2decddf00c69", + "version": "V1.0", + "content": "#!/bin/bash***", + "status": 1, + "version_desc": "version description", + "creator": "admin", + "create_time": 1600746078520, + "last_modify_user": "admin", + "last_modify_time": 1600746078520, + "script_language": 1, + "description": "script description" + } + ], + "start": 0, + "length": 10, + "total": 1 + } +} +``` + +### Response Description + +#### response +| Fields | Type | Description | +|-----------|-----------|-----------| +| result | bool | Whether the request succeeded or not. True: request succeeded;False: request failed| +| code | int | Error code. 0 indicates success, >0 indicates failure| +| message | string |Error message| +| data | object |Data returned by request| +| permission | object |Permission information| +| request_id | string |Request chain id| + +#### data + +| Fields | Type | Description | +|-----------|-----------|-----------| +| id | long | Script version ID| +| script_id | string | Script ID to which the script version belongs| +| version | string | Version| +| content | string | Script content| +| status | int | Script version status (0: not online, 1: Online, 2: offline, 3: Disabled)| +| version_desc | string | Version description| +| creator | string | Creator| +| create_time | long | Creation time Unix timestamp (ms)| +| last_modify_user | string | Last modify user| +| last_modify_time | long | Last modified time Unix timestamp (ms)| +| script_language | int | Script language:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - sql | +| description | string | Description | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_script_list.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_script_list.md new file mode 100644 index 0000000000..64fda155ef --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_script_list.md @@ -0,0 +1,90 @@ +### Function Description + +Query business script list + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +|----------------------|------------|--------|------------| +| bk_scope_type | string | yes | Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | yes | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| name | string | no | Script name, supports fuzzy query| +| script_language | int | no | Scripting language. 0: all script types, 1: shell, 2: bat，3：perl，4： python，5：powershell，6： sql. Default 0| +| start | int | no | Start position of paging record, default 0| +| length | int | no | The maximum number of records returned in a single time is 1000 at most, default 20| + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "name": "a.sh", + "script_language": 1, + "start": 0, + "length": 10 +} +``` + +### Example of responses + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "data": [ + { + "id": "000dbdddc06c453baf1f2decddf00c69", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "name": "a.sh", + "script_language": 1, + "online_script_version_id": 100, + "creator": "admin", + "create_time": 1600746078520, + "last_modify_user": "admin", + "last_modify_time": 1600746078520 + } + ], + "start": 0, + "length": 10, + "total": 1 + } +} +``` + +### Response Description + +#### response +| Fields | Type | Description | +|-----------|-----------|-----------| +| result | bool | Whether the request succeeded or not. True: request succeeded;False: request failed| +| code | int | Error code. 0 indicates success, >0 indicates failure| +| message | string |Error message| +| data | object |Data returned by request| +| permission | object |Permission information| +| request_id | string |Request chain id| + +#### data + +| Fields | Type | Description | +|-----------|-----------|-----------| +| id | string | Script ID| +| bk_scope_type | string |Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| name | string | Script name| +| script_language | int | Scripting language. 1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - SQL | +| online_script_version_id | long | Online script version ID; null if the script does not have an online version| +| creator | string | Creator| +| create_time | long | Creation time Unix timestamp (ms)| +| last_modify_user| string | Last modify user| +| last_modify_time| long | Last modified time Unix timestamp (ms)| diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_script_version_detail.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_script_version_detail.md new file mode 100644 index 0000000000..2f048768ef --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_script_version_detail.md @@ -0,0 +1,87 @@ +### Function Description + +Query business script version details + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +|----------------------|------------|--------|------------| +| bk_scope_type | string | yes | Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | yes | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| id | long | yes |Script version ID. If passed in, other conditions will be masked based on this condition| +| script_id | string | no | Script ID (can be passed in with version to locate a script version)| +| version | string | no | Script version (can be passed in with script_id to locate a script version)| + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "id": 1 +} +``` + +### Example of responses + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "id": 1, + "bk_scope_type": "biz", + "bk_scope_id": "1", + "script_id": "000dbdddc06c453baf1f2decddf00c69", + "version": "V1.0", + "content": "#!/bin/bash***", + "status": 1, + "version_desc": "version description", + "creator": "admin", + "create_time": 1600746078520, + "last_modify_user": "admin", + "last_modify_time": 1600746078520, + "script_language": 1, + "description": "script description" + } +} +``` + +### Response Description + +#### response +| Fields | Type | Description | +|-----------|-----------|-----------| +| result | bool | Whether the request was successful or not. True: request succeeded;False: request failed| +| code | int | Error code. 0 indicates success, >0 indicates failure| +| message | string |Error message| +| data | object |Data returned by request| +| permission | object |Permission information| +| request_id | string |Request chain id| + +#### data + +| Fields | Type | Description | +|-----------|-----------|-----------| +| id | long | Script version ID| +| bk_scope_type | string |Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| script_id | string | Script ID to which the script version belongs| +| version | string | Version| +| content | string | Script version content| +| status | int | Script version status (0: not online, 1: Online, 2: offline, 3: Disabled)| +| version_desc | string | Version description| +| creator | string | Creator| +| create_time | long | Creation time Unix timestamp (ms)| +| last_modify_user | string | Last modify user| +| last_modify_time | long | Last modified time Unix timestamp (ms)| +| script_language | int | Script language:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - sql | +| description | string | Description | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_script_version_list.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_script_version_list.md new file mode 100644 index 0000000000..e03b482014 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/get_script_version_list.md @@ -0,0 +1,98 @@ +### Function Description + +Query business script version list + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +|----------------------|------------|--------|------------| +| bk_scope_type | string | yes | Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | yes | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| script_id | string | yes |Script ID| +| return_script_content | bool | no | Whether the script content needs to be returned. True: returns script content; False: does not return script content. The default is false. | +| start | int | no | Start position of paging record, default 0| +| length | int | no | The maximum number of records returned in a single time is 1000 at most, default 20| + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "script_id": "000dbdddc06c453baf1f2decddf00c69", + "return_script_content": true, + "start": 0, + "length": 10 +} +``` + +### Example of responses + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "data": [ + { + "id": 1, + "bk_scope_type": "biz", + "bk_scope_id": "1", + "script_id": "000dbdddc06c453baf1f2decddf00c69", + "version": "V1.0", + "content": "#!/bin/bash***", + "status": 1, + "version_desc": "version description", + "creator": "admin", + "create_time": 1600746078520, + "last_modify_user": "admin", + "last_modify_time": 1600746078520, + "script_language": 1, + "description": "script description" + } + ], + "start": 0, + "length": 10, + "total": 1 + } +} +``` + +### Response Description + +#### response +| Fields | Type | Description | +|-----------|-----------|-----------| +| result | bool | Whether the request was successful or not. True: request succeeded;False: request failed| +| code | int | Error code. 0 indicates success, >0 indicates failure| +| message | string |Error message| +| data | object |Data returned by request| +| permission | object |Permission information| +| request_id | string |Request chain id| + +#### data + +| Fields | Type | Description | +|-----------|-----------|-----------| +| id | long | Script version ID| +| bk_scope_type | string |Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| script_id | string | Script ID to which the script version belongs| +| version | string | Version| +| content | string | Script content| +| status | int | Script version status (0: not online, 1: Online, 2: offline, 3: Disabled)| +| version_desc | string | Version description| +| creator | string | Creator| +| create_time | long | Creation time Unix timestamp (ms)| +| last_modify_user | string | Last modify user| +| last_modify_time | long | Last modified time Unix timestamp (ms)| +| script_language | int | Script language:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - sql | +| description | string | Description | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/operate_job_instance.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/operate_job_instance.md new file mode 100644 index 0000000000..088ba8fe5b --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/operate_job_instance.md @@ -0,0 +1,61 @@ +### Function Description + +Use to perform actions on an executed job instance, such as terminating a job. + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +|-----------|------------|--------|------------| +| bk_scope_type | string | yes | Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | yes | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| job_instance_id | long | yes |Job instance ID| +| operation_code | int | yes |Operation Code: 1. Terminate job| + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "job_instance_id": 100, + "operation_code": 1 +} +``` + +### Example of responses + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "job_instance_id": 100 + } +} +``` + +### Response Description + +#### response +| Fields | Type | Description | +|-----------|-----------|-----------| +| result | bool | Whether the request succeeded or not. True: request succeeded;False: request failed| +| code | int | Error code. 0 indicates success, >0 indicates failure| +| message | string |Error message| +| data | object |Data returned by request| +| permission | object |Permission information| +| request_id | string |Request chain id| + +#### data + +| Fields | Type | Description | +|-----------|-----------|-----------| +| job_instance_id | long | Job instance ID| diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/operate_step_instance.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/operate_step_instance.md new file mode 100644 index 0000000000..0ca624eb23 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/operate_step_instance.md @@ -0,0 +1,78 @@ +### Function Description + +Used to perform actions on the steps of an executed instance, such as retry, ignore errors, and so on. + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +|-----------|------------|--------|------------| +| bk_scope_type | string | yes | Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | yes | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| job_instance_id | long | yes |Job instance ID| +| step_instance_id | long | yes |Step instance ID| +| operation_code | int | yes |Operation Code: 2. Retry failed IP; 3. Ignore error; 6. Confirm continue; 8. Retry all ; 9. Terminate the confirmation process; 10. Restart the confirmation; 11. Enter the next step; 12. Forcibly skip| + + +##### Operation_code details +| operation_code |Operation type| Applicable steps| Description| +|-----------|------------|--------|------------| +| 2 |Retry failed IP | Script execution, File transfer steps | Redistribute task to failed IP| +| 3 |Ignore error | Script execution, File transfer steps | Ignore error and continue | +| 6 |Confirmed | Manual confirmation step | Confirm to proceed | +| 8 |Retry all | Script execution, File transfer steps | Re-issue tasks to all IPS| +| 9 |Termination confirmation| Manual confirmation step | Confirm termination of execution | +| 10 |Re-initiate confirmation| Manual confirmation step | Re-initiate confirmation | +| 11 |Go to the next step. | Script execution, File transfer steps | If that state of the step is terminate successfully, the method is used for continue executing the subsequent steps| +| 12 |Forced skip | Script execution, File transfer steps | When the step status is terminated, it is used to forcibly skip the current step and execute the subsequent step| + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "job_instance_id": 100, + "step_instance_id": 200, + "operation_code": 2 +} +``` + +### Example of responses + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "step_instance_id": 200, + "job_instance_id": 100 + } +} +``` + +### Response Description + +#### response +| Fields | Type | Description | +|-----------|-----------|-----------| +| result | bool | Whether the request was successful or not. True: request succeeded;False: request failed| +| code | int | Error code. 0 indicates success, >0 indicates failure| +| message | string |Error message| +| data | object |Data returned by request| +| permission | object |Permission information| +| request_id | string |Request chain id| + +#### data + +| Fields | Type | Description | +|-----------|-----------|-----------| +| job_instance_id | long | Job instance ID| +| step_instance_id | long | Step instance ID| diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/publish_public_script_version.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/publish_public_script_version.md new file mode 100644 index 0000000000..a8143b5472 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/publish_public_script_version.md @@ -0,0 +1,61 @@ +### Function Description + +After the public script version is launched, the previous online script will be set to offline status, but it will not affect the use of tasks. + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +| ----------------- | ------ | -------- | ------------------------ | +| script_id | string | yes | Public script id | +| script_version_id | long | yes | Public script version id | + + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "script_id":"4537fb49ec0840a1b91cef4179c99f9c", + "script_version_id": 1000018 +} +``` + +### Example of responses + +```json +{ + "code": 0, + "result": true, + "data": { + "id": 1000018, + "script_id": "4537fb49ec0840a1b91cef4179c99f9c", + "status": 2 + } +} +``` + +### Response Description + +#### response + +| Fields | Type | Description | +| ---------- | ------ | ------------------------------------------------------------ | +| result | bool | Whether the request succeeded or not. True: request succeeded;False: request failed | +| code | int | Error code. 0 indicates success, >0 indicates failure | +| message | string | Error message | +| data | object | Data returned by request | +| permission | object | Permission information | + +#### data + +| Fields | Type | Description | +| ----------------- | ------ | ------------------------------------------------------------ | +| script_id | string | Script id | +| id | long | Script version id | +| status | int | Script version status (0: Not online, 1: Online, 2: Offline, 3: Disabled) | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/publish_script_version.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/publish_script_version.md new file mode 100644 index 0000000000..1777dbd2f9 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/publish_script_version.md @@ -0,0 +1,65 @@ +### Function Description + +After the online script version is launched, the previous online script will be set to offline status, but it will not affect the use of homework. + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +| ----------------- | ------ | -------- | ------------------------------------------------------------ | +| bk_scope_type | string | yes | Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | yes | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| script_id | string | yes | Script id | +| script_version_id | long | yes | Script version id | + + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type":"biz", + "bk_scope_id":"2", + "script_id": "4a350b0e0707450e93326f6ace921072", + "script_version_id": 1000019 +} +``` + +### Example of responses + +```json +{ + "code": 0, + "result": true, + "data": { + "id": 1000019, + "script_id": "4a350b0e0707450e93326f6ace921072", + "status": 1 + } +} +``` + +### Response Description + +#### response + +| Fields | Type | Description | +| ---------- | ------ | ------------------------------------------------------------ | +| result | bool | Whether the request succeeded or not. True: request succeeded;False: request failed | +| code | int | Error code. 0 indicates success, >0 indicates failure | +| message | string | Error message | +| data | object | Data returned by request | +| permission | object | Permission information | + +#### data + +| Fields | Type | Description | +| ----------------- | ------ | ------------------------------------------------------------ | +| script_id | string | Script id | +| id | long | Script version id | +| status | int | Script version status (0: Not online, 1: Online, 2: Offline, 3: Disabled) | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/push_config_file.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/push_config_file.md new file mode 100644 index 0000000000..644c858586 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/push_config_file.md @@ -0,0 +1,125 @@ +### Function Description + +Distribute configuration files, this interface is used to distribute small plain text files such as configuration files + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +|-------------|------------|--------|------------| +| bk_scope_type | string | yes | Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | yes | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| task_name | string | no | Custom job name| +| account_alias | string | yes |Execute account alias| +| file_target_path | string | yes |File transfer destination path| +| file_list | array | yes |Source file object array, see file definition below| +| target_server | object | yes |Target server, see server definition | + +#### file + +| Fields | Type | Required | Description | +|-----------|------------|--------|------------| +| file_name | string | yes |File name| +| content | string | yes |File content Base64| + +#### server + +| Fields | Type | Required | Description | +| ------------------ | ----- | -------- | ------------------------------------------------------- | +| host_id_list | array | no | Host ID list | +| ip_list | array | no | Static IP list, see ip for definition. ***Deprecated, it is recommended to use the host_id_list parameter***; if host_id_list and ip_list exist at the same time, the ip_list parameter will be ignored. | +| dynamic_group_list | array | no | Dynamic grouping list, see dynamic_group for definition | +| topo_node_list | array | no | Dynamic topo node list, see topo_node for definition | + + +#### ip + +| Fields | Type | Required | Description | +| ----------- | ------ | -------- | ------------- | +| bk_cloud_id | long | yes | BK-Net ID | +| ip | string | yes | IP Address | + +#### dynamic_group + +| Fields | Type | Required | Description | +| ------ | ------ | -------- | -------------- | +| id | string | yes | CMDB dynamic grouping ID| + +#### topo_node_list + +| Fields | Type | Required | Description | +| --------- | ------ | -------- | ------------------------------------------------------------ | +| id | long | yes | Dynamic topo node ID, corresponding to bk_inst_id in CMDB API | +| node_type | string | yes | Dynamic topo node type, corresponding to bk_obj_id in CMDB API, such as "module,""set"| + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "account_alias": "root", + "file_target_path": "/tmp/", + "file_list": [ + { + "file_name": "a.txt", + "content": "aGVsbG8gd29ybGQh" + } + ], + "target_server": { + "dynamic_group_list": [ + { + "id": "blo8gojho0skft7pr5q0" + } + ], + "host_id_list": [ + 101, + 102 + ], + "topo_node_list": [ + { + "id": 1000, + "node_type": "module" + } + ] + } +} +``` + +### Example of responses + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "job_instance_name": "API GSE PUSH FILE1521107826893", + "job_instance_id": 10000 + } +} +``` + +### Response Description + +#### response +| Fields | Type | Description | +|-----------|-----------|-----------| +| result | bool | Whether the request succeeded or not. True: request succeeded;False: request failed| +| code | int | Error code. 0 indicates success, >0 indicates failure| +| message | string |Error message| +| data | object |Data returned by request| +| permission | object |Permission information| +| request_id | string |Request chain id| + +#### data + +| Fields | Type | Description | +|-----------|-----------|-----------| +| job_instance_id | long | Job instance ID| +| job_instance_name | long | Job instance name| +| step_instance_id | long | Step instance ID| diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/save_cron.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/save_cron.md new file mode 100644 index 0000000000..d0423758e9 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/save_cron.md @@ -0,0 +1,381 @@ +### Function Description + +Create or update Cron Job; Create Cron Job, Cron Job status is paused by default, you can call update_cron_status interface to turn it on if needed. + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +|-----------------|------------|--------|------------| +| bk_scope_type | string | yes | Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | yes | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| job_plan_id | long | yes |Job Plan ID of the job to be executed regularly| +| id | long | no | Cron Job ID, when updating a Cron Job, you must pass this value Cron Job | +| name | string | no | Cron job name, required for new creation and optional for modification | +| expression | string | no |Timing rules for Cron Job crontab, optional when modifying. The meaning of each field is: minute hour day month week, for example: 0/5 * * * * means execute every 5 minutes. caution: ? not supported. This field and execute_time cannot be empty at the same time when creating Cron Job | +| execute_time | long | no |Single execution execution time, Unix timestamp, this field and expression cannot be empty at the same time when creating Cron Job | +| global_var_list | array | no | Global variable information, you can use the Query Job Plan Details interface to query the variable information which can be set for the plan | + +#### global_var + +| Fields | Type | Required | Description | +|-----------|-----------|--------|------------| +| id | long | no | Global variable id, unique identification. If the id is empty, then name is used as the unique identification| +| name | string | no | Global variable name| +| value | string | no | Character, password, value of global variable of array type| +| server | object | no | The value of the host type global variable| + +#### server +| Fields | Type | Required | Description | +| ------------------ | ----- | -------- | ------------------------------------------------------- | +| host_id_list | array | no | Host ID list | +| ip_list | array | no | Static IP list, see ip for definition. ***Deprecated, it is recommended to use the host_id_list parameter***; if host_id_list and ip_list exist at the same time, the ip_list parameter will be ignored. | +| dynamic_group_list | array | no | Dynamic grouping list, see dynamic_group for definition | +| topo_node_list | array | no | Dynamic topo node list, see topo_node for definition | + +#### ip + +| Fields | Type | Required | Description | +|-------------|---------|--------|---------| +| bk_cloud_id | int | yes | BK-Net ID | +| ip | string | yes | IP Address | + +#### topo_node +| Fields | Type | Required | Description | +|------------------|--------|--------|------------| +| id | long | yes |Dynamic topo node ID, corresponding to bk_inst_id in CMDB API| +| node_type | string | yes |Dynamic topo node type, corresponding to bk_obj_id in CMDB API, such as "module" and "set"| + + +### Example of request +#### 1. Create Cron Job +```json +{ + "bk_app_code": "xxx", + "bk_app_secret": "xxx", + "access_token": "xxx", + "bk_username": "admin", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "expression": "0 0/5 * * *", + "job_plan_id": 1023060, + "name": "test API", + "global_var_list": [ + { + "name": "stringVar", + "value": "value11112" + }, + { + "name": "nsVar", + "value": "nsvalue11112" + }, + { + "name": "secretVar", + "value": "secretvalue11112" + }, + { + "name": "dictVar", + "value": "([\"var1\"]=1, [\"var2\"]=2)" + }, + { + "name": "indexArrVar", + "value": "(2 3 4)" + }, + { + "name": "hostVar", + "server": { + "host_id_list": [1,2,3] + } + } + ] +} +``` +#### 2. Update Cron Job +```json +{ + "id": 1000064, + "bk_app_code": "xxx", + "bk_app_secret": "xxx", + "access_token": "xxx", + "bk_username": "admin", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "expression": "0 0/5 * * *", + "job_plan_id": 1023060, + "name": "test API", + "global_var_list": [ + { + "name": "stringVar", + "value": "value111333312" + }, + { + "name": "nsVar", + "value": "nsvalue111333312" + }, + { + "name": "secretVar", + "value": "secretvalue111333312" + }, + { + "name": "dictVar", + "value": "([\"var1\"]=1, [\"var2\"]=2)" + }, + { + "name": "indexArrVar", + "value": "(22 3 4)" + }, + { + "name": "hostVar", + "server": { + "host_id_list": [2,3,4] + } + } + ] +} +``` + +### Example of responses +#### 1. Create Cron Job +```json +{ + "code": 0, + "result": true, + "data": { + "id": 1000067, + "name": "test API", + "status": 2, + "creator": "admin", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "job_plan_id": 1023060, + "expression": "0 0/5 * * *", + "global_var_list": [ + { + "id": 1001101, + "name": "stringVar", + "value": "value11112", + "description": null, + "type": 1, + "required": null, + "server": null + }, + { + "id": 1001102, + "name": "nsVar", + "value": "nsvalue11112", + "description": null, + "type": 2, + "required": null, + "server": null + }, + { + "id": 1001104, + "name": "secretVar", + "value": "secretvalue11112", + "description": null, + "type": 4, + "required": null, + "server": null + }, + { + "id": 1001105, + "name": "dictVar", + "value": "([\"var1\"]=1, [\"var2\"]=2)", + "description": null, + "type": 5, + "required": null, + "server": null + }, + { + "id": 1001106, + "name": "indexArrVar", + "value": "(2 3 4)", + "description": null, + "type": 6, + "required": null, + "server": null + }, + { + "id": 1001103, + "name": "hostVar", + "value": null, + "description": null, + "type": 3, + "required": null, + "server": { + "ip_list": [ + { + "bk_host_id": 101, + "bk_cloud_id": 0, + "ip": "127.0.0.1" + } + ], + "dynamic_group_list": null, + "topo_node_list": null + } + } + ], + "create_time": 1642045370, + "last_modify_user": "admin", + "last_modify_time": 1642045370 + } +} +``` +#### 2. Update Cron Job +```json +{ + "code": 0, + "result": true, + "data": { + "id": 1000064, + "name": "test API", + "status": 2, + "creator": "admin", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "job_plan_id": 1023060, + "expression": "0 0/5 * * *", + "global_var_list": [ + { + "id": 1001101, + "name": "stringVar", + "value": "value111333312", + "description": null, + "type": 1, + "required": null, + "server": null + }, + { + "id": 1001102, + "name": "nsVar", + "value": "nsvalue111333312", + "description": null, + "type": 2, + "required": null, + "server": null + }, + { + "id": 1001104, + "name": "secretVar", + "value": "secretvalue111333312", + "description": null, + "type": 4, + "required": null, + "server": null + }, + { + "id": 1001105, + "name": "dictVar", + "value": "([\"var1\"]=1, [\"var2\"]=2)", + "description": null, + "type": 5, + "required": null, + "server": null + }, + { + "id": 1001106, + "name": "indexArrVar", + "value": "(22 3 4)", + "description": null, + "type": 6, + "required": null, + "server": null + }, + { + "id": 1001103, + "name": "hostVar", + "value": null, + "description": null, + "type": 3, + "required": null, + "server": { + "ip_list": [ + { + "bk_host_id": 101, + "bk_cloud_id": 0, + "ip": "127.0.0.1" + } + ], + "dynamic_group_list": null, + "topo_node_list": null + } + } + ], + "create_time": 1641990674, + "last_modify_user": "admin", + "last_modify_time": 1641995052 + } +} +``` + +### Response Description + +#### response +| Fields | Type | Description | +|-----------|-----------|-----------| +| result | bool | Whether the request succeeded or not. True: request succeeded;False: request failed| +| code | int | Error code. 0 indicates success, >0 indicates failure| +| message | string |Error message| +| data | object |Data returned by request| +| permission | object |Permission information| +| request_id | string |Request chain id| + +#### data + +| Fields | Type | Description | +|-----------------|------------|-------------| +| id | long | Cron Job ID | +| name | string | Cron job name | +| status | int | Cron Job Status | +| bk_scope_type | string |Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| job_plan_id | long | Job Plan ID of the job to be executed regularly | +| creator | string | Creator| +| create_time | long | Created time, Unix timestamp| +| last_modify_user | string | Last modify user| +| last_modify_time | long | Last modified time, Unix timestamp| +| expression | string | Timing rules for Cron Job crontab | +| execute_time | long | Execution time of a single Cron Job execution, Unix timestamp | +| global_var_list | array | For global variable information, see global_var| + +#### global_var + +| Fields | Type | Description | +|-------------|-----------|------------| +| id | long | Global variable id, unique identification. If the id is empty, then name is used as the unique identification| +| name | string | Global variable name| +| description | string | Global variable description| +| type | int | Global variable type| +| required | int | Whether the variable is required in the template/Job Plan | +| value | string | Character, password, value of global variable of array type| +| server | object | Host type global variable value| + +#### server +| Fields | Type | Description | +|-----------------------|-------|------------| +| variable | string |Referenced variable name| +| host_id_list | array | Host ID list | +| ip_list | array |Static IP list| +| dynamic_group_list | array |Dynamic group ID list| +| topo_node_list | array |Dynamic topo node list| + +#### ip +| Fields | Type | Description | +|-------------|---------|---------| +| bk_host_id | long | Host ID | +| bk_cloud_id | int | BK-Net ID | +| ip | string | IP Address | + +#### dynamic_group +| Fields | Type | Description | +|-----|---------|------------| +| id | string | Dynamic Group ID | + +#### topo_node +| Fields | Type | Description | +|------------------|--------|------------| +| id | long | Dynamic topo node ID, corresponding to bk_inst_id in CMDB API| +| node_type | string |Dynamic topo node type, corresponding to bk_obj_id in CMDB API, such as "module" and "set"| diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/update_credential.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/update_credential.md new file mode 100644 index 0000000000..15957ceb0b --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/update_credential.md @@ -0,0 +1,72 @@ +### Function Description + +Update credentials. + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +|----------------------------|------------|--------|------------| +| bk_scope_type | string | yes | Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | yes | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| id | string | yes |Credential ID | +| name | string | no | Credential name | +| type | string | no | Credential type, which can be ACCESS_KEY_SECRET_KEY,PASSWORD,USERNAME_PASSWORD,SECRET_KEY| +| description | string | no | Credential description| +| credential_access_key | string | no | Required when the credential type is ACCESS_KEY_SECRET_KEY| +| credential_secret_key | string | no | Required when the credential type is ACCESS_KEY_SECRET_KEY/SECRET_KEY| +| credential_username | string | no | Required when the credential type is USERNAME_PASSWORD| +| credential_password | string | no | Required when the credential type is USERNAME_PASSWORD/PASSWORD| + + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "id": "06644309e10e4068b3c7b32799668210", + "name": "testCredential", + "type": "USERNAME_PASSWORD", + "description": "This is a new credential", + "credential_username": "admin", + "credential_password": "newPassword" +} +``` + +### Example of responses + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "id": "06644309e10e4068b3c7b32799668210" + } +} +``` + +### Response Description + +#### response +| Fields | Type | Description | +|-----------|-----------|-----------| +| result | bool | Whether the request was successful or not. True: request succeeded;False: request failed| +| code | int | Error code. 0 indicates success, >0 indicates failure| +| message | string |Error message| +| data | object |Data returned by request| +| permission | object |Permission information| +| request_id | string |Request chain id| + +#### data + +| Fields | Type |Whether the field must exist | Description | +|-----------|-------|---------------|---------| +| id | string |yes | Credential ID| diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/update_cron_status.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/update_cron_status.md new file mode 100644 index 0000000000..9016a25958 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/update_cron_status.md @@ -0,0 +1,59 @@ +### Function Description + +Update Cron job status, such as started or paused + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +|----------- |------------|--------|------------| +| bk_scope_type | string | yes | Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | yes | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| id | long | yes |Cron job ID| +| status | int | yes |State, 1. Start, 2. Paused| + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "id": 2, + "status": 1 +} +``` + +### Example of responses + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": 2 +} +``` + +### Response Description + +#### response +| Fields | Type | Description | +|-----------|-----------|-----------| +| result | bool | Whether the request succeeded or not. True: request succeeded;False: request failed| +| code | int | Error code. 0 indicates success, >0 indicates failure| +| message | string |Error message| +| data | object |Data returned by request| +| permission | object |Permission information| +| request_id | string |Request chain id| + +#### data + +| Fields | Type | Description | +|-----------|-----------|-----------| +| data | long | Cron Job ID | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/update_dangerous_rule.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/update_dangerous_rule.md new file mode 100644 index 0000000000..19ff32aece --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/update_dangerous_rule.md @@ -0,0 +1,81 @@ +### Function Description + +update dangerous rule + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +| -------------------- | ------ | -------- | ------------------------------------------------------------ | +| id | long | yes | Rule id | +| expression | string | yes | Expression | +| script_language_list | array | yes | Script language:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - sql | +| description | string | yes | Rule description | +| action | int | yes | Processing actions: 1- Scan, 2- Intercept | + + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "id": 1, + "expression": "rm", + "script_language_list": [1], + "description": "drangerous!!!", + "action": 2 +} +``` + +### Example of responses + +```json +{ + "code": 0, + "result": true, + "data": { + "id": 1, + "expression": "rm", + "script_language_list": [1], + "description": "drangerous!!!", + "action": 2, + "status": 0, + "creator": "admin", + "create_time": 1695193968000, + "last_modify_user": "admin", + "last_modify_time": 1695302417000 + } +} +``` + +### Response Description + +#### response + +| Fields | Type | Description | +| ---------- | ------ | ------------------------------------------------------------ | +| result | bool | Whether the request succeeded or not. True: request succeeded;False: request failed | +| code | int | Error code. 0 indicates success, >0 indicates failure | +| message | string | Error message | +| data | object | Data returned by request | +| permission | object | Permission information | + +#### data + +| Fields | Type | Description | +| -------------------- | ------ | ------------------------------------------------------------ | +| id | long | Rule id | +| expression | string | expression | +| script_language_list | array | Script language:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - sql | +| description | string | Description | +| action | int | Processing actions: 1- Scan, 2- Intercept | +| status | int | Enabling status: 0- disabled, 1- enabled | +| creator | string | Creator | +| create_time | long | Created time, Unix timestamp | +| last_modify_user | string | Last modify user | +| last_modify_time | long | Last modified time, Unix timestamp | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/update_file_source.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/update_file_source.md new file mode 100644 index 0000000000..f82ff26c3c --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/update_file_source.md @@ -0,0 +1,79 @@ +### Function Description + +Update the file source. + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +|-----------------|------------|--------|------------| +| bk_scope_type | string | yes | Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | yes | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| id | int | no | File source ID, at least one of which is required with code, and id preferred | +| code | string | no | At least one of the document source identification and id is required, and id preferred. It must start with English characters and consist of 1 32 English characters, underscores or numbers. It can not be changed after creation| +| alias | string | no | File source alias| +| type | string | no | File source type, currently only BlueKing Repository is supported, BLUEKING_ARTIFACTORY| +| access_params | object | no | File source access parameter. Different objects are passed in according to type. See subsequent description| +| credential_id | string | no | Credential Id used by the file source| +| file_prefix | string | no | The Job prefix is added to the file distributed from the file source. The default prefix is not added if it is not passed| + +### access_params +**When Type is BLUEKING_ARTIFACTORY** + +| Fields | Type | Required | Description | +|-----------------|------------|--------|------------| +| base_url | string | yes | The root url of the BlueKing Repository instance, for example: https://bkrepo.com | + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "code": "sopsArtifactory", + "alias": "SOPS_Artifactory", + "type": "BLUEKING_ARTIFACTORY", + "access_params": { + "base_url": "https://bkrepo.com" + }, + "credential_id": "06644309e10e4068b3c7b32799668210", + "file_prefix": "" +} +``` + +### Example of responses + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "id": 1 + } +} +``` + +### Response Description + +#### response +| Fields | Type | Description | +|-----------|-----------|-----------| +| result | bool | Whether the request succeeded or not. True: request succeeded;False: request failed| +| code | int | Error code. 0 indicates success, >0 indicates failure| +| message | string |Error message| +| data | object |Data returned by request| +| permission | object |Permission information| +| request_id | string |Request chain id| + +#### data + +| Fields | Type |Whether the field must exist | Description | +|------------|--------|---------------|-----------| +| id | int | yes | File source ID| diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/update_public_script_basic.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/update_public_script_basic.md new file mode 100644 index 0000000000..5295955167 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/update_public_script_basic.md @@ -0,0 +1,73 @@ +### Function Description + +Update basic information of public scripts + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +| ----------- | ------ | -------- | ------------------ | +| script_id | string | yes | Script id | +| name | string | yes | Script name | +| description | string | no | Script description | + + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "script_id": "4537fb49ec0840a1b91cef4179c99f9c", + "name": "public script test", + "description": "public script test" +} +``` + +### Example of responses + +```json +{ + "code": 0, + "result": true, + "data": { + "id": "4537fb49ec0840a1b91cef4179c99f9c", + "name": "public script test", + "script_language": 1, + "creator": "admin", + "create_time": 1691739630000, + "last_modify_user": "admin", + "last_modify_time": 1691740230000, + "description": "public script test" + } +} +``` + +### Response Description + +#### response + +| Fields | Type | Description | +| ---------- | ------ | ------------------------------------------------------------ | +| result | bool | Whether the request succeeded or not. True: request succeeded;False: request failed | +| code | int | Error code. 0 indicates success, >0 indicates failure | +| message | string | Error message | +| data | object | Data returned by request | +| permission | object | Permission information | + +#### data + +| Fields | Type | Description | +| ---------------- | ------ | ------------------------------------------------------------ | +| id | string | Script id | +| name | string | Script name | +| script_language | int | Script language:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - sql | +| creator | string | Creator | +| create_time | long | Created time, Unix timestamp | +| last_modify_user | string | Last modify user | +| last_modify_time | long | Last modified time, Unix timestamp | +| description | string | Description | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/update_public_script_version.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/update_public_script_version.md new file mode 100644 index 0000000000..5a623e6e7a --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/update_public_script_version.md @@ -0,0 +1,85 @@ +### Function Description + +Update public script version + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +| ----------------- | ------ | -------- | ----------------------------------------- | +| script_id | string | yes | Script id | +| script_version_id | long | yes | Script version id | +| content | string | yes | Script content, requiring Base64 encoding | +| version_desc | string | no | Version Description | + + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "script_id": "4537fb49ec0840a1b91cef4179c99f9c", + "script_version_id": 1000020, + "content": "IyEvYmluL2Jhc2gKbHM=", + "version_desc": "version 2" +} +``` + +### Example of responses + +```json +{ + "code": 0, + "result": true, + "data": { + "id": 1000020, + "script_id": "4537fb49ec0840a1b91cef4179c99f9c", + "name": "public script test", + "script_language": 1, + "content": "#!/bin/bash\nls", + "creator": "admin", + "create_time": 1691739630000, + "last_modify_user": "admin", + "last_modify_time": 1691740290000, + "version": "1.0", + "version_desc": "version 2", + "status": 0, + "description": "public script test" + } +} +``` + +### Response Description + +#### response + +| Fields | Type | Description | +| ---------- | ------ | ------------------------------------------------------------ | +| result | bool | Whether the request succeeded or not. True: request succeeded;False: request failed | +| code | int | Error code. 0 indicates success, >0 indicates failure | +| message | string | Error message | +| data | object | Data returned by request | +| permission | object | Permission information | + +#### data + +| Fields | Type | Description | +| ----------------- | ------ | ------------------------------------------------------------ | +| id | long | Script version id | +| script_id | string | Script id | +| name | string | Script name | +| script_language | int | Script language:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - sql | +| content | string | Script content | +| creator | string | Creator | +| create_time | long | Created time, Unix timestamp | +| last_modify_user | string | Last modify user | +| last_modify_time | long | Last modified time, Unix timestamp | +| version | string | Script version | +| version_desc | string | Version description | +| status | int | Script version status (0: Not online, 1: Online, 2: Offline, 3: Disabled) | +| description | string | Description | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/update_script_basic.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/update_script_basic.md new file mode 100644 index 0000000000..0a07ef2b4f --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/update_script_basic.md @@ -0,0 +1,80 @@ +### Function Description + +Update script basic information + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +| ------------- | ------ | -------- | ------------------------------------------------------------ | +| bk_scope_type | string | yes | Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | yes | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| script_id | string | yes | Script id | +| name | string | yes | Script name | +| description | string | no | Script description | + + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "2", + "script_id": "4a350b0e0707450e93326f6ace921072", + "name": "script test" +} +``` + +### Example of responses + +```json +{ + "code": 0, + "result": true, + "data": { + "id": "4a350b0e0707450e93326f6ace921072", + "name": "script test", + "script_language": 1, + "bk_scope_type": "biz", + "bk_scope_id": "2", + "creator": "admin", + "create_time": 1691743535000, + "last_modify_user": "admin", + "last_modify_time": 1691743715000, + "description": "script test" + } +} +``` + +### Response Description + +#### response + +| Fields | Type | Description | +| ---------- | ------ | ------------------------------------------------------------ | +| result | bool | Whether the request succeeded or not. True: request succeeded;False: request failed | +| code | int | Error code. 0 indicates success, >0 indicates failure | +| message | string | Error message | +| data | object | Data returned by request | +| permission | object | Permission information | + +#### data + +| Fields | Type | Description | +| ---------------- | ------ | ------------------------------------------------------------ | +| id | string | Script id | +| name | string | Script name | +| script_language | int | Script language:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - sql | +| bk_scope_type | string | Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| creator | string | Creator | +| create_time | long | Created time, Unix timestamp | +| last_modify_user | string | Last modify user | +| last_modify_time | long | Last modified time, Unix timestamp | +| description | string | Description | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/en/update_script_version.md b/docs/apidoc/esb/jobv3-confapis/apidocs/en/update_script_version.md new file mode 100644 index 0000000000..eb93e59b37 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/en/update_script_version.md @@ -0,0 +1,93 @@ +### Function Description + +Update Script Version Information + +### Request Parameters + +{{ common_args_desc }} + +#### Interface parameters + +| Fields | Type | Required | Description | +| ----------------- | ------ | -------- | ------------------------------------------------------------ | +| bk_scope_type | string | yes | Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | yes | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| script_id | string | yes | Script id | +| script_version_id | long | yes | Script version id | +| content | string | yes | Script content, requiring Base64 encoding | +| version_desc | string | no | Version Description | + + +### Example of request + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type":"biz", + "bk_scope_id":"2", + "script_id": "4a350b0e0707450e93326f6ace921072", + "script_version_id": 1000019, + "content": "IyEvYmluL2Jhc2gKZGF0ZQo=", + "version_desc": "version 1.0" +} +``` + +### Example of responses + +```json +{ + "code": 0, + "result": true, + "data": { + "id": 1000019, + "script_id": "4a350b0e0707450e93326f6ace921072", + "name": "script test", + "script_language": 1, + "bk_scope_type": "biz", + "bk_scope_id": "2", + "content": "#!/bin/bash\ndate\n", + "creator": "admin", + "create_time": 1691741073000, + "last_modify_user": "admin", + "last_modify_time": 1691741313000, + "version": "1.0", + "version_desc": "version 1.0", + "status": 0, + "description": "script test" + } +} +``` + +### Response Description + +#### response + +| Fields | Type | Description | +| ---------- | ------ | ------------------------------------------------------------ | +| result | bool | Whether the request succeeded or not. True: request succeeded;False: request failed | +| code | int | Error code. 0 indicates success, >0 indicates failure | +| message | string | Error message | +| data | object | Data returned by request | +| permission | object | Permission information | + +#### data + +| Fields | Type | Description | +| ----------------- | ------ | ------------------------------------------------------------ | +| id | long | Script version id | +| script_id | string | Script id | +| name | string | Script name | +| script_language | int | Script language:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - sql | +| bk_scope_type | string | Resource scope type. Optional values: biz - Business，biz_set - Business Set | +| bk_scope_id | string | Resource scope ID. Corresponds to bk_scope_type, which means business ID or business set ID | +| content | string | Script content | +| creator | string | Creator | +| create_time | long | Created time, Unix timestamp | +| last_modify_user | string | Last modify user | +| last_modify_time | long | Last modified time, Unix timestamp | +| version | string | Script version | +| version_desc | string | Version description | +| status | int | Script version status (0: Not online, 1: Online, 2: Offline, 3: Disabled) | +| description | string | Description | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/batch_get_job_instance_ip_log.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/batch_get_job_instance_ip_log.md new file mode 100644 index 0000000000..f5a0a69d44 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/batch_get_job_instance_ip_log.md @@ -0,0 +1,204 @@ +### 功能描述 + +根据主机列表批量查询作业执行日志 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| job_instance_id | long | 是 | 作业实例ID | +| step_instance_id | long | 是 | 步骤实例ID | +| host_id_list | array | 否 | 主机ID列表 | +| ip_list | array | 否 | ***不推荐使用，建议使用host_id_list参数***;如果host_id_list与ip_list同时存在，将忽略ip_list参数。主机IP 列表，定义见ip | + + +##### ip + +| 字段 | 类型 | 必选 | 描述 | +| ----------- | ------ | ---- | -------- | +| bk_cloud_id | int | 是 | 管控区域ID | +| ip | string | 是 | IP地址 | + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "job_instance_id": 100, + "step_instance_id": 200, + "host_id_list": [ + 101,102 + ] +} +``` + +### 返回结果示例 + +#### 脚本执行步骤 +```json +{ + "result": true, + "code": 0, + "message": "", + "data": { + "log_type": 1, + "job_instance_id": 100, + "step_instance_id": 200, + "script_task_logs": [ + { + "bk_host_id": 101, + "ip": "127.0.0.1", + "bk_cloud_id": 0, + "log_content": "[2018-03-15 14:39:30][PID:56875] job_start\n" + }, + { + "bk_host_id": 102, + "ip": "127.0.0.2", + "bk_cloud_id": 0, + "log_content": "[2018-03-15 14:39:30][PID:16789] job_start\n" + } + ] + } +} +``` + +#### 文件分发步骤 +```json +{ + "result": true, + "code": 0, + "message": "", + "data": { + "log_type": 2, + "job_instance_id": 100, + "step_instance_id": 200, + "file_task_logs": [ + { + "bk_host_id": 101, + "ip": "127.0.0.1", + "bk_cloud_id": 0, + "file_logs": [ + { + "mode": 1, + "src_ip": { + "bk_host_id": 102, + "bk_cloud_id": 0, + "ip": "127.0.0.2" + }, + "src_path": "/data/1.log", + "dest_ip": { + "bk_host_id": 101, + "bk_cloud_id": 0, + "ip": "127.0.0.1" + }, + "dest_path": "/tmp/1.log", + "status": 4, + "log_content": "[2021-06-28 11:32:16] FileName: /tmp/1.log FileSize: 9.0 Bytes State: dest agent success download file Speed: 1 KB/s Progress: 100% StatusDesc: dest agent success download file Detail: success", + "size": "1.0 Bytes", + "speed": "0 KB/s", + "process": "100%" + } + ] + }, + { + "bk_host_id": 102, + "ip": "127.0.0.2", + "bk_cloud_id": 0, + "file_logs": [ + { + "mode": 0, + "src_ip": { + "bk_host_id": 102, + "bk_cloud_id": 0, + "ip": "127.0.0.2" + }, + "src_path": "/data/1.log", + "status": 4, + "log_content": "[2021-06-28 11:32:16] FileName: /data/1.log FileSize: 9.0 Bytes State: source agent success upload file Speed: 1 KB/s Progress: 100% StatusDesc: source agent success upload file Detail: success upload", + "size": "1.0 Bytes", + "speed": "0 KB/s", + "process": "100%" + } + ] + } + ] + } +} +``` + +**文件任务返回结果说明** + +- 如果需要返回文件源的上传日志，需要在 host_id_list/ip_list添加源文件主机信息 + +### 返回结果说明 + +#### response +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + +#### data + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| bk_host_id | long | 主机ID | +| bk_cloud_id | int | 目标服务器管控区域ID | +| ip | string | 目标服务器IP地址 | +| log_type | int | 日志类型。1-脚本执行任务日志;2-文件分发任务日志 | +| script_task_logs | array | 脚本执行任务日志。定义见script_task_log| +| file_task_logs | array | 文件分发任务日志。定义见file_task_log| + +#### script_task_log + +| 字段 | 类型 | 描述 | +|-----------|------------|--------| +| bk_host_id | long | 主机ID | +| bk_cloud_id | long | 管控区域ID | +| ip | string | 目标IP地址 | +| log_content | string | 脚本执行日志内容 | + +#### file_task_log + +| 字段 | 类型 | 描述 | +|-----------|------------|--------| +| bk_host_id | long | 主机ID | +| bk_cloud_id | long | 管控区域ID | +| ip | string | 源/目标IP地址 | +| file_logs | array | 文件分发日志内容。定义见file_log | + +#### file_log + +| 字段 | 类型 | 描述 | +|--------------|-----------|-----------| +| mode | 分发模式 | 0:上传;1:下载| +| src_ip | object |文件源主机IP。定义见ip | +| src_path | string | 源文件路径 | +| dest_ip | object | 分发目标主机IP，mode=1时有值。定义见ip | +| dest_path | string | 目标路径，mode=1时有值 | +| status | int | 任务状态。1-等待开始;2-上传中;3-下载中;4-成功;5-失败 | +| log_content | string | 文件分发日志内容 | +| size | string | 文件大小 | +| speed | string | 文件传输速率 | +| process | string | 文件传输进度 | + +#### ip + +| 字段 | 类型 | 描述 | +|-----------|------------|--------| +| bk_host_id | long | 主机ID | +| bk_cloud_id | long | 管控区域ID | +| ip | string | IP地址 | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/callback_protocol.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/callback_protocol.md new file mode 100644 index 0000000000..0d4949dbfc --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/callback_protocol.md @@ -0,0 +1,43 @@ +### 功能描述 + +此组件仅用于展示回调协议文档。 + +对作业执行类的请求传入的回调 callback_url 地址进行回调时所传递的报文结构描述。 + +### 请求参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------------|------------|--------|------------| +| job_instance_id | long | 是 | 作业实例ID | +| status | int | 是 | 作业状态码: 1.等待执行; 2.正在执行; 3.执行成功; 4.执行失败; 5.跳过; 6.忽略错误; 7.等待用户; 8.强制终止; 9.状态异常; 10.强制终止中; 11.强制终止成功,13.确认终止 | +| step_instance_list | array | 是 | 步骤块中包含的各个步骤执行状态 | + +#### step_instances + +| 字段 | 类型 | 必选 | 描述 | +|-----------------|------------|--------|------------| +| step_instance_id | long | 是 | 作业步骤实例ID | +| status | int | 是 | 作业步骤状态码: 1.等待执行; 2.正在执行; 3.执行成功; 4.执行失败; 5.跳过; 6.忽略错误; 7.等待用户; 8.强制终止; 9.状态异常; 10.强制终止中; 11.强制终止成功,13.确认终止 | + +### 请求参数示例 + +```json +{ + "job_instance_id": 12345, + "status": 2, + "step_instance_list": [ + { + "step_instance_id": 16271, + "status": 3 + }, + { + "step_instance_id": 16272, + "status": 2 + } + ] +} +``` + +### 回调响应 + +回调成功以HTTP状态为准，如果成功，则状态码200，其他表示失败，Job会对失败的做一次重试，如还失败，则忽略失败，不再回调。 \ No newline at end of file diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/check_script.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/check_script.md new file mode 100644 index 0000000000..c90dd557be --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/check_script.md @@ -0,0 +1,74 @@ +### 功能描述 + +高危脚本检测。 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +| --------------- | ------ | ---- | ------------------------------------------------------------ | +| script_language | int | 是 | 脚本语言:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - sql | +| content | string | 是 | 脚本内容，需Base64编码 | + + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "script_language": 1, + "content": "cm0gLXJmIC8=" +} +``` + +### 返回结果示例 + +```json +{ + "code": 0, + "result": true, + "data": [ + { + "line": 1, + "line_content": "rm /tmp", + "match_content": "rm /tmp", + "level": 1, + "description": "The first line of the script does not define a valid script type, for example: #!/bin/bash" + }, + { + "line": 1, + "line_content": "rm /tmp", + "match_content": "rm", + "level": 3, + "description": "dangerous！！！" + } + ] +} +``` + +### 返回结果参数说明 + +#### response + +| 字段 | 类型 | 描述 | +| ---------- | ------ | ------------------------------------------ | +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息 | +| data | object | 请求返回的数据 | +| permission | object | 权限信息 | + +#### data + +| 字段 | 类型 | 描述 | +| ------------- | ------ | -------------------------------------- | +| line | int | 错误所在行数 | +| line_content | string | 脚本所在行的内容 | +| match_content | string | 匹配的内容 | +| level | int | 错误级别：1 - 警告，2 - 错误，3 - 致命 | +| description | string | 检查项描述 | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/common.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/common.md new file mode 100644 index 0000000000..76f6c0b527 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/common.md @@ -0,0 +1,47 @@ +## 通用字段和约定 + +#### global_var + +| 字段 | 类型 | 必选 | 描述 | +|-------------|------------|--------|------------| +| id | long | 否 | 全局变量 id，唯一标识。如果 id 为空，那么使用 name 作为唯一标识 | +| name | string | 否 | 全局变量名 | +| description | string | 否 | 全局变量描述 | +| type | int | 否 | 全局变量类型 | +| required | int | 否 | 变量是否必填 | +| value | string | 否 | 字符、密码、命名空间、数组类型的全局变量的值 | +| server | object | 否 | 主机类型全局变量的值 | + +#### server +| 字段 | 类型 | 必选 | 描述 | +|-----------------------|-------|--------|------------| +| host_id_list | array | 否 | 主机ID列表 | +| ip_list | array | 否 | ***不推荐使用，建议使用host_id_list参数***;如果host_id_list与ip_list同时存在，将忽略ip_list参数。主机IP 列表，定义见ip | +| dynamic_group_list | array | 否 | 动态分组列表，定义见dynamic_group | +| topo_node_list | array | 否 | 动态 topo 节点列表，定义见topo_node | + +#### ip +| 字段 | 类型 | 必选 | 描述 | +|-------------|---------|--------|---------| +| bk_host_id | long | 是 | 主机ID | +| bk_cloud_id | int | 是 | 管控区域ID | +| ip | string | 是 | IP地址 | + +#### dynamic_group + +| 字段 | 类型 | 必选 | 描述 | +| ---- | ------ | ---- | -------------- | +| id | string | 是 | CMDB动态分组ID | + +#### topo_node + +| 字段 | 类型 | 必选 | 描述 | +|------------------|--------|--------|------------| +| id | long | 是 | 动态topo节点ID，对应CMDB API 中的 bk_inst_id | +| node_type | string | 是 | 动态topo节点类型，对应CMDB API 中的 bk_obj_id,比如"module","set"| + +#### account +| 字段 | 类型 | 必选 | 描述 | +|-------|--------|--------|------------| +| id | long | 否 | 账号 ID | +| name | string | 否 | 账号名 | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/create_account.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/create_account.md new file mode 100644 index 0000000000..c0858a012d --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/create_account.md @@ -0,0 +1,94 @@ +### 功能描述 + +创建账号（当前仅支持系统账号） + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +|---------------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID，与bk_scope_type对应，表示业务ID或者业务集ID | +| account | string | 是 | 账号名称 | +| type | int | 是 | 账号类型（1：Linux，2：Windows）| +| category | int | 是 | 账号用途（1：系统账号）| +| password | string | 否 | 系统账号密码，账号用途为系统账号且账号类型为Windows时必传 | +| alias | string | 否 | 别名，不传则以账号名称作为别名 | +| description | string | 否 | 描述 | + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "account": "Admin", + "type": 2, + "category": 1, + "password": "AdminPassword", + "alias": "Admin", + "description": "An account for windows" +} +``` + +### 返回结果示例 + +```json +{ + "code": 0, + "message": null, + "result": true, + "data": { + "id": 70, + "bk_scope_type": "biz", + "bk_scope_id": "1", + "account": "Admin", + "alias": "Admin", + "category": 1, + "type": 2, + "os": "Windows", + "description": "An account for windows", + "creator": "admin", + "create_time": 1614659536108, + "last_modify_user": "admin", + "last_modify_time": 1614659536116 + }, + "job_request_id": "4e7acb216087eb96" +} +``` + +### 返回结果参数说明 + +#### response +| 字段 | 类型 | 描述 | +|--------------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + + +#### data + +| 字段 | 类型 | 描述 | +|-----------------------|-----------|-----------| +| id | long | 账号ID | +| bk_scope_type | string | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| account | string | 账号名称 | +| alias | string | 账号别名 | +| category | int | 账号用途（1：系统账号） | +| type | int | 账号类型（1：Linux，2：Windows）| +| os | string | 账号用途为系统账号时该字段生效，账号对应的OS | +| description | string | 账号描述 | +| creator | string | 创建人 | +| create_time | long | 创建时间Unix时间戳（ms） | +| last_modify_user | string | 最近一次修改人 | +| last_modify_time | long | 最近一次修改时间Unix时间戳（ms） | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/create_credential.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/create_credential.md new file mode 100644 index 0000000000..b24aa2010b --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/create_credential.md @@ -0,0 +1,70 @@ +### 功能描述 + +新建凭证。 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +|----------------------------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| name | string | 是 | 凭证名称 | +| type | string | 是 | 凭证类型，取值可为ACCESS_KEY_SECRET_KEY,PASSWORD,USERNAME_PASSWORD,SECRET_KEY | +| description | string | 否 | 凭证描述 | +| credential_access_key | string | 否 | 凭证类型为ACCESS_KEY_SECRET_KEY时填写 | +| credential_secret_key | string | 否 | 凭证类型为ACCESS_KEY_SECRET_KEY/SECRET_KEY时填写 | +| credential_username | string | 否 | 凭证类型为USERNAME_PASSWORD时填写 | +| credential_password | string | 否 | 凭证类型为USERNAME_PASSWORD/PASSWORD时填写 | + + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "name": "testCredential", + "type": "USERNAME_PASSWORD", + "description": "This is a test credential", + "credential_username": "admin", + "credential_password": "password" +} +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "id": "06644309e10e4068b3c7b32799668210" + } +} +``` + +### 返回结果参数说明 + +#### response +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + + +#### data + +| 字段 | 类型 |字段是否一定存在 | 描述 | +|-----------|-------|---------------|---------| +| id | string |是 | 凭证ID | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/create_dangerous_rule.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/create_dangerous_rule.md new file mode 100644 index 0000000000..6f482388a2 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/create_dangerous_rule.md @@ -0,0 +1,79 @@ +### 功能描述 + +新建高危语句检测规则。 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +| -------------------- | ------ | ---- | ------------------------------------------------------------ | +| expression | string | 是 | 表达式 | +| script_language_list | array | 是 | 脚本语言列表:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - sql | +| description | string | 是 | 规则描述 | +| action | int | 是 | 处理动作: 1 - 扫描, 2 - 拦截 | + + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "expression": "rm -rf", + "script_language_list": [1], + "description": "drangerous!!!", + "action": 2 +} +``` + +### 返回结果示例 + +```json +{ + "code": 0, + "result": true, + "data": { + "id": 1, + "expression": "rm -rf", + "script_language_list": [1], + "description": "drangerous!!!", + "action": 2, + "status": 0, + "creator": "admin", + "create_time": 1695193968000, + "last_modify_user": "admin", + "last_modify_time": 1695193968000 + } +} +``` + +### 返回结果参数说明 + +#### response + +| 字段 | 类型 | 描述 | +| ---------- | ------ | ------------------------------------------ | +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息 | +| data | object | 请求返回的数据 | +| permission | object | 权限信息 | + +#### data + +| 字段 | 类型 | 描述 | +| -------------------- | ------ | ------------------------------------------------------------ | +| id | long | 高危语句规则ID | +| expression | string | 表达式 | +| script_language_list | array | 脚本语言列表:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - sql | +| description | string | 规则描述 | +| action | int | 处理动作: 1 - 扫描, 2 - 拦截 | +| status | int | 启用状态: 0 - 停用, 1 - 启用 | +| creator | string | 创建人 | +| create_time | long | 创建时间Unix时间戳（ms） | +| last_modify_user | string | 最近一次修改人 | +| last_modify_time | long | 最近一次修改时间Unix时间戳（ms） | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/create_file_source.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/create_file_source.md new file mode 100644 index 0000000000..f2a7d0f1cc --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/create_file_source.md @@ -0,0 +1,87 @@ +### 功能描述 + +新建文件源，当前仅支持创建关联自动选择公共接入点的蓝鲸制品库文件源。 +创建的文件源具备以下默认特性： + +| 特性 | 取值 | +|-----------------|------------| +|存储类型|对象存储| +|文件源类型|蓝鲸制品库| +|是否为公共文件源|否| +|接入点选择范围|公共接入点| +|接入点选择模式|自动| + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| code | string | 是 | 文件源标识，英文字符开头，1-32位英文字符、下划线、数字组成，创建后不可更改 | +| alias | string | 是 | 文件源别名 | +| type | string | 是 | 文件源类型，当前仅支持蓝鲸制品库，BLUEKING_ARTIFACTORY | +| access_params | object | 是 | 文件源接入参数，根据type传入不同的对象，见后续说明 | +| credential_id | string | 否 | 文件源使用的凭证Id | +| file_prefix | string | 否 | Job对从该文件源分发的文件加上的前缀，不传默认不加前缀 | + +### access_params +**type为BLUEKING_ARTIFACTORY** + +| 字段 | 类型 | 必选 | 描述 | +|-----------------|------------|--------|------------| +| base_url | string | 是 | 对接的制品库实例根地址，例如：https://bkrepo.com | + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "code": "sopsArtifactory", + "alias": "标准运维制品库文件源", + "type": "BLUEKING_ARTIFACTORY", + "access_params": { + "base_url": "https://bkrepo.com" + }, + "credential_id": "06644309e10e4068b3c7b32799668210", + "file_prefix": "" +} +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "id": 1 + } +} +``` + +### 返回结果参数说明 + +#### response +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + + +#### data + +| 字段 | 类型 |字段是否一定存在 | 描述 | +|-----------|-------|---------------|---------| +| id | int |是 | 文件源ID | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/create_public_script.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/create_public_script.md new file mode 100644 index 0000000000..3a0b99d0ce --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/create_public_script.md @@ -0,0 +1,87 @@ +### 功能描述 + +新建公共脚本。 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +| --------------- | ------ | ---- | ------------------------------------------------------------ | +| name | string | 是 | 公共脚本名称 | +| description | string | 否 | 公共脚本描述 | +| script_language | int | 是 | 脚本语言:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - sql | +| content | string | 是 | 脚本内容，需Base64编码 | +| version | string | 是 | 版本号 | +| version_desc | string | 否 | 版本描述 | + + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "name": "public script test", + "description": "public script test", + "script_language": "1", + "content": "IyEvYmluL2Jhc2gKbHM=", + "version": "1.0" +} +``` + +### 返回结果示例 + +```json +{ + "code": 0, + "result": true, + "data": { + "id": 1000018, + "script_id": "4537fb49ec0840a1b91cef4179c99f9c", + "name": "public script test", + "script_language": 1, + "content": "#!/bin/bash\nls", + "creator": "admin", + "create_time": 1691739630000, + "last_modify_user": "admin", + "last_modify_time": 1691739630000, + "version": "1.0", + "status": 0, + "description": "public script test" + } +} +``` + +### 返回结果参数说明 + +#### response + +| 字段 | 类型 | 描述 | +| ---------- | ------ | ------------------------------------------ | +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息 | +| data | object | 请求返回的数据 | +| permission | object | 权限信息 | + +#### data + +| 字段 | 类型 | 描述 | +| ----------------- | ------ | ------------------------------------------------------------ | +| id | long | 公共脚本版本ID | +| script_id | string | 公共脚本ID | +| name | string | 公共脚本名称 | +| script_language | int | 脚本语言:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - sql | +| content | string | 公共脚本版本内容 | +| creator | string | 创建人 | +| create_time | long | 创建时间Unix时间戳（ms） | +| last_modify_user | string | 最近一次修改人 | +| last_modify_time | long | 最近一次修改时间Unix时间戳（ms） | +| version | string | 公共脚本版本 | +| version_desc | string | 公共脚本版本描述 | +| status | int | 公共脚本版本状态（0：未上线，1：已上线，2：已下线，3：已禁用） | +| description | string | 公共脚本描述 | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/create_public_script_version.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/create_public_script_version.md new file mode 100644 index 0000000000..067a72f070 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/create_public_script_version.md @@ -0,0 +1,85 @@ +### 功能描述 + +新建公共脚本版本。 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +| ------------ | ------ | ---- | ---------------------- | +| script_id | string | 是 | 公共脚本ID | +| content | string | 是 | 脚本内容，需Base64编码 | +| version | string | 是 | 版本号 | +| version_desc | string | 否 | 版本描述 | + + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "script_id": "4537fb49ec0840a1b91cef4179c99f9c", + "content": "IyEvYmluL2Jhc2gKbHM=", + "version": "2.0", + "version_desc": "version 2" +} +``` + +### 返回结果示例 + +```json +{ + "code": 0, + "result": true, + "data": { + "id": 1000020, + "script_id": "4537fb49ec0840a1b91cef4179c99f9c", + "name": "public script test", + "script_language": 1, + "content": "#!/bin/bash\nls", + "creator": "admin", + "create_time": 1691739630000, + "last_modify_user": "admin", + "last_modify_time": 1691739630000, + "version": "1.0", + "version_desc": "version 2", + "status": 0, + "description": "public script test" + } +} +``` + +### 返回结果参数说明 + +#### response + +| 字段 | 类型 | 描述 | +| ---------- | ------ | ------------------------------------------ | +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息 | +| data | object | 请求返回的数据 | +| permission | object | 权限信息 | + +#### data + +| 字段 | 类型 | 描述 | +| ----------------- | ------ | ------------------------------------------------------------ | +| id | long | 公共脚本版本ID | +| script_id | string | 公共脚本ID | +| name | string | 公共脚本名称 | +| script_language | int | 脚本语言:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - sql | +| content | string | 公共脚本版本内容 | +| creator | string | 创建人 | +| create_time | long | 创建时间Unix时间戳（ms） | +| last_modify_user | string | 最近一次修改人 | +| last_modify_time | long | 最近一次修改时间Unix时间戳（ms） | +| version | string | 公共脚本版本 | +| version_desc | string | 公共脚本版本描述 | +| status | int | 公共脚本版本状态（0：未上线，1：已上线，2：已下线，3：已禁用） | +| description | string | 公共脚本描述 | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/create_script.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/create_script.md new file mode 100644 index 0000000000..ad9cee9b14 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/create_script.md @@ -0,0 +1,96 @@ +### 功能描述 + +新建脚本。 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +| --------------- | ------ | ---- | ------------------------------------------------------------ | +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| name | string | 是 | 脚本名称 | +| description | string | 否 | 脚本描述 | +| script_language | int | 是 | 脚本语言:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - sql | +| content | string | 是 | 脚本内容，需Base64编码 | +| version | string | 是 | 版本号 | +| version_desc | string | 否 | 版本描述 | + + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type":"biz", + "bk_scope_id":"2", + "name": "script test", + "description": "script test", + "script_language": "1", + "content": "IyEvYmluL2Jhc2gKZGF0ZQo=", + "version": "1.0" +} +``` + +### 返回结果示例 + +```json +{ + "code": 0, + "result": true, + "data": { + "id": 1000019, + "script_id": "4a350b0e0707450e93326f6ace921072", + "name": "script test", + "script_language": 1, + "bk_scope_type": "biz", + "bk_scope_id": "2", + "content": "#!/bin/bash\ndate\n", + "creator": "admin", + "create_time": 1691741073000, + "last_modify_user": "admin", + "last_modify_time": 1691741073000, + "version": "1.0", + "status": 0, + "description": "script test" + } +} +``` + +### 返回结果参数说明 + +#### response + +| 字段 | 类型 | 描述 | +| ---------- | ------ | ------------------------------------------ | +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息 | +| data | object | 请求返回的数据 | +| permission | object | 权限信息 | + +#### data + +| 字段 | 类型 | 描述 | +| ----------------- | ------ | ------------------------------------------------------------ | +| id | long | 脚本版本ID | +| script_id | string | 脚本ID | +| name | string | 脚本名称 | +| script_language | int | 脚本语言:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell | +| bk_scope_type | string | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| content | string | 脚本版本内容 | +| creator | string | 创建人 | +| create_time | long | 创建时间Unix时间戳（ms） | +| last_modify_user | string | 最近一次修改人 | +| last_modify_time | long | 最近一次修改时间Unix时间戳（ms） | +| version | string | 脚本版本 | +| version_desc | string | 版本描述 | +| status | int | 脚本版本状态（0：未上线，1：已上线，2：已下线，3：已禁用） | +| description | string | 脚本描述 | + diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/create_script_version.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/create_script_version.md new file mode 100644 index 0000000000..695861cd84 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/create_script_version.md @@ -0,0 +1,93 @@ +### 功能描述 + +新建脚本版本。 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +| ------------- | ------ | ---- | ------------------------------------------------------- | +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| script_id | string | 是 | 脚本ID | +| content | string | 是 | 脚本内容，需Base64编码 | +| version | string | 是 | 版本号 | +| version_desc | string | 否 | 版本描述 | + + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "2", + "script_id": "4a350b0e0707450e93326f6ace921072", + "content": "IyEvYmluL2Jhc2gKZGF0ZQo=", + "version": "2.0", + "version_desc": "version 2" +} +``` + +### 返回结果示例 + +```json +{ + "code": 0, + "result": true, + "data": { + "id": 1000021, + "script_id": "4a350b0e0707450e93326f6ace921072", + "name": "script test", + "script_language": 1, + "bk_scope_type": "biz", + "bk_scope_id": "2", + "content": "#!/bin/bash\ndate\n", + "creator": "admin", + "create_time": 1691743535000, + "last_modify_user": "admin", + "last_modify_time": 1691743535000, + "version": "2.0", + "version_desc": "version 2", + "status": 0, + "description": "script test" + } +} +``` + +### 返回结果参数说明 + +#### response + +| 字段 | 类型 | 描述 | +| ---------- | ------ | ------------------------------------------ | +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息 | +| data | object | 请求返回的数据 | +| permission | object | 权限信息 | + +#### data + +| 字段 | 类型 | 描述 | +| ----------------- | ------ | ------------------------------------------------------------ | +| id | long | 脚本版本ID | +| script_id | string | 脚本ID | +| name | string | 脚本名称 | +| script_language | int | 脚本语言:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - sql | +| bk_scope_type | string | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| content | string | 脚本版本内容 | +| creator | string | 创建人 | +| create_time | long | 创建时间Unix时间戳（ms） | +| last_modify_user | string | 最近一次修改人 | +| last_modify_time | long | 最近一次修改时间Unix时间戳（ms） | +| version | string | 脚本版本 | +| version_desc | string | 版本描述 | +| status | int | 脚本版本状态（0：未上线，1：已上线，2：已下线，3：已禁用） | +| description | string | 脚本描述 | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/delete_account.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/delete_account.md new file mode 100644 index 0000000000..6db8d984ed --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/delete_account.md @@ -0,0 +1,86 @@ +### 功能描述 + +删除账号。 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +|---------------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| id | long | 否 | 账号ID | + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "id": 70 +} +``` + +### 返回结果示例 + +```json +{ + "code": 0, + "message": null, + "result": true, + "data": { + "id": 70, + "bk_scope_type": "biz", + "bk_scope_id": "1", + "account": "Admin", + "alias": "Admin", + "category": 1, + "type": 2, + "db_system_account_id": null, + "os": "Windows", + "description": "An account for windows", + "creator": "admin", + "create_time": 1614659536108, + "last_modify_user": "admin", + "last_modify_time": 1614659536116 + }, + "job_request_id": "4e7acb216087eb96" +} +``` + +### 返回结果参数说明 + +#### response +| 字段 | 类型 | 描述 | +|--------------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + + +#### data + +| 字段 | 类型 | 描述 | +|-----------------------|-----------|-----------| +| id | long | 账号ID | +| bk_scope_type | string | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| account | string | 账号名称 | +| alias | string | 账号别名 | +| category | int | 账号用途（1：系统账号，2：数据库账号） | +| type | int | 账号类型（1：Linux，2：Windows，9：MySQL，10：Oracle，11：DB2）| +| db_system_account_id | long | 账号用途为数据库账号时该字段生效，表示数据库账号对应的系统账号ID | +| os | string | 账号用途为系统账号时该字段生效，账号对应的OS | +| description | string | 描述 | +| creator | string | 创建人 | +| create_time | long | 创建时间Unix时间戳（ms） | +| last_modify_user | string | 最近一次修改人 | +| last_modify_time | long | 最近一次修改时间Unix时间戳（ms） | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/delete_cron.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/delete_cron.md new file mode 100644 index 0000000000..aa4ef8c67a --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/delete_cron.md @@ -0,0 +1,53 @@ +### 功能描述 + +删除定时任务 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +| ------------- | ------ | ---- | ------------------------------------------------------- | +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| id | long | 是 | 定时任务ID | + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type":"biz", + "bk_scope_id":"2", + "id": 1000045 +} +``` + +### 返回结果示例 + +```json +{ + "code": 0, + "result": true +} +``` + +### 返回结果参数说明 + +#### response + +| 字段 | 类型 | 描述 | +| ---------- | ------ | ------------------------------------------ | +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息 | +| data | object | 请求返回的数据 | +| permission | object | 权限信息 | + +#### data + +无 diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/delete_dangerous_rule.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/delete_dangerous_rule.md new file mode 100644 index 0000000000..e4696f4550 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/delete_dangerous_rule.md @@ -0,0 +1,50 @@ +### 功能描述 + +删除高危语句检测规则。 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +| ---- | ---- | ---- | -------------- | +| id | long | 是 | 高危语句规则ID | + + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "id": 1 +} +``` + +### 返回结果示例 + +```json +{ + "code": 0, + "result": true +} +``` + +### 返回结果参数说明 + +#### response + +| 字段 | 类型 | 描述 | +| ---------- | ------ | ------------------------------------------ | +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息 | +| data | object | 请求返回的数据 | +| permission | object | 权限信息 | + +#### data + +无 diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/delete_public_script.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/delete_public_script.md new file mode 100644 index 0000000000..92528e1d25 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/delete_public_script.md @@ -0,0 +1,50 @@ +### 功能描述 + +删除公共脚本，同时会删除该脚本下的所有的脚本版本。 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +| --------- | ------ | ---- | ---------- | +| script_id | string | 是 | 公共脚本ID | + + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "script_id": "4537fb49ec0840a1b91cef4179c99f9c" +} +``` + +### 返回结果示例 + +```json +{ + "code": 0, + "result": true +} +``` + +### 返回结果参数说明 + +#### response + +| 字段 | 类型 | 描述 | +| ---------- | ------ | ------------------------------------------ | +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息 | +| data | object | 请求返回的数据 | +| permission | object | 权限信息 | + +#### data + +无 diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/delete_public_script_version.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/delete_public_script_version.md new file mode 100644 index 0000000000..14eae102f1 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/delete_public_script_version.md @@ -0,0 +1,52 @@ +### 功能描述 + +删除公共脚本版本。 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +| ----------------- | ------ | ---- | -------------- | +| script_id | string | 是 | 公共脚本ID | +| script_version_id | long | 是 | 公共脚本版本ID | + + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "script_id":"4537fb49ec0840a1b91cef4179c99f9c", + "script_version_id": "1000017" +} +``` + +### 返回结果示例 + +```json +{ + "code": 0, + "result": true +} +``` + +### 返回结果参数说明 + +#### response + +| 字段 | 类型 | 描述 | +| ---------- | ------ | ------------------------------------------ | +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息 | +| data | object | 请求返回的数据 | +| permission | object | 权限信息 | + +#### data + +无 diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/delete_script.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/delete_script.md new file mode 100644 index 0000000000..baf0122066 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/delete_script.md @@ -0,0 +1,54 @@ +### 功能描述 + +删除脚本，同时会删除该脚本下的所有的脚本版本。 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +| ------------- | ------ | ---- | ------------------------------------------------------- | +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| script_id | long | 是 | 脚本ID | + + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type":"biz", + "bk_scope_id":"2", + "script_id": "4a350b0e0707450e93326f6ace921072" +} +``` + +### 返回结果示例 + +```json +{ + "code": 0, + "result": true +} +``` + +### 返回结果参数说明 + +#### response + +| 字段 | 类型 | 描述 | +| ---------- | ------ | ------------------------------------------ | +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息 | +| data | object | 请求返回的数据 | +| permission | object | 权限信息 | + +#### data + +无 diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/delete_script_version.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/delete_script_version.md new file mode 100644 index 0000000000..d2cfdc4960 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/delete_script_version.md @@ -0,0 +1,56 @@ +### 功能描述 + +删除脚本版本。 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +| ----------------- | ------ | ---- | ------------------------------------------------------- | +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| script_id | string | 是 | 脚本ID | +| script_version_id | long | 是 | 脚本版本ID | + + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type":"biz", + "bk_scope_id":"2", + "script_id": "4a350b0e0707450e93326f6ace921072", + "script_version_id": 1000019 +} +``` + +### 返回结果示例 + +```json +{ + "code": 0, + "result": true +} +``` + +### 返回结果参数说明 + +#### response + +| 字段 | 类型 | 描述 | +| ---------- | ------ | ------------------------------------------ | +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息 | +| data | object | 请求返回的数据 | +| permission | object | 权限信息 | + +#### data + +无 diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/disable_dangerous_rule.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/disable_dangerous_rule.md new file mode 100644 index 0000000000..1d8e788bed --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/disable_dangerous_rule.md @@ -0,0 +1,57 @@ +### 功能描述 + +停用高危语句检测规则。 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +| ---- | ---- | ---- | -------------- | +| id | long | 是 | 高危语句规则ID | + + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "id": 1 +} +``` + +### 返回结果示例 + +```json +{ + "code": 0, + "result": true, + "data": { + "id": 1, + "status": 0 + } +} +``` + +### 返回结果参数说明 + +#### response + +| 字段 | 类型 | 描述 | +| ---------- | ------ | ------------------------------------------ | +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息 | +| data | object | 请求返回的数据 | +| permission | object | 权限信息 | + +#### data + +| 字段 | 类型 | 描述 | +| ------ | ---- | ---------------------------- | +| id | long | 高危语句规则ID | +| status | int | 启用状态: 0 - 停用, 1 - 启用 | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/disable_public_script_version.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/disable_public_script_version.md new file mode 100644 index 0000000000..9015340d83 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/disable_public_script_version.md @@ -0,0 +1,61 @@ +### 功能描述 + +禁用公共脚本版本，一旦禁用成功，不可恢复！且线上引用该版本的作业步骤都会无法执行，请务必谨慎操作。 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +| ----------------- | ------ | ---- | -------------- | +| script_id | string | 是 | 公共脚本ID | +| script_version_id | long | 是 | 公共脚本版本ID | + + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "script_id":"4537fb49ec0840a1b91cef4179c99f9c", + "script_version_id": 1000018 +} +``` + +### 返回结果示例 + +```json +{ + "code": 0, + "result": true, + "data": { + "id": 1000018, + "script_id": "4537fb49ec0840a1b91cef4179c99f9c", + "status": 3 + } +} +``` + +### 返回结果参数说明 + +#### response + +| 字段 | 类型 | 描述 | +| ---------- | ------ | ------------------------------------------ | +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息 | +| data | object | 请求返回的数据 | +| permission | object | 权限信息 | + +#### data + +| 字段 | 类型 | 描述 | +| ----------------- | ------ | ------------------------------------------------------------ | +| script_id | string | 脚本ID | +| id | long | 脚本版本ID | +| status | int | 脚本版本当前状态（0：未上线，1：已上线，2：已下线，3：已禁用） | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/disable_script_version.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/disable_script_version.md new file mode 100644 index 0000000000..ab649bfe29 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/disable_script_version.md @@ -0,0 +1,65 @@ +### 功能描述 + +禁用脚本版本，一旦禁用成功，不可恢复！且线上引用该版本的作业步骤都会无法执行，请务必谨慎操作。 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +| ----------------- | ------ | ---- | ------------------------------------------------------- | +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| script_id | string | 是 | 脚本ID | +| script_version_id | long | 是 | 脚本版本ID | + + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type":"biz", + "bk_scope_id":"2", + "script_id": "4a350b0e0707450e93326f6ace921072", + "script_version_id": 1000019 +} +``` + +### 返回结果示例 + +```json +{ + "code": 0, + "result": true, + "data": { + "id": 1000019, + "script_id": "4a350b0e0707450e93326f6ace921072", + "status": 3 + } +} +``` + +### 返回结果参数说明 + +#### response + +| 字段 | 类型 | 描述 | +| ---------- | ------ | ------------------------------------------ | +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息 | +| data | object | 请求返回的数据 | +| permission | object | 权限信息 | + +#### data + +| 字段 | 类型 | 描述 | +| ----------------- | ------ | ------------------------------------------------------------ | +| script_id | string | 脚本版本ID | +| id | long | 脚本版本ID | +| status | int | 脚本版本当前状态（0：未上线，1：已上线，2：已下线，3：已禁用） | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/enable_dangerous_rule.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/enable_dangerous_rule.md new file mode 100644 index 0000000000..b552aa83bd --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/enable_dangerous_rule.md @@ -0,0 +1,57 @@ +### 功能描述 + +启用高危语句检测规则。 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +| ---- | ---- | ---- | -------------- | +| id | long | 是 | 高危语句规则ID | + + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "id": 1 +} +``` + +### 返回结果示例 + +```json +{ + "code": 0, + "result": true, + "data": { + "id": 1, + "status": 1 + } +} +``` + +### 返回结果参数说明 + +#### response + +| 字段 | 类型 | 描述 | +| ---------- | ------ | ------------------------------------------ | +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息 | +| data | object | 请求返回的数据 | +| permission | object | 权限信息 | + +#### data + +| 字段 | 类型 | 描述 | +| ------ | ---- | ---------------------------- | +| id | long | 高危语句规则ID | +| status | int | 启用状态: 0 - 停用, 1 - 启用 | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/execute_job_plan.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/execute_job_plan.md new file mode 100644 index 0000000000..1b41069642 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/execute_job_plan.md @@ -0,0 +1,127 @@ +### 功能描述 + +启动作业执行方案 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| job_plan_id | long | 是 | 作业执行方案ID | +| global_var_list | array | 否 | 全局变量。对于作业执行方案中的全局变量值，如果请求参数中包含该变量，则使用传入的变量值；否则使用执行方案当前已配置的默认值。定义见global_var | +| callback_url | string | 否 | 回调URL，当任务执行完成后，JOB会调用该URL告知任务执行结果。回调协议参考callback_protocol组件文档 | + +#### global_var + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| id | long | 否 | 全局变量id，唯一标识。如果id为空，那么使用name作为唯一标识 | +| name | string | 否 | 全局变量name | +| value | string | 否 | 字符、密码、数组、命名空间类型的全局变量的值 | +| server | object | 否 | 主机类型全局变量的值，见server定义 | + +#### server + +| 字段 | 类型 | 必选 | 描述 | +| ------------------ | ----- | ---- | ----------------------------------- | +| host_id_list | array | 否 | 主机ID列表 | +| ip_list | array | 否 | ***不推荐使用，建议使用host_id_list参数***;如果host_id_list与ip_list同时存在，将忽略ip_list参数。主机IP 列表，定义见ip | +| dynamic_group_list | array | 否 | 动态分组列表，定义见dynamic_group | +| topo_node_list | array | 否 | 动态 topo 节点列表，定义见topo_node | + +#### ip + +| 字段 | 类型 | 必选 | 描述 | +| ----------- | ------ | ---- | -------- | +| bk_cloud_id | int | 是 | 管控区域ID | +| ip | string | 是 | IP地址 | + +#### dynamic_group + +| 字段 | 类型 | 必选 | 描述 | +| ---- | ------ | ---- | -------------- | +| id | string | 是 | CMDB动态分组ID | + +#### topo_node + +| 字段 | 类型 | 必选 | 描述 | +| --------- | ------ | ---- | ------------------------------------------------------------ | +| id | long | 是 | 动态topo节点ID，对应CMDB API 中的 bk_inst_id | +| node_type | string | 是 | 动态topo节点类型，对应CMDB API 中的 bk_obj_id,比如"module","set" | + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "job_plan_id": 100, + "global_var_list": [ + { + "id": 436, + "server": { + "dynamic_group_list": [ + { + "id": "blo8gojho0skft7pr5q0" + } + ], + "host_id_list": [ + 101, + 102 + ], + "topo_node_list": [ + { + "id": 1000, + "node_type": "module" + } + ] + } + }, + { + "name": "param_name", + "value": "param_value" + } + ] +} +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "job_instance_name": "Test", + "job_instance_id": 10000 + } +} +``` + +### 返回结果参数说明 + +#### response +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + +#### data + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| job_instance_id | long | 作业实例ID | +| job_instance_name | long | 作业实例名称 | + diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/fast_execute_script.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/fast_execute_script.md new file mode 100644 index 0000000000..415c4708c8 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/fast_execute_script.md @@ -0,0 +1,125 @@ +### 功能描述 + +快速执行脚本 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +|---------------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| script_version_id | long | 否 | 脚本版本ID。当script_version_id不为空的时候，使用script_version_id对应的脚本版本 | +| script_id | string | 否 | 脚本ID。当传入script_id，且script_version_id为空的时候，使用脚本的上线版本 | +| script_content | string | 否 | 脚本内容Base64。如果不存在script_version_id和script_id,那么使用script_content。优先级：script_version_id>script_id>script_content | +| task_name | string | 否 | 自定义作业名称 | +| script_param | string | 否 | 脚本参数Base64。注意：如果有多个参数，比如"param1 param2"这种，需要对"param1 param2"整体进行base64编码，而不是对每个参数进行base64编码再拼接起来 | +| timeout | long | 否 | 脚本执行超时时间，秒。默认7200，取值范围1-86400 | +| account_alias | string | 否 | 执行账号别名。与account_id必须存在一个。当同时存在account_alias和account_id时，account_id优先。 | +| account_id | long | 否 | 执行账号ID。与account_alias必须存在一个。当同时存在account_alias和account_id时，account_id优先。 | +| is_param_sensitive | int | 否 | 敏感参数将会在执行详情页面上隐藏, 0:不是（默认），1:是 | +| script_language | int | 否 | 脚本语言：1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell。当使用script_content传入自定义脚本的时候，需要指定script_language | +| target_server | object | 否 | 目标服务器，见server定义 | +| callback_url | string | 否 | 回调URL，当任务执行完成后，JOB会调用该URL告知任务执行结果。回调协议参考callback_protocol组件文档 | + +#### server + +| 字段 | 类型 | 必选 | 描述 | +| ------------------ | ----- | ---- | ------------------------------------------------------------ | +| host_id_list | array | 否 | 主机ID列表 | +| ip_list | array | 否 | ***不推荐使用，建议使用host_id_list参数***。主机IP 列表，定义见ip | +| dynamic_group_list | array | 否 | 动态分组列表，定义见dynamic_group | +| topo_node_list | array | 否 | 动态 topo 节点列表，定义见topo_node | + +#### ip + +| 字段 | 类型 | 必选 | 描述 | +| ----------- | ------ | ---- | -------- | +| bk_cloud_id | long | 是 | 管控区域ID | +| ip | string | 是 | IP地址 | + +#### dynamic_group + +| 字段 | 类型 | 必选 | 描述 | +| ---- | ------ | ---- | -------------- | +| id | string | 是 | CMDB动态分组ID | + +#### topo_node_list + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| id | long | 是 | 动态topo节点ID，对应CMDB API 中的 bk_inst_id | +| node_type | string | 是 | 动态topo节点类型，对应CMDB API 中的 bk_obj_id,比如"module","set"| + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "script_version_id": 1, + "script_content": "ZWNobyAkMQ==", + "script_param": "aGVsbG8=", + "timeout": 1000, + "account_id": 1000, + "is_param_sensitive": 0, + "script_language": 1, + "target_server": { + "dynamic_group_list": [ + { + "id": "blo8gojho0skft7pr5q0" + } + ], + "host_id_list": [ + 101, + 102 + ], + "topo_node_list": [ + { + "id": 1000, + "node_type": "module" + } + ] + } +} +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "job_instance_name": "API Quick execution script1521100521303", + "job_instance_id": 10000, + "step_instance_id": 10001 + } +} +``` + +### 返回结果参数说明 + +#### response +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + +#### data + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| job_instance_id | long | 作业实例ID | +| job_instance_name | long | 作业实例名称 | +| step_instance_id | long | 步骤实例ID | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/fast_execute_sql.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/fast_execute_sql.md new file mode 100644 index 0000000000..373cb79966 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/fast_execute_sql.md @@ -0,0 +1,115 @@ +### 功能描述 + +快速执行SQL脚本 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +|---------------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| script_version_id | long | 否 | SQL脚本版本ID | +| script_id | string | 否 | 脚本ID。当传入script_id，且script_version_id为空的时候，使用当前脚本的上线版本 | +| script_content | string | 否 | 脚本内容Base64。如果不存在script_version_id和script_id,那么使用script_content。优先级：script_version_id>script_id>script_content | +| timeout | int | 否 | 脚本超时时间，秒。默认7200，取值范围1-86400 | +| db_account_id | long | 是 | SQL执行的db账号ID，必填, 从账号管理-DB账号处获得。 | +| target_server | object | 否 | 目标服务器，见server定义 | +| callback_url | string | 否 | 回调URL，当任务执行完成后，JOB会调用该URL告知任务执行结果。回调协议参考callback_protocol组件文档 | + +#### server + +| 字段 | 类型 | 必选 | 描述 | +| ------------------ | ----- | ---- | ------------------------------------------------------------ | +| host_id_list | array | 否 | 主机ID列表 | +| ip_list | array | 否 | ***不推荐使用，建议使用host_id_list参数***。主机IP 列表，定义见ip | +| dynamic_group_list | array | 否 | 动态分组列表，定义见dynamic_group | +| topo_node_list | array | 否 | 动态 topo 节点列表，定义见topo_node | + +#### ip + +| 字段 | 类型 | 必选 | 描述 | +| ----------- | ------ | ---- | -------- | +| bk_cloud_id | long | 是 | 管控区域ID | +| ip | string | 是 | IP地址 | + +#### dynamic_group + +| 字段 | 类型 | 必选 | 描述 | +| ---- | ------ | ---- | -------------- | +| id | string | 是 | CMDB动态分组ID | + +#### topo_node_list + +| 字段 | 类型 | 必选 | 描述 | +| --------- | ------ | ---- | ------------------------------------------------------------ | +| id | long | 是 | 动态topo节点ID，对应CMDB API 中的 bk_inst_id | +| node_type | string | 是 | 动态topo节点类型，对应CMDB API 中的 bk_obj_id,比如"module","set" | + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "script_version_id": 1, + "timeout": 1000, + "db_account_id": 32, + "target_server": { + "dynamic_group_list": [ + { + "id": "blo8gojho0skft7pr5q0" + } + ], + "host_id_list": [ + 101, + 102 + ], + "topo_node_list": [ + { + "id": 1000, + "node_type": "module" + } + ] + } +} +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "job_instance_name": "API Quick SQL Execution1524454292038", + "job_instance_id": 10000, + "step_instance_id": 10001 + } +} +``` + +### 返回结果参数说明 + +#### response +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| +#### data + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| job_instance_id | long | 作业实例ID | +| job_instance_name | long | 作业实例名称 | +| step_instance_id | long | 步骤实例ID | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/fast_transfer_file.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/fast_transfer_file.md new file mode 100644 index 0000000000..816bffce67 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/fast_transfer_file.md @@ -0,0 +1,172 @@ +### 功能描述 +快速分发文件 + +### 请求参数 +{{ common_args_desc }} + +#### 接口参数 +| 字段 | 类型 |
必选
| 描述 | +|------------------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| account_alias | string | 否 | 目标执行账号别名，可从账号页面获取，推荐使用。与account_id必须存在一个。当同时存在account_alias和account_id时，account_id优先。 | +| account_id | long | 否 | 目标执行账号ID，可从get_account_list接口获取。与account_alias必须存在一个。当同时存在account_alias和account_id时，account_id优先。 | +| file_target_path | string | 是 | 文件传输目标路径 | +| file_source_list | array | 是 | 源文件对象数组，见下面file_source定义 | +| timeout | int | 否 | 任务超时时间，秒，默认值为7200。取值范围1-86400。| +| download_speed_limit| int | 否 | 下载限速，单位MB。如果未传入该参数，表示不限速| +| upload_speed_limit| int | 否 | 上传限速，单位MB。如果未传入该参数，表示不限速| +| transfer_mode | int | 否 | 传输模式。1-严谨模式，2-强制模式。默认使用强制模式 | +| target_server | object | 否 | 目标服务器，见server定义 | +| callback_url | string | 否 | 回调URL，当任务执行完成后，JOB会调用该URL告知任务执行结果。回调协议参考callback_protocol组件文档 | + +#### file_source +| 字段 | 类型 |
必选
| 描述 | +|---------------|------------|--------|------------| +| file_list | array | 是 | 支持多个文件，若文件源类型为服务器文件，填写源文件的绝对路径数组；若文件源类型为第三方文件源，COS文件源填写的路径为"bucket名称/文件路径"，例如：testbucket/test.txt | +| account | object | 是 | 文件源账号，见account定义，文件源类型为服务器文件源时必填，文件源类型为第三方文件源时无需填写 | +| server | object | 否 | 源文件服务器，见server定义 | +| file_type | int | 否 | 文件源类型，1：服务器文件，3：第三方文件源文件，不传默认为1 | +| file_source_id | int | 否 | file_type为3时，file_source_id与file_source_code选择一个填写，若都填写，优先使用file_source_id，第三方文件源Id，可从get_job_detail接口返回结果中的步骤详情获取 | +| file_source_code| string | 否 | file_type为3时，file_source_id与file_source_code选择一个填写，若都填写，优先使用file_source_id，第三方文件源标识，可从作业平台的文件分发页面->选择文件源文件弹框中获取 | + +#### account + +| 字段 | 类型 | 必选 | 描述 | +| ----- | ------ | ---- | ------------------------------------------------------------ | +| id | long | 否 | 源执行账号ID，可从get_account_list接口获取。与alias必须存在一个。当同时存在alias和id时，id优先。 | +| alias | string | 否 | 源执行账号别名，可从账号页面获取，推荐使用。与alias必须存在一个。当同时存在alias和id时，id优先。 | + +#### server + +| 字段 | 类型 | 必选 | 描述 | +| ------------------ | ----- | ---- | ------------------------------------------------------------ | +| host_id_list | array | 否 | 主机ID列表 | +| ip_list | array | 否 | ***不推荐使用，建议使用host_id_list参数***。主机IP 列表，定义见ip | +| dynamic_group_list | array | 否 | 动态分组列表，定义见dynamic_group | +| topo_node_list | array | 否 | 动态 topo 节点列表，定义见topo_node | + +#### ip + +| 字段 | 类型 | 必选 | 描述 | +| ----------- | ------ | ---- | -------- | +| bk_cloud_id | long | 是 | 管控区域ID | +| ip | string | 是 | IP地址 | + +#### topo_node_list + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| id | long | 是 | 动态topo节点ID，对应CMDB API 中的 bk_inst_id | +| node_type | string | 是 | 动态topo节点类型，对应CMDB API 中的 bk_obj_id,比如"module","set"| + +### 请求参数示例 +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "file_target_path": "/tmp/", + "transfer_mode": 1, + "file_source_list": [ + { + "file_list": [ + "/tmp/REGEX:[a-z]*.txt" + ], + "account": { + "id": 100 + }, + "server": { + "dynamic_group_list": [ + { + "id": "blo8gojho0skft7pr5q0" + }, + { + "id": "blo8gojho0sabc7priuy" + } + ], + "host_id_list": [ + 101, + 102 + ], + "topo_node_list": [ + { + "id": 1000, + "node_type": "module" + } + ] + }, + "file_type": 1 + }, + { + "file_list": [ + "testbucket/test.txt" + ], + "file_type": 3, + "file_source_id": 1 + }, + { + "file_list": [ + "testbucket/test2.txt" + ], + "file_type": 3, + "file_source_code": "testInnerCOS" + } + ], + "target_server": { + "dynamic_group_list": [ + { + "id": "blo8gojho0skft7pr5q0" + }, + { + "id": "blo8gojho0sabc7priuy" + } + ], + "host_id_list": [ + 103, + 104 + ], + "topo_node_list": [ + { + "id": 1000, + "node_type": "module" + } + ] + }, + "account_id": 101 +} +``` +### 返回结果示例 +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "job_instance_name": "API Quick Distribution File1521101427176", + "job_instance_id": 10000, + "step_instance_id": 10001 + } +} +``` + +### 返回结果参数说明 + +#### response +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + +#### data + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| job_instance_id | long | 作业实例ID | +| job_instance_name | long | 作业实例名称 | +| step_instance_id | long | 步骤实例ID | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/generate_local_file_upload_url.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/generate_local_file_upload_url.md new file mode 100644 index 0000000000..84f49cacdd --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/generate_local_file_upload_url.md @@ -0,0 +1,72 @@ +### 功能描述 + +生成本地文件上传URL。 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +|----------------------------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| file_name_list | string[] | 是 | 要上传的文件名列表 | + + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "file_name_list": [ + "file1.txt", + "file2.txt" + ] +} +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "url_map": { + "file1.txt": { + "upload_url": "http://bkrepo.com/generic/temporary/upload/bkjob/localupload/1/008f821f-259b-4f62-bd84-1e89d6f05f0d/admin/file1.txt?token=30adf862fdce4b02b909e6a1a1c762c6", + "path": "1/008f821f-259b-4f62-bd84-1e89d6f05f0d/admin/file1.txt" + }, + "file2.txt": { + "upload_url": "http://bkrepo.com/generic/temporary/upload/bkjob/localupload/1/008f821f-259b-4f62-bd84-1e89d6f05f0d/admin/file2.txt?token=30adf862fdce4b02b909e6a1a1c762c6", + "path": "1/008f821f-259b-4f62-bd84-1e89d6f05f0d/admin/file2.txt" + } + } + } +} +``` + +### 返回结果参数说明 + +#### response +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + + +#### data + +| 字段 | 类型 |字段是否一定存在 | 描述 | +|-----------|----------|---------------|---------| +| url_map | map | 是 | key:传入的文件名，value:upload_url为带凭据的文件上传地址，path为分发该文件时要传给文件分发接口的路径 | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_account_list.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_account_list.md new file mode 100644 index 0000000000..e61bf5c5e6 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_account_list.md @@ -0,0 +1,100 @@ +### 功能描述 + +查询业务下用户有权限的执行账号列表 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +|----------------------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| category | int | 否 | 账号用途（1：系统账号，2：DB账号），不传则不区分 | +| account | string | 否 | 账号名称 | +| alias | string | 否 | 账号别名 | +| start | int | 否 | 分页记录起始位置，不传默认为0 | +| length | int | 否 | 单次返回最大记录数，最大1000，不传默认为20 | + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "category": 1, + "account": "aaa", + "alias": "aaa", + "start": 0, + "length": 1 +} +``` + +### 返回结果示例 + +```json +{ + "code": 0, + "message": null, + "result": true, + "data": { + "start": 0, + "total": 12, + "data": [ + { + "id": 70, + "account": "aaa", + "alias": "aaa", + "category": 1, + "type": 1, + "os": "Linux", + "description":"xxx", + "creator": "admin", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "create_time": 1614659536108, + "last_modify_user": "admin", + "last_modify_time": 1614659536116 + } + ], + "length": 1 + }, + "request_id": "4e7acb216087eb96" +} +``` + +### 返回结果参数说明 + +#### response +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + + +#### data.data + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| id | long | 账号ID | +| bk_scope_type | string |资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| account | string | 账号名称 | +| alias | string | 账号别名 | +| category | int | 账号用途（1：系统账号，2：DB账号） | +| type | int | 账号类型（1：Linux，2：Windows，9：MySQL，10：Oracle，11：DB2）| +| db_system_account_id | long | 账号用途为DB账号时该字段生效，表示DB账号对应的系统账号ID | +| os | string | 账号用途为系统账号时该字段生效，账号对应的OS | +| description | string | 账号描述 | +| creator | string | 创建人 | +| create_time | long | 创建时间Unix时间戳（ms） | +| last_modify_user | string | 最近一次修改人 | +| last_modify_time | long | 最近一次修改时间Unix时间戳（ms） | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_cron_detail.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_cron_detail.md new file mode 100644 index 0000000000..69ed0460bc --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_cron_detail.md @@ -0,0 +1,148 @@ +### 功能描述 + +查询定时作业详情 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +|------------------------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| id | long | 否 | 定时任务 ID | + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "id": 1 +} +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "", + "data": { + "bk_scope_type": "biz", + "bk_scope_id": "1", + "job_plan_id": 100, + "id": 1, + "name": "test", + "status": 1, + "expression": "0/5 * * * ?", + "global_var_list": [ + { + "id": 436, + "name": "ip", + "server": { + "dynamic_group_list": [ + { + "id": "blo8gojho0skft7pr5q0" + }, + { + "id": "blo8gojho0sabc7priuy" + } + ], + "ip_list": [ + { + "bk_host_id": 101, + "bk_cloud_id": 0, + "ip": "127.0.0.1" + }, + { + "bk_host_id": 102, + "bk_cloud_id": 0, + "ip": "127.0.0.2" + } + ], + "topo_node_list": [ + { + "id": 1000, + "node_type": "module" + } + ] + } + }, + { + "id": 437, + "name": "text", + "value": "new String value" + } + ], + "creator": "admin", + "create_time": 1546272000000, + "last_modify_user": "admin", + "last_modify_time": 1577807999999 + } +} +``` + +### 返回结果参数说明 + +#### response +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + + +#### data +| 字段 | 类型 | 描述 | +|------------------|-----------|-----------| +| bk_scope_type | string |资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| job_plan_id | long | 执行方案 ID | +| id | long | 定时作业 ID | +| name | string | 定时作业名称 | +| status | int | 定时作业状态：1.已启动、2.已暂停 | +| expression | string | 定时任务crontab的定时规则，新建时必填，修改时选填。各字段含义为：分 时 日 月 周，如: 0/5 * * * ? 表示每5分钟执行一次 | +| global_var_list | array | 全局变量信息 | +| creator | string | 作业创建人账号 | +| create_time | long | 创建时间，Unix 时间戳 | +| last_modify_user | string | 作业修改人账号 | +| last_modify_time | long | 最后修改时间，Unix 时间戳 | + +#### global_var + +| 字段 | 类型 | 描述 | +|-----------|-----------|------------| +| id | long | 全局变量 id，唯一标识。如果 id 为空，那么使用 name 作为唯一标识 | +| name | string | 全局变量 name | +| value | string | 字符、密码、数组类型的全局变量的值 | +| server | object | 主机类型全局变量的值 | + +#### server +| 字段 | 类型 | 描述 | +|-----------------------|-------|------------| +| ip_list | array | 静态 IP 列表 | +| dynamic_group_list | array | 动态分组 列表 | +| topo_node_list | array | 动态 topo 节点列表 | + +#### ip + +| 字段 | 类型 | 描述 | +|-------------|---------|---------| +| bk_host_id | long | 主机ID | +| bk_cloud_id | int | 管控区域ID | +| ip | string | IP地址 | + +#### topo_node +| 字段 | 类型 | 描述 | +|------------------|--------|------------| +| id | long | 动态topo节点ID，对应CMDB API 中的 bk_inst_id | +| node_type | string | 动态topo节点类型，对应CMDB API 中的 bk_obj_id,比如"module","set"| diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_cron_list.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_cron_list.md new file mode 100644 index 0000000000..e34d2a4797 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_cron_list.md @@ -0,0 +1,105 @@ +### 功能描述 + +查询业务下定时作业信息 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +|------------------------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| name | string | 否 | 定时作业名称 | +| id | long | 否 | 定时任务 ID，如果存在则忽略其他筛选条件，只查询这个指定的作业信息 | +| status | int | 否 | 定时作业状态：1.已启动、2.已暂停 | +| creator | string | 否 | 定时作业创建人账号 | +| create_time_start | long | 否 | 创建起始时间，Unix 时间戳(s) | +| create_time_end | long | 否 | 创建结束时间，Unix 时间戳(s) | +| last_modify_user | string | 否 | 作业修改人账号 | +| last_modify_time_start | long | 否 | 最后修改起始时间，Unix 时间戳(s) | +| last_modify_time_end | long | 否 | 最后修改结束时间，Unix 时间戳(s) | +| start | int | 否 | 默认 0 表示从第 1 条记录开始返回 | +| length | int | 否 | 单次返回最大记录数，最大1000，不传默认为20 | + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "name": "test", + "id": 1000031, + "status": 1, + "creator": "admin", + "create_time_start": 1601371525, + "create_time_end": 1617285956, + "last_modify_user": "admin", + "last_modify_time_start": 1601371525, + "last_modify_time_end": 1617286227, + "start": 0, + "length": 1 +} +``` + +### 返回结果示例 + +```json +{ + "code": 0, + "result": true, + "data": { + "start": 0, + "total": 8, + "data": [ + { + "id": 1000031, + "name": "test", + "status": 1, + "creator": "admin", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "job_plan_id": 1000193, + "expression": "* * * * * *", + "create_time": 1617285956, + "last_modify_user": "admin", + "last_modify_time": 1617286227 + } + ], + "length": 1 + }, + "request_id": "9720d3549c49a48a" +} +``` + +### 返回结果参数说明 + +#### response +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + + +#### data +| 字段 | 类型 | 描述 | +|------------------|-----------|-----------| +| bk_scope_type | string |资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| job_plan_id | long | 执行方案 ID | +| id | long | 定时作业 ID | +| name | string | 定时作业名称 | +| status | int | 定时作业状态：1.已启动、2.已暂停 | +| expression | string | 定时任务crontab的定时规则，新建时必填，修改时选填。各字段含义为：分 时 日 月 周，如: 0/5 * * * ? 表示每5分钟执行一次 | +| creator | string | 作业创建人账号 | +| create_time | long | 创建时间，Unix 时间戳(s) | +| last_modify_user | string | 作业修改人账号 | +| last_modify_time | long | 最后修改时间，Unix 时间戳(s) | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_dangerous_rule_list.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_dangerous_rule_list.md new file mode 100644 index 0000000000..c5b0042dbc --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_dangerous_rule_list.md @@ -0,0 +1,79 @@ +### 功能描述 + +查看高危语句检测规则列表。 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +| -------------------- | ------ | ---- | ------------------------------------------------------------ | +| expression | string | 否 | 表达式 | +| script_language_list | array | 否 | 脚本语言列表:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - sql | +| description | string | 否 | 规则描述 | +| action | int | 否 | 处理动作: 1 - 扫描, 2 - 拦截 | + + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "script_language_list": [1], + "action": 2 +} +``` + +### 返回结果示例 + +```json +{ + "code": 0, + "result": true, + "data": [ + { + "id": 1, + "expression": "rm", + "script_language_list": [1], + "description": "drangerous!!!", + "action": 2, + "status": 0, + "creator": "admin", + "create_time": 1695193968000, + "last_modify_user": "admin", + "last_modify_time": 1695302417000 + } + ] +} +``` + +### 返回结果参数说明 + +#### response + +| 字段 | 类型 | 描述 | +| ---------- | ------ | ------------------------------------------ | +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息 | +| data | object | 请求返回的数据 | +| permission | object | 权限信息 | + +#### data + +| 字段 | 类型 | 描述 | +| -------------------- | ------ | ------------------------------------------------------------ | +| id | long | 高危语句规则ID | +| expression | string | 表达式 | +| script_language_list | array | 脚本语言列表:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - sql | +| description | string | 规则描述 | +| action | int | 处理动作: 1 - 扫描, 2 - 拦截 | +| status | int | 启用状态: 0 - 停用, 1 - 启用 | +| creator | string | 创建人 | +| create_time | long | 创建时间Unix时间戳（ms） | +| last_modify_user | string | 最近一次修改人 | +| last_modify_time | long | 最近一次修改时间Unix时间戳（ms） | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_job_instance_global_var_value.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_job_instance_global_var_value.md new file mode 100644 index 0000000000..a44aa87b39 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_job_instance_global_var_value.md @@ -0,0 +1,136 @@ +### 功能描述 + +获取作业实例全局变量的值 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| job_instance_id | long | 是 | 作业实例ID | + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "job_instance_id": 100 +} +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "", + "data": { + "job_instance_id": 100, + "step_instance_var_list": [ + { + "step_instance_id": 292778, + "global_var_list": [ + { + "name": "aa", + "value": "AA", + "type": 1 + }, + { + "name": "password", + "value": "mypassword", + "type": 4 + } + ] + }, + { + "step_instance_id": 292779, + "global_var_list": [ + { + "name": "aa", + "value": "AAAA", + "type": 1 + }, + { + "name": "password", + "value": "mypassword", + "type": 4 + } + ] + } + ] + } +} +``` + +### 返回结果参数说明 + +#### response +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + +#### data + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| job_instance_id | long | 作业实例ID | +| step_instance_var_list | array | 作业步骤实例全局变量值。定义见step_instance_var | + +#### step_instance_var + +| 字段 | 类型 | 描述 | +| ---------------- | ----- | -------------------------------- | +| step_instance_id | long | 步骤实例ID | +| global_var_list | array | 全局变量值列表，定义见global_var | + +#### global_var + +| 字段 | 类型 | 必选 | 描述 | +| ------ | ------ | ---- | ---------------------------------------------------------- | +| id | long | 否 | 全局变量id，唯一标识。如果id为空，那么使用name作为唯一标识 | +| name | string | 否 | 全局变量name | +| value | string | 否 | 字符、密码、数组类型的全局变量的值 | +| server | object | 否 | 主机类型全局变量的值，见server定义 | + +#### server + +| 字段 | 类型 | 必选 | 描述 | +| ------------------ | ----- | ---- | ----------------------------------- | +| ip_list | array | 否 | 静态 IP 列表，定义见ip | +| dynamic_group_list | array | 否 | 动态分组列表，定义见dynamic_group | +| topo_node_list | array | 否 | 动态 topo 节点列表，定义见topo_node | + +#### ip + +| 字段 | 类型 | 必选 | 描述 | +| ----------- | ------ | ---- | -------- | +| bk_host_id | long | 主机ID | +| bk_cloud_id | int | 是 | 管控区域ID | +| ip | string | 是 | IP地址 | + +#### dynamic_group + +| 字段 | 类型 | 必选 | 描述 | +| ---- | ------ | ---- | -------------- | +| id | string | 是 | CMDB动态分组ID | + +#### topo_node + +| 字段 | 类型 | 必选 | 描述 | +| --------- | ------ | ---- | ------------------------------------------------------------ | +| id | long | 是 | 动态topo节点ID，对应CMDB API 中的 bk_inst_id | +| node_type | string | 是 | 动态topo节点类型，对应CMDB API 中的 bk_obj_id,比如"module","set" | \ No newline at end of file diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_job_instance_ip_log.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_job_instance_ip_log.md new file mode 100644 index 0000000000..36fc7dad16 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_job_instance_ip_log.md @@ -0,0 +1,145 @@ +### 功能描述 + +根据主机查询作业执行日志 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| job_instance_id | long | 是 | 作业实例ID | +| step_instance_id | long | 是 | 步骤实例ID | +| bk_host_id | long | 否 | 目标主机host_id | +| bk_cloud_id | int | 否 | ***不推荐使用，建议使用host_id参数，如果存在bk_host_id将忽略该参数***。目标服务器管控区域ID | +| ip | string | 否 | ***不推荐使用，建议使用host_id参数，如果存在bk_host_id将忽略该参数***。目标主机IP | + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "job_instance_id": 50, + "step_instance_id": 100, + "bk_host_id": 101 +} +``` + +### 返回结果示例 + +#### 脚本执行步骤 +```json +{ + "result": true, + "code": 0, + "message": "", + "data": { + "log_type": 1, + "bk_host_id": 101, + "ip": "127.0.0.1", + "bk_cloud_id": 0, + "log_content": "[2018-03-15 14:39:30][PID:56875] job_start\n" + } +} +``` + +#### 文件分发步骤 + +```json +{ + "result": true, + "code": 0, + "message": "", + "data": { + "log_type": 2, + "bk_host_id": 101, + "ip": "127.0.0.1", + "bk_cloud_id": 0, + "file_logs": [ + { + "mode": 1, + "src_ip": { + "host_id": 102, + "bk_cloud_id": 0, + "ip": "127.0.0.2" + }, + "src_path": "/data/1.log", + "dest_ip": { + "bk_host_id": 101, + "bk_cloud_id": 0, + "ip": "127.0.0.1" + }, + "dest_path": "/tmp/1.log", + "status": 4, + "log_content": "[2021-06-28 11:32:16] FileName: /tmp/1.log FileSize: 9.0 Bytes State: dest agent success download file Speed: 1 KB/s Progress: 100% StatusDesc: dest agent success download file Detail: success" + }, + { + "mode": 0, + "src_ip": { + "bk_host_id": 102, + "bk_cloud_id": 0, + "ip": "127.0.0.2" + }, + "src_path": "/data/1.log", + "status": 4, + "log_content": "[2021-06-28 11:32:16] FileName: /data/1.log FileSize: 9.0 Bytes State: source agent success upload file Speed: 1 KB/s Progress: 100% StatusDesc: source agent success upload file Detail: success upload" + } + ] + } +} +``` + +**返回结果说明** + +- 文件分发日志，除了目标服务器的文件下载任务日志，也会返回源服务器的文件上传任务日志(mode=0) +- dest_ip 与请求参数的bk_cloud_id/ip对应 + +### 返回结果参数说明 + +#### response +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + +#### data + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| bk_host_id | long | 主机ID | +| bk_cloud_id | int | 目标服务器管控区域ID | +| ip | string | 目标服务器IP地址 | +| log_type | int | 日志类型。1-脚本执行任务日志;2-文件分发任务日志 | +| log_content | string | 作业脚本输出的日志内容 | +| file_logs | array | 文件分发任务日志。定义见file_log| + +#### file_log + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| mode | 分发模式 | 0:上传;1:下载| +| src_ip | object |文件源主机，定义见host | +| src_path | string | 源文件路径 | +| dest_ip | object | 分发目标主机，mode=1时有值。定义见host | +| dest_path | string | 目标路径，mode=1时有值 | +| status | int | 任务状态。1-等待开始;2-上传中;3-下载中;4-成功;5-失败 | +| log_content | string | 文件分发日志内容 | + +#### Host + +| 字段 | 类型 | 描述 | +|-----------|------------|--------| +| bk_host_id | long | 主机ID | +| bk_cloud_id | long | 管控区域ID | +| ip | string | IP地址 | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_job_instance_list.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_job_instance_list.md new file mode 100644 index 0000000000..e1680166cf --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_job_instance_list.md @@ -0,0 +1,111 @@ +### 功能描述 + +查询作业实例列表（执行历史) + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +| ----------------- | ------ | ---- | ------------------------------------------------------------ | +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| create_time_start | long | 是 | 创建起始时间，Unix 时间戳，单位毫秒 | +| create_time_end | long | 是 | 创建结束时间，Unix 时间戳，单位毫秒 | +| job_instance_id | long | 否 | 任务实例ID。 如果出入job_instance_id，将忽略其他查询条件 | +| job_cron_id | long | 否 | 定时任务ID | +| operator | string | 否 | 执行人，精准匹配 | +| name | string | 否 | 任务名称，模糊匹配 | +| launch_mode | int | 否 | 执行方式。1 - 页面执行，2 - API调用，3 - 定时执行 | +| type | int | 否 | 任务类型。0 - 作业执行，1 - 脚本执行，2 - 文件分发 | +| status | int | 否 | 任务状态。1 - 等待执行，2 - 正在执行，3 - 执行成功，4 - 执行失败，7 - 等待确认，10 - 强制终止中，11 - 强制终止成功，13 - 确认终止 | +| ip | string | 否 | 执行目标服务器IP, 精准匹配 | +| start | int | 否 | 默认0表示从第1条记录开始返回 | +| length | int | 否 | 单次返回最大记录数，最大1000，不传默认为20 | + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "type": 0, + "launch_mode": 1, + "status": 3, + "operator": "admin", + "name": "test", + "create_time_start": 1546272000000, + "create_time_end": 1577807999999, + "start": 0, + "length": 20 +} +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "data": [ + { + "bk_scope_type": "biz", + "bk_scope_id": "1", + "id": 100, + "job_template_id": 1, + "job_plan_id": 1, + "name": "test", + "operator": "admin", + "create_time": 1546272000000, + "start_time": 1546272000000, + "end_time": 1546272001000, + "total_time": 1000, + "launch_mode": 1, + "task_status": 3, + "task_type": 0 + } + ], + "start": 0, + "length": 20, + "total": 1 + } +} +``` + +### 返回结果参数说明 + +#### response +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + + +#### data + +| 字段 | 类型 | 描述 | +| --------------- | ------ | ------------------------------------------------------------ | +| bk_scope_type | string |资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| id | long | 执行方案 ID | +| job_template_id | long | 作业模版 ID，当任务为执行方案的时候有值 | +| job_plan_id | long | 作业执行方案 ID，当任务为执行方案的时候有值 | +| name | string | 任务名称 | +| operator | string | 操作者 | +| create_time | long | 创建时间，Unix 时间戳，单位毫秒 | +| status | int | 任务状态。1 - 等待执行，2 - 正在执行，3 - 执行成功，4 - 执行失败，7 - 等待确认，10 - 强制终止中，11 - 强制终止成功，13 - 确认终止 | +| type | int | 任务类型。0 - 作业执行，1 - 脚本执行，2 - 文件分发 | +| launch_mode | int | 执行方式。1 - 页面执行，2 - API调用，3 - 定时执行 | +| start_time | long | 任务启动时间，Unix 时间戳，单位毫秒 | +| end_time | long | 任务结束时间，Unix 时间戳，单位毫秒 | +| total_time | long | 任务执行时间，Unix 时间戳，单位毫秒 | \ No newline at end of file diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_job_instance_status.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_job_instance_status.md new file mode 100644 index 0000000000..a22306836e --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_job_instance_status.md @@ -0,0 +1,143 @@ +### 功能描述 + +根据作业实例 ID 查询作业执行状态 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +|------------------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| job_instance_id | long | 是 | 作业实例ID | +| return_ip_result | boolean | 否 | 是否返回每个主机上的任务详情，对应返回结果中的step_ip_result_list。默认值为false。 | + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "job_instance_id": 100 +} +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "", + "data": { + "finished": true, + "job_instance": { + "job_instance_id": 100, + "bk_scope_type": "biz", + "bk_scope_id": "1", + "name": "API Quick execution script1521089795887", + "create_time": 1605064271000, + "status": 4, + "start_time": 1605064271000, + "end_time": 1605064272000, + "total_time": 1000 + }, + "step_instance_list": [ + { + "status": 4, + "total_time": 1000, + "name": "API Quick execution scriptxxx", + "step_instance_id": 75, + "execute_count": 0, + "create_time": 1605064271000, + "end_time": 1605064272000, + "type": 1, + "start_time": 1605064271000, + "step_ip_result_list": [ + { + "bk_host_id": 101, + "ip": "127.0.0.1", + "bk_cloud_id": 0, + "status": 9, + "tag": "", + "exit_code": 0, + "error_code": 0, + "start_time": 1605064271000, + "end_time": 1605064272000, + "total_time": 1000 + } + ] + } + ] + } +} +``` +### 返回结果参数说明 + +#### response +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + +#### data + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| finished | bool | 作业是否结束 | +| job_instance | object | 作业实例基本信息。见job_instance定义 | +| step_instance_list | array | 作业步骤列表。见step_instance定义 | + +#### job_instance + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| name | string | 作业实例名称 | +| status | int | 作业状态码: 1.未执行; 2.正在执行; 3.执行成功; 4.执行失败; 5.跳过; 6.忽略错误; 7.等待用户; 8.手动结束; 9.状态异常; 10.步骤强制终止中; 11.步骤强制终止成功 | +| create_time | long | 作业创建时间，Unix时间戳，单位毫秒 | +| start_time | long | 开始执行时间，Unix时间戳，单位毫秒 | +| end_time | long | 执行结束时间，Unix时间戳，单位毫秒 | +| total_time | int | 总耗时，单位毫秒 | +| bk_scope_type | string |资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| job_instance_id | long | 作业实例ID | + +#### step_instance + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| step_instance_id | long | 作业步骤实例ID | +| type | int | 步骤类型：1.脚本步骤; 2.文件步骤; 4.SQL步骤 | +| name | string | 步骤名称 | +| status | int | 作业步骤状态码: 1.未执行; 2.正在执行; 3.执行成功; 4.执行失败; 5.跳过; 6.忽略错误; 7.等待用户; 8.手动结束; 9.状态异常; 10.步骤强制终止中; 11.步骤强制终止成功; 12.步骤强制终止失败 | +| create_time | long | 作业步骤实例创建时间，Unix时间戳，单位毫秒 | +| start_time | long | 开始执行时间，Unix时间戳，单位毫秒 | +| end_time | long | 执行结束时间，Unix时间戳，单位毫秒 | +| total_time | int | 总耗时，单位毫秒 | +| execute_count | int | 步骤重试次数 | +| step_ip_result_list | array | 每个主机的任务执行结果，定义见step_ip_result | + + +#### step_ip_result + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| bk_host_id | long | 主机ID | +| ip | string | IP | +| bk_cloud_id | long | 管控区域ID | +| status | int | 作业执行状态:1.Agent异常; 5.等待执行; 7.正在执行; 9.执行成功; 11.执行失败; 12.任务下发失败; 403.任务强制终止成功; 404.任务强制终止失败 | +| tag | string | 用户通过job_success/job_fail函数模板自定义输出的结果。仅脚本任务存在该参数 | +| exit_code | int | 脚本任务exit code | +| error_code | int | 主机任务状态码，1.Agent异常; 3.上次已成功; 5.等待执行; 7.正在执行; 9.执行成功; 11.任务失败; 12.任务下发失败; 13.任务超时; 15.任务日志错误; 101.脚本执行失败; 102.脚本执行超时; 103.脚本执行被终止; 104.脚本返回码非零; 202.文件传输失败; 203.源文件不存在; 310.Agent异常; 311.用户名不存在; 320.文件获取失败; 321.文件超出限制; 329.文件传输错误; 399.任务执行出错 | +| start_time | long | 开始执行时间，Unix时间戳，单位毫秒 | +| end_time | long | 执行结束时间，Unix时间戳，单位毫秒 | +| total_time | int | 总耗时，单位毫秒 | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_job_plan_detail.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_job_plan_detail.md new file mode 100644 index 0000000000..d6cb4c609f --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_job_plan_detail.md @@ -0,0 +1,310 @@ +### 功能描述 + +根据作业执行方案 ID 查询作业执行方案详情 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +|-------------|-------------|--------|--------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| job_plan_id | long | 是 | 作业执行方案 ID | + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "job_plan_id": 100 +} +``` + +### 返回结果示例 +```json +{ + "result": true, + "code": 0, + "message": "", + "data": { + "bk_scope_type": "biz", + "bk_scope_id": "1", + "job_plan_id": 100, + "name": "test", + "creator": "admin", + "create_time": 1546272000000, + "last_modify_user": "admin", + "last_modify_time": 1577807999999, + "global_var_list": [ + { + "id": 11, + "type": 1, + "name": "varName", + "value": "value is Me", + "description": "hello", + "required": 1 + }, + { + "id": 12, + "type": 3, + "name": "servers", + "description": "", + "required": 0, + "server": { + "dynamic_group_list": [ + { + "id": "blo8gojho0skft7pr5q0" + }, + { + "id": "blo8gojho0sabc7priuy" + } + ], + "ip_list": [ + { + "bk_host_id": 101, + "bk_cloud_id": 0, + "ip": "127.0.0.1" + }, + { + "bk_host_id": 102, + "bk_cloud_id": 0, + "ip": "127.0.0.2" + } + ], + "topo_node_list": [ + { + "id": 1000, + "node_type": "module" + } + ] + } + } + ], + "step_list": [ + { + "id": 1059, + "type": 1, + "name": "run local script", + "script_info": { + "script_type": 1, + "script_timeout": 1000, + "script_content": "ZWNobyAkMSAkMiAkMw==", + "script_param": "YTEgYTIgYTM=", + "is_param_sensitive": 0, + "account": "root" + } + }, + { + "id": 1060, + "type": 1, + "name": "run cite script", + "script_info": { + "script_type": 2, + "script_id": "aaaaa-bbb-ccc-ddddd", + "script_version_id": 1078, + "script_timeout": 1000, + "script_param": "YTEgYTIgYTM=", + "is_param_sensitive": 1, + "account": "root" + } + }, + { + "id": 1061, + "type": 2, + "name": "xxx", + "file_info": { + "file_source": [ + { + "file_list": [ + "/tmp/REGEX:[a-z]*.txt" + ], + "server": { + "variable": "servers" + }, + "account": { + "id": 1, + "name": "root" + }, + "file_type": 1 + }, + { + "file_list": [ + "testbucket/test.txt" + ], + "file_type": 3, + "file_source_id": 1 + } + ], + "file_destination": { + "path": "/tmp/", + "account": { + "id": 1, + "name": "root" + }, + "server": { + "variable": "", + "dynamic_group_list": [ + { + "id": "blo8gojho0skft7pr5q0" + }, + { + "id": "blo8gojho0sabc7priuy" + } + ], + "ip_list": [ + { + "bk_host_id": 103, + "bk_cloud_id": 0, + "ip": "127.0.0.3" + }, + { + "bk_host_id": 104, + "bk_cloud_id": 0, + "ip": "127.0.0.4" + } + ], + "topo_node_list": [ + { + "id": 1000, + "node_type": "module" + } + ] + } + }, + "timeout": 60, + "transfer_mode": 1, + "upload_speed_limit": 1000, + "download_speed_limit": 1000 + } + } + ] + } +} +``` + + +### 返回结果参数说明 + +#### response +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + +#### data + +| 字段 | 类型 | 描述 | +|------------------|-----------|-----------| +| bk_scope_type | string |资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| job_plan_id | long | 执行方案 ID | +| name | string | 作业名称 | +| creator | string | 作业创建人账号 | +| create_time | long | 创建时间，Unix 时间戳 | +| last_modify_user | string | 作业修改人账号 | +| last_modify_time | long | 最后修改时间，Unix 时间戳 | +| step_list | array | 步骤对象 | +| global_var_list | array | 全局变量信息 | + +#### step + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| id | long | 作业步骤ID | +| name | string | 作业步骤名称 | +| type | int | 步骤类型：1.脚本步骤; 2.文件步骤; 3.人工确认步骤 | +| script_info | object | 脚本信息。当 type=1 时才有这个字段。 | +| file_info | object | 文件传输步骤信息。当 type=2 时才有这个字段 | + +#### script_info +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| script_type | int | 脚本类型：1 本地脚本 2 引用脚本 3 公共脚本 | +| script_id | string | 脚本 ID。当 script_type=2,3 时才有这个字段。 | +| script_version_id | long | 脚本版本 ID。当 script_type=2,3 时才有这个字段。 | +| script_content | string | 脚本内容。当 type=1 时才有这个字段。 | +| script_language | int | 脚本语言：1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - SQL | +| script_param | string | 脚本参数 | +| script_timeout | int | 脚本超时时间，秒。默认3600，取值范围60-86400 | +| is_param_sensitive | int | 是否敏感参数, 0.不是（默认），1.是。| +| account | object | 执行账号名/别名 | + +#### file_info +| 字段 | 类型 | 描述 | +|----------------------|--------|-----------| +| file_source_list | array | 源文件信息 | +| file_destination | object | 目标信息 | +| timeout | int | 文件传输超时设置 | +| transfer_mode | int | 文件传输模式 | +| upload_speed_limit | int | 上传限速 | +| download_speed_limit | int | 下载限速 | + +#### global_var + +| 字段 | 类型 | 描述 | +|-------------|-----------|------------| +| id | long | 全局变量 id，唯一标识。如果 id 为空，那么使用 name 作为唯一标识 | +| name | string | 全局变量名 | +| description | string | 全局变量描述 | +| type | int | 全局变量类型 | +| required | int | 模版/执行方案中该变量是否必填 | +| value | string | 字符、密码、数组类型的全局变量的值 | +| server | object | 主机类型全局变量的值 | + +#### server +| 字段 | 类型 | 描述 | +|-----------------------|-------|------------| +| variable | string | 引用的变量名 | +| ip_list | array | 静态 IP 列表 | +| dynamic_group_list | array | 动态分组 ID 列表 | +| topo_node_list | array | 动态 topo 节点列表 | + +#### ip +| 字段 | 类型 | 描述 | +|-------------|---------|---------| +| bk_host_id | long | 主机ID | +| bk_cloud_id | int | 管控区域ID | +| ip | string | IP地址 | + +#### dynamic_group +| 字段 | 类型 | 描述 | +|-----|---------|------------| +| id | string | 动态分组 ID | + +#### topo_node +| 字段 | 类型 | 描述 | +|------------------|--------|------------| +| id | long | 动态topo节点ID，对应CMDB API 中的 bk_inst_id | +| node_type | string | 动态topo节点类型，对应CMDB API 中的 bk_obj_id,比如"module","set"| + +#### account +| 字段 | 类型 | 必选 | 描述 | +|-------|--------|--------|------------| +| id | long | 否 | 账号 ID | +| name | string | 否 | 账号名 | + +#### file_source +| 字段 | 类型 | 描述 | +|-----------------|---------|-----------| +| file_list | array | 源文件的绝对路径数组，支持多个文件 | +| account | object | 执行账号名/别名 | +| server | object | 目标服务器 | +| file_type | int | 文件源类型，1：服务器文件，2：本地文件，3：第三方文件源文件 | +| file_source_id | int | 文件源类型为第三方文件源文件时的第三方文件源Id | + +#### file_destination +| 字段 | 类型 | 描述 | +|-----------------|---------|-----------| +| path | string | 目标文件存放的路径 | +| account | object | 执行账号名/别名| +| server | object | 目标服务器 | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_job_plan_list.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_job_plan_list.md new file mode 100644 index 0000000000..b8f174c7a4 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_job_plan_list.md @@ -0,0 +1,99 @@ +### 功能描述 + +查询执行方案列表 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +|------------------------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| job_template_id | long | 否 | 作业模版 ID | +| creator | string | 否 | 作业执行方案创建人账号 | +| name | string | 否 | 作业执行方案名称，模糊匹配 | +| create_time_start | long | 否 | 创建起始时间，Unix 时间戳 | +| create_time_end | long | 否 | 创建结束时间，Unix 时间戳 | +| last_modify_user | string | 否 | 作业执行方案修改人账号 | +| last_modify_time_start | long | 否 | 最后修改起始时间，Unix 时间戳 | +| last_modify_time_end | long | 否 | 最后修改结束时间，Unix 时间戳 | +| start | int | 否 | 默认0表示从第1条记录开始返回 | +| length | int | 否 | 单次返回最大记录数，最大1000，不传默认为20 | + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "job_template_id": 1, + "creator": "admin", + "name": "test", + "create_time_start": 1546272000000, + "create_time_end": 1577807999999, + "last_modify_user": "admin", + "last_modify_time_start": 1546272000000, + "last_modify_time_end": 1577807999999, + "start": 0, + "length": 20 +} +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "data": [ + { + "bk_scope_type": "biz", + "bk_scope_id": "1", + "id": 100, + "job_template_id": 1, + "name": "test", + "creator": "admin", + "create_time": 1546272000000, + "last_modify_user": "admin", + "last_modify_time": 1546272000000 + } + ], + "start": 0, + "length": 20, + "total": 1 + } +} +``` + +### 返回结果参数说明 + +#### response +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + +#### data + +| 字段 | 类型 | 描述 | +|------------------|-----------|-----------| +| bk_scope_type | string |资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| id | long | 执行方案 ID | +| job_template_id | long | 作业模版 ID | +| name | string | 执行方案名称 | +| creator | string | 创建人账号 | +| create_time | long | 创建时间，Unix 时间戳 | +| last_modify_user | string | 修改人账号 | +| last_modify_time | long | 最后修改时间，Unix 时间戳 | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_job_template_list.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_job_template_list.md new file mode 100644 index 0000000000..ca97a90963 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_job_template_list.md @@ -0,0 +1,95 @@ +### 功能描述 + +查询作业模版列表 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +|------------------------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| creator | string | 否 | 作业执行方案创建人账号 | +| name | string | 否 | 作业执行方案名称，模糊匹配 | +| create_time_start | long | 否 | 创建起始时间，Unix 时间戳 | +| create_time_end | long | 否 | 创建结束时间，Unix 时间戳 | +| last_modify_user | string | 否 | 作业执行方案修改人账号 | +| last_modify_time_start | long | 否 | 最后修改起始时间，Unix 时间戳 | +| last_modify_time_end | long | 否 | 最后修改结束时间，Unix 时间戳 | +| start | int | 否 | 默认0表示从第1条记录开始返回 | +| length | int | 否 | 单次返回最大记录数，最大1000，不传默认为20 | + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "creator": "admin", + "name": "test", + "create_time_start": 1546272000000, + "create_time_end": 1577807999999, + "last_modify_user": "admin", + "last_modify_time_start": 1546272000000, + "last_modify_time_end": 1577807999999, + "start": 0, + "length": 20 +} +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "data": [ + { + "bk_scope_type": "biz", + "bk_scope_id": "1", + "id": 100, + "name": "test", + "creator": "admin", + "create_time": 1546272000000, + "last_modify_user": "admin", + "last_modify_time": 1546272000000 + } + ], + "start": 0, + "length": 20, + "total": 1 + } +} +``` + +### 返回结果参数说明 + +#### response +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + +#### data + +| 字段 | 类型 | 描述 | +|------------------|-----------|-----------| +| bk_scope_type | string |资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| id | long | 作业模版 ID | +| name | string | 作业模版名称 | +| creator | string | 创建人账号 | +| create_time | long | 创建时间，Unix 时间戳 | +| last_modify_user | string | 修改人账号 | +| last_modify_time | long | 最后修改时间，Unix 时间戳 | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_latest_service_version.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_latest_service_version.md new file mode 100644 index 0000000000..da2eb33eaf --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_latest_service_version.md @@ -0,0 +1,53 @@ +### 功能描述 + +查询作业平台最新的服务版本号。 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +无 + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_username": "admin" +} +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "version": "3.6.4" + } +} +``` + +### 返回结果参数说明 + +#### response + +| 字段 | 类型 | 描述 | +| ---------- | ------ | ------------------------------------------ | +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息 | +| data | object | 请求返回的数据 | +| permission | object | 权限信息 | + +#### data + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| version | string |服务最新版本号| diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_public_script_list.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_public_script_list.md new file mode 100644 index 0000000000..09210ad311 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_public_script_list.md @@ -0,0 +1,81 @@ +### 功能描述 + +查询公共脚本列表 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +|----------------------|------------|--------|------------| +| name | string | 否 | 脚本名称，支持模糊查询 | +| script_language | int | 否 | 脚本语言。1：shell，2：bat，3：perl，4：python，5：powershell，6：sql。如果不传，默认返回所有脚本语言 | +| start | int | 否 | 分页记录起始位置，不传默认为0 | +| length | int | 否 | 单次返回最大记录数，最大1000，不传默认为20 | + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "name": "a.sh", + "script_language": 1, + "start": 0, + "length": 10 +} +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "data": [ + { + "id": "000dbdddc06c453baf1f2decddf00c69", + "name": "a.sh", + "script_language": 1, + "online_script_version_id": 100, + "creator": "admin", + "create_time": 1600746078520, + "last_modify_user": "admin", + "last_modify_time": 1600746078520 + } + ], + "start": 0, + "length": 10, + "total": 1 + } +} +``` + +### 返回结果参数说明 + +#### response +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + +#### data + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| id | string | 脚本ID | +| name | string | 脚本名称 | +| script_language | int | 脚本语言。1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - SQL | +| online_script_version_id | long | 已上线脚本版本ID;如果脚本没有已上线版本，该值为空 | +| creator | string | 创建人 | +| create_time | long | 创建时间Unix时间戳（ms） | +| last_modify_user| string | 最近一次修改人 | +| last_modify_time| long | 最近一次修改时间Unix时间戳（ms） | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_public_script_version_detail.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_public_script_version_detail.md new file mode 100644 index 0000000000..45aa36c1f5 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_public_script_version_detail.md @@ -0,0 +1,79 @@ +### 功能描述 + +查询公共脚本版本详情 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +|----------------------|------------|--------|------------| +| id | long | 是 | 脚本版本ID，若传入则以此条件为准屏蔽其他条件 | +| script_id | string | 否 | 脚本ID（可与version一起传入定位某个脚本版本） | +| version | string | 否 | 脚本版本（可与script_id一起传入定位某个脚本版本） | + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "id": 1 +} +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "id": 1, + "script_id": "000dbdddc06c453baf1f2decddf00c69", + "version": "V1.0", + "content": "#!/bin/bash***", + "status": 1, + "version_desc": "版本描述", + "creator": "admin", + "create_time": 1600746078520, + "last_modify_user": "admin", + "last_modify_time": 1600746078520, + "script_language": 1, + "description": "脚本描述" + } +} +``` + +### 返回结果参数说明 + +#### response +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + + +#### data + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| id | long | 脚本版本ID | +| script_id | string | 脚本版本所属的脚本ID | +| version | string | 版本号 | +| content | string | 脚本版本内容 | +| status | int | 脚本版本状态（0：未上线，1：已上线，2：已下线，3：已禁用） | +| version_desc | string | 版本描述 | +| creator | string | 创建人 | +| create_time | long | 创建时间Unix时间戳（ms） | +| last_modify_user | string | 最近一次修改人 | +| last_modify_time | long | 最近一次修改时间Unix时间戳（ms） | +| script_language | int | 脚本语言:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell | +| description | string | 脚本描述 | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_public_script_version_list.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_public_script_version_list.md new file mode 100644 index 0000000000..680fe3be95 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_public_script_version_list.md @@ -0,0 +1,90 @@ +### 功能描述 + +查询公共脚本版本列表 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +|----------------------|------------|--------|------------| +| script_id | string | 是 | 脚本ID | +| return_script_content | bool | 否 | 是否需要返回脚本内容。true:返回脚本内容；false：不返回脚本内容。默认为false。 | +| start | int | 否 | 分页记录起始位置，不传默认为0 | +| length | int | 否 | 单次返回最大记录数，最大1000，不传默认为20 | + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "script_id": "000dbdddc06c453baf1f2decddf00c69", + "return_script_content": true, + "start": 0, + "length": 10 +} +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "data": [ + { + "id": 1, + "script_id": "000dbdddc06c453baf1f2decddf00c69", + "version": "V1.0", + "content": "#!/bin/bash***", + "status": 1, + "version_desc": "版本描述", + "creator": "admin", + "create_time": 1600746078520, + "last_modify_user": "admin", + "last_modify_time": 1600746078520, + "script_language": 1, + "description": "脚本描述" + } + ], + "start": 0, + "length": 10, + "total": 1 + } +} +``` + +### 返回结果参数说明 + +#### response +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + + +#### data + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| id | long | 脚本版本ID | +| script_id | string | 脚本版本所属的脚本ID | +| version | string | 版本号 | +| content | string | 脚本内容 | +| status | int | 脚本版本状态（0：未上线，1：已上线，2：已下线，3：已禁用） | +| version_desc | string | 版本描述 | +| creator | string | 创建人 | +| create_time | long | 创建时间Unix时间戳（ms） | +| last_modify_user | string | 最近一次修改人 | +| last_modify_time | long | 最近一次修改时间Unix时间戳（ms） | +| script_language | int | 脚本语言:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell | +| description | string | 脚本描述 | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_script_list.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_script_list.md new file mode 100644 index 0000000000..be669e0bbe --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_script_list.md @@ -0,0 +1,90 @@ +### 功能描述 + +查询业务脚本列表 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +|----------------------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| name | string | 否 | 脚本名称，支持模糊查询 | +| script_language | int | 否 | 脚本语言。0：所有脚本类型，1：shell，2：bat，3：perl，4：python，5：powershell，6：sql。默认值为0 | +| start | int | 否 | 分页记录起始位置，不传默认为0 | +| length | int | 否 | 单次返回最大记录数，最大1000，不传默认为20 | + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "name": "脚本1", + "script_language": 1, + "start": 0, + "length": 10 +} +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "data": [ + { + "id": "000dbdddc06c453baf1f2decddf00c69", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "name": "a.sh", + "script_language": 1, + "online_script_version_id": 100, + "creator": "admin", + "create_time": 1600746078520, + "last_modify_user": "admin", + "last_modify_time": 1600746078520 + } + ], + "start": 0, + "length": 10, + "total": 1 + } +} +``` + +### 返回结果参数说明 + +#### response +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + + +#### data + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| id | string | 脚本ID | +| bk_scope_type | string |资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| name | string | 脚本名称 | +| script_language | int | 脚本语言。1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - SQL | +| online_script_version_id | long | 已上线脚本版本ID;如果脚本没有已上线版本，该值为空 | +| creator | string | 创建人 | +| create_time | long | 创建时间Unix时间戳（ms） | +| last_modify_user| string | 最近一次修改人 | +| last_modify_time| long | 最近一次修改时间Unix时间戳（ms） | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_script_version_detail.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_script_version_detail.md new file mode 100644 index 0000000000..daf9428804 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_script_version_detail.md @@ -0,0 +1,87 @@ +### 功能描述 + +查询业务脚本版本详情 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +|----------------------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| id | long | 是 | 脚本版本ID，若传入则以此条件为准屏蔽其他条件 | +| script_id | string | 否 | 脚本ID（可与version一起传入定位某个脚本版本） | +| version | string | 否 | 脚本版本（可与script_id一起传入定位某个脚本版本） | + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "id": 1 +} +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "id": 1, + "bk_scope_type": "biz", + "bk_scope_id": "1", + "script_id": "000dbdddc06c453baf1f2decddf00c69", + "version": "V1.0", + "content": "#!/bin/bash***", + "status": 1, + "version_desc": "版本描述", + "creator": "admin", + "create_time": 1600746078520, + "last_modify_user": "admin", + "last_modify_time": 1600746078520, + "script_language": 1, + "description": "脚本描述" + } +} +``` + +### 返回结果参数说明 + +#### response +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + + +#### data + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| id | long | 脚本版本ID | +| bk_scope_type | string |资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| script_id | string | 脚本版本所属的脚本ID | +| version | string | 版本号 | +| content | string | 脚本版本内容 | +| status | int | 脚本版本状态（0：未上线，1：已上线，2：已下线，3：已禁用） | +| version_desc | string | 版本描述 | +| creator | string | 创建人 | +| create_time | long | 创建时间Unix时间戳（ms） | +| last_modify_user | string | 最近一次修改人 | +| last_modify_time | long | 最近一次修改时间Unix时间戳（ms） | +| script_language | int | 脚本语言:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell | +| description | string | 脚本描述 | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_script_version_list.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_script_version_list.md new file mode 100644 index 0000000000..7b8f00ce04 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/get_script_version_list.md @@ -0,0 +1,98 @@ +### 功能描述 + +查询业务脚本版本列表 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +|----------------------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| script_id | string | 是 | 脚本ID | +| return_script_content | bool | 否 | 是否需要返回脚本内容。true:返回脚本内容；false：不返回脚本内容。默认为false。 | +| start | int | 否 | 分页记录起始位置，不传默认为0 | +| length | int | 否 | 单次返回最大记录数，最大1000，不传默认为20 | + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "script_id": "000dbdddc06c453baf1f2decddf00c69", + "return_script_content": true, + "start": 0, + "length": 10 +} +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "data": [ + { + "id": 1, + "bk_scope_type": "biz", + "bk_scope_id": "1", + "script_id": "000dbdddc06c453baf1f2decddf00c69", + "version": "V1.0", + "content": "#!/bin/bash***", + "status": 1, + "version_desc": "版本描述", + "creator": "admin", + "create_time": 1600746078520, + "last_modify_user": "admin", + "last_modify_time": 1600746078520, + "script_language": 1, + "description": "脚本描述" + } + ], + "start": 0, + "length": 10, + "total": 1 + } +} +``` + +### 返回结果参数说明 + +#### response +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + + +#### data + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| id | long | 脚本版本ID | +| bk_scope_type | string |资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| script_id | string | 脚本版本所属的脚本ID | +| version | string | 版本号 | +| content | string | 脚本内容 | +| status | int | 脚本版本状态（0：未上线，1：已上线，2：已下线，3：已禁用） | +| version_desc | string | 版本描述 | +| creator | string | 创建人 | +| create_time | long | 创建时间Unix时间戳（ms） | +| last_modify_user | string | 最近一次修改人 | +| last_modify_time | long | 最近一次修改时间Unix时间戳（ms） | +| script_language | int | 脚本语言:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell | +| description | string | 脚本描述 | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/operate_job_instance.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/operate_job_instance.md new file mode 100644 index 0000000000..271cff048f --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/operate_job_instance.md @@ -0,0 +1,61 @@ +### 功能描述 + +用于对执行的作业实例进行操作，例如终止作业。 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| job_instance_id | long | 是 | 作业实例ID | +| operation_code | int | 是 | 操作类型：1、终止作业 | + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "job_instance_id": 100, + "operation_code": 1 +} +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "job_instance_id": 100 + } +} +``` + +### 返回结果参数说明 + +#### response +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + + +#### data + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| job_instance_id | long | 作业实例ID | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/operate_step_instance.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/operate_step_instance.md new file mode 100644 index 0000000000..c11cc809e8 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/operate_step_instance.md @@ -0,0 +1,78 @@ +### 功能描述 + +用于对执行的实例的步骤进行操作，例如重试，忽略错误等。 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| job_instance_id | long | 是 | 作业实例ID | +| step_instance_id | long | 是 | 步骤实例ID | +| operation_code | int | 是 | 操作类型：2、失败IP重做，3、忽略错误 6、确认继续 8、全部重试，9、终止确认流程，10-重新发起确认，11、进入下一步，12、强制跳过 | + + +##### operation_code 详细说明 +| operation_code | 操作类型 | 适用步骤 | 描述 | +|-----------|------------|--------|------------| +| 2 | 失败IP重做 | 脚本执行，文件分发步骤 | 对失败的IP重新下发任务 | +| 3 | 忽略错误 | 脚本执行，文件分发步骤 | 忽略错误，继续执行 | +| 6 | 确认继续 | 人工确认步骤 | 确认继续执行 | +| 8 | 全部重试 | 脚本执行，文件分发步骤 | 对所有的IP重新下发任务 | +| 9 | 终止确认流程 | 人工确认步骤 | 确认终止执行 | +| 10 | 重新发起确认 | 人工确认步骤 | 重新发起确认 | +| 11 | 进入下一步 | 脚本执行，文件分发步骤 | 当步骤状态为终止成功，用于继续执行后续步骤 | +| 12 | 强制跳过 | 脚本执行，文件分发步骤 | 当步骤状态为终止中，用于强制跳过当前步骤，执行后续步骤| + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "job_instance_id": 100, + "step_instance_id": 200, + "operation_code": 2 +} +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "step_instance_id": 200, + "job_instance_id": 100 + } +} +``` + +### 返回结果参数说明 + +#### response +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + + +#### data + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| job_instance_id | long | 作业实例ID | +| step_instance_id | long | 步骤实例ID | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/publish_public_script_version.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/publish_public_script_version.md new file mode 100644 index 0000000000..363aa1a437 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/publish_public_script_version.md @@ -0,0 +1,61 @@ +### 功能描述 + +上线公共脚本版本，上线后，之前的线上脚本将被置为已下线状态，但不影响作业使用。 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +| ----------------- | ------ | ---- | -------------- | +| script_id | string | 是 | 公共脚本ID | +| script_version_id | long | 是 | 公共脚本版本ID | + + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "script_id":"4537fb49ec0840a1b91cef4179c99f9c", + "script_version_id": 1000018 +} +``` + +### 返回结果示例 + +```json +{ + "code": 0, + "result": true, + "data": { + "id": 1000018, + "script_id": "4537fb49ec0840a1b91cef4179c99f9c", + "status": 2 + } +} +``` + +### 返回结果参数说明 + +#### response + +| 字段 | 类型 | 描述 | +| ---------- | ------ | ------------------------------------------ | +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息 | +| data | object | 请求返回的数据 | +| permission | object | 权限信息 | + +#### data + +| 字段 | 类型 | 描述 | +| ----------------- | ------ | ------------------------------------------------------------ | +| script_id | string | 脚本ID | +| id | long | 脚本版本ID | +| status | int | 脚本版本当前状态（0：未上线，1：已上线，2：已下线，3：已禁用） | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/publish_script_version.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/publish_script_version.md new file mode 100644 index 0000000000..5a274793b3 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/publish_script_version.md @@ -0,0 +1,65 @@ +### 功能描述 + +上线脚本版本，上线后，之前的线上脚本将被置为已下线状态，但不影响作业使用。 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +| ----------------- | ------ | ---- | ------------------------------------------------------- | +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| script_id | string | 是 | 脚本ID | +| script_version_id | long | 是 | 脚本版本ID | + + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type":"biz", + "bk_scope_id":"2", + "script_id": "4a350b0e0707450e93326f6ace921072", + "script_version_id": 1000019 +} +``` + +### 返回结果示例 + +```json +{ + "code": 0, + "result": true, + "data": { + "id": 1000019, + "script_id": "4a350b0e0707450e93326f6ace921072", + "status": 1 + } +} +``` + +### 返回结果参数说明 + +#### response + +| 字段 | 类型 | 描述 | +| ---------- | ------ | ------------------------------------------ | +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息 | +| data | object | 请求返回的数据 | +| permission | object | 权限信息 | + +#### data + +| 字段 | 类型 | 描述 | +| ----------------- | ------ | ------------------------------------------------------------ | +| script_id | string | 脚本ID | +| id | long | 脚本版本ID | +| status | int | 脚本版本当前状态（0：未上线，1：已上线，2：已下线，3：已禁用） | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/push_config_file.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/push_config_file.md new file mode 100644 index 0000000000..0d6f0353ea --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/push_config_file.md @@ -0,0 +1,125 @@ +### 功能描述 + +分发配置文件，此接口用于分发配置文件等小的纯文本文件 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +|-------------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| task_name | string | 否 | 自定义作业名称 | +| account_alias | string | 是 | 执行账号别名 | +| file_target_path | string | 是 | 文件传输目标路径 | +| file_list | array | 是 | 源文件对象数组，见下面file定义 | +| target_server | object | 是 | 目标服务器，见server定义 | + +#### file + +| 字段 | 类型 | 必选 | 描述 | +|-----------|------------|--------|------------| +| file_name | string | 是 | 文件名称 | +| content | string | 是 | 文件内容Base64 | + +#### server + +| 字段 | 类型 | 必选 | 描述 | +| ------------------ | ----- | ---- | ------------------------------------------------------------ | +| host_id_list | array | 否 | 主机ID列表 | +| ip_list | array | 否 | ***不推荐使用，建议使用host_id_list参数***;如果host_id_list与ip_list同时存在，将忽略ip_list参数。主机IP 列表，定义见ip | +| dynamic_group_list | array | 否 | 动态分组列表，定义见dynamic_group | +| topo_node_list | array | 否 | 动态 topo 节点列表，定义见topo_node | + + +#### ip + +| 字段 | 类型 | 必选 | 描述 | +| ----------- | ------ | ---- | -------- | +| bk_cloud_id | long | 是 | 管控区域ID | +| ip | string | 是 | IP地址 | + +#### dynamic_group + +| 字段 | 类型 | 必选 | 描述 | +| ---- | ------ | ---- | -------------- | +| id | string | 是 | CMDB动态分组ID | + +#### topo_node_list + +| 字段 | 类型 | 必选 | 描述 | +| --------- | ------ | ---- | ------------------------------------------------------------ | +| id | long | 是 | 动态topo节点ID，对应CMDB API 中的 bk_inst_id | +| node_type | string | 是 | 动态topo节点类型，对应CMDB API 中的 bk_obj_id,比如"module","set" | + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "account_alias": "root", + "file_target_path": "/tmp/", + "file_list": [ + { + "file_name": "a.txt", + "content": "aGVsbG8gd29ybGQh" + } + ], + "target_server": { + "dynamic_group_list": [ + { + "id": "blo8gojho0skft7pr5q0" + } + ], + "host_id_list": [ + 101, + 102 + ], + "topo_node_list": [ + { + "id": 1000, + "node_type": "module" + } + ] + } +} +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "job_instance_name": "API GSE PUSH FILE1521107826893", + "job_instance_id": 10000 + } +} +``` + +### 返回结果参数说明 + +#### response +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + + +#### data + +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| job_instance_id | long | 作业实例ID | +| job_instance_name | long | 作业实例名称 | +| step_instance_id | long | 步骤实例ID | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/save_cron.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/save_cron.md new file mode 100644 index 0000000000..e7e881a677 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/save_cron.md @@ -0,0 +1,382 @@ +### 功能描述 + +新建或保存定时任务；新建定时任务，定时任务状态默认为暂停，如有需要可调用update_cron_status接口开启。 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| job_plan_id | long | 是 | 要定时执行的作业的执行方案 ID | +| id | long | 否 | 定时任务 ID，更新定时任务时，必须传这个值 | +| name | string | 否 | 定时作业名称，新建时必填，修改时选填 | +| expression | string | 否 | 定时任务 crontab 的定时规则，新建时必填，修改时选填，各字段含义为：分 时 日 月 周，如: 0/5 * * * * 表示每5分钟执行一次，注意：不支持? ，新建时与execute_time互斥，二者只可传入一个| +| execute_time | long | 否 | 定时任务单次执行的执行时间，Unix时间戳，新建定时任务时不可与expression同时为空，且二者只可传入一个 | +| global_var_list | array | 否 | 全局变量信息，可使用 查询执行方案详情 接口查询方案可设置的变量信息 | + +#### global_var + +| 字段 | 类型 | 必选 | 描述 | +|-----------|-----------|--------|------------| +| id | long | 否 | 全局变量 id，唯一标识。如果 id 为空，那么使用 name 作为唯一标识 | +| name | string | 否 | 全局变量 name | +| value | string | 否 | 字符、密码、数组类型的全局变量的值 | +| server | object | 否 | 主机类型全局变量的值 | + +#### server +| 字段 | 类型 | 必选 | 描述 | +|-----------------------|-------|--------|------------| +| host_id_list | array | 否 | 主机ID列表 | +| ip_list | array | 否 | ***不推荐使用，建议使用host_id_list参数***;如果host_id_list与ip_list同时存在，将忽略ip_list参数。主机IP 列表，定义见ip | +| host_id_list | array | 否 | 静态 主机ID 列表，元素为Long类型，指定的主机将与ip_list指定的主机取并集 | +| dynamic_group_list | array | 否 | 动态分组列表 | +| topo_node_list | array | 否 | 动态 topo 节点列表 | + +#### ip + +| 字段 | 类型 | 必选 | 描述 | +|-------------|---------|--------|---------| +| bk_cloud_id | int | 是 | 管控区域ID | +| ip | string | 是 | IP地址 | + +#### topo_node +| 字段 | 类型 | 必选 | 描述 | +|------------------|--------|--------|------------| +| id | long | 是 | 动态topo节点ID，对应CMDB API 中的 bk_inst_id | +| node_type | string | 是 | 动态topo节点类型，对应CMDB API 中的 bk_obj_id,比如"module","set"| + + +### 请求参数示例 +- POST +#### 1.创建定时任务 +```json +{ + "bk_app_code": "xxx", + "bk_app_secret": "xxx", + "access_token": "xxx", + "bk_username": "admin", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "expression": "0 0/5 * * *", + "job_plan_id": 1023060, + "name": "test API", + "global_var_list": [ + { + "name": "stringVar", + "value": "value11112" + }, + { + "name": "nsVar", + "value": "nsvalue11112" + }, + { + "name": "secretVar", + "value": "secretvalue11112" + }, + { + "name": "dictVar", + "value": "([\"var1\"]=1, [\"var2\"]=2)" + }, + { + "name": "indexArrVar", + "value": "(2 3 4)" + }, + { + "name": "hostVar", + "server": { + "host_id_list": [1,2,3] + } + } + ] +} +``` +#### 2.更新定时任务 +```json +{ + "id": 1000064, + "bk_app_code": "xxx", + "bk_app_secret": "xxx", + "access_token": "xxx", + "bk_username": "admin", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "expression": "0 0/5 * * *", + "job_plan_id": 1023060, + "name": "test API", + "global_var_list": [ + { + "name": "stringVar", + "value": "value111333312" + }, + { + "name": "nsVar", + "value": "nsvalue111333312" + }, + { + "name": "secretVar", + "value": "secretvalue111333312" + }, + { + "name": "dictVar", + "value": "([\"var1\"]=1, [\"var2\"]=2)" + }, + { + "name": "indexArrVar", + "value": "(22 3 4)" + }, + { + "name": "hostVar", + "server": { + "host_id_list": [2,3,4] + } + } + ] +} +``` + +### 返回结果示例 +#### 1.创建定时任务 +```json +{ + "code": 0, + "result": true, + "data": { + "id": 1000067, + "name": "test API", + "status": 2, + "creator": "admin", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "job_plan_id": 1023060, + "expression": "0 0/5 * * *", + "global_var_list": [ + { + "id": 1001101, + "name": "stringVar", + "value": "value11112", + "description": null, + "type": 1, + "required": null, + "server": null + }, + { + "id": 1001102, + "name": "nsVar", + "value": "nsvalue11112", + "description": null, + "type": 2, + "required": null, + "server": null + }, + { + "id": 1001104, + "name": "secretVar", + "value": "secretvalue11112", + "description": null, + "type": 4, + "required": null, + "server": null + }, + { + "id": 1001105, + "name": "dictVar", + "value": "([\"var1\"]=1, [\"var2\"]=2)", + "description": null, + "type": 5, + "required": null, + "server": null + }, + { + "id": 1001106, + "name": "indexArrVar", + "value": "(2 3 4)", + "description": null, + "type": 6, + "required": null, + "server": null + }, + { + "id": 1001103, + "name": "hostVar", + "value": null, + "description": null, + "type": 3, + "required": null, + "server": { + "ip_list": [ + { + "bk_host_id": 101, + "bk_cloud_id": 0, + "ip": "127.0.0.1" + } + ], + "dynamic_group_list": null, + "topo_node_list": null + } + } + ], + "create_time": 1642045370, + "last_modify_user": "admin", + "last_modify_time": 1642045370 + } +} +``` +#### 2.更新定时任务 +```json +{ + "code": 0, + "result": true, + "data": { + "id": 1000064, + "name": "test API", + "status": 2, + "creator": "admin", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "job_plan_id": 1023060, + "expression": "0 0/5 * * *", + "global_var_list": [ + { + "id": 1001101, + "name": "stringVar", + "value": "value111333312", + "description": null, + "type": 1, + "required": null, + "server": null + }, + { + "id": 1001102, + "name": "nsVar", + "value": "nsvalue111333312", + "description": null, + "type": 2, + "required": null, + "server": null + }, + { + "id": 1001104, + "name": "secretVar", + "value": "secretvalue111333312", + "description": null, + "type": 4, + "required": null, + "server": null + }, + { + "id": 1001105, + "name": "dictVar", + "value": "([\"var1\"]=1, [\"var2\"]=2)", + "description": null, + "type": 5, + "required": null, + "server": null + }, + { + "id": 1001106, + "name": "indexArrVar", + "value": "(22 3 4)", + "description": null, + "type": 6, + "required": null, + "server": null + }, + { + "id": 1001103, + "name": "hostVar", + "value": null, + "description": null, + "type": 3, + "required": null, + "server": { + "ip_list": [ + { + "bk_host_id": 101, + "bk_cloud_id": 0, + "ip": "127.0.0.1" + } + ], + "dynamic_group_list": null, + "topo_node_list": null + } + } + ], + "create_time": 1641990674, + "last_modify_user": "admin", + "last_modify_time": 1641995052 + } +} +``` + +### 返回结果参数说明 + +#### response +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| message | string | 请求失败返回的错误信息（成功时无此字段）| +| data | object | 请求返回的数据| +| permission | object | 权限信息（无权限情况下才返回）| + +#### data + +| 字段 | 类型 | 描述 | +|-----------------|------------|-------------| +| id | long | 定时任务 ID| +| name | string | 定时作业名称| +| status | int | 定时任务状态| +| bk_scope_type | string |资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| job_plan_id | long | 要定时执行的作业的执行方案 ID | +| creator | string | 创建人 | +| create_time | long | 创建时间Unix时间戳（s） | +| last_modify_user | string | 最近一次修改人 | +| last_modify_time | long | 最近一次修改时间Unix时间戳（s） | +| expression | string | 定时任务 crontab 的定时规则| +| execute_time | long | 定时任务单次执行的执行时间，Unix时间戳| +| global_var_list | array | 全局变量信息，定义见global_var | + +#### global_var + +| 字段 | 类型 | 描述 | +|-------------|-----------|------------| +| id | long | 全局变量 id，唯一标识。如果 id 为空，那么使用 name 作为唯一标识 | +| name | string | 全局变量名 | +| description | string | 全局变量描述 | +| type | int | 全局变量类型 | +| required | int | 模版/执行方案中该变量是否必填 | +| value | string | 字符、密码、数组类型的全局变量的值 | +| server | object | 主机类型全局变量的值 | + +#### server +| 字段 | 类型 | 描述 | +|-----------------------|-------|------------| +| variable | string | 引用的变量名 | +| ip_list | array | 静态 IP 列表 | +| host_id_list | array | 静态 主机ID 列表，元素为Long类型 | +| dynamic_group_list | array | 动态分组 ID 列表 | +| topo_node_list | array | 动态 topo 节点列表 | + +#### ip +| 字段 | 类型 | 描述 | +|-----------|------------|--------| +| bk_host_id | long | 主机ID | +| bk_cloud_id | long | 管控区域ID | +| ip | string | IP地址 | + +#### dynamic_group +| 字段 | 类型 | 描述 | +|-----|---------|------------| +| id | string | 动态分组 ID | + +#### topo_node +| 字段 | 类型 | 描述 | +|------------------|--------|------------| +| id | long | 动态topo节点ID，对应CMDB API 中的 bk_inst_id | +| node_type | string | 动态topo节点类型，对应CMDB API 中的 bk_obj_id,比如"module","set"| diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/update_credential.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/update_credential.md new file mode 100644 index 0000000000..12e27ac57a --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/update_credential.md @@ -0,0 +1,72 @@ +### 功能描述 + +更新凭证。 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +|----------------------------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| id | string | 是 | 凭证 ID | +| name | string | 否 | 凭证名称 | +| type | string | 否 | 凭证类型，取值可为ACCESS_KEY_SECRET_KEY,PASSWORD,USERNAME_PASSWORD,SECRET_KEY | +| description | string | 否 | 凭证描述 | +| credential_access_key | string | 否 | 凭证类型为ACCESS_KEY_SECRET_KEY时填写 | +| credential_secret_key | string | 否 | 凭证类型为ACCESS_KEY_SECRET_KEY/SECRET_KEY时填写 | +| credential_username | string | 否 | 凭证类型为USERNAME_PASSWORD时填写 | +| credential_password | string | 否 | 凭证类型为USERNAME_PASSWORD/PASSWORD时填写 | + + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "id": "06644309e10e4068b3c7b32799668210", + "name": "testCredential", + "type": "USERNAME_PASSWORD", + "description": "This is a new credential", + "credential_username": "admin", + "credential_password": "newPassword" +} +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "id": "06644309e10e4068b3c7b32799668210" + } +} +``` + +### 返回结果参数说明 + +#### response +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + + +#### data + +| 字段 | 类型 |字段是否一定存在 | 描述 | +|-----------|-------|---------------|---------| +| id | string |是 | 凭证ID | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/update_cron_status.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/update_cron_status.md new file mode 100644 index 0000000000..4ebb035293 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/update_cron_status.md @@ -0,0 +1,181 @@ +### 功能描述 + +更新定时作业状态，如启动或暂停 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +|----------- |------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| id | long | 是 | 定时作业 ID | +| status | int | 是 | 定时状态，1.启动、2.暂停 | + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "id": 2, + "status": 1 +} +``` + +### 返回结果示例 + +```json +{ + "code": 0, + "result": true, + "data": { + "id": 2, + "name": "test API", + "status": 1, + "creator": "admin", + "bk_biz_id": 1, + "job_plan_id": 1023060, + "expression": "0 0/5 * * *", + "global_var_list": [ + { + "id": 1001101, + "name": "stringVar", + "value": "value111333312", + "description": null, + "type": 1, + "required": null, + "server": null + }, + { + "id": 1001102, + "name": "nsVar", + "value": "nsvalue111333312", + "description": null, + "type": 2, + "required": null, + "server": null + }, + { + "id": 1001104, + "name": "secretVar", + "value": "secretvalue111333312", + "description": null, + "type": 4, + "required": null, + "server": null + }, + { + "id": 1001105, + "name": "dictVar", + "value": "([\"var1\"]=1, [\"var2\"]=2)", + "description": null, + "type": 5, + "required": null, + "server": null + }, + { + "id": 1001106, + "name": "indexArrVar", + "value": "(22 3 4)", + "description": null, + "type": 6, + "required": null, + "server": null + }, + { + "id": 1001103, + "name": "hostVar", + "value": null, + "description": null, + "type": 3, + "required": null, + "server": { + "ip_list": [ + { + "bk_cloud_id": 0, + "ip": "127.0.0.1" + } + ], + "dynamic_group_list": null, + "topo_node_list": null + } + } + ], + "create_time": 1641990674, + "last_modify_user": "admin", + "last_modify_time": 1641995052 + } +} +``` + +### 返回结果参数说明 + +#### response +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| message | string | 请求失败返回的错误信息（成功时无此字段）| +| data | object | 请求返回的数据| +| permission | object | 权限信息（无权限情况下才返回）| + +#### data + +| 字段 | 类型 | 描述 | +|-----------------|------------|-------------| +| id | long | 定时任务 ID| +| name | string | 定时作业名称| +| status | int | 定时任务状态| +| bk_biz_id | long | 业务 ID | +| job_plan_id | long | 要定时执行的作业的执行方案 ID | +| creator | string | 创建人 | +| create_time | long | 创建时间Unix时间戳（s） | +| last_modify_user | string | 最近一次修改人 | +| last_modify_time | long | 最近一次修改时间Unix时间戳（s） | +| expression | string | 定时任务 crontab 的定时规则，| +| execute_time | long | 定时任务单次执行的执行时间，Unix时间戳| +| global_var_list | array | 全局变量信息，定义见global_var | + +#### global_var + +| 字段 | 类型 | 描述 | +|-------------|-----------|------------| +| id | long | 全局变量 id，唯一标识。如果 id 为空，那么使用 name 作为唯一标识 | +| name | string | 全局变量名 | +| description | string | 全局变量描述 | +| type | int | 全局变量类型 | +| required | int | 模版/执行方案中该变量是否必填 | +| value | string | 字符、密码、数组类型的全局变量的值 | +| server | object | 主机类型全局变量的值 | + +#### server +| 字段 | 类型 | 描述 | +|-----------------------|-------|------------| +| variable | string | 引用的变量名 | +| ip_list | array | 静态 IP 列表 | +| dynamic_group_list | array | 动态分组 ID 列表 | +| topo_node_list | array | 动态 topo 节点列表 | + +#### ip +| 字段 | 类型 | 描述 | +|-------------|---------|---------| +| bk_cloud_id | int | 云区域ID | +| ip | string | IP地址 | + +#### dynamic_group +| 字段 | 类型 | 描述 | +|-----|---------|------------| +| id | string | 动态分组 ID | + +#### topo_node +| 字段 | 类型 | 描述 | +|------------------|--------|------------| +| id | long | 动态topo节点ID，对应CMDB API 中的 bk_inst_id | +| node_type | string | 动态topo节点类型，对应CMDB API 中的 bk_obj_id,比如"module","set"| diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/update_dangerous_rule.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/update_dangerous_rule.md new file mode 100644 index 0000000000..2e14a831c4 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/update_dangerous_rule.md @@ -0,0 +1,81 @@ +### 功能描述 + +修改高危语句检测规则。 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +| -------------------- | ------ | ---- | ------------------------------------------------------------ | +| id | long | 是 | 高危语句规则ID | +| expression | string | 是 | 表达式 | +| script_language_list | array | 是 | 脚本语言列表:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - sql | +| description | string | 是 | 规则描述 | +| action | int | 是 | 处理动作: 1 - 扫描, 2 - 拦截 | + + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "id": 1, + "expression": "rm", + "script_language_list": [1], + "description": "drangerous!!!", + "action": 2 +} +``` + +### 返回结果示例 + +```json +{ + "code": 0, + "result": true, + "data": { + "id": 1, + "expression": "rm", + "script_language_list": [1], + "description": "drangerous!!!", + "action": 2, + "status": 0, + "creator": "admin", + "create_time": 1695193968000, + "last_modify_user": "admin", + "last_modify_time": 1695302417000 + } +} +``` + +### 返回结果参数说明 + +#### response + +| 字段 | 类型 | 描述 | +| ---------- | ------ | ------------------------------------------ | +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息 | +| data | object | 请求返回的数据 | +| permission | object | 权限信息 | + +#### data + +| 字段 | 类型 | 描述 | +| -------------------- | ------ | ------------------------------------------------------------ | +| id | long | 高危语句规则ID | +| expression | string | 表达式 | +| script_language_list | array | 脚本语言列表:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - sql | +| description | string | 规则描述 | +| action | int | 处理动作: 1 - 扫描, 2 - 拦截 | +| status | int | 启用状态: 0 - 停用, 1 - 启用 | +| creator | string | 创建人 | +| create_time | long | 创建时间Unix时间戳（ms） | +| last_modify_user | string | 最近一次修改人 | +| last_modify_time | long | 最近一次修改时间Unix时间戳（ms） | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/update_file_source.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/update_file_source.md new file mode 100644 index 0000000000..43e2c0c6da --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/update_file_source.md @@ -0,0 +1,78 @@ +### 功能描述 + +更新文件源。 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +|-----------------|------------|--------|------------| +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| code | string | 是 | 文件源标识，英文字符开头，1-32位英文字符、下划线、数字组成 | +| alias | string | 否 | 文件源别名 | +| type | string | 否 | 文件源类型，当前仅支持蓝鲸制品库，BLUEKING_ARTIFACTORY | +| access_params | object | 否 | 文件源接入参数，根据type传入不同的对象，见后续说明 | +| credential_id | string | 否 | 文件源使用的凭证Id | +| file_prefix | string | 否 | Job对从该文件源分发的文件加上的前缀，不传默认不加前缀 | + +### access_params +**type为BLUEKING_ARTIFACTORY** + +| 字段 | 类型 | 必选 | 描述 | +|-----------------|------------|--------|------------| +| base_url | string | 是 | 对接的制品库实例根地址，例如：https://bkrepo.com | + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "1", + "code": "sopsArtifactory", + "alias": "标准运维制品库文件源", + "type": "BLUEKING_ARTIFACTORY", + "access_params": { + "base_url": "https://bkrepo.com" + }, + "credential_id": "06644309e10e4068b3c7b32799668210", + "file_prefix": "" +} +``` + +### 返回结果示例 + +```json +{ + "result": true, + "code": 0, + "message": "success", + "data": { + "id": 1 + } +} +``` + +### 返回结果参数说明 + +#### response +| 字段 | 类型 | 描述 | +|-----------|-----------|-----------| +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息| +| data | object | 请求返回的数据| +| permission | object | 权限信息| + + +#### data + +| 字段 | 类型 |字段是否一定存在 | 描述 | +|------------|--------|---------------|-----------| +| id | int |是 | 文件源ID | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/update_public_script_basic.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/update_public_script_basic.md new file mode 100644 index 0000000000..db6c41a42e --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/update_public_script_basic.md @@ -0,0 +1,73 @@ +### 功能描述 + +更新公共脚本基础信息。 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +| ----------- | ------ | ---- | ------------ | +| script_id | string | 是 | 公共脚本ID | +| name | string | 是 | 公共脚本名称 | +| description | string | 否 | 公共脚本描述 | + + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "script_id": "4537fb49ec0840a1b91cef4179c99f9c", + "name": "public script test", + "description": "public script test" +} +``` + +### 返回结果示例 + +```json +{ + "code": 0, + "result": true, + "data": { + "id": "4537fb49ec0840a1b91cef4179c99f9c", + "name": "public script test", + "script_language": 1, + "creator": "admin", + "create_time": 1691739630000, + "last_modify_user": "admin", + "last_modify_time": 1691740230000, + "description": "public script test" + } +} +``` + +### 返回结果参数说明 + +#### response + +| 字段 | 类型 | 描述 | +| ---------- | ------ | ------------------------------------------ | +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息 | +| data | object | 请求返回的数据 | +| permission | object | 权限信息 | + +#### data + +| 字段 | 类型 | 描述 | +| ---------------- | ------ | ------------------------------------------------------------ | +| id | string | 脚本ID | +| name | string | 脚本名称 | +| script_language | int | 脚本语言:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - sql | +| creator | string | 创建人 | +| create_time | long | 创建时间Unix时间戳（ms） | +| last_modify_user | string | 最近一次修改人 | +| last_modify_time | long | 最近一次修改时间Unix时间戳（ms） | +| description | string | 脚本描述 | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/update_public_script_version.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/update_public_script_version.md new file mode 100644 index 0000000000..322538cd0d --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/update_public_script_version.md @@ -0,0 +1,85 @@ +### 功能描述 + +修改公共脚本版本。 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +| ----------------- | ------ | ---- | ---------------------- | +| script_id | string | 是 | 公共脚本ID | +| script_version_id | long | 是 | 公共脚本版本ID | +| content | string | 是 | 脚本内容，需Base64编码 | +| version_desc | string | 否 | 版本描述 | + + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "script_id": "4537fb49ec0840a1b91cef4179c99f9c", + "script_version_id": 1000020, + "content": "IyEvYmluL2Jhc2gKbHM=", + "version_desc": "version 2" +} +``` + +### 返回结果示例 + +```json +{ + "code": 0, + "result": true, + "data": { + "id": 1000020, + "script_id": "4537fb49ec0840a1b91cef4179c99f9c", + "name": "public script test", + "script_language": 1, + "content": "#!/bin/bash\nls", + "creator": "admin", + "create_time": 1691739630000, + "last_modify_user": "admin", + "last_modify_time": 1691740290000, + "version": "1.0", + "version_desc": "version 2", + "status": 0, + "description": "public script test" + } +} +``` + +### 返回结果参数说明 + +#### response + +| 字段 | 类型 | 描述 | +| ---------- | ------ | ------------------------------------------ | +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息 | +| data | object | 请求返回的数据 | +| permission | object | 权限信息 | + +#### data + +| 字段 | 类型 | 描述 | +| ----------------- | ------ | ------------------------------------------------------------ | +| id | long | 公共脚本版本ID | +| script_id | string | 公共脚本ID | +| name | string | 公共脚本名称 | +| script_language | int | 脚本语言:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - sql | +| content | string | 公共脚本版本内容 | +| creator | string | 创建人 | +| create_time | long | 创建时间Unix时间戳（ms） | +| last_modify_user | string | 最近一次修改人 | +| last_modify_time | long | 最近一次修改时间Unix时间戳（ms） | +| version | string | 公共脚本版本 | +| version_desc | string | 公共脚本版本描述 | +| status | int | 公共脚本版本状态（0：未上线，1：已上线，2：已下线，3：已禁用） | +| description | string | 公共脚本描述 | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/update_script_basic.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/update_script_basic.md new file mode 100644 index 0000000000..39c51d4dc4 --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/update_script_basic.md @@ -0,0 +1,80 @@ +### 功能描述 + +更新脚本基础信息。 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +| ------------- | ------ | ---- | ------------------------------------------------------- | +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| script_id | string | 是 | 脚本ID | +| name | string | 是 | 脚本名称 | +| description | string | 否 | 脚本描述 | + + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type": "biz", + "bk_scope_id": "2", + "script_id": "4a350b0e0707450e93326f6ace921072", + "name": "script test" +} +``` + +### 返回结果示例 + +```json +{ + "code": 0, + "result": true, + "data": { + "id": "4a350b0e0707450e93326f6ace921072", + "name": "script test", + "script_language": 1, + "bk_scope_type": "biz", + "bk_scope_id": "2", + "creator": "admin", + "create_time": 1691743535000, + "last_modify_user": "admin", + "last_modify_time": 1691743715000, + "description": "script test" + } +} +``` + +### 返回结果参数说明 + +#### response + +| 字段 | 类型 | 描述 | +| ---------- | ------ | ------------------------------------------ | +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息 | +| data | object | 请求返回的数据 | +| permission | object | 权限信息 | + +#### data + +| 字段 | 类型 | 描述 | +| ---------------- | ------ | ------------------------------------------------------------ | +| id | string | 脚本ID | +| name | string | 脚本名称 | +| script_language | int | 脚本语言:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - sql | +| bk_scope_type | string | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| creator | string | 创建人 | +| create_time | long | 创建时间Unix时间戳（ms） | +| last_modify_user | string | 最近一次修改人 | +| last_modify_time | long | 最近一次修改时间Unix时间戳（ms） | +| description | string | 脚本描述 | diff --git a/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/update_script_version.md b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/update_script_version.md new file mode 100644 index 0000000000..57076f88ca --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/apidocs/zh_hans/update_script_version.md @@ -0,0 +1,93 @@ +### 功能描述 + +修改脚本版本信息。 + +### 请求参数 + +{{ common_args_desc }} + +#### 接口参数 + +| 字段 | 类型 | 必选 | 描述 | +| ----------------- | ------ | ---- | ------------------------------------------------------- | +| bk_scope_type | string | 是 | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 是 | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| script_id | string | 是 | 脚本ID | +| script_version_id | long | 是 | 脚本版本ID | +| content | string | 是 | 脚本内容，需Base64编码 | +| version_desc | string | 否 | 版本描述 | + + +### 请求参数示例 + +```json +{ + "bk_app_code": "esb_test", + "bk_app_secret": "xxx", + "bk_token": "xxx", + "bk_scope_type":"biz", + "bk_scope_id":"2", + "script_id": "4a350b0e0707450e93326f6ace921072", + "script_version_id": 1000019, + "content": "IyEvYmluL2Jhc2gKZGF0ZQo=", + "version_desc": "version 1.0" +} +``` + +### 返回结果示例 + +```json +{ + "code": 0, + "result": true, + "data": { + "id": 1000019, + "script_id": "4a350b0e0707450e93326f6ace921072", + "name": "script test", + "script_language": 1, + "bk_scope_type": "biz", + "bk_scope_id": "2", + "content": "#!/bin/bash\ndate\n", + "creator": "admin", + "create_time": 1691741073000, + "last_modify_user": "admin", + "last_modify_time": 1691741313000, + "version": "1.0", + "version_desc": "version 1.0", + "status": 0, + "description": "script test" + } +} +``` + +### 返回结果参数说明 + +#### response + +| 字段 | 类型 | 描述 | +| ---------- | ------ | ------------------------------------------ | +| result | bool | 请求成功与否。true:请求成功；false请求失败 | +| code | int | 错误编码。 0表示success，>0表示失败错误 | +| message | string | 请求失败返回的错误信息 | +| data | object | 请求返回的数据 | +| permission | object | 权限信息 | + +#### data + +| 字段 | 类型 | 描述 | +| ----------------- | ------ | ------------------------------------------------------------ | +| id | long | 脚本版本ID | +| script_id | string | 脚本ID | +| name | string | 脚本名称 | +| script_language | int | 脚本语言:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell, 6 - sql | +| bk_scope_type | string | 资源范围类型。可选值: biz - 业务，biz_set - 业务集 | +| bk_scope_id | string | 资源范围ID, 与bk_scope_type对应, 表示业务ID或者业务集ID | +| content | string | 脚本版本内容 | +| creator | string | 创建人 | +| create_time | long | 创建时间Unix时间戳（ms） | +| last_modify_user | string | 最近一次修改人 | +| last_modify_time | long | 最近一次修改时间Unix时间戳（ms） | +| version | string | 脚本版本 | +| version_desc | string | 版本描述 | +| status | int | 脚本版本状态（0：未上线，1：已上线，2：已下线，3：已禁用） | +| description | string | 脚本描述 | diff --git a/docs/apidoc/esb/jobv3-confapis/job.yaml b/docs/apidoc/esb/jobv3-confapis/job.yaml new file mode 100644 index 0000000000..d877135e4f --- /dev/null +++ b/docs/apidoc/esb/jobv3-confapis/job.yaml @@ -0,0 +1,585 @@ +- path: /v2/jobv3/execute_job_plan/ + name: execute_job_plan + label: 执行作业执行方案 + label_en: Execute job plan + suggest_method: POST + api_type: operate + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-execute/execute_job_plan/ + dest_http_method: POST + +- path: /v2/jobv3/fast_execute_script/ + name: fast_execute_script + label: 快速执行脚本 + label_en: Fast execute script + suggest_method: POST + api_type: operate + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-execute/fast_execute_script/ + dest_http_method: POST + +- path: /v2/jobv3/fast_execute_sql/ + name: fast_execute_sql + label: 快速执行SQL + label_en: Fast execute SQL + suggest_method: POST + api_type: operate + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-execute/fast_execute_sql/ + dest_http_method: POST + +- path: /v2/jobv3/fast_transfer_file/ + name: fast_transfer_file + label: 快速分发文件 + label_en: Fast transfer file + suggest_method: POST + api_type: operate + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-execute/fast_transfer_file/ + dest_http_method: POST + +- path: /v2/jobv3/push_config_file/ + name: push_config_file + label: 分发配置文件 + label_en: Distribute config file + suggest_method: POST + api_type: operate + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-execute/push_config_file/ + dest_http_method: POST + is_hidden: True + +- path: /v2/jobv3/operate_step_instance/ + name: operate_step_instance + label: 步骤实例操作 + label_en: Operate step instance + suggest_method: POST + api_type: operate + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-execute/operate_step_instance/ + dest_http_method: POST + +- path: /v2/jobv3/operate_job_instance/ + name: operate_job_instance + label: 作业实例操作 + label_en: Operate job instance + suggest_method: POST + api_type: operate + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-execute/operate_job_instance/ + dest_http_method: POST + +- path: /v2/jobv3/get_job_instance_status/ + name: get_job_instance_status + label: 根据作业实例 ID 查询作业执行状态 + label_en: Get the job execution status based on the job instance ID + suggest_method: POST + api_type: query + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-execute/get_job_instance_status/ + dest_http_method: POST + +- path: /v2/jobv3/get_job_instance_ip_log/ + name: get_job_instance_ip_log + label: 根据作业实例ID查询作业执行日志 + label_en: Get job instance ip log + suggest_method: POST + api_type: query + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-execute/get_job_instance_ip_log/ + dest_http_method: POST + +- path: /v2/jobv3/batch_get_job_instance_ip_log/ + name: batch_get_job_instance_ip_log + label: 根据ip列表批量查询作业执行日志 + label_en: Batch get job instance ip log + suggest_method: POST + api_type: query + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-execute/batch_get_job_instance_ip_log/ + dest_http_method: POST + +- path: /v2/jobv3/get_job_instance_list/ + name: get_job_instance_list + label: 查询作业实例列表(执行历史) + label_en: Get job instance list + suggest_method: POST + api_type: query + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-execute/get_job_instance_list/ + dest_http_method: POST + +- path: /v2/jobv3/get_job_instance_global_var_value/ + name: get_job_instance_global_var_value + label: 获取作业实例全局变量值 + label_en: Get job instance global variable value + suggest_method: POST + api_type: query + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-execute/get_job_instance_global_var_value/ + dest_http_method: POST + +- path: /v2/jobv3/get_job_template_list/ + name: get_job_template_list + label: 查询作业模版列表 + label_en: Get job template list + suggest_method: POST + api_type: query + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-manage/get_job_template_list/ + dest_http_method: POST + +- path: /v2/jobv3/get_job_plan_list/ + name: get_job_plan_list + label: 查询执行方案列表 + label_en: Get job execution plan list + suggest_method: POST + api_type: query + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-manage/get_job_plan_list/ + dest_http_method: POST + +- path: /v2/jobv3/get_job_plan_detail/ + name: get_job_plan_detail + label: 查询执行方案详情 + label_en: Get job execution plan detail + suggest_method: POST + api_type: query + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-manage/get_job_plan_detail/ + dest_http_method: POST + +- path: /v2/jobv3/get_account_list/ + name: get_account_list + label: 查询业务下的执行账号 + label_en: Get accounts + suggest_method: POST + api_type: query + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-manage/get_account_list/ + dest_http_method: POST + +- path: /v2/jobv3/get_public_script_list/ + name: get_public_script_list + label: 查询公共脚本列表 + label_en: Get public script list + suggest_method: POST + api_type: query + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-manage/get_public_script_list/ + dest_http_method: POST + +- path: /v2/jobv3/get_public_script_version_list/ + name: get_public_script_version_list + label: 查询公共脚本版本列表 + label_en: Get public script version list + suggest_method: POST + api_type: query + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-manage/get_public_script_version_list/ + dest_http_method: POST + +- path: /v2/jobv3/get_public_script_version_detail/ + name: get_public_script_version_detail + label: 查询公共脚本详情 + label_en: Get public script detail + suggest_method: POST + api_type: query + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-manage/get_public_script_version_detail/ + dest_http_method: POST + +- path: /v2/jobv3/get_script_list/ + name: get_script_list + label: 查询脚本列表 + label_en: Get script list + suggest_method: POST + api_type: query + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-manage/get_script_list/ + dest_http_method: POST + +- path: /v2/jobv3/get_script_version_list/ + name: get_script_version_list + label: 查询脚本版本列表 + label_en: Get script version list + suggest_method: POST + api_type: query + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-manage/get_script_version_list/ + dest_http_method: POST + +- path: /v2/jobv3/get_script_version_detail/ + name: get_script_version_detail + label: 查询脚本详情 + label_en: Get script version detail + suggest_method: POST + api_type: query + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-manage/get_script_version_detail/ + dest_http_method: POST + +- path: /v2/jobv3/get_cron_list/ + name: get_cron_list + label: 查询业务下定时作业信息 + label_en: Get crontab job list + suggest_method: POST + api_type: query + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-crontab/get_cron_list/ + dest_http_method: POST + +- path: /v2/jobv3/get_cron_detail/ + name: get_cron_detail + label: 查询定时作业详情 + label_en: Get crontab job detail + suggest_method: POST + api_type: query + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-crontab/get_cron_detail/ + dest_http_method: POST + +- path: /v2/jobv3/save_cron/ + name: save_cron + label: 新建或保存定时作业 + label_en: Create or update crontab job + suggest_method: POST + api_type: operate + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-crontab/save_cron/ + dest_http_method: POST + +- path: /v2/jobv3/update_cron_status/ + name: update_cron_status + label: 更新定时作业状态，如启动或暂停 + label_en: Update crontab job status, such as startup or stop + suggest_method: POST + api_type: operate + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-crontab/update_cron_status/ + dest_http_method: POST + +- path: /v2/jobv3/delete_cron/ + name: delete_cron + label: 删除定时任务 + label_en: Delete cron + suggest_method: POST + api_type: operate + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-crontab/delete_cron/ + dest_http_method: POST + +- path: /v2/jobv3/create_credential/ + name: create_credential + label: 新建凭证 + label_en: Create credential + suggest_method: POST + api_type: operate + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-manage/create_credential/ + dest_http_method: POST + is_hidden: True + +- path: /v2/jobv3/update_credential/ + name: update_credential + label: 更新凭证 + label_en: Update credential + suggest_method: POST + api_type: operate + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-manage/update_credential/ + dest_http_method: POST + is_hidden: True + +- path: /v2/jobv3/create_file_source/ + name: create_file_source + label: 新建文件源 + label_en: Create file source + suggest_method: POST + api_type: operate + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-file-gateway/create_file_source/ + dest_http_method: POST + is_hidden: True + +- path: /v2/jobv3/update_file_source/ + name: update_file_source + label: 更新文件源 + label_en: Update file source + suggest_method: POST + api_type: operate + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-file-gateway/update_file_source/ + dest_http_method: POST + is_hidden: True + +- path: /v2/jobv3/generate_local_file_upload_url/ + name: generate_local_file_upload_url + label: 生成本地文件上传URL + label_en: Generate local file upload url + suggest_method: POST + api_type: operate + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-manage/generate_local_file_upload_url/ + dest_http_method: POST + is_hidden: True + +- path: /v2/jobv3/create_public_script/ + name: create_public_script + label: 创建公共脚本 + label_en: Save public script + suggest_method: POST + api_type: operate + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-manage/create_public_script/ + dest_http_method: POST + +- path: /v2/jobv3/update_public_script_basic/ + name: update_public_script_basic + label: 更新公共脚本基础信息 + label_en: Update public script + suggest_method: POST + api_type: operate + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-manage/update_public_script_basic/ + dest_http_method: POST + +- path: /v2/jobv3/delete_public_script_version/ + name: delete_public_script_version + label: 删除公共脚本版本 + label_en: Delete public script version + suggest_method: POST + api_type: operate + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-manage/delete_public_script_version/ + dest_http_method: POST + +- path: /v2/jobv3/create_public_script_version/ + name: create_public_script_version + label: 新建公共脚本版本 + label_en: Save public script version + suggest_method: POST + api_type: operate + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-manage/create_public_script_version/ + dest_http_method: POST + +- path: /v2/jobv3/update_public_script_version/ + name: update_public_script_version + label: 修改公共脚本版本 + label_en: Update public script version + suggest_method: POST + api_type: operate + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-manage/update_public_script_version/ + dest_http_method: POST + +- path: /v2/jobv3/publish_public_script_version/ + name: publish_public_script_version + label: 发布公共脚本版本 + label_en: Publish public script version + suggest_method: POST + api_type: operate + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-manage/publish_public_script_version/ + dest_http_method: POST + +- path: /v2/jobv3/disable_public_script_version/ + name: disable_public_script_version + label: 禁用公共脚本版本 + label_en: Disable public script version + suggest_method: POST + api_type: operate + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-manage/disable_public_script_version/ + dest_http_method: POST + +- path: /v2/jobv3/delete_public_script/ + name: delete_public_script + label: 删除公共脚本 + label_en: Delete public script + suggest_method: POST + api_type: operate + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-manage/delete_public_script/ + dest_http_method: POST + +- path: /v2/jobv3/create_script/ + name: create_script + label: 创建脚本 + label_en: Save script + suggest_method: POST + api_type: operate + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-manage/create_script/ + dest_http_method: POST + +- path: /v2/jobv3/update_script_basic/ + name: update_script_basic + label: 更新脚本基础信息 + label_en: Update script basic + suggest_method: POST + api_type: operate + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-manage/update_script_basic/ + dest_http_method: POST + +- path: /v2/jobv3/delete_script_version/ + name: delete_script_version + label: 删除脚本版本 + label_en: Delete script version + suggest_method: POST + api_type: operate + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-manage/delete_script_version/ + dest_http_method: POST + +- path: /v2/jobv3/create_script_version/ + name: create_script_version + label: 新建脚本版本 + label_en: Save script version + suggest_method: POST + api_type: operate + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-manage/create_script_version/ + dest_http_method: POST + +- path: /v2/jobv3/update_script_version/ + name: update_script_version + label: 修改脚本版本信息 + label_en: Update script version + suggest_method: POST + api_type: operate + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-manage/update_script_version/ + dest_http_method: POST + +- path: /v2/jobv3/publish_script_version/ + name: publish_script_version + label: 发布脚本版本 + label_en: Publish script version + suggest_method: POST + api_type: operate + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-manage/publish_script_version/ + dest_http_method: POST + +- path: /v2/jobv3/disable_script_version/ + name: disable_script_version + label: 禁用脚本版本 + label_en: Disable script version + suggest_method: POST + api_type: operate + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-manage/disable_script_version/ + dest_http_method: POST + +- path: /v2/jobv3/delete_script/ + name: delete_script + label: 删除脚本 + label_en: Delete script + suggest_method: POST + api_type: operate + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-manage/delete_script/ + dest_http_method: POST + +- path: /v2/jobv3/create_account/ + name: create_account + label: 创建账号 + label_en: Create account + suggest_method: POST + api_type: operate + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-manage/create_account/ + dest_http_method: POST + +- path: /v2/jobv3/delete_account/ + name: delete_account + label: 删除账号 + label_en: Delete account + suggest_method: POST + api_type: operate + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-manage/delete_account/ + dest_http_method: POST + +- path: /v2/jobv3/create_dangerous_rule/ + name: create_dangerous_rule + label: 创建高危语句规则 + label_en: Create dangerous rule + suggest_method: POST + api_type: operate + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-manage/create_dangerous_rule/ + dest_http_method: POST + +- path: /v2/jobv3/update_dangerous_rule/ + name: update_dangerous_rule + label: 修改高危语句规则 + label_en: Update dangerous rule + suggest_method: POST + api_type: operate + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-manage/update_dangerous_rule/ + dest_http_method: POST + +- path: /v2/jobv3/delete_dangerous_rule/ + name: delete_dangerous_rule + label: 删除高危语句规则 + label_en: Delete dangerous rule + suggest_method: POST + api_type: operate + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-manage/delete_dangerous_rule/ + dest_http_method: POST + +- path: /v2/jobv3/get_dangerous_rule_list/ + name: get_dangerous_rule_list + label: 查看高危语句规则列表 + label_en: Get dangerous rule list + suggest_method: POST + api_type: query + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-manage/get_dangerous_rule_list/ + dest_http_method: POST + +- path: /v2/jobv3/enable_dangerous_rule/ + name: enable_dangerous_rule + label: 启用高危语句规则 + label_en: Enable dangerous rule + suggest_method: POST + api_type: operate + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-manage/enable_dangerous_rule/ + dest_http_method: POST + +- path: /v2/jobv3/disable_dangerous_rule/ + name: disable_dangerous_rule + label: 停用高危语句规则 + label_en: Disable dangerous rule + suggest_method: POST + api_type: operate + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-manage/disable_dangerous_rule/ + dest_http_method: POST + +- path: /v2/jobv3/check_script/ + name: check_script + label: 高危脚本检测 + label_en: Check script + suggest_method: POST + api_type: operate + comp_codename: generic.v2.jobv3.job_component + dest_path: /api/job/v3/job-manage/check_script/ + dest_http_method: POST + +- path: /esb/api/v3/get_latest_service_version/ + name: get_latest_service_version + label: 查询最新服务版本号 + label_en: Get latest service version + suggest_method: POST + api_type: operate + comp_codename: generic.v2.jobv3.job_component + dest_path: /esb/api/v3/get_latest_service_version/ + dest_http_method: POST diff --git a/docs/overview/ai_chat_stream.md b/docs/overview/ai_chat_stream.md new file mode 100644 index 0000000000..77a3bc20a3 --- /dev/null +++ b/docs/overview/ai_chat_stream.md @@ -0,0 +1,15 @@ +# AI对话流式接口实现技术方案 + +**（1）流式数据通道建立** +为了让用户能在页面上及时看到大模型生成的内容，必须在【用户浏览器】-【作业平台后台】-【大模型服务】之间建立一条流式数据通道。其工作原理如下图所示： +![AIDataRoute](../resource/img/ai_data_route.png) +后台一方面需要从大模型服务获取流式数据，另一方面需要将获取到的流式数据及时写入到用户浏览器发起的请求响应中。 +经过调研，采用前端Fetch API分块读取流式数据+后台Spring MVC异步接口来提供流式数据。 +Spring MVC中的异步接口使用单独的异步任务线程池来执行异步任务（调用大模型服务获取流式数据），在异步任务执行过程中可以向响应输出流写入数据，并在异步任务完成后将请求的处理交还给底层Tomcat容器。 +在实现过程中发现，使用openai4j框架调用大模型获取流式数据的过程本身也是异步的，框架使用了OKHttp的Dispatcher线程池发起请求，收到的流式数据通过回调接口的方式传递给调用方（在Dispatcher工作线程中）。要将Dispatcher工作线程收到的数据传递给Spring MVC的异步任务线程，才能及时写入到输出流中传递给浏览器让用户看到。 +因此这里需要用到线程间的数据同步技术，使用JDK并发包中提供的阻塞队列来实现，建立流式数据通道，在收到分块数据后立即将其发送至Web端，延迟P95<2ms，整体技术方案如下图所示： +![AIDataArchitecture](../resource/img/ai_data_architecture.png) + +**（2）流式数据卡顿问题排查解决** +在部分环境中，我们发现页面上的流式接口需要间隔几秒才能收到一波数据，导致回答内容一批一批地出现几十个字，内容呈现非常不连续，用户体验较差。 +经过对比分析与排查，我们发现问题在于请求链路中存在Nginx转发环节，而**Nginx的默认配置中，对于响应内容存在8K的缓冲区**，只有缓冲区写满了才会输出到客户端从而造成卡顿，缓冲区存在的初衷是为了减少IO操作提高传输效率，但它同时也带来了延迟，这对于流式接口并不适用，因此在非正式环境中我们配置关闭该缓冲区，卡顿问题也由此得到解决，在性能要求较高的环境中，可以针对具体的流式接口关闭该缓冲区。 diff --git a/docs/overview/architecture.en.md b/docs/overview/architecture.en.md index bf4624f71a..56b2a17979 100644 --- a/docs/overview/architecture.en.md +++ b/docs/overview/architecture.en.md @@ -1,16 +1,24 @@ -# Blueking (BK-JOB) Architecture Design +# Blueking (BK-JOB) Architecture Design English | [简体中文](architecture.md) -![Architecture](../resource/img/architecture.png) +## Introduction +The Blueking Job Platform (aka **bk-job**) is one of the **basic atomic platforms** of Tencent Blueking. Through the **"file pipeline"** and **"command pipeline"** of the BK-GSE , it provides capabilities such as job orchestration, script execution, and file distribution. -The BlueKing Platform (aka **bk-job** ) is written in multiple languages including java/js/html/python/shell/gradle, featuring separation of frontend and backend, with highly-accessible, scalabe, service-oriented architecture: -- **Frontend (FrontEnd):** +![BluekingArchitecture](../resource/img/bk_architecture_en.png) + + +## BK-JOB Architecture + +![Architecture](../resource/img/architecture_en.png) + + +### Frontend Features ES6 grammar, MVVM architecture vue.js, and webpack4 package. -- **Backend (BackEnd):** +### Backend Written in Java, featuring microservice architecture of Spring Cloud. Here is a brief rundown on each of the microservice modules: @@ -25,11 +33,12 @@ Written in Java, featuring microservice architecture of Spring Cloud. Here is a - **job-file-gateway:** File Gateway Microservice: Connects with various third party file sources via FileWorker (object-based storage, file system storage, etc.); schedules the file downloading tasks from file source; works with the execution engine to distribute files from third party file source. - **job-file-worker:** An access point, an expandable module independent of other bk-job backend microservice; multiple instances can be set up; communicate with the file gateway to connect to various types of third party file sources; executes file downloading tasks. -- **Resource Service (Resource):** Provides the necessary basic middleware for storage. - - **Consul:** Used as a service discovery server; You need to build a Consul Server and install Consul on the machine where bk-job microservices are deployed, and run it as Agent. - - **RabbitMQ:** Core Message Queue Service: bk-job's task execution engine uses RabbitMQ to deliver the message of task status changes. - - **MySQL:** bk-job's primary database. Uses mysql 5.7.2 to store the relational data of all microservices listed above. - - **Redis:** Core Service Cache 4.0.14: Provides host information for distributed locks and cache service. - - **MongoDB:** BK-job's log database. MongoDB 4.2.2. Used to store the log data generated from script execution/file distribution. - - **NFS:** Used to store user-uploaded local files in local file distribution scenario and store temporary files generated from job import and export. +### Storage And Middleware + + - **Consul:** Phisical deployment only, used as service discovery. You need to build a Consul Server and install Consul on the machine where bk-job microservices are deployed, and run it as Agent. + - **RabbitMQ:** Core Message Queue Service. Job-execute module uses RabbitMQ to implement a message-based event-driven task scheduling engine. + - **MySQL:** bk-job's primary database. It store the relational data of job microservices. + - **Redis:** Distributed Cache. Used for distributed locks and data caching. + - **MongoDB:** BK-job's job execution log database. Used to store the log data generated from script execution/file distribution. + - **NFS:** Used to store user-uploaded local files in local file distribution scenario and store temporary files generated from job import and export. diff --git a/docs/overview/architecture.md b/docs/overview/architecture.md index 8c7a0e55ae..f674402dc9 100644 --- a/docs/overview/architecture.md +++ b/docs/overview/architecture.md @@ -2,34 +2,45 @@ [English](architecture.en.md) | 简体中文 -![Architecture](../resource/img/architecture.png) +## 介绍 +蓝鲸作业平台(简称**bk-job** )是蓝鲸的**基础原子平台**之一，通过蓝鲸管控平台(GSE)的**“文件管道”和“命令管道”**，提供了作业编排、脚本执行、文件分发等能力。 + + +![BluekingArchitecture](../resource/img/bk_architecture_cn.png) + + -蓝鲸作业平台(简称**bk-job** )是基于 java/js/html/python/shell/gradle等多种语言编写实现的，采用完全前后端分离，具备高可用可扩展的服务架构设计： -- **前端(FrontEnd) ：** + +## 作业平台整体架构 + +![Architecture](../resource/img/architecture.png) + +### 前端(FrontEnd) 使用ES6语法，采用MVVM框架vue.js，通过webpack4打包构建。 -- **后端(BackEnd)：** - -基于Java编写，采用SpringCloud框架的微服务架构设计，以下按各微服务模块进行介绍： - - - **job-config：** 配置中心，统一管理所有微服务的配置信息。 - - **job-gateway：** 微服务网关，负责认证、限流、请求路由等。 - - **job-manage：** 作业管理微服务，负责管理作业平台中的多种资源，具体包含脚本、账号、作业模板、执行方案、消息通知、全局设置等。 - - **job-execute：** 作业执行微服务，对接蓝鲸GSE，负责向GSE提交文件分发/脚本执行任务、拉取任务日志、流转任务状态。 - - **job-logsvr：** 日志管理微服务，对接底层MongoDB，负责存储脚本执行、文件分发及文件源文件下载过程中产生的日志。 - - **job-crontab：** 定时任务微服务，基于Quartz定时引擎，实现bk-job的定时任务调度与管理。 - - **job-backup：** 备份管理微服务，负责bk-job中的作业导入导出及作业执行流水历史数据的定期归档任务。 - - **job-analysis：** 统计分析微服务，为首页异常作业提示与运营分析模块提供后台接口，调度大量定时任务从其他微服务模块获取元数据进行分析与统计，生成分析结果数据与统计数据，为bk-job提供运营数据支撑、提高平台易用性。 - - **job-file-gateway：** 文件网关微服务，通过与FileWorker通信对接多种不同类型的第三方文件源（如对象存储、文件系统存储等），负责调度文件源文件下载任务，与执行引擎配合完成第三方文件源文件分发。 - - **job-file-worker：** 文件源接入点，独立于其他bk-job后台微服务的可扩展模块，可部署多个实例，与文件网关进行通信，对接不同类型的第三方文件源，是文件下载任务的执行者。 - -- **资源服务层(Resource)：** 包括提供存储和必须的基础中间件等。 - - **Consul：** 用作服务发现Server，需要搭建Consul Server， 以及在bk-job微服务部署的所在机器上同时安装Consul并以 Agent方式运行。 - - **RabbitMQ：** 核心消息队列服务，bk-job的任务执行引擎使用RabbitMQ来传递作业任务状态切换消息。 - - **MySQL：** bk-job的主数据库存储，可用mysql 5.7.2存储以上所有微服务的关系型数据。 - - **Redis：** 核心服务缓存，4.0.14版本，用于提供分布式锁、缓存业务主机信息等。 - - **MongoDB：** bk-job的日志数据库，MongoDB 4.2.2版本，用于存储脚本执行/文件分发过程中产生的日志数据。 - - **NFS：** 一方面用于本地分发文件场景中存储用户上传的本地文件，另一方面用于存储作业导入或导出时生成的临时文件。 +### 后端(BackEnd) + +基于Java编写，采用主流的SpringBoot+SpringCloud框架的微服务架构设计。以下按各微服务模块进行介绍： + + - **配置中心：** 二进制版本的配置中心，统一管理所有微服务的配置信息。 + - **Job 网关：** 微服务网关，负责认证、限流、请求路由等。 + - **作业管理：** 作业管理微服务，负责管理作业平台中的多种资源，具体包含脚本、账号、作业模板、执行方案、消息通知、全局设置等。 + - **执行引擎：** 作业执行微服务，对接蓝鲸GSE，负责向GSE提交文件分发/脚本执行任务、拉取任务日志、流转任务状态。 + - **执行日志服务：** 日志管理微服务，对接底层MongoDB，负责存储脚本执行、文件分发及文件源文件下载过程中产生的日志。 + - **定时任务：** 定时任务微服务，基于Quartz定时引擎，实现bk-job的定时任务调度与管理。 + - **备份归档：** 备份管理微服务，负责bk-job中的作业导入导出及作业执行流水历史数据的定期归档任务。 + - **统计分析：** 统计分析微服务，为首页异常作业提示与运营分析模块提供后台接口，调度大量定时任务从其他微服务模块获取元数据进行分析与统计，生成分析结果数据与统计数据，为bk-job提供运营数据支撑、提高平台易用性。 + - **文件源网关：** 文件网关微服务，通过与FileWorker通信对接多种不同类型的第三方文件源（如对象存储、文件系统存储等），负责调度文件源文件下载任务，与执行引擎配合完成第三方文件源文件分发。 + - **文件源接入点：** 文件源接入点，独立于其他bk-job后台微服务的可扩展模块，可部署多个实例，与文件网关进行通信，对接不同类型的第三方文件源，是文件下载任务的执行者。 + +### 存储、基础中间件 + + - **Consul：** 仅用于物理部署方式，用作服务发现。需要搭建Consul Server， 以及在bk-job微服务部署的所在机器上同时安装Consul并以 Agent方式运行。 + - **RabbitMQ：** 消息队列服务。作业执行模块使用 RabbitMQ 实现基于消息的事件驱动任务调度引擎 + - **MySQL：** bk-job的主数据库存储，存储微服务的关系型数据。 + - **Redis：** 分布式缓存，用于实现分布式锁、数据缓存等。 + - **MongoDB：** bk-job的作业执行日志数据库，用于存储脚本执行/文件分发过程中产生的日志数据。 + - **NFS：** 一方面用于本地分发文件场景中存储用户上传的本地文件，另一方面用于存储作业导入或导出时生成的临时文件。 diff --git a/docs/overview/installation.en.md b/docs/overview/installation.en.md deleted file mode 100644 index 1f9ce92c0f..0000000000 --- a/docs/overview/installation.en.md +++ /dev/null @@ -1,138 +0,0 @@ -# Installation Setup - -English | [简体中文](installation.md) - -## 1. About Setup Directory - -The directory should conform to Blueking rules of operation. We're taking /data/bkee as the primary directory. Users can feel free to change them. The details are shown as follows: - -``` -|- /data/bkee # Blueking Root Directory - |- job # job Setup Program Directory - |- etc # Blueking Config File Catalog - |- job # job Config File Menu -``` - -Detailed explanation of the following chapters: - -### 1.1 job Setup Directory - -``` -|- /data/bkee/job # Home Directory - |- frontend # Static Resource Directory for Frontend Programs - |- index.html # Frontend Home Page File - |- __init__ # Initialization File Directory - |- js # Static JS Directory - |- static # Other Static File Directory - |- support-files # Resource Files - |- backend # The program that stores backend microservices and file source access points - |- job-manage # job-manage Microservice Jar package and ops script. There are 11 directories in total, which have already been introduced above. - |- job-manage.jar # job-manage Microservice's SpringBoot.jar - |- bin - |- job-manage.sh # job-manage Microservice's ops script - |- job-xxx -``` - -### 1.2 job Config File Directory - -``` -|- /data/bkee/etc # Blueking Config File Catalog - |- job # job Config File Menu - |- job.env # Specified environment variables when rendering job configuration files through template - |- application-manage.yml # job-manage's basic configuration. Each microservice has 1 yaml file. If there are new microservices, please put them here. - |- job-manage # job-manage's extended configuration. Each microservice has 1 corresponding directory for the extended configuration files. If there are new microservices, new directories should be added accordingly. - |- job-manage.properties # job-manage Extended configuration properties file -``` - -## 2. Basic Environment Setup - -### 2.1 System Requirement - -- CentOS 7.X -- jdk: 1.8 -- gradle: 6.3 -- redis: 4.0.14 -- mysql 5.7 -- rabbitmq: 3.7.14 -- mongoDB: 4.2.2 -- nginx: 1.16.1 -- Consul: 1.0+ - -### 2.2 Database Initialization - -Run the files under support-files/sql/* by the file sequence. - - -## 3 Program Setup - -### 3.1 support-files/template Initialize configuration file - -The underlined variables in the configuration file should be replaced with real data (e.g. MySQL account and password.) When the placement is made, the configuration file should be moved to the correct file directory to be read by the Configuration Center Microservice. - -### 3.2 Backend Microservice Setup - -- [Backend Service Setup] (../install/backend.md) - -### 3.3 Frontend Setup - -First, place all compiled static frontend files under the correct frontend directory. Refer to chapter 1.1 - Directory Setup. -Second, replace the variables of index.html in the frontend setup directory: -Location: /data/bkee/job/frontend/index.html -Change {{JOB_API_GATEWAY_URL}} into backend job API address (usually an independent API domain.) -And that's all the preparations for frontend setup. Later, you can set Nginx as frontend static service. - -### 3.4 Nginx Configuration and Setup -Job uses Nginx as the frontend static resource server, and forwards backend requests to the job-gateway microservice. The configuration is listed as follows: -```shell script -# Server Configuration of Frontend Static Resources -server { - listen 80; - server_name {{JOB_FRONTEND_HOST}}; - - gzip on; - client_max_body_size 150M; - root /data/bkee/job/frontend; - index index.html; - - location / { - add_header Cache-Control no-cache; - expires 0; - try_files $uri $uri/ @rewrites; - } - - location @rewrites { - rewrite ^(.+)$ /index.html last; - } -} -# Backend API Forwards Server Configuration -upstream job_gateway_servers { - server {{JOB_GATEWAY_IP0}}:19802; - server {{JOB_GATEWAY_IP1}}:19802; -} -server { - listen 80; - server_name {{JOB_API_HOST}}; - location / { - proxy_pass http://job_gateway_servers; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Host $http_host; - } - access_log /data/bkee/logs/nginx-access.log json_combined; - client_max_body_size 2048M; - error_log /data/bkee/logs/nginx-error.log; - error_page 404 403 500 502 503 504 /job_error.html; - location = /job_error.html { - root /data/html; - } -} -``` -When the configuration is completed, reload nginx to make the settings effective: -```shell script -nginx -s reload -c /path/to/nginx.conf -``` - -### 3.5 Access Job's Home Page -Access Job's home page via {{JOB_FRONTEND_HOST}} to use Job features. diff --git a/docs/overview/installation.md b/docs/overview/installation.md deleted file mode 100644 index e6e57a5dca..0000000000 --- a/docs/overview/installation.md +++ /dev/null @@ -1,138 +0,0 @@ -# 安装部署 - -[English](installation.en.md) | 简体中文 - -## 1. 部署目录说明 - -部署的目录遵循蓝鲸运营规范，这里举例以/data/bkee 作为主目录，用户可以自行更换，具体如下： - -``` -|- /data/bkee # 蓝鲸根目录 - |- job # job部署程序目录 - |- etc # 蓝鲸配置文件总目录 - |- job # job配置文件目录 -``` - -具体说明下以下章节。 - -### 1.1 job部署目录 - -``` -|- /data/bkee/job # 程序主目录 - |- frontend # 存放前端程序的静态资源目录 - |- index.html # 前端首页文件 - |- __init__ # 初始化文件目录 - |- js # 静态JS目录 - |- static # 其他静态文件目录 - |- support-files # 资源文件 - |- backend # 存放后台微服务及文件源接入点程序 - |- job-manage # job-manage微服务相关jar包与运维脚本，共有11个目录，不再一一列举 - |- job-manage.jar # job-manage微服务的SpringBoot.jar - |- bin - |- job-manage.sh # job-manage微服务的运维脚本 - |- job-xxx -``` - -### 1.2 job配置文件目录 - -``` -|- /data/bkee/etc # 蓝鲸配置文件总目录 - |- job # job配置文件目录 - |- job.env # 通过模板渲染job配置文件时所需要指定的环境变量 - |- application-manage.yml # job-manage的基础配置，每个微服务各有1个基础配置yaml文件，如有增加微服务也放此处 - |- job-manage # job-manage的扩展配置，每个微服务各有1个对应目录存放扩展配置文件，如有增加微服务须增加对应目录 - |- job-manage.properties # job-manage扩展配置properties文件 -``` - -## 2. 基础环境部署 - -### 2.1 系统要求 - -- CentOS 7.X -- jdk: 1.8 -- gradle: 6.3 -- redis: 4.0.14 -- mysql 5.7 -- rabbitmq: 3.7.14 -- mongoDB: 4.2.2 -- nginx: 1.16.1 -- Consul: 1.0+ - -### 2.2 数据库初始化 - -将support-files/sql/* 目录下按文件序号顺序执行。 - - -## 3 程序部署 - -### 3.1 support-files/template配置文件初始化 - -涉及到配置文件里面有双"_"下划线定义的变量需要根据实际数据（如MySQL账号密码等）做占位符号替换，完成替换后需将配置文件移动至正确的配置文件目录下，供配置中心微服务启动后读取。 - -### 3.2 后端微服务部署 - -- [后端服务部署](../install/backend.md) - -### 3.3 前端部署 - -首先，将编译生成的所有前端静态文件放置于正确的前端目录下，可参考1.1节部署目录。 -其次，对前端部署目录中的index.html文件做变量替换： -文件位置：/data/bkee/job/frontend/index.html -将其中的{{JOB_API_GATEWAY_URL}}替换为为Job后台API地址(通常为分配的独立API域名)。 -前端部署准备工作至此完成，后续通过配置Nginx作为前端静态服务即可。 - -### 3.4 Nginx配置与部署 -Job使用Nginx作为前端静态资源服务器并转发后端请求至job-gateway微服务，其配置可参考如下： -```shell script -# 前端静态资源服务server配置 -server { - listen 80; - server_name {{JOB_FRONTEND_HOST}}; - - gzip on; - client_max_body_size 150M; - root /data/bkee/job/frontend; - index index.html; - - location / { - add_header Cache-Control no-cache; - expires 0; - try_files $uri $uri/ @rewrites; - } - - location @rewrites { - rewrite ^(.+)$ /index.html last; - } -} -# 后台API转发server配置 -upstream job_gateway_servers { - server {{JOB_GATEWAY_IP0}}:19802; - server {{JOB_GATEWAY_IP1}}:19802; -} -server { - listen 80; - server_name {{JOB_API_HOST}}; - location / { - proxy_pass http://job_gateway_servers; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-Proto $scheme; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Host $http_host; - } - access_log /data/bkee/logs/nginx-access.log json_combined; - client_max_body_size 2048M; - error_log /data/bkee/logs/nginx-error.log; - error_page 404 403 500 502 503 504 /job_error.html; - location = /job_error.html { - root /data/html; - } -} -``` -配置完成后reload nginx使其生效： -```shell script -nginx -s reload -c /path/to/nginx.conf -``` - -### 3.5 访问Job首页 -通过{{JOB_FRONTEND_HOST}}访问Job首页，即可开始体验Job功能。 diff --git a/docs/overview/job_execute_engine_design.md b/docs/overview/job_execute_engine_design.md new file mode 100644 index 0000000000..66400e9517 --- /dev/null +++ b/docs/overview/job_execute_engine_design.md @@ -0,0 +1,20 @@ +## 介绍 + +作业执行引擎是作业平台的核心模块，负责作业任务的创建、调度、执行。 + +用户从 Job Web 页面、定时触发、第三方应用(OpenApi)触发的任务，通过执行引擎调度之后，下发给蓝鲸管控平台（GSE），完成命令执行、文件分发的能力。 + +## 设计 + + + +![EngineArchitecture](../resource/img/job_execute_engine_architecture.png) + +从执行引擎架构图可以看出，执行引擎基于 **RabbitMQ**， 实现了**基于事件的任务调度引擎**。任务执行的不同阶段触发对应的事件(JobEvent\StepEvent\GseTaskEvent), 驱动对应的**事件处理器**(JobEventHandler\StepEventHandler\GseTaskEventHandler)完成业务逻辑。 + +### 依赖平台说明 +- 权限中心：鉴权 +- cmdb: 作业平台使用 cmdb 的资源，比如业务、主机（主机拓扑、动态分组等）、容器（容器拓扑） +- 审计中心：操作审计 +- GSE：提供“命令管道”和“文件管道”。作业平台通过GSE Server向执行目标上的GSE Agent下发任务，并从GSE获取任务执行结果 +- 制品库：用于存储用户上传的本地文件、应用生成的文件 \ No newline at end of file diff --git a/docs/overview/job_execute_engine_design_en.md b/docs/overview/job_execute_engine_design_en.md new file mode 100644 index 0000000000..8efb0baf85 --- /dev/null +++ b/docs/overview/job_execute_engine_design_en.md @@ -0,0 +1,22 @@ +## Introduction + +The job execution engine is the core module of the job platform and is responsible for the creation, scheduling, and execution of job tasks. + +Tasks triggered by users from the Job Web page, cronJob, and third-party applications (OpenApi) are scheduled by the execution engine and then sent to the GSE to complete command execution and file distribution capabilities. + +## Design + + +### Overview + +![EngineArchitecture](../resource/img/job_execute_engine_architecture_en.png) + +As can be seen from the execution engine architecture diagram, the execution engine is based on **RabbitMQ** and implements an **event-based task scheduling engine**. Different stages of task execution trigger corresponding events (JobEvent\StepEvent\GseTaskEvent), driving the corresponding **event handler** (JobEventHandler\StepEventHandler\GseTaskEventHandler) to complete the business logic. + +### Dependent platform description + +- BK-IAM: Operation authentication +- BK-CMDB: The Job platform uses cmdb resources, such as business, host (host topology, dynamic grouping, etc.), container (container topology) +- BK-Audit: Audit records of job execution related operations +- GSE: Provides "command pipeline" and "file pipeline". The job platform delivers tasks to the GSE Agent on the execution target through the GSE Server, and obtains the task execution results from the GSE +- Bk-Repo: Used to store local files uploaded by users and files generated by job platform \ No newline at end of file diff --git a/docs/overview/operation.md b/docs/overview/operation.md new file mode 100644 index 0000000000..ce0bb1d972 --- /dev/null +++ b/docs/overview/operation.md @@ -0,0 +1,77 @@ +# 蓝鲸作业平台部署与运维文档 + +[English](operation_en.md) | 简体中文 + +## 一、部署 +蓝鲸作业平台（后续简称作业平台）是蓝鲸原子平台之一，其底层依赖包括： +### 1.强依赖 +以下的依赖系统是作业平台的强依赖系统，若缺失或异常将导致作业平台核心功能不可用： +**PaaS平台**：作业平台调用其接口对接统一登录服务； +**用户管理**：作业平台调用其接口获取用户信息； +**权限中心**：作业平台调用其接口获取权限策略对用户操作进行鉴权； +**APIGateway**：作业平台通过APIGateway调用底层平台接口并为其他上层平台提供接口； +**配置平台**：作业平台调用其接口获取业务、主机等资源数据； +**管控平台**：作业平台调用其接口下发脚本执行、文件分发任务并获取任务执行结果与日志。 +### 2.弱依赖 +以下的依赖系统是作业平台的弱依赖系统，若缺失或异常将导致作业平台部分功能不可用： +**蓝鲸制品库**：作业平台调用其接口存储本地文件、导入导出、执行日志导出产生的临时文件数据； +**审计中心**：作业平台输出审计数据日志文件供其采集，用于审计用户操作； +**日志平台**：作业平台输出的后台日志文件会被其采集器采集，便于在日志平台上进行统一检索； +**监控平台**：作业平台通过Prometheus协议暴露出指标数据，会被其采集器采集，便于在监控平台上配置指标面板与告警，此外，作业平台通过OTEL协议调用其接口向其上报调用链APM数据，便于在监控平台上进行调用链观测与分析； +**消息通知中心**：作业平台调用其接口获取通知与公告信息并在页面进行展示。 + +由于依赖众多，仅部署作业平台本身是无法正常工作的，需要首先部署其依赖的底层平台，因此请参考[蓝鲸文档中心](https://bk.tencent.com/docs/)部署维护版块的基础套餐相关内容进行部署。 + +## 二、运维 +### 1.资源占用管理 +在成功部署作业平台后，随着系统运行时长增加，使用数据随之增加，会导致作业平台占用的资源增加，出于成本与性能考虑，需要对资源占用进行管理，必要时按需清理部分过期数据。 +#### （1）MySQL数据 +作业平台的不同功能模块使用不同的数据库存储数据，其中需要重点关注的是任务执行引擎使用的job_execute库，该库存储了任务执行过程中产生的流水数据，数据量随着任务量的增长而增长，通常在平台运行一段时间后，job_execute库会成为占用存储空间最多的库，需要考虑数据的归档与清理。 +作业平台自身支持了任务流水数据的自动归档与清理机制，可通过helm chart values中的backupConfig.archive下的属性进行配置，默认的归档策略为：不启用归档，永久保留所有数据。如果有清理过期数据的需求，可以将策略配置为不备份数据，直接删除超出过期时间（默认为30天）的数据。如果有备份过期数据的需求，可以将策略配置为先备份数据再删除。 + +#### （2）MongoDB数据 +作业平台使用MongoDB存储任务执行过程中产生的业务脚本执行日志及文件分发任务执行日志数据，默认存储在joblog库中（可通过helm chart values中的相关属性配置），日志数据量随着任务量的增长而增长，通常在平台运行一段时间后，joblog库会成为占用存储空间最多的库，需要考虑数据的归档与清理。 +作业平台使用任务步骤类型（脚本/文件）及日期来组织存储日志数据，其中脚本任务的日志存储在job_log_script_{date}集合中，文件任务的日志存储在job_log_file_{date}集合中，其中{date}为任务创建时的日期，如：job_log_script_2024_07_19、job_log_file_2024_07_20。 +作业平台当前暂未提供自动归档与清理机制，需要用户自行手动执行命令或编写脚本导出/删除需要处理的集合。 + +#### （3）蓝鲸制品库数据 +作业平台使用蓝鲸制品库存储用户上传的本地文件、导入导出、执行日志导出过程产生的临时文件，默认使用的蓝鲸制品库项目为bkjob（可通过helm chart values中的artifactory相关属性进行配置），使用的仓库为localupload(本地文件)、backup(导入导出临时文件)、filedata(执行日志导出临时文件)，其中localupload仓库中的文件如果被作业/执行方案的本地文件分发步骤引用，则会永久保存，未被引用的文件将在超出过期时间后被自动清理，默认过期时间为7天（可通过helm chart values中的localFile相关属性进行配置），backup与filedata仓库中的文件为临时文件，若系统未自动清理可按需清理。 + +#### （4）其他数据 +**后台日志：** 系统运行过程中产生的后台日志默认打印在容器内的文件中，超出限定容量或过期时间后会被自动清理，通常无需处理，容量与过期时间可通过helm chart values中的log下的属性配置。 +**日志平台数据：** 如果配置了日志采集（helm chart values中的bkLogConfig下的属性），日志平台将通过采集器将后台日志采集到日志平台并存放于其后台存储组件中用于检索，这部分数据需要在日志平台配置清理策略。 +**监控指标数据：** 如果配置了监控数据采集（helm chart values中的serviceMonitor下的属性），监控平台将通过采集器采集后台服务指标数据用于在监控平台通过仪表盘进行查看或配置告警，这部分数据需要在监控平台配置清理策略。 +**APM数据：** 如果配置了APM数据采集（helm chart values中的job.trace下的属性），作业平台将使用OTEL协议向监控平台主动上报后台服务调用链数据用于在监控平台进行调用链的可视化分析，这部分数据需要在监控平台配置清理策略。 + +### 2.系统异常排查与恢复 +（1）服务状态查看 +作业平台成功部署后，系统管理员可以在Web页面的【平台管理-服务状态】子页面查看系统各服务模块及各个实例的状态，一般情况下各实例状态应当全部为“正常”，若非正常则需要对相应的实例进一步排查。 + +（2）Pod状态查看 +在部署作业平台的命名空间中，可以使用kubectl get pod命令结合“bk-job”关键字grep出作业平台的所有Pod，正常情况下作业平台所有Pod状态都应为Running/Completed，若不正常则需要进一步排查对应Pod内的服务日志或者尝试重启Pod。 + +（3）服务日志分析 +在容器化部署下，可进入各服务Pod中的容器内查看服务日志，作业平台各服务的日志存储位置为：/data/logs/{服务名}/，例如执行引擎的日志位置为/data/logs/job-execute/，目录下的日志按照类别分为多个文件并且会按时间进行滚动，以下分别介绍各类日志文件的用途： +error.log: 错误日志汇总，其内容在其余常规日志文件中也会有一份，需要重点关注； +{服务名}.log: 服务主日志，如execute.log，为服务的核心业务逻辑日志，需要重点关注； +openapi.log：作业平台对外提供的API（ESB、APIGW等）调用日志； +schedule.log：执行引擎任务调度日志； +monitor_task.log：超大任务（涉及的主机数量大）监控日志； +sync_app_host.log：业务（集）、主机等CMDB资源同步任务与事件监控日志； +access.log：网关接口访问日志； +esb_access.log：网关ESB接口访问日志； +cmdb.log：配置平台（CMDB）相关接口调用日志； +gse.log：管控平台（GSE）相关接口调用日志； +iam.log：权限中心（IAM）相关接口调用日志； +paas.log：PaaS平台相关接口（登录、用户等）调用日志； +notice.log：消息通知中心相关接口（通知、公告等）调用日志； +audit_event.log：审计日志，如果配置了对接审计中心，会被审计中心通过采集器进行采集； +gc.log*：JVM垃圾回收日志，出现JVM级别的性能问题时才需关注。 + +**日志分析方法：** +作业平台的绝大多数服务日志中都包含traceId与spanId，其在一行日志中的位置位于日志级别后，以下通过示例进行说明： +```shell +[2024-07-19 19:00:00.001] INFO [job-execute,84a2e275a062728b9b54aab16dea4289,4f786ec86329458a] 6 --- [execute-scheduler-3] e.e.r.h.ResultHandleTaskKeepaliveManager : Refresh task keepalive info start... +``` +在上述日志中，traceId为84a2e275a062728b9b54aab16dea4289，spanId为4f786ec86329458a，通过traceId可以过滤出一次请求产生的所有日志（支持跨服务，但网关日志除外），通过spanId可以过滤出一个单独的处理阶段的日志，为了快速排查问题，可以先通过业务特征信息（例如taskInstanceId、stepInstanceId等）找出对应的核心流程的traceId，再用traceId过滤出核心流程所有日志进行分析，建议配置日志采集、清洗并结合日志平台高效检索日志。 +在作业平台的接口（Web、Service、ESB接口等）响应中traceId会作为单独的字段（Web与Service接口为requestId，ESB接口为job_request_id）返回，因此可从接口响应中拿到traceId用于检索日志。 diff --git a/docs/overview/operation_en.md b/docs/overview/operation_en.md new file mode 100644 index 0000000000..a598ca9f9d --- /dev/null +++ b/docs/overview/operation_en.md @@ -0,0 +1,81 @@ +# BlueKing Job Platform Deployment and Maintenance Documentation + +English | [简体中文](operation.md) + +## I. Deployment +The BlueKing Job Platform (Job) is one of the BlueKing atomic platforms. Its underlying dependencies include: + +### 1. Strong Dependencies +The following systems are strong dependencies of Job. If they are missing or malfunctioning, the core functionalities of Job will be unavailable: +**PaaS**: Job calls its interfaces to integrate with the unified login service; +**User Management**: Job calls its interfaces to obtain user information; +**Permission Center**: Job calls its interfaces to get permission policies for user operation authentication; +**APIGateway**: Job calls underlying platform interfaces through APIGateway and provides interfaces for other upper-level platforms; +**CMDB**: Job calls its interfaces to obtain business, host, and other resource data; +**GSE**: Job calls its interfaces to distribute script execution, file distribution tasks, and obtain task execution results and logs. + +### 2. Weak Dependencies +The following systems are weak dependencies of Job. If they are missing or malfunctioning, some functionalities of Job will be unavailable: +**BlueKing Artifact Repository**: Job calls its interfaces to store local files, import/export, and temporary file data generated by execution log export; +**Audit Center**: Job outputs audit data log files for collection, used for auditing user operations; +**Log Platform**: Job's backend log files will be collected by its collector, facilitating unified search on the log platform; +**Monitoring Platform**: Job exposes metric data through the Prometheus protocol, which will be collected by its collector, facilitating the configuration of metric panels and alerts on the monitoring platform. Additionally, Job calls its interfaces through the OTEL protocol to report call chain APM data, facilitating call chain observation and analysis on the monitoring platform; +**Message Notification Center**: Job calls its interfaces to obtain notification and announcement information and displays them on the page. + +Due to the numerous dependencies, deploying Job alone will not work properly. It is necessary to first deploy its underlying platforms. Please refer to the basic package content in the deployment and maintenance section of the [BlueKing Documentation Center](https://bk.tencent.com/docs/) for deployment. + +## II. Maintenance +### 1. Resource Usage Management +After successfully deploying Job, as the system runs longer, the usage data increases, leading to increased resource usage by Job. For cost and performance considerations, resource usage needs to be managed, and some expired data should be cleaned up as needed. + +#### (1) MySQL Data +Different functional modules of Job use different databases to store data. The job_execute database used by the task execution engine needs special attention. This database stores the flow data generated during task execution, and the data volume increases with the number of tasks. Usually, after the platform runs for a period, the job_execute database will become the largest storage space consumer, requiring data archiving and cleaning. +Job supports an automatic archiving and cleaning mechanism for task flow data, which can be configured through the backupConfig.archive properties in the helm chart values. The default archiving policy is: do not enable archiving, permanently retain all data. If there is a need to clean up expired data, the policy can be configured to delete data directly after the expiration time (default is 30 days) without backup. If there is a need to back up expired data, the policy can be configured to back up data before deletion. + +#### (2) MongoDB Data +Job uses MongoDB to store business script execution logs and file distribution task execution logs generated during task execution, which are stored in the joblog database by default (configurable through related properties in the helm chart values). The log data volume increases with the number of tasks. Usually, after the platform runs for a period, the joblog database will become the largest storage space consumer, requiring data archiving and cleaning. +Job organizes and stores log data by task step type (script/file) and date. Script task logs are stored in the job_log_script_{date} collection, and file task logs are stored in the job_log_file_{date} collection, where {date} is the task creation date, such as job_log_script_2024_07_19, job_log_file_2024_07_20. +Currently, Job does not provide an automatic archiving and cleaning mechanism. Users need to manually execute commands or write scripts to export/delete the collections that need to be processed. + +#### (3) BlueKing Artifact Repository Data +Job uses the BlueKing Artifact Repository to store local files uploaded by users, import/export, and temporary files generated during execution log export. The default BlueKing Artifact Repository project used is bkjob (configurable through related properties in the helm chart values), and the repositories used are localupload (local files), backup (import/export temporary files), and filedata (execution log export temporary files). Files in the localupload repository will be permanently retained if referenced by the job/execution plan's local file distribution step. Unreferenced files will be automatically cleaned up after the expiration time, which is 7 days by default (configurable through related properties in the helm chart values). Files in the backup and filedata repositories are temporary files and can be cleaned up as needed if not automatically cleaned by the system. + +#### (4) Other Data +**Backend Logs:** Backend logs generated during system operation are printed in files within the container by default and will be automatically cleaned up after exceeding the limited capacity or expiration time. Usually, no manual handling is required. The capacity and expiration time can be configured through the log properties in the helm chart values. +**Log Platform Data:** If log collection is configured (properties under bkLogConfig in the helm chart values), the log platform will collect backend logs through the collector and store them in its backend storage components for search. This data needs to be configured for cleanup on the log platform. +**Monitoring Metric Data:** If monitoring data collection is configured (properties under serviceMonitor in the helm chart values), the monitoring platform will collect backend service metric data through the collector for viewing on the monitoring platform via dashboards or configuring alerts. This data needs to be configured for cleanup on the monitoring platform. +**APM Data:** If APM data collection is configured (properties under job.trace in the helm chart values), Job will use the OTEL protocol to actively report backend service call chain data to the monitoring platform for visual analysis of the call chain. This data needs to be configured for cleanup on the monitoring platform. + +### 2. System Exception Troubleshooting and Recovery +(1) Service Status Check +After successfully deploying Job, system administrators can check the status of each service module and instance on the "Platform Management - Service Status" subpage of the web page. Generally, the status of all instances should be "Normal". If not, further troubleshooting of the corresponding instance is required. + +(2) Pod Status Check +In the namespace where Job is deployed, you can use the `kubectl get pod` command combined with the "bk-job" keyword to grep out all Pods of Job. Normally, the status of all Pods of Job should be Running/Completed. If not, further troubleshooting of the service logs within the corresponding Pod or attempting to restart the Pod is required. + +(3) Service Log Analysis +In a containerized deployment, you can enter the containers within each service Pod to view service logs. The log storage location for each service of Job is: /data/logs/{service_name}/. For example, the log location for the execution engine is /data/logs/job-execute/. The logs in the directory are divided into multiple files by category and will roll over by time. The purposes of various log files are introduced below: +error.log: Summary of error logs, which will also be present in other regular log files and need special attention; +{service_name}.log: Main service log, such as execute.log, which is the core business logic log of the service and needs special attention; +openapi.log: API (ESB, APIGW, etc.) call logs provided by Job; +schedule.log: Task scheduling logs of the execution engine; +monitor_task.log: Monitoring logs for large tasks (involving a large number of hosts); +sync_app_host.log: Monitoring logs for synchronization tasks and events of business (sets), hosts, and other CMDB resources; +access.log: Gateway interface access logs; +esb_access.log: Gateway ESB interface access logs; +cmdb.log: Logs for calls to CMDB related interfaces; +gse.log: Logs for calls to GSE related interfaces; +iam.log: Logs for calls to Permission Center (IAM) related interfaces; +paas.log: Logs for calls to PaaS platform related interfaces (login, user, etc.); +notice.log: Logs for calls to Message Notification Center related interfaces (notifications, announcements, etc.); +audit_event.log: Audit logs, which will be collected by the Audit Center through its collector if configured; +gc.log*: JVM garbage collection logs, which need attention only when JVM-level performance issues occur. + +**Log Analysis Method:** +Most service logs of Job contain traceId and spanId, located after the log level in a log line. Here is an example for illustration: +```shell +[2024-07-19 19:00:00.001] INFO [job-execute,84a2e275a062728b9b54aab16dea4289,4f786ec86329458a] 6 --- [execute-scheduler-3] e.e.r.h.ResultHandleTaskKeepaliveManager : Refresh task keepalive info start... +``` +In the above log, traceId is 84a2e275a062728b9b54aab16dea4289, and spanId is 4f786ec86329458a. Using traceId, you can filter out all logs generated by a single request (supporting cross-service, but excluding gateway logs). Using spanId, you can filter out logs for a single processing stage. To quickly troubleshoot issues, you can first find the traceId of the corresponding core process through business characteristic information (such as taskInstanceId, stepInstanceId, etc.), then use traceId to filter out all logs of the core process for analysis. It is recommended to configure log collection, cleaning, and use the log platform for efficient log retrieval. + +In Job's interface responses (Web, Service, ESB interfaces, etc.), traceId will be returned as a separate field (requestId for Web and Service interfaces, job_request_id for ESB interfaces). Therefore, you can obtain traceId from the interface response for log retrieval. diff --git a/docs/overview/trouble_shooting.md b/docs/overview/trouble_shooting.md new file mode 100644 index 0000000000..34e2c6ebb3 --- /dev/null +++ b/docs/overview/trouble_shooting.md @@ -0,0 +1,6 @@ +# 蓝鲸作业平台问题排查与处理文档 + +### 一、运维操作相关 +#### 1.如何批量触发某个动作（例如：强制终止一批异常作业，或者批量关闭/开启定时任务）？ +答：可以使用[蓝鲸作业平台代码库](https://github.com/TencentBlueKing/bk-job/tree/master/op-tools)下op-tools目录下的诊断运维工具集进行操作，具体使用方法请参考对应的README文档。 + diff --git a/docs/overview/trouble_shooting_en.md b/docs/overview/trouble_shooting_en.md new file mode 100644 index 0000000000..52c1dc634b --- /dev/null +++ b/docs/overview/trouble_shooting_en.md @@ -0,0 +1,6 @@ +# BlueKing Job Platform Troubleshooting Documentation + +### I. Operations and Maintenance +#### 1. How to Batch Trigger an Action (e.g., Force Terminate a Batch of Abnormal Jobs, or Batch Close/Open Scheduled Tasks)? +Answer: You can use the diagnostic and maintenance toolset under the `op-tools` directory in the [BlueKing Job Platform Code Repository](https://github.com/TencentBlueKing/bk-job/tree/master/op-tools) to perform these operations. For specific usage instructions, please refer to the corresponding README documentation. + diff --git a/docs/resource/img/ai_data_architecture.png b/docs/resource/img/ai_data_architecture.png new file mode 100644 index 0000000000..bc3504c0cc Binary files /dev/null and b/docs/resource/img/ai_data_architecture.png differ diff --git a/docs/resource/img/ai_data_route.png b/docs/resource/img/ai_data_route.png new file mode 100644 index 0000000000..04d3524698 Binary files /dev/null and b/docs/resource/img/ai_data_route.png differ diff --git a/docs/resource/img/architecture.png b/docs/resource/img/architecture.png index 59f226ac04..1f1361fe06 100644 Binary files a/docs/resource/img/architecture.png and b/docs/resource/img/architecture.png differ diff --git a/docs/resource/img/architecture_en.png b/docs/resource/img/architecture_en.png new file mode 100644 index 0000000000..e28b28f3cd Binary files /dev/null and b/docs/resource/img/architecture_en.png differ diff --git a/docs/resource/img/bk_architecture_cn.png b/docs/resource/img/bk_architecture_cn.png new file mode 100644 index 0000000000..2caa69f327 Binary files /dev/null and b/docs/resource/img/bk_architecture_cn.png differ diff --git a/docs/resource/img/bk_architecture_en.png b/docs/resource/img/bk_architecture_en.png new file mode 100644 index 0000000000..520da933eb Binary files /dev/null and b/docs/resource/img/bk_architecture_en.png differ diff --git a/docs/resource/img/job_execute_architecture_en.png b/docs/resource/img/job_execute_architecture_en.png new file mode 100644 index 0000000000..9cbf476d88 Binary files /dev/null and b/docs/resource/img/job_execute_architecture_en.png differ diff --git a/docs/resource/img/job_execute_engine_architecture.png b/docs/resource/img/job_execute_engine_architecture.png new file mode 100644 index 0000000000..3197d3210a Binary files /dev/null and b/docs/resource/img/job_execute_engine_architecture.png differ diff --git a/genProjectsYaml.py b/genProjectsYaml.py index 328997bba5..2b26fbbf5c 100644 --- a/genProjectsYaml.py +++ b/genProjectsYaml.py @@ -2,7 +2,7 @@ """ Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. -Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +Copyright (C) 2021 Tencent. All rights reserved. BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. diff --git a/op-tools/batch-invoke/run.py b/op-tools/batch-invoke/run.py index d361d56475..606038df74 100644 --- a/op-tools/batch-invoke/run.py +++ b/op-tools/batch-invoke/run.py @@ -2,7 +2,7 @@ """ Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. -Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +Copyright (C) 2021 Tencent. All rights reserved. BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. diff --git a/package.sh b/package.sh old mode 100644 new mode 100755 index 8c83efda31..9aa5a8ce96 --- a/package.sh +++ b/package.sh @@ -1,9 +1,9 @@ #!/bin/bash set -e echo 'Begin to package job' -BACKEND_MODULES=(job-config job-crontab job-execute job-gateway job-logsvr job-manage job-backup job-file-gateway job-file-worker job-analysis) +BACKEND_MODULES=(job-config job-crontab job-execute job-gateway job-logsvr job-manage job-backup job-file-gateway job-file-worker job-analysis job-assemble) FRONTEND_MODULES=(job-frontend) -ALL_MODULES=(job-config job-crontab job-execute job-gateway job-logsvr job-manage job-backup job-file-gateway job-file-worker job-analysis job-frontend) +ALL_MODULES=(job-config job-crontab job-execute job-gateway job-logsvr job-manage job-backup job-file-gateway job-file-worker job-analysis job-frontend job-assemble) JOB_EDITION=ce if [[ ! -d "release" ]]; then @@ -16,7 +16,7 @@ usage () { cat < 0 )); do esac shift done - +echo ${JOB_MODULES} JOB_MODULES=${JOB_MODULES,,} # to lower case # 判断参数 if [[ -z $JOB_MODULES ]] || ! [[ $JOB_MODULES =~ ^[A-Za-z,-]+$ ]]; then @@ -179,6 +179,24 @@ if [[ -d "support-files/templates" ]]; then continue fi simpleName=${m:4} + # job-assemble 配置文件单独处理 + if [[ "${m}" == "job-assemble" ]]; then + moduleConfigFilePath="support-files/templates/#etc#job#job-${simpleName}#application-${simpleName}.yml" + if [[ -f "${moduleConfigFilePath}" ]]; then + cp "${moduleConfigFilePath}" release/job/support-files/templates + else + echo "cannot find yml template of #etc#job#job-assemble#application-assemble.yml" + exit 1 + fi + moduleConfigFilePath="support-files/templates/#etc#job#job-assemble#application-gateway.yml" + if [[ -f "${moduleConfigFilePath}" ]]; then + cp "${moduleConfigFilePath}" release/job/support-files/templates + else + echo "cannot find yml template of #etc#job#job-assemble#application-gateway.yml" + exit 1 + fi + continue + fi # Copy yml templates moduleConfigFilePath="support-files/templates/#etc#job#job-${simpleName}#job-${simpleName}.yml" if [[ -f "${moduleConfigFilePath}" ]]; then @@ -189,7 +207,7 @@ if [[ -d "support-files/templates" ]]; then exit 1 fi fi - # Copy application-{module}.yml templates + # Copy application-{module}.yml templates moduleConfigFilePath="support-files/templates/#etc#job#application-${simpleName}.yml" if [[ -f "${moduleConfigFilePath}" ]]; then cp "${moduleConfigFilePath}" release/job/support-files/templates @@ -218,7 +236,12 @@ if [[ -d "support-files/templates" ]]; then else echo "warn: cannot find ${jobEnvFile}, ignore" fi + + # Copy deploy.yml/deploy_assemble.yml + cp "support-files/templates/#job#deploy.yml" release/job/support-files/templates + cp "support-files/templates/#job#deploy_lite.yml" release/job/support-files/templates fi + # readme.md、requirements.txt for fileName in "readme.md" "requirements.txt"; do diff --git a/package_modules.yml b/package_modules.yml new file mode 100644 index 0000000000..0e16abffa4 --- /dev/null +++ b/package_modules.yml @@ -0,0 +1,96 @@ +modules: + - name: job-frontend + module: job + # 打包方式：binary - 二进制部署包，k8s: 容器化部署包 + package: + - binary + - k8s + description: "Job Web Frontend" + + - name: job-config + module: job + package: + - binary + description: "Job Configuration Center" + + - name: job-gateway + module: job + package: + - binary + - k8s + description: "Job Microservice Gateway" + + - name: job-manage + module: job + package: + - binary + - k8s + description: "Job Management Microservice" + + - name: job-execute + module: job + package: + - binary + - k8s + description: "Job Execution Microservice" + + - name: job-crontab + module: job + package: + - binary + - k8s + description: "Job Cron Task Microservice" + + - name: job-logsvr + module: job + package: + - binary + - k8s + description: "Job Execution Log Microservice" + + - name: job-file-gateway + module: job + package: + - binary + - k8s + description: "Job File Source Gateway Microservice" + + - name: job-file-worker + module: job + package: + - binary + - k8s + description: "Job File Source worker" + + - name: job-analysis + module: job + package: + - binary + - k8s + description: "Job Statistical Analysis Microservice" + + - name: job-backup + module: job + package: + - binary + - k8s + description: "Job Backup Microservice" + + - name: job-migration + module: job + package: + - k8s + description: "Job Migration Kubernetes Job" + + - name: startup-controller + module: job + package: + - k8s + description: "Job Kubernetes Mocroservice startup controller" + + - name: job-assemble + module: job + package: + - binary + - k8s + description: "Job Assembled Microservice" diff --git a/projects.yaml b/projects.yaml index c41699f27b..dc8ef6b52f 100644 --- a/projects.yaml +++ b/projects.yaml @@ -54,18 +54,10 @@ version: 1.0.0 version_type: ee -- name: job-ticket - module: job - project_dir: job/job-ticket - alias: job-ticket - language: java/1.8 - version: 1.0.0 - version_type: ee - - name: job-analysis module: job project_dir: job/job-analysis alias: job-analysis language: java/1.8 version: 1.0.0 - version_type: ee + version_type: ee \ No newline at end of file diff --git a/scripts/job-assemble/job-assemble.sh b/scripts/job-assemble/job-assemble.sh new file mode 100644 index 0000000000..819e177785 --- /dev/null +++ b/scripts/job-assemble/job-assemble.sh @@ -0,0 +1,186 @@ +#!/bin/bash +PROJECT_NAME=job +SERVICE_NAME=job-assemble + +SELF_DIR=$(readlink -f "$(dirname $0)") +SERVICE_HOME=$(cd $SELF_DIR/.. && pwd ) +if [[ -z "${BK_HOME}" ]]; then + BK_HOME=$(cd $SELF_DIR/../../../../ && pwd) +fi +JOB_HOME=${BK_HOME}/${PROJECT_NAME} +if [[ -z "${JOB_CONF_HOME}" ]]; then + JOB_CONF_HOME=${BK_HOME}/etc/${PROJECT_NAME} +fi +if [[ -z "${LOGS_HOME}" ]]; then + LOGS_HOME=${BK_HOME}/logs/${PROJECT_NAME}/${SERVICE_NAME}/ +fi + +SHELL_FILE_NAME=${0##*/} +JAR_FILE=${SERVICE_HOME}/${SERVICE_NAME}.jar + + +if [[ ! -n "${JAVA_HOME}" ]]||[[ ! -d "${JAVA_HOME}" ]]; then + JAVA_HOME="${BK_HOME}/service/java" + if [[ ! -d "${JAVA_HOME}" ]]; then + JAVA_HOME="${BK_HOME}/common/java" + fi + export JAVA_HOME + export PATH=${JAVA_HOME}/bin:$PATH +fi + +# Java process pid +PID=0 +# Spring profile param +SPRING_PROFILE="$2" + +# Get current java process pid +function getPID(){ + javaps=`${JAVA_HOME}/bin/jps -l | grep "${JAR_FILE}"` + if [[ -n "$javaps" ]]; then + PID=`echo ${javaps} | awk '{print $1}'` + else + PID=0 + fi +} + + + +function startup() { + JAVA_OPTS="$JAVA_OPTS -server -Dfile.encoding=UTF-8" + JAVA_OPTS="$JAVA_OPTS -Dspring.config.additional-location=file://${JOB_CONF_HOME}/job-assemble/application-job-assemble.yml" + + if [ -n "${SPRING_PROFILE}" ];then + JAVA_OPTS="$JAVA_OPTS -Dspring.profiles.active=${SPRING_PROFILE}" + else + JAVA_OPTS="$JAVA_OPTS -Dspring.profiles.active=prod" + fi + + getPID + if [[ ${PID} -ne 0 ]]; then + echo "${PROJECT_NAME}:${SERVICE_NAME} already started(PID=$PID)" + else + echo -n "Starting ${PROJECT_NAME}:${SERVICE_NAME}" + if [[ ! -d "${LOGS_HOME}" ]]; then + mkdir -p "${LOGS_HOME}" + fi + + NOHUPLOG=/dev/null + #NOHUPLOG=${PROJECT_NAME}_${SERVICE_NAME}_console.log + nohup ${JAVA_HOME}/bin/java ${JAVA_OPTS} -jar ${JAR_FILE} > ${NOHUPLOG} 2>&1 & + + # Process listening port status, 0:not listen, 1:listen + HTTP_PORT_STATUS=0 + + for i in $(seq 60) + do + sleep 0.5 + echo -e ".\c" + getPID + if [[ ${PID} -ne 0 ]]; then + checkPortStatus "${PID}" + HTTP_PORT_STATUS=$? + if [[ ${HTTP_PORT_STATUS} -gt 0 ]]; then + break; + fi + fi + done + + if [[ ${HTTP_PORT_STATUS} -eq 0 ]]; then + echo "[Failed]-- http port ${HTTP_PORT} start fail!" + exit 1 + fi + + echo "(PID=$PID)...[Success]" + + fi +} + +function checkPortStatus() { + # Process listening port status, 0:not listen, 1:listen + HTTP_PORT_STATUS=0 + # $1 is pid + PID_TMP="$1" + PID_PNAME="${PID_TMP}/java" + + LISTEN_PID_COUNT=`netstat -atnp|grep "${PID_PNAME}"|wc -l` + if [[ ${LISTEN_PID_COUNT} -ne 0 ]]; then + HTTP_PORT_STATUS=1 + else + HTTP_PORT_STATUS=0 + fi + return $HTTP_PORT_STATUS +} + +function shutdown(){ + getPID + if [[ ${PID} -ne 0 ]]; then + echo -n "Stopping ${PROJECT_NAME}:${SERVICE_NAME}(PID=${PID})..." + kill ${PID} + if [[ $? -ne 0 ]]; then + echo "[Failed]" + exit 1 + fi + for i in $(seq 20) + do + sleep 0.5 + getPID + if [[ ${PID} -eq 0 ]]; then + + checkPortStatus "${PID}" + HTTP_PORT_STATUS=$? + + if [[ ${HTTP_PORT_STATUS} -eq 0 ]]; then + break + fi + fi + echo -e ".\c" + done + getPID + if [[ ${PID} -eq 0 ]]; then + echo "[Success]" + else + kill -9 ${PID} + if [[ $? -ne 0 ]]; then + echo "[Failed]" + exit 1 + fi + echo "some task is running in background,force stop it.[Success]" + fi + else + echo "${PROJECT_NAME}:${SERVICE_NAME} is not running" + fi +} + +function getServerStatus(){ + getPID + if [[ ${PID} -ne 0 ]]; then + checkPortStatus "${PID}" + HTTP_PORT_STATUS=$? + if [[ ${HTTP_PORT_STATUS} -eq 0 ]]; then + echo "${PROJECT_NAME}:${SERVICE_NAME} port ${HTTP_PORT} is not listening(PID=${PID})"; + exit 99; + fi + echo "${PROJECT_NAME}:${SERVICE_NAME} is running(PID=${PID})" + else + echo "${PROJECT_NAME}:${SERVICE_NAME} is not running" + fi +} + +function restart(){ + shutdown + sleep 1 + startup +} + +case "$1" in +restart) + restart;; +start) + startup;; +stop) + shutdown;; +status) + getServerStatus;; +*) +echo $"Usage: ${SHELL_FILE_NAME} {start|stop|status|restart}" +esac diff --git a/scripts/job-gateway/job-gateway_assemble.sh b/scripts/job-gateway/job-gateway_assemble.sh new file mode 100644 index 0000000000..6698681d51 --- /dev/null +++ b/scripts/job-gateway/job-gateway_assemble.sh @@ -0,0 +1,188 @@ +#!/bin/bash +PROJECT_NAME=job +SERVICE_NAME=job-gateway-assemble + +SELF_DIR=$(readlink -f "$(dirname $0)") +SERVICE_HOME=$(cd $SELF_DIR/.. && pwd ) +if [[ -z "${BK_HOME}" ]]; then + BK_HOME=$(cd $SELF_DIR/../../../../ && pwd) +fi +JOB_HOME=${BK_HOME}/${PROJECT_NAME} +if [[ -z "${JOB_CONF_HOME}" ]]; then + JOB_CONF_HOME=${BK_HOME}/etc/${PROJECT_NAME} +fi +if [[ -z "${LOGS_HOME}" ]]; then + LOGS_HOME=${BK_HOME}/logs/${PROJECT_NAME}/${SERVICE_NAME}/ +fi + +SHELL_FILE_NAME=${0##*/} +JAR_FILE=${SERVICE_HOME}/job-gateway.jar + + +if [[ ! -n "${JAVA_HOME}" ]]||[[ ! -d "${JAVA_HOME}" ]]; then + JAVA_HOME="${BK_HOME}/service/java" + if [[ ! -d "${JAVA_HOME}" ]]; then + JAVA_HOME="${BK_HOME}/common/java" + fi + export JAVA_HOME + export PATH=${JAVA_HOME}/bin:$PATH +fi + +# Java process pid +PID=0 +# Spring profile param +SPRING_PROFILE="$2" + +# Get current java process pid +function getPID(){ + javaps=`${JAVA_HOME}/bin/jps -l | grep "${JAR_FILE}"` + if [[ -n "$javaps" ]]; then + PID=`echo ${javaps} | awk '{print $1}'` + else + PID=0 + fi +} + + + +function startup() { + JAVA_OPTS="$JAVA_OPTS -server -Dfile.encoding=UTF-8" + JAVA_OPTS="$JAVA_OPTS -Dspring.config.additional-location=file://${JOB_CONF_HOME}/job-assemble/application-gateway-assemble.yml" + JAVA_OPTS="$JAVA_OPTS -Dreactor.netty.http.server.accessLogEnabled=true" + JAVA_OPTS="$JAVA_OPTS -DAPP_NAME=job-gateway-assemble" + + if [ -n "${SPRING_PROFILE}" ];then + JAVA_OPTS="$JAVA_OPTS -Dspring.profiles.active=${SPRING_PROFILE}" + else + JAVA_OPTS="$JAVA_OPTS -Dspring.profiles.active=prod" + fi + + getPID + if [[ ${PID} -ne 0 ]]; then + echo "${PROJECT_NAME}:${SERVICE_NAME} already started(PID=$PID)" + else + echo -n "Starting ${PROJECT_NAME}:${SERVICE_NAME}" + if [[ ! -d "${LOGS_HOME}" ]]; then + mkdir -p "${LOGS_HOME}" + fi + + NOHUPLOG=/dev/null + #NOHUPLOG=${PROJECT_NAME}_${SERVICE_NAME}_console.log + nohup ${JAVA_HOME}/bin/java ${JAVA_OPTS} -jar ${JAR_FILE} > ${NOHUPLOG} 2>&1 & + + # Process listening port status, 0:not listen, 1:listen + HTTP_PORT_STATUS=0 + + for i in $(seq 60) + do + sleep 0.5 + echo -e ".\c" + getPID + if [[ ${PID} -ne 0 ]]; then + checkPortStatus "${PID}" + HTTP_PORT_STATUS=$? + if [[ ${HTTP_PORT_STATUS} -gt 0 ]]; then + break; + fi + fi + done + + if [[ ${HTTP_PORT_STATUS} -eq 0 ]]; then + echo "[Failed]-- http port ${HTTP_PORT} start fail!" + exit 1 + fi + + echo "(PID=$PID)...[Success]" + + fi +} + +function checkPortStatus() { + # Process listening port status, 0:not listen, 1:listen + HTTP_PORT_STATUS=0 + # $1 is pid + PID_TMP="$1" + PID_PNAME="${PID_TMP}/java" + + LISTEN_PID_COUNT=`netstat -atnp|grep "${PID_PNAME}"|wc -l` + if [[ ${LISTEN_PID_COUNT} -ne 0 ]]; then + HTTP_PORT_STATUS=1 + else + HTTP_PORT_STATUS=0 + fi + return $HTTP_PORT_STATUS +} + +function shutdown(){ + getPID + if [[ ${PID} -ne 0 ]]; then + echo -n "Stopping ${PROJECT_NAME}:${SERVICE_NAME}(PID=${PID})..." + kill ${PID} + if [[ $? -ne 0 ]]; then + echo "[Failed]" + exit 1 + fi + for i in $(seq 20) + do + sleep 0.5 + getPID + if [[ ${PID} -eq 0 ]]; then + + checkPortStatus "${PID}" + HTTP_PORT_STATUS=$? + + if [[ ${HTTP_PORT_STATUS} -eq 0 ]]; then + break + fi + fi + echo -e ".\c" + done + getPID + if [[ ${PID} -eq 0 ]]; then + echo "[Success]" + else + kill -9 ${PID} + if [[ $? -ne 0 ]]; then + echo "[Failed]" + exit 1 + fi + echo "some task is running in background,force stop it.[Success]" + fi + else + echo "${PROJECT_NAME}:${SERVICE_NAME} is not running" + fi +} + +function getServerStatus(){ + getPID + if [[ ${PID} -ne 0 ]]; then + checkPortStatus "${PID}" + HTTP_PORT_STATUS=$? + if [[ ${HTTP_PORT_STATUS} -eq 0 ]]; then + echo "${PROJECT_NAME}:${SERVICE_NAME} port ${HTTP_PORT} is not listening(PID=${PID})"; + exit 99; + fi + echo "${PROJECT_NAME}:${SERVICE_NAME} is running(PID=${PID})" + else + echo "${PROJECT_NAME}:${SERVICE_NAME} is not running" + fi +} + +function restart(){ + shutdown + sleep 1 + startup +} + +case "$1" in +restart) + restart;; +start) + startup;; +stop) + shutdown;; +status) + getServerStatus;; +*) +echo $"Usage: ${SHELL_FILE_NAME} {start|stop|status|restart}" +esac diff --git a/src/backend/build.gradle b/src/backend/build.gradle index b3bab50370..4330a10891 100644 --- a/src/backend/build.gradle +++ b/src/backend/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,104 +27,158 @@ import com.dorongold.gradle.tasktree.TaskTreePlugin buildscript { ext { set('springDependencyManagePluginVersion', "1.0.11.RELEASE") - set("springBootVersion", "2.4.6") + set("springBootVersion", "2.7.11") set("gradleJooqVersion", "3.0.0") } - def mavenRepoUrl = System.getProperty("mavenRepoUrl") - - if (mavenRepoUrl == null) { - mavenRepoUrl = System.getenv("mavenRepoUrl") - } - - if (mavenRepoUrl == null) { - println "use default maven repo url." - mavenRepoUrl = MAVEN_REPO_URL - } repositories { mavenLocal() - maven { url mavenRepoUrl } + + // 用户通过命令行方式指定的 maven repo + def mavenRepoUrls = System.getProperty("mavenRepoUrl") + if (mavenRepoUrls == null || mavenRepoUrls.trim().length() == 0) { + // gradle.properties 中的 maven repo。优先级从高到低: + // 1. command line, as set using -D. + // 2. gradle.properties in GRADLE_USER_HOME directory (/.gradle by default) + // 3. gradle.properties in the project’s directory, then its parent project’s directory up to the build’s root directory. + // 4. gradle.properties in Gradle installation directory. + mavenRepoUrls = project.findProperty("MAVEN_REPO_URL") + } + if (mavenRepoUrls != null && mavenRepoUrls.trim().length() > 0) { + String[] repoUrls = mavenRepoUrls.trim().replace(" ", "").split(",") + for (String repoUrl : repoUrls) { + println("Add maven repo:" + repoUrl) + maven { url repoUrl } + } + } + maven { url "https://plugins.gradle.org/m2/" } mavenCentral() + maven { url "https://oss.sonatype.org/content/repositories/snapshots" } } dependencies { classpath "io.spring.gradle:dependency-management-plugin:$springDependencyManagePluginVersion" classpath "nu.studer:gradle-jooq-plugin:$gradleJooqVersion" classpath 'org.junit.platform:junit-platform-gradle-plugin:1.1.0' classpath "gradle.plugin.com.dorongold.plugins:task-tree:1.5" + classpath "com.vanniktech:gradle-dependency-graph-generator-plugin:0.8.0" } } plugins { id "java-library" id "io.spring.dependency-management" version '1.0.11.RELEASE' apply false - id 'org.springframework.boot' version '2.4.6' apply false + id 'org.springframework.boot' version '2.7.11' apply false id "idea" id 'nu.studer.jooq' version '3.0.0' } ext { - // https://mvnrepository.com/artifact/org.springframework/spring-jdbc - set("springVersion", "5.3.9") + // 是否容器化部署 + def assemblyMode = System.getProperty("assemblyMode") + def k8sMode = assemblyMode == "k8s" + set("k8sMode", k8sMode) + println("assembly mode: $assemblyMode") + + set("springVersion", "5.3.25") // https://mvnrepository.com/artifact/org.springframework.boot/spring-boot-dependencies - set("springBootVersion", "2.4.6") + set("springBootVersion", "2.7.11") // https://mvnrepository.com/artifact/org.springframework.cloud/spring-cloud-dependencies - set('springCloudVersion', "2020.0.3") - set('springCloudOtelVersion', "1.0.0-M13") - // https://mvnrepository.com/artifact/io.springfox/springfox-swagger-ui - set('swaggerVersion', "2.9.2") + set('springCloudVersion', "2021.0.5") + set('springCloudOtelVersion', "1.1.3") + set('swaggerVersion', "3.0.0") set('junitVersion', "5.5.2") // https://mvnrepository.com/artifact/org.projectlombok/lombok - set('lombokVersion', "1.18.10") + set('lombokVersion', "1.18.24") // https://mvnrepository.com/artifact/com.fasterxml.jackson.core/jackson-core - set('jacksonVersion', "2.11.0") + set('jacksonVersion', "2.15.4") set('jaxrsVersion', "2.0") // https://mvnrepository.com/artifact/ch.qos.logback/logback-core - set('logbackVersion', "1.2.10") + // Fix CVE-2023-6378 1.2.11->1.2.13 + set('logbackVersion', "1.2.13") // https://mvnrepository.com/artifact/org.slf4j/slf4j-api set('slf4jVersion', "1.7.30") set('servletVersion', "3.0.1") + // apache commons // https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 - set('apacheCommonsVersion', "3.9") + set('apacheCommonsLang3Version', "3.12.0") // https://mvnrepository.com/artifact/org.apache.commons/commons-collections4 set('apacheCommonsCollectionVersion', "4.4") // https://mvnrepository.com/artifact/org.apache.commons/commons-pool2 - set('apacheCommonsPoolVersion', "2.8.0") + set('apacheCommonsPoolVersion', "2.11.1") set('apacheCommonsCodecVersion', "1.14") - set('dozerVersion', "5.5.1") + set('apacheCuratorFrameworkVersion', "5.4.0") + set('apacheCuratorRecipesVersion', "5.4.0") set('commonsCollectionsVersion', "3.2.2") + set('dozerVersion', "5.5.1") set('jedisVersion', "3.0.1") // https://mvnrepository.com/artifact/org.jooq/jooq set('jooqVersion', "3.14.8") set('h2Version', "1.4.199") set('gradleJooqVersion', "2.0.11") // https://mvnrepository.com/artifact/org.apache.httpcomponents/httpclient - set('apacheHttpClientVersion', "4.5.10") + set('apacheHttpClientVersion', "4.5.13") set('apacheThriftVersion', "0.15.0") - set('commonsIOVersion', "2.6") + // Fix CVE-2024-47554 CVE-2021-29425 2.6->2.14.0 + set('commonsIOVersion', "2.14.0") set('javaxServletVersion', "4.0.1") - set('guavaVersion', "28.2-jre") + // Fix CVE-2023-2976 CVE-2020-8908 28.2-jre->32.0.0-android + set('guavaVersion', "32.0.0-android") + set('caffeineVersion', "2.9.3") set('jjwtVersion', "0.9.1") set('hibernateValidatorVersion', "6.1.4.Final") - set('micrometerPrometheusVersion', "1.5.1") - set('flapdoodleEmbeddedMongdbDBVersion', "2.2.0") + set('flapdoodleEmbeddedMongdbDBVersion', "4.4.0") set('jodaTimeVersion', "2.10.5") - set('bcprovVersion', "1.64") - set('reflectionsVersion', "0.9.12") + set('bcprovVersion', "1.70") + set('reflectionsVersion', "0.10.2") set('cronUtilsVersion', "9.1.6") - set('otelJdbcVersion', "1.9.2-alpha") + set('otelJdbcVersion', "1.24.0-alpha") set('commonsValidatorVersion', "1.6") - set('jcommanderVersion', "1.71") + set('okHttpVersion', "4.12.0") + set('jcommanderVersion', "1.72") + set('kubernetesJavaClientVersion', "11.0.4") + set('springCloudKubernetesVersion', "2.0.6") + set('cryptoJavaSDKVersion', "1.1.3") + // Fix CVE-2019-10086,CVE-2014-0114 + set('commonsBeanutilsVersion', "1.9.4") if (System.getProperty("bkjobVersion")) { set('bkjobVersion', System.getProperty("bkjobVersion")) - println "bkjobVersoin:" + bkjobVersion + println "bkjobVersion:" + bkjobVersion } else { set('bkjobVersion', "1.0.0") - println "bkjobVersoin:" + bkjobVersion + println "bkjobVersion:" + bkjobVersion } - // Fix Spring Cloud Function Spel表达式注入漏洞(CVE-2022-22963) - // Fix Spring Cloud Function 拒绝服务漏洞(CVE-2022-22979) - set('spring-cloud-function.version', "3.2.8") + set('bkAuditJavaSdkVersion', "1.0.8") + set('mockitoVersion', "4.0.0") + set('embeddedRedisVersion', "0.6") + set('openai4jVersion', "0.18.0") + set('shardingJdbcVersion', "5.4.1") + set('bkDevOpsLeafVersion', "1.0.2-RELEASE") + // Fix CVE-2023-44981,CVE-2024-23944 3.7.1->3.8.4 + set('zookeeperVersion', "3.8.4") + // Fix CVE-2023-3635 2.8.0->3.4.0 + set('okioVersion', "3.4.0") + // Fix CVE-2023-34062 1.0.24->1.0.39 + set('reactorNettyHttpVersion', "1.0.39") + set('reactorNettyCoreVersion', "1.0.39") + // Fix CVE-2023-44487 4.1.84.Final->4.1.100.Final + set('nettyCodecHttp2Version', "4.1.100.Final") + // Fix CVE-2024-38809 CVE-2024-22262 CVE-2024-22259 CVE-2024-22243 5.3.23->5.3.39 + set('springWebVersion', "5.3.39") + // Fix CVE-2021-0341 4.9.1->4.9.2 + set('okHttpVersion', "4.9.2") + // Fix CVE-2023-46120 5.13.1->5.18.0 + set('amqpClientVersion', "5.18.0") + // Fix CVE-2023-4759 5.13.1.202206130422-r->6.6.1.202309021850-r + set('jgitVersion', "6.6.1.202309021850-r") + + // 使用Spring Boot内置的版本号变量，不采用驼峰命名，保持与官方BOM文件一致 + // Fix CVE-2023-22102 8.0.33->8.2.0 + set('mysql.version', "8.2.0") + // Fix CVE-2023-44487, CVE-2024-52316 + set('tomcat.version', "9.0.96") + // Fix CVE-2022-41854 CVE-2022-38752 CVE-2022-38751 CVE-2022-38749 CVE-2022-25857 CVE-2022-1471 1.29->1.33 + set('snakeyaml.version', "1.33") } group "com.tencent.bk.job" @@ -138,6 +192,7 @@ allprojects { apply plugin: "eclipse" apply plugin: "java-library" apply plugin: "io.spring.dependency-management" + apply plugin: "com.vanniktech.dependency.graph.generator" sourceCompatibility = '1.8' targetCompatibility = '1.8' @@ -147,28 +202,42 @@ allprojects { options.compilerArgs.add '-parameters' } - def mavenRepoUrl = System.getProperty("mavenRepoUrl") - - if (mavenRepoUrl == null) { - mavenRepoUrl = System.getenv("mavenRepoUrl") - } - - if (mavenRepoUrl == null) { - println "use default maven repo url." - mavenRepoUrl = MAVEN_REPO_URL - } - repositories { mavenLocal() - maven { url mavenRepoUrl } - maven { url "https://plugins.gradle.org/m2/" } + + // 用户通过命令行方式指定的 maven repo + def mavenRepoUrls = System.getProperty("mavenRepoUrl") + if (mavenRepoUrls == null || mavenRepoUrls.trim().length() == 0) { + // gradle.properties 中的 maven repo。优先级从高到低: + // 1. command line, as set using -D. + // 2. gradle.properties in GRADLE_USER_HOME directory. (/.gradle by default) + // 3. gradle.properties in the project’s directory, then its parent project’s directory up to the build’s root directory. + // 4. gradle.properties in Gradle installation directory. + mavenRepoUrls = project.findProperty("MAVEN_REPO_URL") + } + if (mavenRepoUrls != null && mavenRepoUrls.trim().length() > 0) { + String[] repoUrls = mavenRepoUrls.trim().replace(" ", "").split(",") + for (String repoUrl : repoUrls) { + println("Add maven repo:" + repoUrl) + maven { url repoUrl } + } + } + mavenCentral() maven { url "https://repo.spring.io/snapshot" } maven { url "https://repo.spring.io/milestone" } - mavenCentral() + maven { url "https://plugins.gradle.org/m2/" } + maven { url "https://oss.sonatype.org/content/repositories/snapshots" } + } + + configurations.all { + resolutionStrategy { + cacheChangingModulesFor 0, "seconds" + cacheDynamicVersionsFor 0, "seconds" + } } } @@ -193,8 +262,11 @@ subprojects { include '**/*Test.class' exclude '**/*IntegrationTest.class' } - task integrationTest(type: Test, dependsOn: test) { + task integrationTest(type: Test) { useJUnitPlatform() + testLogging { + events 'PASSED', 'FAILED', 'SKIPPED' + } include '**/*IntegrationTest.class' } @@ -205,6 +277,7 @@ subprojects { } include '**/*Test.class' } + build.dependsOn testAll dependencyManagement { applyMavenExclusions = false @@ -214,6 +287,19 @@ subprojects { mavenBom "org.springframework.cloud:spring-cloud-dependencies:${springCloudVersion}" mavenBom "org.springframework.cloud:spring-cloud-sleuth-otel-dependencies:${springCloudOtelVersion}" } + // 由于spring cloud (2021.0.5)的 kubernetes 组件在特性版本jdk(8u252）下会出现不兼容（http2协议)，并且由于历史原因jdk版本无法升级，所以把spring cloud kubernetes 降低版本 + dependency "org.springframework.cloud:spring-cloud-kubernetes-client-autoconfig:$springCloudKubernetesVersion" + dependency "org.springframework.cloud:spring-cloud-kubernetes-client-config:$springCloudKubernetesVersion" + dependency "org.springframework.cloud:spring-cloud-kubernetes-client-discovery:$springCloudKubernetesVersion" + dependency "org.springframework.cloud:spring-cloud-kubernetes-client-loadbalancer:$springCloudKubernetesVersion" + dependency "org.springframework.cloud:spring-cloud-kubernetes-commons:$springCloudKubernetesVersion" + dependency "io.kubernetes:client-java:$kubernetesJavaClientVersion" + dependency "io.kubernetes:client-java-api:$kubernetesJavaClientVersion" + dependency "io.kubernetes:client-java-api-fluent:$kubernetesJavaClientVersion" + dependency "io.kubernetes:client-java-extended:$kubernetesJavaClientVersion" + dependency "io.kubernetes:client-java-proto:$kubernetesJavaClientVersion" + dependency "io.kubernetes:client-java-spring-integration:$kubernetesJavaClientVersion" + dependency "org.junit.jupiter:junit-jupiter:$junitVersion" dependency "org.springframework.boot:spring-boot-starter-web:$springBootVersion" dependency "org.springframework.boot:spring-boot-starter-jdbc:$springBootVersion" @@ -225,16 +311,20 @@ subprojects { dependency "org.projectlombok:lombok:$lombokVersion" dependency "javax.ws.rs:javax.ws.rs-api:$jaxrsVersion" dependency "org.slf4j:slf4j-api:$slf4jVersion" - dependency "io.springfox:springfox-swagger2:$swaggerVersion" - dependency "io.springfox:springfox-swagger-ui:$swaggerVersion" + dependency "io.springfox:springfox-boot-starter:$swaggerVersion" + // apache commons dependency "org.apache.commons:commons-collections4:$apacheCommonsCollectionVersion" dependency "org.apache.commons:commons-pool2:$apacheCommonsPoolVersion" - dependency "net.sf.dozer:dozer:$dozerVersion" + dependency "org.apache.commons:commons-lang3:$apacheCommonsLang3Version" + dependency "org.apache.curator:curator-framework:$apacheCuratorFrameworkVersion" + dependency "org.apache.curator:curator-recipes:$apacheCuratorRecipesVersion" dependency "commons-collections:commons-collections:$commonsCollectionsVersion" - dependency "org.jedis:jedis:$jedisVersion" - dependency "com.h2database:h2:$h2Version" dependency "commons-io:commons-io:$commonsIOVersion" dependency "commons-codec:commons-codec:$apacheCommonsCodecVersion" + dependency "commons-beanutils:commons-beanutils:$commonsBeanutilsVersion" + dependency "net.sf.dozer:dozer:$dozerVersion" + dependency "org.jedis:jedis:$jedisVersion" + dependency "com.h2database:h2:$h2Version" dependency "javax.servlet:javax.servlet-api:$javaxServletVersion" // https://mvnrepository.com/artifact/org.apache.httpcomponents/httpclient dependencySet(group: 'org.apache.httpcomponents', version: "$apacheHttpClientVersion") { @@ -244,9 +334,10 @@ subprojects { // https://mvnrepository.com/artifact/org.apache.thrift/libthrift dependency group: 'org.apache.thrift', name: 'libthrift', version: "$apacheThriftVersion" dependency group: 'com.google.guava', name: 'guava', version: "$guavaVersion" + dependency "com.github.ben-manes.caffeine:caffeine:$caffeineVersion" dependency group: 'io.jsonwebtoken', name: 'jjwt', version: "$jjwtVersion" dependency "net.sourceforge.jchardet:jchardet:1.0" - dependency "io.micrometer:micrometer-registry-prometheus:$micrometerPrometheusVersion" + dependency "de.flapdoodle.embed:de.flapdoodle.embed.mongo.spring26x:$flapdoodleEmbeddedMongdbDBVersion" dependency "de.flapdoodle.embed:de.flapdoodle.embed.mongo:$flapdoodleEmbeddedMongdbDBVersion" dependency "joda-time:joda-time:$jodaTimeVersion" dependency "org.bouncycastle:bcprov-jdk15on:$bcprovVersion" @@ -255,6 +346,9 @@ subprojects { dependency "com.cronutils:cron-utils:$cronUtilsVersion" dependency "commons-validator:commons-validator:$commonsValidatorVersion" dependency "io.opentelemetry.instrumentation:opentelemetry-jdbc:$otelJdbcVersion" + dependency "com.squareup.okhttp3:logging-interceptor:$okHttpVersion" + dependency "com.squareup.okhttp3:okhttp:$okHttpVersion" + dependency "com.squareup.okhttp3:okhttp-sse:$okHttpVersion" dependencySet(group: "org.jooq", version: "$jooqVersion") { entry "jooq" entry "jooq-codegen" @@ -265,24 +359,47 @@ subprojects { entry "jackson-databind" entry "jackson-annotations" } + dependency "com.fasterxml.jackson.datatype:jackson-datatype-jsr310:$jacksonVersion" + dependency "com.fasterxml.jackson.datatype:jackson-datatype-joda:$jacksonVersion" dependencySet(group: 'ch.qos.logback', version: "$logbackVersion") { entry "logback-core" entry "logback-classic" } - dependencySet(group: 'org.apache.commons', version: "$apacheCommonsVersion") { - entry "commons-lang3" - } dependencySet(group: 'org.hibernate.validator', version: "$hibernateValidatorVersion") { entry "hibernate-validator" } dependency "com.beust:jcommander:$jcommanderVersion" + dependency "com.tencent.bk.sdk:crypto-java-sdk:$cryptoJavaSDKVersion" + dependency "com.tencent.bk.sdk:spring-boot-bk-audit-starter:$bkAuditJavaSdkVersion" + dependency "org.mockito:mockito-inline:$mockitoVersion" + dependency "com.github.kstyrc:embedded-redis:$embeddedRedisVersion" + dependency "dev.ai4j:openai4j:$openai4jVersion" + dependency "org.apache.shardingsphere:shardingsphere-jdbc-core:$shardingJdbcVersion" + dependency "com.tencent.devops.leaf:leaf-boot-starter:$bkDevOpsLeafVersion" + dependency "org.apache.zookeeper:zookeeper:$zookeeperVersion" + dependency "com.squareup.okio:okio:$okioVersion" + dependency "io.projectreactor.netty:reactor-netty-http:$reactorNettyHttpVersion" + dependency "io.projectreactor.netty:reactor-netty-core:$reactorNettyCoreVersion" + dependency "io.netty:netty-codec-http2:$nettyCodecHttp2Version" + dependency "org.springframework:spring-web:$springWebVersion" + dependency "com.squareup.okhttp3:okhttp:$okHttpVersion" + dependency "com.rabbitmq:amqp-client:$amqpClientVersion" + dependency "org.eclipse.jgit:org.eclipse.jgit:$jgitVersion" } } + dependencies { + constraints { + implementation('commons-fileupload:commons-fileupload:1.6.0') { + because 'version 1.4 pulled from spring-cloud-starter-openfeign has vulnerabilities(CVE-2023-24998)' + } + } + } + configurations { all*.exclude group: 'junit', module: 'junit' all*.exclude group: 'org.junit.vintage', module: 'junit-vintage-engine' all*.exclude group: 'org.springframework.cloud', module: 'spring-cloud-sleuth-brave' + all*.exclude group: 'org.springframework.data', module: 'spring-data-redis' } } apply plugin: TaskTreePlugin - diff --git a/src/backend/buildSrc/build.gradle b/src/backend/buildSrc/build.gradle index 5c45f5be6b..b200b8f922 100644 --- a/src/backend/buildSrc/build.gradle +++ b/src/backend/buildSrc/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,15 @@ plugins { id 'groovy' + id "java-gradle-plugin" +} + +sourceSets { + main { + java{ + srcDir('src/main/java') + } + } } dependencies { @@ -33,3 +42,15 @@ dependencies { tasks.withType(JavaCompile) { options.encoding = "UTF-8" } + +gradlePlugin { + plugins { + optionalPlugin { + id = 'com.tencent.bk.job.gradle.optional-dependencies' + displayName = "optional-dependencies" + implementationClass = 'com.tencent.bk.gradle.optional.OptionalDependenciesPlugin' + description = 'Added support for Maven-style optional dependencies.' +// tags.set (['dependencies', 'optional', 'OptionalDependencies']) + } + } +} diff --git a/src/backend/buildSrc/src/main/java/com/tencent/bk/gradle/optional/OptionalDependenciesPlugin.java b/src/backend/buildSrc/src/main/java/com/tencent/bk/gradle/optional/OptionalDependenciesPlugin.java new file mode 100644 index 0000000000..8770f85d97 --- /dev/null +++ b/src/backend/buildSrc/src/main/java/com/tencent/bk/gradle/optional/OptionalDependenciesPlugin.java @@ -0,0 +1,65 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.gradle.optional; + +import org.gradle.api.Plugin; +import org.gradle.api.Project; +import org.gradle.api.artifacts.Configuration; +import org.gradle.api.attributes.Usage; +import org.gradle.api.plugins.JavaPlugin; +import org.gradle.api.plugins.JavaPluginConvention; +import org.gradle.api.tasks.SourceSetContainer; +import org.gradle.api.tasks.javadoc.Javadoc; +import org.gradle.plugins.ide.eclipse.EclipsePlugin; +import org.gradle.plugins.ide.eclipse.model.EclipseModel; + +public class OptionalDependenciesPlugin implements Plugin { + + public static final String OPTIONAL_CONFIGURATION_NAME = "optional"; + + @Override + public void apply(Project project) { + Configuration optional = project.getConfigurations().create(OPTIONAL_CONFIGURATION_NAME); + optional.attributes((attribute) -> { + attribute.attribute(Usage.USAGE_ATTRIBUTE, project.getObjects().named(Usage.class, Usage.JAVA_RUNTIME)); + }); + + project.getPlugins().withType(JavaPlugin.class, javaPlugin -> { + SourceSetContainer sourceSets = + project.getConvention().getPlugin(JavaPluginConvention.class).getSourceSets(); + sourceSets.all(sourceSet -> { + sourceSet.setCompileClasspath(sourceSet.getCompileClasspath().plus(optional)); + sourceSet.setRuntimeClasspath(sourceSet.getRuntimeClasspath().plus(optional)); + }); + + project.getTasks().withType(Javadoc.class) + .all(javadoc -> javadoc.setClasspath(javadoc.getClasspath().plus(optional))); + }); + + project.getPlugins().withType(EclipsePlugin.class, + (eclipsePlugin) -> project.getExtensions().getByType(EclipseModel.class) + .classpath((classpath) -> classpath.getPlusConfigurations().add(optional))); + } +} diff --git a/src/backend/buildSrc/src/main/java/com/tencent/bk/gradle/tasks/CommonUtil.java b/src/backend/buildSrc/src/main/java/com/tencent/bk/gradle/tasks/CommonUtil.java index 120dfc4dd1..5c8279e137 100644 --- a/src/backend/buildSrc/src/main/java/com/tencent/bk/gradle/tasks/CommonUtil.java +++ b/src/backend/buildSrc/src/main/java/com/tencent/bk/gradle/tasks/CommonUtil.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/buildSrc/src/main/java/com/tencent/bk/gradle/tasks/DependJarInfo.java b/src/backend/buildSrc/src/main/java/com/tencent/bk/gradle/tasks/DependJarInfo.java index 6cc47f55ad..65bae11256 100644 --- a/src/backend/buildSrc/src/main/java/com/tencent/bk/gradle/tasks/DependJarInfo.java +++ b/src/backend/buildSrc/src/main/java/com/tencent/bk/gradle/tasks/DependJarInfo.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/buildSrc/src/main/java/com/tencent/bk/gradle/tasks/GenCompleteDependJarListAndRemoveInfectedJarsTask.java b/src/backend/buildSrc/src/main/java/com/tencent/bk/gradle/tasks/GenCompleteDependJarListAndRemoveInfectedJarsTask.java index ff7226c822..c35cea1417 100644 --- a/src/backend/buildSrc/src/main/java/com/tencent/bk/gradle/tasks/GenCompleteDependJarListAndRemoveInfectedJarsTask.java +++ b/src/backend/buildSrc/src/main/java/com/tencent/bk/gradle/tasks/GenCompleteDependJarListAndRemoveInfectedJarsTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,6 @@ package com.tencent.bk.gradle.tasks; -import kotlin.text.Charsets; import org.gradle.api.DefaultTask; import org.gradle.api.Project; import org.gradle.api.file.FileCollection; @@ -44,6 +43,7 @@ import java.io.InputStreamReader; import java.io.OutputStreamWriter; import java.io.UnsupportedEncodingException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -53,6 +53,7 @@ import java.util.regex.Pattern; import java.util.zip.ZipEntry; import java.util.zip.ZipInputStream; + class GenCompleteDependJarListAndRemoveInfectedJarsTask extends DefaultTask { @InputFiles @@ -170,7 +171,7 @@ private List readFileAsList(File file) { private void writeListToFile(List list, File file) { BufferedWriter bw = null; try { - bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file), "UTF-8")); + bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file), StandardCharsets.UTF_8)); for (int i = 0; i < list.size(); i++) { String line = list.get(i); if (i == 0) { @@ -292,7 +293,7 @@ private void genJarList(String jarListDirPath, String projectNameWithoutBoot, Li // 写入Jar列表至文件 BufferedWriter bw = null; try { - bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(jarListFile), "UTF-8")); + bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(jarListFile), StandardCharsets.UTF_8)); for (int i = 0; i < libJarNameList.size(); i++) { String libJarName = libJarNameList.get(i); String libJarPath = libJarPathPrefix + libJarName; @@ -339,7 +340,7 @@ private void removeInfectedJars(List removeJarNameList, File bootJarFile System.out.println("exec command:\n" + command); Process process = Runtime.getRuntime().exec(command, null, bootJarFile.getParentFile()); ins = process.getInputStream(); - BufferedReader br = new BufferedReader(new InputStreamReader(ins, Charsets.UTF_8)); + BufferedReader br = new BufferedReader(new InputStreamReader(ins, StandardCharsets.UTF_8)); String line = ""; while ((line = br.readLine()) != null) { System.out.println(line); diff --git a/src/backend/buildSrc/src/main/java/com/tencent/bk/gradle/tasks/ReplaceEditionTextTask.java b/src/backend/buildSrc/src/main/java/com/tencent/bk/gradle/tasks/ReplaceEditionTextTask.java index 9c3de67da7..0ba008d5af 100644 --- a/src/backend/buildSrc/src/main/java/com/tencent/bk/gradle/tasks/ReplaceEditionTextTask.java +++ b/src/backend/buildSrc/src/main/java/com/tencent/bk/gradle/tasks/ReplaceEditionTextTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/ai-dev-sdk/build.gradle b/src/backend/commons/ai-dev-sdk/build.gradle new file mode 100644 index 0000000000..e869010a68 --- /dev/null +++ b/src/backend/commons/ai-dev-sdk/build.gradle @@ -0,0 +1,43 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +dependencies { + api project(':commons:common') + api project(':commons:esb-sdk') + api project(':commons:common-i18n') + api project(':commons:common-otel') + implementation 'com.fasterxml.jackson.core:jackson-core' + implementation 'com.fasterxml.jackson.core:jackson-databind' + implementation 'com.fasterxml.jackson.core:jackson-annotations' + implementation 'org.apache.commons:commons-lang3' + implementation "net.sf.dozer:dozer" + implementation 'io.micrometer:micrometer-registry-prometheus' + implementation 'org.apache.commons:commons-collections4' + implementation 'org.apache.httpcomponents:httpclient' + implementation 'dev.ai4j:openai4j' + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' + testImplementation 'org.junit.jupiter:junit-jupiter' + testImplementation 'org.springframework.boot:spring-boot-starter-test' +} diff --git a/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/IBkOpenAIClient.java b/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/IBkOpenAIClient.java new file mode 100644 index 0000000000..1e496bd6a3 --- /dev/null +++ b/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/IBkOpenAIClient.java @@ -0,0 +1,49 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.aidev; + +import com.tencent.bk.job.common.aidev.model.common.AIDevMessage; + +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.function.Consumer; + +public interface IBkOpenAIClient { + + /** + * 获取AI流式回答，最终返回完整的回答结果，流式数据由partialRespConsumer接收处理 + * + * @param token 用户身份凭据 + * @param messageHistoryList 历史消息列表 + * @param userInput 用户输入 + * @param partialRespConsumer 分块消息处理器 + * @return 完整的回答结果Future + */ + CompletableFuture getAIAnswerStream(String token, + List messageHistoryList, + String userInput, + Consumer partialRespConsumer); + +} diff --git a/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/config/AIDevAutoConfiguration.java b/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/config/AIDevAutoConfiguration.java new file mode 100644 index 0000000000..69d9332426 --- /dev/null +++ b/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/config/AIDevAutoConfiguration.java @@ -0,0 +1,64 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.aidev.config; + +import com.tencent.bk.job.common.aidev.impl.BkChatCompletionModel; +import com.tencent.bk.job.common.aidev.impl.BkOpenAIClient; +import com.tencent.bk.job.common.esb.config.AppProperties; +import com.tencent.bk.job.common.esb.config.BkApiGatewayProperties; +import io.micrometer.core.instrument.MeterRegistry; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.cloud.sleuth.SpanNamer; +import org.springframework.cloud.sleuth.Tracer; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Slf4j +@Configuration(proxyBeanMethods = false) +@EnableConfigurationProperties({BkAIDevProperties.class, CustomPaasLoginProperties.class}) +public class AIDevAutoConfiguration { + + @Bean + @ConditionalOnMissingBean + public BkOpenAIClient bkOpenAIClient(Tracer tracer, + SpanNamer spanNamer, + MeterRegistry meterRegistry, + AppProperties appProperties, + CustomPaasLoginProperties customPaasLoginProperties, + BkApiGatewayProperties bkApiGatewayProperties) { + return new BkOpenAIClient( + tracer, + spanNamer, + meterRegistry, + appProperties, + customPaasLoginProperties, + bkApiGatewayProperties, + BkChatCompletionModel.HUNYUAN.toString() + ); + } + +} diff --git a/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/config/BkAIDevProperties.java b/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/config/BkAIDevProperties.java new file mode 100644 index 0000000000..19d9dc5cfc --- /dev/null +++ b/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/config/BkAIDevProperties.java @@ -0,0 +1,41 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.aidev.config; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; + +/** + * AI开发中心配置 + */ +@ConfigurationProperties(prefix = "bk-ai-dev") +@Getter +@Setter +public class BkAIDevProperties { + + private boolean enabled = true; + +} diff --git a/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/config/CustomPaasLoginProperties.java b/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/config/CustomPaasLoginProperties.java new file mode 100644 index 0000000000..e9325f2c2d --- /dev/null +++ b/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/config/CustomPaasLoginProperties.java @@ -0,0 +1,48 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.aidev.config; + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.springframework.boot.context.properties.ConfigurationProperties; + +@ConfigurationProperties(prefix = "paas.login.custom") +@Getter +@Setter +@ToString +public class CustomPaasLoginProperties { + + /** + * 是否使用第三方登录系统 + */ + private boolean enabled = false; + + /** + * 固定Token值，不提供则从Cookie中获取 + */ + private String token; + +} diff --git a/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/exception/BkOpenAIException.java b/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/exception/BkOpenAIException.java new file mode 100644 index 0000000000..d0d6ef66c9 --- /dev/null +++ b/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/exception/BkOpenAIException.java @@ -0,0 +1,41 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.aidev.exception; + +import com.tencent.bk.job.common.exception.InternalException; +import lombok.Getter; +import lombok.ToString; + +/** + * 调用蓝鲸OpenAI接口异常 + */ +@Getter +@ToString +public class BkOpenAIException extends InternalException { + + public BkOpenAIException(Throwable cause, Integer errorCode) { + super(cause, errorCode); + } +} diff --git a/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/impl/BkChatCompletionModel.java b/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/impl/BkChatCompletionModel.java new file mode 100644 index 0000000000..12428501dc --- /dev/null +++ b/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/impl/BkChatCompletionModel.java @@ -0,0 +1,42 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.aidev.impl; + +public enum BkChatCompletionModel { + + HUNYUAN("hunyuan"), + HUNYUAN_TURBO("hunyuan-turbo"); + + private final String value; + + BkChatCompletionModel(String value) { + this.value = value; + } + + @Override + public String toString() { + return value; + } +} diff --git a/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/impl/BkOpenAIClient.java b/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/impl/BkOpenAIClient.java new file mode 100644 index 0000000000..08dffce807 --- /dev/null +++ b/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/impl/BkOpenAIClient.java @@ -0,0 +1,447 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.aidev.impl; + +import com.tencent.bk.job.common.aidev.IBkOpenAIClient; +import com.tencent.bk.job.common.aidev.config.CustomPaasLoginProperties; +import com.tencent.bk.job.common.aidev.metrics.MetricsConstants; +import com.tencent.bk.job.common.aidev.model.common.AIDevMessage; +import com.tencent.bk.job.common.esb.config.AppProperties; +import com.tencent.bk.job.common.esb.config.BkApiGatewayProperties; +import com.tencent.bk.job.common.esb.model.BkApiAuthorization; +import com.tencent.bk.job.common.util.JobContextUtil; +import com.tencent.bk.job.common.util.StringUtil; +import com.tencent.bk.job.common.util.json.JsonUtils; +import dev.ai4j.openai4j.OpenAiClient; +import dev.ai4j.openai4j.chat.ChatCompletionChoice; +import dev.ai4j.openai4j.chat.ChatCompletionModel; +import dev.ai4j.openai4j.chat.ChatCompletionRequest; +import dev.ai4j.openai4j.chat.ChatCompletionResponse; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tag; +import io.micrometer.core.instrument.Tags; +import io.micrometer.core.instrument.Timer; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.NotNull; +import org.slf4j.helpers.MessageFormatter; +import org.springframework.cloud.sleuth.Span; +import org.springframework.cloud.sleuth.SpanNamer; +import org.springframework.cloud.sleuth.Tracer; +import org.springframework.cloud.sleuth.instrument.async.TraceRunnable; + +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; + +import static java.util.Collections.singletonMap; + +@Slf4j +@SuppressWarnings("SameParameterValue") +public class BkOpenAIClient implements IBkOpenAIClient { + + private static final String URI_LLM_V1 = "/appspace/gateway/llm/v1/"; + // 存储于内存中的最大消息长度：5MB + private static final int IN_MEMORY_MAX_MESSAGE_SIZE = 5 * 1024 * 1024; + // 最大日志长度：500KB + private static final int MAX_LOG_LENGTH = 500 * 1024; + + private final Tracer tracer; + private final SpanNamer spanNamer; + private final MeterRegistry meterRegistry; + private final AppProperties appProperties; + private final BkApiGatewayProperties.ApiGwConfig bkAIDevConfig; + private final CustomPaasLoginProperties customPaasLoginProperties; + private final String bkAIDevUrl; + private final String model; + + public BkOpenAIClient(Tracer tracer, + SpanNamer spanNamer, + MeterRegistry meterRegistry, + AppProperties appProperties, + CustomPaasLoginProperties customPaasLoginProperties, + BkApiGatewayProperties bkApiGatewayProperties, + String model) { + this.tracer = tracer; + this.spanNamer = spanNamer; + this.meterRegistry = meterRegistry; + this.appProperties = appProperties; + this.bkAIDevUrl = getBkAIDevUrlSafely(bkApiGatewayProperties); + this.bkAIDevConfig = bkApiGatewayProperties.getBkAIDev(); + this.customPaasLoginProperties = customPaasLoginProperties; + this.model = getValidModel(model); + log.info("BkOpenAIClient inited using model {}", this.model); + } + + /** + * 获取有效的模型类型 + * + * @param model 待校验的模型类型 + * @return 有效的模型类型 + * @throws IllegalArgumentException 传入的模型不被支持时抛出 + */ + private String getValidModel(String model) { + if (StringUtils.isBlank(model)) { + throw new IllegalArgumentException("model cannot be blank"); + } + model = model.trim(); + Set availableModels = new HashSet<>(); + for (BkChatCompletionModel bkChatCompletionModel : BkChatCompletionModel.values()) { + String availableModel = bkChatCompletionModel.toString(); + if (availableModel.equals(model)) { + return model; + } + availableModels.add(availableModel); + } + for (ChatCompletionModel chatCompletionModel : ChatCompletionModel.values()) { + String availableModel = chatCompletionModel.toString(); + if (availableModel.equals(model)) { + return model; + } + availableModels.add(availableModel); + } + String message = MessageFormatter.format( + "invalid model: {}, available models: {}", + model, + StringUtil.concatCollection(availableModels) + ).getMessage(); + throw new IllegalArgumentException(message); + } + + /** + * 获取AI平台接口根地址 + * + * @param bkApiGatewayProperties 蓝鲸API网关配置 + * @return AI平台接口根地址 + */ + private static String getBkAIDevUrlSafely(BkApiGatewayProperties bkApiGatewayProperties) { + if (bkApiGatewayProperties == null || bkApiGatewayProperties.getBkAIDev() == null) { + return null; + } + return bkApiGatewayProperties.getBkAIDev().getUrl(); + } + + private String getLLMV1Url() { + return bkAIDevUrl + URI_LLM_V1; + } + + + /** + * 构造蓝鲸网关认证信息 + * + * @param token 用户身份token + * @return 认证信息 + */ + private BkApiAuthorization buildAuthorization(String token) { + if (customPaasLoginProperties.isEnabled()) { + return BkApiAuthorization.bkTicketUserAuthorization( + getAppCode(), + getAppSecret(), + token + ); + } else { + return BkApiAuthorization.bkTokenUserAuthorization( + getAppCode(), + getAppSecret(), + token + ); + } + } + + private String getAppCode() { + String appCode = bkAIDevConfig.getAppCode(); + if (StringUtils.isNotBlank(appCode)) { + return appCode; + } + return appProperties.getCode(); + } + + private String getAppSecret() { + String appSecret = bkAIDevConfig.getAppSecret(); + if (StringUtils.isNotBlank(appSecret)) { + return appSecret; + } + return appProperties.getSecret(); + } + + /** + * 流式获取AI回复数据 + * + * @param token 用户身份凭据 + * @param messageHistoryList 历史消息列表 + * @param userInput 用户输入 + * @param partialRespConsumer 分块消息处理器 + * @return 包含AI完整回复内容的Future + */ + @Override + public CompletableFuture getAIAnswerStream(String token, + List messageHistoryList, + String userInput, + Consumer partialRespConsumer) { + final OpenAiClient client = buildOpenAiClient(token); + OpenAiClient.OpenAiClientContext context = new OpenAiClient.OpenAiClientContext(); + // 构造AI大模型接口请求 + ChatCompletionRequest request = buildRequest(messageHistoryList, userInput); + String username = JobContextUtil.getUsername(); + logRequest(username, request); + ChatCompletionRequest streamRequest = ChatCompletionRequest.builder().from(request).stream(true).build(); + CompletableFuture future = new CompletableFuture<>(); + StringBuilder responseBuilder = new StringBuilder(); + Consumer tracedPartialResponseHandler = getTracedConsumer( + getPartialResponseHandler(username, partialRespConsumer, responseBuilder) + ); + long startTime = System.currentTimeMillis(); + // 构造流式响应回调处理器 + Runnable streamingCompletionCallback = () -> { + recordAIRespAllBlockDelay( + System.currentTimeMillis() - startTime, + Tags.of(Tag.of(MetricsConstants.TAG_KEY_STATUS, MetricsConstants.TAG_VALUE_STATUS_SUCCEED)) + ); + String respStr = responseBuilder.toString(); + logRespStr(username, respStr); + future.complete(respStr); + }; + Runnable tracedStreamingCompletionCallback = new TraceRunnable(tracer, spanNamer, streamingCompletionCallback); + // 构造错误处理器 + Consumer errorHandler = throwable -> { + recordAIRespAllBlockDelay( + System.currentTimeMillis() - startTime, + Tags.of(Tag.of(MetricsConstants.TAG_KEY_STATUS, MetricsConstants.TAG_VALUE_STATUS_ERROR)) + ); + String message = MessageFormatter.format( + "Fail to get stream response, username={}", + username + ).getMessage(); + log.warn(message, throwable); + future.completeExceptionally(throwable); + }; + Consumer tracedErrorHandler = getTracedConsumer(errorHandler); + // 调用AI大模型接口 + client.chatCompletion(context, streamRequest) + .onPartialResponse(tracedPartialResponseHandler) + .onComplete(tracedStreamingCompletionCallback) + .onError(tracedErrorHandler) + .execute(); + return future; + } + + /** + * 根据凭证信息构造OpenAiClient + * + * @param token 用户身份凭证 + * @return OpenAiClient + */ + @SuppressWarnings("unchecked") + private OpenAiClient buildOpenAiClient(String token) { + return OpenAiClient.builder() + .baseUrl(getLLMV1Url()) + .openAiApiKey("empty") + .customHeaders(singletonMap("X-Bkapi-Authorization", JsonUtils.toJson(buildAuthorization(token)))) + .logRequests() + .logResponses() + .build(); + } + + /** + * 按需打印请求内容 + * + * @param username 用户名 + * @param request 请求 + */ + private void logRequest(String username, ChatCompletionRequest request) { + if (log.isDebugEnabled()) { + String requestStr = request.toString(); + log.debug( + "username={}, request={}, length={}", + username, + getLimitedLog(requestStr), + requestStr.length() + ); + } + } + + /** + * 打印响应内容 + * + * @param username 用户名 + * @param respStr 响应内容字符串 + */ + private void logRespStr(String username, String respStr) { + if (log.isDebugEnabled()) { + log.debug( + "username={}, response={}, length={}", + username, + getLimitedLog(respStr), + respStr.length() + ); + } + } + + /** + * 对原始Consumer进行包装，传递Trace数据 + * + * @param originConsumer 原始Consumer + * @param Consumer消费的数据类型 + * @return 包装后的Consumer + */ + private Consumer getTracedConsumer(Consumer originConsumer) { + Span parentSpan = tracer.currentSpan(); + return response -> { + Span childSpan = tracer.nextSpan(parentSpan).start(); + try (Tracer.SpanInScope ignored = tracer.withSpan(childSpan)) { + originConsumer.accept(response); + } finally { + childSpan.end(); + } + }; + } + + /** + * 根据传入的上层消费者生成底层框架需要的部分响应处理器 + * + * @param username 用户名 + * @param partialRespConsumer 上层部分响应消费者 + * @param responseBuilder 请求构造器 + * @return 底层框架需要的部分响应处理器 + */ + @NotNull + private Consumer getPartialResponseHandler(String username, + Consumer partialRespConsumer, + StringBuilder responseBuilder) { + final AtomicInteger bytesReceived = new AtomicInteger(0); + final AtomicBoolean firstBlockRecorded = new AtomicBoolean(false); + long startTime = System.currentTimeMillis(); + return response -> { + List choices = response.choices(); + if (choices == null || choices.isEmpty()) { + if (log.isDebugEnabled()) { + log.debug("username={}, choices is null or empty", username); + } + return; + } + String content = choices.get(0).delta().content(); + if (content != null) { + if (!firstBlockRecorded.get()) { + recordAIRespFirstBlockDelay(System.currentTimeMillis() - startTime); + firstBlockRecorded.set(true); + } + if (bytesReceived.get() > IN_MEMORY_MAX_MESSAGE_SIZE) { + log.warn( + "username={}, ignore content: {}, bytesReceived={}, exceed max message size {}", + username, + content, + bytesReceived.get(), + IN_MEMORY_MAX_MESSAGE_SIZE + ); + return; + } + responseBuilder.append(content); + bytesReceived.addAndGet(content.getBytes(StandardCharsets.UTF_8).length); + if (partialRespConsumer != null) { + partialRespConsumer.accept(content); + } + } + if (log.isDebugEnabled()) { + log.debug("username={}, receive content: {}", username, content); + } + }; + } + + /** + * 构建AI对话请求 + * + * @param messageHistoryList 历史对话记录 + * @param userInput 用户输入 + * @return AI对话请求 + */ + private ChatCompletionRequest buildRequest(List messageHistoryList, + String userInput) { + ChatCompletionRequest.Builder builder = ChatCompletionRequest.builder() + .model(model); + if (CollectionUtils.isNotEmpty(messageHistoryList)) { + for (AIDevMessage message : messageHistoryList) { + if (message.isUserMessage()) { + builder.addUserMessage(message.getContent()); + } else if (message.isAssistantMessage()) { + builder.addAssistantMessage(message.getContent()); + } else if (message.isSystemMessage()) { + builder.addSystemMessage(message.getContent()); + } + } + } + builder.addUserMessage(userInput); + return builder.build(); + } + + /** + * 对日志进行超长截断 + * + * @param rawLog 原始日志 + * @return 截断后的日志 + */ + private String getLimitedLog(String rawLog) { + return StringUtil.substring(rawLog, MAX_LOG_LENGTH); + } + + /** + * 记录AI首次响应延迟指标 + * + * @param delayMillis 延迟的毫秒数 + */ + private void recordAIRespFirstBlockDelay(long delayMillis) { + Timer.builder(MetricsConstants.NAME_AI_RESPONSE_DELAY_FIRST_BLOCK) + .description("AI Response First Block Delay") + .publishPercentileHistogram(true) + .minimumExpectedValue(Duration.ofMillis(500)) + .maximumExpectedValue(Duration.ofSeconds(30L)) + .register(meterRegistry) + .record(delayMillis, TimeUnit.MILLISECONDS); + } + + /** + * 记录AI完成响应延迟指标 + * + * @param delayMillis 延迟的毫秒数 + * @param tags 指标数据标签 + */ + private void recordAIRespAllBlockDelay(long delayMillis, Iterable tags) { + Timer.builder(MetricsConstants.NAME_AI_RESPONSE_DELAY_ALL_BLOCK) + .description("AI Response All Block Delay") + .tags(tags) + .publishPercentileHistogram(true) + .minimumExpectedValue(Duration.ofSeconds(1)) + .maximumExpectedValue(Duration.ofSeconds(180L)) + .register(meterRegistry) + .record(delayMillis, TimeUnit.MILLISECONDS); + } +} diff --git a/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/impl/JobOpenAiClientBuilderFactory.java b/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/impl/JobOpenAiClientBuilderFactory.java new file mode 100644 index 0000000000..82432234ca --- /dev/null +++ b/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/impl/JobOpenAiClientBuilderFactory.java @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.aidev.impl; + +import dev.ai4j.openai4j.DefaultOpenAiClient; +import dev.ai4j.openai4j.OpenAiClient; +import dev.ai4j.openai4j.spi.OpenAiClientBuilderFactory; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class JobOpenAiClientBuilderFactory implements OpenAiClientBuilderFactory { + @Override + public OpenAiClient.Builder get() { + return DefaultOpenAiClient.builder(); + } +} diff --git a/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/metrics/MetricsConstants.java b/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/metrics/MetricsConstants.java new file mode 100644 index 0000000000..42ffb4fa26 --- /dev/null +++ b/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/metrics/MetricsConstants.java @@ -0,0 +1,40 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.aidev.metrics; + +public class MetricsConstants { + + // metric name + public static final String NAME_AI_RESPONSE_DELAY_FIRST_BLOCK = "ai.responseDelay.firstBlock"; + public static final String NAME_AI_RESPONSE_DELAY_ALL_BLOCK = "ai.responseDelay.allBlock"; + + // tag + public static final String TAG_KEY_STATUS = "status"; + + // value + public static final String TAG_VALUE_STATUS_SUCCEED = "succeed"; + public static final String TAG_VALUE_STATUS_ERROR = "error"; + +} diff --git a/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/model/common/AIDevMessage.java b/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/model/common/AIDevMessage.java new file mode 100644 index 0000000000..34aba1f62a --- /dev/null +++ b/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/model/common/AIDevMessage.java @@ -0,0 +1,55 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.aidev.model.common; + +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@NoArgsConstructor +@Data +public class AIDevMessage { + + public static String ROLE_SYSTEM = "system"; + public static String ROLE_USER = "user"; + public static String ROLE_ASSISTANT = "assistant"; + + private String role; + + private String content; + + public boolean isSystemMessage() { + return role.equals(ROLE_SYSTEM); + } + + public boolean isUserMessage() { + return role.equals(ROLE_USER); + } + + public boolean isAssistantMessage() { + return role.equals(ROLE_ASSISTANT); + } +} diff --git a/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/model/req/AIDevInput.java b/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/model/req/AIDevInput.java new file mode 100644 index 0000000000..d2d4c58e7f --- /dev/null +++ b/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/model/req/AIDevInput.java @@ -0,0 +1,41 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.aidev.model.req; + +import com.tencent.bk.job.common.aidev.model.common.AIDevMessage; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.util.List; + +@Slf4j +@NoArgsConstructor +@Data +public class AIDevInput { + + private List messages; + +} diff --git a/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/model/req/AIDevReq.java b/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/model/req/AIDevReq.java new file mode 100644 index 0000000000..e2da1aa13b --- /dev/null +++ b/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/model/req/AIDevReq.java @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.aidev.model.req; + +import lombok.Data; +import lombok.NoArgsConstructor; + +@NoArgsConstructor +@Data +public class AIDevReq { + + private AIDevReqData data; + + private AIDevReqConfig config; + +} diff --git a/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/model/req/AIDevReqConfig.java b/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/model/req/AIDevReqConfig.java new file mode 100644 index 0000000000..5af31f213b --- /dev/null +++ b/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/model/req/AIDevReqConfig.java @@ -0,0 +1,46 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.aidev.model.req; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@AllArgsConstructor +@NoArgsConstructor +@Data +public class AIDevReqConfig { + + public static String ACTION_CREATE = "create"; + public static String MODEL_HUN_YUAN = "hunyuan"; + + private String action; + + private String model; + + public static AIDevReqConfig hunyuanConfig() { + return new AIDevReqConfig(ACTION_CREATE, MODEL_HUN_YUAN); + } +} diff --git a/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/model/req/AIDevReqData.java b/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/model/req/AIDevReqData.java new file mode 100644 index 0000000000..bef2874712 --- /dev/null +++ b/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/model/req/AIDevReqData.java @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.aidev.model.req; + +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +@NoArgsConstructor +@Data +public class AIDevReqData { + + private List inputs; + +} diff --git a/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/model/resp/AIDevChoice.java b/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/model/resp/AIDevChoice.java new file mode 100644 index 0000000000..71d3376846 --- /dev/null +++ b/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/model/resp/AIDevChoice.java @@ -0,0 +1,43 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.aidev.model.resp; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.aidev.model.common.AIDevMessage; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@NoArgsConstructor +@Data +public class AIDevChoice { + + @JsonProperty("finish_reason") + private String finishReason; + + private AIDevMessage message; + +} diff --git a/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/model/resp/AIDevData.java b/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/model/resp/AIDevData.java new file mode 100644 index 0000000000..7371c0ab54 --- /dev/null +++ b/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/model/resp/AIDevData.java @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.aidev.model.resp; + +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@NoArgsConstructor +@Data +public class AIDevData { + + private AIDevResult result; + +} diff --git a/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/model/resp/AIDevResp.java b/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/model/resp/AIDevResp.java new file mode 100644 index 0000000000..f828a23bbc --- /dev/null +++ b/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/model/resp/AIDevResp.java @@ -0,0 +1,41 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.aidev.model.resp; + +import lombok.Data; +import lombok.NoArgsConstructor; + +@NoArgsConstructor +@Data +public class AIDevResp { + + private String code; + + private String name; + + private Boolean result; + + private T data; +} diff --git a/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/model/resp/AIDevResult.java b/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/model/resp/AIDevResult.java new file mode 100644 index 0000000000..9a52b58831 --- /dev/null +++ b/src/backend/commons/ai-dev-sdk/src/main/java/com/tencent/bk/job/common/aidev/model/resp/AIDevResult.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.aidev.model.resp; + +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.util.List; + +@Slf4j +@NoArgsConstructor +@Data +public class AIDevResult { + + private String id; + private String model; + private String version; + private Long created; + private List choices; + +} diff --git a/src/backend/commons/ai-dev-sdk/src/main/resources/META-INF/services/dev.ai4j.openai4j.spi.OpenAiClientBuilderFactory b/src/backend/commons/ai-dev-sdk/src/main/resources/META-INF/services/dev.ai4j.openai4j.spi.OpenAiClientBuilderFactory new file mode 100644 index 0000000000..ab0db59478 --- /dev/null +++ b/src/backend/commons/ai-dev-sdk/src/main/resources/META-INF/services/dev.ai4j.openai4j.spi.OpenAiClientBuilderFactory @@ -0,0 +1 @@ +com.tencent.bk.job.common.aidev.impl.JobOpenAiClientBuilderFactory diff --git a/src/backend/commons/ai-dev-sdk/src/main/resources/META-INF/spring.factories b/src/backend/commons/ai-dev-sdk/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..c52dab5923 --- /dev/null +++ b/src/backend/commons/ai-dev-sdk/src/main/resources/META-INF/spring.factories @@ -0,0 +1,2 @@ +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ +com.tencent.bk.job.common.aidev.config.AIDevAutoConfiguration diff --git a/src/backend/commons/artifactory-sdk/build.gradle b/src/backend/commons/artifactory-sdk/build.gradle index 7cb6e3c970..0351c8da3d 100644 --- a/src/backend/commons/artifactory-sdk/build.gradle +++ b/src/backend/commons/artifactory-sdk/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,7 +26,6 @@ dependencies { implementation 'com.fasterxml.jackson.core:jackson-core' implementation 'com.fasterxml.jackson.core:jackson-databind' implementation 'com.fasterxml.jackson.core:jackson-annotations' - implementation 'io.springfox:springfox-swagger2' api 'org.apache.commons:commons-lang3' implementation "org.springframework:spring-jdbc" implementation 'org.springframework:spring-web' diff --git a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/config/ArtifactoryAutoConfiguration.java b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/config/ArtifactoryAutoConfiguration.java new file mode 100644 index 0000000000..8652af4da0 --- /dev/null +++ b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/config/ArtifactoryAutoConfiguration.java @@ -0,0 +1,67 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.artifactory.config; + +import com.tencent.bk.job.common.artifactory.sdk.ArtifactoryClient; +import io.micrometer.core.instrument.MeterRegistry; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; + +@Configuration(proxyBeanMethods = false) +@Import({ArtifactoryConfig.class}) +@ConditionalOnProperty(value = "artifactory.enable", havingValue = "true", matchIfMissing = true) +@Slf4j +public class ArtifactoryAutoConfiguration { + + @Bean("jobArtifactoryClient") + public ArtifactoryClient jobArtifactoryClient(ArtifactoryConfig artifactoryConfig, + ObjectProvider meterRegistryProvider) { + ArtifactoryClient client = new ArtifactoryClient( + artifactoryConfig.getArtifactoryBaseUrl(), + artifactoryConfig.getArtifactoryJobUsername(), + artifactoryConfig.getArtifactoryJobPassword(), + meterRegistryProvider.getIfAvailable() + ); + log.info("Init JobArtifactoryClient success"); + return client; + } + + @Bean("adminArtifactoryClient") + public ArtifactoryClient adminArtifactoryClient(ArtifactoryConfig artifactoryConfig, + ObjectProvider meterRegistryProvider) { + ArtifactoryClient client = new ArtifactoryClient( + artifactoryConfig.getArtifactoryBaseUrl(), + artifactoryConfig.getArtifactoryAdminUsername(), + artifactoryConfig.getArtifactoryAdminPassword(), + meterRegistryProvider.getIfAvailable() + ); + log.info("Init AdminArtifactoryClient success"); + return client; + } +} diff --git a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/config/ArtifactoryConfig.java b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/config/ArtifactoryConfig.java new file mode 100644 index 0000000000..8f903e3898 --- /dev/null +++ b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/config/ArtifactoryConfig.java @@ -0,0 +1,53 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.artifactory.config; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Configuration; + +@Data +@Configuration +public class ArtifactoryConfig { + + @Value("${artifactory.base-url:}") + private String artifactoryBaseUrl; + + @Value("${artifactory.admin.username:admin}") + private String artifactoryAdminUsername; + + @Value("${artifactory.admin.password:blueking}") + private String artifactoryAdminPassword; + + @Value("${artifactory.job.username:bkjob}") + private String artifactoryJobUsername; + + @Value("${artifactory.job.password:bkjob}") + private String artifactoryJobPassword; + + @Value("${artifactory.job.project:bkjob}") + private String artifactoryJobProject; + +} diff --git a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/constants/ArtifactoryInterfaceConsts.java b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/constants/ArtifactoryInterfaceConsts.java index a2faffdcf4..c795cc0a50 100644 --- a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/constants/ArtifactoryInterfaceConsts.java +++ b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/constants/ArtifactoryInterfaceConsts.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/constants/MetricsConstants.java b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/constants/MetricsConstants.java new file mode 100644 index 0000000000..4fb6c348f5 --- /dev/null +++ b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/constants/MetricsConstants.java @@ -0,0 +1,48 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.artifactory.constants; + +public class MetricsConstants { + + /** + * 仅统计调用制品库 API的HTTP请求过程 + */ + public static final String METRICS_NAME_BKREPO_API_HTTP = "job.client.bkrepo.api.http"; + /** + * 统计调用制品库 API整个过程，含反序列化 + */ + public static final String METRICS_NAME_BKREPO_API = "job.client.bkrepo.api"; + + public static final String TAG_KEY_API_NAME = "api_name"; + public static final String TAG_KEY_STATUS = "status"; + + public static final String TAG_VALUE_OK = "ok"; + public static final String TAG_VALUE_ERROR = "error"; + public static final String TAG_VALUE_NONE = "none"; + /** + * 用户请求参数错误：请求的节点路径在制品库中不存在 + */ + public static final String TAG_VALUE_CLIENT_ERROR_NODE_NOT_FOUND = "client.nodeNotFound"; +} diff --git a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/dto/ArtifactoryResp.java b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/dto/ArtifactoryResp.java index f2896c677e..c18ddee649 100644 --- a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/dto/ArtifactoryResp.java +++ b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/dto/ArtifactoryResp.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/dto/NodeDTO.java b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/dto/NodeDTO.java index 107cdea3c6..793ded10f8 100644 --- a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/dto/NodeDTO.java +++ b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/dto/NodeDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/dto/PageData.java b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/dto/PageData.java index 333252060b..7ad67c2e59 100644 --- a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/dto/PageData.java +++ b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/dto/PageData.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/dto/ProjectDTO.java b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/dto/ProjectDTO.java index c9cbe22dec..87fa2ffa63 100644 --- a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/dto/ProjectDTO.java +++ b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/dto/ProjectDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/dto/RepoConfiguration.java b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/dto/RepoConfiguration.java index be1ae5e002..8102f8eeb1 100644 --- a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/dto/RepoConfiguration.java +++ b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/dto/RepoConfiguration.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/dto/RepoDTO.java b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/dto/RepoDTO.java index cd7a28df27..7216858de4 100644 --- a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/dto/RepoDTO.java +++ b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/dto/RepoDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/ArtifactoryReq.java b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/ArtifactoryReq.java index eae96b9b6a..d7cdcbeb5c 100644 --- a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/ArtifactoryReq.java +++ b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/ArtifactoryReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/CheckRepoExistReq.java b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/CheckRepoExistReq.java index d6cbf57ce4..f06dc1f75c 100644 --- a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/CheckRepoExistReq.java +++ b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/CheckRepoExistReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/CreateProjectReq.java b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/CreateProjectReq.java index e0ee7b4416..4dba07ad1e 100644 --- a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/CreateProjectReq.java +++ b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/CreateProjectReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/CreateRepoReq.java b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/CreateRepoReq.java index f8d63be31d..68275930e7 100644 --- a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/CreateRepoReq.java +++ b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/CreateRepoReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/CreateUserReq.java b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/CreateUserReq.java index a0b556d06e..96d833d931 100644 --- a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/CreateUserReq.java +++ b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/CreateUserReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/CreateUserToProjectReq.java b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/CreateUserToProjectReq.java index 95532e6d1d..6df53e76c5 100644 --- a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/CreateUserToProjectReq.java +++ b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/CreateUserToProjectReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/DeleteNodeReq.java b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/DeleteNodeReq.java index b463da79ae..9a91f9f1ab 100644 --- a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/DeleteNodeReq.java +++ b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/DeleteNodeReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/DeleteRepoReq.java b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/DeleteRepoReq.java index 2862553d4e..fb3369a099 100644 --- a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/DeleteRepoReq.java +++ b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/DeleteRepoReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/DownloadGenericFileReq.java b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/DownloadGenericFileReq.java index b1aeba9cad..c04a6db36a 100644 --- a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/DownloadGenericFileReq.java +++ b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/DownloadGenericFileReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/ListNodePageReq.java b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/ListNodePageReq.java index 12d484d6fb..400ae413f6 100644 --- a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/ListNodePageReq.java +++ b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/ListNodePageReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/ListProjectReq.java b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/ListProjectReq.java index 7c6b93a0cc..b7581a6b75 100644 --- a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/ListProjectReq.java +++ b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/ListProjectReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/ListRepoPageReq.java b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/ListRepoPageReq.java index 2056843b47..cf0cd49eeb 100644 --- a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/ListRepoPageReq.java +++ b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/ListRepoPageReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/PageLimit.java b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/PageLimit.java index b3ebf5356e..1c8242afbe 100644 --- a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/PageLimit.java +++ b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/PageLimit.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/QueryNodeDetailReq.java b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/QueryNodeDetailReq.java index 9b92d3aa80..2111a56d3e 100644 --- a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/QueryNodeDetailReq.java +++ b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/QueryNodeDetailReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/Rule.java b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/Rule.java index bd67fe2974..75a1a2947a 100644 --- a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/Rule.java +++ b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/Rule.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/SearchNodePageReq.java b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/SearchNodePageReq.java index c6cd00d4f7..ca950d8f27 100644 --- a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/SearchNodePageReq.java +++ b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/SearchNodePageReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/Sort.java b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/Sort.java index 5ba423d262..bf1f8fe23e 100644 --- a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/Sort.java +++ b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/Sort.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/UploadGenericFileReq.java b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/UploadGenericFileReq.java index 954b6b7328..4dd0fedc4a 100644 --- a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/UploadGenericFileReq.java +++ b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/UploadGenericFileReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/UserDetailReq.java b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/UserDetailReq.java index 23ee91ecb6..7de4fbbe73 100644 --- a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/UserDetailReq.java +++ b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/model/req/UserDetailReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/sdk/ArtifactoryClient.java b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/sdk/ArtifactoryClient.java index e7bc4f2bc5..b98841afc2 100644 --- a/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/sdk/ArtifactoryClient.java +++ b/src/backend/commons/artifactory-sdk/src/main/java/com/tencent/bk/job/common/artifactory/sdk/ArtifactoryClient.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,6 +26,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.tencent.bk.job.common.artifactory.constants.ArtifactoryInterfaceConsts; +import com.tencent.bk.job.common.artifactory.constants.MetricsConstants; import com.tencent.bk.job.common.artifactory.model.dto.ArtifactoryResp; import com.tencent.bk.job.common.artifactory.model.dto.NodeDTO; import com.tencent.bk.job.common.artifactory.model.dto.PageData; @@ -51,18 +52,21 @@ import com.tencent.bk.job.common.artifactory.model.req.Sort; import com.tencent.bk.job.common.artifactory.model.req.UploadGenericFileReq; import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.constant.HttpMethodEnum; +import com.tencent.bk.job.common.exception.HttpStatusException; import com.tencent.bk.job.common.exception.InternalException; import com.tencent.bk.job.common.exception.NotImplementedException; import com.tencent.bk.job.common.exception.ServiceException; -import com.tencent.bk.job.common.metrics.CommonMetricNames; import com.tencent.bk.job.common.util.Base64Util; import com.tencent.bk.job.common.util.StringUtil; -import com.tencent.bk.job.common.util.http.ExtHttpHelper; +import com.tencent.bk.job.common.util.http.HttpHelper; import com.tencent.bk.job.common.util.http.HttpHelperFactory; import com.tencent.bk.job.common.util.http.HttpMetricUtil; +import com.tencent.bk.job.common.util.http.HttpRequest; import com.tencent.bk.job.common.util.json.JsonUtils; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Tag; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; @@ -76,6 +80,7 @@ import org.apache.http.entity.FileEntity; import org.apache.http.entity.InputStreamEntity; import org.apache.http.message.BasicHeader; +import org.slf4j.helpers.MessageFormatter; import org.springframework.web.util.UriUtils; import java.io.File; @@ -85,6 +90,8 @@ import java.util.List; import java.util.Map; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + @Slf4j public class ArtifactoryClient { @@ -105,13 +112,14 @@ public class ArtifactoryClient { public static final String URL_CREATE_REPO = "/repository/api/repo/create"; public static final String URL_CHECK_REPO_EXIST = "/repository/api/repo/exist/{projectId}/{repoName}"; + @Getter private final String baseUrl; private final String username; private final String password; private final MeterRegistry meterRegistry; - ExtHttpHelper httpHelper = HttpHelperFactory.getDefaultHttpHelper(); - ExtHttpHelper longHttpHelper = HttpHelperFactory.getLongRetryableHttpHelper(); + private final HttpHelper httpHelper = HttpHelperFactory.getDefaultHttpHelper(); + private final HttpHelper longHttpHelper = HttpHelperFactory.getLongRetryableHttpHelper(); public ArtifactoryClient(String baseUrl, String username, String password, MeterRegistry meterRegistry) { this.baseUrl = StringUtil.removeSuffix(baseUrl, "/"); @@ -170,28 +178,30 @@ private Header[] getUploadFileHeaders() { return headerList.toArray(headers); } - private String doHttpGet(String url, ArtifactoryReq reqBody, ExtHttpHelper httpHelper) { - if (null == reqBody) { - return httpHelper.get(url, getJsonHeaders()); - } else { - return httpHelper.get(url + reqBody.toUrlParams(), getJsonHeaders()); - } + private String doHttpGet(String url, ArtifactoryReq reqBody, HttpHelper httpHelper) { + return httpHelper.requestForSuccessResp( + HttpRequest.builder(HttpMethodEnum.GET, reqBody == null ? url : url + reqBody.toUrlParams()) + .setHeaders(getJsonHeaders()) + .build()) + .getEntity(); } - private String doHttpPost(String url, ArtifactoryReq reqBody, ExtHttpHelper httpHelper) { - if (null == reqBody) { - return httpHelper.post(url, "{}", getJsonHeaders()); - } else { - return httpHelper.post(url, JsonUtils.toJson(reqBody), getJsonHeaders()); - } + private String doHttpPost(String url, ArtifactoryReq reqBody, HttpHelper httpHelper) { + return httpHelper.requestForSuccessResp( + HttpRequest.builder(HttpMethodEnum.POST, url) + .setStringEntity(reqBody == null ? "{}" : JsonUtils.toJson(reqBody)) + .setHeaders(getJsonHeaders()) + .build()) + .getEntity(); } - private String doHttpDelete(String url, ArtifactoryReq reqBody, ExtHttpHelper httpHelper) { - if (null == reqBody) { - return httpHelper.delete(url, "{}", getJsonHeaders()); - } else { - return httpHelper.delete(url + reqBody.toUrlParams(), JsonUtils.toJson(reqBody), getJsonHeaders()); - } + private String doHttpDelete(String url, ArtifactoryReq reqBody, HttpHelper httpHelper) { + return httpHelper.requestForSuccessResp( + HttpRequest.builder(HttpMethodEnum.DELETE, reqBody == null ? url : url + reqBody.toUrlParams()) + .setStringEntity(reqBody == null ? "{}" : JsonUtils.toJson(reqBody)) + .setHeaders(getJsonHeaders()) + .build()) + .getEntity(); } @SuppressWarnings("unchecked") @@ -236,7 +246,7 @@ private R getArtifactoryRespByReq( String urlTemplate, ArtifactoryReq reqBody, TypeReference typeReference, - ExtHttpHelper httpHelper + HttpHelper httpHelper ) throws ServiceException { // URL模板变量替换 String url = StringUtil.replacePathVariables(urlTemplate, reqBody); @@ -247,10 +257,10 @@ private R getArtifactoryRespByReq( } String respStr; long start = System.nanoTime(); - String status = "none"; + AtomicReference statusRef = new AtomicReference<>(MetricsConstants.TAG_VALUE_NONE); try { - HttpMetricUtil.setHttpMetricName(CommonMetricNames.BKREPO_API_HTTP); - HttpMetricUtil.addTagForCurrentMetric(Tag.of("api_name", urlTemplate)); + HttpMetricUtil.setHttpMetricName(MetricsConstants.METRICS_NAME_BKREPO_API_HTTP); + HttpMetricUtil.addTagForCurrentMetric(Tag.of(MetricsConstants.TAG_KEY_API_NAME, urlTemplate)); switch (method) { case HttpGet.METHOD_NAME: respStr = doHttpGet(url, reqBody, httpHelper); @@ -277,23 +287,33 @@ private R getArtifactoryRespByReq( ); } R result = JsonUtils.fromJson(respStr, typeReference); - try { - checkResult(result, method, url, reqStr, respStr); - } catch (Exception e) { - status = "error"; - throw e; - } - status = "ok"; + checkResult(result, method, url, reqStr, respStr); + statusRef.set(MetricsConstants.TAG_VALUE_OK); return result; } catch (Exception e) { - log.error("Fail to request ARTIFACTORY data|method={}|url={}|reqStr={}", method, url, reqStr, e); - status = "error"; - throw new InternalException("Fail to request ARTIFACTORY data", ErrorCode.ARTIFACTORY_API_DATA_ERROR); + String msg = MessageFormatter.arrayFormat( + "Fail to request ARTIFACTORY data|method={}|url={}|reqStr={}", + new String[]{ + method, + url, + reqStr + } + ).getMessage(); + log.error(msg, e); + statusRef.set(MetricsConstants.TAG_VALUE_ERROR); + + // 特殊处理文件 NotFound 导致的 HttpStatusException + return convertException(statusRef, e); + } finally { HttpMetricUtil.clearHttpMetric(); long end = System.nanoTime(); if (null != meterRegistry) { - meterRegistry.timer(CommonMetricNames.BKREPO_API, "api_name", urlTemplate, "status", status) + meterRegistry.timer( + MetricsConstants.METRICS_NAME_BKREPO_API, + MetricsConstants.TAG_KEY_API_NAME, urlTemplate, + MetricsConstants.TAG_KEY_STATUS, statusRef.get() + ) .record(end - start, TimeUnit.NANOSECONDS); } } @@ -420,12 +440,12 @@ public NodeDTO queryNodeDetail(String projectId, String repoName, String fullPat return resp.getData(); } - public Boolean deleteProject(String projectId) { + public boolean deleteProject(String projectId) { log.info("deleteProject:{}", projectId); throw new NotImplementedException("Not support feature", ErrorCode.NOT_SUPPORT_FEATURE); } - public Boolean deleteRepo(String projectId, String repoName, Boolean forced) { + public boolean deleteRepo(String projectId, String repoName, Boolean forced) { DeleteRepoReq req = new DeleteRepoReq(); req.setProjectId(projectId); req.setRepoName(repoName); @@ -436,7 +456,7 @@ public Boolean deleteRepo(String projectId, String repoName, Boolean forced) { return resp.getCode() == 0; } - public Boolean deleteNode(String projectId, String repoName, String fullPath) { + public boolean deleteNode(String projectId, String repoName, String fullPath) { DeleteNodeReq req = new DeleteNodeReq(); req.setProjectId(projectId); req.setRepoName(repoName); @@ -468,6 +488,7 @@ private List parsePath(String filePath) { public NodeDTO getFileNode(String filePath) { List pathList = parsePath(filePath); NodeDTO nodeDTO = queryNodeDetail(pathList.get(0), pathList.get(1), pathList.get(2)); + if (null == nodeDTO) { throw new InternalException( "can not find node by filePath", @@ -493,7 +514,7 @@ public Pair getFileInputStream(String projectId, S url = getCompleteUrl(url); CloseableHttpResponse resp; try { - HttpMetricUtil.setHttpMetricName(CommonMetricNames.BKREPO_API_HTTP); + HttpMetricUtil.setHttpMetricName(MetricsConstants.METRICS_NAME_BKREPO_API_HTTP); HttpMetricUtil.addTagForCurrentMetric(Tag.of("api_name", "download:" + URL_DOWNLOAD_GENERIC_FILE)); Pair pair = longHttpHelper.getRawResp(false, url, getJsonHeaders()); resp = pair.getRight(); @@ -501,6 +522,7 @@ public Pair getFileInputStream(String projectId, S return Pair.of(resp.getEntity().getContent(), pair.getLeft()); } else { log.info("resp.statusLine={},resp.entity={}", resp.getStatusLine(), resp.getEntity()); + resp.close(); throw new InternalException(ErrorCode.FAIL_TO_REQUEST_THIRD_FILE_SOURCE_DOWNLOAD_GENERIC_FILE); } } catch (IOException e) { @@ -541,9 +563,15 @@ private NodeDTO uploadGenericFileWithEntity( url = getCompleteUrl(url); String respStr; try { - HttpMetricUtil.setHttpMetricName(CommonMetricNames.BKREPO_API_HTTP); + HttpMetricUtil.setHttpMetricName(MetricsConstants.METRICS_NAME_BKREPO_API_HTTP); HttpMetricUtil.addTagForCurrentMetric(Tag.of("api_name", "upload:" + URL_UPLOAD_GENERIC_FILE)); - respStr = longHttpHelper.put(url, reqEntity, getUploadFileHeaders()); + + respStr = longHttpHelper.requestForSuccessResp( + HttpRequest.builder(HttpMethodEnum.PUT, url) + .setHttpEntity(reqEntity) + .setHeaders(getUploadFileHeaders()) + .build()) + .getEntity(); if (log.isDebugEnabled()) { log.debug("respStr={}", getSimplifiedStrForLog(respStr)); } @@ -638,12 +666,31 @@ public boolean createUserToProject(CreateUserToProjectReq req) { public void shutdown() { } - private String getSimplifiedStrForLog(String respStr) { - String simplifiedStr = StringUtils.deleteWhitespace(respStr); - int maxLength = 20000; - if (simplifiedStr.length() > maxLength) { - simplifiedStr = simplifiedStr.substring(0, maxLength); + private String getSimplifiedStrForLog(String rawStr) { + return StringUtil.substring(rawStr, 20000); + } + + private R convertException(AtomicReference statusRef, Exception e) { + if (e instanceof HttpStatusException) { + String httpStatusExceptionRespStr = ((HttpStatusException) e).getRespBodyStr(); + ArtifactoryResp artifactoryResp = JsonUtils.fromJson(httpStatusExceptionRespStr, + new TypeReference>() { + }); + if (artifactoryResp != null + && artifactoryResp.getCode() == ArtifactoryInterfaceConsts.RESULT_CODE_NODE_NOT_FOUND) { + statusRef.set(MetricsConstants.TAG_VALUE_CLIENT_ERROR_NODE_NOT_FOUND); + throw new InternalException( + artifactoryResp.getMessage(), + ErrorCode.CAN_NOT_FIND_NODE_IN_ARTIFACTORY + ); + } else { + throw new InternalException( + "Fail to request ARTIFACTORY data", + ErrorCode.ARTIFACTORY_API_DATA_ERROR + ); + } + } else { + throw new InternalException("Fail to request ARTIFACTORY data", ErrorCode.ARTIFACTORY_API_DATA_ERROR); } - return simplifiedStr; } } diff --git a/src/backend/commons/artifactory-sdk/src/main/resources/META-INF/spring.factories b/src/backend/commons/artifactory-sdk/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..a5ebae5469 --- /dev/null +++ b/src/backend/commons/artifactory-sdk/src/main/resources/META-INF/spring.factories @@ -0,0 +1,2 @@ +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ +com.tencent.bk.job.common.artifactory.config.ArtifactoryAutoConfiguration diff --git a/src/backend/commons/build.gradle b/src/backend/commons/build.gradle index 7648f1778c..8b4c8c90f9 100644 --- a/src/backend/commons/build.gradle +++ b/src/backend/commons/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -36,7 +36,7 @@ subprojects { version "${jobCommonVersion}" dependencies { compileOnly 'javax.servlet:javax.servlet-api:3.1.0' - compileOnly 'ch.qos.logback:logback-classic:1.1.11' + compileOnly 'ch.qos.logback:logback-classic' compileOnly 'org.projectlombok:lombok' annotationProcessor 'org.projectlombok:lombok' testImplementation 'org.junit.jupiter:junit-jupiter' diff --git a/src/backend/commons/cmdb-sdk-ext/build.gradle b/src/backend/commons/cmdb-sdk-ext/build.gradle deleted file mode 100644 index c6b742fa53..0000000000 --- a/src/backend/commons/cmdb-sdk-ext/build.gradle +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -dependencies { - api project(':commons:cmdb-sdk') - api project(':commons:common-redis') -} diff --git a/src/backend/commons/cmdb-sdk-ext/src/main/java/com/tencent/bk/job/common/cc/config/CMDBFlowControllerConfig.java b/src/backend/commons/cmdb-sdk-ext/src/main/java/com/tencent/bk/job/common/cc/config/CMDBFlowControllerConfig.java deleted file mode 100644 index 91244d5132..0000000000 --- a/src/backend/commons/cmdb-sdk-ext/src/main/java/com/tencent/bk/job/common/cc/config/CMDBFlowControllerConfig.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.cc.config; - -import com.tencent.bk.job.common.redis.util.RedisSlideWindowFlowController; -import lombok.extern.slf4j.Slf4j; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Lazy; - -@Slf4j -@Configuration -@Lazy(false) -public class CMDBFlowControllerConfig { - - @Bean - public RedisSlideWindowFlowController cmdbGlobalFlowController() { - return new RedisSlideWindowFlowController(); - } -} diff --git a/src/backend/commons/cmdb-sdk-ext/src/main/java/com/tencent/bk/job/common/cc/config/CMDBFlowControllerInitListener.java b/src/backend/commons/cmdb-sdk-ext/src/main/java/com/tencent/bk/job/common/cc/config/CMDBFlowControllerInitListener.java deleted file mode 100644 index d02e5792b5..0000000000 --- a/src/backend/commons/cmdb-sdk-ext/src/main/java/com/tencent/bk/job/common/cc/config/CMDBFlowControllerInitListener.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.cc.config; - -import com.tencent.bk.job.common.redis.util.RedisSlideWindowFlowController; -import com.tencent.bk.job.common.util.StringUtil; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.context.event.ApplicationReadyEvent; -import org.springframework.context.ApplicationListener; -import org.springframework.data.redis.core.StringRedisTemplate; -import org.springframework.lang.NonNull; -import org.springframework.stereotype.Component; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -/** - * CMDB全局限流器初始化 - */ -@Slf4j -@Component -public class CMDBFlowControllerInitListener implements ApplicationListener { - - private final RedisSlideWindowFlowController cmdbGlobalFlowController; - private final StringRedisTemplate redisTemplate; - private final CmdbConfig cmdbConfig; - - @Autowired - public CMDBFlowControllerInitListener(StringRedisTemplate redisTemplate, - CmdbConfig cmdbConfig, - RedisSlideWindowFlowController cmdbGlobalFlowController) { - this.redisTemplate = redisTemplate; - this.cmdbConfig = cmdbConfig; - this.cmdbGlobalFlowController = cmdbGlobalFlowController; - } - - @Override - public void onApplicationEvent(@NonNull ApplicationReadyEvent event) { - initCMDBGlobalFlowController(); - } - - public void initCMDBGlobalFlowController() { - if (!cmdbConfig.getEnableFlowControl()) { - log.info("FlowControl not enabled, do not init flowController."); - return; - } - String flowControlResourcesStr = cmdbConfig.getFlowControlResourcesStr(); - flowControlResourcesStr = flowControlResourcesStr.trim(); - Map map = new HashMap<>(); - try { - List resourceLimitList = StringUtil.strToList(flowControlResourcesStr, String.class, ","); - for (String resourceLimitStr : resourceLimitList) { - String[] arr = resourceLimitStr.split(":"); - String resourceId = arr[0]; - Integer limitNum = Integer.parseInt(arr[1]); - map.put(resourceId, limitNum); - } - } catch (Throwable t) { - log.error("CMDB Flow control resources config invalid:{}, right format:{resourceId1}:{limit1},...," + - "{resourceId2}:{limit2}", flowControlResourcesStr, t); - } - try { - log.info("CMDB Flow control initializing,map={},flowControlDefaultLimit={},getFlowControlPrecision={}", - map, cmdbConfig.getFlowControlDefaultLimit(), cmdbConfig.getFlowControlPrecision()); - cmdbGlobalFlowController.init(redisTemplate, map, cmdbConfig.getFlowControlDefaultLimit(), - cmdbConfig.getFlowControlPrecision()); - } catch (Exception e) { - log.error("Fail to init globalFlowController", e); - } - } -} diff --git a/src/backend/commons/cmdb-sdk-model/build.gradle b/src/backend/commons/cmdb-sdk-model/build.gradle deleted file mode 100644 index a57aaca7f4..0000000000 --- a/src/backend/commons/cmdb-sdk-model/build.gradle +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -dependencies { - api project(':commons:common') - api project(':commons:esb-sdk') - implementation 'com.fasterxml.jackson.core:jackson-core' - implementation 'com.fasterxml.jackson.core:jackson-databind' - implementation 'com.fasterxml.jackson.core:jackson-annotations' - compileOnly 'org.projectlombok:lombok' - annotationProcessor 'org.projectlombok:lombok' -} diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/BaseRuleDTO.java b/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/BaseRuleDTO.java deleted file mode 100644 index 2da2a0119e..0000000000 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/BaseRuleDTO.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.cc.model; - -import lombok.Getter; -import lombok.Setter; -import lombok.ToString; - -/** - * cmdb 查询规则 - */ -@Getter -@Setter -@ToString -public class BaseRuleDTO implements IRule { - /** - * 字段名 - */ - private String field; - /** - * 操作符,可选值 equal,not_equal,in,not_in,less,less_or_equal,greater,greater_or_equal,between,not_between - */ - private String operator; - /** - * 操作值，不同的operator对应不同的value格式 - */ - private Object value; -} diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/CcGroupHostInfoDTO.java b/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/CcGroupHostInfoDTO.java deleted file mode 100644 index 62ce39cac7..0000000000 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/CcGroupHostInfoDTO.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.cc.model; - -import lombok.Data; - -/** - * CC动态分组-主机 - * - * @description - * @date 2019/3/7 - */ -@Data -public class CcGroupHostInfoDTO { - private CcGroupHostPropDTO host; -} diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/CcGroupHostPropDTO.java b/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/CcGroupHostPropDTO.java deleted file mode 100644 index 272dc8ce33..0000000000 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/CcGroupHostPropDTO.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.cc.model; - -import com.fasterxml.jackson.annotation.JsonIgnore; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.tencent.bk.job.common.util.ip.IpUtils; -import lombok.Data; - -import java.util.List; - -/** - * CC HOST - * - * @description - * @date 2019/3/7 - */ -@Data -public class CcGroupHostPropDTO { - - @JsonProperty("bk_host_id") - private Long id; - - @JsonProperty("bk_host_name") - private String name; - - // 可能为多IP - @JsonProperty("bk_host_innerip") - private String innerIp; - - @JsonProperty("bk_cloud_id") - private List cloudIdList; - - @JsonIgnore - public String getFirstIp() { - return IpUtils.getFirstIpFromMultiIp(innerIp, ","); - } -} diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/IRule.java b/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/IRule.java deleted file mode 100644 index c61affe841..0000000000 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/IRule.java +++ /dev/null @@ -1,7 +0,0 @@ -package com.tencent.bk.job.common.cc.model; - -/** - * CMDB 资源过滤规则 - */ -public interface IRule { -} diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/Page.java b/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/Page.java deleted file mode 100644 index 01b9dcd840..0000000000 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/Page.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.cc.model.bizset; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Getter; -import lombok.Setter; -import lombok.ToString; - -/** - * CMDB接口请求实体类，定义分页参数 - */ -@Setter -@Getter -@ToString -public class Page { - - /** - * 数据起始位置 - */ - private int start = 0; - - /** - * 单次拉取数据量，取值范围：[0,500] - */ - private int limit = 500; - - /** - * 请求是否获取数据总量 - */ - @JsonProperty("enable_count") - private boolean enableCount = false; - - /** - * 排序字段，可在首位加-表示降序 - */ - private String sort; - -} diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/Rule.java b/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/Rule.java deleted file mode 100644 index ddadc1a4fb..0000000000 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/Rule.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.cc.model.bizset; - -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; -import lombok.ToString; - -/** - * CMDB接口请求实体类，定义业务集过滤业务的规则 - */ -@NoArgsConstructor -@AllArgsConstructor -@Setter -@Getter -@ToString -public class Rule { - - public static final String OPERATOR_EQUAL = "equal"; - public static final String OPERATOR_NOT_EQUAL = "not_equal"; - public static final String OPERATOR_IN = "in"; - public static final String OPERATOR_NOT_IN = "not_in"; - public static final String OPERATOR_LESS = "less"; - public static final String OPERATOR_LESS_OR_EQUAL = "less_or_equal"; - public static final String OPERATOR_GREATER = "greater"; - public static final String OPERATOR_GREATER_OR_EQUAL = "greater_or_equal"; - public static final String OPERATOR_BETWEEN = "between"; - public static final String OPERATOR_NOT_BETWEEN = "not_between"; - - /** - * 业务字段名 - */ - private String field; - - /** - * 操作符，可选值 equal,not_equal,in,not_in, - * less,less_or_equal,greater,greater_or_equal, - * between,not_between - */ - private String operator; - - /** - * 业务字段取值，根据字段不同可为不同类型 - */ - private Object value; -} diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/Page.java b/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/Page.java deleted file mode 100644 index 28578c8b7f..0000000000 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/Page.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.cc.model.req; - -import lombok.*; - -/** - * @Description - * @Date 2020/1/13 - * @Version 1.0 - */ -@Getter -@Setter -@ToString -@AllArgsConstructor -@NoArgsConstructor -public class Page { - private Integer start; - - private Integer limit; -} diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/AbstractCcSearchResult.java b/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/AbstractCcSearchResult.java deleted file mode 100644 index 1bfe84a455..0000000000 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/AbstractCcSearchResult.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.cc.model.result; - -import lombok.Data; - -import java.util.List; - -/** - * @since 19/12/2019 15:16 - */ -@Data -public abstract class AbstractCcSearchResult { - private int count; - private List info; -} diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/FindHostByModuleResult.java b/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/FindHostByModuleResult.java deleted file mode 100644 index 243ace4493..0000000000 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/FindHostByModuleResult.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.cc.model.result; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Getter; -import lombok.Setter; -import lombok.ToString; - -import java.util.List; - -public class FindHostByModuleResult extends AbstractCcSearchResult { - @Getter - @Setter - @ToString - public static class HostWithTopoInfo { - private List biz; - private HostProp host; - private List module; - private List set; - } - - @Getter - @Setter - @ToString - public static class BizProp { - @JsonProperty("bk_biz_id") - private Long appId; - @JsonProperty("bk_biz_name") - private String appName; - @JsonProperty("bk_biz_maintainer") - private String appMaintainer; - } - - @Getter - @Setter - @ToString - public static class HostProp { - @JsonProperty("bk_host_id") - private Long hostId; - @JsonProperty("bk_host_innerip") - private String ip; - @JsonProperty("bk_host_name") - private String hostName; - @JsonProperty("bk_os_name") - private String os; - @JsonProperty("bk_cloud_id") - private List cloudAreaList; - } - - @Getter - @Setter - @ToString - public static class CloudAreaProp { - @JsonProperty("bk_inst_id") - private Long cloudId; - @JsonProperty("bk_inst_name") - private String cloudAreaName; - } - - @Getter - @Setter - @ToString - public static class ModuleProp { - @JsonProperty("bk_module_id") - private Long moduleId; - @JsonProperty("bk_set_id") - private Long setId; - @JsonProperty("bk_module_name") - private String moduleName; - @JsonProperty("bk_module_type") - private String moduleType; - } - - @Getter - @Setter - @ToString - public static class SetProp { - @JsonProperty("bk_biz_id") - private Long appId; - @JsonProperty("bk_set_id") - private Long setId; - @JsonProperty("bk_set_name") - private String setName; - } -} diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/FindModuleHostRelationResult.java b/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/FindModuleHostRelationResult.java deleted file mode 100644 index a100a73393..0000000000 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/FindModuleHostRelationResult.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.cc.model.result; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; -import lombok.Getter; -import lombok.Setter; -import lombok.ToString; - -import java.util.List; - -@Data -public class FindModuleHostRelationResult { - - private Integer count; - - private List relation; - - @Getter - @Setter - @ToString - public static class HostWithModules { - private HostProp host; - private List modules; - } - - @Getter - @Setter - @ToString - public static class HostProp { - @JsonProperty("bk_host_id") - private Long hostId; - @JsonProperty("bk_host_innerip") - private String ip; - @JsonProperty("bk_host_name") - private String hostName; - @JsonProperty("bk_os_name") - private String os; - @JsonProperty("bk_os_type") - private String osType; - @JsonProperty("bk_cloud_id") - private Long cloudAreaId = 0L; - } - - @Getter - @Setter - @ToString - public static class ModuleProp { - @JsonProperty("bk_module_id") - private Long moduleId; - @JsonProperty("bk_set_id") - private Long setId; - @JsonProperty("bk_module_type") - private String moduleType; - } -} diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/HostEventDetail.java b/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/HostEventDetail.java deleted file mode 100644 index a250032c3b..0000000000 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/HostEventDetail.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.cc.model.result; - - -import com.fasterxml.jackson.annotation.JsonProperty; -import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; -import com.tencent.bk.job.common.util.StringUtil; -import lombok.Data; - -import java.util.List; - -@Data -public class HostEventDetail { - - @JsonProperty("bk_host_id") - private Long hostId; - - @JsonProperty("bk_host_innerip") - private String hostInnerIp; - - @JsonProperty("bk_host_name") - private String hostName; - - @JsonProperty("bk_os_name") - private String osName; - - @JsonProperty("bk_os_type") - private String osType; - - @JsonProperty("bk_cloud_id") - private String cloudId; - - public static ApplicationHostDTO toHostInfoDTO(HostEventDetail eventDetail) { - ApplicationHostDTO hostInfoDTO = new ApplicationHostDTO(); - hostInfoDTO.setHostId(eventDetail.hostId); - List ipList = StringUtil.strToList(eventDetail.hostInnerIp, String.class, ","); - hostInfoDTO.setDisplayIp(eventDetail.hostInnerIp); - hostInfoDTO.setIpList(ipList); - if (ipList != null && !ipList.isEmpty()) { - hostInfoDTO.setIp(ipList.get(0)); - } - hostInfoDTO.setIpDesc(eventDetail.hostName); - hostInfoDTO.setOs(eventDetail.osName); - hostInfoDTO.setOsType(eventDetail.osType); - hostInfoDTO.setCloudAreaId(Long.parseLong(eventDetail.getCloudId())); - return hostInfoDTO; - } -} diff --git a/src/backend/commons/cmdb-sdk-model/src/test/java/com/tencent/bk/job/common/cc/TopologyUtilTest.java b/src/backend/commons/cmdb-sdk-model/src/test/java/com/tencent/bk/job/common/cc/TopologyUtilTest.java deleted file mode 100644 index 841a08526a..0000000000 --- a/src/backend/commons/cmdb-sdk-model/src/test/java/com/tencent/bk/job/common/cc/TopologyUtilTest.java +++ /dev/null @@ -1,43 +0,0 @@ -package com.tencent.bk.job.common.cc; - -import com.tencent.bk.job.common.cc.model.InstanceTopologyDTO; -import com.tencent.bk.job.common.cc.util.TopologyUtil; -import org.junit.jupiter.api.Test; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -public class TopologyUtilTest { - - @Test - void testPrint() { - InstanceTopologyDTO appTopology = new InstanceTopologyDTO(); - appTopology.setObjectId("biz"); - appTopology.setInstanceId(2L); - List childList = new ArrayList<>(); - InstanceTopologyDTO child1 = new InstanceTopologyDTO(); - child1.setObjectId("set"); - child1.setInstanceId(3L); - childList.add(child1); - InstanceTopologyDTO child2 = new InstanceTopologyDTO(); - child2.setObjectId("set"); - child2.setInstanceId(4L); - InstanceTopologyDTO child3 = new InstanceTopologyDTO(); - child3.setObjectId("module"); - child3.setInstanceId(5L); - child2.setChild(Collections.singletonList(child3)); - childList.add(child2); - appTopology.setChild(childList); - InstanceTopologyDTO child4 = new InstanceTopologyDTO(); - child4.setObjectId("set"); - child4.setInstanceId(6L); - InstanceTopologyDTO child5 = new InstanceTopologyDTO(); - child5.setObjectId("module"); - child5.setInstanceId(7L); - child4.setChild(Collections.singletonList(child5)); - childList.add(child4); - appTopology.setChild(childList); - TopologyUtil.printTopo(appTopology); - } -} diff --git a/src/backend/commons/cmdb-sdk/build.gradle b/src/backend/commons/cmdb-sdk/build.gradle index 9528af3746..582458a3e2 100644 --- a/src/backend/commons/cmdb-sdk/build.gradle +++ b/src/backend/commons/cmdb-sdk/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -21,13 +21,16 @@ * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. */ +plugins { + id 'com.tencent.bk.job.gradle.optional-dependencies' +} +apply plugin: 'com.tencent.bk.job.gradle.optional-dependencies' dependencies { api project(':commons:common') api project(':commons:esb-sdk') - api project(':commons:cmdb-sdk-model') - api project(':commons:gse-sdk') api project(':commons:common-i18n') + optional(api(project(':commons:common-redis'))) implementation 'com.fasterxml.jackson.core:jackson-core' implementation 'com.fasterxml.jackson.core:jackson-databind' implementation 'com.fasterxml.jackson.core:jackson-annotations' diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/config/BizCmdbClientAutoConfig.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/config/BizCmdbClientAutoConfig.java deleted file mode 100644 index 33d2de8d98..0000000000 --- a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/config/BizCmdbClientAutoConfig.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.cc.config; - -import com.tencent.bk.job.common.cc.sdk.BizCmdbClient; -import com.tencent.bk.job.common.esb.config.EsbConfig; -import com.tencent.bk.job.common.esb.constants.EsbLang; -import com.tencent.bk.job.common.gse.service.QueryAgentStatusClient; -import com.tencent.bk.job.common.util.FlowController; -import io.micrometer.core.instrument.MeterRegistry; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; - -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -@Slf4j -@Configuration -public class BizCmdbClientAutoConfig { - - @Bean("cmdbThreadPoolExecutor") - public ThreadPoolExecutor cmdbThreadPoolExecutor(CmdbConfig cmdbConfig) { - int cmdbQueryThreadsNum = cmdbConfig.getCmdbQueryThreadsNum(); - return new ThreadPoolExecutor( - cmdbQueryThreadsNum, - cmdbQueryThreadsNum, - 180L, - TimeUnit.SECONDS, - new LinkedBlockingQueue<>(cmdbQueryThreadsNum * 4), (r, executor) -> { - //使用请求的线程直接拉取数据 - log.error("cmdb request runnable rejected, use current thread({}), plz add more threads", - Thread.currentThread().getName()); - r.run(); - }); - } - - @Bean("cmdbLongTermThreadPoolExecutor") - public ThreadPoolExecutor cmdbLongTermThreadPoolExecutor(CmdbConfig cmdbConfig) { - int longTermCmdbQueryThreadsNum = cmdbConfig.getFindHostRelationLongTermConcurrency(); - return new ThreadPoolExecutor( - longTermCmdbQueryThreadsNum, - longTermCmdbQueryThreadsNum, - 180L, - TimeUnit.SECONDS, - new LinkedBlockingQueue<>(longTermCmdbQueryThreadsNum * 4), (r, executor) -> { - //使用请求的线程直接拉取数据 - log.warn("cmdb long term request runnable rejected, use current thread({}), plz add more threads", - Thread.currentThread().getName()); - r.run(); - }); - } - - @Bean - @Primary - public BizCmdbClient bizCmdbClient(EsbConfig esbConfig, - CmdbConfig cmdbConfig, - ThreadPoolExecutor cmdbThreadPoolExecutor, - ThreadPoolExecutor cmdbLongTermThreadPoolExecutor, - @Autowired(required = false) QueryAgentStatusClient queryAgentStatusClient, - MeterRegistry meterRegistry, - @Autowired(required = false) FlowController flowController) { - return new BizCmdbClient( - esbConfig, - cmdbConfig, - EsbLang.EN, - cmdbThreadPoolExecutor, - cmdbLongTermThreadPoolExecutor, - queryAgentStatusClient, - flowController, - meterRegistry - ); - } - - @Bean("cnBizCmdbClient") - public BizCmdbClient cnBizCmdbClient(EsbConfig esbConfig, - CmdbConfig cmdbConfig, - ThreadPoolExecutor cmdbThreadPoolExecutor, - ThreadPoolExecutor cmdbLongTermThreadPoolExecutor, - @Autowired(required = false) QueryAgentStatusClient queryAgentStatusClient, - MeterRegistry meterRegistry, - @Autowired(required = false) FlowController flowController) { - return new BizCmdbClient( - esbConfig, - cmdbConfig, - EsbLang.CN, - cmdbThreadPoolExecutor, - cmdbLongTermThreadPoolExecutor, - queryAgentStatusClient, - flowController, - meterRegistry - ); - } -} diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/config/CmdbAutoConfiguration.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/config/CmdbAutoConfiguration.java new file mode 100644 index 0000000000..2042737c39 --- /dev/null +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/config/CmdbAutoConfiguration.java @@ -0,0 +1,158 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.cc.config; + +import com.tencent.bk.job.common.WatchableThreadPoolExecutor; +import com.tencent.bk.job.common.cc.sdk.BizCmdbClient; +import com.tencent.bk.job.common.cc.sdk.BizSetCmdbClient; +import com.tencent.bk.job.common.cc.sdk.BkNetClient; +import com.tencent.bk.job.common.cc.sdk.IBizCmdbClient; +import com.tencent.bk.job.common.esb.config.AppProperties; +import com.tencent.bk.job.common.esb.config.BkApiAutoConfiguration; +import com.tencent.bk.job.common.esb.config.BkApiGatewayProperties; +import com.tencent.bk.job.common.esb.config.EsbProperties; +import com.tencent.bk.job.common.esb.constants.EsbLang; +import com.tencent.bk.job.common.util.FlowController; +import io.micrometer.core.instrument.MeterRegistry; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.boot.autoconfigure.AutoConfigureAfter; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.context.annotation.Primary; + +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +@Configuration(proxyBeanMethods = false) +@Import({CmdbConfig.class}) +@AutoConfigureAfter(BkApiAutoConfiguration.class) +@Slf4j +public class CmdbAutoConfiguration { + @Bean("cmdbThreadPoolExecutor") + public ThreadPoolExecutor cmdbThreadPoolExecutor(MeterRegistry meterRegistry, CmdbConfig cmdbConfig) { + int cmdbQueryThreadsNum = cmdbConfig.getCmdbQueryThreadsNum(); + return new WatchableThreadPoolExecutor( + meterRegistry, + "cmdbThreadPoolExecutor", + cmdbQueryThreadsNum, + cmdbQueryThreadsNum, + 180L, + TimeUnit.SECONDS, + new LinkedBlockingQueue<>(cmdbQueryThreadsNum * 4), (r, executor) -> { + //使用请求的线程直接拉取数据 + log.error("cmdb request runnable rejected, use current thread({}), plz add more threads", + Thread.currentThread().getName()); + r.run(); + }); + } + + @Bean("cmdbLongTermThreadPoolExecutor") + public ThreadPoolExecutor cmdbLongTermThreadPoolExecutor(CmdbConfig cmdbConfig, MeterRegistry meterRegistry) { + int longTermCmdbQueryThreadsNum = cmdbConfig.getFindHostRelationLongTermConcurrency(); + return new WatchableThreadPoolExecutor( + meterRegistry, + "cmdbLongTermThreadPoolExecutor", + longTermCmdbQueryThreadsNum, + longTermCmdbQueryThreadsNum, + 180L, + TimeUnit.SECONDS, + new LinkedBlockingQueue<>(longTermCmdbQueryThreadsNum * 4), (r, executor) -> { + //使用请求的线程直接拉取数据 + log.warn("cmdb long term request runnable rejected, use current thread({}), plz add more threads", + Thread.currentThread().getName()); + r.run(); + }); + } + + @Bean + @Primary + public BizCmdbClient bizCmdbClient(AppProperties appProperties, + EsbProperties esbProperties, + BkApiGatewayProperties bkApiGatewayProperties, + CmdbConfig cmdbConfig, + ThreadPoolExecutor cmdbThreadPoolExecutor, + ThreadPoolExecutor cmdbLongTermThreadPoolExecutor, + MeterRegistry meterRegistry, + ObjectProvider flowControllerProvider) { + return new BizCmdbClient( + appProperties, + esbProperties, + bkApiGatewayProperties, + cmdbConfig, + EsbLang.EN, + cmdbThreadPoolExecutor, + cmdbLongTermThreadPoolExecutor, + flowControllerProvider.getIfAvailable(), + meterRegistry + ); + } + + @Bean("cnBizCmdbClient") + public BizCmdbClient cnBizCmdbClient(AppProperties appProperties, + EsbProperties esbProperties, + BkApiGatewayProperties bkApiGatewayProperties, + CmdbConfig cmdbConfig, + ThreadPoolExecutor cmdbThreadPoolExecutor, + ThreadPoolExecutor cmdbLongTermThreadPoolExecutor, + MeterRegistry meterRegistry, + ObjectProvider flowControllerProvider) { + return new BizCmdbClient( + appProperties, + esbProperties, + bkApiGatewayProperties, + cmdbConfig, + EsbLang.CN, + cmdbThreadPoolExecutor, + cmdbLongTermThreadPoolExecutor, + flowControllerProvider.getIfAvailable(), + meterRegistry + ); + } + + @Bean + public BizSetCmdbClient bizSetCmdbClient(AppProperties appProperties, + EsbProperties esbProperties, + BkApiGatewayProperties bkApiGatewayProperties, + CmdbConfig cmdbConfig, + MeterRegistry meterRegistry, + ObjectProvider flowControllerProvider) { + return new BizSetCmdbClient( + appProperties, + esbProperties, + bkApiGatewayProperties, + cmdbConfig, + flowControllerProvider.getIfAvailable(), + meterRegistry + ); + } + + @Bean + public BkNetClient cloudAreaService(IBizCmdbClient bizCmdbClient) { + return new BkNetClient(bizCmdbClient); + } +} diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/config/CmdbConfig.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/config/CmdbConfig.java index 69833448b3..09c811f80e 100644 --- a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/config/CmdbConfig.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/config/CmdbConfig.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,10 +26,10 @@ import lombok.Data; import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; +import org.springframework.context.annotation.Configuration; @Data -@Component +@Configuration public class CmdbConfig { @Value("${cmdb.default.supplier.account:0}") diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/config/CmdbFlowControlAutoConfiguration.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/config/CmdbFlowControlAutoConfiguration.java new file mode 100644 index 0000000000..aa480dbe29 --- /dev/null +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/config/CmdbFlowControlAutoConfiguration.java @@ -0,0 +1,91 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.cc.config; + +import com.tencent.bk.job.common.redis.config.JobRedisAutoConfiguration; +import com.tencent.bk.job.common.redis.util.RedisSlideWindowFlowController; +import com.tencent.bk.job.common.util.StringUtil; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.boot.autoconfigure.AutoConfigureAfter; +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.data.redis.core.StringRedisTemplate; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +@Configuration(proxyBeanMethods = false) +@Import({CmdbConfig.class}) +@Slf4j +@ConditionalOnProperty(value = "cmdb.interface.flowControl.enabled", havingValue = "true", matchIfMissing = true) +@AutoConfigureAfter({JobRedisAutoConfiguration.class}) +public class CmdbFlowControlAutoConfiguration { + @Bean + @ConditionalOnClass(RedisSlideWindowFlowController.class) + public RedisSlideWindowFlowController cmdbGlobalFlowController( + ObjectProvider redisTemplateProvider, + CmdbConfig cmdbConfig) { + log.info("Init Cmdb global flow controller!"); + return initCMDBGlobalFlowController(redisTemplateProvider.getIfAvailable(), cmdbConfig); + } + + private RedisSlideWindowFlowController initCMDBGlobalFlowController(StringRedisTemplate redisTemplate, + CmdbConfig cmdbConfig) { + RedisSlideWindowFlowController flowController = new RedisSlideWindowFlowController(); + + String flowControlResourcesStr = cmdbConfig.getFlowControlResourcesStr(); + flowControlResourcesStr = flowControlResourcesStr.trim(); + Map map = new HashMap<>(); + try { + List resourceLimitList = StringUtil.strToList(flowControlResourcesStr, String.class, ","); + for (String resourceLimitStr : resourceLimitList) { + String[] arr = resourceLimitStr.split(":"); + String resourceId = arr[0]; + Integer limitNum = Integer.parseInt(arr[1]); + map.put(resourceId, limitNum); + } + } catch (Throwable t) { + log.error("CMDB Flow control resources config invalid:{}, right format:{resourceId1}:{limit1},...," + + "{resourceId2}:{limit2}", flowControlResourcesStr, t); + } + try { + log.info("CMDB Flow control initializing,map={},flowControlDefaultLimit={},getFlowControlPrecision={}", + map, cmdbConfig.getFlowControlDefaultLimit(), cmdbConfig.getFlowControlPrecision()); + flowController.init(redisTemplate, + map, + cmdbConfig.getFlowControlDefaultLimit(), + cmdbConfig.getFlowControlPrecision()); + } catch (Exception e) { + log.error("Fail to init globalFlowController", e); + } + + + return flowController; + } +} diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/config/CmdbMetricsAutoConfiguration.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/config/CmdbMetricsAutoConfiguration.java new file mode 100644 index 0000000000..2d038aabbe --- /dev/null +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/config/CmdbMetricsAutoConfiguration.java @@ -0,0 +1,56 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.cc.config; + +import com.tencent.bk.job.common.cc.constants.CmdbMetricNames; +import io.micrometer.core.instrument.Meter; +import io.micrometer.core.instrument.config.MeterFilter; +import io.micrometer.core.instrument.distribution.DistributionStatisticConfig; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * cmdb 监控指标配置 + */ +@Configuration +public class CmdbMetricsAutoConfiguration { + @Bean("cmdbMeterFilter") + public MeterFilter distributionMeterFilter() { + return new MeterFilter() { + @Override + public DistributionStatisticConfig configure(Meter.Id id, DistributionStatisticConfig config) { + String metricName = id.getName(); + if (metricName.startsWith(CmdbMetricNames.CMDB_API_PREFIX)) { + return DistributionStatisticConfig.builder().percentilesHistogram(true) + // [10ms,1s] + .minimumExpectedValue(10_000_000.0).maximumExpectedValue(1_000_000_000.0) + .build().merge(config); + } else { + return config; + } + } + }; + } +} diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/constants/CmdbConstants.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/constants/CmdbConstants.java index fa56bbc77a..24baad1fa3 100644 --- a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/constants/CmdbConstants.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/constants/CmdbConstants.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/constants/CmdbMetricNames.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/constants/CmdbMetricNames.java new file mode 100644 index 0000000000..b24d5c8c21 --- /dev/null +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/constants/CmdbMetricNames.java @@ -0,0 +1,30 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.cc.constants; + +public interface CmdbMetricNames { + + String CMDB_API_PREFIX = "job.client.cmdb.api"; +} diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/constants/KubeTopoNodeTypeEnum.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/constants/KubeTopoNodeTypeEnum.java new file mode 100644 index 0000000000..2d7a8baa34 --- /dev/null +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/constants/KubeTopoNodeTypeEnum.java @@ -0,0 +1,48 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.cc.constants; + +/** + * 容器拓扑节点资源类型 + */ +public enum KubeTopoNodeTypeEnum { + CLUSTER("cluster"), + NAMESPACE("namespace"), + DEPLOYMENT("deployment"), + DAEMON_SET("daemonSet"), + STATEFUL_SET("statefulSet"), + CRON_JOB("cronJob"), + JOB("job"); + + private final String value; + + KubeTopoNodeTypeEnum(String val) { + this.value = val; + } + + public String getValue() { + return value; + } +} diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/exception/CmdbException.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/exception/CmdbException.java index 07adebc022..449c927a01 100644 --- a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/exception/CmdbException.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/exception/CmdbException.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/AppRoleDTO.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/AppRoleDTO.java similarity index 90% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/AppRoleDTO.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/AppRoleDTO.java index 0b9596645e..e6fff364ea 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/AppRoleDTO.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/AppRoleDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,11 @@ package com.tencent.bk.job.common.cc.model; -import lombok.*; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; /** * CMDB定义的业务角色DTO diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/BriefTopologyDTO.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/BriefTopologyDTO.java similarity index 97% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/BriefTopologyDTO.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/BriefTopologyDTO.java index e84357c694..925914e13d 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/BriefTopologyDTO.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/BriefTopologyDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/BusinessInfoDTO.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/BusinessInfoDTO.java similarity index 96% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/BusinessInfoDTO.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/BusinessInfoDTO.java index bfc3b8c029..821385d7e0 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/BusinessInfoDTO.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/BusinessInfoDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/CcCloudAreaInfoDTO.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/CcCloudAreaInfoDTO.java similarity index 97% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/CcCloudAreaInfoDTO.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/CcCloudAreaInfoDTO.java index 4e8072c072..8fa85e455e 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/CcCloudAreaInfoDTO.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/CcCloudAreaInfoDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/CcCloudIdDTO.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/CcCloudIdDTO.java similarity index 95% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/CcCloudIdDTO.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/CcCloudIdDTO.java index 3cfabb47af..4c80521eeb 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/CcCloudIdDTO.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/CcCloudIdDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/CcConditionExprDTO.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/CcConditionExprDTO.java similarity index 93% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/CcConditionExprDTO.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/CcConditionExprDTO.java index 07f59c377e..e7949b4314 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/CcConditionExprDTO.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/CcConditionExprDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -31,8 +31,6 @@ /** * CC的Condition表达式 * - * @description - * @date 2019/3/4 */ @Getter @Setter diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/CcDynamicGroupDTO.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/CcDynamicGroupDTO.java similarity index 97% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/CcDynamicGroupDTO.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/CcDynamicGroupDTO.java index 14ab6c60ca..b3c332c65d 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/CcDynamicGroupDTO.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/CcDynamicGroupDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -51,6 +51,9 @@ public class CcDynamicGroupDTO { */ private String name; + @JsonProperty("last_time") + private String lastTime; + @JsonProperty("bk_obj_id") private String objId; diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/CcHostInfoDTO.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/CcHostInfoDTO.java similarity index 85% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/CcHostInfoDTO.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/CcHostInfoDTO.java index 97c0e949bd..30bac35ece 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/CcHostInfoDTO.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/CcHostInfoDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -40,12 +40,22 @@ public class CcHostInfoDTO { private Long hostId; @JsonProperty("bk_host_innerip") private String innerIp; + @JsonProperty("bk_host_innerip_v6") + private String ipv6; + @JsonProperty("bk_agent_id") + private String agentId; @JsonProperty("bk_host_name") private String hostName; @JsonProperty("bk_os_name") private String os; + @JsonProperty("bk_os_type") + private String osType; @JsonProperty("bk_cloud_id") private Long cloudId; + @JsonProperty("bk_cloud_vendor") + private String cloudVendorId; + @JsonProperty("last_time") + private String lastTime; @JsonIgnore public String getFirstIp() { diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/CcInstanceDTO.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/CcInstanceDTO.java similarity index 95% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/CcInstanceDTO.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/CcInstanceDTO.java index dd4a5cfdfa..25a3d46c4b 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/CcInstanceDTO.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/CcInstanceDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/CcObjAttributeDTO.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/CcObjAttributeDTO.java similarity index 80% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/CcObjAttributeDTO.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/CcObjAttributeDTO.java index 4a7ff36450..bd60374aee 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/CcObjAttributeDTO.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/CcObjAttributeDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,7 +25,11 @@ package com.tencent.bk.job.common.cc.model; import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.*; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; /** * CMDB 对象模型属性 @@ -59,4 +63,24 @@ public class CcObjAttributeDTO { */ @JsonProperty("creator") private String creator; + + private Object option; + + @NoArgsConstructor + @AllArgsConstructor + @Getter + @Setter + @ToString + public static class Option { + + private String id; + + @JsonProperty("is_default") + private Boolean default_; + + private String name; + + private String type; + + } } diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/DynamicGroupHostInfoDTO.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/DynamicGroupHostInfoDTO.java new file mode 100644 index 0000000000..717a5807b1 --- /dev/null +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/DynamicGroupHostInfoDTO.java @@ -0,0 +1,35 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.cc.model; + +import lombok.Data; + +/** + * CMDB动态分组-主机 + */ +@Data +public class DynamicGroupHostInfoDTO { + private DynamicGroupHostPropDTO host; +} diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/DynamicGroupHostPropDTO.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/DynamicGroupHostPropDTO.java new file mode 100644 index 0000000000..3747d8465b --- /dev/null +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/DynamicGroupHostPropDTO.java @@ -0,0 +1,65 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.cc.model; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.util.ip.IpUtils; +import lombok.Data; + +import java.util.List; + +/** + * 接口实体类，CMDB动态分组中的主机属性 + */ +@Data +public class DynamicGroupHostPropDTO { + + @JsonProperty("bk_host_id") + private Long id; + + @JsonProperty("bk_host_name") + private String name; + + // 可能为多个ip + @JsonProperty("bk_host_innerip") + private String innerIp; + + // 可能为多个ip + @JsonProperty("bk_host_innerip_v6") + private String ipv6; + + // 可能为多个ip + @JsonProperty("bk_agent_id") + private String agentId; + + @JsonProperty("bk_cloud_id") + private List cloudIdList; + + @JsonIgnore + public String getFirstIp() { + return IpUtils.getFirstIpFromMultiIp(innerIp, ","); + } +} diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/GetHostConditionDTO.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/GetHostConditionDTO.java similarity index 96% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/GetHostConditionDTO.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/GetHostConditionDTO.java index 0364a07542..55d84b4d4c 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/GetHostConditionDTO.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/GetHostConditionDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/InstanceTopologyDTO.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/InstanceTopologyDTO.java similarity index 95% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/InstanceTopologyDTO.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/InstanceTopologyDTO.java index f4c65eb865..acb8f6ee1b 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/InstanceTopologyDTO.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/InstanceTopologyDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -79,5 +79,8 @@ public int hashCode() { return Objects.hash(objectId, instanceId); } + public String getUniqueKey() { + return objectId + "_" + instanceId; + } } diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/TopoNodePathDTO.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/TopoNodePathDTO.java similarity index 95% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/TopoNodePathDTO.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/TopoNodePathDTO.java index 21174b1b35..d6d3f633b3 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/TopoNodePathDTO.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/TopoNodePathDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/TopologyNodeInfoDTO.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/TopologyNodeInfoDTO.java similarity index 95% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/TopologyNodeInfoDTO.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/TopologyNodeInfoDTO.java index 758680abd3..fe8775ac17 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/TopologyNodeInfoDTO.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/TopologyNodeInfoDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/bizset/BizFilter.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/bizset/BizFilter.java new file mode 100644 index 0000000000..bddb9952cb --- /dev/null +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/bizset/BizFilter.java @@ -0,0 +1,54 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.cc.model.bizset; + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +import java.util.List; + +/** + * CMDB接口请求实体类，定义业务通过组合规则选择业务的过滤器 + */ +@Setter +@Getter +@ToString +public class BizFilter { + + public static final String CONDITION_AND = "AND"; + public static final String CONDITION_OR = "OR"; + + /** + * 多个规则之间的组合条件，取值为：AND/OR + */ + private String condition; + + /** + * 规则列表 + */ + private List rules; + +} diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/BizInfo.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/bizset/BizInfo.java similarity index 95% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/BizInfo.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/bizset/BizInfo.java index ed4b549e29..feaf28f465 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/BizInfo.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/bizset/BizInfo.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/BizSetFilter.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/bizset/BizSetFilter.java similarity index 96% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/BizSetFilter.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/bizset/BizSetFilter.java index e7ee722d7a..e7c2183d4c 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/BizSetFilter.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/bizset/BizSetFilter.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/BizSetInfo.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/bizset/BizSetInfo.java similarity index 97% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/BizSetInfo.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/bizset/BizSetInfo.java index d99c0513ef..cc7aad2409 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/BizSetInfo.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/bizset/BizSetInfo.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/BizSetScope.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/bizset/BizSetScope.java similarity index 95% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/BizSetScope.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/bizset/BizSetScope.java index 82b19990fa..bf37905170 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/BizSetScope.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/bizset/BizSetScope.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/bizset/Page.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/bizset/Page.java new file mode 100644 index 0000000000..24cd857ba2 --- /dev/null +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/bizset/Page.java @@ -0,0 +1,61 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.cc.model.bizset; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +/** + * CMDB接口请求实体类，定义分页参数 + */ +@Setter +@Getter +@ToString +public class Page { + + /** + * 数据起始位置 + */ + private int start = 0; + + /** + * 单次拉取数据量，取值范围：[0,500] + */ + private int limit = 500; + + /** + * 请求是否获取数据总量 + */ + @JsonProperty("enable_count") + private boolean enableCount = false; + + /** + * 排序字段，可在首位加-表示降序 + */ + private String sort; + +} diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/bizset/Rule.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/bizset/Rule.java new file mode 100644 index 0000000000..ed7c8d0a54 --- /dev/null +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/bizset/Rule.java @@ -0,0 +1,57 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.cc.model.bizset; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; + +/** + * CMDB接口请求实体类，定义业务集过滤业务的规则 + */ +@NoArgsConstructor +@AllArgsConstructor +@Setter +@Getter +@ToString +public class Rule { + + /** + * 业务字段名 + */ + private String field; + + /** + * 操作符 + */ + private String operator; + + /** + * 业务字段取值，根据字段不同可为不同类型 + */ + private Object value; +} diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/SearchBizInBusinessReq.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/bizset/SearchBizInBusinessReq.java similarity index 96% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/SearchBizInBusinessReq.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/bizset/SearchBizInBusinessReq.java index e6ddcabce5..02660b54f3 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/SearchBizInBusinessReq.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/bizset/SearchBizInBusinessReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/SearchBizInBusinessSetResp.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/bizset/SearchBizInBusinessSetResp.java similarity index 96% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/SearchBizInBusinessSetResp.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/bizset/SearchBizInBusinessSetResp.java index 969ac91a80..73a72de8b8 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/SearchBizInBusinessSetResp.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/bizset/SearchBizInBusinessSetResp.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/SearchBizSetReq.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/bizset/SearchBizSetReq.java similarity index 96% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/SearchBizSetReq.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/bizset/SearchBizSetReq.java index a2ebcf6e96..d4c744b23f 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/SearchBizSetReq.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/bizset/SearchBizSetReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/SearchBizSetResp.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/bizset/SearchBizSetResp.java similarity index 96% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/SearchBizSetResp.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/bizset/SearchBizSetResp.java index ccbe3a3206..57893513a8 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/bizset/SearchBizSetResp.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/bizset/SearchBizSetResp.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/container/ContainerDTO.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/container/ContainerDTO.java new file mode 100644 index 0000000000..fb2a23733f --- /dev/null +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/container/ContainerDTO.java @@ -0,0 +1,74 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.cc.model.container; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +import java.util.Arrays; +import java.util.List; +import java.util.Objects; + +/** + * CMDB 容器基础信息 + */ +@Data +public class ContainerDTO { + + /** + * 集群模型 cmdb 字段 + */ + public interface Fields { + String ID = "id"; + String CONTAINER_UID = "container_uid"; + String NAME = "name"; + + /** + * CMDB 模型的字段名 + */ + List ALL = Arrays.asList(ID, CONTAINER_UID, NAME); + } + + + private Long id; + + @JsonProperty("container_uid") + private String containerUID; + + private String name; + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ContainerDTO that = (ContainerDTO) o; + return Objects.equals(id, that.id); + } + + @Override + public int hashCode() { + return Objects.hash(id); + } +} diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/container/ContainerDetailDTO.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/container/ContainerDetailDTO.java new file mode 100644 index 0000000000..d2b0fe3d5c --- /dev/null +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/container/ContainerDetailDTO.java @@ -0,0 +1,85 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.cc.model.container; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.model.dto.Container; +import lombok.Data; + +import java.util.Objects; + +/** + * CMDB 容器详细信息 + */ +@Data +public class ContainerDetailDTO { + /** + * 容器基础信息 + */ + @JsonProperty("container") + private ContainerDTO container; + + /** + * 容器所在 Pod 信息 + */ + private PodDTO pod; + + /** + * 容器所在拓扑节点信息 + */ + private ContainerTopoDTO topo; + + public Container toContainer() { + Container container = new Container(); + container.setId(this.container.getId()); + container.setContainerId(this.container.getContainerUID()); + container.setName(this.container.getName()); + if (pod != null) { + container.setPodName(pod.getName()); + container.setPodLabels(pod.getLabels()); + } + if (topo != null) { + container.setNodeHostId(topo.getHostId()); + container.setClusterId(topo.getClusterId()); + container.setNamespaceId(topo.getNamespaceId()); + container.setWorkloadType(topo.getWorkloadType()); + container.setWorkloadId(topo.getWorkloadId()); + } + return container; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ContainerDetailDTO that = (ContainerDetailDTO) o; + return Objects.equals(container, that.container); + } + + @Override + public int hashCode() { + return Objects.hash(container); + } +} diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/container/ContainerTopoDTO.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/container/ContainerTopoDTO.java new file mode 100644 index 0000000000..3f1b44d617 --- /dev/null +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/container/ContainerTopoDTO.java @@ -0,0 +1,57 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.cc.model.container; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +/** + * 容器所在的拓扑节点信息 + */ +@Data +public class ContainerTopoDTO { + @JsonProperty("bk_biz_id") + private Long bizId; + + @JsonProperty("bk_cluster_id") + private Long clusterId; + + @JsonProperty("bk_namespace_id") + private Long namespaceId; + + @JsonProperty("bk_workload_id") + private Long workloadId; + + @JsonProperty("workload_type") + private String workloadType; + + /** + * 容器所在 node 的主机 ID + */ + @JsonProperty("bk_host_id") + private Long hostId; + + +} diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/container/KubeBizDTO.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/container/KubeBizDTO.java new file mode 100644 index 0000000000..98ac082f25 --- /dev/null +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/container/KubeBizDTO.java @@ -0,0 +1,51 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.cc.model.container; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +/** + * CMDB 容器拓扑 - 业务信息 + */ +@Data +@JsonInclude(JsonInclude.Include.NON_NULL) +public class KubeBizDTO { + /** + * 业务 ID + */ + private Long id; + /** + * 业务名 + */ + @JsonProperty("nm") + private String name; + /** + * 业务下的容器个数 + */ + @JsonProperty("cnt") + private Integer count; +} diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/container/KubeClusterDTO.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/container/KubeClusterDTO.java new file mode 100644 index 0000000000..615ea80579 --- /dev/null +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/container/KubeClusterDTO.java @@ -0,0 +1,85 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.cc.model.container; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +import java.util.Arrays; +import java.util.List; + +/** + * 容器集群 + */ +@Data +public class KubeClusterDTO { + + /** + * 集群模型 cmdb 字段 + */ + public interface Fields { + String BK_BIZ_ID = "bk_biz_id"; + String ID = "id"; + String NAME = "name"; + String CLUSTER_UID = "uid"; + String BK_PROJECT_ID = "bk_project_id"; + String BK_PROJECT_NAME = "bk_project_name"; + + /** + * CMDB 模型的字段名 + */ + List ALL = Arrays.asList(BK_BIZ_ID, ID, NAME, CLUSTER_UID, BK_PROJECT_ID, BK_PROJECT_NAME); + } + + + /** + * 业务 ID + */ + @JsonProperty("bk_biz_id") + private Long bizId; + + /** + * 集群资源 ID（集群在 cmdb 中注册资源的 ID） + */ + private Long id; + + /** + * 集群名称 + */ + private String name; + + /** + * 集群 ID，比如： BCS-K8S-00000 + */ + private String uid; + + @JsonProperty("bk_project_id") + private String bkProjectId; + + @JsonProperty("bk_project_name") + private String bkProjectName; + + +} diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/container/KubeNamespaceDTO.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/container/KubeNamespaceDTO.java new file mode 100644 index 0000000000..40f682c3a5 --- /dev/null +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/container/KubeNamespaceDTO.java @@ -0,0 +1,84 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.cc.model.container; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +import java.util.Arrays; +import java.util.List; + +/** + * 容器 Namespace + */ +@Data +public class KubeNamespaceDTO { + + /** + * 集群模型 cmdb 字段 + */ + public interface Fields { + String BK_BIZ_ID = "bk_biz_id"; + String ID = "id"; + String NAME = "name"; + String BK_CLUSTER_ID = "bk_cluster_id"; + String CLUSTER_UID = "cluster_uid"; + + /** + * CMDB 模型的字段名 + */ + List ALL = Arrays.asList(BK_BIZ_ID, ID, NAME, CLUSTER_UID, BK_CLUSTER_ID); + } + + + /** + * 业务 ID + */ + @JsonProperty("bk_biz_id") + private Long bizId; + + /** + * namespace 资源 ID（namespace 在 cmdb 中注册资源的 ID） + */ + private Long id; + + /** + * 名称 + */ + private String name; + + /** + * 所在集群的资源 ID（集群 在 cmdb 中注册资源的 ID） + */ + @JsonProperty("bk_cluster_id") + private Long bkClusterId; + + /** + * 所在集群的UID + */ + @JsonProperty("cluster_uid") + private String clusterUID; + +} diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/container/KubeNodeDTO.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/container/KubeNodeDTO.java new file mode 100644 index 0000000000..8821243811 --- /dev/null +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/container/KubeNodeDTO.java @@ -0,0 +1,77 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.cc.model.container; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +import java.util.List; +import java.util.Objects; + +/** + * 容器拓扑节点信息 + */ +@Data +@JsonInclude(JsonInclude.Include.NON_NULL) +public class KubeNodeDTO { + /** + * 节点的资源类型 + */ + private String kind; + /** + * 节点 ID + */ + private Long id; + /** + * 节点名称 + */ + @JsonProperty("nm") + private String name; + /** + * 该节点下的Container数量 + */ + @JsonProperty("cnt") + private Integer count; + /** + * 节点下的子节点信息 + */ + @JsonProperty("nds") + private List nodes; + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + KubeNodeDTO kubeNode = (KubeNodeDTO) o; + return kind.equals(kubeNode.kind) && + id.equals(kubeNode.id); + } + + @Override + public int hashCode() { + return Objects.hash(kind, id); + } +} diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/container/KubeNodeID.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/container/KubeNodeID.java new file mode 100644 index 0000000000..b4af6308f2 --- /dev/null +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/container/KubeNodeID.java @@ -0,0 +1,67 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.cc.model.container; + +import com.fasterxml.jackson.annotation.JsonInclude; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.Objects; + +/** + * 容器拓扑节点ID + */ +@Data +@NoArgsConstructor +@JsonInclude(JsonInclude.Include.NON_NULL) +public class KubeNodeID { + /** + * 节点的资源类型 + */ + private String kind; + /** + * 节点 ID + */ + private Long id; + + public KubeNodeID(String kind, Long id) { + this.kind = kind; + this.id = id; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + KubeNodeID kubeNode = (KubeNodeID) o; + return kind.equals(kubeNode.kind) && + id.equals(kubeNode.id); + } + + @Override + public int hashCode() { + return Objects.hash(kind, id); + } +} diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/container/KubeTopologyDTO.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/container/KubeTopologyDTO.java new file mode 100644 index 0000000000..9eb405b600 --- /dev/null +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/container/KubeTopologyDTO.java @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.cc.model.container; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +import java.util.List; + +/** + * CMDB 容器拓扑 + */ +@Data +@JsonInclude(JsonInclude.Include.NON_NULL) +public class KubeTopologyDTO { + + @JsonProperty("biz") + private KubeBizDTO biz; + + @JsonProperty("nds") + private List nodes; +} diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/container/KubeWorkloadDTO.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/container/KubeWorkloadDTO.java new file mode 100644 index 0000000000..ab18f849ef --- /dev/null +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/container/KubeWorkloadDTO.java @@ -0,0 +1,104 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.cc.model.container; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +import java.util.Arrays; +import java.util.List; + +/** + * 容器 workload + */ +@Data +public class KubeWorkloadDTO { + + /** + * 集群模型 cmdb 字段 + */ + public interface Fields { + String BK_BIZ_ID = "bk_biz_id"; + String ID = "id"; + String NAME = "name"; + String KIND = "kind"; + String BK_CLUSTER_ID = "bk_cluster_id"; + String CLUSTER_UID = "cluster_uid"; + String BK_NAMESPACE_ID = "bk_namespace_id"; + String NAMESPACE = "namespace"; + + /** + * CMDB 模型的字段名 + */ + List ALL = Arrays.asList(BK_BIZ_ID, ID, NAME, KIND, BK_CLUSTER_ID, CLUSTER_UID, + BK_NAMESPACE_ID, NAMESPACE); + } + + /** + * 业务 ID + */ + @JsonProperty("bk_biz_id") + private Long bizId; + + /** + * workload 资源 ID（wordload 在 cmdb 中注册资源的 ID） + */ + private Long id; + + /** + * 名称 + */ + private String name; + + /** + * workload 类型 + */ + private String kind; + + /** + * 所在集群的资源 ID（集群 在 cmdb 中注册资源的 ID） + */ + @JsonProperty("bk_cluster_id") + private Long bkClusterId; + + /** + * 所在集群的UID + */ + @JsonProperty("cluster_uid") + private String clusterUID; + + /** + * 所在namespace的资源 ID（namespace 在 cmdb 中注册资源的 ID） + */ + @JsonProperty("bk_namespace_id") + private Long bkNamespaceId; + + /** + * 所在 namespace 名称 + */ + @JsonProperty("namespace") + private String namespace; + +} diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/container/PodDTO.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/container/PodDTO.java new file mode 100644 index 0000000000..5a506142ca --- /dev/null +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/container/PodDTO.java @@ -0,0 +1,56 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.cc.model.container; + +import lombok.Data; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +/** + * 容器 Pod + */ +@Data +public class PodDTO { + + /** + * 集群模型 cmdb 字段 + */ + public interface Fields { + String ID = "id"; + String NAME = "name"; + String LABELS = "labels"; + + List ALL = Arrays.asList(ID, LABELS, NAME); + } + + + private Long id; + + private Map labels; + + private String name; +} diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/filter/BaseRuleDTO.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/filter/BaseRuleDTO.java new file mode 100644 index 0000000000..37d6f38223 --- /dev/null +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/filter/BaseRuleDTO.java @@ -0,0 +1,106 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.cc.model.filter; + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +/** + * cmdb 查询规则 + */ +@Getter +@Setter +@ToString +public class BaseRuleDTO implements IRule { + /** + * 字段名 + */ + private String field; + /** + * 操作符,可选值 equal,not_equal,in,not_in,less,less_or_equal,greater,greater_or_equal,between,not_between,contains + */ + private String operator; + /** + * 操作值，不同的operator对应不同的value格式 + */ + private Object value; + + public BaseRuleDTO() { + } + + public BaseRuleDTO(String field, String operator, Object value) { + this.field = field; + this.operator = operator; + this.value = value; + } + + public static BaseRuleDTO in(String field, Object value) { + return new BaseRuleDTO(field, RuleOperatorEnum.IN.getOperator(), value); + } + + public static BaseRuleDTO notIn(String field, Object value) { + return new BaseRuleDTO(field, RuleOperatorEnum.NOT_IN.getOperator(), value); + } + + public static BaseRuleDTO equals(String field, Object value) { + return new BaseRuleDTO(field, RuleOperatorEnum.EQUAL.getOperator(), value); + } + + public static BaseRuleDTO notEquals(String field, Object value) { + return new BaseRuleDTO(field, RuleOperatorEnum.NOT_EQUAL.getOperator(), value); + } + + public static BaseRuleDTO contains(String field, Object value) { + return new BaseRuleDTO(field, RuleOperatorEnum.CONTAINS.getOperator(), value); + } + + public static BaseRuleDTO filterObject(String field, Object value) { + return new BaseRuleDTO(field, RuleOperatorEnum.FILTER_OBJECT.getOperator(), value); + } + + public static BaseRuleDTO exists(String field) { + // cmdb 统一解析逻辑会解析value，没填的话会报错;传入 true(对过滤结果没有影响)临时规避该问题 + return new BaseRuleDTO(field, RuleOperatorEnum.EXIST.getOperator(), true); + } + + public static BaseRuleDTO notExists(String field) { + // cmdb 统一解析逻辑会解析value，没填的话会报错;传入 true(对过滤结果没有影响)临时规避该问题 + return new BaseRuleDTO(field, RuleOperatorEnum.NOT_EXIST.getOperator(), true); + } + + public static BaseRuleDTO greaterThan(String field, Object value) { + return new BaseRuleDTO(field, RuleOperatorEnum.GREATER.getOperator(), value); + } + + public static BaseRuleDTO lessThan(String field, Object value) { + return new BaseRuleDTO(field, RuleOperatorEnum.LESS.getOperator(), value); + } + + @Override + protected BaseRuleDTO clone() { + return new BaseRuleDTO(field, operator, value); + } +} diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/ComposeRuleDTO.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/filter/ComposeRuleDTO.java similarity index 84% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/ComposeRuleDTO.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/filter/ComposeRuleDTO.java index b9f800a321..e2e3b1f293 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/ComposeRuleDTO.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/filter/ComposeRuleDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,9 +22,10 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.common.cc.model; +package com.tencent.bk.job.common.cc.model.filter; import lombok.Getter; +import lombok.NoArgsConstructor; import lombok.Setter; import lombok.ToString; @@ -37,6 +38,7 @@ @Getter @Setter @ToString +@NoArgsConstructor public class ComposeRuleDTO implements IRule { /** * 组合查询条件 @@ -47,6 +49,14 @@ public class ComposeRuleDTO implements IRule { */ private List rules = new ArrayList<>(); + public ComposeRuleDTO(String condition) { + this.condition = condition; + } + + public ComposeRuleDTO(RuleConditionEnum condition) { + this.condition = condition.getCondition(); + } + /** * 新增过滤规则 * diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/filter/IRule.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/filter/IRule.java new file mode 100644 index 0000000000..c615bce632 --- /dev/null +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/filter/IRule.java @@ -0,0 +1,31 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.cc.model.filter; + +/** + * CMDB 资源过滤规则 + */ +public interface IRule { +} diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/PropertyFilterDTO.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/filter/PropertyFilterDTO.java similarity index 88% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/PropertyFilterDTO.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/filter/PropertyFilterDTO.java index 307bc8e016..19791b961c 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/PropertyFilterDTO.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/filter/PropertyFilterDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,9 +22,10 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.common.cc.model; +package com.tencent.bk.job.common.cc.model.filter; import lombok.Data; +import org.apache.commons.collections4.CollectionUtils; import java.util.ArrayList; import java.util.List; @@ -51,4 +52,8 @@ public class PropertyFilterDTO { public void addRule(IRule rule) { this.rules.add(rule); } + + public boolean hasRule() { + return CollectionUtils.isNotEmpty(rules); + } } diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/filter/RuleConditionEnum.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/filter/RuleConditionEnum.java new file mode 100644 index 0000000000..d804f18d68 --- /dev/null +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/filter/RuleConditionEnum.java @@ -0,0 +1,46 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.cc.model.filter; + +/** + * cmdb 组合查询条件 + *

+ * cmdb 文档：https://github.com/TencentBlueKing/bk-cmdb/blob/master/pkg/filter/README.md + */ +public enum RuleConditionEnum { + + AND("AND"), + OR("OR"); + + RuleConditionEnum(String condition) { + this.condition = condition; + } + + private final String condition; + + public String getCondition() { + return condition; + } +} diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/filter/RuleOperatorEnum.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/filter/RuleOperatorEnum.java new file mode 100644 index 0000000000..3d1f504873 --- /dev/null +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/filter/RuleOperatorEnum.java @@ -0,0 +1,77 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.cc.model.filter; + +/** + * cmdb 查询规则操作符 + *

+ * cmdb 文档：https://github.com/TencentBlueKing/bk-cmdb/blob/master/pkg/filter/README.md + */ +public enum RuleOperatorEnum { + + EQUAL("equal"), + NOT_EQUAL("not_equal"), + IN("in"), + NOT_IN("not_in"), + LESS("less"), + LESS_OR_EQUAL("less_or_equal"), + GREATER("greater"), + GREATER_OR_EQUAL("greater_or_equal"), + BETWEEN("between"), + NOT_BETWEEN("not_between"), + CONTAINS("contains"), + EXIST("exist"), + NOT_EXIST("not_exist"), + FILTER_OBJECT("filter_object"), + FILTER_ARRAY("filter_array"); + + RuleOperatorEnum(String operator) { + this.operator = operator; + } + + private final String operator; + + public String getOperator() { + return operator; + } + + public static RuleOperatorEnum valOf(String operator) { + for (RuleOperatorEnum operatorEnum : values()) { + if (operatorEnum.getOperator().equals(operator)) { + return operatorEnum; + } + } + throw new IllegalArgumentException("No RuleOperatorEnum constant: " + operator); + } + + public static boolean isValid(String operator) { + for (RuleOperatorEnum operatorEnum : values()) { + if (operatorEnum.getOperator().equals(operator)) { + return true; + } + } + return false; + } +} diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/query/KubeClusterQuery.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/query/KubeClusterQuery.java new file mode 100644 index 0000000000..ca54b08750 --- /dev/null +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/query/KubeClusterQuery.java @@ -0,0 +1,78 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.cc.model.query; + +import lombok.Getter; +import lombok.ToString; + +import java.util.List; + +/** + * 容器集群查询 + */ +@Getter +@ToString +public class KubeClusterQuery { + private final Long bizId; + + private final List ids; + + private final List bkClusterUIDs; + + private KubeClusterQuery(Builder builder) { + bizId = builder.bizId; + ids = builder.ids; + bkClusterUIDs = builder.bkClusterUIDs; + } + + + public static final class Builder { + private final Long bizId; + private List ids; + private List bkClusterUIDs; + + private Builder(long bizId) { + this.bizId = bizId; + } + + public static Builder builder(long bizId) { + return new Builder(bizId); + } + + public Builder ids(List ids) { + this.ids = ids; + return this; + } + + public Builder bkClusterUIDs(List bkClusterUIDs) { + this.bkClusterUIDs = bkClusterUIDs; + return this; + } + + public KubeClusterQuery build() { + return new KubeClusterQuery(this); + } + } +} diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/query/NamespaceQuery.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/query/NamespaceQuery.java new file mode 100644 index 0000000000..f1155dda57 --- /dev/null +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/query/NamespaceQuery.java @@ -0,0 +1,87 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.cc.model.query; + +import lombok.Getter; +import lombok.ToString; + +import java.util.List; + +/** + * namespace 查询 + */ +@Getter +@ToString +public class NamespaceQuery { + private final Long bizId; + + private final List ids; + + private final List names; + + private final List bkClusterIds; + + private NamespaceQuery(Builder builder) { + bizId = builder.bizId; + ids = builder.ids; + names = builder.names; + bkClusterIds = builder.bkClusterIds; + } + + + public static final class Builder { + private final Long bizId; + private List ids; + private List names; + private List bkClusterIds; + + private Builder(long bizId) { + this.bizId = bizId; + } + + public static Builder builder(long bizId) { + return new Builder(bizId); + } + + public Builder ids(List ids) { + this.ids = ids; + return this; + } + + public Builder names(List names) { + this.names = names; + return this; + } + + public Builder bkClusterIds(List bkClusterIds) { + this.bkClusterIds = bkClusterIds; + return this; + } + + public NamespaceQuery build() { + return new NamespaceQuery(this); + } + } +} diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/query/WorkloadQuery.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/query/WorkloadQuery.java new file mode 100644 index 0000000000..c5b440543a --- /dev/null +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/query/WorkloadQuery.java @@ -0,0 +1,101 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.cc.model.query; + +import lombok.Getter; +import lombok.ToString; + +import java.util.List; + +/** + * workload 查询 + */ +@Getter +@ToString +public class WorkloadQuery { + private final Long bizId; + + private final List ids; + + private final List names; + + private final List bkClusterIds; + + private final List bkNamespaceIds; + + private final String kind; + + private WorkloadQuery(Builder builder) { + bizId = builder.bizId; + ids = builder.ids; + names = builder.names; + bkClusterIds = builder.bkClusterIds; + bkNamespaceIds = builder.bkNamespaceIds; + kind = builder.kind; + } + + + public static final class Builder { + private final Long bizId; + private List ids; + private List names; + private List bkClusterIds; + private List bkNamespaceIds; + private final String kind; + + private Builder(long bizId, String kind) { + this.bizId = bizId; + this.kind = kind; + } + + public static Builder builder(long bizId, String kind) { + return new Builder(bizId, kind); + } + + public Builder ids(List ids) { + this.ids = ids; + return this; + } + + public Builder names(List names) { + this.names = names; + return this; + } + + public Builder bkClusterIds(List bkClusterIds) { + this.bkClusterIds = bkClusterIds; + return this; + } + + public Builder bkNamespaceIds(List bkNamespaceIds) { + this.bkNamespaceIds = bkNamespaceIds; + return this; + } + + public WorkloadQuery build() { + return new WorkloadQuery(this); + } + } +} diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/AbstractCcReq.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/AbstractCcReq.java similarity index 92% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/AbstractCcReq.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/AbstractCcReq.java index 7ffffc2a69..b77004df64 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/AbstractCcReq.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/AbstractCcReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,7 +27,6 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.tencent.bk.job.common.esb.model.EsbReq; -import com.tencent.bk.job.common.model.dto.PageDTO; import lombok.Getter; import lombok.Setter; import lombok.ToString; @@ -46,6 +45,6 @@ public abstract class AbstractCcReq extends EsbReq { @JsonProperty("bk_biz_id") private Long bizId; private Map condition; - private PageDTO page; + private Page page; private List fields; } diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/CmdbPageReq.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/CmdbPageReq.java new file mode 100644 index 0000000000..ea4046f119 --- /dev/null +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/CmdbPageReq.java @@ -0,0 +1,43 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.cc.model.req; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.tencent.bk.job.common.esb.model.EsbReq; +import lombok.Data; +import lombok.EqualsAndHashCode; + +/** + * cmdb 分页查询基础类 + */ +@EqualsAndHashCode(callSuper = true) +@Data +@JsonInclude(JsonInclude.Include.NON_NULL) +public abstract class CmdbPageReq extends EsbReq { + /** + * 分页设置 + */ + private Page page; +} diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/ExecuteDynamicGroupReq.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/ExecuteDynamicGroupReq.java similarity index 88% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/ExecuteDynamicGroupReq.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/ExecuteDynamicGroupReq.java index e146fb4544..dd8978c31a 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/ExecuteDynamicGroupReq.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/ExecuteDynamicGroupReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -44,8 +44,14 @@ public class ExecuteDynamicGroupReq extends EsbReq { @JsonProperty("id") private String groupId; - private List fields = Arrays.asList("bk_host_id", "bk_host_name", "bk_host_innerip", "bk_cloud_id", - "bk_os_name"); + private List fields = Arrays.asList( + "bk_host_id", + "bk_agent_id", + "bk_host_name", + "bk_host_innerip", + "bk_cloud_id", + "bk_os_name" + ); private Page page = new Page(); } diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/FindHostBizRelationsReq.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/FindHostBizRelationsReq.java similarity index 95% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/FindHostBizRelationsReq.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/FindHostBizRelationsReq.java index 43b26d68a4..55f9d4abe4 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/FindHostBizRelationsReq.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/FindHostBizRelationsReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/FindHostByModuleReq.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/FindHostByModuleReq.java similarity index 96% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/FindHostByModuleReq.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/FindHostByModuleReq.java index 4e3274b753..c7cfc9f383 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/FindHostByModuleReq.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/FindHostByModuleReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/FindHostTopoRelationReq.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/FindHostTopoRelationReq.java similarity index 96% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/FindHostTopoRelationReq.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/FindHostTopoRelationReq.java index eafc37b5b5..1314898547 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/FindHostTopoRelationReq.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/FindHostTopoRelationReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/FindModuleHostRelationReq.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/FindModuleHostRelationReq.java similarity index 89% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/FindModuleHostRelationReq.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/FindModuleHostRelationReq.java index c521dd8a5c..1b0af31903 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/FindModuleHostRelationReq.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/FindModuleHostRelationReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -43,11 +43,11 @@ public class FindModuleHostRelationReq extends EsbReq { private List moduleIdList; @JsonProperty("module_fields") - private List moduleFields = Arrays.asList("bk_module_id", "bk_set_id"); + private List moduleFields = Arrays.asList("bk_module_id", "bk_set_id", "last_time"); @JsonProperty("host_fields") - private List hostFields = Arrays.asList("bk_host_id", "bk_host_innerip", "bk_host_name", "bk_os_name", - "bk_cloud_id", "bk_os_type"); + private List hostFields = Arrays.asList("bk_host_id", "bk_host_innerip", "bk_host_innerip_v6", + "bk_agent_id", "bk_host_name", "bk_os_name", "bk_cloud_id", "bk_os_type", "bk_cloud_vendor", "last_time"); private Page page; } diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/GetAppReq.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/GetAppReq.java similarity index 83% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/GetAppReq.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/GetAppReq.java index 11ec16cb22..7ffaaa921c 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/GetAppReq.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/GetAppReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,9 +24,19 @@ package com.tencent.bk.job.common.cc.model.req; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.cc.model.bizset.BizFilter; +import lombok.Getter; +import lombok.Setter; + /** * @description * @date 2019/3/1 */ +@Getter +@Setter public class GetAppReq extends AbstractCcReq { + + @JsonProperty("biz_property_filter") + private BizFilter bizFilter; } diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/GetBizInstTopoReq.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/GetBizInstTopoReq.java similarity index 95% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/GetBizInstTopoReq.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/GetBizInstTopoReq.java index c5fcb14455..93b838f636 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/GetBizInstTopoReq.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/GetBizInstTopoReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/GetBizInternalModuleReq.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/GetBizInternalModuleReq.java similarity index 95% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/GetBizInternalModuleReq.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/GetBizInternalModuleReq.java index 4595213563..8f4b3a9943 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/GetBizInternalModuleReq.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/GetBizInternalModuleReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/GetBizKubeCacheTopoReq.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/GetBizKubeCacheTopoReq.java new file mode 100644 index 0000000000..02274383e4 --- /dev/null +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/GetBizKubeCacheTopoReq.java @@ -0,0 +1,40 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.cc.model.req; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.esb.model.EsbReq; +import lombok.Getter; +import lombok.Setter; + +/** + * CMDB API 请求 - 根据业务ID,查询该业务的全量容器拓扑树信息 + */ +@Getter +@Setter +public class GetBizKubeCacheTopoReq extends EsbReq { + @JsonProperty("bk_biz_id") + private long bizId; +} diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/GetBriefCacheTopoReq.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/GetBriefCacheTopoReq.java similarity index 95% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/GetBriefCacheTopoReq.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/GetBriefCacheTopoReq.java index 2b66cc79c1..72939e9e78 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/GetBriefCacheTopoReq.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/GetBriefCacheTopoReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/GetCloudAreaInfoReq.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/GetCloudAreaInfoReq.java similarity index 95% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/GetCloudAreaInfoReq.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/GetCloudAreaInfoReq.java index bfb3a7d191..3ee6d67bfa 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/GetCloudAreaInfoReq.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/GetCloudAreaInfoReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/GetCustomGroupListReq.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/GetCustomGroupListReq.java similarity index 95% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/GetCustomGroupListReq.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/GetCustomGroupListReq.java index d499cd6895..3225ec3804 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/GetCustomGroupListReq.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/GetCustomGroupListReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/GetGroupIpReq.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/GetGroupIpReq.java similarity index 95% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/GetGroupIpReq.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/GetGroupIpReq.java index ea8a15fa18..5a3178c285 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/GetGroupIpReq.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/GetGroupIpReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/GetHostReq.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/GetHostReq.java similarity index 92% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/GetHostReq.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/GetHostReq.java index efaba4b585..c1431cb3ba 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/GetHostReq.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/GetHostReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,7 +26,6 @@ import com.tencent.bk.job.common.cc.model.GetHostConditionDTO; import com.tencent.bk.job.common.esb.model.EsbReq; -import com.tencent.bk.job.common.model.dto.PageDTO; import lombok.Getter; import lombok.Setter; import lombok.ToString; @@ -44,5 +43,5 @@ @ToString public class GetHostReq extends EsbReq { private List condition; - private PageDTO page; + private Page page; } diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/GetInstanceReq.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/GetInstanceReq.java similarity index 91% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/GetInstanceReq.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/GetInstanceReq.java index a08b48c080..4d3155f9b8 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/GetInstanceReq.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/GetInstanceReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,7 +26,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.tencent.bk.job.common.esb.model.EsbReq; -import com.tencent.bk.job.common.model.dto.PageDTO; import lombok.Getter; import lombok.Setter; @@ -39,5 +38,5 @@ public class GetInstanceReq extends EsbReq { @JsonProperty("bk_obj_id") private String objectId; - private PageDTO page; + private Page page; } diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/GetObjAttributeReq.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/GetObjAttributeReq.java similarity index 95% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/GetObjAttributeReq.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/GetObjAttributeReq.java index 89deefdd01..2d02fe71e2 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/GetObjAttributeReq.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/GetObjAttributeReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/GetTopoNodePathReq.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/GetTopoNodePathReq.java similarity index 96% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/GetTopoNodePathReq.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/GetTopoNodePathReq.java index 0103cdfc07..aff2438ef4 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/GetTopoNodePathReq.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/GetTopoNodePathReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/HostCondition.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/HostCondition.java similarity index 90% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/HostCondition.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/HostCondition.java index 019ebf76ca..d9e2f64560 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/HostCondition.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/HostCondition.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,7 +25,11 @@ package com.tencent.bk.job.common.cc.model.req; import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.*; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; /** * @Description diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/ListBizHostReq.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/ListBizHostReq.java similarity index 89% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/ListBizHostReq.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/ListBizHostReq.java index 38b2df679e..cbe9cc334f 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/ListBizHostReq.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/ListBizHostReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,10 +25,9 @@ package com.tencent.bk.job.common.cc.model.req; import com.fasterxml.jackson.annotation.JsonProperty; -import com.tencent.bk.job.common.cc.model.BaseRuleDTO; -import com.tencent.bk.job.common.cc.model.PropertyFilterDTO; +import com.tencent.bk.job.common.cc.model.filter.BaseRuleDTO; +import com.tencent.bk.job.common.cc.model.filter.PropertyFilterDTO; import com.tencent.bk.job.common.esb.model.EsbReq; -import com.tencent.bk.job.common.model.dto.PageDTO; import lombok.Getter; import lombok.Setter; @@ -60,5 +59,5 @@ public class ListBizHostReq extends EsbReq { private List fields = Arrays.asList("bk_host_id", "bk_host_innerip", "bk_host_name", "bk_os_name", "bk_cloud_id"); - private PageDTO page; + private Page page; } diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/ListBizHostsTopoReq.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/ListBizHostsTopoReq.java similarity index 96% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/ListBizHostsTopoReq.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/ListBizHostsTopoReq.java index 020acbe863..3f868a823f 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/ListBizHostsTopoReq.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/ListBizHostsTopoReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/ListHostsWithoutBizReq.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/ListHostsWithoutBizReq.java similarity index 80% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/ListHostsWithoutBizReq.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/ListHostsWithoutBizReq.java index 0d38f8c8b0..78ee8f6960 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/ListHostsWithoutBizReq.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/ListHostsWithoutBizReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,9 +25,8 @@ package com.tencent.bk.job.common.cc.model.req; import com.fasterxml.jackson.annotation.JsonProperty; -import com.tencent.bk.job.common.cc.model.PropertyFilterDTO; +import com.tencent.bk.job.common.cc.model.filter.PropertyFilterDTO; import com.tencent.bk.job.common.esb.model.EsbReq; -import com.tencent.bk.job.common.model.dto.PageDTO; import lombok.Getter; import lombok.Setter; @@ -41,8 +40,18 @@ public class ListHostsWithoutBizReq extends EsbReq { private PropertyFilterDTO condition; @JsonProperty("fields") - private List fields = Arrays.asList("bk_host_id", "bk_host_innerip", "bk_host_name", "bk_os_name", - "bk_cloud_id"); + private List fields = Arrays.asList( + "bk_host_id", + "bk_host_innerip", + "bk_host_innerip_v6", + "bk_agent_id", + "bk_host_name", + "bk_os_name", + "bk_os_type", + "bk_cloud_id", + "bk_cloud_vendor", + "last_time" + ); - private PageDTO page; + private Page page; } diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/ListKubeClusterReq.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/ListKubeClusterReq.java new file mode 100644 index 0000000000..c970b9fb3f --- /dev/null +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/ListKubeClusterReq.java @@ -0,0 +1,62 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.cc.model.req; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.cc.model.filter.PropertyFilterDTO; +import lombok.Getter; +import lombok.Setter; + +import java.util.List; + +/** + * CMDB API 请求 - 查询容器集群 + */ +@Getter +@Setter +@JsonInclude(JsonInclude.Include.NON_NULL) +public class ListKubeClusterReq extends CmdbPageReq { + + @JsonProperty("bk_biz_id") + private Long bizId; + + @JsonProperty("filter") + private PropertyFilterDTO filter; + + + /** + * 所要查询的容器集群属性，如果不写代表搜索全部数据 + */ + @JsonProperty("fields") + private List fields; + + /** + * 分页条件 + */ + @JsonProperty("page") + private Page page; + +} diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/ListKubeContainerByTopoReq.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/ListKubeContainerByTopoReq.java new file mode 100644 index 0000000000..bcfa6df463 --- /dev/null +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/ListKubeContainerByTopoReq.java @@ -0,0 +1,64 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.cc.model.req; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.cc.model.container.KubeNodeID; +import com.tencent.bk.job.common.cc.model.filter.PropertyFilterDTO; +import lombok.Getter; +import lombok.Setter; + +import java.util.List; + +/** + * CMDB API 请求 - 根据容器拓扑获取container信息 + */ +@Getter +@Setter +@JsonInclude(JsonInclude.Include.NON_NULL) +public class ListKubeContainerByTopoReq extends CmdbPageReq { + @JsonProperty("bk_biz_id") + private Long bizId; + + /** + * 容器拓扑节点信息，数组限制条件为200 + */ + @JsonProperty("bk_kube_nodes") + private List nodeIdList; + + @JsonProperty("container_filter") + private PropertyFilterDTO containerFilter; + + @JsonProperty("pod_filter") + private PropertyFilterDTO podFilter; + + @JsonProperty("container_fields") + private List containerFields; + + @JsonProperty("pod_fields") + private List podFields; + +} diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/ListKubeNamespaceReq.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/ListKubeNamespaceReq.java new file mode 100644 index 0000000000..c85522a692 --- /dev/null +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/ListKubeNamespaceReq.java @@ -0,0 +1,62 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.cc.model.req; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.cc.model.filter.PropertyFilterDTO; +import lombok.Getter; +import lombok.Setter; + +import java.util.List; + +/** + * CMDB API 请求 - 查询 namespace + */ +@Getter +@Setter +@JsonInclude(JsonInclude.Include.NON_NULL) +public class ListKubeNamespaceReq extends CmdbPageReq { + + @JsonProperty("bk_biz_id") + private Long bizId; + + @JsonProperty("filter") + private PropertyFilterDTO filter; + + + /** + * 属性列表，控制返回结果里有哪些字段，能够加速接口请求和减少网络流量传输 + */ + @JsonProperty("fields") + private List fields; + + /** + * 分页条件 + */ + @JsonProperty("page") + private Page page; + +} diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/ListKubeWorkloadReq.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/ListKubeWorkloadReq.java new file mode 100644 index 0000000000..8d502d643e --- /dev/null +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/ListKubeWorkloadReq.java @@ -0,0 +1,67 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.cc.model.req; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.cc.model.filter.PropertyFilterDTO; +import lombok.Getter; +import lombok.Setter; + +import java.util.List; + +/** + * CMDB API 请求 - 查询 workload + */ +@Getter +@Setter +@JsonInclude(JsonInclude.Include.NON_NULL) +public class ListKubeWorkloadReq extends CmdbPageReq { + + @JsonProperty("bk_biz_id") + private Long bizId; + + /** + * workload类型 + */ + private String kind; + + @JsonProperty("filter") + private PropertyFilterDTO filter; + + + /** + * 属性列表，控制返回结果里有哪些字段，能够加速接口请求和减少网络流量传输 + */ + @JsonProperty("fields") + private List fields; + + /** + * 分页条件 + */ + @JsonProperty("page") + private Page page; + +} diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/Page.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/Page.java new file mode 100644 index 0000000000..6b80f382c4 --- /dev/null +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/Page.java @@ -0,0 +1,82 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.cc.model.req; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; + +/** + * cmdb 请求分页信息 + */ +@Getter +@Setter +@ToString +@NoArgsConstructor +@JsonInclude(JsonInclude.Include.NON_NULL) +public class Page { + /** + * 记录开始位置 + */ + private Integer start; + + /** + * 每页限制条数 + */ + private Integer limit; + + /** + * 排序字段 + */ + private String sort; + /** + * 是否获取查询对象数量的标记。如果此标记为true那么表示此次请求是获取数量，此时其余字段必须为初始化值，start为0，limit为:0，sort为"" + */ + @JsonProperty("enable_count") + private boolean enableCount; + + public Page(Integer start, Integer limit) { + this.start = start; + this.limit = limit; + } + + public Page(Integer start, Integer limit, String sort) { + this.start = start; + this.limit = limit; + this.sort = sort; + } + + /** + * 构造查询记录总数的分页条件 + */ + public static Page buildQueryCountPage() { + Page page = new Page(0, 0, ""); + page.setEnableCount(true); + return page; + } +} diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/ResourceWatchReq.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/ResourceWatchReq.java similarity index 96% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/ResourceWatchReq.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/ResourceWatchReq.java index 6f76400ed7..c3f78e4beb 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/ResourceWatchReq.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/ResourceWatchReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/SearchHostDynamicGroupReq.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/SearchHostDynamicGroupReq.java similarity index 95% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/SearchHostDynamicGroupReq.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/SearchHostDynamicGroupReq.java index fb2f6d0355..98f8e65cdb 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/SearchHostDynamicGroupReq.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/SearchHostDynamicGroupReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/input/GetHostByIpInput.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/input/GetHostByIpInput.java similarity index 95% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/input/GetHostByIpInput.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/input/GetHostByIpInput.java index 5a52cf8734..e7dcba2606 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/req/input/GetHostByIpInput.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/req/input/GetHostByIpInput.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/response/CcCountInfo.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/response/CcCountInfo.java similarity index 90% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/response/CcCountInfo.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/response/CcCountInfo.java index c7aad6f5a7..f0f275140f 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/response/CcCountInfo.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/response/CcCountInfo.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,11 @@ package com.tencent.bk.job.common.cc.model.response; -import lombok.*; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; import java.util.List; import java.util.Map; diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/response/ListResp.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/response/ListResp.java similarity index 95% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/response/ListResp.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/response/ListResp.java index c45b7f8330..460ead0f83 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/response/ListResp.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/response/ListResp.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/BaseCcSearchResult.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/BaseCcSearchResult.java new file mode 100644 index 0000000000..9d63d44d39 --- /dev/null +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/BaseCcSearchResult.java @@ -0,0 +1,35 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.cc.model.result; + +import lombok.Data; + +import java.util.List; + +@Data +public class BaseCcSearchResult { + private int count; + private List info; +} diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/BizEventDetail.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/BizEventDetail.java similarity index 97% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/BizEventDetail.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/BizEventDetail.java index 2205294bb0..d03294d845 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/BizEventDetail.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/BizEventDetail.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/BizSetEventDetail.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/BizSetEventDetail.java similarity index 97% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/BizSetEventDetail.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/BizSetEventDetail.java index 1db627a2fa..d8a8961075 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/BizSetEventDetail.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/BizSetEventDetail.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/BizSetRelationEventDetail.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/BizSetRelationEventDetail.java similarity index 96% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/BizSetRelationEventDetail.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/BizSetRelationEventDetail.java index 82c1774dd0..0436d38222 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/BizSetRelationEventDetail.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/BizSetRelationEventDetail.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/ExecuteDynamicGroupHostResult.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/ExecuteDynamicGroupHostResult.java similarity index 89% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/ExecuteDynamicGroupHostResult.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/ExecuteDynamicGroupHostResult.java index c12fd89d8c..33adde3f2d 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/ExecuteDynamicGroupHostResult.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/ExecuteDynamicGroupHostResult.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,5 +26,5 @@ import com.tencent.bk.job.common.cc.model.CcHostInfoDTO; -public class ExecuteDynamicGroupHostResult extends AbstractCcSearchResult { +public class ExecuteDynamicGroupHostResult extends BaseCcSearchResult { } diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/FindModuleHostRelationResult.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/FindModuleHostRelationResult.java new file mode 100644 index 0000000000..265148ae12 --- /dev/null +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/FindModuleHostRelationResult.java @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.cc.model.result; + +import lombok.Data; + +import java.util.List; + +@Data +public class FindModuleHostRelationResult { + + private Integer count; + + private List relation; + +} diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/GetBizInternalModuleResult.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/GetBizInternalModuleResult.java similarity index 96% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/GetBizInternalModuleResult.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/GetBizInternalModuleResult.java index 7b240019d0..3b991ea575 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/GetBizInternalModuleResult.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/GetBizInternalModuleResult.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/HostBizRelationDTO.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/HostBizRelationDTO.java similarity index 95% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/HostBizRelationDTO.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/HostBizRelationDTO.java index f2be3bc2fd..990f05ab6f 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/HostBizRelationDTO.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/HostBizRelationDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/HostEventDetail.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/HostEventDetail.java new file mode 100644 index 0000000000..1b68bbc6cc --- /dev/null +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/HostEventDetail.java @@ -0,0 +1,93 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.cc.model.result; + + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; +import com.tencent.bk.job.common.util.StringUtil; +import com.tencent.bk.job.common.util.TimeUtil; +import lombok.Data; +import org.apache.commons.lang3.StringUtils; + +import java.util.List; + +@Data +public class HostEventDetail { + + @JsonProperty("bk_host_id") + private Long hostId; + + @JsonProperty("bk_host_innerip") + private String hostInnerIp; + + @JsonProperty("bk_host_innerip_v6") + private String innerIpv6; + + @JsonProperty("bk_agent_id") + private String agentId; + + @JsonProperty("bk_host_name") + private String hostName; + + @JsonProperty("bk_os_name") + private String osName; + + @JsonProperty("bk_os_type") + private String osType; + + @JsonProperty("bk_cloud_id") + private String cloudId; + + @JsonProperty("bk_cloud_vendor") + private String cloudVendorId; + + @JsonProperty("last_time") + private String lastTime; + + public static ApplicationHostDTO toHostInfoDTO(HostEventDetail eventDetail) { + ApplicationHostDTO hostInfoDTO = new ApplicationHostDTO(); + hostInfoDTO.setHostId(eventDetail.hostId); + List ipList = StringUtil.strToList(eventDetail.hostInnerIp, String.class, ","); + hostInfoDTO.setDisplayIp(eventDetail.hostInnerIp); + hostInfoDTO.setIpv6(eventDetail.innerIpv6); + hostInfoDTO.setAgentId(eventDetail.agentId); + hostInfoDTO.setIpList(ipList); + if (ipList != null && !ipList.isEmpty()) { + hostInfoDTO.setIp(ipList.get(0)); + } + hostInfoDTO.setHostName(eventDetail.hostName); + hostInfoDTO.setOsName(eventDetail.osName); + hostInfoDTO.setOsType(eventDetail.osType); + hostInfoDTO.setCloudAreaId(Long.parseLong(eventDetail.getCloudId())); + hostInfoDTO.setCloudVendorId(eventDetail.cloudVendorId); + Long lastTimeMills = null; + if (StringUtils.isNotBlank(eventDetail.getLastTime())) { + lastTimeMills = TimeUtil.parseIsoZonedTimeToMillis(eventDetail.getLastTime()); + } + hostInfoDTO.setLastTime(lastTimeMills); + return hostInfoDTO; + } +} diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/HostProp.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/HostProp.java new file mode 100644 index 0000000000..fd0d081c54 --- /dev/null +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/HostProp.java @@ -0,0 +1,88 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.cc.model.result; + + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; +import com.tencent.bk.job.common.util.StringUtil; +import com.tencent.bk.job.common.util.TimeUtil; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.apache.commons.lang3.StringUtils; + +import java.util.List; + +@Getter +@Setter +@ToString +public class HostProp { + @JsonProperty("bk_host_id") + private Long hostId; + @JsonProperty("bk_host_innerip") + private String ip; + @JsonProperty("bk_host_innerip_v6") + private String ipv6; + @JsonProperty("bk_agent_id") + private String agentId; + @JsonProperty("bk_host_name") + private String hostName; + @JsonProperty("bk_os_name") + private String osName; + @JsonProperty("bk_os_type") + private String osType; + @JsonProperty("bk_cloud_id") + private Long cloudAreaId = 0L; + @JsonProperty("bk_cloud_vendor") + private String cloudVendorId; + @JsonProperty("last_time") + private String lastTime; + + public ApplicationHostDTO toApplicationHostDTO() { + ApplicationHostDTO applicationHostDTO = new ApplicationHostDTO(); + applicationHostDTO.setHostId(hostId); + List ipList = StringUtil.strToList(ip, String.class, ","); + applicationHostDTO.setIpList(ipList); + if (ipList != null && !ipList.isEmpty()) { + applicationHostDTO.setIp(ipList.get(0)); + } + applicationHostDTO.setDisplayIp(ip); + applicationHostDTO.setIpv6(ipv6); + applicationHostDTO.setAgentId(agentId); + int hostNameMaxLength = 2000; + int osNameMaxLength = 512; + int osTypeNameMaxLength = 32; + applicationHostDTO.setHostName(StringUtil.substring(hostName, hostNameMaxLength)); + applicationHostDTO.setOsName(StringUtil.substring(osName, osNameMaxLength)); + applicationHostDTO.setOsType(StringUtil.substring(osType, osTypeNameMaxLength)); + applicationHostDTO.setCloudAreaId(cloudAreaId); + applicationHostDTO.setCloudVendorId(cloudVendorId); + if (StringUtils.isNotBlank(lastTime)) { + applicationHostDTO.setLastTime(TimeUtil.parseIsoZonedTimeToMillis(lastTime)); + } + return applicationHostDTO; + } +} diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/HostRelationEventDetail.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/HostRelationEventDetail.java similarity index 93% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/HostRelationEventDetail.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/HostRelationEventDetail.java index 4a3d6577f3..0de59a94c9 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/HostRelationEventDetail.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/HostRelationEventDetail.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -42,4 +42,7 @@ public class HostRelationEventDetail { @JsonProperty("bk_module_id") private Long moduleId; + + @JsonProperty("last_time") + private String lastTime; } diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/HostWithModules.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/HostWithModules.java new file mode 100644 index 0000000000..e2c0c7beb4 --- /dev/null +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/HostWithModules.java @@ -0,0 +1,40 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.cc.model.result; + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +import java.util.List; + + +@Getter +@Setter +@ToString +public class HostWithModules { + private HostProp host; + private List modules; +} diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/ListBizHostResult.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/ListBizHostResult.java similarity index 90% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/ListBizHostResult.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/ListBizHostResult.java index c6e955d0f4..28ca98afe3 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/ListBizHostResult.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/ListBizHostResult.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -29,5 +29,5 @@ /** * @since 19/12/2019 21:51 */ -public class ListBizHostResult extends AbstractCcSearchResult { +public class ListBizHostResult extends BaseCcSearchResult { } diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/ListBizHostsTopoResult.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/ListBizHostsTopoResult.java similarity index 93% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/ListBizHostsTopoResult.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/ListBizHostsTopoResult.java index ae446a7bf6..f05df0213e 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/ListBizHostsTopoResult.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/ListBizHostsTopoResult.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -31,7 +31,7 @@ import java.util.List; -public class ListBizHostsTopoResult extends AbstractCcSearchResult { +public class ListBizHostsTopoResult extends BaseCcSearchResult { @Getter @Setter @ToString diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/ListHostsWithoutBizResult.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/ListHostsWithoutBizResult.java similarity index 90% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/ListHostsWithoutBizResult.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/ListHostsWithoutBizResult.java index df05c5c0db..76380fabc4 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/ListHostsWithoutBizResult.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/ListHostsWithoutBizResult.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -29,5 +29,5 @@ /** * @since 19/12/2019 21:51 */ -public class ListHostsWithoutBizResult extends AbstractCcSearchResult { +public class ListHostsWithoutBizResult extends BaseCcSearchResult { } diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/ModuleProp.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/ModuleProp.java new file mode 100644 index 0000000000..d0dfeecb32 --- /dev/null +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/ModuleProp.java @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.cc.model.result; + + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +@Getter +@Setter +@ToString +public class ModuleProp { + @JsonProperty("bk_module_id") + private Long moduleId; + @JsonProperty("bk_set_id") + private Long setId; + @JsonProperty("bk_module_type") + private String moduleType; + @JsonProperty("last_time") + private String lastTime; +} diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/ResourceEvent.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/ResourceEvent.java similarity index 96% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/ResourceEvent.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/ResourceEvent.java index 7e2bd032c3..edc160eb6a 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/ResourceEvent.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/ResourceEvent.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/ResourceWatchResult.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/ResourceWatchResult.java similarity index 95% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/ResourceWatchResult.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/ResourceWatchResult.java index fc90b4dd53..ab5de92aaa 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/ResourceWatchResult.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/ResourceWatchResult.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/SearchAppResult.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/SearchAppResult.java similarity index 90% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/SearchAppResult.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/SearchAppResult.java index c7e7cdb03d..2981da85a1 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/SearchAppResult.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/SearchAppResult.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -29,5 +29,5 @@ /** * @date 2019/2/28 */ -public class SearchAppResult extends AbstractCcSearchResult { +public class SearchAppResult extends BaseCcSearchResult { } diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/SearchCloudAreaResult.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/SearchCloudAreaResult.java similarity index 90% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/SearchCloudAreaResult.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/SearchCloudAreaResult.java index 076ef92727..30364e5877 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/SearchCloudAreaResult.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/SearchCloudAreaResult.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -29,5 +29,5 @@ /** * @since 19/12/2019 10:38 */ -public class SearchCloudAreaResult extends AbstractCcSearchResult { +public class SearchCloudAreaResult extends BaseCcSearchResult { } diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/SearchDynamicGroupResult.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/SearchDynamicGroupResult.java similarity index 89% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/SearchDynamicGroupResult.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/SearchDynamicGroupResult.java index 27e0d26be0..ef8a76939e 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/SearchDynamicGroupResult.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/SearchDynamicGroupResult.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,5 +26,5 @@ import com.tencent.bk.job.common.cc.model.CcDynamicGroupDTO; -public class SearchDynamicGroupResult extends AbstractCcSearchResult { +public class SearchDynamicGroupResult extends BaseCcSearchResult { } diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/SearchHostResult.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/SearchHostResult.java similarity index 97% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/SearchHostResult.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/SearchHostResult.java index 994bb7b22c..29337e859a 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/SearchHostResult.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/SearchHostResult.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/SearchPlatResult.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/SearchPlatResult.java similarity index 95% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/SearchPlatResult.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/SearchPlatResult.java index 82f71de09c..4387b6d30b 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/model/result/SearchPlatResult.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/model/result/SearchPlatResult.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/BaseCmdbApiClient.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/BaseCmdbApiClient.java new file mode 100644 index 0000000000..f45675fa04 --- /dev/null +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/BaseCmdbApiClient.java @@ -0,0 +1,232 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.cc.sdk; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.tencent.bk.job.common.cc.config.CmdbConfig; +import com.tencent.bk.job.common.cc.constants.CmdbMetricNames; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.constant.HttpMethodEnum; +import com.tencent.bk.job.common.esb.config.AppProperties; +import com.tencent.bk.job.common.esb.config.BkApiGatewayProperties; +import com.tencent.bk.job.common.esb.config.EsbProperties; +import com.tencent.bk.job.common.esb.constants.ApiGwType; +import com.tencent.bk.job.common.esb.model.BkApiAuthorization; +import com.tencent.bk.job.common.esb.model.EsbReq; +import com.tencent.bk.job.common.esb.model.EsbResp; +import com.tencent.bk.job.common.esb.model.OpenApiRequestInfo; +import com.tencent.bk.job.common.esb.sdk.BkApiClient; +import com.tencent.bk.job.common.exception.InternalCmdbException; +import com.tencent.bk.job.common.exception.InternalException; +import com.tencent.bk.job.common.metrics.CommonMetricNames; +import com.tencent.bk.job.common.util.ApiUtil; +import com.tencent.bk.job.common.util.FlowController; +import com.tencent.bk.job.common.util.http.HttpHelper; +import com.tencent.bk.job.common.util.http.HttpHelperFactory; +import com.tencent.bk.job.common.util.http.HttpMetricUtil; +import com.tencent.bk.job.common.util.http.WatchableHttpHelper; +import com.tencent.bk.job.common.util.json.JsonUtils; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tag; +import lombok.extern.slf4j.Slf4j; +import org.slf4j.LoggerFactory; + +import java.util.HashMap; +import java.util.Map; + +/** + * CMDB API 基础实现 + */ +@Slf4j +public class BaseCmdbApiClient { + protected static final String SEARCH_BIZ_INST_TOPO = "/api/c/compapi/v2/cc/search_biz_inst_topo/"; + protected static final String GET_BIZ_INTERNAL_MODULE = "/api/c/compapi/v2/cc/get_biz_internal_module/"; + protected static final String LIST_BIZ_HOSTS = "/api/c/compapi/v2/cc/list_biz_hosts/"; + protected static final String LIST_HOSTS_WITHOUT_BIZ = "/api/c/compapi/v2/cc/list_hosts_without_biz/"; + protected static final String FIND_HOST_BIZ_RELATIONS = "/api/c/compapi/v2/cc/find_host_biz_relations/"; + protected static final String LIST_BIZ_HOSTS_TOPO = "/api/c/compapi/v2/cc/list_biz_hosts_topo/"; + protected static final String FIND_MODULE_HOST_RELATION = "/api/c/compapi/v2/cc/find_module_host_relation/"; + protected static final String RESOURCE_WATCH = "/api/c/compapi/v2/cc/resource_watch/"; + protected static final String SEARCH_BUSINESS = "/api/c/compapi/v2/cc/search_business/"; + protected static final String GET_CLOUD_AREAS = "/api/c/compapi/v2/cc/search_cloud_area/"; + protected static final String GET_OBJ_ATTRIBUTES = "/api/c/compapi/v2/cc/search_object_attribute/"; + protected static final String GET_TOPO_NODE_PATHS = "/api/c/compapi/v2/cc/find_topo_node_paths/"; + protected static final String SEARCH_DYNAMIC_GROUP = "/api/c/compapi/v2/cc/search_dynamic_group/"; + protected static final String EXECUTE_DYNAMIC_GROUP = "/api/c/compapi/v2/cc/execute_dynamic_group/"; + protected static final String GET_BIZ_BRIEF_CACHE_TOPO = "/api/c/compapi/v2/cc/get_biz_brief_cache_topo/"; + + // 容器相关 API + protected static final String LIST_KUBE_CONTAINER_BY_TOPO = "/api/v3/findmany/kube/container/by_topo"; + protected static final String GET_BIZ_KUBE_CACHE_TOPO = "/api/v3/cache/find/biz/kube/topo"; + protected static final String LIST_KUBE_CLUSTER = "/api/v3/findmany/kube/cluster"; + protected static final String LIST_KUBE_NAMESPACE = "/api/v3/findmany/kube/namespace"; + protected static final String LIST_KUBE_WORKLOAD = "/api/v3/findmany/kube/workload/{kind}"; + + // 业务集相关 API + protected static final String SEARCH_BUSINESS_SET = "/api/c/compapi/v2/cc/list_business_set/"; + protected static final String SEARCH_BIZ_IN_BUSINESS_SET = "/api/c/compapi/v2/cc/list_business_in_business_set/"; + + private static final Map interfaceNameMap = new HashMap<>(); + + protected final String cmdbSupplierAccount; + protected final BkApiAuthorization cmdbBkApiAuthorization; + + /** + * 对整个应用中所有的CMDB调用进行限流 + */ + protected final FlowController globalFlowController; + protected final CmdbConfig cmdbConfig; + /** + * CMDB ESB API 客户端 + */ + protected BkApiClient esbCmdbApiClient; + /** + * CMDB 蓝鲸网关 API 客户端 + */ + protected BkApiClient apiGwCmdbApiClient; + + static { + interfaceNameMap.put(SEARCH_BIZ_INST_TOPO, "search_biz_inst_topo"); + interfaceNameMap.put(GET_BIZ_INTERNAL_MODULE, "get_biz_internal_module"); + interfaceNameMap.put(LIST_BIZ_HOSTS, "list_biz_hosts"); + interfaceNameMap.put(LIST_HOSTS_WITHOUT_BIZ, "list_hosts_without_biz"); + interfaceNameMap.put(FIND_HOST_BIZ_RELATIONS, "find_host_biz_relations"); + interfaceNameMap.put(LIST_BIZ_HOSTS_TOPO, "list_biz_hosts_topo"); + interfaceNameMap.put(FIND_MODULE_HOST_RELATION, "find_module_host_relation"); + interfaceNameMap.put(RESOURCE_WATCH, "resource_watch"); + interfaceNameMap.put(SEARCH_BUSINESS, "search_business"); + interfaceNameMap.put(GET_CLOUD_AREAS, "search_cloud_area"); + interfaceNameMap.put(GET_OBJ_ATTRIBUTES, "search_object_attribute"); + interfaceNameMap.put(GET_TOPO_NODE_PATHS, "find_topo_node_paths"); + interfaceNameMap.put(SEARCH_DYNAMIC_GROUP, "search_dynamic_group"); + interfaceNameMap.put(EXECUTE_DYNAMIC_GROUP, "execute_dynamic_group"); + interfaceNameMap.put(GET_BIZ_BRIEF_CACHE_TOPO, "get_biz_brief_cache_topo"); + interfaceNameMap.put(SEARCH_BUSINESS_SET, "list_business_set"); + interfaceNameMap.put(LIST_KUBE_CONTAINER_BY_TOPO, "list_kube_container_by_topo"); + interfaceNameMap.put(GET_BIZ_KUBE_CACHE_TOPO, "get_biz_kube_cache_topo"); + } + + protected BaseCmdbApiClient(FlowController flowController, + AppProperties appProperties, + EsbProperties esbProperties, + BkApiGatewayProperties bkApiGatewayProperties, + CmdbConfig cmdbConfig, + MeterRegistry meterRegistry, + String lang) { + WatchableHttpHelper httpHelper = HttpHelperFactory.getLongRetryableHttpHelper(); + this.esbCmdbApiClient = new BkApiClient(meterRegistry, + CmdbMetricNames.CMDB_API_PREFIX, + esbProperties.getService().getUrl(), + httpHelper, + lang + ); + this.esbCmdbApiClient.setLogger(LoggerFactory.getLogger(this.getClass())); + this.apiGwCmdbApiClient = new BkApiClient(meterRegistry, + CmdbMetricNames.CMDB_API_PREFIX, + bkApiGatewayProperties.getCmdb().getUrl(), + httpHelper, + lang + ); + this.apiGwCmdbApiClient.setLogger(LoggerFactory.getLogger(this.getClass())); + this.globalFlowController = flowController; + this.cmdbConfig = cmdbConfig; + this.cmdbSupplierAccount = cmdbConfig.getDefaultSupplierAccount(); + this.cmdbBkApiAuthorization = BkApiAuthorization.appAuthorization( + appProperties.getCode(), appProperties.getSecret(), "admin"); + } + + + protected T makeCmdbBaseReq(Class reqClass) { + return EsbReq.buildRequest(reqClass, cmdbSupplierAccount); + } + + protected EsbResp requestCmdbApi(ApiGwType apiGwType, + HttpMethodEnum method, + String uri, + String queryParams, + EsbReq reqBody, + TypeReference> typeReference) { + return requestCmdbApi(apiGwType, method, uri, queryParams, reqBody, typeReference, null); + } + + protected EsbResp requestCmdbApi(ApiGwType apiGwType, + HttpMethodEnum method, + String uri, + String queryParams, + EsbReq reqBody, + TypeReference> typeReference, + HttpHelper httpHelper) { + + String apiName = ApiUtil.getApiNameByUri(interfaceNameMap, uri); + if (cmdbConfig != null && cmdbConfig.getEnableFlowControl()) { + if (globalFlowController != null && globalFlowController.isReady()) { + log.debug("Flow control apiName={}", apiName); + long startTime = System.currentTimeMillis(); + globalFlowController.acquire(apiName); + long duration = System.currentTimeMillis() - startTime; + if (duration >= 5000) { + log.warn("Request resource {} wait flowControl for {}ms", apiName, duration); + } else if (duration >= 1000) { + log.info("Request resource {} wait flowControl for {}ms", apiName, duration); + } + } else { + log.debug("globalFlowController not set or not ready, ignore this time"); + } + } + try { + HttpMetricUtil.setHttpMetricName(CommonMetricNames.ESB_CMDB_API_HTTP); + HttpMetricUtil.addTagForCurrentMetric(Tag.of("api_name", apiName)); + OpenApiRequestInfo requestInfo = OpenApiRequestInfo + .builder() + .method(method) + .uri(uri) + .queryParams(queryParams) + .body(reqBody) + .authorization(cmdbBkApiAuthorization) + .build(); + return getApiClientByApiGwType(apiGwType).doRequest(requestInfo, typeReference, httpHelper); + } catch (Throwable e) { + String errorMsg = "Fail to request CMDB data|method=" + method + "|uri=" + uri + "|queryParams=" + + queryParams + "|body=" + + JsonUtils.toJsonWithoutSkippedFields(JsonUtils.toJsonWithoutSkippedFields(reqBody)); + log.error(errorMsg, e); + throw new InternalCmdbException(e.getMessage(), e, ErrorCode.CMDB_API_DATA_ERROR); + } finally { + HttpMetricUtil.clearHttpMetric(); + } + } + + private BkApiClient getApiClientByApiGwType(ApiGwType apiGwType) { + switch (apiGwType) { + case ESB: + return esbCmdbApiClient; + case BK_APIGW: + return apiGwCmdbApiClient; + default: + log.error("BkApiClient for type: {} not found", apiGwType.name()); + throw new InternalException(ErrorCode.INTERNAL_ERROR); + } + } +} diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/BizCmdbClient.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/BizCmdbClient.java index 7d05577210..037a119639 100644 --- a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/BizCmdbClient.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/BizCmdbClient.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -32,84 +32,108 @@ import com.tencent.bk.job.common.cc.config.CmdbConfig; import com.tencent.bk.job.common.cc.exception.CmdbException; import com.tencent.bk.job.common.cc.model.AppRoleDTO; -import com.tencent.bk.job.common.cc.model.BaseRuleDTO; import com.tencent.bk.job.common.cc.model.BriefTopologyDTO; import com.tencent.bk.job.common.cc.model.BusinessInfoDTO; import com.tencent.bk.job.common.cc.model.CcCloudAreaInfoDTO; import com.tencent.bk.job.common.cc.model.CcCloudIdDTO; import com.tencent.bk.job.common.cc.model.CcDynamicGroupDTO; -import com.tencent.bk.job.common.cc.model.CcGroupHostPropDTO; import com.tencent.bk.job.common.cc.model.CcHostInfoDTO; import com.tencent.bk.job.common.cc.model.CcInstanceDTO; import com.tencent.bk.job.common.cc.model.CcObjAttributeDTO; -import com.tencent.bk.job.common.cc.model.ComposeRuleDTO; +import com.tencent.bk.job.common.cc.model.DynamicGroupHostPropDTO; import com.tencent.bk.job.common.cc.model.InstanceTopologyDTO; -import com.tencent.bk.job.common.cc.model.PropertyFilterDTO; import com.tencent.bk.job.common.cc.model.TopoNodePathDTO; +import com.tencent.bk.job.common.cc.model.bizset.BizFilter; +import com.tencent.bk.job.common.cc.model.bizset.Rule; +import com.tencent.bk.job.common.cc.model.container.ContainerDTO; +import com.tencent.bk.job.common.cc.model.container.ContainerDetailDTO; +import com.tencent.bk.job.common.cc.model.container.KubeClusterDTO; +import com.tencent.bk.job.common.cc.model.container.KubeNamespaceDTO; +import com.tencent.bk.job.common.cc.model.container.KubeNodeID; +import com.tencent.bk.job.common.cc.model.container.KubeTopologyDTO; +import com.tencent.bk.job.common.cc.model.container.KubeWorkloadDTO; +import com.tencent.bk.job.common.cc.model.container.PodDTO; +import com.tencent.bk.job.common.cc.model.filter.BaseRuleDTO; +import com.tencent.bk.job.common.cc.model.filter.ComposeRuleDTO; +import com.tencent.bk.job.common.cc.model.filter.IRule; +import com.tencent.bk.job.common.cc.model.filter.PropertyFilterDTO; +import com.tencent.bk.job.common.cc.model.filter.RuleConditionEnum; +import com.tencent.bk.job.common.cc.model.filter.RuleOperatorEnum; +import com.tencent.bk.job.common.cc.model.query.KubeClusterQuery; +import com.tencent.bk.job.common.cc.model.query.NamespaceQuery; +import com.tencent.bk.job.common.cc.model.query.WorkloadQuery; +import com.tencent.bk.job.common.cc.model.req.CmdbPageReq; import com.tencent.bk.job.common.cc.model.req.ExecuteDynamicGroupReq; import com.tencent.bk.job.common.cc.model.req.FindHostBizRelationsReq; import com.tencent.bk.job.common.cc.model.req.FindModuleHostRelationReq; import com.tencent.bk.job.common.cc.model.req.GetAppReq; import com.tencent.bk.job.common.cc.model.req.GetBizInstTopoReq; import com.tencent.bk.job.common.cc.model.req.GetBizInternalModuleReq; +import com.tencent.bk.job.common.cc.model.req.GetBizKubeCacheTopoReq; import com.tencent.bk.job.common.cc.model.req.GetBriefCacheTopoReq; import com.tencent.bk.job.common.cc.model.req.GetCloudAreaInfoReq; import com.tencent.bk.job.common.cc.model.req.GetObjAttributeReq; import com.tencent.bk.job.common.cc.model.req.GetTopoNodePathReq; import com.tencent.bk.job.common.cc.model.req.ListBizHostReq; import com.tencent.bk.job.common.cc.model.req.ListHostsWithoutBizReq; +import com.tencent.bk.job.common.cc.model.req.ListKubeClusterReq; +import com.tencent.bk.job.common.cc.model.req.ListKubeContainerByTopoReq; +import com.tencent.bk.job.common.cc.model.req.ListKubeNamespaceReq; +import com.tencent.bk.job.common.cc.model.req.ListKubeWorkloadReq; import com.tencent.bk.job.common.cc.model.req.Page; import com.tencent.bk.job.common.cc.model.req.ResourceWatchReq; import com.tencent.bk.job.common.cc.model.req.SearchHostDynamicGroupReq; import com.tencent.bk.job.common.cc.model.req.input.GetHostByIpInput; import com.tencent.bk.job.common.cc.model.response.CcCountInfo; +import com.tencent.bk.job.common.cc.model.result.BaseCcSearchResult; import com.tencent.bk.job.common.cc.model.result.BizEventDetail; import com.tencent.bk.job.common.cc.model.result.ExecuteDynamicGroupHostResult; import com.tencent.bk.job.common.cc.model.result.FindModuleHostRelationResult; import com.tencent.bk.job.common.cc.model.result.GetBizInternalModuleResult; import com.tencent.bk.job.common.cc.model.result.HostBizRelationDTO; import com.tencent.bk.job.common.cc.model.result.HostEventDetail; +import com.tencent.bk.job.common.cc.model.result.HostProp; import com.tencent.bk.job.common.cc.model.result.HostRelationEventDetail; +import com.tencent.bk.job.common.cc.model.result.HostWithModules; import com.tencent.bk.job.common.cc.model.result.ListBizHostResult; import com.tencent.bk.job.common.cc.model.result.ListHostsWithoutBizResult; +import com.tencent.bk.job.common.cc.model.result.ModuleProp; import com.tencent.bk.job.common.cc.model.result.ResourceWatchResult; import com.tencent.bk.job.common.cc.model.result.SearchAppResult; import com.tencent.bk.job.common.cc.model.result.SearchCloudAreaResult; import com.tencent.bk.job.common.cc.model.result.SearchDynamicGroupResult; import com.tencent.bk.job.common.cc.util.TopologyUtil; import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.constant.HttpMethodEnum; import com.tencent.bk.job.common.constant.ResourceScopeTypeEnum; -import com.tencent.bk.job.common.esb.config.EsbConfig; +import com.tencent.bk.job.common.esb.config.AppProperties; +import com.tencent.bk.job.common.esb.config.BkApiGatewayProperties; +import com.tencent.bk.job.common.esb.config.EsbProperties; +import com.tencent.bk.job.common.esb.constants.ApiGwType; import com.tencent.bk.job.common.esb.model.EsbReq; import com.tencent.bk.job.common.esb.model.EsbResp; -import com.tencent.bk.job.common.esb.sdk.AbstractEsbSdkClient; import com.tencent.bk.job.common.exception.InternalCmdbException; import com.tencent.bk.job.common.exception.InternalException; -import com.tencent.bk.job.common.gse.service.QueryAgentStatusClient; -import com.tencent.bk.job.common.metrics.CommonMetricNames; +import com.tencent.bk.job.common.model.PageData; import com.tencent.bk.job.common.model.dto.ApplicationDTO; import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; import com.tencent.bk.job.common.model.dto.HostDTO; -import com.tencent.bk.job.common.model.dto.PageDTO; import com.tencent.bk.job.common.model.dto.ResourceScope; import com.tencent.bk.job.common.model.error.ErrorType; -import com.tencent.bk.job.common.util.ApiUtil; +import com.tencent.bk.job.common.util.CollectionUtil; import com.tencent.bk.job.common.util.FlowController; import com.tencent.bk.job.common.util.JobContextUtil; +import com.tencent.bk.job.common.util.PageUtil; import com.tencent.bk.job.common.util.ThreadUtils; +import com.tencent.bk.job.common.util.TimeUtil; import com.tencent.bk.job.common.util.Utils; -import com.tencent.bk.job.common.util.http.ExtHttpHelper; import com.tencent.bk.job.common.util.http.HttpHelperFactory; -import com.tencent.bk.job.common.util.http.HttpMetricUtil; import com.tencent.bk.job.common.util.json.JsonUtils; import io.micrometer.core.instrument.MeterRegistry; -import io.micrometer.core.instrument.Tag; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; import org.springframework.util.StopWatch; import java.util.ArrayList; @@ -129,31 +153,15 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.locks.ReentrantLock; +import java.util.function.Function; import java.util.stream.Collectors; /** - * ESB-CMDB接口调用客户端 + * CMDB API 调用客户端 */ @Slf4j -public class BizCmdbClient extends AbstractEsbSdkClient implements IBizCmdbClient { - - private static final String SEARCH_BIZ_INST_TOPO = "/api/c/compapi/v2/cc/search_biz_inst_topo/"; - private static final String GET_BIZ_INTERNAL_MODULE = "/api/c/compapi/v2/cc/get_biz_internal_module/"; - private static final String LIST_BIZ_HOSTS = "/api/c/compapi/v2/cc/list_biz_hosts/"; - private static final String LIST_HOSTS_WITHOUT_BIZ = "/api/c/compapi/v2/cc/list_hosts_without_biz/"; - private static final String FIND_HOST_BIZ_RELATIONS = "/api/c/compapi/v2/cc/find_host_biz_relations/"; - private static final String LIST_BIZ_HOSTS_TOPO = "/api/c/compapi/v2/cc/list_biz_hosts_topo/"; - private static final String FIND_MODULE_HOST_RELATION = "/api/c/compapi/v2/cc/find_module_host_relation/"; - private static final String RESOURCE_WATCH = "/api/c/compapi/v2/cc/resource_watch/"; - private static final String SEARCH_BUSINESS = "/api/c/compapi/v2/cc/search_business/"; - private static final String GET_CLOUD_AREAS = "/api/c/compapi/v2/cc/search_cloud_area/"; - private static final String GET_OBJ_ATTRIBUTES = "/api/c/compapi/v2/cc/search_object_attribute/"; - private static final String GET_TOPO_NODE_PATHS = "/api/c/compapi/v2/cc/find_topo_node_paths/"; - private static final String SEARCH_DYNAMIC_GROUP = "/api/c/compapi/v2/cc/search_dynamic_group/"; - private static final String EXECUTE_DYNAMIC_GROUP = "/api/c/compapi/v2/cc/execute_dynamic_group/"; - private static final String GET_BIZ_BRIEF_CACHE_TOPO = "/api/c/compapi/v2/cc/get_biz_brief_cache_topo/"; - - private static final Map interfaceNameMap = new HashMap<>(); +public class BizCmdbClient extends BaseCmdbApiClient implements IBizCmdbClient { + private static final ConcurrentHashMap> bizInstTopoMap = new ConcurrentHashMap<>(); private static final ConcurrentHashMap bizInstTopoLockMap = new ConcurrentHashMap<>(); @@ -162,61 +170,37 @@ public class BizCmdbClient extends AbstractEsbSdkClient implements IBizCmdbClien private static final ConcurrentHashMap bizInternalTopoLockMap = new ConcurrentHashMap<>(); private final ThreadPoolExecutor threadPoolExecutor; private final ThreadPoolExecutor longTermThreadPoolExecutor; - private final CmdbConfig cmdbConfig; - /** - * 对整个应用中所有的CMDB调用进行限流 - */ - private final FlowController globalFlowController; - - static { - interfaceNameMap.put(SEARCH_BIZ_INST_TOPO, "search_biz_inst_topo"); - interfaceNameMap.put(GET_BIZ_INTERNAL_MODULE, "get_biz_internal_module"); - interfaceNameMap.put(LIST_BIZ_HOSTS, "list_biz_hosts"); - interfaceNameMap.put(LIST_HOSTS_WITHOUT_BIZ, "list_hosts_without_biz"); - interfaceNameMap.put(FIND_HOST_BIZ_RELATIONS, "find_host_biz_relations"); - interfaceNameMap.put(LIST_BIZ_HOSTS_TOPO, "list_biz_hosts_topo"); - interfaceNameMap.put(FIND_MODULE_HOST_RELATION, "find_module_host_relation"); - interfaceNameMap.put(RESOURCE_WATCH, "resource_watch"); - interfaceNameMap.put(SEARCH_BUSINESS, "search_business"); - interfaceNameMap.put(GET_CLOUD_AREAS, "search_cloud_area"); - interfaceNameMap.put(GET_OBJ_ATTRIBUTES, "search_object_attribute"); - interfaceNameMap.put(GET_TOPO_NODE_PATHS, "find_topo_node_paths"); - interfaceNameMap.put(SEARCH_DYNAMIC_GROUP, "search_dynamic_group"); - interfaceNameMap.put(EXECUTE_DYNAMIC_GROUP, "execute_dynamic_group"); - interfaceNameMap.put(GET_BIZ_BRIEF_CACHE_TOPO, "get_biz_brief_cache_topo"); - } - - protected String defaultSupplierAccount; - protected String defaultUin = "admin"; - private final QueryAgentStatusClient queryAgentStatusClient; - private final MeterRegistry meterRegistry; + private final LoadingCache bizInstCompleteTopologyCache = CacheBuilder.newBuilder() .maximumSize(1000).expireAfterWrite(30, TimeUnit.SECONDS). - build(new CacheLoader() { - @Override - public InstanceTopologyDTO load(@SuppressWarnings("NullableProblems") Long bizId) { - return getBizInstCompleteTopology(bizId); - } + build(new CacheLoader() { + @Override + public InstanceTopologyDTO load(@SuppressWarnings("NullableProblems") Long bizId) { + return getBizInstCompleteTopology(bizId); } - ); + } + ); - public BizCmdbClient(EsbConfig esbConfig, + public BizCmdbClient(AppProperties appProperties, + EsbProperties esbProperties, + BkApiGatewayProperties bkApiGatewayProperties, CmdbConfig cmdbConfig, String lang, ThreadPoolExecutor threadPoolExecutor, ThreadPoolExecutor longTermThreadPoolExecutor, - QueryAgentStatusClient queryAgentStatusClient, FlowController flowController, MeterRegistry meterRegistry) { - super(esbConfig.getEsbUrl(), esbConfig.getAppCode(), esbConfig.getAppSecret(), lang, - esbConfig.isUseEsbTestEnv()); - this.cmdbConfig = cmdbConfig; - this.defaultSupplierAccount = cmdbConfig.getDefaultSupplierAccount(); + super( + flowController, + appProperties, + esbProperties, + bkApiGatewayProperties, + cmdbConfig, + meterRegistry, + lang + ); this.threadPoolExecutor = threadPoolExecutor; this.longTermThreadPoolExecutor = longTermThreadPoolExecutor; - this.queryAgentStatusClient = queryAgentStatusClient; - this.globalFlowController = flowController; - this.meterRegistry = meterRegistry; } @Override @@ -284,19 +268,29 @@ public InstanceTopologyDTO getBizInstTopologyWithoutInternalTopoWithLock(long bi } public InstanceTopologyDTO getBriefCacheTopo(long bizId) { - GetBriefCacheTopoReq req = makeBaseReq(GetBriefCacheTopoReq.class, defaultUin, defaultSupplierAccount); + GetBriefCacheTopoReq req = makeCmdbBaseReq(GetBriefCacheTopoReq.class); req.setBizId(bizId); - EsbResp esbResp = requestCmdbApi(HttpGet.METHOD_NAME, GET_BIZ_BRIEF_CACHE_TOPO, req, + EsbResp esbResp = requestCmdbApi( + ApiGwType.ESB, + HttpMethodEnum.GET, + GET_BIZ_BRIEF_CACHE_TOPO, + req.toUrlParams(), + null, new TypeReference>() { }); return TopologyUtil.convert(esbResp.getData()); } public InstanceTopologyDTO getBizInstTopologyWithoutInternalTopoFromCMDB(long bizId) { - GetBizInstTopoReq req = makeBaseReq(GetBizInstTopoReq.class, defaultUin, defaultSupplierAccount); + GetBizInstTopoReq req = makeCmdbBaseReq(GetBizInstTopoReq.class); req.setBizId(bizId); - EsbResp> esbResp = requestCmdbApi(HttpGet.METHOD_NAME, SEARCH_BIZ_INST_TOPO, - req, new TypeReference>>() { + EsbResp> esbResp = requestCmdbApi( + ApiGwType.ESB, + HttpMethodEnum.GET, + SEARCH_BIZ_INST_TOPO, + req.toUrlParams(), + null, + new TypeReference>>() { }); if (esbResp.getData().size() > 0) { return esbResp.getData().get(0); @@ -305,56 +299,6 @@ public InstanceTopologyDTO getBizInstTopologyWithoutInternalTopoFromCMDB(long bi } } - public EsbResp requestCmdbApi(String method, - String uri, - EsbReq reqBody, - TypeReference> typeReference) { - return requestCmdbApi(method, uri, reqBody, typeReference, null); - } - - public EsbResp requestCmdbApi(String method, - String uri, - EsbReq reqBody, - TypeReference> typeReference, - ExtHttpHelper httpHelper) { - - String resourceId = ApiUtil.getApiNameByUri(interfaceNameMap, uri); - if (cmdbConfig != null && cmdbConfig.getEnableFlowControl()) { - if (globalFlowController != null && globalFlowController.isReady()) { - log.debug("Flow control resourceId={}", resourceId); - long startTime = System.currentTimeMillis(); - globalFlowController.acquire(resourceId); - long duration = System.currentTimeMillis() - startTime; - if (duration >= 5000) { - log.warn("Request resource {} wait flowControl for {}ms", resourceId, duration); - } else if (duration >= 1000) { - log.info("Request resource {} wait flowControl for {}ms", resourceId, duration); - } - } else { - log.debug("globalFlowController not set or not ready, ignore this time"); - } - } - long start = System.nanoTime(); - String status = "none"; - try { - HttpMetricUtil.setHttpMetricName(CommonMetricNames.ESB_CMDB_API_HTTP); - HttpMetricUtil.addTagForCurrentMetric(Tag.of("api_name", uri)); - EsbResp esbResp = getEsbRespByReq(method, uri, reqBody, typeReference, httpHelper); - status = "ok"; - return esbResp; - } catch (Throwable e) { - String reqStr = JsonUtils.toJsonWithoutSkippedFields(reqBody); - String errorMsg = "Fail to request CMDB data|method=" + method + "|uri=" + uri + "|reqStr=" + reqStr; - log.error(errorMsg, e); - status = "error"; - throw new InternalCmdbException(e.getMessage(), e, ErrorCode.CMDB_API_DATA_ERROR); - } finally { - HttpMetricUtil.clearHttpMetric(); - long end = System.nanoTime(); - meterRegistry.timer(CommonMetricNames.ESB_CMDB_API, "api_name", uri, "status", status) - .record(end - start, TimeUnit.NANOSECONDS); - } - } @Override public InstanceTopologyDTO getBizInternalModule(long bizId) { @@ -392,10 +336,15 @@ public InstanceTopologyDTO getBizInternalModuleWithLock(long bizId) { } public InstanceTopologyDTO getBizInternalModuleFromCMDB(long bizId) { - GetBizInternalModuleReq req = makeBaseReq(GetBizInternalModuleReq.class, defaultUin, defaultSupplierAccount); + GetBizInternalModuleReq req = makeCmdbBaseReq(GetBizInternalModuleReq.class); req.setBizId(bizId); - EsbResp esbResp = requestCmdbApi(HttpGet.METHOD_NAME, - GET_BIZ_INTERNAL_MODULE, req, new TypeReference>() { + EsbResp esbResp = requestCmdbApi( + ApiGwType.ESB, + HttpMethodEnum.GET, + GET_BIZ_INTERNAL_MODULE, + req.toUrlParams(), + null, + new TypeReference>() { }); GetBizInternalModuleResult setInfo = esbResp.getData(); //将结果转换为拓扑树 @@ -428,12 +377,13 @@ public InstanceTopologyDTO getBizInternalModuleFromCMDB(long bizId) { @Override public List getHosts(long bizId, List ccInstList) { - return getHostsByTopology(bizId, ccInstList); + List hostWithModuleList = getHostRelationsByTopology(bizId, ccInstList); + return convertToHostInfoDTOList(bizId, hostWithModuleList); } @Override - public List getHostsByTopology(long bizId, List ccInstList) { - StopWatch watch = new StopWatch("getHostsByTopology"); + public List getHostRelationsByTopology(long bizId, List ccInstList) { + StopWatch watch = new StopWatch("getHostRelationsByTopology"); watch.start("getCachedBizInstCompleteTopology"); InstanceTopologyDTO appCompleteTopology = getCachedBizInstCompleteTopology(bizId); watch.stop(); @@ -453,22 +403,33 @@ public List getHostsByTopology(long bizId, List applicationHostDTOList = findHostByModule(bizId, + watch.start("findHostRelationByModule"); + List hostWithModulesList = findHostRelationByModule(bizId, new ArrayList<>(moduleIdSet)); watch.stop(); if (watch.getTotalTimeMillis() > 1000L) { - log.warn("Get hosts by topo is slow, bizId: {}, ccInsts: {}, watchInfo: {}", bizId, ccInstList, + log.warn("Get hostRelations by topo is slow, bizId: {}, ccInsts: {}, watchInfo: {}", bizId, ccInstList, watch.prettyPrint()); } - return applicationHostDTOList; + hostWithModulesList = hostWithModulesList.stream().filter(hostWithModules -> { + boolean valid = hostWithModules.getHost() != null && hostWithModules.getHost().getHostId() != null; + if (!valid) { + log.warn("Ignore hostWithModules because of null host/hostId:{}", hostWithModules); + } + if (hostWithModules.getModules() == null) { + log.warn("Ignore hostWithModules because of null modules:{}", hostWithModules); + valid = false; + } + return valid; + }).collect(Collectors.toList()); + return hostWithModulesList; } @Override - public List findHostByModule(long bizId, List moduleIdList) { + public List findHostRelationByModule(long bizId, List moduleIdList) { //moduleId分批 - List resultList = new ArrayList<>(); + List resultList = new ArrayList<>(); int batchSize = 200; int start = 0; int end = start + batchSize; @@ -477,7 +438,6 @@ public List findHostByModule(long bizId, List moduleId do { List moduleIdSubList = moduleIdList.subList(start, end); if (moduleIdSubList.size() > 0) { - // 使用find_module_host_relation接口 resultList.addAll(findModuleHostRelationConcurrently(bizId, moduleIdSubList)); } start += batchSize; @@ -488,16 +448,20 @@ public List findHostByModule(long bizId, List moduleId } private FindModuleHostRelationResult getHostsByReq(FindModuleHostRelationReq req) { - EsbResp esbResp = requestCmdbApi(HttpPost.METHOD_NAME, - FIND_MODULE_HOST_RELATION, req, new TypeReference>() { + EsbResp esbResp = requestCmdbApi( + ApiGwType.ESB, + HttpMethodEnum.POST, + FIND_MODULE_HOST_RELATION, + null, + req, + new TypeReference>() { }); return esbResp.getData(); } private FindModuleHostRelationReq genFindModuleHostRelationReq(long bizId, List moduleIdList, int start, int limit) { - FindModuleHostRelationReq req = makeBaseReq(FindModuleHostRelationReq.class, defaultUin, - defaultSupplierAccount); + FindModuleHostRelationReq req = makeCmdbBaseReq(FindModuleHostRelationReq.class); req.setBizId(bizId); req.setModuleIdList(moduleIdList); Page page = new Page(start, limit); @@ -512,20 +476,19 @@ private FindModuleHostRelationReq genFindModuleHostRelationReq(long bizId, List< * @param moduleIdList 模块ID列表 * @return 主机列表 */ - private List findModuleHostRelationConcurrently(long bizId, List moduleIdList) { + private List findModuleHostRelationConcurrently(long bizId, + List moduleIdList) { if (moduleIdList == null || moduleIdList.isEmpty()) { return Collections.emptyList(); } - - List applicationHostDTOList; int start = 0; //已调优 int limit = 500; FindModuleHostRelationReq req = genFindModuleHostRelationReq(bizId, moduleIdList, start, limit); //先拉一次获取总数 FindModuleHostRelationResult pageData = getHostsByReq(req); - List hostWithModulesList = pageData.getRelation(); - LinkedBlockingQueue resultQueue = + List hostWithModulesList = pageData.getRelation(); + LinkedBlockingQueue resultQueue = new LinkedBlockingQueue<>(hostWithModulesList); // 如果该页未达到limit，说明是最后一页 if (pageData.getCount() <= limit) { @@ -562,61 +525,29 @@ private List findModuleHostRelationConcurrently(long bizId, log.warn("bizId {}:{} hosts in total, {} hosts indeed, CMDB interface params invalid", bizId, pageData.getCount(), resultQueue.size()); } - Long startTime = System.currentTimeMillis(); - applicationHostDTOList = convertToHostInfoDTOList(bizId, new ArrayList<>(resultQueue)); - Long endTime = System.currentTimeMillis(); - long timeConsuming = endTime - startTime; - if (timeConsuming >= 1000) { - log.info("convertToHostInfoDTOList time consuming:" + timeConsuming); - } - return applicationHostDTOList; + return new ArrayList<>(resultQueue); } private void fillAgentInfo( ApplicationHostDTO applicationHostDTO, - FindModuleHostRelationResult.HostProp host + HostProp host ) { String multiIp = host.getIp(); multiIp = multiIp.trim(); - if (queryAgentStatusClient != null) { - if (multiIp.contains(",")) { - Pair pair = queryAgentStatusClient.getHostIpWithAgentStatus(multiIp, - host.getCloudAreaId()); - if (pair != null) { - log.debug("query agent status:{}:{}", pair.getLeft(), pair.getRight()); - String ipWithCloudId = pair.getLeft(); - applicationHostDTO.setGseAgentAlive(pair.getRight()); - if (ipWithCloudId.contains(":")) { - String[] arr = ipWithCloudId.split(":"); - applicationHostDTO.setCloudAreaId(Long.parseLong(arr[0])); - applicationHostDTO.setIp(arr[1]); - } else { - applicationHostDTO.setIp(ipWithCloudId); - } - } else { - log.warn("Fail to get agentStatus, host={}", JsonUtils.toJson(host)); - } - } else { - applicationHostDTO.setGseAgentAlive(false); - applicationHostDTO.setCloudAreaId(host.getCloudAreaId()); - applicationHostDTO.setIp(multiIp); - } + applicationHostDTO.setCloudAreaId(host.getCloudAreaId()); + List ipList = Utils.getNotBlankSplitList(multiIp, ","); + if (ipList.size() > 0) { + applicationHostDTO.setIp(ipList.get(0)); } else { - log.warn("queryAgentStatusClient==null, please check!"); - List ipList = Utils.getNotBlankSplitList(multiIp, ","); - if (ipList.size() > 0) { - applicationHostDTO.setIp(ipList.get(0)); - } else { - log.warn("no available ip after queryAgentStatusClient"); - } + log.warn("no available ip, raw multiIp={}", multiIp); } } private ApplicationHostDTO convertToHostInfoDTO( Long bizId, - FindModuleHostRelationResult.HostWithModules hostWithModules + HostWithModules hostWithModules ) { - FindModuleHostRelationResult.HostProp host = hostWithModules.getHost(); + HostProp host = hostWithModules.getHost(); String multiIp = host.getIp(); if (multiIp != null) { multiIp = multiIp.trim(); @@ -627,24 +558,27 @@ private ApplicationHostDTO convertToHostInfoDTO( ApplicationHostDTO applicationHostDTO = new ApplicationHostDTO(); applicationHostDTO.setBizId(bizId); applicationHostDTO.setDisplayIp(multiIp); + applicationHostDTO.setIpv6(host.getIpv6()); + applicationHostDTO.setAgentId(host.getAgentId()); applicationHostDTO.setCloudAreaId(host.getCloudAreaId()); applicationHostDTO.setHostId(host.getHostId()); + applicationHostDTO.setCloudVendorId(host.getCloudVendorId()); fillAgentInfo(applicationHostDTO, host); - List modules = hostWithModules.getModules(); - for (FindModuleHostRelationResult.ModuleProp module : modules) { + List modules = hostWithModules.getModules(); + for (ModuleProp module : modules) { if (module == null || null == module.getModuleId()) { log.warn("invalid host:" + JsonUtils.toJson(applicationHostDTO)); } } - List validModules = + List validModules = hostWithModules.getModules().stream().filter(Objects::nonNull).collect(Collectors.toList()); applicationHostDTO.setModuleId( validModules.stream() - .map(FindModuleHostRelationResult.ModuleProp::getModuleId) + .map(ModuleProp::getModuleId) .collect(Collectors.toList())); applicationHostDTO.setSetId( validModules.stream() - .map(FindModuleHostRelationResult.ModuleProp::getSetId) + .map(ModuleProp::getSetId) .collect(Collectors.toList())); applicationHostDTO.setModuleType(validModules.stream().map(it -> { try { @@ -653,12 +587,12 @@ private ApplicationHostDTO convertToHostInfoDTO( return 0L; } }).collect(Collectors.toList())); - applicationHostDTO.setIpDesc(host.getHostName()); - String os = host.getOs(); - if (os != null && os.length() > 512) { - applicationHostDTO.setOs(os.substring(0, 512)); + applicationHostDTO.setHostName(host.getHostName()); + String osName = host.getOsName(); + if (osName != null && osName.length() > 512) { + applicationHostDTO.setOsName(osName.substring(0, 512)); } else { - applicationHostDTO.setOs(os); + applicationHostDTO.setOsName(osName); } applicationHostDTO.setOsType(host.getOsType()); return applicationHostDTO; @@ -666,60 +600,64 @@ private ApplicationHostDTO convertToHostInfoDTO( private List convertToHostInfoDTOList( long bizId, - List hostWithModulesList) { + List hostWithModulesList + ) { List applicationHostDTOList = new ArrayList<>(); Set ipSet = new HashSet<>(); - for (FindModuleHostRelationResult.HostWithModules hostWithModules : hostWithModulesList) { - FindModuleHostRelationResult.HostProp host = hostWithModules.getHost(); + for (HostWithModules hostWithModules : hostWithModulesList) { + HostProp host = hostWithModules.getHost(); if (host == null) { log.warn("host=null,hostWithTopoInfo={}", JsonUtils.toJson(hostWithModules)); continue; } ipSet.add(host.getCloudAreaId() + ":" + host.getIp()); - String multiIp = host.getIp(); - if (!StringUtils.isBlank(multiIp)) { + Long hostId = host.getHostId(); + if (hostId != null) { ApplicationHostDTO applicationHostDTO = convertToHostInfoDTO(bizId, hostWithModules); applicationHostDTOList.add(applicationHostDTO); } else { - log.info("bk_host_innerip is blank, ignore, hostId={},host={}", host.getHostId(), - JsonUtils.toJson(host)); + log.info("bk_host_id is null, ignore, host={}", JsonUtils.toJson(host)); } } log.info("ipSet.size=" + ipSet.size()); return applicationHostDTOList; } - private ApplicationHostDTO convertHost(long bizId, CcHostInfoDTO hostInfo) { - ApplicationHostDTO ipInfo = new ApplicationHostDTO(); - ipInfo.setHostId(hostInfo.getHostId()); - // 部分从cmdb同步过来的资源没有ip，需要过滤掉 - if (StringUtils.isBlank(hostInfo.getInnerIp())) { + private ApplicationHostDTO convertHost(long bizId, CcHostInfoDTO ccHostInfo) { + // 部分从cmdb同步过来的资源没有hostId，需要过滤掉 + if (ccHostInfo.getHostId() == null) { + log.warn("host with no hostId ignored:{}", ccHostInfo); return null; } + ApplicationHostDTO hostDTO = new ApplicationHostDTO(); + hostDTO.setHostId(ccHostInfo.getHostId()); - if (hostInfo.getOs() != null && hostInfo.getOs().length() > 512) { - ipInfo.setOs(hostInfo.getOs().substring(0, 512)); + if (ccHostInfo.getOs() != null && ccHostInfo.getOs().length() > 512) { + log.warn("osName truncated to 512, host={}", ccHostInfo); + hostDTO.setOsName(ccHostInfo.getOs().substring(0, 512)); } else { - ipInfo.setOs(hostInfo.getOs()); + hostDTO.setOsName(ccHostInfo.getOs()); } - if (hostInfo.getCloudId() != null) { - ipInfo.setCloudAreaId(hostInfo.getCloudId()); + if (ccHostInfo.getCloudId() != null) { + hostDTO.setCloudAreaId(ccHostInfo.getCloudId()); } else { - log.warn("Host does not have cloud area id!|{}", hostInfo); + log.warn("Host does not have cloud area id!|{}", ccHostInfo); return null; } - - ipInfo.setDisplayIp(hostInfo.getInnerIp()); - - if (queryAgentStatusClient != null) { - ipInfo.setIp(queryAgentStatusClient.getHostIpByAgentStatus(hostInfo.getInnerIp(), hostInfo.getCloudId())); - } else { - ipInfo.setIp(hostInfo.getFirstIp()); + hostDTO.setDisplayIp(ccHostInfo.getInnerIp()); + hostDTO.setIp(ccHostInfo.getFirstIp()); + hostDTO.setIpv6(ccHostInfo.getIpv6()); + hostDTO.setAgentId(ccHostInfo.getAgentId()); + hostDTO.setBizId(bizId); + hostDTO.setHostName(ccHostInfo.getHostName()); + hostDTO.setOsType(ccHostInfo.getOsType()); + hostDTO.setCloudVendorId(ccHostInfo.getCloudVendorId()); + Long lastTimeMills = null; + if (StringUtils.isNotBlank(ccHostInfo.getLastTime())) { + lastTimeMills = TimeUtil.parseIsoZonedTimeToMillis(ccHostInfo.getLastTime()); } - ipInfo.setBizId(bizId); - ipInfo.setIpDesc(hostInfo.getHostName()); - - return ipInfo; + hostDTO.setLastTime(lastTimeMills); + return hostDTO; } @Override @@ -730,10 +668,15 @@ public List getAllBizApps() { boolean isLastPage = false; String orderField = "bk_biz_id"; while (!isLastPage) { - GetAppReq req = makeBaseReq(GetAppReq.class, defaultUin, defaultSupplierAccount); - PageDTO page = new PageDTO(start, limit, orderField); + GetAppReq req = makeCmdbBaseReq(GetAppReq.class); + Page page = new Page(start, limit, orderField); req.setPage(page); - EsbResp esbResp = requestCmdbApi(HttpPost.METHOD_NAME, SEARCH_BUSINESS, req, + EsbResp esbResp = requestCmdbApi( + ApiGwType.ESB, + HttpMethodEnum.POST, + SEARCH_BUSINESS, + null, + req, new TypeReference>() { }); SearchAppResult data = esbResp.getData(); @@ -768,13 +711,32 @@ private ApplicationDTO convertToAppInfo(BusinessInfoDTO businessInfo) { return appInfo; } + private List convertToAppInfoList(List businessInfoList) { + List appInfoList = new ArrayList<>(); + for (BusinessInfoDTO businessInfo : businessInfoList) { + ApplicationDTO appInfo = new ApplicationDTO(); + appInfo.setName(businessInfo.getBizName()); + appInfo.setBkSupplierAccount(businessInfo.getSupplierAccount()); + appInfo.setTimeZone(businessInfo.getTimezone()); + appInfo.setScope(new ResourceScope(ResourceScopeTypeEnum.BIZ, businessInfo.getBizId().toString())); + appInfo.setLanguage(businessInfo.getLanguage()); + appInfoList.add(appInfo); + } + return appInfoList; + } + @Override public ApplicationDTO getBizAppById(long bizId) { - GetAppReq req = makeBaseReq(GetAppReq.class, defaultUin, defaultSupplierAccount); + GetAppReq req = makeCmdbBaseReq(GetAppReq.class); Map conditionMap = new HashMap<>(); conditionMap.put("bk_biz_id", bizId); req.setCondition(conditionMap); - EsbResp esbResp = requestCmdbApi(HttpPost.METHOD_NAME, SEARCH_BUSINESS, req, + EsbResp esbResp = requestCmdbApi( + ApiGwType.ESB, + HttpMethodEnum.POST, + SEARCH_BUSINESS, + null, + req, new TypeReference>() { }); SearchAppResult data = esbResp.getData(); @@ -788,10 +750,42 @@ public ApplicationDTO getBizAppById(long bizId) { return convertToAppInfo(businessInfos.get(0)); } + @Override + public List ListBizAppByIds(List bizIds) { + GetAppReq req = makeCmdbBaseReq(GetAppReq.class); + // in查询filter + BizFilter filter = new BizFilter(); + Rule rule = new Rule(); + rule.setField("bk_biz_id"); + rule.setOperator(RuleOperatorEnum.IN.getOperator()); + rule.setValue(bizIds); + filter.setRules(Collections.singletonList(rule)); + filter.setCondition(BizFilter.CONDITION_AND); + req.setBizFilter(filter); + + EsbResp esbResp = requestCmdbApi( + ApiGwType.ESB, + HttpMethodEnum.POST, + SEARCH_BUSINESS, + null, + req, + new TypeReference>() { + }); + SearchAppResult data = esbResp.getData(); + if (data == null) { + throw new InternalCmdbException("data is null", ErrorCode.CMDB_API_DATA_ERROR); + } + List businessInfos = data.getInfo(); + if (businessInfos == null || businessInfos.isEmpty()) { + log.info("Query biz from cmdb through bizIds, return data is null, bizIdz={}", bizIds); + return new ArrayList<>(); + } + return convertToAppInfoList(businessInfos); + } + @Override public List getDynamicGroupList(long bizId) { - SearchHostDynamicGroupReq req = makeBaseReq(SearchHostDynamicGroupReq.class, defaultUin, - defaultSupplierAccount); + SearchHostDynamicGroupReq req = makeCmdbBaseReq(SearchHostDynamicGroupReq.class); req.setBizId(bizId); int start = 0; int limit = 200; @@ -802,8 +796,13 @@ public List getDynamicGroupList(long bizId) { while (!isLastPage) { req.getPage().setStart(start); - EsbResp esbResp = requestCmdbApi(HttpPost.METHOD_NAME, - SEARCH_DYNAMIC_GROUP, req, new TypeReference>() { + EsbResp esbResp = requestCmdbApi( + ApiGwType.ESB, + HttpMethodEnum.POST, + SEARCH_DYNAMIC_GROUP, + null, + req, + new TypeReference>() { }); if (!esbResp.getResult()) { // 由于参数问题导致的CMDB返回数据异常 @@ -827,8 +826,8 @@ public List getDynamicGroupList(long bizId) { return ccDynamicGroupList; } - private List convertToCcGroupHostPropList(List hostInfoList) { - List ccGroupHostList = new ArrayList<>(); + private List convertToCcGroupHostPropList(List hostInfoList) { + List ccGroupHostList = new ArrayList<>(); for (CcHostInfoDTO ccHostInfo : hostInfoList) { if (ccHostInfo.getCloudId() == null || ccHostInfo.getCloudId() < 0) { log.warn( @@ -836,12 +835,8 @@ private List convertToCcGroupHostPropList(List convertToCcGroupHostPropList(List getDynamicGroupIp(long bizId, String groupId) { - ExecuteDynamicGroupReq req = makeBaseReq(ExecuteDynamicGroupReq.class, defaultUin, defaultSupplierAccount); + public List getDynamicGroupIp(long bizId, String groupId) { + ExecuteDynamicGroupReq req = makeCmdbBaseReq(ExecuteDynamicGroupReq.class); req.setBizId(bizId); req.setGroupId(groupId); int limit = 200; int start = 0; req.getPage().setLimit(limit); - List ccGroupHostList = new ArrayList<>(); + List ccGroupHostList = new ArrayList<>(); boolean isLastPage = false; while (!isLastPage) { req.getPage().setStart(start); - EsbResp esbResp = requestCmdbApi(HttpPost.METHOD_NAME, - EXECUTE_DYNAMIC_GROUP, req, new TypeReference>() { + EsbResp esbResp = requestCmdbApi( + ApiGwType.ESB, + HttpMethodEnum.POST, + EXECUTE_DYNAMIC_GROUP, + null, + req, + new TypeReference>() { }); ExecuteDynamicGroupHostResult ccRespData = esbResp.getData(); if (!esbResp.getResult()) { @@ -890,30 +890,56 @@ public List getDynamicGroupIp(long bizId, String groupId) { return ccGroupHostList; } - private CcGroupHostPropDTO convertToCcHost(CcHostInfoDTO ccHostInfo) { - CcGroupHostPropDTO ccGroupHostPropDTO = new CcGroupHostPropDTO(); - ccGroupHostPropDTO.setId(ccHostInfo.getHostId()); - ccGroupHostPropDTO.setName(ccHostInfo.getHostName()); - ccGroupHostPropDTO.setInnerIp(ccHostInfo.getInnerIp()); + private DynamicGroupHostPropDTO convertToCcHost(CcHostInfoDTO ccHostInfo) { + DynamicGroupHostPropDTO dynamicGroupHostPropDTO = new DynamicGroupHostPropDTO(); + dynamicGroupHostPropDTO.setId(ccHostInfo.getHostId()); + dynamicGroupHostPropDTO.setName(ccHostInfo.getHostName()); + dynamicGroupHostPropDTO.setInnerIp(ccHostInfo.getInnerIp()); + dynamicGroupHostPropDTO.setIpv6(ccHostInfo.getIpv6()); + dynamicGroupHostPropDTO.setAgentId(ccHostInfo.getAgentId()); CcCloudIdDTO ccCloudIdDTO = new CcCloudIdDTO(); // 仅使用CloudId其余属性未用到，暂不设置 ccCloudIdDTO.setInstanceId(ccHostInfo.getCloudId()); - ccGroupHostPropDTO.setCloudIdList(Collections.singletonList(ccCloudIdDTO)); - return ccGroupHostPropDTO; + dynamicGroupHostPropDTO.setCloudIdList(Collections.singletonList(ccCloudIdDTO)); + return dynamicGroupHostPropDTO; } @Override public List getCloudAreaList() { + return getCloudAreaByCondition(null); + } + + @Override + public CcCloudAreaInfoDTO getCloudAreaByBkCloudId(Long bkCloudId) { + Map fieldConditions = new HashMap<>(); + fieldConditions.put("bk_cloud_id", bkCloudId); + List cloudAreas = getCloudAreaByCondition(fieldConditions); + if (CollectionUtils.isEmpty(cloudAreas)) { + return null; + } + return cloudAreas.get(0); + } + + private List getCloudAreaByCondition(Map fieldConditions) { List appCloudAreaList = new ArrayList<>(); boolean isLastPage = false; int limit = 200; int start = 0; while (!isLastPage) { - GetCloudAreaInfoReq req = makeBaseReq(GetCloudAreaInfoReq.class, defaultUin, defaultSupplierAccount); - PageDTO page = new PageDTO(start, limit, null); + GetCloudAreaInfoReq req = makeCmdbBaseReq(GetCloudAreaInfoReq.class); + Page page = new Page(start, limit, null); req.setPage(page); - req.setCondition(Collections.emptyMap()); - EsbResp esbResp = requestCmdbApi(HttpPost.METHOD_NAME, GET_CLOUD_AREAS, req, + if (fieldConditions != null && !fieldConditions.isEmpty()) { + req.setCondition(fieldConditions); + } else { + req.setCondition(Collections.emptyMap()); + } + EsbResp esbResp = requestCmdbApi( + ApiGwType.ESB, + HttpMethodEnum.POST, + GET_CLOUD_AREAS, + null, + req, new TypeReference>() { }); SearchCloudAreaResult data = esbResp.getData(); @@ -949,12 +975,23 @@ public List listBizHosts(long bizId, Collection ipL ipList.contains(new HostDTO(host.getCloudAreaId(), host.getIp()))).collect(Collectors.toList()); } + /** + * 根据hostId查询主机业务关系信息 + * + * @param hostIdList 主机id列表，数量<=500 + * @return 主机业务关系列表 + */ @Override - public List findHostBizRelations(String uin, List hostIdList) { - FindHostBizRelationsReq req = makeBaseReq(FindHostBizRelationsReq.class, defaultUin, defaultSupplierAccount); + public List findHostBizRelations(List hostIdList) { + FindHostBizRelationsReq req = makeCmdbBaseReq(FindHostBizRelationsReq.class); req.setHostIdList(hostIdList); - EsbResp> esbResp = requestCmdbApi(HttpPost.METHOD_NAME, - FIND_HOST_BIZ_RELATIONS, req, new TypeReference>>() { + EsbResp> esbResp = requestCmdbApi( + ApiGwType.ESB, + HttpMethodEnum.POST, + FIND_HOST_BIZ_RELATIONS, + null, + req, + new TypeReference>>() { }); List results = esbResp.getData(); if (esbResp.getData() == null) { @@ -966,25 +1003,26 @@ public List findHostBizRelations(String uin, List host @Override public List getHostByIp(GetHostByIpInput input) { List hostInfoList = new ArrayList<>(); - ListBizHostReq req = makeBaseReq(ListBizHostReq.class, defaultUin, defaultSupplierAccount); + ListBizHostReq req = makeCmdbBaseReq(ListBizHostReq.class); req.setBizId(input.getBizId()); PropertyFilterDTO condition = new PropertyFilterDTO(); - condition.setCondition("AND"); + condition.setCondition(RuleConditionEnum.AND.getCondition()); input.ipList.removeIf(StringUtils::isBlank); - BaseRuleDTO baseRuleDTO = new BaseRuleDTO(); - baseRuleDTO.setField("bk_host_innerip"); - baseRuleDTO.setOperator("in"); - baseRuleDTO.setValue(input.ipList); - condition.addRule(baseRuleDTO); + condition.addRule(BaseRuleDTO.in("bk_host_innerip", input.ipList)); req.setCondition(condition); int limit = 200; int start = 0; boolean isLastPage = false; while (!isLastPage) { - PageDTO page = new PageDTO(start, limit, ""); + Page page = new Page(start, limit, ""); req.setPage(page); - EsbResp esbResp = requestCmdbApi(HttpPost.METHOD_NAME, LIST_BIZ_HOSTS, req, + EsbResp esbResp = requestCmdbApi( + ApiGwType.ESB, + HttpMethodEnum.POST, + LIST_BIZ_HOSTS, + null, + req, new TypeReference>() { }); ListBizHostResult pageData = esbResp.getData(); @@ -1008,19 +1046,11 @@ public List getHostByIp(GetHostByIpInput input) { @Override public ApplicationHostDTO getHostByIp(Long cloudAreaId, String ip) { - ListHostsWithoutBizReq req = makeBaseReq(ListHostsWithoutBizReq.class, defaultUin, defaultSupplierAccount); + ListHostsWithoutBizReq req = makeCmdbBaseReq(ListHostsWithoutBizReq.class); PropertyFilterDTO condition = new PropertyFilterDTO(); - condition.setCondition("AND"); - BaseRuleDTO ipRule = new BaseRuleDTO(); - ipRule.setField("bk_host_innerip"); - ipRule.setOperator("equal"); - ipRule.setValue(ip); - condition.addRule(ipRule); - BaseRuleDTO bkCloudIdRule = new BaseRuleDTO(); - bkCloudIdRule.setField("bk_cloud_id"); - bkCloudIdRule.setOperator("equal"); - bkCloudIdRule.setValue(cloudAreaId); - condition.addRule(bkCloudIdRule); + condition.setCondition(RuleConditionEnum.AND.getCondition()); + condition.addRule(BaseRuleDTO.equals("bk_host_innerip", ip)); + condition.addRule(BaseRuleDTO.equals("bk_cloud_id", cloudAreaId)); req.setCondition(condition); List hosts = listHostsWithoutBiz(req); @@ -1028,25 +1058,18 @@ public ApplicationHostDTO getHostByIp(Long cloudAreaId, String ip) { } @Override - public List listHostsByIps(List cloudIps) { - ListHostsWithoutBizReq req = makeBaseReq(ListHostsWithoutBizReq.class, defaultUin, defaultSupplierAccount); + public List listHostsByCloudIps(List cloudIps) { + if (CollectionUtils.isEmpty(cloudIps)) { + return Collections.emptyList(); + } + ListHostsWithoutBizReq req = makeCmdbBaseReq(ListHostsWithoutBizReq.class); PropertyFilterDTO condition = new PropertyFilterDTO(); - condition.setCondition("OR"); + condition.setCondition(RuleConditionEnum.OR.getCondition()); Map> hostGroups = groupHostsByBkCloudId(cloudIps); hostGroups.forEach((bkCloudId, ips) -> { - ComposeRuleDTO hostRule = new ComposeRuleDTO(); - hostRule.setCondition("AND"); - BaseRuleDTO bkCloudIdRule = new BaseRuleDTO(); - bkCloudIdRule.setField("bk_cloud_id"); - bkCloudIdRule.setOperator("equal"); - bkCloudIdRule.setValue(bkCloudId); - hostRule.addRule(bkCloudIdRule); - - BaseRuleDTO ipRule = new BaseRuleDTO(); - ipRule.setField("bk_host_innerip"); - ipRule.setOperator("in"); - ipRule.setValue(ips); - hostRule.addRule(ipRule); + ComposeRuleDTO hostRule = new ComposeRuleDTO(RuleConditionEnum.AND.getCondition()); + hostRule.addRule(buildCloudIdRule(bkCloudId)); + hostRule.addRule(BaseRuleDTO.in("bk_host_innerip", ips)); condition.addRule(hostRule); }); @@ -1055,31 +1078,75 @@ public List listHostsByIps(List cloudIps) { return listHostsWithoutBiz(req); } + @Override + public List listHostsByCloudIpv6s(List cloudIpv6s) { + if (CollectionUtils.isEmpty(cloudIpv6s)) { + return Collections.emptyList(); + } + ListHostsWithoutBizReq req = makeCmdbBaseReq(ListHostsWithoutBizReq.class); + PropertyFilterDTO condition = new PropertyFilterDTO(); + condition.setCondition(RuleConditionEnum.OR.getCondition()); + Map> hostGroups = groupHostsByBkCloudId(cloudIpv6s); + hostGroups.forEach((bkCloudId, ipv6s) -> + ipv6s.forEach(ipv6 -> { + ComposeRuleDTO hostRule = new ComposeRuleDTO(RuleConditionEnum.AND.getCondition()); + hostRule.addRule(buildCloudIdRule(bkCloudId)); + + BaseRuleDTO ipv6Rule = buildIpv6Rule(ipv6); + hostRule.addRule(ipv6Rule); + condition.addRule(hostRule); + })); + req.setCondition(condition); + return listHostsWithoutBiz(req); + } + + private BaseRuleDTO buildIpv6Rule(String ipv6) { + // ipv6字段可能包含多个IPv6地址，故此处使用contains + return BaseRuleDTO.contains("bk_host_innerip_v6", ipv6); + } + + private BaseRuleDTO buildCloudIdRule(Long bkCloudId) { + return BaseRuleDTO.equals("bk_cloud_id", bkCloudId); + } + private List listHostsWithoutBiz(ListHostsWithoutBizReq req) { int limit = 500; int start = 0; - PageDTO page = new PageDTO(start, limit, ""); - req.setPage(page); - EsbResp esbResp = requestCmdbApi(HttpPost.METHOD_NAME, LIST_HOSTS_WITHOUT_BIZ, - req, new TypeReference>() { - }); - ListHostsWithoutBizResult pageData = esbResp.getData(); - if (esbResp.getData() == null || CollectionUtils.isEmpty(esbResp.getData().getInfo())) { - return Collections.emptyList(); - } + int total; + List hosts = new ArrayList<>(); + do { + Page page = new Page(start, limit, ""); + req.setPage(page); + EsbResp esbResp = requestCmdbApi( + ApiGwType.ESB, + HttpMethodEnum.POST, + LIST_HOSTS_WITHOUT_BIZ, + null, + req, + new TypeReference>() { + }); + ListHostsWithoutBizResult pageData = esbResp.getData(); + total = pageData.getCount(); + start += limit; + if (esbResp.getData() == null || CollectionUtils.isEmpty(esbResp.getData().getInfo())) { + break; + } - List hosts = - pageData.getInfo().stream().map(host -> convertHost(-1, host)).collect(Collectors.toList()); + hosts.addAll(pageData.getInfo().stream() + .map(host -> convertHost(-1, host)) + .filter(Objects::nonNull) + .collect(Collectors.toList())); - // 设置主机业务信息 - setBizRelationInfo(hosts); + // 设置主机业务信息 + setBizRelationInfo(hosts); + } while (start < total); return hosts; } private void setBizRelationInfo(List hosts) { List hostIds = hosts.stream().map(ApplicationHostDTO::getHostId).collect(Collectors.toList()); - List hostBizRelations = findHostBizRelations(null, hostIds); + List hostBizRelations = findHostBizRelations(hostIds); Map> hostBizRelationMap = hostBizRelations.stream().collect( Collectors.groupingBy(HostBizRelationDTO::getHostId)); @@ -1104,9 +1171,9 @@ private void setBizRelationInfo(List hosts) { private Map> groupHostsByBkCloudId(List cloudIps) { Map> hostGroup = new HashMap<>(); cloudIps.forEach(cloudIp -> { - String[] cloudIdAndIp = cloudIp.split(":"); - Long bkCloudId = Long.valueOf(cloudIdAndIp[0]); - String ip = cloudIdAndIp[1]; + int i = cloudIp.indexOf(":"); + Long bkCloudId = Long.valueOf(cloudIp.substring(0, i)); + String ip = cloudIp.substring(i + 1); List ipList = hostGroup.computeIfAbsent(bkCloudId, (k) -> new ArrayList<>()); ipList.add(ip); }); @@ -1115,24 +1182,28 @@ private Map> groupHostsByBkCloudId(List cloudIps) { @Override public List listHostsByHostIds(List hostIds) { - ListHostsWithoutBizReq req = makeBaseReq(ListHostsWithoutBizReq.class, defaultUin, defaultSupplierAccount); + if (CollectionUtils.isEmpty(hostIds)) { + return Collections.emptyList(); + } + + ListHostsWithoutBizReq req = makeCmdbBaseReq(ListHostsWithoutBizReq.class); PropertyFilterDTO condition = new PropertyFilterDTO(); - condition.setCondition("AND"); - BaseRuleDTO ipRule = new BaseRuleDTO(); - ipRule.setField("bk_host_id"); - ipRule.setOperator("in"); - ipRule.setValue(hostIds); - condition.addRule(ipRule); + condition.setCondition(RuleConditionEnum.AND.getCondition()); + condition.addRule(BaseRuleDTO.in("bk_host_id", hostIds)); req.setCondition(condition); return listHostsWithoutBiz(req); } @Override public List getObjAttributeList(String objId) { - GetObjAttributeReq req = makeBaseReqByWeb( - GetObjAttributeReq.class, null, defaultUin, defaultSupplierAccount); + GetObjAttributeReq req = makeCmdbBaseReq(GetObjAttributeReq.class); req.setObjId(objId); - EsbResp> esbResp = requestCmdbApi(HttpPost.METHOD_NAME, GET_OBJ_ATTRIBUTES, req, + EsbResp> esbResp = requestCmdbApi( + ApiGwType.ESB, + HttpMethodEnum.POST, + GET_OBJ_ATTRIBUTES, + null, + req, new TypeReference>>() { }); return esbResp.getData(); @@ -1141,12 +1212,17 @@ public List getObjAttributeList(String objId) { @Override public Set listUsersByRole(Long bizId, String role) { CcCountInfo searchResult; - GetAppReq req = makeBaseReqByWeb(GetAppReq.class, null, defaultUin, defaultSupplierAccount); + GetAppReq req = makeCmdbBaseReq(GetAppReq.class); Map condition = new HashMap<>(); condition.put("bk_biz_id", bizId); req.setCondition(condition); req.setFields(Collections.singletonList(role)); - EsbResp esbResp = requestCmdbApi(HttpPost.METHOD_NAME, SEARCH_BUSINESS, req, + EsbResp esbResp = requestCmdbApi( + ApiGwType.ESB, + HttpMethodEnum.POST, + SEARCH_BUSINESS, + null, + req, new TypeReference>() { }); searchResult = esbResp.getData(); @@ -1194,9 +1270,48 @@ public List listRoles() { ).collect(Collectors.toList()); } + @Override + public Map getCloudVendorIdNameMap() { + List esbObjAttributeDTO = getObjAttributeList("host"); + List cloudVendorAttrList = esbObjAttributeDTO.stream().filter(it -> + it.getBkPropertyId().equals("bk_cloud_vendor") + ).collect(Collectors.toList()); + if (CollectionUtils.isEmpty(cloudVendorAttrList)) { + return Collections.emptyMap(); + } + List optionList = parseOptionList(cloudVendorAttrList.get(0).getOption()); + Map map = new HashMap<>(); + for (CcObjAttributeDTO.Option option : optionList) { + map.put(option.getId(), option.getName()); + } + return map; + } + + private List parseOptionList(Object option) { + return JsonUtils.fromJson(JsonUtils.toJson(option), new TypeReference>() { + }); + } + + @Override + public Map getOsTypeIdNameMap() { + List esbObjAttributeDTO = getObjAttributeList("host"); + List osTypeAttrList = esbObjAttributeDTO.stream().filter(it -> + it.getBkPropertyId().equals("bk_os_type") + ).collect(Collectors.toList()); + if (CollectionUtils.isEmpty(osTypeAttrList)) { + return Collections.emptyMap(); + } + List optionList = parseOptionList(osTypeAttrList.get(0).getOption()); + Map map = new HashMap<>(); + for (CcObjAttributeDTO.Option option : optionList) { + map.put(option.getId(), option.getName()); + } + return map; + } + @Override public List getTopoInstancePath(GetTopoNodePathReq getTopoNodePathReq) { - GetTopoNodePathReq req = makeBaseReq(GetTopoNodePathReq.class, defaultUin, defaultSupplierAccount); + GetTopoNodePathReq req = makeCmdbBaseReq(GetTopoNodePathReq.class); // 由于cmdb传入业务节点(topo根节点)会报错，所以job自己处理 List nonAppNodes = new ArrayList<>(); @@ -1214,8 +1329,13 @@ public List getTopoInstancePath(GetTopoNodePathReq getTopoN List hierarchyTopoList = new ArrayList<>(); if (!nonAppNodes.isEmpty()) { - EsbResp> esbResp = requestCmdbApi(HttpPost.METHOD_NAME, GET_TOPO_NODE_PATHS, - req, new TypeReference>>() { + EsbResp> esbResp = requestCmdbApi( + ApiGwType.ESB, + HttpMethodEnum.POST, + GET_TOPO_NODE_PATHS, + null, + req, + new TypeReference>>() { }); if (esbResp == null || esbResp.getData() == null || esbResp.getData().isEmpty()) { return Collections.emptyList(); @@ -1228,10 +1348,10 @@ public List getTopoInstancePath(GetTopoNodePathReq getTopoN hierarchyTopo.setInstanceId(nodePath.getInstanceId()); hierarchyTopo.setInstanceName(nodePath.getObjectName()); if (!CollectionUtils.isEmpty(nodePath.getTopoPaths())) { - List parents = nodePath.getTopoPaths().get(0); - if (!CollectionUtils.isEmpty(parents)) { - Collections.reverse(parents); - parents.forEach(hierarchyTopo::addParent); + List parentNodeList = nodePath.getTopoPaths().get(0); + if (!CollectionUtils.isEmpty(parentNodeList)) { + Collections.reverse(parentNodeList); + parentNodeList.forEach(hierarchyTopo::addParent); } } hierarchyTopoList.add(hierarchyTopo); @@ -1250,55 +1370,70 @@ public List getTopoInstancePath(GetTopoNodePathReq getTopoN @Override public ResourceWatchResult getHostEvents(Long startTime, String cursor) { - ResourceWatchReq req = makeBaseReqByWeb( - ResourceWatchReq.class, null, defaultUin, defaultSupplierAccount); - req.setFields(Arrays.asList("bk_host_id", "bk_host_innerip", "bk_host_name", "bk_os_name", "bk_os_type", - "bk_cloud_id")); + ResourceWatchReq req = makeCmdbBaseReq(ResourceWatchReq.class); + req.setFields(Arrays.asList("bk_host_id", "bk_host_innerip", "bk_host_innerip_v6", "bk_agent_id", + "bk_host_name", "bk_os_name", "bk_os_type", "bk_cloud_id", "bk_cloud_vendor", "last_time")); req.setResource("host"); req.setCursor(cursor); req.setStartTime(startTime); - EsbResp> esbResp = requestCmdbApi(HttpPost.METHOD_NAME, RESOURCE_WATCH, - req, new TypeReference>>() { - }, HttpHelperFactory.getLongRetryableHttpHelper()); + EsbResp> esbResp = requestCmdbApi( + ApiGwType.ESB, + HttpMethodEnum.POST, + RESOURCE_WATCH, + null, + req, + new TypeReference>>() { + }, + HttpHelperFactory.getLongRetryableHttpHelper()); return esbResp.getData(); } @Override public ResourceWatchResult getHostRelationEvents(Long startTime, String cursor) { - ResourceWatchReq req = makeBaseReqByWeb( - ResourceWatchReq.class, null, defaultUin, defaultSupplierAccount); - req.setFields(Arrays.asList("bk_host_id", "bk_biz_id", "bk_set_id", "bk_module_id")); + ResourceWatchReq req = makeCmdbBaseReq(ResourceWatchReq.class); + req.setFields(Arrays.asList("bk_host_id", "bk_biz_id", "bk_set_id", "bk_module_id", "last_time")); req.setResource("host_relation"); req.setCursor(cursor); req.setStartTime(startTime); - EsbResp> esbResp = requestCmdbApi(HttpPost.METHOD_NAME, - RESOURCE_WATCH, req, new TypeReference>>() { - }, HttpHelperFactory.getLongRetryableHttpHelper()); + EsbResp> esbResp = requestCmdbApi( + ApiGwType.ESB, + HttpMethodEnum.POST, + RESOURCE_WATCH, + null, + req, + new TypeReference>>() { + }, + HttpHelperFactory.getLongRetryableHttpHelper()); return esbResp.getData(); } @Override public ResourceWatchResult getAppEvents(Long startTime, String cursor) { - ResourceWatchReq req = makeBaseReqByWeb( - ResourceWatchReq.class, null, defaultUin, defaultSupplierAccount); + ResourceWatchReq req = makeCmdbBaseReq(ResourceWatchReq.class); req.setFields(Arrays.asList("bk_biz_id", "bk_biz_name", "bk_supplier_account", "time_zone", "language")); req.setResource("biz"); req.setCursor(cursor); req.setStartTime(startTime); - EsbResp> esbResp = requestCmdbApi(HttpPost.METHOD_NAME, RESOURCE_WATCH, - req, new TypeReference>>() { - }, HttpHelperFactory.getLongRetryableHttpHelper()); + EsbResp> esbResp = requestCmdbApi( + ApiGwType.ESB, + HttpMethodEnum.POST, + RESOURCE_WATCH, + null, + req, + new TypeReference>>() { + }, + HttpHelperFactory.getLongRetryableHttpHelper()); return esbResp.getData(); } class FindModuleHostRelationTask implements Runnable { //结果队列 - LinkedBlockingQueue resultQueue; + LinkedBlockingQueue resultQueue; FindModuleHostRelationReq req; String requestId; - FindModuleHostRelationTask(LinkedBlockingQueue resultQueue, + FindModuleHostRelationTask(LinkedBlockingQueue resultQueue, FindModuleHostRelationReq req, String requestId) { this.resultQueue = resultQueue; this.req = req; @@ -1315,4 +1450,455 @@ public void run() { } } } + + /** + * 根据业务 ID 查询容器拓扑（缓存) + * + * @param bizId 业务 ID + * @return 容器拓扑 + */ + public KubeTopologyDTO getBizKubeCacheTopo(long bizId) { + GetBizKubeCacheTopoReq req = makeCmdbBaseReq(GetBizKubeCacheTopoReq.class); + req.setBizId(bizId); + + EsbResp esbResp = requestCmdbApi( + ApiGwType.BK_APIGW, + HttpMethodEnum.POST, + GET_BIZ_KUBE_CACHE_TOPO, + null, + req, + new TypeReference>() { + }); + return esbResp.getData(); + } + + /** + * 根据容器拓扑获取container信息(分页) + * + * @param req 请求 + * @return 容器列表（分页） + */ + public PageData listPageKubeContainerByTopo(ListKubeContainerByTopoReq req) { + return listPageKubeContainerByTopo(req, true); + } + + private PageData listPageKubeContainerByTopo(ListKubeContainerByTopoReq req, + boolean withCount) { + setSupplierAccount(req); + req.setContainerFields(ContainerDTO.Fields.ALL); + req.setPodFields(PodDTO.Fields.ALL); + + return listPage( + req, + withCount, + cmdbPageReq -> requestCmdbApi( + ApiGwType.BK_APIGW, + HttpMethodEnum.POST, + LIST_KUBE_CONTAINER_BY_TOPO, + null, + cmdbPageReq, + new TypeReference>>() { + }) + ); + } + + /** + * 根据容器拓扑获取container信息 + * + * @param req 请求 + * @return 容器列表 + */ + public List listKubeContainerByTopo(ListKubeContainerByTopoReq req) { + setSupplierAccount(req); + req.setContainerFields(ContainerDTO.Fields.ALL); + req.setPodFields(PodDTO.Fields.ALL); + // 根据容器 ID 升序排列返回的数据，避免由于分页查询期间数据变更导致返回数据重复或者遗漏 + req.setPage(new Page(0, 500, ContainerDTO.Fields.ID)); + + if (req.getNodeIdList() == null || req.getNodeIdList().size() <= 200) { + return loopPageListKubeContainerByTopo(req); + } else { + // 超过 cmdb API 单次查询最大 node 数量限制，需要按照拓扑节点分批 + List batchReqs = partitionListKubeContainerByTopoReq(req); + return batchReqs.stream() + .flatMap(batchReq -> loopPageListKubeContainerByTopo(batchReq).stream()) + .distinct() + .collect(Collectors.toList()); + } + } + + private List loopPageListKubeContainerByTopo(ListKubeContainerByTopoReq req) { + return PageUtil.queryAllWithLoopPageQueryInOrder( + 500, + (ContainerDetailDTO latestElement) -> { + if (latestElement == null) { + // 第一页使用原始的请求 + return req; + } else { + // 从第二页开始，需要构造 offset 条件，避免由于分页查询期间数据变更导致返回数据重复或者遗漏 + return buildNextPageListKubeContainerByTopoReq(req, latestElement.getContainer().getId()); + } + }, + pageReq -> listPageKubeContainerByTopo(pageReq, false), + PageData::getData, + container -> container + ); + } + + private List partitionListKubeContainerByTopoReq(ListKubeContainerByTopoReq req) { + List reqs = new ArrayList<>(); + List> nodeIDListBatches = + CollectionUtil.partitionCollection(req.getNodeIdList(), 200); + nodeIDListBatches.forEach(nodeIDListBatch -> { + ListKubeContainerByTopoReq newReq = new ListKubeContainerByTopoReq(); + newReq.setBkSupplierAccount(req.getBkSupplierAccount()); + newReq.setBizId(req.getBizId()); + newReq.setNodeIdList(nodeIDListBatch); + newReq.setContainerFilter(req.getContainerFilter()); + newReq.setPodFilter(req.getPodFilter()); + newReq.setPage(req.getPage()); + newReq.setContainerFields(req.getContainerFields()); + newReq.setPodFields(req.getPodFields()); + reqs.add(newReq); + }); + return reqs; + } + + private ListKubeContainerByTopoReq buildNextPageListKubeContainerByTopoReq( + ListKubeContainerByTopoReq originReq, + long lastIdForCurrentPage + ) { + ListKubeContainerByTopoReq nextPageReq = new ListKubeContainerByTopoReq(); + nextPageReq.setPage(originReq.getPage()); + nextPageReq.setNodeIdList(originReq.getNodeIdList()); + nextPageReq.setPodFilter(originReq.getPodFilter()); + nextPageReq.setBizId(originReq.getBizId()); + nextPageReq.setBkSupplierAccount(originReq.getBkSupplierAccount()); + nextPageReq.setPodFields(originReq.getPodFields()); + nextPageReq.setContainerFields(originReq.getContainerFields()); + + // 添加 container ID 作为分页查询 offset 条件 + PropertyFilterDTO rewriteContainerPropertyFilter = addRuleThenReCreate(originReq.getContainerFilter(), + BaseRuleDTO.greaterThan(KubeNamespaceDTO.Fields.ID, lastIdForCurrentPage)); + nextPageReq.setContainerFilter(rewriteContainerPropertyFilter); + + return nextPageReq; + } + + private void setSupplierAccount(EsbReq esbReq) { + if (StringUtils.isEmpty(cmdbSupplierAccount)) { + esbReq.setBkSupplierAccount("0"); + } else { + esbReq.setBkSupplierAccount(cmdbSupplierAccount); + } + } + + /** + * 根据容器ID批量查询容器 + * + * @param bizId CMDB 业务 ID + * @param containerIds 容器 ID 集合 + * @return 容器列表 + */ + public List listKubeContainerByIds(long bizId, Collection containerIds) { + return listBizKubeContainerByContainerFieldWithInCondition( + bizId, ContainerDTO.Fields.ID, containerIds); + } + + /** + * 根据容器ID批量查询容器 + * + * @param bizId CMDB 业务 ID + * @param containerUIds 容器 UID 集合 + * @return 容器列表 + */ + public List listKubeContainerByUIds(long bizId, Collection containerUIds) { + return listBizKubeContainerByContainerFieldWithInCondition( + bizId, ContainerDTO.Fields.CONTAINER_UID, containerUIds); + } + + private List listBizKubeContainerByContainerFieldWithInCondition( + long bizId, + String containerField, + Collection fieldValues) { + + int maxFieldValuesPerBatch = 500; // cmdb 限制每次查询传入的字段值 + List> containerFieldValueBatches = + CollectionUtil.partitionCollection(fieldValues, maxFieldValuesPerBatch); + + List containers = new ArrayList<>(); + + containerFieldValueBatches.forEach(containerFieldValueBatch -> { + ListKubeContainerByTopoReq req = makeCmdbBaseReq(ListKubeContainerByTopoReq.class); + + // 查询条件 + req.setBizId(bizId); + PropertyFilterDTO containerFilter = new PropertyFilterDTO(); + containerFilter.setCondition(RuleConditionEnum.AND.getCondition()); + containerFilter.addRule(BaseRuleDTO.in(containerField, containerFieldValueBatch)); + req.setContainerFilter(containerFilter); + + // 返回参数设置 + req.setContainerFields(ContainerDTO.Fields.ALL); + req.setPodFields(PodDTO.Fields.ALL); + + // 分页设置 + req.setPage(new Page(0, maxFieldValuesPerBatch)); + + PageData pageData = listPageKubeContainerByTopo(req, false); + if (CollectionUtils.isNotEmpty(pageData.getData())) { + containers.addAll(pageData.getData()); + } + }); + + return containers; + } + + public List listKubeClusters(KubeClusterQuery query) { + ListKubeClusterReq req = makeCmdbBaseReq(ListKubeClusterReq.class); + + // 查询条件 + req.setBizId(query.getBizId()); + + PropertyFilterDTO clusterPropFilter = new PropertyFilterDTO(); + clusterPropFilter.setCondition(RuleConditionEnum.AND.getCondition()); + if (CollectionUtils.isNotEmpty(query.getIds())) { + clusterPropFilter.addRule(BaseRuleDTO.in(KubeClusterDTO.Fields.ID, query.getIds())); + } + if (CollectionUtils.isNotEmpty(query.getBkClusterUIDs())) { + clusterPropFilter.addRule(BaseRuleDTO.in(KubeClusterDTO.Fields.CLUSTER_UID, query.getBkClusterUIDs())); + } + if (clusterPropFilter.hasRule()) { + req.setFilter(clusterPropFilter); + } + + // 返回参数设置 + req.setFields(KubeClusterDTO.Fields.ALL); + + // 根据集群 ID 升序排列返回的数据，避免由于分页查询期间数据变更导致返回数据重复或者遗漏 + req.setPage(new Page(0, 500, KubeClusterDTO.Fields.ID)); + + return PageUtil.queryAllWithLoopPageQueryInOrder( + 500, + (KubeClusterDTO latestElement) -> { + if (latestElement == null) { + // 第一页使用原始的请求 + return req; + } else { + // 从第二页开始，需要构造 offset 条件，避免由于分页查询期间数据变更导致返回数据重复或者遗漏 + return buildNextPageListKubeClusterReq(req, latestElement.getId()); + } + }, + cmdbPageReq -> requestCmdbApi( + ApiGwType.BK_APIGW, + HttpMethodEnum.POST, + LIST_KUBE_CLUSTER, + null, + cmdbPageReq, + new TypeReference>>() { + }), + resp -> resp.getData().getInfo(), + cluster -> cluster + ); + } + + private ListKubeClusterReq buildNextPageListKubeClusterReq(ListKubeClusterReq originReq, + long lastIdForCurrentPage) { + ListKubeClusterReq nextPageReq = new ListKubeClusterReq(); + nextPageReq.setPage(originReq.getPage()); + nextPageReq.setBizId(originReq.getBizId()); + nextPageReq.setFields(originReq.getFields()); + nextPageReq.setBkSupplierAccount(originReq.getBkSupplierAccount()); + + // 添加 cluster ID 作为分页查询 offset 条件 + PropertyFilterDTO rewritePropertyFilter = addRuleThenReCreate(originReq.getFilter(), + BaseRuleDTO.greaterThan(KubeClusterDTO.Fields.ID, lastIdForCurrentPage)); + nextPageReq.setFilter(rewritePropertyFilter); + + return nextPageReq; + } + + private PageData listPage(CmdbPageReq req, + boolean withCount, + Function>> query) { + setSupplierAccount(req); + + Page originPage = req.getPage(); + long count = 0; + if (withCount) { + // 设置为查询总数量的分页条件 + req.setPage(Page.buildQueryCountPage()); + + // 查询总数量 + EsbResp> response = query.apply(req); + count = response.getData().getCount(); + } + + // 查询数据 + req.setPage(originPage); + EsbResp> response = query.apply(req); + return new PageData<>(originPage.getStart(), originPage.getLimit(), count, response.getData().getInfo()); + } + + public List listKubeNamespaces(NamespaceQuery query) { + ListKubeNamespaceReq req = makeCmdbBaseReq(ListKubeNamespaceReq.class); + + // 查询条件 + req.setBizId(query.getBizId()); + + PropertyFilterDTO namespacePropFilter = new PropertyFilterDTO(); + namespacePropFilter.setCondition(RuleConditionEnum.AND.getCondition()); + if (CollectionUtils.isNotEmpty(query.getIds())) { + namespacePropFilter.addRule(BaseRuleDTO.in(KubeNamespaceDTO.Fields.ID, query.getIds())); + } + if (CollectionUtils.isNotEmpty(query.getBkClusterIds())) { + namespacePropFilter.addRule(BaseRuleDTO.in(KubeNamespaceDTO.Fields.BK_CLUSTER_ID, query.getBkClusterIds())); + } + if (CollectionUtils.isNotEmpty(query.getNames())) { + namespacePropFilter.addRule(BaseRuleDTO.in(KubeNamespaceDTO.Fields.NAME, query.getNames())); + } + if (namespacePropFilter.hasRule()) { + req.setFilter(namespacePropFilter); + } + + // 返回参数设置 + req.setFields(KubeNamespaceDTO.Fields.ALL); + + // 根据 namespace ID 升序排列返回的数据，避免由于分页查询期间数据变更导致返回数据重复或者遗漏 + req.setPage(new Page(0, 500, KubeNamespaceDTO.Fields.ID)); + + return PageUtil.queryAllWithLoopPageQueryInOrder( + 500, + (KubeNamespaceDTO latestElement) -> { + if (latestElement == null) { + // 第一页使用原始的请求 + return req; + } else { + // 从第二页开始，需要构造 offset 条件，避免由于分页查询期间数据变更导致返回数据重复或者遗漏 + return buildNextPageListKubeNamespaceReq(req, latestElement.getId()); + } + }, + cmdbPageReq -> requestCmdbApi( + ApiGwType.BK_APIGW, + HttpMethodEnum.POST, + LIST_KUBE_NAMESPACE, + null, + cmdbPageReq, + new TypeReference>>() { + }), + resp -> resp.getData().getInfo(), + namespace -> namespace + ); + } + + private ListKubeNamespaceReq buildNextPageListKubeNamespaceReq(ListKubeNamespaceReq originReq, + long lastIdForCurrentPage) { + ListKubeNamespaceReq nextPageReq = new ListKubeNamespaceReq(); + nextPageReq.setPage(originReq.getPage()); + nextPageReq.setBizId(originReq.getBizId()); + nextPageReq.setFields(originReq.getFields()); + nextPageReq.setBkSupplierAccount(originReq.getBkSupplierAccount()); + + // 添加 namespace ID 作为分页查询 offset 条件 + PropertyFilterDTO rewritePropertyFilter = addRuleThenReCreate(originReq.getFilter(), + BaseRuleDTO.greaterThan(KubeNamespaceDTO.Fields.ID, lastIdForCurrentPage)); + nextPageReq.setFilter(rewritePropertyFilter); + + return nextPageReq; + } + + public List listKubeWorkloads(WorkloadQuery query) { + ListKubeWorkloadReq req = makeCmdbBaseReq(ListKubeWorkloadReq.class); + + // 查询条件 + req.setBizId(query.getBizId()); + req.setKind(query.getKind()); + // 根据 Workload ID 升序排列返回的数据，避免由于分页查询期间数据变更导致返回数据重复或者遗漏 + req.setPage(new Page(0, 500, KubeWorkloadDTO.Fields.ID)); + + PropertyFilterDTO workloadPropFilter = new PropertyFilterDTO(); + workloadPropFilter.setCondition(RuleConditionEnum.AND.getCondition()); + if (CollectionUtils.isNotEmpty(query.getIds())) { + workloadPropFilter.addRule(BaseRuleDTO.in(KubeWorkloadDTO.Fields.ID, query.getIds())); + } + if (CollectionUtils.isNotEmpty(query.getBkClusterIds())) { + workloadPropFilter.addRule(BaseRuleDTO.in(KubeWorkloadDTO.Fields.BK_CLUSTER_ID, query.getBkClusterIds())); + } + if (CollectionUtils.isNotEmpty(query.getBkNamespaceIds())) { + workloadPropFilter.addRule(BaseRuleDTO.in(KubeWorkloadDTO.Fields.BK_NAMESPACE_ID, + query.getBkNamespaceIds())); + } + if (CollectionUtils.isNotEmpty(query.getNames())) { + workloadPropFilter.addRule(BaseRuleDTO.in(KubeWorkloadDTO.Fields.NAME, query.getNames())); + } + if (workloadPropFilter.hasRule()) { + req.setFilter(workloadPropFilter); + } + + // 返回参数设置 + req.setFields(KubeWorkloadDTO.Fields.ALL); + + String requestUrl = LIST_KUBE_WORKLOAD.replace("{kind}", query.getKind()); + + return PageUtil.queryAllWithLoopPageQueryInOrder( + 500, + (KubeWorkloadDTO latestElement) -> { + if (latestElement == null) { + // 第一页使用原始的请求 + return req; + } else { + // 从第二页开始，需要构造 offset 条件，避免由于分页查询期间数据变更导致返回数据重复或者遗漏 + return buildNextPageListKubeWorkloadReq(req, latestElement.getId()); + } + }, + cmdbPageReq -> requestCmdbApi( + ApiGwType.BK_APIGW, + HttpMethodEnum.POST, + requestUrl, + null, + cmdbPageReq, + new TypeReference>>() { + }), + resp -> resp.getData().getInfo(), + workload -> { + // cmdb API 返回的数据没有包含 kind 信息，需要补全 + workload.setKind(req.getKind()); + return workload; + } + ); + } + + private ListKubeWorkloadReq buildNextPageListKubeWorkloadReq(ListKubeWorkloadReq originReq, + long lastIdForCurrentPage) { + ListKubeWorkloadReq nextPageReq = new ListKubeWorkloadReq(); + nextPageReq.setPage(originReq.getPage()); + nextPageReq.setBkSupplierAccount(originReq.getBkSupplierAccount()); + nextPageReq.setKind(originReq.getKind()); + nextPageReq.setBizId(originReq.getBizId()); + nextPageReq.setFields(originReq.getFields()); + + // 添加 Workload ID 作为分页查询 offset 条件 + PropertyFilterDTO rewritePropertyFilter = addRuleThenReCreate(originReq.getFilter(), + BaseRuleDTO.greaterThan(KubeWorkloadDTO.Fields.ID, lastIdForCurrentPage)); + nextPageReq.setFilter(rewritePropertyFilter); + + return nextPageReq; + } + + private PropertyFilterDTO addRuleThenReCreate(PropertyFilterDTO originPropertyFilter, IRule rule) { + PropertyFilterDTO newPropertyFilter = new PropertyFilterDTO(); + newPropertyFilter.setCondition(RuleConditionEnum.AND.getCondition()); + List newRules = new ArrayList<>(); + if (originPropertyFilter != null) { + if (originPropertyFilter.getCondition().equals(RuleConditionEnum.OR.getCondition())) { + // 目前场景下属性过滤仅支持 AND + throw new IllegalStateException("Unexpected condition for property filter"); + } + newRules.addAll(originPropertyFilter.getRules()); + } + newRules.add(rule); + newPropertyFilter.setRules(newRules); + + return newPropertyFilter; + } } diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/BizSetCmdbClient.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/BizSetCmdbClient.java index 5bd464d56a..f49c7b954f 100644 --- a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/BizSetCmdbClient.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/BizSetCmdbClient.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -36,21 +36,27 @@ import com.tencent.bk.job.common.cc.model.bizset.SearchBizInBusinessSetResp; import com.tencent.bk.job.common.cc.model.bizset.SearchBizSetReq; import com.tencent.bk.job.common.cc.model.bizset.SearchBizSetResp; +import com.tencent.bk.job.common.cc.model.filter.RuleConditionEnum; +import com.tencent.bk.job.common.cc.model.filter.RuleOperatorEnum; import com.tencent.bk.job.common.cc.model.req.ResourceWatchReq; import com.tencent.bk.job.common.cc.model.result.BizSetEventDetail; import com.tencent.bk.job.common.cc.model.result.BizSetRelationEventDetail; import com.tencent.bk.job.common.cc.model.result.ResourceWatchResult; import com.tencent.bk.job.common.constant.ErrorCode; -import com.tencent.bk.job.common.esb.config.EsbConfig; +import com.tencent.bk.job.common.constant.HttpMethodEnum; +import com.tencent.bk.job.common.esb.config.AppProperties; +import com.tencent.bk.job.common.esb.config.BkApiGatewayProperties; +import com.tencent.bk.job.common.esb.config.EsbProperties; +import com.tencent.bk.job.common.esb.constants.ApiGwType; import com.tencent.bk.job.common.esb.model.EsbReq; import com.tencent.bk.job.common.esb.model.EsbResp; -import com.tencent.bk.job.common.esb.sdk.AbstractEsbSdkClient; import com.tencent.bk.job.common.exception.InternalCmdbException; +import com.tencent.bk.job.common.util.FlowController; import com.tencent.bk.job.common.util.http.HttpHelperFactory; +import io.micrometer.core.instrument.MeterRegistry; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.http.client.methods.HttpPost; import java.util.ArrayList; import java.util.Arrays; @@ -63,22 +69,17 @@ * cmdb API Client - 业务集相关 */ @Slf4j -public class BizSetCmdbClient extends AbstractEsbSdkClient implements IBizSetCmdbClient { +public class BizSetCmdbClient extends BaseCmdbApiClient implements IBizSetCmdbClient { - private final String cmdbSupplierAccount; - private static final String SEARCH_BUSINESS_SET = "/api/c/compapi/v2/cc/list_business_set/"; - private static final String SEARCH_BIZ_IN_BUSINESS_SET = "/api/c/compapi/v2/cc/list_business_in_business_set/"; - private static final String RESOURCE_WATCH = "/api/c/compapi/v2/cc/resource_watch/"; - - public BizSetCmdbClient(EsbConfig esbConfig, CmdbConfig cmdbConfig) { - super(esbConfig.getEsbUrl(), esbConfig.getAppCode(), - esbConfig.getAppSecret(), null, esbConfig.isUseEsbTestEnv()); - this.cmdbSupplierAccount = cmdbConfig.getDefaultSupplierAccount(); - } - - public T makeCmdbBaseReq(Class reqClass) { - return makeBaseReqByWeb(reqClass, null, "admin", cmdbSupplierAccount); + public BizSetCmdbClient(AppProperties appProperties, + EsbProperties esbProperties, + BkApiGatewayProperties bkApiGatewayProperties, + CmdbConfig cmdbConfig, + FlowController flowController, + MeterRegistry meterRegistry) { + super(flowController, appProperties, esbProperties, + bkApiGatewayProperties, cmdbConfig, meterRegistry, null); } /** @@ -87,7 +88,7 @@ public T makeCmdbBaseReq(Class reqClass) { * @return 业务集数量 */ public int searchBizSetCount() { - SearchBizSetReq req = makeCmdbBaseReq(SearchBizSetReq.class); + SearchBizSetReq req = EsbReq.buildRequest(SearchBizSetReq.class, cmdbSupplierAccount); Page page = new Page(); page.setStart(0); page.setLimit(0); @@ -95,9 +96,11 @@ public int searchBizSetCount() { req.setPage(page); req.setFilter(null); try { - EsbResp resp = getEsbRespByReq( - HttpPost.METHOD_NAME, + EsbResp resp = requestCmdbApi( + ApiGwType.ESB, + HttpMethodEnum.POST, SEARCH_BUSINESS_SET, + null, req, new TypeReference>() { }); @@ -129,6 +132,36 @@ public List searchAllBizSet() { return bizSetInfoList; } + /** + * 查询业务集信息 + * + * @return 业务集信息列表 + */ + public List ListBizSetByIds(List bizSetIds) { + BizSetFilter filter = new BizSetFilter(); + filter.setCondition(BizSetFilter.CONDITION_AND); + Rule bizSetIdRule = new Rule(); + bizSetIdRule.setField("bk_biz_set_id"); + bizSetIdRule.setOperator(RuleOperatorEnum.IN.getOperator()); + bizSetIdRule.setValue(bizSetIds); + filter.setRules(Collections.singletonList(bizSetIdRule)); + List bizSetInfoList = searchBizSet(filter, 0, bizSetIds.size()); + + if (bizSetInfoList == null) { + return new ArrayList<>(); + } + + bizSetInfoList.forEach(bizSetInfo -> { + // 查询业务集下包含的子业务(全业务除外) + BizSetScope scope = bizSetInfo.getScope(); + if (scope != null && !scope.isMatchAll()) { + List bizList = searchBizInBizSet(bizSetInfo.getId()); + bizSetInfo.setBizList(bizList); + } + }); + return bizSetInfoList; + } + /** * 查询业务集信息 * @@ -146,9 +179,11 @@ private List searchBizSet(BizSetFilter filter, int start, int limit) req.setPage(page); req.setFilter(filter); try { - EsbResp resp = getEsbRespByReq( - HttpPost.METHOD_NAME, + EsbResp resp = requestCmdbApi( + ApiGwType.ESB, + HttpMethodEnum.POST, SEARCH_BUSINESS_SET, + null, req, new TypeReference>() { }); @@ -176,9 +211,11 @@ private int searchBizCountInBusinessSet(long bizSetId) { page.setLimit(0); req.setPage(page); try { - EsbResp resp = getEsbRespByReq( - HttpPost.METHOD_NAME, + EsbResp resp = requestCmdbApi( + ApiGwType.ESB, + HttpMethodEnum.POST, SEARCH_BIZ_IN_BUSINESS_SET, + null, req, new TypeReference>() { }); @@ -219,9 +256,11 @@ private List searchBizInBizSet(long bizSetId, int start, int limit) { page.setLimit(limit); req.setPage(page); try { - EsbResp resp = getEsbRespByReq( - HttpPost.METHOD_NAME, + EsbResp resp = requestCmdbApi( + ApiGwType.ESB, + HttpMethodEnum.POST, SEARCH_BIZ_IN_BUSINESS_SET, + null, req, new TypeReference>() { }); @@ -237,7 +276,7 @@ private List searchBizInBizSet(long bizSetId, int start, int limit) { @Override public List listAllBizSets() { List bizSetInfoList = searchAllBizSet(); - bizSetInfoList.parallelStream().forEach(bizSetInfo -> { + bizSetInfoList.forEach(bizSetInfo -> { // 查询业务集下包含的子业务(全业务除外) BizSetScope scope = bizSetInfo.getScope(); if (scope != null && !scope.isMatchAll()) { @@ -258,9 +297,11 @@ public ResourceWatchResult getBizSetEvents(Long startTime, St req.setCursor(cursor); req.setStartTime(startTime); try { - EsbResp> resp = getEsbRespByReq( - HttpPost.METHOD_NAME, + EsbResp> resp = requestCmdbApi( + ApiGwType.ESB, + HttpMethodEnum.POST, RESOURCE_WATCH, + null, req, new TypeReference>>() { }, @@ -282,9 +323,11 @@ public ResourceWatchResult getBizSetRelationEvents(Lo req.setCursor(cursor); req.setStartTime(startTime); try { - EsbResp> resp = getEsbRespByReq( - HttpPost.METHOD_NAME, + EsbResp> resp = requestCmdbApi( + ApiGwType.ESB, + HttpMethodEnum.POST, RESOURCE_WATCH, + null, req, new TypeReference>>() { }, @@ -327,7 +370,7 @@ public Set listUsersByRole(Long bizSetId, String role) { @Override public BizSetInfo queryBizSet(Long bizSetId) { BizSetFilter filter = new BizSetFilter(); - filter.setCondition("AND"); + filter.setCondition(RuleConditionEnum.AND.getCondition()); Rule bizSetIdRule = new Rule(); bizSetIdRule.setField("bk_biz_set_id"); bizSetIdRule.setOperator("equal"); diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/BkNetClient.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/BkNetClient.java new file mode 100644 index 0000000000..71bc1cbef7 --- /dev/null +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/BkNetClient.java @@ -0,0 +1,167 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.cc.sdk; + +import com.tencent.bk.job.common.cc.model.CcCloudAreaInfoDTO; +import com.tencent.bk.job.common.util.StringUtil; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.MapUtils; +import org.apache.commons.lang3.StringUtils; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +/** + * 管控区域 Client + */ +@Slf4j +public class BkNetClient { + private static final Map CLOUD_AREA_NAME_MAP = new ConcurrentHashMap<>(); + private static IBizCmdbClient bizCmdbClient; + private static List fullCloudAreaInfoList; + + public BkNetClient(IBizCmdbClient bizCmdbClient) { + BkNetClient.bizCmdbClient = bizCmdbClient; + CloudAreaNameCacheThread cloudAreaNameCacheThread = new CloudAreaNameCacheThread(); + cloudAreaNameCacheThread.start(); + } + + public static String getCloudAreaNameFromCache(Long cloudAreaId) { + if (MapUtils.isNotEmpty(CLOUD_AREA_NAME_MAP)) { + if (StringUtils.isNotBlank(CLOUD_AREA_NAME_MAP.get(cloudAreaId))) { + return CLOUD_AREA_NAME_MAP.get(cloudAreaId); + } + } + return String.valueOf(cloudAreaId); + } + + private static List getCloudAreaListFromCc() { + List cloudAreaInfoList = bizCmdbClient.getCloudAreaList(); + if (cloudAreaInfoList == null) { + return new ArrayList<>(); + } + return cloudAreaInfoList; + } + + public String getCloudAreaName(Long cloudAreaId) { + if (cloudAreaId == null) { + return "Unknown"; + } + + if (MapUtils.isNotEmpty(CLOUD_AREA_NAME_MAP)) { + if (StringUtils.isNotBlank(CLOUD_AREA_NAME_MAP.get(cloudAreaId))) { + return CLOUD_AREA_NAME_MAP.get(cloudAreaId); + } + } else { + List cloudAreaInfoList = getCloudAreaList(); + if (cloudAreaInfoList != null) { + for (CcCloudAreaInfoDTO cloudAreaInfo : cloudAreaInfoList) { + if (cloudAreaId.equals(cloudAreaInfo.getId())) { + if (StringUtils.isNotBlank(cloudAreaInfo.getName())) { + return cloudAreaInfo.getName(); + } else { + return String.valueOf(cloudAreaId); + } + } + } + } + } + return String.valueOf(cloudAreaId); + } + + public List getCloudAreaList() { + return fullCloudAreaInfoList; + } + + private List getAllCloudAreaIds() { + return getCloudAreaList().stream() + .map(CcCloudAreaInfoDTO::getId) + .collect(Collectors.toList()); + } + + /** + * 查找名称符合搜索关键字（与任意一个关键字匹配即可）的云区域，并返回其ID + * + * @param searchContents 搜索关键字集合 + * @return 符合条件的云区域ID列表 + */ + public List getAnyNameMatchedCloudAreaIds(Collection searchContents) { + if (searchContents == null) { + return getAllCloudAreaIds(); + } + List cloudAreaIds; + cloudAreaIds = new ArrayList<>(); + List allCloudAreaInfos = getCloudAreaList(); + for (CcCloudAreaInfoDTO it : allCloudAreaInfos) { + if (StringUtil.matchAnySearchContent(it.getName(), searchContents)) { + cloudAreaIds.add(it.getId()); + } + } + log.debug("filter by cloudAreaIds={}", cloudAreaIds); + return cloudAreaIds; + } + + static class CloudAreaNameCacheThread extends Thread { + @Override + public void run() { + this.setName("Cloud-Area-Info-Sync-Thread"); + while (true) { + long start = System.currentTimeMillis(); + try { + log.debug("Cloud area info syncing start..."); + List cloudAreaInfoList = getCloudAreaListFromCc(); + if (CollectionUtils.isNotEmpty(cloudAreaInfoList)) { + Iterator cloudAreaInfoIterator = cloudAreaInfoList.iterator(); + while (cloudAreaInfoIterator.hasNext()) { + CcCloudAreaInfoDTO cloudAreaInfo = cloudAreaInfoIterator.next(); + if (cloudAreaInfo != null && cloudAreaInfo.getId() != null + && StringUtils.isNotBlank(cloudAreaInfo.getName())) { + CLOUD_AREA_NAME_MAP.put(cloudAreaInfo.getId(), cloudAreaInfo.getName()); + } else { + cloudAreaInfoIterator.remove(); + } + } + fullCloudAreaInfoList = cloudAreaInfoList; + } + log.debug("Cloud area info syncing finished in {}!", System.currentTimeMillis() - start); + } catch (Exception e) { + log.error("Error while process cloud area name!", e); + } + try { + Thread.sleep(60_000L); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + } + } + +} diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/CmdbClientFactory.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/CmdbClientFactory.java index f81e802147..10cdde7d60 100644 --- a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/CmdbClientFactory.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/CmdbClientFactory.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/IBizCmdbClient.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/IBizCmdbClient.java index 502d95105b..2dca91a6f0 100644 --- a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/IBizCmdbClient.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/IBizCmdbClient.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,9 +27,9 @@ import com.tencent.bk.job.common.cc.model.AppRoleDTO; import com.tencent.bk.job.common.cc.model.CcCloudAreaInfoDTO; import com.tencent.bk.job.common.cc.model.CcDynamicGroupDTO; -import com.tencent.bk.job.common.cc.model.CcGroupHostPropDTO; import com.tencent.bk.job.common.cc.model.CcInstanceDTO; import com.tencent.bk.job.common.cc.model.CcObjAttributeDTO; +import com.tencent.bk.job.common.cc.model.DynamicGroupHostPropDTO; import com.tencent.bk.job.common.cc.model.InstanceTopologyDTO; import com.tencent.bk.job.common.cc.model.req.GetTopoNodePathReq; import com.tencent.bk.job.common.cc.model.req.input.GetHostByIpInput; @@ -37,6 +37,7 @@ import com.tencent.bk.job.common.cc.model.result.HostBizRelationDTO; import com.tencent.bk.job.common.cc.model.result.HostEventDetail; import com.tencent.bk.job.common.cc.model.result.HostRelationEventDetail; +import com.tencent.bk.job.common.cc.model.result.HostWithModules; import com.tencent.bk.job.common.cc.model.result.ResourceWatchResult; import com.tencent.bk.job.common.model.dto.ApplicationDTO; import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; @@ -44,6 +45,7 @@ import java.util.Collection; import java.util.List; +import java.util.Map; import java.util.Set; /** @@ -82,22 +84,22 @@ public interface IBizCmdbClient { List getHosts(long bizId, List ccInstList); /** - * 根据topo实例获取hosts + * 根据topo实例获取主机及主机关系 * * @param bizId cmdb业务ID * @param ccInstList topo节点列表 * @return 主机列表 */ - List getHostsByTopology(long bizId, List ccInstList); + List getHostRelationsByTopology(long bizId, List ccInstList); /** - * 根据module获取hosts + * 根据module获取主机及主机关系 * * @param bizId cmdb业务ID * @param moduleIdList 模块ID列表 * @return 主机 */ - List findHostByModule(long bizId, List moduleIdList); + List findHostRelationByModule(long bizId, List moduleIdList); /** * 从CC获取所有业务信息 @@ -114,6 +116,14 @@ public interface IBizCmdbClient { */ ApplicationDTO getBizAppById(long bizId); + /** + * 查询业务列表 + * + * @param bizIds cmdb业务ID + * @return 业务 + */ + List ListBizAppByIds(List bizIds); + /** * 查询业务下的动态分组 * @@ -129,7 +139,7 @@ public interface IBizCmdbClient { * @param groupId 动态分组ID * @return 动态分组下的主机 */ - List getDynamicGroupIp(long bizId, String groupId); + List getDynamicGroupIp(long bizId, String groupId); /** * 获取云区域 @@ -138,6 +148,14 @@ public interface IBizCmdbClient { */ List getCloudAreaList(); + /** + * 根据云区域 ID 获取云区域 + * + * @param bkCloudId 云区域 ID + * @return 云区域 + */ + CcCloudAreaInfoDTO getCloudAreaByBkCloudId(Long bkCloudId); + /** * 根据IP查询主机 * @@ -161,10 +179,18 @@ public interface IBizCmdbClient { * @param cloudIps 云区域+IP列表 * @return 主机列表 */ - List listHostsByIps(List cloudIps); + List listHostsByCloudIps(List cloudIps); /** - * 根据IP批量获取主机 + * 根据IPv6批量获取主机 + * + * @param cloudIpv6s 云区域+IPv6列表 + * @return 主机列表 + */ + List listHostsByCloudIpv6s(List cloudIpv6s); + + /** + * 根据HostId批量获取主机 * * @param hostIds 主机ID列表 * @return 主机列表 @@ -180,7 +206,7 @@ public interface IBizCmdbClient { */ List listBizHosts(long bizId, Collection ipList); - List findHostBizRelations(String uin, List hostIdList); + List findHostBizRelations(List hostIdList); /** * 获取CMDB对象属性信息列表 @@ -202,6 +228,16 @@ public interface IBizCmdbClient { */ List listRoles(); + /** + * 获取主机所属云厂商枚举值，key:id，value:名称 + */ + Map getCloudVendorIdNameMap(); + + /** + * 获取主机类型枚举值，key:id，value:名称 + */ + Map getOsTypeIdNameMap(); + /** * 批量获取topo节点层级 * diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/IBizSetCmdbClient.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/IBizSetCmdbClient.java index 898b138465..656067823d 100644 --- a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/IBizSetCmdbClient.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/sdk/IBizSetCmdbClient.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -52,6 +52,14 @@ public interface IBizSetCmdbClient { */ List listAllBizSets(); + /** + * 从CC获取业务集信息 + * + * @param bizSetIds 业务集ID + * @return 业务集列表 + */ + List ListBizSetByIds(List bizSetIds); + /** * 根据游标获取业务集事件 * diff --git a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/service/CloudAreaService.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/service/CloudAreaService.java deleted file mode 100644 index c7ad08afde..0000000000 --- a/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/service/CloudAreaService.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.cc.service; - -import com.tencent.bk.job.common.cc.model.CcCloudAreaInfoDTO; -import com.tencent.bk.job.common.cc.sdk.IBizCmdbClient; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.collections4.MapUtils; -import org.apache.commons.lang3.StringUtils; -import org.springframework.stereotype.Service; - -import java.util.ArrayList; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; - -/** - * 云区域信息服务 - *

- * 使用前务必在任何一个类引入一次，否则不会初始化 - * - * @since 23/12/2019 22:48 - */ - -@Slf4j -@Service -public class CloudAreaService { - private static final Map CLOUD_AREA_NAME_MAP = new ConcurrentHashMap<>(); - private static IBizCmdbClient bizCmdbClient; - private static List fullCloudAreaInfoList; - - public CloudAreaService(IBizCmdbClient bizCmdbClient) { - CloudAreaService.bizCmdbClient = bizCmdbClient; - CloudAreaNameCacheThread cloudAreaNameCacheThread = new CloudAreaNameCacheThread(); - cloudAreaNameCacheThread.start(); - } - - public static String getCloudAreaNameFromCache(Long cloudAreaId) { - if (MapUtils.isNotEmpty(CLOUD_AREA_NAME_MAP)) { - if (StringUtils.isNotBlank(CLOUD_AREA_NAME_MAP.get(cloudAreaId))) { - return CLOUD_AREA_NAME_MAP.get(cloudAreaId); - } - } - return String.valueOf(cloudAreaId); - } - - private static List getCloudAreaListFromCc() { - List cloudAreaInfoList = bizCmdbClient.getCloudAreaList(); - if (cloudAreaInfoList == null) { - return new ArrayList<>(); - } - return cloudAreaInfoList; - } - - public String getCloudAreaName(Long cloudAreaId) { - if (cloudAreaId == null) { - return "Unknown"; - } - - if (MapUtils.isNotEmpty(CLOUD_AREA_NAME_MAP)) { - if (StringUtils.isNotBlank(CLOUD_AREA_NAME_MAP.get(cloudAreaId))) { - return CLOUD_AREA_NAME_MAP.get(cloudAreaId); - } - } else { - List cloudAreaInfoList = getCloudAreaList(); - if (cloudAreaInfoList != null) { - for (CcCloudAreaInfoDTO cloudAreaInfo : cloudAreaInfoList) { - if (cloudAreaId.equals(cloudAreaInfo.getId())) { - if (StringUtils.isNotBlank(cloudAreaInfo.getName())) { - return cloudAreaInfo.getName(); - } else { - return String.valueOf(cloudAreaId); - } - } - } - } - } - return String.valueOf(cloudAreaId); - } - - public List getCloudAreaList() { - return fullCloudAreaInfoList; - } - - static class CloudAreaNameCacheThread extends Thread { - @Override - public void run() { - this.setName("Cloud-Area-Info-Sync-Thread"); - while (true) { - String uuid = UUID.randomUUID().toString(); - long start = System.currentTimeMillis(); - try { - log.debug("{}|Cloud area info syncing start...", uuid); - List cloudAreaInfoList = getCloudAreaListFromCc(); - if (CollectionUtils.isNotEmpty(cloudAreaInfoList)) { - Iterator cloudAreaInfoIterator = cloudAreaInfoList.iterator(); - while (cloudAreaInfoIterator.hasNext()) { - CcCloudAreaInfoDTO cloudAreaInfo = cloudAreaInfoIterator.next(); - if (cloudAreaInfo != null && cloudAreaInfo.getId() != null - && StringUtils.isNotBlank(cloudAreaInfo.getName())) { - CLOUD_AREA_NAME_MAP.put(cloudAreaInfo.getId(), cloudAreaInfo.getName()); - } else { - cloudAreaInfoIterator.remove(); - } - } - fullCloudAreaInfoList = cloudAreaInfoList; - } - log.debug("{}|Cloud area info syncing finished in {}!", uuid, System.currentTimeMillis() - start); - } catch (Exception e) { - log.error("{}|Error while process cloud area name!", uuid, e); - } - try { - Thread.sleep(60_000L); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - } - } - -} diff --git a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/util/TopologyUtil.java b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/util/TopologyUtil.java similarity index 99% rename from src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/util/TopologyUtil.java rename to src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/util/TopologyUtil.java index ecd39ed289..0c37a4bcce 100644 --- a/src/backend/commons/cmdb-sdk-model/src/main/java/com/tencent/bk/job/common/cc/util/TopologyUtil.java +++ b/src/backend/commons/cmdb-sdk/src/main/java/com/tencent/bk/job/common/cc/util/TopologyUtil.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/cmdb-sdk/src/main/resources/META-INF/spring.factories b/src/backend/commons/cmdb-sdk/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..efaa584316 --- /dev/null +++ b/src/backend/commons/cmdb-sdk/src/main/resources/META-INF/spring.factories @@ -0,0 +1,4 @@ +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ +com.tencent.bk.job.common.cc.config.CmdbAutoConfiguration,\ +com.tencent.bk.job.common.cc.config.CmdbFlowControlAutoConfiguration,\ +com.tencent.bk.job.common.cc.config.CmdbMetricsAutoConfiguration diff --git a/src/backend/commons/common-api/build.gradle b/src/backend/commons/common-api/build.gradle index 8f77a83dde..bbbdc614d1 100644 --- a/src/backend/commons/common-api/build.gradle +++ b/src/backend/commons/common-api/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,12 +24,13 @@ dependencies { api project(':commons:common') + api 'org.springframework.boot:spring-boot-starter-web' + api 'org.springframework.cloud:spring-cloud-starter-openfeign' + api 'io.springfox:springfox-boot-starter' api 'org.hibernate.validator:hibernate-validator' api 'jakarta.validation:jakarta.validation-api' api 'ch.qos.logback:logback-core' api 'ch.qos.logback:logback-classic' - api 'io.springfox:springfox-swagger2' - api 'io.springfox:springfox-swagger-ui' api 'com.fasterxml.jackson.core:jackson-core' api 'com.fasterxml.jackson.core:jackson-databind' api 'com.fasterxml.jackson.core:jackson-annotations' diff --git a/src/backend/commons/common-api/src/main/java/com/tentent/bk/job/common/api/feign/annotation/SmartFeignClient.java b/src/backend/commons/common-api/src/main/java/com/tentent/bk/job/common/api/feign/annotation/SmartFeignClient.java new file mode 100644 index 0000000000..ec62d4b005 --- /dev/null +++ b/src/backend/commons/common-api/src/main/java/com/tentent/bk/job/common/api/feign/annotation/SmartFeignClient.java @@ -0,0 +1,81 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tentent.bk.job.common.api.feign.annotation; + +import com.tentent.bk.job.common.api.feign.condition.SmartFeignClientCondition; +import org.springframework.cloud.openfeign.FeignClient; +import org.springframework.context.annotation.Conditional; +import org.springframework.core.annotation.AliasFor; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * FeignClient 封装，支持 Spring Condition 自定义条件 + */ +@Target({ElementType.TYPE}) +@Retention(RetentionPolicy.RUNTIME) +@Documented +@Inherited +@FeignClient +@Conditional(SmartFeignClientCondition.class) +public @interface SmartFeignClient { + @AliasFor(annotation = FeignClient.class) + String name() default ""; + + @AliasFor(annotation = FeignClient.class) + String value() default ""; + + @AliasFor(annotation = FeignClient.class) + String url() default ""; + + @AliasFor(annotation = FeignClient.class) + String contextId() default ""; + + @AliasFor(annotation = FeignClient.class) + String[] qualifiers() default {}; + + @AliasFor(annotation = FeignClient.class) + boolean decode404() default false; + + @AliasFor(annotation = FeignClient.class) + Class[] configuration() default {}; + + @AliasFor(annotation = FeignClient.class) + Class fallback() default void.class; + + @AliasFor(annotation = FeignClient.class) + Class fallbackFactory() default void.class; + + @AliasFor(annotation = FeignClient.class) + String path() default ""; + + @AliasFor(annotation = FeignClient.class) + boolean primary() default true; +} diff --git a/src/backend/commons/common-api/src/main/java/com/tentent/bk/job/common/api/feign/condition/SmartFeignClientCondition.java b/src/backend/commons/common-api/src/main/java/com/tentent/bk/job/common/api/feign/condition/SmartFeignClientCondition.java new file mode 100644 index 0000000000..55577ea5d7 --- /dev/null +++ b/src/backend/commons/common-api/src/main/java/com/tentent/bk/job/common/api/feign/condition/SmartFeignClientCondition.java @@ -0,0 +1,81 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tentent.bk.job.common.api.feign.condition; + +import com.tencent.bk.job.common.util.JobReflections; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.annotation.Condition; +import org.springframework.context.annotation.ConditionContext; +import org.springframework.core.annotation.AnnotationUtils; +import org.springframework.core.type.AnnotatedTypeMetadata; +import org.springframework.core.type.ClassMetadata; +import org.springframework.web.bind.annotation.RestController; + +import java.util.Set; + +/** + * FeignClient 的生效判断条件 + */ +@Slf4j +public class SmartFeignClientCondition implements Condition { + + @Override + public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) { + // 获取使用@FeignClient的interface的Class + ClassMetadata classMetadata = (ClassMetadata) metadata; + String interfaceClassName = classMetadata.getClassName(); + Class interfaceClass = null; + try { + interfaceClass = Class.forName(interfaceClassName); + } catch (ClassNotFoundException e) { + log.error(e.getMessage(), e); + } + + // 获取interface对应的所有实现类 + Set subTypes = JobReflections.getSubTypesOf(interfaceClass); + + if (subTypes == null || subTypes.isEmpty()) { + log.info("No rpc implementation found for api: {}. FeignClient condition match", interfaceClassName); + return true; + } + // 遍历bean + for (Object subType : subTypes) { + Class subTypeClass = (Class) subType; + boolean isRpcImplementClass = isRpcImplementClass(subTypeClass); + if (isRpcImplementClass) { + // 如果已经有 RPC 请求的实现(RestController), 那么 FeignClient 注解的 bean 无需生效，服务调用走本地进程内调用 + log.info("Found rpc implementation for api: {}. FeignClient condition not match", + interfaceClassName); + return false; + } + } + + return true; + } + + private boolean isRpcImplementClass(Class clazz) { + return AnnotationUtils.findAnnotation(clazz, RestController.class) != null; + } +} diff --git a/src/backend/commons/common-api/src/main/java/com/tentent/bk/job/common/api/package-info.java b/src/backend/commons/common-api/src/main/java/com/tentent/bk/job/common/api/package-info.java new file mode 100644 index 0000000000..833b4d1803 --- /dev/null +++ b/src/backend/commons/common-api/src/main/java/com/tentent/bk/job/common/api/package-info.java @@ -0,0 +1,27 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ +/** + * Job 通用模块 - 微服务内部API调用 + */ +package com.tentent.bk.job.common.api; diff --git a/src/backend/commons/common-audit/build.gradle b/src/backend/commons/common-audit/build.gradle new file mode 100644 index 0000000000..ae37512d1b --- /dev/null +++ b/src/backend/commons/common-audit/build.gradle @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +dependencies { + api project(":commons:common") + api project(":commons:common-iam") + api 'com.tencent.bk.sdk:spring-boot-bk-audit-starter' + implementation 'io.micrometer:micrometer-registry-prometheus' + implementation("org.springframework.boot:spring-boot-autoconfigure") + implementation 'org.springframework:spring-web' + implementation 'org.aspectj:aspectjrt' + implementation 'org.aspectj:aspectjweaver' + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' + testImplementation 'org.junit.jupiter:junit-jupiter' + testImplementation 'org.springframework.boot:spring-boot-starter-test' +} diff --git a/src/backend/commons/common-audit/src/main/java/com/tencent/bk/job/common/audit/AddResourceScopeAuditPostFilter.java b/src/backend/commons/common-audit/src/main/java/com/tencent/bk/job/common/audit/AddResourceScopeAuditPostFilter.java new file mode 100644 index 0000000000..49e17a456c --- /dev/null +++ b/src/backend/commons/common-audit/src/main/java/com/tencent/bk/job/common/audit/AddResourceScopeAuditPostFilter.java @@ -0,0 +1,54 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.audit; + +import com.tencent.bk.audit.filter.AuditPostFilter; +import com.tencent.bk.audit.model.AuditEvent; +import com.tencent.bk.job.common.model.dto.AppResourceScope; +import com.tencent.bk.job.common.util.JobContextUtil; +import lombok.extern.slf4j.Slf4j; + +/** + * 审计事件增加 scopeType, scopeId + */ +@Slf4j +public class AddResourceScopeAuditPostFilter implements AuditPostFilter { + + @Override + public AuditEvent map(AuditEvent auditEvent) { + if (auditEvent == null) { + return null; + } + AppResourceScope appResourceScope = JobContextUtil.getAppResourceScope(); + if (appResourceScope != null) { + if (log.isDebugEnabled()) { + log.debug("Add resource scope for audit event, resourceScope: {}", appResourceScope); + } + auditEvent.setScopeType(appResourceScope.getType().getValue()); + auditEvent.setScopeId(appResourceScope.getId()); + } + return auditEvent; + } +} diff --git a/src/backend/commons/common-audit/src/main/java/com/tencent/bk/job/common/audit/JobAuditAttributeNames.java b/src/backend/commons/common-audit/src/main/java/com/tencent/bk/job/common/audit/JobAuditAttributeNames.java new file mode 100644 index 0000000000..872563afc4 --- /dev/null +++ b/src/backend/commons/common-audit/src/main/java/com/tencent/bk/job/common/audit/JobAuditAttributeNames.java @@ -0,0 +1,35 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.audit; + +public interface JobAuditAttributeNames { + String SCRIPT_ID = "@SCRIPT_ID"; + String SCRIPT_VERSION_ID = "@SCRIPT_VERSION_ID"; + String SCRIPT_NAME = "@SCRIPT_NAME"; + String PLAN_ID = "@PLAN_ID"; + String PLAN_NAME = "@PLAN_NAME"; + String VERSION = "@VERSION"; + String OPERATION = "@OPERATION"; +} diff --git a/src/backend/commons/common-audit/src/main/java/com/tencent/bk/job/common/audit/JobAuditExceptionResolver.java b/src/backend/commons/common-audit/src/main/java/com/tencent/bk/job/common/audit/JobAuditExceptionResolver.java new file mode 100644 index 0000000000..29588e2e7d --- /dev/null +++ b/src/backend/commons/common-audit/src/main/java/com/tencent/bk/job/common/audit/JobAuditExceptionResolver.java @@ -0,0 +1,52 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.audit; + +import com.tencent.bk.audit.AuditExceptionResolver; +import com.tencent.bk.audit.model.ErrorInfo; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.ServiceException; +import com.tencent.bk.job.common.util.I18nUtil; + +import java.util.Locale; + +public class JobAuditExceptionResolver implements AuditExceptionResolver { + @Override + public ErrorInfo resolveException(Throwable e) { + Integer errorCode; + String errorMessage; + if (e instanceof ServiceException) { + ServiceException serviceException = (ServiceException) e; + errorCode = serviceException.getErrorCode(); + // 使用英文描述 + errorMessage = serviceException.getI18nMessage(Locale.ENGLISH); + } else { + errorCode = ErrorCode.INTERNAL_ERROR; + // 使用英文描述 + errorMessage = I18nUtil.getI18nMessage(Locale.ENGLISH, String.valueOf(ErrorCode.INTERNAL_ERROR)); + } + return new ErrorInfo(errorCode, errorMessage); + } +} diff --git a/src/backend/commons/common-audit/src/main/java/com/tencent/bk/job/common/audit/JobAuditExtendDataKeys.java b/src/backend/commons/common-audit/src/main/java/com/tencent/bk/job/common/audit/JobAuditExtendDataKeys.java new file mode 100644 index 0000000000..eb61bcb4d3 --- /dev/null +++ b/src/backend/commons/common-audit/src/main/java/com/tencent/bk/job/common/audit/JobAuditExtendDataKeys.java @@ -0,0 +1,35 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.audit; + +/** + * 审计事件-扩展数据-KEY 定义 + */ +public interface JobAuditExtendDataKeys { + /** + * 作业实例 ID + */ + String JOB_INSTANCE_ID = "job_instance_id"; +} diff --git a/src/backend/commons/common-audit/src/main/java/com/tencent/bk/job/common/audit/JobAuditRequestProvider.java b/src/backend/commons/common-audit/src/main/java/com/tencent/bk/job/common/audit/JobAuditRequestProvider.java new file mode 100644 index 0000000000..251f5c79bb --- /dev/null +++ b/src/backend/commons/common-audit/src/main/java/com/tencent/bk/job/common/audit/JobAuditRequestProvider.java @@ -0,0 +1,72 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.audit; + +import com.tencent.bk.audit.DefaultAuditRequestProvider; +import com.tencent.bk.audit.constants.AccessTypeEnum; +import com.tencent.bk.audit.constants.UserIdentifyTypeEnum; +import com.tencent.bk.audit.model.AuditHttpRequest; +import com.tencent.bk.job.common.util.JobContextUtil; + +public class JobAuditRequestProvider extends DefaultAuditRequestProvider { + + @Override + public String getUsername() { + return JobContextUtil.getUsername(); + } + + @Override + public UserIdentifyTypeEnum getUserIdentifyType() { + // 当前只支持个人账户 + return UserIdentifyTypeEnum.PERSONAL; + } + + @Override + public String getUserIdentifyTenantId() { + // 暂不支持多租户 + return null; + } + + + @Override + public String getRequestId() { + return JobContextUtil.getRequestId(); + } + + @Override + public AccessTypeEnum getAccessType() { + AuditHttpRequest request = getRequest(); + String uri = request.getUri(); + if (uri.startsWith("/web/")) { + return AccessTypeEnum.WEB; + } else if (uri.startsWith("/esb/")) { + return AccessTypeEnum.API; + } else { + return AccessTypeEnum.OTHER; + } + } + + +} diff --git a/src/backend/commons/common-audit/src/main/java/com/tencent/bk/job/common/audit/config/JobAuditAutoConfiguration.java b/src/backend/commons/common-audit/src/main/java/com/tencent/bk/job/common/audit/config/JobAuditAutoConfiguration.java new file mode 100644 index 0000000000..a8615b7e15 --- /dev/null +++ b/src/backend/commons/common-audit/src/main/java/com/tencent/bk/job/common/audit/config/JobAuditAutoConfiguration.java @@ -0,0 +1,65 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.audit.config; + +import com.tencent.bk.audit.AuditRequestProvider; +import com.tencent.bk.audit.config.AuditAutoConfiguration; +import com.tencent.bk.audit.config.AuditProperties; +import com.tencent.bk.job.common.audit.AddResourceScopeAuditPostFilter; +import com.tencent.bk.job.common.audit.JobAuditExceptionResolver; +import com.tencent.bk.job.common.audit.JobAuditRequestProvider; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.AutoConfigureBefore; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + + +@Configuration(proxyBeanMethods = false) +@EnableConfigurationProperties(AuditProperties.class) +@ConditionalOnProperty(name = "audit.enabled", havingValue = "true", matchIfMissing = true) +@AutoConfigureBefore(AuditAutoConfiguration.class) +@Slf4j +public class JobAuditAutoConfiguration { + + @Bean + public AuditRequestProvider auditRequestProvider() { + log.info("Init JobAuditRequestProvider"); + return new JobAuditRequestProvider(); + } + + + @Bean + public JobAuditExceptionResolver auditExceptionResolver() { + log.info("Init JobAuditExceptionResolver"); + return new JobAuditExceptionResolver(); + } + + @Bean + public AddResourceScopeAuditPostFilter addResourceScopeAuditPostFilter() { + return new AddResourceScopeAuditPostFilter(); + } +} diff --git a/src/backend/commons/common-audit/src/main/java/com/tencent/bk/job/common/audit/constants/EventContentConstants.java b/src/backend/commons/common-audit/src/main/java/com/tencent/bk/job/common/audit/constants/EventContentConstants.java new file mode 100644 index 0000000000..de98222e6b --- /dev/null +++ b/src/backend/commons/common-audit/src/main/java/com/tencent/bk/job/common/audit/constants/EventContentConstants.java @@ -0,0 +1,132 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.audit.constants; + +import com.tencent.bk.job.common.audit.JobAuditAttributeNames; + +import static com.tencent.bk.audit.constants.AuditAttributeNames.INSTANCE_ID; +import static com.tencent.bk.audit.constants.AuditAttributeNames.INSTANCE_NAME; +import static com.tencent.bk.job.common.audit.JobAuditAttributeNames.OPERATION; +import static com.tencent.bk.job.common.audit.JobAuditAttributeNames.SCRIPT_NAME; +import static com.tencent.bk.job.common.audit.JobAuditAttributeNames.SCRIPT_VERSION_ID; + +/** + * 审计事件描述 + */ +public interface EventContentConstants { + String INSTANCE_INFO = "[{{" + INSTANCE_NAME + "}}]({{" + INSTANCE_ID + "}})"; + + String QUICK_EXECUTE_SCRIPT = "Run a quick script task"; + String QUICK_TRANSFER_FILE = "Run a quick transfer file task"; + String EXECUTE_SCRIPT = "Launch a script [{{" + SCRIPT_NAME + + "}}]({{" + SCRIPT_VERSION_ID + "}})"; + String EXECUTE_PUBLIC_SCRIPT = "Launch a public script [{{" + SCRIPT_NAME + + "}}]({{" + SCRIPT_VERSION_ID + "}})"; + + String VIEW_JOB_INSTANCE = "View job task " + INSTANCE_INFO; + + String VIEW_CRON_JOB = "View cron task " + INSTANCE_INFO; + String CREATE_CRON_JOB = "Create cron task " + INSTANCE_INFO; + String EDIT_CRON_JOB = "Modify cron task " + INSTANCE_INFO; + String DELETE_CRON_JOB = "Delete cron task " + INSTANCE_INFO; + String SWITCH_CRON_JOB_STATUS = "{{" + OPERATION + "}} cron task " + INSTANCE_INFO; + + String VIEW_FILE_SOURCE = "View file source " + INSTANCE_INFO; + String CREATE_FILE_SOURCE = "Create file source " + INSTANCE_INFO; + String EDIT_FILE_SOURCE = "Modify file source " + INSTANCE_INFO; + String DELETE_FILE_SOURCE = "Delete file source " + INSTANCE_INFO; + String SWITCH_FILE_SOURCE_STATUS = "{{" + OPERATION + "}} file source " + INSTANCE_INFO; + + String USE_ACCOUNT = "View account " + INSTANCE_INFO; + String CREATE_ACCOUNT = "Create account " + INSTANCE_INFO; + String EDIT_ACCOUNT = "Modify account " + INSTANCE_INFO; + String DELETE_ACCOUNT = "Delete account " + INSTANCE_INFO; + + String USE_TICKET = "View credential " + INSTANCE_INFO; + String CREATE_TICKET = "Create credential " + INSTANCE_INFO; + String EDIT_TICKET = "Modify credential " + INSTANCE_INFO; + String DELETE_TICKET = "Delete credential " + INSTANCE_INFO; + + String VIEW_SCRIPT = "View script " + INSTANCE_INFO; + String CREATE_SCRIPT = "Create script " + INSTANCE_INFO; + String EDIT_SCRIPT = "Modify script " + INSTANCE_INFO; + String DELETE_SCRIPT = "Delete script " + INSTANCE_INFO; + String CREATE_SCRIPT_VERSION = "Create a new script version " + INSTANCE_INFO + "({{@VERSION}})"; + String EDIT_SCRIPT_VERSION = "Modify script version " + INSTANCE_INFO + "({{@VERSION}})"; + String DELETE_SCRIPT_VERSION = "Delete script version " + INSTANCE_INFO + "({{@VERSION}})"; + String FORBIDDEN_SCRIPT_VERSION = "Set script version " + INSTANCE_INFO + "({{@VERSION}}) state to [forbidden]"; + String ONLINE_SCRIPT_VERSION = "Set script version " + INSTANCE_INFO + "({{@VERSION}}) state to [online]"; + + String CREATE_PUBLIC_SCRIPT = "Create public script " + INSTANCE_INFO; + String EDIT_PUBLIC_SCRIPT = "Modify public script " + INSTANCE_INFO; + String DELETE_PUBLIC_SCRIPT = "Delete public script " + INSTANCE_INFO; + String CREATE_PUBLIC_SCRIPT_VERSION = "Create a new public script version " + INSTANCE_INFO + "({{@VERSION}})"; + String EDIT_PUBLIC_SCRIPT_VERSION = "Modify public script version " + INSTANCE_INFO + "({{@VERSION}})"; + String DELETE_PUBLIC_SCRIPT_VERSION = "Delete public script version " + INSTANCE_INFO + "({{@VERSION}})"; + String FORBIDDEN_PUBLIC_SCRIPT_VERSION = "Set public script version " + INSTANCE_INFO + "({{@VERSION}}) state to " + + "[forbidden]"; + String ONLINE_PUBLIC_SCRIPT_VERSION = "Set public script version " + INSTANCE_INFO + "({{@VERSION}}) state to " + + "[online]"; + + String CREATE_TAG = "Create tag " + INSTANCE_INFO; + String EDIT_TAG = "Modify tag " + INSTANCE_INFO; + String DELETE_TAG = "Delete tag " + INSTANCE_INFO; + + String VIEW_JOB_TEMPLATE = "View job template " + INSTANCE_INFO; + String CREATE_JOB_TEMPLATE = "Create job template " + INSTANCE_INFO; + String EDIT_JOB_TEMPLATE = "Modify job template " + INSTANCE_INFO; + String DELETE_JOB_TEMPLATE = "Delete job template " + INSTANCE_INFO; + String DEBUG_JOB_TEMPLATE = "Debug job template " + INSTANCE_INFO; + + String CREATE_JOB_PLAN = "Create job plan " + INSTANCE_INFO; + String VIEW_JOB_PLAN = "View job plan " + INSTANCE_INFO; + String EDIT_JOB_PLAN = "Modify job plan " + INSTANCE_INFO; + String DELETE_JOB_PLAN = "Delete job plan " + INSTANCE_INFO; + String SYNC_JOB_PLAN = "Sync job plan " + INSTANCE_INFO; + String LAUNCH_JOB_PLAN = "Launch a plan [{{" + JobAuditAttributeNames.PLAN_NAME + + "}}]({{" + JobAuditAttributeNames.PLAN_ID + "}})"; + + String CREATE_HIGH_RISK_DETECT_RULE = "Create a new high-risk detect rule"; + String VIEW_HIGH_RISK_DETECT_RULE = "View high-risk detect rule settings"; + String EDIT_HIGH_RISK_DETECT_RULE = "Modify a new high-risk detect rule"; + String DELETE_HIGH_RISK_DETECT_RULE = "DELETE a new high-risk detect rule"; + + String VIEW_GLOBAL_SETTINGS = "View the global settings"; + String EDIT_GLOBAL_SETTINGS = "Modify the global settings"; + + String VIEW_ANALYSIS_DASHBOARD = "View the analysis dashboard"; + + String VIEW_PLATFORM_SERVICE_STAT = "View the platform service states"; + + + String VIEW_WHITE_LIST = "View the white list"; + String EDIT_WHITE_LIST = "Modify a white list row"; + + String VIEW_HIGH_RISK_DETECT_RECORD = "View the high-risk detect record"; + + String EDIT_BUSINESS_NOTIFY_SETTINGS = "Modify business notification settings"; + + +} diff --git a/src/backend/commons/common-audit/src/main/resources/META-INF/spring.factories b/src/backend/commons/common-audit/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..a81d3e2e14 --- /dev/null +++ b/src/backend/commons/common-audit/src/main/resources/META-INF/spring.factories @@ -0,0 +1,2 @@ +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ +com.tencent.bk.job.common.audit.config.JobAuditAutoConfiguration diff --git a/src/backend/commons/common-consul/build.gradle b/src/backend/commons/common-consul/build.gradle index 642d04740f..5bda0d1f15 100644 --- a/src/backend/commons/common-consul/build.gradle +++ b/src/backend/commons/common-consul/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-consul/src/main/java/com/tencent/bk/job/common/consul/config/ConsulServiceInfoServiceAutoConfig.java b/src/backend/commons/common-consul/src/main/java/com/tencent/bk/job/common/consul/config/ConsulServiceInfoServiceAutoConfig.java deleted file mode 100644 index c8093a90f1..0000000000 --- a/src/backend/commons/common-consul/src/main/java/com/tencent/bk/job/common/consul/config/ConsulServiceInfoServiceAutoConfig.java +++ /dev/null @@ -1,21 +0,0 @@ -package com.tencent.bk.job.common.consul.config; - -import com.tencent.bk.job.common.consul.provider.ConsulServiceInfoProvider; -import com.tencent.bk.job.common.discovery.ServiceInfoProvider; -import org.springframework.cloud.client.discovery.DiscoveryClient; -import org.springframework.cloud.consul.discovery.ConditionalOnConsulDiscoveryEnabled; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; - -@Configuration -@ConditionalOnConsulDiscoveryEnabled -public class ConsulServiceInfoServiceAutoConfig { - - @Bean - @Primary - public ServiceInfoProvider consulServiceInfoService(DiscoveryClient discoveryClient) { - return new ConsulServiceInfoProvider(discoveryClient); - } - -} diff --git a/src/backend/commons/common-consul/src/main/java/com/tencent/bk/job/common/consul/config/ConsulServiceInfoServiceConfiguration.java b/src/backend/commons/common-consul/src/main/java/com/tencent/bk/job/common/consul/config/ConsulServiceInfoServiceConfiguration.java new file mode 100644 index 0000000000..a99fb9c531 --- /dev/null +++ b/src/backend/commons/common-consul/src/main/java/com/tencent/bk/job/common/consul/config/ConsulServiceInfoServiceConfiguration.java @@ -0,0 +1,21 @@ +package com.tencent.bk.job.common.consul.config; + +import com.tencent.bk.job.common.consul.provider.ConsulServiceInfoProvider; +import com.tencent.bk.job.common.discovery.ServiceInfoProvider; +import org.springframework.cloud.client.discovery.DiscoveryClient; +import org.springframework.cloud.consul.discovery.ConditionalOnConsulDiscoveryEnabled; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; + +@Configuration +@ConditionalOnConsulDiscoveryEnabled +public class ConsulServiceInfoServiceConfiguration { + + @Bean + @Primary + public ServiceInfoProvider consulServiceInfoService(DiscoveryClient discoveryClient) { + return new ConsulServiceInfoProvider(discoveryClient); + } + +} diff --git a/src/backend/commons/common-consul/src/main/java/com/tencent/bk/job/common/consul/config/GracefulShutdownAutoConfiguration.java b/src/backend/commons/common-consul/src/main/java/com/tencent/bk/job/common/consul/config/GracefulShutdownAutoConfiguration.java new file mode 100644 index 0000000000..8c771f80a0 --- /dev/null +++ b/src/backend/commons/common-consul/src/main/java/com/tencent/bk/job/common/consul/config/GracefulShutdownAutoConfiguration.java @@ -0,0 +1,51 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.consul.config; + +import com.tencent.bk.job.common.consul.listener.GracefulShutdown; +import org.springframework.boot.autoconfigure.AutoConfigureAfter; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.cloud.consul.ConditionalOnConsulEnabled; +import org.springframework.cloud.consul.discovery.ConditionalOnConsulDiscoveryEnabled; +import org.springframework.cloud.consul.serviceregistry.ConsulRegistration; +import org.springframework.cloud.consul.serviceregistry.ConsulServiceRegistry; +import org.springframework.cloud.consul.serviceregistry.ConsulServiceRegistryAutoConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration(proxyBeanMethods = false) +@ConditionalOnConsulEnabled +@ConditionalOnConsulDiscoveryEnabled +@AutoConfigureAfter(ConsulServiceRegistryAutoConfiguration.class) +@EnableConfigurationProperties(JobConsulProperties.class) +public class GracefulShutdownAutoConfiguration { + + @Bean + public GracefulShutdown gracefulShutdown(ConsulRegistration consulRegistration, + ConsulServiceRegistry consulServiceRegistry, + JobConsulProperties jobConsulProperties) { + return new GracefulShutdown(consulRegistration, consulServiceRegistry, jobConsulProperties); + } +} diff --git a/src/backend/commons/common-consul/src/main/java/com/tencent/bk/job/common/consul/config/JobApplicationAvailabilityAutoConfiguration.java b/src/backend/commons/common-consul/src/main/java/com/tencent/bk/job/common/consul/config/JobApplicationAvailabilityAutoConfiguration.java new file mode 100644 index 0000000000..d27074aada --- /dev/null +++ b/src/backend/commons/common-consul/src/main/java/com/tencent/bk/job/common/consul/config/JobApplicationAvailabilityAutoConfiguration.java @@ -0,0 +1,15 @@ +package com.tencent.bk.job.common.consul.config; + +import org.springframework.boot.availability.ApplicationAvailabilityBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration(proxyBeanMethods = false) +public class JobApplicationAvailabilityAutoConfiguration { + + @Bean + public ApplicationAvailabilityBean applicationAvailability() { + return new ApplicationAvailabilityBean(); + } + +} diff --git a/src/backend/commons/common-consul/src/main/java/com/tencent/bk/job/common/consul/config/JobConsulAutoConfiguration.java b/src/backend/commons/common-consul/src/main/java/com/tencent/bk/job/common/consul/config/JobConsulAutoConfiguration.java new file mode 100644 index 0000000000..2fc280faf8 --- /dev/null +++ b/src/backend/commons/common-consul/src/main/java/com/tencent/bk/job/common/consul/config/JobConsulAutoConfiguration.java @@ -0,0 +1,52 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.consul.config; + +import com.tencent.bk.job.common.constant.JobDiscoveryConsts; +import org.springframework.boot.info.BuildProperties; +import org.springframework.cloud.consul.ConditionalOnConsulEnabled; +import org.springframework.cloud.consul.discovery.ConditionalOnConsulDiscoveryEnabled; +import org.springframework.cloud.consul.serviceregistry.ConsulRegistrationCustomizer; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; + +@Configuration(proxyBeanMethods = false) +@Import({ConsulServiceInfoServiceConfiguration.class}) +@ConditionalOnConsulEnabled +@ConditionalOnConsulDiscoveryEnabled +public class JobConsulAutoConfiguration { + @Bean + public ConsulRegistrationCustomizer jobConsulRegistrationCustomizer(BuildProperties buildProperties) { + return registration -> { + // 将版本号写入Tag中 + registration.getService().getTags().add( + JobDiscoveryConsts.TAG_KEY_VERSION + "=" + buildProperties.getVersion()); + // 区分Job后台服务与组件（Redis、MQ等） + registration.getService().getTags() + .add(JobDiscoveryConsts.TAG_KEY_TYPE + "=" + JobDiscoveryConsts.TAG_VALUE_TYPE_JOB_BACKEND_SERVICE); + }; + } +} diff --git a/src/backend/commons/common-consul/src/main/java/com/tencent/bk/job/common/consul/config/JobConsulAutoRegistration.java b/src/backend/commons/common-consul/src/main/java/com/tencent/bk/job/common/consul/config/JobConsulAutoRegistration.java deleted file mode 100644 index b30f531b8f..0000000000 --- a/src/backend/commons/common-consul/src/main/java/com/tencent/bk/job/common/consul/config/JobConsulAutoRegistration.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.consul.config; - -import com.ecwid.consul.v1.agent.model.NewService; -import com.tencent.bk.job.common.constant.JobDiscoveryConsts; -import org.springframework.boot.info.BuildProperties; -import org.springframework.cloud.client.serviceregistry.AutoServiceRegistrationProperties; -import org.springframework.cloud.consul.discovery.ConsulDiscoveryProperties; -import org.springframework.cloud.consul.discovery.HeartbeatProperties; -import org.springframework.cloud.consul.serviceregistry.ConsulAutoRegistration; -import org.springframework.cloud.consul.serviceregistry.ConsulManagementRegistrationCustomizer; -import org.springframework.cloud.consul.serviceregistry.ConsulRegistrationCustomizer; -import org.springframework.context.ApplicationContext; - -import java.util.List; - -public class JobConsulAutoRegistration extends ConsulAutoRegistration { - public JobConsulAutoRegistration(NewService service, - AutoServiceRegistrationProperties autoServiceRegistrationProperties, - ConsulDiscoveryProperties properties, ApplicationContext context, - HeartbeatProperties heartbeatProperties, - List managementRegistrationCustomizers) { - super(service, autoServiceRegistrationProperties, properties, context, heartbeatProperties, - managementRegistrationCustomizers); - } - - - public static ConsulAutoRegistration registration( - AutoServiceRegistrationProperties autoServiceRegistrationProperties, - ConsulDiscoveryProperties properties, ApplicationContext context, - List registrationCustomizers, - List managementRegistrationCustomizers, - HeartbeatProperties heartbeatProperties, BuildProperties buildProperties) { - ConsulAutoRegistration registration = ConsulAutoRegistration.registration(autoServiceRegistrationProperties, - properties, context, registrationCustomizers, managementRegistrationCustomizers, heartbeatProperties); - // 将版本号写入Tag中 - registration.getService().getTags().add(JobDiscoveryConsts.TAG_KEY_VERSION + "=" + buildProperties.getVersion()); - // 区分Job后台服务与组件（Redis、MQ等） - registration.getService().getTags() - .add(JobDiscoveryConsts.TAG_KEY_TYPE + "=" + JobDiscoveryConsts.TAG_VALUE_TYPE_JOB_BACKEND_SERVICE); - return registration; - } -} diff --git a/src/backend/commons/common-consul/src/main/java/com/tencent/bk/job/common/consul/config/JobConsulProperties.java b/src/backend/commons/common-consul/src/main/java/com/tencent/bk/job/common/consul/config/JobConsulProperties.java new file mode 100644 index 0000000000..3d8760e7af --- /dev/null +++ b/src/backend/commons/common-consul/src/main/java/com/tencent/bk/job/common/consul/config/JobConsulProperties.java @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.consul.config; + +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; + +@Data +@ConfigurationProperties(prefix = "job.consul.service") +public class JobConsulProperties { + /** + * 服务刷新时间，默认 3 秒 + */ + private int refreshSeconds = 3; + +} diff --git a/src/backend/commons/common-consul/src/main/java/com/tencent/bk/job/common/consul/config/JobConsulServiceRegistrationConfiguration.java b/src/backend/commons/common-consul/src/main/java/com/tencent/bk/job/common/consul/config/JobConsulServiceRegistrationConfiguration.java deleted file mode 100644 index 8ad3df8bbb..0000000000 --- a/src/backend/commons/common-consul/src/main/java/com/tencent/bk/job/common/consul/config/JobConsulServiceRegistrationConfiguration.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.consul.config; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.ObjectProvider; -import org.springframework.boot.info.BuildProperties; -import org.springframework.cloud.client.serviceregistry.AutoServiceRegistrationProperties; -import org.springframework.cloud.consul.ConditionalOnConsulEnabled; -import org.springframework.cloud.consul.discovery.ConditionalOnConsulDiscoveryEnabled; -import org.springframework.cloud.consul.discovery.ConsulDiscoveryProperties; -import org.springframework.cloud.consul.discovery.HeartbeatProperties; -import org.springframework.cloud.consul.serviceregistry.ConsulAutoRegistration; -import org.springframework.cloud.consul.serviceregistry.ConsulManagementRegistrationCustomizer; -import org.springframework.cloud.consul.serviceregistry.ConsulRegistrationCustomizer; -import org.springframework.context.ApplicationContext; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -import java.util.List; - -@Slf4j -@Configuration(proxyBeanMethods = false) -@ConditionalOnConsulEnabled -@ConditionalOnConsulDiscoveryEnabled -public class JobConsulServiceRegistrationConfiguration { - - @Bean - public ConsulAutoRegistration consulRegistration( - AutoServiceRegistrationProperties autoServiceRegistrationProperties, - ConsulDiscoveryProperties properties, ApplicationContext applicationContext, - ObjectProvider> registrationCustomizers, - ObjectProvider> managementRegistrationCustomizers, - HeartbeatProperties heartbeatProperties, BuildProperties buildProperties) { - return JobConsulAutoRegistration.registration(autoServiceRegistrationProperties, - properties, applicationContext, registrationCustomizers.getIfAvailable(), - managementRegistrationCustomizers.getIfAvailable(), heartbeatProperties, buildProperties); - } -} diff --git a/src/backend/commons/common-consul/src/main/java/com/tencent/bk/job/common/consul/listener/GracefulShutdown.java b/src/backend/commons/common-consul/src/main/java/com/tencent/bk/job/common/consul/listener/GracefulShutdown.java index 7a49ffdc5c..7b88270f89 100644 --- a/src/backend/commons/common-consul/src/main/java/com/tencent/bk/job/common/consul/listener/GracefulShutdown.java +++ b/src/backend/commons/common-consul/src/main/java/com/tencent/bk/job/common/consul/listener/GracefulShutdown.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,39 +24,29 @@ package com.tencent.bk.job.common.consul.listener; +import com.tencent.bk.job.common.consul.config.JobConsulProperties; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.AutoConfigureAfter; -import org.springframework.cloud.consul.ConditionalOnConsulEnabled; -import org.springframework.cloud.consul.discovery.ConditionalOnConsulDiscoveryEnabled; import org.springframework.cloud.consul.serviceregistry.ConsulRegistration; import org.springframework.cloud.consul.serviceregistry.ConsulServiceRegistry; -import org.springframework.cloud.consul.serviceregistry.ConsulServiceRegistryAutoConfiguration; import org.springframework.context.ApplicationListener; import org.springframework.context.event.ContextClosedEvent; -import org.springframework.stereotype.Component; import static org.springframework.boot.actuate.health.Status.OUT_OF_SERVICE; @Slf4j -@Component -@ConditionalOnConsulEnabled -@ConditionalOnConsulDiscoveryEnabled -@AutoConfigureAfter(ConsulServiceRegistryAutoConfiguration.class) public class GracefulShutdown implements ApplicationListener { private final ConsulRegistration consulRegistration; private final ConsulServiceRegistry consulServiceRegistry; + private final JobConsulProperties jobConsulProperties; - @Value("${job.consul.gateway.refresh.waitSeconds:3}") - int waitGatewayToRefreshConsulSeconds = 3; - - @Autowired - public GracefulShutdown(ConsulRegistration consulRegistration, ConsulServiceRegistry consulServiceRegistry) { + public GracefulShutdown(ConsulRegistration consulRegistration, + ConsulServiceRegistry consulServiceRegistry, + JobConsulProperties jobConsulProperties) { + log.info("Init consul service graceful shutdown component"); this.consulRegistration = consulRegistration; this.consulServiceRegistry = consulServiceRegistry; - log.debug("consulRegistration={},consulServiceRegistry={}", consulRegistration, consulServiceRegistry); + this.jobConsulProperties = jobConsulProperties; } @Override @@ -70,7 +60,7 @@ public void onApplicationEvent(ContextClosedEvent event) { } // 2.等待网关刷新consul服务状态 try { - Thread.sleep(waitGatewayToRefreshConsulSeconds * 1000L); + Thread.sleep(jobConsulProperties.getRefreshSeconds() * 1000L); } catch (InterruptedException e) { log.error("Error when offline", e); } diff --git a/src/backend/commons/common-consul/src/main/java/com/tencent/bk/job/common/consul/provider/ConsulServiceInfoProvider.java b/src/backend/commons/common-consul/src/main/java/com/tencent/bk/job/common/consul/provider/ConsulServiceInfoProvider.java index 5983346b6b..fbea3838bb 100644 --- a/src/backend/commons/common-consul/src/main/java/com/tencent/bk/job/common/consul/provider/ConsulServiceInfoProvider.java +++ b/src/backend/commons/common-consul/src/main/java/com/tencent/bk/job/common/consul/provider/ConsulServiceInfoProvider.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -36,12 +36,10 @@ import com.tencent.bk.job.common.util.json.JsonUtils; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.actuate.health.Status; import org.springframework.cloud.client.ServiceInstance; import org.springframework.cloud.client.discovery.DiscoveryClient; import org.springframework.cloud.consul.discovery.ConsulServiceInstance; -import org.springframework.stereotype.Component; import java.util.ArrayList; import java.util.HashMap; @@ -55,7 +53,6 @@ import static org.springframework.boot.actuate.health.Status.UP; @Slf4j -@Component public class ConsulServiceInfoProvider implements ServiceInfoProvider { public final String KEY_WORD_MAINTENANCE = "maintenance"; @@ -70,7 +67,6 @@ public class ConsulServiceInfoProvider implements ServiceInfoProvider { private final DiscoveryClient discoveryClient; - @Autowired public ConsulServiceInfoProvider(DiscoveryClient discoveryClient) { this.discoveryClient = discoveryClient; log.debug("ConsulServiceInfoServiceImpl inited"); @@ -146,7 +142,7 @@ public List listServiceInfo() { for (ServiceInstance serviceInstance : serviceInstanceList) { log.debug("serviceInstance={}", JsonUtils.toJson(serviceInstance)); } - return serviceInstanceList.parallelStream().filter(serviceInstance -> { + return serviceInstanceList.stream().filter(serviceInstance -> { ConsulServiceInstance consulServiceInstance = (ConsulServiceInstance) serviceInstance; return consulServiceInstance.getTags().contains( JobDiscoveryConsts.TAG_KEY_TYPE + "=" + JobDiscoveryConsts.TAG_VALUE_TYPE_JOB_BACKEND_SERVICE diff --git a/src/backend/commons/common-consul/src/main/resources/META-INF/spring.factories b/src/backend/commons/common-consul/src/main/resources/META-INF/spring.factories index e69de29bb2..b040d5216d 100644 --- a/src/backend/commons/common-consul/src/main/resources/META-INF/spring.factories +++ b/src/backend/commons/common-consul/src/main/resources/META-INF/spring.factories @@ -0,0 +1,4 @@ +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ +com.tencent.bk.job.common.consul.config.JobApplicationAvailabilityAutoConfiguration,\ +com.tencent.bk.job.common.consul.config.JobConsulAutoConfiguration,\ +com.tencent.bk.job.common.consul.config.GracefulShutdownAutoConfiguration diff --git a/src/backend/commons/common-crypto/build.gradle b/src/backend/commons/common-crypto/build.gradle new file mode 100644 index 0000000000..5f8b683490 --- /dev/null +++ b/src/backend/commons/common-crypto/build.gradle @@ -0,0 +1,31 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +dependencies { + api project(':commons:common') + api project(':commons:common-utils') + api 'com.tencent.bk.sdk:crypto-java-sdk' + testImplementation 'org.junit.jupiter:junit-jupiter' + testImplementation 'org.springframework.boot:spring-boot-starter-test' +} diff --git a/src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/AESCryptor.java b/src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/AESCryptor.java new file mode 100644 index 0000000000..0a24f5fc3b --- /dev/null +++ b/src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/AESCryptor.java @@ -0,0 +1,114 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.crypto; + +import com.tencent.bk.job.common.crypto.util.AESUtils; +import com.tencent.bk.sdk.crypto.annotation.Cryptor; +import com.tencent.bk.sdk.crypto.annotation.CryptorTypeEnum; +import com.tencent.bk.sdk.crypto.cryptor.AbstractSymmetricCryptor; +import com.tencent.bk.sdk.crypto.exception.CryptoException; +import org.slf4j.helpers.FormattingTuple; +import org.slf4j.helpers.MessageFormatter; + +import java.io.InputStream; +import java.io.OutputStream; + +/** + * 使用AES/CBC/PKCS5Padding的加密实现 + */ +@Cryptor(name = JobCryptorNames.AES_CBC, type = CryptorTypeEnum.SYMMETRIC) +public class AESCryptor extends AbstractSymmetricCryptor { + + @Override + public String getName() { + return JobCryptorNames.AES_CBC; + } + + @Override + public byte[] encryptIndeed(byte[] key, byte[] message) { + try { + return AESUtils.encrypt(message, key); + } catch (Exception e) { + FormattingTuple msg = MessageFormatter.arrayFormat( + "Fail to encrypt using {}, key.len={}, message.len={}", + new Object[]{ + getName(), + key.length, + message.length + } + ); + throw new CryptoException(msg.getMessage(), e); + } + } + + @Override + public byte[] decryptIndeed(byte[] key, byte[] encryptedMessage) { + try { + return AESUtils.decrypt(encryptedMessage, key); + } catch (Exception e) { + FormattingTuple msg = MessageFormatter.arrayFormat( + "Fail to decrypt using {}, key.len={}, encryptedMessage.len={}", + new Object[]{ + getName(), + key.length, + encryptedMessage.length + } + ); + throw new CryptoException(msg.getMessage(), e); + } + } + + @Override + public void encryptIndeed(String key, InputStream in, OutputStream out) { + try { + AESUtils.encrypt(in, out, key); + } catch (Exception e) { + FormattingTuple msg = MessageFormatter.arrayFormat( + "Fail to encrypt using {}, key.len={}", + new Object[]{ + getName(), + key.length() + } + ); + throw new CryptoException(msg.getMessage(), e); + } + } + + @Override + public void decryptIndeed(String key, InputStream in, OutputStream out) { + try { + AESUtils.decrypt(in, out, key); + } catch (Exception e) { + FormattingTuple msg = MessageFormatter.arrayFormat( + "Fail to decrypt using {}, key.len={}", + new Object[]{ + getName(), + key.length() + } + ); + throw new CryptoException(msg.getMessage(), e); + } + } +} diff --git a/src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/CryptoConfigService.java b/src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/CryptoConfigService.java new file mode 100644 index 0000000000..f82864ef5f --- /dev/null +++ b/src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/CryptoConfigService.java @@ -0,0 +1,100 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.crypto; + +import com.tencent.bk.sdk.crypto.cryptor.consts.CryptorNames; +import lombok.extern.slf4j.Slf4j; + +import java.util.HashMap; +import java.util.Map; + +/** + * 加密配置服务 + */ +@SuppressWarnings("unused") +@Slf4j +public class CryptoConfigService { + + private final EncryptConfig encryptConfig; + private final Map scenarioAlgorithms; + + public CryptoConfigService(EncryptConfig encryptConfig) { + this.encryptConfig = encryptConfig; + if (encryptConfig != null) { + this.scenarioAlgorithms = trimKeyValues(encryptConfig.getScenarioAlgorithms()); + } else { + this.scenarioAlgorithms = null; + } + } + + private Map trimKeyValues(Map map) { + if (map == null) { + return null; + } + Map resultMap = new HashMap<>(); + map.forEach((key, value) -> { + if (key != null) { + key = key.trim(); + } + if (value != null) { + value = value.trim(); + } + resultMap.put(key, value); + }); + return resultMap; + } + + /** + * 获取对称加密密钥 + * + * @return 对称加密密钥 + */ + public String getSymmetricPassword() { + return encryptConfig.getPassword(); + } + + /** + * 根据加密场景获取需要使用的加密算法 + * + * @param cryptoScenarioEnum 加密场景枚举值 + * @return 加密算法标识 + */ + public String getSymmetricAlgorithmByScenario(CryptoScenarioEnum cryptoScenarioEnum) { + if (cryptoScenarioEnum == null) { + return getDefaultSymmetricAlgorithm(); + } + if (scenarioAlgorithms != null && scenarioAlgorithms.containsKey(cryptoScenarioEnum.getValue())) { + return scenarioAlgorithms.get(cryptoScenarioEnum.getValue()); + } + return getDefaultSymmetricAlgorithm(); + } + + private String getDefaultSymmetricAlgorithm() { + if (encryptConfig.getType() == CryptoTypeEnum.SHANGMI) { + return CryptorNames.SM4; + } + return JobCryptorNames.AES_CBC; + } +} diff --git a/src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/CryptoScenarioEnum.java b/src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/CryptoScenarioEnum.java new file mode 100644 index 0000000000..0b5375c47b --- /dev/null +++ b/src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/CryptoScenarioEnum.java @@ -0,0 +1,61 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.crypto; + +/** + * 加密场景枚举值 + */ +public enum CryptoScenarioEnum { + // 脚本敏感参数 + SCRIPT_SENSITIVE_PARAM((byte) 0, "scriptSensitiveParam"), + // 密文变量 + CIPHER_VARIABLE((byte) 0, "cipherVariable"), + // DB账号的密码 + DATABASE_PASSWORD((byte) 0, "databasePassword"), + // 凭证信息 + CREDENTIAL((byte) 0, "credential"), + // 导出作业的密码 + EXPORT_JOB_PASSWORD((byte) 0, "exportJobPassword"), + // 导出作业的备份文件 + BACKUP_FILE((byte) 0, "backupFile"); + + // 加密类型：0为对称加密，1为非对称加密 + private final byte type; + // 场景标识 + private final String value; + + CryptoScenarioEnum(byte type, String value) { + this.type = type; + this.value = value; + } + + public String getValue() { + return value; + } + + public byte getType() { + return type; + } +} diff --git a/src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/CryptoTypeEnum.java b/src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/CryptoTypeEnum.java new file mode 100644 index 0000000000..379bb6c0b9 --- /dev/null +++ b/src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/CryptoTypeEnum.java @@ -0,0 +1,37 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.crypto; + +/** + * 加密类型枚举值 + */ +public enum CryptoTypeEnum { + + // 经典密码算法（RSA、AES等） + CLASSIC, + // 国家商用密码算法（SM2、SM4等） + SHANGMI + +} diff --git a/src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/EncryptConfig.java b/src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/EncryptConfig.java new file mode 100644 index 0000000000..8c4ea1f13b --- /dev/null +++ b/src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/EncryptConfig.java @@ -0,0 +1,63 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.crypto; + +import com.tencent.bk.job.common.util.json.JsonUtils; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.context.properties.ConfigurationProperties; + +import javax.annotation.PostConstruct; +import java.util.HashMap; +import java.util.Map; + +/** + * 加密配置 + */ +@ConfigurationProperties(prefix = "job.encrypt") +@ToString +@Getter +@Setter +@Slf4j +public class EncryptConfig { + + private CryptoTypeEnum type; + + private String password; + + /** + * 各个场景下使用的加密算法，不配置则使用默认算法 + */ + private Map scenarioAlgorithms = new HashMap<>(); + + @PostConstruct + public void print() { + if (log.isDebugEnabled()) { + log.debug("EncryptConfig init: {}", JsonUtils.toJson(this)); + } + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/encrypt/Encryptor.java b/src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/Encryptor.java similarity index 92% rename from src/backend/commons/common/src/main/java/com/tencent/bk/job/common/encrypt/Encryptor.java rename to src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/Encryptor.java index 24f1a4b748..3d1639d30f 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/encrypt/Encryptor.java +++ b/src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/Encryptor.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.common.encrypt; +package com.tencent.bk.job.common.crypto; /** * 加密器 diff --git a/src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/JobCryptorNames.java b/src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/JobCryptorNames.java new file mode 100644 index 0000000000..2142b0e368 --- /dev/null +++ b/src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/JobCryptorNames.java @@ -0,0 +1,30 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.crypto; + +public class JobCryptorNames { + // 对称加密 + public static final String AES_CBC = "AES_CBC"; +} diff --git a/src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/RSAEncryptor.java b/src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/RSAEncryptor.java new file mode 100644 index 0000000000..4cb0ba0784 --- /dev/null +++ b/src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/RSAEncryptor.java @@ -0,0 +1,50 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.crypto; + +import com.tencent.bk.job.common.crypto.util.RSAUtils; +import com.tencent.bk.sdk.crypto.exception.CryptoException; +import lombok.extern.slf4j.Slf4j; + +import java.security.PublicKey; + +@Slf4j +public class RSAEncryptor implements Encryptor { + + private final PublicKey publicKey; + + public RSAEncryptor(String rsaPublicKeyBase64) { + publicKey = RSAUtils.getPublicKey(rsaPublicKeyBase64); + } + + public String encrypt(String rawText) { + try { + return RSAUtils.encrypt(rawText, publicKey); + } catch (CryptoException e) { + log.error("Fail to encrypt", e); + return null; + } + } +} diff --git a/src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/SymmetricCryptoService.java b/src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/SymmetricCryptoService.java new file mode 100644 index 0000000000..8d54c82270 --- /dev/null +++ b/src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/SymmetricCryptoService.java @@ -0,0 +1,130 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.crypto; + +import com.tencent.bk.sdk.crypto.cryptor.SymmetricCryptor; +import com.tencent.bk.sdk.crypto.cryptor.SymmetricCryptorFactory; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +import java.io.BufferedInputStream; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.HashMap; +import java.util.Map; + +/** + * 对称加密服务 + */ +@SuppressWarnings("unused") +@Slf4j +public class SymmetricCryptoService { + + private final CryptoConfigService cryptoConfigService; + private final Map cryptorMap = new HashMap<>(); + + public SymmetricCryptoService(CryptoConfigService cryptoConfigService) { + this.cryptoConfigService = cryptoConfigService; + } + + /** + * 对流数据加密，加密后的数据写入输出流中 + * + * @param key 密钥 + * @param in 输入流 + * @param out 输出流 + * @param cryptoScenarioEnum 加密场景 + */ + public void encrypt(String key, + InputStream in, + OutputStream out, + CryptoScenarioEnum cryptoScenarioEnum) { + String algorithm = cryptoConfigService.getSymmetricAlgorithmByScenario(cryptoScenarioEnum); + SymmetricCryptor cryptor = cryptorMap.computeIfAbsent(algorithm, SymmetricCryptorFactory::getCryptor); + cryptor.encrypt(key, in, out); + } + + /** + * 对明文信息加密，返回Base64编码的加密后的密文信息 + * + * @param message 要加密的明文信息，不可为空 + * @param cryptoScenarioEnum 加密场景 + * @return Base64编码的加密后的密文信息 + */ + public String encryptToBase64Str(String message, CryptoScenarioEnum cryptoScenarioEnum) { + return encryptToBase64Str(message, cryptoConfigService.getSymmetricAlgorithmByScenario(cryptoScenarioEnum)); + } + + /** + * 对明文信息加密，返回Base64编码的加密后的密文信息，明文为空则原样返回 + * + * @param message 要加密的明文信息 + * @param algorithm 加密算法，不可为空 + * @return Base64编码的加密后的密文信息 + */ + public String encryptToBase64Str(String message, String algorithm) { + assert StringUtils.isNotEmpty(algorithm); + if (StringUtils.isEmpty(message)) { + return message; + } + SymmetricCryptor cryptor = cryptorMap.computeIfAbsent(algorithm, SymmetricCryptorFactory::getCryptor); + return cryptor.encrypt(cryptoConfigService.getSymmetricPassword(), message); + } + + /** + * 对流数据解密，解密后的数据写入输出流中 + * + * @param key 密钥 + * @param in 输入流 + * @param out 输出流 + * @param algorithm 解密算法 + */ + public void decrypt(String key, + BufferedInputStream in, + OutputStream out, + String algorithm) { + SymmetricCryptor cryptor = cryptorMap.computeIfAbsent(algorithm, SymmetricCryptorFactory::getCryptor); + cryptor.decrypt(key, in, out); + } + + /** + * 对Base64编码的加密后的密文信息解密，返回解密后的明文，密文为空则原样返回 + * + * @param base64EncryptedMessage Base64编码的加密后的密文信息 + * @param algorithm 加密算法，不可为空 + * @return 解密后的明文信息 + */ + public String decrypt(String base64EncryptedMessage, String algorithm) { + assert StringUtils.isNotEmpty(algorithm); + if (StringUtils.isEmpty(base64EncryptedMessage)) { + return base64EncryptedMessage; + } + SymmetricCryptor cryptor = cryptorMap.computeIfAbsent(algorithm, SymmetricCryptorFactory::getCryptor); + return cryptor.decrypt( + cryptoConfigService.getSymmetricPassword(), + base64EncryptedMessage + ); + } +} diff --git a/src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/config/CryptoAutoConfiguration.java b/src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/config/CryptoAutoConfiguration.java new file mode 100644 index 0000000000..ba71b85c04 --- /dev/null +++ b/src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/config/CryptoAutoConfiguration.java @@ -0,0 +1,65 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.crypto.config; + +import com.tencent.bk.job.common.crypto.CryptoConfigService; +import com.tencent.bk.job.common.crypto.EncryptConfig; +import com.tencent.bk.job.common.crypto.SymmetricCryptoService; +import com.tencent.bk.job.common.crypto.scenario.CipherVariableCryptoService; +import com.tencent.bk.job.common.crypto.scenario.DbPasswordCryptoService; +import com.tencent.bk.job.common.crypto.scenario.SensitiveParamCryptoService; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration(proxyBeanMethods = false) +@EnableConfigurationProperties(EncryptConfig.class) +public class CryptoAutoConfiguration { + + @Bean + CryptoConfigService cryptoConfigService(EncryptConfig encryptConfig) { + return new CryptoConfigService(encryptConfig); + } + + @Bean + SymmetricCryptoService symmetricCryptoService(CryptoConfigService cryptoConfigService) { + return new SymmetricCryptoService(cryptoConfigService); + } + + @Bean + CipherVariableCryptoService cipherVariableCryptoService(SymmetricCryptoService symmetricCryptoService) { + return new CipherVariableCryptoService(symmetricCryptoService); + } + + @Bean + DbPasswordCryptoService dbPasswordCryptoService(SymmetricCryptoService symmetricCryptoService) { + return new DbPasswordCryptoService(symmetricCryptoService); + } + + @Bean + SensitiveParamCryptoService sensitiveParamCryptoService(SymmetricCryptoService symmetricCryptoService) { + return new SensitiveParamCryptoService(symmetricCryptoService); + } +} diff --git a/src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/scenario/CipherVariableCryptoService.java b/src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/scenario/CipherVariableCryptoService.java new file mode 100644 index 0000000000..9e6cd03bf4 --- /dev/null +++ b/src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/scenario/CipherVariableCryptoService.java @@ -0,0 +1,82 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.crypto.scenario; + +import com.tencent.bk.job.common.constant.TaskVariableTypeEnum; +import com.tencent.bk.job.common.crypto.CryptoScenarioEnum; +import com.tencent.bk.job.common.crypto.SymmetricCryptoService; +import com.tencent.bk.sdk.crypto.cryptor.consts.CryptorNames; +import com.tencent.bk.sdk.crypto.util.CryptorMetaUtil; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +/** + * 密文变量相关加解密服务 + */ +@Slf4j +public class CipherVariableCryptoService { + + private final SymmetricCryptoService symmetricCryptoService; + + public CipherVariableCryptoService(SymmetricCryptoService symmetricCryptoService) { + this.symmetricCryptoService = symmetricCryptoService; + } + + public String getCipherVariableEncryptAlgorithmByCipher(TaskVariableTypeEnum taskVariableTypeEnum, String cipher) { + if (!isCipherVariable(taskVariableTypeEnum) || StringUtils.isEmpty(cipher)) { + return CryptorNames.NONE; + } + String algorithm = CryptorMetaUtil.getCryptorNameFromCipher(cipher); + if (algorithm != null) { + return algorithm; + } + return CryptorNames.NONE; + } + + @SuppressWarnings("BooleanMethodIsAlwaysInverted") + private boolean isCipherVariable(TaskVariableTypeEnum taskVariableTypeEnum) { + return TaskVariableTypeEnum.CIPHER == taskVariableTypeEnum; + } + + public String encryptTaskVariableIfNeeded(TaskVariableTypeEnum taskVariableTypeEnum, String taskVariable) { + if (!isCipherVariable(taskVariableTypeEnum)) { + return taskVariable; + } + return symmetricCryptoService.encryptToBase64Str(taskVariable, CryptoScenarioEnum.CIPHER_VARIABLE); + } + + public String decryptTaskVariableIfNeeded(TaskVariableTypeEnum taskVariableTypeEnum, + String encryptedTaskVariable) { + if (!isCipherVariable(taskVariableTypeEnum)) { + return encryptedTaskVariable; + } + String algorithm = getCipherVariableEncryptAlgorithmByCipher(taskVariableTypeEnum, encryptedTaskVariable); + if (StringUtils.isBlank(algorithm)) { + return encryptedTaskVariable; + } + return symmetricCryptoService.decrypt(encryptedTaskVariable, algorithm); + } + +} diff --git a/src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/scenario/DbPasswordCryptoService.java b/src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/scenario/DbPasswordCryptoService.java new file mode 100644 index 0000000000..579b3f6cb7 --- /dev/null +++ b/src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/scenario/DbPasswordCryptoService.java @@ -0,0 +1,83 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.crypto.scenario; + +import com.tencent.bk.job.common.constant.AccountCategoryEnum; +import com.tencent.bk.job.common.crypto.CryptoScenarioEnum; +import com.tencent.bk.job.common.crypto.JobCryptorNames; +import com.tencent.bk.job.common.crypto.SymmetricCryptoService; +import com.tencent.bk.sdk.crypto.cryptor.consts.CryptorNames; +import com.tencent.bk.sdk.crypto.util.CryptorMetaUtil; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +/** + * DB账号密码相关加解密服务 + */ +@Slf4j +public class DbPasswordCryptoService { + + private final SymmetricCryptoService symmetricCryptoService; + + public DbPasswordCryptoService(SymmetricCryptoService symmetricCryptoService) { + this.symmetricCryptoService = symmetricCryptoService; + } + + public String getDbPasswordEncryptAlgorithmByCipher(AccountCategoryEnum accountCategoryEnum, String cipher) { + if (!isDbAccount(accountCategoryEnum) || StringUtils.isEmpty(cipher)) { + return CryptorNames.NONE; + } + String algorithm = CryptorMetaUtil.getCryptorNameFromCipher(cipher); + if (algorithm != null) { + return algorithm; + } + return JobCryptorNames.AES_CBC; + } + + @SuppressWarnings("BooleanMethodIsAlwaysInverted") + private boolean isDbAccount(AccountCategoryEnum accountCategoryEnum) { + return AccountCategoryEnum.DB == accountCategoryEnum; + } + + public String encryptDbPasswordIfNeeded(AccountCategoryEnum accountCategoryEnum, String dbPassword) { + if (!isDbAccount(accountCategoryEnum)) { + return dbPassword; + } + return symmetricCryptoService.encryptToBase64Str(dbPassword, CryptoScenarioEnum.DATABASE_PASSWORD); + } + + public String decryptDbPasswordIfNeeded(AccountCategoryEnum accountCategoryEnum, + String encryptedDbPassword) { + if (!isDbAccount(accountCategoryEnum)) { + return encryptedDbPassword; + } + String algorithm = getDbPasswordEncryptAlgorithmByCipher(accountCategoryEnum, encryptedDbPassword); + if (StringUtils.isBlank(algorithm)) { + return encryptedDbPassword; + } + return symmetricCryptoService.decrypt(encryptedDbPassword, algorithm); + } + +} diff --git a/src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/scenario/SensitiveParamCryptoService.java b/src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/scenario/SensitiveParamCryptoService.java new file mode 100644 index 0000000000..53982c4a85 --- /dev/null +++ b/src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/scenario/SensitiveParamCryptoService.java @@ -0,0 +1,72 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.crypto.scenario; + +import com.tencent.bk.job.common.crypto.CryptoScenarioEnum; +import com.tencent.bk.job.common.crypto.SymmetricCryptoService; +import com.tencent.bk.sdk.crypto.cryptor.consts.CryptorNames; +import com.tencent.bk.sdk.crypto.util.CryptorMetaUtil; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +/** + * 脚本敏感参数相关加解密服务 + */ +@Slf4j +public class SensitiveParamCryptoService { + + private final SymmetricCryptoService symmetricCryptoService; + + public SensitiveParamCryptoService(SymmetricCryptoService symmetricCryptoService) { + this.symmetricCryptoService = symmetricCryptoService; + } + + public String getSecureParamEncryptAlgorithmByCipher(boolean secureParam, String cipher) { + if (!secureParam || StringUtils.isEmpty(cipher)) { + return CryptorNames.NONE; + } + String algorithm = CryptorMetaUtil.getCryptorNameFromCipher(cipher); + if (algorithm != null) { + return algorithm; + } + return CryptorNames.NONE; + } + + public String encryptParamIfNeeded(boolean secureParam, String param) { + if (!secureParam) { + return param; + } + return symmetricCryptoService.encryptToBase64Str(param, CryptoScenarioEnum.SCRIPT_SENSITIVE_PARAM); + } + + public String decryptParamIfNeeded(boolean secureParam, String encryptedParam) { + if (!secureParam) { + return encryptedParam; + } + String algorithm = getSecureParamEncryptAlgorithmByCipher(secureParam, encryptedParam); + return symmetricCryptoService.decrypt(encryptedParam, algorithm); + } + +} diff --git a/src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/util/AESUtils.java b/src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/util/AESUtils.java new file mode 100644 index 0000000000..d0798c9683 --- /dev/null +++ b/src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/util/AESUtils.java @@ -0,0 +1,219 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.crypto.util; + +import com.tencent.bk.job.common.util.Base64Util; +import com.tencent.bk.sdk.crypto.exception.CryptoException; +import lombok.extern.slf4j.Slf4j; +import org.slf4j.helpers.MessageFormatter; + +import javax.crypto.Cipher; +import javax.crypto.KeyGenerator; +import javax.crypto.spec.IvParameterSpec; +import javax.crypto.spec.SecretKeySpec; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; +import java.security.NoSuchAlgorithmException; +import java.security.NoSuchProviderException; +import java.security.SecureRandom; +import java.util.Arrays; + +@Slf4j +public class AESUtils { + /** + * 加密/解密算法/工作模式/填充方式 + */ + private static final String CIPHER_ALGORITHM = "AES/CBC/PKCS5Padding"; + + /** + * 加密数据，并是使用base64编码 + * + * @param data 待加密数据 + * @param password 密钥 + * @return byte[] 加密后的数据 + */ + public static String encryptToBase64EncodedCipherText(String data, String password) throws CryptoException { + return Base64Util.encodeContentToStr(encrypt(data, password)); + } + + /** + * 解密base64编码的数据，并转换成字符明文 + * + * @param data 待解密数据，base64编码 + * @param password 密钥 + * @return String 解密后的数据 + */ + public static String decryptBase64EncodedDataToPlainText(String data, String password) throws CryptoException { + return decryptToPlainText(Base64Util.decodeContentToByte(data), password); + } + + /** + * 加密数据 + * + * @param data 待加密数据 + * @param key 密钥 + * @return byte[] 加密后的数据 + */ + public static byte[] encrypt(byte[] data, byte[] key) throws CryptoException { + byte[] iv; + byte[] finalData; + try { + Cipher cipher = Cipher.getInstance(CIPHER_ALGORITHM); + cipher.init(Cipher.ENCRYPT_MODE, getKeySpec(cipher, key)); + + iv = cipher.getIV(); + finalData = cipher.doFinal(data); + } catch (Exception e) { + String msg = MessageFormatter.arrayFormat( + "Fail to encrypt using {}, data.len={}, key.len={}", + new Object[]{CIPHER_ALGORITHM, data.length, key.length} + ).getMessage(); + throw new CryptoException(msg, e); + } + if (iv == null) { + throw new RuntimeException(String.format("CIPHER_ALGORITHM %s is invalid", CIPHER_ALGORITHM)); + } + byte[] finalDataWithIv = new byte[finalData.length + iv.length]; + System.arraycopy(iv, 0, finalDataWithIv, 0, iv.length); + System.arraycopy(finalData, 0, finalDataWithIv, iv.length, finalData.length); + return finalDataWithIv; + } + + /** + * 解密数据 + * + * @param data 待解密数据 + * @param key 密钥 + * @return byte[] 解密后的数据 + */ + public static byte[] decrypt(byte[] data, byte[] key) throws CryptoException { + try { + Cipher cipher = Cipher.getInstance(CIPHER_ALGORITHM); + cipher.init(Cipher.DECRYPT_MODE, getKeySpec(cipher, key), getIvSpec(cipher, data)); + byte[] dataWithoutIv = new byte[data.length - cipher.getBlockSize()]; + System.arraycopy(data, cipher.getBlockSize(), dataWithoutIv, + 0, data.length - cipher.getBlockSize()); + return cipher.doFinal(dataWithoutIv); + } catch (Exception e) { + String msg = MessageFormatter.arrayFormat( + "Fail to decrypt using {}, data.len={}, key.len={}", + new Object[]{CIPHER_ALGORITHM, data.length, key.length} + ).getMessage(); + throw new CryptoException(msg, e); + } + } + + public static void encrypt(InputStream in, OutputStream out, String password) throws CryptoException { + byte[] key = password.getBytes(StandardCharsets.UTF_8); + try { + Cipher cipher = Cipher.getInstance(CIPHER_ALGORITHM); + cipher.init(Cipher.ENCRYPT_MODE, getKeySpec(cipher, key)); + byte[] arr = cipher.getIV(); + if (arr == null) { + throw new RuntimeException(String.format("CIPHER_ALGORITHM %s is invalid", CIPHER_ALGORITHM)); + } + out.write(arr); + write(in, out, cipher); + } catch (Exception e) { + String msg = MessageFormatter.format( + "Fail to encrypt using {}, password.len={}", CIPHER_ALGORITHM, password.length() + ).getMessage(); + throw new CryptoException(msg, e); + } + } + + public static void decrypt(InputStream in, OutputStream out, String password) throws CryptoException { + try { + log.debug("decrypt: in.available={}", in.available()); + byte[] key = password.getBytes(StandardCharsets.UTF_8); + Cipher cipher = Cipher.getInstance(CIPHER_ALGORITHM); + byte[] iv = new byte[cipher.getBlockSize()]; + if (in.read(iv) < iv.length) { + throw new RuntimeException(); + } + log.debug("decrypt: iv={}", Arrays.toString(iv)); + cipher.init(Cipher.DECRYPT_MODE, getKeySpec(cipher, key), new IvParameterSpec(iv)); + write(in, out, cipher); + } catch (Exception e) { + String msg = MessageFormatter.format( + "Fail to decrypt using {}, password.len={}", CIPHER_ALGORITHM, password.length() + ).getMessage(); + throw new CryptoException(msg, e); + } + } + + /** + * 解密数据 + * + * @param data 待解密数据 + * @param password 密钥 + * @return byte[] 解密后的数据 + */ + private static String decryptToPlainText(byte[] data, String password) throws CryptoException { + return new String(decrypt(data, password.getBytes(StandardCharsets.UTF_8)), StandardCharsets.UTF_8); + } + + /** + * 加密数据 + * + * @param data 待加密数据 + * @param password 密钥 + * @return byte[] 加密后的数据 + */ + private static byte[] encrypt(String data, String password) throws CryptoException { + return encrypt(data.getBytes(StandardCharsets.UTF_8), password.getBytes(StandardCharsets.UTF_8)); + } + + private static IvParameterSpec getIvSpec(Cipher cipher, byte[] data) { + byte[] iv = new byte[cipher.getBlockSize()]; + System.arraycopy(data, 0, iv, 0, iv.length); + return new IvParameterSpec(iv); + } + + private static SecretKeySpec getKeySpec(Cipher cipher, byte[] key) + throws NoSuchAlgorithmException, NoSuchProviderException { + KeyGenerator kgen = KeyGenerator.getInstance("AES"); + SecureRandom random = SecureRandom.getInstance("SHA1PRNG", "SUN"); + random.setSeed(key); + kgen.init(cipher.getBlockSize() * 8, random); + return new SecretKeySpec(kgen.generateKey().getEncoded(), "AES"); + } + + private static void write(InputStream in, OutputStream out, Cipher cipher) throws Exception { + byte[] iBuffer = new byte[1024]; + int len; + while ((len = in.read(iBuffer)) != -1) { + byte[] oBuffer = cipher.update(iBuffer, 0, len); + if (oBuffer != null) { + out.write(oBuffer); + } + } + byte[] oBuffer = cipher.doFinal(); + if (oBuffer != null) { + out.write(oBuffer); + } + } +} diff --git a/src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/util/RSAUtils.java b/src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/util/RSAUtils.java new file mode 100644 index 0000000000..6ac235a9ab --- /dev/null +++ b/src/backend/commons/common-crypto/src/main/java/com/tencent/bk/job/common/crypto/util/RSAUtils.java @@ -0,0 +1,134 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.crypto.util; + +import com.google.common.collect.Lists; +import com.tencent.bk.job.common.util.Base64Util; +import com.tencent.bk.sdk.crypto.exception.CryptoException; +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.helpers.MessageFormatter; + +import javax.crypto.Cipher; +import java.io.BufferedReader; +import java.io.IOException; +import java.io.StringReader; +import java.security.GeneralSecurityException; +import java.security.KeyFactory; +import java.security.PublicKey; +import java.security.interfaces.RSAPrivateKey; +import java.security.interfaces.RSAPublicKey; +import java.security.spec.PKCS8EncodedKeySpec; +import java.security.spec.X509EncodedKeySpec; +import java.util.List; + +/** + * RSAUtils 加解密 + */ +public class RSAUtils { + private static final String KEY_ALGORITHM = "RSA"; + + private static final String BEGIN_ENCRYPTED_PRIVATE_KEY = "-----BEGIN ENCRYPTED PRIVATE KEY-----"; + private static final String END_ENCRYPTED_PRIVATE_KEY = "-----END ENCRYPTED PRIVATE KEY-----"; + private static final String BEGIN_RSA_PRIVATE_KEY = "-----BEGIN RSA PRIVATE KEY-----"; + private static final String END_RSA_PRIVATE_KEY = "-----END RSA PRIVATE KEY-----"; + + private static final String BEGIN_PRIVATE_KEY = "-----BEGIN PRIVATE KEY-----"; + private static final String END_PRIVATE_KEY = "-----END PRIVATE KEY-----"; + private static final String BEGIN_PUBLIC_KEY = "-----BEGIN PUBLIC KEY-----"; + private static final String END_PUBLIC_KEY = "-----END PUBLIC KEY-----"; + + private static final String CHARSET_NAME = "UTF-8"; + private static final String LINE_SPLIT = "\n"; + + private static final List SKIP_STR = Lists.newArrayList( + BEGIN_PRIVATE_KEY, END_PRIVATE_KEY, BEGIN_PUBLIC_KEY, END_PUBLIC_KEY, + BEGIN_ENCRYPTED_PRIVATE_KEY, END_ENCRYPTED_PRIVATE_KEY, BEGIN_RSA_PRIVATE_KEY, END_RSA_PRIVATE_KEY + ); + + public static RSAPrivateKey getPrivateKey(String rsaPrivateKeyBase64) throws CryptoException { + try { + String privateKeyPEM = getPermKey(rsaPrivateKeyBase64); + byte[] encoded = Base64.decodeBase64(privateKeyPEM); + return (RSAPrivateKey) KeyFactory.getInstance(KEY_ALGORITHM).generatePrivate(new PKCS8EncodedKeySpec(encoded)); + } catch (Exception e) { + String msg = MessageFormatter.arrayFormat( + "Fail to getPrivateKey using {}, rsaPrivateKeyBase64.len={}", + new Object[]{KEY_ALGORITHM, rsaPrivateKeyBase64.length()} + ).getMessage(); + throw new CryptoException(msg, e); + } + } + + public static RSAPublicKey getPublicKey(String rsaPublicKeyBase64) throws CryptoException { + try { + byte[] encoded = Base64.decodeBase64(getPermKey(rsaPublicKeyBase64)); + return (RSAPublicKey) KeyFactory.getInstance(KEY_ALGORITHM).generatePublic(new X509EncodedKeySpec(encoded)); + } catch (Exception e) { + String msg = MessageFormatter.arrayFormat( + "Fail to getPublicKey using {}, rsaPublicKeyBase64={}", + new Object[]{KEY_ALGORITHM, rsaPublicKeyBase64} + ).getMessage(); + throw new CryptoException(msg, e); + } + } + + public static String encrypt(String rawText, PublicKey publicKey) throws CryptoException { + try { + return encrypt(rawText.getBytes(CHARSET_NAME), publicKey); + } catch (Exception e) { + String msg = MessageFormatter.arrayFormat( + "Fail to getPublicKey using {}, rawText.len={}, publicKey={}", + new Object[]{KEY_ALGORITHM, rawText.length(), publicKey} + ).getMessage(); + throw new CryptoException(msg, e); + } + } + + private static String getPermKey(String permBase64) throws IOException { + String perm = Base64Util.decodeContentToStr(permBase64); + if (StringUtils.isEmpty(perm)) { + throw new IOException("Perm content is empty"); + } + StringBuilder strKeyPEM = new StringBuilder(2048); + try (BufferedReader br = new BufferedReader(new StringReader(perm))) { + String line; + while ((line = br.readLine()) != null) { + if (SKIP_STR.contains(line)) { + continue; + } + strKeyPEM.append(line).append(LINE_SPLIT); + } + } + return strKeyPEM.toString(); + } + + private static String encrypt(byte[] messageBytes, + PublicKey publicKey) throws GeneralSecurityException { + Cipher cipher = Cipher.getInstance(KEY_ALGORITHM); + cipher.init(Cipher.ENCRYPT_MODE, publicKey); + return Base64.encodeBase64String(cipher.doFinal(messageBytes)); + } +} diff --git a/src/backend/commons/common-crypto/src/main/resources/META-INF/services/com.tencent.bk.sdk.crypto.cryptor.SymmetricCryptor b/src/backend/commons/common-crypto/src/main/resources/META-INF/services/com.tencent.bk.sdk.crypto.cryptor.SymmetricCryptor new file mode 100644 index 0000000000..e91405d3e9 --- /dev/null +++ b/src/backend/commons/common-crypto/src/main/resources/META-INF/services/com.tencent.bk.sdk.crypto.cryptor.SymmetricCryptor @@ -0,0 +1 @@ +com.tencent.bk.job.common.crypto.AESCryptor diff --git a/src/backend/commons/common-crypto/src/main/resources/META-INF/spring.factories b/src/backend/commons/common-crypto/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..c48c074ab8 --- /dev/null +++ b/src/backend/commons/common-crypto/src/main/resources/META-INF/spring.factories @@ -0,0 +1,2 @@ +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ +com.tencent.bk.job.common.crypto.config.CryptoAutoConfiguration diff --git a/src/backend/commons/common-utils/src/test/java/com/tencent/bk/job/common/util/crypto/AESUtilsTest.java b/src/backend/commons/common-crypto/src/test/java/com/tencent/bk/job/common/crypto/util/AESUtilsTest.java similarity index 81% rename from src/backend/commons/common-utils/src/test/java/com/tencent/bk/job/common/util/crypto/AESUtilsTest.java rename to src/backend/commons/common-crypto/src/test/java/com/tencent/bk/job/common/crypto/util/AESUtilsTest.java index bcc191d214..4e1f9ea11c 100644 --- a/src/backend/commons/common-utils/src/test/java/com/tencent/bk/job/common/util/crypto/AESUtilsTest.java +++ b/src/backend/commons/common-crypto/src/test/java/com/tencent/bk/job/common/crypto/util/AESUtilsTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,27 +22,26 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.common.util.crypto; +package com.tencent.bk.job.common.crypto.util; +import org.assertj.core.api.AssertionsForClassTypes; import org.junit.jupiter.api.Test; -import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThat; - public class AESUtilsTest { @Test - void testEncryptToBase64EncodedCipherText() throws Exception { + void testEncryptToBase64EncodedCipherText() { String text = "job"; String encryptPassword = "job#123"; String encryptedData = AESUtils.encryptToBase64EncodedCipherText(text, encryptPassword); - assertThat(encryptedData).isNotEmpty(); + AssertionsForClassTypes.assertThat(encryptedData).isNotEmpty(); } @Test - void testDecryptBase64EncodedCipherText() throws Exception { + void testDecryptBase64EncodedCipherText() { String encryptedBase64EncodedData = "GQ6kLqtMevL8z/kXGVANQ+VP5o2Bt30yzXALfZbeOoY="; String encryptPassword = "job#123"; String decodeData = AESUtils.decryptBase64EncodedDataToPlainText(encryptedBase64EncodedData, encryptPassword); - assertThat(decodeData).isEqualTo("job"); + AssertionsForClassTypes.assertThat(decodeData).isEqualTo("job"); } } diff --git a/src/backend/commons/common-discovery/build.gradle b/src/backend/commons/common-discovery/build.gradle index 79a442392a..0814abe690 100644 --- a/src/backend/commons/common-discovery/build.gradle +++ b/src/backend/commons/common-discovery/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-discovery/src/main/java/com/tencent/bk/job/common/discovery/ServiceInfoProvider.java b/src/backend/commons/common-discovery/src/main/java/com/tencent/bk/job/common/discovery/ServiceInfoProvider.java index 00fcb4d20e..a3e54bf70c 100644 --- a/src/backend/commons/common-discovery/src/main/java/com/tencent/bk/job/common/discovery/ServiceInfoProvider.java +++ b/src/backend/commons/common-discovery/src/main/java/com/tencent/bk/job/common/discovery/ServiceInfoProvider.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-discovery/src/main/java/com/tencent/bk/job/common/discovery/model/ServiceInstanceInfoDTO.java b/src/backend/commons/common-discovery/src/main/java/com/tencent/bk/job/common/discovery/model/ServiceInstanceInfoDTO.java index 0e48d88691..b74455ccf9 100644 --- a/src/backend/commons/common-discovery/src/main/java/com/tencent/bk/job/common/discovery/model/ServiceInstanceInfoDTO.java +++ b/src/backend/commons/common-discovery/src/main/java/com/tencent/bk/job/common/discovery/model/ServiceInstanceInfoDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-discovery/src/main/java/com/tencent/bk/job/common/discovery/processor/MachineInfoEnvironmentPostProcessor.java b/src/backend/commons/common-discovery/src/main/java/com/tencent/bk/job/common/discovery/processor/MachineInfoEnvironmentPostProcessor.java index 6851e7d1a4..6ec4147793 100644 --- a/src/backend/commons/common-discovery/src/main/java/com/tencent/bk/job/common/discovery/processor/MachineInfoEnvironmentPostProcessor.java +++ b/src/backend/commons/common-discovery/src/main/java/com/tencent/bk/job/common/discovery/processor/MachineInfoEnvironmentPostProcessor.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,22 +25,39 @@ package com.tencent.bk.job.common.discovery.processor; import com.tencent.bk.job.common.util.ip.IpUtils; +import org.apache.commons.lang3.StringUtils; import org.springframework.boot.SpringApplication; +import org.springframework.boot.context.event.ApplicationContextInitializedEvent; import org.springframework.boot.env.EnvironmentPostProcessor; +import org.springframework.boot.logging.DeferredLog; +import org.springframework.context.ApplicationListener; import org.springframework.core.env.ConfigurableEnvironment; import org.springframework.core.env.MapPropertySource; import java.util.HashMap; import java.util.Map; -public class MachineInfoEnvironmentPostProcessor implements EnvironmentPostProcessor { +public class MachineInfoEnvironmentPostProcessor implements EnvironmentPostProcessor, + ApplicationListener { + + private static final DeferredLog log = new DeferredLog(); @Override public void postProcessEnvironment(ConfigurableEnvironment env, SpringApplication application) { Map map = new HashMap<>(); - map.put("machine.ip", IpUtils.getFirstMachineIP()); - env.getPropertySources().addFirst( - new MapPropertySource("machine-info-properties", map)); + String firstIp = IpUtils.getFirstMachineIP(); + if (StringUtils.isBlank(firstIp)) { + log.warn("Got blank ip, ignore to put ip into environment"); + return; + } + map.put("machine.ip", firstIp); + env.getPropertySources().addFirst(new MapPropertySource("machine-info-properties", map)); + } + + @SuppressWarnings("NullableProblems") + @Override + public void onApplicationEvent(ApplicationContextInitializedEvent event) { + log.replayTo(MachineInfoEnvironmentPostProcessor.class); } } diff --git a/src/backend/commons/common-i18n/build.gradle b/src/backend/commons/common-i18n/build.gradle index c89c3a14ea..73acbf3c23 100644 --- a/src/backend/commons/common-i18n/build.gradle +++ b/src/backend/commons/common-i18n/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-i18n/src/main/java/com/tencent/bk/job/common/i18n/config/I18nAutoConfiguration.java b/src/backend/commons/common-i18n/src/main/java/com/tencent/bk/job/common/i18n/config/I18nAutoConfiguration.java new file mode 100644 index 0000000000..67b51bfe07 --- /dev/null +++ b/src/backend/commons/common-i18n/src/main/java/com/tencent/bk/job/common/i18n/config/I18nAutoConfiguration.java @@ -0,0 +1,53 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.i18n.config; + +import com.tencent.bk.job.common.i18n.messagesource.MultiReloadableResourceBundleMessageSource; +import com.tencent.bk.job.common.i18n.service.MessageI18nService; +import com.tencent.bk.job.common.i18n.service.impl.MessageI18nServiceImpl; +import org.springframework.context.MessageSource; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.support.ReloadableResourceBundleMessageSource; + +import java.nio.charset.StandardCharsets; + +@Configuration(proxyBeanMethods = false) +public class I18nAutoConfiguration { + + @Bean("messageSource") + public ReloadableResourceBundleMessageSource messageSource() { + MultiReloadableResourceBundleMessageSource messageSource = new MultiReloadableResourceBundleMessageSource(); + messageSource.addBasenames("classpath*:i18n/message", "classpath*:i18n/exception/message", + "classpath*:i18n/common/message", "classpath*:i18n/validation/ValidationMessages"); + messageSource.setDefaultEncoding(StandardCharsets.UTF_8.name()); + return messageSource; + } + + @Bean("messageI18nService") + public MessageI18nService messageI18nService(MessageSource messageSource) { + return new MessageI18nServiceImpl(messageSource); + } +} diff --git a/src/backend/commons/common-i18n/src/main/java/com/tencent/bk/job/common/i18n/config/MessageSourceConfig.java b/src/backend/commons/common-i18n/src/main/java/com/tencent/bk/job/common/i18n/config/MessageSourceConfig.java deleted file mode 100644 index f4dbf7ebb7..0000000000 --- a/src/backend/commons/common-i18n/src/main/java/com/tencent/bk/job/common/i18n/config/MessageSourceConfig.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.i18n.config; - -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.support.ReloadableResourceBundleMessageSource; - -import java.nio.charset.StandardCharsets; - -@Configuration -public class MessageSourceConfig { - - @Bean("messageSource") - public ReloadableResourceBundleMessageSource messageSource() { - MultiReloadableResourceBundleMessageSource messageSource = new MultiReloadableResourceBundleMessageSource(); - messageSource.addBasenames("classpath:i18n/message", "classpath*:i18n/exception/message", - "classpath*:i18n/common/message", "classpath*:i18n/validation/ValidationMessages"); - messageSource.setDefaultEncoding(StandardCharsets.UTF_8.name()); - return messageSource; - } - -} - diff --git a/src/backend/commons/common-i18n/src/main/java/com/tencent/bk/job/common/i18n/locale/BkConsts.java b/src/backend/commons/common-i18n/src/main/java/com/tencent/bk/job/common/i18n/locale/BkConsts.java new file mode 100644 index 0000000000..ad89cfffb9 --- /dev/null +++ b/src/backend/commons/common-i18n/src/main/java/com/tencent/bk/job/common/i18n/locale/BkConsts.java @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.i18n.locale; + +/** + * 蓝鲸公共协议涉及的常量 + */ +public class BkConsts { + + // 国际化语言Header取值 + public static final String HEADER_VALUE_LANG_ZH = "zh"; + public static final String HEADER_VALUE_LANG_ZH_CN = "zh-cn"; + public static final String HEADER_VALUE_LANG_EN = "en"; + public static final String HEADER_VALUE_LANG_EN_US = "en-us"; + +} diff --git a/src/backend/commons/common-i18n/src/main/java/com/tencent/bk/job/common/i18n/locale/LocaleUtils.java b/src/backend/commons/common-i18n/src/main/java/com/tencent/bk/job/common/i18n/locale/LocaleUtils.java index 1de0bc1294..5d8c21cb4c 100644 --- a/src/backend/commons/common-i18n/src/main/java/com/tencent/bk/job/common/i18n/locale/LocaleUtils.java +++ b/src/backend/commons/common-i18n/src/main/java/com/tencent/bk/job/common/i18n/locale/LocaleUtils.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -38,6 +38,11 @@ public class LocaleUtils { public static final String LANG_ZH = "zh"; public static final String LANG_EN = "en"; public static final String LANG_EN_US = "en_US"; + + /** + * 蓝鲸通用的LANG HEADER + */ + public static final String BLUEKING_LANG_HEADER = "blueking_language"; /** * 定义项目通用的LANG HEADER */ @@ -46,19 +51,28 @@ public class LocaleUtils { private static final Map localeMap = new HashMap<>(); static { - localeMap.put("zh", LANG_ZH); - localeMap.put("zh-cn", LANG_ZH_CN); + localeMap.put(BkConsts.HEADER_VALUE_LANG_ZH, LANG_ZH); + localeMap.put(BkConsts.HEADER_VALUE_LANG_ZH_CN, LANG_ZH_CN); localeMap.put("zh_cn", LANG_ZH_CN); - localeMap.put("en", LANG_EN); - localeMap.put("en-us", LANG_EN_US); + localeMap.put(BkConsts.HEADER_VALUE_LANG_EN, LANG_EN); + localeMap.put(BkConsts.HEADER_VALUE_LANG_EN_US, LANG_EN_US); localeMap.put("en_us", LANG_EN_US); } + private static final Map bkLangMap = new HashMap<>(); + + static { + bkLangMap.put(LANG_ZH, BkConsts.HEADER_VALUE_LANG_ZH); + bkLangMap.put(LANG_ZH_CN, BkConsts.HEADER_VALUE_LANG_ZH_CN); + bkLangMap.put(LANG_EN, BkConsts.HEADER_VALUE_LANG_EN); + bkLangMap.put(LANG_EN_US, BkConsts.HEADER_VALUE_LANG_EN_US); + } + /** * 对不同形式的language Header值进行统一 * - * @param lang - * @return + * @param lang 语言Header取值 + * @return 标准化后的语言Header取值 */ public static String getNormalLang(String lang) { lang = lang.toLowerCase(); @@ -68,6 +82,19 @@ public static String getNormalLang(String lang) { return lang; } + /** + * 根据标准语言取值获取蓝鲸协议语言参数取值 + * + * @param normalLang 标准语言取值 + * @return 蓝鲸协议语言参数取值 + */ + public static String getBkLang(String normalLang) { + if (bkLangMap.containsKey(normalLang)) { + return bkLangMap.get(normalLang); + } + return null; + } + public static Locale getLocale(String lang) { if (StringUtils.isBlank(lang)) { return Locale.SIMPLIFIED_CHINESE; diff --git a/src/backend/commons/common-i18n/src/main/java/com/tencent/bk/job/common/i18n/config/MultiReloadableResourceBundleMessageSource.java b/src/backend/commons/common-i18n/src/main/java/com/tencent/bk/job/common/i18n/messagesource/MultiReloadableResourceBundleMessageSource.java similarity index 93% rename from src/backend/commons/common-i18n/src/main/java/com/tencent/bk/job/common/i18n/config/MultiReloadableResourceBundleMessageSource.java rename to src/backend/commons/common-i18n/src/main/java/com/tencent/bk/job/common/i18n/messagesource/MultiReloadableResourceBundleMessageSource.java index 20a49b3138..a761a81e38 100644 --- a/src/backend/commons/common-i18n/src/main/java/com/tencent/bk/job/common/i18n/config/MultiReloadableResourceBundleMessageSource.java +++ b/src/backend/commons/common-i18n/src/main/java/com/tencent/bk/job/common/i18n/messagesource/MultiReloadableResourceBundleMessageSource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.common.i18n.config; +package com.tencent.bk.job.common.i18n.messagesource; import org.springframework.context.support.ReloadableResourceBundleMessageSource; import org.springframework.core.io.Resource; @@ -37,7 +37,7 @@ public class MultiReloadableResourceBundleMessageSource extends ReloadableResourceBundleMessageSource { private static final String PROPERTIES_SUFFIX = ".properties"; - private PathMatchingResourcePatternResolver resolver = new PathMatchingResourcePatternResolver(); + private final PathMatchingResourcePatternResolver resolver = new PathMatchingResourcePatternResolver(); @Override protected PropertiesHolder refreshProperties(String filename, PropertiesHolder propHolder) { diff --git a/src/backend/commons/common-i18n/src/main/java/com/tencent/bk/job/common/i18n/service/MessageI18nService.java b/src/backend/commons/common-i18n/src/main/java/com/tencent/bk/job/common/i18n/service/MessageI18nService.java index 7f3d074177..19167efa54 100644 --- a/src/backend/commons/common-i18n/src/main/java/com/tencent/bk/job/common/i18n/service/MessageI18nService.java +++ b/src/backend/commons/common-i18n/src/main/java/com/tencent/bk/job/common/i18n/service/MessageI18nService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -50,9 +50,19 @@ public interface MessageI18nService { /** * 根据locale获取国际化 * - * @param msgKey - * @param locale - * @return + * @param locale 语言 + * @param msgKey i18nKey + * @return 国际化之后的值 + */ + String getI18n(Locale locale, String msgKey); + + /** + * 根据locale获取国际化 + * + * @param locale 语言 + * @param msgKey i18nKey + * @param args 参数 + * @return 国际化之后的值 */ - String getI18n(String msgKey, Locale locale); + String getI18nWithArgs(Locale locale, String msgKey, Object... args); } diff --git a/src/backend/commons/common-i18n/src/main/java/com/tencent/bk/job/common/i18n/service/impl/MessageI18nServiceImpl.java b/src/backend/commons/common-i18n/src/main/java/com/tencent/bk/job/common/i18n/service/impl/MessageI18nServiceImpl.java index eafff9d27f..11cc695ca7 100644 --- a/src/backend/commons/common-i18n/src/main/java/com/tencent/bk/job/common/i18n/service/impl/MessageI18nServiceImpl.java +++ b/src/backend/commons/common-i18n/src/main/java/com/tencent/bk/job/common/i18n/service/impl/MessageI18nServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -30,15 +30,13 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.MessageSource; import org.springframework.context.i18n.LocaleContextHolder; -import org.springframework.stereotype.Service; import java.util.Locale; /** - * @date 2019/09/19 + * 国际化 */ @Slf4j -@Service public class MessageI18nServiceImpl implements MessageI18nService { private final MessageSource messageSource; @@ -56,7 +54,7 @@ public String getI18n(String msgKey) { } @Override - public String getI18n(String msgKey, Locale locale) { + public String getI18n(Locale locale, String msgKey) { if (StringUtils.isBlank(msgKey)) { return ""; } @@ -70,4 +68,13 @@ public String getI18nWithArgs(String msgKey, Object... args) { } return messageSource.getMessage(msgKey, args, LocaleContextHolder.getLocale()); } + + + @Override + public String getI18nWithArgs(Locale locale, String msgKey, Object... args) { + if (StringUtils.isBlank(msgKey)) { + return ""; + } + return messageSource.getMessage(msgKey, args, locale); + } } diff --git a/src/backend/commons/common-i18n/src/main/resources/META-INF/spring.factories b/src/backend/commons/common-i18n/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..6cc6311282 --- /dev/null +++ b/src/backend/commons/common-i18n/src/main/resources/META-INF/spring.factories @@ -0,0 +1,2 @@ +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ +com.tencent.bk.job.common.i18n.config.I18nAutoConfiguration diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/common/message.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/common/message.properties index 8d91002035..12f75a23e0 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/common/message.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/common/message.properties @@ -1,7 +1,7 @@ # # Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. # -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +# Copyright (C) 2021 Tencent. All rights reserved. # # BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. # @@ -21,7 +21,6 @@ # CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # - # 权限部分 # 系统 system.bk_job=作业平台 @@ -37,13 +36,14 @@ permission.resource.type.name.account=执行账号 permission.resource.type.name.public_script=公共脚本 permission.resource.type.name.tag=标签 permission.resource.type.name.host=主机 -permission.resource.type.name.sys_cloud_area=云区域 +permission.resource.type.name.sys_cloud_area=管控区域 permission.resource.type.name.set=集群 permission.resource.type.name.module=模块 permission.resource.type.name.biz_custom_query=动态分组 permission.resource.type.name.dashboard_view=运营视图 permission.resource.type.name.file_source=文件源 permission.resource.type.name.ticket=凭证 +permission.resource.type.name.container=容器 # 操作Action # 业务查看 permission.action.name.access_business=业务访问 @@ -109,3 +109,9 @@ permission.action.name.global_settings=全局设置管理 # 高危语句 permission.action.name.high-risk_detect_rule=高危语句规则管理 permission.action.name.high-risk_detect_record=高危语句拦截记录查看 + +resource.topo_node.name=节点 +resource.host.name=主机 +resource.dynamic_group.name=动态分组 +resource.biz_set.name=业务集 +resource.container.name=容器 diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/common/message_en.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/common/message_en.properties index 1bbf203034..7ec532d280 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/common/message_en.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/common/message_en.properties @@ -1,7 +1,7 @@ # # Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. # -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +# Copyright (C) 2021 Tencent. All rights reserved. # # BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. # @@ -24,8 +24,8 @@ # 权限部分 # 系统 -system.bk_job=job -system.bk_cmdb=cmdb +system.bk_job=Job +system.bk_cmdb=Cmdb # 资源类型 permission.resource.type.name.biz=Business permission.resource.type.name.business_set=Business Set @@ -44,6 +44,7 @@ permission.resource.type.name.biz_custom_query=Dynamic Groups permission.resource.type.name.dashboard_view=Dashboard Views permission.resource.type.name.file_source=File Sources permission.resource.type.name.ticket=Tickets +permission.resource.type.name.container=Containers # 操作Action # 业务查看 permission.action.name.access_business=Access Business @@ -109,3 +110,9 @@ permission.action.name.global_settings=Manage Global Settings # 高危语句 permission.action.name.high-risk_detect_rule=High-risk Detect Rule Setting permission.action.name.high-risk_detect_record=High-risk Detect Record + +resource.topo_node.name=node +resource.host.name=host +resource.dynamic_group.name=group +resource.biz_set.name=business set +resource.container.name=container diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/common/message_en_US.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/common/message_en_US.properties index 1bbf203034..7ec532d280 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/common/message_en_US.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/common/message_en_US.properties @@ -1,7 +1,7 @@ # # Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. # -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +# Copyright (C) 2021 Tencent. All rights reserved. # # BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. # @@ -24,8 +24,8 @@ # 权限部分 # 系统 -system.bk_job=job -system.bk_cmdb=cmdb +system.bk_job=Job +system.bk_cmdb=Cmdb # 资源类型 permission.resource.type.name.biz=Business permission.resource.type.name.business_set=Business Set @@ -44,6 +44,7 @@ permission.resource.type.name.biz_custom_query=Dynamic Groups permission.resource.type.name.dashboard_view=Dashboard Views permission.resource.type.name.file_source=File Sources permission.resource.type.name.ticket=Tickets +permission.resource.type.name.container=Containers # 操作Action # 业务查看 permission.action.name.access_business=Access Business @@ -109,3 +110,9 @@ permission.action.name.global_settings=Manage Global Settings # 高危语句 permission.action.name.high-risk_detect_rule=High-risk Detect Rule Setting permission.action.name.high-risk_detect_record=High-risk Detect Record + +resource.topo_node.name=node +resource.host.name=host +resource.dynamic_group.name=group +resource.biz_set.name=business set +resource.container.name=container diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/common/message_zh.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/common/message_zh.properties index 8d91002035..52fb7e1a17 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/common/message_zh.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/common/message_zh.properties @@ -1,7 +1,7 @@ # # Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. # -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +# Copyright (C) 2021 Tencent. All rights reserved. # # BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. # @@ -37,13 +37,14 @@ permission.resource.type.name.account=执行账号 permission.resource.type.name.public_script=公共脚本 permission.resource.type.name.tag=标签 permission.resource.type.name.host=主机 -permission.resource.type.name.sys_cloud_area=云区域 +permission.resource.type.name.sys_cloud_area=管控区域 permission.resource.type.name.set=集群 permission.resource.type.name.module=模块 permission.resource.type.name.biz_custom_query=动态分组 permission.resource.type.name.dashboard_view=运营视图 permission.resource.type.name.file_source=文件源 permission.resource.type.name.ticket=凭证 +permission.resource.type.name.container=容器 # 操作Action # 业务查看 permission.action.name.access_business=业务访问 @@ -109,3 +110,9 @@ permission.action.name.global_settings=全局设置管理 # 高危语句 permission.action.name.high-risk_detect_rule=高危语句规则管理 permission.action.name.high-risk_detect_record=高危语句拦截记录查看 + +resource.topo_node.name=节点 +resource.host.name=主机 +resource.dynamic_group.name=动态分组 +resource.biz_set.name=业务集 +resource.container.name=容器 diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/common/message_zh_CN.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/common/message_zh_CN.properties index 8d91002035..52fb7e1a17 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/common/message_zh_CN.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/common/message_zh_CN.properties @@ -1,7 +1,7 @@ # # Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. # -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +# Copyright (C) 2021 Tencent. All rights reserved. # # BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. # @@ -37,13 +37,14 @@ permission.resource.type.name.account=执行账号 permission.resource.type.name.public_script=公共脚本 permission.resource.type.name.tag=标签 permission.resource.type.name.host=主机 -permission.resource.type.name.sys_cloud_area=云区域 +permission.resource.type.name.sys_cloud_area=管控区域 permission.resource.type.name.set=集群 permission.resource.type.name.module=模块 permission.resource.type.name.biz_custom_query=动态分组 permission.resource.type.name.dashboard_view=运营视图 permission.resource.type.name.file_source=文件源 permission.resource.type.name.ticket=凭证 +permission.resource.type.name.container=容器 # 操作Action # 业务查看 permission.action.name.access_business=业务访问 @@ -109,3 +110,9 @@ permission.action.name.global_settings=全局设置管理 # 高危语句 permission.action.name.high-risk_detect_rule=高危语句规则管理 permission.action.name.high-risk_detect_record=高危语句拦截记录查看 + +resource.topo_node.name=节点 +resource.host.name=主机 +resource.dynamic_group.name=动态分组 +resource.biz_set.name=业务集 +resource.container.name=容器 diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message.properties index dee57d6784..6abd588f5b 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message.properties @@ -1,7 +1,7 @@ # # Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. # -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +# Copyright (C) 2021 Tencent. All rights reserved. # # BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. # @@ -39,52 +39,52 @@ 9900403=用户({0})权限不足，请前往权限中心确认并申请补充后重试 1238002=用户无业务操作权限 -##系统错误-API通用 -1239001=IP:{0}无访问权限 -1239002=用户无访问权限 -1239003=不支持的API接口 -1239004=缺少请求报文或报文不合法 -1239005=调用超时 -1239006=内部服务错误 -1239007=API接口已经过期废弃 -1239008=API服务过载，拒绝服务请求 -1239009=API回调其他接口失败 -1239010=Api.cert文件不可用 - ##系统错误-通用 1240001=服务不可用 1240002=内部服务器异常 1240003=错误的请求 -1240004=Cookie过期或者不存在 +1240004=Cookie 过期或者不存在 1240005=服务认证失败 +1240006=配置异常：{0} ##系统错误-公共组件错误 -1250001=Redis服务连接失败 -1250002=Redis服务内部错误 -1259001=NFS存储不可用 -1252001=DB不可用 -1255001=MQ不可用 +1250001=Redis 服务连接失败 +1250002=Redis 服务内部错误 +1259001=NFS 存储不可用 +1252001=DB 不可用 +1255001=MQ 不可用 #系统错误-平台服务错误 -1210101=License不可用 -1210001=GSE不可用 -1211001=CMDB服务不可用 +1210101=License 不可用 +1210001=GSE 不可用 +1210002=GSE 数据异常：{0} +1211001=CMDB 服务不可用 1211002=CMDB API返回数据异常 -1211003=根据动态分组ID查找主机失败，动态分组ID：{0}，原因：{1}，请确认指定的动态分组在业务下是否存在 -1211004=根据业务ID查找动态分组失败，业务ID：{0}，原因：{1}，请确认指定的业务是否存在动态分组 -1213001=CMSI接口访问异常 +1211003=根据动态分组 ID 查找主机失败，动态分组 ID ：{0}，原因：{1}，请确认指定的动态分组在业务下是否存在 +1211004=根据业务ID查找动态分组失败，业务 ID ：{0}，原因：{1}，请确认指定的业务是否存在动态分组 +1213001=CMSI 接口访问异常 1213002=用户管理接口访问异常 -1213003=调用CMSI接口获取通知渠道数据异常 -1213004=调用CMSI接口发送通知失败，错误码：{0}，错误信息：{1} +1213003=调用 CMSI 接口获取通知渠道数据异常 +1213004=调用 CMSI 接口发送通知失败，错误码：{0}，错误信息：{1} -1214001=ARTIFACTORY API返回数据异常 -1214002=制品库中找不到节点:{0}，请到制品库核实 +1214001=ARTIFACTORY API 返回数据异常 +1214002=制品库中找不到节点：{0}，请到制品库核实 #IAM接口返回数据结构异常 -1215001=IAM接口数据异常 +1215001=IAM 接口数据异常 +1215002=用户({0})账号已被冻结，请处理后再重试 #第三方API请求错误 -1216001=第三方API请求错误 +1216001=第三方 API 请求错误 + +#消息通知中心异常 +1217001=消息通知中心API不存在:{0} +1217002=消息通知中心接口数据异常 + +#AIDev异常 +1218001=AIDev平台接口数据异常 +1218002=蓝鲸OpenAI接口数据异常 +1218003=蓝鲸OpenAI接口数据超时 ##业务错误-通用 @@ -99,6 +99,10 @@ 1241010=请求参数[{0}]不合法，原因：{1} 1241011=尚未支持的功能 1241012=请求参数不合法，原因：{0} +1241013=该功能在业务集下暂不支持 +1241014=IPv6 地址不合法：{0} +1241015=不支持的操作，请勿重复启动作业,job_instance_id={0,number,#} +1241016=非法文件 ##业务错误-作业管理(job-manage) 1243001=脚本不存在 @@ -138,7 +142,7 @@ 1243036=脚本版本已存在 1243037=删除失败！存在关联的定时任务 1243038=删除失败！执行方案存在关联的定时任务 -1243039=脚本版本Id已存在 +1243039=脚本版本 ID 已存在 1243040=作业模板名称已存在 1243041=更新模版失败！并发冲突，请不要多次点击保存 1243045=同步脚本失败 @@ -146,10 +150,15 @@ 1243048=凭证不存在 1243049=业务/业务集不存在 1243050=文件后缀不允许 +1243051=资源范围不存在：{0} +1243052=IP（含管控区域ID）在 CMDB 中不存在：{0} +1243053=主机ID在 CMDB 中不存在：{0} +1243054=脚本被引用不支持删除 +1243055=凭证被引用不支持删除 ##业务错误-作业执行(job-execute) 1244001=主机为空 -1244002=主机[{0}]无效，请检查源或目标主机是否存在于配置平台；另外，主机跨业务需要联系作业平台管理员先将其添加到IP白名单。 +1244002=主机无效，请检查源或目标主机是否存在于配置平台的业务[{0}]下；另外，主机需跨业务执行请联系作业平台管理员将其添加到主机白名单。无效的{1}台主机：[{2}] 1244003=账号[{0}]不存在 1244004=没有账号[{0}]的权限 1244005=启动作业失败 @@ -168,15 +177,28 @@ 1244018=脚本状态[{0}]，不支持执行 1244019=高危脚本，禁止执行 1244020=作业执行历史查询时间范围必须小于30天 -1244021=文件任务数量超过限制:{0} -1244022=脚本任务目标服务器数量超过限制:{0} -1244023=本地文件{0}在后台不存在 +1244021=文件任务数量超过限制：{0} +1244022=脚本任务目标服务器数量超过限制：{0} +1244023=本地文件{0}在后台不存在，可能过期被定时清除(默认过期时间为{1}天)，可重新上传后重试。若要持久化存储，请上传至业务服务器后，从服务器分发。 1244024=保存文件到本地失败 1244025=任务被丢弃 1244026=非法的滚动策略 1244027=滚动批次不能大于{0} +1244028=步骤 [{0}] 的目标执行对象为空 +1244029=步骤 [{0}] 的源文件执行对象为空 +1244030=作业引用的执行对象不存在。不存在的执行对象个数:{0},执行对象列表[{1}] +1244031=Label Selector 不合法 +1244032=当前执行的作业总量超过业务配额限制 +1244033=当前执行的作业总量超过应用配额限制 +1244034=当前执行的作业总量超过系统配额限制 ## 业务错误-定时任务(job-crontab) +1245001=定时任务不存在 +1245002=更新定时任务错误 +1245003=插入定时任务错误 +1245004=定时任务已存在 +1245008=删除定时任务失败 +1245009=保存定时任务(id={0})自定义消息通知策略失败 1245006=定时任务执行时间已失效 1245007=定时任务结束时间或通知时间已失效 1245005=更新定时任务获取锁失败 @@ -187,7 +209,7 @@ 1246003=删除作业执行日志失败 ##业务错误-文件网关服务(job-file-gateway) -1260001=文件源不存在:{0} +1260001=文件源不存在：{0} 1260002=接入点响应异常：ListFileNode，详情：{0} 1260003=接入点响应异常：FileAvailable，详情：{0} 1260008=接入点响应异常：ExecuteAction，详情：{0} @@ -199,11 +221,14 @@ 1260010=不可删除文件未清空的Bucket 1260012=无法根据标识{0}找到文件源 1260013=文件源服务异常 -1260014=通过[{0}]找不到file-worker +1260014=通过[{0}]找不到 file-worker 1260015=文件源标识已存在：{0} 1260016=文件源ID与标识至少指定一个 1260017=文件源[id={0}]不在业务/业务集下 1260018=接入点响应异常，详情：{0} +1260019=文件源code不可为空 +1260020=制品库根地址不在允许范围内（默认仅支持当前环境蓝鲸制品库），如需对接其他环境请联系平台管理员添加白名单 +1260021=白名单记录已存在，请删除旧数据后再添加 ##业务错误-文件Worker服务(job-file-worker) 1262001=第三方文件源响应异常：ListBucket，详情：{0} @@ -213,8 +238,19 @@ 1262005=第三方文件源响应异常：DownloadGenericFile，详情：{0} 1262006=第三方文件源响应异常：GetObject，详情：{0} -##业务错误-用户服务、登录服务 +## 业务错误-升级迁移 +1263001=迁移失败，任务： {0}, 详情： {1} + +## 业务错误-统计分析服务(job-analysis) +1264001=AI分析任务报错信息仅支持脚本或文件任务步骤 +1264002=AI分析任务报错信息内容超过最大值：{0} +1264003=AI对话记录不存在：id={0} + +##业务错误-网关服务 1247001=用户不存在或者未登录 +1247002=缺少有效的用户信息 +1247403=无该应用访问权限 +1247404=缺少有效的AppCode diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en.properties index ae95ef5a65..1623048a94 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en.properties @@ -1,7 +1,7 @@ # # Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. # -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +# Copyright (C) 2021 Tencent. All rights reserved. # # BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. # @@ -35,28 +35,17 @@ 0=Success ## System error - permissions -1238001=User ({0}) permission not allowed, please confirm and apply in IAM and try again -9900403=User ({0}) permission not allowed, please confirm and apply in IAM and try again -1238002=Insufficient user permissions for application - -## System error - API common -1239001=IP: {0} No access -1239002= User does not have access -1239003=Unsupported API -1239004=Missing request param or param is illegal -1239005=Call timeout -1239006=Internal Service Error -1239007=API has expired -1239008=API service overload, denial of service request -1239009=API callback failed -1239010=Api.cert file is not available +1238001=User ({0}) permission not allowed, please confirm and apply permission in IAM and try again +9900403=User ({0}) permission not allowed, please confirm and apply permission in IAM and try again +1238002=User do not have business operation permission ## System error - common 1240001=Service Unavailable 1240002=Internal Server Error 1240003=Bad Request -1240004=Cookies expire or do not exist +1240004=Cookie does not exist or expired 1240005=Service auth fail +1240006=Invalid config:{0} ## System error - common components 1250001=Redis service connection failed @@ -68,6 +57,7 @@ ## System error - platforms 1210101=License is not available 1210001=GSE is not available +1210002=GSE data unexpected:{0} 1211001=CMDB service is unavailable 1211002=CMDB API returned data exception 1211003=Fail to find host by dynamic group, id:{0}, reason:{1}, please confirm the specified dynamic group in business @@ -82,10 +72,20 @@ #IAM接口返回数据结构异常 1215001=IAM API data unexpected +1215002=User({0}) account has been frozen, please fix it and try again #第三方API请求错误 1216001=Third API data unexpected +#消息通知中心异常 +1217001=BK-Notice API not found:{0} +1217002=BK-Notice API data unexpected + +#AIDev异常 +1218001=BK-AIDev API data unexpected +1218002=BK-OpenAI API data unexpected +1218003=BK-OpenAI API data timeout + ## Business error - common 1241001=Missing request parameters 1241002=Invalid request parameters @@ -98,6 +98,10 @@ 1241010=Invalid request parameter [{0}], reason: {1} 1241011=Not support yet 1241012=Invalid request parameter, reason: {0} +1241013=Not support for biz set yet +1241014=Invalid IPv6 address: {0} +1241015=Unsupported operation,do not repeat start the job,job_instance_id={0,number,#} +1241016=Illegal File ## Business error - job-manage 1243001=Script not exist @@ -146,10 +150,15 @@ 1243048=Credential not exist 1243049=Business/Business Set not exist 1243050=File suffix not allow +1243051=Scope not exist: {0} +1243052=IPs(Including BK-Net) not exist in CMDB: {0} +1243053=hostIds not exist in CMDB: {0} +1243054=Script is referenced and cannot be deleted +1243055=Credential is referenced and cannot be deleted ## Business error - job-execute 1244001=Host is empty -1244002=Hosts invalid:{0}, please check them in CMDB, which can come from file source servers or target servers. Besides, please contact JOB administrator to add them to ip whitelist if you have to use hosts between businesses. +1244002=Hosts invalid. Please check whether the source or target host exists under the business [{0}] of CMDB. Besides, please contact JOB administrator to add them to ip whitelist if you have to use hosts between businesses. Invalid {1} hosts:{2} 1244003=Account [{0}] does not exist 1244004=No permission to use this account[{0}] 1244005=Fail to start job @@ -170,13 +179,26 @@ 1244020=The job execution history query time range must be less than 30 days 1244021=The number of file tasks exceeds the limit:{0} 1244022=The number of script task target servers exceeds the limit: {0} -1244023=The local file {0} not exists in backend +1244023=The local file {0} not exists in backend, possibly because it was automatically deleted after expiration (default expiration time is {1} days), please upload it and try it again. To store it permanently, please upload it to the business server and distribute it from there. 1244024=Fail to save file to local 1244025=Task abandoned 1244026=Invalid rolling strategy 1244027=Rolling batch can not be greater than {0} +1244028=Step [{0}] target execute object is empty +1244029=Step [{0}] source execute object is empty +1244030=Execute object referenced by the job does not exist. Number of non-existent execution objects: {0}, execution object list: [{1}] +1244031=Invalid Label Selector +1244032=The total number of jobs currently executed exceeds the resource scope quota limit +1244033=The total number of jobs currently executed exceeds the bk app quota limit +1244034=The total number of jobs currently executed exceeds the system quota limit ## Business error - job-crontab +1245001=Cron job does not exist +1245002=Failed to update cron +1245003=Failed to insert cron +1245004=Cron job already existed +1245008=Failed to delete cron +1245009=Failed to save cron's custom notify policy with id {0} 1245006=Cron job execution time already passed 1245007=Cron job end time or notify time already passed 1245005=Acquire lock failed while update cron job @@ -204,6 +226,9 @@ 1260016=At least one of id/code required 1260017=FileSource[id={0}] does not belong to specified business 1260018=FileWorker response error, Detail:{0} +1260019=File source code can not be empty +1260020=Base url of bk-repo is invalid(Only bk-repo in current environment is supported), please contact administrator to add white records if you want to use bk-repo in other environments +1260021=White info already exists, please add record after deleting old record ## Business error (job-file-worker) 1262001=Third file source response error:ListBucket, Detail:{0} @@ -213,8 +238,19 @@ 1262005=Third file source response error:DownloadGenericFile, Detail:{0} 1262006=Third file source response error:GetObject, Detail:{0} -## Business error - User/Login +## Business error (migration) +1263001=Migration fail，task: {0}, detail: {1} + +## Business error (job-analysis) +1264001=Analyze task error only support Script or File step +1264002=AI analyze task error content exceed max length: {0} +1264003=AI chat history does not exist(id={0}) + +## Business error - (job-gateway) 1247001=User does not exist or is not logged in +1247002=Missing user info +1247403=No access permission for this application +1247404=Missing AppCode ## Business error - Backup 1249001=Fail to get node info from artifactory diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en_US.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en_US.properties index ae95ef5a65..1623048a94 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en_US.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_en_US.properties @@ -1,7 +1,7 @@ # # Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. # -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +# Copyright (C) 2021 Tencent. All rights reserved. # # BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. # @@ -35,28 +35,17 @@ 0=Success ## System error - permissions -1238001=User ({0}) permission not allowed, please confirm and apply in IAM and try again -9900403=User ({0}) permission not allowed, please confirm and apply in IAM and try again -1238002=Insufficient user permissions for application - -## System error - API common -1239001=IP: {0} No access -1239002= User does not have access -1239003=Unsupported API -1239004=Missing request param or param is illegal -1239005=Call timeout -1239006=Internal Service Error -1239007=API has expired -1239008=API service overload, denial of service request -1239009=API callback failed -1239010=Api.cert file is not available +1238001=User ({0}) permission not allowed, please confirm and apply permission in IAM and try again +9900403=User ({0}) permission not allowed, please confirm and apply permission in IAM and try again +1238002=User do not have business operation permission ## System error - common 1240001=Service Unavailable 1240002=Internal Server Error 1240003=Bad Request -1240004=Cookies expire or do not exist +1240004=Cookie does not exist or expired 1240005=Service auth fail +1240006=Invalid config:{0} ## System error - common components 1250001=Redis service connection failed @@ -68,6 +57,7 @@ ## System error - platforms 1210101=License is not available 1210001=GSE is not available +1210002=GSE data unexpected:{0} 1211001=CMDB service is unavailable 1211002=CMDB API returned data exception 1211003=Fail to find host by dynamic group, id:{0}, reason:{1}, please confirm the specified dynamic group in business @@ -82,10 +72,20 @@ #IAM接口返回数据结构异常 1215001=IAM API data unexpected +1215002=User({0}) account has been frozen, please fix it and try again #第三方API请求错误 1216001=Third API data unexpected +#消息通知中心异常 +1217001=BK-Notice API not found:{0} +1217002=BK-Notice API data unexpected + +#AIDev异常 +1218001=BK-AIDev API data unexpected +1218002=BK-OpenAI API data unexpected +1218003=BK-OpenAI API data timeout + ## Business error - common 1241001=Missing request parameters 1241002=Invalid request parameters @@ -98,6 +98,10 @@ 1241010=Invalid request parameter [{0}], reason: {1} 1241011=Not support yet 1241012=Invalid request parameter, reason: {0} +1241013=Not support for biz set yet +1241014=Invalid IPv6 address: {0} +1241015=Unsupported operation,do not repeat start the job,job_instance_id={0,number,#} +1241016=Illegal File ## Business error - job-manage 1243001=Script not exist @@ -146,10 +150,15 @@ 1243048=Credential not exist 1243049=Business/Business Set not exist 1243050=File suffix not allow +1243051=Scope not exist: {0} +1243052=IPs(Including BK-Net) not exist in CMDB: {0} +1243053=hostIds not exist in CMDB: {0} +1243054=Script is referenced and cannot be deleted +1243055=Credential is referenced and cannot be deleted ## Business error - job-execute 1244001=Host is empty -1244002=Hosts invalid:{0}, please check them in CMDB, which can come from file source servers or target servers. Besides, please contact JOB administrator to add them to ip whitelist if you have to use hosts between businesses. +1244002=Hosts invalid. Please check whether the source or target host exists under the business [{0}] of CMDB. Besides, please contact JOB administrator to add them to ip whitelist if you have to use hosts between businesses. Invalid {1} hosts:{2} 1244003=Account [{0}] does not exist 1244004=No permission to use this account[{0}] 1244005=Fail to start job @@ -170,13 +179,26 @@ 1244020=The job execution history query time range must be less than 30 days 1244021=The number of file tasks exceeds the limit:{0} 1244022=The number of script task target servers exceeds the limit: {0} -1244023=The local file {0} not exists in backend +1244023=The local file {0} not exists in backend, possibly because it was automatically deleted after expiration (default expiration time is {1} days), please upload it and try it again. To store it permanently, please upload it to the business server and distribute it from there. 1244024=Fail to save file to local 1244025=Task abandoned 1244026=Invalid rolling strategy 1244027=Rolling batch can not be greater than {0} +1244028=Step [{0}] target execute object is empty +1244029=Step [{0}] source execute object is empty +1244030=Execute object referenced by the job does not exist. Number of non-existent execution objects: {0}, execution object list: [{1}] +1244031=Invalid Label Selector +1244032=The total number of jobs currently executed exceeds the resource scope quota limit +1244033=The total number of jobs currently executed exceeds the bk app quota limit +1244034=The total number of jobs currently executed exceeds the system quota limit ## Business error - job-crontab +1245001=Cron job does not exist +1245002=Failed to update cron +1245003=Failed to insert cron +1245004=Cron job already existed +1245008=Failed to delete cron +1245009=Failed to save cron's custom notify policy with id {0} 1245006=Cron job execution time already passed 1245007=Cron job end time or notify time already passed 1245005=Acquire lock failed while update cron job @@ -204,6 +226,9 @@ 1260016=At least one of id/code required 1260017=FileSource[id={0}] does not belong to specified business 1260018=FileWorker response error, Detail:{0} +1260019=File source code can not be empty +1260020=Base url of bk-repo is invalid(Only bk-repo in current environment is supported), please contact administrator to add white records if you want to use bk-repo in other environments +1260021=White info already exists, please add record after deleting old record ## Business error (job-file-worker) 1262001=Third file source response error:ListBucket, Detail:{0} @@ -213,8 +238,19 @@ 1262005=Third file source response error:DownloadGenericFile, Detail:{0} 1262006=Third file source response error:GetObject, Detail:{0} -## Business error - User/Login +## Business error (migration) +1263001=Migration fail，task: {0}, detail: {1} + +## Business error (job-analysis) +1264001=Analyze task error only support Script or File step +1264002=AI analyze task error content exceed max length: {0} +1264003=AI chat history does not exist(id={0}) + +## Business error - (job-gateway) 1247001=User does not exist or is not logged in +1247002=Missing user info +1247403=No access permission for this application +1247404=Missing AppCode ## Business error - Backup 1249001=Fail to get node info from artifactory diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh.properties index dee57d6784..6abd588f5b 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh.properties @@ -1,7 +1,7 @@ # # Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. # -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +# Copyright (C) 2021 Tencent. All rights reserved. # # BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. # @@ -39,52 +39,52 @@ 9900403=用户({0})权限不足，请前往权限中心确认并申请补充后重试 1238002=用户无业务操作权限 -##系统错误-API通用 -1239001=IP:{0}无访问权限 -1239002=用户无访问权限 -1239003=不支持的API接口 -1239004=缺少请求报文或报文不合法 -1239005=调用超时 -1239006=内部服务错误 -1239007=API接口已经过期废弃 -1239008=API服务过载，拒绝服务请求 -1239009=API回调其他接口失败 -1239010=Api.cert文件不可用 - ##系统错误-通用 1240001=服务不可用 1240002=内部服务器异常 1240003=错误的请求 -1240004=Cookie过期或者不存在 +1240004=Cookie 过期或者不存在 1240005=服务认证失败 +1240006=配置异常：{0} ##系统错误-公共组件错误 -1250001=Redis服务连接失败 -1250002=Redis服务内部错误 -1259001=NFS存储不可用 -1252001=DB不可用 -1255001=MQ不可用 +1250001=Redis 服务连接失败 +1250002=Redis 服务内部错误 +1259001=NFS 存储不可用 +1252001=DB 不可用 +1255001=MQ 不可用 #系统错误-平台服务错误 -1210101=License不可用 -1210001=GSE不可用 -1211001=CMDB服务不可用 +1210101=License 不可用 +1210001=GSE 不可用 +1210002=GSE 数据异常：{0} +1211001=CMDB 服务不可用 1211002=CMDB API返回数据异常 -1211003=根据动态分组ID查找主机失败，动态分组ID：{0}，原因：{1}，请确认指定的动态分组在业务下是否存在 -1211004=根据业务ID查找动态分组失败，业务ID：{0}，原因：{1}，请确认指定的业务是否存在动态分组 -1213001=CMSI接口访问异常 +1211003=根据动态分组 ID 查找主机失败，动态分组 ID ：{0}，原因：{1}，请确认指定的动态分组在业务下是否存在 +1211004=根据业务ID查找动态分组失败，业务 ID ：{0}，原因：{1}，请确认指定的业务是否存在动态分组 +1213001=CMSI 接口访问异常 1213002=用户管理接口访问异常 -1213003=调用CMSI接口获取通知渠道数据异常 -1213004=调用CMSI接口发送通知失败，错误码：{0}，错误信息：{1} +1213003=调用 CMSI 接口获取通知渠道数据异常 +1213004=调用 CMSI 接口发送通知失败，错误码：{0}，错误信息：{1} -1214001=ARTIFACTORY API返回数据异常 -1214002=制品库中找不到节点:{0}，请到制品库核实 +1214001=ARTIFACTORY API 返回数据异常 +1214002=制品库中找不到节点：{0}，请到制品库核实 #IAM接口返回数据结构异常 -1215001=IAM接口数据异常 +1215001=IAM 接口数据异常 +1215002=用户({0})账号已被冻结，请处理后再重试 #第三方API请求错误 -1216001=第三方API请求错误 +1216001=第三方 API 请求错误 + +#消息通知中心异常 +1217001=消息通知中心API不存在:{0} +1217002=消息通知中心接口数据异常 + +#AIDev异常 +1218001=AIDev平台接口数据异常 +1218002=蓝鲸OpenAI接口数据异常 +1218003=蓝鲸OpenAI接口数据超时 ##业务错误-通用 @@ -99,6 +99,10 @@ 1241010=请求参数[{0}]不合法，原因：{1} 1241011=尚未支持的功能 1241012=请求参数不合法，原因：{0} +1241013=该功能在业务集下暂不支持 +1241014=IPv6 地址不合法：{0} +1241015=不支持的操作，请勿重复启动作业,job_instance_id={0,number,#} +1241016=非法文件 ##业务错误-作业管理(job-manage) 1243001=脚本不存在 @@ -138,7 +142,7 @@ 1243036=脚本版本已存在 1243037=删除失败！存在关联的定时任务 1243038=删除失败！执行方案存在关联的定时任务 -1243039=脚本版本Id已存在 +1243039=脚本版本 ID 已存在 1243040=作业模板名称已存在 1243041=更新模版失败！并发冲突，请不要多次点击保存 1243045=同步脚本失败 @@ -146,10 +150,15 @@ 1243048=凭证不存在 1243049=业务/业务集不存在 1243050=文件后缀不允许 +1243051=资源范围不存在：{0} +1243052=IP（含管控区域ID）在 CMDB 中不存在：{0} +1243053=主机ID在 CMDB 中不存在：{0} +1243054=脚本被引用不支持删除 +1243055=凭证被引用不支持删除 ##业务错误-作业执行(job-execute) 1244001=主机为空 -1244002=主机[{0}]无效，请检查源或目标主机是否存在于配置平台；另外，主机跨业务需要联系作业平台管理员先将其添加到IP白名单。 +1244002=主机无效，请检查源或目标主机是否存在于配置平台的业务[{0}]下；另外，主机需跨业务执行请联系作业平台管理员将其添加到主机白名单。无效的{1}台主机：[{2}] 1244003=账号[{0}]不存在 1244004=没有账号[{0}]的权限 1244005=启动作业失败 @@ -168,15 +177,28 @@ 1244018=脚本状态[{0}]，不支持执行 1244019=高危脚本，禁止执行 1244020=作业执行历史查询时间范围必须小于30天 -1244021=文件任务数量超过限制:{0} -1244022=脚本任务目标服务器数量超过限制:{0} -1244023=本地文件{0}在后台不存在 +1244021=文件任务数量超过限制：{0} +1244022=脚本任务目标服务器数量超过限制：{0} +1244023=本地文件{0}在后台不存在，可能过期被定时清除(默认过期时间为{1}天)，可重新上传后重试。若要持久化存储，请上传至业务服务器后，从服务器分发。 1244024=保存文件到本地失败 1244025=任务被丢弃 1244026=非法的滚动策略 1244027=滚动批次不能大于{0} +1244028=步骤 [{0}] 的目标执行对象为空 +1244029=步骤 [{0}] 的源文件执行对象为空 +1244030=作业引用的执行对象不存在。不存在的执行对象个数:{0},执行对象列表[{1}] +1244031=Label Selector 不合法 +1244032=当前执行的作业总量超过业务配额限制 +1244033=当前执行的作业总量超过应用配额限制 +1244034=当前执行的作业总量超过系统配额限制 ## 业务错误-定时任务(job-crontab) +1245001=定时任务不存在 +1245002=更新定时任务错误 +1245003=插入定时任务错误 +1245004=定时任务已存在 +1245008=删除定时任务失败 +1245009=保存定时任务(id={0})自定义消息通知策略失败 1245006=定时任务执行时间已失效 1245007=定时任务结束时间或通知时间已失效 1245005=更新定时任务获取锁失败 @@ -187,7 +209,7 @@ 1246003=删除作业执行日志失败 ##业务错误-文件网关服务(job-file-gateway) -1260001=文件源不存在:{0} +1260001=文件源不存在：{0} 1260002=接入点响应异常：ListFileNode，详情：{0} 1260003=接入点响应异常：FileAvailable，详情：{0} 1260008=接入点响应异常：ExecuteAction，详情：{0} @@ -199,11 +221,14 @@ 1260010=不可删除文件未清空的Bucket 1260012=无法根据标识{0}找到文件源 1260013=文件源服务异常 -1260014=通过[{0}]找不到file-worker +1260014=通过[{0}]找不到 file-worker 1260015=文件源标识已存在：{0} 1260016=文件源ID与标识至少指定一个 1260017=文件源[id={0}]不在业务/业务集下 1260018=接入点响应异常，详情：{0} +1260019=文件源code不可为空 +1260020=制品库根地址不在允许范围内（默认仅支持当前环境蓝鲸制品库），如需对接其他环境请联系平台管理员添加白名单 +1260021=白名单记录已存在，请删除旧数据后再添加 ##业务错误-文件Worker服务(job-file-worker) 1262001=第三方文件源响应异常：ListBucket，详情：{0} @@ -213,8 +238,19 @@ 1262005=第三方文件源响应异常：DownloadGenericFile，详情：{0} 1262006=第三方文件源响应异常：GetObject，详情：{0} -##业务错误-用户服务、登录服务 +## 业务错误-升级迁移 +1263001=迁移失败，任务： {0}, 详情： {1} + +## 业务错误-统计分析服务(job-analysis) +1264001=AI分析任务报错信息仅支持脚本或文件任务步骤 +1264002=AI分析任务报错信息内容超过最大值：{0} +1264003=AI对话记录不存在：id={0} + +##业务错误-网关服务 1247001=用户不存在或者未登录 +1247002=缺少有效的用户信息 +1247403=无该应用访问权限 +1247404=缺少有效的AppCode diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh_CN.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh_CN.properties index dee57d6784..11acefae1b 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh_CN.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/exception/message_zh_CN.properties @@ -1,7 +1,7 @@ # # Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. # -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +# Copyright (C) 2021 Tencent. All rights reserved. # # BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. # @@ -39,52 +39,52 @@ 9900403=用户({0})权限不足，请前往权限中心确认并申请补充后重试 1238002=用户无业务操作权限 -##系统错误-API通用 -1239001=IP:{0}无访问权限 -1239002=用户无访问权限 -1239003=不支持的API接口 -1239004=缺少请求报文或报文不合法 -1239005=调用超时 -1239006=内部服务错误 -1239007=API接口已经过期废弃 -1239008=API服务过载，拒绝服务请求 -1239009=API回调其他接口失败 -1239010=Api.cert文件不可用 - ##系统错误-通用 1240001=服务不可用 1240002=内部服务器异常 1240003=错误的请求 -1240004=Cookie过期或者不存在 +1240004=Cookie 过期或者不存在 1240005=服务认证失败 +1240006=配置异常：{0} ##系统错误-公共组件错误 -1250001=Redis服务连接失败 -1250002=Redis服务内部错误 -1259001=NFS存储不可用 -1252001=DB不可用 -1255001=MQ不可用 +1250001=Redis 服务连接失败 +1250002=Redis 服务内部错误 +1259001=NFS 存储不可用 +1252001=DB 不可用 +1255001=MQ 不可用 #系统错误-平台服务错误 -1210101=License不可用 -1210001=GSE不可用 -1211001=CMDB服务不可用 +1210101=License 不可用 +1210001=GSE 不可用 +1210002=GSE 数据异常：{0} +1211001=CMDB 服务不可用 1211002=CMDB API返回数据异常 -1211003=根据动态分组ID查找主机失败，动态分组ID：{0}，原因：{1}，请确认指定的动态分组在业务下是否存在 -1211004=根据业务ID查找动态分组失败，业务ID：{0}，原因：{1}，请确认指定的业务是否存在动态分组 -1213001=CMSI接口访问异常 +1211003=根据动态分组 ID 查找主机失败，动态分组 ID ：{0}，原因：{1}，请确认指定的动态分组在业务下是否存在 +1211004=根据业务ID查找动态分组失败，业务 ID ：{0}，原因：{1}，请确认指定的业务是否存在动态分组 +1213001=CMSI 接口访问异常 1213002=用户管理接口访问异常 -1213003=调用CMSI接口获取通知渠道数据异常 -1213004=调用CMSI接口发送通知失败，错误码：{0}，错误信息：{1} +1213003=调用 CMSI 接口获取通知渠道数据异常 +1213004=调用 CMSI 接口发送通知失败，错误码：{0}，错误信息：{1} -1214001=ARTIFACTORY API返回数据异常 -1214002=制品库中找不到节点:{0}，请到制品库核实 +1214001=ARTIFACTORY API 返回数据异常 +1214002=制品库中找不到节点：{0}，请到制品库核实 #IAM接口返回数据结构异常 -1215001=IAM接口数据异常 +1215001=IAM 接口数据异常 +1215002=用户({0})账号已被冻结，请处理后再重试 #第三方API请求错误 -1216001=第三方API请求错误 +1216001=第三方 API 请求错误 + +#消息通知中心异常 +1217001=消息通知中心API不存在:{0} +1217002=消息通知中心接口数据异常 + +#AIDev异常 +1218001=AIDev平台接口数据异常 +1218002=蓝鲸OpenAI接口数据异常 +1218003=蓝鲸OpenAI接口数据超时 ##业务错误-通用 @@ -99,6 +99,10 @@ 1241010=请求参数[{0}]不合法，原因：{1} 1241011=尚未支持的功能 1241012=请求参数不合法，原因：{0} +1241013=该功能在业务集下暂不支持 +1241014=IPv6 地址不合法：{0} +1241015=不支持的操作，请勿重复启动作业，job_instance_id={0,number,#} +1241016=非法文件 ##业务错误-作业管理(job-manage) 1243001=脚本不存在 @@ -138,7 +142,7 @@ 1243036=脚本版本已存在 1243037=删除失败！存在关联的定时任务 1243038=删除失败！执行方案存在关联的定时任务 -1243039=脚本版本Id已存在 +1243039=脚本版本 ID 已存在 1243040=作业模板名称已存在 1243041=更新模版失败！并发冲突，请不要多次点击保存 1243045=同步脚本失败 @@ -146,10 +150,16 @@ 1243048=凭证不存在 1243049=业务/业务集不存在 1243050=文件后缀不允许 +1243051=资源范围不存在：{0} +1243052=IP（含管控区域ID）在 CMDB 中不存在：{0} +1243053=主机ID在 CMDB 中不存在：{0} +1243054=脚本被引用不支持删除 +1243055=凭证被引用不支持删除 + ##业务错误-作业执行(job-execute) 1244001=主机为空 -1244002=主机[{0}]无效，请检查源或目标主机是否存在于配置平台；另外，主机跨业务需要联系作业平台管理员先将其添加到IP白名单。 +1244002=主机无效，请检查源或目标主机是否存在于配置平台的业务[{0}]下；另外，主机需跨业务执行请联系作业平台管理员将其添加到主机白名单。无效的{1}台主机：[{2}] 1244003=账号[{0}]不存在 1244004=没有账号[{0}]的权限 1244005=启动作业失败 @@ -168,15 +178,28 @@ 1244018=脚本状态[{0}]，不支持执行 1244019=高危脚本，禁止执行 1244020=作业执行历史查询时间范围必须小于30天 -1244021=文件任务数量超过限制:{0} -1244022=脚本任务目标服务器数量超过限制:{0} -1244023=本地文件{0}在后台不存在 +1244021=文件任务数量超过限制：{0} +1244022=脚本任务目标服务器数量超过限制：{0} +1244023=本地文件{0}在后台不存在，可能过期被定时清除(默认过期时间为{1}天)，可重新上传后重试。若要持久化存储，请上传至业务服务器后，从服务器分发。 1244024=保存文件到本地失败 1244025=任务被丢弃 1244026=非法的滚动策略 1244027=滚动批次不能大于{0} +1244028=步骤 [{0}] 的目标执行对象为空 +1244029=步骤 [{0}] 的源文件执行对象为空 +1244030=作业引用的执行对象不存在。不存在的执行对象个数:{0},执行对象列表[{1}] +1244031=Label Selector 不合法 +1244032=当前执行的作业总量超过业务配额限制 +1244033=当前执行的作业总量超过应用配额限制 +1244034=当前执行的作业总量超过系统配额限制 ## 业务错误-定时任务(job-crontab) +1245001=定时任务不存在 +1245002=更新定时任务错误 +1245003=插入定时任务错误 +1245004=定时任务已存在 +1245008=删除定时任务失败 +1245009=保存定时任务(id={0})自定义消息通知策略失败 1245006=定时任务执行时间已失效 1245007=定时任务结束时间或通知时间已失效 1245005=更新定时任务获取锁失败 @@ -187,7 +210,7 @@ 1246003=删除作业执行日志失败 ##业务错误-文件网关服务(job-file-gateway) -1260001=文件源不存在:{0} +1260001=文件源不存在：{0} 1260002=接入点响应异常：ListFileNode，详情：{0} 1260003=接入点响应异常：FileAvailable，详情：{0} 1260008=接入点响应异常：ExecuteAction，详情：{0} @@ -199,11 +222,14 @@ 1260010=不可删除文件未清空的Bucket 1260012=无法根据标识{0}找到文件源 1260013=文件源服务异常 -1260014=通过[{0}]找不到file-worker +1260014=通过[{0}]找不到 file-worker 1260015=文件源标识已存在：{0} 1260016=文件源ID与标识至少指定一个 1260017=文件源[id={0}]不在业务/业务集下 1260018=接入点响应异常，详情：{0} +1260019=文件源code不可为空 +1260020=制品库根地址不在允许范围内（默认仅支持当前环境蓝鲸制品库），如需对接其他环境请联系平台管理员添加白名单 +1260021=白名单记录已存在，请删除旧数据后再添加 ##业务错误-文件Worker服务(job-file-worker) 1262001=第三方文件源响应异常：ListBucket，详情：{0} @@ -213,8 +239,19 @@ 1262005=第三方文件源响应异常：DownloadGenericFile，详情：{0} 1262006=第三方文件源响应异常：GetObject，详情：{0} -##业务错误-用户服务、登录服务 +## 业务错误-升级迁移 +1263001=迁移失败，任务： {0}, 详情： {1} + +## 统计分析服务job-analysis错误码 +1264001=AI分析任务报错信息仅支持脚本或文件任务步骤 +1264002=AI分析任务报错信息内容超过最大值：{0} +1264003=AI对话记录不存在：id={0} + +##业务错误-网关服务 1247001=用户不存在或者未登录 +1247002=缺少有效的用户信息 +1247403=无该应用访问权限 +1247404=缺少有效的AppCode diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages.properties index 02aaf91a48..1e9f54927a 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages.properties @@ -1,7 +1,7 @@ # # Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. # -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +# Copyright (C) 2021 Tencent. All rights reserved. # # BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. # @@ -22,31 +22,119 @@ # IN THE SOFTWARE. # -validation.constraints.InvalidBkCloudId.message=云区域ID非法 -validation.constraints.InvalidIp.message=IP非法 +## 通用 +validation.constraints.NotContainSpecialChar.message=不可包含 \\|/:*<>"? 特殊字符 +validation.constraints.NotBlankField.message=不能为空 + +## 文件相关 +validation.constraints.FileNameInvalid.message=文件名非法，不可包含路径分隔符 + +## 主机 +validation.constraints.InvalidBkCloudId.message=管控区域 ID 非法 +validation.constraints.InvalidIp.message=IP 非法 +validation.constraints.EmptyDynamicGroupId.message=动态分组 ID 不可为空 +validation.constraints.BkHostId_null.message=主机 ID 不能为空 +validation.constraints.BkHostId_invalid.message=非法的主机 ID +validation.constraints.TopoNodeId_null.message=拓扑节点 ID 不可为空 +validation.constraints.TopoNodeType_null.message=拓扑节点类型不可为空 + +## 作业模版、执行方案、作业实例 validation.constraints.InvalidJobTimeout_empty.message=作业超时时间不能为空 validation.constraints.InvalidJobTimeout_outOfRange.message=作业超时时间必须在{min}-{max}之间 +validation.constraints.TaskName_outOfLength.message=作业名称不能超过 {max} 个字符 +validation.constraints.ExecuteTarget_empty.message=作业执行目标不能为空 +validation.constraints.InvalidFileUploadSettingAmount.message=文件大小只能为大于0的浮点数 +validation.constraints.InvalidPort.message=端口号只能为1-65535之间的整数 +validation.constraints.InvalidJobInstanceId.message=任务实例ID必须为正整数 +validation.constraints.InvalidStepInstanceId.message=步骤实例ID必须为正整数 +validation.constraints.InvalidGlobalVarId.message=非法的全局变量 ID +validation.constraints.InvalidUploadFileSuffix.message=非法的上传文件后缀 +validation.constraints.InvalidUploadFileRestrictMode.message=非法的上传文件限制模式 +validation.constraints.ExecuteTarget_KubeContainerFilterEmpty.message=作业执行目标-容器过滤器不能为空 + +## 高危语句 +validation.constraints.InvalidJobHighRiskGrammarId_empty.message=高危语句规则ID不能为空 validation.constraints.InvalidJobHighRiskGrammarRegex_empty.message=语法检测表达式不能为空 validation.constraints.InvalidJobHighRiskGrammarRegex_outOfLength.message=语法检测表达式不能超过{max}个字符 validation.constraints.InvalidJobHighRiskGrammarRegex_wrongExpr.message=语法检测表达式错误 -validation.constraints.ScriptTypeList_empty.message=脚本类型不能为空 validation.constraints.InvalidHighRiskGrammarHandleAction.message=非法的处理动作 validation.constraints.InvalidHighRiskRegularStatus.message=非法的规则启停状态 validation.constraints.InvalidHighRiskRegularDescription_outOfLength.message=规则说明不能超过{max}个字符 -validation.constraints.InvalidBkBizId.message=非法的业务ID -validation.constraints.InvalidCronId.message=非法的定时任务ID -validation.constraints.InvalidCronJobPlanId.message=非法的作业ID + +## 脚本 +validation.constraints.ScriptTypeList_empty.message=脚本类型不能为空 +validation.constraints.ScriptName_outOfLength.message=脚本名称不能超过{max}个字符 +validation.constraints.ScriptVersion_outOfLength.message=脚本版本不能超过{max}个字符 +validation.constraints.ScriptVersion_illegal.message=脚本版本非法 +validation.constraints.ScriptVersion_empty.message=脚本版本不能为空 +validation.constraints.ScriptVersionId_empty.message=脚本版本ID不能为空 +validation.constraints.ScriptType_empty.message=脚本类型不能为空 +validation.constraints.ScriptType_illegal.message=脚本类型非法 +validation.constraints.ScriptId_empty.message=脚本ID不能为空 +validation.constraints.ScriptName_illegal.message=脚本名称非法 +validation.constraints.ScriptName_empty.message=脚本名称不能为空 +validation.constraints.ScriptContent_empty.message=脚本内容不能为空 +validation.constraints.InvalidScriptId_empty.message=脚本 ID 不能为空 + +## 业务 +validation.constraints.InvalidBkBizId.message=非法的业务 ID +validation.constraints.InvalidBkScopeType.message=非法的资源范围类型 +validation.constraints.InvalidBkScopeId.message=非法的资源范围 ID + +## 定时任务 +validation.constraints.InvalidCronId.message=非法的定时任务 ID +validation.constraints.CronId_null.message=定时任务ID不能为空 +validation.constraints.InvalidCronJobPlanId.message=非法的作业 ID validation.constraints.InvalidCronJobName_empty.message=定时作业名称不能为空 validation.constraints.InvalidCronJobName_outOfLength.message=定时作业名称不能超过{max}个字符 -validation.constraints.InvalidCronExpression.message=非法的Cron表达式 +validation.constraints.InvalidCronExpression.message=非法的 Cron 表达式 validation.constraints.InvalidCronExecuteTime.message=非法的执行时间 -validation.constraints.InvalidGlobalVarId.message=非法的全局变量ID -validation.constraints.InvalidBkScopeType.message=非法的资源范围类型 -validation.constraints.InvalidBkScopeId.message=非法的资源范围ID -validation.constraints.InvalidUploadFileSuffix.message=非法的上传文件后缀 -validation.constraints.InvalidUploadFileRestrictMode.message=非法的上传文件限制模式 -validation.constraints.InvalidScriptId_empty.message=脚本ID不能为空 -validation.constraints.EmptyDynamicGroupId.message=动态分组ID不可为空 -validation.constraints.InvalidFileUploadSettingAmount.message=文件大小只能为大于0的浮点数 -validation.constraints.InvalidPort.message=端口号只能为0-65535之间的整数 -validation.constraints.NotContainSpecialChar.message=不可包含特殊字符（\\|/:*<>"?） + +## 账号 +validation.constraints.AccountId_empty.message=账号ID不能为空 +validation.constraints.AccountName_empty.message=账号名称不能为空 +validation.constraints.AccountType_illegal.message=账号类型取值非法 +validation.constraints.AccountCategory_illegal.message=账号用途取值非法 +validation.constraints.AccountPassword_tooLong.message=账号密码不能超过255字符 +validation.constraints.AccountAlias_tooLong.message=账号别名不能超过255字符 +validation.constraints.AccountDescription_tooLong.message=账号描述不能超过1024字符 +validation.constraints.AccountAlias_empty.message=账号别名不能为空 + + +## 蓝鲸消息通知 +validation.constraints.InvalidAnnouncementsOffset.message=公告偏移量必须大于等于0 +validation.constraints.InvalidAnnouncementsLimit.message=公告获取数量必须大于等于1 +validation.constraints.InvalidWhiteIpScope.message=目标业务不可为空 + +## 其它 +validation.constraints.queryAgentInfoHostIds_tooMany.message=单次查询主机数量不可超过5000 +validation.constraints.queryExecuteObjects_outOfRange.message=单次查询执行对象数量范围必须在 {min}-{max} 之间 + +## 快速执行脚本 +validation.constraints.WindowsInterpreterExceedMaxLength.message=Windows解释器路径长度不可超过260字符 + +## AI对话 +validation.constraints.AIGeneralChat_contentEmpty.message=对话输入内容不能为空 +validation.constraints.AIAnalyzeError_contentEmpty.message=报错信息内容不能为空 +validation.constraints.AIInvalidHistoryStart.message=对话历史记录起始位置start必须大于等于0 +validation.constraints.AIInvalidHistoryLength.message=对话历史记录数量length必须在 {min}-{max} 之间 +validation.constraints.AIInvalidRecordId.message=对话记录ID只能为正整数 +validation.constraints.AIAnalyzeError_taskInstanceIdEmpty.message=任务实例ID不能为空 +validation.constraints.AIAnalyzeError_stepExecuteTypeEmpty.message=步骤执行类型不能为空 +validation.constraints.AIAnalyzeError_stepInstanceIdEmpty.message=步骤实例ID不能为空 +validation.constraints.AIAnalyzeError_executeCountEmpty.message=执行次数不能为空 +validation.constraints.AIAnalyzeError_batchEmpty.message=滚动批次不能为空 +validation.constraints.AIAnalyzeError_executeObjectTypeEmpty.message=执行对象类型不能为空 +validation.constraints.AIAnalyzeError_executeObjectResourceIdEmpty.message=执行对象资源ID不能为空 +validation.constraints.AICheckScript_contentExceedMaxLength.message=检查的脚本内容长度不可超过5MB +validation.constraints.AIGeneralChat_contentExceedMaxLength.message=对话输入内容长度不可超过5MB +validation.constraints.ExceedMaxLength.message=字段超出最大长度 +validation.constraints.MustEndWithSuffix.message=必须以指定后缀结尾 +validation.constraints.WinInterpreterInvalidSuffix.message=解释器路径不合法，需以 .exe 结尾 + + +## MySQL +validation.constraints.NotExceedMySQLFieldLength.message=字段过长 + +## 文件源管理 +validation.constraints.BkRepoBaseUrlInvalid.message=制品库根地址只能是http(s)://domain格式 diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_en.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_en.properties index 6a065c9690..ab42cd4fd9 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_en.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_en.properties @@ -1,7 +1,7 @@ # # Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. # -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +# Copyright (C) 2021 Tencent. All rights reserved. # # BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. # @@ -22,31 +22,119 @@ # IN THE SOFTWARE. # -validation.constraints.InvalidBkCloudId.message=Invalid bk_cloud_id -validation.constraints.InvalidIp.message=Invalid Ip +## common +validation.constraints.NotContainSpecialChar.message=Cannot contain special char, such as \\|/:*<>"? +validation.constraints.NotBlankField.message=Can not be empty + +## 文件相关 +validation.constraints.FileNameInvalid.message=FileName invalid, cannot contain path separator + +## host +validation.constraints.InvalidBkCloudId.message=Invalid BK-Net +validation.constraints.InvalidIp.message=Invalid IP +validation.constraints.EmptyDynamicGroupId.message=DynamicGroup ID cannot be empty +validation.constraints.BkHostId_null.message=Host id must not be null +validation.constraints.BkHostId_invalid.message=Invalid host id +validation.constraints.TopoNodeId_null.message=Topo node ID can not be empty +validation.constraints.TopoNodeType_null.message=Topo node type can not be empty + +## Job template/ Job plan/ Job instance validation.constraints.InvalidJobTimeout_empty.message=Job timeout cannot be empty validation.constraints.InvalidJobTimeout_outOfRange.message=Job timeout must be between {min} and {max} -validation.constraints.InvalidJobHighRiskGrammarRegex_empty.message=High-risk grammar regex can not be empty -validation.constraints.InvalidJobHighRiskGrammarRegex_outOfLength.message=High-risk grammar regex can not exceed {max} characters -validation.constraints.InvalidJobHighRiskGrammarRegex_wrongExpr.message=Wrong high-risk grammar regex -validation.constraints.ScriptTypeList_empty.message=Script type cannot be empty -validation.constraints.InvalidHighRiskGrammarHandleAction.message=Invalid handle action -validation.constraints.InvalidHighRiskRegularStatus.message=Invalid regular status -validation.constraints.InvalidHighRiskRegularDescription_outOfLength.message=Regular description can not exceed {max} characters -validation.constraints.InvalidBkBizId.message=Invalid bk_biz_id -validation.constraints.InvalidCronId.message=Invalid id -validation.constraints.InvalidCronJobPlanId.message=Invalid job_plan_id -validation.constraints.InvalidCronJobName_empty.message=Name cannot be empty -validation.constraints.InvalidCronJobName_outOfLength.message=Name can not exceed {max} characters -validation.constraints.InvalidCronExpression.message=Invalid expression -validation.constraints.InvalidCronExecuteTime.message=Invalid execute time +validation.constraints.TaskName_outOfLength.message=Job task name can not exceed {max} characters +validation.constraints.ExecuteTarget_empty.message=Job execute target can not be empty +validation.constraints.InvalidFileUploadSettingAmount.message=File size must be a positive float +validation.constraints.InvalidPort.message=Port must be a integer which is between 1 and 65535 +validation.constraints.InvalidJobInstanceId.message=Job instance id must be a positive number +validation.constraints.InvalidStepInstanceId.message=Step instance id must be a positive number validation.constraints.InvalidGlobalVarId.message=Invalid id of globalVar -validation.constraints.InvalidBkScopeType.message=Invalid bk_scope_type -validation.constraints.InvalidBkScopeId.message=Invalid bk_scope_id validation.constraints.InvalidUploadFileSuffix.message=Invalid file suffix validation.constraints.InvalidUploadFileRestrictMode.message=Invalid restrict mode +validation.constraints.ExecuteTarget_KubeContainerFilterEmpty.message=Job execute target - container filter cannot be empty + + +## HighRiskGrammar +validation.constraints.InvalidJobHighRiskGrammarId_empty.message=High-risk statement rule id can not be empty +validation.constraints.InvalidJobHighRiskGrammarRegex_empty.message=High-risk statement regex can not be empty +validation.constraints.InvalidJobHighRiskGrammarRegex_outOfLength.message=High-risk statement regex can not exceed {max} characters +validation.constraints.InvalidJobHighRiskGrammarRegex_wrongExpr.message=Wrong High-risk statement regex +validation.constraints.InvalidHighRiskGrammarHandleAction.message=Invalid handle action +validation.constraints.InvalidHighRiskRegularStatus.message=Invalid High-risk explanation status +validation.constraints.InvalidHighRiskRegularDescription_outOfLength.message=High-risk explanation description can not exceed {max} characters + +## script +validation.constraints.ScriptTypeList_empty.message=Script type cannot be empty +validation.constraints.ScriptName_outOfLength.message=Script name can not exceed {max} characters +validation.constraints.ScriptVersion_outOfLength.message=Script version name can not exceed {max} characters +validation.constraints.ScriptVersion_illegal.message=Illegal script version +validation.constraints.ScriptVersion_empty.message=Script version cannot be empty +validation.constraints.ScriptVersionId_empty.message=Script version id cannot be empty +validation.constraints.ScriptType_empty.message=script type cannot be empty +validation.constraints.ScriptType_illegal.message=Illegal script type +validation.constraints.ScriptId_empty.message=Script id cannot be empty +validation.constraints.ScriptName_illegal.message=Illegal Script name +validation.constraints.ScriptName_empty.message=Script name cannot be empty +validation.constraints.ScriptContent_empty.message=Script content cannot be empty validation.constraints.InvalidScriptId_empty.message=Script id cannot be empty -validation.constraints.EmptyDynamicGroupId.message=DynamicGroup ID cannot be empty -validation.constraints.InvalidFileUploadSettingAmount.message=File size must be a positive float -validation.constraints.InvalidPort.message=Port must be a integer which is between 0 and 65535 -validation.constraints.NotContainSpecialChar.message=Cannot contain special char:(\\|/:*<>"?) + +## business +validation.constraints.InvalidBkBizId.message=Invalid bk_biz_id +validation.constraints.InvalidBkScopeType.message=Invalid bk_scope_type +validation.constraints.InvalidBkScopeId.message=Invalid bk_scope_id + +## Cron Job +validation.constraints.InvalidCronId.message=Invalid id +validation.constraints.CronId_null.message=Cron id cannot be null +validation.constraints.InvalidCronJobPlanId.message=Invalid job_plan_id +validation.constraints.InvalidCronJobName_empty.message=Cron name cannot be empty +validation.constraints.InvalidCronJobName_outOfLength.message=Cron name can not exceed {max} characters +validation.constraints.InvalidCronExpression.message=Invalid cron expression +validation.constraints.InvalidCronExecuteTime.message=Invalid cron execute time + +## Account +validation.constraints.AccountId_empty.message=Account id cannot be empty +validation.constraints.AccountName_empty.message=Account name cannot be empty +validation.constraints.AccountType_illegal.message=Account type value is illegal +validation.constraints.AccountCategory_illegal.message=Account category value is illegal +validation.constraints.AccountPassword_tooLong.message=Account password length cannot be larger than 255 +validation.constraints.AccountAlias_tooLong.message=Account alias length cannot be larger than 255 +validation.constraints.AccountDescription_tooLong.message=Account description length cannot be larger than 1024 +validation.constraints.AccountAlias_empty.message=Account alias cannot be empty + +## BK-Notice +validation.constraints.InvalidAnnouncementsOffset.message=Announcements offset must be equal or greater than 0 +validation.constraints.InvalidAnnouncementsLimit.message=Announcements limit must be equal or greater than 1 +validation.constraints.InvalidWhiteIpScope.message=ScopeList cannot be empty + +## Others +validation.constraints.queryAgentInfoHostIds_tooMany.message=HostIds size can not be over 5000 +validation.constraints.queryExecuteObjects_outOfRange.message=Query execute objects size must be between {min} and {max} + +## Fast Execute Script +validation.constraints.WindowsInterpreterExceedMaxLength.message=Windows interpreter path length cannot exceed 260 chars + +## AI Chat +validation.constraints.AIGeneralChat_contentEmpty.message=Chat message cannot be empty +validation.constraints.AIAnalyzeError_contentEmpty.message=Error content cannot be empty +validation.constraints.AIInvalidHistoryStart.message=Start must be equal or greater than 0 +validation.constraints.AIInvalidHistoryLength.message=Length must be between {min} and {max} +validation.constraints.AIInvalidRecordId.message=Record id must be a positive number +validation.constraints.AIAnalyzeError_taskInstanceIdEmpty.message=TaskInstanceId cannot be empty +validation.constraints.AIAnalyzeError_stepExecuteTypeEmpty.message=StepExecuteType cannot be empty +validation.constraints.AIAnalyzeError_stepInstanceIdEmpty.message=StepInstanceId cannot be empty +validation.constraints.AIAnalyzeError_executeCountEmpty.message=ExecuteCount cannot be empty +validation.constraints.AIAnalyzeError_batchEmpty.message=Batch cannot be empty +validation.constraints.AIAnalyzeError_executeObjectTypeEmpty.message=ExecuteObjectType cannot be empty +validation.constraints.AIAnalyzeError_executeObjectResourceIdEmpty.message=ExecuteObjectResourceId cannot be empty +validation.constraints.AICheckScript_contentExceedMaxLength.message=Script content cannot exceed 5MB +validation.constraints.AIGeneralChat_contentExceedMaxLength.message=Chat content cannot exceed 5MB +validation.constraints.ExceedMaxLength.message=String field exceed max length +validation.constraints.MustEndWithSuffix.message=must end with suffix +validation.constraints.WinInterpreterInvalidSuffix.message=Interpreter path is invalid. Must end with .exe + + +## MySQL +validation.constraints.NotExceedMySQLFieldLength.message=Field content too long + +## File-Source Manage +validation.constraints.BkRepoBaseUrlInvalid.message=Pattern of bk-repo base url must be like: http(s)://domain diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_en_US.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_en_US.properties index fcdca650d9..ab42cd4fd9 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_en_US.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_en_US.properties @@ -1,7 +1,7 @@ # # Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. # -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +# Copyright (C) 2021 Tencent. All rights reserved. # # BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. # @@ -22,31 +22,119 @@ # IN THE SOFTWARE. # -validation.constraints.InvalidBkCloudId.message=Invalid bk_cloud_id -validation.constraints.InvalidIp.message=Invalid Ip +## common +validation.constraints.NotContainSpecialChar.message=Cannot contain special char, such as \\|/:*<>"? +validation.constraints.NotBlankField.message=Can not be empty + +## 文件相关 +validation.constraints.FileNameInvalid.message=FileName invalid, cannot contain path separator + +## host +validation.constraints.InvalidBkCloudId.message=Invalid BK-Net +validation.constraints.InvalidIp.message=Invalid IP +validation.constraints.EmptyDynamicGroupId.message=DynamicGroup ID cannot be empty +validation.constraints.BkHostId_null.message=Host id must not be null +validation.constraints.BkHostId_invalid.message=Invalid host id +validation.constraints.TopoNodeId_null.message=Topo node ID can not be empty +validation.constraints.TopoNodeType_null.message=Topo node type can not be empty + +## Job template/ Job plan/ Job instance validation.constraints.InvalidJobTimeout_empty.message=Job timeout cannot be empty validation.constraints.InvalidJobTimeout_outOfRange.message=Job timeout must be between {min} and {max} -validation.constraints.InvalidJobHighRiskGrammarRegex_empty.message=High-risk grammar regex can not be empty -validation.constraints.InvalidJobHighRiskGrammarRegex_outOfLength.message=High-risk grammar regex can not exceed {max} characters -validation.constraints.InvalidJobHighRiskGrammarRegex_wrongExpr.message=Wrong high-risk grammar regex -validation.constraints.ScriptTypeList_empty.message=Script type cannot be empty -validation.constraints.InvalidHighRiskGrammarHandleAction.message=Invalid handle action -validation.constraints.InvalidHighRiskRegularStatus.message=Invalid regular status -validation.constraints.InvalidHighRiskRegularDescription_outOfLength.message=Regular description can not exceed {max} characters -validation.constraints.InvalidBkBizId.message=Invalid bk_biz_id -validation.constraints.InvalidCronId.message=Invalid id -validation.constraints.InvalidCronJobPlanId.message=Invalid job_plan_id -validation.constraints.InvalidCronJobName_empty.message=Name cannot be empty -validation.constraints.InvalidCronJobName_outOfLength.message=Name can not exceed {max} characters -validation.constraints.InvalidCronExpression.message=Invalid expression -validation.constraints.InvalidCronExecuteTime.message=Invalid execute time +validation.constraints.TaskName_outOfLength.message=Job task name can not exceed {max} characters +validation.constraints.ExecuteTarget_empty.message=Job execute target can not be empty +validation.constraints.InvalidFileUploadSettingAmount.message=File size must be a positive float +validation.constraints.InvalidPort.message=Port must be a integer which is between 1 and 65535 +validation.constraints.InvalidJobInstanceId.message=Job instance id must be a positive number +validation.constraints.InvalidStepInstanceId.message=Step instance id must be a positive number validation.constraints.InvalidGlobalVarId.message=Invalid id of globalVar -validation.constraints.InvalidBkScopeType.message=Invalid bk_scope_type -validation.constraints.InvalidBkScopeId.message=Invalid bk_scope_id validation.constraints.InvalidUploadFileSuffix.message=Invalid file suffix validation.constraints.InvalidUploadFileRestrictMode.message=Invalid restrict mode +validation.constraints.ExecuteTarget_KubeContainerFilterEmpty.message=Job execute target - container filter cannot be empty + + +## HighRiskGrammar +validation.constraints.InvalidJobHighRiskGrammarId_empty.message=High-risk statement rule id can not be empty +validation.constraints.InvalidJobHighRiskGrammarRegex_empty.message=High-risk statement regex can not be empty +validation.constraints.InvalidJobHighRiskGrammarRegex_outOfLength.message=High-risk statement regex can not exceed {max} characters +validation.constraints.InvalidJobHighRiskGrammarRegex_wrongExpr.message=Wrong High-risk statement regex +validation.constraints.InvalidHighRiskGrammarHandleAction.message=Invalid handle action +validation.constraints.InvalidHighRiskRegularStatus.message=Invalid High-risk explanation status +validation.constraints.InvalidHighRiskRegularDescription_outOfLength.message=High-risk explanation description can not exceed {max} characters + +## script +validation.constraints.ScriptTypeList_empty.message=Script type cannot be empty +validation.constraints.ScriptName_outOfLength.message=Script name can not exceed {max} characters +validation.constraints.ScriptVersion_outOfLength.message=Script version name can not exceed {max} characters +validation.constraints.ScriptVersion_illegal.message=Illegal script version +validation.constraints.ScriptVersion_empty.message=Script version cannot be empty +validation.constraints.ScriptVersionId_empty.message=Script version id cannot be empty +validation.constraints.ScriptType_empty.message=script type cannot be empty +validation.constraints.ScriptType_illegal.message=Illegal script type +validation.constraints.ScriptId_empty.message=Script id cannot be empty +validation.constraints.ScriptName_illegal.message=Illegal Script name +validation.constraints.ScriptName_empty.message=Script name cannot be empty +validation.constraints.ScriptContent_empty.message=Script content cannot be empty validation.constraints.InvalidScriptId_empty.message=Script id cannot be empty -validation.constraints.EmptyDynamicGroupId.message=DynamicGroup ID cannot be empty -validation.constraints.InvalidFileUploadSettingAmount.message=File size must be a positive float -validation.constraints.InvalidPort.message=Port must be a integer which is between 0 and 65535 -validation.constraints.NotContainSpecialChar.message=Cannot contain special char:(\\|/:*<>"?) + +## business +validation.constraints.InvalidBkBizId.message=Invalid bk_biz_id +validation.constraints.InvalidBkScopeType.message=Invalid bk_scope_type +validation.constraints.InvalidBkScopeId.message=Invalid bk_scope_id + +## Cron Job +validation.constraints.InvalidCronId.message=Invalid id +validation.constraints.CronId_null.message=Cron id cannot be null +validation.constraints.InvalidCronJobPlanId.message=Invalid job_plan_id +validation.constraints.InvalidCronJobName_empty.message=Cron name cannot be empty +validation.constraints.InvalidCronJobName_outOfLength.message=Cron name can not exceed {max} characters +validation.constraints.InvalidCronExpression.message=Invalid cron expression +validation.constraints.InvalidCronExecuteTime.message=Invalid cron execute time + +## Account +validation.constraints.AccountId_empty.message=Account id cannot be empty +validation.constraints.AccountName_empty.message=Account name cannot be empty +validation.constraints.AccountType_illegal.message=Account type value is illegal +validation.constraints.AccountCategory_illegal.message=Account category value is illegal +validation.constraints.AccountPassword_tooLong.message=Account password length cannot be larger than 255 +validation.constraints.AccountAlias_tooLong.message=Account alias length cannot be larger than 255 +validation.constraints.AccountDescription_tooLong.message=Account description length cannot be larger than 1024 +validation.constraints.AccountAlias_empty.message=Account alias cannot be empty + +## BK-Notice +validation.constraints.InvalidAnnouncementsOffset.message=Announcements offset must be equal or greater than 0 +validation.constraints.InvalidAnnouncementsLimit.message=Announcements limit must be equal or greater than 1 +validation.constraints.InvalidWhiteIpScope.message=ScopeList cannot be empty + +## Others +validation.constraints.queryAgentInfoHostIds_tooMany.message=HostIds size can not be over 5000 +validation.constraints.queryExecuteObjects_outOfRange.message=Query execute objects size must be between {min} and {max} + +## Fast Execute Script +validation.constraints.WindowsInterpreterExceedMaxLength.message=Windows interpreter path length cannot exceed 260 chars + +## AI Chat +validation.constraints.AIGeneralChat_contentEmpty.message=Chat message cannot be empty +validation.constraints.AIAnalyzeError_contentEmpty.message=Error content cannot be empty +validation.constraints.AIInvalidHistoryStart.message=Start must be equal or greater than 0 +validation.constraints.AIInvalidHistoryLength.message=Length must be between {min} and {max} +validation.constraints.AIInvalidRecordId.message=Record id must be a positive number +validation.constraints.AIAnalyzeError_taskInstanceIdEmpty.message=TaskInstanceId cannot be empty +validation.constraints.AIAnalyzeError_stepExecuteTypeEmpty.message=StepExecuteType cannot be empty +validation.constraints.AIAnalyzeError_stepInstanceIdEmpty.message=StepInstanceId cannot be empty +validation.constraints.AIAnalyzeError_executeCountEmpty.message=ExecuteCount cannot be empty +validation.constraints.AIAnalyzeError_batchEmpty.message=Batch cannot be empty +validation.constraints.AIAnalyzeError_executeObjectTypeEmpty.message=ExecuteObjectType cannot be empty +validation.constraints.AIAnalyzeError_executeObjectResourceIdEmpty.message=ExecuteObjectResourceId cannot be empty +validation.constraints.AICheckScript_contentExceedMaxLength.message=Script content cannot exceed 5MB +validation.constraints.AIGeneralChat_contentExceedMaxLength.message=Chat content cannot exceed 5MB +validation.constraints.ExceedMaxLength.message=String field exceed max length +validation.constraints.MustEndWithSuffix.message=must end with suffix +validation.constraints.WinInterpreterInvalidSuffix.message=Interpreter path is invalid. Must end with .exe + + +## MySQL +validation.constraints.NotExceedMySQLFieldLength.message=Field content too long + +## File-Source Manage +validation.constraints.BkRepoBaseUrlInvalid.message=Pattern of bk-repo base url must be like: http(s)://domain diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_zh.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_zh.properties index 02aaf91a48..3eb9a6a49f 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_zh.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_zh.properties @@ -1,7 +1,7 @@ # # Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. # -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +# Copyright (C) 2021 Tencent. All rights reserved. # # BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. # @@ -22,31 +22,120 @@ # IN THE SOFTWARE. # -validation.constraints.InvalidBkCloudId.message=云区域ID非法 -validation.constraints.InvalidIp.message=IP非法 +## 通用 +validation.constraints.NotContainSpecialChar.message=不可包含 \\|/:*<>"? 特殊字符 +validation.constraints.NotBlankField.message=不能为空 + +## 文件相关 +validation.constraints.FileNameInvalid.message=文件名非法，不可包含路径分隔符 + +## 主机 +validation.constraints.InvalidBkCloudId.message=管控区域 ID 非法 +validation.constraints.InvalidIp.message=IP 非法 +validation.constraints.EmptyDynamicGroupId.message=动态分组 ID 不可为空 +validation.constraints.BkHostId_null.message=主机 ID 不能为空 +validation.constraints.BkHostId_invalid.message=非法的主机 ID +validation.constraints.TopoNodeId_null.message=拓扑节点 ID 不可为空 +validation.constraints.TopoNodeType_null.message=拓扑节点类型不可为空 + +## 作业模版、执行方案、作业实例 validation.constraints.InvalidJobTimeout_empty.message=作业超时时间不能为空 validation.constraints.InvalidJobTimeout_outOfRange.message=作业超时时间必须在{min}-{max}之间 +validation.constraints.TaskName_outOfLength.message=作业名称不能超过 {max} 个字符 +validation.constraints.ExecuteTarget_empty.message=作业执行目标不能为空 +validation.constraints.InvalidFileUploadSettingAmount.message=文件大小只能为大于0的浮点数 +validation.constraints.InvalidPort.message=端口号只能为1-65535之间的整数 +validation.constraints.InvalidJobInstanceId.message=任务实例ID必须为正整数 +validation.constraints.InvalidStepInstanceId.message=步骤实例ID必须为正整数 +validation.constraints.InvalidGlobalVarId.message=非法的全局变量 ID +validation.constraints.InvalidUploadFileSuffix.message=非法的上传文件后缀 +validation.constraints.InvalidUploadFileRestrictMode.message=非法的上传文件限制模式 +validation.constraints.ExecuteTarget_KubeContainerFilterEmpty.message=作业执行目标-容器过滤器不能为空 + +## 高危语句 +validation.constraints.InvalidJobHighRiskGrammarId_empty.message=高危语句规则ID不能为空 validation.constraints.InvalidJobHighRiskGrammarRegex_empty.message=语法检测表达式不能为空 validation.constraints.InvalidJobHighRiskGrammarRegex_outOfLength.message=语法检测表达式不能超过{max}个字符 validation.constraints.InvalidJobHighRiskGrammarRegex_wrongExpr.message=语法检测表达式错误 -validation.constraints.ScriptTypeList_empty.message=脚本类型不能为空 validation.constraints.InvalidHighRiskGrammarHandleAction.message=非法的处理动作 validation.constraints.InvalidHighRiskRegularStatus.message=非法的规则启停状态 validation.constraints.InvalidHighRiskRegularDescription_outOfLength.message=规则说明不能超过{max}个字符 -validation.constraints.InvalidBkBizId.message=非法的业务ID -validation.constraints.InvalidCronId.message=非法的定时任务ID -validation.constraints.InvalidCronJobPlanId.message=非法的作业ID + +## 脚本 +validation.constraints.ScriptTypeList_empty.message=脚本类型不能为空 +validation.constraints.ScriptName_outOfLength.message=脚本名称不能超过{max}个字符 +validation.constraints.ScriptVersion_outOfLength.message=脚本版本不能超过{max}个字符 +validation.constraints.ScriptVersion_illegal.message=脚本版本非法 +validation.constraints.ScriptVersion_empty.message=脚本版本不能为空 +validation.constraints.ScriptVersionId_empty.message=脚本版本ID不能为空 +validation.constraints.ScriptType_empty.message=脚本类型不能为空 +validation.constraints.ScriptType_illegal.message=脚本类型非法 +validation.constraints.ScriptId_empty.message=脚本ID不能为空 +validation.constraints.ScriptName_illegal.message=脚本名称非法 +validation.constraints.ScriptName_empty.message=脚本名称不能为空 +validation.constraints.ScriptContent_empty.message=脚本内容不能为空 +validation.constraints.InvalidScriptId_empty.message=脚本 ID 不能为空 + +## 业务 +validation.constraints.InvalidBkBizId.message=非法的业务 ID +validation.constraints.InvalidBkScopeType.message=非法的资源范围类型 +validation.constraints.InvalidBkScopeId.message=非法的资源范围 ID + +## 定时任务 +validation.constraints.InvalidCronId.message=非法的定时任务 ID +validation.constraints.CronId_null.message=定时任务ID不能为空 +validation.constraints.InvalidCronJobPlanId.message=非法的作业 ID validation.constraints.InvalidCronJobName_empty.message=定时作业名称不能为空 validation.constraints.InvalidCronJobName_outOfLength.message=定时作业名称不能超过{max}个字符 -validation.constraints.InvalidCronExpression.message=非法的Cron表达式 +validation.constraints.InvalidCronExpression.message=非法的 Cron 表达式 validation.constraints.InvalidCronExecuteTime.message=非法的执行时间 -validation.constraints.InvalidGlobalVarId.message=非法的全局变量ID -validation.constraints.InvalidBkScopeType.message=非法的资源范围类型 -validation.constraints.InvalidBkScopeId.message=非法的资源范围ID -validation.constraints.InvalidUploadFileSuffix.message=非法的上传文件后缀 -validation.constraints.InvalidUploadFileRestrictMode.message=非法的上传文件限制模式 -validation.constraints.InvalidScriptId_empty.message=脚本ID不能为空 -validation.constraints.EmptyDynamicGroupId.message=动态分组ID不可为空 -validation.constraints.InvalidFileUploadSettingAmount.message=文件大小只能为大于0的浮点数 -validation.constraints.InvalidPort.message=端口号只能为0-65535之间的整数 -validation.constraints.NotContainSpecialChar.message=不可包含特殊字符（\\|/:*<>"?） + +## 账号 +validation.constraints.AccountId_empty.message=账号ID不能为空 +validation.constraints.AccountName_empty.message=账号名称不能为空 +validation.constraints.AccountType_illegal.message=账号类型取值非法 +validation.constraints.AccountCategory_illegal.message=账号用途取值非法 +validation.constraints.AccountPassword_tooLong.message=账号密码不能超过255字符 +validation.constraints.AccountAlias_tooLong.message=账号别名不能超过255字符 +validation.constraints.AccountDescription_tooLong.message=账号描述不能超过1024字符 +validation.constraints.AccountAlias_empty.message=账号别名不能为空 + + +## 蓝鲸消息通知 +validation.constraints.InvalidAnnouncementsOffset.message=公告偏移量必须大于等于0 +validation.constraints.InvalidAnnouncementsLimit.message=公告获取数量必须大于等于1 +validation.constraints.InvalidWhiteIpScope.message=目标业务不可为空 + +## 其它 +validation.constraints.queryAgentInfoHostIds_tooMany.message=单次查询主机数量不可超过5000 +validation.constraints.queryExecuteObjects_outOfRange.message=单次查询执行对象数量范围必须在 {min}-{max} 之间 + +## 快速执行脚本 +validation.constraints.WindowsInterpreterExceedMaxLength.message=Windows解释器路径长度不可超过260字符 + + +## AI对话 +validation.constraints.AIGeneralChat_contentEmpty.message=对话输入内容不能为空 +validation.constraints.AIAnalyzeError_contentEmpty.message=报错信息内容不能为空 +validation.constraints.AIInvalidHistoryStart.message=对话历史记录起始位置start必须大于等于0 +validation.constraints.AIInvalidHistoryLength.message=对话历史记录数量length必须在 {min}-{max} 之间 +validation.constraints.AIInvalidRecordId.message=对话记录ID只能为正整数 +validation.constraints.AIAnalyzeError_taskInstanceIdEmpty.message=任务实例ID不能为空 +validation.constraints.AIAnalyzeError_stepExecuteTypeEmpty.message=步骤执行类型不能为空 +validation.constraints.AIAnalyzeError_stepInstanceIdEmpty.message=步骤实例ID不能为空 +validation.constraints.AIAnalyzeError_executeCountEmpty.message=执行次数不能为空 +validation.constraints.AIAnalyzeError_batchEmpty.message=滚动批次不能为空 +validation.constraints.AIAnalyzeError_executeObjectTypeEmpty.message=执行对象类型不能为空 +validation.constraints.AIAnalyzeError_executeObjectResourceIdEmpty.message=执行对象资源ID不能为空 +validation.constraints.AICheckScript_contentExceedMaxLength.message=检查的脚本内容长度不可超过5MB +validation.constraints.AIGeneralChat_contentExceedMaxLength.message=对话输入内容长度不可超过5MB +validation.constraints.ExceedMaxLength.message=字段超出最大长度 +validation.constraints.MustEndWithSuffix.message=必须以指定后缀结尾 +validation.constraints.WinInterpreterInvalidSuffix.message=解释器路径不合法，需以 .exe 结尾 + + +## MySQL +validation.constraints.NotExceedMySQLFieldLength.message=字段过长 + +## 文件源管理 +validation.constraints.BkRepoBaseUrlInvalid.message=制品库根地址只能是http(s)://domain格式 diff --git a/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_zh_CN.properties b/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_zh_CN.properties index 02aaf91a48..3eb9a6a49f 100644 --- a/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_zh_CN.properties +++ b/src/backend/commons/common-i18n/src/main/resources/i18n/validation/ValidationMessages_zh_CN.properties @@ -1,7 +1,7 @@ # # Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. # -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +# Copyright (C) 2021 Tencent. All rights reserved. # # BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. # @@ -22,31 +22,120 @@ # IN THE SOFTWARE. # -validation.constraints.InvalidBkCloudId.message=云区域ID非法 -validation.constraints.InvalidIp.message=IP非法 +## 通用 +validation.constraints.NotContainSpecialChar.message=不可包含 \\|/:*<>"? 特殊字符 +validation.constraints.NotBlankField.message=不能为空 + +## 文件相关 +validation.constraints.FileNameInvalid.message=文件名非法，不可包含路径分隔符 + +## 主机 +validation.constraints.InvalidBkCloudId.message=管控区域 ID 非法 +validation.constraints.InvalidIp.message=IP 非法 +validation.constraints.EmptyDynamicGroupId.message=动态分组 ID 不可为空 +validation.constraints.BkHostId_null.message=主机 ID 不能为空 +validation.constraints.BkHostId_invalid.message=非法的主机 ID +validation.constraints.TopoNodeId_null.message=拓扑节点 ID 不可为空 +validation.constraints.TopoNodeType_null.message=拓扑节点类型不可为空 + +## 作业模版、执行方案、作业实例 validation.constraints.InvalidJobTimeout_empty.message=作业超时时间不能为空 validation.constraints.InvalidJobTimeout_outOfRange.message=作业超时时间必须在{min}-{max}之间 +validation.constraints.TaskName_outOfLength.message=作业名称不能超过 {max} 个字符 +validation.constraints.ExecuteTarget_empty.message=作业执行目标不能为空 +validation.constraints.InvalidFileUploadSettingAmount.message=文件大小只能为大于0的浮点数 +validation.constraints.InvalidPort.message=端口号只能为1-65535之间的整数 +validation.constraints.InvalidJobInstanceId.message=任务实例ID必须为正整数 +validation.constraints.InvalidStepInstanceId.message=步骤实例ID必须为正整数 +validation.constraints.InvalidGlobalVarId.message=非法的全局变量 ID +validation.constraints.InvalidUploadFileSuffix.message=非法的上传文件后缀 +validation.constraints.InvalidUploadFileRestrictMode.message=非法的上传文件限制模式 +validation.constraints.ExecuteTarget_KubeContainerFilterEmpty.message=作业执行目标-容器过滤器不能为空 + +## 高危语句 +validation.constraints.InvalidJobHighRiskGrammarId_empty.message=高危语句规则ID不能为空 validation.constraints.InvalidJobHighRiskGrammarRegex_empty.message=语法检测表达式不能为空 validation.constraints.InvalidJobHighRiskGrammarRegex_outOfLength.message=语法检测表达式不能超过{max}个字符 validation.constraints.InvalidJobHighRiskGrammarRegex_wrongExpr.message=语法检测表达式错误 -validation.constraints.ScriptTypeList_empty.message=脚本类型不能为空 validation.constraints.InvalidHighRiskGrammarHandleAction.message=非法的处理动作 validation.constraints.InvalidHighRiskRegularStatus.message=非法的规则启停状态 validation.constraints.InvalidHighRiskRegularDescription_outOfLength.message=规则说明不能超过{max}个字符 -validation.constraints.InvalidBkBizId.message=非法的业务ID -validation.constraints.InvalidCronId.message=非法的定时任务ID -validation.constraints.InvalidCronJobPlanId.message=非法的作业ID + +## 脚本 +validation.constraints.ScriptTypeList_empty.message=脚本类型不能为空 +validation.constraints.ScriptName_outOfLength.message=脚本名称不能超过{max}个字符 +validation.constraints.ScriptVersion_outOfLength.message=脚本版本不能超过{max}个字符 +validation.constraints.ScriptVersion_illegal.message=脚本版本非法 +validation.constraints.ScriptVersion_empty.message=脚本版本不能为空 +validation.constraints.ScriptVersionId_empty.message=脚本版本ID不能为空 +validation.constraints.ScriptType_empty.message=脚本类型不能为空 +validation.constraints.ScriptType_illegal.message=脚本类型非法 +validation.constraints.ScriptId_empty.message=脚本ID不能为空 +validation.constraints.ScriptName_illegal.message=脚本名称非法 +validation.constraints.ScriptName_empty.message=脚本名称不能为空 +validation.constraints.ScriptContent_empty.message=脚本内容不能为空 +validation.constraints.InvalidScriptId_empty.message=脚本 ID 不能为空 + +## 业务 +validation.constraints.InvalidBkBizId.message=非法的业务 ID +validation.constraints.InvalidBkScopeType.message=非法的资源范围类型 +validation.constraints.InvalidBkScopeId.message=非法的资源范围 ID + +## 定时任务 +validation.constraints.InvalidCronId.message=非法的定时任务 ID +validation.constraints.CronId_null.message=定时任务ID不能为空 +validation.constraints.InvalidCronJobPlanId.message=非法的作业 ID validation.constraints.InvalidCronJobName_empty.message=定时作业名称不能为空 validation.constraints.InvalidCronJobName_outOfLength.message=定时作业名称不能超过{max}个字符 -validation.constraints.InvalidCronExpression.message=非法的Cron表达式 +validation.constraints.InvalidCronExpression.message=非法的 Cron 表达式 validation.constraints.InvalidCronExecuteTime.message=非法的执行时间 -validation.constraints.InvalidGlobalVarId.message=非法的全局变量ID -validation.constraints.InvalidBkScopeType.message=非法的资源范围类型 -validation.constraints.InvalidBkScopeId.message=非法的资源范围ID -validation.constraints.InvalidUploadFileSuffix.message=非法的上传文件后缀 -validation.constraints.InvalidUploadFileRestrictMode.message=非法的上传文件限制模式 -validation.constraints.InvalidScriptId_empty.message=脚本ID不能为空 -validation.constraints.EmptyDynamicGroupId.message=动态分组ID不可为空 -validation.constraints.InvalidFileUploadSettingAmount.message=文件大小只能为大于0的浮点数 -validation.constraints.InvalidPort.message=端口号只能为0-65535之间的整数 -validation.constraints.NotContainSpecialChar.message=不可包含特殊字符（\\|/:*<>"?） + +## 账号 +validation.constraints.AccountId_empty.message=账号ID不能为空 +validation.constraints.AccountName_empty.message=账号名称不能为空 +validation.constraints.AccountType_illegal.message=账号类型取值非法 +validation.constraints.AccountCategory_illegal.message=账号用途取值非法 +validation.constraints.AccountPassword_tooLong.message=账号密码不能超过255字符 +validation.constraints.AccountAlias_tooLong.message=账号别名不能超过255字符 +validation.constraints.AccountDescription_tooLong.message=账号描述不能超过1024字符 +validation.constraints.AccountAlias_empty.message=账号别名不能为空 + + +## 蓝鲸消息通知 +validation.constraints.InvalidAnnouncementsOffset.message=公告偏移量必须大于等于0 +validation.constraints.InvalidAnnouncementsLimit.message=公告获取数量必须大于等于1 +validation.constraints.InvalidWhiteIpScope.message=目标业务不可为空 + +## 其它 +validation.constraints.queryAgentInfoHostIds_tooMany.message=单次查询主机数量不可超过5000 +validation.constraints.queryExecuteObjects_outOfRange.message=单次查询执行对象数量范围必须在 {min}-{max} 之间 + +## 快速执行脚本 +validation.constraints.WindowsInterpreterExceedMaxLength.message=Windows解释器路径长度不可超过260字符 + + +## AI对话 +validation.constraints.AIGeneralChat_contentEmpty.message=对话输入内容不能为空 +validation.constraints.AIAnalyzeError_contentEmpty.message=报错信息内容不能为空 +validation.constraints.AIInvalidHistoryStart.message=对话历史记录起始位置start必须大于等于0 +validation.constraints.AIInvalidHistoryLength.message=对话历史记录数量length必须在 {min}-{max} 之间 +validation.constraints.AIInvalidRecordId.message=对话记录ID只能为正整数 +validation.constraints.AIAnalyzeError_taskInstanceIdEmpty.message=任务实例ID不能为空 +validation.constraints.AIAnalyzeError_stepExecuteTypeEmpty.message=步骤执行类型不能为空 +validation.constraints.AIAnalyzeError_stepInstanceIdEmpty.message=步骤实例ID不能为空 +validation.constraints.AIAnalyzeError_executeCountEmpty.message=执行次数不能为空 +validation.constraints.AIAnalyzeError_batchEmpty.message=滚动批次不能为空 +validation.constraints.AIAnalyzeError_executeObjectTypeEmpty.message=执行对象类型不能为空 +validation.constraints.AIAnalyzeError_executeObjectResourceIdEmpty.message=执行对象资源ID不能为空 +validation.constraints.AICheckScript_contentExceedMaxLength.message=检查的脚本内容长度不可超过5MB +validation.constraints.AIGeneralChat_contentExceedMaxLength.message=对话输入内容长度不可超过5MB +validation.constraints.ExceedMaxLength.message=字段超出最大长度 +validation.constraints.MustEndWithSuffix.message=必须以指定后缀结尾 +validation.constraints.WinInterpreterInvalidSuffix.message=解释器路径不合法，需以 .exe 结尾 + + +## MySQL +validation.constraints.NotExceedMySQLFieldLength.message=字段过长 + +## 文件源管理 +validation.constraints.BkRepoBaseUrlInvalid.message=制品库根地址只能是http(s)://domain格式 diff --git a/src/backend/commons/common-iam/build.gradle b/src/backend/commons/common-iam/build.gradle index cf1dbd7590..9c8eb567fd 100644 --- a/src/backend/commons/common-iam/build.gradle +++ b/src/backend/commons/common-iam/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-iam/libs/iam-sdk-1.0.0.jar b/src/backend/commons/common-iam/libs/iam-sdk-1.0.0.jar index 5e6da9837c..1f9c4ba787 100644 Binary files a/src/backend/commons/common-iam/libs/iam-sdk-1.0.0.jar and b/src/backend/commons/common-iam/libs/iam-sdk-1.0.0.jar differ diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/aspect/IamAppTransferAspect.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/aspect/IamAppTransferAspect.java index 727572ccac..a5be08ab2b 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/aspect/IamAppTransferAspect.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/aspect/IamAppTransferAspect.java @@ -8,8 +8,6 @@ import org.aspectj.lang.annotation.Around; import org.aspectj.lang.annotation.Aspect; import org.aspectj.lang.annotation.Pointcut; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; /** * 鉴权执行前对Job业务与CMDB业务（集）进行转换的切面 @@ -17,25 +15,27 @@ */ @Slf4j @Aspect -@Component public class IamAppTransferAspect { private final AppScopeMappingService appScopeMappingService; - public IamAppTransferAspect(@Autowired(required = false) AppScopeMappingService appScopeMappingService) { + public IamAppTransferAspect(AppScopeMappingService appScopeMappingService) { this.appScopeMappingService = appScopeMappingService; } - @Pointcut("execution (* com.tencent.bk.job.common.iam.service.impl.BusinessAuthServiceImpl.auth*(..))") + @Pointcut("within(com.tencent.bk.job.common.iam.service.impl.BusinessAuthServiceImpl) " + + "&& execution (* com.tencent.bk.job.common.iam.service.impl.BusinessAuthServiceImpl.auth*(..))") public void processAuthBusinessAction() { } - @Pointcut("execution (* com.tencent.bk.job.*.auth..impl.*.auth*(..))") + @Pointcut("within(com.tencent.bk.job..*) " + + "&& execution (* com.tencent.bk.job.*.auth..impl.*.auth*(..))") public void processAuthResourceAction() { } - @Pointcut("execution (* com.tencent.bk.job.*.auth..impl.*.batchAuth*(..))") + @Pointcut("within(com.tencent.bk.job..*) " + + "&& execution (* com.tencent.bk.job.*.auth..impl.*.batchAuth*(..))") public void processBatchAuthResourceAction() { } diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/aspect/IamCallbackAspect.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/aspect/IamCallbackAspect.java index 708887fce8..fef1c67f9f 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/aspect/IamCallbackAspect.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/aspect/IamCallbackAspect.java @@ -7,19 +7,15 @@ import org.aspectj.lang.annotation.Around; import org.aspectj.lang.annotation.Aspect; import org.aspectj.lang.annotation.Pointcut; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; @Slf4j @Aspect -@Component public class IamCallbackAspect { - @Autowired public IamCallbackAspect() { } - @Pointcut("execution (* com.tencent.bk.job.*.api.iam.impl.*.callback(..))") + @Pointcut("within(com.tencent.bk.job..*) && execution (* com.tencent.bk.job.*.api.iam.impl.*.callback(..))") public void processCallbackRequest() { } diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/aspect/IamExceptionHandleAspect.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/aspect/IamExceptionHandleAspect.java new file mode 100644 index 0000000000..2fce0bd86b --- /dev/null +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/aspect/IamExceptionHandleAspect.java @@ -0,0 +1,58 @@ +package com.tencent.bk.job.common.iam.aspect; + +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.FailedPreconditionException; +import com.tencent.bk.job.common.exception.InternalException; +import com.tencent.bk.sdk.iam.exception.IamException; +import lombok.extern.slf4j.Slf4j; +import org.aspectj.lang.ProceedingJoinPoint; +import org.aspectj.lang.annotation.Around; +import org.aspectj.lang.annotation.Aspect; +import org.aspectj.lang.annotation.Pointcut; + +/** + * 调用IAM-SDK鉴权过程中对SDK抛出的异常进行处理 + */ +@Slf4j +@Aspect +public class IamExceptionHandleAspect { + + @Pointcut("within(com.tencent.bk.sdk.iam.service.impl.PolicyServiceImpl) " + + "&& execution (* com.tencent.bk.sdk.iam.service.impl.PolicyServiceImpl.*(..))") + public void processPolicyServiceAction() { + } + + @Pointcut("within(com.tencent.bk.sdk.iam.helper.AuthHelper) " + + "&& execution (* com.tencent.bk.sdk.iam.helper.AuthHelper.isAllowed(..))") + public void processIsAllowedAction() { + } + + @Around("processIsAllowedAction()||processPolicyServiceAction()") + public Object handleIamExceptionDuringAuth(ProceedingJoinPoint pjp) throws Throwable { + String username = null; + try { + Object[] args = pjp.getArgs(); + if (args.length > 0 && args[0] instanceof String) { + username = (String) args[0]; + } + return pjp.proceed(); + } catch (IamException e) { + return handIamException(username, e); + } + } + + /** + * 对IAM异常进行包装处理 + * + * @param username 用户名 + * @param e 权限中心异常 + */ + private Object handIamException(String username, IamException e) { + long errCode = e.getErrorCode(); + long iamErrorCodeUserAccountFrozen = 1901403L; + if (errCode == iamErrorCodeUserAccountFrozen) { + throw new FailedPreconditionException(e, ErrorCode.IAM_USER_ACCOUNT_FROZEN, new String[]{username}); + } + throw new InternalException(e, ErrorCode.IAM_API_DATA_ERROR); + } +} diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/aspect/IamMetricsAspect.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/aspect/IamMetricsAspect.java new file mode 100644 index 0000000000..f4f0f2611a --- /dev/null +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/aspect/IamMetricsAspect.java @@ -0,0 +1,81 @@ +package com.tencent.bk.job.common.iam.aspect; + +import com.tencent.bk.job.common.iam.metrics.MetricsConstants; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tag; +import io.micrometer.core.instrument.Tags; +import io.micrometer.core.instrument.Timer; +import lombok.extern.slf4j.Slf4j; +import org.aspectj.lang.ProceedingJoinPoint; +import org.aspectj.lang.annotation.Around; +import org.aspectj.lang.annotation.Aspect; +import org.aspectj.lang.annotation.Pointcut; + +import java.time.Duration; +import java.util.concurrent.TimeUnit; + +/** + * 记录AuthHelper.isAllow方法调用结果统计数据 + */ +@Slf4j +@Aspect +public class IamMetricsAspect { + + private final MeterRegistry meterRegistry; + + public IamMetricsAspect(MeterRegistry meterRegistry) { + this.meterRegistry = meterRegistry; + } + + @Pointcut("within(com.tencent.bk.sdk.iam.helper.AuthHelper) " + + "&& execution (* com.tencent.bk.sdk.iam.helper.AuthHelper.isAllowed(..))") + public void processIsAllowedAction() { + } + + @Around("processIsAllowedAction()") + public Object recordMetricsDuringAuth(ProceedingJoinPoint pjp) throws Throwable { + String action = MetricsConstants.TAG_VALUE_ACTION_NONE; + String result = MetricsConstants.TAG_VALUE_RESULT_TRUE; + Long startTimeMills = System.currentTimeMillis(); + try { + Object[] args = pjp.getArgs(); + if (args.length >= 2 && args[1] instanceof String) { + action = (String) args[1]; + } + Object methodResult = pjp.proceed(); + if (methodResult instanceof Boolean) { + if (!(Boolean) methodResult) { + result = MetricsConstants.TAG_VALUE_RESULT_FALSE; + } + } + return methodResult; + } catch (Throwable t) { + result = MetricsConstants.TAG_VALUE_RESULT_ERROR; + throw t; + } finally { + tryToRecordMetrics(action, result, startTimeMills); + } + } + + private void tryToRecordMetrics(String action, String status, Long startTimeMills) { + try { + recordMetrics(action, status, startTimeMills); + } catch (Throwable t) { + log.warn("Fail to recordMetrics", t); + } + } + + private void recordMetrics(String action, String status, Long startTimeMills) { + Tags tags = Tags.of( + Tag.of(MetricsConstants.TAG_KEY_ACTION, action), + Tag.of(MetricsConstants.TAG_KEY_STATUS, status) + ); + Timer.builder(MetricsConstants.NAME_AUTH_HELPER_IS_ALLOW) + .tags(tags) + .publishPercentileHistogram(true) + .minimumExpectedValue(Duration.ofMillis(10)) + .maximumExpectedValue(Duration.ofSeconds(30L)) + .register(meterRegistry) + .record(System.currentTimeMillis() - startTimeMills, TimeUnit.MILLISECONDS); + } +} diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/client/EsbIamClient.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/client/EsbIamClient.java index 0ce58f9d17..95ad203fbf 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/client/EsbIamClient.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/client/EsbIamClient.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,9 +26,14 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.constant.HttpMethodEnum; +import com.tencent.bk.job.common.esb.config.AppProperties; +import com.tencent.bk.job.common.esb.config.EsbProperties; +import com.tencent.bk.job.common.esb.model.BkApiAuthorization; import com.tencent.bk.job.common.esb.model.EsbReq; import com.tencent.bk.job.common.esb.model.EsbResp; -import com.tencent.bk.job.common.esb.sdk.AbstractEsbSdkClient; +import com.tencent.bk.job.common.esb.model.OpenApiRequestInfo; +import com.tencent.bk.job.common.esb.sdk.BkApiClient; import com.tencent.bk.job.common.exception.InternalIamException; import com.tencent.bk.job.common.iam.dto.AuthByPathReq; import com.tencent.bk.job.common.iam.dto.BatchAuthByPathReq; @@ -42,22 +47,24 @@ import com.tencent.bk.job.common.iam.dto.GetApplyUrlResponse; import com.tencent.bk.job.common.iam.dto.RegisterResourceRequest; import com.tencent.bk.job.common.metrics.CommonMetricNames; +import com.tencent.bk.job.common.util.http.HttpHelperFactory; import com.tencent.bk.job.common.util.http.HttpMetricUtil; -import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.sdk.iam.constants.SystemId; import com.tencent.bk.sdk.iam.dto.action.ActionDTO; import com.tencent.bk.sdk.iam.dto.resource.ResourceDTO; +import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Tag; import lombok.extern.slf4j.Slf4j; -import org.apache.http.client.methods.HttpPost; import java.util.List; +import static com.tencent.bk.job.common.metrics.CommonMetricNames.IAM_API; + /** - * @since 16/6/2020 21:37 + * IAM API 调用客户端 */ @Slf4j -public class EsbIamClient extends AbstractEsbSdkClient implements IIamClient { +public class EsbIamClient extends BkApiClient implements IIamClient { private static final String API_GET_APPLY_URL = "/api/c/compapi/v2/iam/application/"; private static final String API_REGISTER_RESOURCE_URL = @@ -67,26 +74,30 @@ public class EsbIamClient extends AbstractEsbSdkClient implements IIamClient { private static final String API_BATCH_AUTH_BY_PATH_URL = "/api/c/compapi/v2/iam/authorization/batch_path/"; - public EsbIamClient(String esbHostUrl, String appCode, String appSecret, boolean useEsbTestEnv) { - super(esbHostUrl, appCode, appSecret, null, useEsbTestEnv); - } + private final BkApiAuthorization authorization; - public EsbIamClient(String esbHostUrl, String appCode, String appSecret, String lang, boolean useEsbTestEnv) { - super(esbHostUrl, appCode, appSecret, lang, useEsbTestEnv); + public EsbIamClient(MeterRegistry meterRegistry, + AppProperties appProperties, + EsbProperties esbProperties) { + super( + meterRegistry, + IAM_API, + esbProperties.getService().getUrl(), + HttpHelperFactory.createHttpHelper( + httpClientBuilder -> httpClientBuilder.addInterceptorLast(getLogBkApiRequestIdInterceptor()) + ) + ); + this.authorization = BkApiAuthorization.appAuthorization(appProperties.getCode(), + appProperties.getSecret(), "admin"); } @Override public String getApplyUrl(List actionList) { - GetApplyUrlRequest getApplyUrlRequest = makeBaseReqByWeb( - GetApplyUrlRequest.class, - null, - "admin", - "superadmin" - ); + GetApplyUrlRequest getApplyUrlRequest = EsbReq.buildRequest(GetApplyUrlRequest.class, null); getApplyUrlRequest.setSystem(SystemId.JOB); getApplyUrlRequest.setAction(actionList); EsbResp esbResp = requestIamApi( - HttpPost.METHOD_NAME, + HttpMethodEnum.POST, API_GET_APPLY_URL, getApplyUrlRequest, new TypeReference>() { @@ -101,8 +112,7 @@ public String getApplyUrl(List actionList) { @Override public boolean registerResource(String id, String name, String type, String creator, List ancestor) { - RegisterResourceRequest registerResourceRequest = - makeBaseReqByWeb(RegisterResourceRequest.class, null, creator, "0"); + RegisterResourceRequest registerResourceRequest = EsbReq.buildRequest(RegisterResourceRequest.class, null); registerResourceRequest.setSystem(SystemId.JOB); registerResourceRequest.setId(id); registerResourceRequest.setName(name); @@ -112,7 +122,7 @@ public boolean registerResource(String id, String name, String type, String crea registerResourceRequest.setAncestor(ancestor); } EsbResp> esbResp = requestIamApi( - HttpPost.METHOD_NAME, + HttpMethodEnum.POST, API_REGISTER_RESOURCE_URL, registerResourceRequest, new TypeReference>>() { @@ -126,14 +136,13 @@ public EsbIamAuthedPolicy authByPath( EsbIamSubject esbIamSubject, List esbIamResources ) { - AuthByPathReq authByPathReq = - makeBaseReqByWeb(AuthByPathReq.class, null, "admin", "superadmin"); + AuthByPathReq authByPathReq = EsbReq.buildRequest(AuthByPathReq.class, null); authByPathReq.setAction(esbIamAction); authByPathReq.setSubject(esbIamSubject); authByPathReq.setResources(esbIamResources); EsbResp esbResp = requestIamApi( - HttpPost.METHOD_NAME, + HttpMethodEnum.POST, API_AUTH_BY_PATH_URL, authByPathReq, new TypeReference>() { @@ -149,13 +158,13 @@ public List batchAuthByPath( Long expiredAt ) { BatchAuthByPathReq batchAuthByPathReq = - makeBaseReqByWeb(BatchAuthByPathReq.class, null, "admin", "superadmin"); + EsbReq.buildRequest(BatchAuthByPathReq.class, null); batchAuthByPathReq.setActions(esbIamActions); batchAuthByPathReq.setSubject(esbIamSubject); batchAuthByPathReq.setResources(esbIamBatchPathResources); batchAuthByPathReq.setExpiredAt(expiredAt); EsbResp> esbResp = requestIamApi( - HttpPost.METHOD_NAME, + HttpMethodEnum.POST, API_BATCH_AUTH_BY_PATH_URL, batchAuthByPathReq, new TypeReference>>() { @@ -173,23 +182,25 @@ public List batchAuthByPath( * @param 泛型：返回值类型 * @return 返回值类型实例 */ - private EsbResp requestIamApi(String method, + private EsbResp requestIamApi(HttpMethodEnum method, String uri, EsbReq reqBody, TypeReference> typeReference) { try { - HttpMetricUtil.setHttpMetricName(CommonMetricNames.ESB_IAM_API_HTTP); + HttpMetricUtil.setHttpMetricName(CommonMetricNames.IAM_API_HTTP); HttpMetricUtil.addTagForCurrentMetric(Tag.of("api_name", uri)); - return getEsbRespByReq(method, uri, reqBody, typeReference); + OpenApiRequestInfo requestInfo = OpenApiRequestInfo + .builder() + .method(method) + .uri(uri) + .body(reqBody) + .authorization(authorization) + .build(); + return doRequest(requestInfo, typeReference); } catch (Exception e) { throw new InternalIamException(e, ErrorCode.IAM_API_DATA_ERROR, null); } finally { HttpMetricUtil.clearHttpMetric(); } } - - @Override - protected String buildPostBody(T params) { - return JsonUtils.toNonEmptyJson(params); - } } diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/client/IIamClient.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/client/IIamClient.java index bfdc47ac73..c10195e45c 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/client/IIamClient.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/client/IIamClient.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/config/EsbConfiguration.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/config/EsbConfiguration.java deleted file mode 100644 index fe9d392a58..0000000000 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/config/EsbConfiguration.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.iam.config; - -import lombok.AllArgsConstructor; -import lombok.Data; - -/** - * @since 16/6/2020 22:00 - */ -@Data -@AllArgsConstructor -public class EsbConfiguration { - private String esbUrl; - private boolean useEsbTestEnv; -} diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/config/IamAspectConfiguration.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/config/IamAspectConfiguration.java new file mode 100644 index 0000000000..33994437d2 --- /dev/null +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/config/IamAspectConfiguration.java @@ -0,0 +1,63 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.iam.config; + +import com.tencent.bk.job.common.iam.aspect.IamAppTransferAspect; +import com.tencent.bk.job.common.iam.aspect.IamCallbackAspect; +import com.tencent.bk.job.common.iam.aspect.IamExceptionHandleAspect; +import com.tencent.bk.job.common.iam.aspect.IamMetricsAspect; +import com.tencent.bk.job.common.service.AppScopeMappingService; +import io.micrometer.core.instrument.MeterRegistry; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * Iam 切面配置 + */ +@Configuration(proxyBeanMethods = false) +public class IamAspectConfiguration { + + @Bean + public IamCallbackAspect iamCallbackAspect() { + return new IamCallbackAspect(); + } + + @Bean + public IamAppTransferAspect iamAppTransferAspect( + ObjectProvider appScopeMappingServiceProvider) { + return new IamAppTransferAspect(appScopeMappingServiceProvider.getIfAvailable()); + } + + @Bean + public IamExceptionHandleAspect iamExceptionHandleAspect() { + return new IamExceptionHandleAspect(); + } + + @Bean + public IamMetricsAspect iamMetricsAspect(MeterRegistry meterRegistry) { + return new IamMetricsAspect(meterRegistry); + } +} diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/config/IamAutoConfiguration.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/config/IamAutoConfiguration.java new file mode 100644 index 0000000000..5d784d9b91 --- /dev/null +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/config/IamAutoConfiguration.java @@ -0,0 +1,138 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.iam.config; + +import com.tencent.bk.job.common.esb.config.AppProperties; +import com.tencent.bk.job.common.esb.config.EsbProperties; +import com.tencent.bk.job.common.i18n.service.MessageI18nService; +import com.tencent.bk.job.common.iam.http.IamHttpClientServiceImpl; +import com.tencent.bk.job.common.iam.service.AppAuthService; +import com.tencent.bk.job.common.iam.service.AuthService; +import com.tencent.bk.job.common.iam.service.BusinessAuthService; +import com.tencent.bk.job.common.iam.service.WebAuthService; +import com.tencent.bk.job.common.iam.service.impl.AppAuthServiceImpl; +import com.tencent.bk.job.common.iam.service.impl.AuthServiceImpl; +import com.tencent.bk.job.common.iam.service.impl.BusinessAuthServiceImpl; +import com.tencent.bk.job.common.iam.service.impl.WebAuthServiceImpl; +import com.tencent.bk.job.common.iam.util.BusinessAuthHelper; +import com.tencent.bk.sdk.iam.config.IamConfiguration; +import com.tencent.bk.sdk.iam.helper.AuthHelper; +import com.tencent.bk.sdk.iam.service.HttpClientService; +import com.tencent.bk.sdk.iam.service.PolicyService; +import com.tencent.bk.sdk.iam.service.TokenService; +import com.tencent.bk.sdk.iam.service.TopoPathService; +import com.tencent.bk.sdk.iam.service.impl.PolicyServiceImpl; +import com.tencent.bk.sdk.iam.service.impl.TokenServiceImpl; +import io.micrometer.core.instrument.MeterRegistry; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Import; + +@Import({IamAspectConfiguration.class, IamInterceptorConfiguration.class}) +@EnableConfigurationProperties(JobIamProperties.class) +public class IamAutoConfiguration { + + @Bean + public IamConfiguration iamConfiguration(AppProperties appProperties, JobIamProperties jobIamProperties) { + return new IamConfiguration(jobIamProperties.getSystemId(), appProperties.getCode(), + appProperties.getSecret(), jobIamProperties.getBaseUrl()); + } + + + @Bean + public HttpClientService httpClientService(IamConfiguration iamConfiguration) { + return new IamHttpClientServiceImpl(iamConfiguration); + } + + @Bean + public PolicyService policyService(IamConfiguration iamConfiguration, + HttpClientService httpClientService) { + return new PolicyServiceImpl(iamConfiguration, httpClientService); + } + + @Bean + public TokenService tokenService(IamConfiguration iamConfiguration, + HttpClientService httpClientService) { + return new TokenServiceImpl(iamConfiguration, httpClientService); + } + + @Bean + public AuthHelper authHelper(IamConfiguration iamConfiguration, + TokenService tokenService, + PolicyService policyService, + @Autowired(required = false) TopoPathService topoPathService) { + return new AuthHelper(tokenService, policyService, topoPathService, iamConfiguration); + } + + @Bean + public BusinessAuthHelper businessAuthHelper(IamConfiguration iamConfiguration, + TokenService tokenService, + PolicyService policyService, + @Autowired(required = false) TopoPathService topoPathService) { + return new BusinessAuthHelper(tokenService, policyService, topoPathService, iamConfiguration); + } + + @Bean + public WebAuthService webAuthService(MessageI18nService i18nService, AuthService authService) { + return new WebAuthServiceImpl(i18nService, authService); + } + + @Bean + public BusinessAuthService businessAuthService(AppAuthService appAuthService) { + return new BusinessAuthServiceImpl(appAuthService); + } + + @Bean + public AuthService authService(AuthHelper authHelper, + IamConfiguration iamConfiguration, + EsbProperties esbProperties, + MessageI18nService i18nService, + ObjectProvider meterRegistryObjectProvider) { + return new AuthServiceImpl(authHelper, iamConfiguration, esbProperties, i18nService, + meterRegistryObjectProvider.getIfAvailable()); + } + + @Bean + public AppAuthService appAuthService(AuthHelper authHelper, + BusinessAuthHelper businessAuthHelper, + IamConfiguration iamConfiguration, + PolicyService policyService, + JobIamProperties jobIamProperties, + EsbProperties esbProperties, + ObjectProvider meterRegistryObjectProvider) { + return new AppAuthServiceImpl( + authHelper, + businessAuthHelper, + iamConfiguration, + policyService, + jobIamProperties, + esbProperties, + meterRegistryObjectProvider.getIfAvailable() + ); + } + +} diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/config/IamInterceptorConfiguration.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/config/IamInterceptorConfiguration.java new file mode 100644 index 0000000000..e8e7e58dc4 --- /dev/null +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/config/IamInterceptorConfiguration.java @@ -0,0 +1,49 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.iam.config; + +import com.tencent.bk.job.common.iam.interceptor.AuthAppInterceptor; +import com.tencent.bk.job.common.iam.interceptor.JobIamInterceptor; +import com.tencent.bk.job.common.iam.service.BusinessAuthService; +import com.tencent.bk.sdk.iam.helper.AuthHelper; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * IAM Interceptor 配置 + */ +@Configuration(proxyBeanMethods = false) +public class IamInterceptorConfiguration { + + @Bean + public AuthAppInterceptor authAppInterceptor(BusinessAuthService businessAuthService) { + return new AuthAppInterceptor(businessAuthService); + } + + @Bean + public JobIamInterceptor jobIamInterceptor(AuthHelper authHelper) { + return new JobIamInterceptor(authHelper); + } +} diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/config/JobIamAutoConfiguration.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/config/JobIamAutoConfiguration.java deleted file mode 100644 index 50e615378c..0000000000 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/config/JobIamAutoConfiguration.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.iam.config; - -import com.tencent.bk.job.common.iam.http.IamHttpClientServiceImpl; -import com.tencent.bk.job.common.iam.util.BusinessAuthHelper; -import com.tencent.bk.sdk.iam.config.IamConfiguration; -import com.tencent.bk.sdk.iam.helper.AuthHelper; -import com.tencent.bk.sdk.iam.service.HttpClientService; -import com.tencent.bk.sdk.iam.service.PolicyService; -import com.tencent.bk.sdk.iam.service.TokenService; -import com.tencent.bk.sdk.iam.service.impl.PolicyServiceImpl; -import com.tencent.bk.sdk.iam.service.impl.TokenServiceImpl; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.DependsOn; - -/** - * @since 28/5/2020 17:42 - */ -@Configuration -public class JobIamAutoConfiguration { - - /** - * 权限中心对应的系统 ID - */ - @Value("${iam.system-id:}") - private String systemId; - - /** - * ESB 分配的系统 App Code - */ - @Value("${app.code:}") - private String appCode; - - /** - * ESB 分配的系统 App Secret - */ - @Value("${app.secret:}") - private String appSecret; - - /** - * 权限中心的访问地址 - */ - @Value("${iam.base-url:}") - private String iamBaseUrl; - - /** - * ESB API 的访问地址 - */ - @Value("${esb.service.url:}") - private String esbUrl; - - @Value("${esb.use.test.env:false}") - private boolean useEsbTestEnv; - - @Bean - public IamConfiguration iamConfiguration() { - return new IamConfiguration(systemId, appCode, appSecret, iamBaseUrl); - } - - @Bean - public EsbConfiguration esbConfiguration() { - return new EsbConfiguration(esbUrl, useEsbTestEnv); - } - - @Bean - @DependsOn("httpConfigSetter") - public HttpClientService httpClientService() { - return new IamHttpClientServiceImpl(iamConfiguration()); - } - - @Bean - public PolicyService policyService() { - return new PolicyServiceImpl(iamConfiguration(), httpClientService()); - } - - @Bean - public TokenService tokenService() { - return new TokenServiceImpl(iamConfiguration(), httpClientService()); - } - - @Bean - public AuthHelper authHelper() { - return new AuthHelper(tokenService(), policyService(), iamConfiguration()); - } - - @Bean - public BusinessAuthHelper businessAuthHelper() { - return new BusinessAuthHelper(tokenService(), policyService(), iamConfiguration()); - } -} diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/config/JobIamProperties.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/config/JobIamProperties.java new file mode 100644 index 0000000000..cdadbf3ebc --- /dev/null +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/config/JobIamProperties.java @@ -0,0 +1,54 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.iam.config; + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.springframework.boot.context.properties.ConfigurationProperties; + +/** + * Job IAM 配置 + */ +@Getter +@Setter +@ToString +@ConfigurationProperties(prefix = "iam") +public class JobIamProperties { + + /** + * 权限中心对应的系统 ID + */ + private String systemId; + /** + * 权限中心的后台接口访问地址 + */ + private String baseUrl; + /** + * 权限中心的页面访问地址 + */ + private String webUrl; + +} diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/constant/ActionId.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/constant/ActionId.java index a8b479a920..5baa0e0b98 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/constant/ActionId.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/constant/ActionId.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/constant/ActionInfo.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/constant/ActionInfo.java index 1a980e5a81..0519fb31ee 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/constant/ActionInfo.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/constant/ActionInfo.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/constant/Actions.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/constant/Actions.java index 57f66aa318..176904a8c2 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/constant/Actions.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/constant/Actions.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/constant/ResourceTypeEnum.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/constant/ResourceTypeEnum.java index e3a81fc49a..6c87f17e8c 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/constant/ResourceTypeEnum.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/constant/ResourceTypeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/constant/ResourceTypeId.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/constant/ResourceTypeId.java index 53f0fc2f12..ed10b11502 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/constant/ResourceTypeId.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/constant/ResourceTypeId.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/AppResourceScopeResult.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/AppResourceScopeResult.java index 591d1f44f5..b7ba5570aa 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/AppResourceScopeResult.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/AppResourceScopeResult.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/AuthByPathReq.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/AuthByPathReq.java index d1b8fdb208..8b8ad0f243 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/AuthByPathReq.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/AuthByPathReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/BatchAuthByPathReq.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/BatchAuthByPathReq.java index c2383b7f9c..dae356c86e 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/BatchAuthByPathReq.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/BatchAuthByPathReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/EsbIamAction.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/EsbIamAction.java index c03fb6ae8c..084620405a 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/EsbIamAction.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/EsbIamAction.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/EsbIamAuthedPolicy.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/EsbIamAuthedPolicy.java index be3ed98990..1073d5fa46 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/EsbIamAuthedPolicy.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/EsbIamAuthedPolicy.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/EsbIamBatchAuthedPolicy.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/EsbIamBatchAuthedPolicy.java index d6ef61e03a..068482f184 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/EsbIamBatchAuthedPolicy.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/EsbIamBatchAuthedPolicy.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/EsbIamBatchPathResource.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/EsbIamBatchPathResource.java index b295a6be44..ed470af81a 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/EsbIamBatchPathResource.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/EsbIamBatchPathResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/EsbIamPathItem.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/EsbIamPathItem.java index 90e4adeb29..e22cab9b59 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/EsbIamPathItem.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/EsbIamPathItem.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/EsbIamResource.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/EsbIamResource.java index 4bca5e27ba..189812cbab 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/EsbIamResource.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/EsbIamResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/EsbIamSubject.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/EsbIamSubject.java index b7a0530285..031e4c6d7f 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/EsbIamSubject.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/EsbIamSubject.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/GetApplyUrlRequest.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/GetApplyUrlRequest.java index d4dc856aea..1c46dd0423 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/GetApplyUrlRequest.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/GetApplyUrlRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/GetApplyUrlResponse.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/GetApplyUrlResponse.java index e51a98b6e7..4c76990637 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/GetApplyUrlResponse.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/GetApplyUrlResponse.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/RegisterResourceRequest.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/RegisterResourceRequest.java index 06e7ce578d..2cd2f8210c 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/RegisterResourceRequest.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/dto/RegisterResourceRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/exception/PermissionDeniedException.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/exception/PermissionDeniedException.java index d868be1c50..67816e1c19 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/exception/PermissionDeniedException.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/exception/PermissionDeniedException.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/http/IamHttpClientServiceImpl.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/http/IamHttpClientServiceImpl.java index 1158e47bde..adc0490bde 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/http/IamHttpClientServiceImpl.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/http/IamHttpClientServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,11 +25,13 @@ package com.tencent.bk.job.common.iam.http; import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.constant.HttpMethodEnum; import com.tencent.bk.job.common.exception.InternalIamException; import com.tencent.bk.job.common.metrics.CommonMetricNames; -import com.tencent.bk.job.common.util.http.ExtHttpHelper; +import com.tencent.bk.job.common.util.http.HttpHelper; import com.tencent.bk.job.common.util.http.HttpHelperFactory; import com.tencent.bk.job.common.util.http.HttpMetricUtil; +import com.tencent.bk.job.common.util.http.HttpRequest; import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.sdk.iam.config.IamConfiguration; import com.tencent.bk.sdk.iam.constants.HttpHeader; @@ -39,14 +41,11 @@ import org.apache.http.Header; import org.apache.http.message.BasicHeader; -import java.util.ArrayList; -import java.util.List; - @Slf4j public class IamHttpClientServiceImpl implements HttpClientService { private final String DEFAULT_CHARSET = "UTF-8"; - private final ExtHttpHelper httpHelper = HttpHelperFactory.getDefaultHttpHelper(); + private final HttpHelper httpHelper = HttpHelperFactory.getDefaultHttpHelper(); private final IamConfiguration iamConfiguration; public IamHttpClientServiceImpl(IamConfiguration iamConfiguration) { @@ -59,7 +58,11 @@ public String doHttpGet(String uri) { try { HttpMetricUtil.setHttpMetricName(CommonMetricNames.IAM_API_HTTP); HttpMetricUtil.addTagForCurrentMetric(Tag.of("api_name", uri)); - return httpHelper.get(buildUrl(uri), buildAuthHeader()); + return httpHelper.requestForSuccessResp( + HttpRequest.builder(HttpMethodEnum.GET, buildUrl(uri)) + .setHeaders(buildAuthHeader()) + .build()) + .getEntity(); } catch (Exception e) { throw new InternalIamException(e, ErrorCode.IAM_API_DATA_ERROR, null); } finally { @@ -72,7 +75,12 @@ public String doHttpPost(String uri, Object body) { try { HttpMetricUtil.setHttpMetricName(CommonMetricNames.IAM_API_HTTP); HttpMetricUtil.addTagForCurrentMetric(Tag.of("api_name", uri)); - return httpHelper.post(buildUrl(uri), DEFAULT_CHARSET, JsonUtils.toJson(body), buildAuthHeader()); + return httpHelper.requestForSuccessResp( + HttpRequest.builder(HttpMethodEnum.POST, buildUrl(uri)) + .setHeaders(buildAuthHeader()) + .setStringEntity(JsonUtils.toJson(body)) + .build()) + .getEntity(); } catch (Exception e) { log.error("Fail to request IAM", e); throw new InternalIamException(e, ErrorCode.IAM_API_DATA_ERROR, null); @@ -86,7 +94,12 @@ public String doHttpPut(String uri, Object body) { try { HttpMetricUtil.setHttpMetricName(CommonMetricNames.IAM_API_HTTP); HttpMetricUtil.addTagForCurrentMetric(Tag.of("api_name", uri)); - return httpHelper.put(buildUrl(uri), DEFAULT_CHARSET, JsonUtils.toJson(body), buildAuthHeader()); + return httpHelper.requestForSuccessResp( + HttpRequest.builder(HttpMethodEnum.PUT, buildUrl(uri)) + .setHeaders(buildAuthHeader()) + .setStringEntity(JsonUtils.toJson(body)) + .build()) + .getEntity(); } catch (Exception e) { log.error("Fail to request IAM", e); throw new InternalIamException(e, ErrorCode.IAM_API_DATA_ERROR, null); @@ -100,7 +113,11 @@ public String doHttpDelete(String uri) { try { HttpMetricUtil.setHttpMetricName(CommonMetricNames.IAM_API_HTTP); HttpMetricUtil.addTagForCurrentMetric(Tag.of("api_name", uri)); - return httpHelper.delete(buildUrl(uri), buildAuthHeader()); + return httpHelper.requestForSuccessResp( + HttpRequest.builder(HttpMethodEnum.DELETE, buildUrl(uri)) + .setHeaders(buildAuthHeader()) + .build()) + .getEntity(); } catch (Exception e) { throw new InternalIamException(e, ErrorCode.IAM_API_DATA_ERROR, null); } finally { @@ -112,11 +129,11 @@ private String buildUrl(String uri) { return iamConfiguration.getIamBaseUrl() + uri; } - private List
buildAuthHeader() { - List
headerList = new ArrayList<>(); - headerList.add(new BasicHeader(HttpHeader.BK_APP_CODE, iamConfiguration.getAppCode())); - headerList.add(new BasicHeader(HttpHeader.BK_APP_SECRET, iamConfiguration.getAppSecret())); - return headerList; + private Header[] buildAuthHeader() { + Header[] headers = new Header[2]; + headers[0] = new BasicHeader(HttpHeader.BK_APP_CODE, iamConfiguration.getAppCode()); + headers[1] = new BasicHeader(HttpHeader.BK_APP_SECRET, iamConfiguration.getAppSecret()); + return headers; } } diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/interceptor/AuthAppInterceptor.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/interceptor/AuthAppInterceptor.java index 5b4f11434a..00914721d0 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/interceptor/AuthAppInterceptor.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/interceptor/AuthAppInterceptor.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB 蓝鲸作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB 蓝鲸作业平台 is licensed under the MIT license. * @@ -27,6 +27,8 @@ package com.tencent.bk.job.common.iam.interceptor; +import com.tencent.bk.job.common.annotation.JobInterceptor; +import com.tencent.bk.job.common.constant.InterceptorOrder; import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; import com.tencent.bk.job.common.iam.model.AuthResult; import com.tencent.bk.job.common.iam.service.BusinessAuthService; @@ -34,8 +36,6 @@ import com.tencent.bk.job.common.util.JobContextUtil; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Pair; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; import org.springframework.web.servlet.ModelAndView; import org.springframework.web.servlet.handler.HandlerInterceptorAdapter; @@ -43,12 +43,12 @@ import javax.servlet.http.HttpServletResponse; @Slf4j -@Component +@JobInterceptor(pathPatterns = {"/web/**", "/esb/api/**"}, + order = InterceptorOrder.AUTH.AUTH_GLOBAL) public class AuthAppInterceptor extends HandlerInterceptorAdapter { private final BusinessAuthService businessAuthService; - @Autowired public AuthAppInterceptor(BusinessAuthService businessAuthService) { this.businessAuthService = businessAuthService; } diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/interceptor/JobIamInterceptor.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/interceptor/JobIamInterceptor.java index 91ef6a5abd..3734fdd368 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/interceptor/JobIamInterceptor.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/interceptor/JobIamInterceptor.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB 蓝鲸作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB 蓝鲸作业平台 is licensed under the MIT license. * @@ -27,10 +27,10 @@ package com.tencent.bk.job.common.iam.interceptor; +import com.tencent.bk.job.common.annotation.JobInterceptor; +import com.tencent.bk.job.common.constant.InterceptorOrder; import com.tencent.bk.sdk.iam.helper.AuthHelper; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; import org.springframework.web.servlet.ModelAndView; import org.springframework.web.servlet.handler.HandlerInterceptorAdapter; @@ -38,12 +38,12 @@ import javax.servlet.http.HttpServletResponse; @Slf4j -@Component +@JobInterceptor(pathPatterns = {"/iam/api/v1/resources/**"}, + order = InterceptorOrder.Init.CHECK_VALID) public class JobIamInterceptor extends HandlerInterceptorAdapter { - private AuthHelper authHelper; + private final AuthHelper authHelper; - @Autowired public JobIamInterceptor(AuthHelper authHelper) { this.authHelper = authHelper; } diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/metrics/MetricsConstants.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/metrics/MetricsConstants.java new file mode 100644 index 0000000000..f1184526d5 --- /dev/null +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/metrics/MetricsConstants.java @@ -0,0 +1,42 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.iam.metrics; + +public class MetricsConstants { + + // metric name + public static final String NAME_AUTH_HELPER_IS_ALLOW = "AuthHelper.isAllow"; + + // tag + public static final String TAG_KEY_ACTION = "action"; + public static final String TAG_KEY_STATUS = "status"; + + // value + public static final String TAG_VALUE_ACTION_NONE = "none"; + public static final String TAG_VALUE_RESULT_TRUE = "true"; + public static final String TAG_VALUE_RESULT_FALSE = "false"; + public static final String TAG_VALUE_RESULT_ERROR = "error"; + +} diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/model/ActionAuthRequest.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/model/ActionAuthRequest.java index 9a18e53e77..f12cca361e 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/model/ActionAuthRequest.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/model/ActionAuthRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/model/AuthResult.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/model/AuthResult.java index 06dbbeb7bf..afc9e91153 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/model/AuthResult.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/model/AuthResult.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,6 +25,7 @@ package com.tencent.bk.job.common.iam.model; import com.tencent.bk.job.common.iam.constant.ResourceTypeEnum; +import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; import com.tencent.bk.job.common.model.iam.AuthResultDTO; import com.tencent.bk.job.common.model.iam.PathInfoDTO; import com.tencent.bk.job.common.model.iam.PermissionActionResourceDTO; @@ -301,7 +302,7 @@ public static AuthResult fromAuthResultDTO(AuthResultDTO authResultDTO) { PermissionResourceGroup resourceGroup = new PermissionResourceGroup(); resourceGroup.setResourceType(ResourceTypeEnum.getByResourceTypeId(resourceGroupDTO.getResourceType())); resourceGroup.setSystemId(resourceGroupDTO.getSystemId()); - if (CollectionUtils.isNotEmpty(resourceGroup.getPermissionResources())) { + if (CollectionUtils.isNotEmpty(resourceGroupDTO.getPermissionResources())) { List resources = new ArrayList<>(); for (PermissionResourceDTO resourceDTO : resourceGroupDTO.getPermissionResources()) { resources.add(toPermissionResource(resourceDTO)); @@ -350,4 +351,13 @@ private static com.tencent.bk.sdk.iam.dto.PathInfoDTO toIamPathInfoDTO(PathInfoD return pathBuilder.build(); } + /** + * 处理鉴权结果。如果鉴权不通过，抛出 PermissionDeniedException + */ + public void denyIfNoPermission() throws PermissionDeniedException { + if (!isPass()) { + throw new PermissionDeniedException(this); + } + } + } diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/model/PermissionActionResource.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/model/PermissionActionResource.java index e42decc3db..17e710b963 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/model/PermissionActionResource.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/model/PermissionActionResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/model/PermissionResource.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/model/PermissionResource.java index 3a8290fa05..2481d57c49 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/model/PermissionResource.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/model/PermissionResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -40,11 +40,11 @@ public class PermissionResource { */ private String systemId; /** - * 资源类型 + * 资源类型，用于构造资源路径判断是否有权限 */ private ResourceTypeEnum resourceType; /** - * 子资源类型。比如cmdb主机，具有静态主机、动态topo、动态分组三种子类型。可取值：host/topo/dynamic_group + * 子资源类型。比如cmdb主机，具有静态主机、动态topo、动态分组三种子类型 */ private String subResourceType; /** @@ -52,7 +52,7 @@ public class PermissionResource { */ private String resourceId; /** - * iam path + * iam path，用于构造资源路径判断是否有权限 */ private PathInfoDTO pathInfo; /** @@ -60,12 +60,12 @@ public class PermissionResource { */ private String resourceName; /** - * 层级节点的资源类型 + * 层级节点的资源类型，用于构造获取权限申请链接的请求数据，优先于resourceType */ private String type; /** - * 层级节点（比如CMDB的主机)的父资源 + * 层级节点（比如CMDB的主机)的父资源，用于构造获取权限申请链接的请求数据 */ private List parentHierarchicalResources; diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/model/PermissionResourceGroup.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/model/PermissionResourceGroup.java index bd86cb2c5a..6bd868490f 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/model/PermissionResourceGroup.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/model/PermissionResourceGroup.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/service/AppAuthService.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/service/AppAuthService.java index ed4f0515ba..cb0f0ea370 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/service/AppAuthService.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/service/AppAuthService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/service/AuthService.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/service/AuthService.java index efdb8e83ce..4a6a61cb2d 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/service/AuthService.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/service/AuthService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/service/BaseIamCallbackService.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/service/BaseIamCallbackService.java index 585bbb51a4..35181f8677 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/service/BaseIamCallbackService.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/service/BaseIamCallbackService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -34,6 +34,7 @@ import com.tencent.bk.sdk.iam.dto.callback.request.CallbackRequestDTO; import com.tencent.bk.sdk.iam.dto.callback.request.IamSearchCondition; import com.tencent.bk.sdk.iam.dto.callback.response.CallbackBaseResponseDTO; +import com.tencent.bk.sdk.iam.dto.callback.response.FetchResourceTypeSchemaResponseDTO; import com.tencent.bk.sdk.iam.dto.callback.response.ListAttributeResponseDTO; import com.tencent.bk.sdk.iam.dto.callback.response.ListAttributeValueResponseDTO; import com.tencent.bk.sdk.iam.dto.callback.response.ListInstanceByPolicyResponseDTO; @@ -105,6 +106,9 @@ public PathInfoDTO getPathNodeByAppId(Long appId, Map appId protected abstract CallbackBaseResponseDTO fetchInstanceResp(CallbackRequestDTO callbackRequest); + protected abstract FetchResourceTypeSchemaResponseDTO fetchResourceTypeSchemaResp( + CallbackRequestDTO callbackRequest); + public CallbackBaseResponseDTO baseCallback(CallbackRequestDTO callbackRequest) { CallbackBaseResponseDTO response; switch (callbackRequest.getMethod()) { @@ -129,6 +133,9 @@ public CallbackBaseResponseDTO baseCallback(CallbackRequestDTO callbackRequest) case SEARCH_INSTANCE: response = searchInstanceResp(callbackRequest); break; + case FETCH_RESOURCE_TYPE_SCHEMA: + response = fetchResourceTypeSchemaResp(callbackRequest); + break; default: log.error("Unknown callback method!|{}|{}|{}|{}", callbackRequest.getMethod(), callbackRequest.getType(), callbackRequest.getFilter(), callbackRequest.getPage()); diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/service/BusinessAuthService.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/service/BusinessAuthService.java index 9bf00afc60..81e97de5c1 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/service/BusinessAuthService.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/service/BusinessAuthService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/service/ResourceNameQueryService.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/service/ResourceNameQueryService.java index b9d54c3844..dff67feba7 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/service/ResourceNameQueryService.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/service/ResourceNameQueryService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/service/WebAuthService.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/service/WebAuthService.java index 75a19b54c3..d6332f3260 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/service/WebAuthService.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/service/WebAuthService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/service/impl/AppAuthServiceImpl.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/service/impl/AppAuthServiceImpl.java index 5e05c30c6a..10b64bd4da 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/service/impl/AppAuthServiceImpl.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/service/impl/AppAuthServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,8 +25,10 @@ package com.tencent.bk.job.common.iam.service.impl; import com.tencent.bk.job.common.constant.ResourceScopeTypeEnum; +import com.tencent.bk.job.common.esb.config.AppProperties; +import com.tencent.bk.job.common.esb.config.EsbProperties; import com.tencent.bk.job.common.iam.client.EsbIamClient; -import com.tencent.bk.job.common.iam.config.EsbConfiguration; +import com.tencent.bk.job.common.iam.config.JobIamProperties; import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.iam.constant.ResourceTypeEnum; import com.tencent.bk.job.common.iam.constant.ResourceTypeId; @@ -48,12 +50,11 @@ import com.tencent.bk.sdk.iam.dto.resource.RelatedResourceTypeDTO; import com.tencent.bk.sdk.iam.helper.AuthHelper; import com.tencent.bk.sdk.iam.service.PolicyService; +import io.micrometer.core.instrument.MeterRegistry; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.slf4j.helpers.FormattingTuple; import org.slf4j.helpers.MessageFormatter; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; import java.util.ArrayList; import java.util.Collections; @@ -62,24 +63,29 @@ import java.util.stream.Collectors; @Slf4j -@Service public class AppAuthServiceImpl extends BasicAuthService implements AppAuthService { private final AuthHelper authHelper; private final BusinessAuthHelper businessAuthHelper; private final PolicyService policyService; + private final JobIamProperties jobIamProperties; private final EsbIamClient iamClient; private ResourceNameQueryService resourceNameQueryService; - public AppAuthServiceImpl(@Autowired AuthHelper authHelper, - @Autowired BusinessAuthHelper businessAuthHelper, - @Autowired IamConfiguration iamConfiguration, - @Autowired PolicyService policyService, - @Autowired EsbConfiguration esbConfiguration) { + public AppAuthServiceImpl(AuthHelper authHelper, + BusinessAuthHelper businessAuthHelper, + IamConfiguration iamConfiguration, + PolicyService policyService, + JobIamProperties jobIamProperties, + EsbProperties esbProperties, + MeterRegistry meterRegistry) { this.authHelper = authHelper; this.businessAuthHelper = businessAuthHelper; this.policyService = policyService; - this.iamClient = new EsbIamClient(esbConfiguration.getEsbUrl(), iamConfiguration.getAppCode(), - iamConfiguration.getAppSecret(), esbConfiguration.isUseEsbTestEnv()); + this.jobIamProperties = jobIamProperties; + this.iamClient = new EsbIamClient( + meterRegistry, + new AppProperties(iamConfiguration.getAppCode(), iamConfiguration.getAppSecret()), + esbProperties); } @Override @@ -271,19 +277,18 @@ private List buildAppResourceScopeInstanceList(AppResourceScope app @Override public String getBusinessApplyUrl(AppResourceScope appResourceScope) { + if (appResourceScope == null) { + return jobIamProperties.getWebUrl(); + } ActionDTO action = new ActionDTO(); action.setId(ActionId.ACCESS_BUSINESS); List relatedResourceTypes = new ArrayList<>(); RelatedResourceTypeDTO businessResourceTypeDTO = new RelatedResourceTypeDTO(); businessResourceTypeDTO.setType(ResourceTypeEnum.BUSINESS.getId()); businessResourceTypeDTO.setSystemId(SystemId.CMDB); - if (appResourceScope != null) { - List instanceDTOList = new ArrayList<>(); - instanceDTOList.add(buildInstance(appResourceScope)); - businessResourceTypeDTO.setInstance(Collections.singletonList(instanceDTOList)); - } else { - businessResourceTypeDTO.setInstance(Collections.emptyList()); - } + List instanceDTOList = new ArrayList<>(); + instanceDTOList.add(buildInstance(appResourceScope)); + businessResourceTypeDTO.setInstance(Collections.singletonList(instanceDTOList)); relatedResourceTypes.add(businessResourceTypeDTO); action.setRelatedResourceTypes(relatedResourceTypes); return iamClient.getApplyUrl(Collections.singletonList(action)); diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/service/impl/AuthServiceImpl.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/service/impl/AuthServiceImpl.java index c764591b3d..df2f4ac137 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/service/impl/AuthServiceImpl.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/service/impl/AuthServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,6 +25,8 @@ package com.tencent.bk.job.common.iam.service.impl; import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.esb.config.AppProperties; +import com.tencent.bk.job.common.esb.config.EsbProperties; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.common.esb.model.iam.EsbActionDTO; import com.tencent.bk.job.common.esb.model.iam.EsbApplyPermissionDTO; @@ -33,7 +35,6 @@ import com.tencent.bk.job.common.exception.InternalException; import com.tencent.bk.job.common.i18n.service.MessageI18nService; import com.tencent.bk.job.common.iam.client.EsbIamClient; -import com.tencent.bk.job.common.iam.config.EsbConfiguration; import com.tencent.bk.job.common.iam.constant.ActionInfo; import com.tencent.bk.job.common.iam.constant.Actions; import com.tencent.bk.job.common.iam.constant.ResourceTypeEnum; @@ -53,11 +54,10 @@ import com.tencent.bk.sdk.iam.dto.resource.RelatedResourceTypeDTO; import com.tencent.bk.sdk.iam.dto.resource.ResourceDTO; import com.tencent.bk.sdk.iam.helper.AuthHelper; +import io.micrometer.core.instrument.MeterRegistry; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; import java.util.ArrayList; import java.util.Collections; @@ -68,20 +68,23 @@ import java.util.stream.Collectors; @Slf4j -@Service public class AuthServiceImpl extends BasicAuthService implements AuthService { private final AuthHelper authHelper; private final EsbIamClient iamClient; private final MessageI18nService i18nService; private ResourceNameQueryService resourceNameQueryService; - public AuthServiceImpl(@Autowired AuthHelper authHelper, - @Autowired IamConfiguration iamConfiguration, - @Autowired EsbConfiguration esbConfiguration, MessageI18nService i18nService) { + public AuthServiceImpl(AuthHelper authHelper, + IamConfiguration iamConfiguration, + EsbProperties esbProperties, + MessageI18nService i18nService, + MeterRegistry meterRegistry) { this.authHelper = authHelper; this.i18nService = i18nService; - this.iamClient = new EsbIamClient(esbConfiguration.getEsbUrl(), iamConfiguration.getAppCode(), - iamConfiguration.getAppSecret(), esbConfiguration.isUseEsbTestEnv()); + this.iamClient = new EsbIamClient( + meterRegistry, + new AppProperties(iamConfiguration.getAppCode(), iamConfiguration.getAppSecret()), + esbProperties); } @Override diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/service/impl/BasicAuthService.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/service/impl/BasicAuthService.java index eff903b45f..95573f9d94 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/service/impl/BasicAuthService.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/service/impl/BasicAuthService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/service/impl/BusinessAuthServiceImpl.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/service/impl/BusinessAuthServiceImpl.java index 95f4c80c7b..710e3b2e9f 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/service/impl/BusinessAuthServiceImpl.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/service/impl/BusinessAuthServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -30,19 +30,15 @@ import com.tencent.bk.job.common.iam.service.BusinessAuthService; import com.tencent.bk.job.common.model.dto.AppResourceScope; import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; /** * 账号相关操作鉴权接口 */ @Slf4j -@Service public class BusinessAuthServiceImpl implements BusinessAuthService { private final AppAuthService appAuthService; - @Autowired public BusinessAuthServiceImpl(AppAuthService appAuthService) { this.appAuthService = appAuthService; } diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/service/impl/WebAuthServiceImpl.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/service/impl/WebAuthServiceImpl.java index b54068eace..edf989f309 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/service/impl/WebAuthServiceImpl.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/service/impl/WebAuthServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -37,20 +37,16 @@ import com.tencent.bk.job.common.model.permission.RequiredPermissionVO; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; import java.util.ArrayList; import java.util.List; import java.util.StringJoiner; @Slf4j -@Service public class WebAuthServiceImpl implements WebAuthService { private final MessageI18nService i18nService; private final AuthService authService; - - @Autowired + public WebAuthServiceImpl(MessageI18nService i18nService, AuthService authService) { this.i18nService = i18nService; this.authService = authService; @@ -109,6 +105,7 @@ private String buildResourceName(PermissionResourceGroup resourceGroup) { int topoNodeCount = 0; int dynamicGroupCount = 0; int bizSetCount = 0; + int containerCount = 0; for (PermissionResource resource : resourceGroup.getPermissionResources()) { String subResourceType = resource.getSubResourceType(); switch (subResourceType) { @@ -124,6 +121,9 @@ private String buildResourceName(PermissionResourceGroup resourceGroup) { case "biz_set": bizSetCount++; break; + case "container": + containerCount++; + break; } } StringJoiner stringJoiner = new StringJoiner(","); @@ -139,6 +139,9 @@ private String buildResourceName(PermissionResourceGroup resourceGroup) { if (bizSetCount > 0) { stringJoiner.add(bizSetCount + " " + i18nService.getI18n("resource.biz_set.name")); } + if (containerCount > 0) { + stringJoiner.add(containerCount + " " + i18nService.getI18n("resource.container.name")); + } return stringJoiner.toString(); } else { StringJoiner stringJoiner = new StringJoiner(","); diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/util/BusinessAuthHelper.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/util/BusinessAuthHelper.java index 799157af95..7c06edd446 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/util/BusinessAuthHelper.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/util/BusinessAuthHelper.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -33,6 +33,7 @@ import com.tencent.bk.sdk.iam.helper.AuthHelper; import com.tencent.bk.sdk.iam.service.PolicyService; import com.tencent.bk.sdk.iam.service.TokenService; +import com.tencent.bk.sdk.iam.service.TopoPathService; import com.tencent.bk.sdk.iam.util.PathBuilder; import lombok.extern.slf4j.Slf4j; @@ -44,9 +45,11 @@ @Slf4j public class BusinessAuthHelper extends AuthHelper { - public BusinessAuthHelper(TokenService tokenService, PolicyService policyService, + public BusinessAuthHelper(TokenService tokenService, + PolicyService policyService, + TopoPathService topoPathService, IamConfiguration iamConfiguration) { - super(tokenService, policyService, iamConfiguration); + super(tokenService, policyService, topoPathService, iamConfiguration); } private InstanceDTO buildAppResourceScopeInstance(AppResourceScope appResourceScope) { diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/util/IamRespUtil.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/util/IamRespUtil.java index 11d4dba7b6..f74f05c51f 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/util/IamRespUtil.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/util/IamRespUtil.java @@ -19,7 +19,7 @@ private static List convertToInstanceList( List rawDataList, InstanceInfoDTOConverter converter ) { - return rawDataList.parallelStream().map(converter::convert).collect(Collectors.toList()); + return rawDataList.stream().map(converter::convert).collect(Collectors.toList()); } public static ListInstanceResponseDTO getListInstanceRespFromPageData( diff --git a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/util/IamUtil.java b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/util/IamUtil.java index 23d8b8ad85..4e36cfa66e 100644 --- a/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/util/IamUtil.java +++ b/src/backend/commons/common-iam/src/main/java/com/tencent/bk/job/common/iam/util/IamUtil.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-iam/src/main/resources/META-INF/spring.factories b/src/backend/commons/common-iam/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..16cce93bc4 --- /dev/null +++ b/src/backend/commons/common-iam/src/main/resources/META-INF/spring.factories @@ -0,0 +1,2 @@ +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ +com.tencent.bk.job.common.iam.config.IamAutoConfiguration diff --git a/src/backend/commons/common-jwt/build.gradle b/src/backend/commons/common-jwt/build.gradle new file mode 100644 index 0000000000..19f1fee6b8 --- /dev/null +++ b/src/backend/commons/common-jwt/build.gradle @@ -0,0 +1,29 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +dependencies { + api project(':commons:common-crypto') + api 'com.google.guava:guava' + api 'io.jsonwebtoken:jjwt' +} diff --git a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/jwt/BasicJwtManager.java b/src/backend/commons/common-jwt/src/main/java/com/tencent/bk/job/common/jwt/BasicJwtManager.java similarity index 87% rename from src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/jwt/BasicJwtManager.java rename to src/backend/commons/common-jwt/src/main/java/com/tencent/bk/job/common/jwt/BasicJwtManager.java index d6f3342a1c..4939012256 100644 --- a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/jwt/BasicJwtManager.java +++ b/src/backend/commons/common-jwt/src/main/java/com/tencent/bk/job/common/jwt/BasicJwtManager.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,19 +22,17 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.common.util.jwt; +package com.tencent.bk.job.common.jwt; import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; -import com.tencent.bk.job.common.util.crypto.RSAUtils; +import com.tencent.bk.job.common.crypto.util.RSAUtils; import io.jsonwebtoken.Claims; import io.jsonwebtoken.ExpiredJwtException; import io.jsonwebtoken.Jwts; import io.jsonwebtoken.SignatureAlgorithm; import lombok.extern.slf4j.Slf4j; -import java.io.IOException; -import java.security.GeneralSecurityException; import java.security.PrivateKey; import java.security.PublicKey; import java.time.Instant; @@ -51,17 +49,16 @@ public class BasicJwtManager implements JwtManager { private final Cache tokenCache = CacheBuilder.newBuilder() .maximumSize(9999).expireAfterWrite(5, TimeUnit.MINUTES).build(); - public BasicJwtManager(String privateKeyBase64, - String publicKeyBase64) throws IOException, GeneralSecurityException { + public BasicJwtManager(String privateKeyBase64, String publicKeyBase64) { this.privateKey = RSAUtils.getPrivateKey(privateKeyBase64); this.publicKey = RSAUtils.getPublicKey(publicKeyBase64); log.info("Init JwtManager successfully!"); } /** - * 获取JWT jwt token + * 获取 jwt token * - * @return + * @return jwt token */ @Override public String getToken() { @@ -73,7 +70,7 @@ public String getToken() { @Override public String generateToken(long expireMills) { - // token 超时10min + // token 超时2h long expireAt = System.currentTimeMillis() + expireMills; this.token = Jwts.builder().setSubject("job-service-auth").setExpiration(new Date(expireAt)) .signWith(SignatureAlgorithm.RS512, privateKey).compact(); @@ -81,15 +78,15 @@ public String generateToken(long expireMills) { } public String generateToken() { - // token 默认超时10min - return generateToken(10 * 60 * 1000); + // token 默认超时2h + return generateToken(120 * 60 * 1000); } /** * 验证JWT * * @param token jwt token - * @return + * @return 是否通过 */ @Override public boolean verifyJwt(String token) { diff --git a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/jwt/JwtManager.java b/src/backend/commons/common-jwt/src/main/java/com/tencent/bk/job/common/jwt/JwtManager.java similarity index 75% rename from src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/jwt/JwtManager.java rename to src/backend/commons/common-jwt/src/main/java/com/tencent/bk/job/common/jwt/JwtManager.java index edfbf16267..3af1eacfb1 100644 --- a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/jwt/JwtManager.java +++ b/src/backend/commons/common-jwt/src/main/java/com/tencent/bk/job/common/jwt/JwtManager.java @@ -1,4 +1,4 @@ -package com.tencent.bk.job.common.util.jwt; +package com.tencent.bk.job.common.jwt; public interface JwtManager { String getToken(); diff --git a/src/backend/commons/common-k8s/build.gradle b/src/backend/commons/common-k8s/build.gradle index 688bd5b291..41858ab07f 100644 --- a/src/backend/commons/common-k8s/build.gradle +++ b/src/backend/commons/common-k8s/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,6 +25,5 @@ dependencies { api project(':commons:common-discovery') api 'org.apache.httpcomponents:httpcore' - api 'org.springframework.cloud:spring-cloud-starter-kubernetes-client-all' - api 'org.springframework.cloud:spring-cloud-starter-kubernetes-client-loadbalancer' + api "org.springframework.cloud:spring-cloud-starter-kubernetes-client-all" } diff --git a/src/backend/commons/common-k8s/src/main/java/com/tencent/bk/job/common/k8s/availability/JobApplicationAvailabilityBean.java b/src/backend/commons/common-k8s/src/main/java/com/tencent/bk/job/common/k8s/availability/JobApplicationAvailabilityBean.java new file mode 100644 index 0000000000..b19421dfe5 --- /dev/null +++ b/src/backend/commons/common-k8s/src/main/java/com/tencent/bk/job/common/k8s/availability/JobApplicationAvailabilityBean.java @@ -0,0 +1,23 @@ +package com.tencent.bk.job.common.k8s.availability; + +import com.tencent.bk.job.common.util.ThreadUtils; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.availability.ApplicationAvailabilityBean; +import org.springframework.boot.availability.AvailabilityChangeEvent; +import org.springframework.boot.availability.ReadinessState; + +@Slf4j +public class JobApplicationAvailabilityBean extends ApplicationAvailabilityBean { + @Override + public void onApplicationEvent(AvailabilityChangeEvent event) { + super.onApplicationEvent(event); + if (ReadinessState.REFUSING_TRAFFIC == event.getState()) { + // SpringCloud负载均衡缓存设置为20s，等待调用方缓存刷新后再真正关闭Spring容器 + int waitSeconds = 40; + while (waitSeconds > 0) { + ThreadUtils.sleep(1000); + log.info("wait for GracefulShutdown, {}s left", waitSeconds--); + } + } + } +} diff --git a/src/backend/commons/common-k8s/src/main/java/com/tencent/bk/job/common/k8s/config/JobApplicationAvailabilityAutoConfiguration.java b/src/backend/commons/common-k8s/src/main/java/com/tencent/bk/job/common/k8s/config/JobApplicationAvailabilityAutoConfiguration.java new file mode 100644 index 0000000000..78b9534e33 --- /dev/null +++ b/src/backend/commons/common-k8s/src/main/java/com/tencent/bk/job/common/k8s/config/JobApplicationAvailabilityAutoConfiguration.java @@ -0,0 +1,16 @@ +package com.tencent.bk.job.common.k8s.config; + +import com.tencent.bk.job.common.k8s.availability.JobApplicationAvailabilityBean; +import org.springframework.boot.availability.ApplicationAvailabilityBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration(proxyBeanMethods = false) +public class JobApplicationAvailabilityAutoConfiguration { + + @Bean + public ApplicationAvailabilityBean applicationAvailability() { + return new JobApplicationAvailabilityBean(); + } + +} diff --git a/src/backend/commons/common-k8s/src/main/java/com/tencent/bk/job/common/k8s/config/JobK8sDiscoveryClientAutoConfiguration.java b/src/backend/commons/common-k8s/src/main/java/com/tencent/bk/job/common/k8s/config/JobK8sDiscoveryClientAutoConfiguration.java new file mode 100644 index 0000000000..7caacf190a --- /dev/null +++ b/src/backend/commons/common-k8s/src/main/java/com/tencent/bk/job/common/k8s/config/JobK8sDiscoveryClientAutoConfiguration.java @@ -0,0 +1,149 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.k8s.config; + + +import io.kubernetes.client.informer.SharedInformer; +import io.kubernetes.client.informer.SharedInformerFactory; +import io.kubernetes.client.informer.cache.Lister; +import io.kubernetes.client.openapi.ApiClient; +import io.kubernetes.client.openapi.models.V1Endpoints; +import io.kubernetes.client.openapi.models.V1EndpointsList; +import io.kubernetes.client.openapi.models.V1Service; +import io.kubernetes.client.openapi.models.V1ServiceList; +import io.kubernetes.client.spring.extended.controller.annotation.GroupVersionResource; +import io.kubernetes.client.spring.extended.controller.annotation.KubernetesInformer; +import io.kubernetes.client.spring.extended.controller.annotation.KubernetesInformers; +import io.kubernetes.client.spring.extended.controller.config.KubernetesInformerAutoConfiguration; +import org.springframework.boot.actuate.health.HealthIndicator; +import org.springframework.boot.autoconfigure.AutoConfigureAfter; +import org.springframework.boot.autoconfigure.AutoConfigureBefore; +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.cloud.client.CommonsClientAutoConfiguration; +import org.springframework.cloud.client.ConditionalOnDiscoveryEnabled; +import org.springframework.cloud.client.ConditionalOnDiscoveryHealthIndicatorEnabled; +import org.springframework.cloud.client.discovery.simple.SimpleDiscoveryClientAutoConfiguration; +import org.springframework.cloud.kubernetes.client.KubernetesClientAutoConfiguration; +import org.springframework.cloud.kubernetes.client.discovery.ConditionalOnKubernetesDiscoveryEnabled; +import org.springframework.cloud.kubernetes.client.discovery.KubernetesInformerDiscoveryClient; +import org.springframework.cloud.kubernetes.commons.ConditionalOnKubernetesEnabled; +import org.springframework.cloud.kubernetes.commons.KubernetesNamespaceProvider; +import org.springframework.cloud.kubernetes.commons.PodUtils; +import org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryClientHealthIndicatorInitializer; +import org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryProperties; +import org.springframework.context.ApplicationEventPublisher; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.env.Environment; + +/** + * 负载均衡相关的自定义Bean配置 + * 注意：该类用于使用部分自定义Bean实例覆盖KubernetesDiscoveryClientAutoConfiguration类的配置内容，框架升级时需要同步更新！！！ + */ +@Configuration(proxyBeanMethods = false) +@ConditionalOnKubernetesDiscoveryEnabled +@ConditionalOnKubernetesEnabled +@AutoConfigureBefore({SimpleDiscoveryClientAutoConfiguration.class, CommonsClientAutoConfiguration.class, + // So that CatalogSharedInformerFactory can be processed in prior to the default + // factory + KubernetesInformerAutoConfiguration.class}) +@AutoConfigureAfter({KubernetesClientAutoConfiguration.class}) +@EnableConfigurationProperties(KubernetesDiscoveryProperties.class) +public class JobK8sDiscoveryClientAutoConfiguration { + + @ConditionalOnClass({HealthIndicator.class}) + @ConditionalOnDiscoveryEnabled + @ConditionalOnDiscoveryHealthIndicatorEnabled + @Configuration + public static class KubernetesDiscoveryClientHealthIndicatorConfiguration { + + @Bean + public KubernetesDiscoveryClientHealthIndicatorInitializer indicatorInitializer( + ApplicationEventPublisher applicationEventPublisher, PodUtils podUtils) { + return new KubernetesDiscoveryClientHealthIndicatorInitializer(podUtils, applicationEventPublisher); + } + + } + + @Bean + @ConditionalOnMissingBean + public JobSpringCloudKubernetesInformerFactoryProcessor discoveryInformerConfigurer( + KubernetesNamespaceProvider kubernetesNamespaceProvider, + ApiClient apiClient, + JobCatalogSharedInformerFactory sharedInformerFactory, + Environment environment) { + // Injecting KubernetesDiscoveryProperties here would cause it to be + // initialize too early + // Instead get the all-namespaces property value from the Environment directly + boolean allNamespaces = environment.getProperty("spring.cloud.kubernetes.discovery.all-namespaces", + Boolean.class, false); + return new JobSpringCloudKubernetesInformerFactoryProcessor( + kubernetesNamespaceProvider, + apiClient, + sharedInformerFactory, + allNamespaces + ); + } + + @Bean + public JobCatalogSharedInformerFactory jobCatalogSharedInformerFactory(ApiClient apiClient) { + return new JobCatalogSharedInformerFactory(); + } + + @Bean + public KubernetesInformerDiscoveryClient kubernetesInformerDiscoveryClient( + KubernetesNamespaceProvider kubernetesNamespaceProvider, + JobCatalogSharedInformerFactory jobCatalogSharedInformerFactory, + Lister serviceLister, + Lister endpointsLister, + SharedInformer serviceInformer, + SharedInformer endpointsInformer, + KubernetesDiscoveryProperties properties + ) { + return new JobKubernetesInformerDiscoveryClient( + kubernetesNamespaceProvider.getNamespace(), + jobCatalogSharedInformerFactory, + serviceLister, + endpointsLister, + serviceInformer, + endpointsInformer, + properties + ); + } + + @KubernetesInformers({ + @KubernetesInformer(apiTypeClass = V1Service.class, apiListTypeClass = V1ServiceList.class, + groupVersionResource = @GroupVersionResource(apiGroup = "", apiVersion = "v1", + resourcePlural = "services")), + @KubernetesInformer(apiTypeClass = V1Endpoints.class, apiListTypeClass = V1EndpointsList.class, + groupVersionResource = @GroupVersionResource(apiGroup = "", apiVersion = "v1", + resourcePlural = "endpoints"))}) + class JobCatalogSharedInformerFactory extends SharedInformerFactory { + // TODO: optimization to ease memory pressure from continuous list&watch. + } +} + diff --git a/src/backend/commons/common-k8s/src/main/java/com/tencent/bk/job/common/k8s/config/JobKubernetesInformerDiscoveryClient.java b/src/backend/commons/common-k8s/src/main/java/com/tencent/bk/job/common/k8s/config/JobKubernetesInformerDiscoveryClient.java new file mode 100644 index 0000000000..50a06dcff7 --- /dev/null +++ b/src/backend/commons/common-k8s/src/main/java/com/tencent/bk/job/common/k8s/config/JobKubernetesInformerDiscoveryClient.java @@ -0,0 +1,61 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.k8s.config; + +import io.kubernetes.client.informer.SharedInformer; +import io.kubernetes.client.informer.SharedInformerFactory; +import io.kubernetes.client.informer.cache.Lister; +import io.kubernetes.client.openapi.models.V1Endpoints; +import io.kubernetes.client.openapi.models.V1Service; +import lombok.extern.slf4j.Slf4j; +import org.springframework.cloud.client.ServiceInstance; +import org.springframework.cloud.kubernetes.client.discovery.KubernetesInformerDiscoveryClient; +import org.springframework.cloud.kubernetes.commons.discovery.KubernetesDiscoveryProperties; + +import java.util.List; + +/** + * 支持打印负载均衡实例请求结果数据，用于排查相关问题 + */ +@Slf4j +public class JobKubernetesInformerDiscoveryClient extends KubernetesInformerDiscoveryClient { + public JobKubernetesInformerDiscoveryClient(String namespace, + SharedInformerFactory sharedInformerFactory, + Lister serviceLister, + Lister endpointsLister, + SharedInformer serviceInformer, + SharedInformer endpointsInformer, + KubernetesDiscoveryProperties properties) { + super(namespace, sharedInformerFactory, serviceLister, + endpointsLister, serviceInformer, endpointsInformer, properties); + } + + @Override + public List getInstances(String serviceId) { + List serviceInstanceList = super.getInstances(serviceId); + log.info("serviceId={}, serviceInstanceList={}", serviceId, serviceInstanceList); + return serviceInstanceList; + } +} diff --git a/src/backend/commons/common-k8s/src/main/java/com/tencent/bk/job/common/k8s/config/JobSpringCloudKubernetesInformerFactoryProcessor.java b/src/backend/commons/common-k8s/src/main/java/com/tencent/bk/job/common/k8s/config/JobSpringCloudKubernetesInformerFactoryProcessor.java new file mode 100644 index 0000000000..fe7f6aa2fd --- /dev/null +++ b/src/backend/commons/common-k8s/src/main/java/com/tencent/bk/job/common/k8s/config/JobSpringCloudKubernetesInformerFactoryProcessor.java @@ -0,0 +1,127 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.k8s.config; + +import io.kubernetes.client.informer.SharedIndexInformer; +import io.kubernetes.client.informer.SharedInformer; +import io.kubernetes.client.informer.SharedInformerFactory; +import io.kubernetes.client.informer.cache.Lister; +import io.kubernetes.client.openapi.ApiClient; +import io.kubernetes.client.spring.extended.controller.KubernetesInformerFactoryProcessor; +import io.kubernetes.client.spring.extended.controller.annotation.KubernetesInformer; +import io.kubernetes.client.spring.extended.controller.annotation.KubernetesInformers; +import io.kubernetes.client.util.Namespaces; +import io.kubernetes.client.util.generic.GenericKubernetesApi; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.config.ConfigurableListableBeanFactory; +import org.springframework.beans.factory.support.AbstractBeanDefinition; +import org.springframework.beans.factory.support.BeanDefinitionRegistry; +import org.springframework.beans.factory.support.RootBeanDefinition; +import org.springframework.cloud.kubernetes.commons.KubernetesNamespaceProvider; +import org.springframework.core.ResolvableType; + +import java.time.Duration; + +/** + * SpringCloudKubernetesInformerFactoryProcessor为非公开类，此处将其设置为公开类便于覆盖Bean时使用 + */ +public class JobSpringCloudKubernetesInformerFactoryProcessor extends KubernetesInformerFactoryProcessor { + + private static final Logger log = LoggerFactory.getLogger(JobSpringCloudKubernetesInformerFactoryProcessor.class); + + private BeanDefinitionRegistry beanDefinitionRegistry; + + private final ApiClient apiClient; + + private final SharedInformerFactory sharedInformerFactory; + + private final boolean allNamespaces; + + private final KubernetesNamespaceProvider kubernetesNamespaceProvider; + + JobSpringCloudKubernetesInformerFactoryProcessor(KubernetesNamespaceProvider kubernetesNamespaceProvider, + ApiClient apiClient, + SharedInformerFactory sharedInformerFactory, + boolean allNamespaces) { + super(); + this.apiClient = apiClient; + this.sharedInformerFactory = sharedInformerFactory; + this.kubernetesNamespaceProvider = kubernetesNamespaceProvider; + this.allNamespaces = allNamespaces; + } + + @Override + public void postProcessBeanFactory(ConfigurableListableBeanFactory beanFactory) throws BeansException { + String namespace = allNamespaces ? Namespaces.NAMESPACE_ALL : kubernetesNamespaceProvider.getNamespace() == null + ? Namespaces.NAMESPACE_DEFAULT : kubernetesNamespaceProvider.getNamespace(); + + this.apiClient.setHttpClient(this.apiClient.getHttpClient().newBuilder().readTimeout(Duration.ZERO).build()); + + KubernetesInformers kubernetesInformers = sharedInformerFactory.getClass() + .getAnnotation(KubernetesInformers.class); + if (kubernetesInformers == null || kubernetesInformers.value().length == 0) { + log.info("No informers registered in the sharedInformerFactory.."); + return; + } + for (KubernetesInformer kubernetesInformer : kubernetesInformers.value()) { + final GenericKubernetesApi api = new GenericKubernetesApi(kubernetesInformer.apiTypeClass(), + kubernetesInformer.apiListTypeClass(), kubernetesInformer.groupVersionResource().apiGroup(), + kubernetesInformer.groupVersionResource().apiVersion(), + kubernetesInformer.groupVersionResource().resourcePlural(), apiClient); + SharedIndexInformer sharedIndexInformer = sharedInformerFactory.sharedIndexInformerFor(api, + kubernetesInformer.apiTypeClass(), kubernetesInformer.resyncPeriodMillis(), + kubernetesInformer.namespace().equals(Namespaces.NAMESPACE_ALL) ? namespace + : kubernetesInformer.namespace()); + ResolvableType informerType = ResolvableType.forClassWithGenerics(SharedInformer.class, + kubernetesInformer.apiTypeClass()); + RootBeanDefinition informerBean = new RootBeanDefinition(); + informerBean.setTargetType(informerType); + informerBean.setAutowireMode(AbstractBeanDefinition.AUTOWIRE_BY_TYPE); + informerBean.setAutowireCandidate(true); + String informerBeanName = informerType.toString(); + this.beanDefinitionRegistry.registerBeanDefinition(informerBeanName, informerBean); + beanFactory.registerSingleton(informerBeanName, sharedIndexInformer); + + Lister lister = new Lister(sharedIndexInformer.getIndexer()); + ResolvableType listerType = ResolvableType.forClassWithGenerics(Lister.class, + kubernetesInformer.apiTypeClass()); + RootBeanDefinition listerBean = new RootBeanDefinition(); + listerBean.setTargetType(listerType); + listerBean.setAutowireMode(AbstractBeanDefinition.AUTOWIRE_BY_TYPE); + listerBean.setAutowireCandidate(true); + String listerBeanName = listerType.toString(); + this.beanDefinitionRegistry.registerBeanDefinition(listerBeanName, listerBean); + beanFactory.registerSingleton(listerBeanName, lister); + } + } + + @Override + public void postProcessBeanDefinitionRegistry(BeanDefinitionRegistry registry) throws BeansException { + this.beanDefinitionRegistry = registry; + } + +} diff --git a/src/backend/commons/common-k8s/src/main/java/com/tencent/bk/job/common/k8s/config/K8sServiceInfoServiceAutoConfig.java b/src/backend/commons/common-k8s/src/main/java/com/tencent/bk/job/common/k8s/config/K8sServiceInfoServiceAutoConfig.java deleted file mode 100644 index 0dc765152b..0000000000 --- a/src/backend/commons/common-k8s/src/main/java/com/tencent/bk/job/common/k8s/config/K8sServiceInfoServiceAutoConfig.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.tencent.bk.job.common.k8s.config; - -import com.tencent.bk.job.common.discovery.ServiceInfoProvider; -import com.tencent.bk.job.common.k8s.provider.K8SServiceInfoProvider; -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingClass; -import org.springframework.cloud.client.discovery.DiscoveryClient; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Configuration -@ConditionalOnMissingClass("org.springframework.cloud.consul.discovery.ConsulDiscoveryClient") -public class K8sServiceInfoServiceAutoConfig { - - @Bean - public ServiceInfoProvider k8sServiceInfoService(DiscoveryClient discoveryClient) { - return new K8SServiceInfoProvider(discoveryClient); - } - -} diff --git a/src/backend/commons/common-k8s/src/main/java/com/tencent/bk/job/common/k8s/config/K8sServiceInfoServiceAutoConfiguration.java b/src/backend/commons/common-k8s/src/main/java/com/tencent/bk/job/common/k8s/config/K8sServiceInfoServiceAutoConfiguration.java new file mode 100644 index 0000000000..d9559f26a8 --- /dev/null +++ b/src/backend/commons/common-k8s/src/main/java/com/tencent/bk/job/common/k8s/config/K8sServiceInfoServiceAutoConfiguration.java @@ -0,0 +1,21 @@ +package com.tencent.bk.job.common.k8s.config; + +import com.tencent.bk.job.common.discovery.ServiceInfoProvider; +import com.tencent.bk.job.common.k8s.provider.K8SServiceInfoProvider; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingClass; +import org.springframework.cloud.client.discovery.DiscoveryClient; +import org.springframework.cloud.kubernetes.commons.ConditionalOnKubernetesEnabled; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +@ConditionalOnMissingClass("org.springframework.cloud.consul.discovery.ConsulDiscoveryClient") +@ConditionalOnKubernetesEnabled +public class K8sServiceInfoServiceAutoConfiguration { + + @Bean + public ServiceInfoProvider k8sServiceInfoService(DiscoveryClient discoveryClient) { + return new K8SServiceInfoProvider(discoveryClient); + } + +} diff --git a/src/backend/commons/common-k8s/src/main/java/com/tencent/bk/job/common/k8s/provider/K8SServiceInfoProvider.java b/src/backend/commons/common-k8s/src/main/java/com/tencent/bk/job/common/k8s/provider/K8SServiceInfoProvider.java index fb4d1350fd..a21255add0 100644 --- a/src/backend/commons/common-k8s/src/main/java/com/tencent/bk/job/common/k8s/provider/K8SServiceInfoProvider.java +++ b/src/backend/commons/common-k8s/src/main/java/com/tencent/bk/job/common/k8s/provider/K8SServiceInfoProvider.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -45,6 +45,7 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; @Slf4j @@ -157,22 +158,37 @@ private ServiceInstanceInfoDTO getDetailFromK8s( */ @Override public List listServiceInfo() { + String jobServiceSymbol = "job-"; List serviceIdList = discoveryClient.getServices(); List serviceInstanceList = new ArrayList<>(); for (String serviceId : serviceIdList) { - serviceInstanceList.addAll(discoveryClient.getInstances(serviceId)); - } - for (ServiceInstance serviceInstance : serviceInstanceList) { - log.debug("serviceInstance={}", JsonUtils.toJson(serviceInstance)); - } - return serviceInstanceList.parallelStream().filter(serviceInstance -> { - if (serviceInstance.getServiceId().contains("job-gateway-management")) { - return false; - } else { - return serviceInstance.getServiceId().contains("job-"); + if (serviceId.contains(jobServiceSymbol)) { + serviceInstanceList.addAll(discoveryClient.getInstances(serviceId)); } - }).filter(serviceInstance -> getNameSpace(serviceInstance) != null) + } + tryToLogServiceInstanceList(serviceInstanceList); + return serviceInstanceList.stream().filter( + serviceInstance -> !serviceInstance.getServiceId().contains("job-gateway-management") + ).filter(serviceInstance -> getNameSpace(serviceInstance) != null) .map(this::getDetailFromK8s).collect(Collectors.toList()); } + private void tryToLogServiceInstanceList(List serviceInstanceList) { + try { + for (ServiceInstance serviceInstance : serviceInstanceList) { + Map metaData = serviceInstance.getMetadata(); + // 清除null key，防止后续序列化失败 + if (metaData != null && metaData.containsKey(null)) { + log.debug("Ignore null key value:{}", metaData.get(null)); + metaData.remove(null); + } + if (log.isDebugEnabled()) { + log.debug("serviceInstance={}", JsonUtils.toJson(serviceInstance)); + } + } + } catch (Throwable t) { + log.warn("Fail to logServiceInstanceList", t); + } + } + } diff --git a/src/backend/commons/common-k8s/src/main/resources/META-INF/spring.factories b/src/backend/commons/common-k8s/src/main/resources/META-INF/spring.factories index e69de29bb2..c382e26029 100644 --- a/src/backend/commons/common-k8s/src/main/resources/META-INF/spring.factories +++ b/src/backend/commons/common-k8s/src/main/resources/META-INF/spring.factories @@ -0,0 +1,4 @@ +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ +com.tencent.bk.job.common.k8s.config.K8sServiceInfoServiceAutoConfiguration,\ +com.tencent.bk.job.common.k8s.config.JobApplicationAvailabilityAutoConfiguration, \ + com.tencent.bk.job.common.k8s.config.JobK8sDiscoveryClientAutoConfiguration diff --git a/src/backend/commons/common-log/build.gradle b/src/backend/commons/common-log/build.gradle new file mode 100644 index 0000000000..0ef3056544 --- /dev/null +++ b/src/backend/commons/common-log/build.gradle @@ -0,0 +1,30 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +dependencies { + api project(':commons:common') + api project(':commons:common-utils') + testImplementation 'org.junit.jupiter:junit-jupiter' + testImplementation 'org.springframework.boot:spring-boot-starter-test' +} diff --git a/src/backend/commons/common-log/src/main/java/com/tencent/bk/job/common/log/config/LogClearAutoConfiguration.java b/src/backend/commons/common-log/src/main/java/com/tencent/bk/job/common/log/config/LogClearAutoConfiguration.java new file mode 100644 index 0000000000..ac1aea7116 --- /dev/null +++ b/src/backend/commons/common-log/src/main/java/com/tencent/bk/job/common/log/config/LogClearAutoConfiguration.java @@ -0,0 +1,43 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.log.config; + +import com.tencent.bk.job.common.log.task.LogClearScheduledTasks; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration(proxyBeanMethods = false) +@ConditionalOnProperty(name = "log.clear-by-volume-usage.enabled", havingValue = "true", matchIfMissing = true) +@EnableConfigurationProperties(LogClearByVolumeUsageProperties.class) +public class LogClearAutoConfiguration { + + @Bean + public LogClearScheduledTasks logClearScheduledTasks( + LogClearByVolumeUsageProperties logClearByVolumeUsageProperties) { + return new LogClearScheduledTasks(logClearByVolumeUsageProperties); + } +} diff --git a/src/backend/commons/common-log/src/main/java/com/tencent/bk/job/common/log/config/LogClearByVolumeUsageProperties.java b/src/backend/commons/common-log/src/main/java/com/tencent/bk/job/common/log/config/LogClearByVolumeUsageProperties.java new file mode 100644 index 0000000000..b08d77855f --- /dev/null +++ b/src/backend/commons/common-log/src/main/java/com/tencent/bk/job/common/log/config/LogClearByVolumeUsageProperties.java @@ -0,0 +1,48 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.log.config; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; + +/** + * 根据磁盘使用量清理日志相关配置 + */ +@ConfigurationProperties(prefix = "log.clear-by-volume-usage") +@Getter +@Setter +public class LogClearByVolumeUsageProperties { + + /** + * 是否开启日志清理 + */ + private boolean enabled = true; + + /** + * 最大容量，单位支持B、KB、MB、GB、TB、PB + */ + private String maxVolume = "200GB"; +} diff --git a/src/backend/commons/common-log/src/main/java/com/tencent/bk/job/common/log/pojo/encoder/SpecificLoggerAdjustLevelEncoder.java b/src/backend/commons/common-log/src/main/java/com/tencent/bk/job/common/log/pojo/encoder/SpecificLoggerAdjustLevelEncoder.java new file mode 100644 index 0000000000..f9e911e020 --- /dev/null +++ b/src/backend/commons/common-log/src/main/java/com/tencent/bk/job/common/log/pojo/encoder/SpecificLoggerAdjustLevelEncoder.java @@ -0,0 +1,73 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.log.pojo.encoder; + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.encoder.PatternLayoutEncoder; +import ch.qos.logback.classic.spi.ILoggingEvent; +import com.tencent.bk.job.common.log.pojo.event.MutableLevelLoggingEventWrapper; +import lombok.Setter; + +import java.util.HashSet; +import java.util.Set; + +/** + * 自定义encoder，可对特定logger(s)打出的日志做日志级别的调整，此encoder包含 {@link PatternLayoutEncoder} 所有功能 + * 使用方法： + * 将所需调整日志级别的logger路径用包裹起来（以<包名>.<类名>为路径） + * 将所需要调整的日志级别包裹在内 + * 将调整的目标日志级别包裹在内 + * + */ +public class SpecificLoggerAdjustLevelEncoder extends PatternLayoutEncoder { + private final Set targetLoggerSet = new HashSet<>(); + @Setter + private Level originLoggingLevel; + @Setter + private Level targetLoggingLevel; + + public void addTargetLogger(String targetLogger) { + targetLoggerSet.add(targetLogger); + } + + @Override + public byte[] encode(ILoggingEvent event) { + if (!isBegin()) { + addWarn("[SpecificLoggerAdjustLevelEncoder]Not provide origin or target logging level"); + return super.encode(event); + } + + ILoggingEvent wrappedEvent = event; + if (targetLoggerSet.contains(event.getLoggerName()) && originLoggingLevel.equals(event.getLevel())) { + wrappedEvent = new MutableLevelLoggingEventWrapper(event, targetLoggingLevel); + } + + return super.encode(wrappedEvent); + } + + private boolean isBegin() { + return originLoggingLevel != null && targetLoggingLevel != null; + } +} diff --git a/src/backend/commons/common-log/src/main/java/com/tencent/bk/job/common/log/pojo/event/MutableLevelLoggingEventWrapper.java b/src/backend/commons/common-log/src/main/java/com/tencent/bk/job/common/log/pojo/event/MutableLevelLoggingEventWrapper.java new file mode 100644 index 0000000000..76099e8403 --- /dev/null +++ b/src/backend/commons/common-log/src/main/java/com/tencent/bk/job/common/log/pojo/event/MutableLevelLoggingEventWrapper.java @@ -0,0 +1,50 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.log.pojo.event; + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.spi.ILoggingEvent; +import lombok.experimental.Delegate; + +public class MutableLevelLoggingEventWrapper implements ILoggingEvent { + @Delegate + private final ILoggingEvent originalEvent; + + private Level wrapLevel; + + public MutableLevelLoggingEventWrapper(ILoggingEvent originalEvent, Level level) { + this.originalEvent = originalEvent; + this.wrapLevel = level; + } + + public void setLevel(Level level) { + this.wrapLevel = level; + } + + @Override + public Level getLevel() { + return wrapLevel; + } +} diff --git a/src/backend/commons/common-log/src/main/java/com/tencent/bk/job/common/log/pojo/filter/LoggerExclusionFilter.java b/src/backend/commons/common-log/src/main/java/com/tencent/bk/job/common/log/pojo/filter/LoggerExclusionFilter.java new file mode 100644 index 0000000000..ecd6fa86c2 --- /dev/null +++ b/src/backend/commons/common-log/src/main/java/com/tencent/bk/job/common/log/pojo/filter/LoggerExclusionFilter.java @@ -0,0 +1,60 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.log.pojo.filter; + +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.filter.AbstractMatcherFilter; +import ch.qos.logback.core.spi.FilterReply; + +import java.util.HashSet; +import java.util.Set; + +/** + * 排除特定logger打出的日志 + */ +public class LoggerExclusionFilter extends AbstractMatcherFilter { + private final Set exclusiveLoggerSet = new HashSet<>(); + + public void addExclusiveLogger(String exclusiveLogger) { + this.exclusiveLoggerSet.add(exclusiveLogger); + } + + @Override + public FilterReply decide(ILoggingEvent event) { + if (!isStarted()) { + return FilterReply.NEUTRAL; + } + + if (exclusiveLoggerSet.contains(event.getLoggerName())) { + return onMatch; + } + + return onMismatch; + } + + public void start() { + super.start(); + } +} diff --git a/src/backend/commons/common-log/src/main/java/com/tencent/bk/job/common/log/spi/impl/GetDayFromHourPropertyDefiner.java b/src/backend/commons/common-log/src/main/java/com/tencent/bk/job/common/log/spi/impl/GetDayFromHourPropertyDefiner.java new file mode 100644 index 0000000000..7c4be99eec --- /dev/null +++ b/src/backend/commons/common-log/src/main/java/com/tencent/bk/job/common/log/spi/impl/GetDayFromHourPropertyDefiner.java @@ -0,0 +1,59 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.log.spi.impl; + +import ch.qos.logback.core.CoreConstants; +import ch.qos.logback.core.PropertyDefinerBase; + +/** + * 根据参数传入的小时数换算为对应的天数，不足1天按1天算 + */ +public class GetDayFromHourPropertyDefiner extends PropertyDefinerBase { + + int hours; + + @Override + public String getPropertyValue() { + // 无限期存储 + if (hours == CoreConstants.UNBOUND_HISTORY) { + return String.valueOf(CoreConstants.UNBOUND_HISTORY); + } + int days = new Double(Math.ceil(hours / 24.0)).intValue(); + // 不足1天按1天算 + int minHistory = 1; + if (days <= 0) { + return String.valueOf(minHistory); + } + return String.valueOf(days); + } + + public int getHours() { + return hours; + } + + public void setHours(int hours) { + this.hours = hours; + } +} diff --git a/src/backend/commons/common-log/src/main/java/com/tencent/bk/job/common/log/task/ClearLogFileByVolumeUsageTask.java b/src/backend/commons/common-log/src/main/java/com/tencent/bk/job/common/log/task/ClearLogFileByVolumeUsageTask.java new file mode 100644 index 0000000000..e404d7a062 --- /dev/null +++ b/src/backend/commons/common-log/src/main/java/com/tencent/bk/job/common/log/task/ClearLogFileByVolumeUsageTask.java @@ -0,0 +1,103 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.log.task; + +import ch.qos.logback.classic.LoggerContext; +import com.tencent.bk.job.common.util.file.FileSizeUtil; +import com.tencent.bk.job.common.util.file.FileUtil; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.LoggerFactory; + +import java.util.Collections; + +/** + * 根据磁盘使用量清理过期日志文件任务 + */ +@Slf4j +public class ClearLogFileByVolumeUsageTask { + + private volatile boolean clearTaskRunning = false; + + /** + * 最大容量字符串，单位支持B/KB/MB/GB/TB/PB + */ + private final String maxSizeStr; + + public ClearLogFileByVolumeUsageTask(String maxSizeStr) { + this.maxSizeStr = maxSizeStr; + } + + /** + * 检查磁盘容量并清理最旧的文件 + */ + public void checkVolumeAndClear() { + if (StringUtils.isBlank(maxSizeStr)) { + log.info("maxSizeStr is blank, ignore checkVolumeAndClear"); + return; + } + long maxSizeBytes = FileSizeUtil.parseFileSizeBytes(maxSizeStr); + if (maxSizeBytes <= 0) { + log.error("Cannot parse valid maxSizeBytes from maxSizeStr {}, ignore checkVolumeAndClear", maxSizeStr); + return; + } + // 获取当前服务的日志根路径 + String appLogDirPath = getAppLogDirPath(); + if (StringUtils.isBlank(appLogDirPath)) { + log.error("Cannot find valid appLogDirPath, ignore checkVolumeAndClear"); + return; + } + try { + // 上一次清理任务还未跑完则不清理 + if (clearTaskRunning) { + log.info("Last checkVolumeAndClear task still running, skip"); + return; + } + clearTaskRunning = true; + int count = doCheckVolumeAndClear(maxSizeBytes, appLogDirPath); + log.info("{} log file cleared", count); + } catch (Exception e) { + log.warn("Exception when checkVolumeAndClear", e); + } finally { + clearTaskRunning = false; + } + } + + private int doCheckVolumeAndClear(long maxSizeBytes, String appLogDirPath) { + // 当前正在写入的日志文件不删除，防止日志采集丢失部分日志 + String notDeleteFileSuffix = ".log"; + return FileUtil.checkVolumeAndClearOldestFiles( + maxSizeBytes, + appLogDirPath, + Collections.singleton(notDeleteFileSuffix) + ); + } + + private String getAppLogDirPath() { + LoggerContext loggerContext = (LoggerContext) LoggerFactory.getILoggerFactory(); + String propertyNameAppLogDir = "APP_LOG_DIR"; + return loggerContext.getProperty(propertyNameAppLogDir); + } +} diff --git a/src/backend/commons/common-log/src/main/java/com/tencent/bk/job/common/log/task/LogClearScheduledTasks.java b/src/backend/commons/common-log/src/main/java/com/tencent/bk/job/common/log/task/LogClearScheduledTasks.java new file mode 100644 index 0000000000..bdf588d9a9 --- /dev/null +++ b/src/backend/commons/common-log/src/main/java/com/tencent/bk/job/common/log/task/LogClearScheduledTasks.java @@ -0,0 +1,67 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.log.task; + +import com.tencent.bk.job.common.log.config.LogClearByVolumeUsageProperties; +import lombok.extern.slf4j.Slf4j; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.scheduling.annotation.Scheduled; + +@Slf4j +public class LogClearScheduledTasks { + + private static final Logger logger = LoggerFactory.getLogger(LogClearScheduledTasks.class); + + private final ClearLogFileByVolumeUsageTask clearLogFileByVolumeUsageTask; + + public LogClearScheduledTasks(LogClearByVolumeUsageProperties logClearByVolumeUsageProperties) { + if (logClearByVolumeUsageProperties.isEnabled()) { + this.clearLogFileByVolumeUsageTask = new ClearLogFileByVolumeUsageTask( + logClearByVolumeUsageProperties.getMaxVolume() + ); + } else { + this.clearLogFileByVolumeUsageTask = null; + } + } + + /** + * 清理：每20分钟根据磁盘使用量清理一次日志文件 + */ + @Scheduled(cron = "50 0/20 * * * ?") + public void clearLogFileByVolumeUsage() { + if (clearLogFileByVolumeUsageTask == null) { + log.debug("clearLogFileByVolumeUsage not enabled, ignore clearLogFile"); + return; + } + logger.info("clearLogFileByVolumeUsage start"); + try { + clearLogFileByVolumeUsageTask.checkVolumeAndClear(); + } catch (Exception e) { + logger.error("clearLogFileByVolumeUsage fail", e); + } + } + +} diff --git a/src/backend/commons/common-log/src/main/resources/META-INF/spring.factories b/src/backend/commons/common-log/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..2f685a98ef --- /dev/null +++ b/src/backend/commons/common-log/src/main/resources/META-INF/spring.factories @@ -0,0 +1,2 @@ +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ +com.tencent.bk.job.common.log.config.LogClearAutoConfiguration diff --git a/src/backend/commons/common-log/src/main/resources/logback-default.xml b/src/backend/commons/common-log/src/main/resources/logback-default.xml new file mode 100644 index 0000000000..e6876946f5 --- /dev/null +++ b/src/backend/commons/common-log/src/main/resources/logback-default.xml @@ -0,0 +1,368 @@ + + + + + + + + + + + + + + + + + + + + + + + + ${LOG_KEEP_HOURS} + + + + + + + ${APP_LOG_FILE} + + ${APP_LOG_FILE}-%d{yyyyMMdd_HH}.log.%i + 1GB + ${LOG_KEEP_HOURS} + 100GB + ${CLEAN_HISTORY_ON_START} + + + ${FILE_LOG_PATTERN} + UTF-8 + io.opentelemetry.exporter.internal.grpc.OkHttpGrpcExporter + ERROR + WARN + + + + 10240 + + + + + + ${OPEN_API_LOG_FILE} + + ${OPEN_API_LOG_FILE}-%d{yyyyMMdd}.log.%i + 1GB + ${LOG_KEEP_DAYS} + 20GB + ${CLEAN_HISTORY_ON_START} + + + ${FILE_LOG_PATTERN} + UTF-8 + + + + 10240 + + + + + + ${GSE_LOG_FILE} + + ${GSE_LOG_FILE}-%d{yyyyMMdd_HH}.log.%i + 1GB + ${LOG_KEEP_HOURS} + 100GB + ${CLEAN_HISTORY_ON_START} + + + ${FILE_LOG_PATTERN} + UTF-8 + + + + 10240 + + + + + + ${IAM_LOG_FILE} + + ${IAM_LOG_FILE}-%d{yyyy-MM-dd}.log.%i + 1GB + ${LOG_KEEP_DAYS} + 20GB + ${CLEAN_HISTORY_ON_START} + + + ${FILE_LOG_PATTERN} + UTF-8 + + + + + + ${CMDB_LOG_FILE} + + ${CMDB_LOG_FILE}-%d{yyyyMMdd_HH}.log.%i + 1GB + ${LOG_KEEP_HOURS} + 100GB + ${CLEAN_HISTORY_ON_START} + + + ${FILE_LOG_PATTERN} + UTF-8 + + + + 10240 + + + + + + ${PAAS_LOG_FILE} + + ${PAAS_LOG_FILE}-%d{yyyyMMdd}.log.%i + 1GB + ${LOG_KEEP_DAYS} + 20GB + ${CLEAN_HISTORY_ON_START} + + + ${FILE_LOG_PATTERN} + UTF-8 + + + + + + + ${AUDIT_EVENT_LOG_FILE} + + ${AUDIT_EVENT_LOG_FILE}-%d{yyyy-MM-dd}.log.%i + 1GB + 1 + 5GB + ${CLEAN_HISTORY_ON_START} + + + ${AUDIT_EVENT_LOG_PATTERN} + UTF-8 + + + + + + ${ERROR_LOG_FILE} + + ${ERROR_LOG_FILE}-%d{yyyy-MM-dd}.log.%i + 1GB + ${LOG_KEEP_DAYS} + 20GB + ${CLEAN_HISTORY_ON_START} + + + ${FILE_LOG_PATTERN} + UTF-8 + + + ERROR + NEUTRAL + DENY + + + + io.opentelemetry.exporter.internal.grpc.OkHttpGrpcExporter + DENY + ACCEPT + + + + + + + ${CONSOLE_LOG_PATTERN} + utf8 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/backend/commons/common-log/src/test/java/com/tencent/bk/job/common/log/MutableLevelLoggingEventWrapperTest.java b/src/backend/commons/common-log/src/test/java/com/tencent/bk/job/common/log/MutableLevelLoggingEventWrapperTest.java new file mode 100644 index 0000000000..5847dee306 --- /dev/null +++ b/src/backend/commons/common-log/src/test/java/com/tencent/bk/job/common/log/MutableLevelLoggingEventWrapperTest.java @@ -0,0 +1,64 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.log; + +import ch.qos.logback.classic.Level; +import ch.qos.logback.classic.Logger; +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.classic.spi.LoggingEvent; +import com.tencent.bk.job.common.log.pojo.event.MutableLevelLoggingEventWrapper; +import org.junit.jupiter.api.Test; +import org.slf4j.LoggerFactory; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class MutableLevelLoggingEventWrapperTest { + + @Test + public void testDelegate() { + ILoggingEvent event = new LoggingEvent( + "fqcn", + (Logger) LoggerFactory.getLogger(MutableLevelLoggingEventWrapperTest.class), + Level.WARN, + "message", + new Throwable("simple-t"), + new Object[]{} + ); + ILoggingEvent wrappedEvent = new MutableLevelLoggingEventWrapper(event, Level.DEBUG); + + assertEquals(event.getLoggerName(), wrappedEvent.getLoggerName()); + assertEquals(event.getFormattedMessage(), wrappedEvent.getFormattedMessage()); + assertEquals(event.getMessage(), wrappedEvent.getMessage()); + assertEquals(event.getThreadName(), wrappedEvent.getThreadName()); + assertEquals(event.getArgumentArray(), wrappedEvent.getArgumentArray()); + assertEquals(event.getCallerData(), wrappedEvent.getCallerData()); + assertEquals(event.getLoggerContextVO(), wrappedEvent.getLoggerContextVO()); + assertEquals(event.getMarker(), wrappedEvent.getMarker()); + assertEquals(event.getMDCPropertyMap(), wrappedEvent.getMDCPropertyMap()); + assertEquals(event.getThrowableProxy(), wrappedEvent.getThrowableProxy()); + assertEquals(event.getTimeStamp(), wrappedEvent.getTimeStamp()); + assertEquals(event.hasCallerData(), wrappedEvent.hasCallerData()); + } +} diff --git a/src/backend/commons/common-mongodb/build.gradle b/src/backend/commons/common-mongodb/build.gradle new file mode 100644 index 0000000000..2e8424afd1 --- /dev/null +++ b/src/backend/commons/common-mongodb/build.gradle @@ -0,0 +1,30 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software', to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +dependencies { + implementation project(':commons:common') + api "org.springframework:spring-context" + api "org.springframework.boot:spring-boot-autoconfigure" + api "org.springframework.data:spring-data-mongodb" +} diff --git a/src/backend/commons/common-mongodb/src/main/java/com/tencent/bk/job/common/mongodb/config/JobMongoSslProperties.java b/src/backend/commons/common-mongodb/src/main/java/com/tencent/bk/job/common/mongodb/config/JobMongoSslProperties.java new file mode 100644 index 0000000000..060bf762c1 --- /dev/null +++ b/src/backend/commons/common-mongodb/src/main/java/com/tencent/bk/job/common/mongodb/config/JobMongoSslProperties.java @@ -0,0 +1,41 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.mongodb.config; + +import com.tencent.bk.job.common.properties.JobSslProperties; +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; + +@ConfigurationProperties(prefix = "tls") +@Getter +@Setter +public class JobMongoSslProperties { + + /** + * mongodb的ssl配置 + */ + private JobSslProperties mongodb = new JobSslProperties(); +} diff --git a/src/backend/commons/common-mongodb/src/main/java/com/tencent/bk/job/common/mongodb/config/MongoDBConfig.java b/src/backend/commons/common-mongodb/src/main/java/com/tencent/bk/job/common/mongodb/config/MongoDBConfig.java new file mode 100644 index 0000000000..178b806f1f --- /dev/null +++ b/src/backend/commons/common-mongodb/src/main/java/com/tencent/bk/job/common/mongodb/config/MongoDBConfig.java @@ -0,0 +1,164 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.mongodb.config; + +import com.mongodb.Block; +import com.mongodb.ReadConcern; +import com.mongodb.ReadPreference; +import com.mongodb.WriteConcern; +import com.mongodb.connection.ConnectionPoolSettings; +import com.mongodb.connection.SocketSettings; +import com.tencent.bk.job.common.properties.JobSslProperties; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.mongo.MongoClientSettingsBuilderCustomizer; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.mongodb.core.WriteConcernResolver; +import org.springframework.util.StringUtils; + +import javax.net.ssl.KeyManager; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.TrustManager; +import javax.net.ssl.TrustManagerFactory; +import java.io.FileInputStream; +import java.security.KeyStore; +import java.security.SecureRandom; +import java.util.concurrent.TimeUnit; + +@Configuration +@Slf4j +@EnableConfigurationProperties({JobMongoSslProperties.class}) +public class MongoDBConfig { + @Bean + public MongoClientSettingsBuilderCustomizer mongoClientCustomizer(JobMongoSslProperties mongoSslProperties) { + log.info("Init MongoClientSettingsBuilderCustomizer"); + JobSslProperties jobMongoSslProperties = mongoSslProperties.getMongodb(); + Block socketSettings = builder -> builder + .connectTimeout(15, TimeUnit.SECONDS) + .readTimeout(60, TimeUnit.SECONDS); + Block connectionPoolSettings = builder -> builder + .minSize(200) + .maxSize(500) + .maxConnectionLifeTime(0, TimeUnit.SECONDS) + .maxConnectionIdleTime(0, TimeUnit.SECONDS); + return clientSettingsBuilder -> { + clientSettingsBuilder + .writeConcern(WriteConcern.W1) + .readConcern(ReadConcern.LOCAL) + .readPreference(ReadPreference.primaryPreferred()) + .applyToSocketSettings(socketSettings) + .applyToConnectionPoolSettings(connectionPoolSettings); + + // 根据配置文件判断是否开启ssl + if (jobMongoSslProperties.isEnabled()) { + clientSettingsBuilder.applyToSslSettings(ssl -> { + ssl.enabled(true); + ssl.invalidHostNameAllowed(!jobMongoSslProperties.isVerifyHostname()); + + try { + // 根据配置判断使用单向还是双向TLS + if (jobMongoSslProperties.isMutualTlsConfigured()) { + log.info("Detected client certificate configuration - enabling mutual TLS"); + ssl.context(createMutualTlsSslContext(jobMongoSslProperties)); + } else { + log.info("Using one-way TLS configuration"); + ssl.context(createOnewayTlsSslContext(jobMongoSslProperties)); + } + } catch (Exception e) { + log.error("Failed to configure MongoDB TLS context", e); + throw new RuntimeException("Failed to configure MongoDB TLS context", e); + } + }); + } + }; + } + + @Bean + public WriteConcernResolver writeConcernResolver() { + return action -> { + log.info("Using Write Concern of Unacknowledged"); + return WriteConcern.W1; + }; + } + + /** + * 创建单向TLS的SSLContext + */ + private SSLContext createOnewayTlsSslContext(JobSslProperties sslProps) throws Exception { + TrustManager[] trustManagers = getTrustManagers(sslProps); + + SSLContext sslContext = SSLContext.getInstance("TLS"); + sslContext.init(null, trustManagers, new SecureRandom()); + + return sslContext; + } + + /** + * 创建双向TLS的SSLContext + */ + private SSLContext createMutualTlsSslContext(JobSslProperties sslProps) throws Exception { + TrustManager[] trustManagers = getTrustManagers(sslProps); + KeyManager[] keyManagers = getKeyManagers(sslProps); + + SSLContext sslContext = SSLContext.getInstance("TLS"); + sslContext.init(keyManagers, trustManagers, new SecureRandom()); + + return sslContext; + } + + private TrustManager[] getTrustManagers(JobSslProperties sslProps) throws Exception { + TrustManager[] trustManagers; + if (StringUtils.hasText(sslProps.getTrustStore())) { + log.info("Loading mongo client trust store: {}", sslProps.getTrustStore()); + KeyStore trustStore = KeyStore.getInstance(sslProps.getTrustStoreType()); + try (FileInputStream fis = new FileInputStream(sslProps.getTrustStore())) { + trustStore.load(fis, sslProps.getTrustStorePassword().toCharArray()); + } + + TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + tmf.init(trustStore); + trustManagers = tmf.getTrustManagers(); + } else { + // fallback至JVM默认信任库 + TrustManagerFactory tmf = TrustManagerFactory.getInstance(TrustManagerFactory.getDefaultAlgorithm()); + tmf.init((KeyStore) null); + trustManagers = tmf.getTrustManagers(); + } + return trustManagers; + } + + private KeyManager[] getKeyManagers(JobSslProperties sslProps) throws Exception { + log.info("Loading mongo client key store: {}", sslProps.getKeyStore()); + KeyStore keyStore = KeyStore.getInstance(sslProps.getKeyStoreType()); + try (FileInputStream fis = new FileInputStream(sslProps.getKeyStore())) { + keyStore.load(fis, sslProps.getKeyStorePassword().toCharArray()); + } + KeyManagerFactory kmf = KeyManagerFactory.getInstance(KeyManagerFactory.getDefaultAlgorithm()); + kmf.init(keyStore, sslProps.getKeyStorePassword().toCharArray()); + return kmf.getKeyManagers(); + } +} diff --git a/src/backend/commons/common-mysql-sharding/build.gradle b/src/backend/commons/common-mysql-sharding/build.gradle new file mode 100644 index 0000000000..6884eea1b6 --- /dev/null +++ b/src/backend/commons/common-mysql-sharding/build.gradle @@ -0,0 +1,43 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +dependencies { + api project(":commons:common") + api project(":commons:common-mysql") + implementation 'io.micrometer:micrometer-registry-prometheus' + implementation("org.springframework.boot:spring-boot-autoconfigure") + implementation 'org.springframework:spring-web' + implementation 'org.aspectj:aspectjrt' + implementation 'org.aspectj:aspectjweaver' + api 'org.apache.shardingsphere:shardingsphere-jdbc-core' + api 'org.yaml:snakeyaml' + api 'com.tencent.devops.leaf:leaf-boot-starter' + api 'org.jooq:jooq' + api 'org.springframework.boot:spring-boot-starter-jdbc' + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' + testImplementation 'org.junit.jupiter:junit-jupiter' + testImplementation 'org.springframework.boot:spring-boot-starter-test' +} +apply from: "./leaf_gen_jooq.gradle" diff --git a/src/backend/commons/common-mysql-sharding/leaf_gen_jooq.gradle b/src/backend/commons/common-mysql-sharding/leaf_gen_jooq.gradle new file mode 100644 index 0000000000..25d02e2112 --- /dev/null +++ b/src/backend/commons/common-mysql-sharding/leaf_gen_jooq.gradle @@ -0,0 +1,78 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ +apply plugin: 'nu.studer.jooq' + +dependencies { + api "org.jooq:jooq" + jooqRuntime "com.mysql:mysql-connector-j" +} + +def databaseName = "job_leaf" +def targetPackageName = "com.tencent.bk.job.common.sharding.mysql.jooq.model" + +jooq { + genenrate(sourceSets.main) { + + jdbc { + driver = 'com.mysql.cj.jdbc.Driver' + def mysqlURL = System.getProperty("mysqlURL") + def mysqlUser = System.getProperty("mysqlUser") + def mysqlPasswd = System.getProperty("mysqlPasswd") + + if (mysqlURL == null) { + // gradle.properties 中的 Jooq DB 配置 + mysqlURL = project.findProperty("DB_HOST") + mysqlUser = project.findProperty("DB_USERNAME") + mysqlPasswd = project.findProperty("DB_PASSWORD") + } + + println("mysqlURL=" + mysqlURL) + println("mysqlUser=" + mysqlUser) + url = "jdbc:mysql://${mysqlURL}/${databaseName}?useSSL=false&serverTimezone=UTC" + user = mysqlUser + password = mysqlPasswd + } + + generator { + name = 'org.jooq.codegen.DefaultGenerator' + database { + name = 'org.jooq.meta.mysql.MySQLDatabase' + inputSchema = "${databaseName}" + } + + generate { + relations = false + deprecated = false + fluentSetters = true + generatedAnnotation = false + javaTimeTypes = true + } + + target { + packageName = "${targetPackageName}" + encoding = 'UTF-8' + } + } + } +} diff --git a/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/JooqLeafIdAllocator.java b/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/JooqLeafIdAllocator.java new file mode 100644 index 0000000000..5dd0c201da --- /dev/null +++ b/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/JooqLeafIdAllocator.java @@ -0,0 +1,119 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.sharding.mysql; + +import com.tencent.bk.job.common.sharding.mysql.jooq.model.tables.TLeafAlloc; +import com.tencent.bk.job.common.sharding.mysql.jooq.model.tables.records.TLeafAllocRecord; +import com.tencent.bk.job.common.util.date.DateUtils; +import com.tencent.devops.leaf.segment.dao.IDAllocDao; +import com.tencent.devops.leaf.segment.model.LeafAlloc; +import org.jooq.DSLContext; +import org.jooq.Record1; +import org.jooq.Result; +import org.jooq.impl.DSL; + +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.List; + +/** + * 基于 JOOQ+美团 Leaf分布式 ID 组件，生成分布式 ID + */ +public class JooqLeafIdAllocator implements IDAllocDao { + private final DSLContext dslContext; + + private static final TLeafAlloc T_LEAF_ALLOC = TLeafAlloc.T_LEAF_ALLOC; + + + public JooqLeafIdAllocator(DSLContext dslContext) { + this.dslContext = dslContext; + } + + @Override + public List getAllLeafAllocs() { + Result leafAllocRecords = dslContext.selectFrom(T_LEAF_ALLOC).fetch(); + List leafAllocs = new ArrayList<>(); + leafAllocRecords.forEach(leafAllocRecord -> { + LeafAlloc leafAlloc = generateLeafAlloc(leafAllocRecord); + leafAllocs.add(leafAlloc); + }); + return leafAllocs; + } + + private LeafAlloc generateLeafAlloc(TLeafAllocRecord leafAllocRecord) { + LeafAlloc leafAlloc = new LeafAlloc(); + leafAlloc.setKey(leafAllocRecord.getBizTag()); + leafAlloc.setMaxId(leafAllocRecord.getMaxId()); + leafAlloc.setStep(leafAllocRecord.getStep()); + leafAlloc.setUpdateTime(DateUtils.defaultLocalDateTime(leafAllocRecord.getUpdateTime())); + return leafAlloc; + } + + @Override + public LeafAlloc updateMaxIdAndGetLeafAlloc(String tag) { + TLeafAllocRecord leafAllocRecord; + leafAllocRecord = dslContext.transactionResult(t -> { + DSLContext context = DSL.using(t); + context.update(T_LEAF_ALLOC) + .set(T_LEAF_ALLOC.MAX_ID, T_LEAF_ALLOC.MAX_ID.add(T_LEAF_ALLOC.STEP)) + .set(T_LEAF_ALLOC.UPDATE_TIME, LocalDateTime.now()) + .where(T_LEAF_ALLOC.BIZ_TAG.eq(tag)) + .execute(); + return context.selectFrom(T_LEAF_ALLOC) + .where(T_LEAF_ALLOC.BIZ_TAG + .eq(tag)) + .fetchOne(); + }); + return generateLeafAlloc(leafAllocRecord); + } + + @Override + public LeafAlloc updateMaxIdByCustomStepAndGetLeafAlloc(LeafAlloc leafAlloc) { + TLeafAllocRecord leafAllocRecord; + leafAllocRecord = dslContext.transactionResult(t -> { + DSLContext context = DSL.using(t); + context.update(T_LEAF_ALLOC) + .set(T_LEAF_ALLOC.MAX_ID, T_LEAF_ALLOC.MAX_ID.add(leafAlloc.getStep())) + .set(T_LEAF_ALLOC.UPDATE_TIME, LocalDateTime.now()) + .where(T_LEAF_ALLOC.BIZ_TAG.eq(leafAlloc.getKey())) + .execute(); + return context.selectFrom(T_LEAF_ALLOC) + .where(T_LEAF_ALLOC.BIZ_TAG.eq(leafAlloc.getKey())) + .fetchOne(); + }); + return generateLeafAlloc(leafAllocRecord); + } + + @Override + public List getAllTags() { + Result> tagRecords = dslContext + .select(T_LEAF_ALLOC.BIZ_TAG) + .from(T_LEAF_ALLOC) + .fetch(); + List tags = new ArrayList<>(); + tagRecords.forEach(record -> tags.add(record.value1())); + return tags; + } +} diff --git a/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/SegmentIdKeys.java b/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/SegmentIdKeys.java new file mode 100644 index 0000000000..a554a68303 --- /dev/null +++ b/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/SegmentIdKeys.java @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.sharding.mysql; + +/** + * 分布式 ID KEY + */ +public class SegmentIdKeys { + + public static final String KEY_JOB_EXECUTE_TASK_INSTANCE = "job_execute.task_instance"; + public static final String KEY_JOB_EXECUTE_STEP_INSTANCE = "job_execute.step_instance"; + public static final String KEY_JOB_EXECUTE_GSE_TASK = "job_execute.gse_task"; + public static final String KEY_JOB_EXECUTE_OPERATION_LOG = "job_execute.operation_log"; + public static final String KEY_JOB_EXECUTE_FILE_SOURCE_TASK_LOG = "job_execute.file_source_task_log"; + public static final String KEY_JOB_EXECUTE_GSE_FILE_EXECUTE_OBJ_TASK = "job_execute.gse_file_execute_obj_task"; + public static final String KEY_JOB_EXECUTE_GSE_SCRIPT_EXECUTE_OBJ_TASK = + "job_execute.gse_script_execute_obj_task"; + public static final String KEY_JOB_EXECUTE_ROLLING_CONFIG = "job_execute.rolling_config"; + public static final String KEY_JOB_EXECUTE_STEP_INSTANCE_ROLLING_TASK = + "job_execute.step_instance_rolling_task"; + public static final String KEY_JOB_EXECUTE_STEP_INSTANCE_VARIABLE = "job_execute.step_instance_variable"; + public static final String KEY_JOB_EXECUTE_TASK_INSTANCE_VARIABLE = "job_execute.task_instance_variable"; +} diff --git a/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/ShardingConfigParseException.java b/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/ShardingConfigParseException.java new file mode 100644 index 0000000000..f5e50d66b6 --- /dev/null +++ b/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/ShardingConfigParseException.java @@ -0,0 +1,37 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.sharding.mysql; + +import com.tencent.bk.job.common.exception.InternalException; + +/** + * 分片配置解析异常 + */ +public class ShardingConfigParseException extends InternalException { + + public ShardingConfigParseException(String message) { + super(message); + } +} diff --git a/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/config/LeafAutoConfiguration.java b/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/config/LeafAutoConfiguration.java new file mode 100644 index 0000000000..5fe64c2393 --- /dev/null +++ b/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/config/LeafAutoConfiguration.java @@ -0,0 +1,110 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.sharding.mysql.config; + +import com.tencent.bk.job.common.mysql.util.JooqConfigurationUtil; +import com.tencent.bk.job.common.sharding.mysql.JooqLeafIdAllocator; +import org.jooq.ConnectionProvider; +import org.jooq.DSLContext; +import org.jooq.impl.DataSourceConnectionProvider; +import org.jooq.impl.DefaultDSLContext; +import org.jooq.impl.DefaultExecuteListenerProvider; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.boot.jdbc.DataSourceBuilder; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.DependsOn; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.DataSourceTransactionManager; +import org.springframework.jdbc.datasource.TransactionAwareDataSourceProxy; + +import javax.sql.DataSource; + +/** + * 美团 leaf 分布式 ID 组件自动装配 + */ +@Configuration(value = "leafDbConfig") +@ConditionalOnProperty(value = "leaf.enabled", havingValue = "true") +@EnableConfigurationProperties({LeafProperties.class}) +public class LeafAutoConfiguration { + @Qualifier("leaf-data-source") + @Bean(name = "leaf-data-source") + @ConfigurationProperties(prefix = "spring.datasource.leaf") + public DataSource dataSource() { + return DataSourceBuilder.create().build(); + } + + @Qualifier("leafTransactionManager") + @Bean(name = "leafTransactionManager") + @DependsOn("leaf-data-source") + public DataSourceTransactionManager transactionManager( + @Qualifier("leaf-data-source") DataSource dataSource) { + return new DataSourceTransactionManager(dataSource); + } + + @Qualifier("leaf-jdbc-template") + @Bean(name = "leaf-jdbc-template") + public JdbcTemplate jdbcTemplate( + @Qualifier("leaf-data-source") DataSource dataSource) { + return new JdbcTemplate(dataSource); + } + + @Qualifier("leaf-dsl-context") + @Bean(name = "leaf-dsl-context") + public DSLContext dslContext(@Qualifier("leaf-jooq-conf") org.jooq.Configuration configuration) { + return new DefaultDSLContext(configuration); + } + + @Qualifier("leaf-jooq-conf") + @Bean(name = "leaf-jooq-conf") + public org.jooq.Configuration + jooqConf(@Qualifier("leaf-conn-provider") ConnectionProvider connectionProvider, + DefaultExecuteListenerProvider jooqExecuteListenerProvider + ) { + return JooqConfigurationUtil.getConfiguration(connectionProvider, jooqExecuteListenerProvider); + } + + @Qualifier("leaf-conn-provider") + @Bean(name = "leaf-conn-provider") + public ConnectionProvider connectionProvider( + @Qualifier("leafTransactionAwareDataSource") DataSource dataSource) { + return new DataSourceConnectionProvider(dataSource); + } + + @Qualifier("leafTransactionAwareDataSource") + @Bean(name = "leafTransactionAwareDataSource") + public TransactionAwareDataSourceProxy + transactionAwareDataSourceProxy(@Qualifier("leaf-data-source") DataSource dataSource) { + return new TransactionAwareDataSourceProxy(dataSource); + } + + @Bean("jooqLeafIdAllocator") + public JooqLeafIdAllocator jooqLeafIdAllocator(@Qualifier("leaf-dsl-context") DSLContext dslContext) { + return new JooqLeafIdAllocator(dslContext); + } +} diff --git a/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/config/LeafProperties.java b/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/config/LeafProperties.java new file mode 100644 index 0000000000..2c6c37a2f7 --- /dev/null +++ b/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/config/LeafProperties.java @@ -0,0 +1,41 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.sharding.mysql.config; + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.springframework.boot.context.properties.ConfigurationProperties; + +@ConfigurationProperties(prefix = "leaf") +@Getter +@Setter +@ToString +public class LeafProperties { + /** + * 是否启用leaf + */ + private boolean enabled; +} diff --git a/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/config/ShardingDatasourceAutoConfiguration.java b/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/config/ShardingDatasourceAutoConfiguration.java new file mode 100644 index 0000000000..fd00d47aae --- /dev/null +++ b/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/config/ShardingDatasourceAutoConfiguration.java @@ -0,0 +1,314 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.sharding.mysql.config; + +import com.tencent.bk.job.common.mysql.util.JooqConfigurationUtil; +import com.tencent.bk.job.common.sharding.mysql.ShardingConfigParseException; +import com.zaxxer.hikari.HikariDataSource; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.MapUtils; +import org.apache.shardingsphere.driver.api.ShardingSphereDataSourceFactory; +import org.apache.shardingsphere.infra.config.algorithm.AlgorithmConfiguration; +import org.apache.shardingsphere.infra.config.mode.ModeConfiguration; +import org.apache.shardingsphere.infra.config.rule.RuleConfiguration; +import org.apache.shardingsphere.mode.repository.standalone.StandalonePersistRepositoryConfiguration; +import org.apache.shardingsphere.sharding.api.config.ShardingRuleConfiguration; +import org.apache.shardingsphere.sharding.api.config.rule.ShardingTableReferenceRuleConfiguration; +import org.apache.shardingsphere.sharding.api.config.rule.ShardingTableRuleConfiguration; +import org.apache.shardingsphere.sharding.api.config.strategy.keygen.KeyGenerateStrategyConfiguration; +import org.apache.shardingsphere.sharding.api.config.strategy.sharding.HintShardingStrategyConfiguration; +import org.apache.shardingsphere.sharding.api.config.strategy.sharding.ShardingStrategyConfiguration; +import org.apache.shardingsphere.sharding.api.config.strategy.sharding.StandardShardingStrategyConfiguration; +import org.jooq.ConnectionProvider; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; +import org.jooq.impl.DataSourceConnectionProvider; +import org.jooq.impl.DefaultConfiguration; +import org.jooq.impl.DefaultDSLContext; +import org.jooq.impl.DefaultExecuteListenerProvider; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.DependsOn; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.DataSourceTransactionManager; +import org.springframework.jdbc.datasource.TransactionAwareDataSourceProxy; + +import javax.sql.DataSource; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.stream.Collectors; + +/** + * 分库分表组件 ShardingSphere 配置 + */ +@Configuration(proxyBeanMethods = false) +@EnableConfigurationProperties({ShardingProperties.class}) +@ConditionalOnProperty(value = "sharding.enabled", havingValue = "true") +@Slf4j +public class ShardingDatasourceAutoConfiguration { + + @Bean("jobShardingDataSource") + public DataSource shardingDataSource(ShardingProperties shardingProperties) throws SQLException { + log.info("Init sharding datasource start ..."); + ShardingProperties.ShardingsphereProperties shardingsphereProperties = shardingProperties.getShardingsphere(); + // 指定逻辑 Database 名称 + String databaseName = shardingsphereProperties.getDatabaseName(); + // 构建运行模式 + ModeConfiguration modeConfig = createModeConfiguration(); + // 构建真实数据源 + Map dataSourceMap = createDataSources(shardingsphereProperties.getDataSources()); + // 构建具体规则 + List ruleConfigs = new ArrayList<>(); + ruleConfigs.add(createShardingRuleConfiguration(shardingsphereProperties.getShardingRule())); + // 构建系统级属性配置 + Properties globalProps = createShardingGlobalProps(shardingsphereProperties.getProps()); + DataSource dataSource = ShardingSphereDataSourceFactory.createDataSource(databaseName, modeConfig, + dataSourceMap, ruleConfigs, globalProps); + log.info("Init sharding datasource successfully"); + return dataSource; + } + + private ModeConfiguration createModeConfiguration() { + log.info("Load sharding mode, type: Standalone"); + return new ModeConfiguration("Standalone", + new StandalonePersistRepositoryConfiguration("JDBC", new Properties())); + } + + private Map createDataSources( + Map dataSourcePropMap) { + log.info("Init datasourceMap start ..."); + + Map dataSourceMap = new HashMap<>(); + + dataSourcePropMap.forEach((name, dataSourceProp) -> { + log.info("Create sharding datasource, name : {}, dataSourceProp: {}", name, dataSourceProp); + HikariDataSource dataSource = new HikariDataSource(); + dataSource.setDriverClassName(dataSourceProp.getDriverClassName()); + dataSource.setJdbcUrl(dataSourceProp.getJdbcUrl()); + dataSource.setUsername(dataSourceProp.getUsername()); + dataSource.setPassword(dataSourceProp.getPassword()); + dataSourceMap.put(name, dataSource); + }); + log.info("Init datasourceMap successfully"); + return dataSourceMap; + } + + private ShardingRuleConfiguration createShardingRuleConfiguration( + ShardingProperties.ShardingsphereProperties.ShardingRule shardingRuleProps) { + log.info("Init sharding rule configuration start ..."); + + ShardingRuleConfiguration shardingRuleConfiguration = new ShardingRuleConfiguration(); + + // 分片表规则列表 + shardingRuleConfiguration.getTables() + .addAll(createShardingTableRuleConfigurations(shardingRuleProps.getTables())); + + // 绑定表规则列表 + if (CollectionUtils.isNotEmpty(shardingRuleProps.getBindingTables())) { + shardingRuleProps.getBindingTables().forEach(bindingTables -> { + String ruleName = "binding_rule_" + + Arrays.stream(bindingTables.split(",")).map(String::trim).collect(Collectors.joining("_")); + log.info("Create binding table group, ruleName: {}, bindingTables: {}", ruleName, bindingTables); + shardingRuleConfiguration.getBindingTableGroups().add( + new ShardingTableReferenceRuleConfiguration(ruleName, bindingTables)); + }); + } + + // 默认分库策略 + if (shardingRuleProps.getDefaultDatabaseStrategy() != null) { + log.info("Create default database sharding strategy, strategy: {}", + shardingRuleProps.getDefaultDatabaseStrategy()); + + shardingRuleConfiguration.setDefaultDatabaseShardingStrategy( + createShardingStrategyConfiguration(shardingRuleProps.getDefaultDatabaseStrategy())); + } + + // 默认分表策略 + if (shardingRuleProps.getDefaultTableStrategy() != null) { + log.info("Create default table sharding strategy, strategy: {}", + shardingRuleProps.getDefaultTableStrategy()); + shardingRuleConfiguration.setDefaultTableShardingStrategy( + createShardingStrategyConfiguration(shardingRuleProps.getDefaultTableStrategy())); + } + + // 分片算法配置 + if (MapUtils.isNotEmpty(shardingRuleProps.getShardingAlgorithms())) { + shardingRuleProps.getShardingAlgorithms().forEach((name, algorithm) -> { + log.info("Create sharding algorithms, name: {}, type: {}, props: {}", + name, algorithm.getType(), algorithm.getProps()); + Properties props = toProperties(algorithm.getProps()); + shardingRuleConfiguration.getShardingAlgorithms() + .put(name, new AlgorithmConfiguration(algorithm.getType(), props)); + }); + } + + // 自增列生成算法配置 + if (MapUtils.isNotEmpty(shardingRuleProps.getKeyGenerators())) { + shardingRuleProps.getKeyGenerators().forEach((name, generator) -> { + log.info("Create key generator, name: {}, type: {}", name, generator.getType()); + Properties props = toProperties(generator.getProps()); + shardingRuleConfiguration.getKeyGenerators() + .put(name, new AlgorithmConfiguration(generator.getType(), props)); + }); + } + + log.info("Init sharding rule configuration successfully"); + + return shardingRuleConfiguration; + } + + private Properties toProperties(Map propsMap) { + Properties props = new Properties(); + props.putAll(propsMap); + return props; + } + + + private List createShardingTableRuleConfigurations( + Map tables) { + + List configurations = new ArrayList<>(tables.size()); + + tables.forEach((tableName, tableShardingRule) -> + configurations.add(createShardingTableRuleConfiguration(tableName, tableShardingRule))); + + return configurations; + } + + private ShardingTableRuleConfiguration createShardingTableRuleConfiguration( + String tableName, + ShardingProperties.ShardingsphereProperties.TableShardingRule tableShardingRule) { + log.info("[{}] Create sharding table rule configuration, tableShardingRule: {}", + tableName, tableShardingRule); + + ShardingTableRuleConfiguration configuration = new ShardingTableRuleConfiguration( + tableName, tableShardingRule.getActualDataNodes()); + if (tableShardingRule.getKeyGenerateStrategy() != null) { + log.info("[{}] Create key generate strategy configuration, column: {}, keyGeneratorName: {}", + tableName, + tableShardingRule.getKeyGenerateStrategy().getColumn(), + tableShardingRule.getKeyGenerateStrategy().getKeyGeneratorName()); + configuration.setKeyGenerateStrategy( + new KeyGenerateStrategyConfiguration( + tableShardingRule.getKeyGenerateStrategy().getColumn(), + tableShardingRule.getKeyGenerateStrategy().getKeyGeneratorName() + ) + ); + } + if (tableShardingRule.getDatabaseStrategy() != null) { + log.info("[{}] Create database sharding strategy, strategy: {}", + tableName, tableShardingRule.getDatabaseStrategy()); + configuration.setDatabaseShardingStrategy( + createShardingStrategyConfiguration(tableShardingRule.getDatabaseStrategy())); + } + if (tableShardingRule.getTableStrategy() != null) { + log.info("[{}] Create table sharding strategy, strategy: {}", + tableName, tableShardingRule.getTableStrategy()); + configuration.setTableShardingStrategy( + createShardingStrategyConfiguration(tableShardingRule.getTableStrategy())); + } + + return configuration; + } + + private ShardingStrategyConfiguration createShardingStrategyConfiguration( + ShardingProperties.ShardingsphereProperties.ShardingStrategy shardingStrategyProps) { + String shardingStrategyType = shardingStrategyProps.getType().trim().toLowerCase(); + switch (shardingStrategyType) { + case "standard": + return new StandardShardingStrategyConfiguration( + shardingStrategyProps.getShardingColumn(), + shardingStrategyProps.getShardingAlgorithmName() + ); + case "hint": + return new HintShardingStrategyConfiguration( + shardingStrategyProps.getShardingAlgorithmName() + ); + default: + throw new ShardingConfigParseException("Not support sharding algorithm"); + } + } + + + private Properties createShardingGlobalProps(Map globalPropsMap) { + // 配置 shardingsphere 系统级配置 + if (globalPropsMap == null || globalPropsMap.isEmpty()) { + return null; + } + return toProperties(globalPropsMap); + } + + @Qualifier("jobShardingTransactionManager") + @Bean(name = "jobShardingTransactionManager") + @DependsOn("jobShardingDataSource") + public DataSourceTransactionManager transactionManager( + @Qualifier("jobShardingDataSource") DataSource dataSource) { + return new DataSourceTransactionManager(dataSource); + } + + @Qualifier("job-sharding-jdbc-template") + @Bean(name = "job-sharding-jdbc-template") + public JdbcTemplate jdbcTemplate(@Qualifier("jobShardingDataSource") DataSource dataSource) { + return new JdbcTemplate(dataSource); + } + + @Qualifier("job-sharding-dsl-context") + @Bean(name = "job-sharding-dsl-context") + public DSLContext dslContext(@Qualifier("job-sharding-jooq-conf") org.jooq.Configuration configuration) { + return new DefaultDSLContext(configuration); + } + + @Qualifier("job-sharding-jooq-conf") + @Bean(name = "job-sharding-jooq-conf") + public org.jooq.Configuration jooqConf( + @Qualifier("job-sharding-conn-provider") ConnectionProvider connectionProvider, + DefaultExecuteListenerProvider jooqExecuteListenerProvider + ) { + return JooqConfigurationUtil.getConfiguration(connectionProvider, jooqExecuteListenerProvider); + } + + @Qualifier("job-sharding-conn-provider") + @Bean(name = "job-sharding-conn-provider") + public ConnectionProvider connectionProvider( + @Qualifier("jobShardingTransactionAwareDataSource") DataSource dataSource) { + return new DataSourceConnectionProvider(dataSource); + } + + @Qualifier("jobShardingTransactionAwareDataSource") + @Bean(name = "jobShardingTransactionAwareDataSource") + public TransactionAwareDataSourceProxy + transactionAwareDataSourceProxy(@Qualifier("jobShardingDataSource") DataSource dataSource) { + return new TransactionAwareDataSourceProxy(dataSource); + } +} diff --git a/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/config/ShardingProperties.java b/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/config/ShardingProperties.java new file mode 100644 index 0000000000..229fd1395a --- /dev/null +++ b/src/backend/commons/common-mysql-sharding/src/main/java/com/tencent/bk/job/common/sharding/mysql/config/ShardingProperties.java @@ -0,0 +1,247 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.sharding.mysql.config; + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.springframework.boot.context.properties.ConfigurationProperties; + +import java.util.List; +import java.util.Map; +import java.util.StringJoiner; + +/** + * 分库分表配置 + */ +@ConfigurationProperties(prefix = "sharding") +@Getter +@Setter +@ToString +public class ShardingProperties { + /** + * 是否启用分库分表 + */ + private boolean enabled; + /** + * 数据库实例数量 + */ + private Integer dbNodeCount; + /** + * 每个数据库实例上的表分片数量 + */ + private Integer tableNodeCount; + + /** + * 单 db -> 分库分表迁移配置 + */ + private MigrationProperties migration; + + /** + * ShardingSphere 分片组件配置 + */ + private ShardingsphereProperties shardingsphere; + + @Getter + @Setter + @ToString + public static class MigrationProperties { + private boolean enabled; + } + + @Getter + @Setter + @ToString + public static class ShardingsphereProperties { + /** + * 分片数据库名 + */ + private String databaseName; + /** + * 数据源集群。 key:集群名称；value:集群配置 + */ + private Map dataSources; + /** + * 分片规则 + */ + private ShardingRule shardingRule; + /** + * 系统级属性配置 + */ + private Map props; + + @Getter + @Setter + public static class DataSource { + /** + * 数据库驱动类名，以数据库连接池自身配置为准 + */ + private String driverClassName; + /** + * 数据库 URL 连接，以数据库连接池自身配置为准 + */ + private String jdbcUrl; + private String username; + private String password; + + @Override + public String toString() { + return new StringJoiner(", ", DataSource.class.getSimpleName() + "[", "]") + .add("driverClassName='" + driverClassName + "'") + .add("jdbcUrl='" + jdbcUrl + "'") + .add("username='" + username + "'") + .add("password='******'") + .toString(); + } + } + + + /** + * 数据分片规则配置 + */ + @Getter + @Setter + @ToString + public static class ShardingRule { + /** + * 表对应的分片规则。 key: 表名称;value: 分片规则 + */ + private Map tables; + /** + * 分片算法定义 + */ + private Map shardingAlgorithms; + /** + * 分布式序列算法配置 + */ + private Map keyGenerators; + /** + * 默认数据库分片策略 + */ + private ShardingStrategy defaultDatabaseStrategy; + /** + * 默认表分片策略 + */ + private ShardingStrategy defaultTableStrategy; + /** + * 默认分片列名称 + */ + private String defaultShardingColumn; + /** + * 默认的分布式序列策略 + */ + private String defaultKeyGenerateStrategy; + + private List bindingTables; + } + + + @Getter + @Setter + @ToString + public static class TableShardingRule { + /** + * 由数据源名 + 表名组成 + */ + private String actualDataNodes; + /** + * 分库策略 + */ + private ShardingStrategy databaseStrategy; + /** + * 分表策略，同分库策略 + */ + private ShardingStrategy tableStrategy; + /** + * 分布式序列策略 + */ + private KeyGenerateStrategy keyGenerateStrategy; + } + + /** + * 分片计算策略 + */ + @Getter + @Setter + @ToString + public static class ShardingStrategy { + private String type; + private String shardingColumn; + private String shardingAlgorithmName; + } + + /** + * 分布式序列策略 + */ + @Getter + @Setter + @ToString + public static class KeyGenerateStrategy { + /** + * 自增列名称，缺省表示不使用自增主键生成器 + */ + private String column; + /** + * 分布式序列算法名称 + */ + private String keyGeneratorName; + } + + /** + * 分布式序列生成配置 + */ + @Getter + @Setter + @ToString + public static class KeyGenerator { + /** + * 分布式序列算法类型 + */ + private String type; + /** + * 分布式序列算法属性配置 + */ + private Map props; + } + + /** + * 分片算法配置 + */ + @Getter + @Setter + @ToString + public static class ShardingAlgorithm { + /** + * 分片算法类型 + */ + private String type; + /** + * 分片算法属性配置 + */ + private Map props; + } + + } +} diff --git a/src/backend/commons/common-mysql-sharding/src/main/resources/META-INF/spring.factories b/src/backend/commons/common-mysql-sharding/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..f020f2ef71 --- /dev/null +++ b/src/backend/commons/common-mysql-sharding/src/main/resources/META-INF/spring.factories @@ -0,0 +1,3 @@ +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ +com.tencent.bk.job.common.sharding.mysql.config.LeafAutoConfiguration,\ +com.tencent.bk.job.common.sharding.mysql.config.ShardingDatasourceAutoConfiguration diff --git a/src/backend/commons/common-mysql/build.gradle b/src/backend/commons/common-mysql/build.gradle new file mode 100644 index 0000000000..47417dab21 --- /dev/null +++ b/src/backend/commons/common-mysql/build.gradle @@ -0,0 +1,34 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +dependencies { + api project(':commons:common-utils') + api project(':commons:common') + api 'org.springframework.boot:spring-boot-starter-jdbc' + api 'org.springframework.boot:spring-boot-configuration-processor' + api 'org.jooq:jooq' + api "org.jooq:jooq-codegen" + api "org.jooq:jooq-meta" + api 'io.micrometer:micrometer-registry-prometheus' +} diff --git a/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/JobTransactional.java b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/JobTransactional.java new file mode 100644 index 0000000000..8f6b2a5dd8 --- /dev/null +++ b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/JobTransactional.java @@ -0,0 +1,55 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.mysql; + +import org.springframework.core.annotation.AliasFor; +import org.springframework.transaction.TransactionDefinition; +import org.springframework.transaction.annotation.Transactional; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Job 自定义事务注解 + */ +@Target({ElementType.TYPE, ElementType.METHOD}) +@Retention(RetentionPolicy.RUNTIME) +@Inherited +@Documented +@Transactional +public @interface JobTransactional { + @AliasFor(annotation = Transactional.class) + String transactionManager(); + + @AliasFor(annotation = Transactional.class) + Class[] rollbackFor() default {Throwable.class}; + + @AliasFor(annotation = Transactional.class) + int timeout() default TransactionDefinition.TIMEOUT_DEFAULT; +} diff --git a/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/MySQLAutoConfiguration.java b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/MySQLAutoConfiguration.java new file mode 100644 index 0000000000..f451732237 --- /dev/null +++ b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/MySQLAutoConfiguration.java @@ -0,0 +1,42 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.mysql; + +import com.tencent.bk.job.common.mysql.listener.JooqExecuteListener; +import io.micrometer.core.instrument.MeterRegistry; +import org.jooq.impl.DefaultExecuteListenerProvider; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +@EnableConfigurationProperties(MySQLProperties.class) +public class MySQLAutoConfiguration { + @Bean + public DefaultExecuteListenerProvider jooqExecuteListenerProvider(MySQLProperties mySQLProperties, + MeterRegistry registry) { + return new DefaultExecuteListenerProvider(new JooqExecuteListener(mySQLProperties, registry)); + } +} diff --git a/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/MySQLProperties.java b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/MySQLProperties.java new file mode 100644 index 0000000000..3ae8646161 --- /dev/null +++ b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/MySQLProperties.java @@ -0,0 +1,87 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.mysql; + +import com.tencent.bk.job.common.mysql.dynamic.ds.DataSourceMode; +import com.tencent.bk.job.common.mysql.dynamic.ds.MigrationProperties; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.springframework.boot.context.properties.ConfigurationProperties; + +/** + * mysql db 配置 + */ +@ConfigurationProperties(prefix = "mysql") +@Getter +@Setter +@ToString +public class MySQLProperties { + + /** + * 数据源模式（单库/垂直分库/水平分库) + */ + private String dataSourceMode = DataSourceMode.Constants.STANDALONE; + + private StandaloneProperties standalone; + + private VerticalShardingProperties verticalSharding; + + /** + * DB 迁移配置 + */ + private MigrationProperties migration; + + /** + * 慢查询SQL配置 + */ + private SlowSqlProperties slowSql = new SlowSqlProperties(); + + @Getter + @Setter + public static class SlowSqlProperties { + /** + * 是否在日志中打印慢查询SQL + */ + private boolean logEnabled = true; + /** + * 慢查询阈值（ms) + */ + private Integer thresholdMillis = 5000; + } + + @Getter + @Setter + public static class VerticalShardingProperties { + private boolean enabled; + } + + @Getter + @Setter + public static class StandaloneProperties { + private boolean enabled; + } + +} diff --git a/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dao/BaseDAOImpl.java b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dao/BaseDAOImpl.java new file mode 100644 index 0000000000..ae2b6cb477 --- /dev/null +++ b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dao/BaseDAOImpl.java @@ -0,0 +1,96 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.mysql.dao; + +import lombok.extern.slf4j.Slf4j; +import org.jooq.Record; +import org.jooq.Result; +import org.jooq.ResultQuery; +import org.jooq.SelectConditionStep; +import org.jooq.SelectLimitStep; +import org.jooq.conf.ParamType; +import org.slf4j.helpers.MessageFormatter; + +import java.util.Collections; +import java.util.List; + +/** + * 抽取通用查询方法 + */ +@Slf4j +public class BaseDAOImpl { + public List listPage(SelectLimitStep query, + Integer start, + Integer pageSize, + RecordDTOConverter converter) { + Result records = null; + if (start == null || start < 0) { + start = 0; + } + if (pageSize == null || pageSize < 0) { + pageSize = -1; + } + ResultQuery finalQuery = query; + String sql = ""; + try { + if (pageSize >= 0) { + finalQuery = query.limit(start, pageSize); + } + sql = finalQuery.getSQL(ParamType.INLINED); + log.debug("SQL={}", sql); + records = finalQuery.fetch(); + } catch (Exception e) { + String msg = MessageFormatter.format( + "error SQL={}", + sql + ).getMessage(); + log.error(msg, e); + } + if (records == null || records.isEmpty()) { + return Collections.emptyList(); + } else { + return records.map(converter::convert); + } + } + + public DTOClazz fetchOne(SelectConditionStep query, + RecordDTOConverter converter) { + try { + if (log.isDebugEnabled()) { + log.debug("SQL={}", query.getSQL(ParamType.INLINED)); + } + RecordClazz record = query.fetchOne(); + return record == null ? null : converter.convert(record); + } catch (Exception e) { + String msg = MessageFormatter.format( + "error SQL={}", + query.getSQL(ParamType.INLINED) + ).getMessage(); + log.error(msg, e); + throw e; + } + } + +} diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/RecordDTOConverter.java b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dao/RecordDTOConverter.java similarity index 91% rename from src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/RecordDTOConverter.java rename to src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dao/RecordDTOConverter.java index 1c1f171b98..f4801c0faf 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/RecordDTOConverter.java +++ b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dao/RecordDTOConverter.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.file_gateway.dao.filesource.impl; +package com.tencent.bk.job.common.mysql.dao; public interface RecordDTOConverter { DTOClazz convert(RecordClazz record); diff --git a/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/ds/DSLContextProvider.java b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/ds/DSLContextProvider.java new file mode 100644 index 0000000000..626debfcf6 --- /dev/null +++ b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/ds/DSLContextProvider.java @@ -0,0 +1,35 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.mysql.dynamic.ds; + +import org.jooq.DSLContext; + +/** + * Jooq DSLContext 提供者 + */ +public interface DSLContextProvider { + DSLContext get(String tableName); +} + diff --git a/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/ds/DataSourceMode.java b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/ds/DataSourceMode.java new file mode 100644 index 0000000000..46e3e00a60 --- /dev/null +++ b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/ds/DataSourceMode.java @@ -0,0 +1,82 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.mysql.dynamic.ds; + +import lombok.Getter; + +/** + * 数据源模式 + */ +@Getter +public enum DataSourceMode { + /** + * 单点 DB + */ + STANDALONE(Constants.STANDALONE), + /** + * 垂直分库 + */ + VERTICAL_SHARDING(Constants.VERTICAL_SHARDING), + /** + * 水平分库 + */ + HORIZONTAL_SHARDING(Constants.HORIZONTAL_SHARDING); + + public static class Constants { + public final static String STANDALONE = "standalone"; + public final static String VERTICAL_SHARDING = "vertical_sharding"; + public final static String HORIZONTAL_SHARDING = "horizontal_sharding"; + } + + private final String mode; + + DataSourceMode(String mode) { + this.mode = mode; + } + + public static DataSourceMode valOf(String mode) { + if (mode == null) { + return null; + } + for (DataSourceMode value : values()) { + if (value.getMode().equals(mode)) { + return value; + } + } + throw new IllegalArgumentException("No DataSourceMode constant: " + mode); + } + + public static boolean checkValid(String mode) { + if (mode == null) { + return false; + } + for (DataSourceMode value : values()) { + if (value.getMode().equals(mode)) { + return true; + } + } + return false; + } +} diff --git a/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/ds/DbOperationEnum.java b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/ds/DbOperationEnum.java new file mode 100644 index 0000000000..22a36cc4cc --- /dev/null +++ b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/ds/DbOperationEnum.java @@ -0,0 +1,46 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.mysql.dynamic.ds; + +public enum DbOperationEnum { + /** + * 读操作 + */ + READ(1), + /** + * 写操作 + */ + WRITE(2); + + DbOperationEnum(int op) { + this.op = op; + } + + private final int op; + + public int getValue() { + return op; + } +} diff --git a/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/ds/DynamicDSLContextProvider.java b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/ds/DynamicDSLContextProvider.java new file mode 100644 index 0000000000..299e2063be --- /dev/null +++ b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/ds/DynamicDSLContextProvider.java @@ -0,0 +1,31 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.mysql.dynamic.ds; + +public interface DynamicDSLContextProvider extends DSLContextProvider { + + void setProvider(DSLContextProvider provider); + +} diff --git a/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/ds/HorizontalShardingDSLContextProvider.java b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/ds/HorizontalShardingDSLContextProvider.java new file mode 100644 index 0000000000..00ce966f62 --- /dev/null +++ b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/ds/HorizontalShardingDSLContextProvider.java @@ -0,0 +1,46 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.mysql.dynamic.ds; + +import lombok.extern.slf4j.Slf4j; +import org.jooq.DSLContext; + +/** + * 水平分库 MySQL Jooq DSLContext 提供者 + */ +@Slf4j +public class HorizontalShardingDSLContextProvider implements DSLContextProvider { + + private final DSLContext dslContext; + + public HorizontalShardingDSLContextProvider(DSLContext dslContext) { + this.dslContext = dslContext; + } + + @Override + public DSLContext get(String tableName) { + return dslContext; + } +} diff --git a/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/ds/MigrateDynamicDSLContextProvider.java b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/ds/MigrateDynamicDSLContextProvider.java new file mode 100644 index 0000000000..88b93f3f25 --- /dev/null +++ b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/ds/MigrateDynamicDSLContextProvider.java @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.mysql.dynamic.ds; + +import lombok.extern.slf4j.Slf4j; +import org.jooq.DSLContext; + +/** + * DB 迁移专用 DynamicDSLContextProvider, 用于动态切换数据源 + */ +@Slf4j +public class MigrateDynamicDSLContextProvider implements DynamicDSLContextProvider { + + private final ThreadLocal threadLocalDSLContextProvider = new ThreadLocal<>(); + + @Override + public DSLContext get(String tableName) { + return threadLocalDSLContextProvider.get().get(tableName); + } + + @Override + public void setProvider(DSLContextProvider provider) { + threadLocalDSLContextProvider.set(provider); + } +} diff --git a/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/ds/MigrationProperties.java b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/ds/MigrationProperties.java new file mode 100644 index 0000000000..1e980cf0d5 --- /dev/null +++ b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/ds/MigrationProperties.java @@ -0,0 +1,42 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.mysql.dynamic.ds; + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +/** + * mysql db 切换配置 + */ +@Getter +@Setter +@ToString +public class MigrationProperties { + /** + * 是否启用数据源切换模式 + */ + private boolean enabled; +} diff --git a/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/ds/MigrationStatus.java b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/ds/MigrationStatus.java new file mode 100644 index 0000000000..0af092f016 --- /dev/null +++ b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/ds/MigrationStatus.java @@ -0,0 +1,73 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.mysql.dynamic.ds; + +import lombok.Getter; + +/** + * DB 迁移状态 + */ +@Getter +public enum MigrationStatus { + /** + * 未启动 + */ + NOT_START(1), + /** + * 准备迁移 + */ + PREPARING(2), + /** + * 迁移中 + */ + MIGRATING(3), + /** + * 迁移完成 + */ + MIGRATED(4), + /** + * 迁移失败 + */ + FAIL(5); + + MigrationStatus(int status) { + this.status = status; + } + + private final int status; + + public static MigrationStatus valOf(Integer status) { + if (status == null) { + return null; + } + for (MigrationStatus value : values()) { + if (value.getStatus() == status) { + return value; + } + } + throw new IllegalArgumentException("No MigrationStatus constant: " + status); + } + +} diff --git a/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/ds/MySQLOperation.java b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/ds/MySQLOperation.java new file mode 100644 index 0000000000..a5a51d225e --- /dev/null +++ b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/ds/MySQLOperation.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.mysql.dynamic.ds; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * mysql db 操作注解 + */ +@Retention(RetentionPolicy.RUNTIME) +@Target(ElementType.METHOD) +@Inherited +public @interface MySQLOperation { + String table(); + + DbOperationEnum op(); + +} diff --git a/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/ds/StandaloneDSLContextProvider.java b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/ds/StandaloneDSLContextProvider.java new file mode 100644 index 0000000000..bb910a1d57 --- /dev/null +++ b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/ds/StandaloneDSLContextProvider.java @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.mysql.dynamic.ds; + +import lombok.extern.slf4j.Slf4j; +import org.jooq.DSLContext; + +/** + * 单点 MySQL Jooq DSLContext 提供者 + */ +@Slf4j +public class StandaloneDSLContextProvider implements DSLContextProvider { + + + private final DSLContext dslContext; + + public StandaloneDSLContextProvider(DSLContext dslContext) { + this.dslContext = dslContext; + } + + @Override + public DSLContext get(String tableName) { + return dslContext; + } +} diff --git a/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/ds/VerticalShardingDSLContextProvider.java b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/ds/VerticalShardingDSLContextProvider.java new file mode 100644 index 0000000000..a8c05d169d --- /dev/null +++ b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/ds/VerticalShardingDSLContextProvider.java @@ -0,0 +1,54 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.mysql.dynamic.ds; + +import com.tencent.bk.job.common.exception.InternalException; +import lombok.extern.slf4j.Slf4j; +import org.jooq.DSLContext; + +import java.util.Map; + +/** + * 垂直分库 MySQL Jooq DSLContext 提供者 + */ +@Slf4j +public class VerticalShardingDSLContextProvider implements DSLContextProvider { + + private Map tableNameAndDslContextMap = null; + + public void initTableNameAndDslContextMap(Map tableNameAndDslContextMap) { + this.tableNameAndDslContextMap = tableNameAndDslContextMap; + } + + @Override + public DSLContext get(String tableName) { + DSLContext dslContext = tableNameAndDslContextMap.get(tableName); + if (dslContext == null) { + log.error("DSLContext not found for table : {}", tableName); + throw new InternalException("DSLContext not found"); + } + return dslContext; + } +} diff --git a/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/id/IdGenType.java b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/id/IdGenType.java new file mode 100644 index 0000000000..7dfe5611a1 --- /dev/null +++ b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/dynamic/id/IdGenType.java @@ -0,0 +1,68 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.mysql.dynamic.id; + +import lombok.Getter; + +@Getter +public enum IdGenType { + AUTO_INCREMENT(IdGenType.Constants.AUTO_INCREMENT), + SEGMENT(IdGenType.Constants.LEAF_SEGMENT); + + public static class Constants { + public final static String AUTO_INCREMENT = "auto_increment"; + public final static String LEAF_SEGMENT = "leaf_segment"; + } + + private final String type; + + IdGenType(String type) { + this.type = type; + } + + public static IdGenType valOf(String type) { + if (type == null) { + return null; + } + for (IdGenType value : values()) { + if (value.getType().equals(type)) { + return value; + } + } + throw new IllegalArgumentException("No IdGenType constant: " + type); + } + + public static boolean checkValid(String type) { + if (type == null) { + return false; + } + for (IdGenType value : values()) { + if (value.getType().equals(type)) { + return true; + } + } + return false; + } +} diff --git a/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/jooq/JooqDataTypeUtil.java b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/jooq/JooqDataTypeUtil.java new file mode 100644 index 0000000000..e12dd3b02a --- /dev/null +++ b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/jooq/JooqDataTypeUtil.java @@ -0,0 +1,59 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.mysql.jooq; + +import org.jooq.types.UByte; + +public class JooqDataTypeUtil { + + public static UByte toUByte(Integer value) { + if (value == null) { + return null; + } + return UByte.valueOf(value); + } + + public static Byte toByte(Integer value) { + if (value == null) { + return null; + } + return Byte.valueOf(String.valueOf(value)); + } + + public static Short toShort(Integer value) { + if (value == null) { + return null; + } + return Short.valueOf(String.valueOf(value)); + } + + public static Integer toInteger(Byte value) { + if (value == null) { + return null; + } + return value.intValue(); + } + +} diff --git a/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/listener/JooqExecuteListener.java b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/listener/JooqExecuteListener.java new file mode 100644 index 0000000000..2f99dd3199 --- /dev/null +++ b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/listener/JooqExecuteListener.java @@ -0,0 +1,152 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.mysql.listener; + +import com.tencent.bk.job.common.mysql.MySQLProperties; +import com.tencent.bk.job.common.mysql.metrics.MetricsConstants; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tags; +import io.micrometer.core.instrument.Timer; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.jooq.ExecuteContext; +import org.jooq.impl.DefaultExecuteListener; + +import java.time.Duration; +import java.util.concurrent.TimeUnit; + +/** + * jOOQ执行监听器，用于监控慢SQL + */ +@Slf4j +public class JooqExecuteListener extends DefaultExecuteListener { + + private final MySQLProperties mySQLProperties; + private final MeterRegistry meterRegistry; + + private final String KEY_START_TIME = "sqlStartTimeMillis"; + + public JooqExecuteListener(MySQLProperties mySQLProperties, MeterRegistry meterRegistry) { + this.mySQLProperties = mySQLProperties; + this.meterRegistry = meterRegistry; + } + + @Override + public void executeStart(ExecuteContext ctx) { + super.executeStart(ctx); + ctx.data(KEY_START_TIME, System.currentTimeMillis()); + } + + @Override + public void executeEnd(ExecuteContext ctx) { + super.executeEnd(ctx); + tryToAnalysisExecuteTime(ctx); + } + + /** + * 尝试分析SQL执行时间，如果异常则打印WARN日志，不影响主流程 + * + * @param ctx 执行上下文 + */ + @SuppressWarnings("DataFlowIssue") + private void tryToAnalysisExecuteTime(ExecuteContext ctx) { + try { + Long startTimeMillis = (Long) ctx.data(KEY_START_TIME); + long costTimeMillis = System.currentTimeMillis() - startTimeMillis; + String sql = parseSql(ctx); + recordSqlExecuteTimeConsuming(sql, costTimeMillis); + if (shouldLogSlowSql(costTimeMillis)) { + // 打印慢SQL日志 + log.warn( + "SlowSQL cost {}ms. SQL=[{}]", + costTimeMillis, + sql + ); + } + } catch (Exception e) { + log.warn("tryToAnalysisExecuteTime failed", e); + } + } + + /** + * 判断是否需要打印慢SQL + * + * @param costTimeMillis SQL执行耗时 + * @return 是否需要打印慢SQL + */ + private boolean shouldLogSlowSql(long costTimeMillis) { + return mySQLProperties.getSlowSql().isLogEnabled() + && costTimeMillis > mySQLProperties.getSlowSql().getThresholdMillis(); + } + + /** + * 从执行上下文中解析真实执行的SQL + * + * @param ctx 执行上下文 + * @return 真实执行的SQL + */ + private String parseSql(ExecuteContext ctx) { + return ctx.sql(); + } + + /** + * 记录SQL执行耗时到指标数据 + * + * @param sql 执行的SQL + * @param costTimeMillis 耗时 + */ + private void recordSqlExecuteTimeConsuming(String sql, long costTimeMillis) { + // 1.分析出SQL类型 + String sqlType = parseSqlType(sql); + // 2.记录到MeterRegistry + Timer.builder(MetricsConstants.NAME_SQL_EXECUTE_TIME) + .description("SQL Execute Time") + .tags(Tags.of(MetricsConstants.TAG_KEY_TYPE, sqlType)) + .publishPercentileHistogram(false) + .publishPercentiles(0.5, 0.8, 0.95, 0.99) + .minimumExpectedValue(Duration.ofMillis(10)) + .maximumExpectedValue(Duration.ofMinutes(1L)) + .register(meterRegistry) + .record(costTimeMillis, TimeUnit.MILLISECONDS); + } + + /** + * 解析SQL类型：SELECT、INSERT、UPDATE、DELETE、UNKNOWN + * + * @param sql 待解析的SQL + * @return SQL类型 + */ + private String parseSqlType(String sql) { + if (StringUtils.isBlank(sql)) { + return "UNKNOWN"; + } + sql = sql.trim().replaceAll("[\r\n]+", " "); + int indexOfSpace = sql.indexOf(" "); + if (indexOfSpace < 0) { + return sql; + } + return sql.substring(0, indexOfSpace).toUpperCase(); + } +} diff --git a/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/metrics/MetricsConstants.java b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/metrics/MetricsConstants.java new file mode 100644 index 0000000000..5093b77684 --- /dev/null +++ b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/metrics/MetricsConstants.java @@ -0,0 +1,34 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.mysql.metrics; + +public class MetricsConstants { + + // metric name + public static final String NAME_SQL_EXECUTE_TIME = "sql.execute.time"; + + // tag + public static final String TAG_KEY_TYPE = "type"; +} diff --git a/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/util/JooqConfigurationUtil.java b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/util/JooqConfigurationUtil.java new file mode 100644 index 0000000000..b3a01df7ec --- /dev/null +++ b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/util/JooqConfigurationUtil.java @@ -0,0 +1,52 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.mysql.util; + +import org.jooq.ConnectionProvider; +import org.jooq.SQLDialect; +import org.jooq.impl.DefaultConfiguration; +import org.jooq.impl.DefaultExecuteListenerProvider; + +/** + * jOOQ 配置工具类 + */ +public class JooqConfigurationUtil { + + /** + * 获取jOOQ配置 + * + * @param connectionProvider 连接提供者 + * @param executeListenerProvider 执行监听器提供者 + * @return jOOQ配置 + */ + public static org.jooq.Configuration getConfiguration(ConnectionProvider connectionProvider, + DefaultExecuteListenerProvider executeListenerProvider) { + org.jooq.Configuration configuration = new DefaultConfiguration() + .derive(connectionProvider) + .derive(SQLDialect.MYSQL); + configuration.set(executeListenerProvider); + return configuration; + } +} diff --git a/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/util/JooqDataTypeUtil.java b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/util/JooqDataTypeUtil.java new file mode 100644 index 0000000000..fd0b4525eb --- /dev/null +++ b/src/backend/commons/common-mysql/src/main/java/com/tencent/bk/job/common/mysql/util/JooqDataTypeUtil.java @@ -0,0 +1,101 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.mysql.util; + +import org.jooq.types.UByte; +import org.jooq.types.UInteger; +import org.jooq.types.ULong; + +public class JooqDataTypeUtil { + public static ULong buildULong(Long value) { + if (value == null) { + return null; + } + return ULong.valueOf(value); + } + + public static Long buildLong(ULong value) { + if (value == null) { + return null; + } + return value.longValue(); + } + + public static UInteger buildUInteger(Integer value) { + if (value == null) { + return null; + } + return UInteger.valueOf(value); + } + + public static UByte buildUByte(Integer value) { + if (value == null) { + return null; + } + return UByte.valueOf(value); + } + + public static Byte getByteFromInteger(Integer value) { + if (value == null) { + return null; + } + return Byte.valueOf(String.valueOf(value)); + } + + public static Integer getIntegerFromByte(Byte value) { + if (value == null) { + return null; + } + return value.intValue(); + } + + public static Long getLongFromULong(ULong value) { + if (value == null) { + return null; + } + return value.longValue(); + } + + public static Integer getIntegerFromUInteger(UInteger value) { + if (value == null) { + return null; + } + return value.intValue(); + } + + public static Integer getIntegerFromShort(Short value) { + if (value == null) { + return null; + } + return value.intValue(); + } + + public static Short getShortFromInteger(Integer value) { + if (value == null) { + return null; + } + return value.shortValue(); + } +} diff --git a/src/backend/commons/common-mysql/src/main/resources/META-INF/spring.factories b/src/backend/commons/common-mysql/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..eac6d3c5f2 --- /dev/null +++ b/src/backend/commons/common-mysql/src/main/resources/META-INF/spring.factories @@ -0,0 +1,4 @@ +# AutoConfiguration +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ +com.tencent.bk.job.common.mysql.MySQLAutoConfiguration + diff --git a/src/backend/commons/common-otel/build.gradle b/src/backend/commons/common-otel/build.gradle index 6b4da9a8b9..94f59c741c 100644 --- a/src/backend/commons/common-otel/build.gradle +++ b/src/backend/commons/common-otel/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -23,11 +23,10 @@ */ dependencies { - api "org.springframework.cloud:spring-cloud-sleuth-api" - api "org.springframework.cloud:spring-cloud-sleuth-otel" - api "org.springframework.cloud:spring-cloud-sleuth-otel-autoconfigure" + api('org.springframework.cloud:spring-cloud-starter-sleuth') { + exclude group: 'org.springframework.cloud', module: 'spring-cloud-sleuth-brave' + } + api 'org.springframework.cloud:spring-cloud-sleuth-otel-autoconfigure' api "io.opentelemetry.instrumentation:opentelemetry-jdbc" - api "io.opentelemetry:opentelemetry-exporter-otlp-trace" - // https://mvnrepository.com/artifact/io.opentelemetry/opentelemetry-sdk-extension-resources - api 'io.opentelemetry:opentelemetry-sdk-extension-resources' + api "io.opentelemetry:opentelemetry-exporter-otlp" } diff --git a/src/backend/commons/common-otel/src/main/java/com/tencent/bk/job/common/tracing/JobOtelAutoConfiguration.java b/src/backend/commons/common-otel/src/main/java/com/tencent/bk/job/common/tracing/JobOtelAutoConfiguration.java new file mode 100644 index 0000000000..9d61ae9a5c --- /dev/null +++ b/src/backend/commons/common-otel/src/main/java/com/tencent/bk/job/common/tracing/JobOtelAutoConfiguration.java @@ -0,0 +1,156 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.tracing; + +import io.opentelemetry.api.OpenTelemetry; +import io.opentelemetry.api.common.Attributes; +import io.opentelemetry.api.common.AttributesBuilder; +import io.opentelemetry.context.propagation.ContextPropagators; +import io.opentelemetry.sdk.OpenTelemetrySdk; +import io.opentelemetry.sdk.resources.Resource; +import io.opentelemetry.sdk.trace.SdkTracerProvider; +import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder; +import io.opentelemetry.sdk.trace.SpanLimits; +import io.opentelemetry.sdk.trace.SpanProcessor; +import io.opentelemetry.sdk.trace.export.BatchSpanProcessor; +import io.opentelemetry.sdk.trace.export.BatchSpanProcessorBuilder; +import io.opentelemetry.sdk.trace.export.SpanExporter; +import io.opentelemetry.sdk.trace.samplers.Sampler; +import io.opentelemetry.semconv.resource.attributes.ResourceAttributes; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.AutoConfigureBefore; +import org.springframework.cloud.sleuth.autoconfig.otel.OtelAutoConfiguration; +import org.springframework.cloud.sleuth.autoconfig.otel.OtelProcessorProperties; +import org.springframework.cloud.sleuth.autoconfig.otel.SpanProcessorProvider; +import org.springframework.cloud.sleuth.otel.bridge.SpanExporterCustomizer; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Profile; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +/** + * Otel 配置 + * + *

在跑测试用例启动时，会启动spring容器，会多次初始化OtelConfiguration。由于OtelConfiguration.otel()->OtelConfiguration.otel() + * .buildAndRegisterGlobal()-> GlobalOpenTelemetry.set多次调用，从而抛出异常"Failed to instantiate [io + * .opentelemetry.api.OpenTelemetry]: Factory method 'otel' threw exception; nested exception is java.lang + * .IllegalStateException: GlobalOpenTelemetry.set has already been called. GlobalOpenTelemetry.set must be called + * only once before any calls to GlobalOpenTelemetry.get"

+ * + * @see io.opentelemetry.api.GlobalOpenTelemetry + */ +@Slf4j +@Configuration(proxyBeanMethods = false) +@Profile("!test") +@AutoConfigureBefore(OtelAutoConfiguration.class) +class JobOtelAutoConfiguration { + + @Value("${spring.sleuth.otel.exporter.enabled:false}") + private boolean exporterEnabled; + + @Value("${spring.sleuth.otel.resource.bkDataToken:}") + private String bkDataToken; + + @Bean + Supplier otelBkDataTokenResourceProvider() { + return this::buildResource; + } + + Resource buildResource() { + AttributesBuilder attributes = Attributes.builder(); + attributes.put("bk.data.token", bkDataToken); + return Resource.create(attributes.build(), ResourceAttributes.SCHEMA_URL); + } + + @Bean + SdkTracerProvider otelTracerProvider(SpanLimits spanLimits, + ObjectProvider> spanProcessors, + SpanExporterCustomizer spanExporterCustomizer, + ObjectProvider> spanExporters, + Sampler sampler, + Resource resource, + SpanProcessorProvider spanProcessorProvider) { + log.debug("exporterEnabled={},bkDataToken={}", exporterEnabled, bkDataToken); + SdkTracerProviderBuilder sdkTracerProviderBuilder = SdkTracerProvider.builder().setResource(resource) + .setSampler(sampler).setSpanLimits(spanLimits); + List processors = spanProcessors.getIfAvailable(ArrayList::new); + if (exporterEnabled) { + processors.addAll(spanExporters.getIfAvailable(ArrayList::new).stream() + .map(e -> spanProcessorProvider.toSpanProcessor(spanExporterCustomizer.customize(e))) + .collect(Collectors.toList())); + } + processors.forEach(sdkTracerProviderBuilder::addSpanProcessor); + return sdkTracerProviderBuilder.build(); + } + + @Bean + SpanProcessorProvider otelBatchSpanProcessorProvider(OtelProcessorProperties otelProcessorProperties) { + return new SpanProcessorProvider() { + @Override + public SpanProcessor toSpanProcessor(SpanExporter spanExporter) { + BatchSpanProcessorBuilder builder = BatchSpanProcessor.builder(spanExporter); + setBuilderProperties(otelProcessorProperties, builder); + return builder.build(); + } + + private void setBuilderProperties(OtelProcessorProperties otelProcessorProperties, + BatchSpanProcessorBuilder builder) { + if (otelProcessorProperties.getBatch().getExporterTimeout() != null) { + builder.setExporterTimeout(otelProcessorProperties.getBatch().getExporterTimeout(), + TimeUnit.MILLISECONDS); + } + if (otelProcessorProperties.getBatch().getMaxExportBatchSize() != null) { + builder.setMaxExportBatchSize(otelProcessorProperties.getBatch().getMaxExportBatchSize()); + } + if (otelProcessorProperties.getBatch().getMaxQueueSize() != null) { + builder.setMaxQueueSize(otelProcessorProperties.getBatch().getMaxQueueSize()); + } + if (otelProcessorProperties.getBatch().getScheduleDelay() != null) { + builder.setScheduleDelay(otelProcessorProperties.getBatch().getScheduleDelay(), + TimeUnit.MILLISECONDS); + } + } + }; + } + + @Bean + OpenTelemetry otel(SdkTracerProvider tracerProvider, ContextPropagators contextPropagators) { + OpenTelemetry openTelemetry = + OpenTelemetrySdk + .builder() + .setTracerProvider(tracerProvider) + .setPropagators(contextPropagators) + .buildAndRegisterGlobal(); + log.info("GlobalOpenTelemetry has been set"); + return openTelemetry; + } +} diff --git a/src/backend/commons/common-otel/src/main/java/com/tencent/bk/job/common/tracing/OtelConfiguration.java b/src/backend/commons/common-otel/src/main/java/com/tencent/bk/job/common/tracing/OtelConfiguration.java deleted file mode 100644 index 40a457758c..0000000000 --- a/src/backend/commons/common-otel/src/main/java/com/tencent/bk/job/common/tracing/OtelConfiguration.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.tracing; - -import io.opentelemetry.api.GlobalOpenTelemetry; -import io.opentelemetry.api.OpenTelemetry; -import io.opentelemetry.api.common.Attributes; -import io.opentelemetry.api.common.AttributesBuilder; -import io.opentelemetry.context.propagation.ContextPropagators; -import io.opentelemetry.sdk.OpenTelemetrySdk; -import io.opentelemetry.sdk.resources.Resource; -import io.opentelemetry.sdk.trace.SdkTracerProvider; -import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder; -import io.opentelemetry.sdk.trace.SpanLimits; -import io.opentelemetry.sdk.trace.SpanProcessor; -import io.opentelemetry.sdk.trace.export.SpanExporter; -import io.opentelemetry.sdk.trace.samplers.Sampler; -import io.opentelemetry.semconv.resource.attributes.ResourceAttributes; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.ObjectProvider; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.cloud.sleuth.autoconfig.otel.SpanProcessorProvider; -import org.springframework.cloud.sleuth.otel.bridge.SpanExporterCustomizer; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -import java.util.ArrayList; -import java.util.List; -import java.util.function.Supplier; -import java.util.stream.Collectors; - -@Slf4j -@Configuration(proxyBeanMethods = false) -class OtelConfiguration { - - @Value("${spring.sleuth.otel.exporter.enabled:false}") - private boolean exporterEnabled; - - @Value("${spring.sleuth.otel.resource.bkDataToken:}") - private String bkDataToken; - - @Bean - Supplier otelBkDataTokenResourceProvider() { - return this::buildResource; - } - - Resource buildResource() { - AttributesBuilder attributes = Attributes.builder(); - attributes.put("bk.data.token", bkDataToken); - return Resource.create(attributes.build(), ResourceAttributes.SCHEMA_URL); - } - - @Bean - SdkTracerProvider otelTracerProvider(SpanLimits spanLimits, - ObjectProvider> spanProcessors, - SpanExporterCustomizer spanExporterCustomizer, - ObjectProvider> spanExporters, - Sampler sampler, - Resource resource, - SpanProcessorProvider spanProcessorProvider) { - log.debug("exporterEnabled={},bkDataToken={}", exporterEnabled, bkDataToken); - SdkTracerProviderBuilder sdkTracerProviderBuilder = SdkTracerProvider.builder().setResource(resource) - .setSampler(sampler).setSpanLimits(spanLimits); - List processors = spanProcessors.getIfAvailable(ArrayList::new); - if (exporterEnabled) { - processors.addAll(spanExporters.getIfAvailable(ArrayList::new).stream() - .map(e -> spanProcessorProvider.toSpanProcessor(spanExporterCustomizer.customize(e))) - .collect(Collectors.toList())); - } - processors.forEach(sdkTracerProviderBuilder::addSpanProcessor); - return sdkTracerProviderBuilder.build(); - } - - @Bean - OpenTelemetry otel(SdkTracerProvider tracerProvider, ContextPropagators contextPropagators) { - OpenTelemetry openTelemetry = - OpenTelemetrySdk.builder().setTracerProvider(tracerProvider).setPropagators(contextPropagators).build(); - GlobalOpenTelemetry.set(openTelemetry); - log.info("GlobalOpenTelemetry has been set"); - return openTelemetry; - } -} diff --git a/src/backend/commons/common-otel/src/main/java/com/tencent/bk/job/common/tracing/util/SpanUtil.java b/src/backend/commons/common-otel/src/main/java/com/tencent/bk/job/common/tracing/util/SpanUtil.java index 7e3face06d..526840eae2 100644 --- a/src/backend/commons/common-otel/src/main/java/com/tencent/bk/job/common/tracing/util/SpanUtil.java +++ b/src/backend/commons/common-otel/src/main/java/com/tencent/bk/job/common/tracing/util/SpanUtil.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-otel/src/main/java/com/tencent/bk/job/common/tracing/util/TraceUtil.java b/src/backend/commons/common-otel/src/main/java/com/tencent/bk/job/common/tracing/util/TraceUtil.java new file mode 100644 index 0000000000..685c778f74 --- /dev/null +++ b/src/backend/commons/common-otel/src/main/java/com/tencent/bk/job/common/tracing/util/TraceUtil.java @@ -0,0 +1,52 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.tracing.util; + +import io.opentelemetry.api.trace.Span; +import io.opentelemetry.api.trace.SpanContext; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class TraceUtil { + + /** + * 获取当前Scope内的TraceID + * + * @return TraceId + */ + public static String getTraceIdFromCurrentSpan() { + Span currentSpan = Span.current(); + if (currentSpan == null) { + log.debug("currentSpan is null"); + return null; + } + SpanContext spanContext = currentSpan.getSpanContext(); + if (spanContext == null) { + log.debug("spanContext is null"); + return null; + } + return spanContext.getTraceId(); + } +} diff --git a/src/backend/commons/common-otel/src/main/resources/META-INF/spring.factories b/src/backend/commons/common-otel/src/main/resources/META-INF/spring.factories index e69de29bb2..82e3c4f025 100644 --- a/src/backend/commons/common-otel/src/main/resources/META-INF/spring.factories +++ b/src/backend/commons/common-otel/src/main/resources/META-INF/spring.factories @@ -0,0 +1,2 @@ +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ +com.tencent.bk.job.common.tracing.JobOtelAutoConfiguration diff --git a/src/backend/commons/common-rabbitmq/build.gradle b/src/backend/commons/common-rabbitmq/build.gradle new file mode 100644 index 0000000000..8bfbface40 --- /dev/null +++ b/src/backend/commons/common-rabbitmq/build.gradle @@ -0,0 +1,29 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software', to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +dependencies { + api "org.springframework:spring-context" + api "org.springframework.boot:spring-boot-autoconfigure" + api "com.rabbitmq:amqp-client" +} diff --git a/src/backend/commons/common-rabbitmq/src/main/java/com/tencent/bk/job/common/rabbitmq/config/JobRabbitMQAutoConfig.java b/src/backend/commons/common-rabbitmq/src/main/java/com/tencent/bk/job/common/rabbitmq/config/JobRabbitMQAutoConfig.java new file mode 100644 index 0000000000..359238acba --- /dev/null +++ b/src/backend/commons/common-rabbitmq/src/main/java/com/tencent/bk/job/common/rabbitmq/config/JobRabbitMQAutoConfig.java @@ -0,0 +1,62 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.rabbitmq.config; + +import com.rabbitmq.client.impl.CredentialsProvider; +import com.rabbitmq.client.impl.CredentialsRefreshService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.boot.autoconfigure.AutoConfigureBefore; +import org.springframework.boot.autoconfigure.amqp.RabbitAutoConfiguration; +import org.springframework.boot.autoconfigure.amqp.RabbitConnectionFactoryBeanConfigurer; +import org.springframework.boot.autoconfigure.amqp.RabbitProperties; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.io.FileSystemResourceLoader; + +@Slf4j +@Configuration +@AutoConfigureBefore(RabbitAutoConfiguration.class) +@EnableConfigurationProperties(RabbitProperties.class) +public class JobRabbitMQAutoConfig { + + @Bean + RabbitConnectionFactoryBeanConfigurer rabbitConnectionFactoryBeanConfigurer( + RabbitProperties properties, + ObjectProvider credentialsProvider, + ObjectProvider credentialsRefreshService + ) { + RabbitConnectionFactoryBeanConfigurer configurer = new RabbitConnectionFactoryBeanConfigurer( + // 从系统文件读取truststore与keystore + new FileSystemResourceLoader(), + properties + ); + configurer.setCredentialsProvider(credentialsProvider.getIfUnique()); + configurer.setCredentialsRefreshService(credentialsRefreshService.getIfUnique()); + log.info("rabbitConnectionFactoryBeanConfigurer init"); + return configurer; + } +} diff --git a/src/backend/commons/common-rabbitmq/src/main/resources/META-INF/spring.factories b/src/backend/commons/common-rabbitmq/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..d7f0663f57 --- /dev/null +++ b/src/backend/commons/common-rabbitmq/src/main/resources/META-INF/spring.factories @@ -0,0 +1,2 @@ +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ +com.tencent.bk.job.common.rabbitmq.config.JobRabbitMQAutoConfig diff --git a/src/backend/commons/common-redis/build.gradle b/src/backend/commons/common-redis/build.gradle index abb5245aa6..e066389539 100644 --- a/src/backend/commons/common-redis/build.gradle +++ b/src/backend/commons/common-redis/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -23,10 +23,18 @@ */ dependencies { - api project(':commons:common-utils') - api project(':commons:common') + api files("libs/spring-data-redis-2.7.19-alpha.1.jar") + implementation project(':commons:common-utils') + implementation project(':commons:common') + implementation 'io.micrometer:micrometer-registry-prometheus' api 'org.springframework.boot:spring-boot-starter-data-redis' api 'org.springframework.data:spring-data-redis' api 'org.springframework.boot:spring-boot-configuration-processor' api 'org.apache.commons:commons-pool2' + api 'org.springframework.data:spring-data-keyvalue' + api 'org.springframework:spring-tx' + api 'org.springframework:spring-oxm' + api 'org.springframework:spring-aop' + api 'org.springframework:spring-context-support' + api 'org.slf4j:slf4j-api' } diff --git a/src/backend/commons/common-redis/libs/spring-data-redis-2.7.19-alpha.1.jar b/src/backend/commons/common-redis/libs/spring-data-redis-2.7.19-alpha.1.jar new file mode 100644 index 0000000000..d8233c4c9b Binary files /dev/null and b/src/backend/commons/common-redis/libs/spring-data-redis-2.7.19-alpha.1.jar differ diff --git a/src/backend/commons/common-redis/src/main/java/com/tencent/bk/job/common/redis/BaseRedisCache.java b/src/backend/commons/common-redis/src/main/java/com/tencent/bk/job/common/redis/BaseRedisCache.java new file mode 100644 index 0000000000..87df9ff40e --- /dev/null +++ b/src/backend/commons/common-redis/src/main/java/com/tencent/bk/job/common/redis/BaseRedisCache.java @@ -0,0 +1,64 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.redis; + +import io.micrometer.core.instrument.MeterRegistry; + +/** + * Redis 缓存基础实现类，提供通用能力 + */ +public class BaseRedisCache { + + private final MeterRegistry meterRegistry; + + private final String cacheName; + + private static final String METRIC_NAME_JOB_REDIS_CACHE_HITS_TOTAL = "job_redis_cache_hits_total"; + + private static final String METRIC_NAME_JOB_REDIS_CACHE_MISSES_TOTAL = "job_redis_cache_misses_total"; + + private static final String TAG_CACHE_NAME = "cacheName"; + + public BaseRedisCache(MeterRegistry meterRegistry, String cacheName) { + this.meterRegistry = meterRegistry; + this.cacheName = cacheName; + } + + /** + * 增加缓存命中 key 次数 + */ + public void addHits(long hitCount) { + meterRegistry.counter(METRIC_NAME_JOB_REDIS_CACHE_HITS_TOTAL, TAG_CACHE_NAME, cacheName) + .increment(hitCount); + } + + /** + * 增加缓存命中 key 次数 + */ + public void addMisses(long missCount) { + meterRegistry.counter(METRIC_NAME_JOB_REDIS_CACHE_MISSES_TOTAL, TAG_CACHE_NAME, cacheName) + .increment(missCount); + } +} diff --git a/src/backend/commons/common-redis/src/main/java/com/tencent/bk/job/common/redis/config/JobRedisAutoConfiguration.java b/src/backend/commons/common-redis/src/main/java/com/tencent/bk/job/common/redis/config/JobRedisAutoConfiguration.java index d7f8af8d05..dc6b400215 100644 --- a/src/backend/commons/common-redis/src/main/java/com/tencent/bk/job/common/redis/config/JobRedisAutoConfiguration.java +++ b/src/backend/commons/common-redis/src/main/java/com/tencent/bk/job/common/redis/config/JobRedisAutoConfiguration.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,8 +28,16 @@ import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; +import com.tencent.bk.job.common.properties.JobSslProperties; +import io.lettuce.core.ClientOptions; +import io.lettuce.core.SslOptions; +import io.lettuce.core.SslVerifyMode; +import io.micrometer.core.instrument.util.StringUtils; +import lombok.extern.slf4j.Slf4j; import org.springframework.boot.autoconfigure.AutoConfigureAfter; +import org.springframework.boot.autoconfigure.data.redis.LettuceClientConfigurationBuilderCustomizer; import org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration; +import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Import; import org.springframework.context.annotation.Primary; @@ -38,14 +46,19 @@ import org.springframework.data.redis.serializer.GenericJackson2JsonRedisSerializer; import org.springframework.data.redis.serializer.RedisSerializer; +import java.io.File; +import java.util.Optional; + +@Slf4j +@EnableConfigurationProperties(JobRedisSslProperties.class) @AutoConfigureAfter(RedisAutoConfiguration.class) -@Import({RedisAutoConfiguration.class}) +@Import({RedisLockConfig.class, RedisAutoConfiguration.class}) public class JobRedisAutoConfiguration { @Bean("jsonRedisTemplate") @Primary - public RedisTemplate jsonRedisTemplate(RedisConnectionFactory redisConnectionFactory) { - RedisTemplate redisTemplate = new RedisTemplate<>(); + public RedisTemplate jsonRedisTemplate(RedisConnectionFactory redisConnectionFactory) { + RedisTemplate redisTemplate = new RedisTemplate<>(); redisTemplate.setConnectionFactory(redisConnectionFactory); ObjectMapper objectMapper = new ObjectMapper(); @@ -68,4 +81,59 @@ public RedisTemplate jsonRedisTemplate(RedisConnectionFactory re return redisTemplate; } + + @Bean + public LettuceClientConfigurationBuilderCustomizer lettuceClientConfigurationBuilderCustomizer( + JobRedisSslProperties jobRedisSslProperties + ) { + return clientConfigurationBuilder -> { + Optional optional = clientConfigurationBuilder.build().getClientOptions(); + ClientOptions currentClientOptions; + if (optional.isPresent()) { + currentClientOptions = optional.get(); + } else { + log.info("ClientOptions is null, init one"); + currentClientOptions = ClientOptions.builder().build(); + } + JobSslProperties sslProperties = jobRedisSslProperties.getRedis(); + if (!sslProperties.isEnabled()) { + log.info("SSL for redis is not enabled, use default options"); + return; + } + if (!sslProperties.isVerifyHostname()) { + SslVerifyMode verifyMode = SslVerifyMode.CA; + clientConfigurationBuilder.useSsl().verifyMode(verifyMode); + log.info("Redis sslVerifyMode set: {}", verifyMode); + } + SslOptions sslOptions = createSslOptions(sslProperties); + ClientOptions newClientOptions = currentClientOptions.mutate() + // 设置自定义的 SslOptions + .sslOptions(sslOptions) + .build(); + clientConfigurationBuilder.clientOptions(newClientOptions); + }; + } + + private SslOptions createSslOptions(JobSslProperties sslProperties) { + String trustStoreType = sslProperties.getTrustStoreType(); + String keyStoreType = sslProperties.getKeyStoreType(); + if (!keyStoreType.equals(trustStoreType)) { + log.warn( + "trustStoreType must be same with keyStoreType for redis, " + + "but keyStoreType({}) is not the same with trustStoreType({}) now, " + + "both use keyStoreType({}) finally", + keyStoreType, + trustStoreType, + keyStoreType + ); + } + SslOptions.Builder builder = SslOptions.builder().keyStoreType(keyStoreType); + if (StringUtils.isNotBlank(sslProperties.getTrustStore())) { + builder.truststore(new File(sslProperties.getTrustStore()), sslProperties.getTrustStorePassword()); + } + if (StringUtils.isNotBlank(sslProperties.getKeyStore())) { + builder.keystore(new File(sslProperties.getKeyStore()), sslProperties.getKeyStorePassword().toCharArray()); + } + return builder.build(); + } } diff --git a/src/backend/commons/common-redis/src/main/java/com/tencent/bk/job/common/redis/config/JobRedisSslProperties.java b/src/backend/commons/common-redis/src/main/java/com/tencent/bk/job/common/redis/config/JobRedisSslProperties.java new file mode 100644 index 0000000000..37987a4475 --- /dev/null +++ b/src/backend/commons/common-redis/src/main/java/com/tencent/bk/job/common/redis/config/JobRedisSslProperties.java @@ -0,0 +1,41 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.redis.config; + +import com.tencent.bk.job.common.properties.JobSslProperties; +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; + +@ConfigurationProperties(prefix = "tls") +@Getter +@Setter +public class JobRedisSslProperties { + + /** + * Redis的ssl配置 + */ + private JobSslProperties redis = new JobSslProperties(); +} diff --git a/src/backend/commons/common-redis/src/main/java/com/tencent/bk/job/common/redis/config/RedisLockConfig.java b/src/backend/commons/common-redis/src/main/java/com/tencent/bk/job/common/redis/config/RedisLockConfig.java index 49707a63fb..97ff00379b 100644 --- a/src/backend/commons/common-redis/src/main/java/com/tencent/bk/job/common/redis/config/RedisLockConfig.java +++ b/src/backend/commons/common-redis/src/main/java/com/tencent/bk/job/common/redis/config/RedisLockConfig.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -30,7 +30,7 @@ import org.springframework.data.redis.core.StringRedisTemplate; import org.springframework.data.redis.core.script.RedisScript; -@Configuration +@Configuration(proxyBeanMethods = false) public class RedisLockConfig { private static final String UNLOCK_SCRIPT = diff --git a/src/backend/commons/common-redis/src/main/java/com/tencent/bk/job/common/redis/util/DistributedUniqueTask.java b/src/backend/commons/common-redis/src/main/java/com/tencent/bk/job/common/redis/util/DistributedUniqueTask.java new file mode 100644 index 0000000000..e4a6ba57f5 --- /dev/null +++ b/src/backend/commons/common-redis/src/main/java/com/tencent/bk/job/common/redis/util/DistributedUniqueTask.java @@ -0,0 +1,80 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.redis.util; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.redis.core.RedisTemplate; + +import java.util.concurrent.Callable; + +/** + * 分布式唯一任务：在多个实例同时触发的情况下，只会有一个实例执行，唯一性通过同一个Redis Key保证 + */ +@Slf4j +public class DistributedUniqueTask { + + private final RedisTemplate redisTemplate; + private final String name; + private final String redisKey; + private final String redisValue; + private final Callable task; + + public DistributedUniqueTask(RedisTemplate redisTemplate, + String name, + String redisKey, + String redisValue, + Callable task) { + this.redisTemplate = redisTemplate; + this.name = name; + this.redisKey = redisKey; + this.redisValue = redisValue; + this.task = task; + } + + public V execute() throws Exception { + HeartBeatRedisLockConfig config = HeartBeatRedisLockConfig.getDefault(); + config.setHeartBeatThreadName(name + "-RedisKeyHeartBeatThread"); + HeartBeatRedisLock lock = new HeartBeatRedisLock( + redisTemplate, + redisKey, + redisValue, + config + ); + LockResult lockResult = lock.lock(); + if (!lockResult.isLockGotten()) { + log.info( + "lock {} gotten by another instance: {}, return", + redisKey, + lockResult.getLockValue() + ); + return null; + } + try { + return task.call(); + } finally { + lockResult.tryToRelease(); + } + } +} diff --git a/src/backend/commons/common-redis/src/main/java/com/tencent/bk/job/common/redis/util/HeartBeatRedisLock.java b/src/backend/commons/common-redis/src/main/java/com/tencent/bk/job/common/redis/util/HeartBeatRedisLock.java new file mode 100644 index 0000000000..8fa8aff13e --- /dev/null +++ b/src/backend/commons/common-redis/src/main/java/com/tencent/bk/job/common/redis/util/HeartBeatRedisLock.java @@ -0,0 +1,95 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.redis.util; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.redis.core.RedisTemplate; + +/** + * 支持通过心跳续期的Redis锁，防止进程被杀后锁不释放 + */ +@Slf4j +public class HeartBeatRedisLock { + + private final RedisTemplate redisTemplate; + private final String key; + private final String value; + private final HeartBeatRedisLockConfig config; + + public HeartBeatRedisLock(RedisTemplate redisTemplate, + String key, + String value, + HeartBeatRedisLockConfig config) { + this.redisTemplate = redisTemplate; + this.key = key; + this.value = value; + this.config = config; + } + + public HeartBeatRedisLock(RedisTemplate redisTemplate, + String key, + String value) { + this.redisTemplate = redisTemplate; + this.key = key; + this.value = value; + this.config = HeartBeatRedisLockConfig.getDefault(); + } + + public LockResult lock() { + boolean lockGotten; + try { + lockGotten = LockUtils.tryGetDistributedLock(key, value, config.getExpireTimeMillis()); + if (!lockGotten) { + String realLockKey = getRealLockKey(); + String realLockKeyValue = redisTemplate.opsForValue().get(realLockKey); + log.info("Get lock {} fail, already held by {}", realLockKey, realLockKeyValue); + return LockResult.fail(realLockKeyValue); + } + RedisKeyHeartBeatThread heartBeatThread = startRedisKeyHeartBeatThread(); + return LockResult.succeed(value, heartBeatThread); + } catch (Throwable t) { + log.error("Get lock caught exception", t); + } + return LockResult.fail(null); + } + + private String getRealLockKey() { + return LockUtils.LOCK_KEY_PREFIX + key; + } + + private RedisKeyHeartBeatThread startRedisKeyHeartBeatThread() { + // 开一个心跳子线程，维护当前机器正在WatchResource的状态 + RedisKeyHeartBeatThread redisKeyHeartBeatThread = new RedisKeyHeartBeatThread( + redisTemplate, + getRealLockKey(), + value, + config.getExpireTimeMillis(), + config.getPeriodMillis() + ); + redisKeyHeartBeatThread.setName(config.getHeartBeatThreadName()); + redisKeyHeartBeatThread.start(); + return redisKeyHeartBeatThread; + } +} diff --git a/src/backend/commons/common-redis/src/main/java/com/tencent/bk/job/common/redis/util/HeartBeatRedisLockConfig.java b/src/backend/commons/common-redis/src/main/java/com/tencent/bk/job/common/redis/util/HeartBeatRedisLockConfig.java new file mode 100644 index 0000000000..6a58e7838f --- /dev/null +++ b/src/backend/commons/common-redis/src/main/java/com/tencent/bk/job/common/redis/util/HeartBeatRedisLockConfig.java @@ -0,0 +1,64 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.redis.util; + +import com.tencent.bk.job.common.util.JobUUID; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +/** + * 支持心跳的Redis锁配置 + */ +@Setter +@Getter +@NoArgsConstructor +public class HeartBeatRedisLockConfig { + + /** + * 锁过期时间，单位：毫秒 + */ + private long expireTimeMillis = 5_000L; + /** + * 锁心跳间隔时间，单位：毫秒 + */ + private long periodMillis = 2_000L; + /** + * 心跳线程名称 + */ + private String heartBeatThreadName = "redisKeyHeartBeatThread-" + JobUUID.getUUID().substring(0, 8); + + public static final HeartBeatRedisLockConfig INSTANCE = new HeartBeatRedisLockConfig(); + + public static HeartBeatRedisLockConfig getDefault() { + return INSTANCE; + } + + public HeartBeatRedisLockConfig(String heartBeatThreadName, long expireTimeMillis, long periodMillis) { + this.heartBeatThreadName = heartBeatThreadName; + this.expireTimeMillis = expireTimeMillis; + this.periodMillis = periodMillis; + } +} diff --git a/src/backend/commons/common-redis/src/main/java/com/tencent/bk/job/common/redis/util/LockResult.java b/src/backend/commons/common-redis/src/main/java/com/tencent/bk/job/common/redis/util/LockResult.java new file mode 100644 index 0000000000..36865f4b2e --- /dev/null +++ b/src/backend/commons/common-redis/src/main/java/com/tencent/bk/job/common/redis/util/LockResult.java @@ -0,0 +1,62 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.redis.util; + +import lombok.Data; + +@Data +public class LockResult { + private boolean lockGotten; + private String lockValue; + private RedisKeyHeartBeatThread heartBeatThread; + + public LockResult(boolean lockGotten, + String lockValue, + RedisKeyHeartBeatThread heartBeatThread) { + this.lockGotten = lockGotten; + this.lockValue = lockValue; + this.heartBeatThread = heartBeatThread; + } + + public LockResult(boolean lockGotten, String lockValue) { + this.lockGotten = lockGotten; + this.lockValue = lockValue; + } + + public void tryToRelease() { + if (heartBeatThread != null) { + heartBeatThread.stopAtOnce(); + } + } + + public static LockResult fail(String lockValue) { + return new LockResult(false, lockValue); + } + + public static LockResult succeed(String lockValue, RedisKeyHeartBeatThread heartBeatThread) { + return new LockResult(true, lockValue, heartBeatThread); + } + +} diff --git a/src/backend/commons/common-redis/src/main/java/com/tencent/bk/job/common/redis/util/LockUtils.java b/src/backend/commons/common-redis/src/main/java/com/tencent/bk/job/common/redis/util/LockUtils.java index d4ab2f0bc0..1f16bb7893 100644 --- a/src/backend/commons/common-redis/src/main/java/com/tencent/bk/job/common/redis/util/LockUtils.java +++ b/src/backend/commons/common-redis/src/main/java/com/tencent/bk/job/common/redis/util/LockUtils.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -35,12 +35,15 @@ import java.util.concurrent.TimeUnit; /** - * @since 20/1/2020 10:49 + * 分布式锁工具类 */ @Slf4j public class LockUtils { - private static final String LOCK_PREFIX = "job:util:lock:"; + /** + * 分布式锁 key 的前缀 + */ + public static final String LOCK_KEY_PREFIX = "job:util:lock:"; private static final Long RELEASE_SUCCESS = 1L; private static StringRedisTemplate redisTemplate = null; private static RedisScript unlockScript = null; @@ -69,28 +72,16 @@ public static synchronized void init(StringRedisTemplate template, RedisScript redisTemplate; + @Setter private volatile boolean runFlag; private final String redisKey; private final String redisValue; private final Long expireTimeMillis; private final Long periodMillis; - public RedisKeyHeartBeatThread(RedisTemplate redisTemplate, String redisKey, String redisValue, - Long expireTimeMillis, Long periodMillis) { + public RedisKeyHeartBeatThread(RedisTemplate redisTemplate, + String redisKey, + String redisValue, + Long expireTimeMillis, + Long periodMillis) { this.redisTemplate = redisTemplate; this.redisKey = redisKey; this.redisValue = redisValue; @@ -52,11 +60,9 @@ public RedisKeyHeartBeatThread(RedisTemplate redisTemplate, Stri public void stopAtOnce() { setRunFlag(false); - redisTemplate.delete(redisKey); - } - - public void setRunFlag(boolean runFlag) { - this.runFlag = runFlag; + Boolean result = redisTemplate.delete(redisKey); + log.debug("stopAtOnce, delete redis key:{}, result={}", redisKey, result); + interrupt(); } @Override @@ -64,10 +70,19 @@ public void run() { try { while (runFlag) { redisTemplate.opsForValue().set(redisKey, redisValue, expireTimeMillis, TimeUnit.MILLISECONDS); - ThreadUtils.sleep(periodMillis); + ThreadUtils.sleep(periodMillis, false); } } catch (Throwable t) { - log.error("RedisKeyHeartBeatThread {} quit unexpectedly:", this.getName(), t); + String msg = MessageFormatter.format( + "RedisKeyHeartBeatThread {} quit unexpectedly:", + this.getName() + ).getMessage(); + // 主动终止线程产生的异常只打印调试级别日志 + if (!causeByStopAtOnceInterrupt(t)) { + log.error(msg, t); + } else { + log.debug(msg, t); + } } finally { deleteRedisKeySafely(); } @@ -75,9 +90,27 @@ public void run() { private void deleteRedisKeySafely() { try { - redisTemplate.delete(redisKey); + Boolean result = redisTemplate.delete(redisKey); + log.debug("delete redis key:{}, result={}", redisKey, result); } catch (Throwable e) { - log.error("Delete redis key fail", e); + // 主动终止线程产生的异常只打印调试级别日志 + if (!causeByStopAtOnceInterrupt(e)) { + log.error("Delete redis key fail", e); + } else { + log.debug("Delete redis key fail", e); + } + } + } + + @SuppressWarnings("BooleanMethodIsAlwaysInverted") + private boolean causeByStopAtOnceInterrupt(Throwable t) { + if (runFlag) { + return false; + } + if (t instanceof RedisSystemException) { + Throwable cause = t.getCause(); + return cause instanceof RedisCommandInterruptedException; } + return false; } } diff --git a/src/backend/commons/common-redis/src/main/java/com/tencent/bk/job/common/redis/util/RedisSlideWindowFlowController.java b/src/backend/commons/common-redis/src/main/java/com/tencent/bk/job/common/redis/util/RedisSlideWindowFlowController.java index 5f0315953d..9df9f851bc 100644 --- a/src/backend/commons/common-redis/src/main/java/com/tencent/bk/job/common/redis/util/RedisSlideWindowFlowController.java +++ b/src/backend/commons/common-redis/src/main/java/com/tencent/bk/job/common/redis/util/RedisSlideWindowFlowController.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,6 +28,7 @@ import com.tencent.bk.job.common.util.FlowController; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.slf4j.helpers.MessageFormatter; import org.springframework.data.redis.core.StringRedisTemplate; import org.springframework.data.redis.core.script.RedisScript; @@ -195,7 +196,14 @@ public int updateConfig(Map configMap) { redisTemplate.opsForValue().set(redisKey, redisValue); count += 1; } catch (Exception e) { - log.error("Fail to write redis:key={},value={}", redisKey, redisValue, e); + String msg = MessageFormatter.arrayFormat( + "Fail to write redis:key={},value={}", + new String[]{ + redisKey, + redisValue + } + ).getMessage(); + log.error(msg, e); } } return count; @@ -209,7 +217,14 @@ public Long getCurrentConfig(String resourceId) { try { return Long.parseLong(stringValue); } catch (Exception e) { - log.warn("Fail to parse maxRate, resourceId={},stringValue={}", resourceId, stringValue, e); + String msg = MessageFormatter.arrayFormat( + "Fail to parse maxRate, resourceId={},stringValue={}", + new String[]{ + resourceId, + stringValue + } + ).getMessage(); + log.error(msg, e); return null; } } @@ -227,7 +242,11 @@ public Map getCurrentConfig() { map.put(maxRateKey.replace(REDIS_PREFIX_MAX_RATE, ""), Long.parseLong(value)); } } catch (Exception e) { - log.error("Fail to get redis value:key={}", maxRateKey, e); + String msg = MessageFormatter.format( + "Fail to get redis value:key={}", + maxRateKey + ).getMessage(); + log.error(msg, e); } } return map; @@ -252,7 +271,11 @@ public Map getCurrentRateMap() { try { map.put(resourceId, getCurrentRate(resourceId)); } catch (Exception e) { - log.error("Fail to get current rate:resourceId={}", resourceId, e); + String msg = MessageFormatter.format( + "Fail to get current rate:resourceId={}", + resourceId + ).getMessage(); + log.error(msg, e); } } return map; diff --git a/src/backend/commons/common-security/build.gradle b/src/backend/commons/common-security/build.gradle index e540f42654..5efa12d219 100644 --- a/src/backend/commons/common-security/build.gradle +++ b/src/backend/commons/common-security/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,7 @@ dependencies { api project(':commons:common-utils') + api project(':commons:common-jwt') implementation 'io.jsonwebtoken:jjwt' implementation 'com.google.guava:guava' implementation 'org.springframework.cloud:spring-cloud-starter-openfeign' diff --git a/src/backend/commons/common-security/src/main/java/com/tencent/bk/job/common/security/annotation/ConditionalOnSecurityEnabled.java b/src/backend/commons/common-security/src/main/java/com/tencent/bk/job/common/security/annotation/ConditionalOnSecurityEnabled.java new file mode 100644 index 0000000000..5700da4948 --- /dev/null +++ b/src/backend/commons/common-security/src/main/java/com/tencent/bk/job/common/security/annotation/ConditionalOnSecurityEnabled.java @@ -0,0 +1,46 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.security.annotation; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target(ElementType.METHOD) +@Retention(RetentionPolicy.RUNTIME) +@Documented +@Inherited +@ConditionalOnProperty( + value = "job.security.service.enabled", + havingValue = "true", + matchIfMissing = true +) +public @interface ConditionalOnSecurityEnabled { +} diff --git a/src/backend/commons/common-security/src/main/java/com/tencent/bk/job/common/security/autoconfigure/ServiceSecurityAutoConfiguration.java b/src/backend/commons/common-security/src/main/java/com/tencent/bk/job/common/security/autoconfigure/ServiceSecurityAutoConfiguration.java index 929c828213..8f31c76b86 100644 --- a/src/backend/commons/common-security/src/main/java/com/tencent/bk/job/common/security/autoconfigure/ServiceSecurityAutoConfiguration.java +++ b/src/backend/commons/common-security/src/main/java/com/tencent/bk/job/common/security/autoconfigure/ServiceSecurityAutoConfiguration.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,18 +24,15 @@ package com.tencent.bk.job.common.security.autoconfigure; +import com.tencent.bk.job.common.jwt.JwtManager; +import com.tencent.bk.job.common.security.feign.FeignBasicAuthRequestInterceptor; import com.tencent.bk.job.common.security.jwt.AutoUpdateJwtManager; -import com.tencent.bk.job.common.util.jwt.JwtManager; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.context.properties.EnableConfigurationProperties; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.scheduling.annotation.EnableScheduling; -import java.io.IOException; -import java.security.GeneralSecurityException; - @EnableScheduling @Configuration(proxyBeanMethods = false) @EnableConfigurationProperties(ServiceSecurityProperties.class) @@ -43,12 +40,21 @@ public class ServiceSecurityAutoConfiguration { @Bean @ConditionalOnMissingBean(name = "jwtManager") - JwtManager jwtManager(@Autowired ServiceSecurityProperties securityProperties) throws IOException, - GeneralSecurityException { + JwtManager jwtManager(ServiceSecurityProperties securityProperties) { return new AutoUpdateJwtManager( securityProperties.getPrivateKeyBase64(), securityProperties.getPublicKeyBase64() ); } + /** + * Feign 服务认证拦截 + * + * @param jwtManager jwt管理 + */ + @Bean + public FeignBasicAuthRequestInterceptor basicAuthRequestInterceptor(JwtManager jwtManager) { + return new FeignBasicAuthRequestInterceptor(jwtManager); + } + } diff --git a/src/backend/commons/common-security/src/main/java/com/tencent/bk/job/common/security/autoconfigure/ServiceSecurityProperties.java b/src/backend/commons/common-security/src/main/java/com/tencent/bk/job/common/security/autoconfigure/ServiceSecurityProperties.java index 213efab85f..5ba9866480 100644 --- a/src/backend/commons/common-security/src/main/java/com/tencent/bk/job/common/security/autoconfigure/ServiceSecurityProperties.java +++ b/src/backend/commons/common-security/src/main/java/com/tencent/bk/job/common/security/autoconfigure/ServiceSecurityProperties.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-security/src/main/java/com/tencent/bk/job/common/security/consts/JwtConsts.java b/src/backend/commons/common-security/src/main/java/com/tencent/bk/job/common/security/consts/JwtConsts.java new file mode 100644 index 0000000000..6d4865bc05 --- /dev/null +++ b/src/backend/commons/common-security/src/main/java/com/tencent/bk/job/common/security/consts/JwtConsts.java @@ -0,0 +1,33 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.security.consts; + +/** + * Jwt相关常量 + */ +public class JwtConsts { + // 服务间调用使用的Jwt Token Header Key + public static final String HEADER_KEY_SERVICE_JWT_TOKEN = "x-job-auth-token"; +} diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/exception/ServiceNoAuthException.java b/src/backend/commons/common-security/src/main/java/com/tencent/bk/job/common/security/exception/ServiceNoAuthException.java similarity index 92% rename from src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/exception/ServiceNoAuthException.java rename to src/backend/commons/common-security/src/main/java/com/tencent/bk/job/common/security/exception/ServiceNoAuthException.java index 0ea0600433..6cc63ccc6d 100644 --- a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/exception/ServiceNoAuthException.java +++ b/src/backend/commons/common-security/src/main/java/com/tencent/bk/job/common/security/exception/ServiceNoAuthException.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.common.web.exception; +package com.tencent.bk.job.common.security.exception; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.exception.InternalException; diff --git a/src/backend/commons/common-security/src/main/java/com/tencent/bk/job/common/security/feign/FeignBasicAuthRequestInterceptor.java b/src/backend/commons/common-security/src/main/java/com/tencent/bk/job/common/security/feign/FeignBasicAuthRequestInterceptor.java index cca93f1d90..eb0e9a89ec 100644 --- a/src/backend/commons/common-security/src/main/java/com/tencent/bk/job/common/security/feign/FeignBasicAuthRequestInterceptor.java +++ b/src/backend/commons/common-security/src/main/java/com/tencent/bk/job/common/security/feign/FeignBasicAuthRequestInterceptor.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,8 @@ package com.tencent.bk.job.common.security.feign; -import com.tencent.bk.job.common.util.jwt.JwtManager; +import com.tencent.bk.job.common.jwt.JwtManager; +import com.tencent.bk.job.common.security.consts.JwtConsts; import feign.RequestInterceptor; import feign.RequestTemplate; import lombok.extern.slf4j.Slf4j; @@ -39,9 +40,9 @@ public FeignBasicAuthRequestInterceptor(JwtManager jwtManager) { @Override public void apply(RequestTemplate template) { - if (!template.headers().containsKey("x-job-auth-token")) { + if (!template.headers().containsKey(JwtConsts.HEADER_KEY_SERVICE_JWT_TOKEN)) { String token = jwtManager.getToken(); - template.header("x-job-auth-token", token); + template.header(JwtConsts.HEADER_KEY_SERVICE_JWT_TOKEN, token); } } } diff --git a/src/backend/commons/common-security/src/main/java/com/tencent/bk/job/common/security/feign/FeignSecurityConfiguration.java b/src/backend/commons/common-security/src/main/java/com/tencent/bk/job/common/security/feign/FeignSecurityConfiguration.java deleted file mode 100644 index d9ca0cb5df..0000000000 --- a/src/backend/commons/common-security/src/main/java/com/tencent/bk/job/common/security/feign/FeignSecurityConfiguration.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.security.feign; - -import com.tencent.bk.job.common.util.jwt.JwtManager; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Configuration -public class FeignSecurityConfiguration { - - /** - * Feign 服务认证拦截 - * - * @param jwtManager jwt管理 - * @return - */ - @Bean - public FeignBasicAuthRequestInterceptor basicAuthRequestInterceptor(@Autowired JwtManager jwtManager) { - return new FeignBasicAuthRequestInterceptor(jwtManager); - } - -} - diff --git a/src/backend/commons/common-security/src/main/java/com/tencent/bk/job/common/security/jwt/AutoUpdateJwtManager.java b/src/backend/commons/common-security/src/main/java/com/tencent/bk/job/common/security/jwt/AutoUpdateJwtManager.java index 76dc95b062..81d05b7e8d 100644 --- a/src/backend/commons/common-security/src/main/java/com/tencent/bk/job/common/security/jwt/AutoUpdateJwtManager.java +++ b/src/backend/commons/common-security/src/main/java/com/tencent/bk/job/common/security/jwt/AutoUpdateJwtManager.java @@ -1,16 +1,12 @@ package com.tencent.bk.job.common.security.jwt; -import com.tencent.bk.job.common.util.jwt.BasicJwtManager; +import com.tencent.bk.job.common.jwt.BasicJwtManager; import lombok.extern.slf4j.Slf4j; import org.springframework.scheduling.annotation.Scheduled; -import java.io.IOException; -import java.security.GeneralSecurityException; - @Slf4j public class AutoUpdateJwtManager extends BasicJwtManager { - public AutoUpdateJwtManager(String privateKeyBase64, String publicKeyBase64) - throws IOException, GeneralSecurityException { + public AutoUpdateJwtManager(String privateKeyBase64, String publicKeyBase64) { super(privateKeyBase64, publicKeyBase64); } diff --git a/src/backend/commons/common-service/build.gradle b/src/backend/commons/common-service/build.gradle index 2b8c10d50b..764ef216bb 100644 --- a/src/backend/commons/common-service/build.gradle +++ b/src/backend/commons/common-service/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -21,34 +21,35 @@ * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. */ -def assemblyMode = System.getProperty("assemblyMode") -def k8s = assemblyMode == "k8s" -println("assembly mode: $assemblyMode") - dependencies { api project(':commons:common') + api project(':commons:common-log') + api project(':commons:common-task') api project(':commons:common-security') api project(':commons:common-otel') + api project(':commons:common-rabbitmq') api 'org.hibernate.validator:hibernate-validator' api 'jakarta.validation:jakarta.validation-api' api 'org.springframework.boot:spring-boot-starter-actuator' api 'org.springframework.boot:spring-boot-starter-logging' api 'org.springframework.cloud:spring-cloud-starter-openfeign' api 'org.springframework.cloud:spring-cloud-starter-sleuth' - if (k8s) { - println("Compile with kubernetes mode") + api 'org.springframework.cloud:spring-cloud-loadbalancer' + api 'org.springframework.cloud:spring-cloud-starter-bus-amqp' + api 'org.springframework.cloud:spring-cloud-starter-stream-rabbit' + if (rootProject.ext.k8sMode) { + println("Compile module[common-service] with kubernetes mode") api project(":commons:common-k8s") api 'org.springframework.cloud:spring-cloud-starter-kubernetes-client-all' - api 'org.springframework.cloud:spring-cloud-starter-kubernetes-client-loadbalancer' } else { + println("Compile module[common-service] with binary mode") api project(":commons:common-consul") api 'org.springframework.cloud:spring-cloud-starter-config' api 'org.springframework.cloud:spring-cloud-starter-consul-discovery' } api 'ch.qos.logback:logback-core' api 'ch.qos.logback:logback-classic' - api 'io.springfox:springfox-swagger2' - api 'io.springfox:springfox-swagger-ui' + api 'io.springfox:springfox-boot-starter' api 'com.fasterxml.jackson.core:jackson-core' api 'com.fasterxml.jackson.core:jackson-databind' api 'com.fasterxml.jackson.core:jackson-annotations' @@ -56,4 +57,5 @@ dependencies { api 'io.prometheus:simpleclient_pushgateway' api 'io.micrometer:micrometer-registry-prometheus' testImplementation 'org.springframework.boot:spring-boot-starter-test' + testImplementation 'org.mockito:mockito-inline' } diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/CommonServiceAutoConfiguration.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/CommonServiceAutoConfiguration.java new file mode 100644 index 0000000000..9cb86ae938 --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/CommonServiceAutoConfiguration.java @@ -0,0 +1,49 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service; + +import com.tencent.bk.job.common.service.quota.ResourceQuotaStore; +import com.tencent.bk.job.common.util.toggle.feature.FeatureStore; +import com.tencent.bk.job.common.util.toggle.prop.PropToggleStore; +import io.micrometer.core.instrument.MeterRegistry; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration(proxyBeanMethods = false) +public class CommonServiceAutoConfiguration { + + @Bean + public ConfigRefreshEventListener configRefreshEventListener(MeterRegistry meterRegistry, + FeatureStore featureStore, + ResourceQuotaStore resourceQuotaStore, + PropToggleStore propToggleStore) { + return new ConfigRefreshEventListener(meterRegistry, featureStore, resourceQuotaStore, propToggleStore); + } + + @Bean + public SpringProfile springProfile() { + return new SpringProfile(); + } +} diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/ConfigRefreshEventListener.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/ConfigRefreshEventListener.java new file mode 100644 index 0000000000..388cbd95c6 --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/ConfigRefreshEventListener.java @@ -0,0 +1,138 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service; + +import com.tencent.bk.job.common.service.quota.ResourceQuotaStore; +import com.tencent.bk.job.common.util.toggle.feature.FeatureStore; +import com.tencent.bk.job.common.util.toggle.prop.PropToggleStore; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tags; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.springframework.cloud.context.environment.EnvironmentChangeEvent; +import org.springframework.context.event.EventListener; + +import java.util.Set; +import java.util.StringJoiner; +import java.util.stream.Collectors; + +/** + * 配置刷新监听 + */ +@Slf4j +public class ConfigRefreshEventListener { + + private final MeterRegistry meterRegistry; + + private final FeatureStore featureStore; + + private final ResourceQuotaStore resourceQuotaStore; + + private final PropToggleStore propToggleStore; + + private static final String METRIC_JOB_CONFIG_REFRESH_FAIL_TOTAL = "job_config_refresh_fail_total"; + private static final String METRIC_TAG_CONFIG_NAME = "config_name"; + + + public ConfigRefreshEventListener(MeterRegistry meterRegistry, + FeatureStore featureStore, + ResourceQuotaStore resourceQuotaStore, + PropToggleStore propToggleStore) { + this.meterRegistry = meterRegistry; + this.featureStore = featureStore; + this.resourceQuotaStore = resourceQuotaStore; + this.propToggleStore = propToggleStore; + log.info("Init ConfigRefreshEventListener"); + } + + /** + * 监听并处理Spring cloud 配置更新事件(通过/actuator/refresh 和 /actuator/busrefresh endpoint 触发) + * + * @param event 配置更新事件 + */ + @EventListener + public void onEvent(EnvironmentChangeEvent event) { + if (log.isInfoEnabled()) { + log.info("Handle EnvironmentChangeEvent, event: {}", printEnvironmentChangeEvent(event)); + } + reload(event.getKeys()); + } + + private String printEnvironmentChangeEvent(EnvironmentChangeEvent event) { + return new StringJoiner(", ", EnvironmentChangeEvent.class.getSimpleName() + "[", "]") + .add("source=" + event.getSource()) + .add("timestamp=" + event.getTimestamp()) + .add("keys=" + event.getKeys()) + .toString(); + } + + /** + * 重载配置 + */ + private void reload(Set changedKeys) { + if (CollectionUtils.isEmpty(changedKeys)) { + return; + } + if (changedKeys.stream().anyMatch(changedKey -> changedKey.startsWith(FeatureStore.PROP_KEY_PREFIX))) { + boolean handleResult = featureStore.handleConfigChange( + filterChangedKeys(changedKeys, FeatureStore.PROP_KEY_PREFIX), true); + if (!handleResult) { + meterRegistry.counter( + METRIC_JOB_CONFIG_REFRESH_FAIL_TOTAL, + Tags.of(METRIC_TAG_CONFIG_NAME, "job.feature")) + .increment(); + } + } + + if (changedKeys.stream().anyMatch(changedKey -> changedKey.startsWith(ResourceQuotaStore.PROP_KEY_PREFIX))) { + boolean handleResult = resourceQuotaStore.handleConfigChange( + filterChangedKeys(changedKeys, ResourceQuotaStore.PROP_KEY_PREFIX)); + if (!handleResult) { + meterRegistry.counter( + METRIC_JOB_CONFIG_REFRESH_FAIL_TOTAL, + Tags.of(METRIC_TAG_CONFIG_NAME, "job.resourceQuotaLimit")) + .increment(); + } + } + + if (changedKeys.stream().anyMatch(changedKey -> changedKey.startsWith(PropToggleStore.PROP_KEY_PREFIX))) { + boolean handleResult = propToggleStore.handleConfigChange( + filterChangedKeys(changedKeys, PropToggleStore.PROP_KEY_PREFIX), true); + + if (!handleResult) { + meterRegistry.counter( + METRIC_JOB_CONFIG_REFRESH_FAIL_TOTAL, + Tags.of(METRIC_TAG_CONFIG_NAME, "job.toggle.props")) + .increment(); + } + } + } + + private Set filterChangedKeys(Set changedKeys, String keyPrefix) { + return changedKeys.stream().filter(key -> key.startsWith(keyPrefix)) + .collect(Collectors.toSet()); + } + +} diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/SpringProfile.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/SpringProfile.java new file mode 100644 index 0000000000..a95cc951b1 --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/SpringProfile.java @@ -0,0 +1,75 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service; + +import org.apache.commons.lang3.ArrayUtils; +import org.springframework.beans.BeansException; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class SpringProfile implements ApplicationContextAware { + + private static ApplicationContext context = null; + private transient Boolean isDevProfile; + + + @Override + public void setApplicationContext(ApplicationContext applicationContext) + throws BeansException { + context = applicationContext; + } + + public List getActiveProfiles() { + String[] profiles = context.getEnvironment().getActiveProfiles(); + if (!ArrayUtils.isEmpty(profiles)) { + return Arrays.asList(profiles); + } + return new ArrayList<>(); + } + + public boolean isProfileActive(String profile) { + String[] activeProfiles = context.getEnvironment().getActiveProfiles(); + for (String activeProfile : activeProfiles) { + if (activeProfile.equals(profile)) { + return true; + } + } + return false; + } + + public boolean isDevProfileActive() { + if (isDevProfile != null) { + return isDevProfile; + } + isDevProfile = isProfileActive("dev") || isProfileActive("local"); + return isDevProfile; + } + + +} diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/bean/UniqueBeanNameGenerator.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/bean/UniqueBeanNameGenerator.java new file mode 100644 index 0000000000..82118c0547 --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/bean/UniqueBeanNameGenerator.java @@ -0,0 +1,50 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service.bean; + +import org.springframework.beans.factory.annotation.AnnotatedBeanDefinition; +import org.springframework.beans.factory.config.BeanDefinition; +import org.springframework.beans.factory.support.BeanDefinitionRegistry; +import org.springframework.context.annotation.AnnotationBeanNameGenerator; +import org.springframework.util.StringUtils; + +/** + * 自定义唯一名称 Bean 生成规则，避免Bean名称冲突 + */ +public class UniqueBeanNameGenerator extends AnnotationBeanNameGenerator { + @Override + public String generateBeanName(BeanDefinition definition, BeanDefinitionRegistry registry) { + if (definition instanceof AnnotatedBeanDefinition) { + String beanName = determineBeanNameFromAnnotation((AnnotatedBeanDefinition) definition); + if (StringUtils.hasText(beanName)) { + // Explicit bean name found. + return beanName; + } + } + + // 如果该bean名称没有指定，则按照“package+className”的规则生成 + return definition.getBeanClassName(); + } +} diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/boot/JobBootApplication.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/boot/JobBootApplication.java new file mode 100644 index 0000000000..bc9e3e0c6c --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/boot/JobBootApplication.java @@ -0,0 +1,67 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service.boot; + +import com.tencent.bk.job.common.service.bean.UniqueBeanNameGenerator; +import org.springframework.boot.SpringBootConfiguration; +import org.springframework.boot.autoconfigure.AutoConfigurationExcludeFilter; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.context.TypeExcludeFilter; +import org.springframework.context.annotation.ComponentScan; +import org.springframework.context.annotation.FilterType; +import org.springframework.core.annotation.AliasFor; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Job 微服务启动注解 + */ +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +@Documented +@Inherited +@SpringBootConfiguration +@EnableAutoConfiguration +@ComponentScan( + nameGenerator = UniqueBeanNameGenerator.class, + excludeFilters = { + @ComponentScan.Filter(type = FilterType.CUSTOM, classes = TypeExcludeFilter.class), + @ComponentScan.Filter(type = FilterType.CUSTOM, classes = AutoConfigurationExcludeFilter.class)} +) +public @interface JobBootApplication { + @AliasFor(annotation = ComponentScan.class, attribute = "basePackages") + String[] scanBasePackages() default {}; + + @AliasFor(annotation = EnableAutoConfiguration.class) + Class[] exclude() default {}; + + @AliasFor(annotation = EnableAutoConfiguration.class) + String[] excludeName() default {}; +} diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/constants/DeployModeEnum.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/constants/DeployModeEnum.java new file mode 100644 index 0000000000..d1cc83c861 --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/constants/DeployModeEnum.java @@ -0,0 +1,32 @@ +package com.tencent.bk.job.common.service.constants; + +/** + * Job 部署模式 + */ +public enum DeployModeEnum { + + /** + * 标准模式（微服务） + */ + STANDARD(Constants.STANDARD), + /** + * 轻量化模式(微服务合并） + */ + LITE(Constants.LITE); + + public static class Constants { + public static final String STANDARD = "standard"; + public static final String LITE = "lite"; + } + + + DeployModeEnum(String mode) { + this.mode = mode; + } + + private final String mode; + + public String getValue() { + return mode; + } +} diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/deploy/DeployAutoConfiguration.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/deploy/DeployAutoConfiguration.java new file mode 100644 index 0000000000..33cd27a31c --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/deploy/DeployAutoConfiguration.java @@ -0,0 +1,33 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service.deploy; + +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +@Configuration(proxyBeanMethods = false) +@EnableConfigurationProperties(DeployProperties.class) +public class DeployAutoConfiguration { +} diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/deploy/DeployProperties.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/deploy/DeployProperties.java new file mode 100644 index 0000000000..5239450d1c --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/deploy/DeployProperties.java @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service.deploy; + +import com.tencent.bk.job.common.service.constants.DeployModeEnum; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.springframework.boot.context.properties.ConfigurationProperties; + +/** + * Job 部署配置 + */ +@Getter +@Setter +@ToString +@ConfigurationProperties(prefix = "deploy") +public class DeployProperties { + /** + * 部署模式 + */ + private String mode = DeployModeEnum.STANDARD.getValue(); +} diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/feign/client/WatchableFeignClient.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/feign/client/WatchableFeignClient.java new file mode 100644 index 0000000000..c4ee894f02 --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/feign/client/WatchableFeignClient.java @@ -0,0 +1,67 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service.feign.client; + +import feign.Client; +import feign.Request; +import feign.Response; +import lombok.extern.slf4j.Slf4j; + +import javax.net.ssl.HostnameVerifier; +import javax.net.ssl.SSLSocketFactory; +import java.io.IOException; + +/** + * FeignClient抛出的原始异常信息中仅包含负载均衡解析前的服务名称信息， + * 当前类用于覆盖默认Client.Default以便观察FeignClient调用异常时的真实请求信息（URL等） + */ +@Slf4j +public class WatchableFeignClient extends Client.Default { + + public WatchableFeignClient(SSLSocketFactory sslContextFactory, HostnameVerifier hostnameVerifier) { + super(sslContextFactory, hostnameVerifier); + } + + @Override + public Response execute(Request request, Request.Options options) throws IOException { + try { + Response response = super.execute(request, options); + log.debug( + "SucceedToExecFeignRequest, method={}, url={}", + request.httpMethod().name(), + request.url() + ); + return response; + } catch (Exception e) { + log.warn( + "FailToExecFeignRequest, method={}, url={}", + request.httpMethod().name(), + request.url() + ); + throw e; + } + } + +} diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/feign/config/JobFeignLoadBalancerAutoConfiguration.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/feign/config/JobFeignLoadBalancerAutoConfiguration.java new file mode 100644 index 0000000000..f17c7d0e26 --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/feign/config/JobFeignLoadBalancerAutoConfiguration.java @@ -0,0 +1,69 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service.feign.config; + +import com.tencent.bk.job.common.service.feign.client.WatchableFeignClient; +import feign.Client; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.AutoConfigureAfter; +import org.springframework.boot.autoconfigure.AutoConfigureBefore; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.cloud.client.loadbalancer.LoadBalancedRetryFactory; +import org.springframework.cloud.client.loadbalancer.LoadBalancerClient; +import org.springframework.cloud.loadbalancer.config.BlockingLoadBalancerClientAutoConfiguration; +import org.springframework.cloud.loadbalancer.support.LoadBalancerClientFactory; +import org.springframework.cloud.openfeign.loadbalancer.FeignLoadBalancerAutoConfiguration; +import org.springframework.cloud.openfeign.loadbalancer.RetryableFeignBlockingLoadBalancerClient; +import org.springframework.context.annotation.Bean; + +/** + * FeignBlockingLoadBalancerClient相关的自定义Bean配置 + */ +@Slf4j +@AutoConfigureBefore(FeignLoadBalancerAutoConfiguration.class) +@AutoConfigureAfter(BlockingLoadBalancerClientAutoConfiguration.class) +public class JobFeignLoadBalancerAutoConfiguration { + + @Bean + @ConditionalOnMissingBean + @ConditionalOnClass(name = "org.springframework.retry.support.RetryTemplate") + @ConditionalOnBean(LoadBalancedRetryFactory.class) + @ConditionalOnProperty(value = "spring.cloud.loadbalancer.retry.enabled", havingValue = "true", + matchIfMissing = true) + public Client feignRetryClient(LoadBalancerClient loadBalancerClient, + LoadBalancedRetryFactory loadBalancedRetryFactory, + LoadBalancerClientFactory loadBalancerClientFactory) { + log.info("feignRetryClient init"); + return new RetryableFeignBlockingLoadBalancerClient( + new WatchableFeignClient(null, null), + loadBalancerClient, + loadBalancedRetryFactory, + loadBalancerClientFactory + ); + } +} diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/feign/FeignAddHeaderRequestInterceptor.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/feign/interceptor/FeignAddHeaderRequestInterceptor.java similarity index 93% rename from src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/feign/FeignAddHeaderRequestInterceptor.java rename to src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/feign/interceptor/FeignAddHeaderRequestInterceptor.java index caa70d1f73..6c030250a5 100644 --- a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/feign/FeignAddHeaderRequestInterceptor.java +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/feign/interceptor/FeignAddHeaderRequestInterceptor.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.common.feign; +package com.tencent.bk.job.common.service.feign.interceptor; import com.tencent.bk.job.common.i18n.locale.LocaleUtils; import com.tencent.bk.job.common.util.JobContextUtil; diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/lb/JobLoadBalancerClientAutoConfiguration.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/lb/JobLoadBalancerClientAutoConfiguration.java new file mode 100644 index 0000000000..d6766e1811 --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/lb/JobLoadBalancerClientAutoConfiguration.java @@ -0,0 +1,139 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service.lb; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.boot.autoconfigure.AutoConfigureAfter; +import org.springframework.boot.autoconfigure.AutoConfigureBefore; +import org.springframework.boot.autoconfigure.condition.ConditionalOnBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.cloud.client.ServiceInstance; +import org.springframework.cloud.client.loadbalancer.AsyncLoadBalancerAutoConfiguration; +import org.springframework.cloud.client.loadbalancer.LoadBalancerClient; +import org.springframework.cloud.client.loadbalancer.LoadBalancerProperties; +import org.springframework.cloud.client.loadbalancer.LoadBalancerUriTools; +import org.springframework.cloud.loadbalancer.annotation.LoadBalancerClients; +import org.springframework.cloud.loadbalancer.blocking.client.BlockingLoadBalancerClient; +import org.springframework.cloud.loadbalancer.config.BlockingLoadBalancerClientAutoConfiguration; +import org.springframework.cloud.loadbalancer.config.LoadBalancerAutoConfiguration; +import org.springframework.cloud.loadbalancer.support.LoadBalancerClientFactory; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.web.client.RestTemplate; + +import java.net.URI; +import java.util.Map; + +/** + * Job 自定义请求负载均衡客户端，用于解决在 IPv6 部署环境下，微服务内部调用报错 + */ +@Configuration(proxyBeanMethods = false) +@LoadBalancerClients +@AutoConfigureAfter(LoadBalancerAutoConfiguration.class) +@AutoConfigureBefore({org.springframework.cloud.client.loadbalancer.LoadBalancerAutoConfiguration.class, + AsyncLoadBalancerAutoConfiguration.class, BlockingLoadBalancerClientAutoConfiguration.class}) +@ConditionalOnClass(RestTemplate.class) +@Slf4j +public class JobLoadBalancerClientAutoConfiguration { + + @Bean + @ConditionalOnBean(LoadBalancerClientFactory.class) + @ConditionalOnMissingBean + @Primary + public LoadBalancerClient blockingLoadBalancerClient(LoadBalancerClientFactory loadBalancerClientFactory, + LoadBalancerProperties properties) { + log.info("Init JobLoadBalancerClient."); + return new JobLoadBalancerClient(loadBalancerClientFactory, properties); + } + + private static class JobLoadBalancerClient extends BlockingLoadBalancerClient { + public JobLoadBalancerClient(LoadBalancerClientFactory loadBalancerClientFactory, + LoadBalancerProperties properties) { + super(loadBalancerClientFactory, properties); + } + + @Override + public URI reconstructURI(ServiceInstance serviceInstance, URI original) { + return LoadBalancerUriTools.reconstructURI( + new Ipv6CapableDelegatingServiceInstance(serviceInstance), + original + ); + } + } + + private static class Ipv6CapableDelegatingServiceInstance implements ServiceInstance { + + final ServiceInstance delegate; + + public Ipv6CapableDelegatingServiceInstance(ServiceInstance delegate) { + this.delegate = delegate; + } + + @Override + public String getServiceId() { + return delegate.getServiceId(); + } + + @Override + public String getHost() { + return getAvailableIpv6Host(delegate.getHost()); + } + + private String getAvailableIpv6Host(String host) { + if (StringUtils.isNotBlank(host) && host.contains(":") && !host.startsWith("[")) { + return "[" + host + "]"; + } + return host; + } + + @Override + public int getPort() { + return delegate.getPort(); + } + + @Override + public boolean isSecure() { + return delegate.isSecure(); + } + + @Override + public URI getUri() { + return delegate.getUri(); + } + + @Override + public Map getMetadata() { + return delegate.getMetadata(); + } + + @Override + public String getScheme() { + return delegate.getScheme(); + } + } +} diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/metrics/GlobalMeterRegister.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/metrics/GlobalMeterRegister.java new file mode 100644 index 0000000000..6313908d1d --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/metrics/GlobalMeterRegister.java @@ -0,0 +1,53 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service.metrics; + +import com.tencent.bk.job.common.util.ApplicationContextRegister; +import io.micrometer.core.instrument.MeterRegistry; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class GlobalMeterRegister { + private static volatile MeterRegistry globalMeterRegistry; + + private static final Object lock = new Object(); + + private GlobalMeterRegister() { + + } + + public static MeterRegistry get() { + MeterRegistry meterRegistry = globalMeterRegistry; + if (meterRegistry == null) { + synchronized (lock) { + meterRegistry = globalMeterRegistry; + if (meterRegistry == null) { + meterRegistry = ApplicationContextRegister.getBean(MeterRegistry.class); + } + } + } + return meterRegistry; + } +} diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/package-info.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/package-info.java new file mode 100644 index 0000000000..760e36dbfb --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/package-info.java @@ -0,0 +1,28 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +/** + * Job 通用模块 - 微服务通用能力 + */ +package com.tencent.bk.job.common.service; diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/quota/ResourceQuotaConfigParseException.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/quota/ResourceQuotaConfigParseException.java new file mode 100644 index 0000000000..40340c2148 --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/quota/ResourceQuotaConfigParseException.java @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service.quota; + +/** + * 资源配额限制配置解析异常 + */ +public class ResourceQuotaConfigParseException extends RuntimeException { + public ResourceQuotaConfigParseException() { + } + + public ResourceQuotaConfigParseException(String message) { + super(message); + } + + public ResourceQuotaConfigParseException(String message, Throwable cause) { + super(message, cause); + } + + public ResourceQuotaConfigParseException(Throwable cause) { + super(cause); + } +} diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/quota/ResourceQuotaLoadApplicationRunner.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/quota/ResourceQuotaLoadApplicationRunner.java new file mode 100644 index 0000000000..c6cd400edf --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/quota/ResourceQuotaLoadApplicationRunner.java @@ -0,0 +1,49 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service.quota; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.ApplicationArguments; +import org.springframework.boot.ApplicationRunner; + +/** + * 配额限制配置加载 ApplicationRunner + */ +@Slf4j +public class ResourceQuotaLoadApplicationRunner implements ApplicationRunner { + private final ResourceQuotaStore resourceQuotaStore; + + public ResourceQuotaLoadApplicationRunner(ResourceQuotaStore resourceQuotaStore) { + this.resourceQuotaStore = resourceQuotaStore; + } + + @Override + public void run(ApplicationArguments args) throws Exception { + // 初始化配置；如果初始化错误，那么抛出异常终止程序启动 + log.info("ResourceQuotaLoadApplicationRunner start"); + resourceQuotaStore.load(false); + log.info("ResourceQuotaLoadApplicationRunner run success"); + } +} diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/quota/ResourceQuotaStore.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/quota/ResourceQuotaStore.java new file mode 100644 index 0000000000..267ecd4d50 --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/quota/ResourceQuotaStore.java @@ -0,0 +1,219 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service.quota; + +import com.tencent.bk.job.common.model.dto.ResourceScope; +import com.tencent.bk.job.common.refreshable.config.ConfigRefreshHandler; +import com.tencent.bk.job.common.resource.quota.QuotaResourceId; +import com.tencent.bk.job.common.resource.quota.ResourceQuotaLimit; +import com.tencent.bk.job.common.service.quota.config.ResourceQuotaLimitProperties; +import com.tencent.bk.job.common.service.quota.config.parser.CounterResourceQuotaConfigParser; +import com.tencent.bk.job.common.service.quota.config.parser.ResourceQuotaConfigParser; +import com.tencent.bk.job.common.util.ApplicationContextRegister; +import com.tencent.bk.job.common.util.json.JsonUtils; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +/** + * 配额限制存储 + */ +@Slf4j +public class ResourceQuotaStore implements ConfigRefreshHandler { + + /** + * 属性名称前缀 + */ + public static final String PROP_KEY_PREFIX = "job.resourceQuotaLimit."; + + /** + * key: QuotaResourceId; value: ResourceQuotaLimit + */ + private volatile Map resourceQuotas = new HashMap<>(); + /** + * 是否初始化 + */ + private volatile boolean isInitial = false; + + + public ResourceQuotaLimit getResourceQuota(String resourceId) { + checkLoad(); + return resourceQuotas.get(resourceId); + } + + public boolean isQuotaLimitEnabled(String resourceId) { + ResourceQuotaLimit resourceQuotaLimit = getResourceQuota(resourceId); + return resourceQuotaLimit != null && resourceQuotaLimit.isEnabled(); + } + + /** + * 获取资源管理空间配额限制 + * + * @param resourceId 资源 ID + * @param resourceScope 资源管理空间 + * @return 配额限制 + */ + public long getQuotaLimitByResourceScope(String resourceId, ResourceScope resourceScope) { + ResourceQuotaLimit resourceQuotaLimit = getResourceQuota(resourceId); + if (resourceQuotaLimit == null || !resourceQuotaLimit.isEnabled()) { + return ResourceQuotaLimit.UNLIMITED_VALUE; + } + return resourceQuotaLimit.getLimitByResourceScope(resourceScope); + } + + /** + * 获取应用配额限制 + * + * @param resourceId 资源 ID + * @param appCode 蓝鲸应用 App Code + * @return 配额限制 + */ + public long getQuotaLimitByAppCode(String resourceId, String appCode) { + ResourceQuotaLimit resourceQuotaLimit = getResourceQuota(resourceId); + if (resourceQuotaLimit == null || !resourceQuotaLimit.isEnabled()) { + return ResourceQuotaLimit.UNLIMITED_VALUE; + } + return resourceQuotaLimit.getLimitByBkAppCode(appCode); + + } + + /** + * 获取 Job 系统配额限制 + * + * @param resourceId 资源 ID + * @return 配额限制 + */ + public long getSystemQuotaLimit(String resourceId) { + ResourceQuotaLimit resourceQuotaLimit = getResourceQuota(resourceId); + if (resourceQuotaLimit == null || !resourceQuotaLimit.isEnabled()) { + return ResourceQuotaLimit.UNLIMITED_VALUE; + } + return resourceQuotaLimit.getCapacity() != null ? resourceQuotaLimit.getCapacity() : + ResourceQuotaLimit.UNLIMITED_VALUE; + + } + + public Map getAll() { + checkLoad(); + return resourceQuotas; + } + + private void checkLoad() { + if (!isInitial) { + synchronized (this) { + if (!isInitial) { + load(true); + } + } + } + } + + public boolean load(boolean ignoreException) { + boolean loadResult = true; + try { + loadInternal(); + } catch (Throwable e) { + log.warn("Load ResourceQuota error", e); + loadResult = false; + if (ignoreException) { + log.warn("Ignore ResourceQuota load error"); + } else { + throw e; + } + } + return loadResult; + } + + private void loadInternal() { + synchronized (this) { + log.info("Load ResourceQuota start ..."); + ResourceQuotaLimitProperties resourceQuotaLimitProperties = + ApplicationContextRegister.getBean(ResourceQuotaLimitProperties.class); + + if (resourceQuotaLimitProperties.getResources() == null + || resourceQuotaLimitProperties.getResources().isEmpty()) { + log.info("ResourceQuota empty!"); + return; + } + + log.info("Parse ResourceQuota config: {}", JsonUtils.toJson(resourceQuotaLimitProperties)); + + Map tmpResourceQuotaLimits = new HashMap<>(); + resourceQuotaLimitProperties.getResources() + .forEach((resourceId, resourceQuotaConfig) -> { + ResourceQuotaLimit resourceQuotaLimit = parseResourceQuotaConfig(resourceId, resourceQuotaConfig); + tmpResourceQuotaLimits.put(resourceId, resourceQuotaLimit); + }); + + // 使用新的配置完全替换老的配置 + resourceQuotas = tmpResourceQuotaLimits; + log.info("Load ResourceQuota config done! resourceQuotas: {}", resourceQuotas); + isInitial = true; + } + } + + /** + * 解析特性配置 + * + * @param resourceId 资源 ID + * @param resourceQuotaLimitProp 配置信息 + * @throws ResourceQuotaConfigParseException 如果解析报错，抛出异常 + */ + private ResourceQuotaLimit parseResourceQuotaConfig( + String resourceId, + ResourceQuotaLimitProperties.ResourceQuotaLimitProp resourceQuotaLimitProp + ) throws ResourceQuotaConfigParseException { + + if (StringUtils.isBlank(resourceId)) { + log.error("resourceId is blank"); + throw new ResourceQuotaConfigParseException("resourceId is blank"); + } + + ResourceQuotaConfigParser configParser = chooseResourceQuotaConfigParser(resourceId); + return configParser.parse(resourceQuotaLimitProp); + } + + + private ResourceQuotaConfigParser chooseResourceQuotaConfigParser(String resourceId) { + ResourceQuotaConfigParser configParser; + switch (resourceId) { + case QuotaResourceId.JOB_INSTANCE: + configParser = new CounterResourceQuotaConfigParser(); + break; + default: + log.error("Unsupported resource: {}", resourceId); + throw new ResourceQuotaConfigParseException("Unsupported resource: " + resourceId); + } + return configParser; + } + + @Override + public boolean handleConfigChange(Set changedKeys) { + return load(true); + } +} diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/quota/RunningJobResourceQuotaStore.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/quota/RunningJobResourceQuotaStore.java new file mode 100644 index 0000000000..af8e005b80 --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/quota/RunningJobResourceQuotaStore.java @@ -0,0 +1,73 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service.quota; + +import com.tencent.bk.job.common.model.dto.ResourceScope; +import com.tencent.bk.job.common.resource.quota.QuotaResourceId; + +/** + * 资源配额存储-正在执行中的作业数量配额限制 + */ +public class RunningJobResourceQuotaStore { + + private final ResourceQuotaStore resourceQuotaStore; + + public RunningJobResourceQuotaStore(ResourceQuotaStore resourceQuotaStore) { + this.resourceQuotaStore = resourceQuotaStore; + } + + public boolean isQuotaLimitEnabled() { + return resourceQuotaStore.isQuotaLimitEnabled(QuotaResourceId.JOB_INSTANCE); + } + + /** + * 根据资源管理空间获取正在执行作业配额限制 + * + * @param resourceScope 资源管理空间 + * @return 最大正在执行作业限制 + */ + public long getQuotaLimitByResourceScope(ResourceScope resourceScope) { + return resourceQuotaStore.getQuotaLimitByResourceScope(QuotaResourceId.JOB_INSTANCE, resourceScope); + } + + /** + * 根据应用 Code 获取正在执行作业配额限制 + * + * @param appCode 蓝鲸应用 App Code + * @return 最大正在执行作业限制 + */ + public long getQuotaLimitByAppCode(String appCode) { + return resourceQuotaStore.getQuotaLimitByAppCode(QuotaResourceId.JOB_INSTANCE, appCode); + } + + /** + * 获取整个Job 系统 正在执行作业配额限制 + * + * @return 最大正在执行作业限制 + */ + public long getSystemQuotaLimit() { + return resourceQuotaStore.getSystemQuotaLimit(QuotaResourceId.JOB_INSTANCE); + } +} diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/quota/config/ResourceQuotaAutoConfiguration.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/quota/config/ResourceQuotaAutoConfiguration.java new file mode 100644 index 0000000000..92b633aee2 --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/quota/config/ResourceQuotaAutoConfiguration.java @@ -0,0 +1,56 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service.quota.config; + +import com.tencent.bk.job.common.service.quota.ResourceQuotaLoadApplicationRunner; +import com.tencent.bk.job.common.service.quota.ResourceQuotaStore; +import com.tencent.bk.job.common.service.quota.RunningJobResourceQuotaStore; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * 配额限制 Spring AutoConfiguration + */ +@Configuration(proxyBeanMethods = false) +@EnableConfigurationProperties({ResourceQuotaLimitProperties.class}) +public class ResourceQuotaAutoConfiguration { + + @Bean + public ResourceQuotaStore resourceQuotaStore() { + return new ResourceQuotaStore(); + } + + @Bean + public RunningJobResourceQuotaStore resourceScopeResourceQuotaManager(ResourceQuotaStore resourceQuotaStore) { + return new RunningJobResourceQuotaStore(resourceQuotaStore); + } + + @Bean + public ResourceQuotaLoadApplicationRunner resourceQuotaLoadApplicationRunner( + ResourceQuotaStore resourceQuotaStore) { + return new ResourceQuotaLoadApplicationRunner(resourceQuotaStore); + } +} diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/quota/config/ResourceQuotaLimitProperties.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/quota/config/ResourceQuotaLimitProperties.java new file mode 100644 index 0000000000..3df938798c --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/quota/config/ResourceQuotaLimitProperties.java @@ -0,0 +1,109 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service.quota.config; + +import com.tencent.bk.job.common.resource.quota.QuotaResourceId; +import com.tencent.bk.job.common.util.json.JsonUtils; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.context.properties.ConfigurationProperties; + +import javax.annotation.PostConstruct; +import java.util.Map; + +/** + * 资源配额限制-配置信息 + */ +@ConfigurationProperties(prefix = "job.resource-quota-limit", ignoreInvalidFields = true) +@Data +@Slf4j +public class ResourceQuotaLimitProperties { + + /** + * key: 资源 ID ； value: 资源对应的配额限制 + * + * @see QuotaResourceId + */ + private Map resources; + + @Data + @NoArgsConstructor + public static class ResourceQuotaLimitProp { + /** + * 是否启用配额限制 + */ + private boolean enabled; + /** + * 配额容量（比如任务数量、存储占用大小等） + */ + private String capacity; + + /** + * 资源管理空间-配额 + */ + private QuotaLimitProp resourceScopeQuotaLimit; + + /** + * 蓝鲸应用-配额 + */ + private QuotaLimitProp appQuotaLimit; + + public ResourceQuotaLimitProp(boolean enabled, + String capacity, + QuotaLimitProp resourceScopeQuotaLimit, + QuotaLimitProp appQuotaLimit) { + this.enabled = enabled; + this.capacity = capacity; + this.resourceScopeQuotaLimit = resourceScopeQuotaLimit; + this.appQuotaLimit = appQuotaLimit; + } + } + + @Data + @NoArgsConstructor + public static class QuotaLimitProp { + + /** + * 全局配额表达式 + */ + private String global; + + /** + * 自定义配额表达式。会覆盖全局配额配置 + */ + private String custom; + + public QuotaLimitProp(String global, String custom) { + this.global = global; + this.custom = custom; + } + } + + @PostConstruct + public void print() { + log.info("ResourceQuotaLimitProperties init: {}", JsonUtils.toJson(this)); + } +} diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/quota/config/parser/AbstractResourceQuotaConfigParser.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/quota/config/parser/AbstractResourceQuotaConfigParser.java new file mode 100644 index 0000000000..6eb9d85050 --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/quota/config/parser/AbstractResourceQuotaConfigParser.java @@ -0,0 +1,119 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service.quota.config.parser; + +import com.tencent.bk.job.common.resource.quota.AppQuotaLimit; +import com.tencent.bk.job.common.resource.quota.QuotaLimit; +import com.tencent.bk.job.common.resource.quota.ResourceScopeQuotaLimit; +import org.apache.commons.lang3.StringUtils; + +import java.util.HashMap; +import java.util.Map; +import java.util.function.Function; + +/** + * 资源配额配置解析-抽象实现 + */ +public abstract class AbstractResourceQuotaConfigParser implements ResourceQuotaConfigParser { + + protected abstract Long parseCapacity(String capacityExpr); + + protected ResourceScopeQuotaLimit parseResourceScopeQuotaLimit(Long capacity, + String globalLimitExpr, + String customLimitExpr) { + + ResourceScopeQuotaLimit resourceScopeQuotaLimit = new ResourceScopeQuotaLimit( + globalLimitExpr, + customLimitExpr + ); + + parseGlobalLimit(resourceScopeQuotaLimit, capacity, globalLimitExpr); + parseCustomLimit(resourceScopeQuotaLimit, capacity, customLimitExpr); + + return resourceScopeQuotaLimit; + } + + protected AppQuotaLimit parseAppQuotaLimit(Long capacity, + String globalLimitExpr, + String customLimitExpr) { + AppQuotaLimit appQuotaLimit = new AppQuotaLimit( + globalLimitExpr, + customLimitExpr + ); + + parseGlobalLimit(appQuotaLimit, capacity, globalLimitExpr); + parseCustomLimit(appQuotaLimit, capacity, customLimitExpr); + + return appQuotaLimit; + } + + private void parseGlobalLimit(QuotaLimit quotaLimit, Long capacity, String globalLimitExpr) { + long globalLimit = computeLimitValue(capacity, globalLimitExpr); + quotaLimit.setGlobalLimit(globalLimit); + } + + private void parseCustomLimit(ResourceScopeQuotaLimit quotaLimit, Long capacity, String customLimitExpr) { + quotaLimit.setCustomLimits(parseCustomLimit(capacity, customLimitExpr, key -> key)); + } + + private void parseCustomLimit(AppQuotaLimit quotaLimit, Long capacity, String customLimitExpr) { + quotaLimit.setCustomLimits(parseCustomLimit(capacity, customLimitExpr, key -> key)); + } + + protected Map parseCustomLimit(Long capacity, + String customLimitExpr, + Function customKeyParser) { + Map customLimits = new HashMap<>(); + if (StringUtils.isNotBlank(customLimitExpr)) { + + customLimitExpr = customLimitExpr.trim(); + String[] limitExprParts = customLimitExpr.split(","); + for (String limitExpr : limitExprParts) { + String[] limitExprKV = limitExpr.split("="); + K key = customKeyParser.apply(limitExprKV[0]); + Long limit = computeLimitValue(capacity, limitExprKV[1]); + customLimits.put(key, limit); + } + + } + return customLimits; + } + + protected long computeLimitValue(Long capacity, String limitExpr) { + long limit; + if (limitExpr.endsWith("%")) { + limit = computePercentageLimitValue(capacity, limitExpr); + } else { + limit = Long.parseLong(limitExpr); + } + return limit; + } + + protected long computePercentageLimitValue(Long capacity, String limitExpr) { + String percentageValueStr = limitExpr.substring(0, limitExpr.length() - 1); + int percentageValue = Integer.parseInt(percentageValueStr); + return capacity * percentageValue / 100; + } +} diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/quota/config/parser/CounterResourceQuotaConfigParser.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/quota/config/parser/CounterResourceQuotaConfigParser.java new file mode 100644 index 0000000000..4b83df8296 --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/quota/config/parser/CounterResourceQuotaConfigParser.java @@ -0,0 +1,82 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service.quota.config.parser; + +import com.tencent.bk.job.common.resource.quota.AppQuotaLimit; +import com.tencent.bk.job.common.resource.quota.ResourceQuotaLimit; +import com.tencent.bk.job.common.resource.quota.ResourceScopeQuotaLimit; +import com.tencent.bk.job.common.service.quota.ResourceQuotaConfigParseException; +import com.tencent.bk.job.common.service.quota.config.ResourceQuotaLimitProperties; +import org.apache.commons.lang3.StringUtils; + +/** + * 配额限制配置解析-计数类型 + */ +public class CounterResourceQuotaConfigParser extends AbstractResourceQuotaConfigParser { + + @Override + public ResourceQuotaLimit parse(ResourceQuotaLimitProperties.ResourceQuotaLimitProp resourceQuotaLimitProp) + throws ResourceQuotaConfigParseException { + + ResourceQuotaLimit resourceQuota = new ResourceQuotaLimit(); + try { + resourceQuota.setEnabled(resourceQuotaLimitProp.isEnabled()); + resourceQuota.setCapacityExpr(resourceQuotaLimitProp.getCapacity()); + Long capacity = parseCapacity(resourceQuotaLimitProp.getCapacity()); + resourceQuota.setCapacity(capacity); + + ResourceQuotaLimitProperties.QuotaLimitProp resourceScopeQuotaLimitProp + = resourceQuotaLimitProp.getResourceScopeQuotaLimit(); + if (resourceScopeQuotaLimitProp != null) { + ResourceScopeQuotaLimit resourceScopeQuotaLimit = parseResourceScopeQuotaLimit( + capacity, + resourceScopeQuotaLimitProp.getGlobal(), + resourceScopeQuotaLimitProp.getCustom() + ); + resourceQuota.setResourceScopeQuotaLimit(resourceScopeQuotaLimit); + } + + ResourceQuotaLimitProperties.QuotaLimitProp appQuotaLimitProp + = resourceQuotaLimitProp.getAppQuotaLimit(); + if (resourceQuotaLimitProp.getAppQuotaLimit() != null) { + AppQuotaLimit appQuotaLimit = parseAppQuotaLimit( + capacity, + appQuotaLimitProp.getGlobal(), + appQuotaLimitProp.getCustom() + ); + resourceQuota.setAppQuotaLimit(appQuotaLimit); + } + + return resourceQuota; + } catch (Throwable e) { + throw new ResourceQuotaConfigParseException(e); + } + } + + @Override + protected Long parseCapacity(String capacityExpr) { + return StringUtils.isNotBlank(capacityExpr) ? Long.parseLong(capacityExpr) : null; + } +} diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/quota/config/parser/ResourceQuotaConfigParser.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/quota/config/parser/ResourceQuotaConfigParser.java new file mode 100644 index 0000000000..545b9441b3 --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/quota/config/parser/ResourceQuotaConfigParser.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service.quota.config.parser; + +import com.tencent.bk.job.common.resource.quota.ResourceQuotaLimit; +import com.tencent.bk.job.common.service.quota.ResourceQuotaConfigParseException; +import com.tencent.bk.job.common.service.quota.config.ResourceQuotaLimitProperties; + +/** + * 资源配额配置解析 + */ +public interface ResourceQuotaConfigParser { + + /** + * 解析配额配置信息 + * + * @param resourceQuotaLimitProp 资源配额限制的配置 + * @return 资源配额 + */ + ResourceQuotaLimit parse(ResourceQuotaLimitProperties.ResourceQuotaLimitProp resourceQuotaLimitProp) + throws ResourceQuotaConfigParseException; +} diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/quota/package-info.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/quota/package-info.java new file mode 100644 index 0000000000..3f1ea3cce1 --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/quota/package-info.java @@ -0,0 +1,29 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service.quota; + +/* + * 资源配额限制 + */ diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/feature/DefaultFeatureManager.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/feature/DefaultFeatureManager.java new file mode 100644 index 0000000000..82dcb4e9e6 --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/feature/DefaultFeatureManager.java @@ -0,0 +1,144 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service.toggle.feature; + +import com.tencent.bk.job.common.metrics.CommonMetricTags; +import com.tencent.bk.job.common.model.dto.ResourceScope; +import com.tencent.bk.job.common.util.toggle.ToggleEvaluateContext; +import com.tencent.bk.job.common.util.toggle.ToggleStrategy; +import com.tencent.bk.job.common.util.toggle.ToggleStrategyContextParams; +import com.tencent.bk.job.common.util.toggle.feature.Feature; +import com.tencent.bk.job.common.util.toggle.feature.FeatureManager; +import com.tencent.bk.job.common.util.toggle.feature.FeatureStore; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tags; +import lombok.extern.slf4j.Slf4j; + +import java.util.List; + +@Slf4j +public class DefaultFeatureManager implements FeatureManager { + /** + * 度量指标 - 特性灰度检查总次数 + */ + private static final String METRIC_JOB_FEATURE_TOGGLE_TOTAL = "job.feature.toggle.total"; + /** + * 度量指标 - 特性灰度命中总次数 + */ + private static final String METRIC_JOB_FEATURE_TOGGLE_HIT_TOTAL = "job.feature.toggle.hit.total"; + + private final FeatureStore featureStore; + private final MeterRegistry meterRegistry; + + public DefaultFeatureManager(FeatureStore featureStore, MeterRegistry meterRegistry) { + this.featureStore = featureStore; + this.meterRegistry = meterRegistry; + } + + public boolean isFeatureEnabled(String featureId) { + Feature feature = featureStore.getFeature(featureId); + if (feature == null) { + if (log.isDebugEnabled()) { + log.debug("Feature: {} is not exist!", featureId); + } + return false; + } + return feature.isEnabled(); + } + + /** + * 判断特性是否开启 + * + * @param featureId 特性ID + * @param ctx 特性运行上下文 + * @return 是否开启 + */ + public boolean checkFeature(String featureId, ToggleEvaluateContext ctx) { + String resourceScope = extractResourceScope(ctx); + recordFeatureToggleTotal(featureId, resourceScope); + + Feature feature = featureStore.getFeature(featureId); + if (log.isDebugEnabled()) { + log.debug("Check feature {}", featureId); + } + if (feature == null) { + if (log.isDebugEnabled()) { + log.debug("Feature: {} is not exist!", featureId); + } + return false; + } + if (!feature.isEnabled()) { + if (log.isDebugEnabled()) { + log.debug("Feature: {} is disabled!", featureId); + } + return false; + } + + ToggleStrategy strategy = feature.getStrategy(); + if (strategy == null) { + if (log.isDebugEnabled()) { + log.debug("Feature:{} strategy is empty!", featureId); + } + // 如果没有配置特性开启策略，且enabled=true，判定为特性开启 + recordFeatureToggleHitTotal(featureId, resourceScope); + return true; + } + + boolean result = strategy.evaluate(featureId, ctx); + if (log.isDebugEnabled()) { + log.debug("Apply feature toggle strategy, featureId: {}, context: {}, result: {}", featureId, ctx, result); + } + if (result) { + recordFeatureToggleHitTotal(featureId, resourceScope); + } + return result; + } + + private String extractResourceScope(ToggleEvaluateContext ctx) { + Object resourceScopeObj = ctx.getParam(ToggleStrategyContextParams.CTX_PARAM_RESOURCE_SCOPE); + if (resourceScopeObj == null) { + return "None"; + } + ResourceScope resourceScope = (ResourceScope) resourceScopeObj; + return resourceScope.getType() + ":" + resourceScope.getId(); + } + + private void recordFeatureToggleTotal(String featureId, String resourceScope) { + meterRegistry.counter(METRIC_JOB_FEATURE_TOGGLE_TOTAL, + Tags.of(CommonMetricTags.KEY_RESOURCE_SCOPE, resourceScope).and("feature", featureId)) + .increment(); + } + + private void recordFeatureToggleHitTotal(String featureId, String resourceScope) { + meterRegistry.counter(METRIC_JOB_FEATURE_TOGGLE_HIT_TOTAL, + Tags.of(CommonMetricTags.KEY_RESOURCE_SCOPE, resourceScope).and("feature", featureId)) + .increment(); + } + + @Override + public List listFeatures() { + return featureStore.listFeatures(); + } +} diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/feature/FeatureConfigParseException.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/feature/FeatureConfigParseException.java new file mode 100644 index 0000000000..2faa2580f8 --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/feature/FeatureConfigParseException.java @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service.toggle.feature; + +/** + * 特性开关配置解析异常 + */ +public class FeatureConfigParseException extends RuntimeException { + public FeatureConfigParseException() { + } + + public FeatureConfigParseException(String message) { + super(message); + } + + public FeatureConfigParseException(String message, Throwable cause) { + super(message, cause); + } + + public FeatureConfigParseException(Throwable cause) { + super(cause); + } +} diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/feature/FeatureToggleAutoConfiguration.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/feature/FeatureToggleAutoConfiguration.java new file mode 100644 index 0000000000..3822a76bc4 --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/feature/FeatureToggleAutoConfiguration.java @@ -0,0 +1,48 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service.toggle.feature; + +import com.tencent.bk.job.common.service.toggle.feature.config.FeatureToggleProperties; +import com.tencent.bk.job.common.util.toggle.feature.FeatureManager; +import com.tencent.bk.job.common.util.toggle.feature.FeatureStore; +import io.micrometer.core.instrument.MeterRegistry; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration(proxyBeanMethods = false) +@EnableConfigurationProperties({FeatureToggleProperties.class}) +public class FeatureToggleAutoConfiguration { + + @Bean + public FeatureStore featureStore(FeatureToggleProperties featureToggleProperties) { + return new InMemoryFeatureStore(featureToggleProperties); + } + + @Bean + public FeatureManager featureManager(FeatureStore featureStore, MeterRegistry meterRegistry) { + return new DefaultFeatureManager(featureStore, meterRegistry); + } +} diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/feature/InMemoryFeatureStore.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/feature/InMemoryFeatureStore.java new file mode 100644 index 0000000000..97332d8ca5 --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/feature/InMemoryFeatureStore.java @@ -0,0 +1,154 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service.toggle.feature; + +import com.tencent.bk.job.common.service.toggle.feature.config.FeatureConfig; +import com.tencent.bk.job.common.service.toggle.feature.config.FeatureToggleProperties; +import com.tencent.bk.job.common.service.toggle.strategy.ToggleStrategyParseException; +import com.tencent.bk.job.common.service.toggle.strategy.ToggleStrategyParser; +import com.tencent.bk.job.common.service.toggle.strategy.config.ToggleStrategyConfig; +import com.tencent.bk.job.common.util.ApplicationContextRegister; +import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.common.util.toggle.ToggleStrategy; +import com.tencent.bk.job.common.util.toggle.feature.Feature; +import com.tencent.bk.job.common.util.toggle.feature.FeatureStore; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * 特性开关配置存储实现 + */ +@Slf4j +public class InMemoryFeatureStore implements FeatureStore { + + /** + * key: featureId; value: Feature + */ + private volatile Map features = new HashMap<>(); + + public InMemoryFeatureStore(FeatureToggleProperties featureToggleProperties) { + log.info("Init InMemoryFeatureStore, properties : {}", JsonUtils.toJson(featureToggleProperties)); + loadAllFeatures(featureToggleProperties); + log.info("Init InMemoryFeatureStore successfully"); + } + + @Override + public Feature getFeature(String featureId) { + return features.get(featureId); + } + + @Override + public boolean handleConfigChange(Set changedKeys, boolean ignoreException) { + boolean loadResult = true; + try { + loadAllFeatures(getRealTimeFeatureToggleProperties()); + } catch (Throwable e) { + log.warn("Load feature config error", e); + loadResult = false; + if (ignoreException) { + log.warn("Ignore feature config load error"); + } else { + throw e; + } + } + return loadResult; + } + + /** + * 获取 FeatureToggleProperties 实时配置 + */ + private FeatureToggleProperties getRealTimeFeatureToggleProperties() { + return ApplicationContextRegister.getBean(FeatureToggleProperties.class); + } + + private void loadAllFeatures(FeatureToggleProperties featureToggleProperties) { + synchronized (this) { + log.info("Load feature toggle start ..."); + + if (featureToggleProperties.getFeatures() == null || featureToggleProperties.getFeatures().isEmpty()) { + log.info("Feature toggle config empty!"); + return; + } + + log.info("Parse feature toggle config: {}", JsonUtils.toJson(featureToggleProperties)); + + Map tmpFeatures = new HashMap<>(); + featureToggleProperties.getFeatures().forEach((featureId, featureConfig) -> { + Feature feature = parseFeatureConfig(featureId, featureConfig); + tmpFeatures.put(featureId, feature); + }); + + // 使用新的配置完全替换老的配置 + features = tmpFeatures; + log.info("Load feature toggle config done! features: {}", features); + } + } + + /** + * 解析特性配置 + * + * @param featureId 特性 ID + * @param featureConfig 特性配置信息 + * @return 解析之后的特性 + * @throws ToggleStrategyParseException 如果解析策略报错，抛出异常 + * @throws FeatureConfigParseException 如果解析报错，抛出异常 + */ + private Feature parseFeatureConfig( + String featureId, + FeatureConfig featureConfig) + throws ToggleStrategyParseException, FeatureConfigParseException { + + if (StringUtils.isBlank(featureId)) { + log.error("FeatureId is blank"); + throw new FeatureConfigParseException("FeatureId is blank"); + } + Feature feature = new Feature(); + feature.setId(featureId); + feature.setEnabled(featureConfig.isEnabled()); + + if (featureConfig.isEnabled()) { + ToggleStrategyConfig strategyConfig = featureConfig.getStrategy(); + if (strategyConfig != null) { + ToggleStrategy toggleStrategy = ToggleStrategyParser.parseToggleStrategy(strategyConfig); + if (toggleStrategy != null) { + feature.setStrategy(toggleStrategy); + } + } + } + return feature; + } + + @Override + public List listFeatures() { + return new ArrayList<>(features.values()); + } + +} diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/feature/config/FeatureConfig.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/feature/config/FeatureConfig.java new file mode 100644 index 0000000000..8b457587bf --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/feature/config/FeatureConfig.java @@ -0,0 +1,43 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service.toggle.feature.config; + +import com.tencent.bk.job.common.service.toggle.strategy.config.ToggleStrategyConfig; +import lombok.Data; + +/** + * 特性配置 + */ +@Data +public class FeatureConfig { + /** + * 是否启用特性 + */ + private boolean enabled; + /** + * 特性启用策略；必须enabled=true,策略才会生效 + */ + private ToggleStrategyConfig strategy; +} diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/feature/config/FeatureToggleProperties.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/feature/config/FeatureToggleProperties.java new file mode 100644 index 0000000000..31cd1879f5 --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/feature/config/FeatureToggleProperties.java @@ -0,0 +1,59 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service.toggle.feature.config; + +import com.tencent.bk.job.common.util.json.JsonUtils; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.context.properties.ConfigurationProperties; + +import javax.annotation.PostConstruct; +import java.util.Map; + +/** + * 特性开关配置 + *

+ * ignoreInvalidFields: true, 避免因为错误的配置导致微服务不可用（RefreshScopeHealthIndicator会对ConfigurationProperties + * 进行健康检查，如果配置有问题，会把微服务的状态设置为health=DOWN) + */ +@ConfigurationProperties(prefix = "job", ignoreInvalidFields = true) +@ToString +@Getter +@Setter +@Slf4j +public class FeatureToggleProperties { + + /** + * 特性 + */ + private Map features; + + @PostConstruct + public void print() { + log.info("FeatureToggleProperties init: {}", JsonUtils.toJson(this)); + } +} diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/prop/InMemoryPropToggleStore.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/prop/InMemoryPropToggleStore.java new file mode 100644 index 0000000000..1f3996d6f1 --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/prop/InMemoryPropToggleStore.java @@ -0,0 +1,195 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service.toggle.prop; + +import com.tencent.bk.job.common.service.toggle.strategy.ToggleStrategyParser; +import com.tencent.bk.job.common.service.toggle.strategy.config.ToggleStrategyConfig; +import com.tencent.bk.job.common.util.ApplicationContextRegister; +import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.common.util.toggle.ToggleStrategy; +import com.tencent.bk.job.common.util.toggle.prop.PropChangeEventListener; +import com.tencent.bk.job.common.util.toggle.prop.PropToggle; +import com.tencent.bk.job.common.util.toggle.prop.PropToggleStore; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * 属性开关配置存储实现 + */ +@Slf4j +public class InMemoryPropToggleStore implements PropToggleStore { + + /** + * key: prop_name; value: PropToggle + */ + private volatile Map propToggles = new HashMap<>(); + + private final Map> propEventListeners = new HashMap<>(); + + public InMemoryPropToggleStore(PropToggleProperties propToggleProperties) { + log.info("Init InMemoryPropToggleStore, properties : {}", JsonUtils.toJson(propToggleProperties)); + loadAllPropToggles(propToggleProperties); + log.info("Init InMemoryPropToggleStore successfully"); + } + + public PropToggle getPropToggle(String propName) { + return propToggles.get(propName); + } + + @Override + public boolean handleConfigChange(Set changedKeys, boolean ignoreException) { + boolean loadResult = true; + try { + loadAllPropToggles(getRealTimePropToggleProperties()); + } catch (Throwable e) { + log.warn("Load prop config error", e); + loadResult = false; + if (ignoreException) { + log.warn("Ignore prop config load error"); + } else { + throw e; + } + } + Set uniquePropNames = getUniqueChangedPropNames(changedKeys); + log.info("Changed prop names : {}", uniquePropNames); + uniquePropNames.forEach(propName -> { + List listeners = propEventListeners.get(propName); + if (CollectionUtils.isNotEmpty(listeners)) { + listeners.forEach(listener -> { + log.info("Invoke listener {}", listener.getClass().getSimpleName()); + listener.handlePropChangeEvent(propName, getPropToggle(propName)); + }); + } + }); + return loadResult; + } + + private Set getUniqueChangedPropNames(Set changedKeys) { + Set uniquePropNames = new HashSet<>(); + + for (String changeKey : changedKeys) { + if (!changeKey.startsWith(PROP_KEY_PREFIX)) { + log.error("Invalid key : {}", changeKey); + } + String[] parts = changeKey.split("\\."); + // 格式 job.toggle.props.{propName}.others + if (parts.length >= 4) { + uniquePropNames.add(parts[3]); + } else { + log.error("Invalid key : {}", changeKey); + } + } + + return uniquePropNames; + } + + + /** + * 获取 PropToggleProperties 实时配置 + */ + private PropToggleProperties getRealTimePropToggleProperties() { + return ApplicationContextRegister.getBean(PropToggleProperties.class); + } + + private void loadAllPropToggles(PropToggleProperties propToggleProperties) { + synchronized (this) { + log.info("Load prop toggle start ..."); + + if (propToggleProperties.getProps() == null || propToggleProperties.getProps().isEmpty()) { + log.info("Prop toggle config empty!"); + return; + } + + log.info("Parse prop toggle config: {}", JsonUtils.toJson(propToggleProperties)); + + Map tmpPropToggles = new HashMap<>(); + propToggleProperties.getProps().forEach((propName, propConfig) -> { + PropToggle propToggle = parsePropToggle(propName, propConfig); + tmpPropToggles.put(propName, propToggle); + }); + + // 使用新的配置完全替换老的配置 + propToggles = tmpPropToggles; + log.info("Load prop toggle config done! props: {}", propToggles); + } + } + + /** + * 解析配置 + * + * @param propName 属性名 + * @param propConfig 配置信息 + */ + private PropToggle parsePropToggle(String propName, + PropToggleProperties.PropToggleConfig propConfig) { + + if (StringUtils.isBlank(propName)) { + log.error("PropName is blank"); + throw new PropConfigParseException("PropName is blank"); + } + PropToggle propToggle = new PropToggle(); + propToggle.setDefaultValue(propConfig.getDefaultValue()); + + if (CollectionUtils.isEmpty(propConfig.getConditions())) { + return propToggle; + } + + List valueConditions = new ArrayList<>(propConfig.getConditions().size()); + for (PropToggleProperties.ConditionConfig conditionConfig : propConfig.getConditions()) { + PropToggle.PropValueCondition valueCondition = new PropToggle.PropValueCondition(); + valueCondition.setValue(conditionConfig.getValue()); + ToggleStrategyConfig strategyConfig = conditionConfig.getStrategy(); + if (strategyConfig != null) { + ToggleStrategy toggleStrategy = ToggleStrategyParser.parseToggleStrategy(strategyConfig); + valueCondition.setStrategy(toggleStrategy); + } + valueConditions.add(valueCondition); + } + propToggle.setConditions(valueConditions); + + return propToggle; + } + + + @Override + public void addPropChangeEventListener(String propName, PropChangeEventListener propChangeEventListener) { + propEventListeners.compute(propName, (prop, listeners) -> { + if (listeners == null) { + listeners = new ArrayList<>(); + } + listeners.add(propChangeEventListener); + return listeners; + }); + } + +} diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/prop/PropConfigParseException.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/prop/PropConfigParseException.java new file mode 100644 index 0000000000..dcdc8c5234 --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/prop/PropConfigParseException.java @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service.toggle.prop; + +/** + * 属性开关配置解析异常 + */ +public class PropConfigParseException extends RuntimeException { + public PropConfigParseException() { + } + + public PropConfigParseException(String message) { + super(message); + } + + public PropConfigParseException(String message, Throwable cause) { + super(message, cause); + } + + public PropConfigParseException(Throwable cause) { + super(cause); + } +} diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/prop/PropToggleAutoConfiguration.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/prop/PropToggleAutoConfiguration.java new file mode 100644 index 0000000000..228156bd63 --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/prop/PropToggleAutoConfiguration.java @@ -0,0 +1,40 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service.toggle.prop; + +import com.tencent.bk.job.common.util.toggle.prop.PropToggleStore; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration(proxyBeanMethods = false) +@EnableConfigurationProperties({PropToggleProperties.class}) +public class PropToggleAutoConfiguration { + + @Bean + public PropToggleStore propToggleStore(PropToggleProperties propToggleProperties) { + return new InMemoryPropToggleStore(propToggleProperties); + } +} diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/prop/PropToggleProperties.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/prop/PropToggleProperties.java new file mode 100644 index 0000000000..74bc32ba45 --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/prop/PropToggleProperties.java @@ -0,0 +1,77 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service.toggle.prop; + +import com.tencent.bk.job.common.service.toggle.strategy.config.ToggleStrategyConfig; +import com.tencent.bk.job.common.util.json.JsonUtils; +import lombok.Data; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.context.properties.ConfigurationProperties; + +import javax.annotation.PostConstruct; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * 属性值动态切换开关配置 + */ +@ConfigurationProperties(prefix = "job.toggle", ignoreInvalidFields = true) +@ToString +@Getter +@Setter +@Slf4j +public class PropToggleProperties { + + /** + * key: prop_name; value: PropToggleConfig + */ + private Map props = new HashMap<>(); + + + @Data + @NoArgsConstructor + public static class PropToggleConfig { + private String defaultValue; + + private List conditions; + } + + @Data + @NoArgsConstructor + public static class ConditionConfig { + private String value; + private ToggleStrategyConfig strategy; + } + + @PostConstruct + public void print() { + log.info("PropToggleProperties init: {}", JsonUtils.toJson(this)); + } +} diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/strategy/AbstractCompositeToggleStrategy.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/strategy/AbstractCompositeToggleStrategy.java new file mode 100644 index 0000000000..c0575c868a --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/strategy/AbstractCompositeToggleStrategy.java @@ -0,0 +1,59 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service.toggle.strategy; + +import com.tencent.bk.job.common.util.toggle.ToggleStrategy; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; + +import java.util.List; +import java.util.Map; + +/** + * 组合策略抽象实现 + */ +@Slf4j +public abstract class AbstractCompositeToggleStrategy extends AbstractToggleStrategy { + /** + * 组合策略 + */ + protected final List compositeStrategies; + + public AbstractCompositeToggleStrategy(String strategyId, + List compositeStrategies, + Map initParams) { + super(strategyId, initParams); + this.compositeStrategies = compositeStrategies; + assertRequiredAtLeastOneStrategy(); + } + + protected void assertRequiredAtLeastOneStrategy() { + if (CollectionUtils.isEmpty(this.compositeStrategies)) { + String msg = "Required at least one strategy for this ToggleStrategy"; + log.error(msg); + throw new ToggleStrategyParseException(msg); + } + } +} diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/strategy/AbstractResourceScopeToggleStrategy.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/strategy/AbstractResourceScopeToggleStrategy.java new file mode 100644 index 0000000000..7432f62b86 --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/strategy/AbstractResourceScopeToggleStrategy.java @@ -0,0 +1,131 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service.toggle.strategy; + +import com.tencent.bk.job.common.constant.ResourceScopeTypeEnum; +import com.tencent.bk.job.common.model.dto.ResourceScope; +import com.tencent.bk.job.common.util.toggle.ToggleEvaluateContext; +import com.tencent.bk.job.common.util.toggle.ToggleStrategyContextParams; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.helpers.MessageFormatter; + +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +/** + * 根据资源范围灰度策略-基础实现 + */ +@Slf4j +public abstract class AbstractResourceScopeToggleStrategy extends AbstractToggleStrategy { + /** + * 策略初始化参数-资源范围 + */ + public static final String INIT_PARAM_RESOURCE_SCOPE_LIST = "resourceScopeList"; + + protected final Set resourceScopes = new HashSet<>(); + + public AbstractResourceScopeToggleStrategy(String strategyId, Map initParams) { + super(strategyId, initParams); + assertRequiredInitParam(INIT_PARAM_RESOURCE_SCOPE_LIST); + + String resourceScopesValue = initParams.get(INIT_PARAM_RESOURCE_SCOPE_LIST); + if (StringUtils.isNotEmpty(resourceScopesValue)) { + String[] resourceScopeArray = resourceScopesValue.split(","); + if (resourceScopeArray.length == 0) { + String msg = MessageFormatter.format( + "Parameter {} is invalid, value: {}", + INIT_PARAM_RESOURCE_SCOPE_LIST, resourceScopesValue).getMessage(); + log.error(msg); + throw new ToggleStrategyParseException(msg); + } + for (String resourceScope : resourceScopeArray) { + this.resourceScopes.add(parseResourceScope(resourceScope)); + } + } + } + + private ResourceScope parseResourceScope(String resourceScope) { + String[] scopeTypeAndId = resourceScope.split(":"); + if (scopeTypeAndId.length != 2) { + String msg = MessageFormatter.format( + "Parameter {} is invalid. Invalid resource scope: {}", + INIT_PARAM_RESOURCE_SCOPE_LIST, resourceScope).getMessage(); + log.error(msg); + throw new ToggleStrategyParseException(msg); + } + String scopeType = scopeTypeAndId[0].trim(); + if (!ResourceScopeTypeEnum.isValid(scopeType)) { + String msg = MessageFormatter.format( + "Parameter {} is invalid. Invalid resource scope: {}", + INIT_PARAM_RESOURCE_SCOPE_LIST, resourceScope).getMessage(); + log.error(msg); + throw new ToggleStrategyParseException(msg); + } + + String scopeId = scopeTypeAndId[1].trim(); + if (!isValidScopeId(scopeId)) { + String msg = MessageFormatter.format( + "Parameter {} is invalid. Invalid resource scope: {}", + INIT_PARAM_RESOURCE_SCOPE_LIST, resourceScope).getMessage(); + log.error(msg); + throw new ToggleStrategyParseException(msg); + } + return new ResourceScope(scopeType, scopeId); + } + + private boolean isValidScopeId(String scopeId) { + if (StringUtils.isEmpty(scopeId)) { + return false; + } + + try { + Long.parseLong(scopeId); + } catch (NumberFormatException e) { + return false; + } + + return true; + } + + /** + * 判断是否命中配置的管理空间列表 + * + * @param scope 管理空间 + */ + protected boolean hitResourceScopeList(ResourceScope scope) { + ResourceScope checkScope = scope; + if (!scope.getClass().equals(ResourceScope.class)) { + // 需要转换为 ResourceScope 类型，避免后面 contains() 判断因为 class 不同出现预期之外的结果 + checkScope = new ResourceScope(scope.getType(), scope.getId()); + } + return this.resourceScopes.contains(checkScope); + } + + protected boolean checkFeatureExecuteContext(ToggleEvaluateContext context) { + return checkRequiredContextParam(context, ToggleStrategyContextParams.CTX_PARAM_RESOURCE_SCOPE); + } +} diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/strategy/AbstractToggleStrategy.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/strategy/AbstractToggleStrategy.java new file mode 100644 index 0000000000..c18521de0a --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/strategy/AbstractToggleStrategy.java @@ -0,0 +1,96 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service.toggle.strategy; + +import com.tencent.bk.job.common.util.toggle.ToggleEvaluateContext; +import com.tencent.bk.job.common.util.toggle.ToggleStrategy; +import lombok.extern.slf4j.Slf4j; +import org.slf4j.helpers.MessageFormatter; + +import java.util.HashMap; +import java.util.Map; + +/** + * 特性开关开启策略基础实现抽象类 + */ +@Slf4j +public abstract class AbstractToggleStrategy implements ToggleStrategy { + + /** + * 策略 ID + */ + protected final String id; + + /** + * 策略初始化参数 + */ + protected final Map initParams; + + /** + * 构造策略 + * + * @param strategyId 策略ID + * @param initParams 初始化参数 + */ + public AbstractToggleStrategy(String strategyId, + Map initParams) { + this.id = strategyId; + if (initParams != null) { + this.initParams = initParams; + } else { + this.initParams = new HashMap<>(); + } + } + + @Override + public String getId() { + return id; + } + + @Override + public Map getInitParams() { + return this.initParams; + } + + public void assertRequiredInitParam(String paramName) { + if (!initParams.containsKey(paramName)) { + String msg = MessageFormatter.format( + "Parameter {} is required for this ToggleStrategy", paramName).getMessage(); + log.error(msg); + throw new ToggleStrategyParseException(msg); + } + } + + + public boolean checkRequiredContextParam(ToggleEvaluateContext context, String paramName) { + boolean checkResult = true; + if (context.getParam(paramName) == null) { + log.info("Context param {} is required for evaluate", paramName); + checkResult = false; + } + return checkResult; + } + +} diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/strategy/AllMatchToggleStrategy.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/strategy/AllMatchToggleStrategy.java new file mode 100644 index 0000000000..d7a954f3b5 --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/strategy/AllMatchToggleStrategy.java @@ -0,0 +1,54 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service.toggle.strategy; + +import com.tencent.bk.job.common.util.toggle.ToggleEvaluateContext; +import com.tencent.bk.job.common.util.toggle.ToggleStrategy; + +import java.util.List; +import java.util.Map; + +public class AllMatchToggleStrategy extends AbstractCompositeToggleStrategy { + /** + * 开关开启策略ID + */ + public static final String STRATEGY_ID = "AllMatchToggleStrategy"; + + public AllMatchToggleStrategy(List strategies, + Map initParams) { + super(STRATEGY_ID, strategies, initParams); + } + + @Override + public boolean evaluate(String toggleName, ToggleEvaluateContext ctx) { + for (ToggleStrategy strategy : compositeStrategies) { + boolean isMatch = strategy.evaluate(toggleName, ctx); + if (!isMatch) { + return false; + } + } + return true; + } +} diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/strategy/AnyMatchToggleStrategy.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/strategy/AnyMatchToggleStrategy.java new file mode 100644 index 0000000000..8dbbc144bf --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/strategy/AnyMatchToggleStrategy.java @@ -0,0 +1,56 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service.toggle.strategy; + +import com.tencent.bk.job.common.util.toggle.ToggleEvaluateContext; +import com.tencent.bk.job.common.util.toggle.ToggleStrategy; + +import java.util.List; +import java.util.Map; + +public class AnyMatchToggleStrategy extends AbstractCompositeToggleStrategy { + + /** + * 开关开启策略ID + */ + public static final String STRATEGY_ID = "AnyMatchToggleStrategy"; + + public AnyMatchToggleStrategy(List strategies, + Map initParams) { + super(STRATEGY_ID, strategies, initParams); + } + + @Override + public boolean evaluate(String toggleName, ToggleEvaluateContext ctx) { + assertRequiredAtLeastOneStrategy(); + for (ToggleStrategy strategy : compositeStrategies) { + boolean isMatch = strategy.evaluate(toggleName, ctx); + if (isMatch) { + return true; + } + } + return false; + } +} diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/strategy/JobInstanceAttrToggleStrategy.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/strategy/JobInstanceAttrToggleStrategy.java new file mode 100644 index 0000000000..b630b62ca5 --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/strategy/JobInstanceAttrToggleStrategy.java @@ -0,0 +1,157 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service.toggle.strategy; + +import com.tencent.bk.job.common.util.toggle.ToggleEvaluateContext; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; + +import java.util.Arrays; +import java.util.Map; +import java.util.Set; +import java.util.StringJoiner; +import java.util.stream.Collectors; + +/** + * 按照作业实例属性灰度策略 + */ +@Slf4j +public class JobInstanceAttrToggleStrategy extends AbstractToggleStrategy { + /** + * 特性开关开启策略ID + */ + public static final String STRATEGY_ID = "JobInstanceAttrToggleStrategy"; + + /** + * 策略初始化参数 + */ + public static final String INIT_PARAM_REQUIRE_ALL_GSE_V2_AGENT_AVAILABLE = "requireAllGseV2AgentAvailable"; + public static final String INIT_PARAM_REQUIRE_ANY_GSE_V2_AGENT_AVAILABLE = "requireAnyGseV2AgentAvailable"; + public static final String INIT_PARAM_STARTUP_MODES = "startupModes"; + public static final String INIT_PARAM_OPERATORS = "operators"; + + /** + * 灰度策略上下文参数 + */ + public static final String CTX_PARAM_IS_ALL_GSE_V2_AGENT_AVAILABLE = "isAllGseV2AgentAvailable"; + public static final String CTX_PARAM_IS_ANY_GSE_V2_AGENT_AVAILABLE = "isAnyGseV2AgentAvailable"; + public static final String CTX_PARAM_STARTUP_MODE = "startupMode"; + public static final String CTX_PARAM_OPERATOR = "operator"; + + /** + * 灰度策略生效条件 - 所有 GSE v2 agent 可用 + */ + private Boolean requireAllGseV2AgentAvailable; + /** + * 灰度策略生效条件 - 任一 GSE v2 agent 可用 + */ + private Boolean requireAnyGseV2AgentAvailable; + /** + * 灰度策略生效条件 - 任务启动方式，web: job 页面启动; api: 第三方 API 调用；cron: 定时触发 + */ + private Set startupModes; + /** + * 灰度策略生效条件 - 操作人列表 + */ + private Set operators; + + + public JobInstanceAttrToggleStrategy(Map initParams) { + super(STRATEGY_ID, initParams); + String requireAllGseV2AgentAvailableValue = initParams.get(INIT_PARAM_REQUIRE_ALL_GSE_V2_AGENT_AVAILABLE); + if (StringUtils.isNotBlank(requireAllGseV2AgentAvailableValue)) { + requireAllGseV2AgentAvailable = Boolean.valueOf(requireAllGseV2AgentAvailableValue); + } + + String requireAnyGseV2AgentAvailableValue = initParams.get(INIT_PARAM_REQUIRE_ANY_GSE_V2_AGENT_AVAILABLE); + if (StringUtils.isNotBlank(requireAnyGseV2AgentAvailableValue)) { + requireAnyGseV2AgentAvailable = Boolean.valueOf(requireAnyGseV2AgentAvailableValue); + } + + String startupModesValue = initParams.get(INIT_PARAM_STARTUP_MODES); + if (StringUtils.isNotBlank(startupModesValue)) { + startupModes = Arrays.stream(startupModesValue.split(",")).collect(Collectors.toSet()); + } + + String operatorValue = initParams.get(INIT_PARAM_OPERATORS); + if (StringUtils.isNotBlank(operatorValue)) { + operators = Arrays.stream(operatorValue.split(",")).collect(Collectors.toSet()); + } + } + + @SuppressWarnings("DuplicatedCode") + @Override + public boolean evaluate(String toggleName, ToggleEvaluateContext ctx) { + if (requireAllGseV2AgentAvailable != null && requireAllGseV2AgentAvailable) { + boolean checkResult = checkRequiredContextParam(ctx, CTX_PARAM_IS_ALL_GSE_V2_AGENT_AVAILABLE); + if (!checkResult) { + return false; + } + boolean isAllAgentV2Available = (boolean) ctx.getParam(CTX_PARAM_IS_ALL_GSE_V2_AGENT_AVAILABLE); + if (!isAllAgentV2Available) { + return false; + } + } + if (requireAnyGseV2AgentAvailable != null && requireAnyGseV2AgentAvailable) { + boolean checkResult = checkRequiredContextParam(ctx, CTX_PARAM_IS_ANY_GSE_V2_AGENT_AVAILABLE); + if (!checkResult) { + return false; + } + boolean isAnyAgentV2Available = (boolean) ctx.getParam(CTX_PARAM_IS_ANY_GSE_V2_AGENT_AVAILABLE); + if (!isAnyAgentV2Available) { + return false; + } + } + if (CollectionUtils.isNotEmpty(startupModes)) { + boolean checkResult = checkRequiredContextParam(ctx, CTX_PARAM_STARTUP_MODE); + if (!checkResult) { + return false; + } + String startupMode = (String) ctx.getParam(CTX_PARAM_STARTUP_MODE); + if (!startupModes.contains(startupMode)) { + return false; + } + } + if (CollectionUtils.isNotEmpty(operators)) { + boolean checkResult = checkRequiredContextParam(ctx, CTX_PARAM_OPERATOR); + if (!checkResult) { + return false; + } + String operator = (String) ctx.getParam(CTX_PARAM_OPERATOR); + return operators.contains(operator); + } + return true; + } + + @Override + public String toString() { + return new StringJoiner(", ", JobInstanceAttrToggleStrategy.class.getSimpleName() + + "[", "]") + .add("id='" + id + "'") + .add("initParams=" + initParams) + .toString(); + } +} diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/strategy/ResourceScopeBlackListToggleStrategy.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/strategy/ResourceScopeBlackListToggleStrategy.java new file mode 100644 index 0000000000..611fe24ec6 --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/strategy/ResourceScopeBlackListToggleStrategy.java @@ -0,0 +1,65 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service.toggle.strategy; + +import com.tencent.bk.job.common.model.dto.ResourceScope; +import com.tencent.bk.job.common.util.toggle.ToggleEvaluateContext; +import com.tencent.bk.job.common.util.toggle.ToggleStrategyContextParams; + +import java.util.Map; +import java.util.StringJoiner; + +/** + * 根据资源范围黑名单灰度策略 + */ +public class ResourceScopeBlackListToggleStrategy extends AbstractResourceScopeToggleStrategy { + /** + * 特性开关开启策略ID + */ + public static final String STRATEGY_ID = "ResourceScopeBlackListToggleStrategy"; + + public ResourceScopeBlackListToggleStrategy(Map initParams) { + super(STRATEGY_ID, initParams); + } + + @Override + public boolean evaluate(String toggleName, ToggleEvaluateContext ctx) { + boolean isValidContext = checkFeatureExecuteContext(ctx); + if (!isValidContext) { + return false; + } + ResourceScope scope = (ResourceScope) ctx.getParam(ToggleStrategyContextParams.CTX_PARAM_RESOURCE_SCOPE); + return !hitResourceScopeList(scope); + } + + @Override + public String toString() { + return new StringJoiner(", ", ResourceScopeBlackListToggleStrategy.class.getSimpleName() + "[", "]") + .add("id='" + id + "'") + .add("initParams=" + initParams) + .add("resourceScopes=" + resourceScopes) + .toString(); + } +} diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/strategy/ResourceScopeWhiteListToggleStrategy.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/strategy/ResourceScopeWhiteListToggleStrategy.java new file mode 100644 index 0000000000..15a3291803 --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/strategy/ResourceScopeWhiteListToggleStrategy.java @@ -0,0 +1,65 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service.toggle.strategy; + +import com.tencent.bk.job.common.model.dto.ResourceScope; +import com.tencent.bk.job.common.util.toggle.ToggleEvaluateContext; +import com.tencent.bk.job.common.util.toggle.ToggleStrategyContextParams; + +import java.util.Map; +import java.util.StringJoiner; + +/** + * 根据资源范围白名单灰度策略 + */ +public class ResourceScopeWhiteListToggleStrategy extends AbstractResourceScopeToggleStrategy { + /** + * 特性开关开启策略ID + */ + public static final String STRATEGY_ID = "ResourceScopeWhiteListToggleStrategy"; + + public ResourceScopeWhiteListToggleStrategy(Map initParams) { + super(STRATEGY_ID, initParams); + } + + @Override + public boolean evaluate(String toggleName, ToggleEvaluateContext ctx) { + boolean isValidContext = checkFeatureExecuteContext(ctx); + if (!isValidContext) { + return false; + } + ResourceScope scope = (ResourceScope) ctx.getParam(ToggleStrategyContextParams.CTX_PARAM_RESOURCE_SCOPE); + return hitResourceScopeList(scope); + } + + @Override + public String toString() { + return new StringJoiner(", ", ResourceScopeWhiteListToggleStrategy.class.getSimpleName() + "[", "]") + .add("id='" + id + "'") + .add("initParams=" + initParams) + .add("resourceScopes=" + resourceScopes) + .toString(); + } +} diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/strategy/ToggleStrategyParseException.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/strategy/ToggleStrategyParseException.java new file mode 100644 index 0000000000..7531f35f4c --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/strategy/ToggleStrategyParseException.java @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service.toggle.strategy; + +/** + * 开关策略配置解析异常 + */ +public class ToggleStrategyParseException extends RuntimeException { + public ToggleStrategyParseException() { + } + + public ToggleStrategyParseException(String message) { + super(message); + } + + public ToggleStrategyParseException(String message, Throwable cause) { + super(message, cause); + } + + public ToggleStrategyParseException(Throwable cause) { + super(cause); + } +} diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/strategy/ToggleStrategyParser.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/strategy/ToggleStrategyParser.java new file mode 100644 index 0000000000..6b2e3d447e --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/strategy/ToggleStrategyParser.java @@ -0,0 +1,74 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service.toggle.strategy; + +import com.tencent.bk.job.common.service.toggle.strategy.config.ToggleStrategyConfig; +import com.tencent.bk.job.common.util.toggle.ToggleStrategy; +import lombok.extern.slf4j.Slf4j; + +import java.util.stream.Collectors; + +@Slf4j +public class ToggleStrategyParser { + + public static ToggleStrategy parseToggleStrategy(ToggleStrategyConfig strategyConfig) { + String strategyId = strategyConfig.getId(); + ToggleStrategy toggleStrategy; + switch (strategyId) { + case ResourceScopeWhiteListToggleStrategy.STRATEGY_ID: + toggleStrategy = new ResourceScopeWhiteListToggleStrategy(strategyConfig.getParams()); + break; + case ResourceScopeBlackListToggleStrategy.STRATEGY_ID: + toggleStrategy = new ResourceScopeBlackListToggleStrategy(strategyConfig.getParams()); + break; + case WeightToggleStrategy.STRATEGY_ID: + toggleStrategy = new WeightToggleStrategy(strategyConfig.getParams()); + break; + case JobInstanceAttrToggleStrategy.STRATEGY_ID: + toggleStrategy = new JobInstanceAttrToggleStrategy(strategyConfig.getParams()); + break; + case AllMatchToggleStrategy.STRATEGY_ID: + toggleStrategy = new AllMatchToggleStrategy( + strategyConfig.getStrategies() + .stream() + .map(ToggleStrategyParser::parseToggleStrategy) + .collect(Collectors.toList()), + strategyConfig.getParams()); + break; + case AnyMatchToggleStrategy.STRATEGY_ID: + toggleStrategy = new AnyMatchToggleStrategy( + strategyConfig.getStrategies() + .stream() + .map(ToggleStrategyParser::parseToggleStrategy) + .collect(Collectors.toList()), + strategyConfig.getParams()); + break; + default: + log.error("Unsupported toggle strategy: {}", strategyId); + throw new ToggleStrategyParseException("Unsupported toggle strategy " + strategyId); + } + return toggleStrategy; + } +} diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/strategy/WeightToggleStrategy.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/strategy/WeightToggleStrategy.java new file mode 100644 index 0000000000..56d6cf52d9 --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/strategy/WeightToggleStrategy.java @@ -0,0 +1,100 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service.toggle.strategy; + +import com.tencent.bk.job.common.util.toggle.ToggleEvaluateContext; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +import java.security.SecureRandom; +import java.util.Map; +import java.util.Random; +import java.util.StringJoiner; + +/** + * 根据权重灰度策略 + */ +@Slf4j +public class WeightToggleStrategy extends AbstractToggleStrategy { + /** + * 策略参数-权重 + */ + public static final String INIT_PARAM_WEIGHT = "weight"; + /** + * 特性开关开启策略ID + */ + public static final String STRATEGY_ID = "WeightToggleStrategy"; + + private final int weight; + + private final Random RANDOM = new SecureRandom(); + + public WeightToggleStrategy(Map initParams) { + super(STRATEGY_ID, initParams); + assertRequiredInitParam(INIT_PARAM_WEIGHT); + String weightStrValue = initParams.get(INIT_PARAM_WEIGHT); + this.weight = computeWeight(weightStrValue); + } + + private int computeWeight(String weightStrValue) { + String weightValue = weightStrValue.trim(); + if (StringUtils.isBlank(weightStrValue)) { + log.error("Weight is empty!"); + throw new ToggleStrategyParseException("Weight is empty!"); + } + try { + int weight = Integer.parseInt(weightValue); + if (weight < 0 || weight > 100) { + log.error("Weight should be set between 0 and 100, value: {}", weight); + throw new ToggleStrategyParseException("Weight should be set between 0 and 100"); + } + return weight; + } catch (NumberFormatException e) { + log.error("Invalid weight value: {}, not a valid number", weightValue); + throw new ToggleStrategyParseException("Weight should be a number"); + } + } + + @Override + public boolean evaluate(String toggleName, ToggleEvaluateContext ctx) { + if (weight == 0) { + return false; + } else if (weight == 100) { + return true; + } else { + int random = RANDOM.nextInt(100) + 1; + return random <= weight; + } + } + + @Override + public String toString() { + return new StringJoiner(", ", WeightToggleStrategy.class.getSimpleName() + "[", "]") + .add("id='" + id + "'") + .add("initParams=" + initParams) + .add("weight=" + weight) + .toString(); + } +} diff --git a/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/strategy/config/ToggleStrategyConfig.java b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/strategy/config/ToggleStrategyConfig.java new file mode 100644 index 0000000000..e78b4f68fa --- /dev/null +++ b/src/backend/commons/common-service/src/main/java/com/tencent/bk/job/common/service/toggle/strategy/config/ToggleStrategyConfig.java @@ -0,0 +1,50 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service.toggle.strategy.config; + +import lombok.Data; + +import java.util.List; +import java.util.Map; + +/** + * 特性启用策略配置 + */ +@Data +public class ToggleStrategyConfig { + /** + * 策略ID + */ + private String id; + /** + * 组合策略配置 + */ + private List strategies; + + /** + * 策略初始化参数 + */ + private Map params; +} diff --git a/src/backend/commons/common-service/src/main/resources/META-INF/spring.factories b/src/backend/commons/common-service/src/main/resources/META-INF/spring.factories index e69de29bb2..62a2cd5aef 100644 --- a/src/backend/commons/common-service/src/main/resources/META-INF/spring.factories +++ b/src/backend/commons/common-service/src/main/resources/META-INF/spring.factories @@ -0,0 +1,10 @@ +# AutoConfiguration +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ +com.tencent.bk.job.common.service.lb.JobLoadBalancerClientAutoConfiguration, \ +com.tencent.bk.job.common.service.deploy.DeployAutoConfiguration, \ +com.tencent.bk.job.common.service.toggle.feature.FeatureToggleAutoConfiguration, \ +com.tencent.bk.job.common.service.CommonServiceAutoConfiguration, \ +com.tencent.bk.job.common.service.quota.config.ResourceQuotaAutoConfiguration, \ +com.tencent.bk.job.common.service.feign.config.JobFeignLoadBalancerAutoConfiguration, \ +com.tencent.bk.job.common.service.toggle.prop.PropToggleAutoConfiguration + diff --git a/src/backend/commons/common-service/src/main/resources/logback-spring.xml b/src/backend/commons/common-service/src/main/resources/logback-spring.xml deleted file mode 100644 index d7b1c4fe6d..0000000000 --- a/src/backend/commons/common-service/src/main/resources/logback-spring.xml +++ /dev/null @@ -1,207 +0,0 @@ - - - - - - - - - - - - - - - - logback - - - ${APP_LOG_FILE} - - ${APP_LOG_FILE}-%d{yyyyMMdd_HH}.log.%i - 1GB - 168 - 100GB - - - ${FILE_LOG_PATTERN} - UTF-8 - - - - 10240 - - - - ${ESB_JOB_LOG_FILE} - - ${ESB_JOB_LOG_FILE}-%d{yyyyMMdd}.log.%i - 2GB - 7 - 20GB - - - ${FILE_LOG_PATTERN} - UTF-8 - - - - ${ESB_LOG_FILE} - - ${ESB_LOG_FILE}-%d{yyyyMMdd_HH}.log.%i - 1GB - 168 - 100GB - - - ${FILE_LOG_PATTERN} - UTF-8 - - 1 - - ${ERROR_LOG_FILE} - - ${ERROR_LOG_FILE}-%d{yyyy-MM-dd}.log.%i - 1GB - 7 - 20GB - - - ${FILE_LOG_PATTERN} - UTF-8 - - - ERROR - ACCEPT - DENY - - - - ${IAM_LOG_FILE} - - ${IAM_LOG_FILE}-%d{yyyy-MM-dd}.log.%i - 1GB - 72 - 20GB - - - ${FILE_LOG_PATTERN} - UTF-8 - - - - - - ${CONSOLE_LOG_PATTERN} - utf8 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/backend/commons/common-service/src/test/java/FeatureToggleTest.java b/src/backend/commons/common-service/src/test/java/FeatureToggleTest.java new file mode 100644 index 0000000000..837aa59bcd --- /dev/null +++ b/src/backend/commons/common-service/src/test/java/FeatureToggleTest.java @@ -0,0 +1,132 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +import com.tencent.bk.job.common.constant.ResourceScopeTypeEnum; +import com.tencent.bk.job.common.model.dto.ResourceScope; +import com.tencent.bk.job.common.service.toggle.feature.DefaultFeatureManager; +import com.tencent.bk.job.common.service.toggle.feature.InMemoryFeatureStore; +import com.tencent.bk.job.common.service.toggle.feature.config.FeatureToggleProperties; +import com.tencent.bk.job.common.service.toggle.strategy.JobInstanceAttrToggleStrategy; +import com.tencent.bk.job.common.util.ApplicationContextRegister; +import com.tencent.bk.job.common.util.toggle.ToggleEvaluateContext; +import com.tencent.bk.job.common.util.toggle.feature.FeatureIdConstants; +import com.tencent.bk.job.common.util.toggle.feature.FeatureManager; +import com.tencent.bk.job.common.util.toggle.feature.FeatureStore; +import com.tencent.bk.job.common.util.toggle.feature.FeatureToggle; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.MeterRegistry; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import org.yaml.snakeyaml.LoaderOptions; +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.constructor.Constructor; + +import java.io.InputStream; + +import static com.tencent.bk.job.common.util.toggle.ToggleStrategyContextParams.CTX_PARAM_RESOURCE_SCOPE; +import static org.assertj.core.api.Assertions.assertThat; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class FeatureToggleTest { + + @BeforeAll + static void beforeAll() { + Constructor constructor = new Constructor(FeatureToggleProperties.class, new LoaderOptions()); + Yaml yaml = new Yaml(constructor); + InputStream inputStream = FeatureToggleTest.class.getClassLoader() + .getResourceAsStream("features_1.yaml"); + FeatureToggleProperties featureToggleProperties = yaml.load(inputStream); + + MockedStatic mockedStatic = Mockito.mockStatic(ApplicationContextRegister.class); + mockedStatic.when(() -> ApplicationContextRegister.getBean(FeatureToggleProperties.class)) + .thenReturn(featureToggleProperties); + FeatureStore mockFeatureStore = new InMemoryFeatureStore(featureToggleProperties); + mockFeatureStore.init(); + mockedStatic.close(); + + MeterRegistry mockMeterRegistry = mock(MeterRegistry.class); + Counter mockCounter = mock(Counter.class); + when(mockMeterRegistry.counter(Mockito.anyString(), Mockito.anyIterable())).thenReturn(mockCounter); + FeatureManager featureManager = new DefaultFeatureManager(mockFeatureStore, mockMeterRegistry); + + Mockito.mockStatic(ApplicationContextRegister.class) + .when(() -> ApplicationContextRegister.getBean(FeatureManager.class)) + .thenReturn(featureManager); + } + + @Test + void checkFeature() { + ToggleEvaluateContext ctx = + ToggleEvaluateContext.builder() + .addContextParam(CTX_PARAM_RESOURCE_SCOPE, new ResourceScope(ResourceScopeTypeEnum.BIZ, "1000")) + .addContextParam(JobInstanceAttrToggleStrategy.CTX_PARAM_IS_ALL_GSE_V2_AGENT_AVAILABLE, true) + .addContextParam(JobInstanceAttrToggleStrategy.CTX_PARAM_IS_ANY_GSE_V2_AGENT_AVAILABLE, true) + .addContextParam(JobInstanceAttrToggleStrategy.CTX_PARAM_STARTUP_MODE, "web") + .addContextParam(JobInstanceAttrToggleStrategy.CTX_PARAM_OPERATOR, "admin"); + assertThat(FeatureToggle.checkFeature(FeatureIdConstants.FEATURE_GSE_V2, ctx)).isTrue(); + + ctx = ToggleEvaluateContext.builder() + .addContextParam(CTX_PARAM_RESOURCE_SCOPE, new ResourceScope(ResourceScopeTypeEnum.BIZ, "2")) + .addContextParam(JobInstanceAttrToggleStrategy.CTX_PARAM_IS_ALL_GSE_V2_AGENT_AVAILABLE, true) + .addContextParam(JobInstanceAttrToggleStrategy.CTX_PARAM_IS_ANY_GSE_V2_AGENT_AVAILABLE, true) + .addContextParam(JobInstanceAttrToggleStrategy.CTX_PARAM_STARTUP_MODE, "web") + .addContextParam(JobInstanceAttrToggleStrategy.CTX_PARAM_OPERATOR, "admin"); + assertThat(FeatureToggle.checkFeature(FeatureIdConstants.FEATURE_GSE_V2, ctx)).isFalse(); + + ctx = ToggleEvaluateContext.builder() + .addContextParam(CTX_PARAM_RESOURCE_SCOPE, new ResourceScope(ResourceScopeTypeEnum.BIZ, "100")) + .addContextParam(JobInstanceAttrToggleStrategy.CTX_PARAM_IS_ALL_GSE_V2_AGENT_AVAILABLE, false) + .addContextParam(JobInstanceAttrToggleStrategy.CTX_PARAM_IS_ANY_GSE_V2_AGENT_AVAILABLE, true) + .addContextParam(JobInstanceAttrToggleStrategy.CTX_PARAM_STARTUP_MODE, "web") + .addContextParam(JobInstanceAttrToggleStrategy.CTX_PARAM_OPERATOR, "admin"); + assertThat(FeatureToggle.checkFeature(FeatureIdConstants.FEATURE_GSE_V2, ctx)).isFalse(); + + ctx = ToggleEvaluateContext.builder() + .addContextParam(CTX_PARAM_RESOURCE_SCOPE, new ResourceScope(ResourceScopeTypeEnum.BIZ, "100")) + .addContextParam(JobInstanceAttrToggleStrategy.CTX_PARAM_IS_ALL_GSE_V2_AGENT_AVAILABLE, true) + .addContextParam(JobInstanceAttrToggleStrategy.CTX_PARAM_IS_ANY_GSE_V2_AGENT_AVAILABLE, true) + .addContextParam(JobInstanceAttrToggleStrategy.CTX_PARAM_STARTUP_MODE, "web") + .addContextParam(JobInstanceAttrToggleStrategy.CTX_PARAM_OPERATOR, "admin"); + assertThat(FeatureToggle.checkFeature(FeatureIdConstants.FEATURE_GSE_V2, ctx)).isTrue(); + + ctx = ToggleEvaluateContext.builder() + .addContextParam(CTX_PARAM_RESOURCE_SCOPE, new ResourceScope(ResourceScopeTypeEnum.BIZ, "200")) + .addContextParam(JobInstanceAttrToggleStrategy.CTX_PARAM_IS_ALL_GSE_V2_AGENT_AVAILABLE, false) + .addContextParam(JobInstanceAttrToggleStrategy.CTX_PARAM_IS_ANY_GSE_V2_AGENT_AVAILABLE, true) + .addContextParam(JobInstanceAttrToggleStrategy.CTX_PARAM_STARTUP_MODE, "web") + .addContextParam(JobInstanceAttrToggleStrategy.CTX_PARAM_OPERATOR, "admin"); + assertThat(FeatureToggle.checkFeature(FeatureIdConstants.FEATURE_GSE_V2, ctx)).isTrue(); + ctx = ToggleEvaluateContext.builder() + .addContextParam(CTX_PARAM_RESOURCE_SCOPE, new ResourceScope(ResourceScopeTypeEnum.BIZ, "200")) + .addContextParam(JobInstanceAttrToggleStrategy.CTX_PARAM_IS_ALL_GSE_V2_AGENT_AVAILABLE, false) + .addContextParam(JobInstanceAttrToggleStrategy.CTX_PARAM_IS_ANY_GSE_V2_AGENT_AVAILABLE, true) + .addContextParam(JobInstanceAttrToggleStrategy.CTX_PARAM_STARTUP_MODE, "web") + .addContextParam(JobInstanceAttrToggleStrategy.CTX_PARAM_OPERATOR, "job"); + assertThat(FeatureToggle.checkFeature(FeatureIdConstants.FEATURE_GSE_V2, ctx)).isFalse(); + + } +} diff --git a/src/backend/commons/common-service/src/test/java/com/tencent/bk/job/common/service/quota/CounterResourceQuotaConfigParserTest.java b/src/backend/commons/common-service/src/test/java/com/tencent/bk/job/common/service/quota/CounterResourceQuotaConfigParserTest.java new file mode 100644 index 0000000000..9943961177 --- /dev/null +++ b/src/backend/commons/common-service/src/test/java/com/tencent/bk/job/common/service/quota/CounterResourceQuotaConfigParserTest.java @@ -0,0 +1,164 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service.quota; + +import com.tencent.bk.job.common.resource.quota.ResourceQuotaLimit; +import com.tencent.bk.job.common.service.quota.config.ResourceQuotaLimitProperties; +import com.tencent.bk.job.common.service.quota.config.parser.CounterResourceQuotaConfigParser; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +class CounterResourceQuotaConfigParserTest { + + @Test + void parseByPercentageLimitValue() { + CounterResourceQuotaConfigParser parser = new CounterResourceQuotaConfigParser(); + + ResourceQuotaLimitProperties.QuotaLimitProp resourceScopeQuotaLimit = + new ResourceQuotaLimitProperties.QuotaLimitProp("1%", "biz:2=10%,biz:3=15%"); + ResourceQuotaLimitProperties.QuotaLimitProp appQuotaLimit = + new ResourceQuotaLimitProperties.QuotaLimitProp("2%", "bk-soap=20%,bk-nodeman=10%"); + ResourceQuotaLimitProperties.ResourceQuotaLimitProp resourceQuotaLimitProp = + new ResourceQuotaLimitProperties.ResourceQuotaLimitProp(true, "1000", resourceScopeQuotaLimit, + appQuotaLimit); + + ResourceQuotaLimit resourceQuota = parser.parse(resourceQuotaLimitProp); + + assertThat(resourceQuota.getCapacityExpr()).isEqualTo("1000"); + assertThat(resourceQuota.getCapacity()).isEqualTo(1000L); + + assertThat(resourceQuota.getResourceScopeQuotaLimit().getGlobalLimitExpr()).isEqualTo("1%"); + assertThat(resourceQuota.getResourceScopeQuotaLimit().getCustomLimitExpr()).isEqualTo("biz:2=10%,biz:3=15%"); + assertThat(resourceQuota.getResourceScopeQuotaLimit().getGlobalLimit()).isEqualTo(10L); + assertThat(resourceQuota.getResourceScopeQuotaLimit().getCustomLimits()).isNotEmpty(); + assertThat(resourceQuota.getResourceScopeQuotaLimit().getCustomLimits().get("biz:2")).isEqualTo(100L); + assertThat(resourceQuota.getResourceScopeQuotaLimit().getCustomLimits().get("biz:3")).isEqualTo(150L); + + assertThat(resourceQuota.getAppQuotaLimit().getGlobalLimitExpr()).isEqualTo("2%"); + assertThat(resourceQuota.getAppQuotaLimit().getCustomLimitExpr()).isEqualTo("bk-soap=20%,bk-nodeman=10%"); + assertThat(resourceQuota.getAppQuotaLimit().getGlobalLimit()).isEqualTo(20L); + assertThat(resourceQuota.getAppQuotaLimit().getCustomLimits()).isNotEmpty(); + assertThat(resourceQuota.getAppQuotaLimit().getCustomLimits().get("bk-soap")).isEqualTo(200L); + assertThat(resourceQuota.getAppQuotaLimit().getCustomLimits().get("bk-nodeman")).isEqualTo(100L); + + resourceScopeQuotaLimit = + new ResourceQuotaLimitProperties.QuotaLimitProp("10%", "biz:2=20%"); + appQuotaLimit = + new ResourceQuotaLimitProperties.QuotaLimitProp("20%", "bk-soap=40%"); + resourceQuotaLimitProp = + new ResourceQuotaLimitProperties.ResourceQuotaLimitProp(true, "16", resourceScopeQuotaLimit, + appQuotaLimit); + resourceQuota = parser.parse(resourceQuotaLimitProp); + + assertThat(resourceQuota.getCapacityExpr()).isEqualTo("16"); + assertThat(resourceQuota.getCapacity()).isEqualTo(16L); + + // 测试配额计算值向下取整 + assertThat(resourceQuota.getResourceScopeQuotaLimit().getGlobalLimitExpr()).isEqualTo("10%"); + assertThat(resourceQuota.getResourceScopeQuotaLimit().getGlobalLimit()).isEqualTo(1L); + assertThat(resourceQuota.getResourceScopeQuotaLimit().getCustomLimits().get("biz:2")).isEqualTo(3L); + assertThat(resourceQuota.getAppQuotaLimit().getGlobalLimitExpr()).isEqualTo("20%"); + assertThat(resourceQuota.getAppQuotaLimit().getGlobalLimit()).isEqualTo(3L); + assertThat(resourceQuota.getAppQuotaLimit().getCustomLimits().get("bk-soap")).isEqualTo(6L); + } + + @Test + void parseByExplicitLimitValue() { + CounterResourceQuotaConfigParser parser = new CounterResourceQuotaConfigParser(); + + ResourceQuotaLimitProperties.QuotaLimitProp resourceScopeQuotaLimit = + new ResourceQuotaLimitProperties.QuotaLimitProp("10", "biz:2=100,biz:3=150"); + ResourceQuotaLimitProperties.QuotaLimitProp appQuotaLimit = + new ResourceQuotaLimitProperties.QuotaLimitProp("20", "bk-soap=100,bk-nodeman=150"); + ResourceQuotaLimitProperties.ResourceQuotaLimitProp resourceQuotaLimitProp = + new ResourceQuotaLimitProperties.ResourceQuotaLimitProp(true, null, resourceScopeQuotaLimit, appQuotaLimit); + + ResourceQuotaLimit resourceQuota = parser.parse(resourceQuotaLimitProp); + + assertThat(resourceQuota.getCapacityExpr()).isNull(); + assertThat(resourceQuota.getCapacity()).isNull(); + + assertThat(resourceQuota.getResourceScopeQuotaLimit().getGlobalLimitExpr()).isEqualTo("10"); + assertThat(resourceQuota.getResourceScopeQuotaLimit().getCustomLimitExpr()).isEqualTo("biz:2=100,biz:3=150"); + assertThat(resourceQuota.getResourceScopeQuotaLimit().getGlobalLimit()).isEqualTo(10L); + assertThat(resourceQuota.getResourceScopeQuotaLimit().getCustomLimits()).isNotEmpty(); + assertThat(resourceQuota.getResourceScopeQuotaLimit().getCustomLimits().get("biz:2")).isEqualTo(100L); + assertThat(resourceQuota.getResourceScopeQuotaLimit().getCustomLimits().get("biz:3")).isEqualTo(150L); + + assertThat(resourceQuota.getAppQuotaLimit().getGlobalLimitExpr()).isEqualTo("20"); + assertThat(resourceQuota.getAppQuotaLimit().getCustomLimitExpr()).isEqualTo("bk-soap=100,bk-nodeman=150"); + assertThat(resourceQuota.getAppQuotaLimit().getGlobalLimit()).isEqualTo(20L); + assertThat(resourceQuota.getAppQuotaLimit().getCustomLimits()).isNotEmpty(); + assertThat(resourceQuota.getAppQuotaLimit().getCustomLimits().get("bk-soap")).isEqualTo(100L); + assertThat(resourceQuota.getAppQuotaLimit().getCustomLimits().get("bk-nodeman")).isEqualTo(150L); + } + + @Test + void parseInvalidConfig() { + CounterResourceQuotaConfigParser parser = new CounterResourceQuotaConfigParser(); + + assertThatThrownBy(() -> parser.parse( + new ResourceQuotaLimitProperties.ResourceQuotaLimitProp( + true, + "", + new ResourceQuotaLimitProperties.QuotaLimitProp("2%", null), + null))) + .isInstanceOf(ResourceQuotaConfigParseException.class); + + assertThatThrownBy(() -> parser.parse( + new ResourceQuotaLimitProperties.ResourceQuotaLimitProp( + true, + "", + null, + new ResourceQuotaLimitProperties.QuotaLimitProp("2%", null)))) + .isInstanceOf(ResourceQuotaConfigParseException.class); + + assertThatThrownBy(() -> parser.parse( + new ResourceQuotaLimitProperties.ResourceQuotaLimitProp( + true, + "1000", + new ResourceQuotaLimitProperties.QuotaLimitProp("2%", "error_custom_settings"), + null))) + .isInstanceOf(ResourceQuotaConfigParseException.class); + + assertThatThrownBy(() -> parser.parse( + new ResourceQuotaLimitProperties.ResourceQuotaLimitProp( + true, + "1000", + null, + new ResourceQuotaLimitProperties.QuotaLimitProp("2%", "error_custom_settings")))) + .isInstanceOf(ResourceQuotaConfigParseException.class); + + assertThatThrownBy(() -> parser.parse( + new ResourceQuotaLimitProperties.ResourceQuotaLimitProp( + true, + "1000G", + null, + new ResourceQuotaLimitProperties.QuotaLimitProp("2%", null)))) + .isInstanceOf(ResourceQuotaConfigParseException.class); + } +} diff --git a/src/backend/commons/common-service/src/test/resources/features_1.yaml b/src/backend/commons/common-service/src/test/resources/features_1.yaml new file mode 100644 index 0000000000..3e124ae82e --- /dev/null +++ b/src/backend/commons/common-service/src/test/resources/features_1.yaml @@ -0,0 +1,30 @@ +features: + gseV2: + enabled: true + strategy: + # 作业执行对接 GSE2.0 灰度策略。任一策略满足即可开启灰度 + id: AnyMatchToggleStrategy + strategies: + # 策略 1： 业务黑名单灰度策略，除了 Biz:2,Biz:100,biz:200， 其他业务都使用 GSE2.0 + - id: ResourceScopeBlackListToggleStrategy + params: + resourceScopeList: "biz:2,biz:100,biz:200" + # 策略 2：如果业务ID=100 并且作业任务下的主机都是 2.0 gse agent，那么使用 GSE2.0 + - id: AllMatchToggleStrategy + strategies: + - id: ResourceScopeWhiteListToggleStrategy + params: + resourceScopeList: "biz:100" + - id: JobInstanceAttrToggleStrategy + params: + requireAllGseV2AgentAvailable: true + # 策略 3：如果业务ID=200 并且作业执行人是 admin, 那么使用 GSE2.0 + - id: AllMatchToggleStrategy + strategies: + - id: ResourceScopeWhiteListToggleStrategy + params: + resourceScopeList: "biz:200" + - id: JobInstanceAttrToggleStrategy + params: + operators: admin + diff --git a/src/backend/commons/common-statistics/build.gradle b/src/backend/commons/common-statistics/build.gradle deleted file mode 100644 index 6ba6f17c4e..0000000000 --- a/src/backend/commons/common-statistics/build.gradle +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -dependencies { - implementation project(':commons:common') - implementation 'com.fasterxml.jackson.core:jackson-core' - implementation 'com.fasterxml.jackson.core:jackson-databind' - implementation 'com.fasterxml.jackson.core:jackson-annotations' -} diff --git a/src/backend/commons/common-task/build.gradle b/src/backend/commons/common-task/build.gradle new file mode 100644 index 0000000000..79a45f849d --- /dev/null +++ b/src/backend/commons/common-task/build.gradle @@ -0,0 +1,27 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +dependencies { + api project(':commons:common') +} diff --git a/src/backend/commons/common-task/src/main/java/com/tencent/bk/job/common/task/ClearJvmDiagnosticFileByLastModifyTimeTask.java b/src/backend/commons/common-task/src/main/java/com/tencent/bk/job/common/task/ClearJvmDiagnosticFileByLastModifyTimeTask.java new file mode 100644 index 0000000000..8310db3a0b --- /dev/null +++ b/src/backend/commons/common-task/src/main/java/com/tencent/bk/job/common/task/ClearJvmDiagnosticFileByLastModifyTimeTask.java @@ -0,0 +1,140 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.task; + +import com.tencent.bk.job.common.util.file.PathUtil; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.filefilter.AgeFileFilter; +import org.apache.commons.lang3.StringUtils; + +import java.io.File; +import java.util.Date; +import java.util.Iterator; + +/** + * 根据最后修改时间清理JVM诊断文件（堆Dump文件、JVM错误日志等） + */ +@Slf4j +public class ClearJvmDiagnosticFileByLastModifyTimeTask { + + private volatile boolean clearTaskRunning = false; + + /** + * 需要保留的时间：单位为小时 + */ + private final Integer keepHours; + + public ClearJvmDiagnosticFileByLastModifyTimeTask(Integer keepHours) { + this.keepHours = keepHours; + } + + /** + * 检查最后修改时间并清理过期文件 + */ + public void checkLastModifyTimeAndClear() { + if (keepHours == null || keepHours < 0) { + log.info("keepHours({}) is invalid, ignore checkLastModifyTimeAndClear", keepHours); + return; + } + // 获取Helm Chart Release名称 + String releaseName = getHelmChartReleaseName(); + if (StringUtils.isBlank(releaseName)) { + log.info("application not running in container, ignore checkLastModifyTimeAndClear"); + return; + } + // 获取JVM诊断文件的存储路径 + String jvmDiagnosticFileDirPath = getJvmDiagnosticFileDirPath(); + if (StringUtils.isBlank(jvmDiagnosticFileDirPath)) { + log.error("Cannot find valid jvmDiagnosticFileDirPath, ignore checkLastModifyTimeAndClear"); + return; + } + try { + // 上一次清理任务还未跑完则不清理 + if (clearTaskRunning) { + log.info("Last checkLastModifyTimeAndClear task still running, skip"); + return; + } + clearTaskRunning = true; + int count = doCheckLastModifyTimeAndClear(releaseName, jvmDiagnosticFileDirPath); + log.info("{} jvm diagnostic file cleared", count); + } catch (Exception e) { + log.warn("Exception when checkLastModifyTimeAndClear", e); + } finally { + clearTaskRunning = false; + } + } + + private int doCheckLastModifyTimeAndClear(String releaseName, String jvmDiagnosticFileDirPath) { + log.debug( + "doCheckLastModifyTimeAndClear: releaseName={}, jvmDiagnosticFileDirPath={}", + releaseName, + jvmDiagnosticFileDirPath + ); + Date thresholdDate = new Date(System.currentTimeMillis() - keepHours * 3600L * 1000L); + Iterator fileIterator = FileUtils.iterateFiles( + new File(jvmDiagnosticFileDirPath), + new AgeFileFilter(thresholdDate), + null + ); + int count = 0; + while (fileIterator.hasNext()) { + File file = fileIterator.next(); + if (!file.getName().startsWith(releaseName)) { + // 只清理当前Release下Pod产生的诊断文件 + continue; + } + boolean result = file.delete(); + if (result) { + count += 1; + } + log.info("Delete file:{}, result={}", file.getAbsolutePath(), result); + } + return count; + } + + /** + * 获取从环境变量注入的Helm Chart Release名称 + * + * @return Helm Chart Release名称 + */ + private String getHelmChartReleaseName() { + return System.getenv("BK_JOB_RELEASE_NAME"); + } + + /** + * 获取JVM诊断文件所在路径 + * + * @return JVM诊断文件所在路径 + */ + private String getJvmDiagnosticFileDirPath() { + String storageBaseDirPath = System.getenv("BK_JOB_STORAGE_BASE_DIR"); + if (StringUtils.isBlank(storageBaseDirPath)) { + return null; + } + String jvmDirName = "jvm"; + return PathUtil.joinFilePath(storageBaseDirPath, jvmDirName); + } +} diff --git a/src/backend/commons/common-task/src/main/java/com/tencent/bk/job/common/task/JvmDiagnosticFileClearScheduledTasks.java b/src/backend/commons/common-task/src/main/java/com/tencent/bk/job/common/task/JvmDiagnosticFileClearScheduledTasks.java new file mode 100644 index 0000000000..02407fd2cd --- /dev/null +++ b/src/backend/commons/common-task/src/main/java/com/tencent/bk/job/common/task/JvmDiagnosticFileClearScheduledTasks.java @@ -0,0 +1,68 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.task; + +import com.tencent.bk.job.common.task.config.JvmDiagnosticFileClearByLastModifyTimeProperties; +import lombok.extern.slf4j.Slf4j; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.scheduling.annotation.Scheduled; + +@Slf4j +public class JvmDiagnosticFileClearScheduledTasks { + + private static final Logger logger = LoggerFactory.getLogger(JvmDiagnosticFileClearScheduledTasks.class); + + private final ClearJvmDiagnosticFileByLastModifyTimeTask clearJvmDiagnosticFileByLastModifyTimeTask; + + public JvmDiagnosticFileClearScheduledTasks( + JvmDiagnosticFileClearByLastModifyTimeProperties jvmDiagnosticFileClearByLastModifyTimeProperties) { + if (jvmDiagnosticFileClearByLastModifyTimeProperties.isEnabled()) { + this.clearJvmDiagnosticFileByLastModifyTimeTask = new ClearJvmDiagnosticFileByLastModifyTimeTask( + jvmDiagnosticFileClearByLastModifyTimeProperties.getKeepHours() + ); + } else { + this.clearJvmDiagnosticFileByLastModifyTimeTask = null; + } + } + + /** + * 清理：每小时根据最后修改时间清理一次JVM诊断文件 + */ + @Scheduled(cron = "05 30 * * * ?") + public void clearJvmDiagnosticFileByLastModifyTime() { + if (clearJvmDiagnosticFileByLastModifyTimeTask == null) { + log.debug("clearJvmDiagnosticFileByLastModifyTime not enabled, ignore"); + return; + } + logger.info("clearJvmDiagnosticFileByLastModifyTime start"); + try { + clearJvmDiagnosticFileByLastModifyTimeTask.checkLastModifyTimeAndClear(); + } catch (Exception e) { + logger.error("clearJvmDiagnosticFileByLastModifyTime fail", e); + } + } + +} diff --git a/src/backend/commons/common-task/src/main/java/com/tencent/bk/job/common/task/config/JvmDiagnosticFileClearAutoConfiguration.java b/src/backend/commons/common-task/src/main/java/com/tencent/bk/job/common/task/config/JvmDiagnosticFileClearAutoConfiguration.java new file mode 100644 index 0000000000..71ec2c8744 --- /dev/null +++ b/src/backend/commons/common-task/src/main/java/com/tencent/bk/job/common/task/config/JvmDiagnosticFileClearAutoConfiguration.java @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.task.config; + +import com.tencent.bk.job.common.task.JvmDiagnosticFileClearScheduledTasks; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration(proxyBeanMethods = false) +@ConditionalOnProperty( + name = "jvm-diagnostic-file.clear-by-last-modify-time.enabled", + havingValue = "true", + matchIfMissing = true) +@EnableConfigurationProperties(JvmDiagnosticFileClearByLastModifyTimeProperties.class) +public class JvmDiagnosticFileClearAutoConfiguration { + + @Bean + public JvmDiagnosticFileClearScheduledTasks jvmDiagnosticFileClearScheduledTasks( + JvmDiagnosticFileClearByLastModifyTimeProperties jvmDiagnosticFileClearByLastModifyTimeProperties + ) { + return new JvmDiagnosticFileClearScheduledTasks(jvmDiagnosticFileClearByLastModifyTimeProperties); + } +} diff --git a/src/backend/commons/common-task/src/main/java/com/tencent/bk/job/common/task/config/JvmDiagnosticFileClearByLastModifyTimeProperties.java b/src/backend/commons/common-task/src/main/java/com/tencent/bk/job/common/task/config/JvmDiagnosticFileClearByLastModifyTimeProperties.java new file mode 100644 index 0000000000..ce9f214296 --- /dev/null +++ b/src/backend/commons/common-task/src/main/java/com/tencent/bk/job/common/task/config/JvmDiagnosticFileClearByLastModifyTimeProperties.java @@ -0,0 +1,48 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.task.config; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; + +/** + * JVM诊断文件留存相关配置 + */ +@ConfigurationProperties(prefix = "jvm-diagnostic-file.clear-by-last-modify-time") +@Getter +@Setter +public class JvmDiagnosticFileClearByLastModifyTimeProperties { + + /** + * 是否开启自动清理任务，默认开启 + */ + private boolean enabled = true; + + /** + * JVM诊断文件保留的小时数，默认168小时（7天） + */ + private Integer keepHours = 168; +} diff --git a/src/backend/commons/common-task/src/main/resources/META-INF/spring.factories b/src/backend/commons/common-task/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..cca8e93ccc --- /dev/null +++ b/src/backend/commons/common-task/src/main/resources/META-INF/spring.factories @@ -0,0 +1,2 @@ +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ +com.tencent.bk.job.common.task.config.JvmDiagnosticFileClearAutoConfiguration diff --git a/src/backend/commons/common-utils/build.gradle b/src/backend/commons/common-utils/build.gradle index 9b60dd6cd3..c269b6b36c 100644 --- a/src/backend/commons/common-utils/build.gradle +++ b/src/backend/commons/common-utils/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -23,15 +23,19 @@ */ dependencies { - implementation 'com.fasterxml.jackson.core:jackson-core' - implementation 'com.fasterxml.jackson.core:jackson-databind' - implementation 'com.fasterxml.jackson.core:jackson-annotations' - implementation 'net.sf.dozer:dozer' - implementation 'org.apache.commons:commons-collections4' - implementation 'commons-codec:commons-codec' - implementation 'com.google.guava:guava' - implementation 'io.jsonwebtoken:jjwt' - implementation 'org.apache.commons:commons-lang3' + api 'commons-io:commons-io' + api 'com.fasterxml.jackson.core:jackson-core' + api 'com.fasterxml.jackson.core:jackson-databind' + api 'com.fasterxml.jackson.core:jackson-annotations' + api 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310' + api 'com.fasterxml.jackson.datatype:jackson-datatype-joda' + api 'net.sf.dozer:dozer' + api 'org.apache.commons:commons-collections4' + api 'commons-codec:commons-codec' + api 'com.google.guava:guava' + api 'com.github.ben-manes.caffeine:caffeine' + api 'org.apache.commons:commons-lang3' + api 'org.reflections:reflections' compileOnly 'org.projectlombok:lombok' annotationProcessor 'org.projectlombok:lombok' testImplementation 'org.junit.jupiter:junit-jupiter' diff --git a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/ArrayUtil.java b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/ArrayUtil.java index 98ca5561a3..99245335e5 100644 --- a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/ArrayUtil.java +++ b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/ArrayUtil.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/Base64Util.java b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/Base64Util.java index 1968d4c5c8..4357c1eca5 100644 --- a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/Base64Util.java +++ b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/Base64Util.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -103,4 +103,28 @@ public static byte[] decodeContentToByte(String content) { throw e; } } + + /** + * 根据BASE64编码后的字符串计算原始字节流长度，不用decode字符串 + * + * @param encodedContent 编码后的字符串 + * @return 原始字符串的长度 + */ + public static int calcOriginBytesLength(String encodedContent) { + if (StringUtils.isEmpty(encodedContent)) { + return 0; + } + + // 最多只会填充两个 = + int fillCount = 0; + for (int i = 0; i <= 1 && encodedContent.length() - 1 - i >= 0; i++) { + if (encodedContent.charAt(encodedContent.length() - 1 - i) == '=') { + fillCount++; + } else { + break; + } + } + + return encodedContent.length() * 3 / 4 - fillCount; + } } diff --git a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/BatchUtil.java b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/BatchUtil.java index 71ced5fc1d..27d5ffc747 100644 --- a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/BatchUtil.java +++ b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/BatchUtil.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,45 +24,36 @@ package com.tencent.bk.job.common.util; -import java.util.ArrayList; +import org.apache.commons.collections4.CollectionUtils; + +import java.util.Collection; import java.util.List; +import java.util.function.Consumer; /** - * 分批工具 + * 分批处理工具类 */ public class BatchUtil { + /** - * 分批 + * 分批执行工具 * - * @param elements 用于分批的集合 - * @param batchSize 每一批的最大数量 - * @return 分批结果 + * @param targets 执行目标列表 + * @param batchSize 每批次大小 + * @param execution 执行函数 + * @param 执行目标 */ - public static List> buildBatchList(List elements, int batchSize) { - List> batchList = new ArrayList<>(); - int total = elements.size(); - if (total <= batchSize) { - batchList.add(elements); - return batchList; - } - - int batch = total / batchSize; - int left = total % batchSize; - if (left > 0) { - batch += 1; + public static void executeBatch(Collection targets, + int batchSize, + Consumer> execution) { + if (CollectionUtils.isEmpty(targets)) { + return; } - int startIndex = 0; - for (int i = 1; i <= batch; i++) { - List subList; - if (i == batch && left > 0) { - subList = elements.subList(startIndex, startIndex + left); - startIndex += left; - } else { - subList = elements.subList(startIndex, startIndex + batchSize); - startIndex += batchSize; - } - batchList.add(subList); + if (targets.size() <= batchSize) { + execution.accept(targets); + } else { + List> targetBatches = CollectionUtil.partitionCollection(targets, batchSize); + targetBatches.forEach(execution); } - return batchList; } } diff --git a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/ByteHexUtils.java b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/ByteHexUtils.java index 54c1098562..fdcb77d92f 100644 --- a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/ByteHexUtils.java +++ b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/ByteHexUtils.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/CollectionUtil.java b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/CollectionUtil.java new file mode 100644 index 0000000000..c68cac870a --- /dev/null +++ b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/CollectionUtil.java @@ -0,0 +1,141 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.util; + +import org.apache.commons.collections4.CollectionUtils; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +/** + * 常用的集合工具 + */ +public class CollectionUtil { + /** + * 对 List 进行分片，每一分片的大小相同（最后一个分片可能小于) + * + * @param list 需要分片的集合 + * @param size 分片大小 + * @return 集合分片 + */ + public static List> partitionList(List list, int size) { + List> batchList = new ArrayList<>(); + int total = list.size(); + if (total <= size) { + batchList.add(list); + return batchList; + } + + int batch = total / size; + int left = total % size; + if (left > 0) { + batch += 1; + } + int startIndex = 0; + for (int i = 1; i <= batch; i++) { + List subList; + if (i == batch && left > 0) { + subList = list.subList(startIndex, startIndex + left); + startIndex += left; + } else { + subList = list.subList(startIndex, startIndex + size); + startIndex += size; + } + batchList.add(subList); + } + return batchList; + } + + /** + * 对集合进行分片，每一分片的大小相同（最后一个分片可能小于) + * + * @param collection 需要分片的集合 + * @param size 分片大小 + * @return 集合分片 + */ + public static List> partitionCollection(Collection collection, int size) { + int limit = (collection.size() + size - 1) / size; + return Stream.iterate(0, n -> n + 1) + .limit(limit) + .map(a -> collection.stream() + .skip(a * size) + .limit(size) + .collect(Collectors.toList())) + .collect(Collectors.toList()); + } + + /** + * 从集合元素中抽取两个字段构成Map + * + * @param entityCollection 实体集合 + * @param keyFunc 抽取Key的方法 + * @param valueFunc 抽取Value的方法 + * @param 实体类型 + * @param Key类型 + * @param Value类型 + * @return HashMap，null可以为key或value，entityCollection为null时返回空Map + */ + public static Map convertToMap(List entityCollection, + Function keyFunc, + Function valueFunc) { + if (CollectionUtils.isEmpty(entityCollection)) { + return Collections.emptyMap(); + } + Map map = new HashMap<>(); + for (E entity : entityCollection) { + map.put(keyFunc.apply(entity), valueFunc.apply(entity)); + } + return map; + } + + public static ArrayList mergeToArrayList(List list1, List list2) { + ArrayList mergeList; + + boolean isList1NotEmpty = CollectionUtils.isNotEmpty(list1); + boolean isList2NotEmpty = CollectionUtils.isNotEmpty(list2); + + if (isList1NotEmpty && isList2NotEmpty) { + mergeList = new ArrayList<>(list1.size() + list2.size()); + mergeList.addAll(list1); + mergeList.addAll(list2); + return mergeList; + } else if (isList1NotEmpty) { + mergeList = new ArrayList<>(list1); + return mergeList; + } else if (isList2NotEmpty) { + mergeList = new ArrayList<>(list2); + return mergeList; + } else { + return new ArrayList<>(0); + } + } +} diff --git a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/CompareUtil.java b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/CompareUtil.java index 5cff7dc599..2032781ff3 100644 --- a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/CompareUtil.java +++ b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/CompareUtil.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -98,6 +98,8 @@ private static int compareElement(String ele1, String ele2) { * -1 ： version1在version2之前 */ public static int compareVersion(String version1, String version2) { + version1 = removeLeadingLetters(version1); + version2 = removeLeadingLetters(version2); // 标准化为点分形式 version1 = version1.replace("-", ".").toLowerCase(); version2 = version2.replace("-", ".").toLowerCase(); @@ -118,8 +120,18 @@ public static int compareVersion(String version1, String version2) { int result = compareElement(arr1[i].trim(), arr2[i].trim()); if (result != 0) return result; } - if (len1 < len2) return -1; - else if (len1 > len2) return 1; + // 特殊情况处理，比如3.3.3>3.3.3-beta.1，3.3.3.1>3.3.3-beta.1 + if (len1 < len2) { + if (!isNumberStr(arr2[shortLen])) { + return 1; + } + return -1; + } else if (len1 > len2) { + if (!isNumberStr(arr1[shortLen])) { + return -1; + } + return 1; + } return 0; } @@ -148,4 +160,11 @@ public static int safeCompareNullBack(Comparable p1, Comparable p2) { } return p1.compareTo(p2); } + + /** + * 去掉版本号开头的字母，v1.0 -> 1.0 + */ + public static String removeLeadingLetters(String version) { + return version.replaceFirst("^[a-zA-Z]+", ""); + } } diff --git a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/DataSizeConverter.java b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/DataSizeConverter.java new file mode 100644 index 0000000000..a3fd1daa73 --- /dev/null +++ b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/DataSizeConverter.java @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.util; + +/** + * 计算机数据单位转换工具类 + */ +public class DataSizeConverter { + public static int convertKBToMB(int value) { + return (int) Math.ceil(value / 1024.0); + } + + public static int convertMBToKB(int value) { + return value << 10; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/FilePathUtils.java b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/FilePathUtils.java similarity index 82% rename from src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/FilePathUtils.java rename to src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/FilePathUtils.java index 8a123dae2d..bcd170e829 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/FilePathUtils.java +++ b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/FilePathUtils.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.execute.engine.util; +package com.tencent.bk.job.common.util; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.ImmutablePair; @@ -32,6 +32,21 @@ * 文件路径解析器 */ public class FilePathUtils { + + /** + * 从可能含有路径的文件名称中获取纯净的文件名 + * + * @param fileNameWithPath 可能含有路径的原始文件名 + * @return 纯净的文件名 + */ + public static String getPureFileName(String fileNameWithPath) { + if (StringUtils.isBlank(fileNameWithPath)) { + return ""; + } + Pair dirAndFileNamePair = parseDirAndFileName(fileNameWithPath); + return dirAndFileNamePair.getRight(); + } + /** * 解析全路径文件的文件名和路径 */ @@ -67,33 +82,6 @@ private static boolean isFilePathSeparator(char ch) { return ch == '/' || ch == '\\'; } - /** - * 标准化GSE返回的文件路径 - * - * @param filePath GSE返回的文件路径 - * @return 标准化文件路径 - */ - public static String standardizedGSEFilePath(String filePath) { - if (StringUtils.isEmpty(filePath)) { - return ""; - } - // GSE BUG - // GSE对于Windows的路径，不管下发参数dir中是否包含路径分隔符，都会默认加上/,比如C:\Users\/；这里需要对路径进行标准化 - Pair dirAndFileName = FilePathUtils.parseDirAndFileName(filePath); - String dir = dirAndFileName.getLeft(); - String fileName = dirAndFileName.getRight(); - int lastBackSlashIndex = dir.lastIndexOf("\\"); - if (lastBackSlashIndex == -1) { - return dir + fileName; - } - // "C:\/"这种路径，需要删除掉最后一个GSE添加的"/" - if (lastBackSlashIndex == dir.length() - 2) { - return dir.substring(0, lastBackSlashIndex + 1) + fileName; - } else { - return dir + fileName; - } - } - /** * 构造文件路径 * diff --git a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/FilePathValidateUtil.java b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/FilePathValidateUtil.java new file mode 100644 index 0000000000..10405ce6bd --- /dev/null +++ b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/FilePathValidateUtil.java @@ -0,0 +1,84 @@ +package com.tencent.bk.job.common.util; + +import org.apache.commons.lang3.StringUtils; + +import java.util.regex.Pattern; + +/** + * 文件路径合法性校验工具类 + */ +public class FilePathValidateUtil { + // 传统DOS正则表达式 + private static final String CONVENTIONAL_DOS_PATH_REGEX = "(^[A-Za-z]:\\\\([^\\\\])(([^\\\\/:*?\"<>|])*\\\\?)*)|" + + "(^[A-Za-z]:[\\\\])"; + // Linux路径正则表达式 + private static final String LINUX_PATH_REGEX = "^/(((../)*|(./)*)|(\\.?[^.].*/{0,1}))+"; + + // 传统DOS Pattern + private static final Pattern CONVENTIONAL_DOS_PATH_PATTERN = Pattern.compile(CONVENTIONAL_DOS_PATH_REGEX); + // Linux路径Pattern + private static final Pattern LINUX_PATH_PATTERN = Pattern.compile(LINUX_PATH_REGEX); + + /** + * 验证文件系统绝对路径的合法性 + * + * @param path 绝对路径 + * @return boolean true合法，false非法 + */ + public static boolean validateFileSystemAbsolutePath(String path) { + // 临时规避，后续补齐校验逻辑；目前只判断路径非空 +// if (StringUtils.isBlank(path)) { +// return false; +// } +// if (isLinuxAbsolutePath(path)) { +// return validateLinuxFileSystemAbsolutePath(path); +// } else { +// return validateWindowsFileSystemAbsolutePath(path); +// } + return StringUtils.isNotBlank(path); + } + + /** + * 判断是否Linux绝对路径 + * + * @param path 文件路径 + * @return boolean + */ + private static boolean isLinuxAbsolutePath(String path) { + if (path.startsWith("/")) { + return true; + } + return false; + } + + /** + * 1 传统DOS路径 + * 标准的DOS路径可由以下三部分组成： + * 1)卷号或驱动器号，后跟卷分隔符(:)。 + * 2)目录名称。目录分隔符用来分隔嵌套目录层次结构中的子目录。 + * 3)文件名。目录分隔符用来分隔文件路径和文件名。 + * + * @param path + * @return boolean + */ + private static boolean validateWindowsFileSystemAbsolutePath(String path) { + // 传统DOS + if (CONVENTIONAL_DOS_PATH_PATTERN.matcher(path).matches()) { + return true; + } + return false; + } + + /** + * 文件或目录名，除了/以外，所有的字符都合法 + * + * @param path + * @return boolean + */ + private static boolean validateLinuxFileSystemAbsolutePath(String path) { + if (LINUX_PATH_PATTERN.matcher(path).matches()) { + return true; + } + return false; + } +} diff --git a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/FileUtil.java b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/FileUtil.java deleted file mode 100644 index 9d1a6fe813..0000000000 --- a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/FileUtil.java +++ /dev/null @@ -1,317 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.util; - -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.codec.digest.DigestUtils; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Stream; - -@Slf4j -public class FileUtil { - - /** - * 创建文件的父目录 - * - * @param path 文件路径 - * @return 最终文件父目录是否存在 - */ - private static boolean checkOrCreateParentDirsForFile(String path) { - File theFile = new File(path); - File parentDir = theFile.getParentFile(); - if (!parentDir.exists()) { - if (!parentDir.mkdirs()) { - log.warn( - "mkdir parent dir fail!dir:{}", - parentDir.getAbsolutePath() - ); - return false; - } - if (!parentDir.setWritable(true, false)) { - log.warn( - "set parent dir writeable fail!dir:{}", - parentDir.getAbsolutePath() - ); - return false; - } - } - return true; - } - - /** - * 将字节数组数据保存至文件 - * - * @param path 文件路径 - * @param contentBytes 字节数组 - * @return 是否保存成功 - */ - public static boolean saveBytesToFile(String path, byte[] contentBytes) { - if (!checkOrCreateParentDirsForFile(path)) { - return false; - } - boolean isSuccess = false; - if (contentBytes == null || contentBytes.length == 0) { - return false; - } - try (FileOutputStream out = new FileOutputStream(path)) { - out.write(contentBytes); - out.flush(); - File file = new File(path); - isSuccess = file.setExecutable(true, false); - } catch (IOException e) { - log.warn("Save fail", e); - } - - return isSuccess; - } - - /** - * 将base64编码字符串的内容保存至文件 - * - * @param path 文件路径 - * @param base64Str base64编码字符串内容 - * @return 是否保存成功 - */ - public static boolean saveBase64StrToFile(String path, String base64Str) { - byte[] contentBytes = Base64Util.decodeContentToByte(base64Str); - File theFile = new File(path); - if (theFile.exists() && theFile.isFile()) { - if (!theFile.delete()) { - log.warn( - "delete old file fail!dir:{}", - theFile.getAbsolutePath() - ); - return false; - } - } - if (!saveBytesToFile(path, contentBytes)) { - log.warn( - "save file failed!fileName:{}", - theFile.getAbsolutePath() - ); - return false; - } - return true; - } - - private static void tryToCreateFile(File file) { - try { - boolean flag = file.mkdirs(); - } catch (Exception e) { - //创建目录失败 - String msg = String.format("Fail to create dir:%s", file.getAbsolutePath()); - log.error(msg, e); - throw new RuntimeException(msg, e); - } - } - - /** - * 将 InputStream 流中内容写入文件 - * - * @param ins 流 - * @param targetPath 目标文件路径 - * @return 文件Md5 - * @throws InterruptedException 写入过程中被中断异常 - */ - public static String writeInsToFile(InputStream ins, String targetPath) throws InterruptedException { - File file = new File(targetPath); - File parentFile = file.getParentFile(); - if (!parentFile.exists()) { - tryToCreateFile(parentFile); - } - FileOutputStream fos = null; - FileInputStream fis = null; - try { - fos = new FileOutputStream(targetPath); - int batchSize = 20480; - byte[] content = new byte[batchSize]; - int length; - while ((length = ins.read(content)) > 0) { - fos.write(content, 0, length); - Thread.sleep(0); - } - closeFos(fos); - fis = new FileInputStream(targetPath); - return DigestUtils.md5Hex(fis); - } catch (FileNotFoundException e) { - log.error("File not found:{}", targetPath, e); - throw new RuntimeException(e); - } catch (IOException e) { - log.error("IOException occurred:{}", targetPath, e); - throw new RuntimeException(e); - } catch (InterruptedException e) { - log.info("Download interrupted, targetPath:{}", targetPath); - throw e; - } finally { - closeStreams(ins, fis); - } - } - - /** - * 将 InputStream 流中内容写入文件 - * - * @param ins 流 - * @param targetPath 目标文件路径 - * @param fileSize 文件大小 - * @param speed 用于观测写入速度 - * @param process 用于观测写入进度 - * @return 文件Md5 - * @throws InterruptedException 写入过程中被中断异常 - */ - public static String writeInsToFile(InputStream ins, String targetPath, Long fileSize, AtomicInteger speed, - AtomicInteger process) throws InterruptedException { - File file = new File(targetPath); - File parentFile = file.getParentFile(); - if (!parentFile.exists()) { - tryToCreateFile(parentFile); - } - FileOutputStream fos = null; - FileInputStream fis = null; - try { - fos = new FileOutputStream(targetPath); - int batchSize = 20480; - byte[] content = new byte[batchSize]; - long totalLength = 0; - int length = 0; - long lastSpeedWatchTime = System.currentTimeMillis(); - long lastSpeedWatchFileSize = totalLength; - long currentSpeedWatchTime; - long timeDelta = 0; - while ((length = ins.read(content)) > 0) { - fos.write(content, 0, length); - totalLength += length; - process.set((int) (totalLength / (float) fileSize * 100)); - currentSpeedWatchTime = System.currentTimeMillis(); - timeDelta = currentSpeedWatchTime - lastSpeedWatchTime; - if (timeDelta >= 1000) { - //计算速度 KB/s - long fileSizeDelta = totalLength - lastSpeedWatchFileSize; - speed.set((int) (fileSizeDelta / timeDelta)); - lastSpeedWatchTime = currentSpeedWatchTime; - lastSpeedWatchFileSize = totalLength; - log.info("progress: {}KB/{}KB, speed: {}KB/s", totalLength / 1000, fileSize / 1000, speed.get()); - } - Thread.sleep(0); - } - closeFos(fos); - log.info("targetPath:{},totalLength={},fileSize={}", targetPath, totalLength, fileSize); - currentSpeedWatchTime = System.currentTimeMillis(); - timeDelta = currentSpeedWatchTime - lastSpeedWatchTime; - long fileSizeDelta = totalLength - lastSpeedWatchFileSize; - if (timeDelta > 0) { - speed.set((int) (fileSizeDelta / timeDelta)); - log.info("progress: {}KB/{}KB, speed: {}KB/s", totalLength / 1000, fileSize / 1000, speed.get()); - } - fis = new FileInputStream(targetPath); - return DigestUtils.md5Hex(fis); - } catch (FileNotFoundException e) { - log.error("File not found:{}", targetPath, e); - throw new RuntimeException(e); - } catch (IOException e) { - log.error("IOException occurred:{}", targetPath, e); - throw new RuntimeException(e); - } catch (InterruptedException e) { - log.info("Download interrupted, targetPath:{}", targetPath); - throw e; - } finally { - closeStreams(ins, fis); - } - } - - private static void closeFos(FileOutputStream fos) { - if (fos != null) { - try { - fos.flush(); - fos.close(); - } catch (IOException e) { - log.error("Fail to close fos", e); - } - } - } - - private static void closeStreams( - InputStream ins, - FileInputStream fis - ) { - if (fis != null) { - try { - fis.close(); - } catch (IOException e) { - log.error("Fail to close fis", e); - } - } - if (ins != null) { - try { - ins.close(); - } catch (IOException e) { - log.error("Fail to close ins", e); - } - } - } - - /** - * 删除空目录 - * - * @param directory 目录文件 - */ - public static void deleteEmptyDirectory(File directory) { - if (directory == null) { - log.warn("Directory is null!"); - return; - } - - if (isEmpty(directory.toPath())) { - boolean delete = directory.delete(); - if (delete) { - log.info("Delete empty directory {}", directory.getPath()); - } else { - log.warn("Delete directory {} failed!", directory.getPath()); - } - } else { - log.debug("Directory {} not empty!", directory.getPath()); - } - } - - private static boolean isEmpty(Path path) { - if (Files.isDirectory(path)) { - try (Stream entries = Files.list(path)) { - return !entries.findFirst().isPresent(); - } catch (IOException e) { - return false; - } - } - - return false; - } -} diff --git a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/JobReflections.java b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/JobReflections.java new file mode 100644 index 0000000000..f401955280 --- /dev/null +++ b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/JobReflections.java @@ -0,0 +1,53 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.util; + +import org.reflections.Reflections; + +import java.util.Set; + +/** + * Job 项目范围内的反射信息 + */ +public class JobReflections { + private static final Reflections reflections; + + static { + reflections = new Reflections("com.tencent.bk.job"); + } + + private JobReflections() { + } + + /** + * 根据父类类型获取所有子类类型 + * + * @param type 类型 + * @return 子类型 + */ + public static Set> getSubTypesOf(final Class type) { + return reflections.getSubTypesOf(type); + } +} diff --git a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/ListUtil.java b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/ListUtil.java index d02212074a..91b96f2e23 100644 --- a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/ListUtil.java +++ b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/ListUtil.java @@ -1,6 +1,14 @@ package com.tencent.bk.job.common.util; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.tuple.Pair; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; import java.util.List; +import java.util.function.Function; +import java.util.stream.Collectors; public class ListUtil { @@ -16,4 +24,73 @@ public static boolean isAllTrue(List list) { } return true; } + + /** + * 将列表中的元素按规则分隔为两个列表 + * + * @param list 原始列表 + * @param separateFunction 分隔依据函数 + * @param 列表元素类型 + * @return 列表Pair, 左元素为函数判断为true的元素列表，右元素为函数判断为false的元素列表 + */ + public static Pair, List> separate(List list, Function separateFunction) { + List trueList = new ArrayList<>(); + List falseList = new ArrayList<>(); + for (T t : list) { + Boolean result = separateFunction.apply(t); + if (result != null && result) { + trueList.add(t); + } else { + falseList.add(t); + } + } + return Pair.of(trueList, falseList); + } + + /** + * 合并集合（不去重), 并返回List + * + * @param collection1 集合列表1 + * @param collection2 集合列表2 + * @param 集合中的元素 + * @return 合并后的List + */ + public static List union(Collection collection1, Collection collection2) { + boolean isCollection1Empty = CollectionUtils.isEmpty(collection1); + boolean isCollection2Empty = CollectionUtils.isEmpty(collection2); + if (isCollection1Empty && isCollection2Empty) { + return Collections.emptyList(); + } + + List result = new ArrayList<>((isCollection1Empty ? 0 : collection1.size()) + + (isCollection2Empty ? 0 : collection2.size())); + if (!isCollection1Empty) { + result.addAll(collection1); + } + if (!isCollection2Empty) { + result.addAll(collection2); + } + return result; + } + + /** + * 去除字符串列表中各元素首尾的空字符串，null元素不处理原样返回 + * + * @param list 字符串列表 + * @return 处理后的字符串列表 + */ + public static List trimStringList(List list) { + if (list == null) { + return null; + } + return list.stream().map( + str -> { + if (str == null) { + return null; + } else { + return str.trim(); + } + } + ).collect(Collectors.toList()); + } } diff --git a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/ObjectWrapper.java b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/ObjectWrapper.java index f95bec9160..7ec5f2b812 100644 --- a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/ObjectWrapper.java +++ b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/ObjectWrapper.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/PrefConsts.java b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/PrefConsts.java index ad7deba019..d995841731 100644 --- a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/PrefConsts.java +++ b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/PrefConsts.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/RandomUtil.java b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/RandomUtil.java index 6bd9623801..5aaa643383 100644 --- a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/RandomUtil.java +++ b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/RandomUtil.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,14 +24,20 @@ package com.tencent.bk.job.common.util; -import java.util.Random; +import java.util.concurrent.ThreadLocalRandom; +/** + * 随机工具类 + */ public class RandomUtil { - private static final Random random = new Random(); - + /** + * 生成随机的正整数，可用于多线程环境 + * + * @return 随机数 + */ public static long getRandomPositiveLong() { - long value = random.nextLong(); + long value = ThreadLocalRandom.current().nextLong(); if (value == Long.MIN_VALUE || value == 1L) { return 1; } else if (value < 0) { @@ -39,4 +45,14 @@ public static long getRandomPositiveLong() { } return value; } + + /** + * 生成[0,bound)范围内的随机正整数，可用于多线程环境 + * + * @param bound 边界值，不包含 + * @return 随机数 + */ + public static int nextInt(int bound) { + return ThreadLocalRandom.current().nextInt(bound); + } } diff --git a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/ThreadUtils.java b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/ThreadUtils.java index 94a619fe72..7c8f7fbc9b 100644 --- a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/ThreadUtils.java +++ b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/ThreadUtils.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -43,4 +43,21 @@ public static void sleep(long sleepInMills) { log.warn("Caught InterruptedException", e); } } + + + /** + * 线程sleep + * + * @param sleepInMills sleep时间 + * @param logInterrupted 是否输出线程打断的错误日志 + */ + public static void sleep(long sleepInMills, boolean logInterrupted) { + try { + Thread.sleep(sleepInMills); + } catch (InterruptedException e) { + if (logInterrupted) { + log.warn("Caught InterruptedException", e); + } + } + } } diff --git a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/crypto/AESUtils.java b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/crypto/AESUtils.java deleted file mode 100644 index d8ec577ccc..0000000000 --- a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/crypto/AESUtils.java +++ /dev/null @@ -1,230 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.util.crypto; - -import com.tencent.bk.job.common.util.Base64Util; - -import javax.crypto.Cipher; -import javax.crypto.KeyGenerator; -import javax.crypto.spec.IvParameterSpec; -import javax.crypto.spec.SecretKeySpec; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.nio.charset.StandardCharsets; -import java.security.NoSuchAlgorithmException; -import java.security.NoSuchProviderException; -import java.security.SecureRandom; - -public class AESUtils { - /** - * 加密/解密算法/工作模式/填充方式 - */ - private static final String CIPHER_ALGORITHM = "AES/CBC/PKCS5Padding"; - - /** - * 加密数据 - * - * @param data 待加密数据 - * @param password 密钥 - * @return byte[] 加密后的数据 - */ - public static byte[] encrypt(byte[] data, String password) throws Exception { - return encrypt(data, password.getBytes(StandardCharsets.UTF_8)); - } - - /** - * 加密数据 - * - * @param data 待加密数据 - * @param password 密钥 - * @return byte[] 加密后的数据 - */ - public static byte[] encrypt(String data, String password) throws Exception { - return encrypt(data.getBytes(StandardCharsets.UTF_8), password.getBytes(StandardCharsets.UTF_8)); - } - - /** - * 加密数据 - * - * @param data 待加密数据 - * @param password 密钥 - * @return String 加密后的数据 - */ - public static String encryptToBase64EncodedCipherText(byte[] data, String password) throws Exception { - return Base64Util.encodeContentToStr(encrypt(data, password.getBytes(StandardCharsets.UTF_8))); - } - - /** - * 解密base64编码的数据，并转换成字符明文 - * - * @param data 待解密数据，base64编码 - * @param password 密钥 - * @return String 解密后的数据 - */ - public static String decryptBase64EncodedDataToPlainText(String data, String password) throws Exception { - return decryptToPlainText(Base64Util.decodeContentToByte(data), password); - } - - /** - * 加密数据，并是使用base64编码 - * - * @param data 待加密数据 - * @param password 密钥 - * @return byte[] 加密后的数据 - */ - public static String encryptToBase64EncodedCipherText(String data, String password) throws Exception { - return Base64Util.encodeContentToStr(encrypt(data, password)); - } - - /** - * 解密数据 - * - * @param data 待解密数据 - * @param password 密钥 - * @return byte[] 解密后的数据 - */ - public static byte[] decrypt(byte[] data, String password) throws Exception { - return decrypt(data, password.getBytes(StandardCharsets.UTF_8)); - } - - /** - * 解密数据 - * - * @param data 待解密数据 - * @param password 密钥 - * @return byte[] 解密后的数据 - */ - public static String decryptToPlainText(byte[] data, String password) throws Exception { - return new String(decrypt(data, password.getBytes(StandardCharsets.UTF_8)), StandardCharsets.UTF_8); - } - - /** - * 解密数据 - * - * @param data 待解密数据 - * @param password 密钥 - * @return byte[] 解密后的数据 - */ - public static String decryptToPlainText(String data, String password) throws Exception { - return new String(decrypt(data.getBytes(StandardCharsets.UTF_8), - password.getBytes(StandardCharsets.UTF_8)), StandardCharsets.UTF_8); - } - - /** - * 加密数据 - * - * @param data 待加密数据 - * @param key 密钥 - * @return byte[] 加密后的数据 - */ - public static byte[] encrypt(byte[] data, byte[] key) throws Exception { - Cipher cipher = Cipher.getInstance(CIPHER_ALGORITHM); - cipher.init(Cipher.ENCRYPT_MODE, getKeySpec(cipher, key)); - - byte[] iv = cipher.getIV(); - byte[] finalData = cipher.doFinal(data); - if (iv == null) { - throw new RuntimeException(String.format("CIPHER_ALGORITHM %s is invalid", CIPHER_ALGORITHM)); - } - byte[] finalDataWithIv = new byte[finalData.length + iv.length]; - System.arraycopy(iv, 0, finalDataWithIv, 0, iv.length); - System.arraycopy(finalData, 0, finalDataWithIv, iv.length, finalData.length); - return finalDataWithIv; - } - - /** - * 解密数据 - * - * @param data 待解密数据 - * @param key 密钥 - * @return byte[] 解密后的数据 - */ - public static byte[] decrypt(byte[] data, byte[] key) throws Exception { - Cipher cipher = Cipher.getInstance(CIPHER_ALGORITHM); - cipher.init(Cipher.DECRYPT_MODE, getKeySpec(cipher, key), getIvSpec(cipher, data)); - byte[] dataWithoutIv = new byte[data.length - cipher.getBlockSize()]; - System.arraycopy(data, cipher.getBlockSize(), dataWithoutIv, - 0, data.length - cipher.getBlockSize()); - return cipher.doFinal(dataWithoutIv); - } - - private static IvParameterSpec getIvSpec(Cipher cipher, byte[] data) { - byte[] iv = new byte[cipher.getBlockSize()]; - System.arraycopy(data, 0, iv, 0, iv.length); - return new IvParameterSpec(iv); - } - - private static SecretKeySpec getKeySpec(Cipher cipher, byte[] key) - throws NoSuchAlgorithmException, NoSuchProviderException { - KeyGenerator kgen = KeyGenerator.getInstance("AES"); - SecureRandom random = SecureRandom.getInstance("SHA1PRNG", "SUN"); - random.setSeed(key); - kgen.init(cipher.getBlockSize() * 8, random); - return new SecretKeySpec(kgen.generateKey().getEncoded(), "AES"); - } - - public static void encrypt(File inFile, File outFile, String password) throws Exception { - byte[] key = password.getBytes(StandardCharsets.UTF_8); - Cipher cipher = Cipher.getInstance(CIPHER_ALGORITHM); - cipher.init(Cipher.ENCRYPT_MODE, getKeySpec(cipher, key)); - try (FileInputStream in = new FileInputStream(inFile); FileOutputStream out = new FileOutputStream(outFile)) { - byte[] arr = cipher.getIV(); - if (arr == null) { - throw new RuntimeException(String.format("CIPHER_ALGORITHM %s is invalid", CIPHER_ALGORITHM)); - } - out.write(arr); - write(in, out, cipher); - } - } - - public static void decrypt(File inFile, File outFile, String password) throws Exception { - byte[] key = password.getBytes(StandardCharsets.UTF_8); - Cipher cipher = Cipher.getInstance(CIPHER_ALGORITHM); - try (FileInputStream in = new FileInputStream(inFile); FileOutputStream out = new FileOutputStream(outFile)) { - byte[] iv = new byte[cipher.getBlockSize()]; - if (in.read(iv) < iv.length) { - throw new RuntimeException(); - } - cipher.init(Cipher.DECRYPT_MODE, getKeySpec(cipher, key), new IvParameterSpec(iv)); - write(in, out, cipher); - } - } - - private static void write(FileInputStream in, FileOutputStream out, Cipher cipher) throws Exception { - byte[] iBuffer = new byte[1024]; - int len; - while ((len = in.read(iBuffer)) != -1) { - byte[] oBuffer = cipher.update(iBuffer, 0, len); - if (oBuffer != null) { - out.write(oBuffer); - } - } - byte[] oBuffer = cipher.doFinal(); - if (oBuffer != null) { - out.write(oBuffer); - } - } -} diff --git a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/crypto/RSAUtils.java b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/crypto/RSAUtils.java deleted file mode 100644 index 08d7f5909e..0000000000 --- a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/crypto/RSAUtils.java +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.util.crypto; - -import com.google.common.collect.Lists; -import com.tencent.bk.job.common.util.Base64Util; -import org.apache.commons.codec.binary.Base64; -import org.apache.commons.lang3.StringUtils; - -import javax.crypto.Cipher; -import java.io.*; -import java.nio.charset.StandardCharsets; -import java.security.*; -import java.security.interfaces.RSAPrivateKey; -import java.security.interfaces.RSAPublicKey; -import java.security.spec.PKCS8EncodedKeySpec; -import java.security.spec.X509EncodedKeySpec; -import java.util.List; - -/** - * RSAUtils 加解密 - */ -public class RSAUtils { - private static final String KEY_ALGORITHM = "RSA"; - private static final String SIGNATURE_ALGORITHM = "SHA1withRSA"; - - private static final String BEGIN_ENCRYPTED_PRIVATE_KEY = "-----BEGIN ENCRYPTED PRIVATE KEY-----"; - private static final String END_ENCRYPTED_PRIVATE_KEY = "-----END ENCRYPTED PRIVATE KEY-----"; - private static final String BEGIN_RSA_PRIVATE_KEY = "-----BEGIN RSA PRIVATE KEY-----"; - private static final String END_RSA_PRIVATE_KEY = "-----END RSA PRIVATE KEY-----"; - - private static final String BEGIN_PRIVATE_KEY = "-----BEGIN PRIVATE KEY-----"; - private static final String END_PRIVATE_KEY = "-----END PRIVATE KEY-----"; - private static final String BEGIN_PUBLIC_KEY = "-----BEGIN PUBLIC KEY-----"; - private static final String END_PUBLIC_KEY = "-----END PUBLIC KEY-----"; - - private static final String CHARSET_NAME = "UTF-8"; - private static final String LINE_SPLIT = "\n"; - - private static List SKIP_STR = Lists.newArrayList( - BEGIN_PRIVATE_KEY, END_PRIVATE_KEY, BEGIN_PUBLIC_KEY, END_PUBLIC_KEY, - BEGIN_ENCRYPTED_PRIVATE_KEY, END_ENCRYPTED_PRIVATE_KEY, BEGIN_RSA_PRIVATE_KEY, END_RSA_PRIVATE_KEY - ); - - private static String getPermKey(File permFile) throws IOException { - StringBuilder strKeyPEM = new StringBuilder(2048); - try (BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(permFile), - StandardCharsets.UTF_8))) { - String line; - while ((line = br.readLine()) != null) { - if (SKIP_STR.contains(line)) { - continue; - } - strKeyPEM.append(line).append(LINE_SPLIT); - } - } - return strKeyPEM.toString(); - } - - private static String getPermKey(String permBase64) throws IOException { - String perm = Base64Util.decodeContentToStr(permBase64); - if (StringUtils.isEmpty(perm)) { - throw new IOException("Perm content is empty"); - } - StringBuilder strKeyPEM = new StringBuilder(2048); - try (BufferedReader br = new BufferedReader(new StringReader(perm))) { - String line; - while ((line = br.readLine()) != null) { - if (SKIP_STR.contains(line)) { - continue; - } - strKeyPEM.append(line).append(LINE_SPLIT); - } - } - return strKeyPEM.toString(); - } - - - public static RSAPrivateKey getPrivateKey(File rsaPrivatePermFile) throws IOException, GeneralSecurityException { - String privateKeyPEM = getPermKey(rsaPrivatePermFile); - byte[] encoded = Base64.decodeBase64(privateKeyPEM); - return (RSAPrivateKey) KeyFactory.getInstance(KEY_ALGORITHM).generatePrivate(new PKCS8EncodedKeySpec(encoded)); - } - - public static RSAPrivateKey getPrivateKey(String rsaPrivateKeyBase64) throws IOException, GeneralSecurityException { - String privateKeyPEM = getPermKey(rsaPrivateKeyBase64); - byte[] encoded = Base64.decodeBase64(privateKeyPEM); - return (RSAPrivateKey) KeyFactory.getInstance(KEY_ALGORITHM).generatePrivate(new PKCS8EncodedKeySpec(encoded)); - } - - public static RSAPublicKey getPublicKey(File rsaPublicPermFile) throws IOException, GeneralSecurityException { - String publicKeyPEM = getPermKey(rsaPublicPermFile); - byte[] encoded = Base64.decodeBase64(publicKeyPEM); - return (RSAPublicKey) KeyFactory.getInstance(KEY_ALGORITHM).generatePublic(new X509EncodedKeySpec(encoded)); - } - - public static RSAPublicKey getPublicKey(String rsaPublicKeyBase64) throws IOException, GeneralSecurityException { - byte[] encoded = Base64.decodeBase64(getPermKey(rsaPublicKeyBase64)); - return (RSAPublicKey) KeyFactory.getInstance(KEY_ALGORITHM).generatePublic(new X509EncodedKeySpec(encoded)); - } - - public static String sign(PrivateKey privateKey, - String message) throws NoSuchAlgorithmException, InvalidKeyException, - SignatureException, UnsupportedEncodingException { - Signature sign = Signature.getInstance(SIGNATURE_ALGORITHM); - sign.initSign(privateKey); - sign.update(message.getBytes(CHARSET_NAME)); - return new String(Base64.encodeBase64(sign.sign()), CHARSET_NAME); - } - - public static boolean verify(PublicKey publicKey, String message, - String signature) throws SignatureException, NoSuchAlgorithmException, - UnsupportedEncodingException, InvalidKeyException { - Signature sign = Signature.getInstance(SIGNATURE_ALGORITHM); - sign.initVerify(publicKey); - sign.update(message.getBytes(CHARSET_NAME)); - return sign.verify(Base64.decodeBase64(signature.getBytes(CHARSET_NAME))); - } - - public static String encrypt(String rawText, PublicKey publicKey) throws IOException, GeneralSecurityException { - Cipher cipher = Cipher.getInstance(KEY_ALGORITHM); - cipher.init(Cipher.ENCRYPT_MODE, publicKey); - return Base64.encodeBase64String(cipher.doFinal(rawText.getBytes(CHARSET_NAME))); - } - - public static String decrypt(String cipherText, - PrivateKey privateKey) throws IOException, GeneralSecurityException { - Cipher cipher = Cipher.getInstance(KEY_ALGORITHM); - cipher.init(Cipher.DECRYPT_MODE, privateKey); - return new String(cipher.doFinal(Base64.decodeBase64(cipherText)), CHARSET_NAME); - } -} diff --git a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/date/DateUtils.java b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/date/DateUtils.java index 01e07df14b..26136ff8a1 100644 --- a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/date/DateUtils.java +++ b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/date/DateUtils.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,16 +26,25 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.slf4j.helpers.MessageFormatter; import java.sql.Timestamp; import java.text.DateFormat; import java.text.ParseException; import java.text.SimpleDateFormat; -import java.time.*; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZoneOffset; +import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.time.temporal.ChronoUnit; import java.util.Calendar; import java.util.Date; +import java.util.HashMap; +import java.util.Map; /** * 时间处理 @@ -208,6 +217,45 @@ public static LocalDateTime convertFromMillSeconds(long millSeconds) { ZoneOffset.systemDefault().getRules().getOffset(Instant.now())); } + /** + * 尝试使用多种格式来解析字符串中的时间，只要有任意一种格式匹配即可成功解析 + * + * @param dateTime 日期时间字符串 + * @param patterns 多个可能的日期时间格式 + * @return 本地日期时间对象 + */ + public static LocalDateTime convertFromStringDateByPatterns(String dateTime, String... patterns) { + if (patterns.length == 0) { + throw new IllegalArgumentException("patterns must not be empty"); + } + LocalDateTime localDateTime; + Map exceptionMap = new HashMap<>(); + for (String pattern : patterns) { + try { + localDateTime = convertFromStringDate(dateTime, pattern); + if (localDateTime != null) { + return localDateTime; + } + } catch (Exception e) { + exceptionMap.put(pattern, e); + } + } + String patternsInvalidMsg = MessageFormatter.format( + "Fail to convertFromStringDateByPatterns: dateTime={}, patterns={}, exceptions: ", + dateTime, + patterns + ).getMessage(); + log.warn(patternsInvalidMsg); + exceptionMap.forEach((pattern, e) -> { + String patternInvalidMsg = MessageFormatter.format( + "pattern: {}, exception: ", + pattern + ).getMessage(); + log.warn(patternInvalidMsg, e); + }); + throw new IllegalArgumentException(patternsInvalidMsg); + } + public static LocalDateTime convertFromStringDate(String dateTime, String pattern) { DateTimeFormatter formatter = DateTimeFormatter.ofPattern(pattern); return LocalDateTime.parse(dateTime, formatter); @@ -301,7 +349,15 @@ public static String getPreviousDateStr(String dateStr, String pattern, int prev Date previousDay = getPreviousDate(date, previousDays); return sdf.format(previousDay); } catch (ParseException e) { - log.error("Fail to calc date:{} - {}days, pattern={}", dateStr, previousDays, pattern, e); + String msg = MessageFormatter.arrayFormat( + "Fail to calc date:{} - {}days, pattern={}", + new String[]{ + dateStr, + String.valueOf(previousDays), + pattern + } + ).getMessage(); + log.error(msg, e); return null; } } @@ -327,8 +383,35 @@ public static Long calcDaysBetween(String startDateStr, String endDateStr, Strin Date endDate = sdf.parse(endDateStr); return (endDate.getTime() - startDate.getTime()) / (24 * 60 * 60 * 1000); } catch (ParseException e) { - log.error("Fail to calcDaysBetween:{},{},pattern={}", startDateStr, endDateStr, pattern, e); + String msg = MessageFormatter.arrayFormat( + "Fail to calcDaysBetween:{},{},pattern={}", + new String[]{ + startDateStr, + endDateStr, + pattern + } + ).getMessage(); + log.error(msg, e); return null; } } + + /** + * 计算当天（UTC 时区)的截止时间戳 + * + * @return 当天（UTC 时区)的截止时间戳，单位毫秒 + */ + public static long getUTCCurrentDayEndTimestamp() { + return getUTCDayEndTimestamp(LocalDate.now(ZoneId.of("UTC"))); + } + + /** + * 根据日期（UTC 时区)计算这一天的截止时间戳 + * + * @return 日期（UTC 时区)对应当天截止时间戳，单位毫秒 + */ + public static long getUTCDayEndTimestamp(LocalDate localDateUTC) { + + return 1000 * (localDateUTC.atStartOfDay().toEpochSecond(ZoneOffset.UTC) + 86400); + } } diff --git a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/file/FileSizeUtil.java b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/file/FileSizeUtil.java new file mode 100644 index 0000000000..da45f778d6 --- /dev/null +++ b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/file/FileSizeUtil.java @@ -0,0 +1,88 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.util.file; + +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +@Slf4j +public class FileSizeUtil { + + /** + * 根据字节数获取易于读取的带单位的文件大小描述 + * + * @param byteNum 字节数 + * @return 易于读取的带单位的文件大小描述 + */ + public static String getFileSizeStr(Long byteNum) { + if (byteNum == null) return "--"; + String[] units = new String[]{"B", "KB", "MB", "GB", "TB", "PB"}; + int i = 0; + double value = (float) byteNum; + while (value >= 1024 && i < units.length - 1) { + value = value / 1024.; + i += 1; + } + return String.format("%.2f", value) + units[i]; + } + + /** + * 从带单位的文件大小描述中解析出真实表示的字节数 + * + * @param fileSizeStr 带单位的文件大小描述 + * @return 真实表示的字节数 + */ + public static long parseFileSizeBytes(String fileSizeStr) { + if (StringUtils.isBlank(fileSizeStr)) { + throw new IllegalArgumentException("Invalid fileSize : " + fileSizeStr); + } + long factor = 1L; + fileSizeStr = fileSizeStr.trim().toUpperCase(); + if (fileSizeStr.endsWith("B")) { + fileSizeStr = fileSizeStr.replace("B", ""); + } + if (fileSizeStr.endsWith("P")) { + fileSizeStr = fileSizeStr.replace("P", ""); + factor = 1024L * 1024L * 1024L * 1024L * 1024L; + } + if (fileSizeStr.endsWith("T")) { + fileSizeStr = fileSizeStr.replace("T", ""); + factor = 1024L * 1024L * 1024L * 1024L; + } + if (fileSizeStr.endsWith("G")) { + fileSizeStr = fileSizeStr.replace("G", ""); + factor = 1024L * 1024L * 1024L; + } + if (fileSizeStr.endsWith("M")) { + fileSizeStr = fileSizeStr.replace("M", ""); + factor = 1024L * 1024L; + } + if (fileSizeStr.endsWith("K")) { + fileSizeStr = fileSizeStr.replace("K", ""); + factor = 1024L; + } + return Math.round(Double.parseDouble(fileSizeStr) * factor); + } +} diff --git a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/file/FileUtil.java b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/file/FileUtil.java new file mode 100644 index 0000000000..aae333a2bc --- /dev/null +++ b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/file/FileUtil.java @@ -0,0 +1,497 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.util.file; + +import com.tencent.bk.job.common.util.Base64Util; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.helpers.FormattingTuple; +import org.slf4j.helpers.MessageFormatter; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileNotFoundException; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +@Slf4j +public class FileUtil { + + /** + * 创建文件的父目录 + * + * @param path 文件路径 + * @return 最终文件父目录是否存在 + */ + private static boolean checkOrCreateParentDirsForFile(String path) { + File theFile = new File(path); + File parentDir = theFile.getParentFile(); + if (!parentDir.exists()) { + if (!parentDir.mkdirs()) { + log.warn( + "mkdir parent dir fail!dir:{}", + parentDir.getAbsolutePath() + ); + return false; + } + if (!parentDir.setWritable(true, false)) { + log.warn( + "set parent dir writeable fail!dir:{}", + parentDir.getAbsolutePath() + ); + return false; + } + } + return true; + } + + /** + * 将字节数组数据保存至文件 + * + * @param path 文件路径 + * @param contentBytes 字节数组 + * @return 是否保存成功 + */ + public static boolean saveBytesToFile(String path, byte[] contentBytes) { + if (!checkOrCreateParentDirsForFile(path)) { + return false; + } + boolean isSuccess = false; + if (contentBytes == null || contentBytes.length == 0) { + return false; + } + try (FileOutputStream out = new FileOutputStream(path)) { + out.write(contentBytes); + out.flush(); + File file = new File(path); + isSuccess = file.setExecutable(true, false); + } catch (IOException e) { + log.warn("Save fail", e); + } + + return isSuccess; + } + + /** + * 将base64编码字符串的内容保存至文件 + * + * @param path 文件路径 + * @param base64Str base64编码字符串内容 + * @return 是否保存成功 + */ + public static boolean saveBase64StrToFile(String path, String base64Str) { + byte[] contentBytes = Base64Util.decodeContentToByte(base64Str); + File theFile = new File(path); + if (theFile.exists() && theFile.isFile()) { + if (!theFile.delete()) { + log.warn( + "delete old file fail!dir:{}", + theFile.getAbsolutePath() + ); + return false; + } + } + if (!saveBytesToFile(path, contentBytes)) { + log.warn( + "save file failed!fileName:{}", + theFile.getAbsolutePath() + ); + return false; + } + return true; + } + + private static void tryToCreateFile(File file) { + try { + boolean flag = file.mkdirs(); + } catch (Exception e) { + //创建目录失败 + String msg = String.format("Fail to create dir:%s", file.getAbsolutePath()); + log.error(msg, e); + throw new RuntimeException(msg, e); + } + } + + /** + * 将 InputStream 流中内容写入文件 + * + * @param ins 流 + * @param targetPath 目标文件路径 + * @return 文件Md5 + * @throws InterruptedException 写入过程中被中断异常 + */ + public static String writeInsToFile(InputStream ins, String targetPath) throws InterruptedException { + File file = new File(targetPath); + File parentFile = file.getParentFile(); + if (!parentFile.exists()) { + tryToCreateFile(parentFile); + } + FileOutputStream fos = null; + FileInputStream fis = null; + try { + fos = new FileOutputStream(targetPath); + int batchSize = 20480; + byte[] content = new byte[batchSize]; + int length; + while ((length = ins.read(content)) > 0) { + fos.write(content, 0, length); + Thread.sleep(0); + } + closeFos(fos); + fis = new FileInputStream(targetPath); + return DigestUtils.md5Hex(fis); + } catch (FileNotFoundException e) { + String msg = MessageFormatter.format( + "File not found:{}", + targetPath + ).getMessage(); + log.error(msg, e); + throw new RuntimeException(e); + } catch (IOException e) { + String msg = MessageFormatter.format( + "IOException occurred:{}", + targetPath + ).getMessage(); + log.error(msg, e); + throw new RuntimeException(e); + } catch (InterruptedException e) { + log.info("Download interrupted, targetPath:{}", targetPath); + throw e; + } finally { + closeStreams(ins, fis); + } + } + + /** + * 将 InputStream 流中内容写入文件 + * + * @param ins 流 + * @param targetPath 目标文件路径 + * @param fileSize 文件大小 + * @param speed 用于观测写入速度 + * @param process 用于观测写入进度 + * @return 文件Md5 + * @throws InterruptedException 写入过程中被中断异常 + */ + public static String writeInsToFile(InputStream ins, String targetPath, Long fileSize, AtomicInteger speed, + AtomicInteger process) throws InterruptedException { + File file = new File(targetPath); + File parentFile = file.getParentFile(); + if (!parentFile.exists()) { + tryToCreateFile(parentFile); + } + FileOutputStream fos = null; + FileInputStream fis = null; + try { + fos = new FileOutputStream(targetPath); + int batchSize = 20480; + byte[] content = new byte[batchSize]; + long totalLength = 0; + int length = 0; + long lastSpeedWatchTime = System.currentTimeMillis(); + long lastSpeedWatchFileSize = totalLength; + long currentSpeedWatchTime; + long timeDelta = 0; + while ((length = ins.read(content)) > 0) { + fos.write(content, 0, length); + totalLength += length; + process.set((int) (totalLength / (float) fileSize * 100)); + currentSpeedWatchTime = System.currentTimeMillis(); + timeDelta = currentSpeedWatchTime - lastSpeedWatchTime; + if (timeDelta >= 1000) { + //计算速度 KB/s + long fileSizeDelta = totalLength - lastSpeedWatchFileSize; + speed.set((int) (fileSizeDelta / timeDelta)); + lastSpeedWatchTime = currentSpeedWatchTime; + lastSpeedWatchFileSize = totalLength; + log.info("progress: {}KB/{}KB, speed: {}KB/s", totalLength / 1000, fileSize / 1000, speed.get()); + } + Thread.sleep(0); + } + closeFos(fos); + log.info("targetPath:{},totalLength={},fileSize={}", targetPath, totalLength, fileSize); + currentSpeedWatchTime = System.currentTimeMillis(); + timeDelta = currentSpeedWatchTime - lastSpeedWatchTime; + long fileSizeDelta = totalLength - lastSpeedWatchFileSize; + if (timeDelta > 0) { + speed.set((int) (fileSizeDelta / timeDelta)); + log.info("progress: {}KB/{}KB, speed: {}KB/s", totalLength / 1000, fileSize / 1000, speed.get()); + } + fis = new FileInputStream(targetPath); + return DigestUtils.md5Hex(fis); + } catch (FileNotFoundException e) { + String msg = MessageFormatter.format( + "File not found:{}", + targetPath + ).getMessage(); + log.error(msg, e); + throw new RuntimeException(e); + } catch (IOException e) { + String msg = MessageFormatter.format( + "IOException occurred:{}", + targetPath + ).getMessage(); + log.error(msg, e); + throw new RuntimeException(e); + } catch (InterruptedException e) { + log.info("Download interrupted, targetPath:{}", targetPath); + throw e; + } finally { + closeStreams(ins, fis); + } + } + + private static void closeFos(FileOutputStream fos) { + if (fos != null) { + try { + fos.flush(); + fos.close(); + } catch (IOException e) { + log.error("Fail to close fos", e); + } + } + } + + private static void closeStreams( + InputStream ins, + FileInputStream fis + ) { + if (fis != null) { + try { + fis.close(); + } catch (IOException e) { + log.error("Fail to close fis", e); + } + } + if (ins != null) { + try { + ins.close(); + } catch (IOException e) { + log.error("Fail to close ins", e); + } + } + } + + /** + * 删除空目录 + * + * @param directory 目录文件 + * @return 是否删除了目录 + */ + public static boolean deleteEmptyDirectory(File directory) { + if (directory == null) { + log.warn("Directory is null!"); + return false; + } + + if (isEmpty(directory.toPath())) { + boolean delete = directory.delete(); + if (delete) { + log.info("Delete empty directory {}", directory.getPath()); + } else { + log.warn("Delete directory {} failed!", directory.getPath()); + } + return delete; + } else { + log.debug("Directory {} not empty!", directory.getPath()); + return false; + } + } + + private static boolean isEmpty(Path path) { + if (Files.isDirectory(path)) { + try (Stream entries = Files.list(path)) { + return !entries.findFirst().isPresent(); + } catch (IOException e) { + return false; + } + } + + return false; + } + + /** + * 删除文件，若删除失败则记录路径 + * + * @param file 文件 + * @param failedPathSet 删除失败时用于记录路径的Set + * @return 是否删除成功 + */ + public static boolean deleteFileAndRecordIfFail(File file, Set failedPathSet) { + try { + FileUtils.deleteQuietly(file); + return true; + } catch (Exception e) { + failedPathSet.add(file.getAbsolutePath()); + FormattingTuple message = MessageFormatter.format( + "Fail to delete file {}", + file.getAbsolutePath() + ); + log.warn(message.getMessage(), e); + return false; + } + } + + /** + * 显示某个目录的磁盘使用信息 + * + * @param path 目录路径 + * @param maxSizeBytes 最大可用字节数 + * @param currentSize 当前已使用的字节数 + */ + public static void showVolumeUsage(String path, long maxSizeBytes, long currentSize) { + log.info( + "VolumeUsage: path={},currentSize={},maxSize={}", + path, + FileSizeUtil.getFileSizeStr(currentSize), + FileSizeUtil.getFileSizeStr(maxSizeBytes) + ); + } + + /** + * 检查指定目录下的磁盘使用是否超出限制，超出限制则清理最旧的文件 + * + * @param maxSizeBytes 最大限制字节数 + * @param targetDirPath 目标目录路径 + * @return 被成功清理的文件数量 + */ + public static int checkVolumeAndClearOldestFiles(long maxSizeBytes, String targetDirPath) { + return checkVolumeAndClearOldestFiles(maxSizeBytes, targetDirPath, null); + } + + /** + * 检查指定目录下的磁盘使用是否超出限制，超出限制则清理最旧的文件 + * + * @param maxSizeBytes 最大限制字节数 + * @param targetDirPath 目标目录路径 + * @param exceptSuffixs 不删除的文件后缀名 + * @return 被成功清理的文件数量 + */ + public static int checkVolumeAndClearOldestFiles(long maxSizeBytes, + String targetDirPath, + Set exceptSuffixs) { + if (null == targetDirPath) { + throw new IllegalArgumentException("TargetDirPath cannot be null"); + } + File targetDirFile = new File(targetDirPath); + if (!targetDirFile.exists() && log.isDebugEnabled()) { + log.debug("TargetDir({}) not exists yet, ignore clear", targetDirPath); + return -1; + } + long currentSize = FileUtils.sizeOfDirectory(targetDirFile); + if (log.isDebugEnabled()) { + showVolumeUsage(targetDirFile.getAbsolutePath(), maxSizeBytes, currentSize); + } + File[] files = targetDirFile.listFiles(); + if (files == null || files.length == 0) return 0; + List fileList = new ArrayList<>(Arrays.asList(files)); + fileList.sort(Comparator.comparingLong(File::lastModified)); + // 记录删除失败的文件，下次不再列出 + Set deleteFailedFilePathSet = new HashSet<>(); + // 记录忽略的文件，下次不再列出 + Set ignoredFilePathSet = new HashSet<>(); + int count = 0; + while (currentSize > maxSizeBytes) { + if (fileList.isEmpty()) { + // 上一次拿到的文件列表已删完，空间依然超限，说明删除过程中又新产生了许多文件，重新列出 + files = targetDirFile.listFiles(); + if (files == null || files.length == 0) return count; + fileList.addAll(Arrays.stream(files) + .filter(file -> !deleteFailedFilePathSet.contains(file.getAbsolutePath())) + .filter(file -> !ignoredFilePathSet.contains(file.getAbsolutePath())) + .collect(Collectors.toList()) + ); + fileList.sort(Comparator.comparingLong(File::lastModified)); + } + if (fileList.isEmpty()) { + if (!deleteFailedFilePathSet.isEmpty()) { + log.warn( + "Volume still overlimit after clear, ignoredFilePathSet={}, deleteFailedFilePathSet={}", + ignoredFilePathSet, + deleteFailedFilePathSet + ); + } else { + log.info("Volume still overlimit after clear, ignoredFilePathSet={}", ignoredFilePathSet); + } + return count; + } + File oldestFile = fileList.remove(0); + // 符合指定后缀名的文件不删除 + if (matchSuffixs(oldestFile.getName(), exceptSuffixs)) { + ignoredFilePathSet.add(oldestFile.getAbsolutePath()); + continue; + } + if (deleteFileAndRecordIfFail(oldestFile, deleteFailedFilePathSet)) { + count += 1; + log.info("Delete file {} because of volume overlimit", oldestFile.getAbsolutePath()); + currentSize = FileUtils.sizeOfDirectory(targetDirFile); + showVolumeUsage(targetDirFile.getAbsolutePath(), maxSizeBytes, currentSize); + } + } + if (log.isDebugEnabled()) { + log.debug("{} files deleted because of volume overlimit", count); + } else if (count > 0) { + log.info("{} files deleted because of volume overlimit", count); + } + return count; + } + + /** + * 判断文件名是否以指定的某些后缀名结尾 + * + * @param fileName 文件名 + * @param suffixs 后缀名集合 + * @return 文件名是否以后缀名集合中的任意一个结尾 + */ + private static boolean matchSuffixs(String fileName, Set suffixs) { + if (CollectionUtils.isEmpty(suffixs)) { + return false; + } + if (StringUtils.isBlank(fileName)) { + return false; + } + for (String suffix : suffixs) { + if (fileName.endsWith(suffix)) { + return true; + } + } + return false; + } +} diff --git a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/function/LambdasUtil.java b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/function/LambdasUtil.java index 5c45858af5..1cf5e8e71c 100644 --- a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/function/LambdasUtil.java +++ b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/function/LambdasUtil.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/json/JsonMapper.java b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/json/JsonMapper.java index 0c97e7582a..65220c611c 100644 --- a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/json/JsonMapper.java +++ b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/json/JsonMapper.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -29,6 +29,7 @@ import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.datatype.joda.JodaModule; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -41,7 +42,7 @@ @Slf4j public class JsonMapper { - private ObjectMapper mapper; + private final ObjectMapper mapper; public JsonMapper() { this(null); @@ -57,6 +58,8 @@ public JsonMapper(Include include) { mapper.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES); mapper.disable(SerializationFeature.FAIL_ON_EMPTY_BEANS); mapper.setDateFormat(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")); + mapper.registerModule(new JodaModule()); + mapper.findAndRegisterModules(); } @@ -64,6 +67,10 @@ public static JsonMapper nonEmptyMapper() { return new JsonMapper(Include.NON_EMPTY); } + public static JsonMapper nonNullMapper() { + return new JsonMapper(Include.NON_NULL); + } + public static JsonMapper nonDefaultMapper() { return new JsonMapper(Include.NON_DEFAULT); @@ -74,13 +81,13 @@ public static JsonMapper getAllOutPutMapper() { return new JsonMapper(Include.ALWAYS); } - public String toJson(Object object) { + public String toJson(Object object) throws JsonParseException { try { return mapper.writeValueAsString(object); } catch (IOException e) { log.warn("Write to json string error:" + object, e); - return null; + throw new JsonParseException(e); } } @@ -94,7 +101,7 @@ public String toJson(Object object) { * * @see #fromJson(String, TypeReference) */ - public T fromJson(String jsonString, Class clazz) { + public T fromJson(String jsonString, Class clazz) throws JsonParseException { if (StringUtils.isEmpty(jsonString)) { return null; } @@ -103,7 +110,7 @@ public T fromJson(String jsonString, Class clazz) { return mapper.readValue(jsonString, clazz); } catch (IOException e) { log.warn("Parse json string error:" + jsonString, e); - return null; + throw new JsonParseException(e); } } @@ -113,32 +120,20 @@ public T fromJson(String jsonString, Class clazz) { * 如果JSON字符串为Null或"null"字符串, 返回Null. * 如果JSON字符串为"[]", 返回空集合. */ - public T fromJson(String jsonString, TypeReference typeReference) { + public T fromJson(String jsonString, TypeReference typeReference) throws JsonParseException { if (StringUtils.isEmpty(jsonString)) { return null; } try { return mapper.readValue(jsonString, typeReference); - } catch (IOException e) { + } catch (Throwable e) { + // 捕获所有异常，统一转换为JsonParseException log.warn("parse json string error:" + typeReference, e); - return null; + throw new JsonParseException(e); } } - - /** - * 当JSON里只含有Bean的部分屬性時，更新一個已存在Bean，只覆蓋該部分的屬性. - */ - public void update(String jsonString, Object object) { - try { - mapper.readerForUpdating(object).readValue(jsonString); - } catch (IOException e) { - log.warn("Update json string:" + jsonString + " to object:" + object + " error.", e); - } - } - - /** * 取出Mapper做进一步的设置或使用其他序列化API. */ diff --git a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/json/JsonParseException.java b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/json/JsonParseException.java new file mode 100644 index 0000000000..14c4ca02da --- /dev/null +++ b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/json/JsonParseException.java @@ -0,0 +1,15 @@ +package com.tencent.bk.job.common.util.json; + +/** + * Json 解析异常 + */ +public class JsonParseException extends RuntimeException { + + public JsonParseException(Throwable cause) { + super(cause); + } + + public JsonParseException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/json/JsonUtils.java b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/json/JsonUtils.java index 3dd23b9104..6cadfb2a90 100644 --- a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/json/JsonUtils.java +++ b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/json/JsonUtils.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -34,12 +34,10 @@ import java.io.IOException; import java.lang.reflect.Field; -import java.text.SimpleDateFormat; import java.util.HashMap; import java.util.HashSet; import java.util.Map; import java.util.Set; -import java.util.TimeZone; /** * JSON工具 @@ -54,7 +52,7 @@ public class JsonUtils { * 注意：只对第一层的字段生效，嵌套的不生效 * * @param bean - * @return + * @return Json * @see SkipLogFields */ public static String toJsonWithoutSkippedFields(T bean) { @@ -92,12 +90,12 @@ public static String toJsonWithoutSkippedFields(T bean) { /** * 从Json串中解析成bean对象,支持参数泛型 * - * @param jsonString - * @param typeReference - * @param - * @return + * @param jsonString json + * @param typeReference 类型 + * @param bean Class 类型 + * @return bean */ - public static T fromJson(String jsonString, TypeReference typeReference) { + public static T fromJson(String jsonString, TypeReference typeReference) throws JsonParseException { return JSON_MAPPERS.computeIfAbsent("__all__", s -> JsonMapper.getAllOutPutMapper()).fromJson(jsonString, typeReference); } @@ -105,12 +103,12 @@ public static T fromJson(String jsonString, TypeReference typeReference) /** * 从Json串中解析成bean对象 * - * @param jsonString - * @param beanClass - * @param - * @return + * @param jsonString json + * @param beanClass bean Class 类型 + * @param bean Class 类型 + * @return bean */ - public static T fromJson(String jsonString, Class beanClass) { + public static T fromJson(String jsonString, Class beanClass) throws JsonParseException { return JSON_MAPPERS.computeIfAbsent("__all__", s -> JsonMapper.getAllOutPutMapper()).fromJson(jsonString, beanClass); } @@ -118,11 +116,11 @@ public static T fromJson(String jsonString, Class beanClass) { /** * 创建输出所有字段的Json，不管字段值是默认值 还是等于 null 还是空集合的字段，全输出,可用于外部接口协议输出 * - * @param bean - * @param - * @return + * @param bean bean + * @param bean + * @return json */ - public static String toJson(T bean) { + public static String toJson(T bean) throws JsonParseException { return JSON_MAPPERS.computeIfAbsent("__all__", s -> JsonMapper.getAllOutPutMapper()).toJson(bean); } @@ -134,22 +132,6 @@ public static String toNonDefault(T bean) { return JSON_MAPPERS.computeIfAbsent("__non_default__", s -> JsonMapper.nonDefaultMapper()).toJson(bean); } - public static String toJson(String timeZoneStr, T bean) { - - return JSON_MAPPERS.computeIfAbsent(timeZoneStr + "_tz__all__", s -> { - JsonMapper allOutPutMapper = JsonMapper.getAllOutPutMapper(); - if (timeZoneStr != null) { - try { - TimeZone timeZone = TimeZone.getTimeZone(timeZoneStr); - allOutPutMapper.getMapper().setTimeZone(timeZone).setDateFormat(new SimpleDateFormat("yyyy-MM-dd " + - "HH:mm:ss Z")); - } catch (Exception ignored) { - } - } - return allOutPutMapper; - }).toJson(bean); - } - public static JsonNode toJsonNode(String jsonStr) { try { return JSON_MAPPERS.computeIfAbsent("__all__", s -> JsonMapper.getAllOutPutMapper()) diff --git a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/json/SkipLogFields.java b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/json/SkipLogFields.java index f931b2dc36..0e42451f82 100644 --- a/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/json/SkipLogFields.java +++ b/src/backend/commons/common-utils/src/main/java/com/tencent/bk/job/common/util/json/SkipLogFields.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-utils/src/test/java/com/tencent/bk/job/common/util/Base64UtilTest.java b/src/backend/commons/common-utils/src/test/java/com/tencent/bk/job/common/util/Base64UtilTest.java new file mode 100644 index 0000000000..6e02a7b199 --- /dev/null +++ b/src/backend/commons/common-utils/src/test/java/com/tencent/bk/job/common/util/Base64UtilTest.java @@ -0,0 +1,81 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.util; + +import org.junit.jupiter.api.Test; + +import java.util.Random; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class Base64UtilTest { + + @Test + void testCalcOriginBytesLength() { + for (int i = 0; i < 100; i++) { + String originStr = genRandomString(); + assertCalc(originStr); + } + String specialStrWithChinese = "# 在当前脚本执行时，第一行输出当前时间和进程ID，详见上面函数：job_get_now\n" + + "job_start\n" + + "\n" + + "###### 作业平台中执行脚本成功和失败的标准只取决于脚本最后一条执行语句的返回值\n" + + "###### 如果返回值为0，则认为此脚本执行成功，如果非0，则认为脚本执行失败\n" + + "###### 可在此处开始编写您的脚本逻辑代码"; + assertCalc(specialStrWithChinese); + + } + + private void assertCalc(String originStr) { + + String encodedStr = Base64Util.encodeContentToStr(originStr); + int calcLengthByUtil = Base64Util.calcOriginBytesLength(encodedStr); + + byte[] bytes = Base64Util.decodeContentToByte(encodedStr); + int originBytesLength = bytes.length; + + System.out.println("originStr:" + originStr); + System.out.println("originBytesLength:" + originBytesLength); + System.out.println("calcLengthByUtil:" + calcLengthByUtil); + + assertEquals(originBytesLength, calcLengthByUtil, + "bytes length should be " + originBytesLength + ", but is " + calcLengthByUtil); + } + + private String genRandomString() { + final int ASCII_LOW = 32; + final int ASCII_HIGH = 126; + final int SHORTEST = 1; + final int LONGEST = 500; + Random rand = new Random(); + int length = rand.nextInt(LONGEST - SHORTEST) + SHORTEST; + StringBuilder sb = new StringBuilder(length); + for (int i = 0; i < length; i++) { + int randomAscii = ASCII_LOW + rand.nextInt(ASCII_HIGH - ASCII_LOW + 1); + sb.append((char) randomAscii); + } + return sb.toString(); + } +} diff --git a/src/backend/commons/common-utils/src/test/java/com/tencent/bk/job/common/util/BatchUtilTest.java b/src/backend/commons/common-utils/src/test/java/com/tencent/bk/job/common/util/BatchUtilTest.java deleted file mode 100644 index 872a3c7f3b..0000000000 --- a/src/backend/commons/common-utils/src/test/java/com/tencent/bk/job/common/util/BatchUtilTest.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.util; - -import org.junit.jupiter.api.Test; - -import java.util.ArrayList; -import java.util.List; - -import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThat; - -class BatchUtilTest { - @Test - void testBuildBatchList() { - List list = new ArrayList<>(); - list.add("a"); - List> batchList = BatchUtil.buildBatchList(list, 1); - assertThat(batchList).hasSize(1); - assertThat(batchList.get(0)).hasSize(1); - - list.add("b"); - batchList = BatchUtil.buildBatchList(list, 3); - assertThat(batchList).hasSize(1); - assertThat(batchList.get(0)).hasSize(2); - - list.add("c"); - batchList = BatchUtil.buildBatchList(list, 2); - assertThat(batchList).hasSize(2); - assertThat(batchList.get(0)).hasSize(2); - assertThat(batchList.get(1)).hasSize(1); - - list.add("d"); - batchList = BatchUtil.buildBatchList(list, 2); - assertThat(batchList).hasSize(2); - assertThat(batchList.get(0)).hasSize(2); - assertThat(batchList.get(1)).hasSize(2); - } -} diff --git a/src/backend/commons/common-utils/src/test/java/com/tencent/bk/job/common/util/CollectionUtilTest.java b/src/backend/commons/common-utils/src/test/java/com/tencent/bk/job/common/util/CollectionUtilTest.java new file mode 100644 index 0000000000..4186941269 --- /dev/null +++ b/src/backend/commons/common-utils/src/test/java/com/tencent/bk/job/common/util/CollectionUtilTest.java @@ -0,0 +1,232 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.util; + +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThat; + +class CollectionUtilTest { + @Test + void testPartitionList() { + List list = new ArrayList<>(); + list.add("a"); + List> partitionLists = CollectionUtil.partitionList(list, 1); + assertThat(partitionLists).hasSize(1); + assertThat(partitionLists.get(0)).hasSize(1); + + list.add("b"); + partitionLists = CollectionUtil.partitionList(list, 3); + assertThat(partitionLists).hasSize(1); + assertThat(partitionLists.get(0)).hasSize(2); + + list.add("c"); + partitionLists = CollectionUtil.partitionList(list, 2); + assertThat(partitionLists).hasSize(2); + assertThat(partitionLists.get(0)).hasSize(2); + assertThat(partitionLists.get(1)).hasSize(1); + + list.add("d"); + partitionLists = CollectionUtil.partitionList(list, 2); + assertThat(partitionLists).hasSize(2); + assertThat(partitionLists.get(0)).hasSize(2); + assertThat(partitionLists.get(1)).hasSize(2); + } + + @Nested + class TestPartitionCollection { + @Test + void testPartitionList() { + List list = new ArrayList<>(); + list.add("a"); + List> partitionLists = CollectionUtil.partitionCollection(list, 1); + assertThat(partitionLists).hasSize(1); + assertThat(partitionLists.get(0)).hasSize(1); + + list.add("b"); + partitionLists = CollectionUtil.partitionCollection(list, 3); + assertThat(partitionLists).hasSize(1); + assertThat(partitionLists.get(0)).hasSize(2); + + list.add("c"); + partitionLists = CollectionUtil.partitionCollection(list, 2); + assertThat(partitionLists).hasSize(2); + assertThat(partitionLists.get(0)).hasSize(2); + assertThat(partitionLists.get(1)).hasSize(1); + + list.add("d"); + partitionLists = CollectionUtil.partitionCollection(list, 2); + assertThat(partitionLists).hasSize(2); + assertThat(partitionLists.get(0)).hasSize(2); + assertThat(partitionLists.get(1)).hasSize(2); + + // 测试分区较多的场景 + list = new ArrayList<>(); + for (int i = 1; i <= 2009; i++) { + list.add("test" + i); + } + partitionLists = CollectionUtil.partitionCollection(list, 2); + assertThat(partitionLists).hasSize(1005); + assertThat(partitionLists.get(0)).hasSize(2); + assertThat(partitionLists.get(0).get(0)).isEqualTo("test1"); + assertThat(partitionLists.get(0).get(1)).isEqualTo("test2"); + assertThat(partitionLists.get(1)).hasSize(2); + assertThat(partitionLists.get(1004)).hasSize(1); + assertThat(partitionLists.get(1004).get(0)).isEqualTo("test2009"); + + List mergedElements = + partitionLists.stream().flatMap(Collection::stream).distinct().collect(Collectors.toList()); + // 测试分区之后与原始的在数量上一致 + assertThat(mergedElements).hasSize(2009); + } + + @Test + void testPartitionHashSet() { + Set set1 = new HashSet<>(); + set1.add("a"); + List> partitionLists = CollectionUtil.partitionCollection(set1, 1); + assertThat(partitionLists).hasSize(1); + assertThat(partitionLists.get(0)).hasSize(1); + + Set set2 = new HashSet<>(); + set2.add("a"); + set2.add("b"); + partitionLists = CollectionUtil.partitionCollection(set2, 3); + assertThat(partitionLists).hasSize(1); + + Set set3 = new HashSet<>(); + set3.add("a"); + set3.add("b"); + set3.add("c"); + partitionLists = CollectionUtil.partitionCollection(set3, 2); + assertThat(partitionLists).hasSize(2); + assertThat(partitionLists.get(0)).hasSize(2); + assertThat(partitionLists.get(1)).hasSize(1); + + Set set4 = new HashSet<>(); + set4.add("a"); + set4.add("b"); + set4.add("c"); + set4.add("d"); + partitionLists = CollectionUtil.partitionCollection(set4, 2); + assertThat(partitionLists).hasSize(2); + assertThat(partitionLists.get(0)).hasSize(2); + assertThat(partitionLists.get(1)).hasSize(2); + + Set set5 = new HashSet<>(); + set5.add("a"); + set5.add("b"); + set5.add("c"); + set5.add("d"); + set5.add("e"); + set5.add("f"); + set5.add("g"); + partitionLists = CollectionUtil.partitionCollection(set5, 3); + assertThat(partitionLists).hasSize(3); + assertThat(partitionLists.get(0)).hasSize(3); + assertThat(partitionLists.get(1)).hasSize(3); + assertThat(partitionLists.get(2)).hasSize(1); + + // 测试分区较多的场景 + Set set6 = new HashSet<>(); + for (int i = 1; i <= 2009; i++) { + set6.add("test" + i); + } + partitionLists = CollectionUtil.partitionCollection(set6, 2); + assertThat(partitionLists).hasSize(1005); + assertThat(partitionLists.get(0)).hasSize(2); + assertThat(partitionLists.get(1)).hasSize(2); + assertThat(partitionLists.get(1004)).hasSize(1); + + List mergedElements = + partitionLists.stream().flatMap(Collection::stream).distinct().collect(Collectors.toList()); + // 测试分区之后与原始的在数量上一致 + assertThat(mergedElements).hasSize(2009); + } + + @Test + void testConvertToMap() { + assertThat(CollectionUtil.convertToMap(null, TestEntity::getKey, TestEntity::getStringValue)).isEmpty(); + List entityList = new ArrayList<>(); + entityList.add(new TestEntity(null, null, null)); + assertThat(CollectionUtil.convertToMap(entityList, TestEntity::getKey, TestEntity::getStringValue).size()).isEqualTo(1); + assertThat(CollectionUtil.convertToMap(entityList, TestEntity::getKey, TestEntity::getStringValue).containsKey(null)).isTrue(); + assertThat(CollectionUtil.convertToMap(entityList, TestEntity::getKey, TestEntity::getStringValue).containsValue(null)).isTrue(); + Long key = 1L; + String stringValue = "StringValue"; + Object objectValue = new Object(); + entityList.add(new TestEntity(key, stringValue, objectValue)); + assertThat(CollectionUtil.convertToMap(entityList, TestEntity::getKey, TestEntity::getStringValue).size()).isEqualTo(2); + assertThat(CollectionUtil.convertToMap(entityList, TestEntity::getKey, TestEntity::getStringValue).containsKey(key)).isTrue(); + assertThat(CollectionUtil.convertToMap(entityList, TestEntity::getKey, TestEntity::getStringValue).containsValue(stringValue)).isTrue(); + assertThat(CollectionUtil.convertToMap(entityList, TestEntity::getKey, TestEntity::getObjectValue).size()).isEqualTo(2); + assertThat(CollectionUtil.convertToMap(entityList, TestEntity::getKey, TestEntity::getObjectValue).containsKey(key)).isTrue(); + assertThat(CollectionUtil.convertToMap(entityList, TestEntity::getKey, TestEntity::getObjectValue).containsValue(objectValue)).isTrue(); + String stringValue2 = "StringValue2"; + Object objectValue2 = new Object(); + entityList.add(new TestEntity(2L, stringValue2, objectValue2)); + assertThat(CollectionUtil.convertToMap(entityList, TestEntity::getKey, TestEntity::getStringValue).size()).isEqualTo(3); + assertThat(CollectionUtil.convertToMap(entityList, TestEntity::getKey, TestEntity::getStringValue).containsKey(key)).isTrue(); + assertThat(CollectionUtil.convertToMap(entityList, TestEntity::getKey, TestEntity::getStringValue).containsValue(stringValue)).isTrue(); + assertThat(CollectionUtil.convertToMap(entityList, TestEntity::getKey, TestEntity::getStringValue).containsValue(stringValue2)).isTrue(); + assertThat(CollectionUtil.convertToMap(entityList, TestEntity::getKey, TestEntity::getObjectValue).size()).isEqualTo(3); + assertThat(CollectionUtil.convertToMap(entityList, TestEntity::getKey, TestEntity::getObjectValue).containsKey(key)).isTrue(); + assertThat(CollectionUtil.convertToMap(entityList, TestEntity::getKey, TestEntity::getObjectValue).containsValue(objectValue)).isTrue(); + assertThat(CollectionUtil.convertToMap(entityList, TestEntity::getKey, TestEntity::getObjectValue).containsValue(objectValue2)).isTrue(); + } + + class TestEntity { + private final Long key; + private final String stringValue; + private final Object objectValue; + + public TestEntity(Long key, String stringValue, Object objectValue) { + this.key = key; + this.stringValue = stringValue; + this.objectValue = objectValue; + } + + public Long getKey() { + return key; + } + + public String getStringValue() { + return stringValue; + } + + public Object getObjectValue() { + return objectValue; + } + } + } +} diff --git a/src/backend/commons/common-utils/src/test/java/com/tencent/bk/job/common/util/DataSizeConverterTest.java b/src/backend/commons/common-utils/src/test/java/com/tencent/bk/job/common/util/DataSizeConverterTest.java new file mode 100644 index 0000000000..23c1cfe54a --- /dev/null +++ b/src/backend/commons/common-utils/src/test/java/com/tencent/bk/job/common/util/DataSizeConverterTest.java @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.util; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class DataSizeConverterTest { + + @Test + void convertKBToMB() { + assertEquals(DataSizeConverter.convertKBToMB(0), 0); + assertEquals(DataSizeConverter.convertKBToMB(1023), 1); + assertEquals(DataSizeConverter.convertKBToMB(1024), 1); + assertEquals(DataSizeConverter.convertKBToMB(1025), 2); + assertEquals(DataSizeConverter.convertKBToMB(2048), 2); + } + + @Test + void convertMBToKB() { + assertEquals(DataSizeConverter.convertMBToKB(0), 0); + assertEquals(DataSizeConverter.convertMBToKB(1), 1024); + } +} diff --git a/src/backend/commons/common-utils/src/test/java/com/tencent/bk/job/common/util/FilePathValidateUtilTest.java b/src/backend/commons/common-utils/src/test/java/com/tencent/bk/job/common/util/FilePathValidateUtilTest.java new file mode 100644 index 0000000000..62d759ba24 --- /dev/null +++ b/src/backend/commons/common-utils/src/test/java/com/tencent/bk/job/common/util/FilePathValidateUtilTest.java @@ -0,0 +1,32 @@ +package com.tencent.bk.job.common.util; + +public class FilePathValidateUtilTest { +// @Test +// void testFileSystemAbsolutePath(){ +// // 传统DOS路径 +// assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("C:\\Documents\\abc.txt")).isTrue(); +// assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("c:\\Documents\\abc.txt")).isTrue(); +// assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("C:\\Documents\\嘉 abc.txt")).isTrue(); +// assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath(":\\abc.txt")).isFalse(); +// assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("C:")).isFalse(); +// assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("C:\\\\")).isFalse(); +// assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("C:\\")).isTrue(); +// assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("C:\\logs\\..\\access.log")).isTrue(); +// assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("C:\\.config\\conf")).isTrue(); +// assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("C:\\user\\abc>a")).isFalse(); +// +// // linux路径 +// assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("/data/test_2022-04-12.apk")).isTrue(); +// assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("/data/test_2022 04 12.apk")).isTrue(); +// assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("/")).isTrue(); +// assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("/tmp/")).isTrue(); +// assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("/tmp/.conf/abc")).isTrue(); +// assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("/tmp/test/../test.log")).isTrue(); +// assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("data/test_2022-04-12.apk")).isFalse(); +// assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("///")).isTrue(); // 根目录 +// assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("/tmp////")).isTrue(); // /tmp/ +// assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("/tmp//test/")).isTrue();// /tmp/test/ +// assertThat(FilePathValidateUtil.validateFileSystemAbsolutePath("///")).isTrue(); +// } + +} diff --git a/src/backend/commons/common-utils/src/test/java/com/tencent/bk/job/common/util/ListUtilTest.java b/src/backend/commons/common-utils/src/test/java/com/tencent/bk/job/common/util/ListUtilTest.java index 90591df3b4..43d9660ed7 100644 --- a/src/backend/commons/common-utils/src/test/java/com/tencent/bk/job/common/util/ListUtilTest.java +++ b/src/backend/commons/common-utils/src/test/java/com/tencent/bk/job/common/util/ListUtilTest.java @@ -1,8 +1,12 @@ package com.tencent.bk.job.common.util; +import com.google.common.collect.Lists; +import org.apache.commons.lang3.tuple.Pair; import org.junit.jupiter.api.Test; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.List; import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThat; @@ -23,4 +27,45 @@ void testIsAllTrue() { notAllTrueList.add(false); assertThat(ListUtil.isAllTrue(notAllTrueList)).isFalse(); } + + @Test + void testSeparate() { + List list = Arrays.asList("a", "abc", "acc", "abb", "cc", "dd", "ee", null, "ff"); + Pair, List> pair = ListUtil.separate(list, s -> s != null && s.contains("a")); + assertThat(pair.getLeft().size() + pair.getRight().size() == list.size()); + pair.getLeft().forEach(s -> assertThat(s.contains("a"))); + pair.getRight().forEach(s -> assertThat(s == null || !s.contains("a"))); + } + + @Test + void testUnion() { + List list1 = null; + List list2 = null; + List result = ListUtil.union(list1, list2); + assertThat(result).isNotNull(); + assertThat(result).isEmpty(); + + + list1 = Lists.newArrayList("a", "b"); + result = ListUtil.union(list1, list2); + assertThat(result).containsOnly("a", "b"); + + list2 = Lists.newArrayList("c", "d"); + result = ListUtil.union(list1, list2); + assertThat(result).containsOnly("a", "b", "c", "d"); + } + + @Test + void testTrimStringList() { + assertThat(ListUtil.trimStringList(null)).isNull(); + assertThat(ListUtil.trimStringList(Collections.emptyList())).isNotNull(); + assertThat(ListUtil.trimStringList(Collections.emptyList())).size().isEqualTo(0); + + List list1 = Lists.newArrayList(null, " a ", " b b ", ""); + assertThat(ListUtil.trimStringList(list1)).size().isEqualTo(4); + assertThat(ListUtil.trimStringList(list1).get(0)).isNull(); + assertThat(ListUtil.trimStringList(list1).get(1)).isEqualTo("a"); + assertThat(ListUtil.trimStringList(list1).get(2)).isEqualTo("b b"); + assertThat(ListUtil.trimStringList(list1).get(3)).isEqualTo(""); + } } diff --git a/src/backend/commons/common-utils/src/test/java/com/tencent/bk/job/common/util/RandomUtilTest.java b/src/backend/commons/common-utils/src/test/java/com/tencent/bk/job/common/util/RandomUtilTest.java new file mode 100644 index 0000000000..fe22df7164 --- /dev/null +++ b/src/backend/commons/common-utils/src/test/java/com/tencent/bk/job/common/util/RandomUtilTest.java @@ -0,0 +1,18 @@ +package com.tencent.bk.job.common.util; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThat; + +public class RandomUtilTest { + + @Test + void testNextInt() { + for (int i = 0; i < 100; i++) { + int bound = RandomUtil.nextInt(1000) + 1; + assertThat(RandomUtil.nextInt(bound)).isGreaterThanOrEqualTo(0); + assertThat(RandomUtil.nextInt(bound)).isLessThan(bound); + } + } + +} diff --git a/src/backend/commons/common-utils/src/test/java/com/tencent/bk/job/common/util/date/DateUtilsTest.java b/src/backend/commons/common-utils/src/test/java/com/tencent/bk/job/common/util/date/DateUtilsTest.java index 8d3cde3b28..1ecb9c4ead 100644 --- a/src/backend/commons/common-utils/src/test/java/com/tencent/bk/job/common/util/date/DateUtilsTest.java +++ b/src/backend/commons/common-utils/src/test/java/com/tencent/bk/job/common/util/date/DateUtilsTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,7 +26,12 @@ import org.junit.jupiter.api.Test; +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.time.ZonedDateTime; + import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; public class DateUtilsTest { @@ -42,4 +47,69 @@ public void testGetPreviousDate() { assertThat(DateUtils.calcDaysBetween("2021-2-28", "2021-3-1")).isEqualTo(1); assertThat(DateUtils.calcDaysBetween("2020-2-28", "2020-3-1")).isEqualTo(2); } + + @Test + void getUTCDayEndTimestamp() { + ZonedDateTime dateTime = LocalDateTime.of(2023, 7, 14, 15, 14, 23, 12312) + .atZone(ZoneId.of("UTC")); + long endTime = DateUtils.getUTCDayEndTimestamp(dateTime.toLocalDate()); + // 2023-07-15 00:00:00 UTC + assertThat(endTime).isEqualTo(1689379200000L); + + + ZonedDateTime dateTime2 = LocalDateTime.of(2023, 7, 14, 7, 15, 22, 123) + .atZone(ZoneId.of("UTC")); + long endTime2 = DateUtils.getUTCDayEndTimestamp(dateTime2.toLocalDate()); + // 2023-07-15 00:00:00 UTC + assertThat(endTime2).isEqualTo(1689379200000L); + + ZonedDateTime dateTime3 = LocalDateTime.of(2023, 7, 13, 0, 0, 0, 0) + .atZone(ZoneId.of("UTC")); + long endTime3 = DateUtils.getUTCDayEndTimestamp(dateTime3.toLocalDate()); + // 2023-07-14 00:00:00 UTC + assertThat(endTime3).isEqualTo(1689292800000L); + + ZonedDateTime dateTime4 = LocalDateTime.of(2023, 7, 13, 23, 59, 59, 99999999) + .atZone(ZoneId.of("UTC")); + long endTime4 = DateUtils.getUTCDayEndTimestamp(dateTime4.toLocalDate()); + // 2023-07-14 00:00:00 UTC + assertThat(endTime4).isEqualTo(1689292800000L); + } + + @Test + void convertFromStringDateByPatterns() { + String[] patterns = new String[]{ + "yyyy-MM-dd'T'HH:mm:ss.SSS", + "yyyy-MM-dd'T'HH:mm:ss.SS", + "yyyy-MM-dd'T'HH:mm:ss.S", + "yyyy-MM-dd'T'HH:mm:ss.", + "yyyy-MM-dd'T'HH:mm:ss" + }; + LocalDateTime localDateTime = DateUtils.convertFromStringDateByPatterns( + "2022-04-21T10:55:54.655", patterns + ); + assertThat(localDateTime).isNotNull(); + localDateTime = DateUtils.convertFromStringDateByPatterns( + "2022-04-21T10:55:54.65", patterns + ); + assertThat(localDateTime).isNotNull(); + localDateTime = DateUtils.convertFromStringDateByPatterns( + "2022-04-21T10:55:54.6", patterns + ); + assertThat(localDateTime).isNotNull(); + localDateTime = DateUtils.convertFromStringDateByPatterns( + "2022-04-21T10:55:54.", patterns + ); + assertThat(localDateTime).isNotNull(); + localDateTime = DateUtils.convertFromStringDateByPatterns( + "2022-04-21T10:55:54", patterns + ); + assertThat(localDateTime).isNotNull(); + assertThatThrownBy(() -> DateUtils.convertFromStringDateByPatterns( + "2022-04-21T10:55:54.65" + )).isInstanceOf(IllegalArgumentException.class); + assertThatThrownBy(() -> DateUtils.convertFromStringDateByPatterns( + "2022-04-21 10:55:54.65" + )).isInstanceOf(IllegalArgumentException.class); + } } diff --git a/src/backend/commons/common-utils/src/test/java/com/tencent/bk/job/common/util/file/FileSizeUtilTest.java b/src/backend/commons/common-utils/src/test/java/com/tencent/bk/job/common/util/file/FileSizeUtilTest.java new file mode 100644 index 0000000000..cee58c9602 --- /dev/null +++ b/src/backend/commons/common-utils/src/test/java/com/tencent/bk/job/common/util/file/FileSizeUtilTest.java @@ -0,0 +1,60 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.util.file; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.assertj.core.api.AssertionsForInterfaceTypes.assertThat; + +public class FileSizeUtilTest { + @Test + void testGetFileSizeStr() { + assertThat(FileSizeUtil.getFileSizeStr(null)).isEqualTo("--"); + assertThat(FileSizeUtil.getFileSizeStr(0L)).isEqualTo("0.00B"); + assertThat(FileSizeUtil.getFileSizeStr(1023L)).isEqualTo("1023.00B"); + assertThat(FileSizeUtil.getFileSizeStr(1024L)).isEqualTo("1.00KB"); + assertThat(FileSizeUtil.getFileSizeStr(1024 * 1024L)).isEqualTo("1.00MB"); + assertThat(FileSizeUtil.getFileSizeStr(1024 * 1024L - 1)).isEqualTo("1024.00KB"); + assertThat(FileSizeUtil.getFileSizeStr(1024 * 1024 * 1024L)).isEqualTo("1.00GB"); + assertThat(FileSizeUtil.getFileSizeStr(1024 * 1024 * 1024 * 1024L)).isEqualTo("1.00TB"); + assertThat(FileSizeUtil.getFileSizeStr(1024 * 1024 * 1024 * 1024L * 1024L)).isEqualTo("1.00PB"); + assertThat(FileSizeUtil.getFileSizeStr(1025 * 1024 * 1024 * 1024L * 1024L * 1024L)).isEqualTo("1025.00PB"); + } + + @Test + void testParseFileSizeBytes() { + assertThat(FileSizeUtil.parseFileSizeBytes("1B")).isEqualTo(1L); + assertThat(FileSizeUtil.parseFileSizeBytes("1KB")).isEqualTo(1024L); + assertThat(FileSizeUtil.parseFileSizeBytes("1.5KB")).isEqualTo(1536L); + assertThat(FileSizeUtil.parseFileSizeBytes("1MB")).isEqualTo(1024L * 1024L); + assertThat(FileSizeUtil.parseFileSizeBytes("1GB")).isEqualTo(1024L * 1024L * 1024L); + assertThat(FileSizeUtil.parseFileSizeBytes("1TB")).isEqualTo(1024L * 1024L * 1024L * 1024L); + assertThat(FileSizeUtil.parseFileSizeBytes("1PB")).isEqualTo(1024L * 1024L * 1024L * 1024L * 1024L); + assertThatThrownBy(() -> + FileSizeUtil.parseFileSizeBytes("1XB") + ).isInstanceOf(IllegalArgumentException.class); + } +} diff --git a/src/backend/commons/common-utils/src/test/java/com/tencent/bk/job/common/util/json/JsonUtilsTest.java b/src/backend/commons/common-utils/src/test/java/com/tencent/bk/job/common/util/json/JsonUtilsTest.java index 1c15c09944..0d2146be0f 100644 --- a/src/backend/commons/common-utils/src/test/java/com/tencent/bk/job/common/util/json/JsonUtilsTest.java +++ b/src/backend/commons/common-utils/src/test/java/com/tencent/bk/job/common/util/json/JsonUtilsTest.java @@ -1,9 +1,12 @@ package com.tencent.bk.job.common.util.json; +import com.fasterxml.jackson.annotation.JsonInclude; +import org.joda.time.DateTime; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatCode; public class JsonUtilsTest { @@ -14,4 +17,91 @@ public void testNullToJson() { assertThat(jsonStr).isEqualTo("null"); } + @Test + @DisplayName("测试 Joda Time json field 反序列化") + void jodaTimeToJson() { + class TestObj { + private DateTime dateTime = DateTime.now(); + + public DateTime getDateTime() { + return dateTime; + } + + public void setDateTime(DateTime dateTime) { + this.dateTime = dateTime; + } + } + + TestObj testObj = new TestObj(); + assertThatCode(() -> JsonUtils.toJson(testObj)).doesNotThrowAnyException(); + } + + @Test + @DisplayName("测试序列化非空的属性") + void testToNonEmptyJson() { + Content content = new Content(); + content.setId("abc"); + assertThat(JsonUtils.toNonEmptyJson(content)).doesNotContain("name"); + } + + @Test + @DisplayName("测试序列化所有属性，包括 null/empty 的") + void testAllOutputJson() { + Content content = new Content(); + content.setId("abc"); + assertThat(JsonUtils.toJson(content)).contains("id", "name"); + } + + @Test + @DisplayName("测试 JsonInclude 注解覆盖 JsonUtils 中的默认的Include 配置") + void testJsonIncludeAnnotation() { + Content2 content = new Content2(); + content.setId("abc"); + assertThat(JsonUtils.toJson(content)).doesNotContain("name"); + } + + + private static class Content { + private String id; + private String name; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + } + + + @JsonInclude(JsonInclude.Include.NON_EMPTY) + private static class Content2 { + private String id; + private String name; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + } } diff --git a/src/backend/commons/common-web/build.gradle b/src/backend/commons/common-web/build.gradle index adadaef2d6..24d693af29 100644 --- a/src/backend/commons/common-web/build.gradle +++ b/src/backend/commons/common-web/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,6 +26,7 @@ dependencies { api project(':commons:common-service') api project(':commons:esb-sdk') api project(':commons:common-iam') + api project(':commons:common-audit') api 'org.springframework.boot:spring-boot-starter-web' api 'org.springframework.boot:spring-boot-starter-security' implementation 'com.google.guava:guava' diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/aspects/EsbAppResourceScopeReqAspect.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/aspects/EsbAppResourceScopeReqAspect.java new file mode 100644 index 0000000000..36291c2813 --- /dev/null +++ b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/aspects/EsbAppResourceScopeReqAspect.java @@ -0,0 +1,55 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.web.aspects; + +import com.tencent.bk.job.common.esb.model.EsbAppScopeReq; +import com.tencent.bk.job.common.service.AppScopeMappingService; +import lombok.extern.slf4j.Slf4j; +import org.aspectj.lang.JoinPoint; +import org.aspectj.lang.annotation.Aspect; +import org.aspectj.lang.annotation.Before; + +/** + * 执行ESB Controller之前，对AppResourceScope中的数据进行转换与填充 + */ +@Aspect +@Slf4j +public class EsbAppResourceScopeReqAspect { + private final AppScopeMappingService appScopeMappingService; + + public EsbAppResourceScopeReqAspect(AppScopeMappingService appScopeMappingService) { + this.appScopeMappingService = appScopeMappingService; + } + + @Before("within(com.tencent.bk.job..*) && execution(* com.tencent.bk.job.*.api.esb..*.*(..)))") + public void handleAppScopeReq(JoinPoint joinPoint) { + Object[] args = joinPoint.getArgs(); + for (Object arg : args) { + if (arg instanceof EsbAppScopeReq) { + ((EsbAppScopeReq) arg).fillAppResourceScope(appScopeMappingService); + } + } + } +} diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/ActuatorSecurityConfig.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/ActuatorSecurityConfig.java index db8af636d2..b29634d59a 100644 --- a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/ActuatorSecurityConfig.java +++ b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/ActuatorSecurityConfig.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,24 +25,27 @@ package com.tencent.bk.job.common.web.config; import org.springframework.boot.actuate.autoconfigure.security.servlet.EndpointRequest; +import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.security.config.annotation.web.builders.HttpSecurity; import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; -import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter; +import org.springframework.security.web.SecurityFilterChain; /** * Actuator spring security config */ @Configuration @EnableWebSecurity -public class ActuatorSecurityConfig extends WebSecurityConfigurerAdapter { - @Override - protected void configure(HttpSecurity http) throws Exception { +public class ActuatorSecurityConfig { + @Bean + SecurityFilterChain actuatorChain(HttpSecurity http) throws Exception { http.csrf().disable(); - http.requestMatcher(EndpointRequest.toAnyEndpoint()).authorizeRequests() + http.requestMatcher(EndpointRequest.toAnyEndpoint()) + .authorizeHttpRequests() .antMatchers("/actuator/health/**", "/actuator/info").permitAll() .antMatchers("/actuator/**").authenticated() .and() .httpBasic(); + return http.build(); } } diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/ApiTypeTagsContributor.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/ApiTypeTagsContributor.java index 839c7d9374..bb10c5c440 100644 --- a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/ApiTypeTagsContributor.java +++ b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/ApiTypeTagsContributor.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -30,12 +30,10 @@ import io.micrometer.core.instrument.Tags; import org.springframework.boot.actuate.metrics.web.servlet.WebMvcTags; import org.springframework.boot.actuate.metrics.web.servlet.WebMvcTagsContributor; -import org.springframework.stereotype.Component; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -@Component public class ApiTypeTagsContributor implements WebMvcTagsContributor { @Override diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/CustomTimedMetricsAutoConfiguration.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/CustomTimedMetricsAutoConfiguration.java index 49afc7c412..081051d348 100644 --- a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/CustomTimedMetricsAutoConfiguration.java +++ b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/CustomTimedMetricsAutoConfiguration.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -65,8 +65,8 @@ public DefaultCustomTimedTagsProvider customTimedTagsProvider( @Bean public CustomTimedMetricsInterceptor customTimedMetricsInterceptor( MeterRegistry meterRegistry, - CustomTimedTagsProvider customTimedTagsProvider - ) { + CustomTimedTagsProvider customTimedTagsProvider) { return new CustomTimedMetricsInterceptor(customTimedTagsProvider, meterRegistry); } + } diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/EsbAppReqAspectAutoConfiguration.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/EsbAppReqAspectAutoConfiguration.java new file mode 100644 index 0000000000..7e3294c4c6 --- /dev/null +++ b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/EsbAppReqAspectAutoConfiguration.java @@ -0,0 +1,43 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.web.config; + +import com.tencent.bk.job.common.service.AppScopeMappingService; +import com.tencent.bk.job.common.web.aspects.EsbAppResourceScopeReqAspect; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * Esb请求切面配置 + */ +@Configuration(proxyBeanMethods = false) +public class EsbAppReqAspectAutoConfiguration { + @Bean + public EsbAppResourceScopeReqAspect esbAppResourceScopeReqAspect( + ObjectProvider appScopeMappingServiceProvider) { + return new EsbAppResourceScopeReqAspect(appScopeMappingServiceProvider.getIfAvailable()); + } +} diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/ExceptionAdviceAutoConfiguration.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/ExceptionAdviceAutoConfiguration.java new file mode 100644 index 0000000000..4a02b62ad6 --- /dev/null +++ b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/ExceptionAdviceAutoConfiguration.java @@ -0,0 +1,61 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.web.config; + +import com.tencent.bk.job.common.iam.service.AuthService; +import com.tencent.bk.job.common.iam.service.WebAuthService; +import com.tencent.bk.job.common.web.exception.handler.EsbExceptionControllerAdvice; +import com.tencent.bk.job.common.web.exception.handler.IamCallbackExceptionControllerAdvice; +import com.tencent.bk.job.common.web.exception.handler.ServiceExceptionControllerAdvice; +import com.tencent.bk.job.common.web.exception.handler.WebExceptionControllerAdvice; +import org.springframework.context.annotation.Bean; + +/** + * 全局异常处理自动配置 + */ +public class ExceptionAdviceAutoConfiguration { + + @Bean + public WebExceptionControllerAdvice webExceptionControllerAdvice(WebAuthService webAuthService) { + return new WebExceptionControllerAdvice(webAuthService); + } + + @Bean + public EsbExceptionControllerAdvice esbExceptionControllerAdvice(AuthService authService) { + return new EsbExceptionControllerAdvice(authService); + } + + @Bean + public ServiceExceptionControllerAdvice serviceExceptionControllerAdvice() { + return new ServiceExceptionControllerAdvice(); + } + + @Bean + public IamCallbackExceptionControllerAdvice iamCallbackExceptionControllerAdvice() { + return new IamCallbackExceptionControllerAdvice(); + } + + +} diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/FilterConfig.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/FilterConfig.java new file mode 100644 index 0000000000..b8b8da5cf7 --- /dev/null +++ b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/FilterConfig.java @@ -0,0 +1,74 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.web.config; + +import com.tencent.bk.job.common.web.filter.RepeatableReadWriteServletRequestResponseFilter; +import com.tencent.bk.job.common.web.filter.WebRepeatableReadServletRequestFilter; +import org.springframework.boot.web.servlet.FilterRegistrationBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class FilterConfig { + + /** + * 给/esb/api/*， /service/* 用的过滤器，包装request和response + * + */ + @Bean + public FilterRegistrationBean repeatableRSRRFilterRegister() { + FilterRegistrationBean registration = + new FilterRegistrationBean<>(); + registration.setFilter(repeatableRRRFilter()); + registration.addUrlPatterns("/esb/api/*", "/service/*"); + registration.setName("repeatableReadRequestResponseFilter"); + registration.setOrder(0); + return registration; + } + + /** + * 给/web/* 用的过滤器，仅包装request + * + */ + @Bean + public FilterRegistrationBean webRepeatableRRFilterRegister() { + FilterRegistrationBean registration = new FilterRegistrationBean<>(); + registration.setFilter(webRepeatableReadRequestFilter()); + registration.addUrlPatterns("/web/*"); + registration.setName("webRepeatableReadRequestFilter"); + registration.setOrder(1); + return registration; + } + + @Bean(name = "repeatableReadRequestResponseFilter") + public RepeatableReadWriteServletRequestResponseFilter repeatableRRRFilter() { + return new RepeatableReadWriteServletRequestResponseFilter(); + } + + @Bean(name = "webRepeatableReadRequestFilter") + public WebRepeatableReadServletRequestFilter webRepeatableReadRequestFilter() { + return new WebRepeatableReadServletRequestFilter(); + } +} diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/I18nAutoConfiguration.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/I18nAutoConfiguration.java new file mode 100644 index 0000000000..c7f9749123 --- /dev/null +++ b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/I18nAutoConfiguration.java @@ -0,0 +1,50 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.web.config; + +import com.tencent.bk.job.common.web.i18n.JobLangHeaderLocaleResolver; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.autoconfigure.AutoConfigureBefore; +import org.springframework.boot.autoconfigure.web.servlet.WebMvcAutoConfiguration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Primary; + +import java.util.Locale; + +/** + * 国际化自动配置。由于需要覆盖 LocaleResolver， 所以必须比 WebMvcAutoConfiguration 先解析 + */ +@Slf4j +@AutoConfigureBefore(WebMvcAutoConfiguration.class) +public class I18nAutoConfiguration { + @Bean("localeResolver") + @Primary + public JobLangHeaderLocaleResolver localeResolver() { + log.info("Init JobLangHeaderLocaleResolver"); + JobLangHeaderLocaleResolver resolver = new JobLangHeaderLocaleResolver(); + resolver.setDefaultLocale(Locale.SIMPLIFIED_CHINESE); + return resolver; + } +} diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/OpSecurityConfig.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/OpSecurityConfig.java new file mode 100644 index 0000000000..0e3a5ee3ad --- /dev/null +++ b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/OpSecurityConfig.java @@ -0,0 +1,49 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.web.config; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.annotation.Order; +import org.springframework.security.config.annotation.web.builders.HttpSecurity; +import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; +import org.springframework.security.web.SecurityFilterChain; +import org.springframework.security.web.util.matcher.AntPathRequestMatcher; + +@Order(200) +@Configuration +@EnableWebSecurity +public class OpSecurityConfig { + @Bean + SecurityFilterChain opChain(HttpSecurity http) throws Exception { + http.csrf().disable(); + http.requestMatcher(new AntPathRequestMatcher("/op/**")) + .authorizeHttpRequests() + .anyRequest().authenticated() + .and() + .httpBasic(); + return http.build(); + } +} diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/SwaggerAdapterConfig.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/SwaggerAdapterConfig.java new file mode 100644 index 0000000000..c70ca5fa9f --- /dev/null +++ b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/SwaggerAdapterConfig.java @@ -0,0 +1,85 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.web.config; + +import org.springframework.boot.actuate.autoconfigure.endpoint.web.CorsEndpointProperties; +import org.springframework.boot.actuate.autoconfigure.endpoint.web.WebEndpointProperties; +import org.springframework.boot.actuate.autoconfigure.web.server.ManagementPortType; +import org.springframework.boot.actuate.endpoint.ExposableEndpoint; +import org.springframework.boot.actuate.endpoint.web.EndpointLinksResolver; +import org.springframework.boot.actuate.endpoint.web.EndpointMapping; +import org.springframework.boot.actuate.endpoint.web.EndpointMediaTypes; +import org.springframework.boot.actuate.endpoint.web.ExposableWebEndpoint; +import org.springframework.boot.actuate.endpoint.web.WebEndpointsSupplier; +import org.springframework.boot.actuate.endpoint.web.annotation.ControllerEndpointsSupplier; +import org.springframework.boot.actuate.endpoint.web.annotation.ServletEndpointsSupplier; +import org.springframework.boot.actuate.endpoint.web.servlet.WebMvcEndpointHandlerMapping; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.env.Environment; +import org.springframework.util.StringUtils; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +@Configuration +public class SwaggerAdapterConfig { + + /** + * 增加如下配置可解决Spring Boot 6.x 与Swagger 不兼容问题 + * + *

参考官方解决方案: https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2 + * .6-Release-Notes#pathpattern-based-path-matching-strategy-for-spring-mvc

+ **/ + @Bean + public WebMvcEndpointHandlerMapping webEndpointServletHandlerMapping(WebEndpointsSupplier webEndpointsSupplier, + ServletEndpointsSupplier servletEndpointsSupplier, + ControllerEndpointsSupplier controllerEndpointsSupplier, + EndpointMediaTypes endpointMediaTypes, + CorsEndpointProperties corsProperties, + WebEndpointProperties webEndpointProperties, + Environment environment) { + List> allEndpoints = new ArrayList<>(); + Collection webEndpoints = webEndpointsSupplier.getEndpoints(); + allEndpoints.addAll(webEndpoints); + allEndpoints.addAll(servletEndpointsSupplier.getEndpoints()); + allEndpoints.addAll(controllerEndpointsSupplier.getEndpoints()); + String basePath = webEndpointProperties.getBasePath(); + EndpointMapping endpointMapping = new EndpointMapping(basePath); + boolean shouldRegisterLinksMapping = this.shouldRegisterLinksMapping(webEndpointProperties, environment, + basePath); + return new WebMvcEndpointHandlerMapping(endpointMapping, webEndpoints, endpointMediaTypes, + corsProperties.toCorsConfiguration(), new EndpointLinksResolver(allEndpoints, basePath), + shouldRegisterLinksMapping, null); + } + + private boolean shouldRegisterLinksMapping(WebEndpointProperties webEndpointProperties, + Environment environment, + String basePath) { + return webEndpointProperties.getDiscovery().isEnabled() + && (StringUtils.hasText(basePath) || ManagementPortType.get(environment).equals(ManagementPortType.DIFFERENT)); + } +} diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/WebAutoConfiguration.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/WebAutoConfiguration.java new file mode 100644 index 0000000000..7dd76a14e3 --- /dev/null +++ b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/WebAutoConfiguration.java @@ -0,0 +1,58 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.web.config; + +import com.tencent.bk.job.common.esb.metrics.EsbApiTimedAspect; +import com.tencent.bk.job.common.web.feign.FeignConfiguration; +import com.tencent.bk.job.common.web.validation.ValidationConfiguration; +import io.micrometer.core.instrument.MeterRegistry; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; + +@Import({ + ActuatorSecurityConfig.class, + OpSecurityConfig.class, + SwaggerAdapterConfig.class, + FeignConfiguration.class, + FilterConfig.class, + ValidationConfiguration.class, + WebServerEventListenerConfiguration.class +}) +@Configuration(proxyBeanMethods = false) +public class WebAutoConfiguration { + + @Bean + public EsbApiTimedAspect esbApiTimedAspect(@Autowired MeterRegistry meterRegistry) { + return new EsbApiTimedAspect(meterRegistry); + } + + @Bean + public ApiTypeTagsContributor apiTypeTagsContributor() { + return new ApiTypeTagsContributor(); + } + +} diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/WebInterceptorAutoConfiguration.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/WebInterceptorAutoConfiguration.java new file mode 100644 index 0000000000..1a69a95c9f --- /dev/null +++ b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/WebInterceptorAutoConfiguration.java @@ -0,0 +1,62 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.web.config; + +import com.tencent.bk.job.common.jwt.JwtManager; +import com.tencent.bk.job.common.security.annotation.ConditionalOnSecurityEnabled; +import com.tencent.bk.job.common.service.SpringProfile; +import com.tencent.bk.job.common.web.interceptor.EsbApiLogInterceptor; +import com.tencent.bk.job.common.web.interceptor.JobCommonInterceptor; +import com.tencent.bk.job.common.web.interceptor.ServiceSecurityInterceptor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.cloud.sleuth.Tracer; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + + +/** + * 拦截器 AutoConfiguration + */ +@Slf4j +@Configuration(proxyBeanMethods = false) +public class WebInterceptorAutoConfiguration { + @Bean + public JobCommonInterceptor jobCommonInterceptor(Tracer tracer) { + return new JobCommonInterceptor(tracer); + } + + @Bean + public EsbApiLogInterceptor esbApiLogInterceptor() { + return new EsbApiLogInterceptor(); + } + + + @ConditionalOnSecurityEnabled + @Bean + public ServiceSecurityInterceptor serviceSecurityInterceptor(JwtManager jwtManager, SpringProfile springProfile) { + log.info("ServiceSecurityInterceptor inited"); + return new ServiceSecurityInterceptor(jwtManager, springProfile); + } +} diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/WebInterceptorAutoRegister.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/WebInterceptorAutoRegister.java new file mode 100644 index 0000000000..1659de461b --- /dev/null +++ b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/WebInterceptorAutoRegister.java @@ -0,0 +1,65 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.web.config; + +import com.tencent.bk.job.common.annotation.JobInterceptor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.ApplicationContext; +import org.springframework.web.servlet.HandlerInterceptor; +import org.springframework.web.servlet.config.annotation.InterceptorRegistry; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; + +import java.util.Map; + + +/** + * 拦截器自动注册 + */ +@Slf4j +public class WebInterceptorAutoRegister implements WebMvcConfigurer { + + private final ApplicationContext applicationContext; + + public WebInterceptorAutoRegister(ApplicationContext applicationContext) { + this.applicationContext = applicationContext; + } + + @Override + public void addInterceptors(InterceptorRegistry registry) { + Map interceptorMap = applicationContext.getBeansOfType(HandlerInterceptor.class); + interceptorMap.forEach((name, interceptor) -> { + JobInterceptor jobInterceptor = interceptor.getClass().getAnnotation(JobInterceptor.class); + if (jobInterceptor != null) { + log.info("Add job interceptor: {}, pathPatterns: {}, order: {}", + interceptor.getClass().getName(), + jobInterceptor.pathPatterns(), + jobInterceptor.order()); + registry.addInterceptor(interceptor) + .addPathPatterns(jobInterceptor.pathPatterns()) + .order(jobInterceptor.order()); + } + }); + } +} diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/WebInterceptorRegisterAutoConfiguration.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/WebInterceptorRegisterAutoConfiguration.java new file mode 100644 index 0000000000..9fbab1d2ab --- /dev/null +++ b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/WebInterceptorRegisterAutoConfiguration.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.web.config; + +import org.springframework.boot.autoconfigure.AutoConfigureAfter; +import org.springframework.context.ApplicationContext; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + + +/** + * 拦截器注册 AutoConfiguration + */ +@Configuration(proxyBeanMethods = false) +@AutoConfigureAfter(WebInterceptorAutoConfiguration.class) +public class WebInterceptorRegisterAutoConfiguration { + + @Bean + WebInterceptorAutoRegister webInterceptorAutoRegister(ApplicationContext applicationContext) { + return new WebInterceptorAutoRegister(applicationContext); + } +} diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/WebServerEventListenerConfiguration.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/WebServerEventListenerConfiguration.java new file mode 100644 index 0000000000..bd115be3c7 --- /dev/null +++ b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/config/WebServerEventListenerConfiguration.java @@ -0,0 +1,41 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.web.config; + +import com.tencent.bk.job.common.web.listener.ServletWebServerInitializedEventListener; +import io.micrometer.core.instrument.MeterRegistry; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Slf4j +@Configuration +public class WebServerEventListenerConfiguration { + + @Bean + public ServletWebServerInitializedEventListener servletWebServerInitializedEventListener(MeterRegistry registry) { + return new ServletWebServerInitializedEventListener(registry); + } +} diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/controller/WebVersionResource.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/controller/WebVersionResource.java index 5c93b830c8..264c01bb63 100644 --- a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/controller/WebVersionResource.java +++ b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/controller/WebVersionResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -30,9 +30,6 @@ import java.util.Map; -/** - * @since 17/7/2020 13:45 - */ @RestController @RequestMapping("/web") public interface WebVersionResource { diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/controller/WebVersionResourceImpl.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/controller/WebVersionResourceImpl.java new file mode 100644 index 0000000000..c9b2e27157 --- /dev/null +++ b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/controller/WebVersionResourceImpl.java @@ -0,0 +1,50 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.web.controller; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.actuate.autoconfigure.endpoint.condition.ConditionalOnAvailableEndpoint; +import org.springframework.boot.actuate.info.InfoEndpoint; + +import java.util.Map; + + +@Slf4j +@ConditionalOnAvailableEndpoint(endpoint = InfoEndpoint.class) +public class WebVersionResourceImpl implements WebVersionResource { + + private final InfoEndpoint infoEndpoint; + + @Autowired + public WebVersionResourceImpl(InfoEndpoint infoEndpoint) { + this.infoEndpoint = infoEndpoint; + } + + @Override + public Map getVersion() { + return infoEndpoint.info(); + } +} diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/exception/handler/EsbExceptionControllerAdvice.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/exception/handler/EsbExceptionControllerAdvice.java index 0b9b0631b3..f5cc0806a2 100644 --- a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/exception/handler/EsbExceptionControllerAdvice.java +++ b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/exception/handler/EsbExceptionControllerAdvice.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -32,6 +32,7 @@ import com.tencent.bk.job.common.exception.FailedPreconditionException; import com.tencent.bk.job.common.exception.InternalException; import com.tencent.bk.job.common.exception.InvalidParamException; +import com.tencent.bk.job.common.exception.MissingParameterException; import com.tencent.bk.job.common.exception.NotFoundException; import com.tencent.bk.job.common.exception.ResourceExhaustedException; import com.tencent.bk.job.common.exception.ServiceException; @@ -109,7 +110,10 @@ ResponseEntity handleServiceException(HttpServletRequest request, ServiceExce ResponseEntity handleInternalException(HttpServletRequest request, InternalException ex) { String errorMsg = "Handle InternalException, uri: " + request.getRequestURI(); log.error(errorMsg, ex); - return new ResponseEntity<>(EsbResp.buildCommonFailResp(ex.getErrorCode()), HttpStatus.OK); + return new ResponseEntity<>( + EsbResp.buildCommonFailResp(ex.getErrorCode(), ex.getErrorParams(), null), + HttpStatus.OK + ); } @ExceptionHandler(PermissionDeniedException.class) @@ -160,6 +164,15 @@ ResponseEntity handleUnauthenticatedException(HttpServletRequest request, Una return new ResponseEntity<>(EsbResp.buildCommonFailResp(ex), HttpStatus.OK); } + @ExceptionHandler({MissingParameterException.class}) + @ResponseBody + ResponseEntity handleMissingParameterException (HttpServletRequest request, + MissingParameterException ex) { + String errorMsg = "Handle MissingParameterException , uri: " + request.getRequestURI(); + log.warn(errorMsg, ex); + return new ResponseEntity<>(EsbResp.buildCommonFailResp(ex), HttpStatus.BAD_REQUEST); + } + @Override @SuppressWarnings("all") protected ResponseEntity handleMethodArgumentNotValid(MethodArgumentNotValidException ex, diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/exception/handler/ExceptionControllerAdviceBase.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/exception/handler/ExceptionControllerAdviceBase.java index 34ca5508dd..58346cdd25 100644 --- a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/exception/handler/ExceptionControllerAdviceBase.java +++ b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/exception/handler/ExceptionControllerAdviceBase.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/exception/handler/RemoteExceptionControllerAdvice.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/exception/handler/RemoteExceptionControllerAdvice.java new file mode 100644 index 0000000000..28182b1f8d --- /dev/null +++ b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/exception/handler/RemoteExceptionControllerAdvice.java @@ -0,0 +1,46 @@ +package com.tencent.bk.job.common.web.exception.handler; + +import com.tencent.bk.job.common.annotation.RemoteAPI; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.esb.model.EsbResp; +import com.tencent.bk.job.common.exception.InvalidParamException; +import com.tencent.bk.job.common.util.I18nUtil; +import com.tencent.bk.sdk.iam.constants.CommonResponseCode; +import com.tencent.bk.sdk.iam.dto.callback.response.CallbackBaseResponseDTO; +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.ControllerAdvice; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.ResponseBody; + +import javax.servlet.http.HttpServletRequest; + +/** + * 处理FileWorker上报接口异常 + */ +@ControllerAdvice(annotations = {RemoteAPI.class}) +@Slf4j +public class RemoteExceptionControllerAdvice extends ExceptionControllerAdviceBase { + + @ExceptionHandler(Throwable.class) + @ResponseBody + ResponseEntity handleException(HttpServletRequest request, Throwable ex) { + log.error("Handle Throwable", ex); + CallbackBaseResponseDTO responseDTO = new CallbackBaseResponseDTO(); + responseDTO.setCode(CommonResponseCode.SYSTEM_ERROR); + responseDTO.setMessage(I18nUtil.getI18nMessage(String.valueOf(ErrorCode.INTERNAL_ERROR))); + return new ResponseEntity<>(responseDTO, HttpStatus.INTERNAL_SERVER_ERROR); + } + + @ExceptionHandler(InvalidParamException.class) + @ResponseBody + ResponseEntity handleServiceException(HttpServletRequest request, InvalidParamException ex) { + log.info("Handle InvalidParamException", ex); + CallbackBaseResponseDTO responseDTO = new CallbackBaseResponseDTO(); + responseDTO.setCode(CommonResponseCode.PARAMS_INVALID); + responseDTO.setMessage(ex.getI18nMessage()); + return new ResponseEntity<>(EsbResp.buildCommonFailResp(ex), HttpStatus.BAD_REQUEST); + } + +} diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/exception/handler/ServiceExceptionControllerAdvice.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/exception/handler/ServiceExceptionControllerAdvice.java index 3ce6ce58c1..6e9181bd3b 100644 --- a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/exception/handler/ServiceExceptionControllerAdvice.java +++ b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/exception/handler/ServiceExceptionControllerAdvice.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/exception/handler/WebExceptionControllerAdvice.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/exception/handler/WebExceptionControllerAdvice.java index b4579ef63e..1dcc711642 100644 --- a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/exception/handler/WebExceptionControllerAdvice.java +++ b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/exception/handler/WebExceptionControllerAdvice.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/feign/FeignConfiguration.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/feign/FeignConfiguration.java index 10a0fbc5a0..c3c37200bd 100644 --- a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/feign/FeignConfiguration.java +++ b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/feign/FeignConfiguration.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,7 @@ package com.tencent.bk.job.common.web.feign; -import com.tencent.bk.job.common.feign.FeignAddHeaderRequestInterceptor; +import com.tencent.bk.job.common.service.feign.interceptor.FeignAddHeaderRequestInterceptor; import feign.codec.ErrorDecoder; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/feign/FeignErrorDecoder.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/feign/FeignErrorDecoder.java index d6f2ecfd8e..7e14cb92a0 100644 --- a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/feign/FeignErrorDecoder.java +++ b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/feign/FeignErrorDecoder.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,8 +24,6 @@ package com.tencent.bk.job.common.web.feign; -import org.apache.commons.lang3.StringUtils; - import com.tencent.bk.job.common.exception.AlreadyExistsException; import com.tencent.bk.job.common.exception.FailedPreconditionException; import com.tencent.bk.job.common.exception.InternalException; @@ -36,12 +34,12 @@ import com.tencent.bk.job.common.model.InternalResponse; import com.tencent.bk.job.common.model.error.ErrorType; import com.tencent.bk.job.common.util.json.JsonUtils; - import feign.FeignException; import feign.Response; import feign.RetryableException; import feign.codec.ErrorDecoder; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; @Slf4j public class FeignErrorDecoder extends ErrorDecoder.Default { @@ -94,7 +92,7 @@ private Exception decodeErrorCode(FeignException exception, InternalResponse case UNAVAILABLE: case TIMEOUT: // 微服务内部调用错误，应该转换为外部的错误;例如，从另一个服务接收 INVALID_PARAM 错误,应该将 INTERNAL_ERROR 传播给它自己的调用者。 - return new InternalException(errorMsg, exception, errorCode); + return new InternalException(errorMsg, exception); case PERMISSION_DENIED: return new PermissionDeniedException(AuthResult.fromAuthResultDTO(response.getAuthResult())); case NOT_FOUND: diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/filter/RepeatableReadWriteServletRequestResponseFilter.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/filter/RepeatableReadWriteServletRequestResponseFilter.java index a48aeebe73..fdf30e6517 100644 --- a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/filter/RepeatableReadWriteServletRequestResponseFilter.java +++ b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/filter/RepeatableReadWriteServletRequestResponseFilter.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,7 @@ package com.tencent.bk.job.common.web.filter; +import com.tencent.bk.job.common.web.utils.ServletUtil; import com.tencent.bk.job.common.web.model.RepeatableReadHttpServletResponse; import com.tencent.bk.job.common.web.model.RepeatableReadWriteHttpServletRequest; import lombok.extern.slf4j.Slf4j; @@ -48,9 +49,13 @@ public void init(FilterConfig filterConfig) { @Override public void doFilter(ServletRequest request, ServletResponse response, FilterChain chain) throws IOException, ServletException { - ServletRequest requestWrapper = new RepeatableReadWriteHttpServletRequest((HttpServletRequest) request); - ServletResponse responseWrapper = new RepeatableReadHttpServletResponse((HttpServletResponse) response); - chain.doFilter(requestWrapper, responseWrapper); + if (!ServletUtil.isJsonRequest(request)) { + chain.doFilter(request, response); + return; + } + ServletRequest servletRequest = new RepeatableReadWriteHttpServletRequest((HttpServletRequest) request); + ServletResponse servletResponse = new RepeatableReadHttpServletResponse((HttpServletResponse) response); + chain.doFilter(servletRequest, servletResponse); } @Override diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/filter/WebRepeatableReadServletRequestFilter.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/filter/WebRepeatableReadServletRequestFilter.java new file mode 100644 index 0000000000..2e35231799 --- /dev/null +++ b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/filter/WebRepeatableReadServletRequestFilter.java @@ -0,0 +1,68 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.web.filter; + +import com.tencent.bk.job.common.web.utils.ServletUtil; +import com.tencent.bk.job.common.web.model.RepeatableReadWriteHttpServletRequest; +import lombok.extern.slf4j.Slf4j; + +import javax.servlet.Filter; +import javax.servlet.FilterChain; +import javax.servlet.FilterConfig; +import javax.servlet.ServletException; +import javax.servlet.ServletRequest; +import javax.servlet.ServletResponse; +import javax.servlet.http.HttpServletRequest; +import java.io.IOException; + +@Slf4j +public class WebRepeatableReadServletRequestFilter implements Filter { + @Override + public void init(FilterConfig filterConfig) throws ServletException { + // do nothing + } + + /** + * 仅包装ServletRequest，给/web使用 + * + */ + @Override + public void doFilter(ServletRequest request, ServletResponse response, + FilterChain chain) throws IOException, ServletException { + ServletRequest servletRequest = request; + + if (ServletUtil.isJsonRequest(request)) { + // 仅处理 ContentType: application/json 请求 + servletRequest = new RepeatableReadWriteHttpServletRequest((HttpServletRequest) request); + } + + chain.doFilter(servletRequest, response); + } + + @Override + public void destroy() { + // do nothing + } +} diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/i18n/I18nConfig.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/i18n/I18nConfig.java deleted file mode 100644 index f2337f4fff..0000000000 --- a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/i18n/I18nConfig.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.web.i18n; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -import java.util.Locale; - -@Configuration -@Slf4j -public class I18nConfig { - - @Bean("localeResolver") - public JobLangHeaderLocaleResolver localeResolver() { - log.info("Init JobLangHeaderLocaleResolver"); - JobLangHeaderLocaleResolver resolver = new JobLangHeaderLocaleResolver(); - resolver.setDefaultLocale(Locale.SIMPLIFIED_CHINESE); - return resolver; - } -} - diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/i18n/JobLangHeaderLocaleResolver.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/i18n/JobLangHeaderLocaleResolver.java index d2dfc24d2d..edf6a94ccc 100644 --- a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/i18n/JobLangHeaderLocaleResolver.java +++ b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/i18n/JobLangHeaderLocaleResolver.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/interceptor/AppResourceScopeInterceptor.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/interceptor/AppResourceScopeInterceptor.java new file mode 100644 index 0000000000..2da05750e3 --- /dev/null +++ b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/interceptor/AppResourceScopeInterceptor.java @@ -0,0 +1,339 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.web.interceptor; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.tencent.bk.job.common.annotation.JobInterceptor; +import com.tencent.bk.job.common.constant.HttpRequestSourceEnum; +import com.tencent.bk.job.common.constant.InterceptorOrder; +import com.tencent.bk.job.common.constant.ResourceScopeTypeEnum; +import com.tencent.bk.job.common.model.dto.AppResourceScope; +import com.tencent.bk.job.common.model.dto.ResourceScope; +import com.tencent.bk.job.common.service.AppScopeMappingService; +import com.tencent.bk.job.common.util.JobContextUtil; +import com.tencent.bk.job.common.util.RequestUtil; +import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.common.util.toggle.ToggleEvaluateContext; +import com.tencent.bk.job.common.util.toggle.feature.FeatureIdConstants; +import com.tencent.bk.job.common.util.toggle.feature.FeatureToggle; +import com.tencent.bk.job.common.validation.ScopeValidator; +import com.tencent.bk.job.common.web.model.RepeatableReadWriteHttpServletRequest; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.helpers.MessageFormatter; +import org.springframework.http.HttpMethod; +import org.springframework.lang.NonNull; +import org.springframework.web.servlet.AsyncHandlerInterceptor; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import static com.tencent.bk.job.common.constant.JobConstants.JOB_BUILD_IN_BIZ_SET_ID_MAX; +import static com.tencent.bk.job.common.constant.JobConstants.JOB_BUILD_IN_BIZ_SET_ID_MIN; + +/** + * Job AppResourceScope 处理 + */ +@Slf4j +@JobInterceptor(pathPatterns = {"/web/**", "/service/**", "/esb/api/**"}, + order = InterceptorOrder.Init.REWRITE_REQUEST) +public class AppResourceScopeInterceptor implements AsyncHandlerInterceptor { + private static final Pattern SCOPE_PATTERN = Pattern.compile("/scope/(\\w+)/(\\d+)"); + + private static final Pattern APP_PATTERN = Pattern.compile("/app/(\\d+)"); + + private final AppScopeMappingService appScopeMappingService; + + private final AppResourceScopeParser webAppResourceScopeParser; + + private final AppResourceScopeParser esbAppResourceScopeParser; + + private final AppResourceScopeParser internalAppResourceScopeParser; + + public AppResourceScopeInterceptor(AppScopeMappingService appScopeMappingService) { + this.appScopeMappingService = appScopeMappingService; + this.webAppResourceScopeParser = new WebAppResourceScopeParser(); + this.esbAppResourceScopeParser = new EsbAppResourceScopeParser(); + this.internalAppResourceScopeParser = new InternalAppResourceScopeParser(); + } + + @Override + public boolean preHandle(@NonNull HttpServletRequest request, + @NonNull HttpServletResponse response, + @NonNull Object handler) { + if (!shouldFilter(request)) { + return true; + } + addAppResourceScope(request); + + return true; + } + + private boolean shouldFilter(HttpServletRequest request) { + String uri = request.getRequestURI(); + // 只拦截web/service/esb的API请求 + return uri.startsWith("/web/") || uri.startsWith("/service/") || uri.startsWith("/esb/"); + } + + private void addAppResourceScope(HttpServletRequest request) { + HttpRequestSourceEnum requestSource = RequestUtil.parseHttpRequestSource(request); + if (requestSource == HttpRequestSourceEnum.UNKNOWN) { + return; + } + + AppResourceScope appResourceScope = null; + switch (requestSource) { + case WEB: + appResourceScope = webAppResourceScopeParser.parseAppResourceScope(request); + log.debug("Scope from web:{}", appResourceScope); + break; + case ESB: + appResourceScope = esbAppResourceScopeParser.parseAppResourceScope(request); + log.debug("Scope from esb:{}", appResourceScope); + break; + case INTERNAL: + appResourceScope = internalAppResourceScopeParser.parseAppResourceScope(request); + log.debug("Scope from internal:{}", appResourceScope); + break; + default: + log.debug("Ignore invalid scope: {}", requestSource); + break; + } + if (appResourceScope != null) { + request.setAttribute("appResourceScope", appResourceScope); + JobContextUtil.setAppResourceScope(appResourceScope); + } else { + log.debug("AppResourceScope is empty"); + } + } + + /** + * 从 http 请求中解析 AppResourceScope + */ + public interface AppResourceScopeParser { + AppResourceScope parseAppResourceScope(HttpServletRequest request); + } + + /** + * 从 http 请求中解析 AppResourceScope - Web 请求 + */ + public class WebAppResourceScopeParser implements AppResourceScopeParser { + @Override + public AppResourceScope parseAppResourceScope(HttpServletRequest request) { + return parseAppResourceScopeFromPath(request.getRequestURI()); + } + + private AppResourceScope parseAppResourceScopeFromPath(String requestURI) { + ResourceScope resourceScope = parseResourceScopeFromURI(requestURI); + if (resourceScope != null) { + return buildAppResourceScope(resourceScope); + } + + return null; + } + + private ResourceScope parseResourceScopeFromURI(String requestURI) { + ResourceScope resourceScope = null; + Matcher scopeMatcher = SCOPE_PATTERN.matcher(requestURI); + if (scopeMatcher.find()) { + resourceScope = new ResourceScope(scopeMatcher.group(1), scopeMatcher.group(2)); + } + return resourceScope; + } + + private AppResourceScope buildAppResourceScope(ResourceScope resourceScope) { + Long appId = appScopeMappingService.getAppIdByScope(resourceScope); + return new AppResourceScope(appId, resourceScope); + } + } + + /** + * 从 http 请求中解析 AppResourceScope - ESB/蓝鲸网关请求 + */ + public class EsbAppResourceScopeParser implements AppResourceScopeParser { + @Override + public AppResourceScope parseAppResourceScope(HttpServletRequest request) { + return parseAppResourceScopeFromQueryStringOrBody(request); + } + + private AppResourceScope parseAppResourceScopeFromQueryStringOrBody(HttpServletRequest request) { + Map params = parseMultiValueFromQueryStringOrBody(request, + "bk_scope_type", "bk_scope_id", "bk_biz_id"); + String scopeType = params.get("bk_scope_type"); + String scopeId = params.get("bk_scope_id"); + String bizIdStr = params.get("bk_biz_id"); + + if (StringUtils.isNotBlank(scopeType) && StringUtils.isNotBlank(scopeId)) { + ScopeValidator.validate(null, scopeType, scopeId); + return new AppResourceScope(scopeType, scopeId, null); + } + + // 如果兼容bk_biz_id参数 + if (FeatureToggle.checkFeature(FeatureIdConstants.FEATURE_BK_BIZ_ID_COMPATIBLE, + ToggleEvaluateContext.EMPTY)) { + // 兼容当前业务ID参数 + if (StringUtils.isNotBlank(bizIdStr)) { + long bizId = Long.parseLong(bizIdStr); + // [8000000,9999999]是迁移业务集之前约定的业务集ID范围。为了兼容老的API调用方，在这个范围内的bizId解析为业务集 + scopeId = bizIdStr; + if (bizId >= JOB_BUILD_IN_BIZ_SET_ID_MIN && bizId <= JOB_BUILD_IN_BIZ_SET_ID_MAX) { + Long appId = appScopeMappingService.getAppIdByScope(ResourceScopeTypeEnum.BIZ_SET.getValue(), + scopeId); + return new AppResourceScope(ResourceScopeTypeEnum.BIZ_SET, scopeId, appId); + } else { + Long appId = appScopeMappingService.getAppIdByScope(ResourceScopeTypeEnum.BIZ.getValue(), + scopeId); + return new AppResourceScope(ResourceScopeTypeEnum.BIZ, scopeId, appId); + } + } + } + // 其他情况返回null，后续拦截器会处理null + return null; + } + + /** + * 从请求的解析多个参数 + * + * @param request http请求 + * @param keys 参数名称 + * @return Map + */ + private Map parseMultiValueFromQueryStringOrBody(HttpServletRequest request, String... keys) { + Map params = new HashMap<>(); + try { + if (request.getMethod().equals(HttpMethod.POST.name()) + || request.getMethod().equals(HttpMethod.PUT.name())) { + if (!(request instanceof RepeatableReadWriteHttpServletRequest)) { + return params; + } + RepeatableReadWriteHttpServletRequest wrapperRequest = + (RepeatableReadWriteHttpServletRequest) request; + if (StringUtils.isNotBlank(wrapperRequest.getBody())) { + ObjectNode jsonBody = (ObjectNode) JsonUtils.toJsonNode(wrapperRequest.getBody()); + if (jsonBody == null) { + return params; + } + for (String key : keys) { + JsonNode valueNode = jsonBody.get(key); + String value = (valueNode == null || valueNode.isNull()) ? null : + jsonBody.get(key).asText(); + log.debug("Parsed from POST/PUT: {}={}", key, value); + if (value != null) { + params.put(key, value); + } + } + } + } else if (request.getMethod().equals(HttpMethod.GET.name())) { + for (String key : keys) { + String value = request.getParameter(key); + log.debug("Parsed from GET: {}={}", key, value); + if (value != null) { + params.put(key, value); + } + } + } + return params; + } catch (Exception e) { + String msg = MessageFormatter.format( + "Fail to parse keys: {} from request", + keys + ).getMessage(); + log.warn(msg, e); + } + return params; + } + } + + /** + * 从 http 请求中解析 AppResourceScope - Job 内部请求 + */ + public class InternalAppResourceScopeParser implements AppResourceScopeParser { + @Override + public AppResourceScope parseAppResourceScope(HttpServletRequest request) { + // 优先从 path 解析 + Long appId = parseAppIdFromPath(request.getRequestURI()); + if (appId == null) { + // 从 QueryParam解析 + appId = parseAppIdFromQueryParam(request); + } + if (appId == null) { + // 从 Request Body 解析 + appId = parseAppIdFromRequestBody(request); + } + if (appId == null) { + return null; + } + return appScopeMappingService.getAppResourceScope(appId); + } + + private Long parseAppIdFromPath(String requestURI) { + Long appId = null; + Matcher appIdMatcher = APP_PATTERN.matcher(requestURI); + if (appIdMatcher.find()) { + appId = Long.parseLong(appIdMatcher.group(1)); + } + return appId; + } + + private Long parseAppIdFromQueryParam(HttpServletRequest request) { + String queryParam = "appId"; + String value = request.getParameter(queryParam); + log.debug("Parsed from GET: {}={}", queryParam, value); + return value != null ? Long.parseLong(value) : null; + } + + private Long parseAppIdFromRequestBody(HttpServletRequest request) { + if (!request.getMethod().equals(HttpMethod.POST.name()) + && !request.getMethod().equals(HttpMethod.PUT.name())) { + return null; + } + if (!(request instanceof RepeatableReadWriteHttpServletRequest)) { + return null; + } + + RepeatableReadWriteHttpServletRequest wrapperRequest = + (RepeatableReadWriteHttpServletRequest) request; + if (StringUtils.isNotBlank(wrapperRequest.getBody())) { + ObjectNode jsonBody = (ObjectNode) JsonUtils.toJsonNode(wrapperRequest.getBody()); + if (jsonBody == null) { + return null; + } + String fieldName = "appId"; + JsonNode valueNode = jsonBody.get(fieldName); + String value = (valueNode == null || valueNode.isNull()) ? null : + jsonBody.get(fieldName).asText(); + log.debug("Parsed from POST/PUT: {}={}", fieldName, value); + return value != null ? Long.parseLong(value) : null; + } else { + return null; + } + } + } +} diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/interceptor/CustomTimedMetricsInterceptor.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/interceptor/CustomTimedMetricsInterceptor.java index 78bddb9d97..8fda8005c6 100644 --- a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/interceptor/CustomTimedMetricsInterceptor.java +++ b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/interceptor/CustomTimedMetricsInterceptor.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,8 @@ package com.tencent.bk.job.common.web.interceptor; +import com.tencent.bk.job.common.annotation.JobInterceptor; +import com.tencent.bk.job.common.constant.InterceptorOrder; import com.tencent.bk.job.common.util.JobContextUtil; import com.tencent.bk.job.common.web.metrics.CustomTimed; import com.tencent.bk.job.common.web.metrics.CustomTimedTagsProvider; @@ -45,6 +47,7 @@ * Job自定义Timer指标数据拦截处理器 */ @Slf4j +@JobInterceptor(pathPatterns = "/**", order = InterceptorOrder.Init.METRICS) public class CustomTimedMetricsInterceptor implements HandlerInterceptor { private final MeterRegistry registry; diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/interceptor/EsbApiLogInterceptor.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/interceptor/EsbApiLogInterceptor.java index 05390a6996..1fefdbd689 100644 --- a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/interceptor/EsbApiLogInterceptor.java +++ b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/interceptor/EsbApiLogInterceptor.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,15 +25,16 @@ package com.tencent.bk.job.common.web.interceptor; import com.fasterxml.jackson.databind.node.ObjectNode; -import com.fasterxml.jackson.databind.node.TextNode; +import com.tencent.bk.job.common.annotation.JobInterceptor; +import com.tencent.bk.job.common.constant.InterceptorOrder; import com.tencent.bk.job.common.constant.JobCommonHeaders; +import com.tencent.bk.job.common.util.StringUtil; import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.common.web.model.RepeatableReadHttpServletResponse; import com.tencent.bk.job.common.web.model.RepeatableReadWriteHttpServletRequest; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.http.HttpMethod; -import org.springframework.stereotype.Component; import org.springframework.web.servlet.handler.HandlerInterceptorAdapter; import javax.servlet.http.HttpServletRequest; @@ -41,13 +42,11 @@ import java.util.StringJoiner; @Slf4j -@Component +@JobInterceptor(order = InterceptorOrder.Init.LOG, pathPatterns = "/esb/api/**") public class EsbApiLogInterceptor extends HandlerInterceptorAdapter { private static final String ATTR_REQUEST_START = "request-start"; private static final String ATTR_API_NAME = "api-name"; - private static final String ATTR_USERNAME = "username"; - private static final String ATTR_APP_CODE = "app-code"; @Override public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) { @@ -57,8 +56,8 @@ public boolean preHandle(HttpServletRequest request, HttpServletResponse respons RepeatableReadWriteHttpServletRequest wrapperRequest = (RepeatableReadWriteHttpServletRequest) request; String desensitizedBody = ""; String desensitizedQueryParams = ""; - String username = ""; - String appCode = ""; + String username = request.getHeader(JobCommonHeaders.USERNAME); + String appCode = request.getHeader(JobCommonHeaders.APP_CODE); String apiName = ""; String lang = request.getHeader(JobCommonHeaders.BK_GATEWAY_LANG); String requestId = request.getHeader(JobCommonHeaders.BK_GATEWAY_REQUEST_ID); @@ -67,36 +66,35 @@ public boolean preHandle(HttpServletRequest request, HttpServletResponse respons request.setAttribute(ATTR_REQUEST_START, System.currentTimeMillis()); apiName = getAPIName(wrapperRequest.getRequestURI()); request.setAttribute(ATTR_API_NAME, apiName); - if (request.getMethod().equals(HttpMethod.POST.name()) - || request.getMethod().equals(HttpMethod.PUT.name())) { - if (StringUtils.isNotBlank(wrapperRequest.getBody())) { - ObjectNode jsonBody = (ObjectNode) JsonUtils.toJsonNode(wrapperRequest.getBody()); - if (jsonBody == null) { - return true; - } - username = jsonBody.get("bk_username") == null ? null : jsonBody.get("bk_username").asText(); - appCode = jsonBody.get("bk_app_code") == null ? null : jsonBody.get("bk_app_code").asText(); + desensitizedQueryParams = desensitizeQueryParams(request.getQueryString()); + desensitizedBody = desensitizeRequestBody(wrapperRequest); + } catch (Throwable ignore) { + // do nothing + } finally { + log.info("request-id: {}|lang: {}|API: {}|uri: {}|appCode: {}|username: {}|body: {}|queryParams: {}", + requestId, lang, apiName, request.getRequestURI(), appCode, username, desensitizedBody, + desensitizedQueryParams); + } + return true; + } - // hidden sensitive data - jsonBody.set("bk_app_secret", TextNode.valueOf("***")); - desensitizedBody = jsonBody.toString(); + private String desensitizeRequestBody(RepeatableReadWriteHttpServletRequest request) { + if (request.getMethod().equals(HttpMethod.POST.name()) + || request.getMethod().equals(HttpMethod.PUT.name())) { + if (StringUtils.isNotBlank(request.getBody())) { + ObjectNode jsonBody = (ObjectNode) JsonUtils.toJsonNode(request.getBody()); + if (jsonBody == null) { + return null; } - } else if (request.getMethod().equals(HttpMethod.GET.name())) { - username = request.getParameter("bk_username"); - appCode = request.getParameter("bk_app_code"); - desensitizedQueryParams = desensitizeQueryParams(request.getQueryString()); - request.setAttribute(ATTR_USERNAME, username); - request.setAttribute(ATTR_APP_CODE, appCode); + // 由于历史原因，ESB API 的调用方会在 Body 中直接传入 bk_app_secret 这个敏感参数，需要在日志记录的时候脱敏 + if (jsonBody.get("bk_app_secret") != null) { + jsonBody.remove("bk_app_secret"); + } + return jsonBody.toString(); } - } catch (Throwable e) { - return true; - } finally { - log.info("request-id:{}|lang:{}|API:{}|uri:{}|appCode:{}|username:{}|body:{}|queryParams:{}", requestId, - lang, apiName, - request.getRequestURI(), appCode, username, desensitizedBody, desensitizedQueryParams); } - return true; + return null; } private String desensitizeQueryParams(String queryParams) { @@ -137,13 +135,20 @@ public void afterCompletion(HttpServletRequest request, HttpServletResponse resp try { Long startTimeInMills = (Long) request.getAttribute(ATTR_REQUEST_START); String apiName = (String) request.getAttribute(ATTR_API_NAME); - String appCode = (String) request.getAttribute(ATTR_APP_CODE); - String username = (String) request.getAttribute(ATTR_USERNAME); + String username = request.getHeader(JobCommonHeaders.USERNAME); + String appCode = request.getHeader(JobCommonHeaders.APP_CODE); String requestId = request.getHeader(JobCommonHeaders.BK_GATEWAY_REQUEST_ID); int respStatus = response.getStatus(); long cost = System.currentTimeMillis() - startTimeInMills; - log.info("request-id:{}|API:{}|uri:{}|appCode:{}|username:{}|status:{}|resp:{}|cost:{}", requestId, apiName, - request.getRequestURI(), appCode, username, respStatus, wrapperResponse.getBodyAsText(), cost); + log.info("request-id: {}|API: {}|uri: {}|appCode: {}|username: {}|status: {}|resp: {}|cost: {}", + requestId, + apiName, + request.getRequestURI(), + appCode, + username, + respStatus, + StringUtil.substring(wrapperResponse.getBodyAsText(), 10000), + cost); } catch (Throwable e) { log.warn("Handle after completion fail", e); } finally { diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/interceptor/EsbReqRewriteInterceptor.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/interceptor/EsbReqRewriteInterceptor.java deleted file mode 100644 index 7db4312af7..0000000000 --- a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/interceptor/EsbReqRewriteInterceptor.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.web.interceptor; - -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.fasterxml.jackson.databind.node.TextNode; -import com.tencent.bk.job.common.constant.JobCommonHeaders; -import com.tencent.bk.job.common.util.json.JsonUtils; -import com.tencent.bk.job.common.web.model.RepeatableReadWriteHttpServletRequest; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.http.HttpMethod; -import org.springframework.stereotype.Component; -import org.springframework.web.servlet.handler.HandlerInterceptorAdapter; - -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - -@Slf4j -@Component -public class EsbReqRewriteInterceptor extends HandlerInterceptorAdapter { - - @Override - public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) { - if (!(request instanceof RepeatableReadWriteHttpServletRequest)) { - return true; - } - RepeatableReadWriteHttpServletRequest wrapperRequest = (RepeatableReadWriteHttpServletRequest) request; - - String appCode = request.getHeader(JobCommonHeaders.APP_CODE); - String username = request.getHeader(JobCommonHeaders.USERNAME); - - try { - if (request.getMethod().equals(HttpMethod.POST.name()) - || request.getMethod().equals(HttpMethod.PUT.name())) { - if (StringUtils.isNotBlank(wrapperRequest.getBody())) { - ObjectNode jsonBody = (ObjectNode) JsonUtils.toJsonNode(wrapperRequest.getBody()); - if (jsonBody == null) { - return true; - } - // reset username and appCode - if (StringUtils.isNotBlank(username)) { - JsonNode bkUserNameNode = jsonBody.get("bk_username"); - if (bkUserNameNode != null && StringUtils.isNotBlank(bkUserNameNode.asText()) - && !bkUserNameNode.asText().equals(username)) { - log.error("Invalid username, usernameInBody: {}, username: {}", bkUserNameNode.asText(), - username); - } - jsonBody.set("bk_username", new TextNode(username)); - } else { - log.error("Header {} is missing", JobCommonHeaders.USERNAME); - } - - if (StringUtils.isNotBlank(appCode)) { - JsonNode bkAppCodeNode = jsonBody.get("bk_app_code"); - if (bkAppCodeNode != null && StringUtils.isNotBlank(bkAppCodeNode.asText()) - && !bkAppCodeNode.asText().equals(appCode)) { - log.error("Invalid appCode, appCodeInBody: {}, appCode: {}", bkAppCodeNode.asText(), - appCode); - } - jsonBody.set("bk_app_code", new TextNode(appCode)); - } else { - log.error("Header {} is missing", JobCommonHeaders.APP_CODE); - } - - // hidden sensitive data - jsonBody.set("bk_app_secret", TextNode.valueOf("***")); - wrapperRequest.setBody(jsonBody.toString()); - } - } - } catch (Throwable e) { - log.warn("Rewrite EsbReq error", e); - return true; - } - return true; - } -} diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/interceptor/JobCommonInterceptor.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/interceptor/JobCommonInterceptor.java index 0ce8cbbf38..e82d50f4f5 100644 --- a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/interceptor/JobCommonInterceptor.java +++ b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/interceptor/JobCommonInterceptor.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,64 +26,40 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.tencent.bk.job.common.annotation.JobInterceptor; import com.tencent.bk.job.common.constant.HttpRequestSourceEnum; +import com.tencent.bk.job.common.constant.InterceptorOrder; import com.tencent.bk.job.common.constant.JobCommonHeaders; -import com.tencent.bk.job.common.constant.ResourceScopeTypeEnum; import com.tencent.bk.job.common.i18n.locale.LocaleUtils; -import com.tencent.bk.job.common.model.dto.AppResourceScope; -import com.tencent.bk.job.common.model.dto.ResourceScope; -import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.common.util.JobContextUtil; -import com.tencent.bk.job.common.util.feature.FeatureToggle; +import com.tencent.bk.job.common.util.RequestUtil; import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.common.web.model.RepeatableReadWriteHttpServletRequest; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.http.HttpStatus; +import org.slf4j.helpers.MessageFormatter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.cloud.sleuth.Span; import org.springframework.cloud.sleuth.Tracer; -import org.springframework.context.annotation.Lazy; import org.springframework.http.HttpMethod; import org.springframework.lang.NonNull; -import org.springframework.stereotype.Component; import org.springframework.web.servlet.AsyncHandlerInterceptor; import org.springframework.web.servlet.ModelAndView; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import java.util.HashMap; -import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import static com.tencent.bk.job.common.constant.JobConstants.JOB_BUILD_IN_BIZ_SET_ID_MAX; -import static com.tencent.bk.job.common.constant.JobConstants.JOB_BUILD_IN_BIZ_SET_ID_MIN; /** * Job通用拦截器 */ @Slf4j -@Component +@JobInterceptor(order = InterceptorOrder.Init.HIGHEST, pathPatterns = "/**") public class JobCommonInterceptor implements AsyncHandlerInterceptor { - private static final Pattern SCOPE_PATTERN = Pattern.compile("/scope/(\\w+)/(\\d+)"); private final Tracer tracer; private Tracer.SpanInScope spanInScope = null; - private AppScopeMappingService appScopeMappingService; - - /** - * 通过Set方式，同时使用@Lazy，避免Bean循环依赖 - * - * @param appScopeMappingService 依赖的AppScopeMappingService - */ - @Autowired(required = false) - @Lazy - public void setAppScopeMappingService(AppScopeMappingService appScopeMappingService) { - this.appScopeMappingService = appScopeMappingService; - } - @Autowired public JobCommonInterceptor(Tracer tracer) { this.tracer = tracer; @@ -105,7 +81,6 @@ public boolean preHandle(@NonNull HttpServletRequest request, addUsername(request); addLang(request); - addAppResourceScope(request); return true; } @@ -127,8 +102,8 @@ private void initSpanAndAddRequestId() { } private void addUsername(HttpServletRequest request) { - HttpRequestSourceEnum requestSource = getHttpRequestSource(request); - if (requestSource == null) { + HttpRequestSourceEnum requestSource = RequestUtil.parseHttpRequestSource(request); + if (requestSource == HttpRequestSourceEnum.UNKNOWN) { return; } @@ -154,18 +129,6 @@ private void addUsername(HttpServletRequest request) { } } - private HttpRequestSourceEnum getHttpRequestSource(HttpServletRequest request) { - String uri = request.getRequestURI(); - if (uri.startsWith("/web/")) { - return HttpRequestSourceEnum.WEB; - } else if (uri.startsWith("/esb/")) { - return HttpRequestSourceEnum.ESB; - } else if (uri.startsWith("/service/")) { - return HttpRequestSourceEnum.INTERNAL; - } - return null; - } - private void addLang(HttpServletRequest request) { String userLang = request.getHeader(LocaleUtils.COMMON_LANG_HEADER); @@ -176,134 +139,10 @@ private void addLang(HttpServletRequest request) { } } - private void addAppResourceScope(HttpServletRequest request) { - HttpRequestSourceEnum requestSource = getHttpRequestSource(request); - if (requestSource == null) { - return; - } - - AppResourceScope appResourceScope = null; - switch (requestSource) { - case WEB: - appResourceScope = parseAppResourceScopeFromPath(request.getRequestURI()); - log.debug("Scope from path:{}", appResourceScope); - break; - case ESB: - appResourceScope = parseAppResourceScopeFromQueryStringOrBody(request); - log.debug("Scope from query/body:{}", appResourceScope); - } - if (appResourceScope != null) { - request.setAttribute("appResourceScope", appResourceScope); - JobContextUtil.setAppResourceScope(appResourceScope); - } - } - - private AppResourceScope parseAppResourceScopeFromPath(String requestURI) { - ResourceScope resourceScope = parseResourceScopeFromURI(requestURI); - if (resourceScope != null) { - return buildAppResourceScope(resourceScope); - } - - return null; - } - - private ResourceScope parseResourceScopeFromURI(String requestURI) { - ResourceScope resourceScope = null; - Matcher scopeMatcher = SCOPE_PATTERN.matcher(requestURI); - if (scopeMatcher.find()) { - resourceScope = new ResourceScope(scopeMatcher.group(1), scopeMatcher.group(2)); - } - return resourceScope; - } - - private AppResourceScope buildAppResourceScope(ResourceScope resourceScope) { - Long appId = appScopeMappingService.getAppIdByScope(resourceScope); - return new AppResourceScope(appId, resourceScope); - } - private String parseUsernameFromQueryStringOrBody(HttpServletRequest request) { return parseValueFromQueryStringOrBody(request, "bk_username"); } - private AppResourceScope parseAppResourceScopeFromQueryStringOrBody(HttpServletRequest request) { - Map params = parseMultiValueFromQueryStringOrBody(request, "bk_scope_type", "bk_scope_id", - "bk_biz_id"); - String scopeType = params.get("bk_scope_type"); - String scopeId = params.get("bk_scope_id"); - String bizIdStr = params.get("bk_biz_id"); - - if (StringUtils.isNotBlank(scopeType) && StringUtils.isNotBlank(scopeId)) { - return new AppResourceScope(scopeType, scopeId, null); - } - - // 如果兼容bk_biz_id参数 - if (FeatureToggle.isBkBizIdEnabled()) { - // 兼容当前业务ID参数 - if (StringUtils.isNotBlank(bizIdStr)) { - long bizId = Long.parseLong(bizIdStr); - // [8000000,9999999]是迁移业务集之前约定的业务集ID范围。为了兼容老的API调用方，在这个范围内的bizId解析为业务集 - scopeId = bizIdStr; - if (bizId >= JOB_BUILD_IN_BIZ_SET_ID_MIN && bizId <= JOB_BUILD_IN_BIZ_SET_ID_MAX) { - Long appId = appScopeMappingService.getAppIdByScope(ResourceScopeTypeEnum.BIZ_SET.getValue(), - scopeId); - return new AppResourceScope(ResourceScopeTypeEnum.BIZ_SET, scopeId, appId); - } else { - Long appId = appScopeMappingService.getAppIdByScope(ResourceScopeTypeEnum.BIZ.getValue(), scopeId); - return new AppResourceScope(ResourceScopeTypeEnum.BIZ, scopeId, appId); - } - } - } - // 其他情况返回null，后续拦截器会处理null - return null; - } - - /** - * 从请求的解析多个参数 - * - * @param request http请求 - * @param keys 参数名称 - * @return Map - */ - private Map parseMultiValueFromQueryStringOrBody(HttpServletRequest request, String... keys) { - Map params = new HashMap<>(); - try { - if (request.getMethod().equals(HttpMethod.POST.name()) - || request.getMethod().equals(HttpMethod.PUT.name())) { - if (!(request instanceof RepeatableReadWriteHttpServletRequest)) { - return params; - } - RepeatableReadWriteHttpServletRequest wrapperRequest = - (RepeatableReadWriteHttpServletRequest) request; - if (StringUtils.isNotBlank(wrapperRequest.getBody())) { - ObjectNode jsonBody = (ObjectNode) JsonUtils.toJsonNode(wrapperRequest.getBody()); - if (jsonBody == null) { - return params; - } - for (String key : keys) { - JsonNode valueNode = jsonBody.get(key); - String value = (valueNode == null || valueNode.isNull()) ? null : jsonBody.get(key).asText(); - log.debug("Parsed from POST/PUT: {}={}", key, value); - if (value != null) { - params.put(key, value); - } - } - } - } else if (request.getMethod().equals(HttpMethod.GET.name())) { - for (String key : keys) { - String value = request.getParameter(key); - log.debug("Parsed from GET: {}={}", key, value); - if (value != null) { - params.put(key, value); - } - } - } - return params; - } catch (Exception e) { - log.warn("Fail to parse keys: {} from request", keys, e); - } - return params; - } - private String parseValueFromQueryStringOrBody(HttpServletRequest request, String key) { try { if (request.getMethod().equals(HttpMethod.POST.name()) @@ -329,7 +168,11 @@ private String parseValueFromQueryStringOrBody(HttpServletRequest request, Strin return value; } } catch (Exception e) { - log.warn("Fail to parse {} from request", key, e); + String msg = MessageFormatter.format( + "Fail to parse {} from request", + key + ).getMessage(); + log.warn(msg, e); } return null; } @@ -357,15 +200,24 @@ public void afterCompletion(@NonNull HttpServletRequest request, log.warn("status {} given by {}", response.getStatus(), handler); } if (ex != null) { - log.error("After completion|{}|{}|{}|{}|{}|{}|{}", JobContextUtil.getRequestId(), response.getStatus(), - JobContextUtil.getAppResourceScope(), - JobContextUtil.getUsername(), System.currentTimeMillis() - JobContextUtil.getStartTime(), - request.getRequestURI(), ex.getMessage()); + log.error( + "After completion|{}|{}|{}|{}|{}|{}", + JobContextUtil.getRequestId(), + response.getStatus(), + JobContextUtil.getUsername(), + System.currentTimeMillis() - JobContextUtil.getStartTime(), + request.getRequestURI(), + ex.getMessage() + ); } else { - log.debug("After completion|{}|{}|{}|{}|{}|{}", JobContextUtil.getRequestId(), response.getStatus(), - JobContextUtil.getAppResourceScope(), - JobContextUtil.getUsername(), System.currentTimeMillis() - JobContextUtil.getStartTime(), - request.getRequestURI()); + log.debug( + "After completion|{}|{}|{}|{}|{}", + JobContextUtil.getRequestId(), + response.getStatus(), + JobContextUtil.getUsername(), + System.currentTimeMillis() - JobContextUtil.getStartTime(), + request.getRequestURI() + ); } } finally { if (spanInScope != null) { @@ -376,6 +228,6 @@ public void afterCompletion(@NonNull HttpServletRequest request, } private boolean isClientOrServerError(HttpServletResponse response) { - return response.getStatus() > HttpStatus.SC_BAD_REQUEST; + return response.getStatus() >= HttpStatus.SC_BAD_REQUEST; } } diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/interceptor/ServiceSecurityInterceptor.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/interceptor/ServiceSecurityInterceptor.java index 9162095b2d..683301afef 100644 --- a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/interceptor/ServiceSecurityInterceptor.java +++ b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/interceptor/ServiceSecurityInterceptor.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,38 +24,39 @@ package com.tencent.bk.job.common.web.interceptor; -import com.tencent.bk.job.common.util.jwt.JwtManager; -import com.tencent.bk.job.common.web.exception.ServiceNoAuthException; -import com.tencent.bk.job.common.web.util.ProfileUtil; +import com.tencent.bk.job.common.annotation.JobInterceptor; +import com.tencent.bk.job.common.constant.InterceptorOrder; +import com.tencent.bk.job.common.jwt.JwtManager; +import com.tencent.bk.job.common.security.consts.JwtConsts; +import com.tencent.bk.job.common.security.exception.ServiceNoAuthException; +import com.tencent.bk.job.common.service.SpringProfile; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; import org.springframework.web.servlet.handler.HandlerInterceptorAdapter; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; /** - * 服务认证拦截器 + * 服务间调用请求认证拦截器 */ @Slf4j -@Component +@JobInterceptor(pathPatterns = "/**", order = InterceptorOrder.Init.CHECK_VALID) public class ServiceSecurityInterceptor extends HandlerInterceptorAdapter { + private final SpringProfile springProfile; private final JwtManager jwtManager; - private final ProfileUtil profileUtil; @Autowired - public ServiceSecurityInterceptor(JwtManager jwtManager, ProfileUtil profileUtil) { + public ServiceSecurityInterceptor(JwtManager jwtManager, SpringProfile springProfile) { this.jwtManager = jwtManager; - this.profileUtil = profileUtil; + this.springProfile = springProfile; } - @Override public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) { if (shouldFilter(request)) { - String jwt = request.getHeader("x-job-auth-token"); + String jwt = request.getHeader(JwtConsts.HEADER_KEY_SERVICE_JWT_TOKEN); if (StringUtils.isEmpty(jwt)) { log.error("Invalid request, jwt is empty! url: {}", request.getRequestURI()); throw new ServiceNoAuthException(); @@ -69,14 +70,17 @@ public boolean preHandle(HttpServletRequest request, HttpServletResponse respons return true; } - private boolean shouldFilter(HttpServletRequest request) { + public boolean shouldFilter(HttpServletRequest request) { // dev环境需要支持swagger，请求无需认证 - if (profileUtil.isDevProfileActive()) { + if (springProfile.isDevProfileActive()) { return false; } - String uri = request.getRequestURI(); - // 只拦截web/service/esb的API请求 - return uri.startsWith("/web/") || uri.startsWith("/service/") || uri.startsWith("/esb/"); + String uri = request.getServletPath(); + // 只拦截web/service/esb/remote的API请求 + return uri.startsWith("/web/") + || uri.startsWith("/service/") + || uri.startsWith("/esb/") + || uri.startsWith("/remote/"); } } diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/listener/ServletWebServerInitializedEventListener.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/listener/ServletWebServerInitializedEventListener.java new file mode 100644 index 0000000000..67b6adc6d4 --- /dev/null +++ b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/listener/ServletWebServerInitializedEventListener.java @@ -0,0 +1,88 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.web.listener; + +import io.micrometer.core.instrument.MeterRegistry; +import lombok.extern.slf4j.Slf4j; +import org.apache.coyote.ProtocolHandler; +import org.apache.tomcat.util.threads.ThreadPoolExecutor; +import org.springframework.boot.web.embedded.tomcat.TomcatWebServer; +import org.springframework.boot.web.server.WebServer; +import org.springframework.boot.web.servlet.context.ServletWebServerInitializedEvent; +import org.springframework.context.ApplicationListener; + +import java.util.concurrent.Executor; + +/** + * 监听WebServer初始化完成事件，执行注册线程池监控指标等动作 + */ +@Slf4j +public class ServletWebServerInitializedEventListener implements ApplicationListener { + + private final MeterRegistry registry; + + public ServletWebServerInitializedEventListener(MeterRegistry registry) { + this.registry = registry; + } + + @Override + public void onApplicationEvent(ServletWebServerInitializedEvent event) { + log.info("ServletWebServerInitializedEvent caught"); + WebServer webServer = event.getSource(); + initTomcatExecutorMetrics(webServer); + } + + /** + * 注册Tomcat线程池监控指标 + * + * @param webServer Web服务器 + */ + private void initTomcatExecutorMetrics(WebServer webServer) { + if (!(webServer instanceof TomcatWebServer)) { + log.info("Unknown web server type: {}, ignore tomcat executor metrics", webServer.getClass().getName()); + return; + } + TomcatWebServer tomcatWebServer = (TomcatWebServer) webServer; + ProtocolHandler protocolHandler = tomcatWebServer.getTomcat().getConnector().getProtocolHandler(); + log.debug("protocolHandler: {}", protocolHandler); + if (protocolHandler == null) { + return; + } + Executor executor = protocolHandler.getExecutor(); + log.debug("executor: {}", executor); + if (executor == null) { + return; + } + if (executor instanceof ThreadPoolExecutor) { + ThreadPoolExecutor threadExecutor = (ThreadPoolExecutor) executor; + registry.gauge("tomcat.threads.max", threadExecutor, ThreadPoolExecutor::getMaximumPoolSize); + registry.gauge("tomcat.threads.current", threadExecutor, ThreadPoolExecutor::getPoolSize); + registry.gauge("tomcat.threads.busy", threadExecutor, ThreadPoolExecutor::getActiveCount); + log.info("Tomcat thread pool metrics inited"); + } else { + log.warn("Unknown executor type: {}, ignore tomcat executor metrics", executor.getClass().getName()); + } + } +} diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/metrics/CustomTimed.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/metrics/CustomTimed.java index dacea5f45c..780967efca 100644 --- a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/metrics/CustomTimed.java +++ b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/metrics/CustomTimed.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/metrics/CustomTimedTagsContributor.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/metrics/CustomTimedTagsContributor.java index 41e0e8c9d5..7b0a05804e 100644 --- a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/metrics/CustomTimedTagsContributor.java +++ b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/metrics/CustomTimedTagsContributor.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/metrics/CustomTimedTagsProvider.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/metrics/CustomTimedTagsProvider.java index a3bf3a016f..caf91af256 100644 --- a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/metrics/CustomTimedTagsProvider.java +++ b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/metrics/CustomTimedTagsProvider.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/metrics/DefaultCustomTimedTagsProvider.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/metrics/DefaultCustomTimedTagsProvider.java index 0287d77297..6095c4cbb5 100644 --- a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/metrics/DefaultCustomTimedTagsProvider.java +++ b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/metrics/DefaultCustomTimedTagsProvider.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/model/RepeatableReadHttpServletResponse.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/model/RepeatableReadHttpServletResponse.java index fc7a7c5df5..f28ca1982a 100644 --- a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/model/RepeatableReadHttpServletResponse.java +++ b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/model/RepeatableReadHttpServletResponse.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -42,8 +42,8 @@ * 由于HttpServletRequest的InputStream只允许读取一次，为了能够重复读取body，需要用装饰模式来包装 */ public class RepeatableReadHttpServletResponse extends HttpServletResponseWrapper { - private ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); - private HttpServletResponse response; + private final ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + private final HttpServletResponse response; public RepeatableReadHttpServletResponse(HttpServletResponse response) { super(response); @@ -71,9 +71,7 @@ public PrintWriter getWriter() throws IOException { @Override public void flushBuffer() throws IOException { - if (byteArrayOutputStream != null) { - byteArrayOutputStream.flush(); - } + byteArrayOutputStream.flush(); } @Data diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/model/RepeatableReadWriteHttpServletRequest.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/model/RepeatableReadWriteHttpServletRequest.java index c627dde9a6..6ef79cdafc 100644 --- a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/model/RepeatableReadWriteHttpServletRequest.java +++ b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/model/RepeatableReadWriteHttpServletRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/util/ProfileUtil.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/util/ProfileUtil.java deleted file mode 100644 index 48dd4b302a..0000000000 --- a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/util/ProfileUtil.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.web.util; - -import org.apache.commons.lang3.ArrayUtils; -import org.springframework.beans.BeansException; -import org.springframework.context.ApplicationContext; -import org.springframework.context.ApplicationContextAware; -import org.springframework.stereotype.Component; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; - -@Component -public class ProfileUtil implements ApplicationContextAware { - - private static ApplicationContext context = null; - private transient Boolean isDevProfile; - - - @Override - public void setApplicationContext(ApplicationContext applicationContext) - throws BeansException { - context = applicationContext; - } - - public List getActiveProfiles() { - String[] profiles = context.getEnvironment().getActiveProfiles(); - if (!ArrayUtils.isEmpty(profiles)) { - return Arrays.asList(profiles); - } - return new ArrayList<>(); - } - - public boolean isProfileActive(String profile) { - String[] activeProfiles = context.getEnvironment().getActiveProfiles(); - for (String activeProfile : activeProfiles) { - if (activeProfile.equals(profile)) { - return true; - } - } - return false; - } - - public boolean isDevProfileActive() { - if (isDevProfile != null) { - return isDevProfile; - } - isDevProfile = isProfileActive("dev") || isProfileActive("local"); - return isDevProfile; - } - - -} diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/utils/ServletUtil.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/utils/ServletUtil.java new file mode 100644 index 0000000000..ddc919a5df --- /dev/null +++ b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/utils/ServletUtil.java @@ -0,0 +1,53 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.web.utils; + +import org.apache.commons.lang3.StringUtils; + +import javax.servlet.ServletRequest; + + +/** + * 处理servlet请求和响应的工具类 + */ +public class ServletUtil { + + /** + * 判断 servlet 请求的 Content-Type 是否是 application/json + * @param request 请求 + * @return 请求的 Content-Type 是否是 application/json + */ + public static boolean isJsonRequest(ServletRequest request) { + return isJsonContentType(request.getContentType()); + } + + private static boolean isJsonContentType(String contentType) { + if (StringUtils.isBlank(contentType)) { + return false; + } + contentType = contentType.trim().toLowerCase(); + return contentType.startsWith("application/json"); + } +} diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/validation/JacksonPropertyNodeNameProvider.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/validation/JacksonPropertyNodeNameProvider.java index ce267be26c..653628fd88 100644 --- a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/validation/JacksonPropertyNodeNameProvider.java +++ b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/validation/JacksonPropertyNodeNameProvider.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/validation/ValidationConfiguration.java b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/validation/ValidationConfiguration.java index 6a1f05acba..e7fa8dc0e8 100644 --- a/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/validation/ValidationConfiguration.java +++ b/src/backend/commons/common-web/src/main/java/com/tencent/bk/job/common/web/validation/ValidationConfiguration.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB 蓝鲸作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB 蓝鲸作业平台 is licensed under the MIT license. * @@ -35,7 +35,6 @@ import org.springframework.context.MessageSource; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; import org.springframework.context.i18n.LocaleContextHolder; import org.springframework.core.LocalVariableTableParameterNameDiscoverer; import org.springframework.core.ParameterNameDiscoverer; @@ -55,12 +54,11 @@ import java.util.Locale; import java.util.Set; -@Configuration +@Configuration(proxyBeanMethods = false) @Slf4j public class ValidationConfiguration { - @Bean - @Primary + @Bean("jobLocalValidatorFactoryBean") public LocalValidatorFactoryBean localValidatorFactoryBean( @Qualifier("messageSource") MessageSource messageSource) { LocalValidatorFactoryBean localValidatorFactoryBean = new JobLocalValidatorFactoryBean(messageSource); diff --git a/src/backend/commons/common-web/src/main/resources/META-INF/spring.factories b/src/backend/commons/common-web/src/main/resources/META-INF/spring.factories index 7b95dbdfb5..eb1d70f359 100644 --- a/src/backend/commons/common-web/src/main/resources/META-INF/spring.factories +++ b/src/backend/commons/common-web/src/main/resources/META-INF/spring.factories @@ -1,2 +1,8 @@ org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ -com.tencent.bk.job.common.web.config.CustomTimedMetricsAutoConfiguration +com.tencent.bk.job.common.web.config.CustomTimedMetricsAutoConfiguration,\ +com.tencent.bk.job.common.web.config.WebAutoConfiguration,\ +com.tencent.bk.job.common.web.config.ExceptionAdviceAutoConfiguration,\ +com.tencent.bk.job.common.web.config.WebInterceptorAutoConfiguration,\ +com.tencent.bk.job.common.web.config.WebInterceptorRegisterAutoConfiguration,\ +com.tencent.bk.job.common.web.config.I18nAutoConfiguration,\ +com.tencent.bk.job.common.web.config.EsbAppReqAspectAutoConfiguration diff --git a/src/backend/commons/common/build.gradle b/src/backend/commons/common/build.gradle index 9bebb31084..e0f352c688 100644 --- a/src/backend/commons/common/build.gradle +++ b/src/backend/commons/common/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,13 +25,16 @@ dependencies { api project(':commons:common-i18n') api project(':commons:common-utils') + api 'org.springframework.boot:spring-boot-autoconfigure' api 'com.fasterxml.jackson.core:jackson-core' api 'com.fasterxml.jackson.core:jackson-databind' api 'com.fasterxml.jackson.core:jackson-annotations' + api 'com.fasterxml.jackson.datatype:jackson-datatype-jsr310' api 'org.apache.commons:commons-lang3' api 'org.apache.commons:commons-collections4' api 'org.springframework.boot:spring-boot' - implementation 'io.springfox:springfox-swagger2' + api 'com.google.guava:guava' + implementation 'io.springfox:springfox-boot-starter' implementation 'net.sf.dozer:dozer' implementation 'net.sourceforge.jchardet:jchardet' api 'org.apache.httpcomponents:httpclient' @@ -41,6 +44,8 @@ dependencies { implementation 'io.micrometer:micrometer-registry-prometheus' implementation 'com.cronutils:cron-utils' implementation 'commons-validator:commons-validator' + implementation 'org.springframework.cloud:spring-cloud-sleuth-instrumentation' + implementation 'com.tencent.bk.sdk:crypto-java-sdk' compileOnly 'org.springframework:spring-web' compileOnly 'org.projectlombok:lombok' annotationProcessor 'org.projectlombok:lombok' diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/RequestIdLogger.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/RequestIdLogger.java deleted file mode 100644 index e72eb34bba..0000000000 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/RequestIdLogger.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common; - -import org.slf4j.Logger; - -/** - * @Description - * @Date 2019/12/24 - * @Version 1.0 - */ -public interface RequestIdLogger extends Logger { - void debugWithRequestId(String message); - - void infoWithRequestId(String message); - - void warnWithRequestId(String message); - - void errorWithRequestId(String message); -} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/VersionInfoLogApplicationRunner.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/VersionInfoLogApplicationRunner.java new file mode 100644 index 0000000000..9635f7765f --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/VersionInfoLogApplicationRunner.java @@ -0,0 +1,49 @@ +/* + * + * * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * * + * * Copyright (C) 2021 Tencent. All rights reserved. + * * + * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * * + * * License for BK-JOB蓝鲸智云作业平台: + * * -------------------------------------------------------------------- + * * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * * + * * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * * the Software. + * * + * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * * IN THE SOFTWARE. + * + */ + +package com.tencent.bk.job.common; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.ApplicationArguments; +import org.springframework.boot.ApplicationRunner; +import org.springframework.boot.info.BuildProperties; + +@Slf4j +public class VersionInfoLogApplicationRunner implements ApplicationRunner { + private final String serviceName; + private final BuildProperties buildProperties; + + public VersionInfoLogApplicationRunner(String serviceName, BuildProperties buildProperties) { + this.serviceName = serviceName; + this.buildProperties = buildProperties; + } + + @Override + public void run(ApplicationArguments args) { + String version = buildProperties.getVersion(); + log.info("serviceName= {}, version= {}", serviceName, version); + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/WatchableThreadPoolExecutor.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/WatchableThreadPoolExecutor.java new file mode 100644 index 0000000000..ddd7d22679 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/WatchableThreadPoolExecutor.java @@ -0,0 +1,180 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common; + +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tag; +import io.micrometer.core.instrument.Timer; +import lombok.extern.slf4j.Slf4j; + +import java.util.Collections; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.RejectedExecutionHandler; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +@Slf4j +public class WatchableThreadPoolExecutor extends ThreadPoolExecutor { + /** + * 任务开始时间 + */ + private final ThreadLocal startTimeThreadLocal = new ThreadLocal<>(); + + private MeterRegistry meterRegistry; + + private List tags; + + public WatchableThreadPoolExecutor(MeterRegistry meterRegistry, + String poolName, + int corePoolSize, + int maximumPoolSize, + long keepAliveTime, + TimeUnit unit, + BlockingQueue workQueue) { + super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue); + init(poolName, meterRegistry); + } + + public WatchableThreadPoolExecutor(MeterRegistry meterRegistry, + String poolName, + int corePoolSize, + int maximumPoolSize, + long keepAliveTime, + TimeUnit unit, + BlockingQueue workQueue, + ThreadFactory threadFactory) { + super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory); + init(poolName, meterRegistry); + } + + public WatchableThreadPoolExecutor(MeterRegistry meterRegistry, + String poolName, + int corePoolSize, + int maximumPoolSize, + long keepAliveTime, + TimeUnit unit, + BlockingQueue workQueue, + RejectedExecutionHandler handler) { + super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, handler); + init(poolName, meterRegistry); + } + + public WatchableThreadPoolExecutor(MeterRegistry meterRegistry, + String poolName, + int corePoolSize, + int maximumPoolSize, + long keepAliveTime, + TimeUnit unit, + BlockingQueue workQueue, + ThreadFactory threadFactory, + RejectedExecutionHandler handler) { + super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue, threadFactory, handler); + init(poolName, meterRegistry); + } + + private void init(String poolName, MeterRegistry meterRegistry) { + this.tags = Collections.singletonList(Tag.of("pool_name", poolName)); + this.meterRegistry = meterRegistry; + startWatch(); + } + + private void startWatch() { + meterRegistry.gauge("job_thread_pool_active_thread_size", + getTags(), + this, + threadPoolExecutor -> (double) threadPoolExecutor.getActiveCount()); + meterRegistry.gauge("job_thread_pool_pool_size", + getTags(), + this, + threadPoolExecutor -> (double) threadPoolExecutor.getPoolSize()); + meterRegistry.gauge("job_thread_pool_core_pool_size", + getTags(), + this, + threadPoolExecutor -> (double) threadPoolExecutor.getCorePoolSize()); + meterRegistry.gauge("job_thread_pool_max_pool_size", + getTags(), + this, + threadPoolExecutor -> (double) threadPoolExecutor.getMaximumPoolSize()); + meterRegistry.gauge("job_thread_pool_task_total", + getTags(), + this, + threadPoolExecutor -> (double) threadPoolExecutor.getTaskCount()); + meterRegistry.gauge("job_thread_pool_completed_task_total", + getTags(), + this, + threadPoolExecutor -> (double) threadPoolExecutor.getCompletedTaskCount()); + meterRegistry.gauge("job_thread_pool_largest_pool_size", + getTags(), + this, + threadPoolExecutor -> (double) threadPoolExecutor.getLargestPoolSize()); + meterRegistry.gauge("job_thread_pool_queue_size", + getTags(), + this, + threadPoolExecutor -> (double) (threadPoolExecutor.getQueue() == null ? 0 : + threadPoolExecutor.getQueue().size())); + } + + private List getTags() { + return tags; + } + + + /** + * 任务执行之前，记录任务开始时间 + */ + @Override + protected void beforeExecute(Thread t, Runnable r) { + startTimeThreadLocal.set(System.currentTimeMillis()); + } + + /** + * 任务执行之后，计算任务结束时间 + */ + @Override + protected void afterExecute(Runnable r, Throwable t) { + try { + long costTime = System.currentTimeMillis() - startTimeThreadLocal.get(); + recordTaskCostTime(costTime); + } finally { + startTimeThreadLocal.remove(); + } + } + + + private void recordTaskCostTime(long costInMills) { + try { + Timer.builder("job_thread_pool_tasks") + .tags(getTags()) + .publishPercentileHistogram(true) + .register(meterRegistry) + .record(costInMills, TimeUnit.MILLISECONDS); + } catch (Exception e) { + log.warn("Fail to record thread pool task timer metrics", e); + } + } + +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/annotation/CompatibleImplementation.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/annotation/CompatibleImplementation.java index d66af9c77a..ab0c85cc8a 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/annotation/CompatibleImplementation.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/annotation/CompatibleImplementation.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,10 @@ package com.tencent.bk.job.common.annotation; +import com.tencent.bk.job.common.constant.CompatibleType; + import java.lang.annotation.Documented; +import java.lang.annotation.Inherited; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; import java.lang.annotation.Target; @@ -38,10 +41,11 @@ import static java.lang.annotation.ElementType.TYPE; /** - * 兼容历史版本的实现 + * 用来标识兼容历史版本的实现 */ @Documented @Retention(RetentionPolicy.RUNTIME) +@Inherited @Target(value = {CONSTRUCTOR, FIELD, LOCAL_VARIABLE, METHOD, PACKAGE, PARAMETER, TYPE}) public @interface CompatibleImplementation { /** @@ -50,12 +54,17 @@ String name() default ""; /** - * 说明 + * 兼容说明 */ - String explain(); + String explain() default ""; /** * 兼容实现被废除的版本 */ - String version(); + String deprecatedVersion() default ""; + + /** + * 兼容类型 + */ + CompatibleType type(); } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/annotation/EsbAPI.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/annotation/EsbAPI.java index 2106f9d7ec..cd9c254b8d 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/annotation/EsbAPI.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/annotation/EsbAPI.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB 蓝鲸作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB 蓝鲸作业平台 is licensed under the MIT license. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/annotation/IamCallbackAPI.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/annotation/IamCallbackAPI.java index 1de4c4d0dd..6faaac243b 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/annotation/IamCallbackAPI.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/annotation/IamCallbackAPI.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/annotation/InternalAPI.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/annotation/InternalAPI.java index 38e36385bc..c2c38f33f9 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/annotation/InternalAPI.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/annotation/InternalAPI.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/annotation/JobInterceptor.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/annotation/JobInterceptor.java new file mode 100644 index 0000000000..44949783e1 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/annotation/JobInterceptor.java @@ -0,0 +1,49 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.annotation; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * 注解 Job 自定义的 Interceptor + */ +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +@Documented +public @interface JobInterceptor { + /** + * 拦截器注册 ORDER + */ + int order() default 0; + + /** + * 拦截路径正则 + */ + String[] pathPatterns() default {}; +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/annotation/PersistenceObject.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/annotation/PersistenceObject.java index 14cd6fec0f..e02c5851d8 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/annotation/PersistenceObject.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/annotation/PersistenceObject.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB 蓝鲸作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB 蓝鲸作业平台 is licensed under the MIT license. * @@ -33,7 +33,7 @@ import java.lang.annotation.Target; /** - * 用于表示某个持久对象，该对象会被持久化到DB中,所以对于该对象的改动需要谨慎，避免破坏性的修改(比如修改字段名) + * 用于标识持久化的对象。该对象会被持久化到DB中,所以对于该对象的改动需要谨慎，避免破坏性的修改(比如修改字段名) */ @Target(ElementType.TYPE) @Retention(RetentionPolicy.RUNTIME) diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/annotation/RemoteAPI.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/annotation/RemoteAPI.java new file mode 100644 index 0000000000..84ca81790f --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/annotation/RemoteAPI.java @@ -0,0 +1,36 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ +package com.tencent.bk.job.common.annotation; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +@Documented +public @interface RemoteAPI { +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/annotation/WebAPI.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/annotation/WebAPI.java index 4799a62560..04c1635649 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/annotation/WebAPI.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/annotation/WebAPI.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB 蓝鲸作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB 蓝鲸作业平台 is licensed under the MIT license. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/annotation/WorkerAPI.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/annotation/WorkerAPI.java new file mode 100644 index 0000000000..7e4882663e --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/annotation/WorkerAPI.java @@ -0,0 +1,36 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ +package com.tencent.bk.job.common.annotation; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +@Documented +public @interface WorkerAPI { +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/config/BkConfig.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/config/BkConfig.java new file mode 100644 index 0000000000..de28aa5207 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/config/BkConfig.java @@ -0,0 +1,26 @@ +package com.tencent.bk.job.common.config; + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +/** + * 蓝鲸公共配置 + */ +@Getter +@Setter +@ToString +@ConfigurationProperties(prefix = "bk") +@Configuration("bkCommonConfig") +public class BkConfig { + /** + * 蓝鲸根域名。指蓝鲸产品公共 cookies 写入的目录，同时也是各个系统的公共域名部分。 + */ + private String bkDomain; + /** + * BK助手链接。 + */ + private String bkHelperLink = ""; +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/config/FeatureToggleConfig.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/config/FeatureToggleConfig.java deleted file mode 100644 index f6b6b0f893..0000000000 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/config/FeatureToggleConfig.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.config; - -import com.tencent.bk.job.common.util.json.JsonUtils; -import lombok.Getter; -import lombok.Setter; -import lombok.ToString; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; -import org.springframework.boot.context.properties.ConfigurationProperties; - -import javax.annotation.PostConstruct; -import java.util.List; - -@ConfigurationProperties(prefix = "job.feature") -@ToString -@Getter -@Setter -@Slf4j -public class FeatureToggleConfig { - /** - * 是否兼容ESB bk_biz_id 参数开关 - */ - private ToggleConfig esbApiParamBkBizId = new ToggleConfig(); - - - /** - * 特性开关配置 - */ - @ToString - @Setter - public static class ToggleConfig { - /** - * 特性开关是否开启 - */ - private boolean enabled = true; - /** - * 是否支持灰度 - */ - private boolean gray; - /** - * 灰度-白名单 - */ - private List include; - /** - * 灰度-黑名单 - */ - private List exclude; - - /** - * 判断是否开启特性 - 灰度 - */ - public boolean isOpen(String value) { - if (!enabled) { - // 特性开关未开启 - return false; - } - if (!gray) { - // 未开启灰度，全量开启 - return true; - } - // 如果配置exclude参数，需要优先判断灰度黑名单 - if (CollectionUtils.isNotEmpty(exclude) && exclude.contains(value)) { - return false; - } - // 是否在灰度白名单中 - return CollectionUtils.isNotEmpty(include) && include.contains(value); - } - - /** - * 判断是否开启特性 - */ - public boolean isOpen() { - return enabled; - } - - } - - @PostConstruct - public void print() { - log.info("FeatureToggleConfig init: {}", JsonUtils.toJson(this)); - } - -} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/config/HttpConfig.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/config/HttpConfig.java deleted file mode 100644 index 5133e03f91..0000000000 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/config/HttpConfig.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.config; - -import com.tencent.bk.job.common.util.http.HttpHelperFactory; -import io.micrometer.core.instrument.MeterRegistry; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Slf4j -@Configuration -public class HttpConfig { - - @Bean - public HttpConfigSetter httpConfigSetter(@Autowired MeterRegistry meterRegistry) { - HttpHelperFactory.setMeterRegistry(meterRegistry); - log.info("meterRegistry for HttpHelperFactory init"); - return new HttpConfigSetter(); - } - - static class HttpConfigSetter { - HttpConfigSetter() { - } - } -} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/account/AccountCategoryEnum.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/AccountCategoryEnum.java similarity index 89% rename from src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/account/AccountCategoryEnum.java rename to src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/AccountCategoryEnum.java index cc2042c456..01305038ce 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/account/AccountCategoryEnum.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/AccountCategoryEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,12 +22,10 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.manage.common.consts.account; +package com.tencent.bk.job.common.constant; /** * 账号分类 - * - * @date 2019/09/19 */ public enum AccountCategoryEnum { SYSTEM(1, "system"), @@ -53,6 +51,13 @@ public static AccountCategoryEnum valOf(Integer category) { return null; } + public static boolean isValid(Integer category) { + if (category == null) { + return false; + } + return valOf(category) != null; + } + public Integer getValue() { return value; } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/ArchiveFlag.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/ArchiveFlag.java new file mode 100644 index 0000000000..f75f5f72bd --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/ArchiveFlag.java @@ -0,0 +1,43 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.constant; + +/** + * 数据归档状态 + */ +public enum ArchiveFlag { + HOT(1), + COLD(2); + + ArchiveFlag(int value) { + this.value = value; + } + + private final int value; + + public int getValue() { + return value; + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/Bool.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/Bool.java index 46e43bf79c..0256d38099 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/Bool.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/Bool.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -38,7 +38,7 @@ public static Bool from(Byte value) { return FALSE; } for (Bool boolEnum : values()) { - if (boolEnum.getValue() == value) { + if (boolEnum.byteValue() == value) { return boolEnum; } } @@ -53,10 +53,14 @@ public static boolean isTrue(Byte value) { if (value == null) { return false; } - return value.equals(Bool.TRUE.getValue()); + return value.equals(Bool.TRUE.byteValue()); } - public byte getValue() { + public byte byteValue() { + return this.value; + } + + public int intValue() { return this.value; } } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/CcNodeTypeEnum.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/CcNodeTypeEnum.java index 1d640323fe..c1e6de9440 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/CcNodeTypeEnum.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/CcNodeTypeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -31,8 +31,6 @@ /** * CMDB通用topo节点类型（不包含用户自定义topo节点类型） - * - * @since 10/12/2019 16:11 */ @Getter @AllArgsConstructor @@ -40,21 +38,18 @@ public enum CcNodeTypeEnum { /** * 模块 */ - MODULE("module", "模块"), + MODULE("module"), /** * 集群 */ - SET("set", "集群"), + SET("set"), /** * 业务 */ - BIZ("biz", "业务"), - ; + BIZ("biz"); @JsonValue - private String type; - - private String name; + private final String type; @JsonCreator(mode = JsonCreator.Mode.DELEGATING) public static CcNodeTypeEnum parseString(String type) { diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/CompatibleType.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/CompatibleType.java new file mode 100644 index 0000000000..e74cfdba75 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/CompatibleType.java @@ -0,0 +1,50 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.constant; + +/** + * 兼容类型 + */ +public enum CompatibleType { + /** + * 历史数据兼容。历史数据失效之后可删除 + */ + HISTORY_DATA, + /** + * 历史业务逻辑兼容。该业务逻辑不会再次启用之后可删除 + */ + HISTORY_LOGIC, + /** + * API 协议兼容。协议正式废弃之后可删除 + */ + API, + /** + * 发布兼容。发布完成后可删除 + */ + DEPLOY; + + CompatibleType() { + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/CronJobNotifyType.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/CronJobNotifyType.java new file mode 100644 index 0000000000..5b5cae4d60 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/CronJobNotifyType.java @@ -0,0 +1,46 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.constant; + +import lombok.Getter; + +@Getter +public enum CronJobNotifyType { + /** + * 继承业务 + */ + EXTENDS_APP(1), + + /** + * 自定义通知方式 + */ + CUSTOM(2); + + private final Integer type; + + CronJobNotifyType(Integer type) { + this.type = type; + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/CronJobTypeEnum.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/CronJobTypeEnum.java index d7656b1ee7..a92f868f60 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/CronJobTypeEnum.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/CronJobTypeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/DuplicateHandlerEnum.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/DuplicateHandlerEnum.java index 3d3bf6bbac..adf31c7256 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/DuplicateHandlerEnum.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/DuplicateHandlerEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/ErrorCode.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/ErrorCode.java index 1bbe66b1a0..8dbe8ab61b 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/ErrorCode.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/ErrorCode.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -38,6 +38,98 @@ public class ErrorCode { */ public static final int RESULT_OK = 0; + // ==================================== 系统级错误 ================================================// + + // ======= 系统错误-平台服务错误 =======// + // LICENSE 不可用 + public static final int LICENSE_ERROR = 1210101; + // GSE 不可用 + public static final int GSE_ERROR = 1210001; + // GSE数据异常：{0} + public static final int GSE_API_DATA_ERROR = 1210002; + + // CMDB错误 + // CMDB服务状态不可达 - 地址配置错误或者地址无法正确解析 + public static final int CMDB_UNREACHABLE_SERVER = 1211001; + // CMDB接口返回数据结构异常- 一般是被网关防火墙重定向返回统一登录页面 + public static final int CMDB_API_DATA_ERROR = 1211002; + // 根据动态分组ID查找主机失败，动态分组ID：{0}，原因：{1}，请确认指定的动态分组在业务下是否存在 + public static final int FAIL_TO_FIND_HOST_BY_DYNAMIC_GROUP = 1211003; + // 根据业务ID查找动态分组失败，业务ID：{0}，原因：{1}，请确认指定的业务是否存在动态分组 + public static final int FAIL_TO_FIND_DYNAMIC_GROUP_BY_BIZ = 1211004; + + // PaaS异常 + // CMSI接口访问异常 + public static final int CMSI_API_ACCESS_ERROR = 1213001; + // 用户管理接口访问异常 + public static final int USER_MANAGE_API_ACCESS_ERROR = 1213002; + // 调用CMSI接口获取通知渠道数据异常 + public static final int CMSI_MSG_CHANNEL_DATA_ERROR = 1213003; + // 调用CMSI接口发送通知失败，错误码：{0}，错误信息：{1} + public static final int CMSI_FAIL_TO_SEND_MSG = 1213004; + + // 制品库异常 + // Artifactory接口返回数据结构异常 + public static final int ARTIFACTORY_API_DATA_ERROR = 1214001; + // 制品库中找不到节点:{0}，请到制品库核实 + public static final int CAN_NOT_FIND_NODE_IN_ARTIFACTORY = 1214002; + + // IAM接口数据异常- 一般是被网关防火墙重定向返回统一登录页面 + public static final int IAM_API_DATA_ERROR = 1215001; + // 用户({0})账号已被冻结，请处理后再重试 + public static final int IAM_USER_ACCOUNT_FROZEN = 1215002; + + // 第三方API请求错误 + public static final int API_ERROR = 1216001; + + // 消息通知中心异常 + // 消息通知中心API不存在:{0} + public static final int BK_NOTICE_API_NOT_FOUND = 1217001; + // 消息通知中心接口数据异常 + public static final int BK_NOTICE_API_DATA_ERROR = 1217002; + + // AIDev平台异常 + // AIDev接口数据异常 + public static final int BK_AI_DEV_API_DATA_ERROR = 1218001; + // 蓝鲸OpenAI接口数据异常 + public static final int BK_OPEN_AI_API_DATA_ERROR = 1218002; + // 蓝鲸OpenAI接口数据超时 + public static final int BK_OPEN_AI_API_DATA_TIMEOUT = 1218003; + + // ======== 系统错误-权限错误 ==================// + // 用户({0})权限不足，请前往权限中心确认并申请补充后重试 + public static final int PERMISSION_DENIED = 1238001; + // 蓝鲸统一权限错误码，用户({0})权限不足，请前往权限中心确认并申请补充后重试 + public static final int BK_PERMISSION_DENIED = 9900403; + + // ========= 系统错误-请求 ====================// + // 内部服务异常 + public static final int INTERNAL_ERROR = 1240002; + // 错误的请求 + public static final int BAD_REQUEST = 1240003; + // Cookie过期或者不存在 + public static final int COOKIE_ILLEGAL = 1240004; + // 服务不可用 + public static final int SERVICE_UNAVAILABLE = 1240001; + // 服务认证失败 + public static final int SERVICE_AUTH_FAIL = 1240005; + // 配置异常：{0} + public static final int INVALID_CONFIG = 1240006; + // 命令行参数异常：{0} + public static final int INVALID_CMD_ARGS = 1240007; + + // ======= 系统错误-公共组件错误 =======// + // Redis服务不可用，连接不上 - IP不存在或者配置错误 + public static final int REDIS_CONNECT_FAIL = 1250001; + // Redis服务内存满或者其他问题 - 内存不足够 + public static final int REDIS_DATA_EXCEPTION = 1250002; + // DB 不可用 + public static final int DB_ERROR = 1252001; + // MQ 不可用 + public static final int MQ_ERROR = 1255001; + // NFS存储 不可用 + public static final int NFS_ERROR = 1259001; + // ==================================== 业务级错误 ================================================// /* * 业务通用-1241xxx @@ -46,7 +138,7 @@ public class ErrorCode { * 作业执行-1244xxx * 定时作业-1245xxx * 日志服务-1246xxx - * 用户服务-1247xxx + * 网关服务-1247xxx * 业务网关-1248xxx * 备份服务-1249xxx * 文件网关-1260xxx @@ -77,6 +169,14 @@ public class ErrorCode { public static final int NOT_SUPPORT_FEATURE = 1241011; public static final int ILLEGAL_PARAM_WITH_REASON = 1241012; + // 该功能暂不支持业务集 + public static final int NOT_SUPPORT_FEATURE_FOR_BIZ_SET = 1241013; + // IPv6地址不合法：{0} + public static final int INVALID_IPV6_ADDRESS = 1241014; + // 不支持的操作，请勿重复启动作业：{0} + public static final int UNSUPPORTED_OPERATION_REPEAT_START_JOB = 1241015; + // 非法文件 + public static final int ILLEGAL_FILE = 1241016; // 业务通用 end // 配置服务 start @@ -183,12 +283,20 @@ public class ErrorCode { public static final int UPLOAD_FILE_SUFFIX_NOT_ALLOW = 1243050; // 资源范围不存在:{0} public static final int SCOPE_NOT_EXIST = 1243051; + // IP（含云区域ID）在CMDB中不存在:{0} + public static final int IP_NOT_EXIST_IN_CMDB = 1243052; + // 主机ID在CMDB中不存在:{0} + public static final int HOST_ID_NOT_EXIST_IN_CMDB = 1243053; + // 脚本被引用不支持删除 + public static final int DELETE_REF_SCRIPT_FAIL = 1243054; + // 凭证被引用不支持删除 + public static final int DELETE_REF_CREDENTIAL_FAIL = 1243055; // 作业管理 end // 作业执行 start // 主机为空 public static final int SERVER_EMPTY = 1244001; - // 主机[{0}]无效，请检查源或目标主机是否存在于配置平台；另外，主机跨业务需要联系作业平台管理员先将其添加到IP白名单。 + // 主机无效，请检查源或目标主机的是否存在于配置平台的业务[{0}]下；另外，主机需跨业务执行请联系作业平台管理员将其添加到主机白名单。无效的{1}台主机：[{2}] public static final int HOST_INVALID = 1244002; // 账号不存在 public static final int ACCOUNT_NOT_EXIST = 1244003; @@ -238,16 +346,36 @@ public class ErrorCode { public static final int INVALID_ROLLING_EXPR = 1244026; // 滚动批次不能大于{} public static final int EXCEED_MAX_ALLOWED_BATCH_SIZE = 1244027; + // 步骤：{} 的目标执行对象为空 + public static final int STEP_TARGET_EXECUTE_OBJECT_EMPTY = 1244028; + // 步骤：{} 的源文件执行对象为空 + public static final int STEP_SOURCE_EXECUTE_OBJECT_EMPTY = 1244029; + // 执行对象不存在。无效的{0}个执行对象：[{1}] + public static final int EXECUTE_OBJECT_NOT_EXIST = 1244030; + public static final int INVALID_LABEL_SELECTOR = 1244031; + // 当前执行的作业总量超过业务配额限制 + public static final int RUNNING_JOB_EXCEED_RESOURCE_SCOPE_QUOTA_LIMIT = 1244032; + // 当前执行的作业总量超过应用配额限制 + public static final int RUNNING_JOB_EXCEED_APP_QUOTA_LIMIT = 1244033; + // 当前执行的作业总量超过系统配额限制 + public static final int RUNNING_JOB_EXCEED_SYSTEM_QUOTA_LIMIT = 1244034; // 作业执行 end // 定时作业 start + // 定时任务不存在 public static final int CRON_JOB_NOT_EXIST = 1245001; + // 更新定时任务错误 public static final int UPDATE_CRON_JOB_FAILED = 1245002; + // 插入定时任务错误 public static final int INSERT_CRON_JOB_FAILED = 1245003; + // 定时任务已存在 public static final int CRON_JOB_ALREADY_EXIST = 1245004; public static final int ACQUIRE_CRON_JOB_LOCK_FAILED = 1245005; public static final int CRON_JOB_TIME_PASSED = 1245006; public static final int END_TIME_OR_NOTIFY_TIME_ALREADY_PASSED = 1245007; + public static final int DELETE_CRON_FAILED = 1245008; + // 保存定时任务(id={0})自定义消息通知策略失败 + public static final int SAVE_CRON_CUSTOM_NOTIFY_FAILED = 1245009; // 定时作业 end // 日志服务 @@ -258,10 +386,16 @@ public class ErrorCode { // 删除作业执执行日志失败 public static final int GET_JOB_EXECUTION_LOG_FAIL = 1246001; - // 用户服务 start + // 网关服务 start // 用户不存在或者未登录 public static final int USER_NOT_EXIST_OR_NOT_LOGIN_IN = 1247001; - // 用户服务 end + // 缺少有效的用户信息 + public static final int MISSING_USER_INFO = 1247002; + // 用户认证成功，但用户无应用访问权限 + public static final int USER_ACCESS_APP_FORBIDDEN = 1247403; + // 缺少有效的AppCode + public static final int MISSING_APP_CODE = 1247004; + // 网关服务 end // 业务网关 start // 业务网关 end @@ -273,6 +407,10 @@ public class ErrorCode { public static final int FAIL_TO_DOWNLOAD_NODE_FROM_ARTIFACTORY = 1249002; // 备份服务 end + // ======= 系统错误-公共组件错误 =======// + // 1250xxx-1259xxx被【系统错误-公共组件错误】占用 + // ======= 系统错误-公共组件错误 =======// + // 文件网关 start // 文件源不存在:{0} public static final int FILE_SOURCE_NOT_EXIST = 1260001; @@ -311,6 +449,12 @@ public class ErrorCode { public static final int FILE_SOURCE_ID_NOT_IN_BIZ = 1260017; // 接入点响应异常，详情：{0} public static final int FAIL_TO_REQUEST_FILE_WORKER_WITH_REASON = 1260018; + // 文件源code不可为空 + public static final int FILE_SOURCE_CODE_CAN_NOT_BE_EMPTY = 1260019; + // 制品库根地址不在允许范围内（默认仅支持当前环境蓝鲸制品库），如需对接其他环境请联系平台管理员添加白名单 + public static final int BK_ARTIFACTORY_BASE_URL_INVALID = 1260020; + // 白名单记录已存在，请删除旧数据后再添加 + public static final int FILE_SOURCE_WHITE_INFO_ALREADY_EXISTS = 1260021; // 文件网关 end // 文件代理 start @@ -330,95 +474,16 @@ public class ErrorCode { public static final int FAIL_TO_REQUEST_THIRD_FILE_SOURCE_GET_OBJECT = 1262006; // 文件Worker end - // ==================================== 系统级错误 ================================================// - // ======== 系统错误-权限错误 ==================// - // 用户({0})权限不足，请前往权限中心确认并申请补充后重试 - public static final int PERMISSION_DENIED = 1238001; - // 蓝鲸统一权限错误码，用户({0})权限不足，请前往权限中心确认并申请补充后重试 - public static final int BK_PERMISSION_DENIED = 9900403; - - // ========= 系统错误-请求 ====================// - // 内部服务异常 - public static final int INTERNAL_ERROR = 1240002; - // 错误的请求 - public static final int BAD_REQUEST = 1240003; - // Cookie过期或者不存在 - public static final int COOKIE_ILLEGAL = 1240004; - // 服务不可用 - public static final int SERVICE_UNAVAILABLE = 1240001; - // 服务认证失败 - public static final int SERVICE_AUTH_FAIL = 1240005; - - // ========= 系统错误-API通用 ==================// - // IP:{}无访问权限 - public static final int API_IP_NO_ACCESS = 1239001; - // 用户无访问权限 - public static final int API_USER_NO_ACCESS = 1239002; - // 不支持的API接口 - public static final int API_UNSUPPORTED = 1239003; - // 缺少请求报文或报文不合法 - public static final int API_PARAM_NULL_OR_ILLEGAL = 1239004; - // 异步调用的方法超时了 - public static final int API_INVOKE_TIMEOUT = 1239005; - // 调用方法出错了 - public static final int API_INVOKE_ERROR = 1239006; - // API接口已经过期废弃 - public static final int API_DEPRECATED = 1239007; - // API服务过载，拒绝服务请求 - public static final int API_OVER_LOAD = 1239008; - // API回调其他接口失败 - public static final int API_CALLBACK_FAIL = 1239009; - // Api.cert文件不可用 - public static final int API_CERT_ERROR = 1239010; - - // ======= 系统错误-公共组件错误 =======// - // Redis服务不可用，连接不上 - IP不存在或者配置错误 - public static final int REDIS_CONNECT_FAIL = 1250001; - // Redis服务内存满或者其他问题 - 内存不足够 - public static final int REDIS_DATA_EXCEPTION = 1250002; - // NFS存储 不可用 - public static final int NFS_ERROR = 1259001; - // DB 不可用 - public static final int DB_ERROR = 1252001; - // MQ 不可用 - public static final int MQ_ERROR = 1255001; - - // ======= 系统错误-平台服务错误 =======// - // LICENSE 不可用 - public static final int LICENSE_ERROR = 1210101; - // GSE 不可用 - public static final int GSE_ERROR = 1210001; - - // CMDB错误 - // CMDB服务状态不可达 - 地址配置错误或者地址无法正确解析 - public static final int CMDB_UNREACHABLE_SERVER = 1211001; - // CMDB接口返回数据结构异常- 一般是被网关防火墙重定向返回统一登录页面 - public static final int CMDB_API_DATA_ERROR = 1211002; - // 根据动态分组ID查找主机失败，动态分组ID：{0}，原因：{1}，请确认指定的动态分组在业务下是否存在 - public static final int FAIL_TO_FIND_HOST_BY_DYNAMIC_GROUP = 1211003; - // 根据业务ID查找动态分组失败，业务ID：{0}，原因：{1}，请确认指定的业务是否存在动态分组 - public static final int FAIL_TO_FIND_DYNAMIC_GROUP_BY_BIZ = 1211004; - - // PaaS异常 - // CMSI接口访问异常 - public static final int CMSI_API_ACCESS_ERROR = 1213001; - // 用户管理接口访问异常 - public static final int USER_MANAGE_API_ACCESS_ERROR = 1213002; - // 调用CMSI接口获取通知渠道数据异常 - public static final int CMSI_MSG_CHANNEL_DATA_ERROR = 1213003; - // 调用CMSI接口发送通知失败，错误码：{0}，错误信息：{1} - public static final int CMSI_FAIL_TO_SEND_MSG = 1213004; - - // 制品库异常 - // Artifactory接口返回数据结构异常 - public static final int ARTIFACTORY_API_DATA_ERROR = 1214001; - // 制品库中找不到节点:{0}，请到制品库核实 - public static final int CAN_NOT_FIND_NODE_IN_ARTIFACTORY = 1214002; - - // IAM接口数据异常- 一般是被网关防火墙重定向返回统一登录页面 - public static final int IAM_API_DATA_ERROR = 1215001; - - // 第三方API请求错误 - public static final int API_ERROR = 1216001; + // 迁移升级 + // 迁移失败，任务: {0}, 详情: {1} + public static final int MIGRATION_FAIL = 1263001; + // 统计分析服务job-analysis错误码 start + // AI分析任务报错信息仅支持脚本或文件任务步骤 + public static final int AI_ANALYZE_ERROR_ONLY_SUPPORT_SCRIPT_OR_FILE_STEP = 1264001; + // AI分析任务报错信息内容超过最大值：{0} + public static final int AI_ANALYZE_ERROR_CONTENT_EXCEED_MAX_LENGTH = 1264002; + // AI对话记录不存在：id={0} + public static final int AI_CHAT_HISTORY_NOT_FOUND_BY_ID = 1264003; + // 统计分析服务job-analysis错误码 end } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/ExecuteObjectTypeEnum.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/ExecuteObjectTypeEnum.java new file mode 100644 index 0000000000..cad6e8f303 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/ExecuteObjectTypeEnum.java @@ -0,0 +1,61 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.constant; + +import com.fasterxml.jackson.annotation.JsonValue; + +/** + * 执行对象类型 + */ +public enum ExecuteObjectTypeEnum { + /** + * 主机 + */ + HOST(1), + /** + * 容器 + */ + CONTAINER(2); + + ExecuteObjectTypeEnum(int type) { + this.type = type; + } + + private final int type; + + @JsonValue + public int getValue() { + return this.type; + } + + public static ExecuteObjectTypeEnum valOf(int type) { + for (ExecuteObjectTypeEnum typeEnum : values()) { + if (typeEnum.getValue() == type) { + return typeEnum; + } + } + throw new IllegalArgumentException("No ExecuteObjectTypeEnum constant: " + type); + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/FeatureToggleModeEnum.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/FeatureToggleModeEnum.java index 46eeb3988f..bb66dcdea2 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/FeatureToggleModeEnum.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/FeatureToggleModeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/HttpHeader.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/HttpHeader.java index a918de2c29..6289cb70c1 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/HttpHeader.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/HttpHeader.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/HttpMethodEnum.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/HttpMethodEnum.java new file mode 100644 index 0000000000..dbeb06c78d --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/HttpMethodEnum.java @@ -0,0 +1,32 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.constant; + +/** + * Http 请求方法 + */ +public enum HttpMethodEnum { + GET, HEAD, POST, PUT, PATCH, DELETE, OPTIONS, TRACE +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/HttpRequestSourceEnum.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/HttpRequestSourceEnum.java index 29c9d002d9..9f09c1ce71 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/HttpRequestSourceEnum.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/HttpRequestSourceEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -31,6 +31,10 @@ * Job HTTP请求来源 */ public enum HttpRequestSourceEnum { + /** + * 未知 + */ + UNKNOWN(0), /** * WEB 请求 */ @@ -45,7 +49,7 @@ public enum HttpRequestSourceEnum { INTERNAL(3); @JsonValue - private int value; + private final int value; HttpRequestSourceEnum(int value) { this.value = value; @@ -58,7 +62,7 @@ public static HttpRequestSourceEnum valueOf(int value) { return httpRequestSource; } } - return null; + return UNKNOWN; } public int getValue() { diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/InterceptorOrder.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/InterceptorOrder.java new file mode 100644 index 0000000000..2fd9b48c53 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/InterceptorOrder.java @@ -0,0 +1,95 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.constant; + +/** + * Job 拦截器 ORDER 定义 + */ +public interface InterceptorOrder { + + /** + * 对请求进行预处理（检查、限流、记录、修改请求等） + */ + interface Init { + int HIGHEST = 1; + int LOWEST = 100; + /** + * 检查请求合法性 + */ + int CHECK_VALID = HIGHEST; + /** + * 日志记录 + */ + int LOG = HIGHEST + 1; + /** + * 请求统计 + */ + int METRICS = HIGHEST + 2; + /** + * 请求处理 + */ + int REWRITE_REQUEST = HIGHEST + 2; + } + + /** + * 用户认证 + */ + interface Identification { + int HIGHEST = 101; + int LOWEST = 200; + } + + /** + * 用户鉴权 + */ + interface AUTH { + int HIGHEST = 201; + int LOWEST = 300; + /** + * 全局鉴权 + */ + int AUTH_GLOBAL = HIGHEST; + /** + * 普通鉴权 + */ + int AUTH_COMMON = HIGHEST + 50; + } + + /** + * 业务逻辑 + */ + interface Business { + int HIGHEST = 301; + int LOWEST = 400; + } + + /** + * 请求处理完成 + */ + interface Completion { + int HIGHEST = 401; + int LOWEST = 500; + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/JobCommonHeaders.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/JobCommonHeaders.java index c2692bef8a..000bdf70c7 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/JobCommonHeaders.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/JobCommonHeaders.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -42,4 +42,9 @@ public interface JobCommonHeaders { * 蓝鲸网关-JWT */ String BK_GATEWAY_JWT = "X-Bkapi-JWT"; + + /** + * 蓝鲸网关-从网关来的请求，与ESB请求区分 + */ + String BK_GATEWAY_FROM = "X-Bkapi-From"; } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/JobConstants.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/JobConstants.java index 404999b3ba..2baeed7d81 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/JobConstants.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/JobConstants.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,6 +28,10 @@ * Job 全局通用常量 */ public class JobConstants { + /** + * 不可能取值的action scope id值，用于数据查询时排除 + */ + public static final long UNAVAILABLE_ACTION_SCOPE_ID = -1L; /** * GSE Agent状态：正常 */ @@ -79,7 +83,7 @@ public class JobConstants { /** * 作业最大超时时间，单位秒 */ - public static final int MAX_JOB_TIMEOUT_SECONDS = 86400; + public static final int MAX_JOB_TIMEOUT_SECONDS = 259200; /** * 请求来源：备份服务 */ @@ -92,4 +96,8 @@ public class JobConstants { * Job内置业务集的ID范围的最大值 */ public static final long JOB_BUILD_IN_BIZ_SET_ID_MAX = 9999999L; + /** + * 未知的资源名称 + */ + public static final String UNKNOWN_NAME = "Unknown"; } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/JobDiscoveryConsts.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/JobDiscoveryConsts.java index 783e4c6853..8fdafa8053 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/JobDiscoveryConsts.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/JobDiscoveryConsts.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/JobResourceTypeEnum.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/JobResourceTypeEnum.java index f3ad751b26..76e6394925 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/JobResourceTypeEnum.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/JobResourceTypeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/LabelSelectorOperatorEnum.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/LabelSelectorOperatorEnum.java new file mode 100644 index 0000000000..d45598f5ec --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/LabelSelectorOperatorEnum.java @@ -0,0 +1,67 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.constant; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonValue; + +/** + * Label selector 操作符 + */ +public enum LabelSelectorOperatorEnum { + NOT_EXISTS("not_exists"), + EQUALS("equals"), + IN("in"), + NOT_EQUALS("not_equals"), + NOT_IN("not_in"), + EXISTS("exists"), + GREATER_THAN("gt"), + LESS_THAN("lt"); + + private final String op; + + LabelSelectorOperatorEnum(String op) { + this.op = op; + } + + @JsonValue + public String getOp() { + return op; + } + + public static LabelSelectorOperatorEnum valOf(String op) { + for (LabelSelectorOperatorEnum operatorEnum : values()) { + if (operatorEnum.getOp().equals(op)) { + return operatorEnum; + } + } + throw new IllegalArgumentException("No LabelSelectorOperatorEnum constant: " + op); + } + + @JsonCreator(mode = JsonCreator.Mode.DELEGATING) + public static LabelSelectorOperatorEnum forOp(String op) { + return LabelSelectorOperatorEnum.valOf(op); + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/MySQLTextDataType.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/MySQLTextDataType.java new file mode 100644 index 0000000000..69636fdb4d --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/MySQLTextDataType.java @@ -0,0 +1,63 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.constant; + +import com.tencent.bk.job.common.util.file.FileSizeUtil; +import lombok.Getter; + +/** + * 这里暂时只记录TEXT形式的MySQL类型，用于获取各类型的最大长度。 + * 因为在MySQL中，TEXT类型的存储是以字节流的长度为限制的，而char/varchar则是以字符串的长度为限制的，所以校验长度合法的逻辑是不太一样的。 + * + */ +@Getter +public enum MySQLTextDataType { + TINYTEXT("TINYTEXT", 255L), + TEXT("TEXT", 65535L), + MEDIUMTEXT("MEDIUMTEXT", 16777215L), + LONGTEXT("LONGTEXT", 4294967295L); + + private final String value; + private final Long maximumLength; // MySQL能存的最大字节数 + + MySQLTextDataType(String value, Long maximumLength) { + this.value = value; + this.maximumLength = maximumLength; + } + + public static MySQLTextDataType valOf(String value) { + for (MySQLTextDataType type : MySQLTextDataType.values()) { + if (type.value.equals(value)) { + return type; + } + } + return null; + } + + @Override + public String toString() { + return value + "(" + FileSizeUtil.getFileSizeStr(maximumLength) + ")"; + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/NotExistPathHandlerEnum.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/NotExistPathHandlerEnum.java index 208ad3f6fa..0900f7725c 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/NotExistPathHandlerEnum.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/NotExistPathHandlerEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/NotifyChannelEnum.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/NotifyChannelEnum.java index 880fe036ba..538f0d171f 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/NotifyChannelEnum.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/NotifyChannelEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/Order.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/Order.java index f510ec1ce3..2e5ab7d12c 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/Order.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/Order.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/ProfileEnum.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/ProfileEnum.java new file mode 100644 index 0000000000..303b795ffc --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/ProfileEnum.java @@ -0,0 +1,64 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.constant; + +/** + * Job 使用的 Spring profile 定义 + */ +public enum ProfileEnum { + /** + * 本地 + */ + LOCAL(Constants.LOCAL), + /** + * 开发 + */ + DEV(Constants.DEV), + /** + * 正式 + */ + PROD(Constants.PROD), + /** + * 单元/继承测试 + */ + TEST(Constants.TEST); + + public interface Constants { + String LOCAL = "local"; + String DEV = "dev"; + String PROD = "prod"; + String TEST = "test"; + } + + private final String value; + + ProfileEnum(String value) { + this.value = value; + } + + public String getValue() { + return value; + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/RedisConstants.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/RedisConstants.java index cc68ca5864..71845c6830 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/RedisConstants.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/RedisConstants.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/ResourceScopeTypeEnum.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/ResourceScopeTypeEnum.java index ccd74b4175..3fd82dd82b 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/ResourceScopeTypeEnum.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/ResourceScopeTypeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -57,7 +57,7 @@ public static ResourceScopeTypeEnum from(String type) { return scopeType; } } - return null; + throw new IllegalArgumentException("No ResourceScopeTypeEnum constant: " + type); } public String getValue() { @@ -65,6 +65,11 @@ public String getValue() { } public static boolean isValid(String type) { - return from(type) != null; + try { + from(type); + return true; + } catch (IllegalArgumentException e) { + return false; + } } } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/RollingModeEnum.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/RollingModeEnum.java index 9e666898bd..cd90eb3f56 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/RollingModeEnum.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/RollingModeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/TaskVariableTypeEnum.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/TaskVariableTypeEnum.java index 39e6fffd36..de3fa2fd5a 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/TaskVariableTypeEnum.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/TaskVariableTypeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -29,9 +29,6 @@ import lombok.AllArgsConstructor; import lombok.Getter; -/** - * @since 3/10/2019 17:17 - */ @Getter @AllArgsConstructor public enum TaskVariableTypeEnum { diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/WhiteIpRuleEnum.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/WhiteIpRuleEnum.java index de8ee261d2..9a2ed2a9a5 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/WhiteIpRuleEnum.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/constant/WhiteIpRuleEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/context/JobContext.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/context/JobContext.java index 0050cc9cfb..8fdcb80711 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/context/JobContext.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/context/JobContext.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -37,7 +37,7 @@ import java.util.Map; /** - * @since 6/11/2019 10:26 + * Job http 请求上下文 */ @Data public class JobContext { @@ -52,6 +52,8 @@ public class JobContext { private String userLang; + private String requestFrom; + private List debugMessage; private ZoneId timeZone; diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/context/JobContextThreadLocal.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/context/JobContextThreadLocal.java index 81e1851377..4675f1f4c9 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/context/JobContextThreadLocal.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/context/JobContextThreadLocal.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,7 +25,7 @@ package com.tencent.bk.job.common.context; /** - * @since 6/11/2019 10:26 + * Job http 请求上下文托管（ThreadLocal方式) */ public class JobContextThreadLocal { diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/encrypt/AsymmetricEncryptor.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/encrypt/AsymmetricEncryptor.java deleted file mode 100644 index 629f63456f..0000000000 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/encrypt/AsymmetricEncryptor.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.encrypt; - -/** - * 非对称加密器 - */ -public interface AsymmetricEncryptor extends Encryptor { - - /** - * 校验消息的签名是否一致 通过公钥对消息内容进行校验signature内容（由私钥加签名） - * - * @param message 原消息内容 - * @param signature 消息的签名 - * @return 是否签名一致 - */ - boolean verify(String message, String signature); -} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/encrypt/RSAEncryptor.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/encrypt/RSAEncryptor.java deleted file mode 100644 index aa5ed36945..0000000000 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/encrypt/RSAEncryptor.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.encrypt; - -import com.tencent.bk.job.common.util.crypto.RSAUtils; - -import java.io.File; -import java.io.IOException; -import java.security.GeneralSecurityException; -import java.security.PublicKey; - -public class RSAEncryptor implements AsymmetricEncryptor { - private PublicKey publicKey; - - public RSAEncryptor(File rsaPubPermFile) throws IOException, GeneralSecurityException { - publicKey = RSAUtils.getPublicKey(rsaPubPermFile); - } - - public RSAEncryptor(String rsaPublicKeyBase64) throws IOException, GeneralSecurityException { - publicKey = RSAUtils.getPublicKey(rsaPublicKeyBase64); - } - - public String encrypt(String rawText) { - try { - return RSAUtils.encrypt(rawText, publicKey); - } catch (IOException | GeneralSecurityException e) { - return null; - } - } - - public boolean verify(String message, String signature) { - try { - return RSAUtils.verify(publicKey, message, signature); - } catch (Exception e) { - return false; - } - } -} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/event/Event.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/event/Event.java new file mode 100644 index 0000000000..6ed0549a13 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/event/Event.java @@ -0,0 +1,48 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.event; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.tencent.bk.job.common.util.date.DateUtils; +import lombok.Data; + +import java.time.LocalDateTime; + +@Data +@JsonInclude(JsonInclude.Include.NON_NULL) +public class Event { + /** + * 事件发生时间 + */ + protected LocalDateTime time; + + public long duration() { + if (time != null) { + return DateUtils.calculateMillsBetweenDateTime(time, LocalDateTime.now()); + } else { + return 0; + } + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/AbortedException.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/AbortedException.java index 46f771de77..1f8661038b 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/AbortedException.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/AbortedException.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/AlreadyExistsException.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/AlreadyExistsException.java index 4b9fb69ea0..7f71633a31 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/AlreadyExistsException.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/AlreadyExistsException.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/DAOException.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/DAOException.java index a0e3135a97..735a96b4c3 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/DAOException.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/DAOException.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/DistributeFileFromExternalAgentException.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/DistributeFileFromExternalAgentException.java new file mode 100644 index 0000000000..69a14986cd --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/DistributeFileFromExternalAgentException.java @@ -0,0 +1,42 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.exception; + +import com.tencent.bk.job.common.model.error.ErrorType; + + +/** + * 当使用集群外部的机器作为文件的分发源时，不满足分发条件（没有可用的源机器/...）时抛出的异常 + */ +public class DistributeFileFromExternalAgentException extends ServiceException { + + public DistributeFileFromExternalAgentException(Integer errorCode) { + super(ErrorType.INTERNAL, errorCode); + } + + public DistributeFileFromExternalAgentException(String message, Integer errorCode) { + super(message, ErrorType.INTERNAL, errorCode); + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/ErrorInfo.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/ErrorInfo.java index 0e05bf13e2..6c26f30b38 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/ErrorInfo.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/ErrorInfo.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/FailedPreconditionException.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/FailedPreconditionException.java index 7a36f10a2f..0640e167e5 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/FailedPreconditionException.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/FailedPreconditionException.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/HttpStatusException.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/HttpStatusException.java new file mode 100644 index 0000000000..d1b4ad9128 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/HttpStatusException.java @@ -0,0 +1,57 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.exception; + +import lombok.Getter; +import lombok.ToString; + +/** + * 服务异常 + */ +@Getter +@ToString +public class HttpStatusException extends RuntimeException { + private final String uri; + private final int httpStatus; + private final String reasonPhrase; + private final String respBodyStr; + + public HttpStatusException(String uri, int httpStatus, String reasonPhrase) { + super("http status not ok, uri=" + uri + ", statusCode=" + httpStatus + ", reasonPhrase=" + reasonPhrase); + this.uri = uri; + this.httpStatus = httpStatus; + this.reasonPhrase = reasonPhrase; + respBodyStr = null; + } + + public HttpStatusException(String uri, int httpStatus, String reasonPhrase, String respBodyStr) { + super("http status not ok, uri=" + uri + + ", statusCode=" + httpStatus + ", reasonPhrase=" + reasonPhrase + ", respBodyStr=" + respBodyStr); + this.uri = uri; + this.httpStatus = httpStatus; + this.reasonPhrase = reasonPhrase; + this.respBodyStr = respBodyStr; + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/IncorrectConfigException.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/IncorrectConfigException.java new file mode 100644 index 0000000000..8827c6fb64 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/IncorrectConfigException.java @@ -0,0 +1,16 @@ +package com.tencent.bk.job.common.exception; + +import lombok.Getter; +import lombok.ToString; + +/** + * 内部服务异常--配置异常 + */ +@Getter +@ToString +public class IncorrectConfigException extends InternalException { + + public IncorrectConfigException(String message, Integer errorCode, Object[] errorParams) { + super(message, errorCode, errorParams); + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/InternalException.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/InternalException.java index ce21375b87..ab5e47d101 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/InternalException.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/InternalException.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,7 @@ package com.tencent.bk.job.common.exception; +import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.model.error.ErrorType; import lombok.Getter; import lombok.ToString; @@ -35,6 +36,10 @@ @ToString public class InternalException extends ServiceException { + public InternalException(String message) { + super(message, ErrorType.INTERNAL, ErrorCode.INTERNAL_ERROR); + } + public InternalException(Integer errorCode) { super(ErrorType.INTERNAL, errorCode); } @@ -63,6 +68,10 @@ public InternalException(String message, Throwable cause, Integer errorCode) { super(message, cause, ErrorType.INTERNAL, errorCode); } + public InternalException(String message, Throwable cause) { + super(message, cause, ErrorType.INTERNAL, ErrorCode.INTERNAL_ERROR); + } + public InternalException(String message, Throwable cause, Integer errorCode, Object[] errorParams) { super(message, cause, ErrorType.INTERNAL, errorCode, errorParams); diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/InvalidIpv6Exception.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/InvalidIpv6Exception.java new file mode 100644 index 0000000000..843f8c9e98 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/InvalidIpv6Exception.java @@ -0,0 +1,70 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.exception; + +import com.tencent.bk.job.common.model.error.ErrorType; +import lombok.Getter; +import lombok.ToString; + +/** + * IPv6地址不合法 + */ +@Getter +@ToString +public class InvalidIpv6Exception extends ServiceException { + + public InvalidIpv6Exception(Integer errorCode) { + super(ErrorType.INTERNAL, errorCode); + } + + public InvalidIpv6Exception(Integer errorCode, Object[] errorParams) { + super(ErrorType.INTERNAL, errorCode, errorParams); + } + + public InvalidIpv6Exception(String message, Integer errorCode) { + super(message, ErrorType.INTERNAL, errorCode); + } + + public InvalidIpv6Exception(String message, Integer errorCode, Object[] errorParams) { + super(message, ErrorType.INTERNAL, errorCode, errorParams); + } + + public InvalidIpv6Exception(Throwable cause, Integer errorCode) { + super(cause, ErrorType.INTERNAL, errorCode); + } + + public InvalidIpv6Exception(Throwable cause, Integer errorCode, Object[] errorParams) { + super(cause, ErrorType.INTERNAL, errorCode, errorParams); + } + + public InvalidIpv6Exception(String message, Throwable cause, Integer errorCode) { + super(message, cause, ErrorType.INTERNAL, errorCode); + } + + public InvalidIpv6Exception(String message, Throwable cause, Integer errorCode, + Object[] errorParams) { + super(message, cause, ErrorType.INTERNAL, errorCode, errorParams); + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/InvalidParamException.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/InvalidParamException.java index ee750b6706..f575ce0cdc 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/InvalidParamException.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/InvalidParamException.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/JobMicroServiceBootException.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/JobMicroServiceBootException.java new file mode 100644 index 0000000000..8ca50c308d --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/JobMicroServiceBootException.java @@ -0,0 +1,46 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.exception; + +/** + * 作业平台微服务启动异常 + */ +public class JobMicroServiceBootException extends RuntimeException { + public JobMicroServiceBootException() { + super(); + } + + public JobMicroServiceBootException(String message) { + super(message); + } + + public JobMicroServiceBootException(String message, Throwable cause) { + super(message, cause); + } + + public JobMicroServiceBootException(Throwable cause) { + super(cause); + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/MissingParameterException.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/MissingParameterException.java new file mode 100644 index 0000000000..d01ff7a52e --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/MissingParameterException.java @@ -0,0 +1,79 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.exception; + +import com.tencent.bk.job.common.model.ValidateResult; +import com.tencent.bk.job.common.model.error.ErrorType; +import lombok.Getter; +import lombok.ToString; + +/** + * 请求参数为空异常 + */ +@Getter +@ToString +public class MissingParameterException extends ServiceException { + public MissingParameterException(Integer errorCode) { + super(ErrorType.INVALID_PARAM, errorCode); + } + + public MissingParameterException(Integer errorCode, Object[] errorParams) { + super(ErrorType.INVALID_PARAM, errorCode, errorParams); + } + + public MissingParameterException(Integer errorCode, Object errorParam) { + super(ErrorType.INVALID_PARAM, errorCode, new Object[]{errorParam}); + } + + public MissingParameterException(String message, Integer errorCode) { + super(message, ErrorType.INVALID_PARAM, errorCode); + } + + public MissingParameterException(String message, Integer errorCode, Object[] errorParams) { + super(message, ErrorType.INVALID_PARAM, errorCode, errorParams); + } + + public MissingParameterException(Throwable cause, Integer errorCode) { + super(cause, ErrorType.INVALID_PARAM, errorCode); + } + + public MissingParameterException(Throwable cause, Integer errorCode, Object[] errorParams) { + super(cause, ErrorType.INVALID_PARAM, errorCode, errorParams); + } + + public MissingParameterException(String message, Throwable cause, Integer errorCode) { + super(message, cause, ErrorType.INVALID_PARAM, errorCode); + } + + public MissingParameterException(String message, Throwable cause, Integer errorCode, + Object[] errorParams) { + super(message, cause, ErrorType.INVALID_PARAM, errorCode, errorParams); + } + + public MissingParameterException(ValidateResult validateResult) { + super(ErrorType.INVALID_PARAM, validateResult.getErrorCode(), validateResult.getErrorParams()); + } + +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/NotFoundException.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/NotFoundException.java index a72c4ceee5..698e7a46ca 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/NotFoundException.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/NotFoundException.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/NotImplementedException.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/NotImplementedException.java index feb0dfc65c..0f1bc5c5d2 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/NotImplementedException.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/NotImplementedException.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -64,7 +64,7 @@ public NotImplementedException(String message, Throwable cause, Integer errorCod } public NotImplementedException(String message, Throwable cause, Integer errorCode, - Object[] errorParams) { + Object[] errorParams) { super(message, cause, ErrorType.UNIMPLEMENTED, errorCode, errorParams); } } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/ResourceExhaustedException.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/ResourceExhaustedException.java index 589037999a..ba7c3e4e49 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/ResourceExhaustedException.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/ResourceExhaustedException.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/ServiceException.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/ServiceException.java index 2b866f082c..0b74e37cd3 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/ServiceException.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/ServiceException.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -29,6 +29,8 @@ import lombok.Getter; import lombok.ToString; +import java.util.Locale; + /** * 服务异常 */ @@ -95,4 +97,8 @@ public ServiceException(String message, Throwable cause, ErrorType errorType, In public String getI18nMessage() { return I18nUtil.getI18nMessage(String.valueOf(errorCode), errorParams); } + + public String getI18nMessage(Locale locale) { + return I18nUtil.getI18nMessage(locale, String.valueOf(errorCode), errorParams); + } } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/SubThreadException.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/SubThreadException.java new file mode 100644 index 0000000000..88db026710 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/SubThreadException.java @@ -0,0 +1,43 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.exception; + +import lombok.Getter; +import lombok.ToString; + +/** + * 子线程抛出的异常 + */ +@Getter +@ToString +public class SubThreadException extends RuntimeException { + + private final String message; + + public SubThreadException(String message, Throwable cause) { + super(cause); + this.message = message; + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/TimeoutException.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/TimeoutException.java index 527fbbe813..20a836fcaa 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/TimeoutException.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/TimeoutException.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/UnauthenticatedException.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/UnauthenticatedException.java index 1ac07d1c9d..67bd887976 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/UnauthenticatedException.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/UnauthenticatedException.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/UnavailableException.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/UnavailableException.java index 53abc13d83..d00acf410f 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/UnavailableException.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/exception/UnavailableException.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/metrics/CommonMetricNames.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/metrics/CommonMetricNames.java index 31dc6b6484..b3aa7bc07a 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/metrics/CommonMetricNames.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/metrics/CommonMetricNames.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -32,10 +32,18 @@ public class CommonMetricNames { * 仅统计调用ESB BK-LOGIN API的HTTP请求过程 */ public static final String ESB_BK_LOGIN_API_HTTP = "job.client.bk.login.api.http"; + /** + * 仅统计调用ESB BK-LOGIN API的整个过程，含反序列化 + */ + public static final String ESB_BK_LOGIN_API = "job.client.bk.login.api"; /** * 仅统计调用ESB USER-MANAGE API的HTTP请求过程 */ public static final String ESB_USER_MANAGE_API_HTTP = "job.client.user.manage.api.http"; + /** + * 统计调用ESB 用户管理 API的整个过程，含反序列化 + */ + public static final String ESB_USER_MANAGE_API = "job.client.user.manage.api"; /** * 仅统计调用ESB CMSI API的HTTP请求过程 */ @@ -44,38 +52,37 @@ public class CommonMetricNames { * 统计调用ESB CMSI API的整个过程，含反序列化 */ public static final String ESB_CMSI_API = "job.client.cmsi.api"; - /** - * 仅统计调用ESB IAM API的HTTP请求过程 - */ - public static final String ESB_IAM_API_HTTP = "job.client.iam.api.http"; /** * 仅统计调用ESB CMDB API的HTTP请求过程 */ public static final String ESB_CMDB_API_HTTP = "job.client.cmdb.api.http"; + /** - * 统计调用CMDB API整个过程，含反序列化 + * 仅统计调用权限中心后台 API的HTTP请求过程 */ - public static final String ESB_CMDB_API = "job.client.cmdb.api"; - + public static final String IAM_API_HTTP = "job.client.iam.api.http"; + /** + * 统计调用权限中心后台 API整个过程，含反序列化 + */ + public static final String IAM_API = "job.client.iam.api"; /** - * 仅统计调用制品库 API的HTTP请求过程 + * 仅统计调用消息通知中心后台 API的HTTP请求过程 */ - public static final String BKREPO_API_HTTP = "job.client.bkrepo.api.http"; + public static final String BK_NOTICE_API_HTTP = "job.client.bknotice.api.http"; /** - * 统计调用制品库 API整个过程，含反序列化 + * 统计调用消息通知中心后台 API整个过程，含反序列化 */ - public static final String BKREPO_API = "job.client.bkrepo.api"; - + public static final String BK_NOTICE_API = "job.client.bknotice.api"; /** - * 仅统计调用权限中心后台 API的HTTP请求过程 + * 仅统计调用AIDev后台 API的HTTP请求过程 */ - public static final String IAM_API_HTTP = "job.client.iam.api.http"; + public static final String BK_AI_DEV_API_HTTP = "job.client.bkAIDev.api.http"; /** - * 统计调用权限中心后台 API整个过程，含反序列化 + * 统计调用AIDev后台 API整个过程，含反序列化 */ - public static final String IAM_API = "job.client.iam.api"; + public static final String BK_AI_DEV_API = "job.client.bkAIDev.api"; /** @@ -86,4 +93,14 @@ public class CommonMetricNames { * 被调用WEB API */ public static final String WEB_API = "job.server.web.api"; + + /** + * 仅统计调用ESB GSE API的HTTP请求过程 + */ + public static final String ESB_GSE_API_HTTP = "job.client.gse.api.http"; + /** + * 统计调用 GSE API整个过程，含反序列化 + */ + public static final String ESB_GSE_API = "job.client.gse.api"; + } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/metrics/CommonMetricTags.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/metrics/CommonMetricTags.java index c9cf1b3644..e194a68370 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/metrics/CommonMetricTags.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/metrics/CommonMetricTags.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -44,4 +44,24 @@ public class CommonMetricTags { * API类型：未知 */ public static final String VALUE_API_TYPE_UNKNOWN = "unknown"; + + /** + * Job 资源管理空间 + */ + public static final String KEY_RESOURCE_SCOPE = "resource_scope"; + + /** + * 蓝鲸应用 ID + */ + public static final String KEY_APP_CODE = "app_code"; + + /** + * HTTP请求状态 + */ + public static final String KEY_HTTP_STATUS = "http_status"; + + /** + * HTTP请求状态：未知 + */ + public static final String VALUE_HTTP_STATUS_UNKNOWN = "unknown"; } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/metrics/CommonMetricValues.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/metrics/CommonMetricValues.java new file mode 100644 index 0000000000..72521f33a9 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/metrics/CommonMetricValues.java @@ -0,0 +1,35 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.metrics; + +/** + * 公共监控指标的值 + */ +public class CommonMetricValues { + /** + * 标识一个不存在的值 + */ + public static final String NONE = "none"; +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/BaseSearchCondition.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/BaseSearchCondition.java index 575559b22b..2668533b33 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/BaseSearchCondition.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/BaseSearchCondition.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,6 @@ package com.tencent.bk.job.common.model; -import io.swagger.annotations.ApiModelProperty; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; @@ -41,24 +40,25 @@ public class BaseSearchCondition implements Cloneable { /** * 分页起始 */ - @ApiModelProperty(value = "分页起始", required = false) private Integer start; /** * 分页大小 */ - @ApiModelProperty(value = "分页大小", required = false) private Integer length; + /** + * 分页-是否计算总数；默认计算。分页计算总数可能会影响 API 性能, 必要场景才可使用 + */ + private boolean countPageTotal = true; + /** * 排序 0：降序 1：升序 */ - @ApiModelProperty(value = "排序 0：降序 1：升序", required = false) private Integer order; /** * 排序的字段 */ - @ApiModelProperty(value = "排序的字段", required = false) private String orderField; @@ -94,6 +94,15 @@ public static BaseSearchCondition pageCondition(Integer start, Integer length) { BaseSearchCondition searchCondition = new BaseSearchCondition(); searchCondition.setStart(start); searchCondition.setLength(length); + searchCondition.setCountPageTotal(true); + return searchCondition; + } + + public static BaseSearchCondition pageCondition(Integer start, Integer length, boolean countPageTotal) { + BaseSearchCondition searchCondition = new BaseSearchCondition(); + searchCondition.setStart(start); + searchCondition.setLength(length); + searchCondition.setCountPageTotal(countPageTotal); return searchCondition; } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/HostCompositeKey.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/HostCompositeKey.java new file mode 100644 index 0000000000..eb393bb03d --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/HostCompositeKey.java @@ -0,0 +1,107 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.model; + +import com.tencent.bk.job.common.model.dto.HostDTO; +import lombok.Getter; +import lombok.Setter; + +import java.util.Objects; + +/** + * 主机复合 KEY，用于主机的多种表达方式 + */ +@Getter +@Setter +public class HostCompositeKey { + /** + * Key 类型 + */ + private final HostCompositeKeyType keyType; + + /** + * 主机唯一 key（目前支持 hostId/cloudIp 两种） + */ + private final String key; + + public HostCompositeKey(HostCompositeKeyType keyType, String key) { + this.keyType = keyType; + this.key = key; + } + + public static HostCompositeKey ofHost(HostDTO host) { + if (host.getHostId() != null) { + // 优先使用 hostId + return new HostCompositeKey(HostCompositeKeyType.HOST_ID, String.valueOf(host.getHostId())); + } else if (host.toCloudIp() != null) { + // 没有 hostId, 使用管控区域 ID + ipv4 + return new HostCompositeKey(HostCompositeKeyType.CLOUD_IP, host.toCloudIp()); + } else { + throw new IllegalArgumentException("Invalid host, both hostId or cloudIp are empty"); + } + } + + + @Getter + public enum HostCompositeKeyType { + /** + * HostId 作为 KEY + */ + HOST_ID(1), + /** + * 管控区域 ID+ ipv4 作为 KEY + */ + CLOUD_IP(2); + + private final int value; + + HostCompositeKeyType(int value) { + this.value = value; + } + + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + HostCompositeKey that = (HostCompositeKey) o; + if (this.getKeyType() != that.getKeyType()) { + return false; + } + + return keyType == ((HostCompositeKey) o).getKeyType() && key.equals(that.getKey()); + } + + @Override + public int hashCode() { + return Objects.hash(keyType, key); + } + + @Override + public String toString() { + return keyType.getValue() + ":" + key; + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/InternalResponse.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/InternalResponse.java index c5bf6ccf78..7c347ca1db 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/InternalResponse.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/InternalResponse.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/OrderCondition.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/OrderCondition.java new file mode 100644 index 0000000000..a090f10bc9 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/OrderCondition.java @@ -0,0 +1,60 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.model; + +import com.tencent.bk.job.common.constant.Order; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; + +/** + * 查询排序设置 + */ +@Getter +@Setter +@ToString +@NoArgsConstructor +public class OrderCondition implements Cloneable { + + /** + * 排序 + */ + private Order order; + + /** + * 排序的字段 + */ + private String orderField; + + private OrderCondition(Order order, String orderField) { + this.order = order; + this.orderField = orderField; + } + + public static OrderCondition build(Order order, String orderField) { + return new OrderCondition(order, orderField); + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/PageCondition.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/PageCondition.java new file mode 100644 index 0000000000..814f6b6277 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/PageCondition.java @@ -0,0 +1,78 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.model; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; + +/** + * 分页查询条件 + */ +@Getter +@Setter +@ToString +@NoArgsConstructor +public class PageCondition implements Cloneable { + /** + * 分页起始 + */ + private int start = 0; + /** + * 分页大小 + */ + private int length = 20; + + /** + * 分页-是否计算总数；默认计算。分页计算总数可能会影响 API 性能, 必要场景才可使用 + */ + private boolean countPageTotal = true; + + public static PageCondition build(int start, int length) { + PageCondition pageCondition = new PageCondition(); + pageCondition.setStart(start); + pageCondition.setLength(length); + pageCondition.setCountPageTotal(true); + return pageCondition; + } + + public static PageCondition build(int start, int length, boolean countPageTotal) { + PageCondition pageCondition = new PageCondition(); + pageCondition.setStart(start); + pageCondition.setLength(length); + pageCondition.setCountPageTotal(countPageTotal); + return pageCondition; + } + + public PageCondition clone() { + PageCondition pageCondition = new PageCondition(); + pageCondition.setStart(start); + pageCondition.setLength(length); + pageCondition.setCountPageTotal(countPageTotal); + return pageCondition; + } + +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/PageData.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/PageData.java index db106e152c..45ae29d2b5 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/PageData.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/PageData.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -77,12 +77,12 @@ public static PageData from(PageData pageData, Function conve if (CollectionUtils.isEmpty(pageData.getData())) { return emptyPageData(pageData.getStart(), pageData.getPageSize()); } - PageData esbPageData = new PageData<>(); - esbPageData.setTotal(pageData.getTotal()); - esbPageData.setStart(pageData.getStart()); - esbPageData.setPageSize(pageData.getPageSize()); - esbPageData.setData(pageData.getData().stream().map(converter).collect(Collectors.toList())); - return esbPageData; + PageData newPageData = new PageData<>(); + newPageData.setTotal(pageData.getTotal()); + newPageData.setStart(pageData.getStart()); + newPageData.setPageSize(pageData.getPageSize()); + newPageData.setData(pageData.getData().stream().map(converter).collect(Collectors.toList())); + return newPageData; } public static PageData emptyPageData(Integer start, Integer length) { diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/PageDataWithManagePermission.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/PageDataWithManagePermission.java index 4735e464fc..6a07710db1 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/PageDataWithManagePermission.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/PageDataWithManagePermission.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/Response.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/Response.java index fae8eed059..3bce787be5 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/Response.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/Response.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -58,7 +58,7 @@ public class Response { @ApiModelProperty("错误信息") private String errorMsg; - @ApiModelProperty("请求成功返回的数据") + @ApiModelProperty("请求成功/失败返回的数据") private T data; @ApiModelProperty("请求 ID") diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/SearchCondition.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/SearchCondition.java new file mode 100644 index 0000000000..5a4240f4da --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/SearchCondition.java @@ -0,0 +1,51 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.model; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; + +/** + * 查询基础类 + */ +@Getter +@Setter +@ToString +@NoArgsConstructor +public class SearchCondition implements Cloneable { + private PageCondition pageCondition; + private OrderCondition orderCondition; + + public SearchCondition(PageCondition pageCondition, OrderCondition orderCondition) { + this.pageCondition = pageCondition; + this.orderCondition = orderCondition; + } + + public static SearchCondition build(PageCondition pageCondition, OrderCondition orderCondition) { + return new SearchCondition(pageCondition, orderCondition); + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/ValidateResult.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/ValidateResult.java index 02de6c0f7c..c199a35b97 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/ValidateResult.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/ValidateResult.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/AppResourceScope.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/AppResourceScope.java index 4dea6d55bd..5377364d5a 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/AppResourceScope.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/AppResourceScope.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/ApplicationAttrsDO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/ApplicationAttrsDO.java index d42ec57da7..767359cb1d 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/ApplicationAttrsDO.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/ApplicationAttrsDO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/ApplicationDTO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/ApplicationDTO.java index b9e7bdc59a..273a945a8d 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/ApplicationDTO.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/ApplicationDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/ApplicationHostDTO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/ApplicationHostDTO.java index 3db525dd7f..6c3c8ab359 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/ApplicationHostDTO.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/ApplicationHostDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,23 +24,35 @@ package com.tencent.bk.job.common.model.dto; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.tencent.bk.job.common.annotation.PersistenceObject; import com.tencent.bk.job.common.constant.JobConstants; +import com.tencent.bk.job.common.model.vo.CloudAreaInfoVO; +import com.tencent.bk.job.common.model.vo.HostInfoVO; +import com.tencent.bk.job.common.util.ip.IpUtils; import lombok.AllArgsConstructor; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.slf4j.helpers.MessageFormatter; +import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; /** * 主机 */ +@Slf4j +@PersistenceObject @Data @EqualsAndHashCode @NoArgsConstructor @AllArgsConstructor +@JsonInclude(JsonInclude.Include.NON_EMPTY) public class ApplicationHostDTO { /** @@ -56,26 +68,43 @@ public class ApplicationHostDTO { */ private Long bizId; /** - * IP + * IPv4(主机的第一个IPv4地址,完整的IPv4地址使用displayIp字段) */ private String ip; + /** + * IPv6 + */ + private String ipv6; + /** + * AgentId + */ + private String agentId; /** - * 展示用的IP + * 完整的IPv4地址 */ private String displayIp; /** - * 主机描述 + * 主机名称 + */ + private String hostName; + /** + * 主机Agent状态值 */ - private String ipDesc; + private Integer gseAgentStatus = null; /** - * 主机Agent状态 + * 主机Agent是否正常 */ - private Boolean gseAgentAlive; + private Boolean gseAgentAlive = false; /** * 云区域ID */ private Long cloudAreaId; + /** + * 云区域名称 + */ + @JsonIgnore + private String cloudAreaName; /** * 云区域+ip */ @@ -84,38 +113,145 @@ public class ApplicationHostDTO { /** * 操作系统 */ - private String os; + private String osName; /** * 操作系统类型 */ private String osType; + /** + * 操作系统类型名称 + */ + private String osTypeName; + + /** + * 所属云厂商ID + */ + private String cloudVendorId; + + /** + * 所属云厂商名称 + */ + private String cloudVendorName; + + /** + * CMDB中的上次修改时间 + */ + private Long lastTime; + /** * 集群ID */ - private List setId; + private List setId = new ArrayList<>(); /** * 模块id,一个主机可以属于多个模块 */ - private List moduleId; + private List moduleId = new ArrayList<>(); /** * cc的模块类型集合， 选填0,1,2，0所有模块 1普通模块，2DB模块， 支持多个模块 **/ - private List moduleType; + private List moduleType = new ArrayList<>(); /** * IP 列表，搜索用参数 */ - private List ipList; + private List ipList = new ArrayList<>(); - /** - * 主机AgentId - */ - private String agentId; + public String getCloudVendorId() { + if (cloudVendorId != null && cloudVendorId.length() > 64) { + return cloudVendorId.substring(0, 64); + } + return cloudVendorId; + } + + public void setGseAgentStatus(Integer gseAgentStatus) { + this.gseAgentStatus = gseAgentStatus; + this.gseAgentAlive = gseAgentStatus != null && gseAgentStatus == 2; + } + + public void setGseAgentAlive(Boolean gseAgentAlive) { + this.gseAgentAlive = gseAgentAlive; + if (gseAgentAlive != null && gseAgentAlive) { + // 取值参考AgentStateStatusEnum + this.gseAgentStatus = 2; + } else { + this.gseAgentStatus = -2; + } + } + + private Integer getAgentAliveValue() { + return gseAgentAlive == null ? 0 : (gseAgentAlive ? 1 : 0); + } + + private static boolean isGseAgentAlive(HostInfoVO hostInfoVO) { + if (hostInfoVO.getAlive() != null) { + return hostInfoVO.getAlive() == 1; + } + return false; + } + + public static ApplicationHostDTO fromVO(HostInfoVO hostInfoVO) { + if (hostInfoVO == null) { + return null; + } + ApplicationHostDTO hostInfoDTO = new ApplicationHostDTO(); + hostInfoDTO.setHostId(hostInfoVO.getHostId()); + hostInfoDTO.setIp(hostInfoVO.getIp()); + hostInfoDTO.setDisplayIp(hostInfoVO.getDisplayIp()); + hostInfoDTO.setHostName(hostInfoVO.getHostName()); + if (hostInfoVO.getAgentStatus() != null) { + hostInfoDTO.setGseAgentStatus(hostInfoVO.getAgentStatus()); + } else { + hostInfoDTO.setGseAgentAlive(isGseAgentAlive(hostInfoVO)); + } + if (hostInfoVO.getCloudArea() != null) { + hostInfoDTO.setCloudAreaId(hostInfoVO.getCloudArea().getId()); + hostInfoDTO.setCloudAreaName(hostInfoVO.getCloudArea().getName()); + } + hostInfoDTO.setOsName(hostInfoVO.getOsName()); + hostInfoDTO.setOsTypeName(hostInfoVO.getOsTypeName()); + hostInfoDTO.setAgentId(hostInfoVO.getAgentId()); + hostInfoDTO.setCloudVendorName(hostInfoVO.getCloudVendorName()); + return hostInfoDTO; + } + public HostInfoVO toVO() { + HostInfoVO hostInfoVO = new HostInfoVO(); + hostInfoVO.setHostId(hostId); + hostInfoVO.setCloudArea(new CloudAreaInfoVO(cloudAreaId, cloudAreaName)); + hostInfoVO.setIp(ip); + hostInfoVO.setIpv6(ipv6); + hostInfoVO.setDisplayIp(displayIp); + hostInfoVO.setHostName(hostName); + hostInfoVO.setOsName(osName); + hostInfoVO.setOsTypeName(osTypeName); + hostInfoVO.setAgentStatus(gseAgentStatus); + hostInfoVO.setAlive(getAgentAliveValue()); + hostInfoVO.setAgentId(agentId); + hostInfoVO.setCloudVendorName(cloudVendorName); + return hostInfoVO; + } + + @JsonIgnore + public String getFinalAgentId() { + if (StringUtils.isNotBlank(agentId)) { + return agentId; + } + return getCloudIp(); + } + + public static List buildAgentIdList(List hosts) { + List agentIdList = new ArrayList<>(); + for (ApplicationHostDTO host : hosts) { + agentIdList.add(host.getFinalAgentId()); + } + return agentIdList; + } + + @JsonIgnore public String getCloudIp() { if (StringUtils.isNotBlank(cloudIp)) { return cloudIp; @@ -124,6 +260,7 @@ public String getCloudIp() { } } + @JsonIgnore public String getModuleIdsStr() { if (moduleId != null) { return moduleId.stream().map(Object::toString).collect(Collectors.joining(",")); @@ -131,6 +268,7 @@ public String getModuleIdsStr() { return null; } + @JsonIgnore public String getSetIdsStr() { if (setId != null) { return setId.stream().map(Object::toString).collect(Collectors.joining(",")); @@ -138,6 +276,7 @@ public String getSetIdsStr() { return null; } + @JsonIgnore public String getModuleTypeStr() { if (moduleType != null) { return moduleType.stream().map(Object::toString).collect(Collectors.joining(",")); @@ -145,6 +284,7 @@ public String getModuleTypeStr() { return null; } + @JsonIgnore public int getAgentStatusValue() { if (gseAgentAlive == null || !gseAgentAlive) { return JobConstants.GSE_AGENT_STATUS_VALUE_NOT_ALIVE; @@ -152,4 +292,52 @@ public int getAgentStatusValue() { return JobConstants.GSE_AGENT_STATUS_VALUE_ALIVE; } + @JsonIgnore + public String getHostIdOrCloudIp() { + if (hostId != null && hostId > 0) { + return String.valueOf(hostId); + } + return getCloudIp(); + } + + public HostDTO toHostDTO() { + HostDTO host = new HostDTO(); + host.setHostId(hostId); + host.setBkCloudId(cloudAreaId); + host.setBkCloudName(cloudAreaName); + host.setIp(ip); + host.setIpv6(ipv6); + host.setAgentId(agentId); + host.setAlive(getAgentAliveValue()); + host.setOsName(osName); + host.setOsType(osType); + host.setOsTypeName(osTypeName); + host.setHostname(hostName); + return host; + } + + public String preferFullIpv6() { + try { + if (StringUtils.isNotBlank(ipv6) && IpUtils.checkIpv6(ipv6)) { + return IpUtils.getFullIpv6ByCompressedOne(ipv6); + } + } catch (Exception e) { + String msg = MessageFormatter.format( + "Fail to getFullIpv6ByCompressedOne by {}", + ipv6 + ).getMessage(); + log.warn(msg, e); + } + return ipv6; + } + + /** + * 获取主机的ip，优先返回ipv4 + * + * @return 主机ipv4/ipv6, ipv4 优先 + */ + @JsonIgnore + public String getPrimaryIp() { + return StringUtils.isNotEmpty(ip) ? ip : ipv6; + } } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/BasicDTO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/BasicDTO.java index c1e3dfe50d..815e68b290 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/BasicDTO.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/BasicDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/BasicHostDTO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/BasicHostDTO.java new file mode 100644 index 0000000000..e664c07601 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/BasicHostDTO.java @@ -0,0 +1,52 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.model.dto; + +import com.fasterxml.jackson.annotation.JsonInclude; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + +/** + * 主机基础信息 + */ +@Data +@EqualsAndHashCode +@NoArgsConstructor +@AllArgsConstructor +@JsonInclude(JsonInclude.Include.NON_EMPTY) +public class BasicHostDTO { + + /** + * 主机ID + */ + private Long hostId; + /** + * CMDB数据的上次修改时间 + */ + private Long lastTime; + +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/BkUserDTO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/BkUserDTO.java index 2f71fcb040..423d82c290 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/BkUserDTO.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/BkUserDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/CheckRuleDTO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/CheckRuleDTO.java index a6679df0b8..2694b4f6f6 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/CheckRuleDTO.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/CheckRuleDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/CmdbTopoNodeDTO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/CmdbTopoNodeDTO.java index b99e354a8e..edc1f77560 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/CmdbTopoNodeDTO.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/CmdbTopoNodeDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,7 @@ package com.tencent.bk.job.common.model.dto; +import com.tencent.bk.job.common.annotation.PersistenceObject; import com.tencent.bk.job.common.model.vo.TargetNodeVO; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; @@ -31,11 +32,12 @@ import lombok.Data; import lombok.NoArgsConstructor; +@PersistenceObject @Data @NoArgsConstructor @AllArgsConstructor @ApiModel("业务拓扑节点") -public class CmdbTopoNodeDTO { +public class CmdbTopoNodeDTO implements Cloneable { @ApiModelProperty("节点ID") private Long id; @ApiModelProperty("节点类型，module-模块，set-集群，biz-业务") @@ -46,8 +48,8 @@ public static TargetNodeVO toVO(CmdbTopoNodeDTO cmdbTopoNodeDTO) { return null; } TargetNodeVO targetNode = new TargetNodeVO(); - targetNode.setId(cmdbTopoNodeDTO.getId()); - targetNode.setType(cmdbTopoNodeDTO.getNodeType()); + targetNode.setInstanceId(cmdbTopoNodeDTO.getId()); + targetNode.setObjectId(cmdbTopoNodeDTO.getNodeType()); return targetNode; } @@ -56,8 +58,17 @@ public static CmdbTopoNodeDTO fromVO(TargetNodeVO targetNode) { return null; } CmdbTopoNodeDTO cmdbTopoNodeDTO = new CmdbTopoNodeDTO(); - cmdbTopoNodeDTO.setId(targetNode.getId()); - cmdbTopoNodeDTO.setNodeType(targetNode.getType()); + cmdbTopoNodeDTO.setId(targetNode.getInstanceId()); + cmdbTopoNodeDTO.setNodeType(targetNode.getObjectId()); return cmdbTopoNodeDTO; } + + @SuppressWarnings("MethodDoesntCallSuperMethod") + @Override + public CmdbTopoNodeDTO clone() { + CmdbTopoNodeDTO nodeDTO = new CmdbTopoNodeDTO(); + nodeDTO.setId(id); + nodeDTO.setNodeType(nodeType); + return nodeDTO; + } } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/CommonCredential.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/CommonCredential.java index f35f1463a1..b716736ad1 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/CommonCredential.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/CommonCredential.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/Container.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/Container.java new file mode 100644 index 0000000000..6db3cab040 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/Container.java @@ -0,0 +1,234 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.model.dto; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.annotation.PersistenceObject; +import com.tencent.bk.job.common.model.openapi.v4.OpenApiContainerDTO; +import com.tencent.bk.job.common.model.vo.ContainerVO; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; + +import java.util.Map; +import java.util.Objects; + +/** + * 作业执行对象-容器模型 + */ +@Setter +@Getter +@NoArgsConstructor +@JsonInclude(JsonInclude.Include.NON_NULL) +@PersistenceObject +@Slf4j +public class Container implements Cloneable { + /** + * 容器在 cmdb 注册的 ID + */ + @JsonProperty("id") + private Long id; + + /** + * 容器 UID + */ + @JsonProperty("containerId") + private String containerId; + + /** + * 容器名称 + */ + private String name; + + /** + * 容器所在 Node 对应的主机ID + */ + @JsonProperty("nodeHostId") + private Long nodeHostId; + + /** + * node 主机 ip + */ + @JsonProperty("nodeIp") + private String nodeIp; + + /** + * 容器所在 Node 对应的 Agent ID + */ + @JsonProperty("nodeAgentId") + private String nodeAgentId; + + /** + * cluster在cmdb中的唯一ID + */ + @JsonProperty("clusterId") + private Long clusterId; + + /** + * 集群 ID + */ + @JsonProperty("clusterUID") + private String clusterUID; + + /** + * 集群名称 + */ + @JsonProperty("clusterName") + private String clusterName; + + /** + * namespace在cmdb中的唯一ID + */ + @JsonProperty("namespaceId") + private Long namespaceId; + + /** + * 命名空间名称 + */ + @JsonProperty("namespace") + private String namespace; + + /** + * POD 名称 + */ + @JsonProperty("podName") + private String podName; + + /** + * pod labels + */ + @JsonProperty("podLabels") + private Map podLabels; + + /** + * workload 类型(Deployment/Job ...) + */ + @JsonProperty("workloadType") + private String workloadType; + + /** + * workload在cmdb中的唯一ID + */ + @JsonProperty("workloadId") + private Long workloadId; + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Container container = (Container) o; + return id.equals(container.id); + } + + @Override + public int hashCode() { + return Objects.hash(id); + } + + @Override + @SuppressWarnings("all") + public Container clone() { + Container clone = new Container(); + clone.setId(id); + clone.setNodeHostId(nodeHostId); + clone.setNodeIp(nodeIp); + clone.setNodeAgentId(nodeAgentId); + clone.setContainerId(containerId); + clone.setPodLabels(podLabels); + clone.setClusterId(clusterId); + clone.setClusterUID(clusterUID); + clone.setClusterName(clusterName); + clone.setNamespaceId(namespaceId); + clone.setNamespace(namespace); + clone.setPodName(podName); + clone.setName(name); + clone.setWorkloadType(workloadType); + clone.setWorkloadId(workloadId); + return clone; + } + + public ContainerVO toContainerVO() { + ContainerVO vo = new ContainerVO(); + vo.setId(id); + vo.setName(name); + vo.setUid(containerId); + vo.setNodeHostId(nodeHostId); + vo.setNodeIp(nodeIp); + vo.setPodName(podName); + vo.setPodLabels(podLabels); + vo.setClusterId(clusterId); + vo.setClusterUID(clusterUID); + vo.setClusterName(clusterName); + vo.setNamespaceId(namespaceId); + vo.setNamespace(namespace); + vo.setWorkloadType(workloadType); + return vo; + } + + public OpenApiContainerDTO toOpenApiContainerDTO() { + OpenApiContainerDTO openApiContainerDTO = new OpenApiContainerDTO(); + openApiContainerDTO.setContainerId(containerId); + openApiContainerDTO.setId(id); + openApiContainerDTO.setNodeHostId(nodeHostId); + openApiContainerDTO.setName(name); + return openApiContainerDTO; + } + + public void updatePropsByContainer(Container container) { + this.containerId = container.getContainerId(); + this.nodeHostId = container.getNodeHostId(); + this.nodeAgentId = container.getNodeAgentId(); + this.nodeIp = container.getNodeIp(); + this.clusterId = container.getClusterId(); + this.clusterUID = container.getClusterUID(); + this.clusterName = container.getClusterName(); + this.namespaceId = container.getNamespaceId(); + this.namespace = container.getNamespace(); + this.workloadType = container.getWorkloadType(); + this.workloadId = container.getWorkloadId(); + this.podName = container.getPodName(); + this.podLabels = container.getPodLabels(); + this.name = container.getName(); + } + + /** + * 获取可读性的执行对象名称 + */ + @JsonIgnore + public String getReadabilityName() { + StringBuilder nameBuilder = new StringBuilder(256); + if (clusterId != null) { + nameBuilder.append(clusterId).append("::"); + } + if (namespace != null) { + nameBuilder.append(namespace).append("::"); + } + nameBuilder.append(podName).append("::").append(name).append("::").append(containerId); + return nameBuilder.toString(); + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/DynamicGroupWithHost.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/DynamicGroupWithHost.java index e4b0816b10..514705a22e 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/DynamicGroupWithHost.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/DynamicGroupWithHost.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -53,6 +53,8 @@ public class DynamicGroupWithHost { private String name; + private String lastTime; + private String type; private List ipList; diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/HostDTO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/HostDTO.java index 8bb9951864..ec099022c1 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/HostDTO.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/HostDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,28 +28,31 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.tencent.bk.job.common.annotation.PersistenceObject; +import com.tencent.bk.job.common.model.HostCompositeKey; +import com.tencent.bk.job.common.model.openapi.v4.OpenApiHostDTO; import com.tencent.bk.job.common.model.vo.CloudAreaInfoVO; import com.tencent.bk.job.common.model.vo.HostInfoVO; import com.tencent.bk.job.common.util.ip.IpUtils; -import lombok.AllArgsConstructor; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; import lombok.ToString; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import java.util.Objects; +import java.util.StringJoiner; /** - * 主机通用表示-内部服务使用 + * 作业执行对象-主机模型 */ @Setter @Getter -@AllArgsConstructor @NoArgsConstructor @JsonInclude(JsonInclude.Include.NON_NULL) @ToString @PersistenceObject +@Slf4j public class HostDTO implements Cloneable { /** * 主机ID @@ -76,52 +79,83 @@ public class HostDTO implements Cloneable { private String bkCloudName; /** - * 主机IP - ipv4 + * 主机IP - IPv4 */ @JsonProperty("ip") private String ip; - /** - * 主机显示IP - */ - @JsonProperty("displayIp") - private String displayIp; /** - * 主机IP - ipv6 + * 主机IP - IPv6 */ @JsonProperty("ipv6") private String ipv6; /** - * agent状态，0-异常，1-正常 + * agent存活状态，0-异常，1-正常 */ @JsonInclude(JsonInclude.Include.NON_NULL) private Integer alive; - public HostDTO(Long bkCloudId, String ip) { + /** + * 操作系统名称 + */ + private String osName; + + /** + * 操作系统类型 + */ + private String osType; + + /** + * 操作系统类型名称 + */ + private String osTypeName; + /** + * 主机名称 + */ + private String hostname; + + /** + * 所属云厂商ID + */ + private String cloudVendorId; + + /** + * 所属云厂商名称 + */ + private String cloudVendorName; + + /** + * 管控区域:IPv4 + */ + @JsonIgnore + private String cloudIp; + + @Deprecated + public HostDTO(Long bkCloudId, String ipv4) { this.bkCloudId = bkCloudId; - this.ip = ip; + this.ip = ipv4; + this.cloudIp = buildCloudIp(bkCloudId, ipv4); } - public static HostDTO fromHostId(Long hostId) { - HostDTO hostDTO = new HostDTO(); - hostDTO.setHostId(hostId); - return hostDTO; + public HostDTO(Long hostId) { + this.hostId = hostId; + } + + public HostDTO(Long hostId, Long bkCloudId, String ip) { + this.hostId = hostId; + this.bkCloudId = bkCloudId; + this.ip = ip; } - public static HostDTO fromHostIdAndCloudIp(Long hostId, String cloudIp) { + public static HostDTO fromHostId(Long hostId) { HostDTO hostDTO = new HostDTO(); hostDTO.setHostId(hostId); - if (StringUtils.isNotBlank(cloudIp)) { - String[] ipProps = cloudIp.split(IpUtils.COLON); - hostDTO.setBkCloudId(Long.valueOf(ipProps[0])); - hostDTO.setIp(ipProps[1]); - } - return hostDTO; } + @Deprecated public static HostDTO fromCloudIp(String cloudIp) { if (!IpUtils.checkCloudIp(cloudIp)) { throw new IllegalArgumentException("Invalid cloudIp : " + cloudIp); @@ -130,55 +164,95 @@ public static HostDTO fromCloudIp(String cloudIp) { return new HostDTO(Long.valueOf(ipProps[0]), ipProps[1]); } + @Deprecated + public static HostDTO fromHostIdOrCloudIp(Long hostId, String cloudIp) { + HostDTO host = new HostDTO(); + host.setHostId(hostId); + if (StringUtils.isNotEmpty(cloudIp)) { + String[] ipProps = cloudIp.split(IpUtils.COLON); + host.setBkCloudId(Long.valueOf(ipProps[0])); + host.setIp(ipProps[1]); + } + return host; + } + + /** + * 返回主机 云区域:ipv4 + */ public String toCloudIp() { + if (StringUtils.isNotEmpty(cloudIp)) { + return cloudIp; + } + if (StringUtils.isEmpty(ip)) { + return null; + } else { + cloudIp = buildCloudIp(bkCloudId, ip); + return cloudIp; + } + } + + private String buildCloudIp(Long bkCloudId, String ip) { return bkCloudId + ":" + ip; } - public String getDisplayIp() { - if (StringUtils.isNotEmpty(displayIp)) { - return displayIp; + /** + * 返回主机 云区域:ipv6 + */ + public String toCloudIpv6() { + if (StringUtils.isEmpty(ipv6)) { + return null; } else { - return ip; + return buildCloudIp(bkCloudId, ipv6); } } - public static HostInfoVO toVO(HostDTO host) { - if (host == null) { + public HostInfoVO toHostInfoVO() { + HostInfoVO hostInfoVO = new HostInfoVO(); + hostInfoVO.setHostId(hostId); + hostInfoVO.setCloudArea(new CloudAreaInfoVO(bkCloudId, bkCloudName)); + hostInfoVO.setIp(ip); + hostInfoVO.setIpv6(ipv6); + hostInfoVO.setHostName(hostname); + hostInfoVO.setOsName(osName); + hostInfoVO.setOsTypeName(osTypeName); + hostInfoVO.setAlive(alive); + hostInfoVO.setAgentId(agentId); + hostInfoVO.setCloudVendorName(cloudVendorName); + return hostInfoVO; + } + + public static HostDTO fromHostInfoVO(HostInfoVO hostInfoVO) { + if (hostInfoVO == null) { return null; } - HostInfoVO hostInfo = new HostInfoVO(); - hostInfo.setIp(host.getIp()); - hostInfo.setAlive(host.getAlive()); - CloudAreaInfoVO cloudAreaInfo = new CloudAreaInfoVO(); - cloudAreaInfo.setId(host.getBkCloudId()); - cloudAreaInfo.setName(host.getBkCloudName()); - hostInfo.setCloudAreaInfo(cloudAreaInfo); - return hostInfo; - } - - public static HostDTO fromVO(HostInfoVO hostInfo) { - if (hostInfo == null) { - return null; + HostDTO hostDTO = new HostDTO(); + hostDTO.setHostId(hostInfoVO.getHostId()); + hostDTO.setIp(hostInfoVO.getIp()); + hostDTO.setIpv6(hostInfoVO.getIpv6()); + CloudAreaInfoVO cloudAreaInfo = hostInfoVO.getCloudArea(); + if (cloudAreaInfo != null) { + hostDTO.setBkCloudId(cloudAreaInfo.getId()); + hostDTO.setBkCloudName(cloudAreaInfo.getName()); } - HostDTO host = new HostDTO(); - host.setIp(hostInfo.getIp()); - host.setBkCloudId(hostInfo.getCloudAreaInfo().getId()); - host.setBkCloudName(hostInfo.getCloudAreaInfo().getName()); - host.setAlive(hostInfo.getAlive()); - return host; + hostDTO.setAlive(hostInfoVO.getAgentStatus()); + hostDTO.setOsName(hostInfoVO.getOsName()); + hostDTO.setOsTypeName(hostInfoVO.getOsTypeName()); + return hostDTO; } @Override public boolean equals(Object o) { if (this == o) return true; if (o == null || getClass() != o.getClass()) return false; - HostDTO hostDTO = (HostDTO) o; - if (hostId != null) { - return hostId.equals(hostDTO.getHostId()); + HostDTO otherHost = (HostDTO) o; + if (hostId != null && otherHost.getHostId() != null) { + return hostId.equals(otherHost.getHostId()); + } else if (this.getIp() != null && otherHost.getIp() != null) { + // 兼容没有hostId,只有ip的的场景 + return bkCloudId.equals(otherHost.bkCloudId) && + ip.equals(otherHost.ip); } else { - // 兼容没有hostId的场景 - return bkCloudId.equals(hostDTO.bkCloudId) && - ip.equals(hostDTO.ip); + return false; } } @@ -192,6 +266,7 @@ public int hashCode() { } } + @SuppressWarnings("all") public HostDTO clone() { HostDTO clone = new HostDTO(); clone.setHostId(hostId); @@ -210,11 +285,67 @@ public HostDTO clone() { * @return 主机KEY */ @JsonIgnore - public String getUniqueKey() { - if (hostId != null) { - return "HOST_ID:" + hostId; - } else { - return "HOST_IP:" + toCloudIp(); + public HostCompositeKey getUniqueKey() { + return HostCompositeKey.ofHost(this); + } + + /** + * 获取主机的ip，优先返回ipv4 + * + * @return 主机ipv4/ipv6, ipv4 优先 + */ + @JsonIgnore + public String getPrimaryIp() { + return StringUtils.isNotEmpty(ip) ? ip : ipv6; + } + + /** + * 获取主机的管控区域 ID+ip，优先返回ipv4 + * + * @return 主机ipv4/ipv6, ipv4 优先 + */ + @JsonIgnore + public String getPrimaryIpWithBkNetId() { + return bkCloudId + ":" + (StringUtils.isNotEmpty(ip) ? ip : ipv6); + } + + public String toStringBasic() { + return new StringJoiner(", ", HostDTO.class.getSimpleName() + "[", "]") + .add("hostId=" + hostId) + .add("bkCloudId=" + bkCloudId) + .add("ip='" + ip + "'") + .add("ipv6='" + ipv6 + "'") + .toString(); + } + + public void updateByHost(HostDTO host) { + if (host == null) { + return; } + this.hostId = host.getHostId(); + this.agentId = host.getAgentId(); + this.bkCloudId = host.getBkCloudId(); + this.bkCloudName = host.getBkCloudName(); + this.ip = host.getIp(); + this.ipv6 = host.getIpv6(); + this.osName = host.getOsName(); + this.osType = host.getOsType(); + this.osTypeName = host.getOsTypeName(); + this.alive = host.getAlive(); + this.cloudVendorId = host.getCloudVendorId(); + this.cloudVendorName = host.getCloudVendorName(); + this.hostname = host.getHostname(); + } + + public OpenApiHostDTO toOpenApiHostDTO() { + OpenApiHostDTO openApiHostDTO = new OpenApiHostDTO(); + openApiHostDTO.setHostId(hostId); + openApiHostDTO.setBkCloudId(bkCloudId); + openApiHostDTO.setBkCloudName(bkCloudName); + openApiHostDTO.setIp(ip); + openApiHostDTO.setIpv6(ipv6); + openApiHostDTO.setAlive(alive); + openApiHostDTO.setAgentId(agentId); + return openApiHostDTO; } } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/HostSimpleDTO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/HostSimpleDTO.java new file mode 100644 index 0000000000..dedc8247b7 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/HostSimpleDTO.java @@ -0,0 +1,149 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.model.dto; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +import java.util.ArrayList; +import java.util.List; + +/** + * 主机 + */ +@Data +@EqualsAndHashCode +@NoArgsConstructor +@AllArgsConstructor +@Slf4j +public class HostSimpleDTO { + + /** + * 主机ID + */ + private Long hostId; + /** + * cmdb业务ID + */ + private Long bizId; + /** + * 主机Agent状态，取值参考AgentStatusEnum + */ + private Integer agentAliveStatus; + /** + * 云区域+ip + */ + private String cloudIp; + + /** + * AgentId + */ + private String agentId; + + /** + * Job业务ID + */ + private Long appId; + + /** + * IPv6 + */ + private String ipv6; + + /** + * 主机名称 + */ + private String hostName; + + /** + * 操作系统 + */ + private String osName; + + /** + * 操作系统类型 + */ + private String osType; + + /** + * IPv4(主机的第一个IPv4地址,完整的IPv4地址使用displayIp字段) + */ + private String ip; + + /** + * 云区域ID + */ + private Long cloudAreaId; + + @JsonIgnore + public String getFinalAgentId() { + if (StringUtils.isNotBlank(agentId)) { + return agentId; + } + return cloudIp; + } + + @JsonIgnore + public String getHostIdOrCloudIp() { + if (hostId != null && hostId > 0) { + return String.valueOf(hostId); + } + return cloudIp; + } + + public static List buildAgentIdList(List hosts) { + List agentIdList = new ArrayList<>(); + for (HostSimpleDTO host : hosts) { + if (StringUtils.isBlank(host.getIp()) && StringUtils.isBlank(host.getAgentId())) { + log.warn("buildAgentIdList, ip and agentId is blank, {}", host); + continue; + } + agentIdList.add(host.getFinalAgentId()); + } + return agentIdList; + } + + public ApplicationHostDTO convertToHostDTO() { + ApplicationHostDTO hostDTO = new ApplicationHostDTO(); + hostDTO.setIp(this.getIp()); + hostDTO.setCloudAreaId(this.getCloudAreaId()); + hostDTO.setCloudIp(this.getCloudIp()); + hostDTO.setGseAgentAlive(this.getAgentAliveStatus() == 1); + hostDTO.setBizId(this.getBizId()); + hostDTO.setHostId(this.getHostId()); + hostDTO.setAgentId(this.getAgentId()); + hostDTO.setAppId(this.getAppId()); + hostDTO.setIpv6(this.getIpv6()); + hostDTO.setHostName(this.getHostName()); + hostDTO.setOsName(this.getOsName()); + hostDTO.setOsType(this.getOsType()); + return hostDTO; + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/HostStatusNumStatisticsDTO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/HostStatusNumStatisticsDTO.java new file mode 100644 index 0000000000..9ed8de69a0 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/HostStatusNumStatisticsDTO.java @@ -0,0 +1,29 @@ +package com.tencent.bk.job.common.model.dto; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + +/** + * host状态数量统计 + */ +@Data +@EqualsAndHashCode +@NoArgsConstructor +@AllArgsConstructor +public class HostStatusNumStatisticsDTO { + public static final String KEY_AGENT_ALIVE = "gseAgentAlive"; + public static final String KEY_HOST_NUM = "hostNum"; + + /** + * 主机Agent状态 + */ + private int gseAgentAlive; + + /** + * 主机数量 + */ + private int hostNum; + +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/HostTopoRelationDTO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/HostTopoRelationDTO.java index f69d4639e1..4cdc8275d2 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/HostTopoRelationDTO.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/HostTopoRelationDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/KeyValue.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/KeyValue.java index 9b9b2f896e..68e3cc121e 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/KeyValue.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/KeyValue.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/PageDTO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/PageDTO.java deleted file mode 100644 index 82daa9a5eb..0000000000 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/PageDTO.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.model.dto; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -/** - * @description - * @date 2019/3/4 - */ -@Data -@NoArgsConstructor -@AllArgsConstructor -public class PageDTO { - private Integer start; - private Integer limit = 200; - private String sort; -} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/PlatDTO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/PlatDTO.java index fae977b136..a1c915c254 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/PlatDTO.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/PlatDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/PlatformInfoDTO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/PlatformInfoDTO.java index 7d6c85f545..15dff959f3 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/PlatformInfoDTO.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/PlatformInfoDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/ResourceScope.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/ResourceScope.java index 82fa4f5860..108a7f5f87 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/ResourceScope.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/ResourceScope.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -63,6 +63,21 @@ public ResourceScope(ResourceScopeTypeEnum type, String id) { this.id = id; } + public String toBasicStr() { + return "(" + type + "," + id + ")"; + } + + public boolean isBiz() { + return type == ResourceScopeTypeEnum.BIZ; + } + + public boolean isBizSet() { + return type == ResourceScopeTypeEnum.BIZ_SET; + } + + public String toResourceScopeUniqueId() { + return type.getValue() + ":" + id; + } @Override public String toString() { diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/UserRoleInfoDTO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/UserRoleInfoDTO.java index d6dc7a892f..537bfbad7d 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/UserRoleInfoDTO.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/UserRoleInfoDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -65,8 +65,8 @@ public boolean validate() { if (CollectionUtils.isEmpty(userList) && CollectionUtils.isEmpty(roleList)) { return false; } - userList = userList.parallelStream().filter(Objects::nonNull).collect(Collectors.toList()); - roleList = roleList.parallelStream().filter(Objects::nonNull).collect(Collectors.toList()); + userList = userList.stream().filter(Objects::nonNull).collect(Collectors.toList()); + roleList = roleList.stream().filter(Objects::nonNull).collect(Collectors.toList()); return true; } } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/WhiteListIpDTO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/WhiteListIpDTO.java index 90edfa9d41..a80887ff1f 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/WhiteListIpDTO.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/WhiteListIpDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/notify/CustomNotifyDTO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/notify/CustomNotifyDTO.java new file mode 100644 index 0000000000..7b9c08243a --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/notify/CustomNotifyDTO.java @@ -0,0 +1,82 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.model.dto.notify; + +import lombok.Data; +import org.apache.commons.collections.CollectionUtils; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Data +public class CustomNotifyDTO { + /** + * 自定义通知角色 + */ + private List roleList; + + /** + * 自定义额外通知人 + */ + private List extraObserverList; + + /** + * 自定义通知，当notifyType为CUSTOM时生效 + * 执行状态与对应通知渠道列表 + */ + private List customNotifyChannel; + + public void buildCustomNotifyChannel(Map> map) { + this.setCustomNotifyChannel( + map.entrySet().stream() + .filter(entry -> ExecuteStatusEnum.hasName(entry.getKey())) + .map(entry -> { + StatusNotifyChannel channel = new StatusNotifyChannel(); + channel.setExecuteStatus(ExecuteStatusEnum.valueOf(entry.getKey())); + channel.setChannelList(entry.getValue()); + return channel; + }).collect(Collectors.toList()) + ); + } + + public static CustomNotifyVO toVO(CustomNotifyDTO customNotifyDTO) { + CustomNotifyVO cronJobCustomNotifyVO = new CustomNotifyVO(); + cronJobCustomNotifyVO.setRoleList(customNotifyDTO.getRoleList()); + cronJobCustomNotifyVO.setExtraObserverList(customNotifyDTO.getExtraObserverList()); + if (CollectionUtils.isNotEmpty(customNotifyDTO.getCustomNotifyChannel())) { + Map> customNotifyChannelMap = new HashMap<>(); + customNotifyDTO.getCustomNotifyChannel().forEach(statusNotifyChannel -> + customNotifyChannelMap.put( + statusNotifyChannel.getExecuteStatus().getName(), + statusNotifyChannel.getChannelList()) + ); + cronJobCustomNotifyVO.setResourceStatusChannelMap(customNotifyChannelMap); + } + return cronJobCustomNotifyVO; + } + +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/notify/CustomNotifyVO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/notify/CustomNotifyVO.java new file mode 100644 index 0000000000..d81002e919 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/notify/CustomNotifyVO.java @@ -0,0 +1,52 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.model.dto.notify; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.util.List; +import java.util.Map; + +@Data +@ApiModel("定时任务级别的自定义通知配置") +public class CustomNotifyVO { + + /** + * 自定义通知角色 + */ + @ApiModelProperty(value = "任务角色（通知对象）列表", required = true) + private List roleList; + + /** + * 自定义额外通知人 + */ + @ApiModelProperty(value = "额外通知人列表", required = false) + private List extraObserverList; + + @ApiModelProperty(value = "状态通知渠道列表，key:执行状态(SUCCESS,FAIL),value:通知渠道列表", required = true) + private Map> resourceStatusChannelMap; +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/notify/ExecuteStatusEnum.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/notify/ExecuteStatusEnum.java new file mode 100644 index 0000000000..c37e280a9d --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/notify/ExecuteStatusEnum.java @@ -0,0 +1,55 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.model.dto.notify; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +@Getter +@AllArgsConstructor +public enum ExecuteStatusEnum { + /** + * 成功 + */ + SUCCESS(1, "SUCCESS"), + /** + * 失败 + */ + FAIL(2, "FAIL"); + + private final int value; + + private final String name; + + public static boolean hasName(String name) { + for (ExecuteStatusEnum statusEnum : ExecuteStatusEnum.values()) { + if (statusEnum.getName().equals(name)) { + return true; + } + } + return false; + } + +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/notify/StatusNotifyChannel.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/notify/StatusNotifyChannel.java new file mode 100644 index 0000000000..1912da9f4b --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/dto/notify/StatusNotifyChannel.java @@ -0,0 +1,43 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.model.dto.notify; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.util.List; + + +@Data +@ApiModel("定时任务状态通知渠道") +public class StatusNotifyChannel { + + @ApiModelProperty(value = "资源状态Code（SUCCESS/FAIL）", required = true) + private ExecuteStatusEnum executeStatus; + + @ApiModelProperty(value = "通知渠道Code列表", required = true) + private List channelList; +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/error/BadRequestDetailDTO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/error/BadRequestDetailDTO.java index 76edbf6f5b..adaa5a68f0 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/error/BadRequestDetailDTO.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/error/BadRequestDetailDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB 蓝鲸作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB 蓝鲸作业平台 is licensed under the MIT license. * @@ -40,7 +40,7 @@ @NoArgsConstructor public class BadRequestDetailDTO { - @JsonProperty("field_violations") + @JsonProperty("fieldViolations") private List fieldViolations; /** diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/error/ErrorDetailDTO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/error/ErrorDetailDTO.java index 563ad17611..ce5b677da6 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/error/ErrorDetailDTO.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/error/ErrorDetailDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -37,7 +37,7 @@ public class ErrorDetailDTO { */ private Integer type; - @JsonProperty("bad_request_detail") + @JsonProperty("badRequestDetail") private BadRequestDetailDTO badRequestDetail; public ErrorDetailDTO(BadRequestDetailDTO badRequestDetail) { diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/error/ErrorType.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/error/ErrorType.java index 4d34b4b5c7..a2e780600e 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/error/ErrorType.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/error/ErrorType.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB 蓝鲸作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB 蓝鲸作业平台 is licensed under the MIT license. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/error/FieldViolationDTO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/error/FieldViolationDTO.java index ceee2904e1..55179a66cf 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/error/FieldViolationDTO.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/error/FieldViolationDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -33,7 +33,7 @@ public class FieldViolationDTO { @JsonProperty("field") private String field; - @JsonProperty("rejected_value") + @JsonProperty("rejectedValue") private Object rejectedValue; @JsonProperty("description") private String description; diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/http/HttpReq.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/http/HttpReq.java index 7f9f1777e7..fbf693fe42 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/http/HttpReq.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/http/HttpReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/iam/AuthResultDTO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/iam/AuthResultDTO.java index d975e3b433..f00d6f8e27 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/iam/AuthResultDTO.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/iam/AuthResultDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/iam/PathInfoDTO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/iam/PathInfoDTO.java index ca821aae12..7b7124d009 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/iam/PathInfoDTO.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/iam/PathInfoDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/iam/PermissionActionResourceDTO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/iam/PermissionActionResourceDTO.java index 7a9b8b0faf..c476fd7a94 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/iam/PermissionActionResourceDTO.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/iam/PermissionActionResourceDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/iam/PermissionResourceDTO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/iam/PermissionResourceDTO.java index a64ecf41b9..4b2e92df1f 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/iam/PermissionResourceDTO.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/iam/PermissionResourceDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/iam/PermissionResourceGroupDTO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/iam/PermissionResourceGroupDTO.java index 7f2171b8e8..2610b8399b 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/iam/PermissionResourceGroupDTO.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/iam/PermissionResourceGroupDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/EsbCmdbTopoNodeDTO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/openapi/v3/EsbCmdbTopoNodeDTO.java similarity index 82% rename from src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/EsbCmdbTopoNodeDTO.java rename to src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/openapi/v3/EsbCmdbTopoNodeDTO.java index ba3916a4d4..c30fbccf8f 100644 --- a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/EsbCmdbTopoNodeDTO.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/openapi/v3/EsbCmdbTopoNodeDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,12 +22,15 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.common.esb.model.job; +package com.tencent.bk.job.common.model.openapi.v3; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; import com.tencent.bk.job.common.model.dto.CmdbTopoNodeDTO; import lombok.Data; +import javax.validation.constraints.NotNull; + /** * ESB接口 CMDB topo节点实体类 */ @@ -37,12 +40,16 @@ public class EsbCmdbTopoNodeDTO { * topo节点ID */ @JsonProperty("id") + @JsonPropertyDescription("Cmdb topo node id") + @NotNull(message = "{validation.constraints.TopoNodeId_null.message}") private Long id; /** * topo节点类型 */ @JsonProperty("node_type") + @JsonPropertyDescription("Cmdb topo node type") + @NotNull(message = "{validation.constraints.TopoNodeType_null.message}") private String nodeType; public static EsbCmdbTopoNodeDTO fromCmdbTopoNode(CmdbTopoNodeDTO ccTopoNode) { diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/openapi/v3/EsbDynamicGroupDTO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/openapi/v3/EsbDynamicGroupDTO.java new file mode 100644 index 0000000000..7d486e136e --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/openapi/v3/EsbDynamicGroupDTO.java @@ -0,0 +1,43 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.model.openapi.v3; + +import com.fasterxml.jackson.annotation.JsonPropertyDescription; +import lombok.Data; + +import javax.validation.constraints.NotNull; + +/** + * 动态分组 + */ +@Data +public class EsbDynamicGroupDTO { + /** + * 动态分组ID + */ + @JsonPropertyDescription("Cmdb dynamic group id") + @NotNull(message = "{validation.constraints.EmptyDynamicGroupId.message}") + private String id; +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/openapi/v4/OpenApiContainerDTO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/openapi/v4/OpenApiContainerDTO.java new file mode 100644 index 0000000000..6633211109 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/openapi/v4/OpenApiContainerDTO.java @@ -0,0 +1,67 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.model.openapi.v4; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.extern.slf4j.Slf4j; + +/** + * 作业执行对象-容器模型 + */ +@Setter +@Getter +@NoArgsConstructor +@JsonInclude(JsonInclude.Include.NON_NULL) +@Slf4j +public class OpenApiContainerDTO { + /** + * 容器在 cmdb 注册的 ID + */ + @JsonProperty("id") + private Long id; + + /** + * 容器 ID + */ + @JsonProperty("container_id") + private String containerId; + + + /** + * 容器所在 Node 对应的主机ID + */ + @JsonProperty("node_host_id") + private Long nodeHostId; + + /** + * 容器名称 + */ + @JsonProperty("name") + private String name; +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/openapi/v4/OpenApiExecuteObjectDTO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/openapi/v4/OpenApiExecuteObjectDTO.java new file mode 100644 index 0000000000..61e1a20813 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/openapi/v4/OpenApiExecuteObjectDTO.java @@ -0,0 +1,68 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.model.openapi.v4; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; +import com.tencent.bk.job.common.constant.ExecuteObjectTypeEnum; +import lombok.Data; + +/** + * OpenAPI - 执行对象定义 + */ +@Data +public class OpenApiExecuteObjectDTO { + + /** + * 执行对象类型 + * + * @see ExecuteObjectTypeEnum + */ + @JsonPropertyDescription("Execute object type") + private Integer type; + + /** + * 执行对象 ID，比如主机 ID、容器 ID + */ + @JsonPropertyDescription("Execute object resource id") + @JsonProperty("resource_id") + private String resourceId; + + /** + * 容器 + */ + @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonPropertyDescription("Container") + private OpenApiContainerDTO container; + + /** + * 主机 + */ + @JsonPropertyDescription("Host") + @JsonInclude(JsonInclude.Include.NON_NULL) + private OpenApiHostDTO host; + +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/openapi/v4/OpenApiExecuteTargetDTO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/openapi/v4/OpenApiExecuteTargetDTO.java new file mode 100644 index 0000000000..554a18c349 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/openapi/v4/OpenApiExecuteTargetDTO.java @@ -0,0 +1,87 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.model.openapi.v4; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; +import com.tencent.bk.job.common.model.openapi.v3.EsbCmdbTopoNodeDTO; +import com.tencent.bk.job.common.model.openapi.v3.EsbDynamicGroupDTO; +import com.tencent.bk.job.common.validation.ExecuteTargetNotEmpty; +import lombok.Data; +import org.apache.commons.collections.CollectionUtils; + +import javax.validation.Valid; +import java.util.List; + +/** + * 执行目标 + */ +@Data +@ExecuteTargetNotEmpty +public class OpenApiExecuteTargetDTO { + + /** + * 静态主机列表 + */ + @JsonProperty("host_list") + @JsonInclude(JsonInclude.Include.NON_EMPTY) + @JsonPropertyDescription("Hosts") + @Valid + private List hosts; + + /** + * cmdb 动态分组列表 + */ + @JsonProperty("host_dynamic_group_list") + @JsonPropertyDescription("Cmdb host dynamic groups") + @Valid + private List hostDynamicGroups; + + /** + * cmdb 主机拓扑节点列表 + */ + @JsonProperty("host_topo_node_list") + @JsonPropertyDescription("Cmdb host topo nodes") + @Valid + private List hostTopoNodes; + + /** + * k8s 容器过滤器列表 + */ + @JsonProperty("kube_container_filters") + @JsonPropertyDescription("Kube container filters") + @Valid + private List kubeContainerFilters; + + @JsonIgnore + public Boolean isValidExecuteTarget() { + return CollectionUtils.isNotEmpty(hosts) + || CollectionUtils.isNotEmpty(hostDynamicGroups) + || CollectionUtils.isNotEmpty(hostTopoNodes) + || CollectionUtils.isNotEmpty(kubeContainerFilters); + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/openapi/v4/OpenApiHostDTO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/openapi/v4/OpenApiHostDTO.java new file mode 100644 index 0000000000..9414237c5a --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/openapi/v4/OpenApiHostDTO.java @@ -0,0 +1,136 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.model.openapi.v4; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; +import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.common.util.ip.IpUtils; +import com.tencent.bk.job.common.validation.EsbHostDTOGroupSequenceProvider; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import org.hibernate.validator.group.GroupSequenceProvider; + +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; +import javax.validation.constraints.Pattern; + +/** + * 主机 + */ +@Setter +@Getter +@NoArgsConstructor +@GroupSequenceProvider(EsbHostDTOGroupSequenceProvider.class) +public class OpenApiHostDTO { + + @JsonProperty("bk_host_id") + @JsonPropertyDescription("Host id") + @NotNull(message = "{validation.constraints.BkHostId_null.message}", groups = BkHostIdGroup.class) + @Min(value = 1L, message = "{validation.constraints.BkHostId_invalid.message}", groups = BkHostIdGroup.class) + private Long hostId; + + @JsonProperty("bk_cloud_id") + @NotNull(message = "{validation.constraints.InvalidBkCloudId.message}", groups = BkCloudIpGroup.class) + @Min(value = 0L, message = "{validation.constraints.InvalidBkCloudId.message}", groups = BkCloudIpGroup.class) + @JsonPropertyDescription("BK-Network Area") + private Long bkCloudId; + + @JsonProperty("bk_cloud_name") + @JsonPropertyDescription("BK-Network Area Name") + private String bkCloudName; + + @JsonProperty("ip") + @Pattern(regexp = "\\b((?!\\d\\d\\d)\\d+|1\\d\\d|2[0-4]\\d|25[0-5])\\.((?!\\d\\d\\d)" + + "\\d+|1\\d\\d|2[0-4]\\d|25[0-5])\\.((?!\\d\\d\\d)\\d+|1\\d\\d|2[0-4]\\d|25[0-5])\\.((?!\\d\\d\\d)" + + "\\d+|1\\d\\d|2[0-4]\\d|25[0-5])\\b", + message = "{validation.constraints.InvalidIp.message}", groups = BkCloudIpGroup.class) + @NotNull(message = "{validation.constraints.InvalidIp.message}", groups = BkCloudIpGroup.class) + @JsonPropertyDescription("ip") + private String ip; + + @JsonProperty("ipv6") + @JsonPropertyDescription("ipv6") + private String ipv6; + + @JsonProperty("bk_agent_id") + @JsonPropertyDescription("Agent Id") + private String agentId; + + @JsonProperty("alive") + @JsonPropertyDescription("Agent是否正常") + private Integer alive; + + public interface BkHostIdGroup { + } + + public interface BkCloudIpGroup { + } + + public OpenApiHostDTO(Long hostId, Long bkCloudId, String ip) { + this.hostId = hostId; + this.bkCloudId = bkCloudId; + this.ip = ip; + } + + public static OpenApiHostDTO fromApplicationHostInfo(ApplicationHostDTO applicationHostInfo) { + if (applicationHostInfo == null) { + return null; + } + OpenApiHostDTO esbIp = new OpenApiHostDTO(); + esbIp.setBkCloudId(applicationHostInfo.getCloudAreaId()); + esbIp.setIp(applicationHostInfo.getIp()); + esbIp.setHostId(applicationHostInfo.getHostId()); + return esbIp; + } + + public static OpenApiHostDTO fromHost(HostDTO host) { + if (host == null) { + return null; + } + OpenApiHostDTO esbIp = new OpenApiHostDTO(); + esbIp.setBkCloudId(host.getBkCloudId()); + esbIp.setBkCloudName(host.getBkCloudName()); + esbIp.setIp(host.getIp()); + esbIp.setIpv6(host.getIpv6()); + esbIp.setHostId(host.getHostId()); + esbIp.setAgentId(host.getAgentId()); + esbIp.setAlive(host.getAlive()); + return esbIp; + } + + public static OpenApiHostDTO fromCloudIp(String cloudIp) { + if (!IpUtils.checkCloudIp(cloudIp)) { + return null; + } + String[] ipProps = cloudIp.split(IpUtils.COLON); + OpenApiHostDTO host = new OpenApiHostDTO(); + host.setBkCloudId(Long.valueOf(ipProps[0])); + host.setIp(ipProps[1]); + return host; + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/openapi/v4/OpenApiKubeClusterFilterDTO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/openapi/v4/OpenApiKubeClusterFilterDTO.java new file mode 100644 index 0000000000..461ce8d721 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/openapi/v4/OpenApiKubeClusterFilterDTO.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.model.openapi.v4; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +import java.util.List; + +/** + * 执行目标-容器选择过滤器-按集群过滤 + */ +@Data +public class OpenApiKubeClusterFilterDTO { + + /** + * k8s 集群ID + */ + @JsonProperty("cluster_uid_list") + private List clusterUIDs; + +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/openapi/v4/OpenApiKubeContainerFilterDTO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/openapi/v4/OpenApiKubeContainerFilterDTO.java new file mode 100644 index 0000000000..e294c0ce4a --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/openapi/v4/OpenApiKubeContainerFilterDTO.java @@ -0,0 +1,81 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.model.openapi.v4; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.validation.KubeContainerFilterNotEmpty; +import lombok.Data; + +/** + * 执行目标-容器选择过滤器 + */ +@Data +@KubeContainerFilterNotEmpty +public class OpenApiKubeContainerFilterDTO { + + /** + * 集群过滤器 + */ + @JsonProperty("kube_cluster_filter") + private OpenApiKubeClusterFilterDTO clusterFilter; + + /** + * namespace 过滤器 + */ + @JsonProperty("kube_namespace_filter") + private OpenApiKubeNamespaceFilterDTO namespaceFilter; + + /** + * workload 过滤器 + */ + @JsonProperty("kube_workload_filter") + private OpenApiKubeWorkloadFilterDTO workloadFilter; + + /** + * pod 属性过滤器 + */ + @JsonProperty("kube_pod_filter") + private OpenApiKubePodFilterDTO podFilter; + + /** + * 容器属性过滤器 + */ + @JsonProperty("kube_container_prop_filter") + private OpenApiKubeContainerPropFilterDTO containerPropFilter; + + /** + * 标识一个没有设置任何条件的过滤器；默认值为 false。如果设置为 true, 将忽略其他的条件（kube_cluster_filter/kube_namespace_filter + * /kube_workload_filter/kube_pod_filter/kube_container_prop_filter)，返回业务下的所有容器 + */ + @JsonProperty("is_empty_filter") + private boolean emptyFilter; + + /** + * 是否从过滤结果集中选择任意一个容器作为执行对象（只有一个容器会被执行） + */ + @JsonProperty("fetch_any_one_container") + private boolean fetchAnyOneContainer; + +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/openapi/v4/OpenApiKubeContainerPropFilterDTO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/openapi/v4/OpenApiKubeContainerPropFilterDTO.java new file mode 100644 index 0000000000..e2e6eab6bf --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/openapi/v4/OpenApiKubeContainerPropFilterDTO.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.model.openapi.v4; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +import java.util.List; + +/** + * 执行目标-容器选择过滤器-按容器属性过滤 + */ +@Data +public class OpenApiKubeContainerPropFilterDTO { + + /** + * k8s 容器名称列表 + */ + @JsonProperty("container_name_list") + private List containerNames; + +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/openapi/v4/OpenApiKubeNamespaceFilterDTO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/openapi/v4/OpenApiKubeNamespaceFilterDTO.java new file mode 100644 index 0000000000..0561b8352c --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/openapi/v4/OpenApiKubeNamespaceFilterDTO.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.model.openapi.v4; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +import java.util.List; + +/** + * 执行目标-容器选择过滤器-按命名空间过滤 + */ +@Data +public class OpenApiKubeNamespaceFilterDTO { + + /** + * k8s 命名空间名称 + */ + @JsonProperty("namespace_name_list") + private List namespaces; + +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/openapi/v4/OpenApiKubePodFilterDTO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/openapi/v4/OpenApiKubePodFilterDTO.java new file mode 100644 index 0000000000..4a8e3eaa93 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/openapi/v4/OpenApiKubePodFilterDTO.java @@ -0,0 +1,56 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.model.openapi.v4; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +import java.util.List; + +/** + * 执行目标-容器选择过滤器-按 POD 过滤 + */ +@Data +public class OpenApiKubePodFilterDTO { + + /** + * k8s pod 名称列表 + */ + @JsonProperty("pod_name_list") + private List podNames; + + /** + * k8s pod label selector + */ + @JsonProperty("label_selector") + private List labelSelector; + + /** + * k8s pod label selector 表达式 + */ + @JsonProperty("label_selector_expr") + private String labelSelectorExpr; + +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/openapi/v4/OpenApiKubeWorkloadFilterDTO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/openapi/v4/OpenApiKubeWorkloadFilterDTO.java new file mode 100644 index 0000000000..a7c69ffbbd --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/openapi/v4/OpenApiKubeWorkloadFilterDTO.java @@ -0,0 +1,49 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.model.openapi.v4; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +import java.util.List; + +/** + * 执行目标-容器选择过滤器-按 workload 过滤 + */ +@Data +public class OpenApiKubeWorkloadFilterDTO { + + /** + * workload 类型 + */ + private String kind; + + /** + * k8s workload 名称列表 + */ + @JsonProperty("workload_name_list") + private List workloadNames; + +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/openapi/v4/OpenApiLabelSelectExprDTO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/openapi/v4/OpenApiLabelSelectExprDTO.java new file mode 100644 index 0000000000..2967052de0 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/openapi/v4/OpenApiLabelSelectExprDTO.java @@ -0,0 +1,55 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.model.openapi.v4; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.constant.LabelSelectorOperatorEnum; +import lombok.Data; + +import java.util.List; + +/** + * Label selector 计算表达式 + */ +@Data +public class OpenApiLabelSelectExprDTO { + /** + * Label key + */ + @JsonProperty("key") + private String key; + + /** + * 计算操作符 + */ + @JsonProperty("operator") + private LabelSelectorOperatorEnum operator; + + /** + * Label values + */ + @JsonProperty("values") + private List values; +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/permission/AuthResultVO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/permission/AuthResultVO.java index 83a120533b..1ea48c94a5 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/permission/AuthResultVO.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/permission/AuthResultVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/permission/PermissionResourceVO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/permission/PermissionResourceVO.java index 1906b26093..05a2f1f4b3 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/permission/PermissionResourceVO.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/permission/PermissionResourceVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/permission/RequiredPermissionVO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/permission/RequiredPermissionVO.java index 6998a49d5d..69e40a23ed 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/permission/RequiredPermissionVO.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/permission/RequiredPermissionVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/vo/BasicVO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/vo/BasicVO.java index 09309be673..a0bb3f02ec 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/vo/BasicVO.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/vo/BasicVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/vo/CloudAreaInfoVO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/vo/CloudAreaInfoVO.java index 974aaa22e3..6ce9e8bd73 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/vo/CloudAreaInfoVO.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/vo/CloudAreaInfoVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,7 +28,6 @@ import com.tencent.bk.job.common.util.JobContextUtil; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; -import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @@ -36,7 +35,6 @@ * @since 13/12/2019 17:31 */ @Data -@AllArgsConstructor @NoArgsConstructor @ApiModel("云区域信息") @JsonInclude(JsonInclude.Include.NON_NULL) @@ -50,6 +48,11 @@ public class CloudAreaInfoVO { @ApiModelProperty("云区域名称") private String name; + public CloudAreaInfoVO(Long id, String name) { + this.id = id; + this.name = name; + } + public boolean validate(boolean isCreate) { if (id != null && id >= 0) { return true; diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/vo/ContainerVO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/vo/ContainerVO.java new file mode 100644 index 0000000000..2495aa7bef --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/vo/ContainerVO.java @@ -0,0 +1,87 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.model.vo; + +import com.fasterxml.jackson.annotation.JsonInclude; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +import java.util.Map; + +/** + * 容器 + */ +@Getter +@Setter +@ToString +@JsonInclude(JsonInclude.Include.NON_NULL) +@ApiModel("容器") +public class ContainerVO { + + @ApiModelProperty("容器资源 ID, 容器在 cmdb 中注册资源的 ID") + private Long id; + + @ApiModelProperty(value = "容器 ID", example = "docker://8812391923...") + private String uid; + + @ApiModelProperty("容器名称") + private String name; + + @ApiModelProperty("Pod名称") + private String podName; + + @ApiModelProperty("所属 pod labels") + private Map podLabels; + + @ApiModelProperty("所属 Node hostId") + private Long nodeHostId; + + @ApiModelProperty("所属 Node Ip") + private String nodeIp; + + @ApiModelProperty("所属 Node GSE agent 状态") + private String nodeAgentStatus; + + @ApiModelProperty("cluster在cmdb中的唯一ID") + private Long clusterId; + + @ApiModelProperty(value = "集群 ID", example = "BCS-K8S-00000") + private String clusterUID; + + @ApiModelProperty("集群名称") + private String clusterName; + + @ApiModelProperty("命名空间在 cmdb 的唯一 ID") + private Long namespaceId; + + @ApiModelProperty("命名空间名称") + private String namespace; + + @ApiModelProperty("workload 类型") + private String workloadType; +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/vo/DynamicGroupIdWithMeta.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/vo/DynamicGroupIdWithMeta.java new file mode 100644 index 0000000000..a044525cc1 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/vo/DynamicGroupIdWithMeta.java @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.model.vo; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.Map; + +@NoArgsConstructor +@Data +@ApiModel("动态分组ID及元数据") +public class DynamicGroupIdWithMeta { + @ApiModelProperty(value = "动态分组ID", required = true) + String id; + + @ApiModelProperty(value = "动态分组元数据") + Map meta; + + public DynamicGroupIdWithMeta(String id) { + this.id = id; + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/vo/HostInfoVO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/vo/HostInfoVO.java index 766d1bafe3..dccaf7b2ea 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/vo/HostInfoVO.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/vo/HostInfoVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,21 +24,25 @@ package com.tencent.bk.job.common.model.vo; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; -import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.util.JobContextUtil; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -import org.apache.commons.lang3.StringUtils; +import lombok.extern.slf4j.Slf4j; import java.util.Objects; /** * @since 7/11/2019 16:08 */ +@Slf4j @NoArgsConstructor @AllArgsConstructor @ApiModel("主机信息") @@ -46,59 +50,53 @@ @JsonInclude(JsonInclude.Include.NON_NULL) public class HostInfoVO { - @ApiModelProperty(value = "服务器 ID", required = true) + @ApiModelProperty(value = "主机ID", required = true) private Long hostId; @ApiModelProperty("主机 IP") private String ip; + @ApiModelProperty("主机 IPv6") + private String ipv6; + @ApiModelProperty("展示用的IP，主要针对多内网IP问题") private String displayIp; - @ApiModelProperty("描述") - private String ipDesc; + @ApiModelProperty("主机名称") + private String hostName; + + // agent状态：-2：未找到，-1：查询失败，0：初始安装，1：启动中，2：运行中，3：有损状态，4：繁忙，5：升级中，6：停止中，7：解除安装 + @JsonIgnore + private Integer agentStatus; @ApiModelProperty("agent 状态 0-异常 1-正常") private Integer alive; @ApiModelProperty("云区域信息") - private CloudAreaInfoVO cloudAreaInfo; + private CloudAreaInfoVO cloudArea; /** * 操作系统 */ @ApiModelProperty("操作系统") - private String os; + private String osName; - public static ApplicationHostDTO toDTO(HostInfoVO hostInfo) { - if (hostInfo == null) { - return null; - } - ApplicationHostDTO hostInfoDTO = new ApplicationHostDTO(); - hostInfoDTO.setHostId(hostInfo.getHostId()); - hostInfoDTO.setIp(hostInfo.getIp()); - hostInfoDTO.setDisplayIp(hostInfo.getDisplayIp()); - hostInfoDTO.setIpDesc(hostInfo.getIpDesc()); - if (hostInfo.getAlive() != null) { - hostInfoDTO.setGseAgentAlive(hostInfo.getAlive() == 1); - } else { - hostInfoDTO.setGseAgentAlive(false); - } - hostInfoDTO.setCloudAreaId(hostInfo.getCloudAreaInfo().getId()); - hostInfoDTO.setOs(hostInfo.getOs()); - return hostInfoDTO; - } + @ApiModelProperty("系统类型") + @JsonProperty("osType") + private String osTypeName; - public boolean validate(boolean isCreate) { - if (cloudAreaInfo == null) { - JobContextUtil.addDebugMessage("Missing host info cloud area info!"); - return false; - } - if (StringUtils.isNotBlank(ip) && cloudAreaInfo.validate(isCreate)) { - return true; + @ApiModelProperty("AgentId") + private String agentId; + + @ApiModelProperty("所属云厂商") + @JsonProperty("cloudVendor") + private String cloudVendorName; + + public void validate() throws InvalidParamException { + if (!JobContextUtil.isAllowMigration() && (hostId == null || hostId <= 0)) { + log.warn("Missing host_id!"); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } - JobContextUtil.addDebugMessage("Invalid ip or cloud area info!"); - return false; } @Override diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/vo/NotifyChannelVO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/vo/NotifyChannelVO.java index 4311884686..5ba786de1c 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/vo/NotifyChannelVO.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/vo/NotifyChannelVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/vo/TargetNodeVO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/vo/TargetNodeVO.java index d6178d126f..c8774eb5a1 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/vo/TargetNodeVO.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/vo/TargetNodeVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,30 +25,32 @@ package com.tencent.bk.job.common.model.vo; import com.fasterxml.jackson.annotation.JsonInclude; -import com.tencent.bk.job.common.util.JobContextUtil; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.InvalidParamException; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -/** - * @since 12/12/2019 15:07 - */ +@Slf4j @Data @ApiModel("目标节点信息") @JsonInclude(JsonInclude.Include.NON_NULL) public class TargetNodeVO { @ApiModelProperty(value = "节点 ID，对应拓扑树节点中的instanceId", required = true) - private Long id; + private Long instanceId; @ApiModelProperty(value = "节点类型 biz-业务 set-集群 module-模块 xxx-用户自定义节点类型，对应拓扑树节点中的objectId", required = true) - private String type; + private String objectId; - public boolean validate(boolean isCreate) { - if (id != null && id > 0 && StringUtils.isNotBlank(type)) { - return true; + public void validate() throws InvalidParamException { + if (instanceId == null || instanceId <= 0) { + log.warn("Invalid target node instanceId"); + } + if (StringUtils.isBlank(objectId)) { + log.warn("Invalid target node type"); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } - JobContextUtil.addDebugMessage("Target node info does not have id or type"); - return false; } } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/vo/TaskExecuteObjectsInfoVO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/vo/TaskExecuteObjectsInfoVO.java new file mode 100644 index 0000000000..cfab631e86 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/vo/TaskExecuteObjectsInfoVO.java @@ -0,0 +1,87 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.model.vo; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.InvalidParamException; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; + +import java.util.List; + +@Data +@ApiModel("任务执行对象信息") +@JsonInclude(JsonInclude.Include.NON_NULL) +@Slf4j +public class TaskExecuteObjectsInfoVO { + + @ApiModelProperty("主机列表") + private List hostList; + + @ApiModelProperty("主机拓扑节点 ID") + private List nodeList; + + @ApiModelProperty("主机动态分组") + private List dynamicGroupList; + + @ApiModelProperty("容器列表") + private List containerList; + + public void validate() throws InvalidParamException { + boolean allEmpty = true; + if (CollectionUtils.isNotEmpty(nodeList)) { + allEmpty = false; + for (TargetNodeVO targetNodeVO : nodeList) { + targetNodeVO.validate(); + } + } + if (CollectionUtils.isNotEmpty(dynamicGroupList)) { + allEmpty = false; + for (Object dynamicGroup : dynamicGroupList) { + if (dynamicGroup == null) { + log.warn("Host dynamic group id is empty!"); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); + } + } + } + if (CollectionUtils.isNotEmpty(hostList)) { + allEmpty = false; + for (HostInfoVO hostInfoVO : hostList) { + hostInfoVO.validate(); + } + } + if (CollectionUtils.isNotEmpty(containerList)) { + allEmpty = false; + } + if (allEmpty) { + log.warn("TaskExecuteObjects is empty!"); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); + } + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/vo/TaskHostNodeVO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/vo/TaskHostNodeVO.java index cf9fa87660..63bd23766a 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/vo/TaskHostNodeVO.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/vo/TaskHostNodeVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,62 +24,91 @@ package com.tencent.bk.job.common.model.vo; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; -import com.tencent.bk.job.common.util.JobContextUtil; +import com.fasterxml.jackson.core.type.TypeReference; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.InvalidParamException; +import com.tencent.bk.job.common.util.json.JsonUtils; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; -/** - * @since 1/11/2019 12:15 - */ @Data @ApiModel("主机节点信息") @JsonInclude(JsonInclude.Include.NON_NULL) +@Slf4j public class TaskHostNodeVO { - @ApiModelProperty("机器 IP 列表") - private List ipList; + @ApiModelProperty("主机列表") + private List hostList; - @ApiModelProperty("节点 ID") - private List topoNodeList; + @ApiModelProperty("拓扑节点 ID") + private List nodeList; - @ApiModelProperty("动态分组 ID") - private List dynamicGroupList; + @ApiModelProperty("动态分组") + private List dynamicGroupList; - public boolean validate(boolean isCreate) { + public void validate() throws InvalidParamException { boolean allEmpty = true; - if (CollectionUtils.isNotEmpty(topoNodeList)) { + if (CollectionUtils.isNotEmpty(nodeList)) { allEmpty = false; - for (TargetNodeVO targetNodeVO : topoNodeList) { - if (!targetNodeVO.validate(isCreate)) { - JobContextUtil.addDebugMessage("Host node info validate failed!"); - return false; - } + for (TargetNodeVO targetNodeVO : nodeList) { + targetNodeVO.validate(); } } if (CollectionUtils.isNotEmpty(dynamicGroupList)) { allEmpty = false; - for (String dynamicGroup : dynamicGroupList) { - if (!StringUtils.isNoneBlank(dynamicGroup)) { - JobContextUtil.addDebugMessage("Host dynamic group id is empty!"); - return false; + for (Object dynamicGroup : dynamicGroupList) { + if (dynamicGroup == null) { + log.warn("Host dynamic group id is empty!"); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } } } - if (CollectionUtils.isNotEmpty(ipList)) { + if (CollectionUtils.isNotEmpty(hostList)) { allEmpty = false; - for (HostInfoVO hostInfoVO : ipList) { - if (!hostInfoVO.validate(isCreate)) { - JobContextUtil.addDebugMessage("Host info validate failed!"); - return false; - } + for (HostInfoVO hostInfoVO : hostList) { + hostInfoVO.validate(); } } - return !allEmpty; + if (allEmpty) { + log.warn("TaskHostNode is empty!"); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); + } + } + + @JsonIgnore + public List getDynamicGroupIdList() { + if (org.springframework.util.CollectionUtils.isEmpty(dynamicGroupList)) { + return Collections.emptyList(); + } + List dynamicGroupIdList = new ArrayList<>(); + for (DynamicGroupIdWithMeta dynamicGroup : dynamicGroupList) { + DynamicGroupIdWithMeta dynamicGroupIdWithMeta = JsonUtils.fromJson( + JsonUtils.toJson(dynamicGroup), + new TypeReference() { + } + ); + dynamicGroupIdList.add(dynamicGroupIdWithMeta.getId()); + } + return dynamicGroupIdList; + } + + public void setDynamicGroupIdList(List dynamicGroupIdList) { + if (dynamicGroupIdList == null) { + return; + } + List dynamicGroupList = new ArrayList<>(); + for (String id : dynamicGroupIdList) { + dynamicGroupList.add(new DynamicGroupIdWithMeta(id)); + } + this.dynamicGroupList = dynamicGroupList; } } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/vo/TaskTargetVO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/vo/TaskTargetVO.java index 1bb347891d..bf02ac54f3 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/vo/TaskTargetVO.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/vo/TaskTargetVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,37 +24,67 @@ package com.tencent.bk.job.common.model.vo; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; -import com.tencent.bk.job.common.util.JobContextUtil; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.CompatibleType; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.InvalidParamException; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -/** - * @since 1/11/2019 12:08 - */ @Data -@ApiModel("执行目标信息") +@ApiModel("执行目标") @JsonInclude(JsonInclude.Include.NON_NULL) +@Slf4j public class TaskTargetVO { @ApiModelProperty(value = "全局变量名") private String variable; - @ApiModelProperty(value = "主机节点列表") + @ApiModelProperty(value = "主机节点信息, 版本升级之后作废") + @Deprecated + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.DEPLOY, + explain = "兼容 API， 发布完成后前端使用 executeObjectsInfo 参数，该参数可删除") private TaskHostNodeVO hostNodeInfo; - public boolean validate(boolean isCreate) { + @ApiModelProperty(value = "任务执行对象信息") + private TaskExecuteObjectsInfoVO executeObjectsInfo; + + public void validate() throws InvalidParamException { if (StringUtils.isNoneBlank(variable)) { hostNodeInfo = null; - return true; + executeObjectsInfo = null; + return; + } + + if (executeObjectsInfo != null) { + executeObjectsInfo.validate(); + } else if (hostNodeInfo != null) { + hostNodeInfo.validate(); + } else { + log.warn("TaskTarget is empty!"); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } - if (hostNodeInfo != null) { - return hostNodeInfo.validate(isCreate); + } + + @JsonIgnore + @ApiModelProperty(hidden = true) + public TaskExecuteObjectsInfoVO getExecuteObjectsInfoCompatibly() { + if (executeObjectsInfo != null) { + return executeObjectsInfo; + } else if (hostNodeInfo != null) { + TaskExecuteObjectsInfoVO taskExecuteObjectsInfoVO = new TaskExecuteObjectsInfoVO(); + taskExecuteObjectsInfoVO.setHostList(hostNodeInfo.getHostList()); + taskExecuteObjectsInfoVO.setNodeList(hostNodeInfo.getNodeList()); + taskExecuteObjectsInfoVO.setDynamicGroupList(hostNodeInfo.getDynamicGroupList()); + return taskExecuteObjectsInfoVO; } else { - JobContextUtil.addDebugMessage("Empty target info!"); - return false; + return null; } } + } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/vo/UserRoleInfoVO.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/vo/UserRoleInfoVO.java index 997f52ee20..3646e69bff 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/vo/UserRoleInfoVO.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/model/vo/UserRoleInfoVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,10 +25,12 @@ package com.tencent.bk.job.common.model.vo; import com.fasterxml.jackson.annotation.JsonInclude; -import com.tencent.bk.job.common.util.JobContextUtil; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.InvalidParamException; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import java.util.List; @@ -39,6 +41,7 @@ @Data @ApiModel("用户角色列表") @JsonInclude(JsonInclude.Include.NON_NULL) +@Slf4j public class UserRoleInfoVO { @ApiModelProperty("用户列表") private List userList; @@ -46,11 +49,10 @@ public class UserRoleInfoVO { @ApiModelProperty("角色列表") private List roleList; - public boolean validate() { + public void validate() throws InvalidParamException { if (CollectionUtils.isEmpty(userList) && CollectionUtils.isEmpty(roleList)) { - JobContextUtil.addDebugMessage("Approval user info is empty!"); - return false; + log.warn("Approval user info is empty!"); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } - return true; } } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/properties/JobSslProperties.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/properties/JobSslProperties.java new file mode 100644 index 0000000000..11a875d03d --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/properties/JobSslProperties.java @@ -0,0 +1,78 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.properties; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.util.StringUtils; + +@Getter +@Setter +public class JobSslProperties { + + /** + * 连接mongo是否启用ssl + */ + private boolean enabled = false; + + /** + * 信任库类型，默认PKCS12 + */ + private String trustStoreType = "PKCS12"; + + /** + * 信任库地址，暂时以文件形式（运维提供） + */ + private String trustStore; + + /** + * 信任库密码 + */ + private String trustStorePassword; + + /** + * 客户端证书存储类型，默认PKCS12 + */ + private String keyStoreType = "PKCS12"; + + /** + * 客户端的证书地址，文件形式 + */ + private String keyStore; + + /** + * 客户端证书存储密钥 + */ + private String keyStorePassword; + + /** + * 是否校验主机名 + */ + private boolean verifyHostname = false; + + public boolean isMutualTlsConfigured() { + return StringUtils.hasText(keyStore); + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/refreshable/config/ConfigRefreshHandler.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/refreshable/config/ConfigRefreshHandler.java new file mode 100644 index 0000000000..a14691b584 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/refreshable/config/ConfigRefreshHandler.java @@ -0,0 +1,40 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.refreshable.config; + +import java.util.Set; + +/** + * 配置刷新处理 + */ +public interface ConfigRefreshHandler { + /** + * 处理配置动态刷新 + * + * @param changedKeys 变化的 keys + * @return 是否成功处理 + */ + boolean handleConfigChange(Set changedKeys); +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/refreshable/config/RefreshableConfigStore.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/refreshable/config/RefreshableConfigStore.java new file mode 100644 index 0000000000..46c4617616 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/refreshable/config/RefreshableConfigStore.java @@ -0,0 +1,49 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.refreshable.config; + +import java.util.Set; + +/** + * 动态配置存储仓库 + */ +public interface RefreshableConfigStore { + + /** + * 初始化配置仓库，用于加载初始配置 + */ + default void init() { + + } + + /** + * 重载变更的配置 + * + * @param changedKeys 变化的 keys + * @param ignoreException 是否忽略加载配置异常；true - 忽略异常；false: 抛出异常 + * @return true: 重载成功 + */ + boolean handleConfigChange(Set changedKeys, boolean ignoreException); +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/resource/quota/AppQuotaLimit.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/resource/quota/AppQuotaLimit.java new file mode 100644 index 0000000000..ded0464f57 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/resource/quota/AppQuotaLimit.java @@ -0,0 +1,58 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.resource.quota; + +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + +import java.util.HashMap; +import java.util.Map; + +/** + * 配额限制-蓝鲸应用 + */ +@EqualsAndHashCode(callSuper = true) +@Data +@NoArgsConstructor +public class AppQuotaLimit extends QuotaLimit { + + /** + * 解析之后的自定义应用配额限制 + */ + private Map customLimits = new HashMap<>(); + + public AppQuotaLimit(String globalLimitExpr, String customLimitExpr) { + super(globalLimitExpr, customLimitExpr); + } + + public long getLimit(String bkAppCode) { + Long limit = customLimits.get(bkAppCode); + if (limit == null) { + limit = getGlobalLimit(); + } + return limit; + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/resource/quota/QuotaLimit.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/resource/quota/QuotaLimit.java new file mode 100644 index 0000000000..9dd0f7dc1d --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/resource/quota/QuotaLimit.java @@ -0,0 +1,55 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.resource.quota; + +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * 配额限制基础类 + */ +@Data +@NoArgsConstructor +public class QuotaLimit { + /** + * 全局配额表达式 + */ + protected String globalLimitExpr; + + /** + * 自定义配额表达式，会覆盖全局配额的配置 + */ + protected String customLimitExpr; + + /** + * 解析之后的全局业务配额限制 + */ + protected Long globalLimit; + + public QuotaLimit(String globalLimitExpr, String customLimitExpr) { + this.globalLimitExpr = globalLimitExpr; + this.customLimitExpr = customLimitExpr; + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/resource/quota/QuotaResourceId.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/resource/quota/QuotaResourceId.java new file mode 100644 index 0000000000..8f55595594 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/resource/quota/QuotaResourceId.java @@ -0,0 +1,35 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.resource.quota; + +/** + * 配额限制的资源 ID 定义 + */ +public interface QuotaResourceId { + /** + * 当前正在运行的作业实例 + */ + String JOB_INSTANCE = "runningJob"; +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/resource/quota/ResourceQuotaLimit.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/resource/quota/ResourceQuotaLimit.java new file mode 100644 index 0000000000..51aa0399d9 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/resource/quota/ResourceQuotaLimit.java @@ -0,0 +1,74 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.resource.quota; + +import com.tencent.bk.job.common.model.dto.ResourceScope; +import lombok.Data; + +/** + * 资源配额限制-基础类 + */ +@Data +public class ResourceQuotaLimit { + + public static final long UNLIMITED_VALUE = Long.MAX_VALUE; + /** + * 是否启用配额限制 + */ + private boolean enabled; + /** + * 配额容量表达式 + */ + private String capacityExpr; + /** + * 解析后的配额总量限制 + */ + private Long capacity; + + /** + * 资源管理空间配额限制 + */ + private ResourceScopeQuotaLimit resourceScopeQuotaLimit; + + /** + * 应用配额限制 + */ + private AppQuotaLimit appQuotaLimit; + + public long getLimitByResourceScope(ResourceScope resourceScope) { + if (resourceScopeQuotaLimit == null) { + return Long.MAX_VALUE; + } + return resourceScopeQuotaLimit.getLimit(resourceScope); + } + + public long getLimitByBkAppCode(String bkAppCode) { + if (appQuotaLimit == null) { + return Long.MAX_VALUE; + } + return appQuotaLimit.getLimit(bkAppCode); + } + +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/resource/quota/ResourceScopeQuotaLimit.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/resource/quota/ResourceScopeQuotaLimit.java new file mode 100644 index 0000000000..a3340a1d15 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/resource/quota/ResourceScopeQuotaLimit.java @@ -0,0 +1,73 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.resource.quota; + +import com.tencent.bk.job.common.model.dto.ResourceScope; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import java.util.HashMap; +import java.util.Map; +import java.util.StringJoiner; + +/** + * 配额限制-资源管理空间 + */ +@EqualsAndHashCode(callSuper = true) +@Getter +@Setter +@NoArgsConstructor +public class ResourceScopeQuotaLimit extends QuotaLimit { + + /** + * 解析之后的自定义业务配额限制 + */ + private Map customLimits = new HashMap<>(); + + public ResourceScopeQuotaLimit(String globalLimitExpr, String customLimitExpr) { + super(globalLimitExpr, customLimitExpr); + } + + public long getLimit(ResourceScope resourceScope) { + String resourceScopeUniqueId = resourceScope.toResourceScopeUniqueId(); + Long limit = customLimits.get(resourceScopeUniqueId); + if (limit == null) { + limit = getGlobalLimit(); + } + return limit; + } + + @Override + public String toString() { + return new StringJoiner(", ", ResourceScopeQuotaLimit.class.getSimpleName() + "[", "]") + .add("globalLimitExpr='" + globalLimitExpr + "'") + .add("globalLimit=" + globalLimit) + .add("customLimits=" + customLimits) + .add("customLimitExpr='" + customLimitExpr + "'") + .toString(); + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/service/AppScopeMappingService.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/service/AppScopeMappingService.java index aeb7eaa1c5..cfcd26f831 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/service/AppScopeMappingService.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/service/AppScopeMappingService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -63,6 +63,8 @@ public interface AppScopeMappingService { */ AppResourceScope getAppResourceScope(Long appId, String scopeType, String scopeId); + AppResourceScope getAppResourceScope(Long appId); + /** * 根据scopeType&scopeId获取AppResourceScope * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/service/VariableResolver.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/service/VariableResolver.java index f4c1443a9c..f815bbe7a6 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/service/VariableResolver.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/service/VariableResolver.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/service/config/JacksonConfig.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/service/config/JacksonConfig.java new file mode 100644 index 0000000000..dfda30bacc --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/service/config/JacksonConfig.java @@ -0,0 +1,12 @@ +package com.tencent.bk.job.common.service.config; + +import com.fasterxml.jackson.datatype.jsr310.ser.LocalDateTimeSerializer; +import org.springframework.boot.autoconfigure.jackson.Jackson2ObjectMapperBuilderCustomizer; +import org.springframework.context.annotation.Bean; + +public class JacksonConfig { + @Bean + public Jackson2ObjectMapperBuilderCustomizer jsonCustomizer() { + // 支持 java.time.LocalDateTime 序列化与反序列化 + return builder -> builder.serializers(LocalDateTimeSerializer.INSTANCE).build(); + }} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/service/config/JobCommonAutoConfiguration.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/service/config/JobCommonAutoConfiguration.java new file mode 100644 index 0000000000..f8a7d53b81 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/service/config/JobCommonAutoConfiguration.java @@ -0,0 +1,70 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service.config; + +import com.tencent.bk.job.common.VersionInfoLogApplicationRunner; +import com.tencent.bk.job.common.config.BkConfig; +import com.tencent.bk.job.common.util.ApplicationContextRegister; +import com.tencent.bk.job.common.util.http.HttpHelperFactory; +import io.micrometer.core.instrument.MeterRegistry; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.info.BuildProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.context.annotation.Lazy; + +@Slf4j +@Configuration(proxyBeanMethods = false) +@Import({JobCommonConfig.class, BkConfig.class}) +public class JobCommonAutoConfiguration { + @Bean("applicationContextRegister") + @Lazy(false) + public ApplicationContextRegister applicationContextRegister() { + return new ApplicationContextRegister(); + } + + @Bean + HttpConfigSetter httpConfigSetter(@Autowired MeterRegistry meterRegistry) { + HttpHelperFactory.setMeterRegistry(meterRegistry); + log.info("meterRegistry for HttpHelperFactory init"); + return new HttpConfigSetter(); + } + + static class HttpConfigSetter { + HttpConfigSetter() { + } + } + + @Value("${spring.application.name:bk-job}") + private String serviceName; + + @Bean + public VersionInfoLogApplicationRunner versionInfoLogApplicationRunner(BuildProperties buildProperties) { + return new VersionInfoLogApplicationRunner(serviceName, buildProperties); + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/service/config/JobCommonConfig.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/service/config/JobCommonConfig.java new file mode 100644 index 0000000000..d0930d09ca --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/service/config/JobCommonConfig.java @@ -0,0 +1,46 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.service.config; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Configuration; + +@Getter +@Setter +@Configuration +public class JobCommonConfig { + + /** + * 作业平台web访问地址，可配置多个，用","分隔 + */ + @Value("${job.web.url:}") + private String jobWebUrl; + + @Value("${swagger.url:swagger.job.com}") + private String swaggerUrl; + +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/ApiUtil.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/ApiUtil.java index e3cc585252..ebe64e6e34 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/ApiUtil.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/ApiUtil.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/ApplicationContextRegister.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/ApplicationContextRegister.java index d2a0c9eb7e..facd22a0d3 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/ApplicationContextRegister.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/ApplicationContextRegister.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,15 +28,11 @@ import org.springframework.beans.BeansException; import org.springframework.context.ApplicationContext; import org.springframework.context.ApplicationContextAware; -import org.springframework.context.annotation.Lazy; -import org.springframework.stereotype.Component; /** * Spring容器 */ @Slf4j -@Component -@Lazy(false) public class ApplicationContextRegister implements ApplicationContextAware { /** * Spring应用上下文环境 diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/ClassUtil.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/ClassUtil.java deleted file mode 100644 index 26ce05def9..0000000000 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/ClassUtil.java +++ /dev/null @@ -1,228 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.util; - -import lombok.extern.slf4j.Slf4j; - -import java.lang.reflect.*; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; - -/** - * @Description - * @Date 2020/3/6 - * @Version 1.0 - */ -@Slf4j -public class ClassUtil { - private static boolean isTargetPrimitive( - SrcClass srcInstance, - Class targetClass - ) { - Object[] enumConstants = srcInstance.getClass().getEnumConstants(); - return (targetClass.isPrimitive() - || targetClass == Boolean.class - || targetClass == Byte.class - || targetClass == Character.class - || targetClass == Short.class - || targetClass == Integer.class - || targetClass == Long.class - || targetClass == Float.class - || targetClass == Double.class - || (enumConstants != null && enumConstants.length > 0)); - } - - private static TargetClass convertCollection( - SrcClass srcInstance, - Object targetInstance - ) { - try { - //取得add方法 - Method addMethod = targetInstance.getClass().getDeclaredMethod("add", Object.class); - addMethod.setAccessible(true); - Object finalTargetInstance = targetInstance; - ((Collection) srcInstance).forEach(item -> { - try { - addMethod.invoke(finalTargetInstance, copyAttrsBetweenClasses(item, item.getClass())); - } catch (IllegalAccessException | InvocationTargetException e) { - log.error("error occurred when convert", e); - } - }); - return (TargetClass) targetInstance; - } catch (NoSuchMethodException e) { - log.error("error occurred when convert", e); - return null; - } - } - - private static TargetClass convertMap( - SrcClass srcInstance, - Object targetInstance - ) { - try { - //取得put方法 - Method putMethod = targetInstance.getClass().getDeclaredMethod("put", Object.class, Object.class); - putMethod.setAccessible(true); - Object finalTargetInstance = targetInstance; - ((Map) srcInstance).forEach((key, value) -> { - try { - putMethod.invoke(finalTargetInstance, copyAttrsBetweenClasses(key, key.getClass()), - copyAttrsBetweenClasses(value, value.getClass())); - } catch (IllegalAccessException | InvocationTargetException e) { - log.error("error occurred when convert", e); - } - }); - return (TargetClass) targetInstance; - } catch (NoSuchMethodException e) { - log.error("error occurred when convert", e); - return null; - } - } - - /** - * 通过反射实现有属性交集的不同类实例属性复制 - * - * @param srcInstance 源实例 - * @param targetClass 目标类Class对象 - * @param 源类 - * @param 目标类 - * @return 目标类实例 - */ - public static TargetClass copyAttrsBetweenClasses( - SrcClass srcInstance, - Class targetClass) { - Object targetInstance = null; - if (srcInstance == null) { - return null; - } - //基本数据类型：直接强转 - if (isTargetPrimitive(srcInstance, targetClass)) { - return (TargetClass) srcInstance; - } - //String类型：toString - if (targetClass == String.class) { - return (TargetClass) srcInstance.toString(); - } - //集合与Map类型：分别单独处理 - int targetClassModifiers = targetClass.getModifiers(); - if (Modifier.isAbstract(targetClassModifiers) - || Modifier.isInterface(targetClassModifiers)) { - //目标类为抽象类或接口：用来源对象的类型实例化 - try { - Constructor constructor = srcInstance.getClass().getDeclaredConstructor(); - constructor.setAccessible(true); - targetInstance = constructor.newInstance(); - } catch (InstantiationException - | IllegalAccessException - | NoSuchMethodException - | InvocationTargetException e) { - log.error("error occurred when convert", e); - } - } else { - //目标类可以实例化 - try { - Constructor constructor = targetClass.getDeclaredConstructor(); - constructor.setAccessible(true); - targetInstance = constructor.newInstance(); - } catch (InstantiationException - | IllegalAccessException - | NoSuchMethodException - | InvocationTargetException e - ) { - log.error("error occurred when convert", e); - } - } - if (srcInstance instanceof Collection) { - TargetClass result = convertCollection(srcInstance, targetInstance); - if (result != null) return result; - } else if (srcInstance instanceof Map) { - TargetClass result = convertMap(srcInstance, targetInstance); - if (result != null) return result; - } - //自定义数据类型：遍历所有字段分别处理 - Field[] srcFields = srcInstance.getClass().getDeclaredFields(); - List ignoredFields = new ArrayList<>(); - for (Field srcField : srcFields) { - try { - //设置可访问 - srcField.setAccessible(true); - Field targetField = targetClass.getDeclaredField(srcField.getName()); - targetField.setAccessible(true); - Class targetFieldType = targetField.getType(); - Object srcFieldValue = srcField.get(srcInstance); - Type srcFieldGenericType = srcField.getGenericType(); - Type targetFieldGenericType = targetField.getGenericType(); - //泛型参数的转换 - if (srcFieldGenericType instanceof ParameterizedType) { - //目标泛型参数 - ParameterizedType targetParameterizedType = (ParameterizedType) targetFieldGenericType; - if (srcFieldValue == null) { - targetField.set(targetInstance, null); - } else { - //处理Collection与Map - if (srcFieldValue instanceof Collection) { - Type targetActualType = targetParameterizedType.getActualTypeArguments()[0]; - Collection targetFieldValue = (Collection) srcFieldValue.getClass().newInstance(); - ((Collection) srcFieldValue).forEach( - item -> targetFieldValue.add(copyAttrsBetweenClasses(item, (Class) targetActualType))); - targetField.set(targetInstance, targetFieldValue); - } else if (srcFieldValue instanceof Map) { - Type[] targetActualTypes = targetParameterizedType.getActualTypeArguments(); - Type targetActualKeyType = targetActualTypes[0]; - Type targetActualValueType = targetActualTypes[1]; - Map targetFieldValue = (Map) srcFieldValue.getClass().newInstance(); - ((Map) srcFieldValue).forEach( - (key, value) -> { - targetFieldValue.put(copyAttrsBetweenClasses(key, (Class) targetActualKeyType), - copyAttrsBetweenClasses(value, (Class) targetActualValueType)); - }); - targetField.set(targetInstance, targetFieldValue); - } else { - //自定义类型中含泛型参数：递归拷贝 - targetField.set(targetInstance, copyAttrsBetweenClasses(srcField.get(srcInstance), - targetFieldType)); - } - } - } else { - //递归拷贝 - targetField.set(targetInstance, copyAttrsBetweenClasses(srcField.get(srcInstance), - targetFieldType)); - } - } catch (NoSuchFieldException | IllegalAccessException ignore) { - ignoredFields.add(srcField.getName()); - } catch (InstantiationException e) { - log.error("error occurred when convert", e); - } finally { - if (!ignoredFields.isEmpty()) { - log.debug(String.format("%s-->%s:ignored fields:%s", srcInstance.getClass().getSimpleName(), - targetClass.getSimpleName(), String.join(",", ignoredFields))); - } - } - } - return (TargetClass) targetInstance; - } -} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/ConcurrencyUtil.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/ConcurrencyUtil.java index f0fed89969..60e925aa78 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/ConcurrencyUtil.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/ConcurrencyUtil.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,15 +24,20 @@ package com.tencent.bk.job.common.util; +import com.tencent.bk.job.common.exception.SubThreadException; import lombok.extern.slf4j.Slf4j; +import org.springframework.cloud.sleuth.instrument.async.TraceableExecutorService; import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; /** * @Description @@ -51,11 +56,22 @@ public class ConcurrencyUtil { * @param 输入泛型 * @param 单个输出元素泛型 * @return 输出列表 + * @throws SubThreadException 子线程异常 */ - public static List getResultWithThreads(Collection inputCollection, int threadNum, - Handler handler) { - ExecutorService threadPoolExecutor = Executors.newFixedThreadPool(threadNum); - return getResultWithThreads(inputCollection, threadPoolExecutor, handler); + public static List getResultWithThreads( + Collection inputCollection, + int threadNum, + Handler handler + ) throws SubThreadException { + ExecutorService threadPoolExecutor = new TraceableExecutorService( + ApplicationContextRegister.getContext(), + Executors.newFixedThreadPool(threadNum) + ); + try { + return getResultWithThreads(inputCollection, threadPoolExecutor, handler); + } finally { + threadPoolExecutor.shutdown(); + } } /** @@ -67,27 +83,39 @@ public static List getResultWithThreads(Collection 输入泛型 * @param 单个输出元素泛型 * @return 输出列表 + * @throws SubThreadException 子线程异常 */ - public static List getResultWithThreads(Collection inputCollection, - ExecutorService threadPoolExecutor, Handler handler) { + public static List getResultWithThreads( + Collection inputCollection, + ExecutorService threadPoolExecutor, + Handler handler + ) throws SubThreadException { LinkedBlockingQueue resultQueue = new LinkedBlockingQueue<>(); - List futures = new ArrayList<>(); + List> futures = new ArrayList<>(); for (Input input : inputCollection) { - Future future = threadPoolExecutor.submit(new InnerTask<>(resultQueue, input, + Future future = threadPoolExecutor.submit(new InnerTask<>(resultQueue, input, JobContextUtil.getRequestId(), handler)); futures.add(future); } - futures.forEach(it -> { + for (Future future : futures) { try { - while (!it.isDone()) { - Thread.sleep(10); - } + future.get(5, TimeUnit.MINUTES); } catch (InterruptedException e) { - log.warn("sleep interrupted", e); + throw new SubThreadException("sub thread interrupted", e); + } catch (ExecutionException e) { + String msg = "sub thread throws an exception"; + if (e.getCause() == null) { + throw new SubThreadException(msg, e); + } + throw new SubThreadException(msg, e.getCause()); + } catch (TimeoutException e) { + String msg = "sub thread timed out 5 minutes"; + if (e.getCause() == null) { + throw new SubThreadException(msg, e); + } + throw new SubThreadException(msg, e.getCause()); } - }); - threadPoolExecutor.shutdown(); + } return new ArrayList<>(resultQueue); } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/Counter.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/Counter.java index fcb3d36382..64b0ac55a3 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/Counter.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/Counter.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/CustomCollectionUtils.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/CustomCollectionUtils.java index a06c8f46cb..ba2ca1f243 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/CustomCollectionUtils.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/CustomCollectionUtils.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/FilterUtils.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/FilterUtils.java index 6c08d9151f..338deab804 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/FilterUtils.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/FilterUtils.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/FlowControlUtil.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/FlowControlUtil.java index 667e49efff..84885652e3 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/FlowControlUtil.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/FlowControlUtil.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/FlowController.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/FlowController.java index c8e99b59e0..6c8f3e729c 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/FlowController.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/FlowController.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/I18nUtil.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/I18nUtil.java index 8f40179c7f..541c56b55b 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/I18nUtil.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/I18nUtil.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,9 +27,26 @@ import com.tencent.bk.job.common.i18n.service.MessageI18nService; import lombok.extern.slf4j.Slf4j; +import java.util.Locale; + @Slf4j public class I18nUtil { - private static MessageI18nService i18nService; + private volatile static MessageI18nService i18nService; + + public static String getI18nMessage(Locale locale, String key, Object[] params) { + initI18nService(); + if (i18nService == null) { + log.warn("Can not find available i18nService"); + return ""; + } + if (params != null && params.length > 0) { + // 如果参数中包含需要国际化的字符(格式:{i18n_key})，那么需要国际化参数 + buildI18nParams(params); + return i18nService.getI18nWithArgs(locale, key, params); + } else { + return i18nService.getI18n(locale, key); + } + } public static String getI18nMessage(String key, Object[] params) { initI18nService(); @@ -46,7 +63,7 @@ public static String getI18nMessage(String key, Object[] params) { } } - private static Object[] buildI18nParams(Object[] params) { + private static void buildI18nParams(Object[] params) { for (int i = 0; i < params.length; i++) { Object errorParam = params[i]; if (errorParam instanceof String) { @@ -58,13 +75,16 @@ private static Object[] buildI18nParams(Object[] params) { } } } - return params; } public static String getI18nMessage(String key) { return getI18nMessage(key, null); } + public static String getI18nMessage(Locale locale, String key) { + return getI18nMessage(locale, key, null); + } + private static void initI18nService() { if (i18nService == null) { synchronized (I18nUtil.class) { diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/JobContextUtil.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/JobContextUtil.java index 9ff5bf8b23..10627d291d 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/JobContextUtil.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/JobContextUtil.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,6 +26,7 @@ import com.tencent.bk.job.common.context.JobContext; import com.tencent.bk.job.common.context.JobContextThreadLocal; +import com.tencent.bk.job.common.i18n.locale.LocaleUtils; import com.tencent.bk.job.common.model.dto.AppResourceScope; import io.micrometer.core.instrument.Tag; import lombok.extern.slf4j.Slf4j; @@ -42,7 +43,7 @@ import java.util.Map; /** - * @since 6/11/2019 10:27 + * Job http 请求上下文工具类 */ @Slf4j public class JobContextUtil { @@ -135,6 +136,11 @@ public static void setUserLang(String userLang) { jobContext.setUserLang(userLang); } + public static boolean isEnglishLocale() { + String normalLang = LocaleUtils.getNormalLang(getUserLang()); + return normalLang.equals(LocaleUtils.LANG_EN) || normalLang.equals(LocaleUtils.LANG_EN_US); + } + public static List getDebugMessage() { JobContext jobContext = JobContextThreadLocal.get(); List debugMessage = null; @@ -243,4 +249,18 @@ public static Map>> getOrInitMetricTagsMa } return metricTagsMap; } + + public static String getRequestFrom() { + JobContext jobContext = JobContextThreadLocal.get(); + String requestFrom = null; + if (jobContext != null) { + requestFrom = jobContext.getRequestFrom(); + } + return requestFrom; + } + + public static void setRequestFrom(String requestFrom) { + JobContext jobContext = getOrInitContext(); + jobContext.setRequestFrom(requestFrom); + } } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/JobUUID.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/JobUUID.java index 0f463a2f6c..195a834b8e 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/JobUUID.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/JobUUID.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/LogUtil.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/LogUtil.java index 1a93c7072b..ed16247f18 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/LogUtil.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/LogUtil.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,8 @@ package com.tencent.bk.job.common.util; +import java.util.List; + /** * @Description * @Date 2019/12/23 @@ -31,13 +33,35 @@ */ public class LogUtil { + /** + * 构建日志中需要打印出的部分列表元素字符串 + * + * @param list 原始列表 + * @param maxElementNum 打印出的最大元素个数，-1表示无限制 + * @return 日志字符串 + */ + public static String buildListLog(List list, int maxElementNum) { + if (list == null) { + return ""; + } + if (list.isEmpty() || maxElementNum == 0) { + return "[]"; + } else if (list.size() <= maxElementNum || maxElementNum < 0) { + return "[" + StringUtil.concatCollection(list) + "]"; + } else { + return "(" + list.size() + " elements)[" + + StringUtil.concatCollection(list.subList(0, maxElementNum)) + + ",...]"; + } + } + private static String joinBySeparator(String separator, Object... args) { StringBuilder builder = new StringBuilder(); - for (int i = 0; i < args.length; i++) { - if (args[i] == null) { + for (Object arg : args) { + if (arg == null) { builder.append("null"); } else { - builder.append(args[i].toString()); + builder.append(arg.toString()); } builder.append(separator); } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/PageUtil.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/PageUtil.java index 326286ca0c..56081210d5 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/PageUtil.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/PageUtil.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,13 +24,17 @@ package com.tencent.bk.job.common.util; +import com.tencent.bk.job.common.model.PageCondition; import com.tencent.bk.job.common.model.PageData; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.tuple.Pair; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Objects; +import java.util.function.Function; import java.util.stream.Collectors; /** @@ -190,4 +194,139 @@ private static void putPrioritizedElementsInFrontIfExist(PageData pageDat pageData.setData(templates); } } + + /** + * 对分页数据内含的数据进行类型转换后得到新的分页数据 + * + * @param srcPageData 源分页数据 + * @param mapper 映射函数 + * @param 原始元素类型 + * @param 目标元素类型 + * @return 转换后的分页数据 + */ + public static PageData transferPageData(PageData srcPageData, + Function mapper) { + if (srcPageData == null) { + return null; + } + List data = srcPageData.getData(); + List newDataList = null; + if (data != null) { + newDataList = data.stream().map(mapper).collect(Collectors.toList()); + } + return copyPageWithNewData(srcPageData, newDataList); + } + + /** + * 拷贝分页数据，并使用新的列表数据填充 + * + * @param srcPageData 源分页数据 + * @return 新的分页数据 + */ + public static PageData copyPageWithNewData(PageData srcPageData, List newDataList) { + if (srcPageData == null) { + return null; + } + PageData targetPageData = new PageData<>(); + targetPageData.setStart(srcPageData.getStart()); + targetPageData.setPageSize(srcPageData.getPageSize()); + targetPageData.setTotal(srcPageData.getTotal()); + targetPageData.setCanCreate(srcPageData.getCanCreate()); + targetPageData.setExistAny(srcPageData.getExistAny()); + targetPageData.setData(newDataList); + return targetPageData; + } + + /** + * 查询全量 - 通过循环分页查询 + *

+ * 注意：如果多次分页查询期间原始数据发生变化，使用该方法可能出现数据重复/遗漏的情况。该方法仅适用于对数据准确度要求不高的场景！！！ + * 如果需要准确的查询结果，可以使用 queryAllWithLoopPageQueryInOrder 代替 + * + * @param pageLimit 每页限制大小 + * @param pageQuery 查询操作 + * @param extractElementsFunction 从分页查询结果提取返回的对象列表 + * @param elementConverter 分页查询对象转换为最终对象 + * @param 分页查询返回的对象 + * @param 转换之后作为方法返回值的对象 + * @param 分页查询结果 + * @return 全量对象列表 + * @see #queryAllWithLoopPageQueryInOrder + */ + public static List queryAllWithLoopPageQuery(int pageLimit, + Function pageQuery, + Function> extractElementsFunction, + Function elementConverter) { + int start = 0; + List elements = new ArrayList<>(); + while (true) { + PageCondition pageCondition = PageCondition.build(start, pageLimit); + R result = pageQuery.apply(pageCondition); + + Collection originElements = extractElementsFunction.apply(result); + if (CollectionUtils.isEmpty(originElements)) { + // 如果本次没有获取到数据记录，说明数据已经全部拉取完成 + break; + } + elements.addAll(originElements.stream() + .map(elementConverter) + .filter(Objects::nonNull) + .collect(Collectors.toList())); + + if (originElements.size() < pageLimit) { + // 如果实际查询数据记录的数量小于分页大小，说明数据已经全部拉取完成 + break; + } + start += pageLimit; + } + return elements; + } + + /** + * 查询全量 - 按指定顺序循环分页查询 + * + * @param pageLimit 每页限制大小 + * @param queryInputGenerator 分页查询参数生成。输入: 当前页的最后一个元素；输出：下一页查询输入 + * @param query 查询操作 + * @param extractElementsFunction 从分页查询结果提取返回的对象列表 + * @param elementConverter 分页查询对象转换为最终对象 + * @param 分页查询返回的对象类型 + * @param 转换之后作为方法返回值的对象类型 + * @param 分页查询结果 + * @param 分页查询输入参数 + * @return 全量对象列表 + */ + public static List queryAllWithLoopPageQueryInOrder( + int pageLimit, + Function queryInputGenerator, + Function query, + Function> extractElementsFunction, + Function elementConverter) { + + // 查询结果 + List elements = new ArrayList<>(); + + T1 latestElement = null; // 排序之后的最后一个元素，用于作为下一次分页查询的 offset 条件 + while (true) { + Q queryInput = queryInputGenerator.apply(latestElement); + R result = query.apply(queryInput); + + List originElements = extractElementsFunction.apply(result); + if (CollectionUtils.isEmpty(originElements)) { + // 如果本次没有获取到数据记录，说明数据已经全部拉取完成 + break; + } + elements.addAll(originElements.stream() + .map(elementConverter) + .filter(Objects::nonNull) + .collect(Collectors.toList())); + latestElement = originElements.get(originElements.size() - 1); + + if (originElements.size() < pageLimit) { + // 如果实际查询数据记录的数量小于分页大小，说明数据已经全部拉取完成 + break; + } + } + return elements; + } } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/ReflectUtil.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/ReflectUtil.java index 8d8a538418..5554a3345d 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/ReflectUtil.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/ReflectUtil.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/RequestUtil.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/RequestUtil.java index 18ab9e79b7..2e3f5ebcca 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/RequestUtil.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/RequestUtil.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,7 @@ package com.tencent.bk.job.common.util; +import com.tencent.bk.job.common.constant.HttpRequestSourceEnum; import lombok.extern.slf4j.Slf4j; import org.springframework.http.HttpCookie; import org.springframework.http.HttpHeaders; @@ -31,6 +32,7 @@ import org.springframework.util.CollectionUtils; import org.springframework.util.MultiValueMap; +import javax.servlet.http.HttpServletRequest; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -103,4 +105,22 @@ public static List getCookieValuesFromCookies(List cookieList, S } return cookieValueList; } + + /** + * 解析 http 请求来源 + * + * @param request http 请求 + * @return 请求来源 + */ + public static HttpRequestSourceEnum parseHttpRequestSource(HttpServletRequest request) { + String uri = request.getRequestURI(); + if (uri.startsWith("/web/")) { + return HttpRequestSourceEnum.WEB; + } else if (uri.startsWith("/esb/")) { + return HttpRequestSourceEnum.ESB; + } else if (uri.startsWith("/service/")) { + return HttpRequestSourceEnum.INTERNAL; + } + return HttpRequestSourceEnum.UNKNOWN; + } } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/SimpleRequestIdLogger.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/SimpleRequestIdLogger.java deleted file mode 100644 index aa16852057..0000000000 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/SimpleRequestIdLogger.java +++ /dev/null @@ -1,372 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.util; - -import com.tencent.bk.job.common.RequestIdLogger; -import org.slf4j.Logger; -import org.slf4j.Marker; - -/** - * @Description Logger的加RequestId兼容扩展 - * @Date 2019/12/24 - * @Version 1.0 - */ -public class SimpleRequestIdLogger implements RequestIdLogger { - Logger logger; - - public SimpleRequestIdLogger(Logger logger) { - this.logger = logger; - } - - @Override - public String getName() { - return logger.getName(); - } - - @Override - public boolean isTraceEnabled() { - return logger.isTraceEnabled(); - } - - @Override - public void trace(String s) { - logger.trace(s); - } - - @Override - public void trace(String s, Object o) { - logger.trace(s, o); - } - - @Override - public void trace(String s, Object o, Object o1) { - logger.trace(s, o, o1); - } - - @Override - public void trace(String s, Object... objects) { - logger.trace(s, objects); - } - - @Override - public void trace(String s, Throwable throwable) { - logger.trace(s, throwable); - } - - @Override - public boolean isTraceEnabled(Marker marker) { - return logger.isTraceEnabled(marker); - } - - @Override - public void trace(Marker marker, String s) { - logger.trace(marker, s); - } - - @Override - public void trace(Marker marker, String s, Object o) { - logger.trace(marker, s, o); - } - - @Override - public void trace(Marker marker, String s, Object o, Object o1) { - logger.trace(marker, s, o, o1); - } - - @Override - public void trace(Marker marker, String s, Object... objects) { - logger.trace(marker, s, objects); - } - - @Override - public void trace(Marker marker, String s, Throwable throwable) { - logger.trace(marker, s, throwable); - } - - @Override - public boolean isDebugEnabled() { - return logger.isDebugEnabled(); - } - - @Override - public void debug(String s) { - logger.debug(s); - } - - @Override - public void debug(String s, Object o) { - logger.debug(s, o); - } - - @Override - public void debug(String s, Object o, Object o1) { - logger.debug(s, o, o1); - } - - @Override - public void debug(String s, Object... objects) { - logger.debug(s, objects); - } - - @Override - public void debug(String s, Throwable throwable) { - logger.debug(s, throwable); - } - - @Override - public boolean isDebugEnabled(Marker marker) { - return logger.isDebugEnabled(); - } - - @Override - public void debug(Marker marker, String s) { - logger.debug(marker, s); - } - - @Override - public void debug(Marker marker, String s, Object o) { - logger.debug(marker, s, o); - } - - @Override - public void debug(Marker marker, String s, Object o, Object o1) { - logger.debug(marker, s, o, o1); - } - - @Override - public void debug(Marker marker, String s, Object... objects) { - logger.debug(marker, s, objects); - } - - @Override - public void debug(Marker marker, String s, Throwable throwable) { - logger.debug(marker, s, throwable); - } - - @Override - public boolean isInfoEnabled() { - return logger.isInfoEnabled(); - } - - @Override - public void info(String s) { - logger.info(s); - } - - @Override - public void info(String s, Object o) { - logger.info(s, o); - } - - @Override - public void info(String s, Object o, Object o1) { - logger.info(s, o, o1); - } - - @Override - public void info(String s, Object... objects) { - logger.info(s, objects); - } - - @Override - public void info(String s, Throwable throwable) { - logger.info(s, throwable); - } - - @Override - public boolean isInfoEnabled(Marker marker) { - return logger.isInfoEnabled(); - } - - @Override - public void info(Marker marker, String s) { - logger.info(marker, s); - } - - @Override - public void info(Marker marker, String s, Object o) { - logger.info(marker, s, o); - } - - @Override - public void info(Marker marker, String s, Object o, Object o1) { - logger.info(marker, s, o, o1); - } - - @Override - public void info(Marker marker, String s, Object... objects) { - logger.info(marker, s, objects); - } - - @Override - public void info(Marker marker, String s, Throwable throwable) { - logger.info(marker, s, throwable); - } - - @Override - public boolean isWarnEnabled() { - return logger.isWarnEnabled(); - } - - @Override - public void warn(String s) { - logger.warn(s); - } - - @Override - public void warn(String s, Object o) { - logger.warn(s, o); - } - - @Override - public void warn(String s, Object... objects) { - logger.warn(s, objects); - } - - @Override - public void warn(String s, Object o, Object o1) { - logger.warn(s, o, o1); - } - - @Override - public void warn(String s, Throwable throwable) { - logger.warn(s, throwable); - } - - @Override - public boolean isWarnEnabled(Marker marker) { - return logger.isWarnEnabled(); - } - - @Override - public void warn(Marker marker, String s) { - logger.warn(marker, s); - } - - @Override - public void warn(Marker marker, String s, Object o) { - logger.warn(marker, s, o); - } - - @Override - public void warn(Marker marker, String s, Object o, Object o1) { - logger.warn(marker, s, o, o1); - } - - @Override - public void warn(Marker marker, String s, Object... objects) { - logger.warn(marker, s, objects); - } - - @Override - public void warn(Marker marker, String s, Throwable throwable) { - logger.warn(marker, s, throwable); - } - - @Override - public boolean isErrorEnabled() { - return logger.isErrorEnabled(); - } - - @Override - public void error(String s) { - logger.error(s); - } - - @Override - public void error(String s, Object o) { - logger.error(s, o); - } - - @Override - public void error(String s, Object o, Object o1) { - logger.error(s, o, o1); - } - - @Override - public void error(String s, Object... objects) { - logger.error(s, objects); - } - - @Override - public void error(String s, Throwable throwable) { - logger.error(s, throwable); - } - - @Override - public boolean isErrorEnabled(Marker marker) { - return logger.isErrorEnabled(marker); - } - - @Override - public void error(Marker marker, String s) { - logger.error(marker, s); - } - - @Override - public void error(Marker marker, String s, Object o) { - logger.error(marker, s, o); - } - - @Override - public void error(Marker marker, String s, Object o, Object o1) { - logger.error(marker, s, o, o1); - } - - @Override - public void error(Marker marker, String s, Object... objects) { - logger.error(marker, s, objects); - } - - @Override - public void error(Marker marker, String s, Throwable throwable) { - logger.error(marker, s, throwable); - } - - /** - * 扩展方法：将requestId打印至日志中 - * - * @param message - */ - @Override - public void debugWithRequestId(String message) { - debug(JobContextUtil.getRequestId() + "|" + message); - } - - @Override - public void infoWithRequestId(String message) { - info(JobContextUtil.getRequestId() + "|" + message); - } - - @Override - public void warnWithRequestId(String message) { - warn(JobContextUtil.getRequestId() + "|" + message); - } - - @Override - public void errorWithRequestId(String message) { - error(JobContextUtil.getRequestId() + "|" + message); - } -} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/StringUtil.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/StringUtil.java index fff571b4a0..48e73d8cc2 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/StringUtil.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/StringUtil.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,6 +26,7 @@ import com.tencent.bk.job.common.util.json.JsonUtils; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; @@ -42,13 +43,27 @@ import java.util.stream.Collectors; /** - * @Description 字符串处理工具类 - * @Date 2020/3/6 - * @Version 1.0 + * 字符串处理工具类 */ @Slf4j public class StringUtil { + /** + * 使用常见的多个字符[;,；，\n\s|\u00A0]+分割字符串 + * + * @param str 目标字符串 + * @return 分割后的字符串列表 + */ + public static List splitByNormalSeparator(String str) { + if (str == null) { + return Collections.emptyList(); + } + if (StringUtils.isBlank(str)) { + return Collections.singletonList(str); + } + return Arrays.asList(str.split("[;,；，\\n\\s|\\u00A0]+")); + } + /** * 使用对象中的字段值替换路径中的占位符 * @@ -144,6 +159,23 @@ public static String concatCollection(Collection collection, String separ return str; } + /** + * 将集合拼接为字符串 + * + * @param array 原始数组 + * @param separator 分隔符 + * @param T 原始数据 + * @return 拼接后的字符串 + */ + public static String concatArray(T[] array, String separator) { + String str; + if (array == null || array.length == 0) { + return ""; + } + str = Arrays.stream(array).map(Object::toString).collect(Collectors.joining(separator)); + return str; + } + /** * 从字符串提取List * @@ -153,8 +185,9 @@ public static String concatCollection(Collection collection, String separ * @param 目标类型 * @return list */ + @SuppressWarnings("all") public static List strToList(String str, Class clazz, String separator) { - if (str == null || str.isEmpty() || StringUtils.isBlank(str)) { + if (StringUtils.isBlank(str)) { return Collections.emptyList(); } return Arrays.stream(str.trim().split(separator)).filter(StringUtils::isNotBlank) @@ -217,6 +250,7 @@ public static String replaceByRegex(String rawStr, String pattern, Map variablesMap, int depth) { - log.debug("rawStr={},pattern={},variablesMap={},depth={}", rawStr, pattern, variablesMap, depth); + if (log.isDebugEnabled()) { + log.debug("rawStr={},pattern={},variablesMap={},depth={}", rawStr, pattern, variablesMap, depth); + } String resultStr = rawStr; List> keys = findTwoRegexPatterns(rawStr, pattern); Set keyset = variablesMap.keySet(); for (Pair pair : keys) { String placeHolder = pair.getLeft().trim(); String key = pair.getRight().trim(); - log.debug("resultStr={},placeHolder={},key={}", resultStr, placeHolder, key); + if (log.isDebugEnabled()) { + log.debug("resultStr={},placeHolder={},key={}", resultStr, placeHolder, key); + } if (keyset.contains(key)) { String value = variablesMap.get(key); if (value != null) { resultStr = resultStr.replace(placeHolder, value); } } else { - log.warn("There is no value to replace {} in {},variablesMap:{}", placeHolder, rawStr, variablesMap); + if (log.isDebugEnabled()) { + log.debug("There is no value to replace {} in {},variablesMap:{}", placeHolder, rawStr, + variablesMap); + } } } // 变量值中含有变量，递归替换 @@ -294,4 +335,117 @@ public static String escape(String str, char[] chars, String[] escapeTexts) { return writer.toString(); } + /** + * 判断目标字符串是否与任意一个搜索内容匹配 + * + * @param targetStr 目标字符串 + * @param searchContents 搜索内容集合 + * @return 是否匹配 + */ + public static boolean matchAnySearchContent(String targetStr, + Collection searchContents) { + if (CollectionUtils.isEmpty(searchContents)) { + return true; + } + if (targetStr == null) { + return false; + } + for (String content : searchContents) { + if (StringUtils.isNotBlank(content) + && targetStr.toLowerCase().contains(content.toLowerCase())) { + return true; + } + } + return false; + } + + /** + * 返回限定长度的字符 + * + * @param str 字符串 + * @param maxLength 最大长度(字符) + * @return 限定长度的字符 + */ + public static String substring(String str, int maxLength) { + if (str == null) { + return null; + } + if (maxLength <= 0) { + return ""; + } + if (str.length() <= maxLength) { + return str; + } else { + return str.substring(0, maxLength); + } + } + + /** + * 从字符串集合中抽取指定类型的数据（不抽取null值） + * + * @param rawStrs 原始字符串集合 + * @param clazz 目标类型Class对象 + * @param 目标类型 + * @return Pair<抽取的值列表 ， 抽取后剩余的字符串列表> + */ + public static Pair, List> extractValueFromStrings(Collection rawStrs, + Class clazz) { + List valueList = new ArrayList<>(); + List remainStrList = new ArrayList<>(); + for (String rawStr : rawStrs) { + if (StringUtils.isBlank(rawStr)) { + remainStrList.add(rawStr); + continue; + } + try { + T value = parseStringToType(rawStr.trim(), clazz); + valueList.add(value); + } catch (Exception e) { + remainStrList.add(rawStr); + } + } + return Pair.of(valueList, remainStrList); + } + + /** + * 将字符串内容解析为指定类型的值 + * + * @param str 原始字符串 + * @param clazz 目标类型Class对象 + * @param 目标类型 + * @return 目标类型的值 + */ + private static T parseStringToType(String str, Class clazz) { + if (clazz == Integer.class) { + return clazz.cast(Integer.parseInt(str)); + } else if (clazz == Long.class) { + return clazz.cast(Long.parseLong(str)); + } else if (clazz == Double.class) { + return clazz.cast(Double.parseDouble(str)); + } else if (clazz == Boolean.class) { + return clazz.cast(parseStrictBoolean(str)); + } else if (clazz == Float.class) { + return clazz.cast(Float.parseFloat(str)); + } else { + throw new IllegalArgumentException("Unsupported type: " + clazz.getName()); + } + } + + /** + * 从字符串解析布尔值（仅接受 "true" 或 "false"，大小写不敏感） + * + * @param str 原始字符串 + * @return 布尔值 + */ + private static Boolean parseStrictBoolean(String str) { + if ("true".equalsIgnoreCase(str)) return true; + if ("false".equalsIgnoreCase(str)) return false; + throw new IllegalArgumentException("Invalid boolean value: " + str); + } + + public static String removeHttpOrHttpsSchemeOfUrl(String url) { + String urlWithoutScheme = StringUtil.removePrefix(url, "http://"); + urlWithoutScheme = StringUtil.removePrefix(urlWithoutScheme, "https://"); + return urlWithoutScheme; + } } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/TagUtils.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/TagUtils.java index 639f848cfe..c8ebe9429e 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/TagUtils.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/TagUtils.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,6 +27,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import org.slf4j.helpers.MessageFormatter; import java.util.ArrayList; import java.util.Collections; @@ -85,7 +86,11 @@ public static List decodeDbTag(String tags) { } catch (NumberFormatException e) { log.warn("Tag must be number {}!", tagString); } catch (Exception e) { - log.error("Exception while processing tag {}!", tagString, e); + String msg = MessageFormatter.format( + "Exception while processing tag {}!", + tagString + ).getMessage(); + log.error(msg, e); } } } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/TimeUtil.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/TimeUtil.java index f1c13487dc..abfbc02646 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/TimeUtil.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/TimeUtil.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,21 +24,20 @@ package com.tencent.bk.job.common.util; +import lombok.extern.slf4j.Slf4j; + import java.text.DateFormat; import java.text.SimpleDateFormat; import java.time.Instant; import java.time.LocalDateTime; import java.time.ZoneId; import java.time.ZoneOffset; +import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; import java.util.Calendar; import java.util.Date; -/** - * @Description - * @Date 2020/3/6 - * @Version 1.0 - */ +@Slf4j public class TimeUtil { public static final String DEFAULT_TIME_FORMAT = "yyyy-MM-dd HH:mm:ss"; @@ -155,4 +154,20 @@ public static String formatTime(long timeMills, String format) { DateFormat formatter = new SimpleDateFormat(format); return formatter.format(date); } + + /** + * 解析 DateTimeFormatter.ISO_DATE_TIME 格式的时间 + * + * @param zonedTimeStr 带有时区的时间字符串 + * @return Unix时间戳（ms） + */ + public static Long parseIsoZonedTimeToMillis(String zonedTimeStr) { + try { + ZonedDateTime zonedDateTime = ZonedDateTime.parse(zonedTimeStr, DateTimeFormatter.ISO_DATE_TIME); + return zonedDateTime.toInstant().toEpochMilli(); + } catch (Exception e) { + log.warn("Fail to parseZonedTime from: {}", zonedTimeStr); + return null; + } + } } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/TypeUtil.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/TypeUtil.java index 2ff8886de1..05e4dee9eb 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/TypeUtil.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/TypeUtil.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/Utils.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/Utils.java index 3eaf2c9454..b62c17f603 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/Utils.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/Utils.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/Wrapper.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/Wrapper.java index 263ea72535..46a33f3e06 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/Wrapper.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/Wrapper.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/check/IStringCheckStrategy.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/check/IStringCheckStrategy.java index 9dc90557e0..d468080f5a 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/check/IStringCheckStrategy.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/check/IStringCheckStrategy.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/check/IlegalCharChecker.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/check/IlegalCharChecker.java index 64abf3b957..1964d85f4a 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/check/IlegalCharChecker.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/check/IlegalCharChecker.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/check/MaxLengthChecker.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/check/MaxLengthChecker.java index 4bc2b03ca8..08d314df3a 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/check/MaxLengthChecker.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/check/MaxLengthChecker.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/check/NotEmptyChecker.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/check/NotEmptyChecker.java index fa4eae79b5..f3ec5e7496 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/check/NotEmptyChecker.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/check/NotEmptyChecker.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/check/ParamCheckUtil.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/check/ParamCheckUtil.java index dcb832006e..33632407e6 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/check/ParamCheckUtil.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/check/ParamCheckUtil.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/check/StringCheckHelper.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/check/StringCheckHelper.java index db078b5103..a40424d82b 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/check/StringCheckHelper.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/check/StringCheckHelper.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/check/TrimChecker.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/check/TrimChecker.java index dc75539324..1ce9116cd6 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/check/TrimChecker.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/check/TrimChecker.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/check/WhiteCharChecker.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/check/WhiteCharChecker.java index e5cc7ea4c2..6a33a139de 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/check/WhiteCharChecker.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/check/WhiteCharChecker.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/check/exception/StringCheckException.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/check/exception/StringCheckException.java index dd4a3dd434..4093ae7b72 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/check/exception/StringCheckException.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/check/exception/StringCheckException.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/feature/FeatureToggle.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/feature/FeatureToggle.java deleted file mode 100644 index 139ced7e5c..0000000000 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/feature/FeatureToggle.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.util.feature; - -import com.tencent.bk.job.common.config.FeatureToggleConfig; -import com.tencent.bk.job.common.util.ApplicationContextRegister; - -/** - * 特性开关 - */ -public class FeatureToggle { - - private FeatureToggle() { - } - - private static FeatureToggleConfig get() { - return Inner.instance; - } - - private static class Inner { - private static final FeatureToggleConfig instance = - ApplicationContextRegister.getBean(FeatureToggleConfig.class); - } - - /** - * 是否兼容ESB bk_biz_id 参数 - */ - public static boolean isBkBizIdEnabled() { - FeatureToggleConfig featureToggleConfig = get(); - FeatureToggleConfig.ToggleConfig toggleConfig = featureToggleConfig.getEsbApiParamBkBizId(); - return toggleConfig.isOpen(); - } -} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/file/CharsetDetectHelper.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/file/CharsetDetectHelper.java index 2009b77373..64cfe55dfa 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/file/CharsetDetectHelper.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/file/CharsetDetectHelper.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/file/EncodingUtils.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/file/EncodingUtils.java index aaffc033a9..2244e92aa8 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/file/EncodingUtils.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/file/EncodingUtils.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/file/FileSizeUtil.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/file/FileSizeUtil.java deleted file mode 100644 index 1e20b5d002..0000000000 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/file/FileSizeUtil.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.util.file; - -public class FileSizeUtil { - public static String getFileSizeStr(Long byteNum) { - if (byteNum == null) return "--"; - String[] units = new String[]{"B", "KB", "MB", "GB", "TB", "PB"}; - int i = 0; - double value = (float) byteNum; - while (value >= 1024 && i < units.length - 1) { - value = value / 1024.; - i += 1; - } - return String.format("%.2f", value) + units[i]; - } - - public static void main(String[] args) { - System.out.println(getFileSizeStr(1023L)); - System.out.println(getFileSizeStr(1024L)); - System.out.println(getFileSizeStr(1024 * 1024L)); - System.out.println(getFileSizeStr(1024 * 1024L - 1)); - System.out.println(getFileSizeStr(1024 * 1024 * 1025L)); - System.out.println(getFileSizeStr(1024 * 1024 * 1025 * 1024L)); - System.out.println(getFileSizeStr(1024 * 1024 * 1025 * 1024L * 1024L)); - System.out.println(getFileSizeStr(1024 * 1024 * 1025 * 1024L * 1024L * 1024L)); - } -} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/file/PathUtil.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/file/PathUtil.java index 11fd26916d..b2f11d80ab 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/file/PathUtil.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/file/PathUtil.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/file/ZipUtil.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/file/ZipUtil.java index 6aa04a83ee..5d4b72636e 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/file/ZipUtil.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/file/ZipUtil.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,11 @@ package com.tencent.bk.job.common.util.file; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.InvalidParamException; +import com.tencent.bk.job.common.util.FilePathUtils; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.Pair; import java.io.*; import java.nio.charset.StandardCharsets; @@ -154,7 +158,19 @@ public static List unzip(File source) { zis = new ZipInputStream(in, StandardCharsets.UTF_8); ZipEntry entry; while ((entry = zis.getNextEntry()) != null && !entry.isDirectory()) { - File target = new File(source.getParent(), entry.getName()); + String rawFileName = entry.getName(); + String pureFileName = FilePathUtils.getPureFileName(rawFileName); + if (isIllegalFileName(pureFileName)) { + throw new InvalidParamException(ErrorCode.ILLEGAL_FILE); + } + if (!rawFileName.trim().equals(pureFileName)) { + log.warn( + "AttackAttemptFound|Illegal name of zip file entry: {}, replace it with {}", + rawFileName, + pureFileName + ); + } + File target = new File(source.getParent(), pureFileName); unzipFiles.add(target); if (!target.getParentFile().exists()) {// 创建文件父目录 boolean createDirSuccess = target.getParentFile().mkdirs(); @@ -182,6 +198,9 @@ public static List unzip(File source) { return unzipFiles; } + private static boolean isIllegalFileName(String fileName) { + return (fileName.equals("..") || fileName.equals(".")); + } /** * 关闭一个或多个流对象 diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/BaseHttpHelper.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/BaseHttpHelper.java index 8d80ec02a9..916743504b 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/BaseHttpHelper.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/BaseHttpHelper.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,11 +25,16 @@ package com.tencent.bk.job.common.util.http; import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.constant.HttpMethodEnum; +import com.tencent.bk.job.common.exception.HttpStatusException; import com.tencent.bk.job.common.exception.InternalException; +import com.tencent.bk.job.common.exception.NotImplementedException; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.http.Header; import org.apache.http.HttpEntity; +import org.apache.http.HttpEntityEnclosingRequest; import org.apache.http.HttpStatus; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; @@ -38,10 +43,12 @@ import org.apache.http.client.methods.HttpRequestBase; import org.apache.http.entity.ByteArrayEntity; import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.protocol.HttpContext; +import org.apache.http.protocol.HttpCoreContext; import org.apache.http.util.EntityUtils; +import org.slf4j.helpers.MessageFormatter; import java.io.IOException; -import java.io.UnsupportedEncodingException; @Slf4j public class BaseHttpHelper implements HttpHelper { @@ -49,14 +56,10 @@ public class BaseHttpHelper implements HttpHelper { private final CloseableHttpClient httpClient; - protected BaseHttpHelper(CloseableHttpClient httpClient) { + public BaseHttpHelper(CloseableHttpClient httpClient) { this.httpClient = httpClient; } - CloseableHttpClient getHttpClient() { - return httpClient; - } - @Override public Pair getRawResp(boolean keepAlive, String url, Header[] header) { HttpGet get = new HttpGet(url); @@ -67,7 +70,7 @@ public Pair getRawResp(boolean keepAlive get.setHeaders(header); } try { - return Pair.of(get, getHttpClient().execute(get)); + return Pair.of(get, httpClient.execute(get)); } catch (IOException e) { log.error("Get request fail", e); throw new InternalException(e, ErrorCode.API_ERROR); @@ -79,171 +82,157 @@ public Pair getRawResp(boolean keepAlive } @Override - public Pair get(boolean keepAlive, String url, Header[] header) { - HttpGet get = new HttpGet(url); - if (keepAlive) { - get.setHeader("Connection", "Keep-Alive"); + public HttpResponse requestForSuccessResp(HttpRequest request) throws HttpStatusException { + return requestInternal(request, true); + } + + private HttpResponse requestInternal(HttpRequest request, + boolean throwExceptionWhenClientOrServerError) { + HttpMethodEnum method = request.getMethod(); + HttpContext httpContext = buildHttpContext(request); + HttpRequestBase httpClientRequest; + switch (method) { + case GET: + httpClientRequest = buildHttpGet(request); + break; + case POST: + httpClientRequest = buildHttpPost(request); + break; + case PUT: + httpClientRequest = buildHttpPut(request); + break; + case DELETE: + httpClientRequest = buildHttpDelete(request); + break; + default: + log.warn("Unsupported http method : {}", method); + throw new NotImplementedException(ErrorCode.API_ERROR); } - if (header != null && header.length > 0) { - get.setHeaders(header); + return execute(httpClientRequest, httpContext, throwExceptionWhenClientOrServerError); + } + + @Override + public HttpResponse request(HttpRequest request) { + return requestInternal(request, false); + } + + private HttpGet buildHttpGet(HttpRequest request) { + HttpGet get = new HttpGet(request.getUrl()); + setCommonHttpClientRequest(request, get); + + return get; + } + + private void setCommonHttpClientRequest(HttpRequest request, HttpRequestBase httpClientRequest) { + setConnectionKeepAlive(request, httpClientRequest); + if (request.getHeaders() != null && request.getHeaders().length > 0) { + httpClientRequest.setHeaders(request.getHeaders()); } - int httpStatusCode = -1; - String respStr = null; - try (CloseableHttpResponse response = getHttpClient().execute(get)) { - httpStatusCode = response.getStatusLine().getStatusCode(); - HttpEntity entity = response.getEntity(); - respStr = EntityUtils.toString(entity, CHARSET); - return Pair.of(httpStatusCode, respStr); - } catch (IOException e) { - log.error("Get request fail", e); - throw new InternalException(e, ErrorCode.API_ERROR); - } finally { - get.releaseConnection(); - if (log.isDebugEnabled()) { - log.debug( - "get:keepAlive={},url={},headers={},httpStatusCode={},respStr={}", - keepAlive, - url, - header, - httpStatusCode, - respStr - ); + } + + private HttpPost buildHttpPost(HttpRequest request) { + HttpPost post = new HttpPost(request.getUrl()); + setCommonHttpClientRequest(request, post); + setEntity(post, request); + return post; + } + + private HttpPut buildHttpPut(HttpRequest request) { + HttpPut put = new HttpPut(request.getUrl()); + setCommonHttpClientRequest(request, put); + setEntity(put, request); + return put; + } + + private FakeHttpDelete buildHttpDelete(HttpRequest request) { + FakeHttpDelete delete = new FakeHttpDelete(request.getUrl()); + setCommonHttpClientRequest(request, delete); + setEntity(delete, request); + return delete; + } + + private void setEntity(HttpEntityEnclosingRequest request, HttpRequest httpRequest) { + if (httpRequest.getHttpEntity() != null) { + request.setEntity(httpRequest.getHttpEntity()); + } else if (StringUtils.isNotBlank(httpRequest.getStringEntity())) { + try { + request.setEntity(new ByteArrayEntity(httpRequest.getStringEntity().getBytes(CHARSET))); + } catch (IOException e) { + throw new InternalException(e, ErrorCode.API_ERROR); } } } - @Override - public Pair post(String url, HttpEntity requestEntity, Header... headers) { - HttpPost post = new HttpPost(url); - // 设置为长连接，服务端判断有此参数就不关闭连接。 - post.setHeader("Connection", "Keep-Alive"); - post.setHeaders(headers); - post.setEntity(requestEntity); - int httpStatusCode = -1; - String respStr = null; - try (CloseableHttpResponse httpResponse = getHttpClient().execute(post)) { - httpStatusCode = httpResponse.getStatusLine().getStatusCode(); - if (httpStatusCode != HttpStatus.SC_OK) { - String message = httpResponse.getStatusLine().getReasonPhrase(); - HttpEntity entity = httpResponse.getEntity(); - if (entity != null && entity.getContent() != null) { - respStr = new String(EntityUtils.toByteArray(entity), CHARSET); - } - log.warn( - "Post request fail, httpStatusCode={}, errorReason={}, body={}, url={}, headers={}", - httpStatusCode, - message, - respStr, - url, - headers - ); - throw new InternalException(message, ErrorCode.API_ERROR); - } - HttpEntity entity = httpResponse.getEntity(); - return Pair.of(httpStatusCode, EntityUtils.toByteArray(entity)); - } catch (IOException e) { - log.error("Post request fail", e); - throw new InternalException(e, ErrorCode.API_ERROR); - } finally { - post.releaseConnection(); - if (log.isDebugEnabled()) { - log.debug( - "post:url={},headers={},requestEntity={},httpStatusCode={},respStr={}", - url, - headers, - requestEntity, - httpStatusCode, - respStr - ); - } + private HttpContext buildHttpContext(HttpRequest request) { + HttpCoreContext httpContext = HttpCoreContext.create(); + if (request.getRetryMode() != null) { + httpContext.setAttribute(HttpContextAttributeNames.RETRY_MODE, request.getRetryMode().getValue()); + } + if (request.getIdempotent() != null) { + httpContext.setAttribute(HttpContextAttributeNames.IS_IDEMPOTENT, request.getIdempotent()); } + return httpContext; } - @Override - public Pair put(String url, HttpEntity requestEntity, Header... headers) { - HttpPut put = new HttpPut(url); - // 设置为长连接，服务端判断有此参数就不关闭连接。 - put.setHeader("Connection", "Keep-Alive"); - put.setHeaders(headers); - put.setEntity(requestEntity); + private void setConnectionKeepAlive(HttpRequest request, HttpRequestBase httpRequestBase) { + if (request.isKeepAlive()) { + // 设置为长连接，服务端判断有此参数就不关闭连接。 + httpRequestBase.setHeader("Connection", "Keep-Alive"); + } + } + + private HttpResponse execute(HttpRequestBase httpClientRequest, + HttpContext context, + boolean throwExceptionWhenClientOrServerError) { int httpStatusCode = -1; String respStr = null; - try (CloseableHttpResponse httpResponse = getHttpClient().execute(put)) { + Long contentLength = null; + try (CloseableHttpResponse httpResponse = httpClient.execute(httpClientRequest, context)) { httpStatusCode = httpResponse.getStatusLine().getStatusCode(); HttpEntity entity = httpResponse.getEntity(); - respStr = new String(EntityUtils.toByteArray(entity), CHARSET); - if (httpStatusCode != HttpStatus.SC_OK) { - String message = httpResponse.getStatusLine().getReasonPhrase(); - log.warn( - "Put request fail, httpStatusCode={}, errorReason={}, respStr={}", - httpStatusCode, - message, - respStr - ); - throw new InternalException(message, ErrorCode.API_ERROR); + if (entity != null && entity.getContent() != null) { + contentLength = entity.getContentLength(); + respStr = new String(EntityUtils.toByteArray(entity), CHARSET); } - return Pair.of(httpStatusCode, respStr); - } catch (IOException e) { - log.error("Put request fail", e); - throw new InternalException(e, ErrorCode.API_ERROR); - } finally { - put.releaseConnection(); - if (log.isDebugEnabled()) { - log.debug( - "put:url={},headers={},requestEntity={},httpStatusCode={},respStr={}", - url, - headers, - requestEntity, + // 状态码>=400判定为失败 + if (httpStatusCode >= HttpStatus.SC_BAD_REQUEST) { + String reasonPhrase = httpResponse.getStatusLine().getReasonPhrase(); + log.warn( + "Request fail, method: {}, url={}, httpStatusCode={}, errorReason={}, body={}", + httpClientRequest.getMethod(), + httpClientRequest.getURI().getPath(), httpStatusCode, + reasonPhrase, respStr ); + if (throwExceptionWhenClientOrServerError) { + throw new HttpStatusException( + httpClientRequest.getURI().toString(), + httpStatusCode, + reasonPhrase, + respStr + ); + } else { + return new HttpResponse(httpStatusCode, respStr, httpResponse.getAllHeaders()); + } + } else { + return new HttpResponse(httpStatusCode, respStr, httpResponse.getAllHeaders()); } - } - } - - @Override - public Pair delete(String url, String content, Header... headers) { - FakeHttpDelete delete = new FakeHttpDelete(url); - if (content != null) { - HttpEntity requestEntity; - try { - requestEntity = new ByteArrayEntity(content.getBytes(CHARSET)); - } catch (UnsupportedEncodingException e) { - log.error("Fail to get ByteArrayEntity", e); - throw new InternalException(e, ErrorCode.INTERNAL_ERROR); - } - delete.setEntity(requestEntity); - } - delete.setHeaders(headers); - int httpStatusCode = -1; - String respStr = null; - try (CloseableHttpResponse httpResponse = getHttpClient().execute(delete)) { - httpStatusCode = httpResponse.getStatusLine().getStatusCode(); - if (httpStatusCode != HttpStatus.SC_OK) { - String message = httpResponse.getStatusLine().getReasonPhrase(); - log.info("Delete request fail, url={}, httpStatusCode={}, errorReason={}", url, httpStatusCode, - message); - throw new InternalException(String.format("url=%s,httpStatusCode=%s" + - "，message=%s", url, httpStatusCode, message), ErrorCode.API_ERROR); - } - HttpEntity entity = httpResponse.getEntity(); - byte[] respBytes = EntityUtils.toByteArray(entity); - if (respBytes == null) { - return null; - } - respStr = new String(respBytes, CHARSET); - return Pair.of(httpStatusCode, respStr); } catch (IOException e) { - log.error("Delete request fail", e); + String message = MessageFormatter.format( + "Request fail, httpStatusCode={}, contentLength={}", + httpStatusCode, + contentLength + ).getMessage(); + log.error(message, e); throw new InternalException(e, ErrorCode.API_ERROR); } finally { - delete.releaseConnection(); + httpClientRequest.releaseConnection(); if (log.isDebugEnabled()) { log.debug( - "delete:url={},headers={},body={},httpStatusCode={},respStr={}", - url, - headers, - content, + "Request done, method: {}, url={}, httpStatusCode={}, respStr={}", + httpClientRequest.getMethod(), + httpClientRequest.getURI().getPath(), httpStatusCode, respStr ); diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/BasicHttpReq.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/BasicHttpReq.java index a59e6d89b8..02a3012b6e 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/BasicHttpReq.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/BasicHttpReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/ExtHttpHelper.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/ExtHttpHelper.java deleted file mode 100644 index 0fa3c8cf65..0000000000 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/ExtHttpHelper.java +++ /dev/null @@ -1,205 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.util.http; - -import com.tencent.bk.job.common.constant.ErrorCode; -import com.tencent.bk.job.common.exception.InternalException; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.http.Header; -import org.apache.http.HttpEntity; -import org.apache.http.client.methods.CloseableHttpResponse; -import org.apache.http.client.methods.HttpRequestBase; -import org.apache.http.entity.ByteArrayEntity; -import org.apache.http.message.BasicHeader; - -import java.io.IOException; -import java.util.List; - -@Slf4j -public class ExtHttpHelper { - private final String CHARSET = "UTF-8"; - - private final HttpHelper httpHelper; - - protected ExtHttpHelper(HttpHelper httpHelper) { - this.httpHelper = httpHelper; - } - - /** - * 支持自定义头的POST请求 - * - * @param url 提交的地址 - * @param content 提交的内容字符串 - * @param headers 自定义请求头 - * @return - */ - public String post(String url, String content, Header... headers) { - return post(url, CHARSET, content, headers); - } - - public String post(String url, String charset, String content, List

headerList) throws Exception { - Header[] headers = new Header[headerList.size()]; - return post(url, charset, content, headerList.toArray(headers)); - } - - - /** - * 支持自定义头的POST请求 - * - * @param url 提交的地址 - * @param charset 字符集，用于解析返回的字符串 - * @param content 提交的内容字符串 - * @param headers 自定义请求头 - * @return - */ - public String post(String url, String charset, String content, Header... headers) { - try { - byte[] resp = post(url, new ByteArrayEntity(content.getBytes(charset)), headers); - if (null == resp) { - return null; - } - return new String(resp, charset); - } catch (IOException e) { - log.error("Post request fail", e); - throw new InternalException(e, ErrorCode.API_ERROR); - } - } - - /** - * 提交POST请求，并返回字节数组 - * - * @param url 提交的地址 - * @param content 提交的内容字节数据 - * @param contentType 默认传null则为"application/x-www-form-urlencoded" - * @return 返回字节数组 - */ - public byte[] post(String url, byte[] content, String contentType) { - return post(url, new ByteArrayEntity(content), contentType); - } - - /** - * 提交POST请求，并返回字节数组 - * - * @param url 提交的地址 - * @param requestEntity 封装好的请求实体 - * @param contentType 默认传null则为"application/x-www-form-urlencoded" - * @return 返回字节数组 - */ - public byte[] post(String url, HttpEntity requestEntity, String contentType) { - return post(url, requestEntity, - new BasicHeader("Content-Type", contentType == null ? "application/x-www-form-urlencoded" : contentType)); - } - - public byte[] post(String url, HttpEntity requestEntity, Header... headers) { - return httpHelper.post(url, requestEntity, headers).getRight(); - } - - /** - * GET请求，并返回字符串 - * - * @param url 提交的地址 - * @return - */ - public String get(String url) { - return get(url, (Header[]) null); - } - - public String get(String url, List
headerList) { - Header[] headers = new Header[headerList.size()]; - return get(url, headerList.toArray(headers)); - } - - public String get(String url, Header[] header) { - return get(true, url, header); - } - - public String get(boolean keepAlive, String url, Header[] header) { - return httpHelper.get(keepAlive, url, header).getRight(); - } - - public Pair getRawResp(boolean keepAlive, String url, Header[] header) { - return httpHelper.getRawResp(keepAlive, url, header); - } - - public String put(String url, HttpEntity requestEntity, Header... headers) { - return httpHelper.put(url, requestEntity, headers).getRight(); - } - - /** - * 发起PUT请求并获取响应 - * - * @param url 地址 - * @param charset 字符集名称 - * @param content body内容 - * @param headerList 请求头列表 - * @return 响应字符串 - */ - public String put(String url, String charset, String content, List
headerList) { - Header[] headers = new Header[headerList.size()]; - try { - return put(url, new ByteArrayEntity(content.getBytes(charset)), headerList.toArray(headers)); - } catch (IOException e) { - log.error("Put request fail", e); - throw new InternalException(e, ErrorCode.API_ERROR); - } - } - - /** - * 发起DELETE请求并获取响应 - * - * @param url 地址 - * @param content body内容 - * @param headers 请求头数组 - * @return 响应字符串 - */ - public String delete(String url, String content, Header... headers) { - return httpHelper.delete(url, content, headers).getRight(); - } - - /** - * 发起DELETE请求并获取响应 - * - * @param url 地址 - * @param content body内容 - * @param headerList 请求头列表 - * @return 响应字符串 - */ - public String delete(String url, String content, List
headerList) { - Header[] headers = new Header[headerList.size()]; - return httpHelper.delete(url, content, headerList.toArray(headers)).getRight(); - } - - /** - * 发起DELETE请求并获取响应 - * - * @param url 地址 - * @param headerList 请求头列表 - * @return 响应字符串 - */ - public String delete(String url, List
headerList) { - return delete(url, null, headerList); - } -} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpConPoolUtil.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpConPoolUtil.java index 1557dd79cc..7af797286b 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpConPoolUtil.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpConPoolUtil.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -38,6 +38,7 @@ import org.apache.http.conn.ssl.SSLConnectionSocketFactory; import org.apache.http.conn.ssl.TrustSelfSignedStrategy; import org.apache.http.entity.ByteArrayEntity; +import org.apache.http.entity.ContentType; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClientBuilder; import org.apache.http.message.BasicHeader; @@ -92,7 +93,7 @@ public class HttpConPoolUtil { * @param contentType 默认传null则为"application/x-www-form-urlencoded" * @return 响应 */ - public static String post(String url, String content, String contentType) { + public static HttpResponse post(String url, String content, String contentType) { return post(url, CHARSET, content, contentType); } @@ -105,30 +106,37 @@ public static String post(String url, String content, String contentType) { * @param contentType 默认传null则为"application/x-www-form-urlencoded" * @return 响应 */ - public static String post(String url, String charset, String content, String contentType) { + public static HttpResponse post(String url, String charset, String content, String contentType) { try { - byte[] resp = post(url, content.getBytes(charset), contentType); - if (null == resp) { - return null; - } - return new String(resp, charset); + return post(url, content.getBytes(charset), contentType); } catch (IOException e) { - log.error("Post request fail", e); + log.warn("Post request fail", e); throw new InternalException(e, ErrorCode.API_ERROR); } } /** - * 提交POST请求，并返回数据（默认采用encoding常量指定的字符集解析） + * 提交POST请求，数据格式为表单数据，并返回响应数据 * * @param url 提交的地址 * @param content 提交的内容字符串 * @return 响应 */ - public static String post(String url, String content) { + public static HttpResponse postFormData(String url, String content) { return post(url, CHARSET, content, "application/x-www-form-urlencoded"); } + /** + * 提交POST请求，数据格式为JSON数据，并返回响应数据 + * + * @param url 提交的地址 + * @param content 提交的内容字符串 + * @return 响应 + */ + public static HttpResponse postJson(String url, String content) { + return post(url, CHARSET, content, ContentType.APPLICATION_JSON.getMimeType()); + } + /** * 支持自定义头的POST请求 * @@ -137,7 +145,7 @@ public static String post(String url, String content) { * @param headers 自定义请求头 * @return 响应 */ - public static String post(String url, String content, Header... headers) { + public static HttpResponse post(String url, String content, Header... headers) { return post(url, CHARSET, content, headers); } @@ -150,15 +158,11 @@ public static String post(String url, String content, Header... headers) { * @param headers 自定义请求头 * @return 响应 */ - public static String post(String url, String charset, String content, Header... headers) { + public static HttpResponse post(String url, String charset, String content, Header... headers) { try { - byte[] resp = post(url, new ByteArrayEntity(content.getBytes(charset)), headers); - if (null == resp) { - return null; - } - return new String(resp, charset); + return post(url, new ByteArrayEntity(content.getBytes(charset)), headers); } catch (IOException e) { - log.error("Post request fail", e); + log.warn("Post request fail", e); throw new InternalException(e, ErrorCode.API_ERROR); } } @@ -171,7 +175,7 @@ public static String post(String url, String charset, String content, Header... * @param contentType 默认传null则为"application/x-www-form-urlencoded" * @return 返回字节数组 */ - public static byte[] post(String url, byte[] content, String contentType) { + public static HttpResponse post(String url, byte[] content, String contentType) { return post(url, new ByteArrayEntity(content), contentType); } @@ -183,30 +187,32 @@ public static byte[] post(String url, byte[] content, String contentType) { * @param contentType 默认传null则为"application/x-www-form-urlencoded" * @return 返回字节数组 */ - public static byte[] post(String url, HttpEntity requestEntity, String contentType) { + public static HttpResponse post(String url, HttpEntity requestEntity, String contentType) { return post(url, requestEntity, - new BasicHeader("Content-Type", contentType == null ? "application/x-www-form-urlencoded" : contentType)); + new BasicHeader("Content-Type", + contentType == null ? "application/x-www-form-urlencoded" : contentType)); } - public static byte[] post(String url, HttpEntity requestEntity, Header... headers) { + public static HttpResponse post(String url, HttpEntity requestEntity, Header... headers) { HttpPost post = new HttpPost(url); // 设置为长连接，服务端判断有此参数就不关闭连接。 post.setHeader("Connection", "Keep-Alive"); post.setHeaders(headers); post.setEntity(requestEntity); - try (CloseableHttpResponse httpResponse = HTTP_CLIENT.execute(post)) { - int statusCode = httpResponse.getStatusLine().getStatusCode(); + try (CloseableHttpResponse response = HTTP_CLIENT.execute(post)) { + int statusCode = response.getStatusLine().getStatusCode(); log.info("Post url: {}, statusCode: {}", url, statusCode); if (statusCode != HttpStatus.SC_OK) { - String errorMsg = buildErrorMessage("Post", url, statusCode, - httpResponse.getStatusLine().getReasonPhrase()); - log.error(errorMsg); - throw new InternalException(errorMsg, ErrorCode.API_ERROR); + String errorMsg = buildErrorMessage("Post", + url, + statusCode, + response.getStatusLine().getReasonPhrase()); + log.warn(errorMsg); } - HttpEntity entity = httpResponse.getEntity(); - return EntityUtils.toByteArray(entity); + String entity = response.getEntity() != null ? EntityUtils.toString(response.getEntity()) : null; + return new HttpResponse(statusCode, entity, response.getAllHeaders()); } catch (IOException e) { - log.error("Post request fail", e); + log.warn("Post request fail", e); throw new InternalException(e, ErrorCode.API_ERROR); } } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpContextAttributeNames.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpContextAttributeNames.java new file mode 100644 index 0000000000..cfef283f54 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpContextAttributeNames.java @@ -0,0 +1,39 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.util.http; + +/** + * HTTP Client Context 属性名称定义 + */ +public interface HttpContextAttributeNames { + /** + * 重试模式 + */ + String RETRY_MODE = "RETRY_MODE"; + /** + * 请求是否幂等 + */ + String IS_IDEMPOTENT = "IS_IDEMPOTENT"; +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpHelper.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpHelper.java index 02c66923c2..720a1609b4 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpHelper.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpHelper.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,21 +24,33 @@ package com.tencent.bk.job.common.util.http; +import com.tencent.bk.job.common.exception.HttpStatusException; import org.apache.commons.lang3.tuple.Pair; import org.apache.http.Header; -import org.apache.http.HttpEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpRequestBase; +/** + * Job http 调用基础实现 + */ public interface HttpHelper { Pair getRawResp(boolean keepAlive, String url, Header[] header); - Pair get(boolean keepAlive, String url, Header[] header); - - Pair post(String url, HttpEntity requestEntity, Header... headers); - - Pair put(String url, HttpEntity requestEntity, Header... headers); - - Pair delete(String url, String content, Header... headers); + /** + * 发起 http 请求。如果返回的 http 状态码 >= 400, 抛出 HttpStatusException + * + * @param request 请求 + * @return 响应 + * @throws HttpStatusException 如果返回的 http 状态码 >= 400, 抛出 HttpStatusException + */ + HttpResponse requestForSuccessResp(HttpRequest request) throws HttpStatusException; + + /** + * 发起 http 请求 + * + * @param request 请求 + * @return 响应 + */ + HttpResponse request(HttpRequest request); } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpHelperFactory.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpHelperFactory.java index 9ec28d2b1c..b7fb62f416 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpHelperFactory.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpHelperFactory.java @@ -2,21 +2,10 @@ import io.micrometer.core.instrument.MeterRegistry; import lombok.extern.slf4j.Slf4j; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.conn.socket.LayeredConnectionSocketFactory; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; -import org.apache.http.conn.ssl.TrustSelfSignedStrategy; +import org.apache.http.client.HttpRequestRetryHandler; import org.apache.http.impl.client.CloseableHttpClient; -import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.impl.client.StandardHttpRequestRetryHandler; -import org.apache.http.ssl.SSLContexts; import org.springframework.stereotype.Service; -import java.security.KeyManagementException; -import java.security.KeyStoreException; -import java.security.NoSuchAlgorithmException; -import java.util.concurrent.TimeUnit; - /** * Http请求基础工厂类 */ @@ -30,101 +19,118 @@ public class HttpHelperFactory { private static final CloseableHttpClient RETRYABLE_HTTP_CLIENT; private static final CloseableHttpClient LONG_RETRYABLE_HTTP_CLIENT; - private static CloseableHttpClient getHttpClient(@SuppressWarnings("SameParameterValue") int connRequestTimeout, - @SuppressWarnings("SameParameterValue") int connTimeout, - int socketTimeout, - boolean canRetry) { - return getHttpClient( - connRequestTimeout, - connTimeout, - socketTimeout, + static { + DEFAULT_HTTP_CLIENT = JobHttpClientFactory.createHttpClient( + 15000, + 15000, + 15000, 500, 1000, - canRetry - ); - } - - private static CloseableHttpClient getHttpClient(int connRequestTimeout, - int connTimeout, - int socketTimeout, - @SuppressWarnings("SameParameterValue") int maxConnPerRoute, - @SuppressWarnings("SameParameterValue") int maxConnTotal, - boolean canRetry) { - HttpClientBuilder httpClientBuilder = HttpClientBuilder.create() - .setDefaultRequestConfig( - RequestConfig.custom() - .setConnectionRequestTimeout(connRequestTimeout) - .setConnectTimeout(connTimeout) - .setSocketTimeout(socketTimeout) - .build() - ) - .evictExpiredConnections() - .evictIdleConnections(5, TimeUnit.SECONDS) - .disableAutomaticRetries() - .disableAuthCaching() - .disableCookieManagement(); - if (canRetry) { - httpClientBuilder.setRetryHandler(new StandardHttpRequestRetryHandler()); - } - CloseableHttpClient httpClient; - LayeredConnectionSocketFactory sslSocketFactory = null; - try { - sslSocketFactory = new SSLConnectionSocketFactory( - SSLContexts.custom() - .loadTrustMaterial(null, new TrustSelfSignedStrategy()) - .build() - ); - } catch (NoSuchAlgorithmException | KeyManagementException | KeyStoreException e) { - log.error("", e); - } - httpClientBuilder.setConnectionManager( - JobHttpClientConnectionManagerFactory.createWatchableConnectionManager( - maxConnPerRoute, - maxConnTotal, - sslSocketFactory - )); - httpClient = httpClientBuilder.build(); - return httpClient; - } - - static { - DEFAULT_HTTP_CLIENT = getHttpClient( - 15000, 15000, 15000, false - ); - RETRYABLE_HTTP_CLIENT = getHttpClient( - 15000, 15000, 15000, true - ); - LONG_RETRYABLE_HTTP_CLIENT = getHttpClient( - 15000, 15000, 30000, true - ); + 60, + false, + null); + RETRYABLE_HTTP_CLIENT = JobHttpClientFactory.createHttpClient( + 15000, + 15000, + 15000, + 500, + 1000, + 60, + true, + new JobHttpRequestRetryHandler()); + LONG_RETRYABLE_HTTP_CLIENT = JobHttpClientFactory.createHttpClient( + 15000, + 15000, + 35000, + 500, + 1000, + 60, + true, + new JobHttpRequestRetryHandler()); } public static void setMeterRegistry(MeterRegistry meterRegistry) { HttpHelperFactory.meterRegistry = meterRegistry; } - private static HttpHelper getWatchableHttpHelper(HttpHelper httpHelper) { - return new WatchableHttpHelper(httpHelper, meterRegistry); + private static WatchableHttpHelper getWatchableHttpHelper(HttpHelper httpHelper) { + return new WatchableHttpHelper(httpHelper, () -> meterRegistry); } - private static ExtHttpHelper getWatchableExtHelper(HttpHelper httpHelper) { - HttpHelper watchableHttpHelper = getWatchableHttpHelper(httpHelper); - return new ExtHttpHelper(watchableHttpHelper); + private static WatchableHttpHelper getWatchableExtHelper(HttpHelper httpHelper) { + return getWatchableHttpHelper(httpHelper); } - public static ExtHttpHelper getDefaultHttpHelper() { + public static WatchableHttpHelper getDefaultHttpHelper() { HttpHelper baseHttpHelper = new BaseHttpHelper(DEFAULT_HTTP_CLIENT); return getWatchableExtHelper(baseHttpHelper); } @SuppressWarnings("unused") - public static ExtHttpHelper getRetryableHttpHelper() { + public static WatchableHttpHelper getRetryableHttpHelper() { HttpHelper baseHttpHelper = new BaseHttpHelper(RETRYABLE_HTTP_CLIENT); return getWatchableExtHelper(baseHttpHelper); } - public static ExtHttpHelper getLongRetryableHttpHelper() { + public static WatchableHttpHelper getLongRetryableHttpHelper() { HttpHelper baseHttpHelper = new BaseHttpHelper(LONG_RETRYABLE_HTTP_CLIENT); return getWatchableExtHelper(baseHttpHelper); } + + /** + * 创建自定义的HttpHelper，每创建一次都会生成一个常驻的连接驱逐线程，请在单例中使用，避免线程泄露 + * + * @param customizer 自定义的HttpClient定制器 + * @return 自定义的HttpHelper + */ + public static WatchableHttpHelper createHttpHelper(JobHttpClientFactory.HttpClientCustomizer customizer) { + HttpHelper baseHttpHelper = createHttpHelper( + 15000, + 15000, + 15000, + 500, + 1000, + 60, + false, + null, + customizer + ); + return getWatchableExtHelper(baseHttpHelper); + } + + /** + * 创建自定义的HttpHelper，每创建一次都会生成一个常驻的连接驱逐线程，请在单例中使用，避免线程泄露 + * + * @param connRequestTimeout 连接请求超时时间，单位毫秒 + * @param connTimeout 连接超时时间，单位毫秒 + * @param socketTimeout socket读写超时时间，单位毫秒 + * @param maxConnPerRoute 单个路由最大连接数 + * @param maxConnTotal 总的最大连接数 + * @param timeToLive 连接驱逐线程存活时间，单位秒 + * @param allowRetry 是否允许重试 + * @param retryHandler 重试策略 + * @param customizer 自定义的HttpClient定制器 + * @return 自定义的HttpHelper + */ + public static HttpHelper createHttpHelper(int connRequestTimeout, + int connTimeout, + int socketTimeout, + int maxConnPerRoute, + int maxConnTotal, + int timeToLive, + boolean allowRetry, + HttpRequestRetryHandler retryHandler, + JobHttpClientFactory.HttpClientCustomizer customizer) { + CloseableHttpClient httpClient = JobHttpClientFactory.createHttpClient( + connRequestTimeout, + connTimeout, + socketTimeout, + maxConnPerRoute, + maxConnTotal, + timeToLive, + allowRetry, + retryHandler, + customizer); + return new BaseHttpHelper(httpClient); + } } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpMetricUtil.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpMetricUtil.java index dd75158459..cf81716cd9 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpMetricUtil.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpMetricUtil.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpReqGenUtil.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpReqGenUtil.java index 3679092dbb..d1c432f865 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpReqGenUtil.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpReqGenUtil.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,6 +28,7 @@ import com.tencent.bk.job.common.model.http.HttpReq; import com.tencent.bk.job.common.util.JobContextUtil; import com.tencent.bk.job.common.util.json.JsonUtils; +import org.apache.commons.collections4.CollectionUtils; import org.apache.http.Header; import org.apache.http.message.BasicHeader; @@ -35,16 +36,25 @@ import java.util.List; public class HttpReqGenUtil { - public static HttpReq genSimpleJsonReq(String url, Object body) { + public static HttpReq genSimpleJsonReq(String url, List
headerList, Object body) { HttpReq httpReq = new HttpReq(); httpReq.setUrl(url); String bodyStr = JsonUtils.toJson(body); httpReq.setBody(bodyStr); - List
headerList = new ArrayList<>(); - headerList.add(new BasicHeader("Content-Type", "application/json;charset=UTF-8")); - headerList.add(new BasicHeader(LocaleUtils.COMMON_LANG_HEADER, JobContextUtil.getUserLang())); - Header[] headers = new Header[headerList.size()]; - httpReq.setHeaders(headerList.toArray(headers)); + List
finalHeaderList = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(headerList)) { + finalHeaderList.addAll(headerList); + } + finalHeaderList.add(new BasicHeader("Content-Type", "application/json;charset=UTF-8")); + finalHeaderList.add(new BasicHeader(LocaleUtils.COMMON_LANG_HEADER, JobContextUtil.getUserLang())); + Header[] headers = new Header[finalHeaderList.size()]; + httpReq.setHeaders(finalHeaderList.toArray(headers)); + return httpReq; + } + + public static HttpReq genUrlGetReq(String url) { + HttpReq httpReq = new HttpReq(); + httpReq.setUrl(url); return httpReq; } } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpRequest.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpRequest.java new file mode 100644 index 0000000000..82d94804b9 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpRequest.java @@ -0,0 +1,128 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.util.http; + +import com.tencent.bk.job.common.constant.HttpMethodEnum; +import lombok.Data; +import org.apache.http.Header; +import org.apache.http.HttpEntity; + +/** + * Http 请求 + */ +@Data +public class HttpRequest { + private HttpMethodEnum method; + private String url; + /** + * 文本格式的 entity + */ + private String stringEntity; + /** + * 标准 httpEntity，优先级最高 + */ + private HttpEntity httpEntity; + private Header[] headers; + /** + * 请求重试模式 + */ + private RetryModeEnum retryMode; + /** + * 请求是否幂等 + */ + private final Boolean idempotent; + + private boolean keepAlive; + + public HttpRequest(Builder builder) { + this.method = builder.method; + this.url = builder.url; + this.stringEntity = builder.stringEntity; + this.httpEntity = builder.httpEntity; + this.headers = builder.headers; + this.retryMode = builder.retryMode; + this.idempotent = builder.idempotent; + this.keepAlive = builder.keepAlive; + } + + public static Builder builder(HttpMethodEnum method, String url) { + return new Builder(method, url); + } + + public static class Builder { + private final HttpMethodEnum method; + private final String url; + private String stringEntity; + private HttpEntity httpEntity; + private Header[] headers; + private RetryModeEnum retryMode = RetryModeEnum.SAFE_GUARANTEED; + private Boolean idempotent; + /** + * 按照 http1.1 之后的标准，默认都使用长连接 + */ + private boolean keepAlive = true; + + public Builder(HttpMethodEnum method, String url) { + this.method = method; + this.url = url; + } + + public Builder setStringEntity(String stringEntity) { + this.stringEntity = stringEntity; + return this; + } + + public Builder setHttpEntity(HttpEntity httpEntity) { + this.httpEntity = httpEntity; + return this; + } + + public Builder setHeaders(Header[] headers) { + this.headers = headers; + return this; + } + + public Builder setRetryMode(RetryModeEnum retryMode) { + this.retryMode = retryMode; + return this; + } + + public Builder setIdempotent(Boolean idempotent) { + this.idempotent = idempotent; + return this; + } + + public Builder setKeepAlive(boolean keepAlive) { + this.keepAlive = keepAlive; + return this; + } + + public HttpRequest build() { + return new HttpRequest(this); + } + } + + +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpResponse.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpResponse.java new file mode 100644 index 0000000000..ce451a928a --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/HttpResponse.java @@ -0,0 +1,52 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.util.http; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import org.apache.http.Header; + +@NoArgsConstructor +@Getter +@Setter +public class HttpResponse { + /** + * http status + */ + private int statusCode; + /** + * response entity + */ + private String entity; + + private Header[] headers; + + public HttpResponse(int statusCode, String entity, Header[] headers) { + this.statusCode = statusCode; + this.entity = entity; + this.headers = headers; + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/JobHttpClient.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/JobHttpClient.java index 15c1d64d12..7374e43a86 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/JobHttpClient.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/JobHttpClient.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -31,6 +31,8 @@ */ public interface JobHttpClient { + String get(HttpReq req); + String post(HttpReq req); } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/JobHttpClientConnectionManagerFactory.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/JobHttpClientConnectionManagerFactory.java index da2b2c5c73..d7e6537691 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/JobHttpClientConnectionManagerFactory.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/JobHttpClientConnectionManagerFactory.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -57,11 +57,10 @@ public class JobHttpClientConnectionManagerFactory { public static HttpClientConnectionManager createWatchableConnectionManager( int maxConnPerRoute, int maxConnTotal, + long timeToLive, + TimeUnit timeUnit, LayeredConnectionSocketFactory sslSocketFactory ) { - // esb的keep-alive时间为90s，需要<90s,防止连接超时抛出org.apache.http.NoHttpResponseException: - // The target server failed to respond - // 因此，timeToLive使用34s final PoolingHttpClientConnectionManager poolingmgr = new WatchablePoolingHttpClientConnectionManager( RegistryBuilder.create() @@ -71,8 +70,8 @@ public static HttpClientConnectionManager createWatchableConnectionManager( new WatchableManagedHttpClientConnectionFactory(), null, null, - 34, - TimeUnit.SECONDS + timeToLive, + timeUnit ); if (defaultConnectionConfig != null) { poolingmgr.setDefaultConnectionConfig(defaultConnectionConfig); diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/JobHttpClientFactory.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/JobHttpClientFactory.java new file mode 100644 index 0000000000..7ad0a473c5 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/JobHttpClientFactory.java @@ -0,0 +1,141 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.util.http; + +import lombok.extern.slf4j.Slf4j; +import org.apache.http.client.HttpRequestRetryHandler; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.conn.socket.LayeredConnectionSocketFactory; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.conn.ssl.TrustSelfSignedStrategy; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.client.StandardHttpRequestRetryHandler; +import org.apache.http.ssl.SSLContexts; + +import java.security.KeyManagementException; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.util.concurrent.TimeUnit; + +/** + * Job http client 工厂 + */ +@Slf4j +public class JobHttpClientFactory { + /** + * 创建 HttpClient + * + * @param connRequestTimeout 从连接池获取到连接的超时时间，单位秒 + * @param connTimeout 建立连接的超时时间，单位秒 + * @param socketTimeout 客户端与服务端进行交互的时间，单位秒 + * @param maxConnPerRoute 每个路由的最大连接 + * @param maxConnTotal 连接池最大连接数 + * @param timeToLive 连接存活时间 + * @param retryHandler 请求重试Handler; 如果传入 null，表示不需要重试 + * @return HttpClient + */ + public static CloseableHttpClient createHttpClient(int connRequestTimeout, + int connTimeout, + int socketTimeout, + int maxConnPerRoute, + int maxConnTotal, + int timeToLive, + boolean allowRetry, + HttpRequestRetryHandler retryHandler) { + return createHttpClient( + connRequestTimeout, + connTimeout, + socketTimeout, + maxConnPerRoute, + maxConnTotal, + timeToLive, + allowRetry, + retryHandler, + (httpClientBuilder) -> { + // do nothing + }); + } + + public static CloseableHttpClient createHttpClient(int connRequestTimeout, + int connTimeout, + int socketTimeout, + int maxConnPerRoute, + int maxConnTotal, + int timeToLive, + boolean allowRetry, + HttpRequestRetryHandler retryHandler, + HttpClientCustomizer customizer) { + HttpClientBuilder httpClientBuilder = HttpClientBuilder.create() + .setDefaultRequestConfig( + RequestConfig.custom() + .setConnectionRequestTimeout(connRequestTimeout) + .setConnectTimeout(connTimeout) + .setSocketTimeout(socketTimeout) + .build() + ) + .evictExpiredConnections() + .evictIdleConnections(5, TimeUnit.SECONDS) + .disableAuthCaching() + .disableCookieManagement(); + if (!allowRetry) { + httpClientBuilder.disableAutomaticRetries(); + } else { + if (retryHandler != null) { + httpClientBuilder.setRetryHandler(retryHandler); + } else { + httpClientBuilder.setRetryHandler(new StandardHttpRequestRetryHandler()); + } + } + + LayeredConnectionSocketFactory sslSocketFactory = null; + try { + sslSocketFactory = new SSLConnectionSocketFactory( + SSLContexts.custom() + .loadTrustMaterial(null, new TrustSelfSignedStrategy()) + .build() + ); + } catch (NoSuchAlgorithmException | KeyManagementException | KeyStoreException e) { + log.error("Create HttpClient exception", e); + } + httpClientBuilder.setConnectionManager( + JobHttpClientConnectionManagerFactory.createWatchableConnectionManager( + maxConnPerRoute, + maxConnTotal, + timeToLive, + TimeUnit.SECONDS, + sslSocketFactory + )); + + if (customizer != null) { + customizer.customize(httpClientBuilder); + } + return httpClientBuilder.build(); + } + + public interface HttpClientCustomizer { + void customize(HttpClientBuilder httpClientBuilder); + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/JobHttpClientImpl.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/JobHttpClientImpl.java index d1552c8b9b..f3fa3d03ce 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/JobHttpClientImpl.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/JobHttpClientImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -30,6 +30,7 @@ import com.tencent.bk.job.common.model.http.HttpReq; import com.tencent.bk.job.common.util.json.JsonUtils; import lombok.extern.slf4j.Slf4j; +import org.apache.http.Header; import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpStatus; @@ -46,14 +47,34 @@ public JobHttpClientImpl(RestTemplate restTemplate) { this.restTemplate = restTemplate; } + @Override + public String get(HttpReq req) { + logReq(req); + ResponseEntity respEntity = restTemplate.getForEntity( + req.getUrl(), + String.class + ); + if (respEntity.getStatusCode() == HttpStatus.OK) { + String respStr = respEntity.getBody(); + logRespStr(respStr); + return respStr; + } + logAndThrow(respEntity); + return null; + } + @Override public String post(HttpReq req) { logReq(req); - HttpHeaders headers = new HttpHeaders(); - MediaType type = MediaType.APPLICATION_JSON; - headers.setContentType(type); + HttpHeaders httpHeaders = new HttpHeaders(); + Header[] headers = req.getHeaders(); + for (Header header : headers) { + if (header.getName() != null && header.getValue() != null) { + httpHeaders.add(header.getName(), header.getValue()); + } + } String requestJson = req.getBody(); - HttpEntity entity = new HttpEntity<>(requestJson, headers); + HttpEntity entity = new HttpEntity<>(requestJson, httpHeaders); ResponseEntity respEntity = restTemplate.postForEntity( req.getUrl(), entity, @@ -69,12 +90,15 @@ public String post(HttpReq req) { } private void logReq(HttpReq req) { - log.info( - "url={},body={},headers={}", - req.getUrl(), - req.getBody(), - JsonUtils.toJson(req.getHeaders()) - ); + if (log.isDebugEnabled()) { + // 内容中可能有敏感信息，非必要不开启 + log.debug( + "url={},body={},headers={}", + req.getUrl(), + req.getBody(), + JsonUtils.toJson(req.getHeaders()) + ); + } } private void logRespStr(String respStr) { @@ -82,7 +106,7 @@ private void logRespStr(String respStr) { } private void logAndThrow(ResponseEntity respEntity) { - log.error("Fail to request fileWorker, status={}, msg={}", respEntity.getStatusCode(), respEntity.getBody()); + log.error("Fail to request, status={}, msg={}", respEntity.getStatusCode(), respEntity.getBody()); throw new ServiceException( ErrorType.INTERNAL, ErrorCode.FAIL_TO_REQUEST_FILE_WORKER_WITH_REASON, diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/JobHttpRequestRetryHandler.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/JobHttpRequestRetryHandler.java new file mode 100644 index 0000000000..d17212c5ba --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/JobHttpRequestRetryHandler.java @@ -0,0 +1,175 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.util.http; + +import lombok.extern.slf4j.Slf4j; +import org.apache.http.HttpRequest; +import org.apache.http.NoHttpResponseException; +import org.apache.http.client.HttpRequestRetryHandler; +import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.conn.ConnectTimeoutException; +import org.apache.http.protocol.HttpContext; + +import java.io.IOException; +import java.net.SocketTimeoutException; +import java.util.HashSet; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +import static com.tencent.bk.job.common.util.http.RetryModeEnum.SAFE_GUARANTEED; + +/** + * Job 第三方请求调用重试 Handler + */ +@Slf4j +public class JobHttpRequestRetryHandler implements HttpRequestRetryHandler { + + private final int retryCount; + + private final Map retryMethods; + + private final Set> retryableExceptions; + + public JobHttpRequestRetryHandler() { + this.retryCount = 3; + this.retryMethods = new ConcurrentHashMap<>(); + // 蓝鲸各平台仅能保证 GET、DELETE 是幂等的，跟 RESTful API 规范有差异 + this.retryMethods.put("GET", Boolean.TRUE); + this.retryMethods.put("DELETE", Boolean.TRUE); + retryableExceptions = new HashSet<>(); + // ConnectTimeoutException 建立连接超时/从连接池获取连接超时 + // SocketTimeoutException Socket连接/读取超时 + // NoHttpResponseException 服务器端关闭连接或者服务端负载太高无法响应，nginx reload，以及网络连接问题 + // httpclient 异常处理参考 https://hc.apache.org/httpclient-legacy/exception-handling.html 的说明 + retryableExceptions.add(ConnectTimeoutException.class); + retryableExceptions.add(SocketTimeoutException.class); + retryableExceptions.add(NoHttpResponseException.class); + } + + @Override + public boolean retryRequest(IOException exception, + int executionCount, + HttpContext context) { + if (isExceedMaxRetry(executionCount)) { + log.info("[{}] Retry rejected. Http request exceed max retry times : {}", + exception.getClass().getSimpleName(), this.retryCount); + return false; + } + + if (!isExceptionRetryable(exception)) { + log.info("[{}] Retry rejected. Exception is not retryable", exception.getClass().getSimpleName()); + return false; + } + + return checkByRetryMode(context, exception); + + } + + private boolean isExceptionRetryable(IOException exception) { + if (log.isDebugEnabled()) { + log.debug("Http exception : {}", exception.getClass().getName()); + } + return retryableExceptions.contains(exception.getClass()); + } + + private boolean checkByRetryMode(HttpContext context, IOException exception) { + RetryModeEnum retryMode = getRetryMode(context); + + switch (retryMode) { + case ALWAYS: + log.info("[{}] Retry accepted. Retry mode is ALWAYS", exception.getClass().getSimpleName()); + return true; + case NEVER: + log.info("[{}] Retry rejected. Retry mode is NEVER", exception.getClass().getSimpleName()); + return false; + case SAFE_GUARANTEED: + // 判断重试请求是否安全 + boolean isRetryable = checkIsRetrySafe(context, exception); + if (isRetryable) { + log.info("[{}] Retry accepted. Retry is safe", exception.getClass().getSimpleName()); + return true; + } else { + log.info("[{}] Retry rejected. Retry is not safe", exception.getClass().getSimpleName()); + return false; + } + default: + return false; + } + + } + + private boolean checkIsRetrySafe(HttpContext context, IOException exception) { + boolean isRequestIdempotent = isRequestIdempotent(context); + if (isRequestIdempotent) { + return true; + } + // ConnectTimeoutException/NoHttpResponseException，服务端并没有开始接受和正式处理请求，可以重试 + // SocketTimeoutException 可能服务端已经接受处理了请求，如果二次请求可能引发问题 + return exception instanceof ConnectTimeoutException || exception instanceof NoHttpResponseException; + } + + private boolean isRequestIdempotent(HttpContext context) { + // 判断方法使用幂等 + Object isIdempotentAttrVal = context.getAttribute(HttpContextAttributeNames.IS_IDEMPOTENT); + if (log.isDebugEnabled()) { + log.debug("HttpContext::IS_IDEMPOTENT: {}", isIdempotentAttrVal); + } + boolean isIdempotent = isIdempotentAttrVal != null && (boolean) isIdempotentAttrVal; + if (isIdempotent) { + // 如果上下文主动设置了该请求的幂等参数，那么优先使用 + return true; + } + + // 上下文没有主动设置幂等参数，那么按照 http method 的实际幂等情况判断 + final HttpClientContext clientContext = HttpClientContext.adapt(context); + final HttpRequest request = clientContext.getRequest(); + final String method = request.getRequestLine().getMethod().toUpperCase(Locale.ROOT); + final Boolean b = this.retryMethods.get(method); + if (log.isDebugEnabled()) { + log.debug("Method: {}, isRequestIdempotent: {}",method, isIdempotentAttrVal); + } + return b != null && b; + } + + + private RetryModeEnum getRetryMode(HttpContext context) { + Object retryModeObj = context.getAttribute(HttpContextAttributeNames.RETRY_MODE); + RetryModeEnum retryMode = SAFE_GUARANTEED; + if (retryModeObj != null) { + retryMode = RetryModeEnum.valOf((int) retryModeObj); + } + if (log.isDebugEnabled()) { + log.debug("RetryMode : {}", retryMode); + } + return retryMode; + } + + private boolean isExceedMaxRetry(int executionCount) { + return executionCount > this.retryCount; + } + +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/RetryModeEnum.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/RetryModeEnum.java new file mode 100644 index 0000000000..cf2f2c66bf --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/RetryModeEnum.java @@ -0,0 +1,62 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.util.http; + +/** + * http 请求重试方式 + */ +public enum RetryModeEnum { + /** + * 总是重试 + */ + ALWAYS(1), + /** + * 禁止重试 + */ + NEVER(2), + /** + * 在保证安全的情况下尽可能重试 + */ + SAFE_GUARANTEED(3); + + private final int value; + + RetryModeEnum(int value) { + this.value = value; + } + + public static RetryModeEnum valOf(int val) { + for (RetryModeEnum modeEnum : values()) { + if (modeEnum.value == val) { + return modeEnum; + } + } + throw new IllegalArgumentException("No RetryModeEnum constant: " + val); + } + + public int getValue() { + return value; + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/WatchableConnectionRequestProxy.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/WatchableConnectionRequestProxy.java index 947a2f0102..a1dc8f9bf7 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/WatchableConnectionRequestProxy.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/WatchableConnectionRequestProxy.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/WatchableHttpHelper.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/WatchableHttpHelper.java index fc3515b1df..d8bc52c640 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/WatchableHttpHelper.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/WatchableHttpHelper.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,17 +24,19 @@ package com.tencent.bk.job.common.util.http; +import com.tencent.bk.job.common.exception.HttpStatusException; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Tag; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.http.Header; -import org.apache.http.HttpEntity; import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpRequestBase; import java.util.AbstractList; import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import java.util.function.Supplier; /** * 对Http请求状态码与异常情况进行统计便于监控 @@ -42,11 +44,11 @@ public class WatchableHttpHelper implements HttpHelper { private final HttpHelper httpHelper; - private final MeterRegistry meterRegistry; + private final Supplier meterRegistrySupplier; - public WatchableHttpHelper(HttpHelper httpHelper, MeterRegistry meterRegistry) { + public WatchableHttpHelper(HttpHelper httpHelper, Supplier meterRegistrySupplier) { this.httpHelper = httpHelper; - this.meterRegistry = meterRegistry; + this.meterRegistrySupplier = meterRegistrySupplier; } @Override @@ -55,106 +57,33 @@ public Pair getRawResp(boolean keepAlive } @Override - public Pair get(boolean keepAlive, String url, Header[] header) { - String httpMetricName = HttpMetricUtil.getHttpMetricName(); - long start = System.nanoTime(); - String httpStatus = null; - try { - Pair pair = httpHelper.get(keepAlive, url, header); - if (pair != null) { - httpStatus = "" + pair.getLeft(); - } else { - httpStatus = "null"; - } - return pair; - } catch (Throwable t) { - httpStatus = "error"; - throw t; - } finally { - long end = System.nanoTime(); - AbstractList httpMetricTags = HttpMetricUtil.getCurrentMetricTags(); - httpMetricTags.add(Tag.of("http_status", httpStatus)); - if (meterRegistry != null && StringUtils.isNotBlank(httpMetricName)) { - meterRegistry.timer(httpMetricName, httpMetricTags) - .record(end - start, TimeUnit.NANOSECONDS); - } - } - } - - @Override - public Pair post(String url, HttpEntity requestEntity, Header... headers) { - String httpMetricName = HttpMetricUtil.getHttpMetricName(); - long start = System.nanoTime(); - String httpStatus = null; - try { - Pair pair = httpHelper.post(url, requestEntity, headers); - if (pair != null) { - httpStatus = "" + pair.getLeft(); - } else { - httpStatus = "null"; - } - return pair; - } catch (Throwable t) { - httpStatus = "error"; - throw t; - } finally { - long end = System.nanoTime(); - AbstractList httpMetricTags = HttpMetricUtil.getCurrentMetricTags(); - httpMetricTags.add(Tag.of("http_status", httpStatus)); - if (meterRegistry != null && StringUtils.isNotBlank(httpMetricName)) { - meterRegistry.timer(httpMetricName, httpMetricTags) - .record(end - start, TimeUnit.NANOSECONDS); - } - } + public HttpResponse requestForSuccessResp(HttpRequest request) throws HttpStatusException { + return requestInternal(request, httpHelper::requestForSuccessResp); } @Override - public Pair put(String url, HttpEntity requestEntity, Header... headers) { - String httpMetricName = HttpMetricUtil.getHttpMetricName(); - long start = System.nanoTime(); - String httpStatus = null; - try { - Pair pair = httpHelper.put(url, requestEntity, headers); - if (pair != null) { - httpStatus = "" + pair.getLeft(); - } else { - httpStatus = "null"; - } - return pair; - } catch (Throwable t) { - httpStatus = "error"; - throw t; - } finally { - long end = System.nanoTime(); - AbstractList httpMetricTags = HttpMetricUtil.getCurrentMetricTags(); - httpMetricTags.add(Tag.of("http_status", httpStatus)); - if (meterRegistry != null && StringUtils.isNotBlank(httpMetricName)) { - meterRegistry.timer(httpMetricName, httpMetricTags) - .record(end - start, TimeUnit.NANOSECONDS); - } - } + public HttpResponse request(HttpRequest request) { + return requestInternal(request, httpHelper::request); } - @Override - public Pair delete(String url, String content, Header... headers) { + private HttpResponse requestInternal(HttpRequest request, + Function requestImpl) { String httpMetricName = HttpMetricUtil.getHttpMetricName(); long start = System.nanoTime(); - String httpStatus = null; + String httpStatusTagValue = null; try { - Pair pair = httpHelper.delete(url, content, headers); - if (pair != null) { - httpStatus = "" + pair.getLeft(); - } else { - httpStatus = "null"; - } - return pair; - } catch (Throwable t) { - httpStatus = "error"; + HttpResponse response = requestImpl.apply(request); + httpStatusTagValue = String.valueOf(response.getStatusCode()); + return response; + } catch (HttpStatusException t) { + httpStatusTagValue = String.valueOf(t.getHttpStatus()); throw t; } finally { long end = System.nanoTime(); AbstractList httpMetricTags = HttpMetricUtil.getCurrentMetricTags(); - httpMetricTags.add(Tag.of("http_status", httpStatus)); + httpMetricTags.add(Tag.of("http_status", + StringUtils.isNotEmpty(httpStatusTagValue) ? httpStatusTagValue : "UNKNOWN")); + MeterRegistry meterRegistry = meterRegistrySupplier.get(); if (meterRegistry != null && StringUtils.isNotBlank(httpMetricName)) { meterRegistry.timer(httpMetricName, httpMetricTags) .record(end - start, TimeUnit.NANOSECONDS); diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/WatchableManagedHttpClientConnectionFactory.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/WatchableManagedHttpClientConnectionFactory.java index fac186ecb3..e478ffe281 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/WatchableManagedHttpClientConnectionFactory.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/WatchableManagedHttpClientConnectionFactory.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/WatchablePoolingHttpClientConnectionManager.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/WatchablePoolingHttpClientConnectionManager.java index c7d6bd220a..cb812e6b0c 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/WatchablePoolingHttpClientConnectionManager.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/http/WatchablePoolingHttpClientConnectionManager.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/ip/IpUtils.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/ip/IpUtils.java index 74ea5783bc..56fa1f1d4f 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/ip/IpUtils.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/ip/IpUtils.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,19 +24,27 @@ package com.tencent.bk.job.common.util.ip; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.InvalidIpv6Exception; import com.tencent.bk.job.common.model.dto.HostDTO; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.math.NumberUtils; +import org.apache.commons.lang3.tuple.Pair; import org.apache.commons.validator.routines.InetAddressValidator; +import org.springframework.util.CollectionUtils; import java.net.Inet4Address; +import java.net.Inet6Address; import java.net.InetAddress; import java.net.NetworkInterface; import java.net.UnknownHostException; import java.util.Enumeration; import java.util.HashMap; +import java.util.HashSet; +import java.util.List; import java.util.Map; +import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -45,6 +53,8 @@ */ @Slf4j public class IpUtils { + public static final String PROTOCOL_IP_V4 = "v4"; + public static final String PROTOCOL_IP_V6 = "v6"; public static final String COLON = ":"; /** @@ -54,9 +64,6 @@ public class IpUtils { private static final Pattern IP_PATTERN = Pattern.compile( "\\b((?!\\d\\d\\d)\\d+|1\\d\\d|2[0-4]\\d|25[0-5])\\.((?!\\d\\d\\d)\\d+|1\\d\\d|2[0-4]\\d|25[0-5])\\.(" + "(?!\\d\\d\\d)\\d+|1\\d\\d|2[0-4]\\d|25[0-5])\\.((?!\\d\\d\\d)\\d+|1\\d\\d|2[0-4]\\d|25[0-5])\\b"); - private static final Pattern pattern = Pattern.compile( - "\\b((?!\\d\\d\\d)\\d+|1\\d\\d|2[0-4]\\d|25[0-5])\\.((?!\\d\\d\\d)\\d+|1\\d\\d|2[0-4]\\d|25[0-5])\\.(" + - "(?!\\d\\d\\d)\\d+|1\\d\\d|2[0-4]\\d|25[0-5])\\.((?!\\d\\d\\d)\\d+|1\\d\\d|2[0-4]\\d|25[0-5])\\b"); /** * 校验云区域:服务器IP @@ -166,7 +173,7 @@ public static long getStringIpToLong(String ip) { } /** - * 将long转换为ip + * 将数值转换为ip * * @return ip */ @@ -179,13 +186,13 @@ public static String revertIpFromLong(long ipLong) { } /** - * 将long转换为ip + * 将数值转换为ip * * @return ip */ - public static String revertIpFromLongStr(String longStr) { + public static String revertIpFromNumericalStr(String numericalStr) { try { - InetAddress addr = InetAddress.getByName(longStr); + InetAddress addr = InetAddress.getByName(numericalStr); return addr.getHostAddress(); } catch (UnknownHostException e) { return ""; @@ -194,16 +201,79 @@ public static String revertIpFromLongStr(String longStr) { public static boolean compileIP(String ip) { - Matcher matcher = pattern.matcher(ip); + Matcher matcher = IP_PATTERN.matcher(ip); return matcher.matches(); } /** - * 获取当前服务器IP地址 + * 从网卡获取首个机器IP，优先获取IPv4地址 + * 若获取到的值为v6协议的IP，默认提供完整无压缩的IPv6地址 * - * @return 返回当前服务器的网卡对应IP的MAP。 + * @return 首个机器IP地址 */ - public static Map getMachineIP() { + public static String getFirstMachineIP() { + return getFirstMachineIpPreferV4(); + } + + private static String getFirstMachineIpPreferV4() { + Map ipv4Map = getMachineIPv4Map(); + if (!ipv4Map.isEmpty()) { + return ipv4Map.values().iterator().next(); + } + Map ipv6Map = getMachineIPv6Map(); + if (!ipv6Map.isEmpty()) { + String ipv6 = ipv6Map.values().iterator().next(); + /* + * 此处处理原因详情可见Inet6Address.getHostAddress()方法说明。 + * Because link-local and site-local addresses are non-global, it is possible that + * different hosts may have the same destination address and may be reachable through + * different interfaces on the same originating system. In this case, the originating + * system is said to be connected to multiple zones of the same scope. In order to + * disambiguate which is the intended destination zone, it is possible to append a + * zone identifier (or scope_id) to an IPv6 address. + */ + if (ipv6.contains("%")) { + ipv6 = ipv6.substring(0, ipv6.indexOf("%")); + } + return getFullIpv6ByCompressedOne(ipv6); + } + log.error("no available ip, plz check net interface"); + return null; + } + + interface IpExtracter { + String extractIpFromInetAddress(InetAddress inetAddress); + } + + /** + * 获取当前服务器IPv4地址 + * + * @return Map<网卡名称 ， IP> + */ + private static Map getMachineIPv4Map() { + return getMachineIP(inetAddress -> { + if (inetAddress instanceof Inet4Address && !inetAddress.isLoopbackAddress()) { + return inetAddress.getHostAddress(); + } + return null; + }); + } + + /** + * 获取当前服务器IPv6地址 + * + * @return Map<网卡名称 ， IP> + */ + private static Map getMachineIPv6Map() { + return getMachineIP(inetAddress -> { + if (inetAddress instanceof Inet6Address && !inetAddress.isLoopbackAddress()) { + return inetAddress.getHostAddress(); + } + return null; + }); + } + + private static Map getMachineIP(IpExtracter ipExtracter) { log.info("#####################Start getMachineIP"); Map allIp = new HashMap<>(); @@ -212,41 +282,212 @@ public static Map getMachineIP() { Enumeration allNetInterfaces = NetworkInterface.getNetworkInterfaces(); if (null == allNetInterfaces) { log.error("#####################getMachineIP Can not get NetworkInterfaces"); - } else { - // 循环网卡获取网卡的IP地址 - while (allNetInterfaces.hasMoreElements()) { - NetworkInterface netInterface = allNetInterfaces.nextElement(); - String netInterfaceName = netInterface.getName(); - // 过滤掉127.0.0.1的IP - if (StringUtils.isBlank(netInterfaceName) || "lo".equalsIgnoreCase(netInterfaceName)) { - log.info("loopback地址或网卡名称为空"); - } else { - Enumeration addresses = netInterface.getInetAddresses(); - while (addresses.hasMoreElements()) { - InetAddress ip = addresses.nextElement(); - if (ip instanceof Inet4Address && !ip.isLoopbackAddress()) { - String machineIp = ip.getHostAddress(); - log.info("###############" + "netInterfaceName=" + netInterfaceName - + " The Macheine IP=" + machineIp); - allIp.put(netInterfaceName, machineIp); - } - } + return allIp; + } + // 循环网卡获取网卡的IP地址 + while (allNetInterfaces.hasMoreElements()) { + NetworkInterface netInterface = allNetInterfaces.nextElement(); + String netInterfaceName = netInterface.getName(); + // 过滤掉127.0.0.1的IP + if (StringUtils.isBlank(netInterfaceName) || "lo".equalsIgnoreCase(netInterfaceName)) { + log.info("loopback address or net interface name is blank"); + continue; + } + Enumeration addresses = netInterface.getInetAddresses(); + while (addresses.hasMoreElements()) { + InetAddress inetAddress = addresses.nextElement(); + String ip = ipExtracter.extractIpFromInetAddress(inetAddress); + if (StringUtils.isBlank(ip)) { + continue; } + log.info("NetInterfaceName={}, The Machine IP={}", netInterfaceName, ip); + allIp.put(netInterfaceName, ip); } } } catch (Exception e) { - log.error("获取网卡失败", e); + log.error("Fail to get network interfaces", e); } - return allIp; } - public static String getFirstMachineIP() { - String ip = getMachineIP().values().iterator().next(); - if (ip == null) { - ip = "no available ip"; + /** + * 提取IP + * + * @param cloudIp bkCloudId:ip + * @return ip + */ + public static String extractIp(String cloudIp) { + if (cloudIp == null) { + return null; + } + if (cloudIp.contains(":")) { + return cloudIp.substring(cloudIp.indexOf(":") + 1); + } else { + return cloudIp; + } + } + + /** + * 将纯IPv4地址与含云区域的IPv4地址分离开，清洗掉其中的空白字符、中括号并去重 + * + * @param ipv4OrCloudIpv4List ipv4/cloudIpv4列表 + * @return <纯IPv4地址集合，含云区域IPv4地址集合> + */ + public static Pair, Set> parseCleanIpv4AndCloudIpv4s(List ipv4OrCloudIpv4List) { + Set ipv4Set = new HashSet<>(); + Set cloudIpv4Set = new HashSet<>(); + if (CollectionUtils.isEmpty(ipv4OrCloudIpv4List)) { + return Pair.of(ipv4Set, cloudIpv4Set); + } + for (String ipv4OrCloudIpv4 : ipv4OrCloudIpv4List) { + if (ipv4OrCloudIpv4.contains(":")) { + String cloudIpStr = StringUtils.deleteWhitespace(ipv4OrCloudIpv4); + cloudIpv4Set.add(removeBrackets(cloudIpStr)); + } else { + ipv4Set.add(StringUtils.deleteWhitespace(ipv4OrCloudIpv4)); + } + } + return Pair.of(ipv4Set, cloudIpv4Set); + } + + private static String removeBrackets(String s) { + if (StringUtils.isBlank(s)) { + return s; } - return ip; + return s.replace("[", "").replace("]", ""); + } + + /** + * 将纯IPv6地址与含云区域的IPv6地址分离开，清洗掉其中的空白字符、中括号、去重并转为完整无压缩的IPv6地址 + * + * @param ipv6OrCloudIpv6List ipv6/cloudIpv6列表 + * @return <纯IPv6地址集合，含云区域IPv6地址集合> + */ + public static Pair, Set>> parseFullIpv6AndCloudIpv6s( + List ipv6OrCloudIpv6List + ) { + Set ipv6Set = new HashSet<>(); + Set> cloudIpv6Set = new HashSet<>(); + if (CollectionUtils.isEmpty(ipv6OrCloudIpv6List)) { + return Pair.of(ipv6Set, cloudIpv6Set); + } + for (String ipv6OrCloudIpv6 : ipv6OrCloudIpv6List) { + ipv6OrCloudIpv6 = StringUtils.deleteWhitespace(ipv6OrCloudIpv6); + if (ipv6OrCloudIpv6.contains(":[")) { + String[] cloudIpv6Arr = ipv6OrCloudIpv6.split(":\\["); + Long cloudId = Long.parseLong(cloudIpv6Arr[0]); + String ipv6 = removeBrackets(cloudIpv6Arr[1]); + if (!checkIpv6(ipv6)) { + log.warn("{} is not a valid ipv6 addr, ignore", ipv6); + continue; + } + if (StringUtils.isBlank(ipv6)) { + log.warn("Ipv6 address is invalid:{}", ipv6OrCloudIpv6); + } + String fullIpv6 = getFullIpv6ByCompressedOne(ipv6); + cloudIpv6Set.add(Pair.of(cloudId, fullIpv6)); + } else { + String ipv6 = removeBrackets(StringUtils.deleteWhitespace(ipv6OrCloudIpv6)); + if (!checkIpv6(ipv6)) { + log.warn("{} is not a valid ipv6 addr, ignore", ipv6); + continue; + } + String fullIpv6 = getFullIpv6ByCompressedOne(ipv6); + ipv6Set.add(fullIpv6); + } + } + return Pair.of(ipv6Set, cloudIpv6Set); + } + + /** + * 根据IP推断IP协议 + * + * @param ip ip地址 + * @return 协议号，常量值：PROTOCOL_IP_V6/PROTOCOL_IP_V4 + */ + public static String inferProtocolByIp(String ip) { + if (IpUtils.checkIpv6(ip)) { + return PROTOCOL_IP_V6; + } + return PROTOCOL_IP_V4; + } + + /** + * 有压缩的IPv6地址转换成完整无压缩的IPv6 + * + * @param compressedIpv6 有压缩的IPv6地址 + * @return 完整无压缩的IPv6地址 + */ + public static String getFullIpv6ByCompressedOne(String compressedIpv6) { + if (!checkIpv6(compressedIpv6)) { + throw new InvalidIpv6Exception(ErrorCode.INVALID_IPV6_ADDRESS, new String[]{compressedIpv6}); + } + String[] finalSeqArr = new String[]{"0000", "0000", "0000", "0000", "0000", "0000", "0000", "0000"}; + // 连续0标识符 + String continueZeroToken = "::"; + // 段之间分隔符 + String seqSeparator = ":"; + // 一个IPv6地址最多有8段 + int maxSeqNum = 8; + // ::开头的IP首位补0便于后续统一分割处理 + if (compressedIpv6.startsWith(continueZeroToken)) { + compressedIpv6 = "0" + compressedIpv6; + } + // ::结尾的IP末位补0便于后续统一分割处理 + if (compressedIpv6.endsWith(continueZeroToken)) { + compressedIpv6 = compressedIpv6 + "0"; + } + // 统一分割、解析 + if (compressedIpv6.contains(continueZeroToken)) { + String[] seqArr = compressedIpv6.split(continueZeroToken); + String[] leftSeqArr = seqArr[0].split(seqSeparator); + for (int i = 0; i < leftSeqArr.length && i < maxSeqNum; i++) { + finalSeqArr[i] = getStandardIpv6Seq(leftSeqArr[i]); + } + String[] rightSeqArr = seqArr[1].split(seqSeparator); + for (int i = 0; i < rightSeqArr.length && i < maxSeqNum; i++) { + finalSeqArr[i + maxSeqNum - rightSeqArr.length] = getStandardIpv6Seq(rightSeqArr[i]); + } + } else { + String[] seqArr = compressedIpv6.split(seqSeparator); + for (int i = 0; i < seqArr.length && i < maxSeqNum; i++) { + finalSeqArr[i] = getStandardIpv6Seq(seqArr[i]); + } + } + return StringUtils.join(finalSeqArr, ":"); + } + + /** + * 获取含有4个字符的Ipv6标准段，不足4字符则添加前缀0 + * + * @param ipv6Seq ipv6地址的一段 + * @return 4个字符的标准段 + */ + private static String getStandardIpv6Seq(String ipv6Seq) { + // 兼容IPv4地址的最后一个地址段 + if (ipv6Seq.contains(".")) { + return ipv6Seq; + } + String template = "0000"; + return template.substring(0, template.length() - ipv6Seq.length()) + ipv6Seq; + } + + /** + * 提取云区域ID + * + * @param cloudIp 云区域ID:IP + * @return 云区域ID + */ + public static Long extractBkCloudId(String cloudIp) { + if (cloudIp == null) { + throw new IllegalArgumentException("Empty cloudIp"); + } + int idx = cloudIp.indexOf(":"); + if (idx == -1) { + throw new IllegalArgumentException("Invalid cloudIp: " + cloudIp); + } + String bkCloudId = cloudIp.substring(0, idx); + return Long.parseLong(bkCloudId); } /** @@ -265,4 +506,8 @@ public static String getFirstIpFromMultiIp(String multiIp, String separator) { } return multiIp; } + + public static String buildCloudIp(Long bkCloudId, String ipv4) { + return bkCloudId + ":" + ipv4; + } } diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/json/DecimalFormatJsonSerializer.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/json/DecimalFormatJsonSerializer.java index dba92b00dc..343001061d 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/json/DecimalFormatJsonSerializer.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/json/DecimalFormatJsonSerializer.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/json/LongTimestampDeserializer.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/json/LongTimestampDeserializer.java index b863a6dcdc..e83269b22c 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/json/LongTimestampDeserializer.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/json/LongTimestampDeserializer.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/json/LongTimestampSerializer.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/json/LongTimestampSerializer.java index 438772b433..5e9460f4ef 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/json/LongTimestampSerializer.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/json/LongTimestampSerializer.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/json/LongToDecimalJsonSerializer.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/json/LongToDecimalJsonSerializer.java index f313f955ec..d4ea80b48f 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/json/LongToDecimalJsonSerializer.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/json/LongToDecimalJsonSerializer.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/json/PercentageFormatJsonSerializer.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/json/PercentageFormatJsonSerializer.java index 87ff5171e6..f9ff6ddf53 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/json/PercentageFormatJsonSerializer.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/json/PercentageFormatJsonSerializer.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/machine/MachineUtil.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/machine/MachineUtil.java index 0faa5e464b..a987456b60 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/machine/MachineUtil.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/machine/MachineUtil.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/toggle/ToggleEvaluateContext.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/toggle/ToggleEvaluateContext.java new file mode 100644 index 0000000000..7ab9c36a0b --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/toggle/ToggleEvaluateContext.java @@ -0,0 +1,84 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.util.toggle; + +import java.util.HashMap; +import java.util.Map; +import java.util.StringJoiner; +import java.util.function.Supplier; + +/** + * 开关评估上下文 + */ +public class ToggleEvaluateContext { + public static ToggleEvaluateContext EMPTY = new ToggleEvaluateContext(); + + /** + * 运行时参数 + */ + private final Map params = new HashMap<>(); + + /** + * 运行时参数 Supplier, 用于延迟计算可选参数 + */ + private final Map> paramSuppliers = new HashMap<>(); + + public static ToggleEvaluateContext builder() { + return new ToggleEvaluateContext(); + } + + private ToggleEvaluateContext() { + } + + public Object getParam(String paramName) { + Object value = this.params.get(paramName); + if (value != null) { + return value; + } + Supplier paramSupplier = this.paramSuppliers.get(paramName); + if (paramSupplier != null) { + value = paramSupplier.get(); + params.put(paramName, value); + } + return value; + } + + public ToggleEvaluateContext addContextParam(String paramName, Object value) { + this.params.put(paramName, value); + return this; + } + + public ToggleEvaluateContext addContextParam(String paramName, Supplier paramValueSupplier) { + this.paramSuppliers.put(paramName, paramValueSupplier); + return this; + } + + @Override + public String toString() { + return new StringJoiner(", ", ToggleEvaluateContext.class.getSimpleName() + "[", "]") + .add("params=" + params) + .toString(); + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/toggle/ToggleStrategy.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/toggle/ToggleStrategy.java new file mode 100644 index 0000000000..c7827f2d11 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/toggle/ToggleStrategy.java @@ -0,0 +1,51 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.util.toggle; + +import java.util.Map; + +/** + * 开关开启策略 + */ +public interface ToggleStrategy { + /** + * 获取开关开启策略ID + */ + String getId(); + + /** + * 获取初始化参数 + */ + Map getInitParams(); + + /** + * 计算开关是否开启 + * + * @param toggleName 开关名称 + * @param ctx 开关评估上下文 + * @return true: 开启；false: 关闭 + */ + boolean evaluate(String toggleName, ToggleEvaluateContext ctx); +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/toggle/ToggleStrategyContextParams.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/toggle/ToggleStrategyContextParams.java new file mode 100644 index 0000000000..33cd5956fb --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/toggle/ToggleStrategyContextParams.java @@ -0,0 +1,36 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.util.toggle; + +public interface ToggleStrategyContextParams { + /** + * 上下文参数-资源范围 + */ + String CTX_PARAM_RESOURCE_SCOPE = "resourceScope"; + /** + * 上下文参数-用户账号 + */ + String CTX_PARAM_USERNAME = "username"; +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/toggle/feature/Feature.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/toggle/feature/Feature.java new file mode 100644 index 0000000000..95736c9758 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/toggle/feature/Feature.java @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.util.toggle.feature; + +import com.tencent.bk.job.common.util.toggle.ToggleStrategy; +import lombok.Data; + +/** + * 特性 + */ +@Data +public class Feature { + /** + * 特性ID + */ + private String id; + /** + * 是否启用特性 + */ + private boolean enabled; + /** + * 特性启用灰度策略 + */ + private ToggleStrategy strategy; +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/toggle/feature/FeatureIdConstants.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/toggle/feature/FeatureIdConstants.java new file mode 100644 index 0000000000..4d2aa888bb --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/toggle/feature/FeatureIdConstants.java @@ -0,0 +1,64 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.util.toggle.feature; + +/** + * 特性ID定义 + */ +public interface FeatureIdConstants { + /** + * 特性: 任务下发对接 GSE2.0 + */ + String FEATURE_GSE_V2 = "gseV2"; + /** + * 特性: Agent状态对接 GSE2.0 + */ + String FEATURE_AGENT_STATUS_GSE_V2 = "agentStatusGseV2"; + /** + * 特性: OpenAPI 兼容bk_biz_id参数 + */ + String FEATURE_BK_BIZ_ID_COMPATIBLE = "bkBizIdCompatible"; + /** + * 特性-第三方文件源 + */ + String FEATURE_FILE_MANAGE = "fileManage"; + + /** + * 特性-是否支持GSE 获取文件分发任务结果的API协议(2.0版本之前) + */ + String GSE_FILE_PROTOCOL_BEFORE_V2 = "gseFileProtocolBeforeV2"; + /** + * 特性: 执行对象 + */ + String FEATURE_EXECUTE_OBJECT = "executeObject"; + /** + * 特性: 容器执行 + */ + String FEATURE_CONTAINER_EXECUTE = "containerExecution"; + /** + * 特性: 作业执行增加db 所有表增加 task_instance_id 字段作为分库分表 shard_key + */ + String DAO_ADD_TASK_INSTANCE_ID = "daoAddTaskInstanceId"; +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/toggle/feature/FeatureManager.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/toggle/feature/FeatureManager.java new file mode 100644 index 0000000000..ceee9c6c46 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/toggle/feature/FeatureManager.java @@ -0,0 +1,58 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.util.toggle.feature; + +import com.tencent.bk.job.common.util.toggle.ToggleEvaluateContext; + +import java.util.List; + +/** + * 特性管理 + */ +public interface FeatureManager { + /** + * 判断特性是否开启（只考虑特性本身，忽略策略） + * + * @param featureId 特性ID + * @return 是否开启 + */ + boolean isFeatureEnabled(String featureId); + + /** + * 判断特性是否开启 + * + * @param featureId 特性ID + * @param ctx 特性运行上下文 + * @return 是否开启 + */ + boolean checkFeature(String featureId, ToggleEvaluateContext ctx); + + /** + * 查询所有特性开关配置 + * + * @return 特性开关配置列表 + */ + List listFeatures(); +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/toggle/feature/FeatureStore.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/toggle/feature/FeatureStore.java new file mode 100644 index 0000000000..23940b6286 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/toggle/feature/FeatureStore.java @@ -0,0 +1,55 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.util.toggle.feature; + +import com.tencent.bk.job.common.refreshable.config.RefreshableConfigStore; + +import java.util.List; + +/** + * 特性开关配置存储 + */ +public interface FeatureStore extends RefreshableConfigStore { + + /** + * 属性名称前缀 + */ + String PROP_KEY_PREFIX = "job.features."; + + /** + * 返回特性开关配置 + * + * @param featureId 特性 ID + */ + Feature getFeature(String featureId); + + + /** + * 查询所有特性开关配置 + * + * @return 特性开关配置列表 + */ + List listFeatures(); +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/toggle/feature/FeatureToggle.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/toggle/feature/FeatureToggle.java new file mode 100644 index 0000000000..b77692c4d6 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/toggle/feature/FeatureToggle.java @@ -0,0 +1,71 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.util.toggle.feature; + +import com.tencent.bk.job.common.util.ApplicationContextRegister; +import com.tencent.bk.job.common.util.toggle.ToggleEvaluateContext; +import lombok.extern.slf4j.Slf4j; + +/** + * 特性开关工具类 + */ +@Slf4j +public class FeatureToggle { + + private static volatile FeatureManager featureManager = null; + + /** + * 判断特性是否开启 + * + * @param featureId 特性ID + * @param ctx 特性运行上下文 + * @return 是否开启 + */ + public static boolean checkFeature(String featureId, ToggleEvaluateContext ctx) { + ensureFeatureManagerInited(); + return featureManager.checkFeature(featureId, ctx); + } + + /** + * 判断特性是否开启（只考虑特性本身，忽略策略） + * + * @param featureId 特性ID + * @return 是否开启 + */ + public static boolean isFeatureEnabled(String featureId) { + ensureFeatureManagerInited(); + return featureManager.isFeatureEnabled(featureId); + } + + private static void ensureFeatureManagerInited() { + if (featureManager == null) { + synchronized (FeatureToggle.class) { + if (featureManager == null) { + featureManager = ApplicationContextRegister.getBean(FeatureManager.class); + } + } + } + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/toggle/prop/PropChangeEventListener.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/toggle/prop/PropChangeEventListener.java new file mode 100644 index 0000000000..940f71936f --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/toggle/prop/PropChangeEventListener.java @@ -0,0 +1,31 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.util.toggle.prop; + +public interface PropChangeEventListener { + + void handlePropChangeEvent(String propName, PropToggle currentValue); + +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/toggle/prop/PropToggle.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/toggle/prop/PropToggle.java new file mode 100644 index 0000000000..fd9b92806b --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/toggle/prop/PropToggle.java @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.util.toggle.prop; + +import com.tencent.bk.job.common.util.toggle.ToggleStrategy; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +@Data +public class PropToggle { + private String defaultValue; + + private List conditions; + + @Data + @NoArgsConstructor + public static class PropValueCondition { + private String value; + private ToggleStrategy strategy; + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/toggle/prop/PropToggleStore.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/toggle/prop/PropToggleStore.java new file mode 100644 index 0000000000..6b125f2d56 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/util/toggle/prop/PropToggleStore.java @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.util.toggle.prop; + +import com.tencent.bk.job.common.refreshable.config.RefreshableConfigStore; + +public interface PropToggleStore extends RefreshableConfigStore { + + /** + * 属性名称前缀 + */ + String PROP_KEY_PREFIX = "job.toggle.props."; + + /** + * 返回属性开关配置 + * + * @param propName 属性名称 + */ + PropToggle getPropToggle(String propName); + + void addPropChangeEventListener(String propName, PropChangeEventListener propChangeEventListener); + +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/CheckEnum.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/CheckEnum.java index 03cc29fe8e..4bcd3f2d31 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/CheckEnum.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/CheckEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -47,7 +47,7 @@ /** * spring validation枚举校验注解 */ -@Target({ FIELD, METHOD, PARAMETER, ANNOTATION_TYPE, TYPE_USE }) +@Target({FIELD, METHOD, PARAMETER, ANNOTATION_TYPE, TYPE_USE}) @Constraint(validatedBy = CheckEnum.Validator.class) @Documented @Retention(RUNTIME) @@ -61,7 +61,7 @@ Class> enumClass(); - String enumMethod(); + String enumMethod() default "isValid"; class Validator implements ConstraintValidator { diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/Create.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/Create.java index 8bf2454dae..9e46742b43 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/Create.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/Create.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/Delete.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/Delete.java new file mode 100644 index 0000000000..8d6bba799c --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/Delete.java @@ -0,0 +1,32 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.validation; + +/** + * 删除使用(配合spring的@Validated功能分组使用) + */ +public interface Delete { + +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/EndWith.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/EndWith.java new file mode 100644 index 0000000000..7bfdfe1857 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/EndWith.java @@ -0,0 +1,75 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.validation; + +import org.apache.commons.lang3.StringUtils; + +import javax.validation.Constraint; +import javax.validation.ConstraintValidator; +import javax.validation.ConstraintValidatorContext; +import javax.validation.Payload; +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import static java.lang.annotation.ElementType.ANNOTATION_TYPE; +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.ElementType.TYPE_USE; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@Target({FIELD, METHOD, PARAMETER, ANNOTATION_TYPE, TYPE_USE}) +@Retention(RUNTIME) +@Constraint(validatedBy = EndWith.Validator.class) +@Documented +public @interface EndWith { + String fieldName(); + + String value(); + + String message() default "{fieldName} {validation.constraints.MustEndWithSuffix.message}: {value}"; + + Class[] groups() default {}; + + Class[] payload() default {}; + + class Validator implements ConstraintValidator { + private String suffix; + + @Override + public void initialize(EndWith endWithAnnotation) { + this.suffix = endWithAnnotation.value(); + } + + @Override + public boolean isValid(String value, ConstraintValidatorContext constraintValidatorContext) { + if (StringUtils.isBlank(value)) { + return true; + } + return value.trim().endsWith(suffix); + } + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/EsbHostDTOGroupSequenceProvider.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/EsbHostDTOGroupSequenceProvider.java new file mode 100644 index 0000000000..e2dbc34acf --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/EsbHostDTOGroupSequenceProvider.java @@ -0,0 +1,56 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.validation; + +import com.tencent.bk.job.common.model.openapi.v4.OpenApiHostDTO; +import lombok.extern.slf4j.Slf4j; +import org.hibernate.validator.spi.group.DefaultGroupSequenceProvider; + +import java.util.ArrayList; +import java.util.List; + +/** + * 主机联合校验 + */ +@Slf4j +public class EsbHostDTOGroupSequenceProvider implements DefaultGroupSequenceProvider { + + @Override + public List> getValidationGroups(OpenApiHostDTO host) { + List> validationGroups = new ArrayList<>(); + validationGroups.add(OpenApiHostDTO.class); + if (host != null) { + if (host.getHostId() != null) { + validationGroups.add(OpenApiHostDTO.BkHostIdGroup.class); + } else if (host.getBkCloudId() != null || host.getIp() != null) { + validationGroups.add(OpenApiHostDTO.BkCloudIpGroup.class); + } else { + // 其它情况默认使用 bk_host_id 校验方式 + validationGroups.add(OpenApiHostDTO.BkHostIdGroup.class); + } + } + return validationGroups; + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/ExecuteTargetNotEmpty.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/ExecuteTargetNotEmpty.java new file mode 100644 index 0000000000..1d349a9d72 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/ExecuteTargetNotEmpty.java @@ -0,0 +1,48 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.validation; + +import javax.validation.Constraint; +import javax.validation.Payload; +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import static java.lang.annotation.ElementType.TYPE; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@Constraint(validatedBy = ExecuteTargetNotEmptyValidator.class) +@Target({TYPE}) +@Retention(RUNTIME) +@Documented +public @interface ExecuteTargetNotEmpty { + + String message() default "{validation.constraints.ExecuteTarget_empty.message}"; + + Class[] groups() default {}; + + Class[] payload() default {}; + +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/ExecuteTargetNotEmptyValidator.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/ExecuteTargetNotEmptyValidator.java new file mode 100644 index 0000000000..566f5c4c07 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/ExecuteTargetNotEmptyValidator.java @@ -0,0 +1,48 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.validation; + +import com.tencent.bk.job.common.model.openapi.v4.OpenApiExecuteTargetDTO; +import org.apache.commons.collections4.CollectionUtils; + +import javax.validation.ConstraintValidator; +import javax.validation.ConstraintValidatorContext; + +public class ExecuteTargetNotEmptyValidator + implements ConstraintValidator { + + @Override + public boolean isValid(OpenApiExecuteTargetDTO value, ConstraintValidatorContext context) { + return CollectionUtils.isNotEmpty(value.getHosts()) + || CollectionUtils.isNotEmpty(value.getHostDynamicGroups()) + || CollectionUtils.isNotEmpty(value.getHostTopoNodes()) + || CollectionUtils.isNotEmpty(value.getKubeContainerFilters()); + } + + @Override + public void initialize(ExecuteTargetNotEmpty constraintAnnotation) { + + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/FieldError.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/FieldError.java new file mode 100644 index 0000000000..3bcccbe849 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/FieldError.java @@ -0,0 +1,156 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.validation; + +import java.util.Collection; + +/** + * 字段验证错误 + */ +public class FieldError { + + /** + * 字段校验错误类型 + */ + private final ErrorType type; + /** + * 字段名 + */ + private final Path field; + /** + * 校验不通过的值 + */ + private final Object badValue; + /** + * 详细错误 + */ + private final String detail; + + private FieldError(ErrorType type, Path field, Object badValue, String detail) { + this.type = type; + this.field = field; + this.badValue = badValue; + this.detail = detail; + } + + enum ErrorType { + /** + * 必填字段没有值（比如空字符、null, 空的列表） + */ + Required("Required value"), + /** + * 字段值重复 + */ + Duplicate("Duplicate value"), + /** + * 不合法-通用 + */ + Invalid("Invalid value"), + /** + * 无法识别的枚举值 + */ + NotSupported("Unsupported value"), + /** + * 字段对应的字符超长 + */ + TooLong("Too long"), + /** + * 字段对应的列表包含太多的元素 + */ + TooMany("Too many"), + /** + * 表示值与该字段的 schema 定义不匹配 + */ + TypeInvalid("Invalid value"); + + private final String value; + + ErrorType(String value) { + this.value = value; + } + + @Override + public String toString() { + return value; + } + } + + public static FieldError invalid(Path field, Object value, String detail) { + return new FieldError(ErrorType.Invalid, field, value, detail); + } + + public static FieldError required(Path field, String detail) { + return new FieldError(ErrorType.Required, field, "", detail); + } + + public static FieldError duplicate(Path field, Object value) { + return new FieldError(ErrorType.Duplicate, field, value, ""); + } + + public static FieldError notSupported(Path field, Object value, Collection validValues) { + String detail = "supported values: " + String.join(", ", validValues); + return new FieldError(ErrorType.NotSupported, field, value, detail); + } + + public static FieldError tooLong(Path field, Object value, int maxLength) { + String detail = "must have at most " + maxLength + " chars"; + return new FieldError(ErrorType.TooLong, field, value, detail); + } + + public static FieldError tooMany(Path field, int actualSize, int maxSize) { + String detail = "must have at most " + maxSize + " items"; + return new FieldError(ErrorType.TooMany, field, actualSize, detail); + } + + public ErrorType getType() { + return type; + } + + public Path getField() { + return field; + } + + public Object getBadValue() { + return badValue; + } + + public String getDetail() { + return detail; + } + + @Override + public String toString() { + String s = field.toString(); + if (type == ErrorType.Required || type == ErrorType.TooLong) { + s += ": " + type.toString(); + } else { + s += ": " + type.toString() + ": " + (badValue == null ? "null" : badValue.toString()); + } + if (detail != null && !detail.isEmpty()) { + s += ": " + detail; + } + return s; + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/FieldErrors.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/FieldErrors.java new file mode 100644 index 0000000000..54021901c7 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/FieldErrors.java @@ -0,0 +1,56 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.validation; + +import org.apache.commons.collections4.CollectionUtils; + +import java.util.ArrayList; +import java.util.List; + +public class FieldErrors { + private final List errorList = new ArrayList<>(); + + public List getErrorList() { + return errorList; + } + + public FieldErrors add(FieldError fieldError) { + if (fieldError == null) { + return this; + } + this.errorList.add(fieldError); + return this; + } + + public boolean hasError() { + return CollectionUtils.isNotEmpty(errorList); + } + + @Override + public String toString() { + return "errors: " + errorList.toString(); + + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/KubeContainerFilterNotEmpty.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/KubeContainerFilterNotEmpty.java new file mode 100644 index 0000000000..3189c9f64f --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/KubeContainerFilterNotEmpty.java @@ -0,0 +1,48 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.validation; + +import javax.validation.Constraint; +import javax.validation.Payload; +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import static java.lang.annotation.ElementType.TYPE; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@Constraint(validatedBy = KubeContainerFilterNotEmptyValidator.class) +@Target({TYPE}) +@Retention(RUNTIME) +@Documented +public @interface KubeContainerFilterNotEmpty { + + String message() default "{validation.constraints.ExecuteTarget_KubeContainerFilterEmpty.message}"; + + Class[] groups() default {}; + + Class[] payload() default {}; + +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/KubeContainerFilterNotEmptyValidator.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/KubeContainerFilterNotEmptyValidator.java new file mode 100644 index 0000000000..bfefc32d58 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/KubeContainerFilterNotEmptyValidator.java @@ -0,0 +1,49 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.validation; + +import com.tencent.bk.job.common.model.openapi.v4.OpenApiKubeContainerFilterDTO; + +import javax.validation.ConstraintValidator; +import javax.validation.ConstraintValidatorContext; + +public class KubeContainerFilterNotEmptyValidator + implements ConstraintValidator { + + @Override + public boolean isValid(OpenApiKubeContainerFilterDTO value, ConstraintValidatorContext context) { + return value.isEmptyFilter() || + (value.getClusterFilter() != null + || value.getNamespaceFilter() != null + || value.getWorkloadFilter() != null + || value.getPodFilter() != null + || value.getContainerPropFilter() != null); + } + + @Override + public void initialize(KubeContainerFilterNotEmpty constraintAnnotation) { + + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/MaxLength.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/MaxLength.java new file mode 100644 index 0000000000..fee4153fab --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/MaxLength.java @@ -0,0 +1,52 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.validation; + +import javax.validation.Constraint; +import javax.validation.Payload; +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import static java.lang.annotation.ElementType.ANNOTATION_TYPE; +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.ElementType.TYPE_USE; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@Target({FIELD, METHOD, PARAMETER, ANNOTATION_TYPE, TYPE_USE}) +@Retention(RUNTIME) +@Constraint(validatedBy = MaxLengthValidator.class) +@Documented +public @interface MaxLength { + long value(); + + String message() default "{validation.constraints.ExceedMaxLength.message}"; + + Class[] groups() default {}; + + Class[] payload() default {}; +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/MaxLengthValidator.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/MaxLengthValidator.java new file mode 100644 index 0000000000..77f88b3427 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/MaxLengthValidator.java @@ -0,0 +1,46 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.validation; + +import javax.validation.ConstraintValidator; +import javax.validation.ConstraintValidatorContext; + +public class MaxLengthValidator implements ConstraintValidator { + private Long maxLength; + + @Override + public void initialize(MaxLength maxLengthAnnotation) { + this.maxLength = maxLengthAnnotation.value(); + } + + @Override + public boolean isValid(String content, + ConstraintValidatorContext constraintValidatorContext) { + if (content == null) { + return true; + } + return content.length() <= maxLength; + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/NotBlankField.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/NotBlankField.java index cd7b5ef5ae..09663a9810 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/NotBlankField.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/NotBlankField.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/NotBlankFieldValidator.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/NotBlankFieldValidator.java index 2585617884..581bb1a0a0 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/NotBlankFieldValidator.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/NotBlankFieldValidator.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/NotContainSpecialChar.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/NotContainSpecialChar.java index 0dcc532f1a..976b180cb4 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/NotContainSpecialChar.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/NotContainSpecialChar.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/NotExceedMySQLTextFieldLength.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/NotExceedMySQLTextFieldLength.java new file mode 100644 index 0000000000..93ba137c3d --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/NotExceedMySQLTextFieldLength.java @@ -0,0 +1,103 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.validation; + + +import com.tencent.bk.job.common.constant.MySQLTextDataType; +import com.tencent.bk.job.common.util.Base64Util; +import lombok.extern.slf4j.Slf4j; + +import javax.validation.Constraint; +import javax.validation.ConstraintValidator; +import javax.validation.ConstraintValidatorContext; +import javax.validation.Payload; +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.Target; +import java.nio.charset.StandardCharsets; + +import static java.lang.annotation.ElementType.ANNOTATION_TYPE; +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.ElementType.TYPE_USE; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +/** + * 通过在MySQL定义的TEXT一族字段类型（TINYTEXT/TEXT/MEDIUMTEXT/LONGTEXT）判断是否过长 + * 因为在MySQL中，TEXT类型的存储是以字节流的长度为限制的，而char/varchar则是以字符串的长度为限制的，所以校验长度合法的逻辑是不太一样的 + * 因此这个校验器只校验TEXT类型的字节流长度，若需要校验char/varchar类型的字段长度，请用str.length()校验 + * + */ +@Target({FIELD, METHOD, PARAMETER, ANNOTATION_TYPE, TYPE_USE}) +@Constraint(validatedBy = NotExceedMySQLTextFieldLength.FieldLengthValidator.class) +@Documented +@Retention(RUNTIME) +public @interface NotExceedMySQLTextFieldLength { + + String message() default "{fieldName} {validation.constraints.NotExceedMySQLFieldLength.message} limit:{fieldType}"; + + Class[] groups() default {}; + + Class[] payload() default {}; + + String fieldName(); + + MySQLTextDataType fieldType(); + + boolean base64(); + + @Slf4j + class FieldLengthValidator implements ConstraintValidator { + + MySQLTextDataType fieldType = null; + boolean useBase64 = false; + + @Override + public void initialize(NotExceedMySQLTextFieldLength constraintAnnotation) { + fieldType = constraintAnnotation.fieldType(); + useBase64 = constraintAnnotation.base64(); + } + + @Override + public boolean isValid(String value, ConstraintValidatorContext constraintValidatorContext) { + if (value == null || fieldType == null) { + return true; + } + + int currentLength; + if (useBase64) { + currentLength = Base64Util.calcOriginBytesLength(value); + } else { + currentLength = value.getBytes(StandardCharsets.UTF_8).length; + } + + log.debug("[Validate MySQLFieldLength] field type: {}, current length: {}, maximum length: {}]", + fieldType.getValue(), currentLength, fieldType.getMaximumLength()); + + return currentLength <= fieldType.getMaximumLength(); + } + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/Path.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/Path.java new file mode 100644 index 0000000000..a7d9d70f6a --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/Path.java @@ -0,0 +1,141 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.validation; + +import java.util.ArrayList; +import java.util.List; + +/** + * 表示对象图中从一个对象到另一个对象的导航路径, 比如 root.first.second[0].third[key] + */ +public class Path { + /** + * 字段名；当 PATH 为位置索引的时候，值为"" + */ + private final String name; + /** + * 如果 name == ""，这是前一个元素的下标（索引或映射键） + */ + private final String index; + + /** + * 根路径 + */ + private Path parent; + + private Path(String name, String index, Path parent) { + this.name = name; + this.index = index; + this.parent = parent; + } + + /** + * 创建一个根对象导航路径 + * + * @param name 根路径名 + * @param moreNames 子路径名 + */ + public static Path newPath(String name, String... moreNames) { + Path r = new Path(name, null, null); + for (String moreName : moreNames) { + r = new Path(moreName, null, r); + } + return r; + } + + /** + * 返回根路径 + */ + public Path root() { + Path p = this; + while (p.parent != null) { + p = p.parent; + } + return p; + } + + /** + * 返回父路径 + */ + public Path parent() { + return this.parent; + } + + /** + * 在当前路径下创建子路径 + * + * @param name 路径名 + * @param moreNames 子路径名列表 + */ + public Path child(String name, String... moreNames) { + Path r = newPath(name, moreNames); + r.root().parent = this; + return r; + } + + /** + * 在当前路径下创建索引路径(array) + * + * @param index 索引 + */ + public Path index(int index) { + return new Path("", String.valueOf(index), this); + } + + /** + * 在当前路径下创建映射键路径(map) + * + * @param key 映射键 + */ + public Path key(String key) { + return new Path("", key, this); + } + + @Override + public String toString() { + + List elements = new ArrayList<>(); + Path p = this; + while (p != null) { + elements.add(p); + p = p.parent; + } + + StringBuilder buf = new StringBuilder(); + for (int i = elements.size() - 1; i >= 0; i--) { + p = elements.get(i); + if (p.parent != null && p.name.length() > 0) { + buf.append("."); + } + if (p.name.length() > 0) { + buf.append(p.name); + } else { + buf.append("[").append(p.index).append("]"); + } + } + return buf.toString(); + } +} + diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/ScopeValidator.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/ScopeValidator.java new file mode 100644 index 0000000000..4d98fa7227 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/ScopeValidator.java @@ -0,0 +1,58 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.validation; + +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.constant.ResourceScopeTypeEnum; +import com.tencent.bk.job.common.exception.InvalidParamException; +import org.apache.commons.lang3.StringUtils; + +/** + * 资源范围校验工具类 + */ +public class ScopeValidator { + + /** + * 校验资源范围 + * + * @param bizId 兼容字段：业务ID + * @param scopeType 资源范围类型 + * @param scopeId 资源范围ID + */ + public static void validate(Long bizId, String scopeType, String scopeId) { + if (bizId != null) { + return; + } + if (StringUtils.isBlank(scopeType)) { + throw new InvalidParamException(ErrorCode.MISSING_PARAM_WITH_PARAM_NAME, "bk_scope_type"); + } + if (!ResourceScopeTypeEnum.isValid(scopeType)) { + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME, "bk_scope_type"); + } + if (StringUtils.isBlank(scopeId)) { + throw new InvalidParamException(ErrorCode.MISSING_PARAM_WITH_PARAM_NAME, "bk_scope_id"); + } + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/Update.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/Update.java index 9a54233a4e..4eddee35cd 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/Update.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/Update.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/ValidPureFileName.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/ValidPureFileName.java new file mode 100644 index 0000000000..d7114d79f5 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/ValidPureFileName.java @@ -0,0 +1,68 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.validation; + +import javax.validation.Constraint; +import javax.validation.ConstraintValidator; +import javax.validation.ConstraintValidatorContext; +import javax.validation.Payload; +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import static java.lang.annotation.ElementType.ANNOTATION_TYPE; +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.ElementType.TYPE_USE; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +/** + * spring validation java 从用户参数传入的文件名称合法性校验 + */ +@Target({FIELD, METHOD, PARAMETER, ANNOTATION_TYPE, TYPE_USE}) +@Constraint(validatedBy = ValidPureFileName.Validator.class) +@Documented +@Retention(RUNTIME) +public @interface ValidPureFileName { + + String message() default "{validation.constraints.FileNameInvalid.message}"; + + Class[] groups() default {}; + + Class[] payload() default {}; + + class Validator implements ConstraintValidator { + + @Override + public boolean isValid(String value, ConstraintValidatorContext constraintValidatorContext) { + if (value == null || value.isEmpty()) { + return true; + } + return !(value.contains("/") || value.contains("\\")); + } + + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/ValidRegexPattern.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/ValidRegexPattern.java index 04fd735e7d..40b4dda476 100644 --- a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/ValidRegexPattern.java +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/ValidRegexPattern.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/ValidationGroups.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/ValidationGroups.java new file mode 100644 index 0000000000..bd7e3b68e0 --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/validation/ValidationGroups.java @@ -0,0 +1,49 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.validation; + +/** + * 联合校验分组 + */ +public interface ValidationGroups { + interface Script { + interface ScriptVersionId { + } + + interface ScriptContent { + } + + interface ScriptId { + } + } + + interface Account { + interface AccountId { + } + + interface AccountAlias { + } + } +} diff --git a/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/watch/SafeWatch.java b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/watch/SafeWatch.java new file mode 100644 index 0000000000..57886c795b --- /dev/null +++ b/src/backend/commons/common/src/main/java/com/tencent/bk/job/common/watch/SafeWatch.java @@ -0,0 +1,53 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.watch; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.util.StopWatch; + +import javax.annotation.Nonnull; + +/** + * 基于Spring StopWatch封装的安全计时器，防止因watch状态不正确时使用而抛出异常中断外层逻辑 + */ +@Slf4j +public class SafeWatch extends StopWatch { + @Override + public void start(@Nonnull String taskName) { + if (isRunning()) { + super.stop(); + } + super.start(taskName); + } + + @Override + public void stop() { + if (isRunning()) { + super.stop(); + } else { + log.info("watch is not running, do not need stop"); + } + } +} diff --git a/src/backend/commons/common/src/main/resources/META-INF/spring.factories b/src/backend/commons/common/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..09450a20ce --- /dev/null +++ b/src/backend/commons/common/src/main/resources/META-INF/spring.factories @@ -0,0 +1,2 @@ +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ +com.tencent.bk.job.common.service.config.JobCommonAutoConfiguration diff --git a/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/service/VariableResolverTest.java b/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/service/VariableResolverTest.java index 0cf66afbc7..a50daf27c5 100644 --- a/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/service/VariableResolverTest.java +++ b/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/service/VariableResolverTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/CompareUtilTest.java b/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/CompareUtilTest.java index 5bb61d7ea7..9c9f283215 100644 --- a/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/CompareUtilTest.java +++ b/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/CompareUtilTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -61,7 +61,7 @@ void testCompareVersion() { assertThat(CompareUtil.compareVersion("3.3.3.beta.10", "3.3.3.rc")).isEqualTo(-1); assertThat(CompareUtil.compareVersion("3.3.3.rc", "3.3.3.0")).isEqualTo(-1); assertThat(CompareUtil.compareVersion("3.3.3.rc.2", "3.3.3.0")).isEqualTo(-1); - assertThat(CompareUtil.compareVersion("3.3.3", "3.3.3.alpha")).isEqualTo(-1); + assertThat(CompareUtil.compareVersion("3.3.3", "3.3.3.alpha")).isEqualTo(1); assertThat(CompareUtil.compareVersion("3.3.3.10", "3.3.3.2")).isEqualTo(1); assertThat(CompareUtil.compareVersion("3.3.3.alpha.1", "3.3.3.alpha")).isEqualTo(1); @@ -84,5 +84,11 @@ void testCompareVersion() { assertThat(CompareUtil.compareVersion("3.3.3-alpha.1", "3.3.3-alpha.1")).isEqualTo(0); assertThat(CompareUtil.compareVersion("3.3.3.alpha", "3.3.3.alpha")).isEqualTo(0); assertThat(CompareUtil.compareVersion("3.3.3.alpha.1", "3.3.3.alpha.1")).isEqualTo(0); + + assertThat(CompareUtil.compareVersion("3.3.3", "3.3.3-alpha.1")).isEqualTo(1); + assertThat(CompareUtil.compareVersion("3.3.3-beta.1", "3.3.3-alpha.1")).isEqualTo(1); + assertThat(CompareUtil.compareVersion("3.3.3-rc.1", "3.3.3-alpha.1")).isEqualTo(1); + assertThat(CompareUtil.compareVersion("3.3.3", "3.3.3-rc.1")).isEqualTo(1); + assertThat(CompareUtil.compareVersion("v3.3.3", "3.3.3")).isEqualTo(0); } } diff --git a/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/FilterUtilsTest.java b/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/FilterUtilsTest.java index c16960257b..64f4582713 100644 --- a/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/FilterUtilsTest.java +++ b/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/FilterUtilsTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/IpUtilsTest.java b/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/IpUtilsTest.java index 5cc16ac252..309673409b 100644 --- a/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/IpUtilsTest.java +++ b/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/IpUtilsTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,9 +25,15 @@ package com.tencent.bk.job.common.util; import com.tencent.bk.job.common.util.ip.IpUtils; +import org.apache.commons.lang3.tuple.Pair; import org.junit.jupiter.api.Test; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; + import static org.assertj.core.api.AssertionsForClassTypes.assertThat; +import static org.assertj.core.api.AssertionsForClassTypes.assertThatThrownBy; public class IpUtilsTest { @Test @@ -47,10 +53,184 @@ void testIp2Long() { @Test void testLongStr2Ip() { String ipLongStr = "2130706433"; - String ip = IpUtils.revertIpFromLongStr(ipLongStr); + String ip = IpUtils.revertIpFromNumericalStr(ipLongStr); assertThat(ip).isEqualTo("127.0.0.1"); } + @Test + void testCheckIpv6() { + assertThat(IpUtils.checkIpv6("::1")).isTrue(); + assertThat(IpUtils.checkIpv6("0::1")).isTrue(); + assertThat(IpUtils.checkIpv6("1::1")).isTrue(); + assertThat(IpUtils.checkIpv6("1:1::1")).isTrue(); + assertThat(IpUtils.checkIpv6("0:0:0:0:0:0:0:0")).isTrue(); + assertThat(IpUtils.checkIpv6("00:00::00:00:00")).isTrue(); + assertThat(IpUtils.checkIpv6("00:00:00:00:00:00:00:00")).isTrue(); + assertThat(IpUtils.checkIpv6("01:23:45:67:89:ab:cd:ef")).isTrue(); + assertThat(IpUtils.checkIpv6("001:023:45:67:89:ab:cd:ef")).isTrue(); + assertThat(IpUtils.checkIpv6("1101:23:45:67:89:ab:cd:ef")).isTrue(); + assertThat(IpUtils.checkIpv6("::192.168.0.1")).isTrue(); + assertThat(IpUtils.checkIpv6("ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff")).isTrue(); + assertThat(IpUtils.checkIpv6("0ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff")).isFalse(); + assertThat(IpUtils.checkIpv6("::192.168.0.1/96")).isFalse(); + assertThat(IpUtils.checkIpv6("ff:ff:ff:ff:ff:ff:ff:fg")).isFalse(); + assertThat(IpUtils.checkIpv6("1::1::1")).isFalse(); + assertThat(IpUtils.checkIpv6("1::1::123")).isFalse(); + assertThat(IpUtils.checkIpv6("127.0.0.1")).isFalse(); + } + + @Test + void testExtractIp() { + String cloudIp = "0:127.0.0.1"; + String result = IpUtils.extractIp(cloudIp); + assertThat(result).isEqualTo("127.0.0.1"); + + cloudIp = "0:0000:0000:0000:0000:0000:0000:0000:0001"; + result = IpUtils.extractIp(cloudIp); + assertThat(result).isEqualTo("0000:0000:0000:0000:0000:0000:0000:0001"); + + cloudIp = "127.0.0.1"; + result = IpUtils.extractIp(cloudIp); + assertThat(result).isEqualTo("127.0.0.1"); + + result = IpUtils.extractIp(null); + assertThat(result).isNull(); + } + + @Test + void testParseCleanIpAndCloudIps() { + Pair, Set> emptyPair = IpUtils.parseCleanIpv4AndCloudIpv4s(null); + assertThat(emptyPair.getLeft().size()).isEqualTo(0); + assertThat(emptyPair.getRight().size()).isEqualTo(0); + + emptyPair = IpUtils.parseCleanIpv4AndCloudIpv4s(new ArrayList<>()); + assertThat(emptyPair.getLeft().size()).isEqualTo(0); + assertThat(emptyPair.getRight().size()).isEqualTo(0); + + List ipv4OrCloudIpv4List = new ArrayList<>(); + ipv4OrCloudIpv4List.add("127.0.0.1"); + ipv4OrCloudIpv4List.add(" 127.0.0.1"); + ipv4OrCloudIpv4List.add("0:127 .0.0.1"); + ipv4OrCloudIpv4List.add("0:[127 .0.0.2]"); + ipv4OrCloudIpv4List.add("127.0.0.2"); + ipv4OrCloudIpv4List.add(" 127.0.0.2 \n"); + ipv4OrCloudIpv4List.add("1:[127.0.0.3] \r\n"); + Pair, Set> pair = IpUtils.parseCleanIpv4AndCloudIpv4s(ipv4OrCloudIpv4List); + Set ipv4Set = pair.getLeft(); + Set cloudIpv4Set = pair.getRight(); + assertThat(ipv4Set.size()).isEqualTo(2); + assertThat(ipv4Set.contains("127.0.0.1")).isTrue(); + assertThat(ipv4Set.contains("127.0.0.2")).isTrue(); + assertThat(cloudIpv4Set.size()).isEqualTo(3); + assertThat(cloudIpv4Set.contains("0:127.0.0.1")).isTrue(); + assertThat(cloudIpv4Set.contains("0:127.0.0.2")).isTrue(); + assertThat(cloudIpv4Set.contains("1:127.0.0.3")).isTrue(); + } + + @Test + void testParseCleanIpv6AndCloudIpv6s() { + Pair, Set>> emptyPair = IpUtils.parseFullIpv6AndCloudIpv6s(null); + assertThat(emptyPair.getLeft().size()).isEqualTo(0); + assertThat(emptyPair.getRight().size()).isEqualTo(0); + + emptyPair = IpUtils.parseFullIpv6AndCloudIpv6s(new ArrayList<>()); + assertThat(emptyPair.getLeft().size()).isEqualTo(0); + assertThat(emptyPair.getRight().size()).isEqualTo(0); + + List ipv6OrCloudIpv6List = new ArrayList<>(); + ipv6OrCloudIpv6List.add("FFFF::1"); + ipv6OrCloudIpv6List.add(" FFFF:ABCD::1"); + ipv6OrCloudIpv6List.add("0:[FFFF::1]"); + ipv6OrCloudIpv6List.add("0:[FFFF:1234::2]"); + ipv6OrCloudIpv6List.add("[FFFF:1234::2]"); + ipv6OrCloudIpv6List.add(" FFFF:1234::2 \n"); + ipv6OrCloudIpv6List.add("1:[[FFFF:1234::2]] \r\n"); + Pair, Set>> pair = IpUtils.parseFullIpv6AndCloudIpv6s(ipv6OrCloudIpv6List); + Set ipv6Set = pair.getLeft(); + Set> cloudIpv6Set = pair.getRight(); + assertThat(ipv6Set.size()).isEqualTo(3); + assertThat(ipv6Set.contains("FFFF:0000:0000:0000:0000:0000:0000:0001")).isTrue(); + assertThat(ipv6Set.contains("FFFF:ABCD:0000:0000:0000:0000:0000:0001")).isTrue(); + assertThat(ipv6Set.contains("FFFF:1234:0000:0000:0000:0000:0000:0002")).isTrue(); + assertThat(cloudIpv6Set.size()).isEqualTo(3); + assertThat(cloudIpv6Set.contains(Pair.of(0L, "FFFF:0000:0000:0000:0000:0000:0000:0001"))).isTrue(); + assertThat(cloudIpv6Set.contains(Pair.of(0L, "FFFF:1234:0000:0000:0000:0000:0000:0002"))).isTrue(); + assertThat(cloudIpv6Set.contains(Pair.of(1L, "FFFF:1234:0000:0000:0000:0000:0000:0002"))).isTrue(); + } + + @Test + void testInferProtocolByIp() { + // Ipv6正例 + assertThat(IpUtils.inferProtocolByIp("::1")).isEqualTo(IpUtils.PROTOCOL_IP_V6); + assertThat(IpUtils.inferProtocolByIp("0::1")).isEqualTo(IpUtils.PROTOCOL_IP_V6); + assertThat(IpUtils.inferProtocolByIp("1::1")).isEqualTo(IpUtils.PROTOCOL_IP_V6); + assertThat(IpUtils.inferProtocolByIp("1:1::1")).isEqualTo(IpUtils.PROTOCOL_IP_V6); + assertThat(IpUtils.inferProtocolByIp("0:0:0:0:0:0:0:0")).isEqualTo(IpUtils.PROTOCOL_IP_V6); + assertThat(IpUtils.inferProtocolByIp("00:00::00:00:00")).isEqualTo(IpUtils.PROTOCOL_IP_V6); + assertThat(IpUtils.inferProtocolByIp("00:00:00:00:00:00:00:00")).isEqualTo(IpUtils.PROTOCOL_IP_V6); + assertThat(IpUtils.inferProtocolByIp("01:23:45:67:89:ab:cd:ef")).isEqualTo(IpUtils.PROTOCOL_IP_V6); + assertThat(IpUtils.inferProtocolByIp("001:023:45:67:89:ab:cd:ef")).isEqualTo(IpUtils.PROTOCOL_IP_V6); + assertThat(IpUtils.inferProtocolByIp("1101:23:45:67:89:ab:cd:ef")).isEqualTo(IpUtils.PROTOCOL_IP_V6); + assertThat(IpUtils.inferProtocolByIp("ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff")).isEqualTo(IpUtils.PROTOCOL_IP_V6); + // Ipv6负例 + assertThat(IpUtils.inferProtocolByIp("0ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff")).isNotEqualTo(IpUtils.PROTOCOL_IP_V6); + assertThat(IpUtils.inferProtocolByIp("ff:ff:ff:ff:ff:ff:ff:fg")).isNotEqualTo(IpUtils.PROTOCOL_IP_V6); + assertThat(IpUtils.inferProtocolByIp("1::1::1")).isNotEqualTo(IpUtils.PROTOCOL_IP_V6); + assertThat(IpUtils.inferProtocolByIp("1::1::123")).isNotEqualTo(IpUtils.PROTOCOL_IP_V6); + assertThat(IpUtils.inferProtocolByIp("127.0.0.1")).isNotEqualTo(IpUtils.PROTOCOL_IP_V6); + // Ipv4正例 + assertThat(IpUtils.inferProtocolByIp("127.0.0.1")).isEqualTo(IpUtils.PROTOCOL_IP_V4); + assertThat(IpUtils.inferProtocolByIp("192.168.1.1")).isEqualTo(IpUtils.PROTOCOL_IP_V4); + // Ipv4负例 + assertThat(IpUtils.inferProtocolByIp("::1")).isNotEqualTo(IpUtils.PROTOCOL_IP_V4); + assertThat(IpUtils.inferProtocolByIp("01:23:45:67:89:ab:cd:ef")).isNotEqualTo(IpUtils.PROTOCOL_IP_V4); + } + + @Test + void testGetFullIpv6ByCompressedOne() { + assertThatThrownBy(() -> IpUtils.getFullIpv6ByCompressedOne(null)); + assertThatThrownBy(() -> IpUtils.getFullIpv6ByCompressedOne("")); + assertThat(IpUtils.getFullIpv6ByCompressedOne("::")) + .isEqualTo("0000:0000:0000:0000:0000:0000:0000:0000"); + assertThat(IpUtils.getFullIpv6ByCompressedOne("::1")) + .isEqualTo("0000:0000:0000:0000:0000:0000:0000:0001"); + assertThat(IpUtils.getFullIpv6ByCompressedOne("0::1")) + .isEqualTo("0000:0000:0000:0000:0000:0000:0000:0001"); + assertThat(IpUtils.getFullIpv6ByCompressedOne("00::1")) + .isEqualTo("0000:0000:0000:0000:0000:0000:0000:0001"); + assertThat(IpUtils.getFullIpv6ByCompressedOne("0000::1")) + .isEqualTo("0000:0000:0000:0000:0000:0000:0000:0001"); + assertThatThrownBy(() -> IpUtils.getFullIpv6ByCompressedOne("00000::1")); + assertThatThrownBy(() -> IpUtils.getFullIpv6ByCompressedOne("10000::1")); + assertThat(IpUtils.getFullIpv6ByCompressedOne("1:0000::1")) + .isEqualTo("0001:0000:0000:0000:0000:0000:0000:0001"); + assertThat(IpUtils.getFullIpv6ByCompressedOne("1:1:1:1:1:1:1:1")) + .isEqualTo("0001:0001:0001:0001:0001:0001:0001:0001"); + assertThat(IpUtils.getFullIpv6ByCompressedOne("1:1:1:0:0:1:1:1")) + .isEqualTo("0001:0001:0001:0000:0000:0001:0001:0001"); + assertThat(IpUtils.getFullIpv6ByCompressedOne("ffff:1:1:0:0:1:1:ffff")) + .isEqualTo("ffff:0001:0001:0000:0000:0001:0001:ffff"); + assertThat(IpUtils.getFullIpv6ByCompressedOne("FFFF:1:1:0:0:1:1:FFFF")) + .isEqualTo("FFFF:0001:0001:0000:0000:0001:0001:FFFF"); + assertThatThrownBy(() -> IpUtils.getFullIpv6ByCompressedOne("aaffff:1:1:0:0:1:1:bbffff")); + assertThat(IpUtils.getFullIpv6ByCompressedOne("ffff:0001:0001:0000:0000:0001:0001:ffff")) + .isEqualTo("ffff:0001:0001:0000:0000:0001:0001:ffff"); + // 兼容IPv4地址的用例 + assertThat(IpUtils.getFullIpv6ByCompressedOne("::FFFF:192.168.0.1")) + .isEqualTo("0000:0000:0000:0000:0000:0000:FFFF:192.168.0.1"); + } + + @Test + void testExtractBkCloudId() { + String cloudIp = "0:127.0.0.1"; + Long bkCloudId = IpUtils.extractBkCloudId(cloudIp); + assertThat(bkCloudId).isEqualTo(0L); + + String cloudIpv6 = "0:0000:0000:0000:0000:0000:0000:0000:0000"; + bkCloudId = IpUtils.extractBkCloudId(cloudIpv6); + assertThat(bkCloudId).isEqualTo(0L); + } + @Test void testGetFirstIpFromMultiIp() { assertThat(IpUtils.getFirstIpFromMultiIp(null, ",")).isNull(); diff --git a/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/JobUUIDTest.java b/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/JobUUIDTest.java index d59d5f48ca..50d6994d8f 100644 --- a/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/JobUUIDTest.java +++ b/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/JobUUIDTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/LogUtilTest.java b/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/LogUtilTest.java new file mode 100644 index 0000000000..339cf72dbb --- /dev/null +++ b/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/LogUtilTest.java @@ -0,0 +1,71 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.util; + +import com.tencent.bk.job.common.util.ip.IpUtils; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; + +public class LogUtilTest { + @Test + void testBuildListLog() { + assertThat(LogUtil.buildListLog(null, 1)).isEqualTo(""); + List list = new ArrayList<>(); + assertThat(LogUtil.buildListLog(list, 1)).isEqualTo("[]"); + list.add("1"); + assertThat(LogUtil.buildListLog(list, 0)).isEqualTo("[]"); + assertThat(LogUtil.buildListLog(list, -1)).isEqualTo("[1]"); + assertThat(LogUtil.buildListLog(list, 1)).isEqualTo("[1]"); + assertThat(LogUtil.buildListLog(list, 2)).isEqualTo("[1]"); + list.add("2"); + list.add("3"); + assertThat(LogUtil.buildListLog(list, 2)).isEqualTo("(3 elements)[1,2,...]"); + } + + @Test + void testLong2Ip() { + long ipLong = 2130706433L; + String ip = IpUtils.revertIpFromLong(ipLong); + assertThat(ip).isEqualTo("127.0.0.1"); + } + + @Test + void testIp2Long() { + String ip = "127.0.0.1"; + long ipLong = IpUtils.getStringIpToLong(ip); + assertThat(ipLong).isEqualTo(2130706433L); + } + + @Test + void testLongStr2Ip() { + String ipLongStr = "2130706433"; + String ip = IpUtils.revertIpFromNumericalStr(ipLongStr); + assertThat(ip).isEqualTo("127.0.0.1"); + } +} diff --git a/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/StringUtilTest.java b/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/StringUtilTest.java index a0aceb306f..65b0444761 100644 --- a/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/StringUtilTest.java +++ b/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/StringUtilTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,14 +24,20 @@ package com.tencent.bk.job.common.util; +import org.apache.commons.lang3.tuple.Pair; import org.junit.jupiter.api.Test; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; +import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; public class StringUtilTest { @@ -67,18 +73,73 @@ public void removePrefixOrSuffix() { @Test public void testIsDifferent() { - assertEquals(StringUtil.isDifferent(null, null), false); - assertEquals(StringUtil.isDifferent("123", "123"), false); - assertEquals(StringUtil.isDifferent(null, "123"), true); - assertEquals(StringUtil.isDifferent("123", null), true); - assertEquals(StringUtil.isDifferent("123", "123 "), true); + assertFalse(StringUtil.isDifferent(null, null)); + assertFalse(StringUtil.isDifferent("123", "123")); + assertTrue(StringUtil.isDifferent(null, "123")); + assertTrue(StringUtil.isDifferent("123", null)); + assertTrue(StringUtil.isDifferent("123", "123 ")); } @Test public void testEscape() { - assertEquals(StringUtil.escape("a_b", new char[] {'_', '%', '\\'}, new String[] {"\\_", "\\%", "\\\\"}), "a\\_b"); - assertEquals(StringUtil.escape("a%b", new char[] {'_', '%', '\\'}, new String[] {"\\_", "\\%", "\\\\"}), "a\\%b"); - assertEquals(StringUtil.escape("a\\b", new char[] {'_', '%', '\\'}, new String[] {"\\_", "\\%", "\\\\"}), "a\\\\b"); - assertEquals(StringUtil.escape("a_b%c\\d", new char[] {'_', '%', '\\'}, new String[] {"\\_", "\\%", "\\\\"}), "a\\_b\\%c\\\\d"); + assertEquals(StringUtil.escape("a_b", new char[]{'_', '%', '\\'}, new String[]{"\\_", "\\%", "\\\\"}), "a\\_b"); + assertEquals(StringUtil.escape("a%b", new char[]{'_', '%', '\\'}, new String[]{"\\_", "\\%", "\\\\"}), "a\\%b"); + assertEquals(StringUtil.escape("a\\b", new char[]{'_', '%', '\\'}, new String[]{"\\_", "\\%", "\\\\"}), "a" + + "\\\\b"); + assertEquals(StringUtil.escape("a_b%c\\d", new char[]{'_', '%', '\\'}, new String[]{"\\_", "\\%", "\\\\"}), + "a\\_b\\%c\\\\d"); + } + + @Test + void substring() { + assertEquals(StringUtil.substring("abcde", 3), "abc"); + assertEquals(StringUtil.substring("abcde", 5), "abcde"); + assertEquals(StringUtil.substring("abcde", 6), "abcde"); + assertEquals(StringUtil.substring("abcde", 0), ""); + assertEquals(StringUtil.substring("abcde", -1), ""); + assertNull(StringUtil.substring(null, 3)); + + } + + @Test + void concatArray() { + } + + @Test + void extractValueFromStrings() { + Set rawStrs = new HashSet<>(12); + rawStrs.add(null); + rawStrs.add(""); + rawStrs.add("1"); + rawStrs.add("not value"); + rawStrs.add(" 2"); + rawStrs.add("3 "); + rawStrs.add("not value2"); + rawStrs.add("4 xx"); + rawStrs.add("true"); + rawStrs.add("false"); + rawStrs.add(" True "); + rawStrs.add(" FALSE "); + Pair, List> longValuePair = StringUtil.extractValueFromStrings(rawStrs, Long.class); + System.out.println("valueList=" + longValuePair.getLeft() + ", remainStrList=" + longValuePair.getRight()); + assertEquals(3, longValuePair.getLeft().size()); + assertThat(longValuePair.getLeft()).contains(1L, 2L, 3L); + assertEquals(9, longValuePair.getRight().size()); + assertThat(longValuePair.getRight()) + .contains(null, "", "not value", "not value2", "4 xx", "true", "false", " True ", " FALSE "); + Pair, List> intValuePair = StringUtil.extractValueFromStrings(rawStrs, Integer.class); + assertThat(intValuePair.getLeft()).contains(1, 2, 3); + Pair, List> floatValuePair = StringUtil.extractValueFromStrings(rawStrs, Float.class); + assertThat(floatValuePair.getLeft()).contains(1.0f, 2.0f, 3.0f); + Pair, List> doubleValuePair = StringUtil.extractValueFromStrings(rawStrs, Double.class); + assertThat(doubleValuePair.getLeft()).contains(1.0, 2.0, 3.0); + Pair, List> booleanValuePair = StringUtil.extractValueFromStrings(rawStrs, Boolean.class); + System.out.println( + "valueList=" + booleanValuePair.getLeft() + ", remainStrList=" + booleanValuePair.getRight() + ); + assertEquals(8, booleanValuePair.getRight().size()); + assertThat(booleanValuePair.getLeft()).contains(true, false); + assertThat(booleanValuePair.getRight()) + .contains(null, "", "1", "not value", " 2", "3 ", "not value2", "4 xx"); } } diff --git a/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/TagUtilsTest.java b/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/TagUtilsTest.java index 237d4c6514..4ba4214593 100644 --- a/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/TagUtilsTest.java +++ b/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/TagUtilsTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/date/DateUtilTest.java b/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/date/DateUtilTest.java index 41baf55fcb..ceafdd6cac 100644 --- a/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/date/DateUtilTest.java +++ b/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/date/DateUtilTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/http/BaseHttpHelperTest.java b/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/http/BaseHttpHelperTest.java new file mode 100644 index 0000000000..f469ada91a --- /dev/null +++ b/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/http/BaseHttpHelperTest.java @@ -0,0 +1,67 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.util.http; + +import com.tencent.bk.job.common.constant.HttpMethodEnum; +import com.tencent.bk.job.common.exception.InternalException; +import org.apache.http.NoHttpResponseException; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.protocol.HttpContext; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class BaseHttpHelperTest { + @Test + @DisplayName("测试 BaseHttpHelper 异常重试") + void whenGetThrowSocketTimeoutExceptionThenRetry() { + JobHttpRequestRetryHandler mockedRetryHandler = Mockito.spy(new JobHttpRequestRetryHandler()); + + CloseableHttpClient retryableHttpClient = JobHttpClientFactory.createHttpClient( + 15000, + 15000, + 15000, + 1, + 2, + 60, + true, + mockedRetryHandler, + (httpClientBuilder -> { + httpClientBuilder.addInterceptorFirst((org.apache.http.HttpRequest request, HttpContext context) -> { + throw new NoHttpResponseException(""); + }); + })); + HttpHelper httpHelper = new BaseHttpHelper(retryableHttpClient); + assertThrows(InternalException.class, + () -> httpHelper.requestForSuccessResp( + HttpRequest.builder(HttpMethodEnum.GET, "http://localhost:8080/test").build())); + + // GET + NoHttpResponseException 会被重试 + Mockito.verify(mockedRetryHandler, Mockito.times(4)) + .retryRequest(Mockito.any(), Mockito.anyInt(), Mockito.any()); + } +} diff --git a/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/http/JobHttpRequestRetryHandlerTest.java b/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/http/JobHttpRequestRetryHandlerTest.java new file mode 100644 index 0000000000..79ef76e2ee --- /dev/null +++ b/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/http/JobHttpRequestRetryHandlerTest.java @@ -0,0 +1,274 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.util.http; + +import org.apache.http.NoHttpResponseException; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.conn.ConnectTimeoutException; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.protocol.HttpContext; +import org.apache.http.protocol.HttpCoreContext; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.net.SocketTimeoutException; +import java.net.UnknownHostException; + +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * Http Client 组件调用测试 + */ +public class JobHttpRequestRetryHandlerTest { + @Test + @DisplayName("验证ConnectTimeoutException在非幂等请求下能够重试") + void givenNotIdempotentRequestThrowConnectionTimeoutExceptionThenRetry() { + JobHttpRequestRetryHandler mockedRetryHandler = Mockito.spy(new JobHttpRequestRetryHandler()); + + CloseableHttpClient retryableHttpClient = JobHttpClientFactory.createHttpClient( + 15000, + 15000, + 15000, + 1, + 2, + 60, + true, + mockedRetryHandler, + (httpClientBuilder -> { + httpClientBuilder.addInterceptorFirst((org.apache.http.HttpRequest request, HttpContext context) -> { + throw new ConnectTimeoutException(); + }); + })); + + HttpPost post = new HttpPost("http://127.0.0.1:8080/test"); + assertThrows(ConnectTimeoutException.class, () -> retryableHttpClient.execute(post)); + + Mockito.verify(mockedRetryHandler, Mockito.times(4)) + .retryRequest(Mockito.any(), Mockito.anyInt(), Mockito.any()); + } + + @Test + @DisplayName("验证NoHttpResponseException在非幂等请求下能够重试") + void givenNotIdempotentRequestThrowNoHttpResponseExceptionThenRetry() { + JobHttpRequestRetryHandler mockedRetryHandler = Mockito.spy(new JobHttpRequestRetryHandler()); + + CloseableHttpClient retryableHttpClient = JobHttpClientFactory.createHttpClient( + 15000, + 15000, + 15000, + 1, + 2, + 60, + true, + mockedRetryHandler, + (httpClientBuilder -> { + httpClientBuilder.addInterceptorFirst((org.apache.http.HttpRequest request, HttpContext context) -> { + throw new NoHttpResponseException(""); + }); + })); + + HttpPost post = new HttpPost("http://127.0.0.1:8080/test"); + assertThrows(NoHttpResponseException.class, () -> retryableHttpClient.execute(post)); + + Mockito.verify(mockedRetryHandler, Mockito.times(4)) + .retryRequest(Mockito.any(), Mockito.anyInt(), Mockito.any()); + } + + @Test + @DisplayName("验证SocketTimeoutException在非幂等请求下不会被重试") + void givenNotIdempotentRequestThrowSocketTimeoutExceptionThenDoNotRetry() { + JobHttpRequestRetryHandler mockedRetryHandler = Mockito.spy(new JobHttpRequestRetryHandler()); + + CloseableHttpClient retryableHttpClient = JobHttpClientFactory.createHttpClient( + 15000, + 15000, + 15000, + 1, + 2, + 60, + true, + mockedRetryHandler, + (httpClientBuilder -> { + httpClientBuilder.addInterceptorFirst((org.apache.http.HttpRequest request, HttpContext context) -> { + throw new SocketTimeoutException(); + }); + })); + + HttpPost post = new HttpPost("http://127.0.0.1:8080/test"); + assertThrows(SocketTimeoutException.class, () -> retryableHttpClient.execute(post)); + + Mockito.verify(mockedRetryHandler, Mockito.times(1)) + .retryRequest(Mockito.any(), Mockito.anyInt(), Mockito.any()); + } + + @Test + @DisplayName("验证抛出其他不可重试的异常，不会被重试") + void givenThrowNotRetryableExceptionThenDoNotRetry() { + JobHttpRequestRetryHandler mockedRetryHandler = Mockito.spy(new JobHttpRequestRetryHandler()); + + CloseableHttpClient retryableHttpClient = JobHttpClientFactory.createHttpClient( + 15000, + 15000, + 15000, + 1, + 2, + 60, + true, + mockedRetryHandler, + (httpClientBuilder -> { + httpClientBuilder.addInterceptorFirst((org.apache.http.HttpRequest request, HttpContext context) -> { + throw new UnknownHostException(""); + }); + })); + + HttpGet get = new HttpGet("http://127.0.0.1:8080/test"); + assertThrows(UnknownHostException.class, () -> retryableHttpClient.execute(get)); + + Mockito.verify(mockedRetryHandler, Mockito.times(1)) + .retryRequest(Mockito.any(), Mockito.anyInt(), Mockito.any()); + } + + @Test + @DisplayName("测试指定重试模式为RetryModeEnum.ALWAYS, 虽然 POST 不是幂等方法，但是仍然能够重试") + void givenRetryModeAlwaysAndPostThenRetry() { + JobHttpRequestRetryHandler mockedRetryHandler = Mockito.spy(new JobHttpRequestRetryHandler()); + + CloseableHttpClient retryableHttpClient = JobHttpClientFactory.createHttpClient( + 15000, + 15000, + 15000, + 1, + 2, + 60, + true, + mockedRetryHandler, + (httpClientBuilder -> { + httpClientBuilder.addInterceptorFirst((org.apache.http.HttpRequest request, HttpContext context) -> { + throw new SocketTimeoutException(); + }); + })); + + HttpPost post = new HttpPost("http://127.0.0.1:8080/test"); + HttpCoreContext httpContext = HttpCoreContext.create(); + httpContext.setAttribute(HttpContextAttributeNames.RETRY_MODE, RetryModeEnum.ALWAYS.getValue()); + assertThrows(SocketTimeoutException.class, () -> retryableHttpClient.execute(post, httpContext)); + + + // 虽然 POST 不是幂等的方法，但是通过设置重试模式为RetryModeEnum.ALWAYS，依然会被重试3 次，共计调用方法 4 次 + Mockito.verify(mockedRetryHandler, Mockito.times(4)) + .retryRequest(Mockito.any(), Mockito.anyInt(), Mockito.any()); + } + + @Test + @DisplayName("测试指定重试模式为RetryModeEnum.NEVER, 虽然 GET 是幂等方法，但是仍然不能够重试") + void givenRetryModeNeverAndPostThenRetry() { + JobHttpRequestRetryHandler mockedRetryHandler = Mockito.spy(new JobHttpRequestRetryHandler()); + + CloseableHttpClient retryableHttpClient = JobHttpClientFactory.createHttpClient( + 15000, + 15000, + 15000, + 1, + 2, + 60, + true, + mockedRetryHandler, + (httpClientBuilder -> { + httpClientBuilder.addInterceptorFirst((org.apache.http.HttpRequest request, HttpContext context) -> { + throw new SocketTimeoutException(); + }); + })); + + HttpGet get = new HttpGet("http://127.0.0.1:8080/test"); + HttpCoreContext httpContext = HttpCoreContext.create(); + httpContext.setAttribute(HttpContextAttributeNames.RETRY_MODE, RetryModeEnum.NEVER.getValue()); + assertThrows(SocketTimeoutException.class, () -> retryableHttpClient.execute(get, httpContext)); + + + // 虽然 GET 是幂等的方法，但是通过设置重试模式为RetryModeEnum.NEVER，不会被重试 + Mockito.verify(mockedRetryHandler, Mockito.times(1)) + .retryRequest(Mockito.any(), Mockito.anyInt(), Mockito.any()); + } + + @Test + @DisplayName("测试使用默认的重试模式RetryModeEnum.SAFE_GUARANTEED，非幂等方法比如 POST 不会被重试") + void givenNotIdempotentRequestThenDoNotRetry() { + JobHttpRequestRetryHandler mockedRetryHandler = Mockito.spy(new JobHttpRequestRetryHandler()); + + CloseableHttpClient retryableHttpClient = JobHttpClientFactory.createHttpClient( + 15000, + 15000, + 15000, + 1, + 2, + 60, + true, + mockedRetryHandler, + (httpClientBuilder -> { + httpClientBuilder.addInterceptorFirst((org.apache.http.HttpRequest request, HttpContext context) -> { + throw new SocketTimeoutException(); + }); + })); + HttpPost post = new HttpPost("http://127.0.0.1:8080/test"); + HttpCoreContext httpContext = HttpCoreContext.create(); + assertThrows(SocketTimeoutException.class, () -> retryableHttpClient.execute(post, httpContext)); + + + // POST 不幂等的方法，所以不重试，只会调用一次 + Mockito.verify(mockedRetryHandler, Mockito.times(1)) + .retryRequest(Mockito.any(), Mockito.anyInt(), Mockito.any()); + } + + @Test + @DisplayName("测试使用默认的重试模式RetryModeEnum.SAFE_GUARANTEED，幂等方法比如 GET 会被重试") + void givenIdempotentRequestThenRetry() { + JobHttpRequestRetryHandler mockedRetryHandler = Mockito.spy(new JobHttpRequestRetryHandler()); + + CloseableHttpClient retryableHttpClient = JobHttpClientFactory.createHttpClient( + 15000, + 15000, + 15000, + 1, + 2, + 60, + true, + mockedRetryHandler, + (httpClientBuilder -> { + httpClientBuilder.addInterceptorFirst((org.apache.http.HttpRequest request, HttpContext context) -> { + throw new SocketTimeoutException(); + }); + })); + HttpGet get = new HttpGet("http://127.0.0.1:8080/test"); + HttpCoreContext httpContext = HttpCoreContext.create(); + assertThrows(SocketTimeoutException.class, () -> retryableHttpClient.execute(get, httpContext)); + + + // GET 幂等的方法，所以会重试 + Mockito.verify(mockedRetryHandler, Mockito.times(4)) + .retryRequest(Mockito.any(), Mockito.anyInt(), Mockito.any()); + } +} diff --git a/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/json/LongTimestampSerializerTest.java b/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/json/LongTimestampSerializerTest.java index 4ff7624e9c..7f143efe21 100644 --- a/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/json/LongTimestampSerializerTest.java +++ b/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/json/LongTimestampSerializerTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/time/TimeUtilTest.java b/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/time/TimeUtilTest.java new file mode 100644 index 0000000000..af9e7efda5 --- /dev/null +++ b/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/util/time/TimeUtilTest.java @@ -0,0 +1,48 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.util.time; + + +import com.tencent.bk.job.common.util.TimeUtil; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; + +public class TimeUtilTest { + + @Test + @DisplayName("测试解析 DateTimeFormatter.ISO_DATE_TIME 格式的时间") + void testParseIsoZonedTimeToMillis() { + assertThat(TimeUtil.parseIsoZonedTimeToMillis(null)).isNull(); + assertThat(TimeUtil.parseIsoZonedTimeToMillis("")).isNull(); + assertThat(TimeUtil.parseIsoZonedTimeToMillis("2023-01-01T00:00:00.000AA")).isNull(); + assertThat(TimeUtil.parseIsoZonedTimeToMillis("2023-01-01T00:00:00.000Z")).isEqualTo(1672531200000L); + assertThat(TimeUtil.parseIsoZonedTimeToMillis("2023-01-01T00:00:00.000+08:00")).isEqualTo(1672502400000L); + assertThat(TimeUtil.parseIsoZonedTimeToMillis("2023-01-01T00:00:00.000-08:00")).isEqualTo(1672560000000L); + assertThat(TimeUtil.parseIsoZonedTimeToMillis("2023-05-08T19:37:08.191+08:00")).isEqualTo(1683545828191L); + } + +} diff --git a/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/validation/PathTest.java b/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/validation/PathTest.java new file mode 100644 index 0000000000..45beef0985 --- /dev/null +++ b/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/validation/PathTest.java @@ -0,0 +1,53 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.validation; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +public class PathTest { + + @Test + void pathTest() { + Path p = Path.newPath("root"); + assertThat(p.toString()).isEqualTo("root"); + p = p.child("first"); + assertThat(p.toString()).isEqualTo("root.first"); + p = p.child("second"); + assertThat(p.toString()).isEqualTo("root.first.second"); + p = p.index(0); + assertThat(p.toString()).isEqualTo("root.first.second[0]"); + p = p.child("third"); + p = p.index(93); + assertThat(p.toString()).isEqualTo("root.first.second[0].third[93]"); + p = p.parent(); + assertThat(p.toString()).isEqualTo("root.first.second[0].third"); + p = p.parent(); + assertThat(p.toString()).isEqualTo("root.first.second[0]"); + p = p.key("key"); + assertThat(p.toString()).isEqualTo("root.first.second[0][key]"); + } +} diff --git a/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/watch/SafeWatchTest.java b/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/watch/SafeWatchTest.java new file mode 100644 index 0000000000..157fad857e --- /dev/null +++ b/src/backend/commons/common/src/test/java/com/tencent/bk/job/common/watch/SafeWatchTest.java @@ -0,0 +1,69 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.watch; + +import org.junit.jupiter.api.Test; +import org.springframework.util.StopWatch; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; + +/** + * SafeWatch单元测试，主要测试其在非正常状态下操作的鲁棒性 + */ +public class SafeWatchTest { + + @Test + void testStart() { + assertThrows(IllegalStateException.class, () -> { + StopWatch normalWatch = new StopWatch(); + normalWatch.start("task1"); + normalWatch.start("task2"); + normalWatch.stop(); + }); + assertDoesNotThrow(() -> { + StopWatch safeWatch = new SafeWatch(); + safeWatch.start("task1"); + safeWatch.start("task2"); + safeWatch.stop(); + }); + } + + @Test + void testStop() { + assertThrows(IllegalStateException.class, () -> { + StopWatch normalWatch = new StopWatch(); + normalWatch.start("task1"); + normalWatch.stop(); + normalWatch.stop(); + }); + assertDoesNotThrow(() -> { + StopWatch safeWatch = new SafeWatch(); + safeWatch.start("task1"); + safeWatch.stop(); + safeWatch.stop(); + }); + } +} diff --git a/src/backend/commons/esb-sdk/build.gradle b/src/backend/commons/esb-sdk/build.gradle index b9578ad940..61385b85a7 100644 --- a/src/backend/commons/esb-sdk/build.gradle +++ b/src/backend/commons/esb-sdk/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,7 +27,7 @@ dependencies { implementation 'com.fasterxml.jackson.core:jackson-core' implementation 'com.fasterxml.jackson.core:jackson-databind' implementation 'com.fasterxml.jackson.core:jackson-annotations' - implementation 'io.springfox:springfox-swagger2' + implementation 'io.springfox:springfox-boot-starter' implementation 'org.apache.commons:commons-lang3' implementation "org.springframework:spring-jdbc" implementation 'org.springframework:spring-web' diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/config/AppProperties.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/config/AppProperties.java new file mode 100644 index 0000000000..68a3ba7b34 --- /dev/null +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/config/AppProperties.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.esb.config; + +import lombok.Data; +import lombok.NoArgsConstructor; +import org.springframework.boot.context.properties.ConfigurationProperties; + +@Data +@ConfigurationProperties(prefix = "app") +@NoArgsConstructor +public class AppProperties { + + private String code; + + private String secret; + + public AppProperties(String code, String secret) { + this.code = code; + this.secret = secret; + } +} diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/config/BkApiAutoConfiguration.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/config/BkApiAutoConfiguration.java new file mode 100644 index 0000000000..10eeffd173 --- /dev/null +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/config/BkApiAutoConfiguration.java @@ -0,0 +1,33 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.esb.config; + +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +@Configuration(proxyBeanMethods = false) +@EnableConfigurationProperties({BkApiGatewayProperties.class, AppProperties.class, EsbProperties.class}) +public class BkApiAutoConfiguration { +} diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/config/BkApiGatewayProperties.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/config/BkApiGatewayProperties.java new file mode 100644 index 0000000000..014f28eda3 --- /dev/null +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/config/BkApiGatewayProperties.java @@ -0,0 +1,67 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.esb.config; + +import lombok.Data; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.springframework.boot.context.properties.ConfigurationProperties; + +@Data +@ConfigurationProperties(prefix = "bk-api-gateway") +public class BkApiGatewayProperties { + + + private ApiGwConfig gse; + + private ApiGwConfig bkNotice; + + private ApiGwConfig bkAIDev; + + private ApiGwConfig cmdb; + + private ApiGwConfig bkApiGateway; + + @Getter + @Setter + @ToString + public static class ApiGwConfig { + /** + * 蓝鲸Api Gateway url + */ + private String url; + + /** + * 蓝鲸Api Gateway appCode，若配置了则优先使用，覆盖app.code配置项 + */ + private String appCode; + + /** + * 蓝鲸Api Gateway appSecret，若配置了则优先使用，覆盖app.secret配置项 + */ + private String appSecret; + } +} diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/config/EsbConfig.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/config/EsbConfig.java deleted file mode 100644 index 80e9236198..0000000000 --- a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/config/EsbConfig.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.esb.config; - -import lombok.Data; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - -@Data -@Component -public class EsbConfig { - - @Value("${app.code:}") - private String appCode; - - @Value("${app.secret:}") - private String appSecret; - - @Value("${esb.service.url:}") - private String esbUrl; - - @Value("${esb.use.test.env:false}") - private boolean useEsbTestEnv; -} diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/config/EsbProperties.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/config/EsbProperties.java new file mode 100644 index 0000000000..00cb41073f --- /dev/null +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/config/EsbProperties.java @@ -0,0 +1,52 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.esb.config; + +import lombok.Data; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.springframework.boot.context.properties.ConfigurationProperties; + +@Data +@ConfigurationProperties(prefix = "esb") +public class EsbProperties { + + private EsbServiceConfig service; + + @Getter + @Setter + @ToString + public static class EsbServiceConfig { + /** + * ESB 后端调用 URL + */ + private String url; + /** + * ESB 前端调用 URL + */ + private String publicUrl; + } +} diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/constants/ApiGwType.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/constants/ApiGwType.java new file mode 100644 index 0000000000..ff9e130c42 --- /dev/null +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/constants/ApiGwType.java @@ -0,0 +1,39 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.esb.constants; + +/** + * 蓝鲸 API 网关类型 + */ +public enum ApiGwType { + /** + * ESB + */ + ESB, + /** + * 蓝鲸网关，用于替换 ESB + */ + BK_APIGW; +} diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/constants/EsbConsts.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/constants/EsbConsts.java index c0d25468d0..1a060fc66c 100644 --- a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/constants/EsbConsts.java +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/constants/EsbConsts.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/constants/EsbLang.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/constants/EsbLang.java index e672614a50..063676b71b 100644 --- a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/constants/EsbLang.java +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/constants/EsbLang.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/interceptor/LogBkApiRequestIdInterceptor.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/interceptor/LogBkApiRequestIdInterceptor.java new file mode 100644 index 0000000000..e072cae36c --- /dev/null +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/interceptor/LogBkApiRequestIdInterceptor.java @@ -0,0 +1,71 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.esb.interceptor; + +import com.tencent.bk.job.common.constant.JobCommonHeaders; +import lombok.extern.slf4j.Slf4j; +import org.apache.http.HttpResponse; +import org.apache.http.HttpResponseInterceptor; +import org.apache.http.HttpStatus; +import org.apache.http.StatusLine; +import org.apache.http.protocol.HttpContext; + +/** + * 打印APIGW请求响应Header中的X-Bkapi-Request-Id，用于问题定位 + */ +@Slf4j +public class LogBkApiRequestIdInterceptor implements HttpResponseInterceptor { + @Override + public void process(HttpResponse response, HttpContext context) { + StatusLine statusLine = response.getStatusLine(); + // 状态码为2xx的请求会在上层逻辑打印响应头X-Bkapi-Request-Id，此处不打印减少日志量 + if (isStatusCodeSuccess(statusLine)) { + return; + } + // 获取并打印响应头X-Bkapi-Request-Id + String headerName = JobCommonHeaders.BK_GATEWAY_REQUEST_ID; + try { + if (response.containsHeader(headerName)) { + log.info(headerName + "=" + response.getFirstHeader(headerName).getValue()); + } else { + log.info(headerName + " not found in the response"); + } + } catch (Throwable t) { + log.warn("Failed to log header " + headerName, t); + } + } + + /** + * 判断Http状态码是否为成功（2xx）系列状态码 + * + * @param statusLine 状态行 + * @return 是则返回true，否则返回false + */ + private boolean isStatusCodeSuccess(StatusLine statusLine) { + return statusLine != null + && statusLine.getStatusCode() >= HttpStatus.SC_OK + && statusLine.getStatusCode() < HttpStatus.SC_MULTIPLE_CHOICES; + } +} diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/metrics/EsbApiTimed.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/metrics/EsbApiTimed.java index f5a8b2491d..1b0e7ee648 100644 --- a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/metrics/EsbApiTimed.java +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/metrics/EsbApiTimed.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/metrics/EsbApiTimedAspect.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/metrics/EsbApiTimedAspect.java index 156bd6b8ba..b8dc217231 100644 --- a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/metrics/EsbApiTimedAspect.java +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/metrics/EsbApiTimedAspect.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -63,7 +63,7 @@ public static Tag exception(Throwable exception) { return EXCEPTION_NONE; } - @Around("execution (@com.tencent.bk.job.common.esb.metrics.EsbApiTimed * *.*(..))") + @Around("within(com.tencent.bk.job..*) && execution (@com.tencent.bk.job.common.esb.metrics.EsbApiTimed * *.*(..))") public Object timedMethod(ProceedingJoinPoint pjp) throws Throwable { RequestAttributes requestAttributes = RequestContextHolder.getRequestAttributes(); HttpServletRequest request = null; diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/metrics/EsbMetricTags.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/metrics/EsbMetricTags.java index 9f695f57fa..268b8abc98 100644 --- a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/metrics/EsbMetricTags.java +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/metrics/EsbMetricTags.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -36,15 +36,11 @@ public class EsbMetricTags { /** * 接口调用状态：初始值 */ - public static final String VALUE_STATUS_NONE = "none"; + public static final String VALUE_NONE = "none"; /** * 接口调用状态：成功 */ - public static final String VALUE_STATUS_SUCCESS = "success"; - /** - * 接口调用状态：失败 - */ - public static final String VALUE_STATUS_FAIL = "fail"; + public static final String VALUE_STATUS_OK = "ok"; /** * 接口调用状态：超频 */ diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/BkApiAuthorization.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/BkApiAuthorization.java new file mode 100644 index 0000000000..e2dd2bb651 --- /dev/null +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/BkApiAuthorization.java @@ -0,0 +1,140 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.esb.model; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.util.json.SkipLogFields; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * 蓝鲸网关认证信息. 说明文档 + * https://github.com/TencentBlueKing/blueking-apigateway/issues/325 + */ +@Data +@NoArgsConstructor +@JsonInclude(JsonInclude.Include.NON_EMPTY) +public class BkApiAuthorization { + /** + * 应用 ID + */ + @JsonProperty("bk_app_code") + private String appCode; + + /** + * 安全秘钥 + */ + @SkipLogFields + @JsonProperty("bk_app_secret") + private String appSecret; + + /** + * 用户态 access_token，或应用态 access_token + */ + @SkipLogFields + @JsonProperty("access_token") + private String accessToken; + + /** + * 用户登录态 token，用于认证用户；登录蓝鲸，对应 Cookies 中 bk_token 字段的值； + * 与bk_ticket互斥，提供 bk_token 时，不需要再提供 bk_ticket/bk_username + */ + @SkipLogFields + @JsonProperty("bk_token") + private String bkToken; + + /** + * 某些环境下的用户登录态 token，用于认证用户；登录蓝鲸，对应 Cookies 中 bk_ticket 字段的值； + * 与bk_token互斥，提供 bk_ticket 时，不需要再提供 bk_token/bk_username + */ + @SkipLogFields + @JsonProperty("bk_ticket") + private String bkTicket; + + /** + * 当前用户用户名；仅用于应用免用户认证的场景中，用于指定当前用户 + */ + @JsonProperty("bk_username") + private String username; + + /** + * 应用认证 + * + * @param appCode 应用ID + * @param appSecret 安全秘钥 + * @return 认证信息 + */ + public static BkApiAuthorization appAuthorization(String appCode, String appSecret) { + BkApiAuthorization authorization = new BkApiAuthorization(); + authorization.setAppCode(appCode); + authorization.setAppSecret(appSecret); + return authorization; + } + + /** + * 应用认证 - 免用户认证 + * + * @param appCode 应用ID + * @param appSecret 安全秘钥 + * @return 认证信息 + */ + public static BkApiAuthorization appAuthorization(String appCode, String appSecret, String username) { + BkApiAuthorization authorization = new BkApiAuthorization(); + authorization.setAppCode(appCode); + authorization.setAppSecret(appSecret); + authorization.setUsername(username); + return authorization; + } + + /** + * 用户认证 - 根据 bk_token + * + * @param bkToken 用户登录 bk_token + * @return 认证信息 + */ + public static BkApiAuthorization bkTokenUserAuthorization(String appCode, String appSecret, String bkToken) { + BkApiAuthorization authorization = new BkApiAuthorization(); + authorization.setAppCode(appCode); + authorization.setAppSecret(appSecret); + authorization.setBkToken(bkToken); + return authorization; + } + + /** + * 用户认证 - 根据 bk_ticket + * + * @param bkTicket 用户登录 bk_ticket + * @return 认证信息 + */ + public static BkApiAuthorization bkTicketUserAuthorization(String appCode, String appSecret, String bkTicket) { + BkApiAuthorization authorization = new BkApiAuthorization(); + authorization.setAppCode(appCode); + authorization.setAppSecret(appSecret); + authorization.setBkTicket(bkTicket); + return authorization; + } + +} diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/EsbAppScopeDTO.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/EsbAppScopeDTO.java index 6618f51a00..3a4e0449de 100644 --- a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/EsbAppScopeDTO.java +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/EsbAppScopeDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,7 +25,9 @@ package com.tencent.bk.job.common.esb.model; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.CompatibleType; import lombok.Getter; import lombok.Setter; import lombok.ToString; @@ -40,19 +42,23 @@ public class EsbAppScopeDTO { /** * 兼容字段,表示cmdb 业务/业务集ID */ - @CompatibleImplementation(explain = "兼容字段,表示业务ID或者业务集ID", version = "3.6.x") + @CompatibleImplementation(name = "bizId", type = CompatibleType.API, + explain = "兼容字段,表示业务ID或者业务集ID", deprecatedVersion = "3.6.x") @JsonProperty("bk_biz_id") + @JsonPropertyDescription("bk_biz_id") private Long bizId; /** * 资源范围类型 */ @JsonProperty("bk_scope_type") + @JsonPropertyDescription("Resource scope type") private String scopeType; /** * 资源范围ID */ @JsonProperty("bk_scope_id") + @JsonPropertyDescription("Resource scope id") private String scopeId; } diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/EsbAppScopeReq.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/EsbAppScopeReq.java index 19909f29fb..879aae19a8 100644 --- a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/EsbAppScopeReq.java +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/EsbAppScopeReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,7 +27,9 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.CompatibleType; import com.tencent.bk.job.common.constant.ResourceScopeTypeEnum; +import com.tencent.bk.job.common.esb.validate.ValidBkScope; import com.tencent.bk.job.common.esb.validate.EsbAppScopeReqGroupSequenceProvider; import com.tencent.bk.job.common.model.dto.AppResourceScope; import com.tencent.bk.job.common.service.AppScopeMappingService; @@ -45,12 +47,14 @@ @Setter @Getter +@ValidBkScope @GroupSequenceProvider(EsbAppScopeReqGroupSequenceProvider.class) public class EsbAppScopeReq extends EsbJobReq { /** * 兼容字段,表示cmdb 业务/业务集ID */ - @CompatibleImplementation(explain = "兼容字段,表示业务ID或者业务集ID", version = "3.6.x") + @CompatibleImplementation(name = "bizId", type = CompatibleType.API, + explain = "兼容字段,表示业务ID或者业务集ID", deprecatedVersion = "3.6.x") @JsonProperty("bk_biz_id") @Min(value = 1L, message = "{validation.constraints.InvalidBkBizId.message}", groups = UseBkBizIdParam.class) private Long bizId; @@ -108,6 +112,7 @@ public void fillAppResourceScope(AppScopeMappingService appScopeMappingService) } } + @JsonIgnore public AppResourceScope getAppResourceScope() { return new AppResourceScope(this.scopeType, this.scopeId, this.appId); } diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/EsbJobReq.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/EsbJobReq.java index 2476f4fc78..0924cbd304 100644 --- a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/EsbJobReq.java +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/EsbJobReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/EsbPageData.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/EsbPageData.java index 4a445ddf0a..4841442305 100644 --- a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/EsbPageData.java +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/EsbPageData.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/EsbReq.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/EsbReq.java index 64d0db846b..97354ced49 100644 --- a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/EsbReq.java +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/EsbReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,10 +26,12 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.InternalException; import com.tencent.bk.job.common.util.http.BasicHttpReq; -import com.tencent.bk.job.common.util.json.SkipLogFields; import lombok.Getter; import lombok.Setter; +import org.apache.commons.lang3.StringUtils; /** * ESB API 通用请求参数 @@ -37,22 +39,34 @@ @Setter @Getter public class EsbReq extends BasicHttpReq { - @JsonProperty("bk_app_code") - private String appCode; - - @SkipLogFields("bk_app_secret") - @JsonProperty("bk_app_secret") - private String appSecret; - - @JsonProperty("bk_username") - private String userName; - - @JsonProperty("bk_token") - @JsonInclude(JsonInclude.Include.NON_EMPTY) - private String bkToken; + /** + * 租户账号 - 除了 cmdb 之外，其他平台暂未使用 + */ @JsonProperty("bk_supplier_account") @JsonInclude(JsonInclude.Include.NON_EMPTY) private String bkSupplierAccount; + /** + * 构造请求 + * + * @param reqClass 要构建返回的请求 class + * @param bkSupplierAccount 开发商code + * @return EsbReq + */ + public static T buildRequest(Class reqClass, String bkSupplierAccount) { + T esbReq; + try { + esbReq = reqClass.newInstance(); + if (StringUtils.isEmpty(bkSupplierAccount)) { + esbReq.setBkSupplierAccount("0"); + } else { + esbReq.setBkSupplierAccount(bkSupplierAccount); + } + } catch (InstantiationException | IllegalAccessException e) { + throw new InternalException(e, ErrorCode.INTERNAL_ERROR); + } + return esbReq; + } + } diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/EsbResp.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/EsbResp.java index 4893070877..d4d476782f 100644 --- a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/EsbResp.java +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/EsbResp.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,7 @@ package com.tencent.bk.job.common.esb.model; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import com.tencent.bk.job.common.constant.ErrorCode; @@ -102,7 +103,7 @@ public static EsbResp buildCommonFailResp(ServiceException e) { public static EsbResp buildAuthFailResult(EsbApplyPermissionDTO permission) { EsbResp esbResp = buildCommonFailResp(ErrorCode.BK_PERMISSION_DENIED, - new String[]{JobContextUtil.getUsername()}, null); + new String[]{JobContextUtil.getUsername()}, null); esbResp.setPermission(permission); return esbResp; } @@ -133,4 +134,9 @@ public static EsbResp buildValidateFailResp(ErrorDetailDTO errorDetail) { } return resp; } + + @JsonIgnore + public boolean isSuccess() { + return (this.result != null && this.result) || (this.code != null && this.code == ErrorCode.RESULT_OK); + } } diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/OpenApiRequestInfo.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/OpenApiRequestInfo.java new file mode 100644 index 0000000000..ec7d8aeb4d --- /dev/null +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/OpenApiRequestInfo.java @@ -0,0 +1,189 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.esb.model; + +import com.tencent.bk.job.common.constant.HttpMethodEnum; +import com.tencent.bk.job.common.util.http.RetryModeEnum; +import lombok.Data; +import org.apache.commons.lang3.StringUtils; + +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.HashMap; +import java.util.Map; + +/** + * Open API 请求封装 + * + * @param + */ +@Data +public class OpenApiRequestInfo { + private final HttpMethodEnum method; + private final String uri; + /** + * http url 格式 Query 参数，比如 ?name=admin&type=1 + */ + private final String queryParams; + /** + * Query 参数 Map, 优先于 queryParams 参数 + */ + private Map queryParamsMap; + private final T body; + private final BkApiAuthorization authorization; + /** + * 请求重试模式 + */ + private final RetryModeEnum retryMode; + /** + * 请求是否幂等 + */ + private final Boolean idempotent; + + public OpenApiRequestInfo(Builder builder) { + this.method = builder.method; + this.uri = builder.uri; + this.queryParams = builder.queryParams; + this.queryParamsMap = builder.queryParamsMap; + this.body = builder.body; + this.authorization = builder.authorization; + this.retryMode = builder.retryMode; + this.idempotent = builder.idempotent; + } + + public static Builder builder() { + return new Builder<>(); + } + + public static class Builder { + private HttpMethodEnum method; + private String uri; + private String queryParams; + private Map queryParamsMap; + private T body; + private BkApiAuthorization authorization; + private RetryModeEnum retryMode = RetryModeEnum.SAFE_GUARANTEED; + private Boolean idempotent; + + public Builder method(HttpMethodEnum method) { + this.method = method; + return this; + } + + public Builder uri(String uri) { + this.uri = uri; + return this; + } + + public Builder queryParams(String queryParams) { + this.queryParams = queryParams; + return this; + } + + public Builder body(T body) { + this.body = body; + return this; + } + + public Builder authorization(BkApiAuthorization authorization) { + this.authorization = authorization; + return this; + } + + public Builder queryParamsMap(Map queryParamsMap) { + this.queryParamsMap = queryParamsMap; + return this; + } + + public Builder addQueryParam(String name, String value) { + initQueryParamsMapIfNull(); + queryParamsMap.put(name, value); + return this; + } + + private void initQueryParamsMapIfNull() { + if (queryParamsMap == null) { + queryParamsMap = new HashMap<>(); + } + } + + public Builder addQueryParams(Map paramsMap) { + initQueryParamsMapIfNull(); + queryParamsMap.putAll(paramsMap); + return this; + } + + public Builder setRetryMode(RetryModeEnum retryMode) { + this.retryMode = retryMode; + return this; + } + + public Builder setIdempotent(Boolean idempotent) { + this.idempotent = idempotent; + return this; + } + + public OpenApiRequestInfo build() { + return new OpenApiRequestInfo<>(this); + } + } + + /** + * 构造最终请求的 uri + */ + public String buildFinalUri() { + String result = uri; + String queryParamUrl = buildQueryParamUrl(); + if (StringUtils.isNotBlank(queryParamUrl)) { + if (queryParamUrl.startsWith("?")) { + result += queryParamUrl; + } else { + result += "?" + queryParamUrl; + } + } + return result; + } + + private String buildQueryParamUrl() { + if (queryParamsMap != null && !queryParamsMap.isEmpty()) { + StringBuilder urlString = new StringBuilder(512); + queryParamsMap.forEach((name, value) -> + urlString.append('&').append(name).append('=').append(urlEncode(value))); + return urlString.toString().substring(1); + } else if (StringUtils.isNotBlank(this.queryParams)) { + return this.queryParams; + } else { + return null; + } + } + + private String urlEncode(String str) { + try { + return URLEncoder.encode(str, "utf-8"); + } catch (UnsupportedEncodingException e) { + throw new RuntimeException("encode failed"); + } + } +} diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/iam/EsbActionDTO.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/iam/EsbActionDTO.java index 85ff129e36..32293c3a01 100644 --- a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/iam/EsbActionDTO.java +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/iam/EsbActionDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/iam/EsbApplyPermissionDTO.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/iam/EsbApplyPermissionDTO.java index b6c1637029..b8bed9d6f7 100644 --- a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/iam/EsbApplyPermissionDTO.java +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/iam/EsbApplyPermissionDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/iam/EsbInstanceDTO.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/iam/EsbInstanceDTO.java index 0c4c9c09e9..2c35ac2488 100644 --- a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/iam/EsbInstanceDTO.java +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/iam/EsbInstanceDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/iam/EsbRelatedResourceTypeDTO.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/iam/EsbRelatedResourceTypeDTO.java index 8af17d2d00..c9b3ff5af5 100644 --- a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/iam/EsbRelatedResourceTypeDTO.java +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/iam/EsbRelatedResourceTypeDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/EsbFileSourceDTO.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/EsbFileSourceDTO.java index 634a6dfa8c..b85f25b1ca 100644 --- a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/EsbFileSourceDTO.java +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/EsbFileSourceDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/EsbGlobalVarDTO.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/EsbGlobalVarDTO.java index d3c37d7343..c02bbf7e8c 100644 --- a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/EsbGlobalVarDTO.java +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/EsbGlobalVarDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/EsbIpDTO.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/EsbIpDTO.java index 8ed00513a3..1f5f404d01 100644 --- a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/EsbIpDTO.java +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/EsbIpDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,10 +25,10 @@ package com.tencent.bk.job.common.esb.model.job; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.util.ip.IpUtils; -import lombok.AllArgsConstructor; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; @@ -40,21 +40,48 @@ @Setter @Getter @NoArgsConstructor -@AllArgsConstructor public class EsbIpDTO { + @JsonProperty("bk_host_id") + @JsonPropertyDescription("Host id") + private Long hostId; + @JsonProperty("bk_cloud_id") @NotNull(message = "{validation.constraints.InvalidBkCloudId.message}") @Min(value = 0L, message = "{validation.constraints.InvalidBkCloudId.message}") + @JsonPropertyDescription("BK-Network Area") private Long bkCloudId; + @JsonProperty("bk_cloud_name") + @JsonPropertyDescription("BK-Network Area Name") + private String bkCloudName; + @JsonProperty("ip") @Pattern(regexp = "\\b((?!\\d\\d\\d)\\d+|1\\d\\d|2[0-4]\\d|25[0-5])\\.((?!\\d\\d\\d)" + "\\d+|1\\d\\d|2[0-4]\\d|25[0-5])\\.((?!\\d\\d\\d)\\d+|1\\d\\d|2[0-4]\\d|25[0-5])\\.((?!\\d\\d\\d)" + "\\d+|1\\d\\d|2[0-4]\\d|25[0-5])\\b", message = "{validation.constraints.InvalidIp.message}") + @JsonPropertyDescription("ip") private String ip; + @JsonProperty("ipv6") + @JsonPropertyDescription("ipv6") + private String ipv6; + + @JsonProperty("bk_agent_id") + @JsonPropertyDescription("Agent Id") + private String agentId; + + @JsonProperty("alive") + @JsonPropertyDescription("Agent是否正常") + private Integer alive; + + public EsbIpDTO(Long hostId, Long bkCloudId, String ip) { + this.hostId = hostId; + this.bkCloudId = bkCloudId; + this.ip = ip; + } + public static EsbIpDTO fromApplicationHostInfo(ApplicationHostDTO applicationHostInfo) { if (applicationHostInfo == null) { return null; @@ -62,6 +89,7 @@ public static EsbIpDTO fromApplicationHostInfo(ApplicationHostDTO applicationHos EsbIpDTO esbIp = new EsbIpDTO(); esbIp.setBkCloudId(applicationHostInfo.getCloudAreaId()); esbIp.setIp(applicationHostInfo.getIp()); + esbIp.setHostId(applicationHostInfo.getHostId()); return esbIp; } @@ -71,7 +99,12 @@ public static EsbIpDTO fromHost(HostDTO host) { } EsbIpDTO esbIp = new EsbIpDTO(); esbIp.setBkCloudId(host.getBkCloudId()); + esbIp.setBkCloudName(host.getBkCloudName()); esbIp.setIp(host.getIp()); + esbIp.setIpv6(host.getIpv6()); + esbIp.setHostId(host.getHostId()); + esbIp.setAgentId(host.getAgentId()); + esbIp.setAlive(host.getAlive()); return esbIp; } @@ -80,6 +113,9 @@ public static EsbIpDTO fromCloudIp(String cloudIp) { return null; } String[] ipProps = cloudIp.split(IpUtils.COLON); - return new EsbIpDTO(Long.valueOf(ipProps[0]), ipProps[1]); + EsbIpDTO host = new EsbIpDTO(); + host.setBkCloudId(Long.valueOf(ipProps[0])); + host.setIp(ipProps[1]); + return host; } } diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/EsbServerDTO.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/EsbServerDTO.java index 56e3a44faf..dfd8908345 100644 --- a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/EsbServerDTO.java +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/EsbServerDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,6 +25,7 @@ package com.tencent.bk.job.common.esb.model.job; import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.model.openapi.v3.EsbCmdbTopoNodeDTO; import lombok.Data; import java.util.List; diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/v3/EsbAccountV3BasicDTO.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/v3/EsbAccountV3BasicDTO.java index ee481a05c8..17854addb9 100644 --- a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/v3/EsbAccountV3BasicDTO.java +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/v3/EsbAccountV3BasicDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,7 @@ package com.tencent.bk.job.common.esb.model.job.v3; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; import lombok.Data; /** @@ -34,13 +35,16 @@ public class EsbAccountV3BasicDTO { /** * 账号ID */ + @JsonPropertyDescription("Account id") private Long id; /** * 账号名称 */ + @JsonPropertyDescription("Account name") private String name; /** * 账号别名 */ + @JsonPropertyDescription("Account alias") private String alias; } diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/v3/EsbDynamicGroupDTO.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/v3/EsbDynamicGroupDTO.java deleted file mode 100644 index a7cdafc38c..0000000000 --- a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/v3/EsbDynamicGroupDTO.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.esb.model.job.v3; - -import lombok.Data; - -/** - * 动态分组 - */ -@Data -public class EsbDynamicGroupDTO { - /** - * 动态分组ID - */ - private String id; -} diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/v3/EsbFileDestinationV3DTO.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/v3/EsbFileDestinationV3DTO.java index 8a3800442d..c3d44aa5cd 100644 --- a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/v3/EsbFileDestinationV3DTO.java +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/v3/EsbFileDestinationV3DTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,7 @@ package com.tencent.bk.job.common.esb.model.job.v3; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; import lombok.Data; /** @@ -31,9 +32,12 @@ */ @Data public class EsbFileDestinationV3DTO { + @JsonPropertyDescription("Target path") private String path; + @JsonPropertyDescription("Target account") private EsbAccountV3BasicDTO account; + @JsonPropertyDescription("Target hosts") private EsbServerV3DTO server; } diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/v3/EsbFileSourceV3DTO.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/v3/EsbFileSourceV3DTO.java index a24b738de5..0c9fc0c239 100644 --- a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/v3/EsbFileSourceV3DTO.java +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/v3/EsbFileSourceV3DTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,9 @@ package com.tencent.bk.job.common.esb.model.job.v3; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.util.ListUtil; import lombok.Data; import java.util.List; @@ -51,7 +53,7 @@ public class EsbFileSourceV3DTO { /** * 文件源类型，不传默认为服务器文件 * - * @see com.tencent.bk.job.manage.common.consts.task.TaskFileTypeEnum + * @see com.tencent.bk.job.manage.api.common.constants.task.TaskFileTypeEnum */ @JsonProperty("file_type") private Integer fileType; @@ -67,4 +69,9 @@ public class EsbFileSourceV3DTO { */ @JsonProperty("file_source_code") private String fileSourceCode; + + @JsonIgnore + public List getTrimmedFiles() { + return ListUtil.trimStringList(files); + } } diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/v3/EsbGlobalVarV3DTO.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/v3/EsbGlobalVarV3DTO.java index 7ed8a638d5..3c00f0f245 100644 --- a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/v3/EsbGlobalVarV3DTO.java +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/v3/EsbGlobalVarV3DTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,6 +26,7 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; import lombok.Data; import javax.validation.constraints.Min; @@ -39,34 +40,41 @@ public class EsbGlobalVarV3DTO { * 全局变量ID */ @Min(value = 1L, message = "{validation.constraints.InvalidGlobalVarId.message}") + @JsonPropertyDescription("Global variable id") private Long id; /** * 全局变量名称 */ + @JsonPropertyDescription("Global variable name") private String name; /** * 全局变量值，当变量类型为字符、密码、数组时，此变量有效 */ + @JsonPropertyDescription("Global variable value") private String value; @JsonProperty("server") + @JsonPropertyDescription("Value for host variable") private EsbServerV3DTO server; /** * 变量描述 */ + @JsonPropertyDescription("description") private String description; /** * 变量类型 */ + @JsonPropertyDescription("Variable type") private Integer type; /** * 变量是否必填 */ + @JsonPropertyDescription("Required, 1: YES; 0: NO") private Integer required; /** diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/v3/EsbPageDataV3.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/v3/EsbPageDataV3.java index 3f7aff9cd7..deb38f6746 100644 --- a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/v3/EsbPageDataV3.java +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/v3/EsbPageDataV3.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/v3/EsbServerV3DTO.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/v3/EsbServerV3DTO.java index bb9e6bb566..77e1ded34c 100644 --- a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/v3/EsbServerV3DTO.java +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/v3/EsbServerV3DTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,15 +26,18 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; -import com.tencent.bk.job.common.esb.model.job.EsbCmdbTopoNodeDTO; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; import com.tencent.bk.job.common.esb.model.job.EsbIpDTO; +import com.tencent.bk.job.common.model.openapi.v3.EsbCmdbTopoNodeDTO; +import com.tencent.bk.job.common.model.openapi.v3.EsbDynamicGroupDTO; import lombok.Data; +import org.apache.commons.collections4.CollectionUtils; import javax.validation.Valid; import java.util.List; /** - * 服务器定义-ESB + * 主机定义-ESB */ @Data public class EsbServerV3DTO { @@ -42,21 +45,40 @@ public class EsbServerV3DTO { * 目标服务器对应的主机变量 */ @JsonInclude(JsonInclude.Include.NON_NULL) + @JsonPropertyDescription("Host variable name") private String variable; @JsonProperty("ip_list") + @JsonPropertyDescription("Hosts with ip") @Valid private List ips; + @JsonProperty("host_id_list") + @JsonInclude(JsonInclude.Include.NON_EMPTY) + @JsonPropertyDescription("Host ids") + private List hostIds; + /** * 动态分组ID列表 */ @JsonProperty("dynamic_group_list") + @JsonPropertyDescription("Cmdb dynamic groups") private List dynamicGroups; /** * 分布式拓扑节点列表 */ @JsonProperty("topo_node_list") + @JsonPropertyDescription("Cmdb topo nodes") private List topoNodes; + + /** + * 检查执行主机的参数是否非空 + */ + public boolean checkHostParamsNonEmpty() { + return CollectionUtils.isNotEmpty(hostIds) + || CollectionUtils.isNotEmpty(ips) + || CollectionUtils.isNotEmpty(topoNodes) + || CollectionUtils.isNotEmpty(dynamicGroups); + } } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbApprovalStepV3DTO.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/v3/resp/EsbApprovalStepV3DTO.java similarity index 76% rename from src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbApprovalStepV3DTO.java rename to src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/v3/resp/EsbApprovalStepV3DTO.java index d8ba3ce716..0b9d583732 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbApprovalStepV3DTO.java +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/v3/resp/EsbApprovalStepV3DTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,8 +22,10 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.manage.model.esb.v3.response; +package com.tencent.bk.job.common.esb.model.job.v3.resp; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; import lombok.Data; import java.util.List; @@ -39,20 +41,28 @@ public class EsbApprovalStepV3DTO { /** * 确认类型 */ + @JsonProperty("approval_type") + @JsonPropertyDescription("Approval type") private Integer approvalType; /** * 确认角色 */ + @JsonPropertyDescription("Approval user info") + @JsonProperty("approval_user") private EsbUserRoleInfoV3DTO approvalUser; /** * 确认消息 */ + @JsonPropertyDescription("Approval message") + @JsonProperty("approval_message") private String approvalMessage; /** * 通知渠道列表 */ + @JsonPropertyDescription("Notify channel") + @JsonProperty("notify_channel") private List notifyChannel; } diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/v3/resp/EsbFileStepV3DTO.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/v3/resp/EsbFileStepV3DTO.java new file mode 100644 index 0000000000..28e41e67a0 --- /dev/null +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/v3/resp/EsbFileStepV3DTO.java @@ -0,0 +1,86 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.esb.model.job.v3.resp; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; +import com.tencent.bk.job.common.esb.model.job.v3.EsbFileDestinationV3DTO; +import com.tencent.bk.job.common.esb.model.job.v3.EsbFileSourceV3DTO; +import lombok.Data; + +import java.util.List; + +/** + * 文件分发步骤 + * + * @since 17/11/2020 20:37 + */ +@Data +public class EsbFileStepV3DTO { + /** + * 源文件列表 + */ + @JsonProperty("file_source_list") + @JsonPropertyDescription("File source list") + private List fileSourceList; + + /** + * 分发目标信息 + */ + @JsonProperty("file_destination") + @JsonPropertyDescription("File destination") + private EsbFileDestinationV3DTO fileDestination; + + /** + * 超时 + */ + @JsonPropertyDescription("timeout") + private Long timeout; + + /** + * 源机器上传限速 + */ + @JsonProperty("source_speed_limit") + @JsonPropertyDescription("Upload speed limit") + private Long sourceSpeedLimit; + + /** + * 目标机器下载限速 + */ + @JsonProperty("destination_speed_limit") + @JsonPropertyDescription("Download speed limit") + private Long destinationSpeedLimit; + + /** + * 传输模式 + */ + @JsonProperty("transfer_mode") + @JsonPropertyDescription("Transfer mode") + private Integer transferMode; + + @JsonProperty("is_ignore_error") + @JsonPropertyDescription("Is ignore error") + private Integer ignoreError; +} diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/v3/resp/EsbScriptStepV3DTO.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/v3/resp/EsbScriptStepV3DTO.java new file mode 100644 index 0000000000..8cb0c2dfbb --- /dev/null +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/v3/resp/EsbScriptStepV3DTO.java @@ -0,0 +1,83 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.esb.model.job.v3.resp; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; +import com.tencent.bk.job.common.esb.model.job.v3.EsbAccountV3BasicDTO; +import com.tencent.bk.job.common.esb.model.job.v3.EsbServerV3DTO; +import lombok.Data; + +/** + * @since 17/11/2020 20:37 + */ +@Data +public class EsbScriptStepV3DTO { + @JsonProperty("script_type") + @JsonPropertyDescription("Script type") + private Integer type; + + @JsonProperty("script_id") + @JsonPropertyDescription("Script id") + private String scriptId; + + @JsonProperty("script_version_id") + @JsonPropertyDescription("Script version id") + private Long scriptVersionId; + + @JsonProperty("script_content") + @JsonPropertyDescription("Script content") + private String content; + + @JsonProperty("script_language") + @JsonPropertyDescription("Script language") + private Integer language; + + @JsonProperty("script_param") + @JsonPropertyDescription("Script params") + private String scriptParam; + + @JsonProperty("windows_interpreter") + @JsonPropertyDescription("Windows interpreter") + private String windowsInterpreter; + + @JsonProperty("script_timeout") + @JsonPropertyDescription("Timeout") + private Long scriptTimeout; + + @JsonPropertyDescription("Account") + private EsbAccountV3BasicDTO account; + + @JsonPropertyDescription("Hosts") + private EsbServerV3DTO server; + + @JsonProperty("is_param_sensitive") + @JsonPropertyDescription("Is script params sensitive") + private Integer secureParam; + + @JsonProperty("is_ignore_error") + @JsonPropertyDescription("Is ignore error") + private Integer ignoreError; +} diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/v3/resp/EsbStepV3DTO.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/v3/resp/EsbStepV3DTO.java new file mode 100644 index 0000000000..2c9f4a9913 --- /dev/null +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/v3/resp/EsbStepV3DTO.java @@ -0,0 +1,56 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.esb.model.job.v3.resp; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; +import lombok.Data; + +/** + * @since 17/11/2020 16:41 + */ +@Data +public class EsbStepV3DTO { + @JsonPropertyDescription("Step id") + private Long id; + + @JsonPropertyDescription("Step name") + private String name; + + @JsonPropertyDescription("Step type") + private Integer type; + + @JsonProperty("script_info") + @JsonPropertyDescription("Script step info") + private EsbScriptStepV3DTO scriptInfo; + + @JsonProperty("file_info") + @JsonPropertyDescription("File step info") + private EsbFileStepV3DTO fileInfo; + + @JsonProperty("approval_info") + @JsonPropertyDescription("Approval step info") + private EsbApprovalStepV3DTO approvalInfo; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbUserRoleInfoV3DTO.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/v3/resp/EsbUserRoleInfoV3DTO.java similarity index 84% rename from src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbUserRoleInfoV3DTO.java rename to src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/v3/resp/EsbUserRoleInfoV3DTO.java index 7418ceaa96..c7eb054141 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbUserRoleInfoV3DTO.java +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/model/job/v3/resp/EsbUserRoleInfoV3DTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,8 +22,10 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.manage.model.esb.v3.response; +package com.tencent.bk.job.common.esb.model.job.v3.resp; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; import com.tencent.bk.job.common.model.dto.UserRoleInfoDTO; import lombok.Data; @@ -39,11 +41,15 @@ public class EsbUserRoleInfoV3DTO { /** * 用户名列表 */ + @JsonProperty("user_list") + @JsonPropertyDescription("User list") private List userList; /** * 角色 ID 列表 */ + @JsonProperty("role_list") + @JsonPropertyDescription("Job role list ") private List roleList; public static EsbUserRoleInfoV3DTO fromUserRoleInfo(UserRoleInfoDTO approvalUser) { diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/sdk/AbstractEsbSdkClient.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/sdk/AbstractEsbSdkClient.java deleted file mode 100644 index d364f5e3a9..0000000000 --- a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/sdk/AbstractEsbSdkClient.java +++ /dev/null @@ -1,290 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.esb.sdk; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.tencent.bk.job.common.constant.ErrorCode; -import com.tencent.bk.job.common.esb.model.EsbReq; -import com.tencent.bk.job.common.esb.model.EsbResp; -import com.tencent.bk.job.common.exception.InternalException; -import com.tencent.bk.job.common.util.http.ExtHttpHelper; -import com.tencent.bk.job.common.util.http.HttpHelperFactory; -import com.tencent.bk.job.common.util.json.JsonMapper; -import com.tencent.bk.job.common.util.json.JsonUtils; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.apache.http.Header; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.message.BasicHeader; - -import static com.tencent.bk.job.common.constant.HttpHeader.HDR_BK_LANG; -import static com.tencent.bk.job.common.constant.HttpHeader.HDR_CONTENT_TYPE; - -/** - * 所有ESB-API-SDK的抽象类 注意所有接口都有bkToken和userName参数，两个参数用途需要规范 bkToken用于web系统引发的接口调用 - * 是由当前登录用户触发的行为，一概要求传递bkToken - * userName用于后台任务执行由系统触发，传递userName，此时没有用户登录态也就没有bkToken 如果bkToken不为空，则优先以bkToken为准 - *

- * 构建请求参数请用 - * - * @date 2019/11/11 - * @see AbstractEsbSdkClient makeBaseReqByWeb （当前登录用户触发的行为） - * @see AbstractEsbSdkClient makeBaseReq （由后台任务调用的行为） - */ -@Slf4j -public abstract class AbstractEsbSdkClient { - - private static final JsonMapper JSON_MAPPER = JsonMapper.nonDefaultMapper(); - public static final String BK_LANG_EN = "en"; - private final String esbHostUrl; - private final String appSecret; - private final String appCode; - private final String lang; - private final ExtHttpHelper defaultHttpHelper = HttpHelperFactory.getDefaultHttpHelper(); - /** - * 是否对接ESB测试环境 - */ - private final boolean useEsbTestEnv; - - public AbstractEsbSdkClient(String esbHostUrl, String appCode, String appSecret, - String lang, boolean useEsbTestEnv) { - this.esbHostUrl = esbHostUrl; - this.appCode = appCode; - this.appSecret = appSecret; - if (StringUtils.isNotBlank(lang)) { - this.lang = lang; - } else { - this.lang = BK_LANG_EN; - } - this.useEsbTestEnv = useEsbTestEnv; - } - - - public T makeBaseReqByWeb(Class reqClass, String bkToken) { - return makeBaseReqByWeb(reqClass, bkToken, "", ""); - } - - /** - * 生成通过Web界面发过来的生成SDK调用请求构造协议基础参数 - * - * @param reqClass 要构建返回的协议Req类 - * @param bkToken Cookie - * @param userName 用户id - * @param owner 开发商code-如果没有传入null - * @return 如果指定的协议请求Req类有问题，请返回null - */ - public T makeBaseReqByWeb(Class reqClass, String bkToken, String userName, String owner) { - T esbReq = null; - try { - esbReq = reqClass.newInstance(); - esbReq.setBkToken(bkToken); - esbReq.setUserName(userName); - esbReq.setAppCode(appCode); - esbReq.setAppSecret(appSecret); - if (StringUtils.isEmpty(owner)) { - esbReq.setBkSupplierAccount("0"); - } else { - esbReq.setBkSupplierAccount(owner); - } - } catch (InstantiationException | IllegalAccessException e) { - log.error("makeWebReq fail", e); - } - return esbReq; - } - - /** - * 针对无登录态的SDK调用请求，构建协议请求Req类基础参数 - * - * @param reqClass 要构建返回的协议Req类 - * @param userName 用户id - * @param owner 开发商code-如果没有传入null - * @return 如果指定的协议请求Req类有问题，请返回null - */ - public T makeBaseReq(Class reqClass, String userName, String owner) { - T esbReq = null; - try { - esbReq = reqClass.newInstance(); - esbReq.setUserName(userName); - esbReq.setAppCode(appCode); - esbReq.setAppSecret(appSecret); - if (StringUtils.isEmpty(owner)) { - esbReq.setBkSupplierAccount("0"); - } else { - esbReq.setBkSupplierAccount(owner); - } - } catch (InstantiationException | IllegalAccessException e) { - log.error("makeWebReq fail", e); - } - return esbReq; - } - - public String doHttpGet(String uri, EsbReq params) { - return doHttpGet(uri, params, defaultHttpHelper); - } - - public String doHttpGet(String uri, EsbReq params, ExtHttpHelper httpHelper) { - if (httpHelper == null) { - httpHelper = defaultHttpHelper; - } - boolean error = false; - long start = System.currentTimeMillis(); - String responseBody = null; - String url; - try { - if (!esbHostUrl.endsWith("/") && !uri.startsWith("/")) { - url = esbHostUrl + "/" + uri + params.toUrlParams(); - } else { - url = esbHostUrl + uri + params.toUrlParams(); - } - Header[] header; - if (useEsbTestEnv) { - header = new Header[2]; - header[0] = new BasicHeader(HDR_BK_LANG, lang); - header[1] = new BasicHeader("x-use-test-env", "1"); - } else { - header = new Header[1]; - header[0] = new BasicHeader(HDR_BK_LANG, lang); - } - responseBody = httpHelper.get(url, header); - return responseBody; - } catch (Throwable e) { - log.error("Get url {}| params={}| exception={}", esbHostUrl + uri, - JsonUtils.toJsonWithoutSkippedFields(params), - e.getMessage()); - error = true; - throw e; - } finally { - log.info("Get url {}| error={}| params={}| time={}| resp={}", esbHostUrl + uri, error, - JsonUtils.toJsonWithoutSkippedFields(params), (System.currentTimeMillis() - start), responseBody); - } - } - - protected String doHttpPost( - String uri, - T params, - ExtHttpHelper httpHelper) { - - if (httpHelper == null) { - httpHelper = defaultHttpHelper; - } - boolean error = false; - long start = System.currentTimeMillis(); - String responseBody = null; - try { - String url; - if (!esbHostUrl.endsWith("/") && !uri.startsWith("/")) { - url = esbHostUrl + "/" + uri; - } else { - url = esbHostUrl + uri; - } - Header[] header; - if (useEsbTestEnv) { - header = new Header[3]; - header[0] = new BasicHeader(HDR_BK_LANG, lang); - header[1] = new BasicHeader(HDR_CONTENT_TYPE, "application/json"); - header[2] = new BasicHeader("x-use-test-env", "1"); - } else { - header = new Header[2]; - header[0] = new BasicHeader(HDR_BK_LANG, lang); - header[1] = new BasicHeader(HDR_CONTENT_TYPE, "application/json"); - } - responseBody = httpHelper.post(url, "UTF-8", buildPostBody(params), header); - return responseBody; - } catch (Exception e) { - log.error("Post url {}| params={}| exception={}", uri, JsonUtils.toJsonWithoutSkippedFields(params), - e.getMessage()); - error = true; - throw e; - } finally { - log.info("Post url {}| error={}| params={}| time={}| resp={}", uri, error, - JsonUtils.toJsonWithoutSkippedFields(params), (System.currentTimeMillis() - start), responseBody); - } - } - - protected String buildPostBody(T params) { - return JsonUtils.toJson(params); - } - - - public EsbResp getEsbRespByReq(String method, String uri, EsbReq reqBody, - TypeReference> typeReference) { - return getEsbRespByReq(method, uri, reqBody, typeReference, null); - } - - public EsbResp getEsbRespByReq(String method, String uri, EsbReq reqBody, - TypeReference> typeReference, - ExtHttpHelper httpHelper) { - String reqStr = JsonUtils.toJsonWithoutSkippedFields(reqBody); - String respStr = null; - try { - if (method.equals(HttpGet.METHOD_NAME)) { - respStr = doHttpGet(uri, reqBody, httpHelper); - } else if (method.equals(HttpPost.METHOD_NAME)) { - respStr = doHttpPost(uri, reqBody, httpHelper); - } - if (StringUtils.isBlank(respStr)) { - String errorMsg = method + " " + uri + ", error: " + "Response is blank"; - log.error(errorMsg); - throw new InternalException(errorMsg, ErrorCode.API_ERROR); - } else { - log.debug("success|method={}|uri={}|reqStr={}|respStr={}", method, uri, reqStr, respStr); - } - EsbResp esbResp = JSON_MAPPER.fromJson(respStr, typeReference); - if (esbResp == null) { - String errorMsg = method + " " + uri + ", error: " + "Response is blank after parse"; - log.error(errorMsg); - throw new InternalException(errorMsg, ErrorCode.API_ERROR); - } else if (!esbResp.getResult()) { - log.warn( - "fail:esbResp code!=0|esbResp.requestId={}|esbResp.code={}|esbResp" + - ".message={}|method={}|uri={}|reqStr={}|respStr={}", - esbResp.getRequestId(), - esbResp.getCode(), - esbResp.getMessage(), - method, uri, reqStr, respStr - ); - } - if (esbResp.getData() == null) { - log.warn( - "warn:esbResp.getData() == null|esbResp.requestId={}|esbResp.code={}|esbResp" + - ".message={}|method={}|uri={}|reqStr={}|respStr={}" - , esbResp.getRequestId() - , esbResp.getCode() - , esbResp.getMessage() - , method, uri, reqStr, respStr - ); - } - return esbResp; - } catch (Throwable e) { - String errorMsg = "Fail to request ESB data|method=" + method - + "|uri=" + uri - + "|reqStr=" + reqStr - + "|respStr=" + respStr; - log.error(errorMsg, e); - throw new InternalException("Fail to request esb api", e, ErrorCode.API_ERROR); - } - } -} diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/sdk/BkApiClient.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/sdk/BkApiClient.java new file mode 100644 index 0000000000..15ec1a1bf0 --- /dev/null +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/sdk/BkApiClient.java @@ -0,0 +1,405 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.esb.sdk; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.constant.HttpMethodEnum; +import com.tencent.bk.job.common.constant.JobCommonHeaders; +import com.tencent.bk.job.common.esb.constants.EsbLang; +import com.tencent.bk.job.common.esb.interceptor.LogBkApiRequestIdInterceptor; +import com.tencent.bk.job.common.esb.metrics.EsbMetricTags; +import com.tencent.bk.job.common.esb.model.BkApiAuthorization; +import com.tencent.bk.job.common.esb.model.EsbResp; +import com.tencent.bk.job.common.esb.model.OpenApiRequestInfo; +import com.tencent.bk.job.common.exception.InternalException; +import com.tencent.bk.job.common.util.JobContextUtil; +import com.tencent.bk.job.common.util.http.HttpHelper; +import com.tencent.bk.job.common.util.http.HttpRequest; +import com.tencent.bk.job.common.util.http.HttpResponse; +import com.tencent.bk.job.common.util.json.JsonMapper; +import com.tencent.bk.job.common.util.json.JsonUtils; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tag; +import io.micrometer.core.instrument.Tags; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.http.Header; +import org.apache.http.HttpResponseInterceptor; +import org.apache.http.message.BasicHeader; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import javax.servlet.http.HttpServletRequest; +import java.util.Collection; +import java.util.concurrent.TimeUnit; + +import static com.tencent.bk.job.common.constant.HttpHeader.HDR_BK_LANG; +import static com.tencent.bk.job.common.i18n.locale.LocaleUtils.COMMON_LANG_HEADER; + +/** + * 蓝鲸API（组件 API（ESB）、网关 API（蓝鲸 ApiGateway)）调用客户端 + */ +public class BkApiClient { + + private String lang; + private Logger log = LoggerFactory.getLogger(this.getClass()); + private final String baseAccessUrl; + private final HttpHelper defaultHttpHelper; + private final MeterRegistry meterRegistry; + private static final String BK_API_AUTH_HEADER = "X-Bkapi-Authorization"; + /** + * API调用度量指标名称 + */ + private final String metricName; + private JsonMapper jsonMapper = JsonMapper.nonNullMapper(); + + /** + * @param meterRegistry MeterRegistry + * @param metricName API http 请求指标名称 + * @param baseAccessUrl API 服务访问地址 + * @param defaultHttpHelper http 请求处理客户端 + */ + public BkApiClient(MeterRegistry meterRegistry, + String metricName, + String baseAccessUrl, + HttpHelper defaultHttpHelper) { + this.meterRegistry = meterRegistry; + this.metricName = metricName; + this.baseAccessUrl = baseAccessUrl; + this.defaultHttpHelper = defaultHttpHelper; + } + + public BkApiClient(MeterRegistry meterRegistry, + String metricName, + String baseAccessUrl, + HttpHelper defaultHttpHelper, + String lang) { + this(meterRegistry, metricName, baseAccessUrl, defaultHttpHelper); + this.lang = lang; + } + + /** + * 配置自定义的 JsonMapper, 用于序列化 Json 数据 + * + * @param jsonMapper jsonMapper + */ + public void setJsonMapper(JsonMapper jsonMapper) { + this.jsonMapper = jsonMapper; + } + + /** + * 配置自定义的日志 logger + * + * @param logger logger + */ + public void setLogger(Logger logger) { + this.log = logger; + } + + public EsbResp doRequest(OpenApiRequestInfo requestInfo, + TypeReference> typeReference, + HttpHelper httpHelper) { + return doRequest(requestInfo, typeReference, null, httpHelper); + } + + public EsbResp doRequest(OpenApiRequestInfo requestInfo, + TypeReference> typeReference) { + return doRequest(requestInfo, typeReference, null); + } + + public EsbResp doRequest(OpenApiRequestInfo requestInfo, + TypeReference> typeReference, + BkApiLogStrategy logStrategy, + HttpHelper httpHelper) { + HttpMethodEnum httpMethod = requestInfo.getMethod(); + BkApiContext apiContext = new BkApiContext<>(httpMethod.name(), requestInfo.getUri(), + requestInfo.getBody(), null, null, 0, false); + + if (logStrategy != null) { + logStrategy.logReq(log, apiContext); + } else { + if (log.isInfoEnabled()) { + log.info("[AbstractBkApiClient] Request|method={}|uri={}|reqStr={}", + httpMethod.name(), requestInfo.getUri(), + requestInfo.getBody() != null ? JsonUtils.toJsonWithoutSkippedFields(requestInfo.getBody()) : null); + } + } + + try { + return requestApiAndWrapResponse(requestInfo, apiContext, typeReference, httpHelper); + } finally { + if (logStrategy != null) { + logStrategy.logResp(log, apiContext); + } else { + if (log.isInfoEnabled()) { + log.info("[AbstractBkApiClient] Response|bkApiRequestId={}|method={}|uri={}|success={}" + + "|costTime={}|resp={}", + apiContext.getRequestId(), httpMethod.name(), requestInfo.getUri(), apiContext.isSuccess(), + apiContext.getCostTime(), apiContext.getOriginResp()); + } + } + if (apiContext.getCostTime() > 5000L) { + log.info("SlowBkApiRequest|totalCost={}|requestCost={}|deserializeCost={}" + + "|responseBodyLength={}", + apiContext.getCostTime(), + apiContext.getRequestCostTime(), + apiContext.getDeserializeCostTime(), + apiContext.getOriginResp() != null ? apiContext.getOriginResp().length() : 0L + ); + } + } + } + + public R requestApiAndWrapResponse(OpenApiRequestInfo requestInfo, + TypeReference typeReference, + HttpHelper httpHelper) { + if (log.isInfoEnabled()) { + log.info("[AbstractBkApiClient] Request|method={}|uri={}|reqStr={}", + requestInfo.getMethod().name(), requestInfo.getUri(), + requestInfo.getBody() != null ? JsonUtils.toJsonWithoutSkippedFields(requestInfo.getBody()) : null); + } + String uri = requestInfo.getUri(); + String respStr = null; + String status = EsbMetricTags.VALUE_STATUS_OK; + HttpMethodEnum httpMethod = requestInfo.getMethod(); + long start = System.currentTimeMillis(); + String bkApiRequestId = null; + boolean success = true; + try { + HttpResponse response = requestApi(httpHelper, requestInfo); + bkApiRequestId = extractBkApiRequestId(response); + respStr = response.getEntity(); + if (StringUtils.isBlank(respStr)) { + String errorMsg = "[AbstractBkApiClient] " + httpMethod.name() + " " + + uri + ", error: " + "Response is blank"; + log.warn( + "[AbstractBkApiClient] fail: Response is blank| requestId={}|method={}|uri={}", + bkApiRequestId, + httpMethod.name(), + uri + ); + status = EsbMetricTags.VALUE_STATUS_ERROR; + throw new InternalException(errorMsg, ErrorCode.API_ERROR); + } + return jsonMapper.fromJson(respStr, typeReference); + } catch (Throwable e) { + success = false; + String errorMsg = "Fail to request api|method=" + httpMethod.name() + + "|uri=" + uri; + log.error(errorMsg, e); + status = EsbMetricTags.VALUE_STATUS_ERROR; + throw new InternalException("Fail to request bk api", e, ErrorCode.API_ERROR); + } finally { + long cost = System.currentTimeMillis() - start; + if (meterRegistry != null) { + meterRegistry.timer(metricName, buildMetricTags(uri, status)) + .record(cost, TimeUnit.MILLISECONDS); + } + if (log.isInfoEnabled()) { + log.info("[AbstractBkApiClient] Response|requestId={}|method={}|uri={}|success={}" + + "|costTime={}|resp={}", + bkApiRequestId, httpMethod.name(), requestInfo.getUri(), success, cost, respStr); + } + } + } + + private EsbResp requestApiAndWrapResponse(OpenApiRequestInfo requestInfo, + BkApiContext apiContext, + TypeReference> typeReference, + HttpHelper httpHelper) { + String uri = apiContext.getUri(); + EsbResp esbResp; + String respStr; + String status = EsbMetricTags.VALUE_STATUS_OK; + HttpMethodEnum httpMethod = requestInfo.getMethod(); + long startTimestamp = System.currentTimeMillis(); + long responseTimestamp; + try { + HttpResponse response = requestApi(httpHelper, requestInfo); + responseTimestamp = System.currentTimeMillis(); + apiContext.setRequestCostTime(responseTimestamp - startTimestamp); + apiContext.setRequestId(extractBkApiRequestId(response)); + respStr = response.getEntity(); + apiContext.setOriginResp(response.getEntity()); + + if (StringUtils.isBlank(respStr)) { + String errorMsg = "[AbstractBkApiClient] " + httpMethod.name() + " " + + uri + ", error: " + "Response is blank"; + log.warn("[AbstractBkApiClient] fail: Response is blank| bkApiRequestId={}|method={}|uri={}", + apiContext.getRequestId(), httpMethod.name(), uri); + status = EsbMetricTags.VALUE_STATUS_ERROR; + throw new InternalException(errorMsg, ErrorCode.API_ERROR); + } + + long deserializeStartTimestamp = System.currentTimeMillis(); + esbResp = jsonMapper.fromJson(respStr, typeReference); + apiContext.setDeserializeCostTime(System.currentTimeMillis() - deserializeStartTimestamp); + apiContext.setResp(esbResp); + if (!esbResp.isSuccess()) { + log.warn( + "[AbstractBkApiClient] fail:response code!=0" + + "|bkApiRequestId={}|code={}|message={}|method={}|uri={}|reqStr={}|respStr={}", + apiContext.getRequestId(), + esbResp.getCode(), + esbResp.getMessage(), + httpMethod.name(), + uri, + apiContext.getReq() != null ? JsonUtils.toJsonWithoutSkippedFields(apiContext.getReq()) : null, + respStr + ); + status = EsbMetricTags.VALUE_STATUS_ERROR; + } + if (esbResp.getData() == null) { + log.warn( + "[AbstractBkApiClient] warn: response data is null" + + "|bkApiRequestId={}|code={}|message={}|method={}|uri={}|reqStr={}|respStr={}", + apiContext.getRequestId(), + esbResp.getCode(), + esbResp.getMessage(), + httpMethod.name(), + uri, + apiContext.getReq() != null ? JsonUtils.toJsonWithoutSkippedFields(apiContext.getReq()) : null, + respStr + ); + } + apiContext.setSuccess(true); + return esbResp; + } catch (Throwable e) { + String errorMsg = "Fail to request api|method=" + httpMethod.name() + + "|uri=" + uri; + log.error(errorMsg, e); + apiContext.setSuccess(false); + status = EsbMetricTags.VALUE_STATUS_ERROR; + throw new InternalException("Fail to request bk api", e, ErrorCode.API_ERROR); + } finally { + long cost = System.currentTimeMillis() - startTimestamp; + apiContext.setCostTime(cost); + if (meterRegistry != null) { + meterRegistry.timer(metricName, buildMetricTags(uri, status)) + .record(cost, TimeUnit.MILLISECONDS); + } + } + } + + private String extractBkApiRequestId(HttpResponse response) { + if (response.getHeaders() == null || response.getHeaders().length == 0) { + return ""; + } + for (Header header : response.getHeaders()) { + if (JobCommonHeaders.BK_GATEWAY_REQUEST_ID.equalsIgnoreCase(header.getName())) { + return header.getValue(); + } + } + return ""; + } + + private Iterable buildMetricTags(String uri, String status) { + Tags tags = Tags.of(EsbMetricTags.KEY_API_NAME, uri).and(EsbMetricTags.KEY_STATUS, status); + Collection extraTags = getExtraMetricsTags(); + if (CollectionUtils.isNotEmpty(extraTags)) { + extraTags.forEach(tags::and); + } + return tags; + } + + private HttpResponse requestApi(HttpHelper httpHelper, OpenApiRequestInfo requestInfo) { + String url = buildApiUrl(requestInfo.buildFinalUri()); + + Header[] headers = buildBkApiRequestHeaders(requestInfo.getAuthorization()); + HttpRequest httpRequest = HttpRequest.builder(requestInfo.getMethod(), url) + .setHeaders(headers) + .setKeepAlive(true) + .setRetryMode(requestInfo.getRetryMode()) + .setIdempotent(requestInfo.getIdempotent()) + .setStringEntity(requestInfo.getBody() != null ? jsonMapper.toJson(requestInfo.getBody()) : null) + .build(); + + return chooseHttpHelper(httpHelper).requestForSuccessResp(httpRequest); + } + + private HttpHelper chooseHttpHelper(HttpHelper httpHelper) { + return httpHelper != null ? httpHelper : defaultHttpHelper; + } + + private String buildApiUrl(String uri) { + String url; + if (!baseAccessUrl.endsWith("/") && !uri.startsWith("/")) { + url = baseAccessUrl + "/" + uri; + } else { + url = baseAccessUrl + uri; + } + return url; + } + + private Header[] buildBkApiRequestHeaders(BkApiAuthorization authorization) { + Header[] header = new Header[3]; + header[0] = new BasicHeader("Content-Type", "application/json"); + header[1] = buildBkApiAuthorizationHeader(authorization); + if (StringUtils.isNotEmpty(lang)) { + header[2] = new BasicHeader(HDR_BK_LANG, lang); + } else { + header[2] = new BasicHeader(HDR_BK_LANG, getLangFromRequest()); + } + return header; + } + + private Header buildBkApiAuthorizationHeader(BkApiAuthorization authorization) { + return new BasicHeader(BK_API_AUTH_HEADER, jsonMapper.toJson(authorization)); + } + + private String getLangFromRequest() { + try { + HttpServletRequest request = JobContextUtil.getRequest(); + String lang = null; + if (request != null) { + lang = request.getHeader(COMMON_LANG_HEADER); + } + + return StringUtils.isEmpty(lang) ? EsbLang.EN : lang; + } catch (Throwable ignore) { + return EsbLang.EN; + } + } + + /** + * 额外的监控指标 tag + * + * @return tag + */ + protected Collection getExtraMetricsTags() { + return null; + } + + /** + * 获取打印APIGW RequestId的响应拦截器 + * + * @return 响应拦截器 + */ + protected static HttpResponseInterceptor getLogBkApiRequestIdInterceptor() { + return new LogBkApiRequestIdInterceptor(); + } + +} diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/sdk/BkApiContext.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/sdk/BkApiContext.java new file mode 100644 index 0000000000..ff9a833d85 --- /dev/null +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/sdk/BkApiContext.java @@ -0,0 +1,90 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.esb.sdk; + +import com.tencent.bk.job.common.esb.model.EsbResp; +import lombok.Data; + +/** + * BK Open API 调用上下文 + * + * @param 请求 + * @param 响应对象 + */ +@Data +public class BkApiContext { + + /** + * HTTP 请求方法 + */ + private String method; + private String uri; + private T req; + /** + * 原始的 API 响应 + */ + private String originResp; + /** + * 反序列化之后的 API 响应 + */ + private EsbResp resp; + /** + * API 调用耗时 + */ + private long costTime; + + /** + * API 响应耗时 + */ + private long requestCostTime; + /** + * 响应数据反序列化耗时 + */ + private long deserializeCostTime; + /** + * API 是否调用成功并正确响应 + */ + private boolean success; + /** + * 网关 request_id + */ + private String requestId; + + public BkApiContext(String method, + String uri, + T req, + String originResp, + EsbResp resp, + long costTime, + boolean success) { + this.method = method; + this.uri = uri; + this.req = req; + this.originResp = originResp; + this.resp = resp; + this.costTime = costTime; + this.success = success; + } +} diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/sdk/BkApiLogStrategy.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/sdk/BkApiLogStrategy.java new file mode 100644 index 0000000000..bdf3e98e05 --- /dev/null +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/sdk/BkApiLogStrategy.java @@ -0,0 +1,64 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.esb.sdk; + +import com.tencent.bk.job.common.util.json.JsonUtils; +import org.slf4j.Logger; + +/** + * 自定义 BK API 调用日志输出策略 + */ +public interface BkApiLogStrategy { + /** + * 打印请求 + * + * @param log logger + * @param context BK Open API 调用上下文 + * @param 请求 + * @param 响应对象 + */ + default void logReq(Logger log, BkApiContext context) { + if (log.isInfoEnabled()) { + log.info("Request|method={}|uri={}|reqStr={}", context.getMethod(), + context.getUri(), JsonUtils.toJsonWithoutSkippedFields(context.getReq())); + } + } + + /** + * 打印响应 + * + * @param log logger + * @param context BK Open API 调用上下文 + * @param 请求 + * @param 响应对象 + */ + default void logResp(Logger log, BkApiContext context) { + if (log.isInfoEnabled()) { + log.info("Response|method={}|uri={}|success={}|costTime={}|resp={}|", + context.getMethod(), context.getUri(), context.isSuccess(), + context.getCostTime(), context.getOriginResp()); + } + } +} diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/util/CookieUtil.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/util/CookieUtil.java index 49566fc33d..841e1250b5 100644 --- a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/util/CookieUtil.java +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/util/CookieUtil.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/util/EsbDTOAppScopeMappingHelper.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/util/EsbDTOAppScopeMappingHelper.java index 811aafd7f9..36cc2bc4ff 100644 --- a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/util/EsbDTOAppScopeMappingHelper.java +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/util/EsbDTOAppScopeMappingHelper.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,7 +28,9 @@ import com.tencent.bk.job.common.model.dto.ResourceScope; import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.common.util.ApplicationContextRegister; -import com.tencent.bk.job.common.util.feature.FeatureToggle; +import com.tencent.bk.job.common.util.toggle.ToggleEvaluateContext; +import com.tencent.bk.job.common.util.toggle.feature.FeatureIdConstants; +import com.tencent.bk.job.common.util.toggle.feature.FeatureToggle; /** * ESB 业务与资源范围转换工具类 @@ -50,7 +52,8 @@ public static void fillEsbAppScopeDTOByAppId(Long appId, EsbAppScopeDTO esbAppSc esbAppScopeDTO.setScopeType(resourceScope.getType().getValue()); esbAppScopeDTO.setScopeId(resourceScope.getId()); // 如果不兼容bk_biz_id，那么使用bk_scope_type+bk_scope_id参数校验方式 - if (FeatureToggle.isBkBizIdEnabled()) { + if (FeatureToggle.checkFeature(FeatureIdConstants.FEATURE_BK_BIZ_ID_COMPATIBLE, + ToggleEvaluateContext.EMPTY)) { esbAppScopeDTO.setBizId(Long.valueOf(resourceScope.getId())); } } diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/validate/CheckServer.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/validate/CheckServer.java index 4e4ef74aa1..9fa426ec3e 100644 --- a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/validate/CheckServer.java +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/validate/CheckServer.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/validate/CheckServerValidator.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/validate/CheckServerValidator.java index ed142e0888..a0fe32b635 100644 --- a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/validate/CheckServerValidator.java +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/validate/CheckServerValidator.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/validate/EsbAppScopeReqGroupSequenceProvider.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/validate/EsbAppScopeReqGroupSequenceProvider.java index bbc656d8f3..c4a1764a96 100644 --- a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/validate/EsbAppScopeReqGroupSequenceProvider.java +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/validate/EsbAppScopeReqGroupSequenceProvider.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,7 +25,9 @@ package com.tencent.bk.job.common.esb.validate; import com.tencent.bk.job.common.esb.model.EsbAppScopeReq; -import com.tencent.bk.job.common.util.feature.FeatureToggle; +import com.tencent.bk.job.common.util.toggle.ToggleEvaluateContext; +import com.tencent.bk.job.common.util.toggle.feature.FeatureIdConstants; +import com.tencent.bk.job.common.util.toggle.feature.FeatureToggle; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.hibernate.validator.spi.group.DefaultGroupSequenceProvider; @@ -45,7 +47,8 @@ public List> getValidationGroups(EsbAppScopeReq req) { validationGroups.add(EsbAppScopeReq.class); if (req != null) { // 如果不兼容bk_biz_id，那么使用bk_scope_type+bk_scope_id参数校验方式 - if (!FeatureToggle.isBkBizIdEnabled()) { + if (!FeatureToggle.checkFeature(FeatureIdConstants.FEATURE_BK_BIZ_ID_COMPATIBLE, + ToggleEvaluateContext.EMPTY)) { validationGroups.add(EsbAppScopeReq.UseScopeParam.class); return validationGroups; } diff --git a/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/validate/ValidBkScope.java b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/validate/ValidBkScope.java new file mode 100644 index 0000000000..26e88dff96 --- /dev/null +++ b/src/backend/commons/esb-sdk/src/main/java/com/tencent/bk/job/common/esb/validate/ValidBkScope.java @@ -0,0 +1,87 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.esb.validate; + +import com.tencent.bk.job.common.constant.ResourceScopeTypeEnum; +import com.tencent.bk.job.common.esb.model.EsbAppScopeReq; +import org.apache.commons.lang3.StringUtils; + +import javax.validation.Constraint; +import javax.validation.ConstraintValidator; +import javax.validation.ConstraintValidatorContext; +import javax.validation.Payload; +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import static java.lang.annotation.ElementType.ANNOTATION_TYPE; +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.ElementType.TYPE_USE; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +/** + * 资源范围不可为空校验 + */ +@Target({FIELD, METHOD, PARAMETER, ANNOTATION_TYPE, TYPE_USE}) +@Constraint(validatedBy = ValidBkScope.Validator.class) +@Documented +@Retention(RUNTIME) +public @interface ValidBkScope { + + String message() default "资源范围相关参数bk_scope_type或bk_scope_id不正确"; + + Class[] groups() default {}; + + Class[] payload() default {}; + + class Validator implements ConstraintValidator { + + @Override + public boolean isValid(EsbAppScopeReq appScopeReq, ConstraintValidatorContext hibernateContext) { + if (appScopeReq.getBizId() != null) { + return true; + } + if (!ResourceScopeTypeEnum.isValid(appScopeReq.getScopeType())) { + hibernateContext.disableDefaultConstraintViolation(); + hibernateContext + .buildConstraintViolationWithTemplate("{validation.constraints.InvalidBkScopeType.message}") + .addPropertyNode("scopeType") + .addConstraintViolation(); + return false; + } + if (StringUtils.isBlank(appScopeReq.getScopeId())) { + hibernateContext.disableDefaultConstraintViolation(); + hibernateContext + .buildConstraintViolationWithTemplate("{validation.constraints.InvalidBkScopeId.message}") + .addPropertyNode("scopeId") + .addConstraintViolation(); + return false; + } + return true; + } + } +} diff --git a/src/backend/commons/esb-sdk/src/main/resources/META-INF/spring.factories b/src/backend/commons/esb-sdk/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..3e63b4a08a --- /dev/null +++ b/src/backend/commons/esb-sdk/src/main/resources/META-INF/spring.factories @@ -0,0 +1,2 @@ +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ +com.tencent.bk.job.common.esb.config.BkApiAutoConfiguration diff --git a/src/backend/commons/esb-sdk/src/test/java/com/tencent/bk/job/common/esb/EsbRespTest.java b/src/backend/commons/esb-sdk/src/test/java/com/tencent/bk/job/common/esb/EsbRespTest.java index fd76ba89d7..e15e7088bf 100644 --- a/src/backend/commons/esb-sdk/src/test/java/com/tencent/bk/job/common/esb/EsbRespTest.java +++ b/src/backend/commons/esb-sdk/src/test/java/com/tencent/bk/job/common/esb/EsbRespTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/gse-sdk/build.gradle b/src/backend/commons/gse-sdk/build.gradle index 39859bd8d8..a134547ed5 100644 --- a/src/backend/commons/gse-sdk/build.gradle +++ b/src/backend/commons/gse-sdk/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,9 @@ dependencies { api project(':commons:common') - api files('libs/ee-gse-task-3.1.jar') + api project(':commons:common-crypto') + api project(':commons:esb-sdk') + api files('libs/ee-gse-task-3.2.jar') implementation 'com.fasterxml.jackson.core:jackson-core' implementation 'com.fasterxml.jackson.core:jackson-databind' implementation 'com.fasterxml.jackson.core:jackson-annotations' @@ -34,6 +36,11 @@ dependencies { implementation 'org.apache.commons:commons-collections4' implementation 'org.apache.httpcomponents:httpclient' implementation 'io.micrometer:micrometer-registry-prometheus' + // https://mvnrepository.com/artifact/org.apache.curator/curator-framework + implementation(group: 'org.apache.curator', name: 'curator-framework', version: '5.1.0') + // https://mvnrepository.com/artifact/org.apache.curator/curator-recipes + implementation(group: 'org.apache.curator', name: 'curator-recipes', version: '5.1.0') + implementation("org.springframework.boot:spring-boot-autoconfigure") compileOnly 'org.projectlombok:lombok' annotationProcessor 'org.projectlombok:lombok' testImplementation 'org.junit.jupiter:junit-jupiter' diff --git a/src/backend/commons/gse-sdk/libs/ee-gse-task-3.1.jar b/src/backend/commons/gse-sdk/libs/ee-gse-task-3.1.jar deleted file mode 100644 index dbada9e403..0000000000 Binary files a/src/backend/commons/gse-sdk/libs/ee-gse-task-3.1.jar and /dev/null differ diff --git a/src/backend/commons/gse-sdk/libs/ee-gse-task-3.2.jar b/src/backend/commons/gse-sdk/libs/ee-gse-task-3.2.jar new file mode 100644 index 0000000000..8393a67ca9 Binary files /dev/null and b/src/backend/commons/gse-sdk/libs/ee-gse-task-3.2.jar differ diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/GseClient.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/GseClient.java new file mode 100644 index 0000000000..ef3694928f --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/GseClient.java @@ -0,0 +1,143 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.gse; + +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.InternalException; +import com.tencent.bk.job.common.gse.util.AgentUtils; +import com.tencent.bk.job.common.gse.v1.GseV1ApiClient; +import com.tencent.bk.job.common.gse.v2.GseV2ApiClient; +import com.tencent.bk.job.common.gse.v2.model.ExecuteScriptRequest; +import com.tencent.bk.job.common.gse.v2.model.FileTaskResult; +import com.tencent.bk.job.common.gse.v2.model.GetExecuteScriptResultRequest; +import com.tencent.bk.job.common.gse.v2.model.GetTransferFileResultRequest; +import com.tencent.bk.job.common.gse.v2.model.GseReq; +import com.tencent.bk.job.common.gse.v2.model.GseTaskResponse; +import com.tencent.bk.job.common.gse.v2.model.ScriptTaskResult; +import com.tencent.bk.job.common.gse.v2.model.TerminateGseTaskRequest; +import com.tencent.bk.job.common.gse.v2.model.TransferFileRequest; +import com.tencent.bk.job.common.gse.v2.model.req.ListAgentStateReq; +import com.tencent.bk.job.common.gse.v2.model.resp.AgentState; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.util.CollectionUtils; +import org.springframework.util.StopWatch; + +import java.util.Collections; +import java.util.List; + + +@Slf4j +public class GseClient implements IGseClient { + + private final GseV1ApiClient gseV1ApiClient; + private final GseV2ApiClient gseV2ApiClient; + + + public GseClient(GseV1ApiClient gseV1ApiClient, + GseV2ApiClient gseV2ApiClient) { + this.gseV1ApiClient = gseV1ApiClient; + this.gseV2ApiClient = gseV2ApiClient; + log.info("Init gseClient, gseV1ApiClient: {}, gseV2ApiClient: {}", gseV1ApiClient, gseV2ApiClient); + } + + + @Override + public GseTaskResponse asyncExecuteScript(ExecuteScriptRequest request) { + return chooseGseApiClientByGseTaskVersion(request).asyncExecuteScript(request); + } + + private IGseClient chooseGseApiClientByGseTaskVersion(GseReq gseReq) { + if (gseReq.isGseV2Task()) { + return gseV2ApiClient; + } else { + return gseV1ApiClient; + } + } + + @Override + public ScriptTaskResult getExecuteScriptResult(GetExecuteScriptResultRequest request) { + return chooseGseApiClientByGseTaskVersion(request).getExecuteScriptResult(request); + } + + @Override + public List listAgentState(ListAgentStateReq req) { + StopWatch watch = new StopWatch("listAgentState"); + List agentIdList = req.getAgentIdList(); + if (CollectionUtils.isEmpty(agentIdList)) { + log.info("agentIdList is empty"); + return Collections.emptyList(); + } + String firstAgentId = agentIdList.get(0); + + watch.start("chooseGseApiClientByAgentId"); + IGseClient gseClient = chooseGseApiClientByAgentId(firstAgentId); + watch.stop(); + + watch.start("gseClient.listAgentState"); + List resultList = gseClient.listAgentState(req); + watch.stop(); + + if (watch.getTotalTimeMillis() > 3000) { + log.warn("listAgentState slow, statistics: " + watch.prettyPrint()); + } + + return resultList; + } + + private IGseClient chooseGseApiClientByAgentId(String agentId) { + if (StringUtils.isEmpty(agentId)) { + log.error("Empty agentId!"); + throw new InternalException("AgentId is empty", ErrorCode.INTERNAL_ERROR); + } + if (AgentUtils.isGseV1AgentId(agentId)) { + log.debug("Choose GseV1ApiClient, agentId: {}", agentId); + return gseV1ApiClient; + } else { + log.debug("Choose GseV2ApiClient, agentId: {}", agentId); + return gseV2ApiClient; + } + } + + @Override + public GseTaskResponse asyncTransferFile(TransferFileRequest request) { + return chooseGseApiClientByGseTaskVersion(request).asyncTransferFile(request); + } + + @Override + public FileTaskResult getTransferFileResult(GetTransferFileResultRequest request) { + return chooseGseApiClientByGseTaskVersion(request).getTransferFileResult(request); + } + + @Override + public GseTaskResponse terminateGseFileTask(TerminateGseTaskRequest request) { + return chooseGseApiClientByGseTaskVersion(request).terminateGseFileTask(request); + } + + @Override + public GseTaskResponse terminateGseScriptTask(TerminateGseTaskRequest request) { + return chooseGseApiClientByGseTaskVersion(request).terminateGseScriptTask(request); + } +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/IGseClient.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/IGseClient.java new file mode 100644 index 0000000000..3a7ec74429 --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/IGseClient.java @@ -0,0 +1,122 @@ +package com.tencent.bk.job.common.gse; + +import com.tencent.bk.job.common.gse.v2.model.Agent; +import com.tencent.bk.job.common.gse.v2.model.ExecuteScriptRequest; +import com.tencent.bk.job.common.gse.v2.model.FileTaskResult; +import com.tencent.bk.job.common.gse.v2.model.GetExecuteScriptResultRequest; +import com.tencent.bk.job.common.gse.v2.model.GetTransferFileResultRequest; +import com.tencent.bk.job.common.gse.v2.model.GseTaskResponse; +import com.tencent.bk.job.common.gse.v2.model.ScriptTaskResult; +import com.tencent.bk.job.common.gse.v2.model.TerminateGseTaskRequest; +import com.tencent.bk.job.common.gse.v2.model.TransferFileRequest; +import com.tencent.bk.job.common.gse.v2.model.req.ListAgentStateReq; +import com.tencent.bk.job.common.gse.v2.model.resp.AgentState; + +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +/** + * GSE API 客户端 + */ +public interface IGseClient { + + /** + * 执行脚本 + * + * @param request 执行脚本请求 + * @return 下发任务响应 + */ + GseTaskResponse asyncExecuteScript(ExecuteScriptRequest request); + + /** + * 获取 GSE 脚本任务结果 + * + * @param request 查询请求 + */ + ScriptTaskResult getExecuteScriptResult(GetExecuteScriptResultRequest request); + + /** + * 批量获取Agent状态 + * + * @param req 请求体 + * @return Agent状态列表 + */ + List listAgentState(ListAgentStateReq req); + + /** + * 批量构建目标Agent + * + * @param agentIds agentId列表 + * @param user 用户 + * @param password 密码 + * @return Agent + */ + default List buildAgents(Collection agentIds, String user, String password) { + return agentIds.stream() + .map(agentId -> buildAgent(agentId, user, password)) + .collect(Collectors.toList()); + } + + /** + * 设置 Agent 认证信息 + * + * @param agents agent列表 + * @param user 用户 + * @param password 密码 + * @return Agent + */ + default List fillAgentAuthInfo(Collection agents, String user, String password) { + return agents.stream().peek(agent -> { + agent.setUser(user); + agent.setPwd(password); + }).collect(Collectors.toList()); + } + + /** + * 构建目标Agent + * + * @param agentId agentId + * @param user 用户 + * @param password 密码 + * @return Agent + */ + default Agent buildAgent(String agentId, String user, String password) { + Agent agent = new Agent(); + agent.setAgentId(agentId); + agent.setUser(user); + agent.setPwd(password); + return agent; + } + + /** + * 分发文件 + * + * @param request 分发文件请求 + * @return GSE 下发任务响应 + */ + GseTaskResponse asyncTransferFile(TransferFileRequest request); + + /** + * 文件任务结果查询 + * + * @param request 文件任务结果查询请求 + */ + FileTaskResult getTransferFileResult(GetTransferFileResultRequest request); + + /** + * 终止GSE文件任务 + * + * @param request 终止GSE文件任务请求 + */ + GseTaskResponse terminateGseFileTask(TerminateGseTaskRequest request); + + /** + * 终止GSE脚本任务 + * + * @param request 终止GSE脚本任务请求 + */ + GseTaskResponse terminateGseScriptTask(TerminateGseTaskRequest request); + + +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/config/AgentStateQueryConfig.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/config/AgentStateQueryConfig.java new file mode 100644 index 0000000000..51d947567d --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/config/AgentStateQueryConfig.java @@ -0,0 +1,49 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.gse.config; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Configuration; + +/** + * GSE Agent状态查询配置 + */ +@Configuration +@Getter +@Setter +public class AgentStateQueryConfig { + + @Value("${gse.query.threads.num:5}") + private int gseQueryThreadsNum; + + @Value("${gse.query.threads.maxNum:20}") + private int gseQueryThreadsMaxNum; + + @Value("${gse.query.batchSize:5000}") + private int gseQueryBatchSize; + +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/config/GseAutoConfiguration.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/config/GseAutoConfiguration.java new file mode 100644 index 0000000000..402d1f109c --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/config/GseAutoConfiguration.java @@ -0,0 +1,107 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.gse.config; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import com.tencent.bk.job.common.WatchableThreadPoolExecutor; +import com.tencent.bk.job.common.crypto.Encryptor; +import com.tencent.bk.job.common.crypto.RSAEncryptor; +import com.tencent.bk.job.common.gse.GseClient; +import com.tencent.bk.job.common.gse.constants.DefaultBeanNames; +import com.tencent.bk.job.common.gse.constants.GseConstants; +import com.tencent.bk.job.common.gse.service.AgentStateClient; +import com.tencent.bk.job.common.gse.service.PreferV2AgentStateClientImpl; +import com.tencent.bk.job.common.gse.v1.GseV1ApiClient; +import com.tencent.bk.job.common.gse.v1.config.GseV1AutoConfiguration; +import com.tencent.bk.job.common.gse.v2.GseV2ApiClient; +import com.tencent.bk.job.common.gse.v2.GseV2AutoConfiguration; +import io.micrometer.core.instrument.MeterRegistry; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; + +import java.io.IOException; +import java.security.GeneralSecurityException; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadFactory; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +@Slf4j +@Configuration(proxyBeanMethods = false) +@Import( + { + AgentStateQueryConfig.class, + GseV1AutoConfiguration.class, + GseV2AutoConfiguration.class + } +) +public class GseAutoConfiguration { + + @Bean("GseApiClient") + public GseClient gseClient(ObjectProvider gseV1ApiClient, + ObjectProvider gseV2ApiClient) { + return new GseClient(gseV1ApiClient.getIfAvailable(), + gseV2ApiClient.getIfAvailable()); + } + + @ConditionalOnMissingBean(name = DefaultBeanNames.AGENT_STATUS_QUERY_THREAD_POOL_EXECUTOR) + @Bean(DefaultBeanNames.AGENT_STATUS_QUERY_THREAD_POOL_EXECUTOR) + public ThreadPoolExecutor agentStatusQueryExecutor(MeterRegistry meterRegistry, + AgentStateQueryConfig agentStateQueryConfig) { + ThreadFactory threadFactory = new ThreadFactoryBuilder().setNameFormat("agentStatusQuery-thread-%d").build(); + return new WatchableThreadPoolExecutor( + meterRegistry, + "agentStatusQueryExecutor", + agentStateQueryConfig.getGseQueryThreadsNum(), + agentStateQueryConfig.getGseQueryThreadsMaxNum(), + 60, + TimeUnit.SECONDS, + new LinkedBlockingQueue<>(agentStateQueryConfig.getGseQueryThreadsNum()), + threadFactory, + (r, executor) -> { + log.warn("agentStatusQueryExecutor busy, use current thread to query"); + r.run(); + } + ); + } + + @Bean(DefaultBeanNames.PREFER_V2_AGENT_STATE_CLIENT) + public AgentStateClient preferV2AgentStateClientImpl(AgentStateQueryConfig agentStateQueryConfig, + GseClient gseClient, + @Qualifier("agentStatusQueryExecutor") + ThreadPoolExecutor threadPoolExecutor) { + return new PreferV2AgentStateClientImpl(agentStateQueryConfig, gseClient, threadPoolExecutor); + } + + @Bean("gseRsaEncryptor") + public Encryptor rsaEncryptor() throws IOException, GeneralSecurityException { + return new RSAEncryptor(GseConstants.publicKeyPermBase64); + } +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/config/GseConfig.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/config/GseConfig.java deleted file mode 100644 index e7c0393db1..0000000000 --- a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/config/GseConfig.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.gse.config; - -import lombok.Getter; -import lombok.Setter; - -/** - * @since 12/11/2019 11:10 - */ -@Getter -@Setter -public class GseConfig { - /** - * GSE task server host - */ - private String[] gseTaskServerHost; - - /** - * GSE task server port - */ - private int gseTaskServerPort = 48673; - - /** - * GSE cache api service host - */ - private String[] gseCacheApiServerHost; - - /** - * GSE cache api service port - */ - private int gseCacheApiServerPort = 59313; - - /** - * 是否使用ssl - */ - private boolean enableSsl = true; - - - private String keyStore; - - private String keyStorePass; - - - private String trustStore; - - private String trustStorePass; - - - private String trustManagerType = "SunX509"; - - private String trustStoreType = "JKS"; - - private int queryThreadsNum = 5; - - private int queryBatchSize = 5000; - -} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/config/GseMetricsAutoConfiguration.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/config/GseMetricsAutoConfiguration.java new file mode 100644 index 0000000000..16243313c8 --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/config/GseMetricsAutoConfiguration.java @@ -0,0 +1,57 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.gse.config; + +import com.tencent.bk.job.common.gse.constants.GseMetricNames; +import io.micrometer.core.instrument.Meter; +import io.micrometer.core.instrument.config.MeterFilter; +import io.micrometer.core.instrument.distribution.DistributionStatisticConfig; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * GSE 监控指标配置 + */ +@Configuration +public class GseMetricsAutoConfiguration { + @Bean("gseApiMeterFilter") + public MeterFilter distributionMeterFilter() { + return new MeterFilter() { + @Override + public DistributionStatisticConfig configure(Meter.Id id, DistributionStatisticConfig config) { + String metricName = id.getName(); + if (metricName.startsWith(GseMetricNames.GSE_API_METRICS_NAME_PREFIX) + || metricName.startsWith(GseMetricNames.GSE_V2_API_METRICS_NAME_PREFIX)) { + return DistributionStatisticConfig.builder().percentilesHistogram(true) + // [10ms,1s] + .minimumExpectedValue(10_000_000.0).maximumExpectedValue(1_000_000_000.0) + .build().merge(config); + } else { + return config; + } + } + }; + } +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/config/GseV1Properties.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/config/GseV1Properties.java new file mode 100644 index 0000000000..fc1a0c437f --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/config/GseV1Properties.java @@ -0,0 +1,158 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.gse.config; + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.springframework.boot.context.properties.ConfigurationProperties; + +/** + * GSE 配置 + */ +@ConfigurationProperties(prefix = "gse") +@Getter +@Setter +public class GseV1Properties { + + private TaskServer taskServer = new TaskServer(); + + private CacheApiServer cache = new CacheApiServer(); + + private Ssl ssl = new Ssl(); + + private Server server = new Server(); + + /** + * 对接的GSE 版本(v1/v2) + */ + private String version = "v2"; + + /** + * GSE Task Server properties + */ + @Getter + @Setter + @ToString + public static class TaskServer { + private String host; + private Integer port = 48673; + } + + /** + * GSE Cache API Server properties + */ + @Getter + @Setter + @ToString + public static class CacheApiServer { + private ApiServer apiServer = new ApiServer(); + + @Getter + @Setter + @ToString + public static class ApiServer { + private String host; + private Integer port = 59313; + } + } + + /** + * GSE SSL properties + */ + @Getter + @Setter + @ToString + public static class Ssl { + private KeyStore keyStore = new KeyStore(); + private TrustStore trustStore = new TrustStore(); + + @Getter + @Setter + @ToString + public static class KeyStore { + private String path; + private String password; + } + + @Getter + @Setter + @ToString + public static class TrustStore { + private String path; + private String password; + private String managerType = "SunX509"; + private String storeType = "JKS"; + } + } + + /** + * Gse Server connection properties + */ + @Getter + @Setter + @ToString + public static class Server { + private ZooKeeper zooKeeper; + private Discovery discovery; + + @Getter + @Setter + @ToString + public static class ZooKeeper { + private ZooKeeperConnect connect = new ZooKeeperConnect(); + private String path = "/gse/config/server/task/accessv3"; + private ZooKeeperTimeout timeout = new ZooKeeperTimeout(); + + /** + * ZooKeeper timeout + */ + @Getter + @Setter + @ToString + public static class ZooKeeperTimeout { + private Integer session = 60000; + private Integer connect = 60000; + } + + /** + * ZooKeeper connect + */ + @Getter + @Setter + @ToString + public static class ZooKeeperConnect { + private String string; + } + } + + @Getter + @Setter + @ToString + public static class Discovery { + private String type; + } + } +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/config/GseV2Properties.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/config/GseV2Properties.java new file mode 100644 index 0000000000..eb78960a35 --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/config/GseV2Properties.java @@ -0,0 +1,41 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.gse.config; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; + +/** + * GSE 配置 + */ +@ConfigurationProperties(prefix = "gse-v2") +@Getter +@Setter +public class GseV2Properties { + + private boolean enabled = true; + +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/constants/AgentAliveStatusEnum.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/constants/AgentAliveStatusEnum.java new file mode 100644 index 0000000000..78b32f609f --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/constants/AgentAliveStatusEnum.java @@ -0,0 +1,77 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.gse.constants; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonValue; +import com.tencent.bk.job.common.gse.v2.model.resp.AgentState; + +/** + * Agent 正常状态 + */ +public enum AgentAliveStatusEnum { + /** + * 异常 + */ + NOT_ALIVE(0), + /** + * 正常 + */ + ALIVE(1); + @JsonValue + private final int status; + + AgentAliveStatusEnum(int status) { + this.status = status; + } + + @JsonCreator(mode = JsonCreator.Mode.DELEGATING) + public static AgentAliveStatusEnum valOf(int status) { + for (AgentAliveStatusEnum agentAliveStatus : values()) { + if (agentAliveStatus.status == status) { + return agentAliveStatus; + } + } + return null; + } + + public static AgentAliveStatusEnum fromAgentState(AgentState agentState) { + if (agentState == null) { + return NOT_ALIVE; + } + Integer statusCode = agentState.getStatusCode(); + if (statusCode == null) { + return NOT_ALIVE; + } + if (!statusCode.equals(AgentStateStatusEnum.RUNNING.getValue())) { + return NOT_ALIVE; + } + return ALIVE; + } + + public int getStatusValue() { + return status; + } +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/constants/AgentStateStatusEnum.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/constants/AgentStateStatusEnum.java new file mode 100644 index 0000000000..e45d1b5d82 --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/constants/AgentStateStatusEnum.java @@ -0,0 +1,94 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.gse.constants; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonValue; + +/** + * Agent 状态枚举，-2:未找到 -1:查询失败 0:初始安装 1:启动中 2:运行中 3:有损状态 4:繁忙状态 5:升级中 6:停止中 7:解除安装 + */ +public enum AgentStateStatusEnum { + /** + * 未找到 + */ + NOT_FOUND(-2), + /** + * 查询失败 + */ + QUERY_FAIL(-1), + /** + * 初始安装 + */ + INIT_INSTALL(0), + /** + * 启动中 + */ + BOOTING(1), + /** + * 运行中 + */ + RUNNING(2), + /** + * 有损状态 + */ + IN_TROUBLE(3), + /** + * 繁忙状态 + */ + BUSY(4), + /** + * 升级中 + */ + UPGRADING(5), + /** + * 停止中 + */ + STOPPING(6), + /** + * 解除安装 + */ + REMOVING(7); + @JsonValue + private final int status; + + AgentStateStatusEnum(int status) { + this.status = status; + } + + @JsonCreator(mode = JsonCreator.Mode.DELEGATING) + public static AgentStateStatusEnum valOf(int status) { + for (AgentStateStatusEnum agentStatus : values()) { + if (agentStatus.status == status) { + return agentStatus; + } + } + return null; + } + + public int getValue() { + return status; + } +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/constants/AgentStatusEnum.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/constants/AgentStatusEnum.java deleted file mode 100644 index 169744de9a..0000000000 --- a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/constants/AgentStatusEnum.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.gse.constants; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonValue; - -/** - * Agent 状态 - */ -public enum AgentStatusEnum { - /** - * 异常 - */ - NOT_ALIVE(0), - /** - * 正常 - */ - ALIVE(1), - /** - * 未知 - */ - UNKNOWN(2); - @JsonValue - private final int status; - - AgentStatusEnum(int status) { - this.status = status; - } - - @JsonCreator(mode = JsonCreator.Mode.DELEGATING) - public static AgentStatusEnum valOf(int status) { - for (AgentStatusEnum agentStatus : values()) { - if (agentStatus.status == status) { - return agentStatus; - } - } - return null; - } - - public int getValue() { - return status; - } -} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/constants/DefaultBeanNames.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/constants/DefaultBeanNames.java new file mode 100644 index 0000000000..c9794e941b --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/constants/DefaultBeanNames.java @@ -0,0 +1,10 @@ +package com.tencent.bk.job.common.gse.constants; + +public class DefaultBeanNames { + // Agent状态查询线程池Bean名称 + public static final String AGENT_STATUS_QUERY_THREAD_POOL_EXECUTOR = "agentStatusQueryExecutor"; + // 优先使用GSE V2的Agent状态查询Bean名称 + public static final String PREFER_V2_AGENT_STATE_CLIENT = "PreferV2AgentStateClient"; + // 基于特性配置决定是否使用GSE V2的Agent状态查询Bean名称 + public static final String USE_V2_BY_FEATURE_AGENT_STATE_CLIENT = "UseV2ByFeatureAgentStateClient"; +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/FileDistModeEnum.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/constants/FileDistModeEnum.java similarity index 94% rename from src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/FileDistModeEnum.java rename to src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/constants/FileDistModeEnum.java index 69b38c3f71..c60fc26874 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/FileDistModeEnum.java +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/constants/FileDistModeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.execute.common.constants; +package com.tencent.bk.job.common.gse.constants; /** * 文件分发模式 diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/constants/FileTaskTypeEnum.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/constants/FileTaskTypeEnum.java new file mode 100644 index 0000000000..bd6aac3f8c --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/constants/FileTaskTypeEnum.java @@ -0,0 +1,45 @@ +package com.tencent.bk.job.common.gse.constants; + +/** + * GSE 文件任务类型 + */ +public enum FileTaskTypeEnum { + /** + * 文件分发 + */ + FILE(1), + /** + * 目录分发 + */ + DIR(2), + /** + * 按照正则表达式分发源文件 + */ + REGEX(3), + /** + * 按照通配符分发源文件 + */ + WILDCARD(4); + + private final int value; + + FileTaskTypeEnum(int taskType) { + this.value = taskType; + } + + public static FileTaskTypeEnum valueOf(Integer taskType) { + if (taskType == null) { + return null; + } + for (FileTaskTypeEnum inst : values()) { + if (inst.value == taskType) { + return inst; + } + } + return null; + } + + public final int getValue() { + return value; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/GSECode.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/constants/GSECode.java similarity index 97% rename from src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/GSECode.java rename to src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/constants/GSECode.java index b7b10ccb01..e7c2b6b7a2 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/GSECode.java +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/constants/GSECode.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.execute.engine.consts; +package com.tencent.bk.job.common.gse.constants; /** diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/constants/GseConstants.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/constants/GseConstants.java index f8c710061b..0dbc418d64 100644 --- a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/constants/GseConstants.java +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/constants/GseConstants.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -31,7 +31,16 @@ public interface GseConstants { int DEFAULT_CLOUD_ID = 0; /** - * GSE API 度量指标名称前缀 + * GSE 获取文件任务执行结果协议版本V2 - 解除valuekey依赖版本 */ - String GSE_API_METRICS_NAME_PREFIX = "job.client.gse.api"; + int GSE_FILE_PROTOCOL_VERSION_V2 = 2; + + /** + * GSE 公钥证书BASE64，固定值 + */ + String publicKeyPermBase64 = + "LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUlHZk1BMEdDU3FHU0liM0RRRUJBUVVBQTRHTkFEQ0JpUUtCZ1FETEVXZk9" + + "YU2VvMXpNQ1JpRVNFTWs3OXo0cwpHYkw4VmIvZXg5K1RaR2VyN255bEh5Y0Vtb2o5aWE4K2daTmVQOFRRVmRyTExhSz" + + "IzektiT3lja2FiVE5QS0VZCmhQY0NlellEQVdleTZBS2ZHSCtYZGV0MnJDOWtzRWhrM1BqcDVuZDk4QW1KZ0VJeSt6S" + + "0FhaVZEazFvdG5Jc0EKRWxucUdXL24zaWVuN0hmSXN3SURBUUFCCi0tLS0tRU5EIFBVQkxJQyBLRVktLS0tLQo="; } diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/constants/GseMetricNames.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/constants/GseMetricNames.java new file mode 100644 index 0000000000..4f77b3fb1b --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/constants/GseMetricNames.java @@ -0,0 +1,37 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.gse.constants; + +public interface GseMetricNames { + /** + * GSE V1 API 度量指标名称前缀 + */ + String GSE_API_METRICS_NAME_PREFIX = "job.client.gse.api"; + + /** + * GSE V2 API 度量指标名称前缀 + */ + String GSE_V2_API_METRICS_NAME_PREFIX = "job.client.gse.v2.api"; +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/constants/GseTaskTypeEnum.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/constants/GseTaskTypeEnum.java new file mode 100644 index 0000000000..bea1438262 --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/constants/GseTaskTypeEnum.java @@ -0,0 +1,30 @@ +package com.tencent.bk.job.common.gse.constants; + +/** + * GSE 任务类型 + */ +public enum GseTaskTypeEnum { + SCRIPT(1), FILE(2); + + private final Integer value; + + GseTaskTypeEnum(Integer val) { + this.value = val; + } + + public static GseTaskTypeEnum getGseTaskType(Integer type) { + if (type == null) { + return null; + } + for (GseTaskTypeEnum typeEnum : values()) { + if (typeEnum.getValue().equals(type)) { + return typeEnum; + } + } + throw new IllegalArgumentException("No GseTaskTypeEnum constant: " + type); + } + + public Integer getValue() { + return value; + } +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/sdk/BKTSSLTransportFactory.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/sdk/BKTSSLTransportFactory.java deleted file mode 100644 index 3d28a219c1..0000000000 --- a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/sdk/BKTSSLTransportFactory.java +++ /dev/null @@ -1,295 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.gse.sdk; - -import org.apache.thrift.transport.TServerSocket; -import org.apache.thrift.transport.TSocket; -import org.apache.thrift.transport.TTransportException; - -import javax.net.ssl.*; -import java.io.FileInputStream; -import java.io.IOException; -import java.net.InetAddress; -import java.security.KeyStore; -import java.security.SecureRandom; -import java.security.cert.X509Certificate; -import java.util.Arrays; - -/** - * 针对Thrift的SSL做免TrustManager验证，主要是因为centos 7.x以上系统，可能存在认证异常 Caused by: sun.security.validator.ValidatorException: - * KeyUsage does not allow key encipherment at - * sun.security.validator.EndEntityChecker.checkTLSServer(EndEntityChecker.java:264) at - * sun.security.validator.EndEntityChecker.check(EndEntityChecker.java:141) at - * sun.security.validator.Validator.validate(Validator.java:264) at - * sun.security.ssl.X509TrustManagerImpl.validate(X509TrustManagerImpl.java:324) at - * sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:229) at - * sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:124) at - * sun.security.ssl.ClientHandshaker.serverCertificate(ClientHandshaker.java:1496) - * - * @version 1.0 - * @time 2017/6/19. - */ -public class BKTSSLTransportFactory { - private final static X509TrustManager TRUST_MANAGER = new X509TrustManager() { - @SuppressWarnings("all") - @Override - public void checkClientTrusted(X509Certificate[] xcs, String string) { - } - - @SuppressWarnings("all") - @Override - public void checkServerTrusted(X509Certificate[] xcs, String string) { - } - - @SuppressWarnings("all") - @Override - public X509Certificate[] getAcceptedIssuers() { - return null; - } - }; - - public BKTSSLTransportFactory() { - } - - public static TServerSocket getServerSocket(int port) throws TTransportException { - return getServerSocket(port, 0); - } - - public static TServerSocket getServerSocket(int port, int clientTimeout) throws TTransportException { - return getServerSocket(port, clientTimeout, false, (InetAddress) null); - } - - public static TServerSocket getServerSocket(int port, int clientTimeout, boolean clientAuth, InetAddress ifAddress) - throws TTransportException { - SSLServerSocketFactory factory = (SSLServerSocketFactory) SSLServerSocketFactory.getDefault(); - return createServer(factory, port, clientTimeout, clientAuth, ifAddress, null); - } - - public static TServerSocket getServerSocket(int port, int clientTimeout, InetAddress ifAddress, - TSSLTransportParameters params) throws TTransportException { - if (params != null && (params.isKeyStoreSet || params.isTrustStoreSet)) { - SSLContext ctx = createSSLContext(params); - return createServer(ctx.getServerSocketFactory(), port, clientTimeout, params.clientAuth, ifAddress, - params); - } else { - throw new TTransportException( - "Either one of the KeyStore or TrustStore must be set for SSLTransportParameters"); - } - } - - private static TServerSocket createServer(SSLServerSocketFactory factory, int port, int timeout, boolean clientAuth, - InetAddress ifAddress, TSSLTransportParameters params) throws TTransportException { - try { - SSLServerSocket serverSocket = (SSLServerSocket) factory.createServerSocket(port, 100, ifAddress); - serverSocket.setSoTimeout(timeout); - serverSocket.setNeedClientAuth(clientAuth); - if (params != null && params.cipherSuites != null) { - serverSocket.setEnabledCipherSuites(params.cipherSuites); - } - - return new TServerSocket( - (TServerSocket.ServerSocketTransportArgs) (new TServerSocket.ServerSocketTransportArgs()) - .serverSocket(serverSocket).clientTimeout(timeout)); - } catch (Exception var7) { - throw new TTransportException("Could not bind to port " + port, var7); - } - } - - public static TSocket getClientSocket(String host, int port, int timeout) throws TTransportException { - SSLSocketFactory factory = (SSLSocketFactory) SSLSocketFactory.getDefault(); - return createClient(factory, host, port, timeout); - } - - public static TSocket getClientSocket(String host, int port) throws TTransportException { - return getClientSocket(host, port, 0); - } - - public static TSocket getClientSocket(String host, int port, int timeout, TSSLTransportParameters params) - throws TTransportException { - if (params != null && (params.isKeyStoreSet || params.isTrustStoreSet)) { - SSLContext ctx = createSSLContext(params); - return createClient(ctx.getSocketFactory(), host, port, timeout); - } else { - throw new TTransportException( - "Either one of the KeyStore or TrustStore must be set for SSLTransportParameters"); - } - } - - private static SSLContext createSSLContext(TSSLTransportParameters params) throws TTransportException { - FileInputStream fin = null; - FileInputStream fis = null; - - SSLContext ctx; - try { - ctx = SSLContext.getInstance(params.protocol); - TrustManagerFactory tmf = null; - KeyManagerFactory kmf = null; - KeyStore ks; - if (params.isTrustStoreSet) { - tmf = TrustManagerFactory.getInstance(params.trustManagerType); - ks = KeyStore.getInstance(params.trustStoreType); - fin = new FileInputStream(params.trustStore); - ks.load(fin, params.trustPass != null ? params.trustPass.toCharArray() : null); - tmf.init(ks); - } - - if (params.isKeyStoreSet) { - kmf = KeyManagerFactory.getInstance(params.keyManagerType); - ks = KeyStore.getInstance(params.keyStoreType); - fis = new FileInputStream(params.keyStore); - ks.load(fis, params.keyPass.toCharArray()); - kmf.init(ks, params.keyPass.toCharArray()); - } - - if (params.isKeyStoreSet && params.isTrustStoreSet) { - ctx.init(kmf.getKeyManagers(), new TrustManager[]{TRUST_MANAGER}, (SecureRandom) null); - } else if (params.isKeyStoreSet) { - ctx.init(kmf.getKeyManagers(), (TrustManager[]) null, (SecureRandom) null); - } else { - ctx.init((KeyManager[]) null, new TrustManager[]{TRUST_MANAGER}, (SecureRandom) null); - } - } catch (Exception var17) { - throw new TTransportException("Error creating the transport", var17); - } finally { - if (fin != null) { - try { - fin.close(); - } catch (IOException var16) { - var16.printStackTrace(); - } - } - - if (fis != null) { - try { - fis.close(); - } catch (IOException var15) { - var15.printStackTrace(); - } - } - - } - - return ctx; - } - - private static TSocket createClient(SSLSocketFactory factory, String host, int port, int timeout) - throws TTransportException { - try { - SSLSocket socket = (SSLSocket) factory.createSocket(host, port); - socket.setSoTimeout(timeout); - return new TSocket(socket); - } catch (Exception var5) { - throw new TTransportException("Could not connect to " + host + " on port " + port, var5); - } - } - - public static class TSSLTransportParameters { - protected String protocol; - protected String keyStore; - protected String keyPass; - protected String keyManagerType; - protected String keyStoreType; - protected String trustStore; - protected String trustPass; - protected String trustManagerType; - protected String trustStoreType; - protected String[] cipherSuites; - protected boolean clientAuth; - protected boolean isKeyStoreSet; - protected boolean isTrustStoreSet; - - public TSSLTransportParameters() { - this.protocol = "TLS"; - this.keyManagerType = KeyManagerFactory.getDefaultAlgorithm(); - this.keyStoreType = "JKS"; - this.trustManagerType = TrustManagerFactory.getDefaultAlgorithm(); - this.trustStoreType = "JKS"; - this.clientAuth = false; - this.isKeyStoreSet = false; - this.isTrustStoreSet = false; - } - - public TSSLTransportParameters(String protocol, String[] cipherSuites) { - this(protocol, cipherSuites, false); - } - - public TSSLTransportParameters(String protocol, String[] cipherSuites, boolean clientAuth) { - this.protocol = "TLS"; - this.keyManagerType = KeyManagerFactory.getDefaultAlgorithm(); - this.keyStoreType = "JKS"; - this.trustManagerType = TrustManagerFactory.getDefaultAlgorithm(); - this.trustStoreType = "JKS"; - this.clientAuth = false; - this.isKeyStoreSet = false; - this.isTrustStoreSet = false; - if (protocol != null) { - this.protocol = protocol; - } - - this.cipherSuites = (String[]) Arrays.copyOf(cipherSuites, cipherSuites.length); - this.clientAuth = clientAuth; - } - - public void setKeyStore(String keyStore, String keyPass, String keyManagerType, String keyStoreType) { - this.keyStore = keyStore; - this.keyPass = keyPass; - if (keyManagerType != null) { - this.keyManagerType = keyManagerType; - } - - if (keyStoreType != null) { - this.keyStoreType = keyStoreType; - } - - this.isKeyStoreSet = true; - } - - public void setKeyStore(String keyStore, String keyPass) { - this.setKeyStore(keyStore, keyPass, (String) null, (String) null); - } - - public void setTrustStore(String trustStore, String trustPass, String trustManagerType, String trustStoreType) { - this.trustStore = trustStore; - this.trustPass = trustPass; - if (trustManagerType != null) { - this.trustManagerType = trustManagerType; - } - - if (trustStoreType != null) { - this.trustStoreType = trustStoreType; - } - - this.isTrustStoreSet = true; - } - - public void setTrustStore(String trustStore, String trustPass) { - this.setTrustStore(trustStore, trustPass, (String) null, (String) null); - } - - public void requireClientAuth(boolean clientAuth) { - this.clientAuth = clientAuth; - } - } -} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/sdk/GseCacheClientFactory.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/sdk/GseCacheClientFactory.java deleted file mode 100644 index 4e680f4594..0000000000 --- a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/sdk/GseCacheClientFactory.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.gse.sdk; - -import com.tencent.bk.gse.cacheapi.CacheAPI; -import com.tencent.bk.job.common.gse.config.GseConfig; -import com.tencent.bk.job.common.util.ArrayUtil; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.apache.thrift.TException; -import org.apache.thrift.protocol.TBinaryProtocol; -import org.apache.thrift.protocol.TProtocol; -import org.apache.thrift.transport.TSocket; -import org.apache.thrift.transport.TTransport; -import org.apache.thrift.transport.layered.TFramedTransport; -import org.slf4j.helpers.MessageFormatter; - -import java.util.concurrent.atomic.AtomicInteger; - - -@Slf4j -public class GseCacheClientFactory { - - private static final AtomicInteger currentIpIndex = new AtomicInteger(0); - private final GseConfig gseConfig; - - public GseCacheClientFactory(GseConfig gseConfig) { - this.gseConfig = gseConfig; - } - - public GseCacheClient getClient() { - if (gseConfig.getGseCacheApiServerHost().length == 1 - && StringUtils.isBlank(gseConfig.getGseCacheApiServerHost()[0])) { - return null; - } - - int tryTimes = gseConfig.getGseCacheApiServerHost().length; - while (true) { - int ipIndex = currentIpIndex.incrementAndGet() % gseConfig.getGseCacheApiServerHost().length; - String ip = ""; - if (tryTimes > 0) { - try { - ip = gseConfig.getGseCacheApiServerHost()[ipIndex]; - if (StringUtils.isBlank(ip)) { - continue; - } - return getAgent(ip, gseConfig.getGseCacheApiServerPort()); - } catch (TException e) { - String msg = MessageFormatter.format( - "Get GseCacheClient fail|{}:{}|msg={}", - ArrayUtil.toArray(ip, gseConfig.getGseCacheApiServerPort(), e.getMessage())) - .getMessage(); - log.error(msg, e); - if ((--tryTimes) == 0) { - return null; - } - } - } - } - } - - /** - * 构建gse访问客户端, 并连接服务端 - */ - private GseCacheClient getAgent(String ip, int port) throws TException { - log.info("Enter getClient with ip=" + ip + ", port=" + port); - TTransport tTransport; - if (gseConfig.isEnableSsl()) { - BKTSSLTransportFactory.TSSLTransportParameters params = - new BKTSSLTransportFactory.TSSLTransportParameters(); - params.setTrustStore(gseConfig.getTrustStore(), gseConfig.getTrustStorePass(), - gseConfig.getTrustManagerType(), gseConfig.getTrustStoreType()); - params.setKeyStore(gseConfig.getKeyStore(), gseConfig.getKeyStorePass()); - tTransport = new TFramedTransport(BKTSSLTransportFactory.getClientSocket(ip, port, 15000, params)); - } else { - TSocket tSocket = new TSocket(ip, port); - tSocket.setTimeout(15000); - tTransport = new TFramedTransport(tSocket); - } - TProtocol tProtocol = new TBinaryProtocol(tTransport); - CacheAPI.Client gseAgentClient = new CacheAPI.Client(tProtocol); - return new GseCacheClient(gseAgentClient, tTransport); - } - -} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/AbstractAgentStateClientImpl.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/AbstractAgentStateClientImpl.java new file mode 100644 index 0000000000..5ccb701c9a --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/AbstractAgentStateClientImpl.java @@ -0,0 +1,160 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.gse.service; + +import com.tencent.bk.job.common.gse.IGseClient; +import com.tencent.bk.job.common.gse.config.AgentStateQueryConfig; +import com.tencent.bk.job.common.gse.v2.model.req.ListAgentStateReq; +import com.tencent.bk.job.common.gse.v2.model.resp.AgentState; +import com.tencent.bk.job.common.util.ConcurrencyUtil; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.helpers.FormattingTuple; +import org.slf4j.helpers.MessageFormatter; +import org.springframework.util.StopWatch; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ThreadPoolExecutor; + +@Slf4j +public abstract class AbstractAgentStateClientImpl implements AgentStateClient { + + private final AgentStateQueryConfig agentStateQueryConfig; + private final IGseClient gseClient; + private final ThreadPoolExecutor threadPoolExecutor; + + public AbstractAgentStateClientImpl(AgentStateQueryConfig agentStateQueryConfig, + IGseClient gseClient, + ThreadPoolExecutor threadPoolExecutor) { + this.agentStateQueryConfig = agentStateQueryConfig; + this.gseClient = gseClient; + this.threadPoolExecutor = threadPoolExecutor; + } + + protected AgentState getAgentState(String agentId) { + if (StringUtils.isBlank(agentId)) { + return null; + } + ListAgentStateReq req = new ListAgentStateReq(); + req.setAgentIdList(Collections.singletonList(agentId)); + List agentStateList = gseClient.listAgentState(req); + if (CollectionUtils.isEmpty(agentStateList)) { + FormattingTuple msg = MessageFormatter.format( + "cannot find agent state by agentId:{}", + agentId + ); + log.warn(msg.getMessage()); + return null; + } else if (agentStateList.size() > 1) { + FormattingTuple msg = MessageFormatter.format( + "multi({}) agent states by agentId:{}, use the first one", + agentStateList.size(), + agentId + ); + log.warn(msg.getMessage()); + return agentStateList.get(0); + } + return agentStateList.get(0); + } + + public Map batchGetAgentStateConcurrent(List agentIdList) { + StopWatch watch = new StopWatch("batchGetAgentStateConcurrent"); + + watch.start("splitToBatch"); + Map resultMap = new HashMap<>(); + if (CollectionUtils.isEmpty(agentIdList)) { + return resultMap; + } + + // 分批 + int batchSize = agentStateQueryConfig.getGseQueryBatchSize(); + int start = 0; + int end; + int size = agentIdList.size(); + List> ipSubListList = new ArrayList<>(); + while (start < size) { + end = start + batchSize; + end = Math.min(end, size); + List ipSubList = agentIdList.subList(start, end); + ipSubListList.add(ipSubList); + start += batchSize; + } + watch.stop(); + + // 并发查询 + watch.start("ConcurrencyUtil.getResultWithThreads"); + Collection> maps = ConcurrencyUtil.getResultWithThreads( + ipSubListList, + threadPoolExecutor, + ipList1 -> Collections.singletonList(batchGetAgentStatusWithoutLimit(ipList1)) + ); + watch.stop(); + + watch.start("collectResult"); + maps.forEach(resultMap::putAll); + watch.stop(); + + long duration = watch.getTotalTimeMillis(); + FormattingTuple msg = MessageFormatter.format( + "Get status of {} ips, time consuming: {}ms", + resultMap.size(), + duration + ); + if (duration > 1000L) { + log.warn(msg.getMessage() + ", statistics: " + watch.prettyPrint()); + } else { + log.debug(msg.getMessage()); + } + return resultMap; + } + + private Map batchGetAgentStatusWithoutLimit(List agentIdList) { + StopWatch watch = new StopWatch("batchGetAgentStatusWithoutLimit"); + + watch.start("listAgentState"); + Map resultMap = new HashMap<>(); + ListAgentStateReq req = new ListAgentStateReq(); + req.setAgentIdList(agentIdList); + List agentStateList = gseClient.listAgentState(req); + watch.stop(); + + watch.start("collectResult"); + for (AgentState agentState : agentStateList) { + resultMap.put(agentState.getAgentId(), agentState); + } + watch.stop(); + + if (watch.getTotalTimeMillis() > 1000) { + log.warn("batchGetAgentStatusWithoutLimit slow, statistics: " + watch.prettyPrint()); + } + return resultMap; + } +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/AgentStateClient.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/AgentStateClient.java new file mode 100644 index 0000000000..ecf748480a --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/AgentStateClient.java @@ -0,0 +1,69 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.gse.service; + +import com.tencent.bk.job.common.gse.service.model.HostAgentStateQuery; +import com.tencent.bk.job.common.gse.v2.model.resp.AgentState; + +import java.util.List; +import java.util.Map; + +/** + * 封装Agent状态查询的常用操作：判断真实使用的AgentId、单个主机Agent状态查询、批量状态查询 + */ +public interface AgentStateClient { + + /** + * 获取用于查询Agent状态的AgentId，来源于cloudIp或agentId字段（会按需查询并填充hostAgentStateQueryList中各元素的相关字段） + * + * @param hostAgentStateQuery Agent状态查询条件 + * @return 用于查询Agent状态的AgentId + */ + String getEffectiveAgentId(HostAgentStateQuery hostAgentStateQuery); + + /** + * 根据agentId获取agent状态（会按需查询并填充hostAgentStateQueryList中各元素的相关字段） + * + * @param hostAgentStateQuery Agent状态查询条件 + * @return Agent状态对象 + */ + AgentState getAgentState(HostAgentStateQuery hostAgentStateQuery); + + /** + * 根据agentId批量获取agent状态（会按需查询并填充hostAgentStateQueryList中各元素的相关字段） + * + * @param hostAgentStateQueryList Agent状态查询条件列表 + * @return agentId与Agent状态的Map + */ + Map batchGetAgentState(List hostAgentStateQueryList); + + /** + * 根据agentId批量获取agent存活状态（会按需查询并填充hostAgentStateQueryList中各元素的相关字段） + * + * @param hostAgentStateQueryList Agent状态查询条件列表 + * @return agentId与Agent存活状态的Map + */ + Map batchGetAgentAliveStatus(List hostAgentStateQueryList); +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/AutoChoosingAgentStateClientImpl.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/AutoChoosingAgentStateClientImpl.java new file mode 100644 index 0000000000..df62a522e0 --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/AutoChoosingAgentStateClientImpl.java @@ -0,0 +1,84 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.gse.service; + +import com.tencent.bk.job.common.gse.service.model.HostAgentStateQuery; +import com.tencent.bk.job.common.gse.v2.model.resp.AgentState; +import com.tencent.bk.job.common.util.toggle.feature.FeatureIdConstants; +import com.tencent.bk.job.common.util.toggle.feature.FeatureToggle; +import lombok.extern.slf4j.Slf4j; + +import java.util.List; +import java.util.Map; + +/** + * 根据动态变化的特性配置自动选择应当使用的AgentStateClient + */ +@Slf4j +public class AutoChoosingAgentStateClientImpl implements AgentStateClient { + + private final AgentStateClient preferV2AgentStateClient; + private final AgentStateClient useV2ByFeatureAgentStateClient; + + public AutoChoosingAgentStateClientImpl(AgentStateClient preferV2AgentStateClient, + AgentStateClient useV2ByFeatureAgentStateClient) { + this.preferV2AgentStateClient = preferV2AgentStateClient; + this.useV2ByFeatureAgentStateClient = useV2ByFeatureAgentStateClient; + } + + @Override + public String getEffectiveAgentId(HostAgentStateQuery hostAgentStateQuery) { + AgentStateClient agentStateClient = chooseAgentStateByFeatureConfig(); + return agentStateClient.getEffectiveAgentId(hostAgentStateQuery); + } + + @Override + public AgentState getAgentState(HostAgentStateQuery hostAgentStateQuery) { + AgentStateClient agentStateClient = chooseAgentStateByFeatureConfig(); + return agentStateClient.getAgentState(hostAgentStateQuery); + } + + @Override + public Map batchGetAgentState(List hostAgentStateQueryList) { + AgentStateClient agentStateClient = chooseAgentStateByFeatureConfig(); + return agentStateClient.batchGetAgentState(hostAgentStateQueryList); + } + + @Override + public Map batchGetAgentAliveStatus(List hostAgentStateQueryList) { + AgentStateClient agentStateClient = chooseAgentStateByFeatureConfig(); + return agentStateClient.batchGetAgentAliveStatus(hostAgentStateQueryList); + } + + private AgentStateClient chooseAgentStateByFeatureConfig() { + AgentStateClient agentStateClient; + if (FeatureToggle.isFeatureEnabled(FeatureIdConstants.FEATURE_AGENT_STATUS_GSE_V2)) { + agentStateClient = useV2ByFeatureAgentStateClient; + } else { + agentStateClient = preferV2AgentStateClient; + } + return agentStateClient; + } +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/BizHostInfoQueryService.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/BizHostInfoQueryService.java new file mode 100644 index 0000000000..92ff404ae3 --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/BizHostInfoQueryService.java @@ -0,0 +1,32 @@ +package com.tencent.bk.job.common.gse.service; + +import java.util.Collection; +import java.util.Map; + +public interface BizHostInfoQueryService { + + /** + * 根据主机ID批量查询CMDB业务ID + * + * @param hostIds 主机ID集合 + * @return Map<主机ID, CMDB业务ID> + */ + Map queryBizIdsByHostId(Collection hostIds); + + /** + * 根据主机ID批量查询AgentID + * + * @param hostIds 主机ID集合 + * @return Map<主机ID, AgentID> + */ + Map queryAgentIdsByHostId(Collection hostIds); + + /** + * 根据主机ID批量查询云区域IP + * + * @param hostIds 主机ID集合 + * @return Map<主机ID, 云区域IP> + */ + Map queryCloudIpsByHostId(Collection hostIds); + +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/GseV1AgentStateClientImpl.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/GseV1AgentStateClientImpl.java new file mode 100644 index 0000000000..1338df18a5 --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/GseV1AgentStateClientImpl.java @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.gse.service; + +import com.tencent.bk.job.common.gse.config.AgentStateQueryConfig; +import com.tencent.bk.job.common.gse.service.model.HostAgentStateQuery; +import com.tencent.bk.job.common.gse.v1.GseV1ApiClient; +import lombok.extern.slf4j.Slf4j; + +import java.util.concurrent.ThreadPoolExecutor; + +@Slf4j +public class GseV1AgentStateClientImpl extends SingleChannelAgentStateClientImpl { + + public GseV1AgentStateClientImpl(AgentStateQueryConfig agentStateQueryConfig, + GseV1ApiClient gseV1ApiClient, + ThreadPoolExecutor threadPoolExecutor) { + super(agentStateQueryConfig, gseV1ApiClient, threadPoolExecutor); + } + + @Override + public String getEffectiveAgentId(HostAgentStateQuery hostAgentStateQuery) { + return hostAgentStateQuery.getCloudIp(); + } +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/GseV2AgentStateClientImpl.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/GseV2AgentStateClientImpl.java new file mode 100644 index 0000000000..85f8299662 --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/GseV2AgentStateClientImpl.java @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.gse.service; + +import com.tencent.bk.job.common.gse.config.AgentStateQueryConfig; +import com.tencent.bk.job.common.gse.service.model.HostAgentStateQuery; +import com.tencent.bk.job.common.gse.v2.GseV2ApiClient; +import lombok.extern.slf4j.Slf4j; + +import java.util.concurrent.ThreadPoolExecutor; + +@Slf4j +public class GseV2AgentStateClientImpl extends SingleChannelAgentStateClientImpl { + + public GseV2AgentStateClientImpl(AgentStateQueryConfig agentStateQueryConfig, + GseV2ApiClient gseV2ApiClient, + ThreadPoolExecutor threadPoolExecutor) { + super(agentStateQueryConfig, gseV2ApiClient, threadPoolExecutor); + } + + @Override + public String getEffectiveAgentId(HostAgentStateQuery hostAgentStateQuery) { + return hostAgentStateQuery.getAgentId(); + } +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/PreferV2AgentStateClientImpl.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/PreferV2AgentStateClientImpl.java new file mode 100644 index 0000000000..d0b1161bdf --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/PreferV2AgentStateClientImpl.java @@ -0,0 +1,102 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.gse.service; + +import com.tencent.bk.job.common.gse.GseClient; +import com.tencent.bk.job.common.gse.config.AgentStateQueryConfig; +import com.tencent.bk.job.common.gse.service.model.HostAgentStateQuery; +import com.tencent.bk.job.common.gse.util.AgentStateUtil; +import com.tencent.bk.job.common.gse.util.AgentUtils; +import com.tencent.bk.job.common.gse.v2.model.resp.AgentState; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.stream.Collectors; + +@Slf4j +public class PreferV2AgentStateClientImpl extends AbstractAgentStateClientImpl { + + public PreferV2AgentStateClientImpl(AgentStateQueryConfig agentStateQueryConfig, + GseClient gseClient, + ThreadPoolExecutor threadPoolExecutor) { + super(agentStateQueryConfig, gseClient, threadPoolExecutor); + } + + @Override + public String getEffectiveAgentId(HostAgentStateQuery hostAgentStateQuery) { + String agentId = hostAgentStateQuery.getAgentId(); + if (StringUtils.isNotBlank(agentId)) { + return agentId; + } + return hostAgentStateQuery.getCloudIp(); + } + + @Override + public AgentState getAgentState(HostAgentStateQuery hostAgentStateQuery) { + String finalAgentId = getEffectiveAgentId(hostAgentStateQuery); + if (StringUtils.isBlank(finalAgentId)) { + return null; + } + return getAgentState(finalAgentId); + } + + @Override + public Map batchGetAgentState(List hostAgentStateQueryList) { + // 对agentId按照对应的GSE Agent 版本进行分类 + List queryAgentIds = hostAgentStateQueryList.stream() + .map(this::getEffectiveAgentId) + .filter(StringUtils::isNotEmpty) + .collect(Collectors.toList()); + Pair, List> classifiedAgentIdList = classifyGseAgentIds(queryAgentIds); + + Map results = batchGetAgentStateConcurrent(classifiedAgentIdList.getLeft()); + results.putAll(batchGetAgentStateConcurrent(classifiedAgentIdList.getRight())); + return results; + } + + private Pair, List> classifyGseAgentIds(List agentIdList) { + List v1AgentIdList = new ArrayList<>(); + List v2AgentIdList = new ArrayList<>(); + agentIdList.forEach(agentId -> { + if (AgentUtils.isGseV1AgentId(agentId)) { + v1AgentIdList.add(agentId); + } else { + v2AgentIdList.add(agentId); + } + }); + return Pair.of(v1AgentIdList, v2AgentIdList); + } + + @Override + public Map batchGetAgentAliveStatus(List hostAgentStateQueryList) { + Map agentStateMap = batchGetAgentState(hostAgentStateQueryList); + return AgentStateUtil.batchGetAgentAliveStatus(agentStateMap); + } +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/QueryAgentStatusClient.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/QueryAgentStatusClient.java deleted file mode 100644 index 7009546a51..0000000000 --- a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/QueryAgentStatusClient.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.gse.service; - -import com.fasterxml.jackson.annotation.JsonProperty; -import org.apache.commons.lang3.tuple.Pair; - -import java.util.List; -import java.util.Map; - -public interface QueryAgentStatusClient { - - /** - * 批量获取agent状态 - * - * @param ips - * @return Map:key为cloudId:ip，value为AgentStatus，AgentStatus.ip不含云区域Id - */ - Map batchGetAgentStatus(List ips); - - /** - * 获取agent状态 - * - * @param ip - * @return - */ - AgentStatus getAgentStatus(String ip); - - /** - * 获取agent绑定的ip - * - * @param multiIp - * @param cloudAreaId - * @return - */ - String getHostIpByAgentStatus(String multiIp, long cloudAreaId); - - - /** - * 获取agent绑定的ip及Agent状态 - * - * @param multiIp - * @param cloudAreaId - * @return - */ - Pair getHostIpWithAgentStatus(String multiIp, long cloudAreaId); - - /** - * Agent状态返回结果 - */ - class AgentStatus { - // 不含云区域Id - @JsonProperty("ip") - public String ip; - @JsonProperty("plat_id") - public int cloudAreaId; - @JsonProperty("status") - public int status; - } -} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/QueryAgentStatusClientImpl.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/QueryAgentStatusClientImpl.java deleted file mode 100644 index d97d54c100..0000000000 --- a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/QueryAgentStatusClientImpl.java +++ /dev/null @@ -1,261 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.gse.service; - -import com.tencent.bk.gse.cacheapi.AgentStatusRequestInfo; -import com.tencent.bk.gse.cacheapi.AgentStatusResponse; -import com.tencent.bk.gse.cacheapi.CacheIpInfo; -import com.tencent.bk.gse.cacheapi.CacheUser; -import com.tencent.bk.job.common.gse.config.GseConfig; -import com.tencent.bk.job.common.gse.constants.GseConstants; -import com.tencent.bk.job.common.gse.model.AgentStatusDTO; -import com.tencent.bk.job.common.gse.sdk.GseCacheClient; -import com.tencent.bk.job.common.gse.sdk.GseCacheClientFactory; -import com.tencent.bk.job.common.gse.util.AgentUtils; -import com.tencent.bk.job.common.util.ConcurrencyUtil; -import com.tencent.bk.job.common.util.Utils; -import com.tencent.bk.job.common.util.json.JsonUtils; -import io.micrometer.core.instrument.MeterRegistry; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.tuple.Pair; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; - -@Slf4j -public class QueryAgentStatusClientImpl implements QueryAgentStatusClient { - - private final GseCacheClientFactory gseCacheClientFactory; - private final GseConfig gseConfig; - private final MeterRegistry meterRegistry; - - public QueryAgentStatusClientImpl(GseConfig gseConfig, MeterRegistry meterRegistry) { - this.gseConfig = gseConfig; - this.gseCacheClientFactory = new GseCacheClientFactory(gseConfig); - this.meterRegistry = meterRegistry; - } - - @Override - public Map batchGetAgentStatus(List ips) { - Long startTime = System.currentTimeMillis(); - Map resultMap = new HashMap<>(); - // 分批 - int batchSize = gseConfig.getQueryBatchSize(); - int threadNum = gseConfig.getQueryThreadsNum(); - int start = 0; - int end; - int size = ips.size(); - List> ipSubListList = new ArrayList<>(); - while (start < size) { - end = start + batchSize; - end = Math.min(end, size); - List ipSubList = ips.subList(start, end); - ipSubListList.add(ipSubList); - start += batchSize; - } - // 并发查询 - Collection> maps = ConcurrencyUtil.getResultWithThreads(ipSubListList, threadNum, - ipList1 -> Collections.singletonList(batchGetAgentStatusWithoutLimit(ipList1))); - maps.forEach(resultMap::putAll); - Long endTime = System.currentTimeMillis(); - if (endTime - startTime > 100L) { - log.warn("Get status of {} ips, time consuming: {}ms", resultMap.size(), (endTime - startTime)); - } - return resultMap; - } - - public Map batchGetAgentStatusWithoutLimit(Collection ips) { - Map resultMap = new HashMap<>(); - AgentStatusResponse response = queryAgentStatusFromCacheApi(ips); - if (response == null) { - return resultMap; - } - Map responseMap = response.getResult(); - for (Map.Entry item : responseMap.entrySet()) { - String[] ipInfo = item.getKey().split(":"); - if (ipInfo.length != 2) { - log.error("Query Gse agent not return businessId:ip: {}", item.getKey()); - continue; - } - - AgentStatus agentStatus = new AgentStatus(); - agentStatus.ip = ipInfo[1]; - agentStatus.status = parseCacheAgentStatus(item.getValue()); - agentStatus.cloudAreaId = Utils.tryParseInt(ipInfo[0]); - resultMap.put(agentStatus.cloudAreaId + ":" + agentStatus.ip, agentStatus); - } - return resultMap; - } - - /** - * 解析agent状态 预期解析的文本: {"businessid":"","exist":1,"ip":"1.1.1.1","region":"1","version":"NULL"} - */ - private static int parseCacheAgentStatus(String statusStr) { - AgentStatusDTO agentStatus = JsonUtils.fromJson(statusStr, AgentStatusDTO.class); - return agentStatus.getExist(); - } - - - private AgentStatusResponse queryAgentStatusFromCacheApi(Collection ips) { - GseCacheClient gseClient = gseCacheClientFactory.getClient(); - if (null == gseClient) { - log.error("Get GSE cache client connection failed"); - return null; - } - - List ipInfoList = new ArrayList<>(); - for (String ip : ips) { - String[] ipInfo = ip.split(":"); - if (ipInfo.length != 2) { - log.warn("Request ip format error! IP: {}", ip); - continue; - } - - CacheIpInfo cacheIpInfo = new CacheIpInfo(); - String cloudAreaIdStr = "1".equals(ipInfo[0]) ? - String.valueOf(GseConstants.DEFAULT_CLOUD_ID) : ipInfo[0]; - cacheIpInfo.setPlatId(cloudAreaIdStr); - cacheIpInfo.setIp(ipInfo[1]); - ipInfoList.add(cacheIpInfo); - } - if (ipInfoList.isEmpty()) { - return null; - } - - long start = System.currentTimeMillis(); - String status = "ok"; - try { - CacheUser user = new CacheUser(); - user.setUser("bitmap"); - user.setPassword("bitmap"); - - AgentStatusRequestInfo request = new AgentStatusRequestInfo(); - request.setUser(user); - request.setIpinfos(ipInfoList); - - log.debug("QueryAgentStatus request: {}", request); - AgentStatusResponse response = gseClient.getCacheClient().quireAgentStatus(request); - log.debug("QueryAgentStatus response: {}", response); - return response; - } catch (Throwable e) { - log.error("QueryAgentStatus error", e); - status = "error"; - return null; - } finally { - long end = System.currentTimeMillis(); - log.info("BatchGetAgentStatus {} ips, cost: {}ms", ips.size(), (end - start)); - if (this.meterRegistry != null) { - meterRegistry.timer(GseConstants.GSE_API_METRICS_NAME_PREFIX, "api_name", "quireAgentStatus", - "status", status).record(end - start, TimeUnit.MICROSECONDS); - } - gseClient.tearDown(); - } - } - - @Override - public AgentStatus getAgentStatus(String ip) { - if (!ip.contains(":")) { - ip = "0:" + ip; - log.warn("getAgentStatus with ip(no cloudArea), use default cloudAreaId=0"); - } - List ips = new ArrayList<>(); - ips.add(ip); - return batchGetAgentStatus(ips).get(ip); - } - - private String getOneAliveIP(List ipList) { - if (ipList == null || ipList.isEmpty()) { - return null; - } - try { - Map resultMap = batchGetAgentStatus(ipList); - for (QueryAgentStatusClient.AgentStatus agentStatus : resultMap.values()) { - if (AgentUtils.isAgentOkByStatus(agentStatus.status)) { - return agentStatus.ip; - } - } - } catch (Exception e) { - return ipList.get(0); - } - return ipList.get(0); - } - - /** - * 传入的multiIp为逗号分隔的不带云区域ID的IP - * - * @param multiIp 多IP,格式:ip1,ip2,ip3...ipN - * @param cloudAreaId 云区域ID - * @return 返回的单个IP不带云区域ID - */ - public String getHostIpByAgentStatus(String multiIp, long cloudAreaId) { - List ipList = Utils.getNotBlankSplitList(multiIp, ","); - String hostIp = ipList.get(0); - - if (ipList.size() > 1) { - List ipListWithSource = addCloudAreaId(ipList, cloudAreaId); - - String oneAliveIP = getOneAliveIP(ipListWithSource); - if (oneAliveIP != null) { - if (oneAliveIP.contains(":")) { - return oneAliveIP.split(":")[1]; - } - return oneAliveIP; - } - } - return hostIp; - } - - @Override - public Pair getHostIpWithAgentStatus(String multiIp, long cloudAreaId) { - List ipList = Utils.getNotBlankSplitList(multiIp, ","); - if (ipList.isEmpty()) { - return null; - } - String hostIp = ipList.get(0); - - List ipListWithSource = addCloudAreaId(ipList, cloudAreaId); - Map resultMap = batchGetAgentStatus(ipListWithSource); - for (QueryAgentStatusClient.AgentStatus agentStatus : resultMap.values()) { - if (AgentUtils.isAgentOkByStatus(agentStatus.status)) { - return Pair.of(agentStatus.ip, true); - } - } - return Pair.of(hostIp, false); - } - - private List addCloudAreaId(List ipList, long cloudAreaId) { - List newIpList = new ArrayList<>(); - for (String ip : ipList) { - newIpList.add(cloudAreaId + ":" + ip); - } - return newIpList; - } - -} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/SingleChannelAgentStateClientImpl.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/SingleChannelAgentStateClientImpl.java new file mode 100644 index 0000000000..316dd2f9c4 --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/SingleChannelAgentStateClientImpl.java @@ -0,0 +1,70 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.gse.service; + +import com.tencent.bk.job.common.gse.IGseClient; +import com.tencent.bk.job.common.gse.config.AgentStateQueryConfig; +import com.tencent.bk.job.common.gse.service.model.HostAgentStateQuery; +import com.tencent.bk.job.common.gse.util.AgentStateUtil; +import com.tencent.bk.job.common.gse.v2.model.resp.AgentState; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.stream.Collectors; + +@Slf4j +public abstract class SingleChannelAgentStateClientImpl extends AbstractAgentStateClientImpl { + + public SingleChannelAgentStateClientImpl(AgentStateQueryConfig agentStateQueryConfig, + IGseClient gseClient, + ThreadPoolExecutor threadPoolExecutor) { + super(agentStateQueryConfig, gseClient, threadPoolExecutor); + } + + @Override + public AgentState getAgentState(HostAgentStateQuery hostAgentStateQuery) { + String finalAgentId = getEffectiveAgentId(hostAgentStateQuery); + return getAgentState(finalAgentId); + } + + @Override + public Map batchGetAgentState(List hostAgentStateQueryList) { + List queryAgentIds = hostAgentStateQueryList.stream() + .map(this::getEffectiveAgentId) + .filter(StringUtils::isNotEmpty) + .collect(Collectors.toList()); + + return batchGetAgentStateConcurrent(queryAgentIds); + } + + @Override + public Map batchGetAgentAliveStatus(List hostAgentStateQueryList) { + Map agentStateMap = batchGetAgentState(hostAgentStateQueryList); + return AgentStateUtil.batchGetAgentAliveStatus(agentStateMap); + } +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/UseV2ByFeatureAgentStateClientImpl.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/UseV2ByFeatureAgentStateClientImpl.java new file mode 100644 index 0000000000..9f911974ab --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/UseV2ByFeatureAgentStateClientImpl.java @@ -0,0 +1,384 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.gse.service; + +import com.tencent.bk.job.common.constant.ResourceScopeTypeEnum; +import com.tencent.bk.job.common.gse.service.model.HostAgentStateQuery; +import com.tencent.bk.job.common.gse.util.AgentStateUtil; +import com.tencent.bk.job.common.gse.v2.model.resp.AgentState; +import com.tencent.bk.job.common.model.dto.ResourceScope; +import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.common.util.toggle.ToggleEvaluateContext; +import com.tencent.bk.job.common.util.toggle.ToggleStrategyContextParams; +import com.tencent.bk.job.common.util.toggle.feature.FeatureIdConstants; +import com.tencent.bk.job.common.util.toggle.feature.FeatureToggle; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Slf4j +public class UseV2ByFeatureAgentStateClientImpl implements AgentStateClient { + + private final BizHostInfoQueryService bizHostInfoQueryService; + private final SingleChannelAgentStateClientImpl gseV1AgentStateClient; + private final SingleChannelAgentStateClientImpl gseV2AgentStateClient; + + public UseV2ByFeatureAgentStateClientImpl(GseV1AgentStateClientImpl gseV1AgentStateClient, + GseV2AgentStateClientImpl gseV2AgentStateClient, + BizHostInfoQueryService bizHostInfoQueryService) { + this.gseV1AgentStateClient = gseV1AgentStateClient; + this.gseV2AgentStateClient = gseV2AgentStateClient; + this.bizHostInfoQueryService = bizHostInfoQueryService; + } + + @Override + public String getEffectiveAgentId(HostAgentStateQuery hostAgentStateQuery) { + if (hostAgentStateQuery.getBizId() == null) { + fillBizIdByHostId(hostAgentStateQuery); + } + if (needToUseGseV2(hostAgentStateQuery)) { + fillAgentIdIfNeed(hostAgentStateQuery); + return hostAgentStateQuery.getAgentId(); + } + fillCloudIpIfNeed(hostAgentStateQuery); + return hostAgentStateQuery.getCloudIp(); + } + + @Override + public AgentState getAgentState(HostAgentStateQuery hostAgentStateQuery) { + // 填充需要的字段 + String effectiveAgentId = getEffectiveAgentId(hostAgentStateQuery); + if (StringUtils.isBlank(effectiveAgentId)) { + return null; + } + if (needToUseGseV2(hostAgentStateQuery)) { + return gseV2AgentStateClient.getAgentState(hostAgentStateQuery); + } + return gseV1AgentStateClient.getAgentState(hostAgentStateQuery); + } + + private void fillBizIdByHostId(HostAgentStateQuery hostAgentStateQuery) { + Long hostId = hostAgentStateQuery.getHostId(); + Map hostIdBizIdMap = bizHostInfoQueryService.queryBizIdsByHostId( + Collections.singletonList(hostId) + ); + Long bizId = hostIdBizIdMap.get(hostId); + if (log.isDebugEnabled()) { + log.debug("queryBizIdByHostId, hostId={}, bizId={}", hostId, bizId); + } + hostAgentStateQuery.setBizId(bizId); + } + + private void fillAgentIdIfNeed(HostAgentStateQuery hostAgentStateQuery) { + if (StringUtils.isNotBlank(hostAgentStateQuery.getAgentId())) { + return; + } + Long hostId = hostAgentStateQuery.getHostId(); + Map hostIdAgentIdMap = bizHostInfoQueryService.queryAgentIdsByHostId( + Collections.singletonList(hostId) + ); + String agentId = hostIdAgentIdMap.get(hostId); + if (log.isDebugEnabled()) { + log.debug("queryAgentIdByHostId, hostId={}, agentId={}", hostId, agentId); + } + hostAgentStateQuery.setAgentId(agentId); + } + + private void fillCloudIpIfNeed(HostAgentStateQuery hostAgentStateQuery) { + if (StringUtils.isNotBlank(hostAgentStateQuery.getCloudIp())) { + return; + } + Long hostId = hostAgentStateQuery.getHostId(); + Map hostIdCloudIpMap = bizHostInfoQueryService.queryCloudIpsByHostId( + Collections.singletonList(hostId) + ); + String cloudIp = hostIdCloudIpMap.get(hostId); + if (log.isDebugEnabled()) { + log.debug("queryCloudIpByHostId, hostId={}, cloudIp={}", hostId, cloudIp); + } + hostAgentStateQuery.setCloudIp(cloudIp); + } + + private boolean needToUseGseV2(HostAgentStateQuery hostAgentStateQuery) { + ResourceScope resourceScope = new ResourceScope( + ResourceScopeTypeEnum.BIZ.getValue(), + String.valueOf(hostAgentStateQuery.getBizId()) + ); + ToggleEvaluateContext toggleEvaluateContext = + ToggleEvaluateContext.builder() + .addContextParam(ToggleStrategyContextParams.CTX_PARAM_RESOURCE_SCOPE, resourceScope); + + return FeatureToggle.checkFeature( + FeatureIdConstants.FEATURE_AGENT_STATUS_GSE_V2, + toggleEvaluateContext + ); + } + + @Override + public Map batchGetAgentState(List hostAgentStateQueryList) { + // 1.筛选出缺少业务ID的主机 + List queryWithValidBizIdList = new ArrayList<>(); + List queryWithInvalidBizIdList = new ArrayList<>(); + List queryWithInvalidBizAndHostIdList = new ArrayList<>(); + for (HostAgentStateQuery query : hostAgentStateQueryList) { + if (query.getBizId() == null || query.getBizId() <= 0) { + if (query.getHostId() == null || query.getHostId() <= 0) { + queryWithInvalidBizAndHostIdList.add(query); + } else { + queryWithInvalidBizIdList.add(query); + } + } else { + queryWithValidBizIdList.add(query); + } + } + if (!queryWithInvalidBizAndHostIdList.isEmpty()) { + log.warn( + "Ignore {} queryHosts with invalid bizId and hostId:{}", + queryWithInvalidBizAndHostIdList.size(), + queryWithInvalidBizAndHostIdList + ); + } + List queryWithNotFoundBizIdList = new ArrayList<>(); + fillBizIdsByHostId(queryWithInvalidBizIdList); + for (HostAgentStateQuery query : queryWithInvalidBizIdList) { + if (query.getBizId() == null || query.getBizId() <= 0) { + queryWithNotFoundBizIdList.add(query); + } else { + queryWithValidBizIdList.add(query); + } + } + if (!queryWithNotFoundBizIdList.isEmpty()) { + log.warn( + "Ignore {} queryHosts with not found bizId:{}", + queryWithNotFoundBizIdList.size(), + queryWithNotFoundBizIdList + ); + } + Pair, List> pair = classifyQueryAndGetAgentIdList(queryWithValidBizIdList); + List cloudIpList = pair.getLeft(); + List agentIdList = pair.getRight(); + log.info( + "Use {} cloudIps(empty value filtered) and {} agentIds(empty value filtered) to query agent status", + cloudIpList.size(), + agentIdList.size() + ); + Map agentStateMap = gseV1AgentStateClient.batchGetAgentStateConcurrent(cloudIpList); + agentStateMap.putAll(gseV2AgentStateClient.batchGetAgentStateConcurrent(agentIdList)); + if (log.isDebugEnabled()) { + log.debug("agentStateMap={}", JsonUtils.toJson(agentStateMap)); + } + return agentStateMap; + } + + private void fillBizIdsByHostId(List queryList) { + if (CollectionUtils.isEmpty(queryList)) { + return; + } + List hostIdList = queryList.stream().map(HostAgentStateQuery::getHostId).collect(Collectors.toList()); + Map hostIdBizIdMap = bizHostInfoQueryService.queryBizIdsByHostId(hostIdList); + if (hostIdBizIdMap.isEmpty()) { + log.warn("Found empty hostIdBizIdMap by hostIdList:{}", hostIdList); + return; + } + if (log.isDebugEnabled()) { + log.debug( + "queryBizIdsByHostId, hostIdList={}, hostIdBizIdMap={}", + hostIdList, + JsonUtils.toJson(hostIdBizIdMap) + ); + } + for (HostAgentStateQuery query : queryList) { + Long hostId = query.getHostId(); + Long bizId = hostIdBizIdMap.get(hostId); + if (bizId != null && bizId > 0) { + query.setBizId(bizId); + } else { + log.warn("Found unexpected bizId:{} by hostId:{}", bizId, hostId); + } + } + } + + private Pair, List> classifyQueryAndGetAgentIdList(List queryList) { + if (CollectionUtils.isEmpty(queryList)) { + return Pair.of(Collections.emptyList(), Collections.emptyList()); + } + List v1QueryList = new ArrayList<>(); + List v2QueryList = new ArrayList<>(); + for (HostAgentStateQuery query : queryList) { + if (needToUseGseV2(query)) { + v2QueryList.add(query); + } else { + v1QueryList.add(query); + } + } + log.info("v1QueryList.size={},v2QueryList.size={}", v1QueryList.size(), v2QueryList.size()); + List cloudIpList = new ArrayList<>(); + if (!v1QueryList.isEmpty()) { + fillCloudIpsIfNeed(v1QueryList); + } + List agentIdList = new ArrayList<>(); + if (!v2QueryList.isEmpty()) { + fillAgentIdsIfNeed(v2QueryList); + } + if (log.isDebugEnabled()) { + log.debug("classifyQueryAndGetAgentIdList, v1QueryList={}, v2QueryList={}", v1QueryList, v2QueryList); + } + for (HostAgentStateQuery hostAgentStateQuery : v1QueryList) { + String cloudIp = hostAgentStateQuery.getCloudIp(); + if (StringUtils.isNotBlank(cloudIp)) { + cloudIpList.add(cloudIp); + } + } + for (HostAgentStateQuery hostAgentStateQuery : v2QueryList) { + String agentId = hostAgentStateQuery.getAgentId(); + if (StringUtils.isNotBlank(agentId)) { + agentIdList.add(agentId); + } + } + return Pair.of(cloudIpList, agentIdList); + } + + private void fillCloudIpsIfNeed(List hostAgentStateQueryList) { + if (CollectionUtils.isEmpty(hostAgentStateQueryList)) { + return; + } + List queryWithoutCloudIpList = new ArrayList<>(); + List needQueryHostIdList = new ArrayList<>(); + for (HostAgentStateQuery query : hostAgentStateQueryList) { + if (StringUtils.isBlank(query.getCloudIp())) { + queryWithoutCloudIpList.add(query); + needQueryHostIdList.add(query.getHostId()); + } + } + if (CollectionUtils.isEmpty(needQueryHostIdList)) { + return; + } + Map hostIdCloudIpMap = queryCloudIpsByHostId(needQueryHostIdList); + if (hostIdCloudIpMap.isEmpty()) { + log.warn("Found empty hostIdCloudIpMap by needQueryHostIdList:{}", needQueryHostIdList); + return; + } + List hostIdWithBlankCloudIpList = new ArrayList<>(); + for (HostAgentStateQuery query : queryWithoutCloudIpList) { + Long hostId = query.getHostId(); + String cloudIp = hostIdCloudIpMap.get(hostId); + if (StringUtils.isNotBlank(cloudIp)) { + query.setCloudIp(cloudIp); + } else { + hostIdWithBlankCloudIpList.add(hostId); + } + } + if (!hostIdWithBlankCloudIpList.isEmpty()) { + log.warn( + "Found unexpected blank cloudIp by {} hostIds:{}", + hostIdWithBlankCloudIpList.size(), + hostIdWithBlankCloudIpList + ); + } + } + + private Map queryCloudIpsByHostId(List hostIdList) { + if (CollectionUtils.isEmpty(hostIdList)) { + return new HashMap<>(); + } + Map hostIdCloudIpMap = bizHostInfoQueryService.queryCloudIpsByHostId(hostIdList); + if (log.isDebugEnabled()) { + log.debug( + "queryCloudIpsByHostId, hostIdList={}, hostIdCloudIpMap={}", + hostIdList, + JsonUtils.toJson(hostIdCloudIpMap) + ); + } + return hostIdCloudIpMap; + } + + private void fillAgentIdsIfNeed(List hostAgentStateQueryList) { + if (CollectionUtils.isEmpty(hostAgentStateQueryList)) { + return; + } + List queryWithoutAgentIdList = new ArrayList<>(); + List needQueryHostIdList = new ArrayList<>(); + for (HostAgentStateQuery query : hostAgentStateQueryList) { + if (StringUtils.isBlank(query.getAgentId())) { + queryWithoutAgentIdList.add(query); + needQueryHostIdList.add(query.getHostId()); + } + } + if (CollectionUtils.isEmpty(needQueryHostIdList)) { + return; + } + Map hostIdAgentIdMap = queryAgentIdsByHostId(needQueryHostIdList); + if (hostIdAgentIdMap.isEmpty()) { + log.warn("Found empty hostIdAgentIdMap by needQueryHostIdList:{}", needQueryHostIdList); + return; + } + List hostIdWithBlankAgentIdList = new ArrayList<>(); + for (HostAgentStateQuery query : queryWithoutAgentIdList) { + Long hostId = query.getHostId(); + String agentId = hostIdAgentIdMap.get(hostId); + if (StringUtils.isNotBlank(agentId)) { + query.setAgentId(agentId); + } else { + hostIdWithBlankAgentIdList.add(hostId); + } + } + if (!hostIdWithBlankAgentIdList.isEmpty()) { + log.warn( + "Found unexpected blank agentId by {} hostIds:{}", + hostIdWithBlankAgentIdList.size(), + hostIdWithBlankAgentIdList + ); + } + } + + private Map queryAgentIdsByHostId(List hostIdList) { + if (CollectionUtils.isEmpty(hostIdList)) { + return new HashMap<>(); + } + Map hostIdAgentIdMap = bizHostInfoQueryService.queryAgentIdsByHostId(hostIdList); + if (log.isDebugEnabled()) { + log.debug( + "queryAgentIdsByHostId, hostIdList={}, hostIdAgentIdMap={}", + hostIdList, + JsonUtils.toJson(hostIdAgentIdMap) + ); + } + return hostIdAgentIdMap; + } + + @Override + public Map batchGetAgentAliveStatus(List hostAgentStateQueryList) { + Map agentStateMap = batchGetAgentState(hostAgentStateQueryList); + return AgentStateUtil.batchGetAgentAliveStatus(agentStateMap); + } +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/model/HostAgentStateQuery.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/model/HostAgentStateQuery.java new file mode 100644 index 0000000000..fe547f1b8a --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/service/model/HostAgentStateQuery.java @@ -0,0 +1,87 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.gse.service.model; + +import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.common.model.dto.HostSimpleDTO; +import lombok.Data; + + +@SuppressWarnings("DuplicatedCode") +@Data +public class HostAgentStateQuery { + /** + * 主机ID + */ + private Long hostId; + /** + * cmdb业务ID + */ + private Long bizId; + /** + * 云区域IP + */ + private String cloudIp; + /** + * AgentId + */ + private String agentId; + + public static HostAgentStateQuery from(HostDTO hostDTO) { + if (hostDTO == null) { + return null; + } + HostAgentStateQuery query = new HostAgentStateQuery(); + query.setHostId(hostDTO.getHostId()); + query.setCloudIp(hostDTO.toCloudIp()); + query.setAgentId(hostDTO.getAgentId()); + return query; + } + + public static HostAgentStateQuery from(HostSimpleDTO hostSimpleDTO) { + if (hostSimpleDTO == null) { + return null; + } + HostAgentStateQuery query = new HostAgentStateQuery(); + query.setHostId(hostSimpleDTO.getHostId()); + query.setBizId(hostSimpleDTO.getBizId()); + query.setCloudIp(hostSimpleDTO.getCloudIp()); + query.setAgentId(hostSimpleDTO.getAgentId()); + return query; + } + + public static HostAgentStateQuery from(ApplicationHostDTO applicationHostDTO) { + if (applicationHostDTO == null) { + return null; + } + HostAgentStateQuery query = new HostAgentStateQuery(); + query.setHostId(applicationHostDTO.getHostId()); + query.setBizId(applicationHostDTO.getBizId()); + query.setCloudIp(applicationHostDTO.getCloudIp()); + query.setAgentId(applicationHostDTO.getAgentId()); + return query; + } +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/util/AgentStateUtil.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/util/AgentStateUtil.java new file mode 100644 index 0000000000..94acd590e5 --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/util/AgentStateUtil.java @@ -0,0 +1,46 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.gse.util; + +import com.tencent.bk.job.common.gse.constants.AgentAliveStatusEnum; +import com.tencent.bk.job.common.gse.v2.model.resp.AgentState; + +import java.util.HashMap; +import java.util.Map; + +public class AgentStateUtil { + + public static Map batchGetAgentAliveStatus(Map agentStateMap) { + Map agentAliveStatusMap = new HashMap<>(); + for (Map.Entry entry : agentStateMap.entrySet()) { + String agentId = entry.getKey(); + AgentState agentState = entry.getValue(); + agentAliveStatusMap.put(agentId, + AgentAliveStatusEnum.fromAgentState(agentState) == AgentAliveStatusEnum.ALIVE); + } + return agentAliveStatusMap; + } + +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/util/AgentUtils.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/util/AgentUtils.java index f305780ffb..e72ae7b999 100644 --- a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/util/AgentUtils.java +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/util/AgentUtils.java @@ -1,43 +1,48 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - package com.tencent.bk.job.common.gse.util; -import com.tencent.bk.job.common.gse.constants.AgentStatusEnum; +import org.apache.commons.lang3.StringUtils; + +import java.util.regex.Pattern; -public final class AgentUtils { +/** + * GSE Agent 工具类 + */ +public class AgentUtils { + private static final Pattern GSE_V1_AGENT_ID_PATTERN = Pattern.compile( + "(\\d+):((?!\\d\\d\\d)\\d+|1\\d\\d|2[0-4]\\d|25[0-5])\\.((?!\\d\\d\\d)\\d+|1\\d\\d|2[0-4]\\d|25[0-5])\\.(" + + "(?!\\d\\d\\d)\\d+|1\\d\\d|2[0-4]\\d|25[0-5])\\.((?!\\d\\d\\d)\\d+|1\\d\\d|2[0-4]\\d|25[0-5])"); - private AgentUtils() { + /** + * 是否是GSE1.0 AgentId + * + * @param agentId agentId + */ + public static boolean isGseV1AgentId(String agentId) { + if (StringUtils.isEmpty(agentId)) { + return false; + } + return GSE_V1_AGENT_ID_PATTERN.matcher(agentId).matches(); } /** - * 返回Agent的状态是否正常 + * 是否是GSE2.0 AgentId * - * @param status agent的状态 - * @return 正常: true, 异常: false + * @param agentId agentId + */ + public static boolean isGseV2AgentId(String agentId) { + if (StringUtils.isEmpty(agentId)) { + return false; + } + return !isGseV1AgentId(agentId); + } + + /** + * 对接 GSE V1 的 agentId 值为 云区域:ip（内部实现，产品上 GSE V1 Agent 并没有 AgentId 的概念）。 + * 只有GSE V2 Agent 才会在 cmdb 注册真实的 agentId。 + * 为了避免与cmdb 主机 AgentId 属性的理解上的歧义，需要把内部实现上的 GSE V1 agentId 隐藏 */ - public static boolean isAgentOkByStatus(int status) { - return status == AgentStatusEnum.ALIVE.getValue(); + public static String displayAsRealAgentId(String agentId) { + return AgentUtils.isGseV1AgentId(agentId) ? "" : agentId; } + } diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/util/GseFilePathUtils.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/util/GseFilePathUtils.java new file mode 100644 index 0000000000..d06753656e --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/util/GseFilePathUtils.java @@ -0,0 +1,61 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.gse.util; + +import com.tencent.bk.job.common.util.FilePathUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; + +/** + * GSE文件路径解析器 + */ +public class GseFilePathUtils { + + /** + * 标准化GSE返回的文件路径 + * + * @param filePath GSE返回的文件路径 + * @return 标准化文件路径 + */ + public static String standardizedGSEFilePath(String filePath) { + if (StringUtils.isEmpty(filePath)) { + return ""; + } + // GSE对于Windows的路径，不管下发参数dir中是否包含路径分隔符，都会默认加上/,比如C:\Users\/；这里需要对路径进行标准化 + Pair dirAndFileName = FilePathUtils.parseDirAndFileName(filePath); + String dir = dirAndFileName.getLeft(); + String fileName = dirAndFileName.getRight(); + int lastBackSlashIndex = dir.lastIndexOf("\\"); + if (lastBackSlashIndex == -1) { + return dir + fileName; + } + // "C:\/"这种路径，需要删除掉最后一个GSE添加的"/" + if (lastBackSlashIndex == dir.length() - 2) { + return dir.substring(0, lastBackSlashIndex + 1) + fileName; + } else { + return dir + fileName; + } + } +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/util/K8sUtils.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/util/K8sUtils.java new file mode 100644 index 0000000000..006d83c3e5 --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/util/K8sUtils.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.gse.util; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * k8s 工具类 + */ +public class K8sUtils { + private static final Pattern PATTERN_K8S_CONTAINER_ID = Pattern.compile("^([a-zA-Z]+://)([a-f0-9]{64}$)"); + + + /** + * ://, 删除 :// + */ + public static String removeContainerIdType(String containerId) { + Matcher matcher = PATTERN_K8S_CONTAINER_ID.matcher(containerId); + return matcher.find() ? matcher.group(2) : containerId; + } +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/util/ScriptRequestBuilder.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/util/ScriptRequestBuilder.java new file mode 100644 index 0000000000..55b68a0b39 --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/util/ScriptRequestBuilder.java @@ -0,0 +1,120 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.gse.util; + +import com.tencent.bk.job.common.gse.v2.model.Agent; +import com.tencent.bk.job.common.gse.v2.model.AtomicScriptTask; +import com.tencent.bk.job.common.gse.v2.model.AtomicScriptTaskRelation; +import com.tencent.bk.job.common.gse.v2.model.ExecuteScriptRequest; +import com.tencent.bk.job.common.gse.v2.model.GseScript; +import org.apache.commons.lang3.StringUtils; + +import java.util.ArrayList; +import java.util.List; + +/** + * GSE脚本下发请求参数构造器 + */ +public class ScriptRequestBuilder { + private final ExecuteScriptRequest request; + + public ScriptRequestBuilder() { + request = new ExecuteScriptRequest(); + } + + /** + * 添加脚本文件 + * + * @param storeDir 脚本文件存储目录 + * @param scriptFileName 脚本文件名称 + * @param scriptContent 脚本内容 + */ + public ScriptRequestBuilder addScriptFile(String storeDir, String scriptFileName, String scriptContent) { + GseScript gseScript = new GseScript(); + gseScript.setStoreDir(storeDir); + gseScript.setName(scriptFileName); + gseScript.setContent(scriptContent); + request.addScript(gseScript); + return this; + } + + /** + * 新增脚本任务 + * + * @param agents 目标agent列表 + * @param storeDir 脚本文件存储目录 + * @param scriptFileName 脚本文件名称 + * @param scriptParam 脚本执行参数 + * @param timeout 超时时间 + */ + public ScriptRequestBuilder addScriptTask(List agents, String storeDir, String scriptFileName, + String scriptParam, int timeout) { + AtomicScriptTask atomicScriptTask = new AtomicScriptTask(); + atomicScriptTask.setTimeout(timeout); + + int atomicTaskId = request.getAtomicTasks().size(); + atomicScriptTask.setTaskId(atomicTaskId); + + String exeCmd = storeDir + "/" + scriptFileName; + if (StringUtils.isNotBlank(scriptParam)) { + exeCmd += " " + scriptParam; + } + atomicScriptTask.setCommand(exeCmd); + + request.addAtomicScriptTask(atomicScriptTask); + + //为了保证任务串行执行，需要设置relation参数 + if (atomicTaskId > 0) { + AtomicScriptTaskRelation relation = new AtomicScriptTaskRelation(); + relation.setTaskId(atomicTaskId); + + List dependencyTaskIdList = new ArrayList<>(); + int dependencyTaskId = 0; + while (dependencyTaskId < atomicTaskId) { + dependencyTaskIdList.add(dependencyTaskId); + dependencyTaskId++; + } + relation.setIndex(dependencyTaskIdList); + request.addAtomicTaskRelation(relation); + } + + request.setAgents(agents); + + return this; + } + + /** + * 设置Windows执行脚本的命令行解释器 + * + * @param windowsInterpreter Windows执行脚本的命令行解释器 + */ + public void setWindowsInterpreter(String windowsInterpreter) { + request.setWindowsInterpreter(windowsInterpreter); + } + + public ExecuteScriptRequest build() { + return request; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/WindowsHelper.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/util/WindowsHelper.java similarity index 94% rename from src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/WindowsHelper.java rename to src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/util/WindowsHelper.java index 75fa32e92a..f585024db9 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/WindowsHelper.java +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/util/WindowsHelper.java @@ -1,7 +1,7 @@ -/* +package com.tencent.bk.job.common.gse.util;/* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,6 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.execute.engine.util; import java.util.HashSet; import java.util.Set; diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/BKTSSLTransportFactory.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/BKTSSLTransportFactory.java new file mode 100644 index 0000000000..6dd9330202 --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/BKTSSLTransportFactory.java @@ -0,0 +1,298 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.gse.v1; + +import org.apache.thrift.transport.TServerSocket; +import org.apache.thrift.transport.TSocket; +import org.apache.thrift.transport.TTransportException; + +import javax.net.ssl.KeyManager; +import javax.net.ssl.KeyManagerFactory; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLServerSocket; +import javax.net.ssl.SSLServerSocketFactory; +import javax.net.ssl.SSLSocket; +import javax.net.ssl.SSLSocketFactory; +import javax.net.ssl.TrustManager; +import javax.net.ssl.TrustManagerFactory; +import javax.net.ssl.X509TrustManager; +import java.io.FileInputStream; +import java.io.IOException; +import java.net.InetAddress; +import java.security.KeyStore; +import java.security.SecureRandom; +import java.security.cert.X509Certificate; +import java.util.Arrays; + +/** + * 针对Thrift的SSL做免TrustManager验证，主要是因为centos 7.x以上系统，可能存在认证异常 + * Caused by: sun.security.validator.ValidatorException: KeyUsage does not allow key encipherment + * at sun.security.validator.EndEntityChecker.checkTLSServer(EndEntityChecker.java:264) + * at sun.security.validator.EndEntityChecker.check(EndEntityChecker.java:141) + * at sun.security.validator.Validator.validate(Validator.java:264) + * at sun.security.ssl.X509TrustManagerImpl.validate(X509TrustManagerImpl.java:324) + * at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:229) + * at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:124) + * at sun.security.ssl.ClientHandshaker.serverCertificate(ClientHandshaker.java:1496) + * + * @version 1.0 + * @time 2017/6/19. + */ +public class BKTSSLTransportFactory { + private final static X509TrustManager trustManager = new X509TrustManager() { + public void checkClientTrusted(X509Certificate[] xcs, String string) { + } + + public void checkServerTrusted(X509Certificate[] xcs, String string) { + } + + public X509Certificate[] getAcceptedIssuers() { + return null; + } + }; + + public BKTSSLTransportFactory() { + } + + public static TServerSocket getServerSocket(int port) throws TTransportException { + return getServerSocket(port, 0); + } + + public static TServerSocket getServerSocket(int port, int clientTimeout) throws TTransportException { + return getServerSocket(port, clientTimeout, false, (InetAddress) null); + } + + public static TServerSocket getServerSocket(int port, int clientTimeout, boolean clientAuth, + InetAddress ifAddress) throws TTransportException { + SSLServerSocketFactory factory = (SSLServerSocketFactory) SSLServerSocketFactory.getDefault(); + return createServer(factory, port, clientTimeout, clientAuth, ifAddress, null); + } + + public static TServerSocket getServerSocket(int port, int clientTimeout, InetAddress ifAddress, + TSSLTransportParameters params) throws TTransportException { + if (params != null && (params.isKeyStoreSet || params.isTrustStoreSet)) { + SSLContext ctx = createSSLContext(params); + return createServer(ctx.getServerSocketFactory(), port, clientTimeout, params.clientAuth, ifAddress, + params); + } else { + throw new TTransportException("Either one of the KeyStore or TrustStore must be set for " + + "SSLTransportParameters"); + } + } + + private static TServerSocket createServer(SSLServerSocketFactory factory, int port, int timeout, + boolean clientAuth, InetAddress ifAddress, + TSSLTransportParameters params) throws TTransportException { + try { + SSLServerSocket serverSocket = (SSLServerSocket) factory.createServerSocket(port, 100, ifAddress); + serverSocket.setSoTimeout(timeout); + serverSocket.setNeedClientAuth(clientAuth); + if (params != null && params.cipherSuites != null) { + serverSocket.setEnabledCipherSuites(params.cipherSuites); + } + + return new TServerSocket((new TServerSocket.ServerSocketTransportArgs()) + .serverSocket(serverSocket).clientTimeout(timeout)); + } catch (Exception var7) { + throw new TTransportException("Could not bind to port " + port, var7); + } + } + + public static TSocket getClientSocket(String host, int port, int timeout) throws TTransportException { + SSLSocketFactory factory = (SSLSocketFactory) SSLSocketFactory.getDefault(); + return createClient(factory, host, port, timeout); + } + + public static TSocket getClientSocket(String host, int port) throws TTransportException { + return getClientSocket(host, port, 0); + } + + public static TSocket getClientSocket(String host, int port, int timeout, + TSSLTransportParameters params) throws TTransportException { + if (params != null && (params.isKeyStoreSet || params.isTrustStoreSet)) { + SSLContext ctx = createSSLContext(params); + return createClient(ctx.getSocketFactory(), host, port, timeout); + } else { + throw new TTransportException("Either one of the KeyStore or TrustStore must be set for " + + "SSLTransportParameters"); + } + } + + private static SSLContext createSSLContext(TSSLTransportParameters params) throws TTransportException { + FileInputStream fin = null; + FileInputStream fis = null; + + SSLContext ctx; + try { + ctx = SSLContext.getInstance(params.protocol); + TrustManagerFactory tmf = null; + KeyManagerFactory kmf = null; + KeyStore ks; + if (params.isTrustStoreSet) { + tmf = TrustManagerFactory.getInstance(params.trustManagerType); + ks = KeyStore.getInstance(params.trustStoreType); + fin = new FileInputStream(params.trustStore); + ks.load(fin, params.trustPass != null ? params.trustPass.toCharArray() : null); + tmf.init(ks); + } + + if (params.isKeyStoreSet) { + kmf = KeyManagerFactory.getInstance(params.keyManagerType); + ks = KeyStore.getInstance(params.keyStoreType); + fis = new FileInputStream(params.keyStore); + ks.load(fis, params.keyPass.toCharArray()); + kmf.init(ks, params.keyPass.toCharArray()); + } + + if (params.isKeyStoreSet && params.isTrustStoreSet) { + ctx.init(kmf.getKeyManagers(), new TrustManager[]{trustManager}, (SecureRandom) null); + } else if (params.isKeyStoreSet) { + ctx.init(kmf.getKeyManagers(), (TrustManager[]) null, (SecureRandom) null); + } else { + ctx.init((KeyManager[]) null, new TrustManager[]{trustManager}, (SecureRandom) null); + } + } catch (Exception var17) { + throw new TTransportException("Error creating the transport", var17); + } finally { + if (fin != null) { + try { + fin.close(); + } catch (IOException var16) { + var16.printStackTrace(); + } + } + + if (fis != null) { + try { + fis.close(); + } catch (IOException var15) { + var15.printStackTrace(); + } + } + + } + + return ctx; + } + + private static TSocket createClient(SSLSocketFactory factory, String host, int port, + int timeout) throws TTransportException { + try { + SSLSocket socket = (SSLSocket) factory.createSocket(host, port); + socket.setSoTimeout(timeout); + return new TSocket(socket); + } catch (Exception var5) { + throw new TTransportException("Could not connect to " + host + " on port " + port, var5); + } + } + + public static class TSSLTransportParameters { + protected String protocol; + protected String keyStore; + protected String keyPass; + protected String keyManagerType; + protected String keyStoreType; + protected String trustStore; + protected String trustPass; + protected String trustManagerType; + protected String trustStoreType; + protected String[] cipherSuites; + protected boolean clientAuth; + protected boolean isKeyStoreSet; + protected boolean isTrustStoreSet; + + public TSSLTransportParameters() { + this.protocol = "TLS"; + this.keyManagerType = KeyManagerFactory.getDefaultAlgorithm(); + this.keyStoreType = "JKS"; + this.trustManagerType = TrustManagerFactory.getDefaultAlgorithm(); + this.trustStoreType = "JKS"; + this.clientAuth = false; + this.isKeyStoreSet = false; + this.isTrustStoreSet = false; + } + + public TSSLTransportParameters(String protocol, String[] cipherSuites) { + this(protocol, cipherSuites, false); + } + + public TSSLTransportParameters(String protocol, String[] cipherSuites, boolean clientAuth) { + this.protocol = "TLS"; + this.keyManagerType = KeyManagerFactory.getDefaultAlgorithm(); + this.keyStoreType = "JKS"; + this.trustManagerType = TrustManagerFactory.getDefaultAlgorithm(); + this.trustStoreType = "JKS"; + this.clientAuth = false; + this.isKeyStoreSet = false; + this.isTrustStoreSet = false; + if (protocol != null) { + this.protocol = protocol; + } + + this.cipherSuites = (String[]) Arrays.copyOf(cipherSuites, cipherSuites.length); + this.clientAuth = clientAuth; + } + + public void setKeyStore(String keyStore, String keyPass, String keyManagerType, String keyStoreType) { + this.keyStore = keyStore; + this.keyPass = keyPass; + if (keyManagerType != null) { + this.keyManagerType = keyManagerType; + } + + if (keyStoreType != null) { + this.keyStoreType = keyStoreType; + } + + this.isKeyStoreSet = true; + } + + public void setKeyStore(String keyStore, String keyPass) { + this.setKeyStore(keyStore, keyPass, (String) null, (String) null); + } + + public void setTrustStore(String trustStore, String trustPass, String trustManagerType, String trustStoreType) { + this.trustStore = trustStore; + this.trustPass = trustPass; + if (trustManagerType != null) { + this.trustManagerType = trustManagerType; + } + + if (trustStoreType != null) { + this.trustStoreType = trustStoreType; + } + + this.isTrustStoreSet = true; + } + + public void setTrustStore(String trustStore, String trustPass) { + this.setTrustStore(trustStore, trustPass, (String) null, (String) null); + } + + public void requireClientAuth(boolean clientAuth) { + this.clientAuth = clientAuth; + } + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/CuratorFrameworkFactoryBean.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/CuratorFrameworkFactoryBean.java similarity index 88% rename from src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/CuratorFrameworkFactoryBean.java rename to src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/CuratorFrameworkFactoryBean.java index 4f4603c96b..c0b1b725df 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/CuratorFrameworkFactoryBean.java +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/CuratorFrameworkFactoryBean.java @@ -2,7 +2,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -23,8 +23,9 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.execute.config; +package com.tencent.bk.job.common.gse.v1; +import com.tencent.bk.job.common.gse.config.GseV1Properties; import lombok.extern.slf4j.Slf4j; import org.apache.curator.RetryPolicy; import org.apache.curator.framework.CuratorFramework; @@ -55,8 +56,11 @@ public class CuratorFrameworkFactoryBean implements FactoryBean 0) { + try { + ip = gseCacheApiServerHosts[hostIndex]; + if (StringUtils.isBlank(ip)) { + continue; + } + return getAgent(ip, cacheApiServerProperties.getPort()); + } catch (TException e) { + String msg = MessageFormatter.format( + "Get GseCacheClient fail|{}:{}|msg={}", + ArrayUtil.toArray(ip, cacheApiServerProperties.getPort(), e.getMessage())) + .getMessage(); + log.error(msg, e); + if ((--tryTimes) == 0) { + return null; + } + } + } + } + } + + private GseCacheClient getAgent(String ip, int port) throws TException { + log.info("Enter GetGseCacheClient with ip=" + ip + ", port=" + port); + TTransport tTransport; + BKTSSLTransportFactory.TSSLTransportParameters params = + new BKTSSLTransportFactory.TSSLTransportParameters(); + params.setTrustStore(gseV1Properties.getSsl().getTrustStore().getPath(), + gseV1Properties.getSsl().getTrustStore().getPassword(), + gseV1Properties.getSsl().getTrustStore().getManagerType(), + gseV1Properties.getSsl().getTrustStore().getStoreType()); + params.setKeyStore(gseV1Properties.getSsl().getKeyStore().getPath(), + gseV1Properties.getSsl().getKeyStore().getPassword()); + StopWatch watch = new StopWatch("getAgent"); + + watch.start("BKTSSLTransportFactory.getClientSocket"); + TSocket tSocket = BKTSSLTransportFactory.getClientSocket(ip, port, 15000, params); + watch.stop(); + + watch.start("new TFramedTransport"); + tTransport = new TFramedTransport(tSocket); + watch.stop(); + + watch.start("new TBinaryProtocol"); + TProtocol tProtocol = new TBinaryProtocol(tTransport); + watch.stop(); + + watch.start("new CacheAPI.Client"); + CacheAPI.Client gseAgentClient = new CacheAPI.Client(tProtocol); + watch.stop(); + + watch.start("new GseCacheClient"); + GseCacheClient gseCacheClient = new GseCacheClient(gseAgentClient, tTransport); + watch.stop(); + + if (watch.getTotalTimeMillis() > 1000) { + log.warn("getAgent slow, statistics: " + watch.prettyPrint()); + } + return gseCacheClient; + } + +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/GseReadTimeoutException.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/GseReadTimeoutException.java new file mode 100644 index 0000000000..c48a5a5dcc --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/GseReadTimeoutException.java @@ -0,0 +1,39 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.gse.v1; + +/** + * GSE Server 读取超时异常 + */ +public class GseReadTimeoutException extends RuntimeException { + + public GseReadTimeoutException() { + super("Read timeout"); + } + + public GseReadTimeoutException(String errorMessage) { + super(errorMessage); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/gse/GseRequestPrinter.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/GseRequestPrinter.java similarity index 98% rename from src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/gse/GseRequestPrinter.java rename to src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/GseRequestPrinter.java index 1ab9c45cd3..6b09391dc4 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/gse/GseRequestPrinter.java +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/GseRequestPrinter.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.execute.engine.gse; +package com.tencent.bk.job.common.gse.v1; import com.tencent.bk.gse.taskapi.api_agent_task_rst; import com.tencent.bk.gse.taskapi.api_script_file; @@ -155,4 +155,5 @@ public static String printAgentTaskResult(api_agent_task_rst agentTaskRst) { sb.append(")"); return sb.toString(); } + } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/gse/GseServer.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/GseServer.java similarity index 83% rename from src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/gse/GseServer.java rename to src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/GseServer.java index a928788467..6751a52e3a 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/gse/GseServer.java +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/GseServer.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,12 +22,12 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.execute.engine.gse; +package com.tencent.bk.job.common.gse.v1; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.tencent.bk.job.common.gse.config.GseV1Properties; import com.tencent.bk.job.common.util.ArrayUtil; -import com.tencent.bk.job.execute.config.GseConfig; import lombok.extern.slf4j.Slf4j; import org.slf4j.helpers.MessageFormatter; import org.springframework.beans.factory.annotation.Autowired; @@ -44,20 +44,20 @@ public class GseServer { private final transient AtomicBoolean isInit = new AtomicBoolean(false); private List> servers; - @Autowired - private GseConfig gseConfig; - public GseServer() { + private final GseV1Properties gseV1Properties; + @Autowired + public GseServer(GseV1Properties gseV1Properties) { + this.gseV1Properties = gseV1Properties; } @PostConstruct public void init() { if (!isInit.get()) { Map maps = Maps.newHashMap(); - log.info("Init gseConfig, config={}", gseConfig); - String[] gseServerIps = gseConfig.getTaskServerHost().split(","); - int gseServerPort = gseConfig.getTaskServerPort(); + String[] gseServerIps = gseV1Properties.getTaskServer().getHost().split(","); + int gseServerPort = gseV1Properties.getTaskServer().getPort(); for (String gseServerIp : gseServerIps) { maps.put(gseServerIp, gseServerPort); } @@ -69,7 +69,7 @@ public void init() { /** * 构建gse访问客户端, 并连接服务端 */ - public GseClient getClient() { + public GseTaskClient getClient() { return getGseClient(Lists.newArrayList(servers)); } @@ -78,7 +78,7 @@ List> getGseServer() { } @SuppressWarnings("all") - GseClient getGseClient(List> servers) { + GseTaskClient getGseClient(List> servers) { if (servers == null) { return null; } @@ -93,10 +93,10 @@ GseClient getGseClient(List> servers) { } Map.Entry server = servers.get(ipIndex); try { - return GseClient.getClient(server.getKey(), server.getValue()); + return GseTaskClient.getClient(server.getKey(), server.getValue()); } catch (Throwable e) { String msg = MessageFormatter.format( - "Get GseClient fail|{}:{}|msg={}", + "Get GseTaskClient fail|{}:{}|msg={}", ArrayUtil.toArray(server.getKey(), server.getValue(), e.getMessage())) .getMessage(); log.error(msg, e); diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/GseTaskClient.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/GseTaskClient.java new file mode 100644 index 0000000000..08639c0bf3 --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/GseTaskClient.java @@ -0,0 +1,120 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.gse.v1; + +import com.tencent.bk.gse.taskapi.doSomeCmdV3; +import com.tencent.bk.job.common.gse.config.GseV1Properties; +import com.tencent.bk.job.common.util.ApplicationContextRegister; +import lombok.extern.slf4j.Slf4j; +import org.apache.thrift.TException; +import org.apache.thrift.protocol.TBinaryProtocol; +import org.apache.thrift.transport.TTransport; +import org.springframework.beans.BeansException; + +import java.io.Closeable; + +/** + * GSE Task Server Client + */ +@Slf4j +public class GseTaskClient implements Closeable { + + private static final String KEY_STORE; + private static final String TRUST_STORE; + private static final String KEY_STORE_PASS; + private static final String TRUST_STORE_PASS; + private static final String TRUST_MANAGER_TYPE; + private static final String TRUST_STORE_TYPE; + + static { + GseV1Properties gseV1Properties = ApplicationContextRegister.getBean(GseV1Properties.class); + KEY_STORE = gseV1Properties.getSsl().getKeyStore().getPath(); + KEY_STORE_PASS = gseV1Properties.getSsl().getKeyStore().getPassword(); + TRUST_STORE = gseV1Properties.getSsl().getTrustStore().getPath(); + TRUST_STORE_PASS = gseV1Properties.getSsl().getTrustStore().getPassword(); + TRUST_MANAGER_TYPE = gseV1Properties.getSsl().getTrustStore().getManagerType(); + TRUST_STORE_TYPE = gseV1Properties.getSsl().getTrustStore().getStoreType(); + } + + /** + * gse服务接口 + */ + private final doSomeCmdV3.Client gseAgentClient; + private final TTransport transport; + + private GseTaskClient(String ip, int port) throws TException { + BKTSSLTransportFactory.TSSLTransportParameters params = + new BKTSSLTransportFactory.TSSLTransportParameters(); + params.setTrustStore(TRUST_STORE, TRUST_STORE_PASS, TRUST_MANAGER_TYPE, TRUST_STORE_TYPE); + params.setKeyStore(KEY_STORE, KEY_STORE_PASS); + transport = BKTSSLTransportFactory.getClientSocket(ip, port, 60000, params); + TBinaryProtocol tProtocol = new TBinaryProtocol(transport); + this.gseAgentClient = new doSomeCmdV3.Client(tProtocol); + if (!transport.isOpen()) + transport.open(); + } + + public static GseTaskClient getClient() { + try { + GseServer gseServer = ApplicationContextRegister.getBean("gseServer"); + return gseServer.getClient(); + } catch (BeansException ignored) { + log.error("load gseSever failed!", ignored); + } + return null; + } + + /** + * 构建gse访问客户端, 并连接服务端 + * + * @param ip GSE IP + * @param port GSE Port + * @return GseClient + */ + static GseTaskClient getClient(String ip, int port) throws TException { + log.info("Get gseTaskClient, ip: {}, port: {}", ip, port); + return new GseTaskClient(ip, port); + } + + public doSomeCmdV3.Client getGseAgentClient() { + return gseAgentClient; + } + + /** + * 关闭连接 + */ + public void tearDown() { + try { + transport.close(); + } catch (Throwable ignored) { + } + } + + @Override + public void close() { + tearDown(); + } +} + diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/GseV1ApiClient.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/GseV1ApiClient.java new file mode 100644 index 0000000000..d8f7a7d1e1 --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/GseV1ApiClient.java @@ -0,0 +1,916 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.gse.v1; + +import com.tencent.bk.gse.cacheapi.AgentStatusRequestInfo; +import com.tencent.bk.gse.cacheapi.AgentStatusResponse; +import com.tencent.bk.gse.cacheapi.CacheIpInfo; +import com.tencent.bk.gse.cacheapi.CacheUser; +import com.tencent.bk.gse.taskapi.api_agent; +import com.tencent.bk.gse.taskapi.api_auth; +import com.tencent.bk.gse.taskapi.api_auto_task; +import com.tencent.bk.gse.taskapi.api_base_file_info; +import com.tencent.bk.gse.taskapi.api_comm_rsp; +import com.tencent.bk.gse.taskapi.api_copy_fileinfoV2; +import com.tencent.bk.gse.taskapi.api_host; +import com.tencent.bk.gse.taskapi.api_map_rsp; +import com.tencent.bk.gse.taskapi.api_query_agent_info_v2; +import com.tencent.bk.gse.taskapi.api_query_atom_task_info; +import com.tencent.bk.gse.taskapi.api_query_task_info_v2; +import com.tencent.bk.gse.taskapi.api_script_file; +import com.tencent.bk.gse.taskapi.api_script_request; +import com.tencent.bk.gse.taskapi.api_stop_task_request; +import com.tencent.bk.gse.taskapi.api_task_detail_result; +import com.tencent.bk.gse.taskapi.api_task_relation; +import com.tencent.bk.gse.taskapi.api_task_request; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.InternalException; +import com.tencent.bk.job.common.gse.IGseClient; +import com.tencent.bk.job.common.gse.constants.AgentStateStatusEnum; +import com.tencent.bk.job.common.gse.constants.FileDistModeEnum; +import com.tencent.bk.job.common.gse.constants.GseConstants; +import com.tencent.bk.job.common.gse.constants.GseMetricNames; +import com.tencent.bk.job.common.gse.constants.GseTaskTypeEnum; +import com.tencent.bk.job.common.gse.util.WindowsHelper; +import com.tencent.bk.job.common.gse.v1.model.AgentStatusDTO; +import com.tencent.bk.job.common.gse.v1.model.CopyFileRsp; +import com.tencent.bk.job.common.gse.v1.model.GSEFileTaskResult; +import com.tencent.bk.job.common.gse.v2.model.Agent; +import com.tencent.bk.job.common.gse.v2.model.AtomicFileTaskResult; +import com.tencent.bk.job.common.gse.v2.model.AtomicFileTaskResultContent; +import com.tencent.bk.job.common.gse.v2.model.ExecuteScriptRequest; +import com.tencent.bk.job.common.gse.v2.model.FileTaskResult; +import com.tencent.bk.job.common.gse.v2.model.FileTransferTask; +import com.tencent.bk.job.common.gse.v2.model.GetExecuteScriptResultRequest; +import com.tencent.bk.job.common.gse.v2.model.GetTransferFileResultRequest; +import com.tencent.bk.job.common.gse.v2.model.GseTaskResponse; +import com.tencent.bk.job.common.gse.v2.model.ScriptExecuteObjectTaskResult; +import com.tencent.bk.job.common.gse.v2.model.ScriptTaskResult; +import com.tencent.bk.job.common.gse.v2.model.SourceFile; +import com.tencent.bk.job.common.gse.v2.model.TargetFile; +import com.tencent.bk.job.common.gse.v2.model.TerminateGseTaskRequest; +import com.tencent.bk.job.common.gse.v2.model.TransferFileRequest; +import com.tencent.bk.job.common.gse.v2.model.req.ListAgentStateReq; +import com.tencent.bk.job.common.gse.v2.model.resp.AgentState; +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.common.util.DataSizeConverter; +import com.tencent.bk.job.common.util.FilePathUtils; +import com.tencent.bk.job.common.util.ThreadUtils; +import com.tencent.bk.job.common.util.ip.IpUtils; +import com.tencent.bk.job.common.util.json.JsonUtils; +import io.micrometer.core.instrument.MeterRegistry; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.thrift.TException; +import org.apache.thrift.transport.TTransportException; +import org.springframework.util.StopWatch; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +/** + * GSE V1 api client + */ +@Slf4j +public class GseV1ApiClient implements IGseClient { + + private final MeterRegistry meterRegistry; + private final GseCacheClientFactory gseCacheClientFactory; + + public GseV1ApiClient(MeterRegistry meterRegistry, + GseCacheClientFactory gseCacheClientFactory) { + this.meterRegistry = meterRegistry; + this.gseCacheClientFactory = gseCacheClientFactory; + } + + + @Override + public GseTaskResponse asyncExecuteScript(ExecuteScriptRequest request) { + api_script_request requestV1 = toV1ScriptRequest(request); + return sendScriptTaskRequest(requestV1); + } + + /** + * 创建GSE执行脚本请求 + * + * @param request 执行脚本任务通用请求 + * @return 执行脚本任务请求 - V1 + */ + private api_script_request toV1ScriptRequest(ExecuteScriptRequest request) { + api_script_request scriptReq = new api_script_request(); + + // 脚本文件 + List scriptFiles = new ArrayList<>(); + request.getScripts().forEach(script -> { + api_script_file scriptFile = new api_script_file(); + scriptFile.setDownload_path(script.getStoreDir()); + scriptFile.setName(script.getName()); + scriptFile.setContent(script.getContent()); + // 必须设置为空字符，否则GSE报错 + scriptFile.setMd5(""); + scriptFiles.add(scriptFile); + }); + scriptReq.setScripts(scriptFiles); + + // 脚本任务 + api_task_request taskReq = new api_task_request(); + taskReq.setAtomic_task_num(request.getAtomicTaskNum()); + taskReq.setVersion("1.0"); + // 必须设置为空字符，否则GSE报错 + taskReq.setCrond(""); + + // 脚本任务 - 目标 agent + List agents = buildAgents(request.getAgents()); + taskReq.setAgent_list(agents); + + // 脚本任务 - 原子任务 + List taskList = new ArrayList<>(); + request.getAtomicTasks().forEach(atomicScriptTask -> { + api_auto_task autoTask = new api_auto_task(); + autoTask.setAtomic_task_id(atomicScriptTask.getTaskId().byteValue()); + autoTask.setCmd(atomicScriptTask.getCommand()); + autoTask.setTimeout(atomicScriptTask.getTimeout()); + taskList.add(autoTask); + }); + taskReq.setAtomic_tasks(taskList); + + // 任务关系 + if (CollectionUtils.isNotEmpty(request.getAtomicScriptTaskRelations())) { + List relations = + request.getAtomicScriptTaskRelations().stream() + .map(relation -> new api_task_relation(relation.getTaskId().byteValue(), + relation.getIndex().stream().map(Integer::byteValue).collect(Collectors.toList()))) + .collect(Collectors.toList()); + + taskReq.setRel_list(relations); + } else { + // 必须设置为空的集合，否则GSE会报错 + taskReq.setRel_list(Collections.emptyList()); + } + + scriptReq.setTasks(taskReq); + + return scriptReq; + } + + /** + * 下发脚本任务 + * + * @param scriptRequest 请求内容 + * @return GSE Server响应 + */ + private GseTaskResponse sendScriptTaskRequest(api_script_request scriptRequest) { + return sendCmd(new GseTaskResponseCaller() { + @Override + public GseTaskResponse callback(GseTaskClient gseTaskClient) throws TException { + GseTaskResponse gseResponse = new GseTaskResponse(); + if (log.isDebugEnabled()) { + log.debug("RunScriptRequest: {}", scriptRequest); + } else if (log.isInfoEnabled()) { + log.info("RunScriptRequest: {}", + GseRequestPrinter.simplifyScriptRequest(scriptRequest)); + } + api_comm_rsp commRsp = gseTaskClient.getGseAgentClient().run_bash(scriptRequest); + log.info("RunScriptResponse: {}", commRsp.toString()); + + gseResponse.setErrorCode(commRsp.getBk_error_code()); + gseResponse.setErrorMessage(commRsp.getBk_error_msg()); + gseResponse.setGseTaskId(commRsp.getTask_id()); + return gseResponse; + } + + @Override + public String getApiName() { + return "run_bash"; + } + }); + } + + private List buildAgents(Collection agents) { + return agents.stream().map(this::buildAgent).collect(Collectors.toList()); + } + + private api_agent buildAgent(Agent agent) { + api_auth auth = new api_auth(); + auth.setUser(agent.getUser()); + if (agent.getPwd() != null) { + auth.setPassword(agent.getPwd()); + } else { + auth.setPassword(""); + } + + return buildAgent(agent.getAgentId(), auth); + } + + private api_agent buildAgent(String cloudIp, api_auth auth) { + // 在gse api v1, agentId=云区域:IP + int bkCloudId = GseConstants.DEFAULT_CLOUD_ID; + String ip; + String[] ipArray = cloudIp.split(":"); + if (ipArray.length > 1) { + bkCloudId = Integer.parseInt(ipArray[0]); + ip = ipArray[1]; + } else { + ip = ipArray[0]; + } + api_agent apiAgent = new api_agent(); + apiAgent.setIp(ip); + if (bkCloudId > 1) { + apiAgent.setGse_composite_id(bkCloudId); + } + + apiAgent.setAuth(auth); + return apiAgent; + } + + @Override + public ScriptTaskResult getExecuteScriptResult(GetExecuteScriptResultRequest request) { + api_query_task_info_v2 requestV1 = toV1QueryScriptTaskResult(request); + api_task_detail_result resultV1 = getScriptTaskDetailRst(requestV1); + return toScriptTaskResult(resultV1); + } + + private api_query_task_info_v2 toV1QueryScriptTaskResult(GetExecuteScriptResultRequest request) { + List queryAgentList = new ArrayList<>(); + for (GetExecuteScriptResultRequest.AgentTask agentTask : request.getAgentTasks()) { + String cloudIp = agentTask.getAgentId(); + api_query_agent_info_v2 queryAgentInfo = new api_query_agent_info_v2(); + + api_host apiHost = convertToApiHost(cloudIp); + queryAgentInfo.setHost(apiHost); + queryAgentInfo.setAtomic_tasks( + agentTask.getAtomicTasks().stream() + .map(atomicTask -> buildQueryAtomTaskInfo(atomicTask.getOffset(), atomicTask.getAtomicTaskId())) + .collect(Collectors.toList())); + + queryAgentList.add(queryAgentInfo); + } + + api_query_task_info_v2 queryTaskInfo = new api_query_task_info_v2(); + queryTaskInfo.setAgents(queryAgentList); + queryTaskInfo.setTask_id(request.getTaskId()); + return queryTaskInfo; + } + + private static api_host convertToApiHost(String cloudIp) { + String[] ipArray = cloudIp.split(":"); + + int cloudAreaId = 0; + String ip; + if (ipArray.length > 1) { + cloudAreaId = Integer.parseInt(ipArray[0]); + ip = ipArray[1]; + } else { + ip = ipArray[0]; + } + + api_host apiHost = new api_host(); + apiHost.setIp(ip); + if (cloudAreaId > 1) { + apiHost.setGse_composite_id(cloudAreaId); + } else { + apiHost.setGse_composite_id(0); + } + return apiHost; + } + + private ScriptTaskResult toScriptTaskResult(api_task_detail_result resultV1) { + ScriptTaskResult result = new ScriptTaskResult(); + if (CollectionUtils.isNotEmpty(resultV1.getResult())) { + List agentTaskResults = + resultV1.getResult().stream() + .map(agentTaskResultV1 -> { + ScriptExecuteObjectTaskResult agentTaskResult = new ScriptExecuteObjectTaskResult(); + agentTaskResult.setAgentId(agentTaskResultV1.getGse_composite_id() + ":" + + agentTaskResultV1.getIp()); + agentTaskResult.setAtomicTaskId(agentTaskResultV1.getAtomic_task_id()); + agentTaskResult.setErrorCode(agentTaskResultV1.getBk_error_code()); + agentTaskResult.setErrorMsg(agentTaskResultV1.getBk_error_msg()); + agentTaskResult.setStartTime(agentTaskResultV1.getStart_time()); + agentTaskResult.setEndTime(agentTaskResultV1.getEnd_time()); + agentTaskResult.setExitCode(agentTaskResultV1.getExitcode()); + agentTaskResult.setScreen(agentTaskResultV1.getScreen()); + agentTaskResult.setStatus(agentTaskResultV1.getStatus()); + agentTaskResult.setTag(agentTaskResultV1.getTag()); + return agentTaskResult; + }) + .collect(Collectors.toList()); + result.setResult(agentTaskResults); + } + + return result; + } + + @Override + public List listAgentState(ListAgentStateReq req) { + StopWatch watch = new StopWatch("listAgentState"); + + watch.start("queryAgentStatusFromCacheApi"); + List cloudIps = req.getAgentIdList(); + if (CollectionUtils.isEmpty(cloudIps)) { + log.info("cloudIps is empty"); + return Collections.emptyList(); + } + AgentStatusResponse agentStatusResponse = queryAgentStatusFromCacheApi(cloudIps); + watch.stop(); + + watch.start("parseResult"); + List agentStates = new ArrayList<>(); + if (agentStatusResponse.getResult() != null && !agentStatusResponse.getResult().isEmpty()) { + agentStatusResponse.getResult().forEach((agentId, stateStr) -> { + AgentState agentState = new AgentState(); + agentState.setStatusCode(parseCacheAgentStatus(stateStr).getValue()); + agentState.setAgentId(agentId); + agentStates.add(agentState); + }); + } + watch.stop(); + + if (watch.getTotalTimeMillis() > 1000) { + log.warn("listAgentState slow, statistics: " + watch.prettyPrint()); + } + return agentStates; + } + + /** + * 解析agent状态 预期解析的文本: {"businessid":"","exist":1,"ip":"1.1.1.1","region":"1","version":"NULL"} + */ + private AgentStateStatusEnum parseCacheAgentStatus(String statusStr) { + AgentStatusDTO agentStatus = JsonUtils.fromJson(statusStr, AgentStatusDTO.class); + if (agentStatus.getExist() != null && agentStatus.getExist().equals(1)) { + return AgentStateStatusEnum.RUNNING; + } else { + return AgentStateStatusEnum.NOT_FOUND; + } + } + + private AgentStatusResponse queryAgentStatusFromCacheApi(Collection agentIds) { + StopWatch watch = new StopWatch("queryAgentStatusFromCacheApi"); + + watch.start("gseCacheClientFactory.getClient"); + GseCacheClient gseClient = gseCacheClientFactory.getClient(); + watch.stop(); + if (null == gseClient) { + log.error("Get GSE cache client connection failed"); + throw new InternalException(ErrorCode.GSE_API_DATA_ERROR, + new String[]{"Get GSE cache client connection failed"}); + } + + long start = System.currentTimeMillis(); + String status = "ok"; + try { + + AgentStatusRequestInfo request = buildQueryAgentStatusRequest(agentIds); + + log.debug("QueryAgentStatus request: {}", request); + watch.start("quireAgentStatus"); + AgentStatusResponse response = gseClient.getCacheClient().quireAgentStatus(request); + watch.stop(); + log.debug("QueryAgentStatus response: {}", getResponseLogStr(response)); + return response; + } catch (Throwable e) { + log.error("QueryAgentStatus error", e); + status = "error"; + throw new InternalException(e, ErrorCode.GSE_API_DATA_ERROR); + } finally { + long end = System.currentTimeMillis(); + log.info("BatchGetAgentStatus {} agentIds, cost: {}ms", agentIds.size(), (end - start)); + if (this.meterRegistry != null) { + watch.start("reportMetric"); + meterRegistry.timer(GseMetricNames.GSE_API_METRICS_NAME_PREFIX, "api_name", "quireAgentStatus", + "status", status).record(end - start, TimeUnit.MICROSECONDS); + watch.stop(); + } + + watch.start("gseClient.tearDown"); + gseClient.tearDown(); + watch.stop(); + + if (watch.getTotalTimeMillis() > 1000) { + log.warn("queryAgentStatusFromCacheApi slow, statistics: " + watch.prettyPrint()); + } else { + log.info("queryAgentStatusFromCacheApi end"); + } + } + } + + private AgentStatusRequestInfo buildQueryAgentStatusRequest(Collection agentIds) { + AgentStatusRequestInfo request = new AgentStatusRequestInfo(); + + List ipInfoList = new ArrayList<>(); + for (String agentId : agentIds) { + String[] ipInfo = agentId.split(":"); + if (ipInfo.length != 2) { + log.warn("Request agentId format error! IP: {}", agentId); + continue; + } + + CacheIpInfo cacheIpInfo = new CacheIpInfo(); + cacheIpInfo.setPlatId(ipInfo[0]); + cacheIpInfo.setIp(ipInfo[1]); + ipInfoList.add(cacheIpInfo); + } + + CacheUser user = new CacheUser(); + user.setUser("bitmap"); + user.setPassword("bitmap"); + + request.setUser(user); + request.setIpinfos(ipInfoList); + + return request; + } + + private String getResponseLogStr(AgentStatusResponse response) { + return response == null ? null : response.toString().replace("\n", ""); + } + + @Override + public GseTaskResponse asyncTransferFile(TransferFileRequest request) { + List copyFileV1Request = toV1CopyFileInfoRequest(request); + return sendCopyFileTaskRequest(copyFileV1Request); + } + + private List toV1CopyFileInfoRequest(TransferFileRequest request) { + return request.getTasks().stream() + .map(task -> buildCopyFileInfo(task, request.getUploadSpeed(), + request.getDownloadSpeed(), request.getTimeout(), request.isAutoMkdir())) + .collect(Collectors.toList()); + } + + @Override + public FileTaskResult getTransferFileResult(GetTransferFileResultRequest request) { + api_map_rsp rsp; + if (CollectionUtils.isNotEmpty(request.getAgents())) { + rsp = pullCopyFileResult(request.getTaskId(), + request.getAgents().stream().map(Agent::getAgentId).collect(Collectors.toList())); + } else if (StringUtils.isNotBlank(request.getTaskId())) { + rsp = pullCopyFileResult(request.getTaskId()); + } else { + log.error("GetTransferFileResult, invalid request!"); + return null; + } + if (rsp == null) { + log.error("GetTransferFileResult, response is null!"); + return null; + } + return toFileTaskResult(rsp); + } + + private FileTaskResult toFileTaskResult(api_map_rsp rsp) { + FileTaskResult result = new FileTaskResult(); + if (rsp.getResult().isEmpty()) { + return null; + } + + List atomicFileTaskResults = new ArrayList<>(); + Set> ipResults = rsp.getResult().entrySet(); + for (Map.Entry ipResult : ipResults) { + CopyFileRsp copyFileRsp = parseCopyFileRsp(ipResult); + if (copyFileRsp == null) { + continue; + } + AtomicFileTaskResult atomicFileTaskResult = new AtomicFileTaskResult(); + atomicFileTaskResult.setErrorCode(copyFileRsp.getFinalErrorCode()); + atomicFileTaskResult.setErrorMsg(copyFileRsp.getFinalErrorMsg()); + + AtomicFileTaskResultContent content = new AtomicFileTaskResultContent(); + GSEFileTaskResult fileTaskResult = copyFileRsp.getGseFileTaskResult(); + content.setSourceAgentId(fileTaskResult.getSourceAgentId()); + content.setSourceFileDir(fileTaskResult.getSrcDirPath()); + content.setSourceFileName(fileTaskResult.getSrcFileName()); + content.setStandardSourceFilePath(fileTaskResult.getStandardSourceFilePath()); + content.setDestAgentId(fileTaskResult.getDestAgentId()); + content.setDestFileDir(fileTaskResult.getDestDirPath()); + content.setDestFileName(fileTaskResult.getDestFileName()); + content.setStandardDestFilePath(fileTaskResult.getStandardDestFilePath()); + content.setStartTime(fileTaskResult.getStartTime()); + content.setEndTime(fileTaskResult.getEndTime()); + content.setMode(fileTaskResult.getMode()); + content.setProgress(fileTaskResult.getProcess()); + content.setSize(fileTaskResult.getSize()); + content.setSpeed(fileTaskResult.getSpeed()); + content.setStatus(fileTaskResult.getStatus()); + content.setStatusInfo(fileTaskResult.getStatusDesc()); + content.setTaskId(fileTaskResult.getTaskId()); + content.setTaskType(fileTaskResult.getTaskType()); + content.setProtocolVersion(fileTaskResult.getProtocolVersion()); + atomicFileTaskResult.setContent(content); + atomicFileTaskResults.add(atomicFileTaskResult); + } + result.setAtomicFileTaskResults(atomicFileTaskResults); + return result; + } + + private CopyFileRsp parseCopyFileRsp(Map.Entry ipResult) { + log.info("ParseCopyFileRsp: {}", ipResult); + String taskInfo = ipResult.getValue(); + CopyFileRsp copyFileRsp; + try { + copyFileRsp = JsonUtils.fromJson(taskInfo, CopyFileRsp.class); + if (copyFileRsp == null) { + return null; + } + } catch (Throwable e) { + log.error("Parse CopyFileRsp error", e); + return null; + } + + boolean isStandardGSEProtocol = isStandardGSEProtocol(copyFileRsp.getGseFileTaskResult()); + if (!isStandardGSEProtocol) { + log.warn("Not suggested agent file task result protocol version, please upgrade agent to 1.7.21+)"); + parseCopyFileRspFromResultKey(copyFileRsp, ipResult.getKey()); + } + + return copyFileRsp; + } + + /** + * 是否是标准的GSE 文件协议 (协议版本号 > 1) + * + * @param fileTaskResult 文件任务返回协议 + */ + private boolean isStandardGSEProtocol(GSEFileTaskResult fileTaskResult) { + return fileTaskResult != null && fileTaskResult.getProtocolVersion() != null + && fileTaskResult.getProtocolVersion() > 1; + } + + /** + * tmp: 临时兼容，等GSE Agent 全面升级到1.7.6+版本之后，不再支持 + * + * @param copyFileRsp + * @param resultKey + */ + private void parseCopyFileRspFromResultKey(CopyFileRsp copyFileRsp, String resultKey) { + if (!(resultKey.startsWith(FileDistModeEnum.UPLOAD.getName()) + || resultKey.startsWith(FileDistModeEnum.DOWNLOAD.getName()))) { + log.warn("Invalid resultKey: {}, ignore", resultKey); + return; + } + + // 从key中提取任务信息 + String[] taskProps = resultKey.split(":"); + GSEFileTaskResult fileTaskResult = copyFileRsp.getGseFileTaskResult(); + if (fileTaskResult == null) { + fileTaskResult = new GSEFileTaskResult(); + copyFileRsp.setGseFileTaskResult(fileTaskResult); + } + if (fileTaskResult.getMode() == null) { + fileTaskResult.setMode(parseFileTaskModeFromKey(taskProps).getValue()); + } + HostDTO cloudIp = parseCloudIpFromKey(taskProps); + if (FileDistModeEnum.DOWNLOAD.getValue().equals(fileTaskResult.getMode())) { + // 格式: "download:srcIpInt:srcIpInt:destFilePath:destCloudId:destIp" + fileTaskResult.setDestIp(cloudIp.getIp()); + fileTaskResult.setDestCloudId(cloudIp.getBkCloudId()); + // GSE BUG, srcIpInt可能为-1(download:-1:2130706433:/tmp/1.log:0:127.0.0.2) + String srcIpInt = "-1".equals(taskProps[1]) ? taskProps[2] : taskProps[1]; + fileTaskResult.setSourceIp(IpUtils.revertIpFromNumericalStr(srcIpInt)); + // GSE BUG, 只有源主机IP，没有云区域ID + fileTaskResult.setSourceCloudId(null); + // GSE BUG, 只有目标文件信息，没有源文件信息 + String destFilePath = parseFilePathFromKey(taskProps); + Pair dirAndFileName = FilePathUtils.parseDirAndFileName(destFilePath); + fileTaskResult.setDestDirPath(dirAndFileName.getLeft()); + fileTaskResult.setDestFileName(dirAndFileName.getRight()); + } else { + // 格式: "upload:srcIpInt:srcFilePath:srcCloudId:srcIp" + fileTaskResult.setSourceIp(cloudIp.getIp()); + fileTaskResult.setSourceCloudId(cloudIp.getBkCloudId()); + String sourceFilePath = parseFilePathFromKey(taskProps); + Pair dirAndFileName = FilePathUtils.parseDirAndFileName(sourceFilePath); + fileTaskResult.setSrcDirPath(dirAndFileName.getLeft()); + fileTaskResult.setSrcFileName(dirAndFileName.getRight()); + } + } + + private String parseFilePathFromKey(String[] taskProps) { + String filePath = taskProps[taskProps.length - 3]; + if (taskProps.length > 4 && taskProps[taskProps.length - 4] != null) { + // 如果是正则的文件， /tmp/REGEX:abc.*.txt 这种有:，在key中会被分开，要拼回去 + // GSE 的Redis Key问题 可能引入空格变=号，导致key被当成key=value, 所以要判断 taskProps.length > 4 + // Windows路径包含: + if (taskProps[taskProps.length - 4].endsWith("REGEX") + || WindowsHelper.isWindowsDiskPartition(taskProps[taskProps.length - 4])) { + filePath = taskProps[taskProps.length - 4] + ":" + filePath; + } + } + return filePath; + } + + + private HostDTO parseCloudIpFromKey(String[] taskProps) { + String ip = taskProps[taskProps.length - 1]; + long cloudAreaId = Long.parseLong(taskProps[taskProps.length - 2].trim()); + return new HostDTO(cloudAreaId, ip); + } + + private FileDistModeEnum parseFileTaskModeFromKey(String[] taskProps) { + String mode = taskProps[0]; + if (FileDistModeEnum.UPLOAD.getName().equals(mode)) { + return FileDistModeEnum.UPLOAD; + } else if (FileDistModeEnum.DOWNLOAD.getName().equals(mode)) { + return FileDistModeEnum.DOWNLOAD; + } else { + throw new IllegalArgumentException("Invalid file dist mode: " + mode); + } + } + + @Override + public GseTaskResponse terminateGseFileTask(TerminateGseTaskRequest request) { + return sendForceStopTaskRequest(toV1StopTaskRequest(request, GseTaskTypeEnum.FILE)); + } + + private api_stop_task_request toV1StopTaskRequest(TerminateGseTaskRequest request, + GseTaskTypeEnum taskType) { + api_stop_task_request stopRequest = new api_stop_task_request(); + stopRequest.setStop_task_id(request.getTaskId()); + if (CollectionUtils.isNotEmpty(request.getAgents())) { + stopRequest.setAgents(request.getAgents().stream() + .map(agent -> + // 终止任务并不不需要账号密码,此处传入为了绕过thrift协议的校验 + buildAgent(agent.getAgentId(), buildEmptyApiAuth())) + .collect(Collectors.toList())); + } + stopRequest.setType(taskType.getValue()); + return stopRequest; + } + + private api_auth buildEmptyApiAuth() { + api_auth auth = new api_auth(); + auth.setUser("job"); + auth.setPassword(""); + return auth; + } + + @Override + public GseTaskResponse terminateGseScriptTask(TerminateGseTaskRequest request) { + return sendForceStopTaskRequest(toV1StopTaskRequest(request, GseTaskTypeEnum.SCRIPT)); + } + + + /** + * 构建 GSE 拷贝文件请求 + * + * @param task 文件拷贝任务 + * @param uploadSpeedLimit 上传限速，null 表示不限速 + * @param downloadSpeedLimit 下载限速，null 表示不限速 + * @param timeout 任务超时时间 + * @param autoMkdir 目标目录不存在，是否自动创建目录 + * @return 拷贝文件请求 + */ + public api_copy_fileinfoV2 buildCopyFileInfo(FileTransferTask task, + Integer uploadSpeedLimit, + Integer downloadSpeedLimit, + Integer timeout, + boolean autoMkdir) { + api_copy_fileinfoV2 copyFileInfo = new api_copy_fileinfoV2(); + + api_base_file_info baseFileInfo = new api_base_file_info(); + SourceFile sourceFile = task.getSource(); + TargetFile targetFile = task.getTarget(); + baseFileInfo.setName(sourceFile.getFileName()); + baseFileInfo.setPath(sourceFile.getStoreDir()); + baseFileInfo.setDest_path(targetFile.getStoreDir()); + baseFileInfo.setDest_name(targetFile.getFileName()); + baseFileInfo.setBackup_name(StringUtils.EMPTY); + baseFileInfo.setBackup_path(StringUtils.EMPTY); + baseFileInfo.setOwner(StringUtils.EMPTY); + baseFileInfo.setMd5(StringUtils.EMPTY); + + copyFileInfo.setFile(baseFileInfo); + copyFileInfo.setSrc_agent(buildAgent(sourceFile.getAgent())); + copyFileInfo.setDest_agents(buildAgents(targetFile.getAgents())); + if (uploadSpeedLimit != null) { + // MB -> KB + copyFileInfo.setUpload_speed(DataSizeConverter.convertMBToKB(uploadSpeedLimit)); + } + if (downloadSpeedLimit != null) { + // MB -> KB + copyFileInfo.setDownload_speed(DataSizeConverter.convertMBToKB(downloadSpeedLimit)); + } + if (timeout != null) { + copyFileInfo.setTimeout(timeout); + } + copyFileInfo.setMkdirflag(autoMkdir ? 1 : 0); + return copyFileInfo; + } + + /** + * 下发文件分发任务 + * + * @param copyFileInfoList 请求内容 + * @return GSE Server响应 + */ + public GseTaskResponse sendCopyFileTaskRequest(List copyFileInfoList) { + return sendCmd(new GseTaskResponseCaller() { + @Override + public GseTaskResponse callback(GseTaskClient gseTaskClient) throws TException { + GseTaskResponse gseResponse = new GseTaskResponse(); + log.info("FileCopyTaskRequest: {}", copyFileInfoList); + api_comm_rsp commRsp = gseTaskClient.getGseAgentClient().copy_file_v2(copyFileInfoList); + log.info("FileCopyTaskResponse: {}", commRsp); + gseResponse.setErrorCode(commRsp.getBk_error_code()); + gseResponse.setErrorMessage(commRsp.getBk_error_msg()); + gseResponse.setGseTaskId(commRsp.getTask_id()); + return gseResponse; + } + + @Override + public String getApiName() { + return "copy_file_v2"; + } + }); + } + + /** + * 强制终止任务 + * + * @param stopTaskRequest 终止任务请求 + * @return GSE SERVER 响应 + */ + public GseTaskResponse sendForceStopTaskRequest(api_stop_task_request stopTaskRequest) { + return sendCmd(new GseTaskResponseCaller() { + @Override + public GseTaskResponse callback(GseTaskClient gseTaskClient) throws TException { + GseTaskResponse gseResponse = new GseTaskResponse(); + log.info("ForceStopTaskRequest: {}", stopTaskRequest); + api_comm_rsp commRsp = gseTaskClient.getGseAgentClient().stop_task(stopTaskRequest); + log.info("ForceStopTaskResponse: {}", commRsp); + gseResponse.setErrorCode(commRsp.getBk_error_code()); + gseResponse.setErrorMessage(commRsp.getBk_error_msg()); + gseResponse.setGseTaskId(commRsp.getTask_id()); + return gseResponse; + } + + @Override + public String getApiName() { + return "stop_task"; + } + }); + } + + private api_query_atom_task_info buildQueryAtomTaskInfo(int offset, int mid) { + api_query_atom_task_info queryAtomTaskInfo = new api_query_atom_task_info(); + queryAtomTaskInfo.setId((byte) mid); + queryAtomTaskInfo.setOffset(offset); + return queryAtomTaskInfo; + } + + + /** + * 获取文件任务执行结果 + * + * @param gseTaskId gse任务ID + * @return 结果 + */ + private api_map_rsp pullCopyFileResult(String gseTaskId) { + return sendCmd(new GseApiCallback() { + @Override + public api_map_rsp callback(GseTaskClient gseTaskClient) throws TException { + log.info("GetCopyFileResult|request|gseTaskId: {}", gseTaskId); + api_map_rsp copyFileTaskLog = gseTaskClient.getGseAgentClient().get_copy_file_result(gseTaskId); + log.info("GetCopyFileResult|response: {}", copyFileTaskLog); + return copyFileTaskLog; + } + + @Override + public String getApiName() { + return "get_copy_file_result"; + } + }); + } + + /** + * 拉取文件任务执行结果-根据IP + */ + private api_map_rsp pullCopyFileResult(String gseTaskId, Collection cloudIps) { + return sendCmd(new GseApiCallback() { + @Override + public api_map_rsp callback(GseTaskClient gseTaskClient) throws TException { + List hosts = cloudIps.stream() + .map(GseV1ApiClient::convertToApiHost).collect(Collectors.toList()); + log.info("GetCopyFileResultByAgent|gseTaskId:{}|hosts:{}", gseTaskId, hosts); + api_map_rsp copyFileTaskLog = gseTaskClient.getGseAgentClient() + .get_copy_file_result_by_ip_v2(gseTaskId, hosts); + log.info("GetCopyFileResultByAgent|response:{}", copyFileTaskLog); + return copyFileTaskLog; + } + + @Override + public String getApiName() { + return "get_copy_file_result_by_ip_v2"; + } + }); + } + + private api_task_detail_result getScriptTaskDetailRst(api_query_task_info_v2 taskQuery) { + return sendCmd(new GseApiCallback() { + @Override + public api_task_detail_result callback(GseTaskClient gseTaskClient) throws TException { + log.info("GetScriptTaskDetailRequest: {}", taskQuery); + api_task_detail_result taskDetailRst = + gseTaskClient.getGseAgentClient().get_task_detail_result(taskQuery); + log.info("GetScriptTaskDetailResponse: {}", GseRequestPrinter.printScriptTaskResult(taskDetailRst)); + return taskDetailRst; + } + + @Override + public String getApiName() { + return "get_task_detail_result"; + } + }); + } + + private T sendCmd(GseApiCallback caller) { + int retry = 1; + boolean connect = true; + do { + String status = "ok"; + long start = System.nanoTime(); + try (GseTaskClient gseTaskClient = GseTaskClient.getClient()) { + connect = true; + if (gseTaskClient == null) { + connect = false; + // 如果拿不到连接 ，则等待3s重试 + log.info("Get GseTaskClient, retry after 3000ms"); + ThreadUtils.sleep(3000L); + continue; + } + return caller.callback(gseTaskClient); + } catch (TTransportException e) { + // 由于无法捕获到底层的java.net.SocketTimeoutException，所以只能对errorMessage进行判断。读超时无需重试 + status = "error"; + if ("java.net.SocketTimeoutException: Read timed out".equalsIgnoreCase(e.getMessage())) { + log.error("Invoke gse api fail", e); + throw new GseReadTimeoutException(e.getMessage()); + } + } catch (Throwable e) { + log.error("Invoke gse api fail", e); + status = "error"; + } finally { + long end = System.nanoTime(); + meterRegistry.timer(GseMetricNames.GSE_API_METRICS_NAME_PREFIX, "api_name", caller.getApiName(), + "status", status).record(end - start, TimeUnit.NANOSECONDS); + } + } while (retry-- > 0); //重试1次 + return caller.fail(connect); + } + + public interface GseApiCallback { + + T callback(GseTaskClient gseTaskClient) throws TException; + + String getApiName(); + + default T fail(boolean connect) { + log.warn("Call gse api fail, connect: {}", connect); + return null; + } + } + + public interface GseTaskResponseCaller extends GseApiCallback { + + @Override + default GseTaskResponse fail(boolean connect) { + GseTaskResponse gseResponse = new GseTaskResponse(); + gseResponse.setErrorCode(-1); + if (connect) { + gseResponse.setErrorMessage("Send GSE Job failed!"); + } else { + gseResponse.setErrorMessage("Connect GSE Server failed!"); + } + return gseResponse; + } + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/gse/GseZkServer.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/GseZkServer.java similarity index 92% rename from src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/gse/GseZkServer.java rename to src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/GseZkServer.java index 12e2fd98f8..8581583136 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/gse/GseZkServer.java +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/GseZkServer.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,12 +22,13 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.execute.engine.gse; +package com.tencent.bk.job.common.gse.v1; import com.google.common.collect.Lists; import com.google.common.collect.Maps; +import com.tencent.bk.job.common.gse.config.GseV1Properties; +import com.tencent.bk.job.common.gse.v1.model.AccessServerInfoDTO; import com.tencent.bk.job.common.util.json.JsonUtils; -import com.tencent.bk.job.execute.engine.gse.model.AccessServerInfoDTO; import lombok.extern.slf4j.Slf4j; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.recipes.cache.PathChildrenCache; @@ -47,8 +48,9 @@ public class GseZkServer extends GseServer { private PathChildrenCache cache; private List> servers; - public GseZkServer(CuratorFramework client, String gseServerPath) { - super(); + public GseZkServer(GseV1Properties gseV1Properties, CuratorFramework client) { + super(gseV1Properties); + String gseServerPath = gseV1Properties.getServer().getZooKeeper().getPath(); if (client != null) { try { gseServerPath = PathUtils.validatePath(gseServerPath); @@ -73,9 +75,9 @@ public GseZkServer(CuratorFramework client, String gseServerPath) { * 构建gse访问客户端, 并连接服务端 */ @Override - public GseClient getClient() { + public GseTaskClient getClient() { List> servers = getGseServer(); - GseClient client = getGseClient(servers); + GseTaskClient client = getGseClient(servers); if (client == null) { client = super.getClient(); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/gse/IntervalIncrementForeverRetry.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/IntervalIncrementForeverRetry.java similarity index 95% rename from src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/gse/IntervalIncrementForeverRetry.java rename to src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/IntervalIncrementForeverRetry.java index a058e3a4e3..c8c8b635ab 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/gse/IntervalIncrementForeverRetry.java +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/IntervalIncrementForeverRetry.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.execute.engine.gse; +package com.tencent.bk.job.common.gse.v1; import lombok.extern.slf4j.Slf4j; import org.apache.curator.RetryPolicy; diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/config/GseV1AutoConfiguration.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/config/GseV1AutoConfiguration.java new file mode 100644 index 0000000000..8dccfbba74 --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/config/GseV1AutoConfiguration.java @@ -0,0 +1,91 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.gse.v1.config; + +import com.tencent.bk.job.common.gse.config.GseV1Properties; +import com.tencent.bk.job.common.gse.v1.CuratorFrameworkFactoryBean; +import com.tencent.bk.job.common.gse.v1.GseCacheClientFactory; +import com.tencent.bk.job.common.gse.v1.GseServer; +import com.tencent.bk.job.common.gse.v1.GseV1ApiClient; +import com.tencent.bk.job.common.gse.v1.GseZkServer; +import com.tencent.bk.job.common.gse.v1.IntervalIncrementForeverRetry; +import io.micrometer.core.instrument.MeterRegistry; +import org.apache.curator.framework.CuratorFramework; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import java.util.Objects; + + +@Configuration(proxyBeanMethods = false) +@ConditionalOnProperty(name = "gse.enabled", havingValue = "true") +@EnableConfigurationProperties(GseV1Properties.class) +public class GseV1AutoConfiguration { + + private static final String ZOOKEEPER_SERVER_TYPE = "zookeeper"; + + @Bean("gseServer") + @ConditionalOnMissingBean(name = "gseServer") + public GseServer gseServer(GseV1Properties gseV1Properties) { + return new GseServer(gseV1Properties); + } + + /** + * 启用 ZooKeeper 方式，初始化 GseZkServer + * + * @param curatorFramework ZooKeeper 连接组件 + */ + @Bean("gseServer") + @ConditionalOnProperty(name = "gse.server.discovery.type", havingValue = ZOOKEEPER_SERVER_TYPE) + public GseServer gseServer(GseV1Properties gseV1Properties, CuratorFramework curatorFramework) { + return new GseZkServer(gseV1Properties, curatorFramework); + } + + /** + * 启用 ZooKeeper 方式，初始化 ZooKeeper 连接组件 + */ + @Bean(initMethod = "start", destroyMethod = "stop") + @ConditionalOnProperty(name = "gse.server.discovery.type", havingValue = ZOOKEEPER_SERVER_TYPE) + public CuratorFrameworkFactoryBean curatorFrameworkFactoryBean(GseV1Properties gseV1Properties) { + Objects.requireNonNull(gseV1Properties.getServer().getZooKeeper()); + return new CuratorFrameworkFactoryBean(gseV1Properties.getServer().getZooKeeper(), + new IntervalIncrementForeverRetry(60000)); + } + + @Bean("gseCacheClientFactory") + public GseCacheClientFactory gseCacheClientFactory(GseV1Properties gseV1Properties) { + return new GseCacheClientFactory(gseV1Properties); + } + + @Bean("gseV1ApiClient") + public GseV1ApiClient gseV1ApiClient(MeterRegistry meterRegistry, + GseCacheClientFactory gseCacheClientFactory) { + return new GseV1ApiClient(meterRegistry, gseCacheClientFactory); + } + +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/gse/model/AccessServerInfoDTO.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/model/AccessServerInfoDTO.java similarity index 89% rename from src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/gse/model/AccessServerInfoDTO.java rename to src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/model/AccessServerInfoDTO.java index 9db023de13..44167581bf 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/gse/model/AccessServerInfoDTO.java +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/model/AccessServerInfoDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,13 +22,11 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.execute.engine.gse.model; +package com.tencent.bk.job.common.gse.v1.model; import lombok.Data; -/** - * @since 24/12/2020 10:43 - */ + @Data public class AccessServerInfoDTO { private Integer port; diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/model/AgentStatusDTO.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/model/AgentStatusDTO.java similarity index 92% rename from src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/model/AgentStatusDTO.java rename to src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/model/AgentStatusDTO.java index 0c465c42e7..25f7f2b38e 100644 --- a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/model/AgentStatusDTO.java +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/model/AgentStatusDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,11 +22,14 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.common.gse.model; +package com.tencent.bk.job.common.gse.v1.model; import com.fasterxml.jackson.annotation.JsonProperty; import lombok.Data; +/** + * Agent 状态查询结果 + */ @Data public class AgentStatusDTO { @JsonProperty("businessid") diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/gse/model/CopyFileRsp.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/model/CopyFileRsp.java similarity index 93% rename from src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/gse/model/CopyFileRsp.java rename to src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/model/CopyFileRsp.java index 701cb126e6..a0c585819f 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/gse/model/CopyFileRsp.java +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/model/CopyFileRsp.java @@ -1,7 +1,7 @@ -/* +package com.tencent.bk.job.common.gse.v1.model;/* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -21,11 +21,8 @@ * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. */ - -package com.tencent.bk.job.execute.engine.gse.model; - import com.fasterxml.jackson.annotation.JsonProperty; -import com.tencent.bk.job.execute.engine.consts.GSECode; +import com.tencent.bk.job.common.gse.constants.GSECode; import lombok.Getter; import lombok.Setter; import lombok.ToString; diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/model/GSEFileTaskResult.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/model/GSEFileTaskResult.java new file mode 100644 index 0000000000..8c6f28b01b --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v1/model/GSEFileTaskResult.java @@ -0,0 +1,275 @@ +package com.tencent.bk.job.common.gse.v1.model;/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.gse.constants.FileDistModeEnum; +import com.tencent.bk.job.common.gse.constants.FileTaskTypeEnum; +import com.tencent.bk.job.common.gse.util.GseFilePathUtils; +import com.tencent.bk.job.common.util.FilePathUtils; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.apache.commons.lang3.StringUtils; + +import java.util.StringJoiner; + +/** + * GSE 文件任务结果 + */ +@Getter +@Setter +@ToString +public class GSEFileTaskResult { + + /** + * 0:upload， 1:download + */ + private Integer mode; + + /** + * 完成进度，取值0-100 + */ + private Integer process; + + /** + * 文件总字节数 + */ + private Long size; + + /** + * 传输的速度，单位bps + */ + private Integer speed; + + /** + * 任务状态信息，比如checksum/upload/download + */ + private String type; + + /** + * 任务开始时间 + */ + @JsonProperty("start_time") + private Long startTime; + + /** + * 任务开始时间-兼容字段 + */ + @JsonProperty("starttime") + private String startTimeStr; + + /** + * 任务结束时间-兼容字段 + */ + @JsonProperty("endtime") + private String endTimeStr; + + /** + * 任务结束时间 + */ + @JsonProperty("end_time") + private Long endTime; + + /** + * 文件源IP + */ + @JsonProperty("source") + private String sourceIp; + + /** + * 文件源云区域 + */ + @JsonProperty("source_cloudid") + private Long sourceCloudId; + + /** + * 分发目标IP + */ + @JsonProperty("dest") + private String destIp; + + /** + * 分发目标云区域 + */ + @JsonProperty("dest_cloudid") + private Long destCloudId; + + /** + * 源文件目录 + */ + @JsonProperty("source_file_dir") + private String srcDirPath; + + /** + * 源文件名 + */ + @JsonProperty("source_file_name") + private String srcFileName; + + /** + * 目标文件目录 + */ + @JsonProperty("dest_file_dir") + private String destDirPath; + + /** + * 目标文件名 + */ + @JsonProperty("dest_file_name") + private String destFileName; + + /** + * GSE 文件任务状态 + */ + @JsonProperty("status_code") + private Integer status; + + /** + * GSE 文件任务状态描述 + */ + @JsonProperty("status_info") + private String statusDesc; + + /** + * GSE 协议版本(0 - 未知版本；1 - 初始版本 ; 2 - 解除valuekey依赖版本) + */ + @JsonProperty("protover") + private Integer protocolVersion; + + // ------------------ 非协议字段 ---------------------- + /** + * 用来表示文件任务ID + */ + private String taskId; + + /** + * 标准化之后的源文件路径 + */ + private String standardSourceFilePath; + + /** + * 标准化之后的目标文件路径 + */ + private String standardDestFilePath; + + /** + * 任务类型，1-文件分发，2-目录分发，3-正则分发，4-通配符分发 + */ + private FileTaskTypeEnum taskType; + + + public String getSourceAgentId() { + return sourceCloudId + ":" + sourceIp; + } + + public String getDestAgentId() { + if (destCloudId == null || destIp == null) { + return null; + } + return destCloudId + ":" + destIp; + } + + public String getStandardSourceFilePath() { + if (standardSourceFilePath != null) { + return standardSourceFilePath; + } else { + standardSourceFilePath = FilePathUtils.appendFileName(srcDirPath, srcFileName); + } + return standardSourceFilePath; + } + + public String getStandardDestFilePath() { + if (standardDestFilePath != null) { + return standardDestFilePath; + } else { + if (protocolVersion != null && protocolVersion == 2) { + if (StringUtils.isEmpty(srcFileName)) { + //目录分发 + String srcDirName = FilePathUtils.parseDirName(srcDirPath); + standardDestFilePath = FilePathUtils.appendDirName(destDirPath, srcDirName); + } else { + // 文件分发、正则分发、通配符分发 + standardDestFilePath = FilePathUtils.appendFileName(destDirPath, srcFileName); + } + } else { + standardDestFilePath = FilePathUtils.appendFileName(destDirPath, destFileName); + } + } + return standardDestFilePath; + } + + public String getTaskId() { + if (taskId == null) { + this.taskId = buildTaskId(mode, getSourceAgentId(), getStandardSourceFilePath(), getDestAgentId(), + getStandardDestFilePath()); + } + return taskId; + } + + public String buildTaskId(Integer mode, String sourceAgentId, String sourceFilePath, String destAgentId, + String destFilePath) { + String taskId; + if (FileDistModeEnum.getFileDistMode(mode) == FileDistModeEnum.DOWNLOAD) { + taskId = concat(mode.toString(), sourceAgentId, GseFilePathUtils.standardizedGSEFilePath(sourceFilePath), + destAgentId, destFilePath); + } else { + taskId = concat(mode.toString(), sourceAgentId, GseFilePathUtils.standardizedGSEFilePath(sourceFilePath)); + } + return taskId; + } + + private String concat(String... strArgs) { + StringJoiner sj = new StringJoiner(":"); + for (String strArg : strArgs) { + if (StringUtils.isEmpty(strArg)) { + sj.add(""); + } else { + sj.add(strArg); + } + } + return sj.toString(); + } + + public Long getStartTime() { + if (startTime != null) { + return startTime; + } + if (StringUtils.isNotEmpty(startTimeStr)) { + return Long.valueOf(startTimeStr); + } + return null; + } + + public Long getEndTime() { + if (endTime != null) { + return endTime; + } + if (StringUtils.isNotEmpty(endTimeStr)) { + return Long.valueOf(endTimeStr); + } + return null; + } + +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/GseV2ApiClient.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/GseV2ApiClient.java new file mode 100644 index 0000000000..fe7436351f --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/GseV2ApiClient.java @@ -0,0 +1,245 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.gse.v2; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.tencent.bk.job.common.constant.HttpMethodEnum; +import com.tencent.bk.job.common.esb.config.AppProperties; +import com.tencent.bk.job.common.esb.config.BkApiGatewayProperties; +import com.tencent.bk.job.common.esb.model.BkApiAuthorization; +import com.tencent.bk.job.common.esb.model.EsbResp; +import com.tencent.bk.job.common.esb.model.OpenApiRequestInfo; +import com.tencent.bk.job.common.esb.sdk.BkApiClient; +import com.tencent.bk.job.common.esb.sdk.BkApiContext; +import com.tencent.bk.job.common.esb.sdk.BkApiLogStrategy; +import com.tencent.bk.job.common.gse.IGseClient; +import com.tencent.bk.job.common.gse.constants.GseConstants; +import com.tencent.bk.job.common.gse.constants.GseMetricNames; +import com.tencent.bk.job.common.gse.v2.model.AsyncGseTaskResult; +import com.tencent.bk.job.common.gse.v2.model.ExecuteScriptRequest; +import com.tencent.bk.job.common.gse.v2.model.FileTaskResult; +import com.tencent.bk.job.common.gse.v2.model.GetExecuteScriptResultRequest; +import com.tencent.bk.job.common.gse.v2.model.GetTransferFileResultRequest; +import com.tencent.bk.job.common.gse.v2.model.GseTaskResponse; +import com.tencent.bk.job.common.gse.v2.model.ScriptTaskResult; +import com.tencent.bk.job.common.gse.v2.model.TerminateGseTaskRequest; +import com.tencent.bk.job.common.gse.v2.model.TransferFileRequest; +import com.tencent.bk.job.common.gse.v2.model.req.ListAgentStateReq; +import com.tencent.bk.job.common.gse.v2.model.resp.AgentState; +import com.tencent.bk.job.common.util.StringUtil; +import com.tencent.bk.job.common.util.http.HttpHelperFactory; +import com.tencent.bk.job.common.util.http.JobHttpRequestRetryHandler; +import com.tencent.bk.job.common.util.json.JsonUtils; +import io.micrometer.core.instrument.MeterRegistry; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; +import org.slf4j.Logger; + +import java.util.Collections; +import java.util.List; + +@Slf4j +public class GseV2ApiClient extends BkApiClient implements IGseClient { + + private static final String URI_LIST_AGENT_STATE = "/api/v2/cluster/list_agent_state"; + private static final String URI_ASYNC_EXECUTE_SCRIPT = "/api/v2/task/extensions/async_execute_script"; + private static final String URI_GET_EXECUTE_SCRIPT_RESULT = "/api/v2/task/extensions/get_execute_script_result"; + private static final String URI_ASYNC_TRANSFER_FILE = "/api/v2/task/extensions/async_transfer_file"; + private static final String URI_GET_TRANSFER_FILE_RESULT = "/api/v2/task/extensions/get_transfer_file_result"; + private static final String URI_ASYNC_TERMINATE_TRANSFER_FILE = + "/api/v2/task/extensions/async_terminate_transfer_file"; + private static final String URI_ASYNC_TERMINATE_EXECUTE_SCRIPT = + "/api/v2/task/extensions/async_terminate_execute_script"; + private final BkApiAuthorization gseBkApiAuthorization; + + public GseV2ApiClient(MeterRegistry meterRegistry, + AppProperties appProperties, + BkApiGatewayProperties bkApiGatewayProperties) { + + super(meterRegistry, + GseMetricNames.GSE_V2_API_METRICS_NAME_PREFIX, + bkApiGatewayProperties.getGse().getUrl(), + HttpHelperFactory.createHttpHelper( + 15000, + 15000, + 60000, + 1000, + 2000, + 60, + true, + new JobHttpRequestRetryHandler(), + httpClientBuilder -> httpClientBuilder.addInterceptorLast(getLogBkApiRequestIdInterceptor()) + ) + ); + gseBkApiAuthorization = BkApiAuthorization.appAuthorization(appProperties.getCode(), appProperties.getSecret()); + log.info("Init GseV2ApiClient, bkGseApiGatewayUrl: {}, appCode: {}", + bkApiGatewayProperties.getGse().getUrl(), appProperties.getCode()); + } + + @Override + public GseTaskResponse asyncExecuteScript(ExecuteScriptRequest request) { + EsbResp resp = + requestGseApi( + URI_ASYNC_EXECUTE_SCRIPT, + request, + new TypeReference>() { + }, + null, + false); + + return buildGseTaskResponse(resp); + } + + private EsbResp requestGseApi(String uri, + Object reqBody, + TypeReference> typeReference, + BkApiLogStrategy logStrategy, + boolean isRequestIdempotent) { + OpenApiRequestInfo requestInfo = OpenApiRequestInfo + .builder() + .method(HttpMethodEnum.POST) + .uri(uri) + .body(reqBody) + .authorization(gseBkApiAuthorization) + .setIdempotent(isRequestIdempotent) + .build(); + return doRequest(requestInfo, typeReference, logStrategy, null); + } + + private GseTaskResponse buildGseTaskResponse(EsbResp resp) { + AsyncGseTaskResult asyncGseTaskResult = resp.getData(); + GseTaskResponse gseTaskResponse = new GseTaskResponse(); + gseTaskResponse.setErrorCode(resp.getCode()); + gseTaskResponse.setErrorMessage(resp.getMessage()); + gseTaskResponse.setGseTaskId(asyncGseTaskResult.getResult().getTaskId()); + return gseTaskResponse; + } + + @Override + public ScriptTaskResult getExecuteScriptResult(GetExecuteScriptResultRequest request) { + EsbResp resp = + requestGseApi( + URI_GET_EXECUTE_SCRIPT_RESULT, + request, + new TypeReference>() { + }, + new BkApiLogStrategy() { + @Override + public void logResp(Logger log, BkApiContext context) { + if (log.isInfoEnabled()) { + // 自定义输出，防止脚本任务结果中的执行日志字段过大导致内存溢出 + log.info("[AbstractBkApiClient] Response|bkApiRequestId={}|method={}|uri={}|success={}" + + "|costTime={}|resp={}|", + context.getRequestId(), + context.getMethod(), + context.getUri(), + context.isSuccess(), + context.getCostTime(), + context.getResp() != null ? JsonUtils.toJson(context.getResp()) : + StringUtil.substring(context.getOriginResp(), 10000)); + } + } + }, + true); + return resp.getData(); + } + + + @Override + public List listAgentState(ListAgentStateReq req) { + if (CollectionUtils.isEmpty(req.getAgentIdList())) { + log.info("agentIdList is empty"); + return Collections.emptyList(); + } + EsbResp> resp = requestGseApi( + URI_LIST_AGENT_STATE, + req, + new TypeReference>>() { + }, + null, + true); + return resp.getData(); + } + + @Override + public GseTaskResponse asyncTransferFile(TransferFileRequest request) { + EsbResp resp = + requestGseApi( + URI_ASYNC_TRANSFER_FILE, + request, + new TypeReference>() { + }, + null, + false); + + return buildGseTaskResponse(resp); + } + + @Override + public FileTaskResult getTransferFileResult(GetTransferFileResultRequest request) { + EsbResp resp = + requestGseApi( + URI_GET_TRANSFER_FILE_RESULT, + request, + new TypeReference>() { + }, + null, + true); + FileTaskResult fileTaskResult = resp.getData(); + if (fileTaskResult != null && CollectionUtils.isNotEmpty(fileTaskResult.getAtomicFileTaskResults())) { + fileTaskResult.getAtomicFileTaskResults().forEach(atomicFileTaskResult -> { + if (atomicFileTaskResult.getContent() != null) { + // 由于GSE2.0 删除了protocolVersion，会导致Job解析协议版本出问题；按照Job的设计，对接GSE2.0的才会走BK-GSE-API-GATEWAY, 协议版本必定是V2 + atomicFileTaskResult.getContent().setProtocolVersion(GseConstants.GSE_FILE_PROTOCOL_VERSION_V2); + } + }); + } + return fileTaskResult; + } + + @Override + public GseTaskResponse terminateGseFileTask(TerminateGseTaskRequest request) { + EsbResp resp = + requestGseApi(URI_ASYNC_TERMINATE_TRANSFER_FILE, + request, + new TypeReference>() { + }, + null, + true); + return buildGseTaskResponse(resp); + } + + @Override + public GseTaskResponse terminateGseScriptTask(TerminateGseTaskRequest request) { + EsbResp resp = + requestGseApi(URI_ASYNC_TERMINATE_EXECUTE_SCRIPT, + request, + new TypeReference>() { + }, + null, + true); + return buildGseTaskResponse(resp); + } +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/GseV2AutoConfiguration.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/GseV2AutoConfiguration.java new file mode 100644 index 0000000000..a22b843b23 --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/GseV2AutoConfiguration.java @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.gse.v2; + +import com.tencent.bk.job.common.esb.config.AppProperties; +import com.tencent.bk.job.common.esb.config.BkApiGatewayProperties; +import com.tencent.bk.job.common.gse.config.GseV2Properties; +import io.micrometer.core.instrument.MeterRegistry; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration(proxyBeanMethods = false) +@EnableConfigurationProperties({GseV2Properties.class}) +@ConditionalOnProperty(name = "gseV2.enabled", havingValue = "true", matchIfMissing = true) +public class GseV2AutoConfiguration { + + @Bean("gseV2ApiClient") + public GseV2ApiClient gseV2ApiClient(MeterRegistry meterRegistry, + AppProperties appProperties, + BkApiGatewayProperties bkApiGatewayProperties) { + return new GseV2ApiClient(meterRegistry, appProperties, bkApiGatewayProperties); + } +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/Agent.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/Agent.java new file mode 100644 index 0000000000..c4aef5ab0d --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/Agent.java @@ -0,0 +1,32 @@ +package com.tencent.bk.job.common.gse.v2.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +/** + * GSE - 目标Agent + */ +@Data +public class Agent { + /** + * 目标Agent ，数据格式分为两种。1. cloudId:ip（兼容老版本Agent没有agentId的情况) 2. agentId + */ + @JsonProperty("bk_agent_id") + private String agentId; + + /** + * 目标容器 ID, 空则为主机 + */ + @JsonProperty("bk_container_id") + private String containerId; + + /** + * 执行账号名 + */ + private String user; + + /** + * 与user 用户名对应的密码 + */ + private String pwd; +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/AsyncGseTaskResult.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/AsyncGseTaskResult.java new file mode 100644 index 0000000000..6b216aed59 --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/AsyncGseTaskResult.java @@ -0,0 +1,23 @@ +package com.tencent.bk.job.common.gse.v2.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * GSE 任务下发返回 + */ +@Data +@NoArgsConstructor +public class AsyncGseTaskResult { + /** + * 执行结果 + */ + private GseTaskId result; + + @Data + public static class GseTaskId { + @JsonProperty("task_id") + private String taskId; + } +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/AtomicFileTaskResult.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/AtomicFileTaskResult.java new file mode 100644 index 0000000000..d3c10435ff --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/AtomicFileTaskResult.java @@ -0,0 +1,29 @@ +package com.tencent.bk.job.common.gse.v2.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * GSE - 任务执行结果(单个文件) + */ +@Data +@NoArgsConstructor +public class AtomicFileTaskResult { + + /** + * 错误码 + */ + @JsonProperty("error_code") + private Integer errorCode; + + /** + * 错误信息 + */ + @JsonProperty("error_msg") + private String errorMsg; + + @JsonProperty("content") + private AtomicFileTaskResultContent content; + +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/AtomicFileTaskResultContent.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/AtomicFileTaskResultContent.java new file mode 100644 index 0000000000..5a98ef484a --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/AtomicFileTaskResultContent.java @@ -0,0 +1,252 @@ +package com.tencent.bk.job.common.gse.v2.model; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.gse.constants.FileDistModeEnum; +import com.tencent.bk.job.common.gse.constants.FileTaskTypeEnum; +import com.tencent.bk.job.common.gse.util.GseFilePathUtils; +import com.tencent.bk.job.common.util.FilePathUtils; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.apache.commons.lang3.StringUtils; + +import java.util.StringJoiner; + +/** + * GSE - 任务执行结果(单个文件) - 详情 + */ +@Data +@NoArgsConstructor +public class AtomicFileTaskResultContent { + /** + * 传输的任务模式： 0 - 源agent上传任务；1 - 目标agent下载任务 + * + * @see FileDistModeEnum + */ + private Integer mode; + + /** + * 文件源 agent id + */ + @JsonProperty("source_agent_id") + private String sourceAgentId; + + /** + * 文件源 container id + */ + @JsonProperty("source_container_id") + private String sourceContainerId; + + /** + * 源文件目录，下发任务时指定的源文件路径 + */ + @JsonProperty("source_file_dir") + private String sourceFileDir; + + /** + * 源文件名，下发任务时，指定的源文件名 + */ + @JsonProperty("source_file_name") + private String sourceFileName; + + /** + * 分发目标 agent id + */ + @JsonProperty("dest_agent_id") + private String destAgentId; + + /** + * 分发目标容器 container id + */ + @JsonProperty("dest_container_id") + private String destContainerId; + + /** + * 目标目录 + */ + @JsonProperty("dest_file_dir") + private String destFileDir; + + /** + * 目标文件名 + */ + @JsonProperty("dest_file_name") + private String destFileName; + + /** + * 任务执行进度, 取值0～100，代表当前传输任务进度的百分比 + */ + @JsonProperty("progress") + private Integer progress; + + /** + * 文件传输速度，单位KB/S + */ + @JsonProperty("speed") + private Integer speed; + + /** + * 文件大小，单位Byte，字节 + */ + @JsonProperty("size") + private Long size; + + /** + * 任务状态 + */ + @JsonProperty("status") + private Integer status; + + /** + * 状态信息描述 + */ + @JsonProperty("status_info") + private String statusInfo; + + + /** + * 任务开始时间 + */ + @JsonProperty("start_time") + private Long startTime; + + /** + * 任务结束时间 + */ + @JsonProperty("end_time") + private Long endTime; + + /** + * GSE 协议版本(0 - 未知版本；1 - 初始版本 ; 2 - 解除valuekey依赖版本) + */ + @JsonProperty("protover") + private Integer protocolVersion; + + // ------------------ 非协议字段 ---------------------- + /** + * 用来表示文件任务ID + */ + private String taskId; + + /** + * 标准化之后的源文件路径 + */ + private String standardSourceFilePath; + + /** + * 标准化之后的目标文件路径 + */ + private String standardDestFilePath; + + /** + * 任务类型，1-文件分发，2-目录分发，3-正则分发，4-通配符分发 + */ + private FileTaskTypeEnum taskType; + + @JsonIgnore + private ExecuteObjectGseKey sourceExecuteObjectGseKey; + @JsonIgnore + private ExecuteObjectGseKey destExecuteObjectGseKey; + + public boolean isDownloadMode() { + return FileDistModeEnum.DOWNLOAD.getValue().equals(this.mode); + } + + public String getStandardSourceFilePath() { + if (standardSourceFilePath != null) { + return standardSourceFilePath; + } else { + standardSourceFilePath = FilePathUtils.appendFileName(sourceFileDir, sourceFileName); + } + return standardSourceFilePath; + } + + public String getStandardDestFilePath() { + if (standardDestFilePath != null) { + return standardDestFilePath; + } else { + if (StringUtils.isEmpty(sourceFileName)) { + //目录分发 + String srcDirName = FilePathUtils.parseDirName(sourceFileDir); + standardDestFilePath = FilePathUtils.appendDirName(destFileDir, srcDirName); + } else { + // 文件分发、正则分发、通配符分发 + standardDestFilePath = FilePathUtils.appendFileName(destFileDir, sourceFileName); + } + } + return standardDestFilePath; + } + + public String getTaskId() { + if (taskId == null) { + this.taskId = buildTaskId(mode, getSourceExecuteObjectGseKey(), getStandardSourceFilePath(), + getDestExecuteObjectGseKey(), getStandardDestFilePath()); + } + return taskId; + } + + public static String buildTaskId(Integer mode, + ExecuteObjectGseKey sourceExecuteObjectGseKey, + String sourceFilePath, + ExecuteObjectGseKey destExecuteObjectGseKey, + String destFilePath) { + String taskId; + if (FileDistModeEnum.getFileDistMode(mode) == FileDistModeEnum.DOWNLOAD) { + taskId = concat(mode.toString(), sourceExecuteObjectGseKey.getKey(), + GseFilePathUtils.standardizedGSEFilePath(sourceFilePath), + destExecuteObjectGseKey.getKey(), destFilePath); + } else { + taskId = concat(mode.toString(), sourceExecuteObjectGseKey.getKey(), + GseFilePathUtils.standardizedGSEFilePath(sourceFilePath)); + } + return taskId; + } + + private static String concat(String... strArgs) { + StringJoiner sj = new StringJoiner(":"); + for (String strArg : strArgs) { + if (StringUtils.isEmpty(strArg)) { + sj.add(""); + } else { + sj.add(strArg); + } + } + return sj.toString(); + } + + + /** + * 判断是否是协议2.0之前的版本。（该版本文件分发协议存在问题，需要兼容) + */ + public boolean isApiProtocolBeforeV2() { + return this.protocolVersion == null || this.protocolVersion < 2; + } + + @JsonIgnore + public ExecuteObjectGseKey getDestExecuteObjectGseKey() { + if (destExecuteObjectGseKey != null) { + return destExecuteObjectGseKey; + } + if (StringUtils.isNotEmpty(destContainerId)) { + // bk_container_id 不为空，说明是容器执行对象 + destExecuteObjectGseKey = ExecuteObjectGseKey.ofContainer(destAgentId, destContainerId); + } else { + destExecuteObjectGseKey = ExecuteObjectGseKey.ofHost(destAgentId); + } + return destExecuteObjectGseKey; + } + + @JsonIgnore + public ExecuteObjectGseKey getSourceExecuteObjectGseKey() { + if (sourceExecuteObjectGseKey != null) { + return sourceExecuteObjectGseKey; + } + if (StringUtils.isNotEmpty(sourceContainerId)) { + // bk_container_id 不为空，说明是容器执行对象 + sourceExecuteObjectGseKey = ExecuteObjectGseKey.ofContainer(sourceAgentId, sourceContainerId); + } else { + sourceExecuteObjectGseKey = ExecuteObjectGseKey.ofHost(sourceAgentId); + } + return sourceExecuteObjectGseKey; + } +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/AtomicScriptTask.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/AtomicScriptTask.java new file mode 100644 index 0000000000..44f983bf31 --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/AtomicScriptTask.java @@ -0,0 +1,27 @@ +package com.tencent.bk.job.common.gse.v2.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +/** + * GSE - 原子脚本任务定义 + */ +@Data +public class AtomicScriptTask { + /** + * id 编号，在当前任务里面唯一，需要取大于等于0的值 + */ + @JsonProperty("atomic_task_id") + private Integer taskId; + + /** + * 目标机器上执行的脚本文件绝对路径，eg：/tmp/bkjob/root/xxxx.sh + */ + private String command; + + /** + * 当前任务执行的超时时间,单位秒 + */ + @JsonProperty("timeout_seconds") + private int timeout; +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/AtomicScriptTaskRelation.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/AtomicScriptTaskRelation.java new file mode 100644 index 0000000000..281fe6448c --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/AtomicScriptTaskRelation.java @@ -0,0 +1,26 @@ +package com.tencent.bk.job.common.gse.v2.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +import java.util.List; + +/** + * GSE - 定义脚本任务之间的依赖关系 + */ +@Data +public class AtomicScriptTaskRelation { + /** + * id 编号，在当前任务里面唯一，需要取大于等于0的值 + * + * @see AtomicScriptTask + */ + @JsonProperty("atomic_task_id") + private Integer taskId; + + /** + * 此值必须已经在atomic_tasks定义的atomic_task_id值且需要按照atomic_task_id依赖的任务的顺序设置此值 + */ + @JsonProperty("atomic_task_id_idx") + private List index; +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/ExecuteObjectGseKey.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/ExecuteObjectGseKey.java new file mode 100644 index 0000000000..0e25a7fb6d --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/ExecuteObjectGseKey.java @@ -0,0 +1,90 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.gse.v2.model; + +import com.tencent.bk.job.common.constant.ExecuteObjectTypeEnum; +import com.tencent.bk.job.common.gse.util.K8sUtils; +import lombok.Getter; + +import java.util.Objects; + +/** + * 执行对象 GSE KEY, 用于跟 GSE 交互 + */ +@Getter +public class ExecuteObjectGseKey { + /** + * GSE Agent ID + */ + private String agentId; + /** + * 容器 ID（比如 docker id） + */ + private String containerId; + /** + * 完整的唯一 KEY + */ + private String key; + + private ExecuteObjectGseKey() { + } + + public static ExecuteObjectGseKey ofHost(String agentId) { + ExecuteObjectGseKey executeObjectGseKey = new ExecuteObjectGseKey(); + // agentId 指定主机对象 + executeObjectGseKey.agentId = agentId; + executeObjectGseKey.key = ExecuteObjectTypeEnum.HOST.getValue() + ":" + agentId; + return executeObjectGseKey; + } + + public static ExecuteObjectGseKey ofContainer(String agentId, String containerId) { + ExecuteObjectGseKey executeObjectGseKey = new ExecuteObjectGseKey(); + // agentId+containerId 唯一指定容器对象 + executeObjectGseKey.agentId = agentId; + executeObjectGseKey.containerId = K8sUtils.removeContainerIdType(containerId); + executeObjectGseKey.key = ExecuteObjectTypeEnum.CONTAINER.getValue() + ":" + agentId + ":" + + executeObjectGseKey.getContainerId(); + return executeObjectGseKey; + } + + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ExecuteObjectGseKey that = (ExecuteObjectGseKey) o; + return key.equals(that.key); + } + + @Override + public int hashCode() { + return Objects.hash(key); + } + + @Override + public String toString() { + return key; + } +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/ExecuteScriptRequest.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/ExecuteScriptRequest.java new file mode 100644 index 0000000000..d3cc1ba6f0 --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/ExecuteScriptRequest.java @@ -0,0 +1,60 @@ +package com.tencent.bk.job.common.gse.v2.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import lombok.EqualsAndHashCode; + +import java.util.ArrayList; +import java.util.List; + +/** + * GSE - 下发脚本任务请求 + */ +@EqualsAndHashCode(callSuper = true) +@Data +public class ExecuteScriptRequest extends GseReq { + /** + * 目标Agent + */ + private List agents = new ArrayList<>(); + + /** + * 任务脚本 + */ + private List scripts = new ArrayList<>(); + + /** + * atomic_tasks 包含的元素的个数 + */ + private int atomicTaskNum; + + /** + * 脚本命令定义 + */ + @JsonProperty("atomic_tasks") + private List atomicTasks = new ArrayList<>(); + + /** + * 任务之间的依赖关系 + */ + @JsonProperty("atomic_tasks_relations") + private List atomicScriptTaskRelations = new ArrayList<>(); + + /** + * 当该值不为空时，在windows中优先使用该解释器运行脚本 + */ + @JsonProperty("windows_interpreter") + private String windowsInterpreter; + + public void addScript(GseScript script) { + scripts.add(script); + } + + public void addAtomicScriptTask(AtomicScriptTask atomicScriptTask) { + atomicTasks.add(atomicScriptTask); + } + + public void addAtomicTaskRelation(AtomicScriptTaskRelation relation) { + atomicScriptTaskRelations.add(relation); + } +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/FileTaskResult.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/FileTaskResult.java new file mode 100644 index 0000000000..191f37b52a --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/FileTaskResult.java @@ -0,0 +1,20 @@ +package com.tencent.bk.job.common.gse.v2.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +/** + * GSE 文件分发结果 + */ +@Data +@NoArgsConstructor +public class FileTaskResult { + /** + * 执行结果 + */ + @JsonProperty("result") + private List atomicFileTaskResults; +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/FileTransferTask.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/FileTransferTask.java new file mode 100644 index 0000000000..deb6d6b3af --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/FileTransferTask.java @@ -0,0 +1,29 @@ +package com.tencent.bk.job.common.gse.v2.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * GSE 文件分发任务 + */ +@Data +@NoArgsConstructor +public class FileTransferTask { + /** + * 源文件 + */ + @JsonProperty("source") + private SourceFile source; + + /** + * 传输目标 + */ + @JsonProperty("target") + private TargetFile target; + + public FileTransferTask(SourceFile source, TargetFile target) { + this.source = source; + this.target = target; + } +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/GetExecuteScriptResultRequest.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/GetExecuteScriptResultRequest.java new file mode 100644 index 0000000000..5dfb13160d --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/GetExecuteScriptResultRequest.java @@ -0,0 +1,97 @@ +package com.tencent.bk.job.common.gse.v2.model; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import lombok.EqualsAndHashCode; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * GSE - 查询脚本任务的执行结果请求 + */ +@Data +@EqualsAndHashCode(callSuper = true) +public class GetExecuteScriptResultRequest extends GseReq { + /** + * GSE 任务ID + */ + @JsonProperty("task_id") + private String taskId; + + /** + * 脚本命令定义 + */ + @JsonProperty("agent_tasks") + private List agentTasks = new ArrayList<>(); + + @Data + public static class AgentTask { + /** + * 目标Agent ，数据格式分为两种。1. cloudId:ip（兼容老版本Agent没有agentId的情况) 2. agentId + */ + @JsonProperty("bk_agent_id") + private String agentId; + + /** + * 目标容器 ID, 空则为主机 + */ + @JsonProperty("bk_container_id") + private String containerId; + + /** + * 脚本任务 + */ + @JsonProperty("atomic_tasks") + private List atomicTasks; + } + + @Data + @JsonInclude(JsonInclude.Include.NON_NULL) + public static class AtomicTask { + /** + * id 编号，在当前任务里面唯一，需要取大于等于0的值 + */ + @JsonProperty("atomic_task_id") + private Integer atomicTaskId; + + /** + * 执行日志读取偏移量，单位byte + */ + private int offset; + + /** + * 执行日志读取大小上限，单位byte + */ + private Long limit; + } + + /** + * 新增 Agent 查询条件 + * + * @param executeObjectGseKey 执行对象 GSE KEY + * @param atomicTaskId id 编号，在当前任务里面唯一，需要取大于等于0的值 + * @param offset 执行日志读取偏移量，单位byte + * @param limit 执行日志读取大小上限，单位byte；传入 null 无效，表示不限制 + */ + public void addAgentTaskQuery(ExecuteObjectGseKey executeObjectGseKey, + Integer atomicTaskId, + int offset, + Long limit) { + AgentTask agentTask = new AgentTask(); + agentTask.setAgentId(executeObjectGseKey.getAgentId()); + agentTask.setContainerId(executeObjectGseKey.getContainerId()); + AtomicTask atomicTask = new AtomicTask(); + atomicTask.setAtomicTaskId(atomicTaskId); + atomicTask.setOffset(offset); + if (limit != null && limit > 0) { + atomicTask.setLimit(limit); + } + agentTask.setAtomicTasks(Collections.singletonList(atomicTask)); + agentTasks.add(agentTask); + } + + +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/GetTransferFileResultRequest.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/GetTransferFileResultRequest.java new file mode 100644 index 0000000000..b9d671528b --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/GetTransferFileResultRequest.java @@ -0,0 +1,52 @@ +package com.tencent.bk.job.common.gse.v2.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * GSE - 查询文件任务的执行结果请求 + */ +@Data +@NoArgsConstructor +@EqualsAndHashCode(callSuper = true) +public class GetTransferFileResultRequest extends GseReq { + /** + * GSE 任务ID + */ + @JsonProperty("task_id") + private String taskId; + + /** + * 过滤结果的agent + */ + @JsonProperty("agents") + private List agents; + + public void addAgentQuery(ExecuteObjectGseKey executeObjectGseKey) { + if (agents == null) { + agents = new ArrayList<>(); + } + Agent agent = new Agent(); + agent.setAgentId(executeObjectGseKey.getAgentId()); + agent.setContainerId(executeObjectGseKey.getContainerId()); + agents.add(agent); + } + + public void batchAddAgentQuery(Collection executeObjectGseKeys) { + if (agents == null) { + agents = new ArrayList<>(); + } + executeObjectGseKeys.forEach(executeObjectGseKey -> { + Agent agent = new Agent(); + agent.setAgentId(executeObjectGseKey.getAgentId()); + agent.setContainerId(executeObjectGseKey.getContainerId()); + agents.add(agent); + }); + } +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/GseReq.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/GseReq.java new file mode 100644 index 0000000000..033b59369a --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/GseReq.java @@ -0,0 +1,19 @@ +package com.tencent.bk.job.common.gse.v2.model; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import lombok.Data; + +/** + * GSE 请求基础类 + */ +@Data +public class GseReq { + + + /** + * 是否是GSE V2 Task; 根据gseV2Task判断请求GSE V1/v2 + */ + @JsonIgnore + private boolean gseV2Task; + +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/GseScript.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/GseScript.java new file mode 100644 index 0000000000..67a854c3d3 --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/GseScript.java @@ -0,0 +1,28 @@ +package com.tencent.bk.job.common.gse.v2.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +/** + * GSE - 脚本 + */ +@Data +public class GseScript { + /** + * 脚本文件名 + */ + @JsonProperty("script_name") + private String name; + + /** + * 脚本文件的存储路径，eg: /tmp/bkjob + */ + @JsonProperty("script_store_dir") + private String storeDir; + + /** + * 脚本内容 + */ + @JsonProperty("script_content") + private String content; +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/GseTaskResponse.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/GseTaskResponse.java similarity index 94% rename from src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/GseTaskResponse.java rename to src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/GseTaskResponse.java index 86a7ce2f13..6619a2e1c2 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/GseTaskResponse.java +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/GseTaskResponse.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.execute.engine.model; +package com.tencent.bk.job.common.gse.v2.model; import lombok.Data; import lombok.NoArgsConstructor; diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/ScriptExecuteObjectTaskResult.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/ScriptExecuteObjectTaskResult.java new file mode 100644 index 0000000000..78cd446b60 --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/ScriptExecuteObjectTaskResult.java @@ -0,0 +1,129 @@ +package com.tencent.bk.job.common.gse.v2.model; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.apache.commons.lang3.StringUtils; + +import java.util.StringJoiner; + +/** + * GSE - 脚本任务执行结果 + */ +@Data +@NoArgsConstructor +public class ScriptExecuteObjectTaskResult { + /** + * agent id + */ + @JsonProperty("bk_agent_id") + private String agentId; + + /** + * container id + */ + @JsonProperty("bk_container_id") + private String containerId; + + /** + * 脚本原子任务ID + */ + @JsonProperty("atomic_task_id") + private Integer atomicTaskId; + + /** + * 任务状态 + */ + @JsonProperty("status") + private Integer status; + + /** + * 错误码 + */ + @JsonProperty("error_code") + private Integer errorCode; + + /** + * 错误信息 + */ + @JsonProperty("error_msg") + private String errorMsg; + + /** + * 任务开始时间 + */ + @JsonProperty("start_time") + private Long startTime; + + /** + * 任务结束时间 + */ + @JsonProperty("end_time") + private Long endTime; + + /** + * 脚本执行 exit code + */ + @JsonProperty("script_exit_code") + private Integer exitCode; + + /** + * 用户自定义结果分组标签 + */ + @JsonProperty("tag") + private String tag; + + /** + * 脚本输出日志 + */ + @JsonProperty(value = "screen", access = JsonProperty.Access.WRITE_ONLY) + private String screen; + + /** + * 非协议内容，仅用于日志输出 + */ + private long contentLength; + + /** + * 执行目标对应的 GSE KEY。非协议内容 + */ + @JsonIgnore + private ExecuteObjectGseKey executeObjectGseKey; + + + public void setScreen(String screen) { + this.screen = screen; + this.contentLength = StringUtils.isEmpty(screen) ? 0 : screen.length(); + } + + @JsonIgnore + public ExecuteObjectGseKey getExecuteObjectGseKey() { + if (executeObjectGseKey != null) { + return executeObjectGseKey; + } + if (StringUtils.isNotEmpty(containerId)) { + // bk_container_id 不为空，说明是容器执行对象 + executeObjectGseKey = ExecuteObjectGseKey.ofContainer(agentId, containerId); + } else { + executeObjectGseKey = ExecuteObjectGseKey.ofHost(agentId); + } + return executeObjectGseKey; + } + + @Override + public String toString() { + return new StringJoiner(", ", ScriptExecuteObjectTaskResult.class.getSimpleName() + "[", "]") + .add("agentId=" + agentId) + .add("atomicTaskId=" + atomicTaskId) + .add("status=" + status) + .add("errorCode=" + errorCode) + .add("errorMsg='" + errorMsg + "'") + .add("startTime=" + startTime) + .add("endTime=" + endTime) + .add("exitCode=" + exitCode) + .add("tag='" + tag + "'") + .add("contentLength=" + contentLength) + .toString(); + } +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/ScriptTaskResult.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/ScriptTaskResult.java new file mode 100644 index 0000000000..7c8c554733 --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/ScriptTaskResult.java @@ -0,0 +1,18 @@ +package com.tencent.bk.job.common.gse.v2.model; + +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +/** + * GSE 脚本执行结果 + */ +@Data +@NoArgsConstructor +public class ScriptTaskResult { + /** + * 执行结果 + */ + private List result; +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/SourceFile.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/SourceFile.java new file mode 100644 index 0000000000..7ea30c1b44 --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/SourceFile.java @@ -0,0 +1,36 @@ +package com.tencent.bk.job.common.gse.v2.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * GSE 源文件 + */ +@Data +@NoArgsConstructor +public class SourceFile { + /** + * 文件名 + */ + @JsonProperty("file_name") + private String fileName = ""; + + /** + * 文件所在目录 + */ + @JsonProperty("store_dir") + private String storeDir; + + /** + * 源agent + */ + @JsonProperty("agent") + private Agent agent; + + public SourceFile(String fileName, String storeDir, Agent agent) { + this.fileName = fileName; + this.storeDir = storeDir; + this.agent = agent; + } +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/TargetFile.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/TargetFile.java new file mode 100644 index 0000000000..582fc1dffc --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/TargetFile.java @@ -0,0 +1,53 @@ +package com.tencent.bk.job.common.gse.v2.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +/** + * GSE 目标文件 + */ +@Data +@NoArgsConstructor +public class TargetFile { + /** + * 目标文件名 + */ + @JsonProperty("file_name") + private String fileName; + + /** + * 目标文件目录 + */ + @JsonProperty("store_dir") + private String storeDir; + + /** + * 目标文件所有者 + */ + @JsonProperty("owner") + private String owner; + + /** + * 目标文件的权限配置 + */ + @JsonProperty("permission") + private Integer permission; + + + /** + * 目标agent列表 + */ + @JsonProperty("agents") + private List agents; + + public TargetFile(String fileName, + String storeDir, + List agents) { + this.fileName = fileName; + this.storeDir = storeDir; + this.agents = agents; + } +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/TerminateGseTaskRequest.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/TerminateGseTaskRequest.java new file mode 100644 index 0000000000..1c1966a3cf --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/TerminateGseTaskRequest.java @@ -0,0 +1,44 @@ +package com.tencent.bk.job.common.gse.v2.model; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + +import java.util.ArrayList; +import java.util.List; + +/** + * GSE - 终止GSE任务执行请求 + */ +@Data +@NoArgsConstructor +@EqualsAndHashCode(callSuper = true) +public class TerminateGseTaskRequest extends GseReq { + /** + * GSE 任务ID + */ + @JsonProperty("task_id") + private String taskId; + + /** + * 目标 Agent 列表 + */ + @JsonProperty("agents") + private List agents = new ArrayList<>(); + + /** + * 是否是GSE V2 Task; 根据gseV2Task判断请求GSE V1/v2 + */ + @JsonIgnore + private boolean gseV2Task; + + public TerminateGseTaskRequest(String taskId, + List agents, + boolean gseV2Task) { + this.taskId = taskId; + this.agents = agents; + this.gseV2Task = gseV2Task; + } +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/TransferFileRequest.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/TransferFileRequest.java new file mode 100644 index 0000000000..2fd4e05097 --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/TransferFileRequest.java @@ -0,0 +1,49 @@ +package com.tencent.bk.job.common.gse.v2.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import lombok.EqualsAndHashCode; + +import java.util.ArrayList; +import java.util.List; + +/** + * GSE - 下发文件任务请求 + */ +@EqualsAndHashCode(callSuper = true) +@Data +public class TransferFileRequest extends GseReq { + + /** + * 文件任务 + */ + private List tasks = new ArrayList<>(); + + /** + * 任务超时秒数 + */ + @JsonProperty("timeout_seconds") + private int timeout; + + /** + * 目录创建策略，true：自动创建，false：即使目录不存在也不自动创建 + */ + @JsonProperty("auto_mkdir") + private boolean autoMkdir; + + /** + * 文件上传速度限制(MB) + */ + @JsonProperty("upload_speed") + private Integer uploadSpeed; + + /** + * 文件下载速度限制(MB)，0：无限制 + */ + @JsonProperty("download_speed") + private Integer downloadSpeed; + + public void addFileTask(FileTransferTask task) { + tasks.add(task); + } +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/req/ListAgentStateReq.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/req/ListAgentStateReq.java new file mode 100644 index 0000000000..07f5a9eb27 --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/req/ListAgentStateReq.java @@ -0,0 +1,19 @@ +package com.tencent.bk.job.common.gse.v2.model.req; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +import java.util.List; + +/** + * GSE - 查询Agent状态请求 + */ +@Data +public class ListAgentStateReq { + /** + * Agent ID列表 + */ + @JsonProperty("agent_id_list") + private List agentIdList; + +} diff --git a/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/resp/AgentState.java b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/resp/AgentState.java new file mode 100644 index 0000000000..817edee291 --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/java/com/tencent/bk/job/common/gse/v2/model/resp/AgentState.java @@ -0,0 +1,63 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.gse.v2.model.resp; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +/** + * GSE - Agent状态 + */ +@Data +public class AgentState { + /** + * 目标Agent ，数据格式分为两种。1. cloudId:ip（兼容老版本Agent没有agentId的情况) 2. agentId + */ + @JsonProperty("bk_agent_id") + private String agentId; + + /** + * 云区域ID + */ + @JsonProperty("bk_cloud_id") + private Integer cloudId; + + /** + * Agent版本 + */ + private String version; + + /** + * Agent运行模式 + */ + @JsonProperty("run_mode") + private Integer runMode; + + /** + * Agent状态 + */ + @JsonProperty("status_code") + private Integer statusCode; +} diff --git a/src/backend/commons/gse-sdk/src/main/resources/META-INF/spring.factories b/src/backend/commons/gse-sdk/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..93749a4152 --- /dev/null +++ b/src/backend/commons/gse-sdk/src/main/resources/META-INF/spring.factories @@ -0,0 +1,3 @@ +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ +com.tencent.bk.job.common.gse.config.GseAutoConfiguration,\ +com.tencent.bk.job.common.gse.config.GseMetricsAutoConfiguration diff --git a/src/backend/commons/notice-sdk/build.gradle b/src/backend/commons/notice-sdk/build.gradle new file mode 100644 index 0000000000..ebb876d970 --- /dev/null +++ b/src/backend/commons/notice-sdk/build.gradle @@ -0,0 +1,41 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +dependencies { + api project(':commons:common') + api project(':commons:esb-sdk') + api project(':commons:common-i18n') + implementation 'com.fasterxml.jackson.core:jackson-core' + implementation 'com.fasterxml.jackson.core:jackson-databind' + implementation 'com.fasterxml.jackson.core:jackson-annotations' + implementation 'org.apache.commons:commons-lang3' + implementation "net.sf.dozer:dozer" + implementation 'io.micrometer:micrometer-registry-prometheus' + implementation 'org.apache.commons:commons-collections4' + implementation 'org.apache.httpcomponents:httpclient' + compileOnly 'org.projectlombok:lombok' + annotationProcessor 'org.projectlombok:lombok' + testImplementation 'org.junit.jupiter:junit-jupiter' + testImplementation 'org.springframework.boot:spring-boot-starter-test' +} diff --git a/src/backend/commons/notice-sdk/src/main/java/com/tencent/bk/job/common/notice/IBkNoticeClient.java b/src/backend/commons/notice-sdk/src/main/java/com/tencent/bk/job/common/notice/IBkNoticeClient.java new file mode 100644 index 0000000000..e5eea34c7f --- /dev/null +++ b/src/backend/commons/notice-sdk/src/main/java/com/tencent/bk/job/common/notice/IBkNoticeClient.java @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.notice; + +import com.tencent.bk.job.common.notice.model.AnnouncementDTO; +import com.tencent.bk.job.common.notice.model.BkNoticeApp; + +import java.util.List; + +public interface IBkNoticeClient { + + BkNoticeApp registerApplication(); + + List getCurrentAnnouncements(String bkLanguage, Integer offset, Integer limit); + +} diff --git a/src/backend/commons/notice-sdk/src/main/java/com/tencent/bk/job/common/notice/config/BkNoticeProperties.java b/src/backend/commons/notice-sdk/src/main/java/com/tencent/bk/job/common/notice/config/BkNoticeProperties.java new file mode 100644 index 0000000000..72529c375d --- /dev/null +++ b/src/backend/commons/notice-sdk/src/main/java/com/tencent/bk/job/common/notice/config/BkNoticeProperties.java @@ -0,0 +1,41 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.notice.config; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; + +/** + * 消息通知中心配置 + */ +@ConfigurationProperties(prefix = "bk-notice") +@Getter +@Setter +public class BkNoticeProperties { + + private boolean enabled = true; + +} diff --git a/src/backend/commons/notice-sdk/src/main/java/com/tencent/bk/job/common/notice/config/NoticeAutoConfiguration.java b/src/backend/commons/notice-sdk/src/main/java/com/tencent/bk/job/common/notice/config/NoticeAutoConfiguration.java new file mode 100644 index 0000000000..11fded1a13 --- /dev/null +++ b/src/backend/commons/notice-sdk/src/main/java/com/tencent/bk/job/common/notice/config/NoticeAutoConfiguration.java @@ -0,0 +1,48 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.notice.config; + +import com.tencent.bk.job.common.esb.config.AppProperties; +import com.tencent.bk.job.common.esb.config.BkApiGatewayProperties; +import com.tencent.bk.job.common.notice.impl.BkNoticeClient; +import io.micrometer.core.instrument.MeterRegistry; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Slf4j +@Configuration(proxyBeanMethods = false) +@EnableConfigurationProperties({BkNoticeProperties.class}) +public class NoticeAutoConfiguration { + + @Bean + public BkNoticeClient bkNoticeClient(MeterRegistry meterRegistry, + AppProperties appProperties, + BkApiGatewayProperties bkApiGatewayProperties) { + return new BkNoticeClient(meterRegistry, appProperties, bkApiGatewayProperties); + } + +} diff --git a/src/backend/commons/notice-sdk/src/main/java/com/tencent/bk/job/common/notice/exception/BkNoticeException.java b/src/backend/commons/notice-sdk/src/main/java/com/tencent/bk/job/common/notice/exception/BkNoticeException.java new file mode 100644 index 0000000000..14addf24b7 --- /dev/null +++ b/src/backend/commons/notice-sdk/src/main/java/com/tencent/bk/job/common/notice/exception/BkNoticeException.java @@ -0,0 +1,25 @@ +package com.tencent.bk.job.common.notice.exception; + +import com.tencent.bk.job.common.exception.InternalException; +import lombok.Getter; +import lombok.ToString; + +/** + * 调用蓝鲸消息通知中心异常 + */ +@Getter +@ToString +public class BkNoticeException extends InternalException { + + public BkNoticeException(Throwable cause, Integer errorCode, Object[] errorParams) { + super(cause, errorCode, errorParams); + } + + public BkNoticeException(String message, Integer errorCode) { + super(message, errorCode); + } + + public BkNoticeException(String message, Throwable cause, Integer errorCode) { + super(message, cause, errorCode); + } +} diff --git a/src/backend/commons/notice-sdk/src/main/java/com/tencent/bk/job/common/notice/impl/BkNoticeClient.java b/src/backend/commons/notice-sdk/src/main/java/com/tencent/bk/job/common/notice/impl/BkNoticeClient.java new file mode 100644 index 0000000000..e8fea2cbd9 --- /dev/null +++ b/src/backend/commons/notice-sdk/src/main/java/com/tencent/bk/job/common/notice/impl/BkNoticeClient.java @@ -0,0 +1,172 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.notice.impl; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.constant.HttpMethodEnum; +import com.tencent.bk.job.common.esb.config.AppProperties; +import com.tencent.bk.job.common.esb.config.BkApiGatewayProperties; +import com.tencent.bk.job.common.esb.model.BkApiAuthorization; +import com.tencent.bk.job.common.esb.model.EsbReq; +import com.tencent.bk.job.common.esb.model.EsbResp; +import com.tencent.bk.job.common.esb.model.OpenApiRequestInfo; +import com.tencent.bk.job.common.esb.sdk.BkApiClient; +import com.tencent.bk.job.common.exception.HttpStatusException; +import com.tencent.bk.job.common.exception.InternalException; +import com.tencent.bk.job.common.metrics.CommonMetricNames; +import com.tencent.bk.job.common.notice.IBkNoticeClient; +import com.tencent.bk.job.common.notice.exception.BkNoticeException; +import com.tencent.bk.job.common.notice.model.AnnouncementDTO; +import com.tencent.bk.job.common.notice.model.BkNoticeApp; +import com.tencent.bk.job.common.util.http.HttpHelperFactory; +import com.tencent.bk.job.common.util.http.HttpMetricUtil; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tag; +import org.apache.commons.lang3.StringUtils; +import org.apache.http.HttpStatus; + +import java.util.List; + +@SuppressWarnings("SameParameterValue") +public class BkNoticeClient extends BkApiClient implements IBkNoticeClient { + + private static final String URI_REGISTER_APPLICATION = "/apigw/v1/register/"; + private static final String URI_GET_CURRENT_ANNOUNCEMENTS = "/apigw/v1/announcement/get_current_announcements/"; + + private final AppProperties appProperties; + private final BkApiAuthorization authorization; + + public BkNoticeClient(MeterRegistry meterRegistry, + AppProperties appProperties, + BkApiGatewayProperties bkApiGatewayProperties) { + super( + meterRegistry, + CommonMetricNames.BK_NOTICE_API, + getBkNoticeUrlSafely(bkApiGatewayProperties), + HttpHelperFactory.createHttpHelper( + httpClientBuilder -> httpClientBuilder.addInterceptorLast(getLogBkApiRequestIdInterceptor()) + ) + ); + this.appProperties = appProperties; + authorization = BkApiAuthorization.appAuthorization(appProperties.getCode(), appProperties.getSecret()); + } + + private static String getBkNoticeUrlSafely(BkApiGatewayProperties bkApiGatewayProperties) { + if (bkApiGatewayProperties == null || bkApiGatewayProperties.getBkNotice() == null) { + return null; + } + return bkApiGatewayProperties.getBkNotice().getUrl(); + } + + @Override + public BkNoticeApp registerApplication() { + EsbResp resp = requestBkNoticeApi( + HttpMethodEnum.POST, + URI_REGISTER_APPLICATION, + null, + new TypeReference>() { + }, + true + ); + return resp.getData(); + } + + @Override + public List getCurrentAnnouncements(String bkLanguage, Integer offset, Integer limit) { + EsbResp> resp = requestBkNoticeApi( + HttpMethodEnum.GET, + buildUriWithParams(bkLanguage, offset, limit), + null, + new TypeReference>>() { + }, + true + ); + return resp.getData(); + } + + private String buildUriWithParams(String bkLanguage, Integer offset, Integer limit) { + StringBuilder sb = new StringBuilder(); + sb.append(URI_GET_CURRENT_ANNOUNCEMENTS); + sb.append("?platform="); + sb.append(appProperties.getCode()); + if (StringUtils.isNotBlank(bkLanguage)) { + sb.append("&language="); + sb.append(bkLanguage); + } + if (offset != null) { + sb.append("&offset="); + sb.append(offset); + } + if (limit != null) { + sb.append("&limit="); + sb.append(limit); + } + return sb.toString(); + } + + /** + * 通过ESB请求消息通知中心API的统一入口，监控数据埋点位置 + * + * @param method Http方法 + * @param uri 请求地址 + * @param reqBody 请求体内容 + * @param typeReference 指定了返回值类型的EsbResp TypeReference对象 + * @param 泛型：返回值类型 + * @return 返回值类型实例 + */ + private EsbResp requestBkNoticeApi(HttpMethodEnum method, + String uri, + EsbReq reqBody, + TypeReference> typeReference, + Boolean idempotent) { + try { + HttpMetricUtil.setHttpMetricName(CommonMetricNames.BK_NOTICE_API_HTTP); + HttpMetricUtil.addTagForCurrentMetric(Tag.of("api_name", uri)); + OpenApiRequestInfo requestInfo = OpenApiRequestInfo + .builder() + .method(method) + .uri(uri) + .body(reqBody) + .authorization(authorization) + .setIdempotent(idempotent) + .build(); + return doRequest(requestInfo, typeReference); + } catch (InternalException e) { + // 接口不存在的场景需要使用指定错误码以便前端兼容处理 + if (e.getCause() instanceof HttpStatusException) { + HttpStatusException httpStatusException = (HttpStatusException) e.getCause(); + if (httpStatusException.getHttpStatus() == HttpStatus.SC_NOT_FOUND) { + throw new BkNoticeException(e, ErrorCode.BK_NOTICE_API_NOT_FOUND, new String[]{uri}); + } + } + throw new BkNoticeException(e, ErrorCode.BK_NOTICE_API_DATA_ERROR, null); + } catch (Exception e) { + throw new BkNoticeException(e, ErrorCode.BK_NOTICE_API_DATA_ERROR, null); + } finally { + HttpMetricUtil.clearHttpMetric(); + } + } +} diff --git a/src/backend/commons/notice-sdk/src/main/java/com/tencent/bk/job/common/notice/model/AnnouncementDTO.java b/src/backend/commons/notice-sdk/src/main/java/com/tencent/bk/job/common/notice/model/AnnouncementDTO.java new file mode 100644 index 0000000000..2cb510d923 --- /dev/null +++ b/src/backend/commons/notice-sdk/src/main/java/com/tencent/bk/job/common/notice/model/AnnouncementDTO.java @@ -0,0 +1,89 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.notice.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.util.List; + +@Slf4j +@NoArgsConstructor +@Data +public class AnnouncementDTO { + + private Long id; + + /** + * 标题 + */ + private String title; + + /** + * 内容 + */ + private String content; + + /** + * 内容列表（国际化，各种语言的内容） + */ + @JsonProperty("content_list") + private List contentList; + + /** + * 公告类型: event(活动通知)/announce(平台公告) + */ + @JsonProperty("announce_type") + private String announceType; + + /** + * 开始时间 + */ + @JsonProperty("start_time") + private String startTime; + + /** + * 结束时间 + */ + @JsonProperty("end_time") + private String endTime; + + @NoArgsConstructor + @Data + public static class ContentWithLanguage { + /** + * 内容 + */ + private String content; + + /** + * 语言 + */ + private String language; + } + +} diff --git a/src/backend/commons/notice-sdk/src/main/java/com/tencent/bk/job/common/notice/model/BkNoticeApp.java b/src/backend/commons/notice-sdk/src/main/java/com/tencent/bk/job/common/notice/model/BkNoticeApp.java new file mode 100644 index 0000000000..94b49818e9 --- /dev/null +++ b/src/backend/commons/notice-sdk/src/main/java/com/tencent/bk/job/common/notice/model/BkNoticeApp.java @@ -0,0 +1,37 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.notice.model; + +import lombok.Data; +import lombok.NoArgsConstructor; + +@NoArgsConstructor +@Data +public class BkNoticeApp { + + private String code; + + private String name; +} diff --git a/src/backend/commons/notice-sdk/src/main/resources/META-INF/spring.factories b/src/backend/commons/notice-sdk/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..dde166dd38 --- /dev/null +++ b/src/backend/commons/notice-sdk/src/main/resources/META-INF/spring.factories @@ -0,0 +1,2 @@ +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ +com.tencent.bk.job.common.notice.config.NoticeAutoConfiguration diff --git a/src/backend/commons/paas-sdk/build.gradle b/src/backend/commons/paas-sdk/build.gradle index 907fb546e8..ebb876d970 100644 --- a/src/backend/commons/paas-sdk/build.gradle +++ b/src/backend/commons/paas-sdk/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/cmsi/CmsiApiClient.java b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/cmsi/CmsiApiClient.java new file mode 100644 index 0000000000..d9d79fdc17 --- /dev/null +++ b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/cmsi/CmsiApiClient.java @@ -0,0 +1,167 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.paas.cmsi; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.constant.HttpMethodEnum; +import com.tencent.bk.job.common.esb.config.AppProperties; +import com.tencent.bk.job.common.esb.config.EsbProperties; +import com.tencent.bk.job.common.esb.metrics.EsbMetricTags; +import com.tencent.bk.job.common.esb.model.BkApiAuthorization; +import com.tencent.bk.job.common.esb.model.EsbReq; +import com.tencent.bk.job.common.esb.model.EsbResp; +import com.tencent.bk.job.common.esb.model.OpenApiRequestInfo; +import com.tencent.bk.job.common.esb.sdk.BkApiClient; +import com.tencent.bk.job.common.exception.InternalCmsiException; +import com.tencent.bk.job.common.metrics.CommonMetricNames; +import com.tencent.bk.job.common.model.error.ErrorType; +import com.tencent.bk.job.common.paas.exception.PaasException; +import com.tencent.bk.job.common.paas.model.EsbNotifyChannelDTO; +import com.tencent.bk.job.common.paas.model.PostSendMsgReq; +import com.tencent.bk.job.common.util.http.HttpHelperFactory; +import com.tencent.bk.job.common.util.http.HttpMetricUtil; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tag; +import lombok.extern.slf4j.Slf4j; +import org.slf4j.helpers.MessageFormatter; + +import java.util.List; +import java.util.Set; + +import static com.tencent.bk.job.common.metrics.CommonMetricNames.ESB_CMSI_API; + +/** + * 消息通知 API 客户端 + */ +@Slf4j +public class CmsiApiClient extends BkApiClient { + + private static final String API_GET_NOTIFY_CHANNEL_LIST = "/api/c/compapi/cmsi/get_msg_type/"; + private static final String API_POST_SEND_MSG = "/api/c/compapi/cmsi/send_msg/"; + + private final BkApiAuthorization authorization; + + public CmsiApiClient(EsbProperties esbProperties, + AppProperties appProperties, + MeterRegistry meterRegistry) { + super( + meterRegistry, + ESB_CMSI_API, + esbProperties.getService().getUrl(), + HttpHelperFactory.createHttpHelper( + httpClientBuilder -> httpClientBuilder.addInterceptorLast(getLogBkApiRequestIdInterceptor()) + ) + ); + this.authorization = BkApiAuthorization.appAuthorization(appProperties.getCode(), + appProperties.getSecret(), "admin"); + } + + public List getNotifyChannelList() { + try { + HttpMetricUtil.setHttpMetricName(CommonMetricNames.ESB_CMSI_API_HTTP); + HttpMetricUtil.addTagForCurrentMetric( + Tag.of(EsbMetricTags.KEY_API_NAME, API_GET_NOTIFY_CHANNEL_LIST) + ); + EsbResp> esbResp = doRequest( + OpenApiRequestInfo.builder() + .method(HttpMethodEnum.GET) + .uri(API_GET_NOTIFY_CHANNEL_LIST) + .authorization(authorization) + .build(), + new TypeReference>>() { + } + ); + return esbResp.getData(); + } catch (Exception e) { + String errorMsg = "Get " + API_GET_NOTIFY_CHANNEL_LIST + " error"; + log.error(errorMsg, e); + throw new InternalCmsiException(errorMsg, e, ErrorCode.CMSI_MSG_CHANNEL_DATA_ERROR); + } finally { + HttpMetricUtil.clearHttpMetric(); + } + } + + public void sendMsg(String msgType, + String sender, + Set receivers, + String title, + String content) { + PostSendMsgReq req = buildSendMsgReq(msgType, sender, receivers, title, content); + String uri = API_POST_SEND_MSG; + try { + HttpMetricUtil.setHttpMetricName(CommonMetricNames.ESB_CMSI_API_HTTP); + HttpMetricUtil.addTagForCurrentMetric(Tag.of(EsbMetricTags.KEY_API_NAME, uri)); + EsbResp esbResp = doRequest( + OpenApiRequestInfo.builder() + .method(HttpMethodEnum.POST) + .uri(uri) + .body(req) + .authorization(authorization) + .build(), + new TypeReference>() { + } + ); + + if (esbResp.getResult() == null || !esbResp.getResult() || esbResp.getCode() != 0) { + throw new PaasException( + ErrorType.INTERNAL, + ErrorCode.CMSI_FAIL_TO_SEND_MSG, + new Object[]{ + esbResp.getCode().toString(), + esbResp.getMessage() + }); + } + } catch (PaasException e) { + throw e; + } catch (Exception e) { + String msg = MessageFormatter.format( + "Fail to request {}", + uri + ).getMessage(); + log.error(msg, e); + throw new PaasException(e, ErrorType.INTERNAL, ErrorCode.CMSI_API_ACCESS_ERROR, new Object[]{}); + } finally { + HttpMetricUtil.clearHttpMetric(); + } + } + + private PostSendMsgReq buildSendMsgReq(String msgType, + String sender, + Set receivers, + String title, + String content) { + PostSendMsgReq req = EsbReq.buildRequest(PostSendMsgReq.class, "superadmin"); + if (title == null || title.isEmpty()) { + title = "Default Title"; + } + req.setMsgType(msgType); + req.setSender(sender); + req.setReceiverUsername(String.join(",", receivers)); + req.setTitle(title); + req.setContent(content); + return req; + } +} diff --git a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/config/CmsiAutoConfiguration.java b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/config/CmsiAutoConfiguration.java new file mode 100644 index 0000000000..7a1900740d --- /dev/null +++ b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/config/CmsiAutoConfiguration.java @@ -0,0 +1,48 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.paas.config; + +import com.tencent.bk.job.common.esb.config.AppProperties; +import com.tencent.bk.job.common.esb.config.EsbProperties; +import com.tencent.bk.job.common.paas.cmsi.CmsiApiClient; +import io.micrometer.core.instrument.MeterRegistry; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration(proxyBeanMethods = false) +@Slf4j +public class CmsiAutoConfiguration { + + @Bean + public CmsiApiClient cmsiApiClient(AppProperties appProperties, + EsbProperties esbProperties, + ObjectProvider meterRegistryObjectProvider) { + log.info("Init CmsiApiClient"); + return new CmsiApiClient(esbProperties, appProperties, meterRegistryObjectProvider.getIfAvailable()); + } + +} diff --git a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/config/LoginAutoConfiguration.java b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/config/LoginAutoConfiguration.java new file mode 100644 index 0000000000..86feba7336 --- /dev/null +++ b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/config/LoginAutoConfiguration.java @@ -0,0 +1,68 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.paas.config; + +import com.tencent.bk.job.common.esb.config.AppProperties; +import com.tencent.bk.job.common.esb.config.EsbProperties; +import com.tencent.bk.job.common.paas.login.CustomLoginClient; +import com.tencent.bk.job.common.paas.login.ILoginClient; +import com.tencent.bk.job.common.paas.login.StandardLoginClient; +import io.micrometer.core.instrument.MeterRegistry; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.context.annotation.Primary; + +@Configuration(proxyBeanMethods = false) +@Slf4j +@Import(LoginConfiguration.class) +public class LoginAutoConfiguration { + + @Bean + @ConditionalOnProperty(value = "paas.login.custom.enabled", havingValue = "true") + public ILoginClient customLoginClient(@Autowired LoginConfiguration loginConfiguration) { + log.info("Init CustomLoginClient"); + return new CustomLoginClient(loginConfiguration.getCustomLoginApiUrl()); + } + + @Bean + @ConditionalOnProperty(value = "paas.login.custom.enabled", havingValue = "false", matchIfMissing = true) + @Primary + public ILoginClient standardLoginClient(AppProperties appProperties, + EsbProperties esbProperties, + ObjectProvider meterRegistryObjectProvider) { + + log.info("Init StandardLoginClient"); + return new StandardLoginClient( + esbProperties, + appProperties, + meterRegistryObjectProvider.getIfAvailable() + ); + } +} diff --git a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/config/LoginConfiguration.java b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/config/LoginConfiguration.java new file mode 100644 index 0000000000..fffa79c2ff --- /dev/null +++ b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/config/LoginConfiguration.java @@ -0,0 +1,67 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.paas.config; + +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Configuration; + +@Configuration(proxyBeanMethods = false) +@Getter +@Setter +@ToString +public class LoginConfiguration { + /** + * 蓝鲸标准的登录url + */ + @Value("${paas.login.url:}") + private String loginUrl; + + /** + * 是否使用第三方登录系统 + */ + @Value("${paas.login.custom.enabled:false}") + private boolean customPaasLoginEnabled; + + /** + * 第三方登录系统用户token的cookie名称 + */ + @Value("${paas.login.custom.token-name:bk_token}") + private String customLoginToken; + + /** + * 第三方登录系统登录url + */ + @Value("${paas.login.custom.login-url:}") + private String customLoginUrl; + + /** + * 第三方登录系统API url，用于根据token获取用户信息 + */ + @Value("${paas.login.custom.api-url:}") + private String customLoginApiUrl; +} diff --git a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/config/UserMgrAutoConfiguration.java b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/config/UserMgrAutoConfiguration.java new file mode 100644 index 0000000000..7c334df6f7 --- /dev/null +++ b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/config/UserMgrAutoConfiguration.java @@ -0,0 +1,48 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.paas.config; + +import com.tencent.bk.job.common.esb.config.AppProperties; +import com.tencent.bk.job.common.esb.config.EsbProperties; +import com.tencent.bk.job.common.paas.user.UserMgrApiClient; +import io.micrometer.core.instrument.MeterRegistry; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration(proxyBeanMethods = false) +@Slf4j +public class UserMgrAutoConfiguration { + + @Bean + public UserMgrApiClient userMgrApiClient(AppProperties appProperties, + EsbProperties esbProperties, + ObjectProvider meterRegistryObjectProvider) { + log.info("Init UserMgrApiClient"); + return new UserMgrApiClient(esbProperties, appProperties, meterRegistryObjectProvider.getIfAvailable()); + } + +} diff --git a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/exception/AppPermissionDeniedException.java b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/exception/AppPermissionDeniedException.java new file mode 100644 index 0000000000..2378a3e407 --- /dev/null +++ b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/exception/AppPermissionDeniedException.java @@ -0,0 +1,51 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.paas.exception; + +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.ServiceException; +import com.tencent.bk.job.common.model.error.ErrorType; +import lombok.Getter; +import lombok.ToString; + +/** + * 应用权限不足异常 + */ +@Getter +@ToString +public class AppPermissionDeniedException extends ServiceException { + + // 源于用户管理接口：进一步的操作提示 + private final String message; + + public AppPermissionDeniedException(String message) { + super(ErrorType.PERMISSION_DENIED, ErrorCode.USER_ACCESS_APP_FORBIDDEN); + this.message = message; + } + + public String getMessage() { + return message; + } +} diff --git a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/exception/PaasException.java b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/exception/PaasException.java index e8f343c943..b606c3f0ff 100644 --- a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/exception/PaasException.java +++ b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/exception/PaasException.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/login/CustomLoginClient.java b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/login/CustomLoginClient.java index 5f8ba9ed9f..a4ff2a9707 100644 --- a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/login/CustomLoginClient.java +++ b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/login/CustomLoginClient.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -45,7 +45,7 @@ @Slf4j public class CustomLoginClient implements ILoginClient { private static final String API_GET_USER_INFO = "/user/get_info/"; - private String customLoginApiUrl; + private final String customLoginApiUrl; public CustomLoginClient(String customLoginApiUrl) { if (customLoginApiUrl.endsWith("/")) { @@ -145,11 +145,6 @@ private String urlEncode(String str) { } } - @Override - public BkUserDTO getUserInfoByUserName(String userName) { - return null; - } - @Getter @Setter @ToString diff --git a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/login/EELoginClient.java b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/login/EELoginClient.java deleted file mode 100644 index 4c55d2f528..0000000000 --- a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/login/EELoginClient.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.paas.login; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.tencent.bk.job.common.constant.ErrorCode; -import com.tencent.bk.job.common.esb.model.EsbReq; -import com.tencent.bk.job.common.esb.model.EsbResp; -import com.tencent.bk.job.common.esb.sdk.AbstractEsbSdkClient; -import com.tencent.bk.job.common.exception.InternalUserManageException; -import com.tencent.bk.job.common.metrics.CommonMetricNames; -import com.tencent.bk.job.common.model.dto.BkUserDTO; -import com.tencent.bk.job.common.paas.model.EsbUserDto; -import com.tencent.bk.job.common.util.http.HttpMetricUtil; -import io.micrometer.core.instrument.Tag; -import lombok.extern.slf4j.Slf4j; -import org.apache.http.client.methods.HttpGet; - -@Slf4j -public class EELoginClient extends AbstractEsbSdkClient implements ILoginClient { - private static final String API_GET_USER_INFO = "/api/c/compapi/v2/bk_login/get_user/"; - - public EELoginClient(String esbHostUrl, String appCode, String appSecret, String lang, boolean useEsbTestEnv) { - super(esbHostUrl, appCode, appSecret, lang, useEsbTestEnv); - } - - public EELoginClient(String esbHostUrl, String appCode, String appSecret, boolean useEsbTestEnv) { - super(esbHostUrl, appCode, appSecret, "en", useEsbTestEnv); - } - - /** - * 获取指定用户信息 - * - * @param bkToken - * @return - */ - @Override - public BkUserDTO getUserInfoByToken(String bkToken) { - EsbReq esbReq = makeBaseReqByWeb(EsbReq.class, bkToken); - return getUserInfo(esbReq); - } - - @Override - public BkUserDTO getUserInfoByUserName(String userName) { - EsbReq esbReq = makeBaseReqByWeb(EsbReq.class, null, userName, null); - return getUserInfo(esbReq); - } - - private BkUserDTO getUserInfo(EsbReq esbReq) { - try { - HttpMetricUtil.setHttpMetricName(CommonMetricNames.ESB_BK_LOGIN_API_HTTP); - HttpMetricUtil.addTagForCurrentMetric( - Tag.of("api_name", API_GET_USER_INFO) - ); - EsbResp esbResp = getEsbRespByReq( - HttpGet.METHOD_NAME, - API_GET_USER_INFO, - esbReq, - new TypeReference>() { - } - ); - return convertToBkUserDTO(esbResp.getData()); - } catch (Exception e) { - String errorMsg = "Get " + API_GET_USER_INFO + " error"; - log.error(errorMsg, e); - throw new InternalUserManageException(errorMsg, e, ErrorCode.USER_MANAGE_API_ACCESS_ERROR); - } finally { - HttpMetricUtil.clearHttpMetric(); - } - } - - private BkUserDTO convertToBkUserDTO(EsbUserDto esbUserDto) { - BkUserDTO bkUserDTO = new BkUserDTO(); - bkUserDTO.setUsername(esbUserDto.getUsername()); - bkUserDTO.setEmail(esbUserDto.getEmail()); - bkUserDTO.setPhone(esbUserDto.getPhone()); - bkUserDTO.setWxUserId(esbUserDto.getWxUserId()); - bkUserDTO.setTimeZone(esbUserDto.getTimeZone()); - return bkUserDTO; - } -} diff --git a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/login/ILoginClient.java b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/login/ILoginClient.java index f5a6d58765..35eaeea815 100644 --- a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/login/ILoginClient.java +++ b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/login/ILoginClient.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -34,15 +34,7 @@ public interface ILoginClient { * 根据token获取用户信息 * * @param token 用户登录token - * @return + * @return 用户信息 */ BkUserDTO getUserInfoByToken(String token); - - /** - * 根据用户名获取用户信息 - * - * @param userName 用户名 - * @return - */ - BkUserDTO getUserInfoByUserName(String userName); } diff --git a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/login/StandardLoginClient.java b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/login/StandardLoginClient.java new file mode 100644 index 0000000000..a0a4ff6349 --- /dev/null +++ b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/login/StandardLoginClient.java @@ -0,0 +1,135 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.paas.login; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.constant.HttpMethodEnum; +import com.tencent.bk.job.common.esb.config.AppProperties; +import com.tencent.bk.job.common.esb.config.EsbProperties; +import com.tencent.bk.job.common.esb.model.BkApiAuthorization; +import com.tencent.bk.job.common.esb.model.EsbResp; +import com.tencent.bk.job.common.esb.model.OpenApiRequestInfo; +import com.tencent.bk.job.common.esb.sdk.BkApiClient; +import com.tencent.bk.job.common.exception.InternalUserManageException; +import com.tencent.bk.job.common.metrics.CommonMetricNames; +import com.tencent.bk.job.common.model.dto.BkUserDTO; +import com.tencent.bk.job.common.paas.exception.AppPermissionDeniedException; +import com.tencent.bk.job.common.paas.model.EsbUserDto; +import com.tencent.bk.job.common.util.http.HttpHelperFactory; +import com.tencent.bk.job.common.util.http.HttpMetricUtil; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tag; +import lombok.extern.slf4j.Slf4j; + +import static com.tencent.bk.job.common.metrics.CommonMetricNames.ESB_BK_LOGIN_API; + +@Slf4j +public class StandardLoginClient extends BkApiClient implements ILoginClient { + + // 用户认证失败，即用户登录态无效 + private static final Integer ESB_CODE_USER_NOT_LOGIN = 1302100; + // 用户不存在 + private static final Integer ESB_CODE_USER_NOT_EXIST = 1302103; + // 用户认证成功，但用户无应用访问权限 + private static final Integer ESB_CODE_USER_NO_APP_PERMISSION = 1302403; + + private static final String API_GET_USER_INFO = "/api/c/compapi/v2/bk_login/get_user/"; + + private final AppProperties appProperties; + + public StandardLoginClient(EsbProperties esbProperties, AppProperties appProperties, MeterRegistry meterRegistry) { + super( + meterRegistry, + ESB_BK_LOGIN_API, + esbProperties.getService().getUrl(), + HttpHelperFactory.createHttpHelper( + httpClientBuilder -> httpClientBuilder.addInterceptorLast(getLogBkApiRequestIdInterceptor()) + ) + ); + this.appProperties = appProperties; + } + + /** + * 获取指定用户信息 + * + * @param bkToken 用户登录 token + * @return 用户信息 + */ + @Override + public BkUserDTO getUserInfoByToken(String bkToken) { + try { + HttpMetricUtil.setHttpMetricName(CommonMetricNames.ESB_BK_LOGIN_API_HTTP); + HttpMetricUtil.addTagForCurrentMetric( + Tag.of("api_name", API_GET_USER_INFO) + ); + EsbResp esbResp = doRequest( + OpenApiRequestInfo.builder() + .method(HttpMethodEnum.GET) + .uri(API_GET_USER_INFO) + .addQueryParam("bk_token", bkToken) + .authorization(BkApiAuthorization.bkTokenUserAuthorization( + appProperties.getCode(), appProperties.getSecret(), bkToken)) + .build(), + new TypeReference>() { + } + ); + Integer code = esbResp.getCode(); + if (ErrorCode.RESULT_OK == code) { + return convertToBkUserDTO(esbResp.getData()); + } else { + handleNotOkResp(esbResp); + return null; + } + } catch (Exception e) { + String errorMsg = "Get " + API_GET_USER_INFO + " error"; + log.error(errorMsg, e); + throw new InternalUserManageException(errorMsg, e, ErrorCode.USER_MANAGE_API_ACCESS_ERROR); + } finally { + HttpMetricUtil.clearHttpMetric(); + } + } + + private void handleNotOkResp(EsbResp esbResp) { + Integer code = esbResp.getCode(); + if (ESB_CODE_USER_NO_APP_PERMISSION.equals(code)) { + throw new AppPermissionDeniedException(esbResp.getMessage()); + } else if (ESB_CODE_USER_NOT_LOGIN.equals(code)) { + log.info("User not login, esbResp.code={}, esbResp.message={}", esbResp.getCode(), esbResp.getMessage()); + } else if (ESB_CODE_USER_NOT_EXIST.equals(code)) { + log.info("User not exist, esbResp.code={}, esbResp.message={}", esbResp.getCode(), esbResp.getMessage()); + } + } + + private BkUserDTO convertToBkUserDTO(EsbUserDto esbUserDto) { + BkUserDTO bkUserDTO = new BkUserDTO(); + bkUserDTO.setUsername(esbUserDto.getUsername()); + bkUserDTO.setEmail(esbUserDto.getEmail()); + bkUserDTO.setPhone(esbUserDto.getPhone()); + bkUserDTO.setWxUserId(esbUserDto.getWxUserId()); + bkUserDTO.setTimeZone(esbUserDto.getTimeZone()); + return bkUserDTO; + } +} diff --git a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/metrics/PaaSMetricTags.java b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/metrics/PaaSMetricTags.java index e70f10fb45..44457cd7e4 100644 --- a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/metrics/PaaSMetricTags.java +++ b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/metrics/PaaSMetricTags.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/model/EsbListUsersResult.java b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/model/EsbListUsersResult.java index b8c425b8e4..039d232615 100644 --- a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/model/EsbListUsersResult.java +++ b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/model/EsbListUsersResult.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/model/EsbNotifyChannelDTO.java b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/model/EsbNotifyChannelDTO.java index 50806437e0..710c0f9c64 100644 --- a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/model/EsbNotifyChannelDTO.java +++ b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/model/EsbNotifyChannelDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/model/EsbUserDto.java b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/model/EsbUserDto.java index 4a52af4419..bf9d658b19 100644 --- a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/model/EsbUserDto.java +++ b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/model/EsbUserDto.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/model/GetEsbNotifyChannelReq.java b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/model/GetEsbNotifyChannelReq.java index f877add11c..c21ea7fb58 100644 --- a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/model/GetEsbNotifyChannelReq.java +++ b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/model/GetEsbNotifyChannelReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/model/GetUserListReq.java b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/model/GetUserListReq.java index 1d93f235fa..0798947e71 100644 --- a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/model/GetUserListReq.java +++ b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/model/GetUserListReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/model/PaasUserInfoDTO.java b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/model/PaasUserInfoDTO.java index a5e864ae50..3bd4b64fbd 100644 --- a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/model/PaasUserInfoDTO.java +++ b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/model/PaasUserInfoDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/model/PostSendMsgReq.java b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/model/PostSendMsgReq.java index c0f99d452a..13303bfe63 100644 --- a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/model/PostSendMsgReq.java +++ b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/model/PostSendMsgReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/user/EEPaasClient.java b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/user/EEPaasClient.java deleted file mode 100644 index c6a0e6f146..0000000000 --- a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/user/EEPaasClient.java +++ /dev/null @@ -1,275 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.paas.user; - - -import com.fasterxml.jackson.core.type.TypeReference; -import com.tencent.bk.job.common.constant.ErrorCode; -import com.tencent.bk.job.common.esb.metrics.EsbMetricTags; -import com.tencent.bk.job.common.esb.model.EsbResp; -import com.tencent.bk.job.common.esb.sdk.AbstractEsbSdkClient; -import com.tencent.bk.job.common.exception.InternalCmsiException; -import com.tencent.bk.job.common.exception.InternalUserManageException; -import com.tencent.bk.job.common.metrics.CommonMetricNames; -import com.tencent.bk.job.common.model.dto.BkUserDTO; -import com.tencent.bk.job.common.model.error.ErrorType; -import com.tencent.bk.job.common.paas.exception.PaasException; -import com.tencent.bk.job.common.paas.metrics.PaaSMetricTags; -import com.tencent.bk.job.common.paas.model.EsbListUsersResult; -import com.tencent.bk.job.common.paas.model.EsbNotifyChannelDTO; -import com.tencent.bk.job.common.paas.model.GetEsbNotifyChannelReq; -import com.tencent.bk.job.common.paas.model.GetUserListReq; -import com.tencent.bk.job.common.paas.model.PostSendMsgReq; -import com.tencent.bk.job.common.util.http.HttpMetricUtil; -import io.micrometer.core.instrument.MeterRegistry; -import io.micrometer.core.instrument.Tag; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.springframework.util.CollectionUtils; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; - -/** - * 标准企业版对接paas api 客户端 - */ -@Slf4j -public class EEPaasClient extends AbstractEsbSdkClient implements IPaasClient { - - private static final Integer ESB_CODE_RATE_LIMIT_RESTRICTION_BY_STAGE = 1642902; - private static final Integer ESB_CODE_RATE_LIMIT_RESTRICTION_BY_RESOURCE = 1642903; - - private static final String API_GET_USER_LIST = "/api/c/compapi/v2/usermanage/list_users/"; - private static final String API_GET_NOTIFY_CHANNEL_LIST = "/api/c/compapi/cmsi/get_msg_type/"; - private static final String API_POST_SEND_MSG = "/api/c/compapi/cmsi/send_msg/"; - - private static final HashMap todayMsgStatisticsMap = new HashMap<>(); - - private final MeterRegistry meterRegistry; - - public EEPaasClient( - String esbHostUrl, - String appCode, - String appSecret, - String lang, - boolean useEsbTestEnv, - MeterRegistry meterRegistry - ) { - super(esbHostUrl, appCode, appSecret, lang, useEsbTestEnv); - this.meterRegistry = meterRegistry; - } - - @Override - public void resetTodayStatistics() { - todayMsgStatisticsMap.forEach((key, value) -> value.set(0)); - } - - @Override - public List getUserList(String fields, - String bkToken, - String uin) { - List esbUserList; - try { - GetUserListReq req = buildGetUserListReq(uin, fields); - - HttpMetricUtil.setHttpMetricName(CommonMetricNames.ESB_USER_MANAGE_API_HTTP); - HttpMetricUtil.addTagForCurrentMetric( - Tag.of(EsbMetricTags.KEY_API_NAME, API_GET_USER_LIST) - ); - EsbResp> esbResp = getEsbRespByReq( - HttpGet.METHOD_NAME, - API_GET_USER_LIST, - req, - new TypeReference>>() { - } - ); - esbUserList = esbResp.getData(); - } catch (Exception e) { - String errorMsg = "Get " + API_GET_USER_LIST + " error"; - log.error(errorMsg, e); - throw new InternalUserManageException(errorMsg, e, ErrorCode.USER_MANAGE_API_ACCESS_ERROR); - } finally { - HttpMetricUtil.clearHttpMetric(); - } - return convert(esbUserList); - } - - private GetUserListReq buildGetUserListReq(String uin, String fields) { - GetUserListReq req = makeBaseReqByWeb(GetUserListReq.class, null, uin, null); - if (StringUtils.isNotBlank(fields)) { - req.setFields(fields); - } - req.setPage(0L); - req.setPageSize(0L); - req.setNoPage(true); - return req; - } - - private List convert(List esbUserList) { - if (CollectionUtils.isEmpty(esbUserList)) { - return Collections.emptyList(); - } - List userList = new ArrayList<>(); - for (EsbListUsersResult esbUser : esbUserList) { - BkUserDTO user = new BkUserDTO(); - user.setId(esbUser.getId()); - user.setUsername(esbUser.getUsername()); - user.setDisplayName(esbUser.getDisplayName()); - user.setLogo(esbUser.getLogo()); - user.setUid(esbUser.getUid()); - userList.add(user); - } - return userList; - } - - @Override - public List getNotifyChannelList(String uin) { - GetEsbNotifyChannelReq req = makeBaseReqByWeb(GetEsbNotifyChannelReq.class, null, uin, null); - try { - HttpMetricUtil.setHttpMetricName(CommonMetricNames.ESB_CMSI_API_HTTP); - HttpMetricUtil.addTagForCurrentMetric( - Tag.of(EsbMetricTags.KEY_API_NAME, API_GET_NOTIFY_CHANNEL_LIST) - ); - EsbResp> esbResp = getEsbRespByReq( - HttpGet.METHOD_NAME, - API_GET_NOTIFY_CHANNEL_LIST, - req, - new TypeReference>>() { - } - ); - return esbResp.getData(); - } catch (Exception e) { - String errorMsg = "Get " + API_GET_NOTIFY_CHANNEL_LIST + " error"; - log.error(errorMsg, e); - throw new InternalCmsiException(errorMsg, e, ErrorCode.CMSI_MSG_CHANNEL_DATA_ERROR); - } finally { - HttpMetricUtil.clearHttpMetric(); - } - } - - @Override - public void sendMsg( - String msgType, - String sender, - Set receivers, - String title, - String content - ) { - PostSendMsgReq req = buildSendMsgReq(msgType, sender, receivers, title, content); - long start = System.nanoTime(); - String status = EsbMetricTags.VALUE_STATUS_NONE; - String uri = API_POST_SEND_MSG; - try { - HttpMetricUtil.setHttpMetricName(CommonMetricNames.ESB_CMSI_API_HTTP); - HttpMetricUtil.addTagForCurrentMetric(Tag.of(EsbMetricTags.KEY_API_NAME, uri)); - EsbResp esbResp = getEsbRespByReq( - HttpPost.METHOD_NAME, - uri, - req, - new TypeReference>() { - } - ); - - if (esbResp.getResult() == null || !esbResp.getResult() || esbResp.getCode() != 0) { - status = checkRespAndGetStatus(uri, esbResp); - throw new PaasException( - ErrorType.INTERNAL, - ErrorCode.CMSI_FAIL_TO_SEND_MSG, - new Object[]{ - esbResp.getCode().toString(), - esbResp.getMessage() - }); - } - status = EsbMetricTags.VALUE_STATUS_SUCCESS; - } catch (PaasException e) { - throw e; - } catch (Exception e) { - log.error("Fail to request {}", uri, e); - status = EsbMetricTags.VALUE_STATUS_ERROR; - throw new PaasException(e, ErrorType.INTERNAL, ErrorCode.CMSI_API_ACCESS_ERROR, new Object[]{}); - } finally { - HttpMetricUtil.clearHttpMetric(); - recordMetrics(start, status, msgType); - } - } - - private String checkRespAndGetStatus(String uri, EsbResp esbResp) { - Integer code = esbResp.getCode(); - log.warn( - "{}|requestId={}|result={}|code={}|msg={}|esbResp.getCode() != 0", - uri, - esbResp.getRequestId(), - esbResp.getResult(), - esbResp.getCode(), - esbResp.getMessage() - ); - if (code.equals(ESB_CODE_RATE_LIMIT_RESTRICTION_BY_STAGE) - || code.equals(ESB_CODE_RATE_LIMIT_RESTRICTION_BY_RESOURCE)) { - return EsbMetricTags.VALUE_STATUS_OVER_RATE; - } else { - return EsbMetricTags.VALUE_STATUS_FAIL; - } - } - - private PostSendMsgReq buildSendMsgReq(String msgType, - String sender, - Set receivers, - String title, - String content) { - PostSendMsgReq req = makeBaseReqByWeb(PostSendMsgReq.class, null, "admin", "superadmin"); - if (title == null || title.isEmpty()) { - title = "Default Title"; - } - req.setMsgType(msgType); - req.setSender(sender); - req.setReceiverUsername(String.join(",", receivers)); - req.setTitle(title); - req.setContent(content); - return req; - } - - private void recordMetrics(long startTimeNanos, String status, String msgType) { - long end = System.nanoTime(); - meterRegistry.timer( - CommonMetricNames.ESB_CMSI_API, - EsbMetricTags.KEY_API_NAME, API_POST_SEND_MSG, - EsbMetricTags.KEY_STATUS, status, - PaaSMetricTags.KEY_MSG_TYPE, msgType - ).record(end - startTimeNanos, TimeUnit.NANOSECONDS); - String key = "today.msg." + msgType + "." + status; - AtomicInteger valueWrapper = todayMsgStatisticsMap.computeIfAbsent(key, - str -> new AtomicInteger(0)); - Integer value = valueWrapper.incrementAndGet(); - log.debug("statistics:{}->{}", key, value); - meterRegistry.gauge(key, valueWrapper); - } -} diff --git a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/user/IPaasClient.java b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/user/IPaasClient.java deleted file mode 100644 index b662396711..0000000000 --- a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/user/IPaasClient.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.common.paas.user; - - -import com.tencent.bk.job.common.model.dto.BkUserDTO; -import com.tencent.bk.job.common.paas.model.EsbNotifyChannelDTO; - -import java.util.List; -import java.util.Set; - -/** - * Paas客户端 - */ -public interface IPaasClient { - - /** - * 重置当天的统计数据 - */ - void resetTodayStatistics(); - - /** - * 获取用户列表 - */ - List getUserList(String fields, String bkToken, String uin); - - /** - * 获取消息通知渠道 - */ - List getNotifyChannelList(String uin); - - /** - * ESB发通知信息接口 - */ - void sendMsg(String msgType, String sender, Set receivers, String title, String content); -} diff --git a/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/user/UserMgrApiClient.java b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/user/UserMgrApiClient.java new file mode 100644 index 0000000000..79b77f488e --- /dev/null +++ b/src/backend/commons/paas-sdk/src/main/java/com/tencent/bk/job/common/paas/user/UserMgrApiClient.java @@ -0,0 +1,139 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.paas.user; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.constant.HttpMethodEnum; +import com.tencent.bk.job.common.esb.config.AppProperties; +import com.tencent.bk.job.common.esb.config.EsbProperties; +import com.tencent.bk.job.common.esb.constants.EsbLang; +import com.tencent.bk.job.common.esb.metrics.EsbMetricTags; +import com.tencent.bk.job.common.esb.model.BkApiAuthorization; +import com.tencent.bk.job.common.esb.model.EsbResp; +import com.tencent.bk.job.common.esb.model.OpenApiRequestInfo; +import com.tencent.bk.job.common.esb.sdk.BkApiClient; +import com.tencent.bk.job.common.exception.InternalUserManageException; +import com.tencent.bk.job.common.metrics.CommonMetricNames; +import com.tencent.bk.job.common.model.dto.BkUserDTO; +import com.tencent.bk.job.common.paas.model.EsbListUsersResult; +import com.tencent.bk.job.common.paas.model.GetUserListReq; +import com.tencent.bk.job.common.util.http.HttpHelperFactory; +import com.tencent.bk.job.common.util.http.HttpMetricUtil; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tag; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.util.CollectionUtils; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +import static com.tencent.bk.job.common.metrics.CommonMetricNames.ESB_USER_MANAGE_API; + +/** + * 用户管理 API 客户端 + */ +@Slf4j +public class UserMgrApiClient extends BkApiClient { + + private static final String API_GET_USER_LIST = "/api/c/compapi/v2/usermanage/list_users/"; + + private final BkApiAuthorization authorization; + + public UserMgrApiClient(EsbProperties esbProperties, + AppProperties appProperties, + MeterRegistry meterRegistry) { + super(meterRegistry, + ESB_USER_MANAGE_API, + esbProperties.getService().getUrl(), + HttpHelperFactory.getRetryableHttpHelper(), + EsbLang.EN + ); + this.authorization = BkApiAuthorization.appAuthorization(appProperties.getCode(), + appProperties.getSecret(), "admin"); + } + + public List getAllUserList() { + String fields = "id,username,display_name,logo"; + List esbUserList; + try { + GetUserListReq req = buildGetUserListReq(fields); + + HttpMetricUtil.setHttpMetricName(CommonMetricNames.ESB_USER_MANAGE_API_HTTP); + HttpMetricUtil.addTagForCurrentMetric( + Tag.of(EsbMetricTags.KEY_API_NAME, API_GET_USER_LIST) + ); + EsbResp> esbResp = doRequest( + OpenApiRequestInfo.builder() + .method(HttpMethodEnum.GET) + .uri(API_GET_USER_LIST) + .queryParams(req.toUrlParams()) + .authorization(authorization) + .build(), + new TypeReference>>() { + } + ); + esbUserList = esbResp.getData(); + } catch (Exception e) { + String errorMsg = "Get " + API_GET_USER_LIST + " error"; + log.error(errorMsg, e); + throw new InternalUserManageException(errorMsg, e, ErrorCode.USER_MANAGE_API_ACCESS_ERROR); + } finally { + HttpMetricUtil.clearHttpMetric(); + } + return convert(esbUserList); + } + + private GetUserListReq buildGetUserListReq(String fields) { + GetUserListReq req = new GetUserListReq(); + if (StringUtils.isNotBlank(fields)) { + req.setFields(fields); + } + req.setPage(0L); + req.setPageSize(0L); + req.setNoPage(true); + return req; + } + + private List convert(List esbUserList) { + if (CollectionUtils.isEmpty(esbUserList)) { + return Collections.emptyList(); + } + List userList = new ArrayList<>(); + for (EsbListUsersResult esbUser : esbUserList) { + BkUserDTO user = new BkUserDTO(); + user.setId(esbUser.getId()); + user.setUsername(esbUser.getUsername()); + user.setDisplayName(esbUser.getDisplayName()); + user.setLogo(esbUser.getLogo()); + user.setUid(esbUser.getUid()); + userList.add(user); + } + return userList; + } + +} diff --git a/src/backend/commons/paas-sdk/src/main/resources/META-INF/spring.factories b/src/backend/commons/paas-sdk/src/main/resources/META-INF/spring.factories new file mode 100644 index 0000000000..e0f37e466c --- /dev/null +++ b/src/backend/commons/paas-sdk/src/main/resources/META-INF/spring.factories @@ -0,0 +1,4 @@ +org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ +com.tencent.bk.job.common.paas.config.LoginAutoConfiguration,\ +com.tencent.bk.job.common.paas.config.CmsiAutoConfiguration,\ +com.tencent.bk.job.common.paas.config.UserMgrAutoConfiguration diff --git a/src/backend/gradle.properties b/src/backend/gradle.properties index 2e9367317e..22327fc23a 100644 --- a/src/backend/gradle.properties +++ b/src/backend/gradle.properties @@ -1,7 +1,7 @@ # # Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. # -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +# Copyright (C) 2021 Tencent. All rights reserved. # # BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. # @@ -21,12 +21,12 @@ # CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # - -org.gradle.jvmargs=-Xmx2048m +org.gradle.caching=true +org.gradle.parallel=true +org.gradle.jvmargs=-Xmx2g -Dfile.encoding=UTF-8 org.gradle.daemon=true org.gradle.daemon.idleTimeout=3600000 MAVEN_REPO_URL= -MAVEN_REPO_DEPLOY_URL= MAVEN_REPO_USERNAME= MAVEN_REPO_PASSWORD= DB_HOST= diff --git a/src/backend/gradle/wrapper/gradle-wrapper.properties b/src/backend/gradle/wrapper/gradle-wrapper.properties index 0d770c08b8..b1ae827f2a 100644 --- a/src/backend/gradle/wrapper/gradle-wrapper.properties +++ b/src/backend/gradle/wrapper/gradle-wrapper.properties @@ -1,7 +1,7 @@ # -# Tencent is pleased to support the open source community by making BK-JOB����������ҵƽ̨ available. +# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. # -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +# Copyright (C) 2021 Tencent. All rights reserved. # # BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. # diff --git a/src/backend/job-analysis/api-common-job-analysis/build.gradle b/src/backend/job-analysis/api-common-job-analysis/build.gradle new file mode 100644 index 0000000000..8313411560 --- /dev/null +++ b/src/backend/job-analysis/api-common-job-analysis/build.gradle @@ -0,0 +1,29 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +dependencies { + implementation 'com.fasterxml.jackson.core:jackson-core' + implementation 'com.fasterxml.jackson.core:jackson-databind' + implementation 'com.fasterxml.jackson.core:jackson-annotations' +} diff --git a/src/backend/commons/common-statistics/src/main/java/com/tencent/bk/job/common/statistics/consts/StatisticsConstants.java b/src/backend/job-analysis/api-common-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/consts/StatisticsConstants.java similarity index 92% rename from src/backend/commons/common-statistics/src/main/java/com/tencent/bk/job/common/statistics/consts/StatisticsConstants.java rename to src/backend/job-analysis/api-common-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/consts/StatisticsConstants.java index 287320f010..296eb4f4c9 100644 --- a/src/backend/commons/common-statistics/src/main/java/com/tencent/bk/job/common/statistics/consts/StatisticsConstants.java +++ b/src/backend/job-analysis/api-common-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/consts/StatisticsConstants.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.common.statistics.consts; +package com.tencent.bk.job.analysis.api.consts; import java.util.Collections; import java.util.List; @@ -63,6 +63,8 @@ public class StatisticsConstants { public static final String RESOURCE_ACCOUNT_OF_ALL_APP = "accountOfAllApp"; // 所有业务一天的任务执行量 public static final String RESOURCE_ONE_DAY_EXECUTED_TASK_OF_ALL_APP = "oneDayExecutedTaskOfAllApp"; + // 所有业务一天的滚动任务执行量 + public static final String RESOURCE_ONE_DAY_ROLLING_TASK_OF_ALL_APP = "oneDayRollingTaskOfAllApp"; // 所有业务一天的失败任务量 public static final String RESOURCE_ONE_DAY_FAILED_TASK_OF_ALL_APP = "oneDayFailedTaskOfAllApp"; // 所有业务一天的脚本执行量 @@ -75,6 +77,10 @@ public class StatisticsConstants { public static final String RESOURCE_EXECUTED_TASK = "executedTask"; // 失败的任务（含快速脚本、快速文件、作业） public static final String RESOURCE_FAILED_TASK = "failedTask"; + // 滚动执行过的任务（含快速脚本、快速文件、作业） + public static final String RESOURCE_ROLLING_TASK = "rollingTask"; + // 滚动执行失败的任务（含快速脚本、快速文件、作业） + public static final String RESOURCE_ROLLING_FAILED_TASK = "rollingFailedTask"; // 执行过的快速执行脚本 public static final String RESOURCE_EXECUTED_FAST_SCRIPT = "executedFastScript"; // 执行过的快速分发文件 @@ -144,10 +150,14 @@ public class StatisticsConstants { public static final String DIMENSION_VALUE_ACCOUNT_TYPE_LINUX = "LINUX"; public static final String DIMENSION_VALUE_ACCOUNT_TYPE_WINDOWS = "WINDOWS"; public static final String DIMENSION_VALUE_ACCOUNT_TYPE_DB = "DB"; - // 维度取值：主机系统类型 + // 维度取值：Job关注的主机系统类型 public static final String DIMENSION_VALUE_HOST_SYSTEM_TYPE_LINUX = "LINUX"; public static final String DIMENSION_VALUE_HOST_SYSTEM_TYPE_WINDOWS = "WINDOWS"; public static final String DIMENSION_VALUE_HOST_SYSTEM_TYPE_AIX = "AIX"; + public static final String DIMENSION_VALUE_HOST_SYSTEM_TYPE_UNIX = "UNIX"; + public static final String DIMENSION_VALUE_HOST_SYSTEM_TYPE_SOLARIS = "SOLARIS"; + public static final String DIMENSION_VALUE_HOST_SYSTEM_TYPE_FREEBSD = "FREEBSD"; + public static final String DIMENSION_VALUE_HOST_SYSTEM_TYPE_MACOS = "MACOS"; public static final String DIMENSION_VALUE_HOST_SYSTEM_TYPE_OTHERS = "OTHERS"; // 维度取值：任务启动方式 public static final String DIMENSION_VALUE_TASK_STARTUP_MODE_NORMAL = "NORMAL"; diff --git a/src/backend/commons/common-statistics/src/main/java/com/tencent/bk/job/common/statistics/model/dto/StatisticsDTO.java b/src/backend/job-analysis/api-common-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/dto/StatisticsDTO.java similarity index 92% rename from src/backend/commons/common-statistics/src/main/java/com/tencent/bk/job/common/statistics/model/dto/StatisticsDTO.java rename to src/backend/job-analysis/api-common-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/dto/StatisticsDTO.java index 870a9f93ea..d7a71c3cb7 100644 --- a/src/backend/commons/common-statistics/src/main/java/com/tencent/bk/job/common/statistics/model/dto/StatisticsDTO.java +++ b/src/backend/job-analysis/api-common-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/dto/StatisticsDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.common.statistics.model.dto; +package com.tencent.bk.job.analysis.api.dto; import lombok.AllArgsConstructor; import lombok.Data; @@ -62,7 +62,7 @@ public class StatisticsDTO implements Cloneable { */ private String date; /** - * 统计值 + * 序列化的统计数据 */ private String value; /** @@ -74,6 +74,10 @@ public class StatisticsDTO implements Cloneable { */ private Long lastModifyTime; + public StatisticsDTO(String value) { + this.value = value; + } + @Override public StatisticsDTO clone() { StatisticsDTO statisticsDTO = new StatisticsDTO(); diff --git a/src/backend/job-analysis/api-job-analysis/build.gradle b/src/backend/job-analysis/api-job-analysis/build.gradle index 22f48b65bc..dd548dd55d 100644 --- a/src/backend/job-analysis/api-job-analysis/build.gradle +++ b/src/backend/job-analysis/api-job-analysis/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,7 +28,7 @@ dependencies { api project(':commons:common-i18n') api project(':commons:cmdb-sdk') api project(':commons:common-iam') - api project(':commons:common-statistics') + api project(':job-analysis:api-common-job-analysis') api project(':job-execute:api-job-execute') api project(':job-manage:api-job-manage') api(project(":commons:common-api")) diff --git a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/iam/IamDashBoardViewCallbackResource.java b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/iam/IamDashBoardViewCallbackResource.java index 9123a5de81..2936f73ca5 100644 --- a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/iam/IamDashBoardViewCallbackResource.java +++ b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/iam/IamDashBoardViewCallbackResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/op/OpResource.java b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/op/OpResource.java index bbfc817040..6d5d303d03 100644 --- a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/op/OpResource.java +++ b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/op/OpResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/web/WebAIResource.java b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/web/WebAIResource.java new file mode 100644 index 0000000000..3e193268dd --- /dev/null +++ b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/web/WebAIResource.java @@ -0,0 +1,254 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.api.web; + +import com.tencent.bk.job.analysis.model.web.req.AIAnalyzeErrorReq; +import com.tencent.bk.job.analysis.model.web.req.AICheckScriptReq; +import com.tencent.bk.job.analysis.model.web.req.AIGeneralChatReq; +import com.tencent.bk.job.analysis.model.web.req.GenerateChatStreamReq; +import com.tencent.bk.job.analysis.model.web.req.TerminateChatReq; +import com.tencent.bk.job.analysis.model.web.resp.AIChatRecord; +import com.tencent.bk.job.analysis.model.web.resp.ClearChatHistoryResp; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.annotation.WebAPI; +import com.tencent.bk.job.common.constant.CompatibleType; +import com.tencent.bk.job.common.model.Response; +import com.tencent.bk.job.common.model.dto.AppResourceScope; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import org.hibernate.validator.constraints.Range; +import org.springframework.http.ResponseEntity; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; +import org.springframework.web.bind.annotation.RequestAttribute; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; +import springfox.documentation.annotations.ApiIgnore; + +import javax.validation.constraints.Min; +import java.util.List; +import java.util.Map; + +/** + * Web端使用的AI功能相关接口定义 + */ +@Validated +@Api(tags = {"job-analysis:web:AI"}) +@RequestMapping("/web/ai") +@RestController +@WebAPI +public interface WebAIResource { + + @CompatibleImplementation( + name = "ai_config", + deprecatedVersion = "3.11.5", + type = CompatibleType.DEPLOY, + explain = "兼容 API， 发布完成后前端使用 getAIConfig 接口，该接口可删除" + ) + @ApiOperation(value = "获取AI相关的配置参数，取值：\n" + + "enabled：表示是否启用AI功能；\n" + + "analyzeErrorLogMaxLength：表示分析报错信息时支持的最大日志长度，单位为字符；", + produces = "application/json") + @GetMapping("/scope/{scopeType}/{scopeId}/config") + Response> getAIConfigOfScope( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId + ); + + @ApiOperation(value = "获取AI相关的配置参数，取值：\n" + + "enabled：表示是否启用AI功能；\n" + + "analyzeErrorLogMaxLength：表示分析报错信息时支持的最大日志长度，单位为字符；", + produces = "application/json") + @GetMapping("/config") + Response> getAIConfig( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username + ); + + @ApiOperation(value = "获取最近的AI对话记录历史（按产生时间倒序排列）", produces = "application/json") + @GetMapping("/scope/{scopeType}/{scopeId}/latestChatHistoryList") + Response> getLatestChatHistoryList( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId, + @ApiParam(value = "对话记录起始位置，不传默认为0") + @RequestParam(value = "start", defaultValue = "0") + @Min(value = 0L, message = "{validation.constraints.AIInvalidHistoryStart.message}") + Integer start, + @ApiParam(value = "需要获取的对话记录条数，最大200条，不传默认20条") + @RequestParam(value = "length", defaultValue = "20") + @Range(max = 200L, message = "{validation.constraints.AIInvalidHistoryLength.message}") + Integer length + ); + + @ApiOperation(value = "通用对话接口", produces = "application/json") + @PostMapping("/scope/{scopeType}/{scopeId}/general/chat") + Response generalChat( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId, + @ApiParam(value = "AI通用对话参数", required = true) + @Validated + @RequestBody AIGeneralChatReq req + ); + + @ApiOperation(value = "检查脚本", produces = "application/json") + @PostMapping("/scope/{scopeType}/{scopeId}/checkScript") + Response checkScript( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId, + @ApiParam(value = "AI检查脚本参数", required = true) + @Validated + @RequestBody AICheckScriptReq req + ); + + @ApiOperation(value = "分析报错信息", produces = "application/json") + @PostMapping("/scope/{scopeType}/{scopeId}/analyzeError") + Response analyzeError( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId, + @ApiParam(value = "AI分析报错信息参数", required = true) + @Validated + @RequestBody AIAnalyzeErrorReq req + ); + + @ApiOperation(value = "获取单次对话流式数据（流式接口），返回换行符分隔的多条JSON数据，可分块读取，单条JSON数据格式：{\"success\":true,\"code\":0," + + "\"errorMsg\":\"成功\",\"data\":{\"errorCode\":\"0\",\"errorMessage\":null,\"content\":\"hello world\"," + + "\"time\":\"2024-08-14 12:00:00\"},\"requestId\":\"fb991170da868b2a1eb5835bc426e992\",\"authResult\": null," + + "\"errorDetail\": null}", produces = "application/json") + @PostMapping("/scope/{scopeType}/{scopeId}/chatStream") + ResponseEntity generateChatStream( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId, + @ApiParam(value = "生成流式响应数据请求参数", required = true) + @Validated + @RequestBody GenerateChatStreamReq req + ); + + @ApiOperation(value = "终止对话", produces = "application/json") + @PutMapping("/scope/{scopeType}/{scopeId}/terminateChat") + Response terminateChat( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId, + @ApiParam(value = "终止对话请求参数", required = true) + @Validated + @RequestBody TerminateChatReq req + ); + + @ApiOperation(value = "清空对话记录", produces = "application/json") + @DeleteMapping("/scope/{scopeType}/{scopeId}/clearChatHistory") + Response clearChatHistory( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId + ); +} diff --git a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/web/WebIndexResource.java b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/web/WebIndexResource.java index 1850912ba1..fdaea532c8 100644 --- a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/web/WebIndexResource.java +++ b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/web/WebIndexResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/web/WebScriptStatisticsResource.java b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/web/WebScriptStatisticsResource.java index deb9c52818..3d68e8ceae 100644 --- a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/web/WebScriptStatisticsResource.java +++ b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/web/WebScriptStatisticsResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/web/WebStatisticsResource.java b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/web/WebStatisticsResource.java index cb7b5c4902..55db3bb43e 100644 --- a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/web/WebStatisticsResource.java +++ b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/web/WebStatisticsResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -142,7 +142,7 @@ Response distributionStatistics( @RequestHeader("username") String username, @ApiParam("统计量Code，取值：(\n" + - "HOST_SYSTEM_TYPE:主机操作系统类型（结果label取值：LINUX，WINDOWS，AIX，OTHERS）\n" + + "HOST_SYSTEM_TYPE:主机操作系统类型（结果label取值：LINUX，WINDOWS，AIX，UNIX，SOLARIS，FREEBSD，MACOS，OTHERS）\n" + "STEP_TYPE:作业步骤类型（结果label取值：FILE_LOCAL，FILE_SERVER，SCRIPT_MANUAL，SCRIPT_REF，CONFIRM）\n" + "SCRIPT_TYPE:脚本类型（结果label取值：Bat，Shell，Perl，Powershell，Python，SQL）\n" + "SCRIPT_VERSION_STATUS:脚本版本状态（结果label取值：ONLINE，OFFLINE，PREPARING，FORBIDDEN）\n" + @@ -170,6 +170,8 @@ Response> dayDetailStatistics( @ApiParam("资源类型，取值：(\n" + "EXECUTED_TASK:执行过的任务（包含快速执行脚本、快速分发文件、作业）（对应dimension取值：TASK_STARTUP_MODE，TASK_TYPE，TASK_TIME_CONSUMING" + "）\n" + + "EXECUTED_ROLLING_TASK:滚动执行过的任务（包含快速执行脚本、快速分发文件、作业）（对应dimension取值：TASK_TYPE" + + "）\n" + "EXECUTED_FAST_SCRIPT:执行过的快速执行脚本（对应dimension取值：SCRIPT_TYPE）\n" + "EXECUTED_FAST_FILE:执行过的快速分发文件（对应dimension取值：FILE_TRANSFER_MODE）\n" + ")") diff --git a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/AIChatStatusEnum.java b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/AIChatStatusEnum.java new file mode 100644 index 0000000000..cfd49a4f2c --- /dev/null +++ b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/AIChatStatusEnum.java @@ -0,0 +1,70 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.consts; + +import lombok.Getter; + +/** + * AI对话状态 + */ +public enum AIChatStatusEnum { + /** + * 初始状态 + */ + INIT(0, "Chat initialized."), + /** + * 正在回答 + */ + REPLYING(1, "Chat is replying."), + /** + * 已完成 + */ + FINISHED(2, "Chat finished."), + /** + * 已终止 + */ + TERMINATED(3, "Chat terminated."); + + @Getter + private final int status; + + @Getter + private final String description; + + AIChatStatusEnum(int status, String description) { + this.status = status; + this.description = description; + } + + public static AIChatStatusEnum getAIChatStatus(int status) { + for (AIChatStatusEnum aiChatStatusEnum : AIChatStatusEnum.values()) { + if (aiChatStatusEnum.getStatus() == status) { + return aiChatStatusEnum; + } + } + throw new RuntimeException("Unknown AIChat status " + status); + } + +} diff --git a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/AIConsts.java b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/AIConsts.java new file mode 100644 index 0000000000..131ee86fce --- /dev/null +++ b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/AIConsts.java @@ -0,0 +1,39 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.consts; + +/** + * AI功能相关的常量 + */ +public class AIConsts { + // AI回答错误码：OK + public static final String AI_ANSWER_ERROR_CODE_OK = "0"; + // AI回答错误码：Failed + public static final String AI_ANSWER_ERROR_CODE_FAILED = "1"; + // AI回答最大长度：1.5w字 + public static final int MAX_LENGTH_AI_ANSWER = 15_000; + // AI回答报错信息最大长度：512字 + public static final int MAX_LENGTH_AI_ANSWER_ERROR_MESSAGE = 512; +} diff --git a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/AccountTypeEnum.java b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/AccountTypeEnum.java index 185a90dcee..ff992d7d31 100644 --- a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/AccountTypeEnum.java +++ b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/AccountTypeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/AnalysisConsts.java b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/AnalysisConsts.java index aeb4ea2568..884757c14c 100644 --- a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/AnalysisConsts.java +++ b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/AnalysisConsts.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/DimensionEnum.java b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/DimensionEnum.java index 8af4484d20..853d7ac789 100644 --- a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/DimensionEnum.java +++ b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/DimensionEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/DistributionMetricEnum.java b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/DistributionMetricEnum.java index f208290914..a8e6c0da06 100644 --- a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/DistributionMetricEnum.java +++ b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/DistributionMetricEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/PromptTemplateCodeEnum.java b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/PromptTemplateCodeEnum.java new file mode 100644 index 0000000000..e57cca5c51 --- /dev/null +++ b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/PromptTemplateCodeEnum.java @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.consts; + +import lombok.AllArgsConstructor; +import lombok.Getter; + +@Getter +@AllArgsConstructor +public enum PromptTemplateCodeEnum { + /** + * 检查脚本 + */ + CHECK_SCRIPT, + /** + * 分析脚本执行任务报错信息 + */ + ANALYZE_SCRIPT_EXECUTE_TASK_ERROR, + /** + * 分析文件分发任务报错信息 + */ + ANALYZE_FILE_TRANSFER_TASK_ERROR +} diff --git a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/ResourceEnum.java b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/ResourceEnum.java index 76efef14c1..03868199dd 100644 --- a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/ResourceEnum.java +++ b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/ResourceEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -34,6 +34,10 @@ public enum ResourceEnum { * 执行过的任务（包含快速执行脚本、快速分发文件、作业） */ EXECUTED_TASK("job.analysis.resources.executedTask"), + /** + * 滚动执行过的任务（包含快速执行脚本、快速分发文件、作业） + */ + EXECUTED_ROLLING_TASK("job.analysis.resources.executedRollingTask"), /** * 执行过的快速执行脚本 */ diff --git a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/StepTypeEnum.java b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/StepTypeEnum.java index b106f7b5a3..57003df404 100644 --- a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/StepTypeEnum.java +++ b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/StepTypeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/TotalMetricEnum.java b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/TotalMetricEnum.java index 6b7d69b6fe..4210a31e2b 100644 --- a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/TotalMetricEnum.java +++ b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/TotalMetricEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/inner/AnalysisTaskResultItemLocation.java b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/inner/AnalysisTaskResultItemLocation.java index e3246fbd8b..bcf23d8ca6 100644 --- a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/inner/AnalysisTaskResultItemLocation.java +++ b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/inner/AnalysisTaskResultItemLocation.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/inner/PerAppStatisticDTO.java b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/inner/PerAppStatisticDTO.java index b8a7df3d19..c3e9302427 100644 --- a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/inner/PerAppStatisticDTO.java +++ b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/inner/PerAppStatisticDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/op/CancelTasksReq.java b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/op/CancelTasksReq.java index 2bfa74b69a..77eef18902 100644 --- a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/op/CancelTasksReq.java +++ b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/op/CancelTasksReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/op/ClearStatisticsReq.java b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/op/ClearStatisticsReq.java index b6d3e31cac..e959aaf47b 100644 --- a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/op/ClearStatisticsReq.java +++ b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/op/ClearStatisticsReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/op/ConfigStatisticsReq.java b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/op/ConfigStatisticsReq.java index a860db5421..bd907945d5 100644 --- a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/op/ConfigStatisticsReq.java +++ b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/op/ConfigStatisticsReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/op/ConfigThreadsReq.java b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/op/ConfigThreadsReq.java index 35d4988a46..7d4e2dff24 100644 --- a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/op/ConfigThreadsReq.java +++ b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/op/ConfigThreadsReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/op/StartTasksReq.java b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/op/StartTasksReq.java index 92e713682f..03913f5617 100644 --- a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/op/StartTasksReq.java +++ b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/op/StartTasksReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/AnalysisResultItemVO.java b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/AnalysisResultItemVO.java index b322b00eb5..73edcb2ea3 100644 --- a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/AnalysisResultItemVO.java +++ b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/AnalysisResultItemVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/AnalysisResultVO.java b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/AnalysisResultVO.java index 3830f81bd3..b3b884a63d 100644 --- a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/AnalysisResultVO.java +++ b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/AnalysisResultVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/CommonDistributionVO.java b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/CommonDistributionVO.java index 31a74d25be..2a4c7c7c40 100644 --- a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/CommonDistributionVO.java +++ b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/CommonDistributionVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/CommonStatisticWithRateVO.java b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/CommonStatisticWithRateVO.java index 5963d09091..1298a7ccce 100644 --- a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/CommonStatisticWithRateVO.java +++ b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/CommonStatisticWithRateVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -52,9 +52,9 @@ public class CommonStatisticWithRateVO { private Float momRate; @ApiModelProperty("同比趋势：1：上升，0：不变，-1：下降") - private Long yoyTrend; + private Integer yoyTrend; @ApiModelProperty("环比趋势：1：上升，0：不变，-1：下降") - private Long momTrend; + private Integer momTrend; } diff --git a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/CommonTrendElementVO.java b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/CommonTrendElementVO.java index bf4e13bd43..96edea2f69 100644 --- a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/CommonTrendElementVO.java +++ b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/CommonTrendElementVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/DayDistributionElementVO.java b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/DayDistributionElementVO.java index 0593731281..8e1cd28362 100644 --- a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/DayDistributionElementVO.java +++ b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/DayDistributionElementVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/HostStatisticVO.java b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/HostStatisticVO.java index 2f30ce8376..edc7e2e006 100644 --- a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/HostStatisticVO.java +++ b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/HostStatisticVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/PerAppStatisticVO.java b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/PerAppStatisticVO.java index c052a1f2d4..71cc0300be 100644 --- a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/PerAppStatisticVO.java +++ b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/PerAppStatisticVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/ScriptCiteStatisticVO.java b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/ScriptCiteStatisticVO.java index 35b9eb1735..19589391f6 100644 --- a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/ScriptCiteStatisticVO.java +++ b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/ScriptCiteStatisticVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/req/AIAnalyzeErrorReq.java b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/req/AIAnalyzeErrorReq.java new file mode 100644 index 0000000000..33eab6d110 --- /dev/null +++ b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/req/AIAnalyzeErrorReq.java @@ -0,0 +1,73 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.model.web.req; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import javax.validation.constraints.NotNull; + +@AllArgsConstructor +@NoArgsConstructor +@ApiModel("AI分析报错信息请求体") +@Data +public class AIAnalyzeErrorReq { + + @ApiModelProperty(value = "任务ID") + @NotNull(message = "{validation.constraints.AIAnalyzeError_taskInstanceIdEmpty.message}") + private Long taskInstanceId; + + @ApiModelProperty("步骤执行类型：1-脚本，2-文件") + private Integer stepExecuteType; + + @ApiModelProperty(value = "步骤ID") + @NotNull(message = "{validation.constraints.AIAnalyzeError_stepInstanceIdEmpty.message}") + private Long stepInstanceId; + + @ApiModelProperty(value = "执行次数") + @NotNull(message = "{validation.constraints.AIAnalyzeError_executeCountEmpty.message}") + private Integer executeCount; + + @ApiModelProperty(value = "滚动批次，非滚动步骤不需要传入") + @NotNull(message = "{validation.constraints.AIAnalyzeError_batchEmpty.message}") + private Integer batch; + + @ApiModelProperty(value = "执行对象类型") + @NotNull(message = "{validation.constraints.AIAnalyzeError_executeObjectTypeEmpty.message}") + private Integer executeObjectType; + + @ApiModelProperty(value = "执行对象资源 ID") + @NotNull(message = "{validation.constraints.AIAnalyzeError_executeObjectResourceIdEmpty.message}") + private Long executeObjectResourceId; + + @ApiModelProperty(value = "文件任务上传下载标识,0-上传,1-下载") + private Integer mode; + + @ApiModelProperty(value = "脚本任务报错信息内容") + private String content; +} diff --git a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/req/AICheckScriptReq.java b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/req/AICheckScriptReq.java new file mode 100644 index 0000000000..5f54ab17ce --- /dev/null +++ b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/req/AICheckScriptReq.java @@ -0,0 +1,62 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.model.web.req; + +import com.tencent.bk.job.common.validation.MaxLength; +import com.tencent.bk.job.common.validation.CheckEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.NotNull; + +@AllArgsConstructor +@NoArgsConstructor +@ApiModel("AI检查脚本请求体") +@Data +public class AICheckScriptReq { + + /** + * 脚本类型 + */ + @ApiModelProperty(value = "脚本类型，1：shell，2：bat，3：perl，4：python，5：powershell，6：SQL") + @NotNull(message = "{validation.constraints.ScriptType_empty.message}") + @CheckEnum(enumClass = ScriptTypeEnum.class, enumMethod = "isValid", + message = "{validation.constraints.ScriptType_illegal.message}") + private Integer type; + + /** + * 脚本内容 + */ + @ApiModelProperty(value = "脚本内容，BASE64编码") + @NotEmpty(message = "{validation.constraints.ScriptContent_empty.message}") + @MaxLength(value = 5 * 1024L * 1024L, + message = "{validation.constraints.AICheckScript_contentExceedMaxLength.message}") + private String content; +} diff --git a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/req/AIGeneralChatReq.java b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/req/AIGeneralChatReq.java new file mode 100644 index 0000000000..8636470d81 --- /dev/null +++ b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/req/AIGeneralChatReq.java @@ -0,0 +1,50 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.model.web.req; + +import com.tencent.bk.job.common.validation.MaxLength; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import javax.validation.constraints.NotEmpty; + +@AllArgsConstructor +@NoArgsConstructor +@ApiModel("AI通用对话请求体") +@Data +public class AIGeneralChatReq { + + /** + * 用户输入内容 + */ + @ApiModelProperty(value = "用户输入内容") + @NotEmpty(message = "{validation.constraints.AIGeneralChat_contentEmpty.message}") + @MaxLength(value = 5 * 1024L * 1024L, + message = "{validation.constraints.AIGeneralChat_contentExceedMaxLength.message}") + private String content; +} diff --git a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/req/GenerateChatStreamReq.java b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/req/GenerateChatStreamReq.java new file mode 100644 index 0000000000..7736974621 --- /dev/null +++ b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/req/GenerateChatStreamReq.java @@ -0,0 +1,43 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.model.web.req; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiParam; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import javax.validation.constraints.Min; + +@AllArgsConstructor +@NoArgsConstructor +@ApiModel("生成流式响应请求体") +@Data +public class GenerateChatStreamReq { + @ApiParam(value = "对话记录ID") + @Min(value = 1L, message = "{validation.constraints.AIInvalidRecordId.message}") + Long recordId; +} diff --git a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/req/TerminateChatReq.java b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/req/TerminateChatReq.java new file mode 100644 index 0000000000..fd88c6a31e --- /dev/null +++ b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/req/TerminateChatReq.java @@ -0,0 +1,43 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.model.web.req; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiParam; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import javax.validation.constraints.Min; + +@AllArgsConstructor +@NoArgsConstructor +@ApiModel("终止对话请求体") +@Data +public class TerminateChatReq { + @ApiParam(value = "对话记录ID") + @Min(value = 1L, message = "{validation.constraints.AIInvalidRecordId.message}") + Long recordId; +} diff --git a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/resp/AIAnswer.java b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/resp/AIAnswer.java new file mode 100644 index 0000000000..84a13ea712 --- /dev/null +++ b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/resp/AIAnswer.java @@ -0,0 +1,83 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.model.web.resp; + +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.tencent.bk.job.analysis.consts.AIConsts; +import com.tencent.bk.job.common.util.json.LongTimestampSerializer; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@AllArgsConstructor +@NoArgsConstructor +@ApiModel("AI回答内容") +@Data +public class AIAnswer { + + /** + * 错误码 + */ + @ApiModelProperty(value = "错误码") + private String errorCode; + + /** + * 错误信息 + */ + @ApiModelProperty(value = "错误信息") + private String errorMessage; + + /** + * 内容 + */ + @ApiModelProperty(value = "内容") + private String content; + + /** + * 回答时间 + */ + @ApiModelProperty("回答时间") + @JsonSerialize(using = LongTimestampSerializer.class) + private Long time; + + public static AIAnswer successAnswer(String content) { + AIAnswer aiAnswer = new AIAnswer(); + aiAnswer.setErrorCode(AIConsts.AI_ANSWER_ERROR_CODE_OK); + aiAnswer.setTime(System.currentTimeMillis()); + aiAnswer.setContent(content); + return aiAnswer; + } + + public static AIAnswer failAnswer(String content, String errorMessage) { + AIAnswer aiAnswer = new AIAnswer(); + aiAnswer.setErrorCode(AIConsts.AI_ANSWER_ERROR_CODE_FAILED); + aiAnswer.setErrorMessage(errorMessage); + aiAnswer.setTime(System.currentTimeMillis()); + aiAnswer.setContent(content); + return aiAnswer; + } +} diff --git a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/resp/AIChatRecord.java b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/resp/AIChatRecord.java new file mode 100644 index 0000000000..024e86aad5 --- /dev/null +++ b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/resp/AIChatRecord.java @@ -0,0 +1,54 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.model.web.resp; + +import com.tencent.bk.job.analysis.consts.AIChatStatusEnum; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@AllArgsConstructor +@NoArgsConstructor +@ApiModel("一条AI对话记录") +@Data +public class AIChatRecord { + + @ApiModelProperty(value = "ID") + private Long id; + + @ApiModelProperty(value = "用户输入") + private UserInput userInput; + + @ApiModelProperty("AI回答") + private AIAnswer aiAnswer; + + /** + * 对话状态，取值源于{@link AIChatStatusEnum}. + */ + @ApiModelProperty("对话状态：1-正在回答，2-已完成") + private Integer status; +} diff --git a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/resp/ClearChatHistoryResp.java b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/resp/ClearChatHistoryResp.java new file mode 100644 index 0000000000..f4a550aa1f --- /dev/null +++ b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/resp/ClearChatHistoryResp.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.model.web.resp; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@AllArgsConstructor +@NoArgsConstructor +@ApiModel("清空对话记录响应数据") +@Data +public class ClearChatHistoryResp { + + /** + * 被清空的对话记录数量 + */ + @ApiModelProperty(value = "被清空的对话记录数量") + private Integer count; +} diff --git a/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/resp/UserInput.java b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/resp/UserInput.java new file mode 100644 index 0000000000..57b336053d --- /dev/null +++ b/src/backend/job-analysis/api-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/web/resp/UserInput.java @@ -0,0 +1,52 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.model.web.resp; + +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.tencent.bk.job.common.util.json.LongTimestampSerializer; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@AllArgsConstructor +@NoArgsConstructor +@ApiModel("用户提问") +@Data +public class UserInput { + /** + * 内容 + */ + @ApiModelProperty(value = "内容") + private String content; + + /** + * 提问时间 + */ + @ApiModelProperty("提问时间") + @JsonSerialize(using = LongTimestampSerializer.class) + private Long time; +} diff --git a/src/backend/job-analysis/boot-job-analysis/build.gradle b/src/backend/job-analysis/boot-job-analysis/build.gradle index 6e99aed05e..14c38a8600 100644 --- a/src/backend/job-analysis/boot-job-analysis/build.gradle +++ b/src/backend/job-analysis/boot-job-analysis/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -32,7 +32,7 @@ dependencies { implementation 'org.springframework.cloud:spring-cloud-starter-bootstrap' implementation 'org.springframework:spring-webmvc' implementation(group: 'org.springframework.boot', name: 'spring-boot-starter-data-redis') - runtimeOnly('mysql:mysql-connector-java') + runtimeOnly('com.mysql:mysql-connector-j') testImplementation("com.h2database:h2") } @@ -46,8 +46,7 @@ task renameArtifacts(type: Copy) { destinationDir file('build/libs/') rename "boot-job-analysis-${version}.jar", "job-analysis-${version}.jar" } -renameArtifacts.dependsOn jar -assemble.dependsOn renameArtifacts +renameArtifacts.dependsOn assemble task copyToLatestJar(type: Copy) { group = "local" @@ -59,3 +58,4 @@ task copyToLatestJar(type: Copy) { copyToLatestJar.dependsOn assemble apply from: "$rootDir/task_job_package.gradle" +copyToRelease.dependsOn renameArtifacts diff --git a/src/backend/job-analysis/boot-job-analysis/src/main/java/com/tencent/bk/job/analysis/JobAnalysisBootApplication.java b/src/backend/job-analysis/boot-job-analysis/src/main/java/com/tencent/bk/job/analysis/JobAnalysisBootApplication.java index 6b350c53e5..8c52280c84 100644 --- a/src/backend/job-analysis/boot-job-analysis/src/main/java/com/tencent/bk/job/analysis/JobAnalysisBootApplication.java +++ b/src/backend/job-analysis/boot-job-analysis/src/main/java/com/tencent/bk/job/analysis/JobAnalysisBootApplication.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,22 +24,28 @@ package com.tencent.bk.job.analysis; -import com.tencent.bk.job.common.config.FeatureToggleConfig; +import com.tencent.bk.job.common.service.boot.JobBootApplication; import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration; -import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.boot.autoconfigure.availability.ApplicationAvailabilityAutoConfiguration; +import org.springframework.boot.autoconfigure.jooq.JooqAutoConfiguration; import org.springframework.cache.annotation.EnableCaching; import org.springframework.cloud.openfeign.EnableFeignClients; -import org.springframework.context.annotation.DependsOn; import org.springframework.scheduling.annotation.EnableScheduling; -@SpringBootApplication(scanBasePackages = "com.tencent.bk.job", exclude = {RedisAutoConfiguration.class}) +@JobBootApplication( + scanBasePackages = "com.tencent.bk.job.analysis", + exclude = {JooqAutoConfiguration.class, ApplicationAvailabilityAutoConfiguration.class}, + excludeName = {"org.springframework.cloud.kubernetes.client.discovery.KubernetesDiscoveryClientAutoConfiguration"}) @EnableCaching -@EnableFeignClients +@EnableFeignClients( + basePackages = { + "com.tencent.bk.job.manage.api", + "com.tencent.bk.job.execute.api", + "com.tencent.bk.job.logsvr.api", + "com.tencent.bk.job.crontab.api" + } +) @EnableScheduling -@EnableConfigurationProperties({FeatureToggleConfig.class}) -@DependsOn("applicationContextRegister") public class JobAnalysisBootApplication { public static void main(String[] args) { diff --git a/src/backend/job-analysis/boot-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/DbConfig.java b/src/backend/job-analysis/boot-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/DbConfig.java deleted file mode 100644 index 6a925901b0..0000000000 --- a/src/backend/job-analysis/boot-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/DbConfig.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.analysis.config; - -import org.jooq.ConnectionProvider; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; -import org.jooq.impl.DataSourceConnectionProvider; -import org.jooq.impl.DefaultConfiguration; -import org.jooq.impl.DefaultDSLContext; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.boot.jdbc.DataSourceBuilder; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.DependsOn; -import org.springframework.context.annotation.Primary; -import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.jdbc.datasource.DataSourceTransactionManager; -import org.springframework.transaction.annotation.EnableTransactionManagement; - -import javax.sql.DataSource; - -@Configuration -@EnableTransactionManagement -public class DbConfig { - @Qualifier("job-analysis-data-source") - @Primary - @Bean(name = "job-analysis-data-source") - @ConfigurationProperties(prefix = "spring.datasource.job-analysis") - public DataSource dataSource() { - return DataSourceBuilder.create().build(); - } - - @Qualifier("transactionManager") - @Bean(name = "transactionManager") - @DependsOn("job-analysis-data-source") - @Primary - public DataSourceTransactionManager transactionManager( - @Qualifier("job-analysis-data-source") DataSource dataSource) { - return new DataSourceTransactionManager(dataSource); - } - - @Qualifier("job-analysis-jdbc-template") - @Bean(name = "job-analysis-jdbc-template") - public JdbcTemplate jdbcTemplate( - @Qualifier("job-analysis-data-source") DataSource dataSource) { - return new JdbcTemplate(dataSource); - } - - @Qualifier("job-analysis-dsl-context") - @Bean(name = "job-analysis-dsl-context") - public DSLContext dslContext(@Qualifier("job-analysis-jooq-conf") org.jooq.Configuration configuration) { - return new DefaultDSLContext(configuration); - } - - @Qualifier("job-analysis-jooq-conf") - @Bean(name = "job-analysis-jooq-conf") - public org.jooq.Configuration - jooqConf(@Qualifier("job-analysis-conn-provider") ConnectionProvider connectionProvider) { - return new DefaultConfiguration().derive(connectionProvider).derive(SQLDialect.MYSQL); - } - - @Qualifier("job-analysis-conn-provider") - @Bean(name = "job-analysis-conn-provider") - public ConnectionProvider connectionProvider(@Qualifier("job-analysis-data-source") DataSource dataSource) { - return new DataSourceConnectionProvider(dataSource); - } - -} diff --git a/src/backend/job-analysis/boot-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/FilterConfig.java b/src/backend/job-analysis/boot-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/FilterConfig.java deleted file mode 100644 index 9b4c38af10..0000000000 --- a/src/backend/job-analysis/boot-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/FilterConfig.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.analysis.config; - -import com.tencent.bk.job.common.web.filter.RepeatableReadWriteServletRequestResponseFilter; -import org.springframework.boot.web.servlet.FilterRegistrationBean; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Configuration -public class FilterConfig { - @Bean - public FilterRegistrationBean repeatableRSRRFilterRegister() { - FilterRegistrationBean registration = - new FilterRegistrationBean<>(); - registration.setFilter(repeatableRRRFilter()); - registration.addUrlPatterns("/esb/api/*"); - registration.setName("repeatableReadRequestResponseFilter"); - registration.setOrder(0); - return registration; - } - - @Bean(name = "repeatableReadRequestResponseFilter") - public RepeatableReadWriteServletRequestResponseFilter repeatableRRRFilter() { - return new RepeatableReadWriteServletRequestResponseFilter(); - } -} diff --git a/src/backend/job-analysis/boot-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/InterceptorConfiguration.java b/src/backend/job-analysis/boot-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/InterceptorConfiguration.java deleted file mode 100644 index 0e7abd692d..0000000000 --- a/src/backend/job-analysis/boot-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/InterceptorConfiguration.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.analysis.config; - -import com.tencent.bk.job.analysis.interceptor.UriPermissionInterceptor; -import com.tencent.bk.job.common.iam.interceptor.AuthAppInterceptor; -import com.tencent.bk.job.common.web.interceptor.JobCommonInterceptor; -import com.tencent.bk.job.common.web.interceptor.ServiceSecurityInterceptor; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Configuration; -import org.springframework.web.servlet.config.annotation.InterceptorRegistry; -import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; - -@Configuration -public class InterceptorConfiguration implements WebMvcConfigurer { - - private final JobCommonInterceptor jobCommonInterceptor; - private final AuthAppInterceptor authAppInterceptor; - private final ServiceSecurityInterceptor serviceSecurityInterceptor; - private final UriPermissionInterceptor uriPermissionInterceptor; - - @Autowired - public InterceptorConfiguration( - JobCommonInterceptor jobCommonInterceptor, - AuthAppInterceptor authAppInterceptor, - ServiceSecurityInterceptor serviceSecurityInterceptor, - UriPermissionInterceptor uriPermissionInterceptor - ) { - this.jobCommonInterceptor = jobCommonInterceptor; - this.authAppInterceptor = authAppInterceptor; - this.serviceSecurityInterceptor = serviceSecurityInterceptor; - this.uriPermissionInterceptor = uriPermissionInterceptor; - } - - @Override - public void addInterceptors(InterceptorRegistry registry) { - // 注册拦截器 - registry.addInterceptor(serviceSecurityInterceptor).addPathPatterns("/**").order(0); - registry.addInterceptor(jobCommonInterceptor).addPathPatterns("/**").order(10); - registry.addInterceptor(uriPermissionInterceptor) - .addPathPatterns(uriPermissionInterceptor.getControlUriPatterns()).order(30); - registry.addInterceptor(authAppInterceptor).addPathPatterns("/web/**", "/esb/api/**").order(40); - } -} diff --git a/src/backend/job-analysis/boot-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/JobAnalysisConfiguration.java b/src/backend/job-analysis/boot-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/JobAnalysisConfiguration.java new file mode 100644 index 0000000000..69236cf25a --- /dev/null +++ b/src/backend/job-analysis/boot-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/JobAnalysisConfiguration.java @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.config; + +import com.tencent.bk.job.common.service.AppScopeMappingService; +import com.tencent.bk.job.common.web.interceptor.AppResourceScopeInterceptor; +import com.tencent.bk.job.manage.AppScopeMappingServiceImpl; +import com.tencent.bk.job.manage.api.inner.ServiceApplicationResource; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class JobAnalysisConfiguration { + @Bean + AppScopeMappingService appScopeMappingService(ServiceApplicationResource applicationResource) { + return new AppScopeMappingServiceImpl(applicationResource); + } + + @Bean + public AppResourceScopeInterceptor appResourceScopeInterceptor(AppScopeMappingService appScopeMappingService) { + return new AppResourceScopeInterceptor(appScopeMappingService); + } +} diff --git a/src/backend/job-analysis/boot-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/RedisCacheConfig.java b/src/backend/job-analysis/boot-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/RedisCacheConfig.java deleted file mode 100644 index 06ef4e05c2..0000000000 --- a/src/backend/job-analysis/boot-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/RedisCacheConfig.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.analysis.config; - -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.PropertyAccessor; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.springframework.cache.CacheManager; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; -import org.springframework.data.redis.cache.RedisCacheConfiguration; -import org.springframework.data.redis.cache.RedisCacheManager; -import org.springframework.data.redis.cache.RedisCacheWriter; -import org.springframework.data.redis.connection.RedisConnectionFactory; -import org.springframework.data.redis.serializer.GenericJackson2JsonRedisSerializer; -import org.springframework.data.redis.serializer.RedisSerializationContext; -import org.springframework.data.redis.serializer.RedisSerializer; - -import java.time.Duration; -import java.util.HashMap; -import java.util.Map; - -@Configuration -public class RedisCacheConfig { - - RedisCacheConfig() { - } - - /** - * 使用GenericJackson2JsonRedisSerializer序列化器 - */ - private RedisSerializer valueSerializer() { - ObjectMapper objectMapper = new ObjectMapper(); - objectMapper.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.ANY); - objectMapper.enableDefaultTyping(ObjectMapper.DefaultTyping.NON_FINAL); - return new GenericJackson2JsonRedisSerializer(objectMapper); - } - - private RedisCacheConfiguration getRedisCacheConfigurationWithTtl(Integer seconds) { - RedisCacheConfiguration redisCacheConfiguration = RedisCacheConfiguration.defaultCacheConfig(); - redisCacheConfiguration = redisCacheConfiguration.serializeValuesWith( - RedisSerializationContext - .SerializationPair - .fromSerializer(valueSerializer()) - ).entryTtl(Duration.ofSeconds(seconds)); - return redisCacheConfiguration; - } - - /** - * 缓存过期时间自定义配置 - */ - @Bean - @Primary - public CacheManager cacheManager(RedisConnectionFactory redisConnectionFactory) { - // 初始化一个RedisCacheWriter - RedisCacheWriter redisCacheWriter = RedisCacheWriter.nonLockingRedisCacheWriter(redisConnectionFactory); - //默认5秒过期 - RedisCacheConfiguration defaultCacheConfig = - RedisCacheConfiguration.defaultCacheConfig() - .serializeValuesWith(RedisSerializationContext.SerializationPair.fromSerializer(valueSerializer())) - .entryTtl(Duration.ofSeconds(5)); - // 每一类信息进行缓存配置 - Map redisCacheConfigurationMap = new HashMap<>(); - //单个业务信息3秒过期 - redisCacheConfigurationMap.put("appInfoCache", this.getRedisCacheConfigurationWithTtl(3)); - //拓扑信息10秒过期 - redisCacheConfigurationMap.put("appTopologyCache", this.getRedisCacheConfigurationWithTtl(10)); - // 初始化RedisCacheManager - RedisCacheManager cacheManager = new RedisCacheManager(redisCacheWriter, defaultCacheConfig, - redisCacheConfigurationMap); - return cacheManager; - } -} diff --git a/src/backend/job-analysis/boot-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/ScheduleConfig.java b/src/backend/job-analysis/boot-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/ScheduleConfig.java deleted file mode 100644 index 0239af29dc..0000000000 --- a/src/backend/job-analysis/boot-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/ScheduleConfig.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.analysis.config; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.context.annotation.Configuration; -import org.springframework.scheduling.annotation.SchedulingConfigurer; -import org.springframework.scheduling.config.ScheduledTaskRegistrar; - -import java.util.concurrent.ScheduledThreadPoolExecutor; - -//设定一个长度5的定时任务线程池 -@Slf4j -@Configuration -public class ScheduleConfig implements SchedulingConfigurer { - @Override - public void configureTasks(ScheduledTaskRegistrar taskRegistrar) { - taskRegistrar.setScheduler(new ScheduledThreadPoolExecutor(5, (r, executor) -> log.error( - "ScheduledThreadPoolExecutor rejected a runnable"))); - } -} diff --git a/src/backend/job-analysis/boot-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/SwaggerConfig.java b/src/backend/job-analysis/boot-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/SwaggerConfig.java index 8965677bc8..3ffbd051bb 100644 --- a/src/backend/job-analysis/boot-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/SwaggerConfig.java +++ b/src/backend/job-analysis/boot-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/SwaggerConfig.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,35 +24,40 @@ package com.tencent.bk.job.analysis.config; +import com.tencent.bk.job.common.service.config.JobCommonConfig; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Profile; import springfox.documentation.builders.PathSelectors; import springfox.documentation.builders.RequestHandlerSelectors; +import springfox.documentation.oas.annotations.EnableOpenApi; import springfox.documentation.spi.DocumentationType; import springfox.documentation.spring.web.plugins.Docket; -import springfox.documentation.swagger2.annotations.EnableSwagger2; import java.util.Arrays; import java.util.HashSet; +/** + * Swagger 配置 + */ @Configuration -@EnableSwagger2 -@Profile({"dev", "test", "local"}) +@EnableOpenApi +@Profile({"dev", "local"}) public class SwaggerConfig { - private final BkConfig bkConfig; + private final JobCommonConfig jobCommonConfig; @Autowired - public SwaggerConfig(BkConfig bkConfig) { - this.bkConfig = bkConfig; + public SwaggerConfig(JobCommonConfig jobCommonConfig) { + this.jobCommonConfig = jobCommonConfig; } @Bean public Docket api() { - return new Docket(DocumentationType.SWAGGER_2) - .host(bkConfig.getSwaggerUrl() + "/job-analysis") + return new Docket(DocumentationType.OAS_30) + .host(jobCommonConfig.getSwaggerUrl()) + .pathMapping("job-analysis") .protocols(new HashSet<>(Arrays.asList("http", "https"))) .select() .apis(RequestHandlerSelectors.basePackage("com.tencent.bk.job.analysis.api")) diff --git a/src/backend/job-analysis/boot-job-analysis/src/main/resources/application.yml b/src/backend/job-analysis/boot-job-analysis/src/main/resources/application.yml index c11c61c74c..1c960c0650 100644 --- a/src/backend/job-analysis/boot-job-analysis/src/main/resources/application.yml +++ b/src/backend/job-analysis/boot-job-analysis/src/main/resources/application.yml @@ -1,20 +1,22 @@ spring: profiles: active: prod - -ribbon: - ServerListRefreshInterval: 3000 - + mvc: + pathmatch: + matching-strategy: ant_path_matcher management: health: livenessstate: enabled: true readinessstate: enabled: true + # 使用spring cloud stream rabbitmq, 禁用掉springboot的rabbitmq health检查 + rabbit: + enabled: false endpoints: web: exposure: - include: health,configprops,env,beans,conditions,loggers,metrics,mappings,prometheus,scheduledtasks,refresh + include: health,configprops,env,beans,conditions,loggers,metrics,mappings,prometheus,scheduledtasks,refresh,busrefresh,bindings base-path: /actuator enabled-by-default: false endpoint: @@ -43,6 +45,8 @@ management: enabled: true refresh: enabled: true + busrefresh: + enabled: true server: port: 19807 shutdown: graceful diff --git a/src/backend/job-analysis/boot-job-analysis/src/main/resources/bootstrap.yml b/src/backend/job-analysis/boot-job-analysis/src/main/resources/bootstrap.yml index 687925871c..a525b829ee 100644 --- a/src/backend/job-analysis/boot-job-analysis/src/main/resources/bootstrap.yml +++ b/src/backend/job-analysis/boot-job-analysis/src/main/resources/bootstrap.yml @@ -11,3 +11,5 @@ spring: name: ${JOB_APPLICATION_CONFIGMAP_NAME} - namespace: ${KUBERNETES_NAMESPACE} name: ${JOB_COMMON_CONFIGMAP_NAME} + - namespace: ${KUBERNETES_NAMESPACE} + name: ${JOB_COMMON_REFRESHABLE_CONFIGMAP_NAME} diff --git a/src/backend/job-analysis/boot-job-analysis/src/main/resources/i18n/message.properties b/src/backend/job-analysis/boot-job-analysis/src/main/resources/i18n/message.properties deleted file mode 100644 index 47cc1c7ca0..0000000000 --- a/src/backend/job-analysis/boot-job-analysis/src/main/resources/i18n/message.properties +++ /dev/null @@ -1,32 +0,0 @@ -# -# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. -# -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. -# -# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. -# -# License for BK-JOB蓝鲸智云作业平台: -# -------------------------------------------------------------------- -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF -# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - -# 离线分析任务部分 -job.analysis.analysistask.result.ItemLocation.description.CommaSeparatedTplIdAndPlanId=逗号分隔的作业模板Id与执行方案Id -job.analysis.analysistask.result.ItemLocation.description.TemplateId=作业模板Id -job.analysis.analysistask.result.ItemLocation.description.TimerTaskName=定时任务名称 -job.analysis.analysistask.result.BadTplPlanInfo.typeName.TaskPlan=作业执行方案 -job.analysis.analysistask.result.BadTplPlanInfo.typeName.Template=作业模板 -job.analysis.analysistask.result.AbnormalTargetPlanInfo.stepName.scope.Step=步骤 -job.analysis.analysistask.result.AbnormalTargetPlanInfo.description.AbnormalTarget=执行目标异常 diff --git a/src/backend/job-analysis/boot-job-analysis/src/main/resources/i18n/message_en.properties b/src/backend/job-analysis/boot-job-analysis/src/main/resources/i18n/message_en.properties deleted file mode 100644 index 86bdee6fa3..0000000000 --- a/src/backend/job-analysis/boot-job-analysis/src/main/resources/i18n/message_en.properties +++ /dev/null @@ -1,32 +0,0 @@ -# -# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. -# -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. -# -# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. -# -# License for BK-JOB蓝鲸智云作业平台: -# -------------------------------------------------------------------- -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF -# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - -# 离线分析任务部分 -job.analysis.analysistask.result.ItemLocation.description.CommaSeparatedTplIdAndPlanId=Job template ID and Plan ID separated by comma -job.analysis.analysistask.result.ItemLocation.description.TemplateId=TemplateId -job.analysis.analysistask.result.ItemLocation.description.TimerTaskName=Cron Job Name -job.analysis.analysistask.result.BadTplPlanInfo.typeName.TaskPlan=TaskPlan -job.analysis.analysistask.result.BadTplPlanInfo.typeName.Template=Template -job.analysis.analysistask.result.AbnormalTargetPlanInfo.stepName.scope.Step=Steps -job.analysis.analysistask.result.AbnormalTargetPlanInfo.description.AbnormalTarget=Abnormal hosts diff --git a/src/backend/job-analysis/boot-job-analysis/src/main/resources/i18n/message_en_US.properties b/src/backend/job-analysis/boot-job-analysis/src/main/resources/i18n/message_en_US.properties deleted file mode 100644 index 86bdee6fa3..0000000000 --- a/src/backend/job-analysis/boot-job-analysis/src/main/resources/i18n/message_en_US.properties +++ /dev/null @@ -1,32 +0,0 @@ -# -# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. -# -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. -# -# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. -# -# License for BK-JOB蓝鲸智云作业平台: -# -------------------------------------------------------------------- -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF -# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - -# 离线分析任务部分 -job.analysis.analysistask.result.ItemLocation.description.CommaSeparatedTplIdAndPlanId=Job template ID and Plan ID separated by comma -job.analysis.analysistask.result.ItemLocation.description.TemplateId=TemplateId -job.analysis.analysistask.result.ItemLocation.description.TimerTaskName=Cron Job Name -job.analysis.analysistask.result.BadTplPlanInfo.typeName.TaskPlan=TaskPlan -job.analysis.analysistask.result.BadTplPlanInfo.typeName.Template=Template -job.analysis.analysistask.result.AbnormalTargetPlanInfo.stepName.scope.Step=Steps -job.analysis.analysistask.result.AbnormalTargetPlanInfo.description.AbnormalTarget=Abnormal hosts diff --git a/src/backend/job-analysis/boot-job-analysis/src/main/resources/i18n/message_zh.properties b/src/backend/job-analysis/boot-job-analysis/src/main/resources/i18n/message_zh.properties deleted file mode 100644 index 47cc1c7ca0..0000000000 --- a/src/backend/job-analysis/boot-job-analysis/src/main/resources/i18n/message_zh.properties +++ /dev/null @@ -1,32 +0,0 @@ -# -# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. -# -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. -# -# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. -# -# License for BK-JOB蓝鲸智云作业平台: -# -------------------------------------------------------------------- -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF -# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - -# 离线分析任务部分 -job.analysis.analysistask.result.ItemLocation.description.CommaSeparatedTplIdAndPlanId=逗号分隔的作业模板Id与执行方案Id -job.analysis.analysistask.result.ItemLocation.description.TemplateId=作业模板Id -job.analysis.analysistask.result.ItemLocation.description.TimerTaskName=定时任务名称 -job.analysis.analysistask.result.BadTplPlanInfo.typeName.TaskPlan=作业执行方案 -job.analysis.analysistask.result.BadTplPlanInfo.typeName.Template=作业模板 -job.analysis.analysistask.result.AbnormalTargetPlanInfo.stepName.scope.Step=步骤 -job.analysis.analysistask.result.AbnormalTargetPlanInfo.description.AbnormalTarget=执行目标异常 diff --git a/src/backend/job-analysis/boot-job-analysis/src/main/resources/i18n/message_zh_CN.properties b/src/backend/job-analysis/boot-job-analysis/src/main/resources/i18n/message_zh_CN.properties deleted file mode 100644 index 47cc1c7ca0..0000000000 --- a/src/backend/job-analysis/boot-job-analysis/src/main/resources/i18n/message_zh_CN.properties +++ /dev/null @@ -1,32 +0,0 @@ -# -# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. -# -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. -# -# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. -# -# License for BK-JOB蓝鲸智云作业平台: -# -------------------------------------------------------------------- -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF -# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - -# 离线分析任务部分 -job.analysis.analysistask.result.ItemLocation.description.CommaSeparatedTplIdAndPlanId=逗号分隔的作业模板Id与执行方案Id -job.analysis.analysistask.result.ItemLocation.description.TemplateId=作业模板Id -job.analysis.analysistask.result.ItemLocation.description.TimerTaskName=定时任务名称 -job.analysis.analysistask.result.BadTplPlanInfo.typeName.TaskPlan=作业执行方案 -job.analysis.analysistask.result.BadTplPlanInfo.typeName.Template=作业模板 -job.analysis.analysistask.result.AbnormalTargetPlanInfo.stepName.scope.Step=步骤 -job.analysis.analysistask.result.AbnormalTargetPlanInfo.description.AbnormalTarget=执行目标异常 diff --git a/src/backend/job-analysis/boot-job-analysis/src/main/resources/logback-spring.xml b/src/backend/job-analysis/boot-job-analysis/src/main/resources/logback-spring.xml new file mode 100644 index 0000000000..b2ccff7236 --- /dev/null +++ b/src/backend/job-analysis/boot-job-analysis/src/main/resources/logback-spring.xml @@ -0,0 +1,13 @@ + + + logback + + + + + + + + + + diff --git a/src/backend/job-analysis/boot-job-analysis/src/main/resources/logback/logback-app-props.xml b/src/backend/job-analysis/boot-job-analysis/src/main/resources/logback/logback-app-props.xml deleted file mode 100644 index 2ec806f590..0000000000 --- a/src/backend/job-analysis/boot-job-analysis/src/main/resources/logback/logback-app-props.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - diff --git a/src/backend/job-analysis/boot-job-analysis/src/test/java/com/tencent/bk/job/analysis/package-info.java b/src/backend/job-analysis/boot-job-analysis/src/test/java/com/tencent/bk/job/analysis/package-info.java index 19a644cea4..f4d7822c4f 100644 --- a/src/backend/job-analysis/boot-job-analysis/src/test/java/com/tencent/bk/job/analysis/package-info.java +++ b/src/backend/job-analysis/boot-job-analysis/src/test/java/com/tencent/bk/job/analysis/package-info.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/boot-job-analysis/src/test/resources/application-test.yml b/src/backend/job-analysis/boot-job-analysis/src/test/resources/application-test.yml index d73dd10503..be70020eb5 100644 --- a/src/backend/job-analysis/boot-job-analysis/src/test/resources/application-test.yml +++ b/src/backend/job-analysis/boot-job-analysis/src/test/resources/application-test.yml @@ -1,4 +1,7 @@ spring: + mvc: + pathmatch: + matching-strategy: ant_path_matcher datasource: job-analysis: driver-class-name: org.h2.Driver @@ -12,8 +15,22 @@ spring: idle-timeout: 600000 poolName: "job-analysis" validationTimeout: 5000 + cloud: + config: + enabled: false + kubernetes: + discovery: + enabled: false job: security: service: public-key-base64: LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FROEFNSUlCQ2dLQ0FRRUFyTXN3emh3QkVUTktQOThTcElna0VjZThPNVlQWm5aUDVHVW1ZUVhhSHdHWmg5UHJwNDFseS9xbWk5enEwTlRCODRyaXhSZHZyU1c0cCtCUjk4TVRzRmpYaUlRYVBxRTdRUitpdDdoT1VQNVJMQW9KcWlvL3NlZlRIdlZxVW1GcitYU3NST05rek5selRVaVcvMFRNY0cwNVdIS3hORno5YTB2aXo1T01PeE5lUzhqS1ZiRStIMHlQS0JhR1JHMEdZNTZTS1BLbXh1RHk5a2tyVUtGSXZycldSZkh2bzVsK0xsT3IyQmVGT3FHcEhmeW1FaXlkbFRxWHNlbGlsYTRxMUoxbnBZRDYwY3JIY1ZRV3d2Ri9CZUkrZytxVkF5YzNaWmhYeEhoNWNMcmZsK3pqMktsUEFEZVIvZktubHFqKzFBQm4yaFRoVWJ2WjRCdXhTdWxOa1FJREFRQUIKLS0tLS1FTkQgUFVCTElDIEtFWS0tLS0t private-key-base64: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV1d0lCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktVd2dnU2hBZ0VBQW9JQkFRQ3N5ekRPSEFFUk0wby8zeEtraUNRUng3dzdsZzltZGsva1pTWmhCZG9mQVptSDArdW5qV1hMK3FhTDNPclExTUh6aXVMRkYyK3RKYmluNEZIM3d4T3dXTmVJaEJvK29UdEJINkszdUU1US9sRXNDZ21xS2oreDU5TWU5V3BTWVd2NWRLeEU0MlRNMlhOTlNKYi9STXh3YlRsWWNyRTBYUDFyUytMUGs0dzdFMTVMeU1wVnNUNGZUSThvRm9aRWJRWmpucElvOHFiRzRQTDJTU3RRb1VpK3V0WkY4ZStqbVg0dVU2dllGNFU2b2FrZC9LWVNMSjJWT3BleDZXS1ZyaXJVbldlbGdQclJ5c2R4VkJiQzhYOEY0ajZENnBVREp6ZGxtRmZFZUhsd3V0K1g3T1BZcVU4QU41SDk4cWVXcVA3VUFHZmFGT0ZSdTluZ0c3Rks2VTJSQWdNQkFBRUNnZ0VBZlkvZGREaUhVZjlxV2dWNG41VEtXUjFDSEM2TUhGR205bm5HTE1TNjNzTWoxTDdiUkZNTWdQQXY2L3hwcWJ5Zy81K1I3OVI2ODJBdzBkVEEyNENuSXdNOEE5aXkxWVlGNytuVUxSckIxanNuc3cvTVBCb2RWT0wzMkV4UHFYTmZxZEx0aDlXUm53WDRzbFVvWEhDYStxdHorMkRFZ2g5OGkyYUdkMzVwYlZjT2p0czlWc3FtZE5uQTdLNFVQWkd4aVFheG1rLzBOVEVNY2RqMUl4bWNlZjNqL1RUVkFiR05IdWVtYTJKeUdZdk52N09BZUN3VGJSdW5TNW1PSmptRHlFSmp4dndaVWZwanZoM0w0SkZzMGRJUm5wZ2VtTkY3NytqUmFTTy9WUVlqc1FLelB0SnEwcHFneHUwd0RoQUVBeXpjYWI2NW8wSm1hZzNJUjZNQzdRS0JnUUR1MThHNFRVaTlSVk8yZS9YdWtDSUNiOVZISHlMeVFIUDh4NDhBVFZmVENtSVBEMUduTWp5TlFmSnhsUlNma3UyUjJIM1IyRC81YUNpNUYwTi9USG51OWVqYlMvS1NoYjFxMFl5TWF1TDkxQlZjMkEyeDZlRzRPNFFpT2daT0gzaE0raUVSREJxOExNQ1dILzVtYVJDeDhVRmRQTnpHMUtXVVNwTEJXM3l2ZXdLQmdRQzVOTS9yN09Wc3F6OGlDUjBXa2JramN2eWU4MkJNcGFsZVkyRHNWQVMwRHNMNlMvZlAzY2VUdEJQVmdJL244Sk16b1ZNZE5OK3htbXM4Wi9rMGM5OGlLTzRUeE0zN2daYkIwUE8rWGRyeUZkRFhTRDZXTm1heWpYUG5MUndBZmRwbFlWdFNqVGxWMWJzWWIySnhXNXY2RUVSaVlLYnNnRGNpWTA5ZnYva0RZd0ovUnVmR212QldwOW50QUQvTXd3WUhFcklnbnZ5ZlkwdS9JMHdiSi92T0Z0aitRM3BJdzFvbW44ajBNTVFSVzA1RE9Ra01VVC9odmlrdDdjVDZSTkJ2WW9HZW4zdnNoNU1zcUltTk1DS0xRTSsxaDlxY05qVTR6WGpkd1V0NGs3akQvaFpEdXN2ZEpBQWxMR3hUR2hRVzRMeStxdTltbTZDRmg0Q2NFTXNZT1FLQmdCa0pEWTRyd0x1V3pucndWbkZGazR3NkwvSGdua1RlSSszeGF4bW5KZGVkSXpnb2FzQTdqNDNreXNXU2ttRmRBTmo2MXJ4YytGd09ycTVFczhnbVhhUkZlaUZMK2pna1JWZS93aU1hbHMwZ3E5RmVINis0Zk50b0NhSFhWREN3MG9QZnpETjFpYW1Uc2EzVTZLN0FIbTZJSW1aV0FGcDUzbm9VaUpjOTRSQlhBb0dCQUxMcmQrbnljNzVEbjZ0ZWdVT0JQRy9HR3FYWkZGdmd6emJRRnR0SmR0NTlMZ0NBanUxMjJMRExsRjZreTB6Q0JsK0h6Q0kyUXJXVEp4cXUxSTZvbndaZDlOMllKVlNWQlFOV285NTJUamRkOHcvSkExYlNqK3JpTU1WQ0txT1BNRHNiOENEaGlEbVM3RlBEM0trTVlmcUxJOVhNRTVvNVUvNFJzcEZUZWozRQotLS0tLUVORCBQUklWQVRFIEtFWS0tLS0t +bk-api-gateway: + gse: + url: gse.apigw.com + cmdb: + url: cmdb.apigw.com +esb: + service: + url: esb.service diff --git a/src/backend/job-analysis/boot-job-analysis/src/test/resources/bootstrap.yml b/src/backend/job-analysis/boot-job-analysis/src/test/resources/bootstrap.yml index d58fcb38ae..ded6c7f048 100644 --- a/src/backend/job-analysis/boot-job-analysis/src/test/resources/bootstrap.yml +++ b/src/backend/job-analysis/boot-job-analysis/src/test/resources/bootstrap.yml @@ -2,4 +2,7 @@ spring: application: name: job-analysis profiles: - active: test \ No newline at end of file + active: test + cloud: + kubernetes: + enabled: false diff --git a/src/backend/job-analysis/boot-job-analysis/src/test/resources/init_analysis_task_data.sql b/src/backend/job-analysis/boot-job-analysis/src/test/resources/init_analysis_task_data.sql index 6b5893959a..02152da498 100644 --- a/src/backend/job-analysis/boot-job-analysis/src/test/resources/init_analysis_task_data.sql +++ b/src/backend/job-analysis/boot-job-analysis/src/test/resources/init_analysis_task_data.sql @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/boot-job-analysis/src/test/resources/init_analysis_task_instance_data.sql b/src/backend/job-analysis/boot-job-analysis/src/test/resources/init_analysis_task_instance_data.sql index cd7fc06d2b..1f1974f1e4 100644 --- a/src/backend/job-analysis/boot-job-analysis/src/test/resources/init_analysis_task_instance_data.sql +++ b/src/backend/job-analysis/boot-job-analysis/src/test/resources/init_analysis_task_instance_data.sql @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/boot-job-analysis/src/test/resources/init_schema.sql b/src/backend/job-analysis/boot-job-analysis/src/test/resources/init_schema.sql index 414500968d..041e0bba9a 100644 --- a/src/backend/job-analysis/boot-job-analysis/src/test/resources/init_schema.sql +++ b/src/backend/job-analysis/boot-job-analysis/src/test/resources/init_schema.sql @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/boot-job-analysis/src/test/resources/test.properties b/src/backend/job-analysis/boot-job-analysis/src/test/resources/test.properties index 58493ca4e6..a0f8d5250f 100644 --- a/src/backend/job-analysis/boot-job-analysis/src/test/resources/test.properties +++ b/src/backend/job-analysis/boot-job-analysis/src/test/resources/test.properties @@ -1,7 +1,7 @@ # # Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. # -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +# Copyright (C) 2021 Tencent. All rights reserved. # # BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. # diff --git a/src/backend/job-analysis/build.gradle b/src/backend/job-analysis/build.gradle index 31694869b9..adfca7d121 100644 --- a/src/backend/job-analysis/build.gradle +++ b/src/backend/job-analysis/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/model-job-analysis/build.gradle b/src/backend/job-analysis/model-job-analysis/build.gradle index 53cd347e49..8006176375 100644 --- a/src/backend/job-analysis/model-job-analysis/build.gradle +++ b/src/backend/job-analysis/model-job-analysis/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/service-job-analysis/build.gradle b/src/backend/job-analysis/service-job-analysis/build.gradle index 04c4b14bbc..0ad3a59a29 100644 --- a/src/backend/job-analysis/service-job-analysis/build.gradle +++ b/src/backend/job-analysis/service-job-analysis/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,15 +28,18 @@ dependencies { api project(":job-analysis:api-job-analysis") api project(":job-analysis:model-job-analysis") api project(":job-crontab:api-job-crontab") + api project(":job-logsvr:api-job-logsvr") api project(":commons:common") api project(":commons:common-web") api project(":commons:common-iam") api project(":commons:common-security") api project(":commons:common-redis") + api project(":commons:common-mysql") api project(":commons:cmdb-sdk") api project(":commons:paas-sdk") api project(":commons:esb-sdk") api project(":commons:gse-sdk") + api project(":commons:ai-dev-sdk") implementation "org.springframework.boot:spring-boot-starter-web" implementation "org.springframework.boot:spring-boot-starter-jdbc" implementation "org.springframework.boot:spring-boot-starter-jooq" diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/iam/impl/IamDashBoardViewCallbackResourceImpl.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/iam/impl/IamDashBoardViewCallbackResourceImpl.java index c9e296df89..0416e03a2d 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/iam/impl/IamDashBoardViewCallbackResourceImpl.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/iam/impl/IamDashBoardViewCallbackResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -34,6 +34,7 @@ import com.tencent.bk.sdk.iam.dto.callback.response.BaseDataResponseDTO; import com.tencent.bk.sdk.iam.dto.callback.response.CallbackBaseResponseDTO; import com.tencent.bk.sdk.iam.dto.callback.response.FetchInstanceInfoResponseDTO; +import com.tencent.bk.sdk.iam.dto.callback.response.FetchResourceTypeSchemaResponseDTO; import com.tencent.bk.sdk.iam.dto.callback.response.InstanceInfoDTO; import com.tencent.bk.sdk.iam.dto.callback.response.ListInstanceResponseDTO; import com.tencent.bk.sdk.iam.dto.callback.response.SearchInstanceResponseDTO; @@ -141,4 +142,11 @@ protected CallbackBaseResponseDTO fetchInstanceResp( public CallbackBaseResponseDTO callback(CallbackRequestDTO callbackRequest) { return baseCallback(callbackRequest); } + + @Override + protected FetchResourceTypeSchemaResponseDTO fetchResourceTypeSchemaResp( + CallbackRequestDTO callbackRequest) { + // 无需实现 + return null; + } } diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/op/impl/OpResourceImpl.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/op/impl/OpResourceImpl.java index 19ada02e16..fea42a1c85 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/op/impl/OpResourceImpl.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/op/impl/OpResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -41,7 +41,7 @@ import java.util.List; -@RestController +@RestController("jobAnalysisOpResourceImpl") public class OpResourceImpl implements OpResource { private final StatisticsTaskScheduler statisticsTaskScheduler; diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/web/impl/WebAIResourceImpl.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/web/impl/WebAIResourceImpl.java new file mode 100644 index 0000000000..1129a6876a --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/web/impl/WebAIResourceImpl.java @@ -0,0 +1,319 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.api.web.impl; + +import com.tencent.bk.job.analysis.api.web.WebAIResource; +import com.tencent.bk.job.analysis.config.AIProperties; +import com.tencent.bk.job.analysis.consts.AIChatStatusEnum; +import com.tencent.bk.job.analysis.consts.AIConsts; +import com.tencent.bk.job.analysis.model.dto.AIChatHistoryDTO; +import com.tencent.bk.job.analysis.model.web.req.AIAnalyzeErrorReq; +import com.tencent.bk.job.analysis.model.web.req.AICheckScriptReq; +import com.tencent.bk.job.analysis.model.web.req.AIGeneralChatReq; +import com.tencent.bk.job.analysis.model.web.req.GenerateChatStreamReq; +import com.tencent.bk.job.analysis.model.web.req.TerminateChatReq; +import com.tencent.bk.job.analysis.model.web.resp.AIChatRecord; +import com.tencent.bk.job.analysis.model.web.resp.ClearChatHistoryResp; +import com.tencent.bk.job.analysis.service.ai.AIAnalyzeErrorService; +import com.tencent.bk.job.analysis.service.ai.AIChatHistoryService; +import com.tencent.bk.job.analysis.service.ai.AICheckScriptService; +import com.tencent.bk.job.analysis.service.ai.ChatService; +import com.tencent.bk.job.analysis.service.ai.impl.AIConfigService; +import com.tencent.bk.job.analysis.service.ai.impl.AIMessageI18nService; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.ServiceException; +import com.tencent.bk.job.common.model.Response; +import com.tencent.bk.job.common.model.dto.AppResourceScope; +import com.tencent.bk.job.common.model.error.ErrorType; +import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.RestController; +import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * 提供Web端使用的AI功能相关接口实现 + */ +@RestController("jobAnalysisWebAIResource") +@Slf4j +public class WebAIResourceImpl implements WebAIResource { + + private final AIConfigService aiConfigService; + private final ChatService chatService; + private final AICheckScriptService aiCheckScriptService; + private final AIAnalyzeErrorService aiAnalyzeErrorService; + private final AIChatHistoryService aiChatHistoryService; + private final AIMessageI18nService aiMessageI18nService; + private final AIProperties aiProperties; + + @Autowired + public WebAIResourceImpl(AIConfigService aiConfigService, + ChatService chatService, + AICheckScriptService aiCheckScriptService, + AIAnalyzeErrorService aiAnalyzeErrorService, + AIChatHistoryService aiChatHistoryService, + AIMessageI18nService aiMessageI18nService, + AIProperties aiProperties) { + this.aiConfigService = aiConfigService; + this.chatService = chatService; + this.aiCheckScriptService = aiCheckScriptService; + this.aiAnalyzeErrorService = aiAnalyzeErrorService; + this.aiChatHistoryService = aiChatHistoryService; + this.aiMessageI18nService = aiMessageI18nService; + this.aiProperties = aiProperties; + } + + /** + * 获取AI相关配置信息 + * + * @param username 用户名 + * @param appResourceScope 资源范围（业务/业务集） + * @param scopeType 资源范围类型 + * @param scopeId 资源范围ID + * @return AI相关配置信息 + */ + @Override + public Response> getAIConfigOfScope(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId) { + return Response.buildSuccessResp(aiConfigService.getAIConfig()); + } + + /** + * 获取AI相关配置信息 + * + * @param username 用户名 + * @return AI相关配置信息 + */ + @Override + public Response> getAIConfig(String username) { + return Response.buildSuccessResp(aiConfigService.getAIConfig()); + } + + /** + * 获取最新对话记录列表 + * + * @param username 用户名 + * @param appResourceScope 资源范围（业务/业务集） + * @param scopeType 资源范围类型 + * @param scopeId 资源范围ID + * @param start 起始位置 + * @param length 长度 + * @return 最新对话记录列表 + */ + @Override + public Response> getLatestChatHistoryList(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + Integer start, + Integer length) { + if (!aiChatHistoryService.existsChatHistory(username)) { + AIChatHistoryDTO greetingChatHistory = getGreetingChatHistory(username, appResourceScope.getAppId()); + Long id = aiChatHistoryService.insertChatHistory(greetingChatHistory); + log.debug("Greeting chat history created, username={}, id={}", username, id); + } + List chatRecordList = chatService.getLatestChatHistoryList(username, start, length); + List aiChatRecordList = chatRecordList.stream() + .map(AIChatHistoryDTO::toAIChatRecord) + .collect(Collectors.toList()); + return Response.buildSuccessResp(aiChatRecordList); + } + + /** + * 获取开场白对话记录 + * + * @param username 用户名 + * @param appId Job业务ID + * @return 开场白对话记录 + */ + private AIChatHistoryDTO getGreetingChatHistory(String username, Long appId) { + AIChatHistoryDTO greetingChatHistory = new AIChatHistoryDTO(); + greetingChatHistory.setUsername(username); + greetingChatHistory.setAppId(appId); + greetingChatHistory.setUserInput(""); + greetingChatHistory.setStartTime(System.currentTimeMillis()); + greetingChatHistory.setPromptTemplateId(null); + greetingChatHistory.setAiInput(""); + greetingChatHistory.setStatus(AIChatStatusEnum.FINISHED.getStatus()); + greetingChatHistory.setAiAnswer(aiMessageI18nService.getAIGreetingMessage()); + greetingChatHistory.setAnswerTime(System.currentTimeMillis()); + greetingChatHistory.updateTotalTime(); + greetingChatHistory.setErrorCode(AIConsts.AI_ANSWER_ERROR_CODE_OK); + greetingChatHistory.setErrorMessage(null); + greetingChatHistory.setIsDeleted(false); + return greetingChatHistory; + } + + /** + * 与AI进行通用对话 + * + * @param username 用户名 + * @param appResourceScope 资源范围（业务/业务集） + * @param scopeType 资源范围类型 + * @param scopeId 资源范围ID + * @param req 请求体 + * @return AI对话记录响应 + */ + @Override + public Response generalChat(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + AIGeneralChatReq req) { + AIChatRecord aiChatRecord = chatService.chatWithAI(username, appResourceScope.getAppId(), req.getContent()); + return Response.buildSuccessResp(aiChatRecord); + } + + /** + * 使用AI检查脚本内容 + * + * @param username 用户名 + * @param appResourceScope 资源范围（业务/业务集） + * @param scopeType 资源范围类型 + * @param scopeId 资源范围ID + * @param req 请求体 + * @return AI对话记录响应 + */ + @Override + public Response checkScript(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + AICheckScriptReq req) { + AIChatRecord aiChatRecord = aiCheckScriptService.check( + username, + appResourceScope.getAppId(), + req.getType(), + req.getContent() + ); + return Response.buildSuccessResp(aiChatRecord); + } + + /** + * 使用AI分析任务报错信息 + * + * @param username 用户名 + * @param appResourceScope 资源范围（业务/业务集） + * @param scopeType 资源范围类型 + * @param scopeId 资源范围ID + * @param req 请求体 + * @return AI对话记录响应 + */ + @Override + public Response analyzeError(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + AIAnalyzeErrorReq req) { + checkScriptLogContentLength(req); + AIChatRecord aiChatRecord = aiAnalyzeErrorService.analyze(username, appResourceScope.getAppId(), req); + return Response.buildSuccessResp(aiChatRecord); + } + + /** + * 生成AI对话流式数据 + * + * @param username 用户名 + * @param appResourceScope 资源范围（业务/业务集） + * @param scopeType 资源范围类型 + * @param scopeId 资源范围ID + * @param req 请求体 + * @return AI对话流式数据 + */ + @Override + public ResponseEntity generateChatStream(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + GenerateChatStreamReq req) { + StreamingResponseBody streamingResponseBody = chatService.generateChatStream(username, req.getRecordId()); + return ResponseEntity.ok().contentType(MediaType.APPLICATION_OCTET_STREAM).body(streamingResponseBody); + } + + /** + * 终止对话 + * + * @param username 用户名 + * @param appResourceScope 资源范围（业务/业务集） + * @param scopeType 资源范围类型 + * @param scopeId 资源范围ID + * @param req 请求体 + * @return 是否终止成功 + */ + @Override + public Response terminateChat(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + TerminateChatReq req) { + boolean result = chatService.triggerTerminateChat(username, req.getRecordId()); + return Response.buildSuccessResp(result); + } + + /** + * 结合动态配置的限制值检查脚本日志内容长度是否超限 + * + * @param req 请求体 + */ + private void checkScriptLogContentLength(AIAnalyzeErrorReq req) { + if (StepExecuteTypeEnum.EXECUTE_SCRIPT.getValue().equals(req.getStepExecuteType())) { + Long logMaxLengthBytes = aiProperties.getAnalyzeErrorLog().getLogMaxLengthBytes(); + if (req.getContent().length() > logMaxLengthBytes) { + throw new ServiceException( + ErrorType.INVALID_PARAM, + ErrorCode.AI_ANALYZE_ERROR_CONTENT_EXCEED_MAX_LENGTH, + new Object[]{aiProperties.getAnalyzeErrorLog().getLogMaxLength()} + ); + } + } + } + + /** + * 清空对话记录 + * + * @param username 用户名 + * @param appResourceScope 资源范围（业务/业务集） + * @param scopeType 资源范围类型 + * @param scopeId 资源范围ID + * @return 清空对话记录响应数据（被清空数据量等） + */ + @Override + public Response clearChatHistory(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId) { + int deletedTotalCount = aiChatHistoryService.softDeleteChatHistory(username); + return Response.buildSuccessResp(new ClearChatHistoryResp(deletedTotalCount)); + } + +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/web/impl/WebIndexResourceImpl.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/web/impl/WebIndexResourceImpl.java index 1d31c5730c..341d1edfdb 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/web/impl/WebIndexResourceImpl.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/web/impl/WebIndexResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -35,7 +35,7 @@ import java.util.List; -@RestController +@RestController("jobAnalysisWebIndexResource") @Slf4j public class WebIndexResourceImpl implements WebIndexResource { diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/web/impl/WebScriptStatisticsResourceImpl.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/web/impl/WebScriptStatisticsResourceImpl.java index 61fbc9133c..d63ea37e8b 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/web/impl/WebScriptStatisticsResourceImpl.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/web/impl/WebScriptStatisticsResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,9 +24,13 @@ package com.tencent.bk.job.analysis.api.web.impl; +import com.tencent.bk.audit.annotations.ActionAuditRecord; +import com.tencent.bk.audit.annotations.AuditEntry; import com.tencent.bk.job.analysis.api.web.WebScriptStatisticsResource; import com.tencent.bk.job.analysis.model.web.ScriptCiteStatisticVO; import com.tencent.bk.job.analysis.service.ScriptStatisticService; +import com.tencent.bk.job.common.audit.constants.EventContentConstants; +import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.model.Response; import com.tencent.bk.job.common.model.dto.ResourceScope; import com.tencent.bk.job.common.service.AppScopeMappingService; @@ -55,6 +59,11 @@ public WebScriptStatisticsResourceImpl(ScriptStatisticService scriptStatisticSer } @Override + @AuditEntry(actionId = ActionId.DASHBOARD_VIEW) + @ActionAuditRecord( + actionId = ActionId.DASHBOARD_VIEW, + content = EventContentConstants.VIEW_ANALYSIS_DASHBOARD + ) public Response scriptCiteInfo(String username, List scopes, String date) { if (StringUtils.isBlank(date)) { date = DateUtils.getCurrentDateStr(); diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/web/impl/WebStatisticsResourceImpl.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/web/impl/WebStatisticsResourceImpl.java index 6afe388497..3e80cb4dae 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/web/impl/WebStatisticsResourceImpl.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/api/web/impl/WebStatisticsResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,9 @@ package com.tencent.bk.job.analysis.api.web.impl; +import com.tencent.bk.audit.annotations.ActionAuditRecord; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.job.analysis.api.consts.StatisticsConstants; import com.tencent.bk.job.analysis.api.web.WebStatisticsResource; import com.tencent.bk.job.analysis.config.StatisticConfig; import com.tencent.bk.job.analysis.consts.DimensionEnum; @@ -41,19 +44,22 @@ import com.tencent.bk.job.analysis.service.ExecutedTaskStatisticService; import com.tencent.bk.job.analysis.service.FastFileStatisticService; import com.tencent.bk.job.analysis.service.FastScriptStatisticService; +import com.tencent.bk.job.analysis.service.RollingTaskStatisticService; import com.tencent.bk.job.analysis.service.TagStatisticService; +import com.tencent.bk.job.common.audit.constants.EventContentConstants; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.exception.InvalidParamException; +import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.model.Response; import com.tencent.bk.job.common.model.dto.ResourceScope; import com.tencent.bk.job.common.service.AppScopeMappingService; -import com.tencent.bk.job.common.statistics.consts.StatisticsConstants; import com.tencent.bk.job.common.util.TimeUtil; import com.tencent.bk.job.common.util.date.DateUtils; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.jooq.tools.StringUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.web.bind.annotation.RestController; @@ -70,6 +76,7 @@ public class WebStatisticsResourceImpl implements WebStatisticsResource { private final AppStatisticService appStatisticService; private final ExecutedTaskStatisticService executedTaskStatisticService; + private final RollingTaskStatisticService rollingTaskStatisticService; private final FastScriptStatisticService fastScriptStatisticService; private final FastFileStatisticService fastFileStatisticService; private final TagStatisticService tagStatisticService; @@ -79,17 +86,19 @@ public class WebStatisticsResourceImpl implements WebStatisticsResource { private final AppScopeMappingService appScopeMappingService; @Autowired - public WebStatisticsResourceImpl(AppStatisticService appStatisticService, + public WebStatisticsResourceImpl(@Qualifier("appStatisticService") AppStatisticService appStatisticService, ExecutedTaskStatisticService executedTaskStatisticService, + RollingTaskStatisticService rollingTaskStatisticService, FastScriptStatisticService fastScriptStatisticService, FastFileStatisticService fastFileStatisticService, TagStatisticService tagStatisticService, - CommonStatisticService commonStatisticService, + @Qualifier("commonStatisticService") CommonStatisticService commonStatisticService, StatisticConfig statisticConfig, RedisTemplate redisTemplate, AppScopeMappingService appScopeMappingService) { this.appStatisticService = appStatisticService; this.executedTaskStatisticService = executedTaskStatisticService; + this.rollingTaskStatisticService = rollingTaskStatisticService; this.fastScriptStatisticService = fastScriptStatisticService; this.fastFileStatisticService = fastFileStatisticService; this.tagStatisticService = tagStatisticService; @@ -100,6 +109,11 @@ public WebStatisticsResourceImpl(AppStatisticService appStatisticService, } @Override + @AuditEntry(actionId = ActionId.DASHBOARD_VIEW) + @ActionAuditRecord( + actionId = ActionId.DASHBOARD_VIEW, + content = EventContentConstants.VIEW_ANALYSIS_DASHBOARD + ) public Response totalStatistics(String username, TotalMetricEnum metric, List scopes, @@ -120,6 +134,11 @@ public Response totalStatistics(String username, } @Override + @AuditEntry(actionId = ActionId.DASHBOARD_VIEW) + @ActionAuditRecord( + actionId = ActionId.DASHBOARD_VIEW, + content = EventContentConstants.VIEW_ANALYSIS_DASHBOARD + ) public Response> trends(String username, TotalMetricEnum metric, List scopes, @@ -173,6 +192,11 @@ public int compare(CommonTrendElementVO o1, CommonTrendElementVO o2) { } @Override + @AuditEntry(actionId = ActionId.DASHBOARD_VIEW) + @ActionAuditRecord( + actionId = ActionId.DASHBOARD_VIEW, + content = EventContentConstants.VIEW_ANALYSIS_DASHBOARD + ) public Response> listByPerApp(String username, TotalMetricEnum metric, List scopes, @@ -197,13 +221,18 @@ public Response> listByPerApp(String username, appIdList, date); } return Response.buildSuccessResp( - perAppStatisticDTOList.parallelStream() + perAppStatisticDTOList.stream() .map(PerAppStatisticDTO::toPerAppStatisticVO) .collect(Collectors.toList()) ); } @Override + @AuditEntry(actionId = ActionId.DASHBOARD_VIEW) + @ActionAuditRecord( + actionId = ActionId.DASHBOARD_VIEW, + content = EventContentConstants.VIEW_ANALYSIS_DASHBOARD + ) public Response distributionStatistics(String username, DistributionMetricEnum metric, List scopes, @@ -242,6 +271,11 @@ private List executedTaskByTimeConsumingDayDetail(Stri return executedTaskStatisticService.getByTimeConsumingDayDetail(appIdList, startDate, endDate); } + private List rollingTaskByTaskTypeDayDetail(String username, List appIdList, + String startDate, String endDate) { + return rollingTaskStatisticService.rollingTaskByTaskTypeDayDetail(appIdList, startDate, endDate); + } + private List fastScriptByScriptTypeDayDetail(String username, List appIdList, String startDate, @@ -257,6 +291,11 @@ private List fastFileByTransferModeDayDetail(String us } @Override + @AuditEntry(actionId = ActionId.DASHBOARD_VIEW) + @ActionAuditRecord( + actionId = ActionId.DASHBOARD_VIEW, + content = EventContentConstants.VIEW_ANALYSIS_DASHBOARD + ) public Response> dayDetailStatistics(String username, ResourceEnum resource, DimensionEnum dimension, @@ -278,6 +317,8 @@ public Response> dayDetailStatistics(String usern } else if (ResourceEnum.EXECUTED_TASK == resource && DimensionEnum.TASK_TIME_CONSUMING == dimension) { dayDistributionElementVOList = executedTaskByTimeConsumingDayDetail(username, appIdList, startDate, endDate); + } else if (ResourceEnum.EXECUTED_ROLLING_TASK == resource && DimensionEnum.TASK_TYPE == dimension) { + dayDistributionElementVOList = rollingTaskByTaskTypeDayDetail(username, appIdList, startDate, endDate); } else if (ResourceEnum.EXECUTED_FAST_SCRIPT == resource && DimensionEnum.SCRIPT_TYPE == dimension) { dayDistributionElementVOList = fastScriptByScriptTypeDayDetail(username, appIdList, startDate, endDate); } else if (ResourceEnum.EXECUTED_FAST_FILE == resource && DimensionEnum.FILE_TRANSFER_MODE == dimension) { @@ -290,6 +331,11 @@ public Response> dayDetailStatistics(String usern } @Override + @AuditEntry(actionId = ActionId.DASHBOARD_VIEW) + @ActionAuditRecord( + actionId = ActionId.DASHBOARD_VIEW, + content = EventContentConstants.VIEW_ANALYSIS_DASHBOARD + ) public Response> getStatisticsDataInfo(String username) { Map statisticsDataInfoMap = new HashMap<>(); LocalDateTime now = LocalDateTime.now(); diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/client/ApplicationResourceClient.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/client/ApplicationResourceClient.java deleted file mode 100644 index 551280d478..0000000000 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/client/ApplicationResourceClient.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.analysis.client; - -import com.tencent.bk.job.manage.api.inner.ServiceApplicationResource; -import org.springframework.cloud.openfeign.FeignClient; - -/** - * 业务服务远程调用客户端 - */ -@FeignClient(value = "job-manage", contextId = "manageApp") -public interface ApplicationResourceClient extends ServiceApplicationResource { -} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/client/CronJobResourceClient.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/client/CronJobResourceClient.java deleted file mode 100644 index 261c33ba32..0000000000 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/client/CronJobResourceClient.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.analysis.client; - -import com.tencent.bk.job.crontab.api.inner.ServiceCronJobResource; -import org.springframework.cloud.openfeign.FeignClient; - -@FeignClient(value = "job-crontab", contextId = "cronJob") -public interface CronJobResourceClient extends ServiceCronJobResource { -} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/client/CronMetricsResourceClient.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/client/CronMetricsResourceClient.java deleted file mode 100644 index e9d849981e..0000000000 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/client/CronMetricsResourceClient.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.analysis.client; - -import com.tencent.bk.job.crontab.api.inner.ServiceCronMetricsResource; -import org.springframework.cloud.openfeign.FeignClient; - -@FeignClient(value = "job-crontab", contextId = "cronMetrics") -public interface CronMetricsResourceClient extends ServiceCronMetricsResource { -} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/client/ExecuteMetricsClient.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/client/ExecuteMetricsClient.java deleted file mode 100644 index e4a9528fc7..0000000000 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/client/ExecuteMetricsClient.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.analysis.client; - -import com.tencent.bk.job.execute.api.inner.ServiceMetricsResource; -import org.springframework.cloud.openfeign.FeignClient; - -@FeignClient(value = "job-execute", contextId = "executeMetrics") -public interface ExecuteMetricsClient extends ServiceMetricsResource { -} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/client/GlobalSettingsResourceClient.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/client/GlobalSettingsResourceClient.java deleted file mode 100644 index 622c0e04e4..0000000000 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/client/GlobalSettingsResourceClient.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.analysis.client; - -import com.tencent.bk.job.manage.api.inner.ServiceGlobalSettingsResource; -import org.springframework.cloud.openfeign.FeignClient; - -/** - * 全局设置服务远程调用客户端 - */ -@FeignClient(value = "job-manage", contextId = "globalSettings") -public interface GlobalSettingsResourceClient extends ServiceGlobalSettingsResource { -} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/client/HostResourceClient.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/client/HostResourceClient.java deleted file mode 100644 index 0b3b977428..0000000000 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/client/HostResourceClient.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.analysis.client; - -import com.tencent.bk.job.manage.api.inner.ServiceHostResource; -import org.springframework.cloud.openfeign.FeignClient; - -/** - * 主机服务远程调用客户端 - */ -@FeignClient(value = "job-manage", contextId = "host") -public interface HostResourceClient extends ServiceHostResource { -} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/client/InnerCronJobResourceClient.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/client/InnerCronJobResourceClient.java deleted file mode 100644 index c775577f13..0000000000 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/client/InnerCronJobResourceClient.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.analysis.client; - -import com.tencent.bk.job.crontab.api.inner.ServiceInnerCronJobResource; -import org.springframework.cloud.openfeign.FeignClient; - -/** - * 内置定时任务远程调用客户端 - */ -@FeignClient(value = "job-crontab", contextId = "innerCronJob") -public interface InnerCronJobResourceClient extends ServiceInnerCronJobResource { -} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/client/ManageMetricsClient.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/client/ManageMetricsClient.java deleted file mode 100644 index 27c26f3460..0000000000 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/client/ManageMetricsClient.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.analysis.client; - -import com.tencent.bk.job.manage.api.inner.ServiceMetricsResource; -import org.springframework.cloud.openfeign.FeignClient; - -@FeignClient(value = "job-manage", contextId = "manageMetrics") -public interface ManageMetricsClient extends ServiceMetricsResource { -} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/client/ScriptResourceClient.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/client/ScriptResourceClient.java deleted file mode 100644 index 448ca63118..0000000000 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/client/ScriptResourceClient.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.analysis.client; - -import com.tencent.bk.job.manage.api.inner.ServiceScriptResource; -import org.springframework.cloud.openfeign.FeignClient; - -@FeignClient(value = "job-manage", contextId = "script") -public interface ScriptResourceClient extends ServiceScriptResource { -} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/client/ServiceTaskTemplateResourceClient.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/client/ServiceTaskTemplateResourceClient.java deleted file mode 100644 index 79a1cf6746..0000000000 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/client/ServiceTaskTemplateResourceClient.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.analysis.client; - -import com.tencent.bk.job.manage.api.inner.ServiceTaskTemplateResource; -import org.springframework.cloud.openfeign.FeignClient; - -/** - * 业务服务远程调用客户端 - */ -@FeignClient(value = "job-manage", contextId = "taskTemplate") -public interface ServiceTaskTemplateResourceClient extends ServiceTaskTemplateResource { -} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/client/TagClient.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/client/TagClient.java deleted file mode 100644 index 48c960811b..0000000000 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/client/TagClient.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.analysis.client; - -import com.tencent.bk.job.manage.api.inner.ServiceTagResource; -import org.springframework.cloud.openfeign.FeignClient; - -@FeignClient(value = "job-manage", contextId = "tag") -public interface TagClient extends ServiceTagResource { -} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/client/TaskExecuteResultResourceClient.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/client/TaskExecuteResultResourceClient.java deleted file mode 100644 index 6d1c22f079..0000000000 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/client/TaskExecuteResultResourceClient.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.analysis.client; - -import com.tencent.bk.job.execute.api.inner.ServiceTaskExecuteResultResource; -import org.springframework.cloud.openfeign.FeignClient; - -/** - * @Description - * @Date 2020/3/9 - * @Version 1.0 - */ - -@FeignClient(value = "job-execute", contextId = "taskExecuteResult") -public interface TaskExecuteResultResourceClient extends ServiceTaskExecuteResultResource { -} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/client/TaskPlanResourceClient.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/client/TaskPlanResourceClient.java deleted file mode 100644 index e6910d618e..0000000000 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/client/TaskPlanResourceClient.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.analysis.client; - -import com.tencent.bk.job.manage.api.inner.ServiceTaskPlanResource; -import org.springframework.cloud.openfeign.FeignClient; - -@FeignClient(value = "job-manage", contextId = "taskPlan") -public interface TaskPlanResourceClient extends ServiceTaskPlanResource { -} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/AIConfig.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/AIConfig.java new file mode 100644 index 0000000000..09d5419066 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/AIConfig.java @@ -0,0 +1,62 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.config; + +import com.tencent.bk.job.common.aidev.config.CustomPaasLoginProperties; +import com.tencent.bk.job.common.aidev.impl.BkOpenAIClient; +import com.tencent.bk.job.common.esb.config.AppProperties; +import com.tencent.bk.job.common.esb.config.BkApiGatewayProperties; +import io.micrometer.core.instrument.MeterRegistry; +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.cloud.sleuth.SpanNamer; +import org.springframework.cloud.sleuth.Tracer; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Slf4j +@Configuration(value = "jobAnalysisAIConfig") +@EnableConfigurationProperties(AIProperties.class) +public class AIConfig { + + @Bean("jobAnalysisBkOpenAIClient") + public BkOpenAIClient bkOpenAIClient(Tracer tracer, + SpanNamer spanNamer, + MeterRegistry meterRegistry, + AppProperties appProperties, + CustomPaasLoginProperties customPaasLoginProperties, + BkApiGatewayProperties bkApiGatewayProperties, + AIProperties aiProperties) { + return new BkOpenAIClient( + tracer, + spanNamer, + meterRegistry, + appProperties, + customPaasLoginProperties, + bkApiGatewayProperties, + aiProperties.getModel() + ); + } +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/AIProperties.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/AIProperties.java new file mode 100644 index 0000000000..97a1575f90 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/AIProperties.java @@ -0,0 +1,86 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.config; + +import com.tencent.bk.job.common.util.file.FileSizeUtil; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.springframework.boot.context.properties.ConfigurationProperties; + +/** + * AI相关配置 + */ +@Getter +@Setter +@ToString +@ConfigurationProperties(prefix = "ai") +public class AIProperties { + + private Boolean enabled = false; + + /** + * 使用的大模型类别，默认腾讯混元 + */ + private String model = "hunyuan"; + + /** + * 错误日志分析相关配置 + */ + private AnalyzeErrorLogConfig analyzeErrorLog = new AnalyzeErrorLogConfig(); + + /** + * 对话记录相关配置 + */ + private ChatHistoryConfig chatHistory = new ChatHistoryConfig(); + + @Getter + @Setter + @ToString + public static class AnalyzeErrorLogConfig { + /** + * 支持分析的错误日志最大长度 + */ + private String logMaxLength = "5MB"; + + public Long getLogMaxLengthBytes() { + return FileSizeUtil.parseFileSizeBytes(logMaxLength); + } + } + + @Getter + @Setter + @ToString + public static class ChatHistoryConfig { + /** + * 最大保留天数 + */ + private Integer maxKeepDays = 31; + /** + * 单个用户最大保留的对话记录数量 + */ + private Integer maxHistoryPerUser = 1000; + } +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/BkConfig.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/BkConfig.java deleted file mode 100644 index 9e790f56bb..0000000000 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/BkConfig.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.analysis.config; - -import lombok.Getter; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Configuration; - -@Getter -@Configuration -public class BkConfig { - - @Value("${swagger.url:swagger.job.com}") - private String swaggerUrl; - -} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/DbConfig.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/DbConfig.java new file mode 100644 index 0000000000..bb661012c1 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/DbConfig.java @@ -0,0 +1,102 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.config; + +import com.tencent.bk.job.common.mysql.util.JooqConfigurationUtil; +import org.jooq.ConnectionProvider; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; +import org.jooq.impl.DataSourceConnectionProvider; +import org.jooq.impl.DefaultConfiguration; +import org.jooq.impl.DefaultDSLContext; +import org.jooq.impl.DefaultExecuteListenerProvider; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.jdbc.DataSourceBuilder; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.DependsOn; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.DataSourceTransactionManager; +import org.springframework.jdbc.datasource.TransactionAwareDataSourceProxy; +import org.springframework.transaction.annotation.EnableTransactionManagement; + +import javax.sql.DataSource; + +@Configuration(value = "jobAnalysisDbConfig") +@EnableTransactionManagement +public class DbConfig { + @Qualifier("job-analysis-data-source") + @Bean(name = "job-analysis-data-source") + @ConfigurationProperties(prefix = "spring.datasource.job-analysis") + public DataSource dataSource() { + return DataSourceBuilder.create().build(); + } + + @Qualifier("jobAnalysisTransactionManager") + @Bean(name = "jobAnalysisTransactionManager") + @DependsOn("job-analysis-data-source") + public DataSourceTransactionManager transactionManager( + @Qualifier("job-analysis-data-source") DataSource dataSource) { + return new DataSourceTransactionManager(dataSource); + } + + @Qualifier("job-analysis-jdbc-template") + @Bean(name = "job-analysis-jdbc-template") + public JdbcTemplate jdbcTemplate( + @Qualifier("job-analysis-data-source") DataSource dataSource) { + return new JdbcTemplate(dataSource); + } + + @Qualifier("job-analysis-dsl-context") + @Bean(name = "job-analysis-dsl-context") + public DSLContext dslContext(@Qualifier("job-analysis-jooq-conf") org.jooq.Configuration configuration) { + return new DefaultDSLContext(configuration); + } + + @Qualifier("job-analysis-jooq-conf") + @Bean(name = "job-analysis-jooq-conf") + public org.jooq.Configuration + jooqConf(@Qualifier("job-analysis-conn-provider") ConnectionProvider connectionProvider, + DefaultExecuteListenerProvider jooqExecuteListenerProvider + ) { + return JooqConfigurationUtil.getConfiguration(connectionProvider, jooqExecuteListenerProvider); + } + + @Qualifier("job-analysis-conn-provider") + @Bean(name = "job-analysis-conn-provider") + public ConnectionProvider connectionProvider( + @Qualifier("jobAnalysisTransactionAwareDataSource") DataSource dataSource) { + return new DataSourceConnectionProvider(dataSource); + } + + @Qualifier("jobAnalysisTransactionAwareDataSource") + @Bean(name = "jobAnalysisTransactionAwareDataSource") + public TransactionAwareDataSourceProxy + transactionAwareDataSourceProxy(@Qualifier("job-analysis-data-source") DataSource dataSource) { + return new TransactionAwareDataSourceProxy(dataSource); + } + +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/ExecutorConfiguration.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/ExecutorConfiguration.java index 1b71f1959e..120dba7fea 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/ExecutorConfiguration.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/ExecutorConfiguration.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,21 +25,40 @@ package com.tencent.bk.job.analysis.config; import com.tencent.bk.job.analysis.task.statistics.StatisticsTaskScheduler; +import com.tencent.bk.job.common.WatchableThreadPoolExecutor; +import io.micrometer.core.instrument.MeterRegistry; import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; @Slf4j -@Configuration +@Configuration(value = "jobAnalysisExecutorConfig") public class ExecutorConfiguration { + @Bean("analysisAsyncTaskExecutor") + public ThreadPoolExecutor analysisAsyncTaskExecutor(MeterRegistry meterRegistry) { + return new WatchableThreadPoolExecutor( + meterRegistry, + "analysisAsyncTaskExecutor", + 0, + 50, + 180L, + TimeUnit.SECONDS, + new SynchronousQueue<>(), + (r, executor) -> log.warn("AsyncTask runnable rejected!") + ); + } + @Bean("analysisScheduleExecutor") - public ThreadPoolExecutor analysisScheduleExecutor() { - return new ThreadPoolExecutor( + public ThreadPoolExecutor analysisScheduleExecutor(MeterRegistry meterRegistry) { + return new WatchableThreadPoolExecutor( + meterRegistry, + "analysisScheduleExecutor", 5, 10, 60L, @@ -49,8 +68,10 @@ public ThreadPoolExecutor analysisScheduleExecutor() { } @Bean("currentStatisticsTaskExecutor") - public ThreadPoolExecutor currentStatisticsTaskExecutor() { - return new ThreadPoolExecutor( + public ThreadPoolExecutor currentStatisticsTaskExecutor(MeterRegistry meterRegistry) { + return new WatchableThreadPoolExecutor( + meterRegistry, + "currentStatisticsTaskExecutor", StatisticsTaskScheduler.defaultCorePoolSize, StatisticsTaskScheduler.defaultMaximumPoolSize, StatisticsTaskScheduler.defaultKeepAliveTime, @@ -60,8 +81,10 @@ public ThreadPoolExecutor currentStatisticsTaskExecutor() { } @Bean("pastStatisticsTaskExecutor") - public ThreadPoolExecutor pastStatisticsTaskExecutor() { - return new ThreadPoolExecutor( + public ThreadPoolExecutor pastStatisticsTaskExecutor(MeterRegistry meterRegistry) { + return new WatchableThreadPoolExecutor( + meterRegistry, + "pastStatisticsTaskExecutor", StatisticsTaskScheduler.defaultCorePoolSize, StatisticsTaskScheduler.defaultMaximumPoolSize, StatisticsTaskScheduler.defaultKeepAliveTime, diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/JobAnalysisAutoConfiguration.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/JobAnalysisAutoConfiguration.java deleted file mode 100644 index 0096f849b4..0000000000 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/JobAnalysisAutoConfiguration.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.analysis.config; - -import com.tencent.bk.job.analysis.client.ApplicationResourceClient; -import com.tencent.bk.job.common.service.AppScopeMappingService; -import com.tencent.bk.job.manage.AppScopeMappingServiceImpl; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Configuration -public class JobAnalysisAutoConfiguration { - - @Bean - AppScopeMappingService appScopeMappingService(ApplicationResourceClient applicationResource) { - return new AppScopeMappingServiceImpl(applicationResource); - } -} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/JobFunctionConfiguration.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/JobFunctionConfiguration.java new file mode 100644 index 0000000000..777dc1d0d4 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/JobFunctionConfiguration.java @@ -0,0 +1,52 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.config; + +import com.tencent.bk.job.analysis.listener.AIChatOperationEventListener; +import com.tencent.bk.job.analysis.listener.event.AIChatOperationEvent; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import java.util.function.Consumer; + +/** + * spring cloud function 定义 + *

+ * 注意：方法名与配置文件中的spring.cloud.stream.function.definition对应，修改需要注意！！！ + */ +@Configuration(value = "jobAnalysisFunctionConfig") +@Slf4j +public class JobFunctionConfiguration { + @Bean + public Consumer handleAIChatOperationEvent( + @Autowired AIChatOperationEventListener aiChatOperationEventListener + ) { + log.info("Init handleAIChatOperationEvent consumer"); + return aiChatOperationEventListener::handleEvent; + } + +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/ScheduleConfig.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/ScheduleConfig.java new file mode 100644 index 0000000000..84c85f6e50 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/ScheduleConfig.java @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.config; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.annotation.SchedulingConfigurer; +import org.springframework.scheduling.config.ScheduledTaskRegistrar; + +import java.util.concurrent.ScheduledThreadPoolExecutor; + +/** + * 设定一个长度5的定时任务线程池 + */ +@Slf4j +@Configuration +public class ScheduleConfig implements SchedulingConfigurer { + @Override + public void configureTasks(ScheduledTaskRegistrar taskRegistrar) { + taskRegistrar.setScheduler(new ScheduledThreadPoolExecutor(5, (r, executor) -> log.error( + "ScheduledThreadPoolExecutor rejected a runnable"))); + } +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/StatisticConfig.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/StatisticConfig.java index 75671a4acd..f1063f5312 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/StatisticConfig.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/StatisticConfig.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/WebMvcConfig.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/WebMvcConfig.java new file mode 100644 index 0000000000..2d2c46f1e6 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/WebMvcConfig.java @@ -0,0 +1,63 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.config; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.task.support.TaskExecutorAdapter; +import org.springframework.web.servlet.config.annotation.AsyncSupportConfigurer; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; + +import java.util.concurrent.ThreadPoolExecutor; + +/** + * 自定义WebMvc配置 + */ +@Slf4j +@Configuration +public class WebMvcConfig { + + @Bean + public WebMvcConfigurer jobWebMvcConfigurer( + @Qualifier("analysisAsyncTaskExecutor") ThreadPoolExecutor threadPoolExecutor + ) { + return new JobWebMvcConfigurer(threadPoolExecutor); + } + + static class JobWebMvcConfigurer implements WebMvcConfigurer { + ThreadPoolExecutor threadPoolExecutor; + + public JobWebMvcConfigurer(ThreadPoolExecutor threadPoolExecutor) { + this.threadPoolExecutor = threadPoolExecutor; + } + + @Override + public void configureAsyncSupport(AsyncSupportConfigurer configurer) { + configurer.setTaskExecutor(new TaskExecutorAdapter(threadPoolExecutor)); + } + } +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/listener/ApplicationReadyEventListener.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/listener/ApplicationReadyEventListener.java index 8b99b62e76..251eed5642 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/listener/ApplicationReadyEventListener.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/config/listener/ApplicationReadyEventListener.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/AIChatOperationEnum.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/AIChatOperationEnum.java new file mode 100644 index 0000000000..8dd465e736 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/AIChatOperationEnum.java @@ -0,0 +1,54 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.consts; + +import lombok.Getter; + +/** + * AI对话相关操作 + */ +@Getter +public enum AIChatOperationEnum { + /** + * 终止对话 + */ + TERMINATE_CHAT(1); + + private final int value; + + AIChatOperationEnum(int val) { + this.value = val; + } + + public static AIChatOperationEnum valueOf(int value) { + for (AIChatOperationEnum aiChatOperationEnum : values()) { + if (aiChatOperationEnum.getValue() == value) { + return aiChatOperationEnum; + } + } + throw new IllegalArgumentException("No AIChatOperationEnum constant: " + value); + } + +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/DataTrendEnum.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/DataTrendEnum.java new file mode 100644 index 0000000000..a0a7e9a5bb --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/DataTrendEnum.java @@ -0,0 +1,52 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.consts; + +import lombok.Getter; + +/** + * 数据趋势 + */ +@Getter +public enum DataTrendEnum { + /** + * 上升 + */ + UP(1), + /** + * 不变 + */ + NOT_CHANGE(0), + /** + * 下降 + */ + DOWN(-1); + + private final int value; + + DataTrendEnum(int value) { + this.value = value; + } +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/JobAnalysisConsts.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/JobAnalysisConsts.java new file mode 100644 index 0000000000..741ffb0655 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/consts/JobAnalysisConsts.java @@ -0,0 +1,36 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.consts; + +public class JobAnalysisConsts { + // 分析结果描述中的占位符：执行方案名称 + public static final String PLACEHOLDER_PLAN_NAME = "${planName}"; + // 分析结果描述中的占位符：步骤名称 + public static final String PLACEHOLDER_STEP_NAME = "${stepName}"; + // 分析结果描述中的占位符：描述 + public static final String PLACEHOLDER_DESCRIPTION = "${description}"; + // 分析结果描述中的占位符：条目数量 + public static final String PLACEHOLDER_ITEMS_COUNT = "${itemsCount}"; +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/AIAnalyzeErrorContextDAO.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/AIAnalyzeErrorContextDAO.java new file mode 100644 index 0000000000..56ecb72771 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/AIAnalyzeErrorContextDAO.java @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.dao; + +import com.tencent.bk.job.analysis.model.dto.AIAnalyzeErrorContextDTO; + +import java.util.List; + +public interface AIAnalyzeErrorContextDAO { + /** + * 插入报错分析上下文记录 + * + * @param aiAnalyzeErrorContextDTO 报错分析上下文记录 + * @return 受影响的行数 + */ + int insert(AIAnalyzeErrorContextDTO aiAnalyzeErrorContextDTO); + + /** + * 删除报错分析上下文记录 + * + * @param chatHistoryIdList 记录ID列表 + * @return 删除的记录条数 + */ + int delete(List chatHistoryIdList); +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/AIChatHistoryDAO.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/AIChatHistoryDAO.java new file mode 100644 index 0000000000..ee655d5e54 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/AIChatHistoryDAO.java @@ -0,0 +1,147 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.dao; + +import com.tencent.bk.job.analysis.model.dto.AIChatHistoryDTO; + +import java.util.List; + +public interface AIChatHistoryDAO { + /** + * 插入对话记录 + * + * @param aiChatHistoryDTO AI对话记录 + * @return 插入记录的id + */ + Long insertAIChatHistory(AIChatHistoryDTO aiChatHistoryDTO); + + /** + * 判断用户是否存在对话记录 + * + * @param username 用户名 + * @return 是否存在对话记录 + */ + boolean existsChatHistory(String username); + + /** + * 设置对话记录状态 + * + * @param historyId AI对话记录ID + * @param status AI对话记录状态 + * @return 受影响的行数 + */ + int updateChatHistoryStatus(Long historyId, int status); + + /** + * 更新对话记录状态 + * + * @param historyId AI对话记录ID + * @param status AI对话记录状态 + * @param aiAnswer AI回答 + * @param errorCode 错误码 + * @param errorMessage 错误信息 + * @param aiAnswerTime AI回答时间 + * @return 受影响的行数 + */ + int updateChatHistoryStatusAndAIAnswer(Long historyId, + Integer status, + String aiAnswer, + String errorCode, + String errorMessage, + Long aiAnswerTime); + + /** + * 获取最近的对话记录列表 + * + * @param username 用户名 + * @param start 起始位置 + * @param length 长度 + * @return 最近的对话记录列表 + */ + List getLatestChatHistoryList(String username, Integer start, Integer length); + + /** + * 获取最近已完成的对话记录列表 + * + * @param username 用户名 + * @param start 起始位置 + * @param length 长度 + * @return 最近已完成的对话记录列表 + */ + List getLatestFinishedChatHistoryList(String username, Integer start, Integer length); + + /** + * 获取对话记录 + * + * @param username 用户名 + * @param id 对话记录ID + * @return 对话记录 + */ + AIChatHistoryDTO getChatHistory(String username, Long id); + + /** + * 软删除对话记录（优先删除创建时间较早的） + * + * @param username 用户名 + * @param limit 最大删除数量 + * @return 删除的记录条数 + */ + int softDeleteChatHistory(String username, Integer limit); + + /** + * 硬删除id小于指定id的对话记录（按id从小到大的顺序删除） + * + * @param maxStartTime 最大开始时间 + * @param limit 最大删除数量 + * @return 删除的记录条数 + */ + int deleteChatHistory(long maxStartTime, int limit); + + /** + * 硬删除某个用户的id小于指定id的对话记录（按id从小到大的顺序删除） + * + * @param username 用户名 + * @param maxId 最大id + * @param limit 最大删除数量 + * @return 删除的记录条数 + */ + int deleteChatHistory(String username, long maxId, int limit); + + /** + * 获取所有有对话记录（未删除）的用户 + * + * @return 所有有对话记录（未删除）的用户 + */ + List listAllUserOfChatHistory(); + + /** + * 获取offset后第一条记录的Id + * + * @param username 用户名 + * @param offset 偏移量 + * @return 第一条记录的Id + */ + Long getFirstIdAfterOffset(String username, int offset); +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/AIPromptTemplateDAO.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/AIPromptTemplateDAO.java new file mode 100644 index 0000000000..f0232625af --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/AIPromptTemplateDAO.java @@ -0,0 +1,31 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.dao; + +import com.tencent.bk.job.analysis.model.dto.AIPromptTemplateDTO; + +public interface AIPromptTemplateDAO { + AIPromptTemplateDTO getAIPromptTemplate(String code, String locale); +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/AnalysisTaskDAO.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/AnalysisTaskDAO.java index a33d8b3889..cdbb90cf8b 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/AnalysisTaskDAO.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/AnalysisTaskDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,22 +25,21 @@ package com.tencent.bk.job.analysis.dao; import com.tencent.bk.job.analysis.model.dto.AnalysisTaskDTO; -import org.jooq.DSLContext; import java.util.List; public interface AnalysisTaskDAO { - Long insertAnalysisTask(DSLContext dslContext, AnalysisTaskDTO analysisTaskDTO); + Long insertAnalysisTask(AnalysisTaskDTO analysisTaskDTO); - int updateAnalysisTaskById(DSLContext dslContext, AnalysisTaskDTO analysisTaskDTO); + int updateAnalysisTaskById(AnalysisTaskDTO analysisTaskDTO); - int deleteAnalysisTaskById(DSLContext dslContext, Long id); + int deleteAnalysisTaskById(Long id); - AnalysisTaskDTO getAnalysisTaskById(DSLContext dslContext, Long id); + AnalysisTaskDTO getAnalysisTaskById(Long id); - AnalysisTaskDTO getAnalysisTaskByCode(DSLContext dslContext, String code); + AnalysisTaskDTO getAnalysisTaskByCode(String code); - List listAllAnalysisTask(DSLContext dslContext); + List listAllAnalysisTask(); - List listActiveAnalysisTask(DSLContext dslContext); + List listActiveAnalysisTask(); } diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/AnalysisTaskInstanceDAO.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/AnalysisTaskInstanceDAO.java index 5650c41a5c..680af8c1a9 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/AnalysisTaskInstanceDAO.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/AnalysisTaskInstanceDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,33 +26,30 @@ import com.tencent.bk.job.analysis.model.dto.AnalysisTaskInstanceDTO; import com.tencent.bk.job.analysis.model.dto.AnalysisTaskInstanceWithTpl; -import org.jooq.DSLContext; import java.util.List; public interface AnalysisTaskInstanceDAO { - Long insertAnalysisTaskInstance(DSLContext dslContext, AnalysisTaskInstanceDTO analysisTaskInstanceDTO); + Long insertAnalysisTaskInstance(AnalysisTaskInstanceDTO analysisTaskInstanceDTO); - int updateAnalysisTaskInstanceById(DSLContext dslContext, AnalysisTaskInstanceDTO analysisTaskInstanceDTO); + int updateAnalysisTaskInstanceById(AnalysisTaskInstanceDTO analysisTaskInstanceDTO); - int deleteHistoryAnalysisTaskInstance(DSLContext dslContext, Long appId, Long taskId); + int deleteHistoryAnalysisTaskInstance(Long appId, Long taskId); - int deleteAnalysisTaskInstanceById(DSLContext dslContext, Long id); + int deleteAnalysisTaskInstanceById(Long id); - AnalysisTaskInstanceDTO getAnalysisTaskInstanceById(DSLContext dslContext, Long id); + AnalysisTaskInstanceDTO getAnalysisTaskInstanceById(Long id); - List listAllAnalysisTaskInstance(DSLContext dslContext); + List listAllAnalysisTaskInstance(); - List listActiveAnalysisTaskInstance(DSLContext dslContext, Long appId, Long limit); + List listActiveAnalysisTaskInstance(Long appId, Long limit); /** * 查询最新的分析结果，每个任务一条 * - * @param dslContext * @param appId * @param limit * @return 任务分析结果列表 */ - List listNewestActiveInstance(DSLContext dslContext, Long appId, - Long limit); + List listNewestActiveInstance(Long appId, Long limit); } diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/AnalysisTaskStaticInstanceDAO.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/AnalysisTaskStaticInstanceDAO.java index 6139f64485..0aef09aa0d 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/AnalysisTaskStaticInstanceDAO.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/AnalysisTaskStaticInstanceDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,20 +25,17 @@ package com.tencent.bk.job.analysis.dao; import com.tencent.bk.job.analysis.model.dto.AnalysisTaskStaticInstanceDTO; -import org.jooq.DSLContext; import java.util.List; public interface AnalysisTaskStaticInstanceDAO { - Long insert(DSLContext dslContext, AnalysisTaskStaticInstanceDTO analysisTaskInstanceDTO); + Long insert(AnalysisTaskStaticInstanceDTO analysisTaskInstanceDTO); - int updateById(DSLContext dslContext, - AnalysisTaskStaticInstanceDTO analysisTaskInstanceDTO); + int updateById(AnalysisTaskStaticInstanceDTO analysisTaskInstanceDTO); - int deleteById(DSLContext dslContext, Long id); + int deleteById(Long id); - AnalysisTaskStaticInstanceDTO getById(DSLContext dslContext, Long id); + AnalysisTaskStaticInstanceDTO getById(Long id); - List listActiveInstance(DSLContext dslContext, Long offset, - Long limit); + List listActiveInstance(Long offset, Long limit); } diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/StatisticsDAO.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/StatisticsDAO.java index 74b0415569..26c206cdc0 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/StatisticsDAO.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/StatisticsDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,7 @@ package com.tencent.bk.job.analysis.dao; -import com.tencent.bk.job.common.statistics.model.dto.StatisticsDTO; +import com.tencent.bk.job.analysis.api.dto.StatisticsDTO; import org.jooq.DSLContext; import java.util.List; diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/impl/AIAnalyzeErrorContextDAOImpl.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/impl/AIAnalyzeErrorContextDAOImpl.java new file mode 100644 index 0000000000..08c43550fd --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/impl/AIAnalyzeErrorContextDAOImpl.java @@ -0,0 +1,111 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.dao.impl; + +import com.tencent.bk.job.analysis.dao.AIAnalyzeErrorContextDAO; +import com.tencent.bk.job.analysis.model.dto.AIAnalyzeErrorContextDTO; +import com.tencent.bk.job.analysis.model.tables.AiAnalyzeErrorContext; +import com.tencent.bk.job.common.mysql.util.JooqDataTypeUtil; +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.jooq.Condition; +import org.jooq.DSLContext; +import org.jooq.Record; +import org.jooq.conf.ParamType; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Repository; + +import java.util.ArrayList; +import java.util.List; + +@Slf4j +@Repository +public class AIAnalyzeErrorContextDAOImpl implements AIAnalyzeErrorContextDAO { + + private static final AiAnalyzeErrorContext defaultTable = AiAnalyzeErrorContext.AI_ANALYZE_ERROR_CONTEXT; + + private final DSLContext dslContext; + + public AIAnalyzeErrorContextDAOImpl(@Qualifier("job-analysis-dsl-context") DSLContext dslContext) { + this.dslContext = dslContext; + } + + @Override + public int insert(AIAnalyzeErrorContextDTO aiAnalyzeErrorContextDTO) { + val query = dslContext.insertInto(defaultTable, + defaultTable.AI_CHAT_HISTORY_ID, + defaultTable.TASK_INSTANCE_ID, + defaultTable.STEP_INSTANCE_ID, + defaultTable.EXECUTE_COUNT, + defaultTable.BATCH, + defaultTable.EXECUTE_OBJECT_TYPE, + defaultTable.EXECUTE_OBJECT_RESOURCE_ID, + defaultTable.MODE + ) + .values( + aiAnalyzeErrorContextDTO.getAiChatHistoryId(), + aiAnalyzeErrorContextDTO.getTaskInstanceId(), + aiAnalyzeErrorContextDTO.getStepInstanceId(), + aiAnalyzeErrorContextDTO.getExecuteCount(), + JooqDataTypeUtil.getShortFromInteger(aiAnalyzeErrorContextDTO.getBatch()), + JooqDataTypeUtil.getByteFromInteger(aiAnalyzeErrorContextDTO.getExecuteObjectType()), + aiAnalyzeErrorContextDTO.getExecuteObjectResourceId(), + JooqDataTypeUtil.getByteFromInteger(aiAnalyzeErrorContextDTO.getMode()) + ); + val sql = query.getSQL(ParamType.INLINED); + try { + return query.execute(); + } catch (Exception e) { + log.error(sql); + throw e; + } + } + + @Override + public int delete(List chatHistoryIdList) { + List conditions = new ArrayList<>(); + conditions.add(defaultTable.AI_CHAT_HISTORY_ID.in(chatHistoryIdList)); + return dslContext.deleteFrom(defaultTable) + .where(conditions) + .execute(); + } + + private AIAnalyzeErrorContextDTO convertRecordToDto(Record record) { + AIAnalyzeErrorContextDTO aiAnalyzeErrorContextDTO = new AIAnalyzeErrorContextDTO(); + aiAnalyzeErrorContextDTO.setTaskInstanceId(record.get(defaultTable.TASK_INSTANCE_ID)); + aiAnalyzeErrorContextDTO.setStepInstanceId(record.get(defaultTable.STEP_INSTANCE_ID)); + aiAnalyzeErrorContextDTO.setExecuteCount(record.get(defaultTable.EXECUTE_COUNT)); + aiAnalyzeErrorContextDTO.setBatch(JooqDataTypeUtil.getIntegerFromShort(record.get(defaultTable.BATCH))); + aiAnalyzeErrorContextDTO.setExecuteObjectType( + JooqDataTypeUtil.getIntegerFromByte(record.get(defaultTable.EXECUTE_OBJECT_TYPE)) + ); + aiAnalyzeErrorContextDTO.setExecuteObjectResourceId(record.get(defaultTable.EXECUTE_OBJECT_RESOURCE_ID)); + aiAnalyzeErrorContextDTO.setMode( + JooqDataTypeUtil.getIntegerFromByte(record.get(defaultTable.MODE)) + ); + return aiAnalyzeErrorContextDTO; + } + +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/impl/AIChatHistoryDAOImpl.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/impl/AIChatHistoryDAOImpl.java new file mode 100644 index 0000000000..c43d85acc8 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/impl/AIChatHistoryDAOImpl.java @@ -0,0 +1,342 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.dao.impl; + +import com.tencent.bk.job.analysis.consts.AIChatStatusEnum; +import com.tencent.bk.job.analysis.dao.AIAnalyzeErrorContextDAO; +import com.tencent.bk.job.analysis.dao.AIChatHistoryDAO; +import com.tencent.bk.job.analysis.model.dto.AIAnalyzeErrorContextDTO; +import com.tencent.bk.job.analysis.model.dto.AIChatHistoryDTO; +import com.tencent.bk.job.analysis.model.tables.AiChatHistory; +import com.tencent.bk.job.analysis.util.ai.AIAnswerUtil; +import com.tencent.bk.job.common.mysql.dao.BaseDAOImpl; +import com.tencent.bk.job.common.mysql.util.JooqDataTypeUtil; +import io.micrometer.core.instrument.util.StringUtils; +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.apache.commons.collections.CollectionUtils; +import org.jooq.Condition; +import org.jooq.DSLContext; +import org.jooq.Record; +import org.jooq.conf.ParamType; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Repository; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +@Slf4j +@Repository +public class AIChatHistoryDAOImpl extends BaseDAOImpl implements AIChatHistoryDAO { + + // IS_DELETED字段取值：0-未删除，1-已删除 + private static final int VALUE_NOT_DELETED = 0; + private static final int VALUE_DELETED = 1; + + private static final AiChatHistory defaultTable = AiChatHistory.AI_CHAT_HISTORY; + + private final DSLContext dslContext; + private final AIAnalyzeErrorContextDAO aiAnalyzeErrorContextDAO; + + public AIChatHistoryDAOImpl(@Qualifier("job-analysis-dsl-context") + DSLContext dslContext, + AIAnalyzeErrorContextDAO aiAnalyzeErrorContextDAO) { + this.dslContext = dslContext; + this.aiAnalyzeErrorContextDAO = aiAnalyzeErrorContextDAO; + } + + /** + * 插入AI对话历史记录 + * + * @param aiChatHistoryDTO AI对话历史记录 + * @return 记录ID + */ + @SuppressWarnings("DataFlowIssue") + @Override + public Long insertAIChatHistory(AIChatHistoryDTO aiChatHistoryDTO) { + val query = dslContext.insertInto(defaultTable, + defaultTable.USERNAME, + defaultTable.APP_ID, + defaultTable.USER_INPUT, + defaultTable.PROMPT_TEMPLATE_ID, + defaultTable.AI_INPUT, + defaultTable.STATUS, + defaultTable.AI_ANSWER, + defaultTable.ERROR_CODE, + defaultTable.ERROR_MESSAGE, + defaultTable.START_TIME, + defaultTable.ANSWER_TIME, + defaultTable.TOTAL_TIME, + defaultTable.IS_DELETED + ) + .values( + aiChatHistoryDTO.getUsername(), + aiChatHistoryDTO.getAppId(), + aiChatHistoryDTO.getUserInput(), + aiChatHistoryDTO.getPromptTemplateId(), + aiChatHistoryDTO.getAiInput(), + JooqDataTypeUtil.getByteFromInteger(aiChatHistoryDTO.getStatus()), + aiChatHistoryDTO.getLimitedAIAnswer(), + aiChatHistoryDTO.getErrorCode(), + aiChatHistoryDTO.getLimitedErrorMessage(), + JooqDataTypeUtil.buildULong(aiChatHistoryDTO.getStartTime()), + JooqDataTypeUtil.buildULong(aiChatHistoryDTO.getAnswerTime()), + JooqDataTypeUtil.buildULong(aiChatHistoryDTO.getTotalTime()), + JooqDataTypeUtil.buildUByte(aiChatHistoryDTO.getIsDeleted() ? VALUE_DELETED : VALUE_NOT_DELETED) + ) + .returning(defaultTable.ID); + val sql = query.getSQL(ParamType.INLINED); + try { + Long chatHistoryId = query.fetchOne().getId(); + AIAnalyzeErrorContextDTO aiAnalyzeErrorContext = aiChatHistoryDTO.getAiAnalyzeErrorContext(); + if (aiAnalyzeErrorContext != null) { + aiAnalyzeErrorContext.setAiChatHistoryId(chatHistoryId); + insertAIAnalyzeErrorContextIfNeed(aiAnalyzeErrorContext); + } + return chatHistoryId; + } catch (Exception e) { + log.error(sql); + throw e; + } + } + + private void insertAIAnalyzeErrorContextIfNeed(AIAnalyzeErrorContextDTO aiAnalyzeErrorContext) { + if (aiAnalyzeErrorContext == null) { + return; + } + int insertedNum = aiAnalyzeErrorContextDAO.insert(aiAnalyzeErrorContext); + log.debug("{} AnalyzeErrorContext record inserted", insertedNum); + } + + @Override + public boolean existsChatHistory(String username) { + Collection conditions = new ArrayList<>(); + conditions.add(defaultTable.USERNAME.eq(username)); + conditions.add(defaultTable.IS_DELETED.eq(JooqDataTypeUtil.buildUByte(VALUE_NOT_DELETED))); + return dslContext.fetchExists(defaultTable, conditions); + } + + @Override + public int updateChatHistoryStatus(Long historyId, int status) { + return dslContext.update(defaultTable) + .set(defaultTable.STATUS, JooqDataTypeUtil.getByteFromInteger(status)) + .where(defaultTable.ID.eq(historyId)) + .execute(); + } + + @Override + public int updateChatHistoryStatusAndAIAnswer(Long historyId, + Integer status, + String aiAnswer, + String errorCode, + String errorMessage, + Long aiAnswerTime) { + return dslContext.update(defaultTable) + .set(defaultTable.STATUS, JooqDataTypeUtil.getByteFromInteger(status)) + .set(defaultTable.AI_ANSWER, AIAnswerUtil.getLimitedAIAnswer(aiAnswer)) + .set(defaultTable.ERROR_CODE, errorCode) + .set(defaultTable.ERROR_MESSAGE, errorMessage) + .set(defaultTable.ANSWER_TIME, JooqDataTypeUtil.buildULong(aiAnswerTime)) + .where(defaultTable.ID.eq(historyId)) + .execute(); + } + + @Override + public List getLatestChatHistoryList(String username, Integer start, Integer length) { + Collection conditions = new ArrayList<>(); + conditions.add(defaultTable.USERNAME.eq(username)); + conditions.add(defaultTable.IS_DELETED.eq(JooqDataTypeUtil.buildUByte(VALUE_NOT_DELETED))); + return listByConditions(conditions, start, length); + } + + @Override + public List getLatestFinishedChatHistoryList(String username, Integer start, Integer length) { + Collection conditions = new ArrayList<>(); + conditions.add(defaultTable.USERNAME.eq(username)); + conditions.add(defaultTable.STATUS.eq((byte) AIChatStatusEnum.FINISHED.getStatus())); + conditions.add(defaultTable.IS_DELETED.eq(JooqDataTypeUtil.buildUByte(VALUE_NOT_DELETED))); + return listByConditions(conditions, start, length); + } + + @Override + public AIChatHistoryDTO getChatHistory(String username, Long id) { + Collection conditions = new ArrayList<>(); + conditions.add(defaultTable.USERNAME.eq(username)); + conditions.add(defaultTable.ID.eq(id)); + conditions.add(defaultTable.IS_DELETED.eq(JooqDataTypeUtil.buildUByte(VALUE_NOT_DELETED))); + val query = dslContext.select( + defaultTable.ID, + defaultTable.USERNAME, + defaultTable.USER_INPUT, + defaultTable.PROMPT_TEMPLATE_ID, + defaultTable.AI_INPUT, + defaultTable.STATUS, + defaultTable.AI_ANSWER, + defaultTable.ERROR_CODE, + defaultTable.ERROR_MESSAGE, + defaultTable.START_TIME, + defaultTable.ANSWER_TIME, + defaultTable.TOTAL_TIME + ) + .from(defaultTable) + .where(conditions); + val record = query.fetchOne(); + if (record == null) { + return null; + } + return convertRecordToDto(record); + } + + @Override + public int softDeleteChatHistory(String username, Integer limit) { + Collection conditions = new ArrayList<>(); + conditions.add(defaultTable.USERNAME.eq(username)); + conditions.add(defaultTable.IS_DELETED.eq(JooqDataTypeUtil.buildUByte(VALUE_NOT_DELETED))); + return dslContext.update(defaultTable) + .set(defaultTable.IS_DELETED, JooqDataTypeUtil.buildUByte(VALUE_DELETED)) + .where(conditions) + .orderBy(defaultTable.START_TIME) + .limit(limit) + .execute(); + } + + @Override + public int deleteChatHistory(long maxStartTime, int limit) { + List conditions = new ArrayList<>(); + conditions.add(defaultTable.START_TIME.lessOrEqual(JooqDataTypeUtil.buildULong(maxStartTime))); + val idList = dslContext.select(defaultTable.ID) + .from(defaultTable) + .where(conditions) + .orderBy(defaultTable.START_TIME) + .limit(limit) + .fetch() + .map(record -> record.get(defaultTable.ID)); + deleteAnalyzeErrorContext(idList); + return deleteChatHistoryById(idList); + } + + @Override + public int deleteChatHistory(String username, long maxId, int limit) { + List conditions = new ArrayList<>(); + if (StringUtils.isNotBlank(username)) { + conditions.add(defaultTable.USERNAME.eq(username)); + } + conditions.add(defaultTable.ID.lessOrEqual(maxId)); + val idList = dslContext.select(defaultTable.ID) + .from(defaultTable) + .where(conditions) + .orderBy(defaultTable.ID) + .limit(limit) + .fetch() + .map(record -> record.get(defaultTable.ID)); + deleteAnalyzeErrorContext(idList); + return deleteChatHistoryById(idList); + } + + private void deleteAnalyzeErrorContext(List chatHistoryIdList) { + if (CollectionUtils.isEmpty(chatHistoryIdList)) { + return; + } + int deletedNum = aiAnalyzeErrorContextDAO.delete(chatHistoryIdList); + log.debug("{} AnalyzeErrorContext record deleted", deletedNum); + } + + private int deleteChatHistoryById(List idList) { + List conditions = new ArrayList<>(); + if (CollectionUtils.isEmpty(idList)) { + return 0; + } + conditions.add(defaultTable.ID.in(idList)); + return dslContext.deleteFrom(defaultTable) + .where(conditions) + .execute(); + } + + @Override + public List listAllUserOfChatHistory() { + List conditions = new ArrayList<>(); + conditions.add(defaultTable.IS_DELETED.eq(JooqDataTypeUtil.buildUByte(VALUE_NOT_DELETED))); + return dslContext.selectDistinct(defaultTable.USERNAME) + .from(defaultTable) + .where(conditions) + .fetch() + .into(String.class); + } + + @Override + public Long getFirstIdAfterOffset(String username, int offset) { + List conditions = new ArrayList<>(); + conditions.add(defaultTable.USERNAME.eq(username)); + conditions.add(defaultTable.IS_DELETED.eq(JooqDataTypeUtil.buildUByte(VALUE_NOT_DELETED))); + return dslContext.select(defaultTable.ID) + .from(defaultTable) + .where(conditions) + .orderBy(defaultTable.ID.desc()) + .offset(offset) + .limit(1) + .fetchOneInto(Long.class); + } + + private List listByConditions(Collection conditions, + Integer start, + Integer length) { + val query = dslContext.select( + defaultTable.ID, + defaultTable.USERNAME, + defaultTable.USER_INPUT, + defaultTable.PROMPT_TEMPLATE_ID, + defaultTable.AI_INPUT, + defaultTable.STATUS, + defaultTable.AI_ANSWER, + defaultTable.ERROR_CODE, + defaultTable.ERROR_MESSAGE, + defaultTable.START_TIME, + defaultTable.ANSWER_TIME, + defaultTable.TOTAL_TIME + ) + .from(defaultTable) + .where(conditions) + .orderBy(defaultTable.START_TIME.desc()); + return listPage(query, start, length, this::convertRecordToDto); + } + + private AIChatHistoryDTO convertRecordToDto(Record record) { + AIChatHistoryDTO aiChatHistoryDTO = new AIChatHistoryDTO(); + aiChatHistoryDTO.setId(record.get(defaultTable.ID)); + aiChatHistoryDTO.setUsername(record.get(defaultTable.USERNAME)); + aiChatHistoryDTO.setUserInput(record.get(defaultTable.USER_INPUT)); + aiChatHistoryDTO.setPromptTemplateId(record.get(defaultTable.PROMPT_TEMPLATE_ID)); + aiChatHistoryDTO.setAiInput(record.get(defaultTable.AI_INPUT)); + aiChatHistoryDTO.setStatus(JooqDataTypeUtil.getIntegerFromByte(record.get(defaultTable.STATUS))); + aiChatHistoryDTO.setAiAnswer(record.get(defaultTable.AI_ANSWER)); + aiChatHistoryDTO.setErrorCode(record.get(defaultTable.ERROR_CODE)); + aiChatHistoryDTO.setErrorMessage(record.get(defaultTable.ERROR_MESSAGE)); + aiChatHistoryDTO.setStartTime(JooqDataTypeUtil.buildLong(record.get(defaultTable.START_TIME))); + aiChatHistoryDTO.setAnswerTime(JooqDataTypeUtil.buildLong(record.get(defaultTable.ANSWER_TIME))); + aiChatHistoryDTO.setTotalTime(JooqDataTypeUtil.buildLong(record.get(defaultTable.TOTAL_TIME))); + return aiChatHistoryDTO; + } +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/impl/AIPromptTemplateDAOImpl.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/impl/AIPromptTemplateDAOImpl.java new file mode 100644 index 0000000000..dbf9f02477 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/impl/AIPromptTemplateDAOImpl.java @@ -0,0 +1,81 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.dao.impl; + +import com.tencent.bk.job.analysis.dao.AIPromptTemplateDAO; +import com.tencent.bk.job.analysis.model.dto.AIPromptTemplateDTO; +import com.tencent.bk.job.analysis.model.tables.AiPromptTemplate; +import com.tencent.bk.job.common.mysql.dao.BaseDAOImpl; +import lombok.extern.slf4j.Slf4j; +import lombok.val; +import org.jooq.Condition; +import org.jooq.DSLContext; +import org.jooq.Record; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Repository; + +import java.util.ArrayList; +import java.util.Collection; + +@Slf4j +@Repository +public class AIPromptTemplateDAOImpl extends BaseDAOImpl implements AIPromptTemplateDAO { + private static final AiPromptTemplate defaultTable = AiPromptTemplate.AI_PROMPT_TEMPLATE; + + private final DSLContext dslContext; + + public AIPromptTemplateDAOImpl(@Qualifier("job-analysis-dsl-context") DSLContext dslContext) { + this.dslContext = dslContext; + } + + @Override + public AIPromptTemplateDTO getAIPromptTemplate(String code, String locale) { + Collection conditions = new ArrayList<>(); + conditions.add(defaultTable.CODE.eq(code)); + conditions.add(defaultTable.LOCALE.eq(locale)); + val query = dslContext.select( + defaultTable.ID, + defaultTable.CODE, + defaultTable.LOCALE, + defaultTable.NAME, + defaultTable.RAW_PROMPT, + defaultTable.TEMPLATE + ) + .from(defaultTable) + .where(conditions); + return fetchOne(query, this::convertRecordToDto); + } + + private AIPromptTemplateDTO convertRecordToDto(Record record) { + AIPromptTemplateDTO aiPromptTemplateDTO = new AIPromptTemplateDTO(); + aiPromptTemplateDTO.setId(record.get(defaultTable.ID)); + aiPromptTemplateDTO.setCode(record.get(defaultTable.CODE)); + aiPromptTemplateDTO.setLocale(record.get(defaultTable.LOCALE)); + aiPromptTemplateDTO.setName(record.get(defaultTable.NAME)); + aiPromptTemplateDTO.setRawPrompt(record.get(defaultTable.RAW_PROMPT)); + aiPromptTemplateDTO.setTemplate(record.get(defaultTable.TEMPLATE)); + return aiPromptTemplateDTO; + } +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/impl/AnalysisTaskDAOImpl.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/impl/AnalysisTaskDAOImpl.java index 8ec789f772..6c7f025e1e 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/impl/AnalysisTaskDAOImpl.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/impl/AnalysisTaskDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,6 +26,8 @@ import com.tencent.bk.job.analysis.dao.AnalysisTaskDAO; import com.tencent.bk.job.analysis.model.dto.AnalysisTaskDTO; +import com.tencent.bk.job.analysis.model.tables.AnalysisTask; +import com.tencent.bk.job.analysis.model.tables.records.AnalysisTaskRecord; import com.tencent.bk.job.common.util.StringUtil; import lombok.extern.slf4j.Slf4j; import lombok.val; @@ -34,9 +36,9 @@ import org.jooq.DSLContext; import org.jooq.Result; import org.jooq.conf.ParamType; -import org.jooq.generated.tables.AnalysisTask; -import org.jooq.generated.tables.records.AnalysisTaskRecord; import org.jooq.types.ULong; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; import java.util.ArrayList; @@ -44,19 +46,21 @@ import java.util.Collections; import java.util.List; -/** - * @Description - * @Date 2020/3/6 - * @Version 1.0 - */ @Repository @Slf4j public class AnalysisTaskDAOImpl implements AnalysisTaskDAO { private static final AnalysisTask defaultTable = AnalysisTask.ANALYSIS_TASK; + private final DSLContext dslContext; + + @Autowired + public AnalysisTaskDAOImpl(@Qualifier("job-analysis-dsl-context") DSLContext dslContext) { + this.dslContext = dslContext; + } + @Override - public Long insertAnalysisTask(DSLContext dslContext, AnalysisTaskDTO analysisTaskDTO) { + public Long insertAnalysisTask(AnalysisTaskDTO analysisTaskDTO) { String appIdsStr = StringUtil.concatCollection(analysisTaskDTO.getAppIdList(), ","); val query = dslContext.insertInto(defaultTable, defaultTable.ID, @@ -99,7 +103,7 @@ public Long insertAnalysisTask(DSLContext dslContext, AnalysisTaskDTO analysisTa } @Override - public int updateAnalysisTaskById(DSLContext dslContext, AnalysisTaskDTO analysisTaskDTO) { + public int updateAnalysisTaskById(AnalysisTaskDTO analysisTaskDTO) { val query = dslContext.update(defaultTable) .set(defaultTable.CODE, analysisTaskDTO.getCode()) .set(defaultTable.APP_IDS, StringUtil.concatCollection(analysisTaskDTO.getAppIdList(), ",")) @@ -123,14 +127,14 @@ public int updateAnalysisTaskById(DSLContext dslContext, AnalysisTaskDTO analysi } @Override - public int deleteAnalysisTaskById(DSLContext dslContext, Long id) { + public int deleteAnalysisTaskById(Long id) { return dslContext.deleteFrom(defaultTable).where( defaultTable.ID.eq(id) ).execute(); } @Override - public AnalysisTaskDTO getAnalysisTaskById(DSLContext dslContext, Long id) { + public AnalysisTaskDTO getAnalysisTaskById(Long id) { val record = dslContext.selectFrom(defaultTable).where( defaultTable.ID.eq(id) ).fetchOne(); @@ -142,7 +146,7 @@ val record = dslContext.selectFrom(defaultTable).where( } @Override - public AnalysisTaskDTO getAnalysisTaskByCode(DSLContext dslContext, String code) { + public AnalysisTaskDTO getAnalysisTaskByCode(String code) { val record = dslContext.selectFrom(defaultTable).where( defaultTable.CODE.eq(code) ).fetchOne(); @@ -154,19 +158,18 @@ val record = dslContext.selectFrom(defaultTable).where( } @Override - public List listAllAnalysisTask(DSLContext dslContext) { - return listAnalysisTaskWithConditions(dslContext, Collections.emptyList()); + public List listAllAnalysisTask() { + return listAnalysisTaskWithConditions(Collections.emptyList()); } @Override - public List listActiveAnalysisTask(DSLContext dslContext) { + public List listActiveAnalysisTask() { List conditions = new ArrayList<>(); conditions.add(defaultTable.ACTIVE.eq(true)); - return listAnalysisTaskWithConditions(dslContext, conditions); + return listAnalysisTaskWithConditions(conditions); } - private List listAnalysisTaskWithConditions(DSLContext dslContext, - Collection conditions) { + private List listAnalysisTaskWithConditions(Collection conditions) { var query = dslContext.selectFrom(defaultTable).where( conditions //默认按照优先级排序 @@ -179,7 +182,7 @@ private List listAnalysisTaskWithConditions(DSLContext dslConte log.error(sql); throw e; } - if (records == null || records.isEmpty()) { + if (records.isEmpty()) { return Collections.emptyList(); } else { return records.map(this::convert); diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/impl/AnalysisTaskInstanceDAOImpl.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/impl/AnalysisTaskInstanceDAOImpl.java index d8650f54ba..109db9e6b7 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/impl/AnalysisTaskInstanceDAOImpl.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/impl/AnalysisTaskInstanceDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,6 +27,9 @@ import com.tencent.bk.job.analysis.dao.AnalysisTaskInstanceDAO; import com.tencent.bk.job.analysis.model.dto.AnalysisTaskInstanceDTO; import com.tencent.bk.job.analysis.model.dto.AnalysisTaskInstanceWithTpl; +import com.tencent.bk.job.analysis.model.tables.AnalysisTask; +import com.tencent.bk.job.analysis.model.tables.AnalysisTaskInstance; +import com.tencent.bk.job.analysis.model.tables.records.AnalysisTaskInstanceRecord; import com.tencent.bk.job.analysis.task.analysis.AnalysisTaskStatusEnum; import lombok.extern.slf4j.Slf4j; import lombok.val; @@ -36,11 +39,10 @@ import org.jooq.Record8; import org.jooq.Result; import org.jooq.conf.ParamType; -import org.jooq.generated.tables.AnalysisTask; -import org.jooq.generated.tables.AnalysisTaskInstance; -import org.jooq.generated.tables.records.AnalysisTaskInstanceRecord; import org.jooq.impl.DSL; import org.jooq.types.ULong; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; import java.util.ArrayList; @@ -48,11 +50,7 @@ import java.util.Collections; import java.util.List; -/** - * @Description - * @Date 2020/3/6 - * @Version 1.0 - */ + @Repository @Slf4j public class AnalysisTaskInstanceDAOImpl implements AnalysisTaskInstanceDAO { @@ -60,8 +58,16 @@ public class AnalysisTaskInstanceDAOImpl implements AnalysisTaskInstanceDAO { private static final AnalysisTaskInstance defaultTable = AnalysisTaskInstance.ANALYSIS_TASK_INSTANCE; private static final AnalysisTask tableAnalysisTask = AnalysisTask.ANALYSIS_TASK; + private final DSLContext dslContext; + + @Autowired + public AnalysisTaskInstanceDAOImpl(@Qualifier("job-analysis-dsl-context") DSLContext dslContext) { + this.dslContext = dslContext; + } + + @Override - public Long insertAnalysisTaskInstance(DSLContext dslContext, AnalysisTaskInstanceDTO analysisTaskInstanceDTO) { + public Long insertAnalysisTaskInstance(AnalysisTaskInstanceDTO analysisTaskInstanceDTO) { val query = dslContext.insertInto(defaultTable, defaultTable.ID, defaultTable.APP_ID, @@ -97,7 +103,7 @@ public Long insertAnalysisTaskInstance(DSLContext dslContext, AnalysisTaskInstan } @Override - public int updateAnalysisTaskInstanceById(DSLContext dslContext, AnalysisTaskInstanceDTO analysisTaskInstanceDTO) { + public int updateAnalysisTaskInstanceById(AnalysisTaskInstanceDTO analysisTaskInstanceDTO) { val query = dslContext.update(defaultTable) .set(defaultTable.APP_ID, analysisTaskInstanceDTO.getAppId()) .set(defaultTable.TASK_ID, analysisTaskInstanceDTO.getTaskId()) @@ -118,7 +124,7 @@ public int updateAnalysisTaskInstanceById(DSLContext dslContext, AnalysisTaskIns } @Override - public int deleteHistoryAnalysisTaskInstance(DSLContext dslContext, Long appId, Long taskId) { + public int deleteHistoryAnalysisTaskInstance(Long appId, Long taskId) { // 状态不为Running的全部删除 int notRunningCount = dslContext.deleteFrom(defaultTable).where( defaultTable.APP_ID.eq(appId) @@ -145,14 +151,14 @@ val record = dslContext.select(DSL.max(defaultTable.ID)).from(defaultTable).wher } @Override - public int deleteAnalysisTaskInstanceById(DSLContext dslContext, Long id) { + public int deleteAnalysisTaskInstanceById(Long id) { return dslContext.deleteFrom(defaultTable).where( defaultTable.ID.eq(id) ).execute(); } @Override - public AnalysisTaskInstanceDTO getAnalysisTaskInstanceById(DSLContext dslContext, Long id) { + public AnalysisTaskInstanceDTO getAnalysisTaskInstanceById(Long id) { val record = dslContext.selectFrom(defaultTable).where( defaultTable.ID.eq(id) ).fetchOne(); @@ -164,12 +170,12 @@ val record = dslContext.selectFrom(defaultTable).where( } @Override - public List listAllAnalysisTaskInstance(DSLContext dslContext) { + public List listAllAnalysisTaskInstance() { return listAnalysisTaskInstanceWithConditions(dslContext, Collections.emptyList(), null, null); } @Override - public List listActiveAnalysisTaskInstance(DSLContext dslContext, Long appId, + public List listActiveAnalysisTaskInstance(Long appId, Long limit) { List conditions = new ArrayList<>(); conditions.add(defaultTable.STATUS.eq(AnalysisTaskStatusEnum.SUCCESS.getValue())); @@ -179,7 +185,7 @@ public List listActiveAnalysisTaskInstance(DSLConte } @Override - public List listNewestActiveInstance(DSLContext dslContext, Long appId, + public List listNewestActiveInstance(Long appId, Long limit) { List conditions = new ArrayList<>(); conditions.add(defaultTable.STATUS.eq(AnalysisTaskStatusEnum.SUCCESS.getValue())); diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/impl/AnalysisTaskStaticInstanceDAOImpl.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/impl/AnalysisTaskStaticInstanceDAOImpl.java index b8a5b7576a..485bf6224b 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/impl/AnalysisTaskStaticInstanceDAOImpl.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/impl/AnalysisTaskStaticInstanceDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,6 +26,8 @@ import com.tencent.bk.job.analysis.dao.AnalysisTaskStaticInstanceDAO; import com.tencent.bk.job.analysis.model.dto.AnalysisTaskStaticInstanceDTO; +import com.tencent.bk.job.analysis.model.tables.AnalysisTaskStaticInstance; +import com.tencent.bk.job.analysis.model.tables.records.AnalysisTaskStaticInstanceRecord; import lombok.extern.slf4j.Slf4j; import lombok.val; import lombok.var; @@ -33,10 +35,9 @@ import org.jooq.DSLContext; import org.jooq.Result; import org.jooq.conf.ParamType; -import org.jooq.generated.tables.AnalysisTask; -import org.jooq.generated.tables.AnalysisTaskStaticInstance; -import org.jooq.generated.tables.records.AnalysisTaskStaticInstanceRecord; import org.jooq.types.ULong; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; import java.util.ArrayList; @@ -44,22 +45,22 @@ import java.util.List; -/** - * @Description - * @Date 2020/3/6 - * @Version 1.0 - */ @Repository @Slf4j public class AnalysisTaskStaticInstanceDAOImpl implements AnalysisTaskStaticInstanceDAO { private static final AnalysisTaskStaticInstance defaultTable = AnalysisTaskStaticInstance.ANALYSIS_TASK_STATIC_INSTANCE; - private static final AnalysisTask tableAnalysisTask = AnalysisTask.ANALYSIS_TASK; + + private final DSLContext dslContext; + + @Autowired + public AnalysisTaskStaticInstanceDAOImpl(@Qualifier("job-analysis-dsl-context") DSLContext dslContext) { + this.dslContext = dslContext; + } @Override - public Long insert(DSLContext dslContext, - AnalysisTaskStaticInstanceDTO analysisTaskStaticInstanceDTO) { + public Long insert(AnalysisTaskStaticInstanceDTO analysisTaskStaticInstanceDTO) { val query = dslContext.insertInto(defaultTable, defaultTable.ID, defaultTable.APP_ID, @@ -97,8 +98,7 @@ public Long insert(DSLContext dslContext, } @Override - public int updateById(DSLContext dslContext, - AnalysisTaskStaticInstanceDTO analysisTaskInstanceDTO) { + public int updateById(AnalysisTaskStaticInstanceDTO analysisTaskInstanceDTO) { val query = dslContext.update(defaultTable) .set(defaultTable.APP_ID, analysisTaskInstanceDTO.getAppId()) .set(defaultTable.TASK_ID, analysisTaskInstanceDTO.getTaskId()) @@ -121,8 +121,7 @@ public int updateById(DSLContext dslContext, @Override - public List listActiveInstance(DSLContext dslContext, - Long offset, Long limit) { + public List listActiveInstance(Long offset, Long limit) { List conditions = new ArrayList<>(); conditions.add(defaultTable.ACTIVE.eq(true)); var query = dslContext.selectFrom(defaultTable) @@ -142,7 +141,7 @@ public List listActiveInstance(DSLContext dslCont log.error(sql); throw e; } - if (records == null || records.isEmpty()) { + if (records.isEmpty()) { return Collections.emptyList(); } else { return records.map(this::convert); @@ -150,14 +149,14 @@ public List listActiveInstance(DSLContext dslCont } @Override - public int deleteById(DSLContext dslContext, Long id) { + public int deleteById(Long id) { return dslContext.deleteFrom(defaultTable).where( defaultTable.ID.eq(id) ).execute(); } @Override - public AnalysisTaskStaticInstanceDTO getById(DSLContext dslContext, Long id) { + public AnalysisTaskStaticInstanceDTO getById(Long id) { val record = dslContext.selectFrom(defaultTable).where( defaultTable.ID.eq(id) ).fetchOne(); diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/impl/StatisticsDAOImpl.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/impl/StatisticsDAOImpl.java index 8ca0df84fc..1c4a630ef1 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/impl/StatisticsDAOImpl.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/dao/impl/StatisticsDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,8 +24,10 @@ package com.tencent.bk.job.analysis.dao.impl; +import com.tencent.bk.job.analysis.api.dto.StatisticsDTO; import com.tencent.bk.job.analysis.dao.StatisticsDAO; -import com.tencent.bk.job.common.statistics.model.dto.StatisticsDTO; +import com.tencent.bk.job.analysis.model.tables.Statistics; +import com.tencent.bk.job.analysis.model.tables.records.StatisticsRecord; import com.tencent.bk.job.common.util.Wrapper; import lombok.extern.slf4j.Slf4j; import lombok.val; @@ -38,11 +40,10 @@ import org.jooq.Result; import org.jooq.TransactionalRunnable; import org.jooq.conf.ParamType; -import org.jooq.generated.tables.Statistics; -import org.jooq.generated.tables.records.StatisticsRecord; import org.jooq.impl.DSL; import org.jooq.types.ULong; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; import java.math.BigDecimal; @@ -51,11 +52,6 @@ import java.util.Collections; import java.util.List; -/** - * @Description - * @Date 2020/3/6 - * @Version 1.0 - */ @Repository @Slf4j public class StatisticsDAOImpl implements StatisticsDAO { @@ -64,12 +60,13 @@ public class StatisticsDAOImpl implements StatisticsDAO { private final DSLContext defaultDSLContext; @Autowired - public StatisticsDAOImpl(DSLContext dslContext) { + public StatisticsDAOImpl(@Qualifier("job-analysis-dsl-context") DSLContext dslContext) { this.defaultDSLContext = dslContext; } @Override - public Long insertStatistics(DSLContext dslContext, StatisticsDTO statisticsDTO) { + public Long insertStatistics(DSLContext dslContext, + StatisticsDTO statisticsDTO) { if (statisticsDTO == null) { return -1L; } diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/interceptor/JobAnalysisUriPermissionInterceptor.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/interceptor/JobAnalysisUriPermissionInterceptor.java new file mode 100644 index 0000000000..8e6166b49c --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/interceptor/JobAnalysisUriPermissionInterceptor.java @@ -0,0 +1,90 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.interceptor; + +import com.tencent.bk.job.analysis.consts.AnalysisConsts; +import com.tencent.bk.job.common.annotation.JobInterceptor; +import com.tencent.bk.job.common.constant.InterceptorOrder; +import com.tencent.bk.job.common.iam.constant.ActionId; +import com.tencent.bk.job.common.iam.constant.ResourceTypeEnum; +import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; +import com.tencent.bk.job.common.iam.model.AuthResult; +import com.tencent.bk.job.common.iam.service.AuthService; +import com.tencent.bk.job.common.util.JobContextUtil; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.util.AntPathMatcher; +import org.springframework.util.PathMatcher; +import org.springframework.web.servlet.handler.HandlerInterceptorAdapter; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/** + * Uri权限控制拦截 + */ +@Slf4j +@Component +@JobInterceptor(pathPatterns = {"/web/statistics/**"}, + order = InterceptorOrder.AUTH.AUTH_COMMON) +public class JobAnalysisUriPermissionInterceptor extends HandlerInterceptorAdapter { + private final String URI_PATTERN_WEB_STATISTICS = "/web/statistics/**"; + private final AuthService authService; + private final PathMatcher pathMatcher; + + @Autowired + public JobAnalysisUriPermissionInterceptor(AuthService authService) { + this.authService = authService; + this.pathMatcher = new AntPathMatcher(); + } + + @Override + public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) { + String username = JobContextUtil.getUsername(); + String uri = request.getRequestURI(); + if (pathMatcher.match(URI_PATTERN_WEB_STATISTICS, uri)) { + AuthResult authResult = authService.auth( + username, + ActionId.DASHBOARD_VIEW, + ResourceTypeEnum.DASHBOARD_VIEW, + AnalysisConsts.GLOBAL_DASHBOARD_VIEW_ID, + null + ); + if (!authResult.isPass()) { + throw new PermissionDeniedException(authResult); + } + } + return true; + } + + @Override + public void afterCompletion(HttpServletRequest request, + HttpServletResponse response, + Object handler, + Exception ex) { + + } +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/interceptor/UriPermissionInterceptor.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/interceptor/UriPermissionInterceptor.java deleted file mode 100644 index dcc2c3d411..0000000000 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/interceptor/UriPermissionInterceptor.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.analysis.interceptor; - -import com.tencent.bk.job.analysis.consts.AnalysisConsts; -import com.tencent.bk.job.common.iam.constant.ActionId; -import com.tencent.bk.job.common.iam.constant.ResourceTypeEnum; -import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; -import com.tencent.bk.job.common.iam.model.AuthResult; -import com.tencent.bk.job.common.iam.service.AuthService; -import com.tencent.bk.job.common.util.JobContextUtil; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import org.springframework.util.AntPathMatcher; -import org.springframework.util.PathMatcher; -import org.springframework.web.servlet.handler.HandlerInterceptorAdapter; - -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import java.util.Arrays; -import java.util.List; - -/** - * Uri权限控制拦截 - */ -@Slf4j -@Component -public class UriPermissionInterceptor extends HandlerInterceptorAdapter { - private final String URI_PATTERN_WEB_STATISTICS = "/web/statistics/**"; - private AuthService authService; - private PathMatcher pathMatcher; - - @Autowired - public UriPermissionInterceptor(AuthService authService) { - this.authService = authService; - this.pathMatcher = new AntPathMatcher(); - } - - public String[] getControlUriPatterns() { - Object[] rawArr = getControlUriPatternsList().toArray(); - String[] arr = new String[rawArr.length]; - for (int i = 0; i < rawArr.length; i++) { - arr[i] = (String) rawArr[i]; - } - return arr; - } - - private List getControlUriPatternsList() { - return Arrays.asList( - // 运营视图所有接口 - URI_PATTERN_WEB_STATISTICS - ); - } - - @Override - public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) - throws Exception { - String username = JobContextUtil.getUsername(); - String uri = request.getRequestURI(); - log.info("PermissionControlInterceptor.preHandle:username=" + username + ", uri=" + uri + ", " + - "controlUriPatterns=" + getControlUriPatternsList()); - if (pathMatcher.match(URI_PATTERN_WEB_STATISTICS, uri)) { - AuthResult authResult = authService.auth( - username, - ActionId.DASHBOARD_VIEW, - ResourceTypeEnum.DASHBOARD_VIEW, - AnalysisConsts.GLOBAL_DASHBOARD_VIEW_ID, - null - ); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } - } - return true; - } - - @Override - public void afterCompletion(HttpServletRequest request, HttpServletResponse response, Object handler, Exception ex) - throws Exception { - - } -} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/listener/AIChatOperationEventListener.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/listener/AIChatOperationEventListener.java new file mode 100644 index 0000000000..a80b173d45 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/listener/AIChatOperationEventListener.java @@ -0,0 +1,75 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.listener; + +import com.tencent.bk.job.analysis.consts.AIChatOperationEnum; +import com.tencent.bk.job.analysis.listener.event.AIChatOperationEvent; +import com.tencent.bk.job.analysis.service.ai.ChatService; +import lombok.extern.slf4j.Slf4j; +import org.slf4j.helpers.MessageFormatter; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +/** + * AI对话事件处理 + */ +@Component("aiChatOperationEvent") +@Slf4j +public class AIChatOperationEventListener { + + private final ChatService chatService; + + @Autowired + public AIChatOperationEventListener(ChatService chatService) { + this.chatService = chatService; + } + + /** + * 处理AI对话操作事件 + * + * @param event AI对话操作事件 + */ + public void handleEvent(AIChatOperationEvent event) { + log.info("Handle aiChatOperation event, event: {}, duration: {}ms", event, event.duration()); + String username = event.getUsername(); + long recordId = event.getRecordId(); + AIChatOperationEnum action = AIChatOperationEnum.valueOf(event.getAction()); + try { + if (action == AIChatOperationEnum.TERMINATE_CHAT) { + chatService.terminateChat(username, recordId); + } else { + log.error("Invalid event action: {}", action); + } + } catch (Throwable e) { + String errorMsg = MessageFormatter.format( + "Handle aiChatOperation event error, username={}, recordId={}", + username, + recordId + ).getMessage(); + log.error(errorMsg, e); + } + } + +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/listener/event/AIChatOperationEvent.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/listener/event/AIChatOperationEvent.java new file mode 100644 index 0000000000..3053d31fee --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/listener/event/AIChatOperationEvent.java @@ -0,0 +1,84 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.listener.event; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.tencent.bk.job.analysis.consts.AIChatOperationEnum; +import com.tencent.bk.job.common.event.Event; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import java.time.LocalDateTime; +import java.util.StringJoiner; + +/** + * AI对话操作事件 + */ +@Getter +@Setter +@NoArgsConstructor +@JsonInclude(JsonInclude.Include.NON_NULL) +public class AIChatOperationEvent extends Event { + /** + * AI对话相关操作 + * + * @see AIChatOperationEnum + */ + private int action; + /** + * 操作用户 + */ + private String username; + /** + * AI对话记录ID + */ + private Long recordId; + + /** + * 构造终止AI对话事件 + * + * @param recordId 对话记录ID + * @return 终止AI对话事件 + */ + public static AIChatOperationEvent terminateChat(String username, long recordId) { + AIChatOperationEvent event = new AIChatOperationEvent(); + event.setAction(AIChatOperationEnum.TERMINATE_CHAT.getValue()); + event.setUsername(username); + event.setRecordId(recordId); + event.setTime(LocalDateTime.now()); + return event; + } + + @Override + public String toString() { + return new StringJoiner(", ", AIChatOperationEvent.class.getSimpleName() + "[", "]") + .add("action=" + action) + .add("username=" + username) + .add("recordId=" + recordId) + .add("time=" + time) + .toString(); + } +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/dto/AIAnalyzeErrorContextDTO.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/dto/AIAnalyzeErrorContextDTO.java new file mode 100644 index 0000000000..a3fd880bdd --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/dto/AIAnalyzeErrorContextDTO.java @@ -0,0 +1,99 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.model.dto; + +import com.tencent.bk.job.analysis.model.web.req.AIAnalyzeErrorReq; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +/** + * AI分析报错信息上下文信息 + */ +@Slf4j +@Data +@EqualsAndHashCode +@NoArgsConstructor +@AllArgsConstructor +public class AIAnalyzeErrorContextDTO { + + /** + * AI对话记录ID + */ + private Long aiChatHistoryId; + + /** + * 任务ID + */ + private Long taskInstanceId; + + /** + * 步骤ID + */ + private Long stepInstanceId; + + /** + * 执行次数 + */ + private Integer executeCount; + + /** + * 滚动批次 + */ + private Integer batch; + + /** + * 执行对象类型 + */ + private Integer executeObjectType; + + /** + * 执行对象资源 ID + */ + private Long executeObjectResourceId; + + /** + * 文件任务上传下载标识,0-上传,1-下载 + */ + private Integer mode; + + public static AIAnalyzeErrorContextDTO fromAIAnalyzeErrorReq(AIAnalyzeErrorReq req) { + if (req == null) { + return null; + } + return new AIAnalyzeErrorContextDTO( + null, + req.getTaskInstanceId(), + req.getStepInstanceId(), + req.getExecuteCount(), + req.getBatch(), + req.getExecuteObjectType(), + req.getExecuteObjectResourceId(), + req.getMode() + ); + } +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/dto/AIChatHistoryDTO.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/dto/AIChatHistoryDTO.java new file mode 100644 index 0000000000..06e65b4c46 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/dto/AIChatHistoryDTO.java @@ -0,0 +1,178 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.model.dto; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.tencent.bk.job.analysis.consts.AIChatStatusEnum; +import com.tencent.bk.job.analysis.model.web.resp.AIAnswer; +import com.tencent.bk.job.analysis.model.web.resp.AIChatRecord; +import com.tencent.bk.job.analysis.model.web.resp.UserInput; +import com.tencent.bk.job.analysis.util.ai.AIAnswerUtil; +import com.tencent.bk.job.common.util.TimeUtil; +import com.tencent.bk.job.common.util.json.LongTimestampSerializer; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +/** + * AI对话历史记录 + */ +@Slf4j +@Data +@EqualsAndHashCode +@NoArgsConstructor +@AllArgsConstructor +public class AIChatHistoryDTO { + /** + * id + */ + private Long id; + + /** + * 用户名 + */ + private String username; + + /** + * Job业务ID + */ + private Long appId; + + /** + * 用户输入内容 + */ + private String userInput; + + /** + * 使用的提示符模板ID + */ + private Integer promptTemplateId; + + /** + * 提交给AI的输入内容 + */ + private String aiInput; + + /** + * AI对话状态，取值源于{@link AIChatStatusEnum}. + */ + private Integer status; + + /** + * AI回答的内容 + */ + private String aiAnswer; + + /** + * AI回答失败时的错误码 + */ + private String errorCode; + + /** + * AI回答失败时的错误信息 + */ + private String errorMessage; + + /** + * 开始时间 + */ + @JsonSerialize(using = LongTimestampSerializer.class) + private Long startTime; + + /** + * AI回答完成时间 + */ + @JsonSerialize(using = LongTimestampSerializer.class) + private Long answerTime; + + /** + * 总耗时 + */ + @JsonSerialize(using = LongTimestampSerializer.class) + private Long totalTime; + + /** + * 是否已删除：0表示未删除，1表示已删除 + */ + private Boolean isDeleted; + + /** + * 报错分析场景的上下文信息 + */ + private AIAnalyzeErrorContextDTO aiAnalyzeErrorContext; + + public void updateTotalTime() { + if (startTime != null && answerTime != null) { + totalTime = answerTime - startTime; + } + } + + public AIChatRecord toAIChatRecord() { + AIChatRecord aiChatRecord = new AIChatRecord(); + aiChatRecord.setId(id); + UserInput userInput = new UserInput(); + userInput.setContent(this.userInput); + userInput.setTime(startTime); + aiChatRecord.setUserInput(userInput); + AIAnswer aiAnswer = new AIAnswer(); + aiAnswer.setContent(this.aiAnswer); + aiAnswer.setTime(answerTime); + aiAnswer.setErrorCode(errorCode); + aiAnswer.setErrorMessage(errorMessage); + aiChatRecord.setAiAnswer(aiAnswer); + aiChatRecord.setStatus(status); + return aiChatRecord; + } + + @JsonIgnore + public boolean isInitOrReplying() { + return status == AIChatStatusEnum.INIT.getStatus() || status == AIChatStatusEnum.REPLYING.getStatus(); + } + + @JsonIgnore + public boolean isFinished() { + return status == AIChatStatusEnum.FINISHED.getStatus(); + } + + @JsonIgnore + public String getLimitedAIAnswer() { + return AIAnswerUtil.getLimitedAIAnswer(aiAnswer); + } + + public String getLimitedErrorMessage() { + return AIAnswerUtil.getLimitedErrorMessage(errorMessage); + } + + @JsonIgnore + public String getAnswerTimeStr() { + if (answerTime == null) { + return null; + } + return TimeUtil.formatTime(answerTime, "yyyy-MM-dd HH:mm:ss.SSS"); + } +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/dto/AIPromptDTO.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/dto/AIPromptDTO.java new file mode 100644 index 0000000000..7799bff285 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/dto/AIPromptDTO.java @@ -0,0 +1,53 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.model.dto; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + +/** + * AI对话历史记录 + */ +@Data +@EqualsAndHashCode +@NoArgsConstructor +@AllArgsConstructor +public class AIPromptDTO { + + /** + * 提示符模板ID + */ + private Integer promptTemplateId; + /** + * 原始提示符 + */ + private String rawPrompt; + /** + * 通过模板渲染的AI提示符 + */ + private String renderedPrompt; +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/dto/AIPromptTemplateDTO.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/dto/AIPromptTemplateDTO.java new file mode 100644 index 0000000000..7dae872632 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/dto/AIPromptTemplateDTO.java @@ -0,0 +1,98 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.model.dto; + +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.tencent.bk.job.common.util.json.LongTimestampSerializer; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + +/** + * AI提示符模板 + */ +@Data +@EqualsAndHashCode +@NoArgsConstructor +@AllArgsConstructor +public class AIPromptTemplateDTO { + /** + * id + */ + private Integer id; + + /** + * 模板代码，用于唯一标识模板 + */ + private String code; + + /** + * 语言 + */ + private String locale; + + /** + * 模板名称 + */ + private String name; + + /** + * 不含上下文的简单提示符 + */ + private String rawPrompt; + + /** + * 模板内容 + */ + private String template; + + /** + * 对模板的描述 + */ + private String description; + + /** + * 创建者 + */ + private String creator; + + /** + * 更新者 + */ + private String lastModifyUser; + + /** + * 创建时间 + */ + @JsonSerialize(using = LongTimestampSerializer.class) + private Long createTime; + + /** + * 更新时间 + */ + @JsonSerialize(using = LongTimestampSerializer.class) + private Long lastModifyTime; +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/dto/AnalysisTaskDTO.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/dto/AnalysisTaskDTO.java index fd158c0de6..2c36496393 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/dto/AnalysisTaskDTO.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/dto/AnalysisTaskDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/dto/AnalysisTaskInstanceDTO.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/dto/AnalysisTaskInstanceDTO.java index 08721dc0fa..b31d28245e 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/dto/AnalysisTaskInstanceDTO.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/dto/AnalysisTaskInstanceDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/dto/AnalysisTaskInstanceWithTpl.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/dto/AnalysisTaskInstanceWithTpl.java index 18a7c4fbc2..fcaa7c1c64 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/dto/AnalysisTaskInstanceWithTpl.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/dto/AnalysisTaskInstanceWithTpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/dto/AnalysisTaskStaticInstanceDTO.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/dto/AnalysisTaskStaticInstanceDTO.java index 4bc5bf24e6..f235d7c830 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/dto/AnalysisTaskStaticInstanceDTO.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/dto/AnalysisTaskStaticInstanceDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/dto/SimpleAppInfoDTO.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/dto/SimpleAppInfoDTO.java index 047d3b4649..47004cee4e 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/dto/SimpleAppInfoDTO.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/model/dto/SimpleAppInfoDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/mq/AIChatOperationEventDispatcher.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/mq/AIChatOperationEventDispatcher.java new file mode 100644 index 0000000000..381e7fac8e --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/mq/AIChatOperationEventDispatcher.java @@ -0,0 +1,55 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.mq; + +import com.tencent.bk.job.analysis.listener.event.AIChatOperationEvent; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cloud.stream.function.StreamBridge; +import org.springframework.stereotype.Component; + +@Component +@Slf4j +public class AIChatOperationEventDispatcher { + + private final StreamBridge streamBridge; + + @Autowired + public AIChatOperationEventDispatcher(StreamBridge streamBridge) { + this.streamBridge = streamBridge; + } + + /** + * 广播AI对话操作事件 + * + * @param event AI对话操作事件 + */ + public void broadCastAIChatOperationEvent(AIChatOperationEvent event) { + log.info("Begin to broadcast aiChatOperation event: {}", event); + String aiChatOperationFanout = "aiChatOperationFanout-out-0"; + streamBridge.send(aiChatOperationFanout, event); + log.info("Broadcast aiChatOperation event successfully, event: {}", event); + } +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/runner/StatisticsInitRunner.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/runner/StatisticsInitRunner.java index 96e9a963c2..6c71e33c76 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/runner/StatisticsInitRunner.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/runner/StatisticsInitRunner.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/AppService.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/AppService.java index 4d941429c4..b77ce269a3 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/AppService.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/AppService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,10 +27,10 @@ import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; -import com.tencent.bk.job.analysis.client.ApplicationResourceClient; import com.tencent.bk.job.analysis.model.dto.SimpleAppInfoDTO; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.exception.InternalException; +import com.tencent.bk.job.manage.api.inner.ServiceApplicationResource; import com.tencent.bk.job.manage.model.inner.resp.ServiceApplicationDTO; import lombok.extern.slf4j.Slf4j; import lombok.val; @@ -50,7 +50,7 @@ public class AppService { private static final String KEY_CACHE_APP_INFO_LIST = "KEY_CACHE_APP_INFO_LIST"; - protected final ApplicationResourceClient applicationResourceClient; + protected final ServiceApplicationResource applicationResource; private LoadingCache appIdNameCache = CacheBuilder.newBuilder() .maximumSize(3000).expireAfterWrite(30, TimeUnit.SECONDS). build(new CacheLoader() { @@ -76,8 +76,8 @@ public List load(String key) throws Exception { ); @Autowired - public AppService(ApplicationResourceClient applicationResourceClient) { - this.applicationResourceClient = applicationResourceClient; + public AppService(ServiceApplicationResource applicationResource) { + this.applicationResource = applicationResource; } public String getAppNameFromCache(Long appId) { @@ -90,7 +90,7 @@ public String getAppNameFromCache(Long appId) { } public String getAppNameById(Long appId) { - ServiceApplicationDTO serviceApplicationDTO = applicationResourceClient.queryAppById(appId); + ServiceApplicationDTO serviceApplicationDTO = applicationResource.queryAppById(appId); if (serviceApplicationDTO == null) { String msg = String.format("Cannot find appName by id=%s, app may be deleted", appId); log.warn(msg); @@ -135,7 +135,7 @@ public synchronized List listLocalDBAppsFromCache() { } public List listLocalDBApps() { - val resp = applicationResourceClient.listApps(null); + val resp = applicationResource.listApps(null); return resp.getData(); } } diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/AppStatisticService.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/AppStatisticService.java index c9c49fe65e..2ffd0212b8 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/AppStatisticService.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/AppStatisticService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,6 +25,8 @@ package com.tencent.bk.job.analysis.service; import com.fasterxml.jackson.core.type.TypeReference; +import com.tencent.bk.job.analysis.api.consts.StatisticsConstants; +import com.tencent.bk.job.analysis.api.dto.StatisticsDTO; import com.tencent.bk.job.analysis.config.StatisticConfig; import com.tencent.bk.job.analysis.dao.StatisticsDAO; import com.tencent.bk.job.analysis.model.dto.SimpleAppInfoDTO; @@ -32,8 +34,6 @@ import com.tencent.bk.job.analysis.model.web.CommonStatisticWithRateVO; import com.tencent.bk.job.analysis.model.web.CommonTrendElementVO; import com.tencent.bk.job.analysis.util.calc.AppMomYoyCalculator; -import com.tencent.bk.job.common.statistics.consts.StatisticsConstants; -import com.tencent.bk.job.common.statistics.model.dto.StatisticsDTO; import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.common.util.json.JsonUtils; import lombok.extern.slf4j.Slf4j; @@ -47,7 +47,7 @@ import java.util.stream.Collectors; @Slf4j -@Service +@Service("appStatisticService") public class AppStatisticService extends CommonStatisticService { @Autowired @@ -65,7 +65,7 @@ public AppStatisticService(StatisticsDAO statisticsDAO, StatisticConfig statisti public CommonStatisticWithRateVO calcAppMomYoyStatistic(StatisticsDTO statisticsDTO, StatisticsDTO momStatisticsDTO, StatisticsDTO yoyStatisticsDTO) { - return new AppMomYoyCalculator(statisticsDTO, momStatisticsDTO, yoyStatisticsDTO).getResult(); + return new AppMomYoyCalculator(statisticsDTO, momStatisticsDTO, yoyStatisticsDTO).calc(); } public CommonStatisticWithRateVO getAppTotalStatistics(String username, List appIdList, String date) { @@ -109,7 +109,7 @@ private List getFilteredAppDTOList(List appIdList, Stati if (appIdList != null) { Set appIdSet = new HashSet<>(appIdList); applicationDTOList = - applicationDTOList.parallelStream().filter( + applicationDTOList.stream().filter( applicationDTO -> appIdSet.contains(applicationDTO.getId()) ).collect(Collectors.toList()); } diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ApplicationService.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ApplicationService.java index bbab5efcfc..e1e0ea2128 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ApplicationService.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ApplicationService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/BaseStatisticService.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/BaseStatisticService.java index c344ae22c9..f64a153b07 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/BaseStatisticService.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/BaseStatisticService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,14 +24,18 @@ package com.tencent.bk.job.analysis.service; +import com.tencent.bk.job.analysis.api.dto.StatisticsDTO; import com.tencent.bk.job.analysis.model.web.CommonDistributionVO; import com.tencent.bk.job.analysis.model.web.DayDistributionElementVO; -import com.tencent.bk.job.common.statistics.model.dto.StatisticsDTO; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Pair; import org.springframework.stereotype.Service; -import java.util.*; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.function.Function; @Slf4j diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/BasicServiceManager.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/BasicServiceManager.java index 5e91e2409f..35526bb825 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/BasicServiceManager.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/BasicServiceManager.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/CommonStatisticService.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/CommonStatisticService.java index 4856fa9fb0..d9985a82de 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/CommonStatisticService.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/CommonStatisticService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,6 +25,8 @@ package com.tencent.bk.job.analysis.service; import com.fasterxml.jackson.core.type.TypeReference; +import com.tencent.bk.job.analysis.api.consts.StatisticsConstants; +import com.tencent.bk.job.analysis.api.dto.StatisticsDTO; import com.tencent.bk.job.analysis.config.StatisticConfig; import com.tencent.bk.job.analysis.consts.DistributionMetricEnum; import com.tencent.bk.job.analysis.consts.TotalMetricEnum; @@ -35,8 +37,6 @@ import com.tencent.bk.job.analysis.model.web.CommonStatisticWithRateVO; import com.tencent.bk.job.analysis.model.web.CommonTrendElementVO; import com.tencent.bk.job.analysis.util.calc.SimpleMomYoyCalculator; -import com.tencent.bk.job.common.statistics.consts.StatisticsConstants; -import com.tencent.bk.job.common.statistics.model.dto.StatisticsDTO; import com.tencent.bk.job.common.util.CustomCollectionUtils; import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.common.util.json.JsonUtils; @@ -52,7 +52,7 @@ import java.util.stream.Collectors; @Slf4j -@Service +@Service("commonStatisticService") public class CommonStatisticService { protected final StatisticsDAO statisticsDAO; @@ -76,7 +76,7 @@ public List getJoinedAppIdList(String date) { List applicationDTOList = JsonUtils.fromJson(statisticsDTO.getValue(), new TypeReference>() { }); - return applicationDTOList.parallelStream().map(SimpleAppInfoDTO::getId).collect(Collectors.toList()); + return applicationDTOList.stream().map(SimpleAppInfoDTO::getId).collect(Collectors.toList()); } /** @@ -87,7 +87,7 @@ public List getJoinedAppIdList(String date) { */ public CommonStatisticWithRateVO calcMomYoyStatistic(StatisticsDTO statisticsDTO, StatisticsDTO momStatisticsDTO, StatisticsDTO yoyStatisticsDTO) { - return new SimpleMomYoyCalculator(statisticsDTO, momStatisticsDTO, yoyStatisticsDTO).getResult(); + return new SimpleMomYoyCalculator(statisticsDTO, momStatisticsDTO, yoyStatisticsDTO).calc(); } public CommonStatisticWithRateVO getCommonTotalStatistics(TotalMetricEnum metric, List appIdList, diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ExecutedTaskStatisticService.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ExecutedTaskStatisticService.java index 4c8f6f5648..9adabb5540 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ExecutedTaskStatisticService.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ExecutedTaskStatisticService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,10 +24,10 @@ package com.tencent.bk.job.analysis.service; +import com.tencent.bk.job.analysis.api.consts.StatisticsConstants; +import com.tencent.bk.job.analysis.api.dto.StatisticsDTO; import com.tencent.bk.job.analysis.dao.StatisticsDAO; import com.tencent.bk.job.analysis.model.web.DayDistributionElementVO; -import com.tencent.bk.job.common.statistics.consts.StatisticsConstants; -import com.tencent.bk.job.common.statistics.model.dto.StatisticsDTO; import com.tencent.bk.job.common.util.date.DateUtils; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/FastFileStatisticService.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/FastFileStatisticService.java index 85c21787be..67a3920253 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/FastFileStatisticService.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/FastFileStatisticService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,11 +24,11 @@ package com.tencent.bk.job.analysis.service; +import com.tencent.bk.job.analysis.api.consts.StatisticsConstants; +import com.tencent.bk.job.analysis.api.dto.StatisticsDTO; import com.tencent.bk.job.analysis.config.StatisticConfig; import com.tencent.bk.job.analysis.dao.StatisticsDAO; import com.tencent.bk.job.analysis.model.web.DayDistributionElementVO; -import com.tencent.bk.job.common.statistics.consts.StatisticsConstants; -import com.tencent.bk.job.common.statistics.model.dto.StatisticsDTO; import com.tencent.bk.job.common.util.date.DateUtils; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/FastScriptStatisticService.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/FastScriptStatisticService.java index e4cafba34f..199311b395 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/FastScriptStatisticService.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/FastScriptStatisticService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,11 +24,11 @@ package com.tencent.bk.job.analysis.service; +import com.tencent.bk.job.analysis.api.consts.StatisticsConstants; +import com.tencent.bk.job.analysis.api.dto.StatisticsDTO; import com.tencent.bk.job.analysis.config.StatisticConfig; import com.tencent.bk.job.analysis.dao.StatisticsDAO; import com.tencent.bk.job.analysis.model.web.DayDistributionElementVO; -import com.tencent.bk.job.common.statistics.consts.StatisticsConstants; -import com.tencent.bk.job.common.statistics.model.dto.StatisticsDTO; import com.tencent.bk.job.common.util.date.DateUtils; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/HostService.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/HostService.java index f1790aa5d2..b0d5b3b0dc 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/HostService.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/HostService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,16 +24,18 @@ package com.tencent.bk.job.analysis.service; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.manage.model.inner.ServiceHostStatusDTO; -import com.tencent.bk.job.manage.model.web.request.ipchooser.AppTopologyTreeNode; +import com.tencent.bk.job.manage.model.web.request.chooser.host.BizTopoNode; import java.util.List; public interface HostService { - List getHostStatusByNode(Long appId, List treeNodeList); + List getHostStatusByNode(Long appId, List treeNodeList); List getHostStatusByDynamicGroup(Long appId, List dynamicGroupIdList); - List getHostStatusByIp(Long appId, List ipList); + List getHostStatusByHost(Long appId, + List hostList); } diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/HostStatisticService.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/HostStatisticService.java index d6780771de..8ddd4f79a4 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/HostStatisticService.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/HostStatisticService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,11 +24,11 @@ package com.tencent.bk.job.analysis.service; +import com.tencent.bk.job.analysis.api.consts.StatisticsConstants; +import com.tencent.bk.job.analysis.api.dto.StatisticsDTO; import com.tencent.bk.job.analysis.config.StatisticConfig; import com.tencent.bk.job.analysis.dao.StatisticsDAO; import com.tencent.bk.job.analysis.model.web.CommonDistributionVO; -import com.tencent.bk.job.common.statistics.consts.StatisticsConstants; -import com.tencent.bk.job.common.statistics.model.dto.StatisticsDTO; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/IndexService.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/IndexService.java index d3cca2d0e6..a5897354d5 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/IndexService.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/IndexService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,6 @@ package com.tencent.bk.job.analysis.service; -import com.tencent.bk.job.analysis.client.GlobalSettingsResourceClient; import com.tencent.bk.job.analysis.dao.AnalysisTaskInstanceDAO; import com.tencent.bk.job.analysis.dao.AnalysisTaskStaticInstanceDAO; import com.tencent.bk.job.analysis.model.inner.AnalysisTaskResultItemLocation; @@ -39,9 +38,9 @@ import com.tencent.bk.job.common.i18n.service.MessageI18nService; import com.tencent.bk.job.common.util.JobContextUtil; import com.tencent.bk.job.common.util.StringUtil; +import com.tencent.bk.job.manage.api.inner.ServiceGlobalSettingsResource; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -import org.jooq.DSLContext; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -56,26 +55,25 @@ @Service public class IndexService { - private final DSLContext dslContext; private final AnalysisTaskInstanceDAO analysisTaskInstanceDAO; private final AnalysisTaskStaticInstanceDAO analysisTaskStaticInstanceDAO; - private final GlobalSettingsResourceClient globalSettingsService; + private final ServiceGlobalSettingsResource globalSettingsResource; private final MessageI18nService i18nService; @Autowired - public IndexService(DSLContext dslContext, AnalysisTaskInstanceDAO analysisTaskInstanceDAO, + public IndexService(AnalysisTaskInstanceDAO analysisTaskInstanceDAO, AnalysisTaskStaticInstanceDAO analysisTaskStaticInstanceDAO, - GlobalSettingsResourceClient globalSettingsService, MessageI18nService i18nService) { - this.dslContext = dslContext; + ServiceGlobalSettingsResource globalSettingsResource, + MessageI18nService i18nService) { this.analysisTaskInstanceDAO = analysisTaskInstanceDAO; this.analysisTaskStaticInstanceDAO = analysisTaskStaticInstanceDAO; - this.globalSettingsService = globalSettingsService; + this.globalSettingsResource = globalSettingsResource; this.i18nService = i18nService; } private Map getVariablesMap() { Map variablesMap = new HashMap<>(); - variablesMap.put("BK_DOCS_CENTER", globalSettingsService.getDocCenterBaseUrl().getData()); + variablesMap.put("BK_DOC_JOB_ROOT_URL", globalSettingsResource.getDocJobRootUrl().getData()); return variablesMap; } @@ -88,7 +86,7 @@ private String parseVariables(String rawTemplate) { public List listAnalysisResult(String username, Long appId, Long limit) { String normalLang = LocaleUtils.getNormalLang(JobContextUtil.getUserLang()); //业务专属分析结果 - List resultList = analysisTaskInstanceDAO.listNewestActiveInstance(dslContext, + List resultList = analysisTaskInstanceDAO.listNewestActiveInstance( appId, limit).stream().map(it -> { //根据任务代码找到对应的任务类组装结果 IAnalysisTask analysisTask = AnalysisTaskScheduler.analysisTaskMap.get(it.getTaskCode()); @@ -124,7 +122,7 @@ public List listAnalysisResult(String username, Long appId, Lo } descriptionTemplate = parseVariables(descriptionTemplate); itemTemplate = parseVariables(itemTemplate); - AnalysisTaskResultVO taskResultVO = analysisTask.generateResultVO(descriptionTemplate, itemTemplate, + AnalysisTaskResultVO taskResultVO = analysisTask.renderResultVO(descriptionTemplate, itemTemplate, resultData); if (taskResultVO == null) { return null; @@ -141,7 +139,7 @@ public List listAnalysisResult(String username, Long appId, Lo }).filter(Objects::nonNull).collect(Collectors.toList()); // 静态文案类分析结果 List tipsResultList = - analysisTaskStaticInstanceDAO.listActiveInstance(dslContext, 0L, limit).stream().map(it -> { + analysisTaskStaticInstanceDAO.listActiveInstance(0L, limit).stream().map(it -> { String resultData; // 国际化处理 if (normalLang.equals(LocaleUtils.LANG_EN) || normalLang.equals(LocaleUtils.LANG_EN_US)) { diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/MeasureServiceImpl.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/MeasureServiceImpl.java index ee2efefa8c..99ce1e5394 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/MeasureServiceImpl.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/MeasureServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,8 +24,8 @@ package com.tencent.bk.job.analysis.service; +import com.tencent.bk.job.analysis.api.consts.StatisticsConstants; import com.tencent.bk.job.analysis.task.statistics.StatisticsTaskScheduler; -import com.tencent.bk.job.common.statistics.consts.StatisticsConstants; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Tag; import lombok.extern.slf4j.Slf4j; @@ -35,7 +35,7 @@ import java.util.Collections; @Slf4j -@Service +@Service("jobAnalysisMeasureService") public class MeasureServiceImpl { @Autowired diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/MetricResourceReslover.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/MetricResourceReslover.java index 50c423ccd7..a7f2276401 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/MetricResourceReslover.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/MetricResourceReslover.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,10 +24,10 @@ package com.tencent.bk.job.analysis.service; +import com.tencent.bk.job.analysis.api.consts.StatisticsConstants; import com.tencent.bk.job.analysis.consts.DistributionMetricEnum; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.exception.InvalidParamException; -import com.tencent.bk.job.common.statistics.consts.StatisticsConstants; import org.springframework.stereotype.Service; @Service diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ResourceNameQueryServiceImpl.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ResourceNameQueryServiceImpl.java index f476714a50..2257ac40e5 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ResourceNameQueryServiceImpl.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ResourceNameQueryServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -34,7 +34,7 @@ import org.springframework.stereotype.Service; @Slf4j -@Service("ResourceNameQueryService") +@Service("jobAnalysisResourceNameQueryService") public class ResourceNameQueryServiceImpl implements ResourceNameQueryService { @Autowired diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/RollingTaskStatisticService.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/RollingTaskStatisticService.java new file mode 100644 index 0000000000..b423ad4bf0 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/RollingTaskStatisticService.java @@ -0,0 +1,151 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.service; + +import com.tencent.bk.job.analysis.api.consts.StatisticsConstants; +import com.tencent.bk.job.analysis.api.dto.StatisticsDTO; +import com.tencent.bk.job.analysis.dao.StatisticsDAO; +import com.tencent.bk.job.analysis.model.web.CommonDistributionVO; +import com.tencent.bk.job.analysis.model.web.DayDistributionElementVO; +import com.tencent.bk.job.common.util.date.DateUtils; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Slf4j +@Service +public class RollingTaskStatisticService extends BaseStatisticService { + + private final StatisticsDAO statisticsDAO; + private final ExecutedTaskStatisticService executedTaskStatisticService; + + @Autowired + public RollingTaskStatisticService(StatisticsDAO statisticsDAO, + ExecutedTaskStatisticService executedTaskStatisticService) { + super(); + this.statisticsDAO = statisticsDAO; + this.executedTaskStatisticService = executedTaskStatisticService; + } + + private List getRollingTaskFailedStatisticsDTOList(List appIdList, String startDate, String endDate) { + List failedStatisticsDTOList; + if (appIdList == null) { + // 全局指标 + failedStatisticsDTOList = + statisticsDAO.getStatisticsListBetweenDate( + Collections.singletonList(StatisticsConstants.DEFAULT_APP_ID), + null, + StatisticsConstants.RESOURCE_ROLLING_FAILED_TASK, + StatisticsConstants.DIMENSION_TIME_UNIT, + StatisticsConstants.DIMENSION_VALUE_TIME_UNIT_DAY, + startDate, + endDate + ); + if (failedStatisticsDTOList == null || failedStatisticsDTOList.isEmpty()) { + failedStatisticsDTOList = statisticsDAO.getStatisticsListBetweenDate(appIdList, null, + StatisticsConstants.RESOURCE_ROLLING_FAILED_TASK, StatisticsConstants.DIMENSION_TIME_UNIT, + StatisticsConstants.DIMENSION_VALUE_TIME_UNIT_DAY, startDate, endDate); + } + } else { + // 按业务聚合 + failedStatisticsDTOList = statisticsDAO.getStatisticsListBetweenDate(appIdList, + StatisticsConstants.GLOBAL_APP_ID_LIST, StatisticsConstants.RESOURCE_ROLLING_FAILED_TASK, + StatisticsConstants.DIMENSION_TIME_UNIT, StatisticsConstants.DIMENSION_VALUE_TIME_UNIT_DAY, startDate + , endDate); + } + log.debug("RollingTaskFailedStatisticsDTOList={}", failedStatisticsDTOList); + return failedStatisticsDTOList; + } + + public List rollingTaskByTaskTypeDayDetail(List appIdList, String startDate, + String endDate) { + // 滚动执行统计 + List rollingTaskDayDetailList = getByTaskTypeDayDetail(appIdList, startDate, endDate); + // 任务执行统计 + List totalTaskDayDetailList = + executedTaskStatisticService.getByTaskTypeDayDetail(appIdList, startDate, endDate); + + // 将任务执行统计(总执行次数)合入到对应的滚动执行统计中,key以'_TOTAL'结尾 + List mixTaskDayDetailList = new ArrayList<>(); + for (int i = 0; i < rollingTaskDayDetailList.size(); i++) { + DayDistributionElementVO rollingEleVO = rollingTaskDayDetailList.get(i); + CommonDistributionVO rollingVO = rollingEleVO.getDistribution(); + Map rollingMap = rollingVO.getLabelAmountMap(); + Map mixMap = new HashMap<>(); + for (String key : rollingMap.keySet()) { + mixMap.put(key, rollingMap.get(key)); + try { + DayDistributionElementVO elementVO = totalTaskDayDetailList.get(i); + CommonDistributionVO distributionVO = elementVO.getDistribution(); + Map amountMap = distributionVO.getLabelAmountMap(); + long total = amountMap.get(key); + mixMap.put(key + "_TOTAL", total); + } catch (Exception e) { + mixMap.put(key + "_TOTAL", 0L); + log.error("taskTypeDayDetail may be empty!", e); + } + } + rollingVO.setLabelAmountMap(mixMap); + rollingEleVO.setDistribution(rollingVO); + mixTaskDayDetailList.add(rollingEleVO); + } + return mixTaskDayDetailList; + } + + private List getByTaskTypeDayDetail(List appIdList, String startDate, String endDate) { + List statisticsDTOList; + if (appIdList == null) { + // 全部业务，直接拿离线聚合后的数据 + statisticsDTOList = statisticsDAO.getStatisticsListBetweenDate(appIdList, null, + StatisticsConstants.RESOURCE_ONE_DAY_ROLLING_TASK_OF_ALL_APP, + StatisticsConstants.DIMENSION_TASK_TYPE, startDate, endDate); + if (statisticsDTOList == null + || statisticsDTOList.size() < DateUtils.calcDaysBetween(startDate, endDate) + 1) { + log.info("offline data not ready, calc in mem, startDate={}, endDate={}", startDate, endDate); + // 离线聚合数据暂未统计完成 + statisticsDTOList = statisticsDAO.getStatisticsListBetweenDate(appIdList, + Collections.singletonList(StatisticsConstants.DEFAULT_APP_ID), + StatisticsConstants.RESOURCE_ROLLING_TASK, StatisticsConstants.DIMENSION_TASK_TYPE, startDate, + endDate); + } + } else { + statisticsDTOList = statisticsDAO.getStatisticsListBetweenDate(appIdList, + Collections.singletonList(StatisticsConstants.DEFAULT_APP_ID), + StatisticsConstants.RESOURCE_ROLLING_TASK, StatisticsConstants.DIMENSION_TASK_TYPE, startDate, + endDate); + } + log.debug("statisticsDTOList={}", statisticsDTOList); + List dayDistributionElementVOList = groupByDateAndDimensionValue(statisticsDTOList, + getRollingTaskFailedStatisticsDTOList(appIdList, startDate, endDate)); + log.debug("dayDistributionElementVOList={}", dayDistributionElementVOList); + return dayDistributionElementVOList; + } +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ScriptService.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ScriptService.java index caec3e4d30..20eeb75185 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ScriptService.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ScriptService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ScriptStatisticService.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ScriptStatisticService.java index 7c60dbaa39..20001d023e 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ScriptStatisticService.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ScriptStatisticService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,11 +24,11 @@ package com.tencent.bk.job.analysis.service; +import com.tencent.bk.job.analysis.api.consts.StatisticsConstants; +import com.tencent.bk.job.analysis.api.dto.StatisticsDTO; import com.tencent.bk.job.analysis.config.StatisticConfig; import com.tencent.bk.job.analysis.dao.StatisticsDAO; import com.tencent.bk.job.analysis.model.web.ScriptCiteStatisticVO; -import com.tencent.bk.job.common.statistics.consts.StatisticsConstants; -import com.tencent.bk.job.common.statistics.model.dto.StatisticsDTO; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/TagStatisticService.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/TagStatisticService.java index cde280943c..16ef7f8c1a 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/TagStatisticService.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/TagStatisticService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,11 +25,11 @@ package com.tencent.bk.job.analysis.service; import com.fasterxml.jackson.core.type.TypeReference; +import com.tencent.bk.job.analysis.api.consts.StatisticsConstants; +import com.tencent.bk.job.analysis.api.dto.StatisticsDTO; import com.tencent.bk.job.analysis.config.StatisticConfig; import com.tencent.bk.job.analysis.dao.StatisticsDAO; import com.tencent.bk.job.analysis.model.web.CommonDistributionVO; -import com.tencent.bk.job.common.statistics.consts.StatisticsConstants; -import com.tencent.bk.job.common.statistics.model.dto.StatisticsDTO; import com.tencent.bk.job.common.util.json.JsonUtils; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Pair; diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/TaskPlanService.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/TaskPlanService.java index 8440503b5a..b508add207 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/TaskPlanService.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/TaskPlanService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/TaskTemplateService.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/TaskTemplateService.java index a9158c3d44..598f3bac14 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/TaskTemplateService.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/TaskTemplateService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/AIAnalyzeErrorService.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/AIAnalyzeErrorService.java new file mode 100644 index 0000000000..5eceba576e --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/AIAnalyzeErrorService.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.service.ai; + +import com.tencent.bk.job.analysis.model.web.req.AIAnalyzeErrorReq; +import com.tencent.bk.job.analysis.model.web.resp.AIChatRecord; + +/** + * AI分析报错信息服务 + */ +public interface AIAnalyzeErrorService { + + /** + * AI分析报错信息 + * + * @param username 用户名 + * @param appId Job业务ID + * @param req 请求内容 + * @return AI对话记录 + */ + AIChatRecord analyze(String username, Long appId, AIAnalyzeErrorReq req); +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/AIChatHistoryService.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/AIChatHistoryService.java new file mode 100644 index 0000000000..368163b4c2 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/AIChatHistoryService.java @@ -0,0 +1,133 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.service.ai; + +import com.tencent.bk.job.analysis.model.dto.AIChatHistoryDTO; +import com.tencent.bk.job.analysis.model.dto.AIPromptDTO; +import com.tencent.bk.job.analysis.model.web.resp.AIAnswer; + +import java.util.List; + +/** + * AI对话记录服务 + */ +public interface AIChatHistoryService { + /** + * 构建AI对话记录 + * + * @param username 用户名 + * @param appId Job业务ID + * @param startTime 开始时间 + * @param aiPromptDTO AI提示符信息 + * @param status 对话状态 + * @param aiAnswer AI回答 + * @return AI对话记录 + */ + AIChatHistoryDTO buildAIChatHistoryDTO(String username, + Long appId, + Long startTime, + AIPromptDTO aiPromptDTO, + Integer status, + AIAnswer aiAnswer); + + /** + * 插入对话记录 + * + * @param aiChatHistoryDTO AI对话记录 + * @return 插入记录的id + */ + Long insertChatHistory(AIChatHistoryDTO aiChatHistoryDTO); + + /** + * 判断用户是否存在对话记录 + * + * @param username 用户名 + * @return 是否存在对话记录 + */ + boolean existsChatHistory(String username); + + /** + * 更新对话记录状态为正在回答中 + * + * @param historyId AI对话记录ID + * @return 受影响的行数 + */ + int setAIAnswerReplying(Long historyId); + + /** + * 更新对话记录状态 + * + * @param historyId AI对话记录ID + * @param aiAnswer AI回答内容 + * @return 受影响的行数 + */ + int finishAIAnswer(Long historyId, AIAnswer aiAnswer); + + /** + * 终止对话 + * + * @param historyId AI对话记录ID + * @return 受影响的行数 + */ + int terminateAIAnswer(Long historyId); + + + /** + * 获取最近的对话记录列表 + * + * @param username 用户名 + * @param start 起始位置 + * @param length 长度 + * @return 最近的对话记录列表 + */ + List getLatestChatHistoryList(String username, Integer start, Integer length); + + /** + * 获取最近已完成的对话记录列表 + * + * @param username 用户名 + * @param start 起始位置 + * @param length 长度 + * @return 最近的对话记录列表 + */ + List getLatestFinishedChatHistoryList(String username, Integer start, Integer length); + + /** + * 根据用户名与ID获取对话记录 + * + * @param username 用户名 + * @param id ID + * @return 对话记录 + */ + AIChatHistoryDTO getChatHistory(String username, Long id); + + /** + * 软删除对话记录 + * + * @param username 用户名 + * @return 删除的对话记录数量 + */ + int softDeleteChatHistory(String username); +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/AICheckScriptService.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/AICheckScriptService.java new file mode 100644 index 0000000000..f8146834f6 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/AICheckScriptService.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.service.ai; + +import com.tencent.bk.job.analysis.model.web.resp.AIChatRecord; + +/** + * AI脚本检查服务 + */ +public interface AICheckScriptService { + + /** + * 检查脚本 + * + * @param username 用户名 + * @param appId Job业务ID + * @param type 脚本类型 + * @param scriptContent 脚本内容 + * @return AI对话记录 + */ + AIChatRecord check(String username, Long appId, Integer type, String scriptContent); +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/AIService.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/AIService.java new file mode 100644 index 0000000000..d9b9512d50 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/AIService.java @@ -0,0 +1,49 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.service.ai; + +import com.tencent.bk.job.analysis.model.dto.AIChatHistoryDTO; + +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.function.Consumer; + +/** + * AI核心能力服务 + */ +public interface AIService { + + /** + * 获取AI回答流（流式接口） + * + * @param chatHistoryDTOList 历史对话记录 + * @param userInput 用户输入 + * @param partialRespConsumer AI回答流回调 + * @return AI回答结果Future + */ + CompletableFuture getAIAnswerStream(List chatHistoryDTOList, + String userInput, + Consumer partialRespConsumer); +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/ChatService.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/ChatService.java new file mode 100644 index 0000000000..4695de0017 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/ChatService.java @@ -0,0 +1,84 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.service.ai; + +import com.tencent.bk.job.analysis.model.dto.AIChatHistoryDTO; +import com.tencent.bk.job.analysis.model.web.resp.AIChatRecord; +import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; + +import java.util.List; + +/** + * AI对话服务 + */ +public interface ChatService { + + /** + * 与AI对话并处理对话记录保存等逻辑 + * + * @param username 用户名 + * @param appId Job业务ID + * @param userInput 用户输入 + * @return AI对话记录 + */ + AIChatRecord chatWithAI(String username, Long appId, String userInput); + + /** + * 获取最近的对话记录列表 + * + * @param username 用户名 + * @param start 起始位置 + * @param length 长度 + * @return 最近的对话记录列表 + */ + List getLatestChatHistoryList(String username, Integer start, Integer length); + + /** + * 获取对话流式数据 + * + * @param username 用户名 + * @param recordId 对话记录ID + * @return 流式数据 + */ + StreamingResponseBody generateChatStream(String username, Long recordId); + + /** + * 终止当前实例中的对话 + * + * @param username 用户名 + * @param recordId 对话记录ID + * @return 是否终止成功 + */ + boolean terminateChat(String username, Long recordId); + + /** + * 触发终止对话 + * + * @param username 用户名 + * @param recordId 对话记录ID + * @return 是否终止成功 + */ + boolean triggerTerminateChat(String username, Long recordId); +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/CheckScriptAIPromptService.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/CheckScriptAIPromptService.java new file mode 100644 index 0000000000..8182e200d7 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/CheckScriptAIPromptService.java @@ -0,0 +1,42 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.service.ai; + +import com.tencent.bk.job.analysis.model.dto.AIPromptDTO; + +/** + * 检查脚本的AI提示符服务 + */ +public interface CheckScriptAIPromptService { + + /** + * 获取检查脚本的AI提示符 + * + * @param type 脚本类型 + * @param scriptContent 脚本内容 + * @return AI提示符 + */ + AIPromptDTO getPrompt(Integer type, String scriptContent); +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/FileTransferTaskErrorAIPromptService.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/FileTransferTaskErrorAIPromptService.java new file mode 100644 index 0000000000..ff6f8cf039 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/FileTransferTaskErrorAIPromptService.java @@ -0,0 +1,42 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.service.ai; + +import com.tencent.bk.job.analysis.model.dto.AIPromptDTO; +import com.tencent.bk.job.analysis.service.ai.context.model.FileTaskContext; + +/** + * 文件分发任务报错信息AI提示符服务 + */ +public interface FileTransferTaskErrorAIPromptService { + + /** + * 获取分析文件分发任务报错信息的AI提示符 + * + * @param context 文件分发任务上下文 + * @return AI提示符 + */ + AIPromptDTO getPrompt(FileTaskContext context); +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/ScriptExecuteTaskErrorAIPromptService.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/ScriptExecuteTaskErrorAIPromptService.java new file mode 100644 index 0000000000..d1b8afe29e --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/ScriptExecuteTaskErrorAIPromptService.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.service.ai; + +import com.tencent.bk.job.analysis.model.dto.AIPromptDTO; +import com.tencent.bk.job.analysis.service.ai.context.model.ScriptTaskContext; + +/** + * 脚本执行任务报错信息的AI提示符服务 + */ +public interface ScriptExecuteTaskErrorAIPromptService { + + /** + * 获取分析脚本执行任务报错信息的AI提示符 + * + * @param context 脚本任务上下文 + * @param errorContent 报错内容 + * @return AI提示符 + */ + AIPromptDTO getPrompt(ScriptTaskContext context, String errorContent); + +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/TaskContextService.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/TaskContextService.java new file mode 100644 index 0000000000..c5f56f001f --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/TaskContextService.java @@ -0,0 +1,43 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.service.ai.context; + +import com.tencent.bk.job.analysis.service.ai.context.model.TaskContext; +import com.tencent.bk.job.analysis.service.ai.context.model.TaskContextQuery; + +/** + * 任务上下文服务接口 + */ +public interface TaskContextService { + + /** + * 获取任务上下文 + * + * @param username 用户名 + * @param contextQuery 任务上下文查询条件 + * @return 任务上下文 + */ + TaskContext getTaskContext(String username, TaskContextQuery contextQuery); +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/constants/FileTaskErrorSourceEnum.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/constants/FileTaskErrorSourceEnum.java new file mode 100644 index 0000000000..bde28218b2 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/constants/FileTaskErrorSourceEnum.java @@ -0,0 +1,65 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.service.ai.context.constants; + +import lombok.Getter; + +/** + * 文件任务失败原因来源枚举 + */ +@Getter +public enum FileTaskErrorSourceEnum { + NO_ERROR( + "job.analysis.ai.fileTaskErrorSource.noError", + "任务未失败" + ), + SOURCE_FILE_UPLOAD_ERROR( + "job.analysis.ai.fileTaskErrorSource.sourceFileUploadError", + "源文件上传出错导致的任务失败" + ), + DOWNLOAD_ERROR( + "job.analysis.ai.fileTaskErrorSource.downloadError", + "目标执行对象下载文件出错导致的任务失败" + ), + UPLOAD_AND_DOWNLOAD_ERROR( + "job.analysis.ai.fileTaskErrorSource.uploadAndDownloadError", + "源文件上传与目标执行对象下载文件均出错导致的任务失败" + ); + + /** + * 任务失败原因描述国际化key + */ + private final String i18nKey; + + /** + * 任务失败原因描述 + */ + private final String description; + + FileTaskErrorSourceEnum(String i18nKey, String description) { + this.i18nKey = i18nKey; + this.description = description; + } +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/impl/FileTaskContextService.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/impl/FileTaskContextService.java new file mode 100644 index 0000000000..1cee0b9669 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/impl/FileTaskContextService.java @@ -0,0 +1,182 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.service.ai.context.impl; + +import com.tencent.bk.job.analysis.service.ai.context.model.FileTaskContext; +import com.tencent.bk.job.analysis.service.ai.context.model.FileTaskErrorSourceResult; +import com.tencent.bk.job.analysis.service.ai.context.model.TaskContext; +import com.tencent.bk.job.analysis.service.ai.context.model.TaskContextQuery; +import com.tencent.bk.job.common.exception.ServiceException; +import com.tencent.bk.job.common.gse.constants.FileDistModeEnum; +import com.tencent.bk.job.common.model.InternalResponse; +import com.tencent.bk.job.common.model.error.ErrorType; +import com.tencent.bk.job.execute.common.constants.FileDistStatusEnum; +import com.tencent.bk.job.execute.model.inner.ServiceStepInstanceDTO; +import com.tencent.bk.job.logsvr.api.ServiceLogResource; +import com.tencent.bk.job.logsvr.consts.FileTaskModeEnum; +import com.tencent.bk.job.logsvr.model.service.ServiceExecuteObjectLogDTO; +import com.tencent.bk.job.logsvr.model.service.ServiceFileLogQueryRequest; +import com.tencent.bk.job.logsvr.model.service.ServiceFileTaskLogDTO; +import com.tencent.bk.job.logsvr.util.LogFieldUtil; +import org.apache.commons.collections.CollectionUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.List; + +/** + * 文件任务上下文服务 + */ +@Service +public class FileTaskContextService { + + private final ServiceLogResource logResource; + private final FileTaskFailLogAnalyzer fileTaskFailLogAnalyzer; + + @Autowired + public FileTaskContextService(ServiceLogResource logResource, + FileTaskFailLogAnalyzer fileTaskFailLogAnalyzer) { + this.logResource = logResource; + this.fileTaskFailLogAnalyzer = fileTaskFailLogAnalyzer; + } + + /** + * 根据步骤实例与上下文查询条件获取对应的任务上下文 + * + * @param stepInstance 步骤实例 + * @param contextQuery 上下文查询条件 + * @return 任务上下文 + */ + public TaskContext getTaskContext(ServiceStepInstanceDTO stepInstance, TaskContextQuery contextQuery) { + String jobCreateDate = LogFieldUtil.buildJobCreateDate(stepInstance.getCreateTime()); + // 上传日志 + ServiceFileLogQueryRequest request = new ServiceFileLogQueryRequest(); + request.setJobCreateDate(jobCreateDate); + request.setStepInstanceId(contextQuery.getStepInstanceId()); + request.setExecuteCount(contextQuery.getExecuteCount()); + request.setMode(FileDistModeEnum.UPLOAD.getValue()); + request.setBatch(contextQuery.getBatch()); + InternalResponse> uploadLogResp = logResource.listFileExecuteObjectLogs( + jobCreateDate, + contextQuery.getStepInstanceId(), + contextQuery.getExecuteCount(), + request + ); + if (!uploadLogResp.isSuccess()) { + throw new ServiceException( + uploadLogResp.getErrorMsg(), + ErrorType.valOf(uploadLogResp.getErrorType()), + uploadLogResp.getCode() + ); + } + List uploadExecuteObjectLogList = uploadLogResp.getData(); + // 下载日志 + InternalResponse downloadLogResp = logResource.getFileLogByExecuteObjectId( + jobCreateDate, + contextQuery.getStepInstanceId(), + contextQuery.getExecuteCount(), + contextQuery.getExecuteObjectId(), + FileDistModeEnum.DOWNLOAD.getValue(), + contextQuery.getBatch() + ); + if (!downloadLogResp.isSuccess()) { + throw new ServiceException( + downloadLogResp.getErrorMsg(), + ErrorType.valOf(downloadLogResp.getErrorType()), + downloadLogResp.getCode() + ); + } + ServiceExecuteObjectLogDTO downloadExecuteObjectLog = downloadLogResp.getData(); + return buildContextForFileTask(stepInstance, uploadExecuteObjectLogList, downloadExecuteObjectLog); + } + + /** + * 构建文件任务上下文 + * + * @param stepInstance 步骤实例 + * @param uploadExecuteObjectLogList 上传执行对象日志 + * @param downloadExecuteObjectLog 下载执行对象日志 + * @return 任务上下文 + */ + private TaskContext buildContextForFileTask(ServiceStepInstanceDTO stepInstance, + List uploadExecuteObjectLogList, + ServiceExecuteObjectLogDTO downloadExecuteObjectLog) { + List fileTaskLogs = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(uploadExecuteObjectLogList)) { + uploadExecuteObjectLogList.forEach(uploadExecuteObjectLog -> { + List uploadFileTaskLogs = uploadExecuteObjectLog.getFileTaskLogs(); + fileTaskLogs.addAll(uploadFileTaskLogs); + }); + } + List downloadFileTaskLogs = downloadExecuteObjectLog.getFileTaskLogs(); + if (CollectionUtils.isNotEmpty(downloadFileTaskLogs)) { + fileTaskLogs.addAll(downloadFileTaskLogs); + } + List uploadFailLogs = new ArrayList<>(); + List downloadFailLogs = new ArrayList<>(); + for (ServiceFileTaskLogDTO fileTaskLog : fileTaskLogs) { + if (isUploadFailLog(fileTaskLog)) { + uploadFailLogs.add(fileTaskLog); + } else if (isDownloadFailLog(fileTaskLog)) { + downloadFailLogs.add(fileTaskLog); + } + } + // 对上传失败和下载失败的日志进行分析，确定导致任务失败的主要原因 + FileTaskErrorSourceResult result = fileTaskFailLogAnalyzer.analyze(uploadFailLogs, downloadFailLogs); + FileTaskContext fileTaskContext = new FileTaskContext( + stepInstance.getName(), + stepInstance.getFileStepInstance(), + result + ); + return new TaskContext(stepInstance.getExecuteType(), stepInstance.getStatus(), null, fileTaskContext); + } + + /** + * 判断是否为上传失败日志 + * + * @param fileTaskLog 文件任务日志 + * @return 是否为上传失败日志 + */ + private boolean isUploadFailLog(ServiceFileTaskLogDTO fileTaskLog) { + Integer mode = fileTaskLog.getMode(); + Integer status = fileTaskLog.getStatus(); + return FileTaskModeEnum.UPLOAD.getValue().equals(mode) + && FileDistStatusEnum.FAILED.getValue().equals(status); + } + + /** + * 判断是否为下载失败日志 + * + * @param fileTaskLog 文件任务日志 + * @return 是否为下载失败日志 + */ + private boolean isDownloadFailLog(ServiceFileTaskLogDTO fileTaskLog) { + Integer mode = fileTaskLog.getMode(); + Integer status = fileTaskLog.getStatus(); + return FileTaskModeEnum.DOWNLOAD.getValue().equals(mode) + && FileDistStatusEnum.FAILED.getValue().equals(status); + } +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/impl/FileTaskFailLogAnalyzer.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/impl/FileTaskFailLogAnalyzer.java new file mode 100644 index 0000000000..49e6326856 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/impl/FileTaskFailLogAnalyzer.java @@ -0,0 +1,90 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.service.ai.context.impl; + +import com.tencent.bk.job.analysis.service.ai.context.constants.FileTaskErrorSourceEnum; +import com.tencent.bk.job.analysis.service.ai.context.model.FileTaskErrorSourceResult; +import com.tencent.bk.job.logsvr.model.service.ServiceFileTaskLogDTO; +import org.springframework.stereotype.Service; + +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * 文件分发任务失败日志分析器 + */ +@Service +public class FileTaskFailLogAnalyzer { + + /** + * 根据上传与下载失败的日志信息分析导致出错的源头 + * + * @param uploadFailLogs 上传失败日志 + * @param downloadFailLogs 下载失败日志 + * @return 分析结果 + */ + public FileTaskErrorSourceResult analyze(List uploadFailLogs, + List downloadFailLogs) { + if (uploadFailLogs.isEmpty() && downloadFailLogs.isEmpty()) { + return new FileTaskErrorSourceResult(FileTaskErrorSourceEnum.NO_ERROR, uploadFailLogs, downloadFailLogs); + } + // 上传失败日志为空：说明是下载出错 + if (uploadFailLogs.isEmpty()) { + return new FileTaskErrorSourceResult(FileTaskErrorSourceEnum.DOWNLOAD_ERROR, uploadFailLogs, + downloadFailLogs); + } + // 下载失败日志为空：说明是源文件上传出错 + if (downloadFailLogs.isEmpty()) { + return new FileTaskErrorSourceResult(FileTaskErrorSourceEnum.SOURCE_FILE_UPLOAD_ERROR, uploadFailLogs, + downloadFailLogs); + } + // 上传失败日志与下载失败日志均不为空 + Set downloadFailSrcFilePathSet = downloadFailLogs.stream().map(fileTaskLog -> + fileTaskLog.getSrcExecuteObjectId() + ":" + fileTaskLog.getSrcFile() + ).collect(Collectors.toSet()); + Set uploadFailFilePathSet = uploadFailLogs.stream().map(fileTaskLog -> + fileTaskLog.getSrcExecuteObjectId() + ":" + fileTaskLog.getSrcFile() + ).collect(Collectors.toSet()); + downloadFailSrcFilePathSet.removeAll(uploadFailFilePathSet); + if (downloadFailSrcFilePathSet.isEmpty()) { + // 下载失败日志关联的上传文件任务均失败：说明根本原因是源文件上传出错 + return new FileTaskErrorSourceResult( + FileTaskErrorSourceEnum.SOURCE_FILE_UPLOAD_ERROR, + uploadFailLogs, + Collections.emptyList() + ); + } else { + // 上传失败日志与下载失败日志均不为空，且下载失败日志并不全是上传失败导致：说明根本原因是上传下载同时出错 + // 筛选出由于下载方出错导致的下载失败日志 + List realDownloadFailLogs = downloadFailLogs.stream().filter(fileTaskLog -> + downloadFailSrcFilePathSet.contains(fileTaskLog.getSrcExecuteObjectId() + ":" + fileTaskLog.getSrcFile()) + ).collect(Collectors.toList()); + return new FileTaskErrorSourceResult(FileTaskErrorSourceEnum.UPLOAD_AND_DOWNLOAD_ERROR, uploadFailLogs, + realDownloadFailLogs); + } + } +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/impl/TaskContextServiceImpl.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/impl/TaskContextServiceImpl.java new file mode 100644 index 0000000000..e7ef6beeb6 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/impl/TaskContextServiceImpl.java @@ -0,0 +1,108 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.service.ai.context.impl; + +import com.tencent.bk.job.analysis.service.ai.context.TaskContextService; +import com.tencent.bk.job.analysis.service.ai.context.model.ScriptTaskContext; +import com.tencent.bk.job.analysis.service.ai.context.model.TaskContext; +import com.tencent.bk.job.analysis.service.ai.context.model.TaskContextQuery; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.InvalidParamException; +import com.tencent.bk.job.common.exception.ServiceException; +import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; +import com.tencent.bk.job.common.iam.model.AuthResult; +import com.tencent.bk.job.common.model.InternalResponse; +import com.tencent.bk.job.common.model.error.ErrorType; +import com.tencent.bk.job.execute.api.inner.ServiceStepInstanceResource; +import com.tencent.bk.job.execute.model.inner.ServiceScriptStepInstanceDTO; +import com.tencent.bk.job.execute.model.inner.ServiceStepInstanceDTO; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +/** + * 任务上下文服务 + */ +@Service +public class TaskContextServiceImpl implements TaskContextService { + + private final ServiceStepInstanceResource serviceStepInstanceResource; + private final FileTaskContextService fileTaskContextService; + + @Autowired + public TaskContextServiceImpl(ServiceStepInstanceResource serviceStepInstanceResource, + FileTaskContextService fileTaskContextService) { + this.serviceStepInstanceResource = serviceStepInstanceResource; + this.fileTaskContextService = fileTaskContextService; + } + + /** + * 根据用户名与上下文查询条件获取对应的任务上下文 + * + * @param username 用户名 + * @param contextQuery 上下文查询条件 + * @return 任务上下文 + */ + @Override + public TaskContext getTaskContext(String username, TaskContextQuery contextQuery) { + InternalResponse resp = serviceStepInstanceResource.getStepInstance( + username, + contextQuery.getAppId(), + contextQuery.getTaskInstanceId(), + contextQuery.getStepInstanceId() + ); + if (resp.isSuccess()) { + ServiceStepInstanceDTO stepInstance = resp.getData(); + if (stepInstance.isScriptStep()) { + return buildContextForScriptTask(stepInstance); + } else if (stepInstance.isFileStep()) { + return fileTaskContextService.getTaskContext(stepInstance, contextQuery); + } else { + throw new InvalidParamException(ErrorCode.AI_ANALYZE_ERROR_ONLY_SUPPORT_SCRIPT_OR_FILE_STEP); + } + } + if (resp.getAuthResult() != null && !resp.getAuthResult().isPass()) { + throw new PermissionDeniedException(AuthResult.fromAuthResultDTO(resp.getAuthResult())); + } + throw new ServiceException(resp.getErrorMsg(), ErrorType.valOf(resp.getErrorType()), resp.getCode()); + } + + /** + * 构建脚本任务上下文 + * + * @param stepInstance 步骤实例 + * @return 脚本任务上下文 + */ + private TaskContext buildContextForScriptTask(ServiceStepInstanceDTO stepInstance) { + ServiceScriptStepInstanceDTO scriptStepInstance = stepInstance.getScriptStepInstance(); + ScriptTaskContext scriptTaskContext = new ScriptTaskContext( + stepInstance.getName(), + scriptStepInstance.getScriptType(), + scriptStepInstance.getScriptContent(), + scriptStepInstance.getScriptParam(), + scriptStepInstance.isSecureParam() + ); + return new TaskContext(stepInstance.getExecuteType(), stepInstance.getStatus(), scriptTaskContext, null); + } +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/model/AsyncConsumerAndStreamingResponseBodyPair.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/model/AsyncConsumerAndStreamingResponseBodyPair.java new file mode 100644 index 0000000000..31a627ed2a --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/model/AsyncConsumerAndStreamingResponseBodyPair.java @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.service.ai.context.model; + +import lombok.AllArgsConstructor; +import lombok.Data; +import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; + +import java.util.function.Consumer; + +/** + * 互相绑定的异步任务消费者与流式响应体对 + */ +@AllArgsConstructor +@Data +public class AsyncConsumerAndStreamingResponseBodyPair { + /** + * 消费者，消费其他数据源产生的数据，将其写入消息队列 + */ + Consumer consumer; + /** + * 流式响应体，读取消息队列中的数据并将其写入输出流中 + */ + StreamingResponseBody streamingResponseBody; +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/model/ContainerDescription.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/model/ContainerDescription.java new file mode 100644 index 0000000000..dbf8635269 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/model/ContainerDescription.java @@ -0,0 +1,42 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.service.ai.context.model; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * 容器描述信息，提交给AI让其进行归纳总结，字段名必须清晰 + */ +@NoArgsConstructor +@AllArgsConstructor +@Data +public class ContainerDescription { + /** + * 容器UID + */ + private String uid; +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/model/DownloadFileErrorInfo.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/model/DownloadFileErrorInfo.java new file mode 100644 index 0000000000..ab546f79bf --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/model/DownloadFileErrorInfo.java @@ -0,0 +1,78 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.service.ai.context.model; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * 单条下载文件错误信息，提交给AI让其进行归纳总结，字段名必须清晰 + */ +@NoArgsConstructor +@AllArgsConstructor +@Data +public class DownloadFileErrorInfo { + + /** + * 源执行对象类型 + */ + private String sourceExecuteObjectType; + + /** + * 源主机描述信息 + */ + private HostDescription sourceHostDescription; + + /** + * 源容器描述信息 + */ + private ContainerDescription sourceContainerDescription; + + /** + * 目标执行对象类型 + */ + private String targetExecuteObjectType; + + /** + * 目标主机描述信息 + */ + private HostDescription targetHostDescription; + + /** + * 目标容器描述信息 + */ + private ContainerDescription targetContainerDescription; + + /** + * 目标文件路径 + */ + private String targetFilePath; + + /** + * 目标文件下载报错信息 + */ + private String errorLog; +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/model/FileTaskContext.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/model/FileTaskContext.java new file mode 100644 index 0000000000..93423a4d8f --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/model/FileTaskContext.java @@ -0,0 +1,223 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.service.ai.context.model; + +import com.tencent.bk.job.common.constant.ExecuteObjectTypeEnum; +import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.execute.model.inner.ServiceExecuteObject; +import com.tencent.bk.job.execute.model.inner.ServiceExecuteTargetDTO; +import com.tencent.bk.job.execute.model.inner.ServiceFileSourceDTO; +import com.tencent.bk.job.execute.model.inner.ServiceFileStepInstanceDTO; +import com.tencent.bk.job.logsvr.model.service.ServiceFileTaskLogDTO; +import lombok.AllArgsConstructor; +import lombok.Data; +import org.apache.commons.collections4.CollectionUtils; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * 文件任务上下文 + */ +@AllArgsConstructor +@Data +public class FileTaskContext { + /** + * 文件任务名称 + */ + private String name; + /** + * 文件分发任务实例 + */ + ServiceFileStepInstanceDTO fileStepInstance; + /** + * 文件分发任务错误根源分析结果 + */ + FileTaskErrorSourceResult errorSourceResult; + /** + * 执行对象表，Key为执行对象ID，Value为执行对象 + */ + Map executeObjectMap; + + public FileTaskContext(String name, + ServiceFileStepInstanceDTO fileStepInstance, + FileTaskErrorSourceResult errorSourceResult) { + this.name = name; + this.fileStepInstance = fileStepInstance; + this.errorSourceResult = errorSourceResult; + } + + /** + * 根据文件步骤实例信息构建执行对象表 + */ + private void buildExecuteObjectMapIfNeeded() { + if (executeObjectMap != null) { + return; + } + executeObjectMap = new HashMap<>(); + List fileSourceList = fileStepInstance.getFileSourceList(); + if (CollectionUtils.isNotEmpty(fileSourceList)) { + for (ServiceFileSourceDTO fileSource : fileSourceList) { + ServiceExecuteTargetDTO servers = fileSource.getServers(); + if (servers == null) { + continue; + } + List executeObjects = servers.getExecuteObjects(); + if (CollectionUtils.isEmpty(executeObjects)) { + continue; + } + for (ServiceExecuteObject executeObject : executeObjects) { + executeObjectMap.put(executeObject.getId(), executeObject); + } + } + } + ServiceExecuteTargetDTO targetExecuteObjects = fileStepInstance.getTargetExecuteObjects(); + if (targetExecuteObjects == null) { + return; + } + List executeObjects = targetExecuteObjects.getExecuteObjects(); + if (CollectionUtils.isEmpty(executeObjects)) { + return; + } + for (ServiceExecuteObject executeObject : executeObjects) { + executeObjectMap.put(executeObject.getId(), executeObject); + } + } + + /** + * 获取文件任务错误根源的国际化Key + * + * @return 国际化Key + */ + public String getFileTaskErrorSourceI18nKey() { + return errorSourceResult.getErrorSource().getI18nKey(); + } + + /** + * 获取上传文件错误数据 + * + * @return 错误数据 + */ + @SuppressWarnings("DuplicatedCode") + public String getUploadFileErrorData() { + buildExecuteObjectMapIfNeeded(); + List uploadFileErrorInfoList = new ArrayList<>(); + List uploadFailLogs = errorSourceResult.getUploadFailLogs(); + if (CollectionUtils.isEmpty(uploadFailLogs)) { + return JsonUtils.toJson(uploadFileErrorInfoList); + } + for (ServiceFileTaskLogDTO uploadFailLog : uploadFailLogs) { + UploadFileErrorInfo uploadFileErrorInfo = new UploadFileErrorInfo(); + String srcExecuteObjectId = uploadFailLog.getSrcExecuteObjectId(); + ServiceExecuteObject executeObject = executeObjectMap.get(srcExecuteObjectId); + ExecuteObjectTypeEnum executeObjectType = ExecuteObjectTypeEnum.valOf(executeObject.getType()); + uploadFileErrorInfo.setSourceExecuteObjectType(executeObjectType.name()); + uploadFileErrorInfo.setSourceFilePath(uploadFailLog.getDisplaySrcFile()); + if (executeObjectType == ExecuteObjectTypeEnum.HOST) { + HostDescription sourceHostDescription = new HostDescription(); + sourceHostDescription.setCloudIp(executeObject.getHost().toCloudIp()); + uploadFileErrorInfo.setSourceHostDescription(sourceHostDescription); + } else if (executeObjectType == ExecuteObjectTypeEnum.CONTAINER) { + ContainerDescription sourceContainerDescription = new ContainerDescription(); + sourceContainerDescription.setUid(executeObject.getContainer().getContainerId()); + uploadFileErrorInfo.setSourceContainerDescription(sourceContainerDescription); + } + // 填充错误日志信息 + String lastLineLog = getLastLineLog(uploadFailLog.getContent()); + uploadFileErrorInfo.setErrorLog(lastLineLog); + + uploadFileErrorInfoList.add(uploadFileErrorInfo); + } + return JsonUtils.toJson(uploadFileErrorInfoList); + } + + /** + * 获取下载文件错误数据 + * + * @return 错误数据 + */ + @SuppressWarnings("DuplicatedCode") + public String getDownloadFileErrorData() { + buildExecuteObjectMapIfNeeded(); + List downloadFileErrorInfoList = new ArrayList<>(); + List downloadFailLogs = errorSourceResult.getDownloadFailLogs(); + if (CollectionUtils.isEmpty(downloadFailLogs)) { + return JsonUtils.toJson(downloadFileErrorInfoList); + } + for (ServiceFileTaskLogDTO downloadFailLog : downloadFailLogs) { + DownloadFileErrorInfo downloadFileErrorInfo = new DownloadFileErrorInfo(); + // 填充下载文件的源执行对象信息 + String srcExecuteObjectId = downloadFailLog.getSrcExecuteObjectId(); + ServiceExecuteObject srcExecuteObject = executeObjectMap.get(srcExecuteObjectId); + ExecuteObjectTypeEnum srcExecuteObjectType = ExecuteObjectTypeEnum.valOf(srcExecuteObject.getType()); + downloadFileErrorInfo.setSourceExecuteObjectType(srcExecuteObjectType.name()); + if (srcExecuteObjectType == ExecuteObjectTypeEnum.HOST) { + HostDescription sourceHostDescription = new HostDescription(); + sourceHostDescription.setCloudIp(srcExecuteObject.getHost().toCloudIp()); + downloadFileErrorInfo.setSourceHostDescription(sourceHostDescription); + } else if (srcExecuteObjectType == ExecuteObjectTypeEnum.CONTAINER) { + ContainerDescription sourceContainerDescription = new ContainerDescription(); + sourceContainerDescription.setUid(srcExecuteObject.getContainer().getContainerId()); + downloadFileErrorInfo.setSourceContainerDescription(sourceContainerDescription); + } + + // 填充下载文件的目标执行对象信息 + String destExecuteObjectId = downloadFailLog.getDestExecuteObjectId(); + ServiceExecuteObject targetExecuteObject = executeObjectMap.get(destExecuteObjectId); + ExecuteObjectTypeEnum targetExecuteObjectType = ExecuteObjectTypeEnum.valOf(targetExecuteObject.getType()); + downloadFileErrorInfo.setTargetExecuteObjectType(targetExecuteObjectType.name()); + downloadFileErrorInfo.setTargetFilePath(downloadFailLog.getDestFile()); + if (targetExecuteObjectType == ExecuteObjectTypeEnum.HOST) { + HostDescription targetHostDescription = new HostDescription(); + targetHostDescription.setCloudIp(targetExecuteObject.getHost().toCloudIp()); + downloadFileErrorInfo.setTargetHostDescription(targetHostDescription); + } else if (targetExecuteObjectType == ExecuteObjectTypeEnum.CONTAINER) { + ContainerDescription targetContainerDescription = new ContainerDescription(); + targetContainerDescription.setUid(targetExecuteObject.getContainer().getContainerId()); + downloadFileErrorInfo.setTargetContainerDescription(targetContainerDescription); + } + // 填充错误日志信息 + downloadFileErrorInfo.setErrorLog(getLastLineLog(downloadFailLog.getContent())); + downloadFileErrorInfoList.add(downloadFileErrorInfo); + } + return JsonUtils.toJson(downloadFileErrorInfoList); + } + + /** + * 获取日志的最后一行 + * + * @param log 原始日志 + * @return 最后一行日志内容 + */ + private String getLastLineLog(String log) { + if (log != null) { + String[] lines = log.split("\n"); + return lines[lines.length - 1]; + } + return null; + } +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/model/FileTaskErrorSourceResult.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/model/FileTaskErrorSourceResult.java new file mode 100644 index 0000000000..f0ff779520 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/model/FileTaskErrorSourceResult.java @@ -0,0 +1,55 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.service.ai.context.model; + +import com.tencent.bk.job.analysis.service.ai.context.constants.FileTaskErrorSourceEnum; +import com.tencent.bk.job.logsvr.model.service.ServiceFileTaskLogDTO; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +/** + * 文件分发任务错误根源分析结果 + */ +@NoArgsConstructor +@AllArgsConstructor +@Data +public class FileTaskErrorSourceResult { + /** + * 错误根源 + */ + FileTaskErrorSourceEnum errorSource; + + /** + * 上传失败的日志 + */ + List uploadFailLogs; + /** + * 下载失败的日志 + */ + List downloadFailLogs; +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/model/HostDescription.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/model/HostDescription.java new file mode 100644 index 0000000000..f3a102f54e --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/model/HostDescription.java @@ -0,0 +1,42 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.service.ai.context.model; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * 主机描述，提交给AI让其进行归纳总结，字段名必须清晰 + */ +@NoArgsConstructor +@AllArgsConstructor +@Data +public class HostDescription { + /** + * 云区域IP + */ + private String cloudIp; +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/model/MessagePartEvent.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/model/MessagePartEvent.java new file mode 100644 index 0000000000..f07c3b320e --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/model/MessagePartEvent.java @@ -0,0 +1,60 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.service.ai.context.model; + +import lombok.AllArgsConstructor; +import lombok.Data; + +/** + * 含有部分消息的事件 + */ +@AllArgsConstructor +@Data +public class MessagePartEvent { + /** + * 是否为消息结束事件 + */ + private boolean end; + /** + * 消息内容 + */ + private String messagePart; + /** + * 事件产生时间 + */ + private Long timeMills; + /** + * 异常 + */ + private Throwable throwable; + + public static MessagePartEvent endEvent(Throwable throwable) { + return new MessagePartEvent(true, null, System.currentTimeMillis(), throwable); + } + + public static MessagePartEvent normalEvent(String messagePart) { + return new MessagePartEvent(false, messagePart, System.currentTimeMillis(), null); + } +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/model/ScriptTaskContext.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/model/ScriptTaskContext.java new file mode 100644 index 0000000000..5d6709e433 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/model/ScriptTaskContext.java @@ -0,0 +1,65 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.service.ai.context.model; + +import lombok.AllArgsConstructor; +import lombok.Data; + +@AllArgsConstructor +@Data +public class ScriptTaskContext { + /** + * 脚本任务名称 + */ + private String name; + /** + * 脚本类型 + */ + private Integer scriptType; + /** + * 脚本内容 + */ + private String scriptContent; + /** + * 脚本参数 + */ + private String scriptParams; + /** + * 脚本参数是否为敏感参数 + */ + private boolean secureParam; + + /** + * 获取脱敏后的脚本参数 + * + * @return 脱敏后的脚本参数 + */ + public String getInsensitiveScriptParamsStr() { + if (!secureParam) { + return scriptParams == null ? "" : scriptParams; + } + return "[Secure params ignored]"; + } +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/model/TaskContext.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/model/TaskContext.java new file mode 100644 index 0000000000..1c707667aa --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/model/TaskContext.java @@ -0,0 +1,70 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.service.ai.context.model; + +import com.tencent.bk.job.execute.common.constants.RunStatusEnum; +import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; +import lombok.AllArgsConstructor; +import lombok.Data; + +/** + * 任务上下文，用作动态数据填充模板，进而构建向AI提问的Prompt + */ +@AllArgsConstructor +@Data +public class TaskContext { + + /** + * 步骤类型 + */ + private Integer executeType; + /** + * 步骤状态 + */ + private Integer status; + /** + * 脚本任务上下文 + */ + private ScriptTaskContext scriptTaskContext; + /** + * 文件任务上下文 + */ + private FileTaskContext fileTaskContext; + + public boolean isScriptTask() { + StepExecuteTypeEnum stepExecuteTypeEnum = StepExecuteTypeEnum.valOf(executeType); + return stepExecuteTypeEnum == StepExecuteTypeEnum.EXECUTE_SCRIPT + || stepExecuteTypeEnum == StepExecuteTypeEnum.EXECUTE_SQL; + } + + public boolean isFileTask() { + StepExecuteTypeEnum stepExecuteTypeEnum = StepExecuteTypeEnum.valOf(executeType); + return stepExecuteTypeEnum == StepExecuteTypeEnum.SEND_FILE; + } + + public boolean isTaskFail() { + return RunStatusEnum.FAIL.getValue().equals(status); + } +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/model/TaskContextQuery.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/model/TaskContextQuery.java new file mode 100644 index 0000000000..00082867ec --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/model/TaskContextQuery.java @@ -0,0 +1,97 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.service.ai.context.model; + +import com.tencent.bk.job.analysis.model.web.req.AIAnalyzeErrorReq; +import com.tencent.bk.job.execute.model.web.vo.ExecuteObjectVO; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * 任务上下文查询条件 + */ +@NoArgsConstructor +@AllArgsConstructor +@Data +public class TaskContextQuery { + + /** + * 作业平台业务ID + */ + private Long appId; + + /** + * 任务ID + */ + private Long taskInstanceId; + + /** + * 步骤ID + */ + private Long stepInstanceId; + + /** + * 执行次数 + */ + private Integer executeCount; + + /** + * 滚动批次，非滚动步骤不需要传入 + */ + private Integer batch; + + /** + * 执行对象类型 + */ + private Integer executeObjectType; + + /** + * 执行对象资源 ID + */ + private Long executeObjectResourceId; + + /** + * 文件任务上传下载标识,0-上传,1-下载 + */ + private Integer mode; + + public static TaskContextQuery fromAIAnalyzeErrorReq(Long appId, AIAnalyzeErrorReq req) { + TaskContextQuery contextQuery = new TaskContextQuery(); + contextQuery.setAppId(appId); + contextQuery.setTaskInstanceId(req.getTaskInstanceId()); + contextQuery.setStepInstanceId(req.getStepInstanceId()); + contextQuery.setExecuteCount(req.getExecuteCount()); + contextQuery.setBatch(req.getBatch()); + contextQuery.setExecuteObjectType(req.getExecuteObjectType()); + contextQuery.setExecuteObjectResourceId(req.getExecuteObjectResourceId()); + contextQuery.setMode(req.getMode()); + return contextQuery; + } + + public String getExecuteObjectId() { + return ExecuteObjectVO.buildExecuteObjectId(executeObjectType, executeObjectResourceId); + } +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/model/UploadFileErrorInfo.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/model/UploadFileErrorInfo.java new file mode 100644 index 0000000000..e6f16afe53 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/context/model/UploadFileErrorInfo.java @@ -0,0 +1,63 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.service.ai.context.model; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * 单条上传文件错误信息，提交给AI让其进行归纳总结，字段名必须清晰 + */ +@NoArgsConstructor +@AllArgsConstructor +@Data +public class UploadFileErrorInfo { + + /** + * 源执行对象类型 + */ + private String sourceExecuteObjectType; + + /** + * 源主机描述信息 + */ + private HostDescription sourceHostDescription; + + /** + * 源容器描述信息 + */ + private ContainerDescription sourceContainerDescription; + + /** + * 源文件路径 + */ + private String sourceFilePath; + + /** + * 源文件上传报错信息 + */ + private String errorLog; +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/AIAnalyzeErrorServiceImpl.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/AIAnalyzeErrorServiceImpl.java new file mode 100644 index 0000000000..dfa1da788d --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/AIAnalyzeErrorServiceImpl.java @@ -0,0 +1,182 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.service.ai.impl; + +import com.tencent.bk.job.analysis.model.dto.AIAnalyzeErrorContextDTO; +import com.tencent.bk.job.analysis.model.dto.AIPromptDTO; +import com.tencent.bk.job.analysis.model.web.req.AIAnalyzeErrorReq; +import com.tencent.bk.job.analysis.model.web.resp.AIChatRecord; +import com.tencent.bk.job.analysis.service.ai.AIAnalyzeErrorService; +import com.tencent.bk.job.analysis.service.ai.AIChatHistoryService; +import com.tencent.bk.job.analysis.service.ai.FileTransferTaskErrorAIPromptService; +import com.tencent.bk.job.analysis.service.ai.ScriptExecuteTaskErrorAIPromptService; +import com.tencent.bk.job.analysis.service.ai.context.TaskContextService; +import com.tencent.bk.job.analysis.service.ai.context.model.TaskContext; +import com.tencent.bk.job.analysis.service.ai.context.model.TaskContextQuery; +import com.tencent.bk.job.common.config.BkConfig; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.InvalidParamException; +import lombok.extern.slf4j.Slf4j; +import org.jooq.tools.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +/** + * 通过AI分析任务报错信息的服务实现类 + */ +@Slf4j +@Service +public class AIAnalyzeErrorServiceImpl extends AIBaseService implements AIAnalyzeErrorService { + + private final TaskContextService taskContextService; + private final ScriptExecuteTaskErrorAIPromptService scriptExecuteTaskErrorAIPromptService; + private final FileTransferTaskErrorAIPromptService fileTransferTaskErrorAIPromptService; + private final AIMessageI18nService aiMessageI18nService; + private final AITemplateVarService aiTemplateVarService; + private final BkConfig bkConfig; + + @Autowired + public AIAnalyzeErrorServiceImpl(TaskContextService taskContextService, + ScriptExecuteTaskErrorAIPromptService scriptExecuteTaskErrorAIPromptService, + FileTransferTaskErrorAIPromptService fileTransferTaskErrorAIPromptService, + AIChatHistoryService aiChatHistoryService, + AIMessageI18nService aiMessageI18nService, + AITemplateVarService aiTemplateVarService, + BkConfig bkConfig) { + super(aiChatHistoryService); + this.taskContextService = taskContextService; + this.scriptExecuteTaskErrorAIPromptService = scriptExecuteTaskErrorAIPromptService; + this.fileTransferTaskErrorAIPromptService = fileTransferTaskErrorAIPromptService; + this.aiMessageI18nService = aiMessageI18nService; + this.aiTemplateVarService = aiTemplateVarService; + this.bkConfig = bkConfig; + } + + /** + * 通过AI分析任务报错信息，并记录对话历史 + * + * @param username 用户名 + * @param appId Job业务ID + * @param req 请求内容 + * @return AI对话记录 + */ + @Override + public AIChatRecord analyze(String username, Long appId, AIAnalyzeErrorReq req) { + TaskContextQuery contextQuery = TaskContextQuery.fromAIAnalyzeErrorReq(appId, req); + TaskContext taskContext = taskContextService.getTaskContext(username, contextQuery); + if (taskContext.isScriptTask()) { + return analyzeScriptTask(username, appId, taskContext, req); + } else if (taskContext.isFileTask()) { + return analyzeFileTask(username, appId, taskContext, req); + } else { + throw new InvalidParamException(ErrorCode.AI_ANALYZE_ERROR_ONLY_SUPPORT_SCRIPT_OR_FILE_STEP); + } + } + + /** + * 分析脚本执行任务报错信息 + * + * @param username 用户名 + * @param appId Job业务ID + * @param taskContext 任务上下文 + * @param req 请求体 + * @return AI对话记录 + */ + private AIChatRecord analyzeScriptTask(String username, + Long appId, + TaskContext taskContext, + AIAnalyzeErrorReq req) { + String errorContent = req.getContent(); + AIPromptDTO aiPromptDTO = scriptExecuteTaskErrorAIPromptService.getPrompt( + taskContext.getScriptTaskContext(), + errorContent + ); + if (!taskContext.isTaskFail()) { + return getDirectlyAIChatRecord( + username, + appId, + aiPromptDTO, + aiMessageI18nService.getNotFailTaskAIAnswerMessage() + ); + } + if (StringUtils.isEmpty(errorContent)) { + return getDirectlyAIChatRecord( + username, + appId, + aiPromptDTO, + getRenderedEmptyLogTaskAIAnswerMessage() + ); + } + AIAnalyzeErrorContextDTO analyzeErrorContext = AIAnalyzeErrorContextDTO.fromAIAnalyzeErrorReq(req); + return getAIChatRecord(username, appId, aiPromptDTO, analyzeErrorContext); + } + + /** + * 分析文件分发任务报错信息 + * + * @param username 用户名 + * @param appId Job业务ID + * @param taskContext 任务上下文 + * @param req 请求体 + * @return AI对话记录 + */ + private AIChatRecord analyzeFileTask(String username, + Long appId, + TaskContext taskContext, + AIAnalyzeErrorReq req) { + AIPromptDTO aiPromptDTO = fileTransferTaskErrorAIPromptService.getPrompt(taskContext.getFileTaskContext()); + if (!taskContext.isTaskFail()) { + return getDirectlyAIChatRecord( + username, + appId, + aiPromptDTO, + aiMessageI18nService.getNotFailTaskAIAnswerMessage() + ); + } + if (StringUtils.isEmpty(req.getContent())) { + return getDirectlyAIChatRecord( + username, + appId, + aiPromptDTO, + getRenderedEmptyLogTaskAIAnswerMessage() + ); + } + AIAnalyzeErrorContextDTO analyzeErrorContext = AIAnalyzeErrorContextDTO.fromAIAnalyzeErrorReq(req); + return getAIChatRecord(username, appId, aiPromptDTO, analyzeErrorContext); + } + + /** + * 获取变量渲染后的空日志报错回复消息 + * + * @return 空日志报错回复消息 + */ + private String getRenderedEmptyLogTaskAIAnswerMessage() { + String messageTemplate = aiMessageI18nService.getEmptyLogTaskAIAnswerMessage(); + if (StringUtils.isBlank(messageTemplate)) { + return messageTemplate; + } + return messageTemplate.replace(aiTemplateVarService.getBkHelperLinkPlaceHolder(), bkConfig.getBkHelperLink()); + } +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/AIAnswerHandler.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/AIAnswerHandler.java new file mode 100644 index 0000000000..a3c96a7edf --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/AIAnswerHandler.java @@ -0,0 +1,102 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.service.ai.impl; + +import com.tencent.bk.job.analysis.model.web.resp.AIAnswer; +import com.tencent.bk.job.analysis.service.ai.AIChatHistoryService; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.util.I18nUtil; +import lombok.extern.slf4j.Slf4j; +import org.slf4j.helpers.MessageFormatter; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.concurrent.CancellationException; + +/** + * AI回答处理器 + */ +@Slf4j +@Service +public class AIAnswerHandler { + + private final AIChatHistoryService aiChatHistoryService; + + @Autowired + public AIAnswerHandler(AIChatHistoryService aiChatHistoryService) { + this.aiChatHistoryService = aiChatHistoryService; + } + + /** + * 处理完整的AI回答 + * + * @param recordId 对话记录ID + * @param content AI回答内容 + * @param throwable AI回答过程产生的异常 + */ + public void handleAIAnswer(Long recordId, String content, Throwable throwable) { + try { + doHandleAIAnswer(recordId, content, throwable); + } catch (Throwable t) { + log.error("Fail to handleAIAnswer", t); + } + } + + public void doHandleAIAnswer(Long recordId, String content, Throwable throwable) { + AIAnswer aiAnswer; + if (throwable == null) { + // 1.对话正常完成 + aiAnswer = AIAnswer.successAnswer(content); + int affectedRow = aiChatHistoryService.finishAIAnswer(recordId, aiAnswer); + log.info( + "AIAnswer finished, recordId={}, length={}, affectedRow={}", + recordId, + content.length(), + affectedRow + ); + } else if (throwable instanceof CancellationException) { + // 2.对话被主动终止 + int affectedRow = aiChatHistoryService.terminateAIAnswer(recordId); + log.info( + "AIAnswer terminated, recordId={}, affectedRow={}", + recordId, + affectedRow + ); + } else { + // 3.对话异常 + String errorContent = I18nUtil.getI18nMessage(String.valueOf(ErrorCode.BK_OPEN_AI_API_DATA_ERROR)); + aiAnswer = AIAnswer.failAnswer(errorContent, throwable.getMessage()); + int affectedRow = aiChatHistoryService.finishAIAnswer(recordId, aiAnswer); + String message = MessageFormatter.arrayFormat( + "AIAnswer finished(fail), recordId={}, affectedRow={}", + new Object[]{ + recordId, + affectedRow + } + ).getMessage(); + log.warn(message, throwable); + } + } +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/AIAnswerStreamSynchronizer.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/AIAnswerStreamSynchronizer.java new file mode 100644 index 0000000000..e70cf0353f --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/AIAnswerStreamSynchronizer.java @@ -0,0 +1,159 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.service.ai.impl; + +import com.tencent.bk.job.analysis.model.web.resp.AIAnswer; +import com.tencent.bk.job.analysis.service.ai.context.model.AsyncConsumerAndStreamingResponseBodyPair; +import com.tencent.bk.job.analysis.service.ai.context.model.MessagePartEvent; +import com.tencent.bk.job.analysis.util.ai.AIAnswerUtil; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.model.Response; +import com.tencent.bk.job.common.util.TimeUtil; +import lombok.extern.slf4j.Slf4j; +import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; + +import java.io.OutputStream; +import java.util.concurrent.CancellationException; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Consumer; + +/** + * AI回答流式响应同步器 + * 使用阻塞队列构建互相绑定的异步消费者与流式响应体组合，消费者从流式数据源读取到数据后将其写入到阻塞队列中并提供给流式响应体进行读取输出 + */ +@Slf4j +public class AIAnswerStreamSynchronizer { + + private final AtomicBoolean isFinished = new AtomicBoolean(false); + private final LinkedBlockingQueue messageEventQueue; + + /** + * 构造函数 + * + * @param capacity 可缓存于内存的消息事件量 + */ + public AIAnswerStreamSynchronizer(int capacity) { + messageEventQueue = new LinkedBlockingQueue<>(capacity); + } + + /** + * 构建异步消费者与流式响应体组合 + * + * @return 异步消费者与流式响应体组合 + */ + public AsyncConsumerAndStreamingResponseBodyPair buildAsyncConsumerAndStreamingResponseBodyPair() { + StreamingResponseBody streamingResponseBody = buildStreamingResponseBody(); + Consumer partialRespConsumer = new AIMessagePartConsumer(messageEventQueue); + return new AsyncConsumerAndStreamingResponseBodyPair(partialRespConsumer, streamingResponseBody); + } + + /** + * 构建当前同步器绑定的流式响应体 + * + * @return 流式响应体 + */ + private StreamingResponseBody buildStreamingResponseBody() { + return outputStream -> { + while (!isFinished.get()) { + // 从当前同步器的队列中获取一个消息事件并处理 + boolean needContinue = pollEventAndHandle(outputStream); + if (!needContinue) { + break; + } + } + outputStream.close(); + }; + } + + /** + * 从队列中获取一个消息事件并处理 + * + * @param outputStream 输出流 + * @return 是否需要继续处理下一个消息事件 + */ + private boolean pollEventAndHandle(OutputStream outputStream) { + try { + final int maxWaitSeconds = 90; + MessagePartEvent event = messageEventQueue.poll(maxWaitSeconds, TimeUnit.SECONDS); + // 远端流式数据超时，构建对应的错误信息输出至本地输出流 + if (event == null) { + Response respBody = + Response.buildCommonFailResp(ErrorCode.BK_OPEN_AI_API_DATA_TIMEOUT); + respBody.setData(AIAnswer.failAnswer(respBody.getErrorMsg(), respBody.getErrorMsg())); + AIAnswerUtil.setRequestIdAndWriteResp(outputStream, respBody); + return false; + } + // 收到数据结束事件，结束处理 + if (event.isEnd()) { + Throwable throwable = event.getThrowable(); + if (throwable != null) { + // 用户主动取消导致的异常，直接结束处理即可 + if (throwable instanceof CancellationException) { + return false; + } + // 其他异常，表明远端流式接口调用失败，构建对应的错误信息输出 + log.warn("Receive end event with throwable", throwable); + Response respBody = Response.buildCommonFailResp(ErrorCode.BK_OPEN_AI_API_DATA_ERROR); + respBody.setData(AIAnswer.failAnswer(respBody.getErrorMsg(), throwable.getMessage())); + AIAnswerUtil.setRequestIdAndWriteResp(outputStream, respBody); + } + return false; + } + // 收到普通块数据，直接输出至本地输出流 + String partMessage = event.getMessagePart(); + Response respBody = Response.buildSuccessResp(AIAnswer.successAnswer(partMessage)); + AIAnswerUtil.setRequestIdAndWriteResp(outputStream, respBody); + if (log.isDebugEnabled()) { + // 记录延迟数据至调试日志，便于排查可能出现的性能相关问题 + log.debug( + "partMessage={}, time={}, delay={}ms", + partMessage, + TimeUtil.formatTime(event.getTimeMills(), "yyyy-MM-dd HH:mm:ss.SSS"), + System.currentTimeMillis() - event.getTimeMills() + ); + } + } catch (InterruptedException e) { + // 线程被中断，通常在发布变更进程关闭时产生，忽略并记录日志即可 + log.debug("Interrupted when take message from queue", e); + } catch (Exception e) { + // 写入本地输出流失败，可能前端主动关闭了连接，忽略并记录日志即可 + log.warn("Write resp to output stream failed", e); + return false; + } + return true; + } + + /** + * 触发结束事件，消费者读取完数据后，通知流式响应体做出相应的停止输出动作并清理 + * + * @param throwable 异常 + */ + public void triggerEndEvent(Throwable throwable) { + isFinished.set(true); + messageEventQueue.offer(MessagePartEvent.endEvent(throwable)); + } +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/AIBasePromptService.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/AIBasePromptService.java new file mode 100644 index 0000000000..67d1061f9b --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/AIBasePromptService.java @@ -0,0 +1,74 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.service.ai.impl; + +import com.tencent.bk.job.analysis.dao.AIPromptTemplateDAO; +import com.tencent.bk.job.analysis.model.dto.AIPromptTemplateDTO; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.InternalException; +import com.tencent.bk.job.common.util.JobContextUtil; +import lombok.extern.slf4j.Slf4j; + +/** + * 基础AI提示符服务 + */ +@Slf4j +public class AIBasePromptService { + + private final AIPromptTemplateDAO aiPromptTemplateDAO; + + public AIBasePromptService(AIPromptTemplateDAO aiPromptTemplateDAO) { + this.aiPromptTemplateDAO = aiPromptTemplateDAO; + } + + /** + * 根据模板代码获取提示符模板 + * + * @param templateCode 模板代码 + * @return AI提示符模板 + */ + protected AIPromptTemplateDTO getPromptTemplate(String templateCode) { + String userLang = JobContextUtil.getUserLang(); + AIPromptTemplateDTO promptTemplate = aiPromptTemplateDAO.getAIPromptTemplate( + templateCode, + userLang + ); + if (promptTemplate == null) { + String message = "Cannot find prompt template for (" + + "code=" + templateCode + + ", userLang=" + userLang + + "), please check template config in DB"; + throw new InternalException(message, ErrorCode.INTERNAL_ERROR); + } + log.info( + "Use prompt template [{}(id={})], userLang={}", + promptTemplate.getName(), + promptTemplate.getId(), + userLang + ); + return promptTemplate; + } + +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/AIBaseService.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/AIBaseService.java new file mode 100644 index 0000000000..89d30e3b1e --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/AIBaseService.java @@ -0,0 +1,130 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.service.ai.impl; + +import com.tencent.bk.job.analysis.consts.AIChatStatusEnum; +import com.tencent.bk.job.analysis.model.dto.AIAnalyzeErrorContextDTO; +import com.tencent.bk.job.analysis.model.dto.AIChatHistoryDTO; +import com.tencent.bk.job.analysis.model.dto.AIPromptDTO; +import com.tencent.bk.job.analysis.model.web.resp.AIAnswer; +import com.tencent.bk.job.analysis.model.web.resp.AIChatRecord; +import com.tencent.bk.job.analysis.service.ai.AIChatHistoryService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +/** + * AI基础服务 + */ +@Slf4j +@Service +public class AIBaseService { + + private final AIChatHistoryService aiChatHistoryService; + + @Autowired + public AIBaseService(AIChatHistoryService aiChatHistoryService) { + this.aiChatHistoryService = aiChatHistoryService; + } + + /** + * 使用AI提示符调用AI接口生成AI回答 + * + * @param username 用户名 + * @param appId Job业务ID + * @param aiPromptDTO AI提示符 + * @return AI对话记录 + */ + public AIChatRecord getAIChatRecord(String username, + Long appId, + AIPromptDTO aiPromptDTO) { + return getAIChatRecord(username, appId, aiPromptDTO, null); + } + + /** + * 使用AI提示符调用AI接口生成AI回答（支持报错分析上下文） + * + * @param username 用户名 + * @param appId Job业务ID + * @param aiPromptDTO AI提示符 + * @param analyzeErrorContext 报错分析上下文信息 + * @return AI对话记录 + */ + public AIChatRecord getAIChatRecord(String username, + Long appId, + AIPromptDTO aiPromptDTO, + AIAnalyzeErrorContextDTO analyzeErrorContext) { + long startTime = System.currentTimeMillis(); + // 1.插入初始对话记录 + AIChatHistoryDTO aiChatHistoryDTO = aiChatHistoryService.buildAIChatHistoryDTO( + username, + appId, + startTime, + aiPromptDTO, + AIChatStatusEnum.INIT.getStatus(), + null + ); + aiChatHistoryDTO.setAiAnalyzeErrorContext(analyzeErrorContext); + Long historyId = aiChatHistoryService.insertChatHistory(aiChatHistoryDTO); + aiChatHistoryDTO.setId(historyId); + return aiChatHistoryDTO.toAIChatRecord(); + } + + /** + * 使用指定内容直接生成AI回答 + * + * @param username 用户名 + * @param appId Job业务ID + * @param aiPromptDTO AI提示符 + * @param content 指定内容 + * @return AI对话记录 + */ + public AIChatRecord getDirectlyAIChatRecord(String username, Long appId, AIPromptDTO aiPromptDTO, String content) { + long startTime = System.currentTimeMillis(); + aiPromptDTO.setRenderedPrompt(buildAIDirectlyAnswerInput(content)); + AIAnswer aiAnswer = new AIAnswer("0", "", content, System.currentTimeMillis()); + AIChatHistoryDTO aiChatHistoryDTO = aiChatHistoryService.buildAIChatHistoryDTO( + username, + appId, + startTime, + aiPromptDTO, + AIChatStatusEnum.FINISHED.getStatus(), + aiAnswer + ); + Long historyId = aiChatHistoryService.insertChatHistory(aiChatHistoryDTO); + aiChatHistoryDTO.setId(historyId); + return aiChatHistoryDTO.toAIChatRecord(); + } + + /** + * 构建直接回答的AI输入（无需调用AI生成回答） + * + * @param content 回答内容 + * @return AI输入 + */ + private String buildAIDirectlyAnswerInput(String content) { + return "Answer This Directly:" + content; + } +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/AIChatHistoryServiceImpl.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/AIChatHistoryServiceImpl.java new file mode 100644 index 0000000000..da77a8d35b --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/AIChatHistoryServiceImpl.java @@ -0,0 +1,209 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.service.ai.impl; + +import com.tencent.bk.job.analysis.consts.AIChatStatusEnum; +import com.tencent.bk.job.analysis.dao.AIChatHistoryDAO; +import com.tencent.bk.job.analysis.model.dto.AIChatHistoryDTO; +import com.tencent.bk.job.analysis.model.dto.AIPromptDTO; +import com.tencent.bk.job.analysis.model.web.resp.AIAnswer; +import com.tencent.bk.job.analysis.service.ai.AIChatHistoryService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.Comparator; +import java.util.List; + +/** + * AI对话记录管理服务 + */ +@Slf4j +@Service +public class AIChatHistoryServiceImpl implements AIChatHistoryService { + + /** + * AI对话记录数据库操作对象 + */ + private final AIChatHistoryDAO aiChatHistoryDAO; + + @Autowired + public AIChatHistoryServiceImpl(AIChatHistoryDAO aiChatHistoryDAO) { + this.aiChatHistoryDAO = aiChatHistoryDAO; + } + + /** + * 构建AI对话记录数据传输对象 + * + * @param username 用户名 + * @param appId Job业务ID + * @param startTime 开始时间 + * @param aiPromptDTO AI提示符信息 + * @param status 对话状态 + * @param aiAnswer AI回答 + * @return AI对话记录数据传输对象 + */ + @Override + public AIChatHistoryDTO buildAIChatHistoryDTO(String username, + Long appId, + Long startTime, + AIPromptDTO aiPromptDTO, + Integer status, + AIAnswer aiAnswer) { + AIChatHistoryDTO aiChatHistoryDTO = new AIChatHistoryDTO(); + aiChatHistoryDTO.setUsername(username); + aiChatHistoryDTO.setAppId(appId); + aiChatHistoryDTO.setUserInput(aiPromptDTO.getRawPrompt()); + aiChatHistoryDTO.setPromptTemplateId(aiPromptDTO.getPromptTemplateId()); + aiChatHistoryDTO.setAiInput(aiPromptDTO.getRenderedPrompt()); + aiChatHistoryDTO.setStatus(status); + if (aiAnswer != null) { + aiChatHistoryDTO.setAiAnswer(aiAnswer.getContent()); + aiChatHistoryDTO.setErrorCode(String.valueOf(aiAnswer.getErrorCode())); + aiChatHistoryDTO.setErrorMessage(aiAnswer.getErrorMessage()); + aiChatHistoryDTO.setAnswerTime(System.currentTimeMillis()); + } + aiChatHistoryDTO.setStartTime(startTime); + aiChatHistoryDTO.updateTotalTime(); + aiChatHistoryDTO.setIsDeleted(false); + return aiChatHistoryDTO; + } + + @Override + public Long insertChatHistory(AIChatHistoryDTO aiChatHistoryDTO) { + return aiChatHistoryDAO.insertAIChatHistory(aiChatHistoryDTO); + } + + @Override + public boolean existsChatHistory(String username) { + return aiChatHistoryDAO.existsChatHistory(username); + } + + /** + * 设置AI对话记录状态为回复中 + * + * @param historyId AI对话记录ID + * @return 受影响行数 + */ + @Override + public int setAIAnswerReplying(Long historyId) { + return aiChatHistoryDAO.updateChatHistoryStatus(historyId, AIChatStatusEnum.REPLYING.getStatus()); + } + + /** + * 设置AI对话记录状态为已完成 + * + * @param historyId AI对话记录ID + * @param aiAnswer AI回答内容 + * @return 受影响行数 + */ + @Override + public int finishAIAnswer(Long historyId, AIAnswer aiAnswer) { + return aiChatHistoryDAO.updateChatHistoryStatusAndAIAnswer( + historyId, + AIChatStatusEnum.FINISHED.getStatus(), + aiAnswer.getContent(), + aiAnswer.getErrorCode(), + aiAnswer.getErrorMessage(), + System.currentTimeMillis() + ); + } + + /** + * 设置AI对话记录状态为已终止 + * + * @param historyId AI对话记录ID + * @return 受影响行数 + */ + @Override + public int terminateAIAnswer(Long historyId) { + return aiChatHistoryDAO.updateChatHistoryStatusAndAIAnswer( + historyId, + AIChatStatusEnum.TERMINATED.getStatus(), + AIChatStatusEnum.TERMINATED.getDescription(), + "1", + null, + System.currentTimeMillis() + ); + } + + /** + * 从DB获取最近的对话记录列表，按起始时间升序排列 + * + * @param username 用户名 + * @param start 起始位置 + * @param length 长度 + * @return 最近的对话记录列表 + */ + @Override + public List getLatestChatHistoryList(String username, Integer start, Integer length) { + List aiChatHistoryList = aiChatHistoryDAO.getLatestChatHistoryList(username, start, length); + aiChatHistoryList.sort(Comparator.comparing(AIChatHistoryDTO::getStartTime)); + return aiChatHistoryList; + } + + /** + * 从DB获取最近已完成的对话记录列表，按起始时间升序排列 + * + * @param username 用户名 + * @param start 起始位置 + * @param length 长度 + * @return 最近已完成的对话记录列表 + */ + @Override + public List getLatestFinishedChatHistoryList(String username, Integer start, Integer length) { + List aiChatHistoryList = aiChatHistoryDAO.getLatestFinishedChatHistoryList( + username, + start, + length + ); + aiChatHistoryList.sort(Comparator.comparing(AIChatHistoryDTO::getStartTime)); + return aiChatHistoryList; + } + + @Override + public AIChatHistoryDTO getChatHistory(String username, Long id) { + return aiChatHistoryDAO.getChatHistory(username, id); + } + + /** + * 从DB中分批软删除对话记录（优先删除创建时间较早的） + * + * @param username 用户名 + * @return 删除的记录条数 + */ + @Override + public int softDeleteChatHistory(String username) { + int batchSize = 1000; + int deletedCount; + int deletedTotalCount = 0; + do { + deletedCount = aiChatHistoryDAO.softDeleteChatHistory(username, batchSize); + deletedTotalCount += deletedCount; + } while (deletedCount == batchSize); + log.info("{} chat history of user {} soft deleted", deletedTotalCount, username); + return deletedTotalCount; + } +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/AICheckScriptServiceImpl.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/AICheckScriptServiceImpl.java new file mode 100644 index 0000000000..69a07f6701 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/AICheckScriptServiceImpl.java @@ -0,0 +1,66 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.service.ai.impl; + +import com.tencent.bk.job.analysis.model.dto.AIPromptDTO; +import com.tencent.bk.job.analysis.model.web.resp.AIChatRecord; +import com.tencent.bk.job.analysis.service.ai.AIChatHistoryService; +import com.tencent.bk.job.analysis.service.ai.AICheckScriptService; +import com.tencent.bk.job.analysis.service.ai.CheckScriptAIPromptService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +/** + * 通过AI大模型检查脚本内容服务实现类 + */ +@Slf4j +@Service +public class AICheckScriptServiceImpl extends AIBaseService implements AICheckScriptService { + + private final CheckScriptAIPromptService checkScriptAIPromptService; + + @Autowired + public AICheckScriptServiceImpl(CheckScriptAIPromptService checkScriptAIPromptService, + AIChatHistoryService aiChatHistoryService) { + super(aiChatHistoryService); + this.checkScriptAIPromptService = checkScriptAIPromptService; + } + + /** + * 通过AI大模型检查脚本内容 + * + * @param username 用户名 + * @param appId Job业务ID + * @param type 脚本类型 + * @param scriptContent 脚本内容 + * @return AI对话记录 + */ + @Override + public AIChatRecord check(String username, Long appId, Integer type, String scriptContent) { + AIPromptDTO aiPromptDTO = checkScriptAIPromptService.getPrompt(type, scriptContent); + return getAIChatRecord(username, appId, aiPromptDTO); + } +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/AIConfigService.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/AIConfigService.java new file mode 100644 index 0000000000..dbc695dcfa --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/AIConfigService.java @@ -0,0 +1,59 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.service.ai.impl; + +import com.tencent.bk.job.analysis.config.AIProperties; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.HashMap; +import java.util.Map; + +/** + * AI配置服务 + */ +@Service +public class AIConfigService { + private final AIProperties aiProperties; + + @Autowired + public AIConfigService(AIProperties aiProperties) { + this.aiProperties = aiProperties; + } + + /** + * 获取AI配置 + * + * @return AI配置表，key为配置名称，value为配置取值 + */ + public Map getAIConfig() { + Map map = new HashMap<>(); + Long logMaxLengthBytes = aiProperties.getAnalyzeErrorLog().getLogMaxLengthBytes(); + map.put("enabled", aiProperties.getEnabled()); + map.put("analyzeErrorLogMaxLength", logMaxLengthBytes); + return map; + } + +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/AIMessageI18nService.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/AIMessageI18nService.java new file mode 100644 index 0000000000..feea7e0f00 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/AIMessageI18nService.java @@ -0,0 +1,90 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.service.ai.impl; + +import com.tencent.bk.job.common.i18n.service.MessageI18nService; +import org.springframework.stereotype.Service; + +/** + * AI消息国际化服务 + */ +@Service +public class AIMessageI18nService { + private final MessageI18nService messageI18nService; + + public AIMessageI18nService(MessageI18nService messageI18nService) { + this.messageI18nService = messageI18nService; + } + + public String getAIGreetingMessage() { + return messageI18nService.getI18n("job.analysis.ai.greetingMessage"); + } + + /** + * 获取无日志输出任务的AI分析结果信息 + * + * @return 国际化的AI分析结果信息 + */ + public String getEmptyLogTaskAIAnswerMessage() { + return messageI18nService.getI18n("job.analysis.ai.emptyLogTaskAnswerMessage"); + } + + /** + * 获取非失败状态任务的AI分析结果信息 + * + * @return 国际化的AI分析结果信息 + */ + public String getNotFailTaskAIAnswerMessage() { + return messageI18nService.getI18n("job.analysis.ai.notFailTaskAnswerMessage"); + } + + /** + * 获取指定环境语言的系统信息 + * + * @return 国际化的指定环境语言的系统信息 + */ + public String getLanguageSpecifySystemMessage() { + return messageI18nService.getI18n("job.analysis.ai.languageSpecifySystemMessage"); + } + + /** + * 获取指定环境语言的AI答复信息 + * + * @return 国际化的指定环境语言的AI答复信息 + */ + public String getLanguageSpecifyAIReplyMessage() { + return messageI18nService.getI18n("job.analysis.ai.languageSpecifyAIReplyMessage"); + } + + /** + * 根据国际化Key获取国际化信息 + * + * @param i18nKey 国际化Key + * @return 国际化信息 + */ + public String getI18nMessage(String i18nKey) { + return messageI18nService.getI18n(i18nKey); + } +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/AIMessagePartConsumer.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/AIMessagePartConsumer.java new file mode 100644 index 0000000000..83360219e4 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/AIMessagePartConsumer.java @@ -0,0 +1,62 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.service.ai.impl; + +import com.tencent.bk.job.analysis.service.ai.context.model.MessagePartEvent; + +import java.util.concurrent.LinkedBlockingQueue; +import java.util.function.Consumer; + +/** + * AI大模型回复的流式消息块消费者 + */ +public class AIMessagePartConsumer implements Consumer { + private final LinkedBlockingQueue messageQueue; + + public AIMessagePartConsumer(LinkedBlockingQueue messageQueue) { + this.messageQueue = messageQueue; + } + + /** + * 消费一块流式消息，将其放入阻塞队列中 + * + * @param s 一块流式消息 + */ + @Override + public void accept(String s) { + messageQueue.offer(MessagePartEvent.normalEvent(s)); + } + + /** + * 将当前消费者和另一个消费者串行执行 + * + * @param after 下一个消费者 + * @return 串行执行的两个消费者组合 + */ + @Override + public Consumer andThen(Consumer after) { + return Consumer.super.andThen(after); + } +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/AIServiceImpl.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/AIServiceImpl.java new file mode 100644 index 0000000000..5835ce9d2f --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/AIServiceImpl.java @@ -0,0 +1,127 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.service.ai.impl; + +import com.tencent.bk.job.analysis.model.dto.AIChatHistoryDTO; +import com.tencent.bk.job.analysis.service.ai.AIService; +import com.tencent.bk.job.analysis.service.login.LoginTokenService; +import com.tencent.bk.job.common.aidev.IBkOpenAIClient; +import com.tencent.bk.job.common.aidev.model.common.AIDevMessage; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.function.Consumer; + +/** + * AI服务实现类 + */ +@Slf4j +@Service +public class AIServiceImpl implements AIService { + + private final LoginTokenService loginTokenService; + private final IBkOpenAIClient bkOpenAIClient; + private final AIMessageI18nService aiMessageI18nService; + + @Autowired + public AIServiceImpl(LoginTokenService loginTokenService, + IBkOpenAIClient bkOpenAIClient, + AIMessageI18nService aiMessageI18nService) { + this.loginTokenService = loginTokenService; + this.bkOpenAIClient = bkOpenAIClient; + this.aiMessageI18nService = aiMessageI18nService; + } + + + /** + * 通过蓝鲸OpenAI接口获取AI回答流 + * + * @param chatHistoryDTOList 历史对话记录 + * @param userInput 用户输入 + * @param partialRespConsumer AI回答流回调 + * @return AI回答结果Future + */ + @Override + public CompletableFuture getAIAnswerStream(List chatHistoryDTOList, + String userInput, + Consumer partialRespConsumer) { + String token = loginTokenService.getToken(); + return bkOpenAIClient.getAIAnswerStream( + token, + buildMessageHistoryList(chatHistoryDTOList), + userInput, + partialRespConsumer + ); + } + + /** + * 构建传递给蓝鲸OpenAI接口的历史消息记录列表，用作问答上下文 + * + * @param chatHistoryDTOList 历史对话记录 + * @return 历史消息列表 + */ + private List buildMessageHistoryList(List chatHistoryDTOList) { + if (CollectionUtils.isEmpty(chatHistoryDTOList)) { + return buildSystemMessageList(); + } + List messageHistoryList = new ArrayList<>(); + for (AIChatHistoryDTO chatHistoryDTO : chatHistoryDTOList) { + AIDevMessage aiDevMessage = new AIDevMessage(); + aiDevMessage.setRole(AIDevMessage.ROLE_USER); + aiDevMessage.setContent(chatHistoryDTO.getAiInput()); + messageHistoryList.add(aiDevMessage); + aiDevMessage = new AIDevMessage(); + aiDevMessage.setRole(AIDevMessage.ROLE_ASSISTANT); + aiDevMessage.setContent(chatHistoryDTO.getAiAnswer()); + messageHistoryList.add(aiDevMessage); + } + messageHistoryList.addAll(buildSystemMessageList()); + return messageHistoryList; + } + + /** + * 构建系统消息列表，用于指定环境语言等基础信息 + * + * @return 系统消息列表 + */ + private List buildSystemMessageList() { + List systemMessageList = new ArrayList<>(); + AIDevMessage languageSpecifyMessage = new AIDevMessage(); + // ROLE_SYSTEM系统角色不生效，使用ROLE_USER消息代替 + languageSpecifyMessage.setRole(AIDevMessage.ROLE_USER); + languageSpecifyMessage.setContent(aiMessageI18nService.getLanguageSpecifySystemMessage()); + systemMessageList.add(languageSpecifyMessage); + AIDevMessage aiReplyMessage = new AIDevMessage(); + aiReplyMessage.setRole(AIDevMessage.ROLE_ASSISTANT); + aiReplyMessage.setContent(aiMessageI18nService.getLanguageSpecifyAIReplyMessage()); + systemMessageList.add(aiReplyMessage); + return systemMessageList; + } +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/AITemplateVarService.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/AITemplateVarService.java new file mode 100644 index 0000000000..2c7355b851 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/AITemplateVarService.java @@ -0,0 +1,135 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.service.ai.impl; + +import org.springframework.stereotype.Service; + +/** + * AI模板变量服务，统一提供模板变量占位符 + */ +@Service +public class AITemplateVarService { + + /** + * 获取模板变量占位符 + * + * @param varName 变量名 + * @return 占位符 + */ + public String getTemplateVarPlaceHolder(String varName) { + String TEMPLATE_VAR_PREFIX = "{BK_JOB_AI_TEMPLATE_VAR"; + return TEMPLATE_VAR_PREFIX + "{" + varName + "}}"; + } + + /** + * 获取脚本类型占位符 + * + * @return 脚本类型占位符 + */ + public String getScriptTypePlaceHolder() { + return getTemplateVarPlaceHolder("script_type"); + } + + /** + * 获取脚本模板占位符 + * + * @return 脚本模板占位符 + */ + public String getScriptTemplatePlaceHolder() { + return getTemplateVarPlaceHolder("script_template"); + } + + /** + * 获取脚本参数占位符 + * + * @return 脚本参数占位符 + */ + public String getScriptParamsPlaceHolder() { + return getTemplateVarPlaceHolder("script_params"); + } + + /** + * 获取脚本内容占位符 + * + * @return 脚本内容占位符 + */ + public String getScriptContentPlaceHolder() { + return getTemplateVarPlaceHolder("script_content"); + } + + /** + * 获取报错信息占位符 + * + * @return 报错信息占位符 + */ + public String getErrorContentPlaceHolder() { + return getTemplateVarPlaceHolder("error_content"); + } + + /** + * 获取文件任务错误源占位符 + * + * @return 文件任务错误源占位符 + */ + public String getFileTaskErrorSourcePlaceHolder() { + return getTemplateVarPlaceHolder("file_task_error_source"); + } + + /** + * 获取上传文件错误数据占位符 + * + * @return 上传文件错误数据占位符 + */ + public String getUploadFileErrorDataPlaceHolder() { + return getTemplateVarPlaceHolder("upload_file_error_data"); + } + + /** + * 获取下载文件错误数据占位符 + * + * @return 下载文件错误数据占位符 + */ + public String getDownloadFileErrorDataPlaceHolder() { + return getTemplateVarPlaceHolder("download_file_error_data"); + } + + /** + * 获取BK助手链接占位符 + * + * @return BK助手链接占位符 + */ + public String getBkHelperLinkPlaceHolder() { + return getTemplateVarPlaceHolder("bk_helper_link"); + } + + /** + * 获取步骤实例名占位符 + * + * @return 步骤实例名占位符 + */ + public String getStepInstanceNamePlaceHolder() { + return getTemplateVarPlaceHolder("step_instance_name"); + } +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/ChatServiceImpl.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/ChatServiceImpl.java new file mode 100644 index 0000000000..6042f97e67 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/ChatServiceImpl.java @@ -0,0 +1,260 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.service.ai.impl; + +import com.tencent.bk.job.analysis.consts.AIChatStatusEnum; +import com.tencent.bk.job.analysis.listener.event.AIChatOperationEvent; +import com.tencent.bk.job.analysis.model.dto.AIChatHistoryDTO; +import com.tencent.bk.job.analysis.model.dto.AIPromptDTO; +import com.tencent.bk.job.analysis.model.web.resp.AIAnswer; +import com.tencent.bk.job.analysis.model.web.resp.AIChatRecord; +import com.tencent.bk.job.analysis.mq.AIChatOperationEventDispatcher; +import com.tencent.bk.job.analysis.service.ai.AIChatHistoryService; +import com.tencent.bk.job.analysis.service.ai.AIService; +import com.tencent.bk.job.analysis.service.ai.ChatService; +import com.tencent.bk.job.analysis.service.ai.context.model.AsyncConsumerAndStreamingResponseBodyPair; +import com.tencent.bk.job.analysis.util.ai.AIAnswerUtil; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.NotFoundException; +import com.tencent.bk.job.common.model.Response; +import io.micrometer.core.instrument.util.StringUtils; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.i18n.LocaleContextHolder; +import org.springframework.stereotype.Service; +import org.springframework.web.servlet.mvc.method.annotation.StreamingResponseBody; + +import java.util.Comparator; +import java.util.List; +import java.util.Locale; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +/** + * AI对话服务实现类 + */ +@Slf4j +@Service +public class ChatServiceImpl implements ChatService { + + private final AIChatHistoryService aiChatHistoryService; + private final AIAnswerHandler aiAnswerHandler; + private final AIService aiService; + private final AIChatOperationEventDispatcher aiChatOperationEventDispatcher; + private final ConcurrentHashMap> futureMap = new ConcurrentHashMap<>(); + + @Autowired + public ChatServiceImpl(AIChatHistoryService aiChatHistoryService, + AIAnswerHandler aiAnswerHandler, + AIService aiService, AIChatOperationEventDispatcher aiChatOperationEventDispatcher) { + this.aiChatHistoryService = aiChatHistoryService; + this.aiAnswerHandler = aiAnswerHandler; + this.aiService = aiService; + this.aiChatOperationEventDispatcher = aiChatOperationEventDispatcher; + } + + /** + * 与AI对话 + * + * @param username 用户名 + * @param appId Job业务ID + * @param userInput 用户输入 + * @return AI对话记录 + */ + @Override + public AIChatRecord chatWithAI(String username, Long appId, String userInput) { + Long startTime = System.currentTimeMillis(); + // 1.保存初始对话记录 + AIPromptDTO aiPromptDTO = new AIPromptDTO(null, userInput, userInput); + AIChatHistoryDTO aiChatHistoryDTO = aiChatHistoryService.buildAIChatHistoryDTO( + username, + appId, + startTime, + aiPromptDTO, + AIChatStatusEnum.INIT.getStatus(), + null + ); + Long historyId = aiChatHistoryService.insertChatHistory(aiChatHistoryDTO); + aiChatHistoryDTO.setId(historyId); + return aiChatHistoryDTO.toAIChatRecord(); + } + + /** + * 获取最新对话记录列表 + * + * @param username 用户名 + * @param start 起始位置 + * @param length 长度 + * @return 最新对话记录列表 + */ + @Override + public List getLatestChatHistoryList(String username, Integer start, Integer length) { + return aiChatHistoryService.getLatestChatHistoryList(username, start, length); + } + + /** + * 生成对话流式数据 + * + * @param username 用户名 + * @param recordId 对话记录ID + * @return 对话流式数据 + */ + @Override + public StreamingResponseBody generateChatStream(String username, Long recordId) { + log.info("Start to generateChatStream, username={}, recordId={}", username, recordId); + // 1.查出指定的对话记录 + AIChatHistoryDTO currentChatHistoryDTO = getChatHistory(username, recordId); + // 2.已经回答完成直接输出 + if (currentChatHistoryDTO.isFinished()) { + log.info( + "Chat is already finished, use existing answer, lastAnswerTime={}", + currentChatHistoryDTO.getAnswerTimeStr() + ); + return generateRespFromFinishedAIAnswer(currentChatHistoryDTO); + } + // 3.未回答完成则调用AI生成回答 + // 获取最近已完成的对话记录作为上下文 + List chatHistoryDTOList = buildChatHistory(username); + int affectedNum = aiChatHistoryService.setAIAnswerReplying(recordId); + if (affectedNum == 0) { + log.info("AIAnswer is already replying, re-reply, recordId={}", recordId); + } + // 4.将AI回答流数据与接口输出流进行同步对接 + final int inMemoryMessageMaxNum = 10000; + AIAnswerStreamSynchronizer aiAnswerStreamSynchronizer = new AIAnswerStreamSynchronizer(inMemoryMessageMaxNum); + AsyncConsumerAndStreamingResponseBodyPair consumerAndStreamingResponseBodyPair = + aiAnswerStreamSynchronizer.buildAsyncConsumerAndStreamingResponseBodyPair(); + CompletableFuture future = aiService.getAIAnswerStream( + chatHistoryDTOList, + currentChatHistoryDTO.getAiInput(), + consumerAndStreamingResponseBodyPair.getConsumer() + ); + Locale locale = LocaleContextHolder.getLocale(); + log.debug("language={}", locale.getLanguage()); + future.whenComplete((content, throwable) -> { + // 5.处理AI回复内容 + LocaleContextHolder.setLocale(locale); + aiAnswerHandler.handleAIAnswer(recordId, content, throwable); + futureMap.remove(recordId); + aiAnswerStreamSynchronizer.triggerEndEvent(throwable); + }); + futureMap.put(recordId, future); + return consumerAndStreamingResponseBodyPair.getStreamingResponseBody(); + } + + /** + * 根据已完成的对话记录直接生成回复流，不再请求AI接口 + * + * @param currentChatHistoryDTO 对话记录 + * @return 回复流 + */ + private StreamingResponseBody generateRespFromFinishedAIAnswer(AIChatHistoryDTO currentChatHistoryDTO) { + return outputStream -> { + AIAnswer aiAnswer = AIAnswer.successAnswer(currentChatHistoryDTO.getAiAnswer()); + Response respBody = Response.buildSuccessResp(aiAnswer); + AIAnswerUtil.setRequestIdAndWriteResp(outputStream, respBody); + outputStream.close(); + }; + } + + /** + * 构建对话历史记录 + * + * @param username 用户名 + * @return 对话历史记录 + */ + private List buildChatHistory(String username) { + final int maxChatHistoryNum = 5; + List chatHistoryDTOList = aiChatHistoryService.getLatestFinishedChatHistoryList( + username, + 0, + maxChatHistoryNum + ); + // 过滤出输入输出均不为空的对话记录作为上下文 + chatHistoryDTOList = chatHistoryDTOList.stream().filter(aiChatHistoryDTO -> { + String userInput = aiChatHistoryDTO.getUserInput(); + String aiAnswer = aiChatHistoryDTO.getAiAnswer(); + return StringUtils.isNotBlank(userInput) && StringUtils.isNotBlank(aiAnswer); + }).collect(Collectors.toList()); + chatHistoryDTOList.sort(Comparator.comparing(AIChatHistoryDTO::getStartTime)); + return chatHistoryDTOList; + } + + /** + * 根据ID获取指定对话记录 + * + * @param username 用户名 + * @param recordId 对话记录ID + * @return 对话记录 + */ + private AIChatHistoryDTO getChatHistory(String username, Long recordId) { + AIChatHistoryDTO chatHistoryDTO = aiChatHistoryService.getChatHistory(username, recordId); + if (chatHistoryDTO == null) { + throw new NotFoundException( + ErrorCode.AI_CHAT_HISTORY_NOT_FOUND_BY_ID, + new String[]{String.valueOf(recordId)} + ); + } + return chatHistoryDTO; + } + + /** + * 终止当前实例上正在进行的指定ID的对话（如果存在） + * + * @param username 用户名 + * @param recordId 对话记录ID + * @return 是否操作成功 + */ + public boolean terminateChat(String username, Long recordId) { + CompletableFuture future = futureMap.get(recordId); + if (future == null) { + log.info("Cannot find future for recordId={}, chat maybe finished or canceled just now", recordId); + return false; + } + boolean result = future.cancel(true); + log.info("Terminate chat, username={}, recordId={}, result={}", username, recordId, result); + return result; + } + + /** + * 通过MQ广播消息，通知所有实例终止对话 + * + * @param username 用户名 + * @param recordId 对话记录ID + * @return 是否操作成功 + */ + public boolean triggerTerminateChat(String username, Long recordId) { + AIChatHistoryDTO chatHistoryDTO = getChatHistory(username, recordId); + if (!chatHistoryDTO.isInitOrReplying()) { + log.info("Cannot terminate chat, chat is already finished or canceled, recordId={}", recordId); + return false; + } + aiChatOperationEventDispatcher.broadCastAIChatOperationEvent( + AIChatOperationEvent.terminateChat(username, recordId) + ); + return true; + } +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/CheckScriptAIPromptServiceImpl.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/CheckScriptAIPromptServiceImpl.java new file mode 100644 index 0000000000..13a15acacd --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/CheckScriptAIPromptServiceImpl.java @@ -0,0 +1,122 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.service.ai.impl; + +import com.tencent.bk.job.analysis.consts.PromptTemplateCodeEnum; +import com.tencent.bk.job.analysis.dao.AIPromptTemplateDAO; +import com.tencent.bk.job.analysis.model.dto.AIPromptDTO; +import com.tencent.bk.job.analysis.model.dto.AIPromptTemplateDTO; +import com.tencent.bk.job.analysis.service.ai.CheckScriptAIPromptService; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.InternalException; +import com.tencent.bk.job.common.util.JobContextUtil; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +/** + * 检查脚本的AI提示符服务实现类 + */ +@Slf4j +@Service +public class CheckScriptAIPromptServiceImpl implements CheckScriptAIPromptService { + + private final ScriptTemplateService scriptTemplateService; + private final AIPromptTemplateDAO aiPromptTemplateDAO; + private final AITemplateVarService aiTemplateVarService; + + @Autowired + public CheckScriptAIPromptServiceImpl(ScriptTemplateService scriptTemplateService, + AIPromptTemplateDAO aiPromptTemplateDAO, + AITemplateVarService aiTemplateVarService) { + this.scriptTemplateService = scriptTemplateService; + this.aiPromptTemplateDAO = aiPromptTemplateDAO; + this.aiTemplateVarService = aiTemplateVarService; + } + + /** + * 根据脚本类型与内容获取检查脚本的AI提示符 + * + * @param type 脚本类型 + * @param scriptContent 脚本内容 + * @return AI提示符 + */ + @Override + public AIPromptDTO getPrompt(Integer type, String scriptContent) { + String userLang = JobContextUtil.getUserLang(); + AIPromptTemplateDTO promptTemplate = aiPromptTemplateDAO.getAIPromptTemplate( + PromptTemplateCodeEnum.CHECK_SCRIPT.name(), + userLang + ); + if (promptTemplate == null) { + String message = "Cannot find prompt template for (code=CHECK_SCRIPT, userLang=" + userLang + "), " + + "please check template config in DB"; + throw new InternalException(message, ErrorCode.INTERNAL_ERROR); + } + log.info( + "Use prompt template [{}(id={})], userLang={}", + promptTemplate.getName(), + promptTemplate.getId(), + userLang + ); + String scriptTemplate = scriptTemplateService.getScriptTemplate(type); + String renderedPrompt = renderCheckScriptPrompt( + promptTemplate.getTemplate(), + type, + scriptTemplate, + scriptContent + ); + return new AIPromptDTO(promptTemplate.getId(), promptTemplate.getRawPrompt(), renderedPrompt); + } + + /** + * 渲染检查脚本的AI提示符 + * + * @param promptTemplateContent 提示符模板内容 + * @param type 脚本类型 + * @param scriptTemplate 脚本模板 + * @param scriptContent 脚本内容 + * @return 渲染后的提示符 + */ + private String renderCheckScriptPrompt(String promptTemplateContent, + Integer type, + String scriptTemplate, + String scriptContent) { + String scriptTypeName = ScriptTypeEnum.getName(type); + promptTemplateContent = promptTemplateContent.replace( + aiTemplateVarService.getScriptTypePlaceHolder(), scriptTypeName + ); + if (scriptTemplate != null) { + promptTemplateContent = promptTemplateContent.replace( + aiTemplateVarService.getScriptTemplatePlaceHolder(), scriptTemplate + ); + } + promptTemplateContent = promptTemplateContent.replace( + aiTemplateVarService.getScriptContentPlaceHolder(), scriptContent + ); + return promptTemplateContent; + } +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/FileTransferTaskErrorAIPromptServiceImpl.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/FileTransferTaskErrorAIPromptServiceImpl.java new file mode 100644 index 0000000000..ce7396272c --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/FileTransferTaskErrorAIPromptServiceImpl.java @@ -0,0 +1,94 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.service.ai.impl; + +import com.tencent.bk.job.analysis.consts.PromptTemplateCodeEnum; +import com.tencent.bk.job.analysis.dao.AIPromptTemplateDAO; +import com.tencent.bk.job.analysis.model.dto.AIPromptDTO; +import com.tencent.bk.job.analysis.model.dto.AIPromptTemplateDTO; +import com.tencent.bk.job.analysis.service.ai.FileTransferTaskErrorAIPromptService; +import com.tencent.bk.job.analysis.service.ai.context.model.FileTaskContext; +import com.tencent.bk.job.common.config.BkConfig; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +/** + * 文件分发任务报错分析AI提示符服务 + */ +@Slf4j +@Service +public class FileTransferTaskErrorAIPromptServiceImpl extends AIBasePromptService + implements FileTransferTaskErrorAIPromptService { + + private final AITemplateVarService aiTemplateVarService; + private final BkConfig bkConfig; + private final AIMessageI18nService aiMessageI18nService; + + @Autowired + public FileTransferTaskErrorAIPromptServiceImpl(AIPromptTemplateDAO aiPromptTemplateDAO, + AITemplateVarService aiTemplateVarService, + BkConfig bkConfig, + AIMessageI18nService aiMessageI18nService) { + super(aiPromptTemplateDAO); + this.aiTemplateVarService = aiTemplateVarService; + this.bkConfig = bkConfig; + this.aiMessageI18nService = aiMessageI18nService; + } + + /** + * 根据文件任务上下文获取文件分发任务报错分析AI提示符 + * + * @param context 文件分发任务上下文 + * @return AI提示符 + */ + @Override + public AIPromptDTO getPrompt(FileTaskContext context) { + String templateCode = PromptTemplateCodeEnum.ANALYZE_FILE_TRANSFER_TASK_ERROR.name(); + AIPromptTemplateDTO promptTemplate = getPromptTemplate(templateCode); + String renderedRawPrompt = renderPrompt(promptTemplate.getRawPrompt(), context); + String renderedPrompt = renderPrompt(promptTemplate.getTemplate(), context); + return new AIPromptDTO(promptTemplate.getId(), renderedRawPrompt, renderedPrompt); + } + + /** + * 渲染AI提示符 + * + * @param promptTemplateContent AI提示符模板内容 + * @param context 文件任务上下文 + * @return 渲染后的AI提示符 + */ + private String renderPrompt(String promptTemplateContent, FileTaskContext context) { + return promptTemplateContent + .replace(aiTemplateVarService.getStepInstanceNamePlaceHolder(), context.getName()) + .replace(aiTemplateVarService.getBkHelperLinkPlaceHolder(), bkConfig.getBkHelperLink()) + .replace( + aiTemplateVarService.getFileTaskErrorSourcePlaceHolder(), + aiMessageI18nService.getI18nMessage(context.getFileTaskErrorSourceI18nKey()) + ) + .replace(aiTemplateVarService.getUploadFileErrorDataPlaceHolder(), context.getUploadFileErrorData()) + .replace(aiTemplateVarService.getDownloadFileErrorDataPlaceHolder(), context.getDownloadFileErrorData()); + } +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/ScriptExecuteTaskErrorAIPromptServiceImpl.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/ScriptExecuteTaskErrorAIPromptServiceImpl.java new file mode 100644 index 0000000000..beafe9cb0f --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/ScriptExecuteTaskErrorAIPromptServiceImpl.java @@ -0,0 +1,94 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.service.ai.impl; + +import com.tencent.bk.job.analysis.consts.PromptTemplateCodeEnum; +import com.tencent.bk.job.analysis.dao.AIPromptTemplateDAO; +import com.tencent.bk.job.analysis.model.dto.AIPromptDTO; +import com.tencent.bk.job.analysis.model.dto.AIPromptTemplateDTO; +import com.tencent.bk.job.analysis.service.ai.ScriptExecuteTaskErrorAIPromptService; +import com.tencent.bk.job.analysis.service.ai.context.model.ScriptTaskContext; +import com.tencent.bk.job.common.config.BkConfig; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +/** + * 脚本任务报错检查AI提示符服务 + */ +@Slf4j +@Service +public class ScriptExecuteTaskErrorAIPromptServiceImpl extends AIBasePromptService + implements ScriptExecuteTaskErrorAIPromptService { + + private final AITemplateVarService aiTemplateVarService; + private final BkConfig bkConfig; + + @Autowired + public ScriptExecuteTaskErrorAIPromptServiceImpl(AIPromptTemplateDAO aiPromptTemplateDAO, + AITemplateVarService aiTemplateVarService, + BkConfig bkConfig) { + super(aiPromptTemplateDAO); + this.aiTemplateVarService = aiTemplateVarService; + this.bkConfig = bkConfig; + } + + /** + * 根据脚本任务上下文与报错信息获取AI提示符 + * + * @param context 脚本任务上下文 + * @param errorContent 报错内容 + * @return AI提示符 + */ + @Override + public AIPromptDTO getPrompt(ScriptTaskContext context, String errorContent) { + String templateCode = PromptTemplateCodeEnum.ANALYZE_SCRIPT_EXECUTE_TASK_ERROR.name(); + AIPromptTemplateDTO promptTemplate = getPromptTemplate(templateCode); + String renderedRawPrompt = renderPrompt(promptTemplate.getRawPrompt(), context, errorContent); + String renderedPrompt = renderPrompt(promptTemplate.getTemplate(), context, errorContent); + return new AIPromptDTO(promptTemplate.getId(), renderedRawPrompt, renderedPrompt); + } + + /** + * 渲染AI提示符 + * + * @param promptTemplateContent 提示符模板内容 + * @param context 脚本任务上下文 + * @param errorContent 报错信息 + * @return 渲染后的提示符 + */ + private String renderPrompt(String promptTemplateContent, + ScriptTaskContext context, + String errorContent) { + return promptTemplateContent + .replace(aiTemplateVarService.getStepInstanceNamePlaceHolder(), context.getName()) + .replace(aiTemplateVarService.getBkHelperLinkPlaceHolder(), bkConfig.getBkHelperLink()) + .replace(aiTemplateVarService.getScriptTypePlaceHolder(), ScriptTypeEnum.getName(context.getScriptType())) + .replace(aiTemplateVarService.getScriptParamsPlaceHolder(), context.getInsensitiveScriptParamsStr()) + .replace(aiTemplateVarService.getErrorContentPlaceHolder(), errorContent) + .replace(aiTemplateVarService.getScriptContentPlaceHolder(), context.getScriptContent()); + } +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/ScriptTemplateService.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/ScriptTemplateService.java new file mode 100644 index 0000000000..adffd53bb4 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/ai/impl/ScriptTemplateService.java @@ -0,0 +1,104 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.service.ai.impl; + +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.tencent.bk.job.common.model.InternalResponse; +import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.manage.api.inner.ServiceScriptTemplateResource; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; + +/** + * 脚本模板服务 + */ +@Slf4j +@Service +public class ScriptTemplateService { + + private final ServiceScriptTemplateResource scriptTemplateResource; + /** + * 最大脚本模板数量 + */ + private final int maxScriptTemplateNum = 6; + /** + * 脚本模板缓存过期时间（天） + */ + private final int scriptTemplateCacheExpireDays = 1; + + /** + * 各种语言类型的脚本模板数量有限且很少变动，使用本地缓存提高加载速率，也避免频繁的服务调用 + */ + private final LoadingCache scriptTemplateCache = CacheBuilder.newBuilder() + .maximumSize(maxScriptTemplateNum).expireAfterWrite(scriptTemplateCacheExpireDays, TimeUnit.DAYS) + .build(new CacheLoader() { + @SuppressWarnings("all") + @Override + public String load(Integer scriptType) { + return getScriptTemplateIndeed(scriptType); + } + } + ); + + @Autowired + public ScriptTemplateService(ServiceScriptTemplateResource scriptTemplateResource) { + this.scriptTemplateResource = scriptTemplateResource; + } + + /** + * 根据脚本类型获取脚本模板，优先从缓存获取，如果缓存未命中则通过服务调用获取 + * + * @param scriptType 脚本类型 + * @return 脚本模板 + */ + public String getScriptTemplate(Integer scriptType) { + try { + return scriptTemplateCache.get(scriptType); + } catch (ExecutionException e) { + log.warn("Fail to getScriptTemplate by type:{}", scriptType); + return getScriptTemplateIndeed(scriptType); + } + } + + /** + * 通过服务调用获取脚本模板 + * + * @param scriptType 脚本类型 + * @return 脚本模板 + */ + public String getScriptTemplateIndeed(Integer scriptType) { + InternalResponse resp = scriptTemplateResource.getScriptTemplate(scriptType); + if (log.isDebugEnabled()) { + log.debug("resp={}", JsonUtils.toJson(resp)); + } + return resp.getData(); + } +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/login/LoginTokenService.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/login/LoginTokenService.java new file mode 100644 index 0000000000..8604da95a6 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/login/LoginTokenService.java @@ -0,0 +1,29 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.service.login; + +public interface LoginTokenService { + String getToken(); +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/login/impl/LoginTokenServiceImpl.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/login/impl/LoginTokenServiceImpl.java new file mode 100644 index 0000000000..205eb618e6 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/login/impl/LoginTokenServiceImpl.java @@ -0,0 +1,99 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.service.login.impl; + +import com.tencent.bk.job.analysis.service.login.LoginTokenService; +import com.tencent.bk.job.common.aidev.config.CustomPaasLoginProperties; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.InternalException; +import com.tencent.bk.job.common.util.JobContextUtil; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import javax.servlet.http.Cookie; +import javax.servlet.http.HttpServletRequest; + +@Slf4j +@Service +public class LoginTokenServiceImpl implements LoginTokenService { + private final CustomPaasLoginProperties customPaasLoginProperties; + + @Autowired + public LoginTokenServiceImpl(CustomPaasLoginProperties customPaasLoginProperties) { + this.customPaasLoginProperties = customPaasLoginProperties; + } + + @Override + public String getToken() { + String token = customPaasLoginProperties.getToken(); + if (StringUtils.isNotBlank(token)) { + return token; + } + if (!customPaasLoginProperties.isEnabled()) { + return getBkToken(); + } + return getBkTicket(); + } + + private String getBkToken() { + return getCookieValue("bk_token"); + } + + private String getBkTicket() { + return getCookieValue("bk_ticket"); + } + + /** + * 从请求中获取cookie + * + * @param cookieName cookie名称 + * @return cookie值 + */ + private String getCookieValue(String cookieName) { + HttpServletRequest request = JobContextUtil.getRequest(); + Cookie[] cookies = request.getCookies(); + if (cookies == null) { + String message = "Cookies is null, please check request in thread context"; + throw new InternalException(message, ErrorCode.INTERNAL_ERROR); + } + for (Cookie cookie : cookies) { + if (cookieName.equals(cookie.getName())) { + return cookie.getValue(); + } + } + if (log.isDebugEnabled()) { + StringBuilder cookiesSb = new StringBuilder(); + for (Cookie cookie : cookies) { + cookiesSb.append(cookie.toString()); + cookiesSb.append(","); + } + log.debug("Cookies={}", cookiesSb); + } + String message = "Cannot find cookie by name: " + cookieName; + throw new InternalException(message, ErrorCode.INTERNAL_ERROR); + } +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/remote/ApplicationServiceImpl.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/remote/ApplicationServiceImpl.java index 165f522ad9..cce3f0ca01 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/remote/ApplicationServiceImpl.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/remote/ApplicationServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,8 +24,8 @@ package com.tencent.bk.job.analysis.service.remote; -import com.tencent.bk.job.analysis.client.ApplicationResourceClient; import com.tencent.bk.job.analysis.service.ApplicationService; +import com.tencent.bk.job.manage.api.inner.ServiceApplicationResource; import com.tencent.bk.job.manage.model.inner.resp.ServiceApplicationDTO; import lombok.extern.slf4j.Slf4j; import lombok.val; @@ -35,18 +35,18 @@ import java.util.List; @Slf4j -@Service +@Service("jobAnalysisApplicationService") public class ApplicationServiceImpl implements ApplicationService { - private final ApplicationResourceClient applicationResourceClient; + private final ServiceApplicationResource applicationResource; @Autowired - public ApplicationServiceImpl(ApplicationResourceClient applicationResourceClient) { - this.applicationResourceClient = applicationResourceClient; + public ApplicationServiceImpl(ServiceApplicationResource applicationResource) { + this.applicationResource = applicationResource; } @Override public List listLocalDBApps() { - val result = applicationResourceClient.listApps(null); + val result = applicationResource.listApps(null); if (result == null) { throw new RuntimeException("Job-manage unavailable, please check"); } else { diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/remote/HostServiceImpl.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/remote/HostServiceImpl.java index 4330611407..04eb9e4ce0 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/remote/HostServiceImpl.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/remote/HostServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,13 +24,14 @@ package com.tencent.bk.job.analysis.service.remote; -import com.tencent.bk.job.analysis.client.HostResourceClient; import com.tencent.bk.job.analysis.service.HostService; +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.manage.api.inner.ServiceHostResource; import com.tencent.bk.job.manage.model.inner.ServiceHostStatusDTO; import com.tencent.bk.job.manage.model.inner.request.ServiceGetHostStatusByDynamicGroupReq; -import com.tencent.bk.job.manage.model.inner.request.ServiceGetHostStatusByIpReq; +import com.tencent.bk.job.manage.model.inner.request.ServiceGetHostStatusByHostReq; import com.tencent.bk.job.manage.model.inner.request.ServiceGetHostStatusByNodeReq; -import com.tencent.bk.job.manage.model.web.request.ipchooser.AppTopologyTreeNode; +import com.tencent.bk.job.manage.model.web.request.chooser.host.BizTopoNode; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -38,28 +39,30 @@ import java.util.List; @Slf4j -@Service +@Service("jobAnalysisHostService") public class HostServiceImpl implements HostService { - private final HostResourceClient hostResourceClient; + private final ServiceHostResource hostResource; @Autowired - public HostServiceImpl(HostResourceClient hostResourceClient) { - this.hostResourceClient = hostResourceClient; + public HostServiceImpl(ServiceHostResource hostResource) { + this.hostResource = hostResource; } @Override - public List getHostStatusByNode(Long appId, List treeNodeList) { - return hostResourceClient.getHostStatusByNode(appId, new ServiceGetHostStatusByNodeReq(treeNodeList)).getData(); + public List getHostStatusByNode(Long appId, List treeNodeList) { + return hostResource.getHostStatusByNode(appId, new ServiceGetHostStatusByNodeReq(treeNodeList)).getData(); } @Override public List getHostStatusByDynamicGroup(Long appId, List dynamicGroupIdList) { - return hostResourceClient.getHostStatusByDynamicGroup(appId, + return hostResource.getHostStatusByDynamicGroup(appId, new ServiceGetHostStatusByDynamicGroupReq(dynamicGroupIdList)).getData(); } @Override - public List getHostStatusByIp(Long appId, List ipList) { - return hostResourceClient.getHostStatusByIp(appId, new ServiceGetHostStatusByIpReq(ipList)).getData(); + public List getHostStatusByHost(Long appId, + List hostList) { + return hostResource.getHostStatusByHost(appId, + new ServiceGetHostStatusByHostReq(hostList)).getData(); } } diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/remote/ScriptServiceImpl.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/remote/ScriptServiceImpl.java index bfcbd65d01..1937d5dd5e 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/remote/ScriptServiceImpl.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/remote/ScriptServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,9 +24,9 @@ package com.tencent.bk.job.analysis.service.remote; -import com.tencent.bk.job.analysis.client.ScriptResourceClient; import com.tencent.bk.job.analysis.service.ScriptService; import com.tencent.bk.job.common.exception.ServiceException; +import com.tencent.bk.job.manage.api.inner.ServiceScriptResource; import com.tencent.bk.job.manage.model.inner.ServiceScriptDTO; import lombok.extern.slf4j.Slf4j; import lombok.val; @@ -34,13 +34,13 @@ import org.springframework.stereotype.Service; @Slf4j -@Service +@Service("jobAnalysisScriptService") public class ScriptServiceImpl implements ScriptService { - private final ScriptResourceClient scriptResourceClient; + private final ServiceScriptResource scriptResource; @Autowired - public ScriptServiceImpl(ScriptResourceClient scriptResourceClient) { - this.scriptResourceClient = scriptResourceClient; + public ScriptServiceImpl(ServiceScriptResource scriptResource) { + this.scriptResource = scriptResource; } @Override @@ -49,7 +49,7 @@ public ServiceScriptDTO getScriptVersion( Long appId, Long scriptVersionId ) throws ServiceException { - val result = scriptResourceClient.getScriptByScriptVersionId(scriptVersionId); + val result = scriptResource.getScriptByScriptVersionId(scriptVersionId); if (result != null) { return result.getData(); } else { diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/remote/TaskPlanServiceImpl.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/remote/TaskPlanServiceImpl.java index 611f890e15..dcd113351c 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/remote/TaskPlanServiceImpl.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/remote/TaskPlanServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,8 +24,8 @@ package com.tencent.bk.job.analysis.service.remote; -import com.tencent.bk.job.analysis.client.TaskPlanResourceClient; import com.tencent.bk.job.analysis.service.TaskPlanService; +import com.tencent.bk.job.manage.api.inner.ServiceTaskPlanResource; import com.tencent.bk.job.manage.model.inner.ServiceTaskPlanDTO; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; @@ -34,22 +34,22 @@ import java.util.List; @Slf4j -@Service +@Service("jobAnalysisTaskPlanService") public class TaskPlanServiceImpl implements TaskPlanService { - private final TaskPlanResourceClient taskPlanResourceClient; + private final ServiceTaskPlanResource taskPlanResource; @Autowired - public TaskPlanServiceImpl(TaskPlanResourceClient taskPlanResourceClient) { - this.taskPlanResourceClient = taskPlanResourceClient; + public TaskPlanServiceImpl(ServiceTaskPlanResource taskPlanResource) { + this.taskPlanResource = taskPlanResource; } @Override public List listTaskPlanIds(Long templateId) { - return taskPlanResourceClient.listPlanIds(templateId).getData(); + return taskPlanResource.listPlanIds(templateId).getData(); } @Override public ServiceTaskPlanDTO getTaskPlanById(Long appId, Long planId) { - return taskPlanResourceClient.getPlanById(appId, planId, true).getData(); + return taskPlanResource.getPlanById(appId, planId, false).getData(); } } diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/remote/TaskTemplateServiceImpl.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/remote/TaskTemplateServiceImpl.java index c0bae6836e..888154e41e 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/remote/TaskTemplateServiceImpl.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/service/remote/TaskTemplateServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,10 +24,10 @@ package com.tencent.bk.job.analysis.service.remote; -import com.tencent.bk.job.analysis.client.ServiceTaskTemplateResourceClient; import com.tencent.bk.job.analysis.service.TaskTemplateService; import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; +import com.tencent.bk.job.manage.api.inner.ServiceTaskTemplateResource; import com.tencent.bk.job.manage.model.inner.ServiceTaskTemplateDTO; import lombok.extern.slf4j.Slf4j; import lombok.val; @@ -35,20 +35,20 @@ import org.springframework.stereotype.Service; @Slf4j -@Service +@Service("jobAnalysisTaskTemplateServiceImpl") public class TaskTemplateServiceImpl implements TaskTemplateService { - private final ServiceTaskTemplateResourceClient taskTemplateResourceClient; + private final ServiceTaskTemplateResource taskTemplateResource; @Autowired - public TaskTemplateServiceImpl(ServiceTaskTemplateResourceClient taskTemplateResourceClient) { - this.taskTemplateResourceClient = taskTemplateResourceClient; + public TaskTemplateServiceImpl(ServiceTaskTemplateResource taskTemplateResource) { + this.taskTemplateResource = taskTemplateResource; } @Override public PageData listPageTaskTemplates(Long appId, BaseSearchCondition baseSearchCondition) { val result = - taskTemplateResourceClient.listPageTaskTemplates(appId, + taskTemplateResource.listPageTaskTemplates(appId, baseSearchCondition.getStart(), baseSearchCondition.getLength()); if (result != null) { return result.getData(); diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/ScheduledTasks.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/ScheduledTasks.java index 108671a489..73b208c1a4 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/ScheduledTasks.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/ScheduledTasks.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,7 @@ package com.tencent.bk.job.analysis.task; +import com.tencent.bk.job.analysis.task.ai.AIChatHistoryCleanTask; import com.tencent.bk.job.analysis.task.analysis.AnalysisTaskScheduler; import com.tencent.bk.job.analysis.task.statistics.StatisticsTaskScheduler; import org.slf4j.Logger; @@ -33,24 +34,22 @@ import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; -/** - * @Description 增加调度需要注意到ScheduleConfig中更新线程池配置 - * @Date 2020/1/3 - * @Version 1.0 - */ -@Component +@Component("jobAnalysisScheduledTasks") @EnableScheduling public class ScheduledTasks { private static final Logger logger = LoggerFactory.getLogger(ScheduledTasks.class); private final AnalysisTaskScheduler analysisTaskScheduler; private final StatisticsTaskScheduler statisticsTaskScheduler; + private final AIChatHistoryCleanTask aiChatHistoryCleanTask; @Autowired public ScheduledTasks(AnalysisTaskScheduler analysisTaskScheduler, - StatisticsTaskScheduler statisticsTaskScheduler) { + StatisticsTaskScheduler statisticsTaskScheduler, + AIChatHistoryCleanTask aiChatHistoryCleanTask) { this.analysisTaskScheduler = analysisTaskScheduler; this.statisticsTaskScheduler = statisticsTaskScheduler; + this.aiChatHistoryCleanTask = aiChatHistoryCleanTask; } /** @@ -82,4 +81,16 @@ public void analysisTaskSchedulerTask() { logger.error("analysisTaskSchedulerTask fail", e); } } + + /** + * 定时清理用户产生的AI对话记录，1h一次 + */ + @Scheduled(cron = "0 20 * * * *") + public void aiChatHistoryCleanTask() { + try { + aiChatHistoryCleanTask.execute(); + } catch (Exception e) { + logger.error("aiChatHistoryCleanTask fail", e); + } + } } diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/ai/AIChatHistoryCleanTask.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/ai/AIChatHistoryCleanTask.java new file mode 100644 index 0000000000..d71a55c5f2 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/ai/AIChatHistoryCleanTask.java @@ -0,0 +1,188 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.task.ai; + + +import com.tencent.bk.job.analysis.config.AIProperties; +import com.tencent.bk.job.analysis.dao.AIChatHistoryDAO; +import com.tencent.bk.job.common.redis.util.DistributedUniqueTask; +import com.tencent.bk.job.common.util.ThreadUtils; +import com.tencent.bk.job.common.util.TimeUtil; +import com.tencent.bk.job.common.util.ip.IpUtils; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.stereotype.Component; +import org.springframework.util.StopWatch; + +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.List; + +/** + * 清理AI对话记录的任务 + */ +@Slf4j +@Component +public class AIChatHistoryCleanTask { + + private static final String machineIp = IpUtils.getFirstMachineIP(); + private static final String REDIS_KEY_AI_CHAT_HISTORY_CLEAN_TASK_RUNNING_MACHINE = + "aiChatHistoryCleanTask-running-machine"; + private static final String TIME_FORMAT = "yyyy-MM-dd HH:mm:ss"; + private final RedisTemplate redisTemplate; + private final AIProperties aiProperties; + private final AIChatHistoryDAO aiChatHistoryDAO; + + @Autowired + public AIChatHistoryCleanTask(RedisTemplate redisTemplate, + AIProperties aiProperties, + AIChatHistoryDAO aiChatHistoryDAO) { + this.redisTemplate = redisTemplate; + this.aiProperties = aiProperties; + this.aiChatHistoryDAO = aiChatHistoryDAO; + } + + /** + * 清理任务执行入口，多实例并发仅随机选择一个实例真正执行 + */ + public void execute() { + log.info("AIChatHistoryCleanTask start"); + StopWatch watch = new StopWatch(); + Boolean successExecuted = false; + try { + // 分布式唯一性保证 + successExecuted = new DistributedUniqueTask<>( + redisTemplate, + AIChatHistoryCleanTask.class.getSimpleName(), + REDIS_KEY_AI_CHAT_HISTORY_CLEAN_TASK_RUNNING_MACHINE, + machineIp, + () -> { + doExecute(watch); + return true; + } + ).execute(); + } catch (Exception e) { + log.error("AIChatHistoryCleanTask failed", e); + } finally { + if (watch.isRunning()) { + watch.stop(); + } + if (successExecuted != null && successExecuted) { + log.info("AIChatHistoryCleanTask finished, timeConsuming={}", watch.prettyPrint()); + } + } + } + + /** + * 根据配置信息执行清理任务 + * + * @param watch 计时器 + */ + public void doExecute(StopWatch watch) { + watch.start("cleanChatHistoryByMaxKeepDays"); + Integer maxKeepDays = aiProperties.getChatHistory().getMaxKeepDays(); + cleanChatHistoryByMaxKeepDays(maxKeepDays); + watch.stop(); + watch.start("cleanChatHistoryByMaxHistoryPerUser"); + Integer maxHistoryPerUser = aiProperties.getChatHistory().getMaxHistoryPerUser(); + cleanChatHistoryByMaxHistoryPerUser(maxHistoryPerUser); + watch.stop(); + } + + /** + * 根据最大保留天数清理对话记录 + * + * @param maxKeepDays 最大保留天数 + */ + private void cleanChatHistoryByMaxKeepDays(Integer maxKeepDays) { + LocalDateTime currentDateTime = LocalDateTime.now(); + LocalDateTime lastKeepDate = currentDateTime.minusDays(maxKeepDays); + LocalDateTime lastKeepDateStartTime = TimeUtil.getDayStartTime(lastKeepDate); + long maxStartTimeMills = lastKeepDateStartTime.atZone(ZoneId.systemDefault()).toInstant().toEpochMilli(); + log.info( + "Begin to delete ai chat history before {}, maxStartTimeMills={}", + TimeUtil.getTimeStr(lastKeepDateStartTime, TIME_FORMAT), + maxStartTimeMills + ); + int batchSize = 1000; + int totalDeletedNum = 0; + int deletedNum; + do { + deletedNum = aiChatHistoryDAO.deleteChatHistory(maxStartTimeMills, batchSize); + totalDeletedNum += deletedNum; + if (deletedNum > 0) { + ThreadUtils.sleep(1000); + } + } while (deletedNum > 0); + log.info( + "Finish to cleanChatHistoryByMaxKeepDays({}), totalDeletedNum={}, maxStartTimeMills={}", + maxKeepDays, + totalDeletedNum, + maxStartTimeMills + ); + } + + /** + * 根据每个用户最大对话记录数清理对话记录 + * + * @param maxHistoryPerUser 每个用户最大对话记录数 + */ + private void cleanChatHistoryByMaxHistoryPerUser(Integer maxHistoryPerUser) { + // 1.查出所有用户 + List userList = aiChatHistoryDAO.listAllUserOfChatHistory(); + // 2.清理每个用户的对话历史记录 + for (String username : userList) { + Long maxId = aiChatHistoryDAO.getFirstIdAfterOffset(username, maxHistoryPerUser); + if (maxId == null) { + continue; + } + int deletedNum = cleanChatHistoryForUser(username, maxId); + log.info("Finish to cleanChatHistoryForUser({}), maxId={}, deletedNum={}", username, maxId, deletedNum); + } + log.info("cleanChatHistoryByMaxHistoryPerUser finished"); + } + + /** + * 清理某个用户的对话记录 + * + * @param username 用户名 + * @param maxId 最大记录id（包含） + * @return 删除的记录条数 + */ + private int cleanChatHistoryForUser(String username, Long maxId) { + int batchSize = 1000; + int totalDeletedNum = 0; + int deletedNum; + do { + deletedNum = aiChatHistoryDAO.deleteChatHistory(username, maxId, batchSize); + totalDeletedNum += deletedNum; + if (deletedNum > 0) { + ThreadUtils.sleep(100); + } + } while (deletedNum > 0); + return totalDeletedNum; + } +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/AnalysisTaskScheduler.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/AnalysisTaskScheduler.java index 450acd827e..af7269802d 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/AnalysisTaskScheduler.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/AnalysisTaskScheduler.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,9 +24,9 @@ package com.tencent.bk.job.analysis.task.analysis; +import com.tencent.bk.job.analysis.api.consts.StatisticsConstants; import com.tencent.bk.job.analysis.task.analysis.anotation.AnalysisTask; import com.tencent.bk.job.analysis.task.analysis.task.IAnalysisTask; -import com.tencent.bk.job.common.statistics.consts.StatisticsConstants; import com.tencent.bk.job.common.util.ApplicationContextRegister; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Tag; @@ -85,7 +85,6 @@ public static void findAnalysisTask() { List beanNames = Arrays.asList(context.getBeanDefinitionNames()); beanNames.forEach(beanName -> { Object bean = context.getBean(beanName); - log.info("check " + beanName); if (bean instanceof IAnalysisTask) { log.info("add " + beanName); IAnalysisTask taskBean = (IAnalysisTask) bean; diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/AnalysisTaskStatusEnum.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/AnalysisTaskStatusEnum.java index a012e7e064..db68ce16f5 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/AnalysisTaskStatusEnum.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/AnalysisTaskStatusEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/BaseAnalysisTask.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/BaseAnalysisTask.java index 425e5ba49b..909d08fff1 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/BaseAnalysisTask.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/BaseAnalysisTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -36,21 +36,14 @@ import com.tencent.bk.job.analysis.task.analysis.task.IAnalysisTask; import com.tencent.bk.job.manage.model.inner.resp.ServiceApplicationDTO; import lombok.extern.slf4j.Slf4j; -import org.jooq.DSLContext; import java.util.List; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -/** - * @Description - * @Date 2020/3/6 - * @Version 1.0 - */ @Slf4j public abstract class BaseAnalysisTask implements IAnalysisTask { - protected final DSLContext dslContext; protected final AnalysisTaskInstanceDAO analysisTaskInstanceDAO; private final String taskCode; private final ApplicationService applicationService; @@ -62,7 +55,7 @@ public abstract class BaseAnalysisTask implements IAnalysisTask { @Override public AnalysisTaskDTO load(String searchKey) throws Exception { if (searchKey.equals(KEY_ANALYSIS_TASK_DTO)) { - return analysisTaskDAO.getAnalysisTaskByCode(dslContext, taskCode); + return analysisTaskDAO.getAnalysisTaskByCode(taskCode); } else { return null; } @@ -70,9 +63,9 @@ public AnalysisTaskDTO load(String searchKey) throws Exception { } ); - public BaseAnalysisTask(DSLContext dslContext, AnalysisTaskDAO analysisTaskDAO, - AnalysisTaskInstanceDAO analysisTaskInstanceDAO, ApplicationService applicationService) { - this.dslContext = dslContext; + public BaseAnalysisTask(AnalysisTaskDAO analysisTaskDAO, + AnalysisTaskInstanceDAO analysisTaskInstanceDAO, + ApplicationService applicationService) { this.analysisTaskDAO = analysisTaskDAO; this.analysisTaskInstanceDAO = analysisTaskInstanceDAO; this.applicationService = applicationService; @@ -91,14 +84,14 @@ public void refreshAnalysisTask() { } public Long insertAnalysisTaskInstance(AnalysisTaskInstanceDTO analysisTaskInstanceDTO) { - return analysisTaskInstanceDAO.insertAnalysisTaskInstance(dslContext, analysisTaskInstanceDTO); + return analysisTaskInstanceDAO.insertAnalysisTaskInstance(analysisTaskInstanceDTO); } public int updateAnalysisTaskInstanceById(AnalysisTaskInstanceDTO analysisTaskInstanceDTO) { // 先将历史任务结果全部清除 - analysisTaskInstanceDAO.deleteHistoryAnalysisTaskInstance(dslContext, analysisTaskInstanceDTO.getAppId(), + analysisTaskInstanceDAO.deleteHistoryAnalysisTaskInstance(analysisTaskInstanceDTO.getAppId(), analysisTaskInstanceDTO.getTaskId()); - return analysisTaskInstanceDAO.updateAnalysisTaskInstanceById(dslContext, analysisTaskInstanceDTO); + return analysisTaskInstanceDAO.updateAnalysisTaskInstanceById(analysisTaskInstanceDTO); } public List getAppInfoList() { diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/anotation/AnalysisTask.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/anotation/AnalysisTask.java index 3dc60301f5..74554c3418 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/anotation/AnalysisTask.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/anotation/AnalysisTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/enums/AnalysisResourceEnum.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/enums/AnalysisResourceEnum.java index 4b79d7f9c9..50f7419e39 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/enums/AnalysisResourceEnum.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/enums/AnalysisResourceEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -49,9 +49,9 @@ public enum AnalysisResourceEnum { public static AnalysisResourceEnum get(int type) { val values = AnalysisResourceEnum.values(); - for (int i = 0; i < values.length; i++) { - if (values[i].type == type) { - return values[i]; + for (AnalysisResourceEnum value : values) { + if (value.type == type) { + return value; } } return null; diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/IAnalysisTask.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/IAnalysisTask.java index 1d694da4c1..e999fa2f1d 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/IAnalysisTask.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/IAnalysisTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -37,8 +37,15 @@ public interface IAnalysisTask extends Runnable { AnalysisTaskDTO getAnalysisTask(); - //生成每一条任务结果数据的结果文本 - AnalysisTaskResultVO generateResultVO(String descriptionTpl, String itemTpl, String data); + /** + * 渲染每一条分析任务结果数据的结果文本 + * + * @param descriptionTpl 总体概括描述模板 + * @param itemTpl 单条数据模板 + * @param data 数据 + * @return 封装好的分析结果数据实例 + */ + AnalysisTaskResultVO renderResultVO(String descriptionTpl, String itemTpl, String data); long getPeriodSeconds(); } diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/impl/AbstractTimerTaskWatcher.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/impl/AbstractTimerTaskWatcher.java index 0fd6e9f984..34cabffe07 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/impl/AbstractTimerTaskWatcher.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/impl/AbstractTimerTaskWatcher.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,8 +24,6 @@ package com.tencent.bk.job.analysis.task.analysis.task.impl; -import com.tencent.bk.job.analysis.client.CronJobResourceClient; -import com.tencent.bk.job.analysis.client.TaskExecuteResultResourceClient; import com.tencent.bk.job.analysis.dao.AnalysisTaskDAO; import com.tencent.bk.job.analysis.dao.AnalysisTaskInstanceDAO; import com.tencent.bk.job.analysis.model.dto.AnalysisTaskDTO; @@ -37,13 +35,14 @@ import com.tencent.bk.job.common.util.Counter; import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.crontab.api.inner.ServiceCronJobResource; import com.tencent.bk.job.crontab.model.inner.ServiceCronJobDTO; +import com.tencent.bk.job.execute.api.inner.ServiceTaskExecuteResultResource; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; import com.tencent.bk.job.execute.model.inner.ServiceTaskInstanceDTO; import com.tencent.bk.job.manage.model.inner.resp.ServiceApplicationDTO; import lombok.extern.slf4j.Slf4j; import lombok.val; -import org.jooq.DSLContext; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -59,15 +58,15 @@ @Slf4j public abstract class AbstractTimerTaskWatcher extends BaseAnalysisTask { - private final CronJobResourceClient cronJobResourceClient; + private final ServiceCronJobResource cronJobResource; @Autowired - public AbstractTimerTaskWatcher(DSLContext dslContext, AnalysisTaskDAO analysisTaskDAO, + public AbstractTimerTaskWatcher(AnalysisTaskDAO analysisTaskDAO, AnalysisTaskInstanceDAO analysisTaskInstanceDAO, ApplicationService applicationService, - CronJobResourceClient cronJobResourceClient) { - super(dslContext, analysisTaskDAO, analysisTaskInstanceDAO, applicationService); - this.cronJobResourceClient = cronJobResourceClient; + ServiceCronJobResource cronJobResource) { + super(analysisTaskDAO, analysisTaskInstanceDAO, applicationService); + this.cronJobResource = cronJobResource; } abstract void analyseAppCrons( @@ -76,10 +75,10 @@ abstract void analyseAppCrons( ); PageData getFailResults( - TaskExecuteResultResourceClient serviceTaskExecuteResultResourceClient, + ServiceTaskExecuteResultResource taskExecuteResultResource, ServiceCronJobDTO cronJobDTO ) { - PageData failResults = serviceTaskExecuteResultResourceClient.getTaskExecuteResult( + PageData failResults = taskExecuteResultResource.getTaskExecuteResult( cronJobDTO.getAppId(), null, null, @@ -139,7 +138,7 @@ public void run() { log.info("taskId:" + id); analysisTaskInstanceDTO.setId(id); //1.拿到业务下所有开启的定时任务 - val resp = cronJobResourceClient.listCronJobs(appId, true); + val resp = cronJobResource.listCronJobs(appId, true); log.info("listAllCronJobs resp:" + JsonUtils.toJson(resp)); List cronJobVOList = resp.getData(); log.info("" + cronJobVOList.size() + " cronJobs found"); diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/impl/DefaultTipsProvider.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/impl/DefaultTipsProvider.java index 299e6b9cdb..0c70f48cb0 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/impl/DefaultTipsProvider.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/impl/DefaultTipsProvider.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -31,7 +31,6 @@ import com.tencent.bk.job.analysis.task.analysis.anotation.AnalysisTask; import com.tencent.bk.job.analysis.task.analysis.task.pojo.AnalysisTaskResultVO; import lombok.extern.slf4j.Slf4j; -import org.jooq.DSLContext; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -48,9 +47,10 @@ public class DefaultTipsProvider extends BaseAnalysisTask { @Autowired - public DefaultTipsProvider(DSLContext dslContext, AnalysisTaskDAO analysisTaskDAO, - AnalysisTaskInstanceDAO analysisTaskInstanceDAO, ApplicationService applicationService) { - super(dslContext, analysisTaskDAO, analysisTaskInstanceDAO, applicationService); + public DefaultTipsProvider(AnalysisTaskDAO analysisTaskDAO, + AnalysisTaskInstanceDAO analysisTaskInstanceDAO, + ApplicationService applicationService) { + super(analysisTaskDAO, analysisTaskInstanceDAO, applicationService); } @Override @@ -58,7 +58,7 @@ public void run() { } @Override - public AnalysisTaskResultVO generateResultVO(String descriptionTpl, String itemTpl, String data) { + public AnalysisTaskResultVO renderResultVO(String descriptionTpl, String itemTpl, String data) { return new AnalysisTaskResultVO(data, Collections.emptyList()); } } diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/impl/ForbiddenScriptFinder.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/impl/ForbiddenScriptFinder.java index a457883c2b..c392499a98 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/impl/ForbiddenScriptFinder.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/impl/ForbiddenScriptFinder.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -47,9 +47,9 @@ import com.tencent.bk.job.common.model.PageData; import com.tencent.bk.job.common.util.Counter; import com.tencent.bk.job.common.util.json.JsonUtils; -import com.tencent.bk.job.manage.common.consts.JobResourceStatusEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskScriptSourceEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskStepTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.JobResourceStatusEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskScriptSourceEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskStepTypeEnum; import com.tencent.bk.job.manage.model.inner.ServiceScriptDTO; import com.tencent.bk.job.manage.model.inner.ServiceTaskPlanDTO; import com.tencent.bk.job.manage.model.inner.ServiceTaskStepDTO; @@ -60,7 +60,6 @@ import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; import lombok.val; -import org.jooq.DSLContext; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -83,14 +82,14 @@ public class ForbiddenScriptFinder extends BaseAnalysisTask { @Autowired public ForbiddenScriptFinder( - DSLContext dslContext, AnalysisTaskDAO analysisTaskDAO, + AnalysisTaskDAO analysisTaskDAO, AnalysisTaskInstanceDAO analysisTaskInstanceDAO, ApplicationService applicationService, TaskPlanService taskPlanService, TaskTemplateService templateService, ScriptService scriptService ) { - super(dslContext, analysisTaskDAO, analysisTaskInstanceDAO, applicationService); + super(analysisTaskDAO, analysisTaskInstanceDAO, applicationService); this.taskPlanService = taskPlanService; this.templateService = templateService; this.scriptService = scriptService; @@ -327,7 +326,7 @@ public void run() { } @Override - public AnalysisTaskResultVO generateResultVO(String descriptionTpl, String itemTpl, String data) { + public AnalysisTaskResultVO renderResultVO(String descriptionTpl, String itemTpl, String data) { AnalysisTaskResultData resultData = JsonUtils.fromJson(data, new TypeReference>() { }); diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/impl/TaskPlanTargetChecker.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/impl/TaskPlanTargetChecker.java index d7a0cda730..f8691ba7ef 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/impl/TaskPlanTargetChecker.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/impl/TaskPlanTargetChecker.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,6 +25,7 @@ package com.tencent.bk.job.analysis.task.analysis.task.impl; import com.fasterxml.jackson.core.type.TypeReference; +import com.tencent.bk.job.analysis.consts.JobAnalysisConsts; import com.tencent.bk.job.analysis.dao.AnalysisTaskDAO; import com.tencent.bk.job.analysis.dao.AnalysisTaskInstanceDAO; import com.tencent.bk.job.analysis.model.dto.AnalysisTaskDTO; @@ -43,9 +44,10 @@ import com.tencent.bk.job.analysis.task.analysis.task.pojo.AnalysisTaskResultVO; import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.util.Counter; import com.tencent.bk.job.common.util.json.JsonUtils; -import com.tencent.bk.job.manage.common.consts.task.TaskStepTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskStepTypeEnum; import com.tencent.bk.job.manage.model.inner.ServiceHostInfoDTO; import com.tencent.bk.job.manage.model.inner.ServiceHostStatusDTO; import com.tencent.bk.job.manage.model.inner.ServiceTaskHostNodeDTO; @@ -54,13 +56,15 @@ import com.tencent.bk.job.manage.model.inner.ServiceTaskStepDTO; import com.tencent.bk.job.manage.model.inner.ServiceTaskTemplateDTO; import com.tencent.bk.job.manage.model.inner.resp.ServiceApplicationDTO; -import com.tencent.bk.job.manage.model.web.request.ipchooser.AppTopologyTreeNode; +import com.tencent.bk.job.manage.model.web.request.chooser.host.BizTopoNode; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; import lombok.val; -import org.jooq.DSLContext; +import org.apache.commons.collections4.CollectionUtils; +import org.slf4j.helpers.FormattingTuple; +import org.slf4j.helpers.MessageFormatter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -86,7 +90,6 @@ public class TaskPlanTargetChecker extends BaseAnalysisTask { @Autowired public TaskPlanTargetChecker( - DSLContext dslContext, AnalysisTaskDAO analysisTaskDAO, AnalysisTaskInstanceDAO analysisTaskInstanceDAO, TaskTemplateService templateService, @@ -94,269 +97,481 @@ public TaskPlanTargetChecker( ApplicationService applicationService, HostService hostService ) { - super(dslContext, analysisTaskDAO, analysisTaskInstanceDAO, applicationService); + super(analysisTaskDAO, analysisTaskInstanceDAO, applicationService); this.templateService = templateService; this.taskPlanService = taskPlanService; this.hostService = hostService; } + @Override + public AnalysisTaskResultVO renderResultVO(String descriptionTpl, String itemTpl, String data) { + AnalysisTaskResultData resultData = JsonUtils.fromJson(data, + new TypeReference>() { + }); + String description = descriptionTpl; + List contents = new ArrayList<>(); + if (resultData.getCount() == 0) { + return null; + } else if (resultData.getCount() == 1) { + AbnormalTargetPlanInfo abnormalPlan = resultData.getData().get(0); + description = description.replace(JobAnalysisConsts.PLACEHOLDER_PLAN_NAME, abnormalPlan.getPlanName()); + description = description.replace(JobAnalysisConsts.PLACEHOLDER_STEP_NAME, abnormalPlan.getStepName()); + description = description.replace(JobAnalysisConsts.PLACEHOLDER_DESCRIPTION, abnormalPlan.getDescription()); + contents.add( + new AnalysisTaskResultItem( + abnormalPlan.analysisResourceType.name(), + abnormalPlan.getLocation(), + description + ) + ); + } else { + description = descriptionTpl.replace( + JobAnalysisConsts.PLACEHOLDER_ITEMS_COUNT, + "" + resultData.getData().size() + ); + resultData.getData().forEach(it -> contents.add(new AnalysisTaskResultItem( + it.getAnalysisResourceType().name(), + it.getLocation(), + it.getDescWithStepInfo() + ))); + } + return new AnalysisTaskResultVO(description, contents); + } + @Override public void run() { try { - log.info("Task " + getTaskCode() + " start"); - List appInfoList = getAppInfoList(); + log.info("Task {} start", getTaskCode()); + // 当前仅支持对普通业务进行分析 + List bizAppList = getAppInfoList().stream() + .filter(ServiceApplicationDTO::isBiz) + .collect(Collectors.toList()); Counter appCounter = new Counter(); AnalysisTaskDTO analysisTask = getAnalysisTask(); - for (ServiceApplicationDTO applicationInfoDTO : appInfoList) { - if (!applicationInfoDTO.isBiz()) { - continue; - } - Long appId = applicationInfoDTO.getId(); - appCounter.addOne(); - log.info( - "begin to analysis app:" + appCounter.getValue() - + "/" + appInfoList.size() - + "," + appId - + "," + applicationInfoDTO.getName() - ); - //初始化 - val analysisTaskInstanceDTO = new AnalysisTaskInstanceDTO( - null, - appId, - analysisTask.getId(), - AnalysisTaskStatusEnum.RUNNING.getValue(), - "", - analysisTask.getPriority(), - analysisTask.isActive(), - analysisTask.getCreator(), - System.currentTimeMillis(), - analysisTask.getCreator(), - System.currentTimeMillis() - ); - try { - Long id = insertAnalysisTaskInstance(analysisTaskInstanceDTO); - log.info("taskId:" + id); - analysisTaskInstanceDTO.setId(id); - List abnormalTargetPlanInfoList = new ArrayList<>(); - //1.拿到所有作业模板 - ServiceTaskTemplateDTO taskTemplateCondition = new ServiceTaskTemplateDTO(); - taskTemplateCondition.setAppId(appId); - BaseSearchCondition baseSearchCondition = new BaseSearchCondition(); - baseSearchCondition.setStart(0); - baseSearchCondition.setLength(Integer.MAX_VALUE); - PageData taskTemplateInfoDTOPageData = - templateService.listPageTaskTemplates(appId, baseSearchCondition); - List taskTemplateInfoDTOList = taskTemplateInfoDTOPageData.getData(); - //2.遍历所有作业模板 - Counter templateCounter = new Counter(); - for (ServiceTaskTemplateDTO taskTemplateBasicInfoDTO : taskTemplateInfoDTOList) { - try { - templateCounter.addOne(); - log.info("beigin to analysis taskTemplate:" - + templateCounter.getValue() - + "/" + taskTemplateInfoDTOList.size() - + "," + taskTemplateBasicInfoDTO.getId() - + "," + taskTemplateBasicInfoDTO.getName() - ); - List taskPlanIdList = - taskPlanService.listTaskPlanIds(taskTemplateBasicInfoDTO.getId()); - if (taskPlanIdList == null || taskPlanIdList.isEmpty()) { - log.info("Cannot find taskPlanIdList by templateId {}", - taskTemplateBasicInfoDTO.getId()); - continue; - } - //遍历作业模板中的执行方案 - for (Long taskPlanId : taskPlanIdList) { - try { - //获得执行计划详情 - ServiceTaskPlanDTO taskPlanInfoDTO = - taskPlanService.getTaskPlanById(appId, taskPlanId); - if (taskPlanInfoDTO == null) { - log.error("Cannot find taskPlanInfoDTO by id {}", taskPlanId); - continue; - } - //检查步骤 - for (ServiceTaskStepDTO taskStepDTO : taskPlanInfoDTO.getStepList()) { - ServiceTaskHostNodeDTO taskHostNodeDTO = null; - if (taskStepDTO.getType() == TaskStepTypeEnum.SCRIPT.getValue()) { - taskHostNodeDTO = - taskStepDTO.getScriptStepInfo().getExecuteTarget().getTargetServer(); - } else if (taskStepDTO.getType() == TaskStepTypeEnum.FILE.getValue()) { - taskHostNodeDTO = taskStepDTO - .getFileStepInfo() - .getExecuteTarget() - .getTargetServer(); - } - if (taskHostNodeDTO != null) { - findAgentAbnormalInTaskHostNodeVO( - abnormalTargetPlanInfoList, - taskHostNodeDTO, - appId, - "" + taskPlanInfoDTO.getTaskTemplateId() - + "," + taskPlanInfoDTO.getId(), - taskPlanInfoDTO.getName(), - "${job.analysis.analysistask.result.AbnormalTargetPlanInfo.stepName" - + ".scope.Step}", - taskStepDTO.getName(), - "${job.analysis.analysistask.result.AbnormalTargetPlanInfo" - + ".description.AbnormalTarget}" - ); - } - } - } catch (Throwable t) { - log.warn( - String.format("Task of app(id=%d) template(id=%d) taskplan(id=%d) failed" - + " because of exception", - appId, - taskTemplateBasicInfoDTO.getId(), - taskPlanId), - t); - } - } - } catch (Throwable t) { - log.warn(String.format("Task of app(id=%d) template(id=%d) failed because of exception", - appId, taskTemplateBasicInfoDTO.getId()), t); - } - } - //结果入库 - analysisTaskInstanceDTO.setResultData( - JsonUtils.toJson( - new AnalysisTaskResultData<>( - (long) abnormalTargetPlanInfoList.size(), - abnormalTargetPlanInfoList) - ) - ); - analysisTaskInstanceDTO.setStatus(AnalysisTaskStatusEnum.SUCCESS.getValue()); - updateAnalysisTaskInstanceById(analysisTaskInstanceDTO); - log.debug(String.format("%d found agentAbnormalItems are recorded:%s", - abnormalTargetPlanInfoList.size(), JsonUtils.toJson(abnormalTargetPlanInfoList))); - } catch (Throwable t) { - analysisTaskInstanceDTO.setStatus(AnalysisTaskStatusEnum.FAIL.getValue()); - updateAnalysisTaskInstanceById(analysisTaskInstanceDTO); - log.warn(String.format("Task of app(id=%d) failed because of exception", appId), t); - } finally { - log.info(String.format("Task:%s of app(id=%d) end", getTaskCode(), appId)); - } + for (ServiceApplicationDTO application : bizAppList) { + tryToAnalysisOneApp(application, analysisTask, appCounter, bizAppList.size()); } } catch (Throwable t) { log.warn("Task failed because of exception", t); } finally { - log.info("Task " + getTaskCode() + " end"); + log.info("Task {} end", getTaskCode()); + } + } + + /** + * 分析单个业务的数据 + * + * @param application 业务信息 + * @param analysisTask 分析任务 + * @param appCounter 业务计数器 + * @param allAppNum 业务总数 + */ + private void tryToAnalysisOneApp(ServiceApplicationDTO application, + AnalysisTaskDTO analysisTask, + Counter appCounter, + int allAppNum) { + Long appId = application.getId(); + appCounter.addOne(); + log.info( + "begin to analysis app:{}/{},{},{}", + appCounter.getValue(), + allAppNum, + appId, + application.getName() + ); + //初始化 + val analysisTaskInstanceDTO = initAnalysisTaskInstance(appId, analysisTask); + try { + analysisOneApp(application, analysisTaskInstanceDTO); + } catch (Throwable t) { + analysisTaskInstanceDTO.setStatus(AnalysisTaskStatusEnum.FAIL.getValue()); + updateAnalysisTaskInstanceById(analysisTaskInstanceDTO); + FormattingTuple msg = MessageFormatter.format( + "Task of app(id={}) failed because of exception", + appId + ); + log.warn(msg.getMessage(), t); + } finally { + log.info("Task:{} of app(id={}) end", getTaskCode(), appId); + } + } + + /** + * 生成初始化分析任务实例 + * + * @param appId 业务对应的appId + * @param analysisTask 分析任务 + * @return 初始化分析任务实例 + */ + private AnalysisTaskInstanceDTO initAnalysisTaskInstance(Long appId, AnalysisTaskDTO analysisTask) { + return new AnalysisTaskInstanceDTO( + null, + appId, + analysisTask.getId(), + AnalysisTaskStatusEnum.RUNNING.getValue(), + "", + analysisTask.getPriority(), + analysisTask.isActive(), + analysisTask.getCreator(), + System.currentTimeMillis(), + analysisTask.getCreator(), + System.currentTimeMillis() + ); + } + + /** + * 分析一个业务的数据 + * + * @param application 业务信息 + * @param analysisTaskInstance 分析任务实例 + */ + private void analysisOneApp(ServiceApplicationDTO application, + AnalysisTaskInstanceDTO analysisTaskInstance) { + Long appId = application.getId(); + Long id = insertAnalysisTaskInstance(analysisTaskInstance); + log.info("taskId:{}", id); + analysisTaskInstance.setId(id); + List abnormalPlanList = new ArrayList<>(); + //1.拿到所有作业模板 + ServiceTaskTemplateDTO taskTemplateCondition = new ServiceTaskTemplateDTO(); + taskTemplateCondition.setAppId(appId); + BaseSearchCondition baseSearchCondition = new BaseSearchCondition(); + baseSearchCondition.setStart(0); + baseSearchCondition.setLength(Integer.MAX_VALUE); + PageData taskTemplateInfoDTOPageData = + templateService.listPageTaskTemplates(appId, baseSearchCondition); + List templateList = taskTemplateInfoDTOPageData.getData(); + //2.遍历所有作业模板 + Counter templateCounter = new Counter(); + for (ServiceTaskTemplateDTO templateBasicInfo : templateList) { + tryToAnalysisOneTemplate( + appId, + templateBasicInfo, + abnormalPlanList, + templateCounter, + templateList.size() + ); + } + //结果入库 + analysisTaskInstance.setResultData( + JsonUtils.toJson( + new AnalysisTaskResultData<>((long) abnormalPlanList.size(), abnormalPlanList) + ) + ); + analysisTaskInstance.setStatus(AnalysisTaskStatusEnum.SUCCESS.getValue()); + updateAnalysisTaskInstanceById(analysisTaskInstance); + log.debug( + "{} found agentAbnormalItems are recorded:{}", + abnormalPlanList.size(), + JsonUtils.toJson(abnormalPlanList) + ); + } + + /** + * 尝试分析一个模板的数据，若有异常则捕获、打印，不阻塞下一个模板分析 + * + * @param appId 业务的appId + * @param templateBasicInfo 模板信息 + * @param abnormalPlanList 异常的执行计划列表 + * @param templateCounter 模板计数器 + * @param allTemplateNum 当前业务下的所有模板总数 + */ + private void tryToAnalysisOneTemplate(Long appId, + ServiceTaskTemplateDTO templateBasicInfo, + List abnormalPlanList, + Counter templateCounter, + int allTemplateNum) { + try { + analysisOneTemplate(appId, templateBasicInfo, abnormalPlanList, templateCounter, allTemplateNum); + } catch (Throwable t) { + FormattingTuple msg = MessageFormatter.format( + "Task of app(id={}) template(id={}) failed because of exception", + appId, + templateBasicInfo.getId() + ); + log.warn(msg.getMessage(), t); + } + } + + /** + * 分析一个模板的数据 + * + * @param appId 业务的appId + * @param templateBasicInfo 模板信息 + * @param abnormalPlanList 异常的执行计划列表 + * @param templateCounter 模板计数器 + * @param allTemplateNum 当前业务下的所有模板总数 + */ + private void analysisOneTemplate(Long appId, + ServiceTaskTemplateDTO templateBasicInfo, + List abnormalPlanList, + Counter templateCounter, + int allTemplateNum) { + templateCounter.addOne(); + log.info( + "begin to analysis taskTemplate:{}/{},{},{}", + templateCounter.getValue(), + allTemplateNum, + templateBasicInfo.getId(), + templateBasicInfo.getName() + ); + List taskPlanIdList = + taskPlanService.listTaskPlanIds(templateBasicInfo.getId()); + if (taskPlanIdList == null || taskPlanIdList.isEmpty()) { + log.info("Cannot find taskPlanIdList by templateId {}", templateBasicInfo.getId()); + return; + } + //遍历作业模板中的执行方案 + for (Long taskPlanId : taskPlanIdList) { + tryToAnalysisOnePlan(appId, templateBasicInfo.getId(), taskPlanId, abnormalPlanList); } } - private AbnormalTargetPlanInfo getAbnormalTargetPlanInfo(AnalysisResourceEnum analysisResourceEnum, - String locationContent, String instanceName, - String scope, String scopeName, String description) { + /** + * 尝试分析一个执行方案的数据，若有异常则捕获、打印，不阻塞下一个执行方案分析 + * + * @param appId 业务的appId + * @param templateId 模板Id + * @param taskPlanId 执行方案Id + * @param abnormalPlanList 异常执行方案列表 + */ + private void tryToAnalysisOnePlan(Long appId, + Long templateId, + Long taskPlanId, + List abnormalPlanList) { + try { + analysisOnePlan(appId, taskPlanId, abnormalPlanList); + } catch (Throwable t) { + FormattingTuple msg = MessageFormatter.format( + "Task of app(id={}) template(id={}) taskPlan(id={}) failed because of exception", + new Object[]{appId, templateId, taskPlanId} + ); + log.warn(msg.getMessage(), t); + } + } + + /** + * 分析一个执行方案的数据 + * + * @param appId 业务的appId + * @param taskPlanId 执行方案Id + * @param abnormalPlanList 异常执行方案列表 + */ + private void analysisOnePlan(Long appId, + Long taskPlanId, + List abnormalPlanList) { + //获得执行计划详情 + ServiceTaskPlanDTO taskPlanInfoDTO = + taskPlanService.getTaskPlanById(appId, taskPlanId); + if (taskPlanInfoDTO == null) { + log.error("Cannot find taskPlanInfoDTO by id {}", taskPlanId); + return; + } + //检查步骤 + for (ServiceTaskStepDTO taskStepDTO : taskPlanInfoDTO.getStepList()) { + ServiceTaskHostNodeDTO targetServer = null; + if (taskStepDTO.getType() == TaskStepTypeEnum.SCRIPT.getValue()) { + targetServer = taskStepDTO.getScriptStepInfo().getExecuteTarget().getTargetServer(); + } else if (taskStepDTO.getType() == TaskStepTypeEnum.FILE.getValue()) { + targetServer = taskStepDTO.getFileStepInfo().getExecuteTarget().getTargetServer(); + } + if (targetServer == null) { + continue; + } + if (existNotAliveHostInTaskHostNode(appId, targetServer)) { + AbnormalTargetPlanInfo abnormalTargetPlanInfo = buildAbnormalTargetPlanInfo( + buildPlanLocationContent(taskPlanInfoDTO), + taskPlanInfoDTO.getName(), + getI18nKeyAbnormalTargetPlanStep(), + taskStepDTO.getName(), + getI18nKeyAbnormalTargetPlanDesc() + ); + abnormalPlanList.add(abnormalTargetPlanInfo); + } + } + } + + /** + * 任务的执行目标中是否存在Agent异常的主机 + * + * @param appId 业务的appId + * @param targetServer 任务的执行目标 + * @return 是否存在Agent异常的主机 + */ + private boolean existNotAliveHostInTaskHostNode(Long appId, + ServiceTaskHostNodeDTO targetServer) { + //（1）拓扑节点 + List targetNodeList = targetServer.getNodeInfoList(); + if (existNotAliveHostInTargetNodes(appId, targetNodeList)) { + return true; + } + //（2）动态分组 + //找到动态分组对应的所有主机 + List dynamicGroupIdList = targetServer.getDynamicGroupId(); + if (existNotAliveHostInDynamicGroups(appId, dynamicGroupIdList)) { + return true; + } + //（3）主机 + List serviceHostInfoDTOList = targetServer.getHostList(); + return existNotAliveHost(appId, serviceHostInfoDTOList); + } + + /** + * 构造一个含有异常执行目标的执行方案实例 + * + * @param locationContent 定位信息 + * @param instanceName 实例名称 + * @param scope 范畴（步骤等） + * @param scopeName 范畴实例名称 + * @param description 描述 + * @return 异常执行目标的执行方案实例 + */ + private AbnormalTargetPlanInfo buildAbnormalTargetPlanInfo(String locationContent, + String instanceName, + String scope, + String scopeName, + String description) { return new AbnormalTargetPlanInfo( - AnalysisResourceEnum.TASK_PLAN - , new AnalysisTaskResultItemLocation( - "${job.analysis.analysistask.result.ItemLocation.description" - + ".CommaSeparatedTplIdAndPlanId}", - locationContent) - , instanceName - , scope + ":" + scopeName - , description + AnalysisResourceEnum.TASK_PLAN, + new AnalysisTaskResultItemLocation(getI18nKeyCommaSeparatedTplIdAndPlanId(), locationContent), + instanceName, + scope + ":" + scopeName, + description ); } - private void findAgentAbnormalInTaskHostNodeVO(List abnormalTargetPlanInfoList, - ServiceTaskHostNodeDTO serviceTaskHostNodeDTO, - Long appId, String locationContent, String instanceName, - String scope, String scopeName, String description) { - List hostStatusDTOList = new ArrayList<>(); - //（1）目标节点 - List targetNodeVOList = serviceTaskHostNodeDTO.getNodeInfoList(); - if (targetNodeVOList != null && !targetNodeVOList.isEmpty()) { - hostStatusDTOList.addAll(hostService.getHostStatusByNode(appId, - targetNodeVOList.stream().map(it -> new AppTopologyTreeNode( + private String getI18nKeyCommaSeparatedTplIdAndPlanId() { + return "${job.analysis.analysistask.result.ItemLocation.description.CommaSeparatedTplIdAndPlanId}"; + } + + /** + * 判断在节点列表下是否存在Agent异常的主机 + * + * @param appId 业务的appId + * @param targetNodeList 拓扑节点列表 + * @return 是否存在Agent异常的主机 + */ + private boolean existNotAliveHostInTargetNodes(Long appId, List targetNodeList) { + if (CollectionUtils.isEmpty(targetNodeList)) { + return false; + } + List hostStatusList = + hostService.getHostStatusByNode( + appId, + targetNodeList.stream().map(it -> new BizTopoNode( it.getType(), "", it.getId(), "", null - )).collect(Collectors.toList()))); - //找到目标节点对应的所有主机 - for (ServiceHostStatusDTO serviceHostStatusDTO : hostStatusDTOList) { - if (serviceHostStatusDTO.getAlive() != 1) { - abnormalTargetPlanInfoList.add(getAbnormalTargetPlanInfo(AnalysisResourceEnum.TASK_PLAN, - locationContent, instanceName, scope, scopeName, description)); - break; - } + )).collect(Collectors.toList())); + //找到目标节点对应的所有主机 + for (ServiceHostStatusDTO serviceHostStatusDTO : hostStatusList) { + if (!serviceHostStatusDTO.isAgentAlive()) { + return true; } } - //（2）动态分组 - //找到动态分组对应的所有主机 - List dynamicGroupIdList = serviceTaskHostNodeDTO.getDynamicGroupId(); - if (dynamicGroupIdList != null && !dynamicGroupIdList.isEmpty()) { - hostStatusDTOList.addAll(hostService.getHostStatusByDynamicGroup(appId, dynamicGroupIdList)); + return false; + } + + /** + * 判断在动态分组下是否存在Agent异常的主机 + * + * @param appId 业务的appId + * @param dynamicGroupIdList 动态分组Id列表 + * @return 是否存在Agent异常的主机 + */ + private boolean existNotAliveHostInDynamicGroups(Long appId, List dynamicGroupIdList) { + if (CollectionUtils.isEmpty(dynamicGroupIdList)) { + return false; } - //找到目标节点对应的所有主机//主机异常 - for (ServiceHostStatusDTO hostStatusDTO : hostStatusDTOList) { - if (hostStatusDTO.getAlive() != 1) { - abnormalTargetPlanInfoList.add(getAbnormalTargetPlanInfo(AnalysisResourceEnum.TASK_PLAN, - locationContent, instanceName, scope, scopeName, description)); - break; + List hostStatusList = hostService.getHostStatusByDynamicGroup(appId, dynamicGroupIdList); + for (ServiceHostStatusDTO hostStatusDTO : hostStatusList) { + if (!hostStatusDTO.isAgentAlive()) { + return true; } } - //（3）主机 - List serviceHostInfoDTOList = serviceTaskHostNodeDTO.getHostList(); - if (serviceHostInfoDTOList == null || serviceHostInfoDTOList.isEmpty()) return; - List ipList = serviceHostInfoDTOList.parallelStream().map(serviceHostInfoDTO -> - serviceHostInfoDTO.getCloudAreaId() + ":" + serviceHostInfoDTO.getIp() + return false; + } + + /** + * 判断在主机列表中是否存在Agent异常的主机 + * + * @param appId 业务的appId + * @param serviceHostList 主机信息列表 + * @return 是否存在Agent异常的主机 + */ + private boolean existNotAliveHost(Long appId, List serviceHostList) { + if (CollectionUtils.isEmpty(serviceHostList)) { + return false; + } + //noinspection deprecation + List hostList = serviceHostList.stream().map(serviceHost -> + // TODO:执行方案数据迁移添加hostId后此处可去除cloudAreaId与ip + new HostDTO( + serviceHost.getHostId(), + serviceHost.getCloudAreaId(), + serviceHost.getIp() + ) ).collect(Collectors.toList()); - List hostStatusDTOListByIp = - hostService.getHostStatusByIp(appId, ipList); - Map map = new HashMap<>(); - hostStatusDTOListByIp.forEach(serviceHostStatusDTO -> { - map.put(serviceHostStatusDTO.getIp(), serviceHostStatusDTO); - }); - if (!ipList.isEmpty()) { - for (String ip : ipList) { - if (!map.containsKey(ip)) { - //IP无效 - abnormalTargetPlanInfoList.add(getAbnormalTargetPlanInfo(AnalysisResourceEnum.TASK_PLAN, - locationContent, instanceName, scope, scopeName, description)); - break; - } else { - if (map.get(ip).getAlive() != 1) { - //主机异常 - abnormalTargetPlanInfoList.add(getAbnormalTargetPlanInfo(AnalysisResourceEnum.TASK_PLAN, - locationContent, instanceName, scope, scopeName, description)); - break; - } - } + List hostStatusDTOListByHost = + hostService.getHostStatusByHost(appId, hostList); + Map hostIdMap = new HashMap<>(); + hostStatusDTOListByHost.forEach(hostStatusDTO -> hostIdMap.put(hostStatusDTO.getHostId(), hostStatusDTO)); + if (hostList.isEmpty()) { + return true; + } + for (HostDTO host : hostList) { + if (!isHostAlive(hostIdMap, host)) { + return true; } } + return false; } - @Override - public AnalysisTaskResultVO generateResultVO(String descriptionTpl, String itemTpl, String data) { - AnalysisTaskResultData resultData = JsonUtils.fromJson(data, - new TypeReference>() { - }); - String description = descriptionTpl; - List contents = new ArrayList<>(); - if (resultData.getCount() == 0) { - return null; - } else if (resultData.getCount() == 1) { - AbnormalTargetPlanInfo abnormalTargetPlanInfo = resultData.getData().get(0); - description = description.replace("${planName}", abnormalTargetPlanInfo.getPlanName()); - description = description.replace("${stepName}", abnormalTargetPlanInfo.getStepName()); - description = description.replace("${description}", abnormalTargetPlanInfo.getDescription()); - contents.add(new AnalysisTaskResultItem(abnormalTargetPlanInfo.analysisResourceType.name(), - abnormalTargetPlanInfo.getLocation(), description)); - } else { - description = descriptionTpl.replace("${itemsCount}", "" + resultData.getData().size()); - resultData.getData().forEach(it -> contents.add(new AnalysisTaskResultItem( - it.getAnalysisResourceType().name(), - it.getLocation(), - it.getPlanName() + ":" + it.getStepName() + ":" + it.getDescription() - ))); + /** + * 根据hostId判断Agent是否正常 + * + * @param hostIdMap 主机Id与主机Agent状态信息映射表 + * @param host 主机信息 + * @return Agent是否正常 + */ + private boolean isHostAlive(Map hostIdMap, HostDTO host) { + Long hostId = host.getHostId(); + if (hostId == null) { + return false; } - return new AnalysisTaskResultVO(description, contents); + if (!hostIdMap.containsKey(hostId)) { + return false; + } + return hostIdMap.get(hostId).getAlive() == 1; + } + + /** + * 构造执行方案定位信息：模板Id，执行方案Id + * + * @param taskPlan 执行方案实例 + * @return 定位信息字符串 + */ + private String buildPlanLocationContent(ServiceTaskPlanDTO taskPlan) { + return taskPlan.getTaskTemplateId() + "," + taskPlan.getId(); + } + + /** + * @return 含有异常执行目标的执行方案步骤I18nKey + */ + private String getI18nKeyAbnormalTargetPlanStep() { + return "${job.analysis.analysistask.result.AbnormalTargetPlanInfo.stepName.scope.Step}"; + } + + /** + * @return 含有异常执行目标的执行方案描述I18nKey + */ + private String getI18nKeyAbnormalTargetPlanDesc() { + return "${job.analysis.analysistask.result.AbnormalTargetPlanInfo.description.AbnormalTarget}"; } + /** * 执行目标异常的执行方案信息 */ @@ -369,5 +584,9 @@ private static class AbnormalTargetPlanInfo { private String planName; private String stepName; private String description; + + public String getDescWithStepInfo() { + return getPlanName() + ":" + getStepName() + ":" + getDescription(); + } } } diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/impl/TimerTaskFailRateWatcher.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/impl/TimerTaskFailRateWatcher.java index 669246798c..c488451125 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/impl/TimerTaskFailRateWatcher.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/impl/TimerTaskFailRateWatcher.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,8 +25,6 @@ package com.tencent.bk.job.analysis.task.analysis.task.impl; import com.fasterxml.jackson.core.type.TypeReference; -import com.tencent.bk.job.analysis.client.CronJobResourceClient; -import com.tencent.bk.job.analysis.client.TaskExecuteResultResourceClient; import com.tencent.bk.job.analysis.dao.AnalysisTaskDAO; import com.tencent.bk.job.analysis.dao.AnalysisTaskInstanceDAO; import com.tencent.bk.job.analysis.model.dto.AnalysisTaskInstanceDTO; @@ -40,14 +38,15 @@ import com.tencent.bk.job.analysis.task.analysis.task.pojo.AnalysisTaskResultVO; import com.tencent.bk.job.common.model.PageData; import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.crontab.api.inner.ServiceCronJobResource; import com.tencent.bk.job.crontab.model.inner.ServiceCronJobDTO; +import com.tencent.bk.job.execute.api.inner.ServiceTaskExecuteResultResource; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; import com.tencent.bk.job.execute.model.inner.ServiceTaskInstanceDTO; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.jooq.DSLContext; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -64,16 +63,16 @@ @Slf4j public class TimerTaskFailRateWatcher extends AbstractTimerTaskWatcher { - private final TaskExecuteResultResourceClient serviceTaskExecuteResultResourceClient; + private final ServiceTaskExecuteResultResource taskExecuteResultResource; @Autowired - public TimerTaskFailRateWatcher(DSLContext dslContext, AnalysisTaskDAO analysisTaskDAO, + public TimerTaskFailRateWatcher(AnalysisTaskDAO analysisTaskDAO, AnalysisTaskInstanceDAO analysisTaskInstanceDAO, ApplicationService applicationService, - CronJobResourceClient cronJobResourceClient, - TaskExecuteResultResourceClient serviceTaskExecuteResultResourceClient) { - super(dslContext, analysisTaskDAO, analysisTaskInstanceDAO, applicationService, cronJobResourceClient); - this.serviceTaskExecuteResultResourceClient = serviceTaskExecuteResultResourceClient; + ServiceCronJobResource cronJobResource, + ServiceTaskExecuteResultResource taskExecuteResultResource) { + super(analysisTaskDAO, analysisTaskInstanceDAO, applicationService, cronJobResource); + this.taskExecuteResultResource = taskExecuteResultResource; } @Override @@ -87,10 +86,10 @@ protected void analyseAppCrons( cronJobVOList.forEach(it -> { //3.拿到每一个定时任务在指定时间段内的执行结果并找出失败的 log.info("begin to find fail result of task:" + it.getId() + "," + it.getName()); - PageData failResult = getFailResults(serviceTaskExecuteResultResourceClient, it); + PageData failResult = getFailResults(taskExecuteResultResource, it); log.info("begin to find success result of task:" + it.getId() + "," + it.getName()); PageData successResult = - serviceTaskExecuteResultResourceClient.getTaskExecuteResult( + taskExecuteResultResource.getTaskExecuteResult( appId, null, null, @@ -137,7 +136,7 @@ protected void analyseAppCrons( } @Override - public AnalysisTaskResultVO generateResultVO(String descriptionTpl, String itemTpl, String data) { + public AnalysisTaskResultVO renderResultVO(String descriptionTpl, String itemTpl, String data) { AnalysisTaskResultData resultData = JsonUtils.fromJson(data, new TypeReference>() { }); diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/impl/TimerTaskFailWatcher.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/impl/TimerTaskFailWatcher.java index c2fd5f8b4b..8e9bffdbea 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/impl/TimerTaskFailWatcher.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/impl/TimerTaskFailWatcher.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,8 +25,6 @@ package com.tencent.bk.job.analysis.task.analysis.task.impl; import com.fasterxml.jackson.core.type.TypeReference; -import com.tencent.bk.job.analysis.client.CronJobResourceClient; -import com.tencent.bk.job.analysis.client.TaskExecuteResultResourceClient; import com.tencent.bk.job.analysis.dao.AnalysisTaskDAO; import com.tencent.bk.job.analysis.dao.AnalysisTaskInstanceDAO; import com.tencent.bk.job.analysis.model.dto.AnalysisTaskInstanceDTO; @@ -40,13 +38,14 @@ import com.tencent.bk.job.analysis.task.analysis.task.pojo.AnalysisTaskResultVO; import com.tencent.bk.job.common.model.PageData; import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.crontab.api.inner.ServiceCronJobResource; import com.tencent.bk.job.crontab.model.inner.ServiceCronJobDTO; +import com.tencent.bk.job.execute.api.inner.ServiceTaskExecuteResultResource; import com.tencent.bk.job.execute.model.inner.ServiceTaskInstanceDTO; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; import lombok.extern.slf4j.Slf4j; -import org.jooq.DSLContext; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -63,17 +62,18 @@ @Slf4j public class TimerTaskFailWatcher extends AbstractTimerTaskWatcher { - private final TaskExecuteResultResourceClient serviceTaskExecuteResultResourceClient; + private final ServiceTaskExecuteResultResource taskExecuteResultResource; @Autowired public TimerTaskFailWatcher( - DSLContext dslContext, AnalysisTaskDAO analysisTaskDAO, + AnalysisTaskDAO analysisTaskDAO, AnalysisTaskInstanceDAO analysisTaskInstanceDAO, - ApplicationService applicationService, CronJobResourceClient cronJobResourceClient, - TaskExecuteResultResourceClient serviceTaskExecuteResultResourceClient + ApplicationService applicationService, + ServiceCronJobResource cronJobResource, + ServiceTaskExecuteResultResource taskExecuteResultResource ) { - super(dslContext, analysisTaskDAO, analysisTaskInstanceDAO, applicationService, cronJobResourceClient); - this.serviceTaskExecuteResultResourceClient = serviceTaskExecuteResultResourceClient; + super(analysisTaskDAO, analysisTaskInstanceDAO, applicationService, cronJobResource); + this.taskExecuteResultResource = taskExecuteResultResource; } @Override @@ -86,7 +86,7 @@ protected void analyseAppCrons( cronJobVOList.forEach(it -> { //3.拿到每一个定时任务在指定时间段内的执行结果并找出失败的 log.info("begin to find fail result of task:" + it.getId() + "," + it.getName()); - PageData failResults = getFailResults(serviceTaskExecuteResultResourceClient, it); + PageData failResults = getFailResults(taskExecuteResultResource, it); if (failResults.getTotal() > 0) { failCronJobBaseInfoList.add( new FailCronJobBaseInfo( @@ -116,7 +116,7 @@ protected void analyseAppCrons( } @Override - public AnalysisTaskResultVO generateResultVO(String descriptionTpl, String itemTpl, String data) { + public AnalysisTaskResultVO renderResultVO(String descriptionTpl, String itemTpl, String data) { AnalysisTaskResultData resultData = JsonUtils.fromJson(data, new TypeReference>() { }); diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/pojo/AnalysisTaskResultData.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/pojo/AnalysisTaskResultData.java index 996d740691..c90bf0e458 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/pojo/AnalysisTaskResultData.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/pojo/AnalysisTaskResultData.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/pojo/AnalysisTaskResultItem.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/pojo/AnalysisTaskResultItem.java index c858c1bc0b..1759fbf87c 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/pojo/AnalysisTaskResultItem.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/pojo/AnalysisTaskResultItem.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/pojo/AnalysisTaskResultVO.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/pojo/AnalysisTaskResultVO.java index e4d1d8c461..f274b4b46a 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/pojo/AnalysisTaskResultVO.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/analysis/task/pojo/AnalysisTaskResultVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/StatisticsTaskScheduler.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/StatisticsTaskScheduler.java index 734eb1a955..848f38310e 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/StatisticsTaskScheduler.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/StatisticsTaskScheduler.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,7 @@ package com.tencent.bk.job.analysis.task.statistics; +import com.tencent.bk.job.analysis.api.consts.StatisticsConstants; import com.tencent.bk.job.analysis.config.StatisticConfig; import com.tencent.bk.job.analysis.task.statistics.task.ClearExpiredStatisticsTask; import com.tencent.bk.job.analysis.task.statistics.task.DefaultTaskStatusListener; @@ -31,9 +32,10 @@ import com.tencent.bk.job.analysis.task.statistics.task.PastStatisticsMakeupTask; import com.tencent.bk.job.analysis.task.statistics.task.TaskInfo; import com.tencent.bk.job.analysis.task.statistics.task.WatchableTask; -import com.tencent.bk.job.common.redis.util.LockUtils; +import com.tencent.bk.job.common.redis.util.HeartBeatRedisLock; +import com.tencent.bk.job.common.redis.util.HeartBeatRedisLockConfig; +import com.tencent.bk.job.common.redis.util.LockResult; import com.tencent.bk.job.common.redis.util.RedisKeyHeartBeatThread; -import com.tencent.bk.job.common.statistics.consts.StatisticsConstants; import com.tencent.bk.job.common.util.ApplicationContextRegister; import com.tencent.bk.job.common.util.TimeUtil; import com.tencent.bk.job.common.util.ip.IpUtils; @@ -74,7 +76,6 @@ public class StatisticsTaskScheduler { private static final String machineIp = IpUtils.getFirstMachineIP(); - private static final String REDIS_KEY_STATISTIC_JOB_LOCK = "statistic-job-lock"; private static final String REDIS_KEY_STATISTIC_JOB_RUNNING_MACHINE = "statistic-job-running-machine"; public static final AtomicLong rejectedStatisticsTaskNum = new AtomicLong(0); private static final String REDIS_KEY_STATISTIC_JOB_INIT_MACHINE = "statistic-job-init-machine"; @@ -94,18 +95,6 @@ public class StatisticsTaskScheduler { private ThreadPoolExecutor currentStatisticsTaskExecutor; private ThreadPoolExecutor pastStatisticsTaskExecutor; - static { - List keyList = Collections.singletonList(REDIS_KEY_STATISTIC_JOB_LOCK); - keyList.forEach(key -> { - try { - //进程重启首先尝试释放上次加上的锁避免死锁 - LockUtils.releaseDistributedLock(key, machineIp); - } catch (Throwable t) { - log.info("Redis key:" + key + " does not need to be released, ignore"); - } - }); - } - private final StatisticConfig statisticConfig; private final RedisTemplate redisTemplate; private final PastStatisticsMakeupTask pastStatisticsMakeupTask; @@ -115,9 +104,9 @@ public class StatisticsTaskScheduler { @Autowired public StatisticsTaskScheduler(MeterRegistry meterRegistry, @Qualifier("currentStatisticsTaskExecutor") - ThreadPoolExecutor currentStatisticsTaskExecutor, + ThreadPoolExecutor currentStatisticsTaskExecutor, @Qualifier("pastStatisticsTaskExecutor") - ThreadPoolExecutor pastStatisticsTaskExecutor, + ThreadPoolExecutor pastStatisticsTaskExecutor, StatisticConfig statisticConfig, RedisTemplate redisTemplate, PastStatisticsMakeupTask pastStatisticsMakeupTask, @@ -171,7 +160,7 @@ public boolean configThreads(Integer currentStatisticThreadsNum, Integer pastSta if (currentStatisticThreadsNum != null && currentStatisticThreadsNum > 0) { log.info("reconfig currentStatisticsTaskExecutor to {} threads", currentStatisticThreadsNum); List canceledRunnableList = currentStatisticsTaskExecutor.shutdownNow(); - if (canceledRunnableList.size() > 0) { + if (!canceledRunnableList.isEmpty()) { log.warn("{} threads of canceled", canceledRunnableList.size()); } currentStatisticsTaskExecutor = new ThreadPoolExecutor( @@ -190,7 +179,7 @@ public boolean configThreads(Integer currentStatisticThreadsNum, Integer pastSta log.info("reconfig pastStatisticsTaskExecutor to {} threads", pastStatisticThreadsNum); pastStatisticsMakeupTask.setRunFlag(false); List canceledRunnableList = pastStatisticsTaskExecutor.shutdownNow(); - if (canceledRunnableList.size() > 0) { + if (!canceledRunnableList.isEmpty()) { log.warn("{} threads of pastStatisticsTaskExecutor canceled", canceledRunnableList.size()); } pastStatisticsTaskExecutor = new ThreadPoolExecutor( @@ -262,7 +251,7 @@ public List startTasks(String startDateStr, String endDateStr, List= 0 && count < maxPreviousDays) { + while (!targetDate.isBefore(startDate) && count < maxPreviousDays) { startByTaskNameList(taskNameList, targetDate, startedTaskNames); targetDate = targetDate.minusDays(1); count += 1; @@ -315,7 +304,7 @@ public List listAllTasks() { if (statisticsTaskList == null || statisticsTaskList.isEmpty()) { findStatisticsTask(); } - return statisticsTaskList.parallelStream() + return statisticsTaskList.stream() .map(it -> it.getClass().getSimpleName()).collect(Collectors.toList()); } @@ -399,19 +388,6 @@ private boolean needToRunStatisticTasks() { statisticConfig.getIntervalHours()); currentCycleHours = 1; } - - // 分布式唯一性保证 - boolean lockGotten = LockUtils.tryGetDistributedLock(REDIS_KEY_STATISTIC_JOB_LOCK, machineIp, 50); - if (!lockGotten) { - log.info("lock {} gotten by another machine, return", REDIS_KEY_STATISTIC_JOB_LOCK); - return false; - } - String runningMachine = redisTemplate.opsForValue().get(REDIS_KEY_STATISTIC_JOB_RUNNING_MACHINE); - if (StringUtils.isNotBlank(runningMachine)) { - //已有同步线程在跑，不再同步 - log.info("StatisticsTaskScheduler thread already running on {}", runningMachine); - return false; - } return true; } @@ -422,18 +398,25 @@ public void schedule() { if (!needToRunStatisticTasks()) { return; } - // 开一个心跳子线程，维护当前机器正在执行后台统计任务的状态 - long expireTimeMillis = 5000L; - long periodTimeMillis = 4000L; - RedisKeyHeartBeatThread statisticTaskSchedulerRedisKeyHeartBeatThread = new RedisKeyHeartBeatThread( + + // 分布式唯一性保证 + HeartBeatRedisLockConfig config = HeartBeatRedisLockConfig.getDefault(); + config.setHeartBeatThreadName("statisticTaskSchedulerRedisKeyHeartBeatThread"); + HeartBeatRedisLock lock = new HeartBeatRedisLock( redisTemplate, REDIS_KEY_STATISTIC_JOB_RUNNING_MACHINE, machineIp, - expireTimeMillis, - periodTimeMillis + config ); - statisticTaskSchedulerRedisKeyHeartBeatThread.setName("statisticTaskSchedulerRedisKeyHeartBeatThread"); - statisticTaskSchedulerRedisKeyHeartBeatThread.start(); + LockResult lockResult = lock.lock(); + if (!lockResult.isLockGotten()) { + log.info( + "lock {} gotten by another machine: {}, return", + REDIS_KEY_STATISTIC_JOB_RUNNING_MACHINE, + lockResult.getLockValue() + ); + return; + } // 统计任务开始 log.info("start StatisticsTaskScheduler at {},{}", TimeUtil.getCurrentTimeStr("HH:mm:ss"), @@ -467,20 +450,26 @@ public void schedule() { // 等待所有统计任务结束 futureList.forEach(future -> { try { - int timeoutMinutes = 10; + // 最长统计任务耗时约30min + int timeoutMinutes = 60; future.get(timeoutMinutes, TimeUnit.MINUTES); + updateDataUpdateTime(TimeUtil.getCurrentTimeStr()); } catch (InterruptedException | ExecutionException | TimeoutException e) { log.error("Exception in statistic task", e); + updateDataUpdateTime("After " + TimeUtil.getCurrentTimeStr()); } }); - // 向Redis写入统计数据更新时间，不过期 - redisTemplate.opsForValue().set(StatisticsConstants.KEY_DATA_UPDATE_TIME, TimeUtil.getCurrentTimeStr()); } catch (Throwable t) { log.error("Exception in StatisticsTaskScheduler", t); } finally { - statisticTaskSchedulerRedisKeyHeartBeatThread.setRunFlag(false); + lockResult.tryToRelease(); stopWatch.stop(); log.info(stopWatch.prettyPrint()); } } + + private void updateDataUpdateTime(String updateTimeStr) { + // 向Redis写入统计数据更新时间，不过期 + redisTemplate.opsForValue().set(StatisticsConstants.KEY_DATA_UPDATE_TIME, updateTimeStr); + } } diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/anotation/StatisticsTask.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/anotation/StatisticsTask.java index 031c0a39f7..bd862290f3 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/anotation/StatisticsTask.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/anotation/StatisticsTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/BasePerAppStatisticsTask.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/BasePerAppStatisticsTask.java index 9cadd4ed39..b6ce9a6021 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/BasePerAppStatisticsTask.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/BasePerAppStatisticsTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,13 +24,14 @@ package com.tencent.bk.job.analysis.task.statistics.task; +import com.tencent.bk.job.analysis.api.dto.StatisticsDTO; import com.tencent.bk.job.analysis.dao.StatisticsDAO; import com.tencent.bk.job.analysis.service.BasicServiceManager; -import com.tencent.bk.job.common.statistics.model.dto.StatisticsDTO; import com.tencent.bk.job.common.util.TimeUtil; import com.tencent.bk.job.manage.model.inner.resp.ServiceApplicationDTO; import lombok.extern.slf4j.Slf4j; import org.jooq.DSLContext; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.util.StopWatch; import java.time.LocalDateTime; @@ -40,8 +41,9 @@ @Slf4j public abstract class BasePerAppStatisticsTask extends BaseStatisticsTask { - protected BasePerAppStatisticsTask(BasicServiceManager basicServiceManager, StatisticsDAO statisticsDAO, - DSLContext dslContext) { + protected BasePerAppStatisticsTask(BasicServiceManager basicServiceManager, + StatisticsDAO statisticsDAO, + @Qualifier("job-analysis-dsl-context") DSLContext dslContext) { super(basicServiceManager, statisticsDAO, dslContext); } @@ -67,7 +69,7 @@ public void genStatisticsByDay(LocalDateTime dateTime) { return; } log.debug("targetApps:{}", - apps.parallelStream().map(ServiceApplicationDTO::getId).collect(Collectors.toList())); + apps.stream().map(ServiceApplicationDTO::getId).collect(Collectors.toList())); apps.forEach(app -> { try { StopWatch stopWatch = new StopWatch(); diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/BaseStatisticsTask.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/BaseStatisticsTask.java index d26ae591dc..1826063606 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/BaseStatisticsTask.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/BaseStatisticsTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,9 +24,9 @@ package com.tencent.bk.job.analysis.task.statistics.task; +import com.tencent.bk.job.analysis.api.consts.StatisticsConstants; import com.tencent.bk.job.analysis.dao.StatisticsDAO; import com.tencent.bk.job.analysis.service.BasicServiceManager; -import com.tencent.bk.job.common.statistics.consts.StatisticsConstants; import com.tencent.bk.job.common.util.TimeUtil; import com.tencent.bk.job.manage.model.inner.resp.ServiceApplicationDTO; import lombok.extern.slf4j.Slf4j; @@ -42,7 +42,8 @@ public abstract class BaseStatisticsTask implements IStatisticsTask { protected final StatisticsDAO statisticsDAO; protected final DSLContext dslContext; - protected BaseStatisticsTask(BasicServiceManager basicServiceManager, StatisticsDAO statisticsDAO, + protected BaseStatisticsTask(BasicServiceManager basicServiceManager, + StatisticsDAO statisticsDAO, DSLContext dslContext) { this.basicServiceManager = basicServiceManager; this.statisticsDAO = statisticsDAO; diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/ClearExpiredStatisticsTask.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/ClearExpiredStatisticsTask.java index 40b5ccc111..8e7ddc2e18 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/ClearExpiredStatisticsTask.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/ClearExpiredStatisticsTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,9 +24,9 @@ package com.tencent.bk.job.analysis.task.statistics.task; +import com.tencent.bk.job.analysis.api.consts.StatisticsConstants; import com.tencent.bk.job.analysis.config.StatisticConfig; import com.tencent.bk.job.analysis.dao.StatisticsDAO; -import com.tencent.bk.job.common.statistics.consts.StatisticsConstants; import com.tencent.bk.job.common.util.TimeUtil; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; @@ -41,7 +41,8 @@ public class ClearExpiredStatisticsTask extends Thread { private final StatisticConfig statisticConfig; @Autowired - public ClearExpiredStatisticsTask(StatisticsDAO statisticsDAO, StatisticConfig statisticConfig) { + public ClearExpiredStatisticsTask(StatisticsDAO statisticsDAO, + StatisticConfig statisticConfig) { this.statisticsDAO = statisticsDAO; this.statisticConfig = statisticConfig; } diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/DefaultTaskStatusListener.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/DefaultTaskStatusListener.java index f8c7aed3b7..99a7cce971 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/DefaultTaskStatusListener.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/DefaultTaskStatusListener.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/ExecuteBasePerAppStatisticsTask.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/ExecuteBasePerAppStatisticsTask.java index e91cbcf2a7..c9a593dde8 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/ExecuteBasePerAppStatisticsTask.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/ExecuteBasePerAppStatisticsTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,29 +24,31 @@ package com.tencent.bk.job.analysis.task.statistics.task; -import com.tencent.bk.job.analysis.client.ExecuteMetricsClient; +import com.tencent.bk.job.analysis.api.dto.StatisticsDTO; import com.tencent.bk.job.analysis.dao.StatisticsDAO; import com.tencent.bk.job.analysis.service.BasicServiceManager; -import com.tencent.bk.job.common.statistics.model.dto.StatisticsDTO; +import com.tencent.bk.job.execute.api.inner.ServiceMetricsResource; import lombok.extern.slf4j.Slf4j; import org.jooq.DSLContext; +import org.springframework.beans.factory.annotation.Qualifier; import java.util.List; @Slf4j public abstract class ExecuteBasePerAppStatisticsTask extends BasePerAppStatisticsTask { - private final ExecuteMetricsClient executeMetricsClient; + private final ServiceMetricsResource executeMetricsResource; - protected ExecuteBasePerAppStatisticsTask(ExecuteMetricsClient executeMetricsClient, - BasicServiceManager basicServiceManager, StatisticsDAO statisticsDAO, - DSLContext dslContext) { + protected ExecuteBasePerAppStatisticsTask(ServiceMetricsResource executeMetricsResource, + BasicServiceManager basicServiceManager, + StatisticsDAO statisticsDAO, + @Qualifier("job-analysis-dsl-context") DSLContext dslContext) { super(basicServiceManager, statisticsDAO, dslContext); - this.executeMetricsClient = executeMetricsClient; + this.executeMetricsResource = executeMetricsResource; } public void addExecuteStatisticsDTO(StatisticsDTO searchStatisticsDTO, List statisticsDTOList) { - StatisticsDTO remoteStatisticsDTO = executeMetricsClient.getStatistics(searchStatisticsDTO.getAppId(), + StatisticsDTO remoteStatisticsDTO = executeMetricsResource.getStatistics(searchStatisticsDTO.getAppId(), searchStatisticsDTO.getResource(), searchStatisticsDTO.getDimension(), searchStatisticsDTO.getDimensionValue(), searchStatisticsDTO.getDate()).getData(); if (remoteStatisticsDTO != null) { diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/IStatisticsTask.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/IStatisticsTask.java index c708959ebf..9a22a7870a 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/IStatisticsTask.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/IStatisticsTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/PastStatisticsMakeupTask.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/PastStatisticsMakeupTask.java index 231b772686..815b64696a 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/PastStatisticsMakeupTask.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/PastStatisticsMakeupTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,8 +24,8 @@ package com.tencent.bk.job.analysis.task.statistics.task; +import com.tencent.bk.job.analysis.api.consts.StatisticsConstants; import com.tencent.bk.job.analysis.config.StatisticConfig; -import com.tencent.bk.job.common.statistics.consts.StatisticsConstants; import com.tencent.bk.job.common.util.TimeUtil; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/TaskInfo.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/TaskInfo.java index f352c714d4..e469938858 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/TaskInfo.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/TaskInfo.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/TaskStatusListener.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/TaskStatusListener.java index be19b98921..a33a476780 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/TaskStatusListener.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/TaskStatusListener.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/Watchable.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/Watchable.java index 5b14bdfa0b..c27ff7aebe 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/Watchable.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/Watchable.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/WatchableTask.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/WatchableTask.java index 8ba43797eb..67b1ff4d79 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/WatchableTask.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/WatchableTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/AccountStatisticsTask.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/AccountStatisticsTask.java index 3406fc1017..ec3f647e5c 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/AccountStatisticsTask.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/AccountStatisticsTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,17 +24,18 @@ package com.tencent.bk.job.analysis.task.statistics.task.impl; -import com.tencent.bk.job.analysis.client.ManageMetricsClient; +import com.tencent.bk.job.analysis.api.consts.StatisticsConstants; +import com.tencent.bk.job.analysis.api.dto.StatisticsDTO; import com.tencent.bk.job.analysis.dao.StatisticsDAO; import com.tencent.bk.job.analysis.service.BasicServiceManager; import com.tencent.bk.job.analysis.task.statistics.anotation.StatisticsTask; import com.tencent.bk.job.analysis.task.statistics.task.BaseStatisticsTask; import com.tencent.bk.job.common.model.InternalResponse; -import com.tencent.bk.job.common.statistics.consts.StatisticsConstants; -import com.tencent.bk.job.common.statistics.model.dto.StatisticsDTO; -import com.tencent.bk.job.manage.common.consts.account.AccountTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.account.AccountTypeEnum; +import com.tencent.bk.job.manage.api.inner.ServiceMetricsResource; import lombok.extern.slf4j.Slf4j; import org.jooq.DSLContext; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; import java.time.LocalDateTime; @@ -47,12 +48,14 @@ @Service public class AccountStatisticsTask extends BaseStatisticsTask { - private final ManageMetricsClient manageMetricsClient; + private final ServiceMetricsResource manageMetricResource; - protected AccountStatisticsTask(BasicServiceManager basicServiceManager, StatisticsDAO statisticsDAO, - DSLContext dslContext, ManageMetricsClient manageMetricsClient) { + protected AccountStatisticsTask(BasicServiceManager basicServiceManager, + StatisticsDAO statisticsDAO, + @Qualifier("job-analysis-dsl-context") DSLContext dslContext, + ServiceMetricsResource manageMetricResource) { super(basicServiceManager, statisticsDAO, dslContext); - this.manageMetricsClient = manageMetricsClient; + this.manageMetricResource = manageMetricResource; } private StatisticsDTO genStatisticsDTO(String dateStr, String value, String dimensionValue) { @@ -80,7 +83,7 @@ private StatisticsDTO genDBStatisticsDTO(String dateStr, String value) { public void calcAndSaveAccountStatistics(String dateStr) { // Linux - InternalResponse resp = manageMetricsClient.countAccounts(AccountTypeEnum.LINUX); + InternalResponse resp = manageMetricResource.countAccounts(AccountTypeEnum.LINUX); if (resp == null || !resp.isSuccess()) { log.warn("Fail to call remote countAccounts, resp:{}", resp); return; @@ -88,7 +91,7 @@ public void calcAndSaveAccountStatistics(String dateStr) { Integer linuxCount = resp.getData(); statisticsDAO.upsertStatistics(dslContext, genLinuxStatisticsDTO(dateStr, linuxCount.toString())); // Windows - resp = manageMetricsClient.countAccounts(AccountTypeEnum.WINDOW); + resp = manageMetricResource.countAccounts(AccountTypeEnum.WINDOW); if (resp == null || !resp.isSuccess()) { log.warn("Fail to call remote countAccounts, resp:{}", resp); return; @@ -96,19 +99,19 @@ public void calcAndSaveAccountStatistics(String dateStr) { Integer windowsCount = resp.getData(); statisticsDAO.upsertStatistics(dslContext, genWindowsStatisticsDTO(dateStr, windowsCount.toString())); // DB - resp = manageMetricsClient.countAccounts(AccountTypeEnum.MYSQL); + resp = manageMetricResource.countAccounts(AccountTypeEnum.MYSQL); if (resp == null || !resp.isSuccess()) { log.warn("Fail to call remote countAccounts, resp:{}", resp); return; } Integer mysqlCount = resp.getData(); - resp = manageMetricsClient.countAccounts(AccountTypeEnum.ORACLE); + resp = manageMetricResource.countAccounts(AccountTypeEnum.ORACLE); if (resp == null || !resp.isSuccess()) { log.warn("Fail to call remote countAccounts, resp:{}", resp); return; } Integer oracleCount = resp.getData(); - resp = manageMetricsClient.countAccounts(AccountTypeEnum.DB2); + resp = manageMetricResource.countAccounts(AccountTypeEnum.DB2); if (resp == null || !resp.isSuccess()) { log.warn("Fail to call remote countAccounts, resp:{}", resp); return; diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/AppStatisticsTask.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/AppStatisticsTask.java index bb837227a1..ec85d486c8 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/AppStatisticsTask.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/AppStatisticsTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,21 +25,22 @@ package com.tencent.bk.job.analysis.task.statistics.task.impl.app; import com.fasterxml.jackson.core.type.TypeReference; -import com.tencent.bk.job.analysis.client.ExecuteMetricsClient; +import com.tencent.bk.job.analysis.api.consts.StatisticsConstants; +import com.tencent.bk.job.analysis.api.dto.StatisticsDTO; import com.tencent.bk.job.analysis.config.StatisticConfig; import com.tencent.bk.job.analysis.dao.StatisticsDAO; import com.tencent.bk.job.analysis.model.dto.SimpleAppInfoDTO; import com.tencent.bk.job.analysis.service.BasicServiceManager; import com.tencent.bk.job.analysis.task.statistics.anotation.StatisticsTask; import com.tencent.bk.job.analysis.task.statistics.task.BaseStatisticsTask; -import com.tencent.bk.job.common.statistics.consts.StatisticsConstants; -import com.tencent.bk.job.common.statistics.model.dto.StatisticsDTO; import com.tencent.bk.job.common.util.TimeUtil; import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.execute.api.inner.ServiceMetricsResource; import com.tencent.bk.job.manage.model.inner.resp.ServiceApplicationDTO; import lombok.extern.slf4j.Slf4j; import lombok.val; import org.jooq.DSLContext; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; import java.time.LocalDateTime; @@ -56,19 +57,21 @@ @Service public class AppStatisticsTask extends BaseStatisticsTask { - private final ExecuteMetricsClient executeMetricsClient; + private final ServiceMetricsResource executeMetricsResource; private final StatisticConfig statisticConfig; - protected AppStatisticsTask(BasicServiceManager basicServiceManager, StatisticsDAO statisticsDAO, - DSLContext dslContext, ExecuteMetricsClient executeMetricsClient, + protected AppStatisticsTask(BasicServiceManager basicServiceManager, + StatisticsDAO statisticsDAO, + @Qualifier("job-analysis-dsl-context") DSLContext dslContext, + ServiceMetricsResource executeMetricsResource, StatisticConfig statisticConfig) { super(basicServiceManager, statisticsDAO, dslContext); - this.executeMetricsClient = executeMetricsClient; + this.executeMetricsResource = executeMetricsResource; this.statisticConfig = statisticConfig; } public void calcJoinedApps(LocalDateTime dateTime) { - val resp = executeMetricsClient.getJoinedAppIdList(); + val resp = executeMetricsResource.getJoinedAppIdList(); if (resp == null || !resp.isSuccess()) { log.warn("Fail to call remote getJoinedAppIdList, resp:{}", resp); return; @@ -118,7 +121,7 @@ public void calcActiveApps(LocalDateTime dateTime) { List activeAppIdList = new ArrayList<>(); for (ServiceApplicationDTO serviceApplicationDTO : appDTOList) { Long appId = serviceApplicationDTO.getId(); - val resp = executeMetricsClient.hasExecuteHistory(appId, -1L, TimeUtil.localDateTime2Long(fromTime), + val resp = executeMetricsResource.hasExecuteHistory(appId, -1L, TimeUtil.localDateTime2Long(fromTime), TimeUtil.localDateTime2Long(dateTime)); if (resp == null || !resp.isSuccess()) { log.warn("Fail to call remote hasExecuteHistory, resp:{}", resp); diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/CronPerAppStatisticsTask.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/CronPerAppStatisticsTask.java index d5ebb2ef8c..f00715ce20 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/CronPerAppStatisticsTask.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/CronPerAppStatisticsTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,17 +24,18 @@ package com.tencent.bk.job.analysis.task.statistics.task.impl.app.per; -import com.tencent.bk.job.analysis.client.CronMetricsResourceClient; +import com.tencent.bk.job.analysis.api.consts.StatisticsConstants; +import com.tencent.bk.job.analysis.api.dto.StatisticsDTO; import com.tencent.bk.job.analysis.dao.StatisticsDAO; import com.tencent.bk.job.analysis.service.BasicServiceManager; import com.tencent.bk.job.analysis.task.statistics.anotation.StatisticsTask; import com.tencent.bk.job.analysis.task.statistics.task.BasePerAppStatisticsTask; import com.tencent.bk.job.common.model.InternalResponse; -import com.tencent.bk.job.common.statistics.consts.StatisticsConstants; -import com.tencent.bk.job.common.statistics.model.dto.StatisticsDTO; +import com.tencent.bk.job.crontab.api.inner.ServiceCronMetricsResource; import com.tencent.bk.job.manage.model.inner.resp.ServiceApplicationDTO; import lombok.extern.slf4j.Slf4j; import org.jooq.DSLContext; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; import java.util.ArrayList; @@ -49,12 +50,14 @@ @Service public class CronPerAppStatisticsTask extends BasePerAppStatisticsTask { - private final CronMetricsResourceClient cronMetricsResourceClient; + private final ServiceCronMetricsResource cronMetricsResource; - protected CronPerAppStatisticsTask(BasicServiceManager applicationResourceClient, StatisticsDAO statisticsDAO, - DSLContext dslContext, CronMetricsResourceClient cronMetricsResourceClient) { - super(applicationResourceClient, statisticsDAO, dslContext); - this.cronMetricsResourceClient = cronMetricsResourceClient; + protected CronPerAppStatisticsTask(BasicServiceManager basicServiceManager, + StatisticsDAO statisticsDAO, + @Qualifier("job-analysis-dsl-context") DSLContext dslContext, + ServiceCronMetricsResource cronMetricsResource) { + super(basicServiceManager, statisticsDAO, dslContext); + this.cronMetricsResource = cronMetricsResource; } private StatisticsDTO genCronStatusStatisticsDTO(String dateStr, Long appId, String value, String dimensionValue) { @@ -99,7 +102,7 @@ private StatisticsDTO genCronCronStatisticsDTO(String dateStr, Long appId, Strin public List calcAppCronStatistics(String dateStr, Long appId) { List statisticsDTOList = new ArrayList<>(); // 开启的 - InternalResponse resp = cronMetricsResourceClient.countCronJob(appId, true, null); + InternalResponse resp = cronMetricsResource.countCronJob(appId, true, null); if (resp == null || !resp.isSuccess()) { log.warn("Fail to call remote countCronJob, resp:{}", resp); return Collections.emptyList(); @@ -107,7 +110,7 @@ public List calcAppCronStatistics(String dateStr, Long appId) { Integer activeCronCount = resp.getData(); statisticsDTOList.add(genActiveCronStatisticsDTO(dateStr, appId, activeCronCount.toString())); // 关闭的 - resp = cronMetricsResourceClient.countCronJob(appId, false, null); + resp = cronMetricsResource.countCronJob(appId, false, null); if (resp == null || !resp.isSuccess()) { log.warn("Fail to call remote countCronJob, resp:{}", resp); return statisticsDTOList; @@ -115,7 +118,7 @@ public List calcAppCronStatistics(String dateStr, Long appId) { Integer inActiveCronCount = resp.getData(); statisticsDTOList.add(genInActiveCronStatisticsDTO(dateStr, appId, inActiveCronCount.toString())); // 简单的 - resp = cronMetricsResourceClient.countCronJob(appId, null, false); + resp = cronMetricsResource.countCronJob(appId, null, false); if (resp == null || !resp.isSuccess()) { log.warn("Fail to call remote countCronJob, resp:{}", resp); return statisticsDTOList; @@ -123,7 +126,7 @@ public List calcAppCronStatistics(String dateStr, Long appId) { Integer simpleCronCount = resp.getData(); statisticsDTOList.add(genSimpleCronStatisticsDTO(dateStr, appId, simpleCronCount.toString())); // 周期的 - resp = cronMetricsResourceClient.countCronJob(appId, null, true); + resp = cronMetricsResource.countCronJob(appId, null, true); if (resp == null || !resp.isSuccess()) { log.warn("Fail to call remote countCronJob, resp:{}", resp); return statisticsDTOList; diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/ExecuteTaskDailyPerAppStatisticsTask.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/ExecuteTaskDailyPerAppStatisticsTask.java index 8c9729571b..a7a6cbc417 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/ExecuteTaskDailyPerAppStatisticsTask.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/ExecuteTaskDailyPerAppStatisticsTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,19 +24,20 @@ package com.tencent.bk.job.analysis.task.statistics.task.impl.app.per; -import com.tencent.bk.job.analysis.client.ExecuteMetricsClient; +import com.tencent.bk.job.analysis.api.consts.StatisticsConstants; +import com.tencent.bk.job.analysis.api.dto.StatisticsDTO; import com.tencent.bk.job.analysis.consts.TotalMetricEnum; import com.tencent.bk.job.analysis.dao.StatisticsDAO; import com.tencent.bk.job.analysis.service.BasicServiceManager; import com.tencent.bk.job.analysis.task.statistics.anotation.StatisticsTask; import com.tencent.bk.job.analysis.task.statistics.task.ExecuteBasePerAppStatisticsTask; -import com.tencent.bk.job.common.statistics.consts.StatisticsConstants; -import com.tencent.bk.job.common.statistics.model.dto.StatisticsDTO; import com.tencent.bk.job.common.util.date.DateUtils; +import com.tencent.bk.job.execute.api.inner.ServiceMetricsResource; import com.tencent.bk.job.manage.model.inner.resp.ServiceApplicationDTO; import lombok.extern.slf4j.Slf4j; import org.jooq.DSLContext; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; import java.time.LocalDateTime; @@ -49,10 +50,11 @@ public class ExecuteTaskDailyPerAppStatisticsTask extends ExecuteBasePerAppStatisticsTask { @Autowired - public ExecuteTaskDailyPerAppStatisticsTask(ExecuteMetricsClient executeMetricsClient, - BasicServiceManager basicServiceManager, StatisticsDAO statisticsDAO, - DSLContext dslContext) { - super(executeMetricsClient, basicServiceManager, statisticsDAO, dslContext); + public ExecuteTaskDailyPerAppStatisticsTask(ServiceMetricsResource executeMetricsResource, + BasicServiceManager basicServiceManager, + StatisticsDAO statisticsDAO, + @Qualifier("job-analysis-dsl-context") DSLContext dslContext) { + super(executeMetricsResource, basicServiceManager, statisticsDAO, dslContext); } private StatisticsDTO getExecutedTaskBaseStatisticsDTO(ServiceApplicationDTO app, String timeTag) { diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/ExecuteTaskStartupModePerAppStatisticsTask.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/ExecuteTaskStartupModePerAppStatisticsTask.java index 36f201b434..3e4ad3c034 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/ExecuteTaskStartupModePerAppStatisticsTask.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/ExecuteTaskStartupModePerAppStatisticsTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,17 +24,18 @@ package com.tencent.bk.job.analysis.task.statistics.task.impl.app.per; -import com.tencent.bk.job.analysis.client.ExecuteMetricsClient; +import com.tencent.bk.job.analysis.api.consts.StatisticsConstants; +import com.tencent.bk.job.analysis.api.dto.StatisticsDTO; import com.tencent.bk.job.analysis.dao.StatisticsDAO; import com.tencent.bk.job.analysis.service.BasicServiceManager; import com.tencent.bk.job.analysis.task.statistics.anotation.StatisticsTask; import com.tencent.bk.job.analysis.task.statistics.task.ExecuteBasePerAppStatisticsTask; -import com.tencent.bk.job.common.statistics.consts.StatisticsConstants; -import com.tencent.bk.job.common.statistics.model.dto.StatisticsDTO; +import com.tencent.bk.job.execute.api.inner.ServiceMetricsResource; import com.tencent.bk.job.manage.model.inner.resp.ServiceApplicationDTO; import lombok.extern.slf4j.Slf4j; import org.jooq.DSLContext; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; import java.time.LocalDateTime; @@ -48,10 +49,11 @@ public class ExecuteTaskStartupModePerAppStatisticsTask extends ExecuteBasePerAppStatisticsTask { @Autowired - public ExecuteTaskStartupModePerAppStatisticsTask(ExecuteMetricsClient executeMetricsClient, + public ExecuteTaskStartupModePerAppStatisticsTask(ServiceMetricsResource executeMetricsResource, BasicServiceManager basicServiceManager, - StatisticsDAO statisticsDAO, DSLContext dslContext) { - super(executeMetricsClient, basicServiceManager, statisticsDAO, dslContext); + StatisticsDAO statisticsDAO, + @Qualifier("job-analysis-dsl-context") DSLContext dslContext) { + super(executeMetricsResource, basicServiceManager, statisticsDAO, dslContext); } private StatisticsDTO getStartupModeBaseDTO(ServiceApplicationDTO app, String timeTag) { diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/ExecuteTaskTypePerAppStatisticsTask.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/ExecuteTaskTypePerAppStatisticsTask.java index 030c16ad4c..e9968a43a1 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/ExecuteTaskTypePerAppStatisticsTask.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/ExecuteTaskTypePerAppStatisticsTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,16 +24,17 @@ package com.tencent.bk.job.analysis.task.statistics.task.impl.app.per; -import com.tencent.bk.job.analysis.client.ExecuteMetricsClient; +import com.tencent.bk.job.analysis.api.consts.StatisticsConstants; +import com.tencent.bk.job.analysis.api.dto.StatisticsDTO; import com.tencent.bk.job.analysis.dao.StatisticsDAO; import com.tencent.bk.job.analysis.service.BasicServiceManager; import com.tencent.bk.job.analysis.task.statistics.anotation.StatisticsTask; import com.tencent.bk.job.analysis.task.statistics.task.ExecuteBasePerAppStatisticsTask; -import com.tencent.bk.job.common.statistics.consts.StatisticsConstants; -import com.tencent.bk.job.common.statistics.model.dto.StatisticsDTO; +import com.tencent.bk.job.execute.api.inner.ServiceMetricsResource; import com.tencent.bk.job.manage.model.inner.resp.ServiceApplicationDTO; import org.jooq.DSLContext; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; import java.time.LocalDateTime; @@ -46,10 +47,11 @@ public class ExecuteTaskTypePerAppStatisticsTask extends ExecuteBasePerAppStatisticsTask { @Autowired - public ExecuteTaskTypePerAppStatisticsTask(ExecuteMetricsClient executeMetricsClient, - BasicServiceManager basicServiceManager, StatisticsDAO statisticsDAO, - DSLContext dslContext) { - super(executeMetricsClient, basicServiceManager, statisticsDAO, dslContext); + public ExecuteTaskTypePerAppStatisticsTask(ServiceMetricsResource executeMetricsResource, + BasicServiceManager basicServiceManager, + StatisticsDAO statisticsDAO, + @Qualifier("job-analysis-dsl-context") DSLContext dslContext) { + super(executeMetricsResource, basicServiceManager, statisticsDAO, dslContext); } private StatisticsDTO getTaskTypeBaseStatisticsDTO(ServiceApplicationDTO app, String timeTag) { diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/ExecuteTimeConsumingPerAppStatisticsTask.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/ExecuteTimeConsumingPerAppStatisticsTask.java index 740043135a..efaf346704 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/ExecuteTimeConsumingPerAppStatisticsTask.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/ExecuteTimeConsumingPerAppStatisticsTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,16 +24,17 @@ package com.tencent.bk.job.analysis.task.statistics.task.impl.app.per; -import com.tencent.bk.job.analysis.client.ExecuteMetricsClient; +import com.tencent.bk.job.analysis.api.consts.StatisticsConstants; +import com.tencent.bk.job.analysis.api.dto.StatisticsDTO; import com.tencent.bk.job.analysis.dao.StatisticsDAO; import com.tencent.bk.job.analysis.service.BasicServiceManager; import com.tencent.bk.job.analysis.task.statistics.anotation.StatisticsTask; import com.tencent.bk.job.analysis.task.statistics.task.ExecuteBasePerAppStatisticsTask; -import com.tencent.bk.job.common.statistics.consts.StatisticsConstants; -import com.tencent.bk.job.common.statistics.model.dto.StatisticsDTO; +import com.tencent.bk.job.execute.api.inner.ServiceMetricsResource; import com.tencent.bk.job.manage.model.inner.resp.ServiceApplicationDTO; import org.jooq.DSLContext; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; import java.time.LocalDateTime; @@ -46,10 +47,11 @@ public class ExecuteTimeConsumingPerAppStatisticsTask extends ExecuteBasePerAppStatisticsTask { @Autowired - public ExecuteTimeConsumingPerAppStatisticsTask(ExecuteMetricsClient executeMetricsClient, + public ExecuteTimeConsumingPerAppStatisticsTask(ServiceMetricsResource executeMetricsResource, BasicServiceManager basicServiceManager, - StatisticsDAO statisticsDAO, DSLContext dslContext) { - super(executeMetricsClient, basicServiceManager, statisticsDAO, dslContext); + StatisticsDAO statisticsDAO, + @Qualifier("job-analysis-dsl-context") DSLContext dslContext) { + super(executeMetricsResource, basicServiceManager, statisticsDAO, dslContext); } private StatisticsDTO getTimeConsumingBaseDTO(ServiceApplicationDTO app, String timeTag) { diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/FastExecuteScriptPerAppStatisticsTask.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/FastExecuteScriptPerAppStatisticsTask.java index 00458d0d80..b98117d29f 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/FastExecuteScriptPerAppStatisticsTask.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/FastExecuteScriptPerAppStatisticsTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,17 +24,18 @@ package com.tencent.bk.job.analysis.task.statistics.task.impl.app.per; -import com.tencent.bk.job.analysis.client.ExecuteMetricsClient; +import com.tencent.bk.job.analysis.api.consts.StatisticsConstants; +import com.tencent.bk.job.analysis.api.dto.StatisticsDTO; import com.tencent.bk.job.analysis.dao.StatisticsDAO; import com.tencent.bk.job.analysis.service.BasicServiceManager; import com.tencent.bk.job.analysis.task.statistics.anotation.StatisticsTask; import com.tencent.bk.job.analysis.task.statistics.task.ExecuteBasePerAppStatisticsTask; -import com.tencent.bk.job.common.statistics.consts.StatisticsConstants; -import com.tencent.bk.job.common.statistics.model.dto.StatisticsDTO; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; +import com.tencent.bk.job.execute.api.inner.ServiceMetricsResource; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; import com.tencent.bk.job.manage.model.inner.resp.ServiceApplicationDTO; import org.jooq.DSLContext; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; import java.time.LocalDateTime; @@ -47,10 +48,11 @@ public class FastExecuteScriptPerAppStatisticsTask extends ExecuteBasePerAppStatisticsTask { @Autowired - public FastExecuteScriptPerAppStatisticsTask(ExecuteMetricsClient executeMetricsClient, - BasicServiceManager basicServiceManager, StatisticsDAO statisticsDAO - , DSLContext dslContext) { - super(executeMetricsClient, basicServiceManager, statisticsDAO, dslContext); + public FastExecuteScriptPerAppStatisticsTask(ServiceMetricsResource executeMetricsResource, + BasicServiceManager basicServiceManager, + StatisticsDAO statisticsDAO, + @Qualifier("job-analysis-dsl-context") DSLContext dslContext) { + super(executeMetricsResource, basicServiceManager, statisticsDAO, dslContext); } private StatisticsDTO getRunStatusBaseStatisticsDTO(ServiceApplicationDTO app, String timeTag) { diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/FastPushFilePerAppStatisticsTask.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/FastPushFilePerAppStatisticsTask.java index 4d482fb6c1..bdce5a9727 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/FastPushFilePerAppStatisticsTask.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/FastPushFilePerAppStatisticsTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,16 +24,17 @@ package com.tencent.bk.job.analysis.task.statistics.task.impl.app.per; -import com.tencent.bk.job.analysis.client.ExecuteMetricsClient; +import com.tencent.bk.job.analysis.api.consts.StatisticsConstants; +import com.tencent.bk.job.analysis.api.dto.StatisticsDTO; import com.tencent.bk.job.analysis.dao.StatisticsDAO; import com.tencent.bk.job.analysis.service.BasicServiceManager; import com.tencent.bk.job.analysis.task.statistics.anotation.StatisticsTask; import com.tencent.bk.job.analysis.task.statistics.task.ExecuteBasePerAppStatisticsTask; -import com.tencent.bk.job.common.statistics.consts.StatisticsConstants; -import com.tencent.bk.job.common.statistics.model.dto.StatisticsDTO; +import com.tencent.bk.job.execute.api.inner.ServiceMetricsResource; import com.tencent.bk.job.manage.model.inner.resp.ServiceApplicationDTO; import org.jooq.DSLContext; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; import java.time.LocalDateTime; @@ -46,10 +47,11 @@ public class FastPushFilePerAppStatisticsTask extends ExecuteBasePerAppStatisticsTask { @Autowired - public FastPushFilePerAppStatisticsTask(ExecuteMetricsClient executeMetricsClient, - BasicServiceManager basicServiceManager, StatisticsDAO statisticsDAO, - DSLContext dslContext) { - super(executeMetricsClient, basicServiceManager, statisticsDAO, dslContext); + public FastPushFilePerAppStatisticsTask(ServiceMetricsResource executeMetricsResource, + BasicServiceManager basicServiceManager, + StatisticsDAO statisticsDAO, + @Qualifier("job-analysis-dsl-context") DSLContext dslContext) { + super(executeMetricsResource, basicServiceManager, statisticsDAO, dslContext); } private StatisticsDTO getRunStatusBaseDTO(ServiceApplicationDTO app, String timeTag) { diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/RollingTaskDailyPerAppStatisticsTask.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/RollingTaskDailyPerAppStatisticsTask.java new file mode 100644 index 0000000000..515bf64647 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/RollingTaskDailyPerAppStatisticsTask.java @@ -0,0 +1,131 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.task.statistics.task.impl.app.per; + +import com.tencent.bk.job.analysis.api.consts.StatisticsConstants; +import com.tencent.bk.job.analysis.api.dto.StatisticsDTO; +import com.tencent.bk.job.analysis.dao.StatisticsDAO; +import com.tencent.bk.job.analysis.service.BasicServiceManager; +import com.tencent.bk.job.analysis.task.statistics.anotation.StatisticsTask; +import com.tencent.bk.job.analysis.task.statistics.task.ExecuteBasePerAppStatisticsTask; +import com.tencent.bk.job.common.util.date.DateUtils; +import com.tencent.bk.job.execute.api.inner.ServiceMetricsResource; +import com.tencent.bk.job.manage.model.inner.resp.ServiceApplicationDTO; +import lombok.extern.slf4j.Slf4j; +import org.jooq.DSLContext; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Service; + +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +@Slf4j +@StatisticsTask +@Service +public class RollingTaskDailyPerAppStatisticsTask extends ExecuteBasePerAppStatisticsTask { + + @Autowired + public RollingTaskDailyPerAppStatisticsTask(ServiceMetricsResource executeMetricsResource, + BasicServiceManager basicServiceManager, + StatisticsDAO statisticsDAO, + @Qualifier("job-analysis-dsl-context") DSLContext dslContext) { + super(executeMetricsResource, basicServiceManager, statisticsDAO, dslContext); + } + + private StatisticsDTO getFailedTaskBaseStatisticsDTO(ServiceApplicationDTO app, String timeTag) { + StatisticsDTO statisticsDTO = getTimeUnitBaseStatisticsDTO(app, timeTag); + statisticsDTO.setResource(StatisticsConstants.RESOURCE_ROLLING_FAILED_TASK);//失败的任务 + return statisticsDTO; + } + + private StatisticsDTO getTimeUnitBaseStatisticsDTO(ServiceApplicationDTO app, String timeTag) { + StatisticsDTO statisticsDTO = new StatisticsDTO(); + statisticsDTO.setAppId(app.getId()); + statisticsDTO.setCreateTime(System.currentTimeMillis()); + statisticsDTO.setDate(timeTag); + statisticsDTO.setDimension(StatisticsConstants.DIMENSION_TIME_UNIT);//统计的单位时间 + return statisticsDTO; + } + + public List getStatisticsFrom(ServiceApplicationDTO app, Long fromTime, Long toTime, + String timeTag) { + List statisticsDTOList = new ArrayList<>(); + StatisticsDTO statisticsDTO = getFailedTaskBaseStatisticsDTO(app, timeTag); + statisticsDTO.setDimensionValue(StatisticsConstants.DIMENSION_VALUE_TIME_UNIT_DAY); + addExecuteStatisticsDTO(statisticsDTO, statisticsDTOList); + return statisticsDTOList; + } + + /** + * 汇总所有业务的失败执行总量 + * + * @param dayTimeStr + */ + public void updateTotalFailedTaskStatistics(String dayTimeStr) { + List statisticsDTOList = statisticsDAO.getStatisticsList( + null, + Collections.singletonList(StatisticsConstants.DEFAULT_APP_ID), + StatisticsConstants.RESOURCE_ROLLING_FAILED_TASK, + StatisticsConstants.DIMENSION_TIME_UNIT, + StatisticsConstants.DIMENSION_VALUE_TIME_UNIT_DAY, + dayTimeStr); + log.debug("statisticsDTOList.size={}", statisticsDTOList.size()); + Long totalValue = 0L; + for (StatisticsDTO dto : statisticsDTOList) { + log.debug("add {} data", dto.getDate()); + totalValue += Long.parseLong(dto.getValue()); + } + StatisticsDTO statisticsDTO = new StatisticsDTO(); + statisticsDTO.setAppId(StatisticsConstants.DEFAULT_APP_ID); + statisticsDTO.setCreateTime(System.currentTimeMillis()); + statisticsDTO.setDate(dayTimeStr); + statisticsDTO.setValue(totalValue.toString()); + statisticsDTO.setResource(StatisticsConstants.RESOURCE_ROLLING_FAILED_TASK); + statisticsDTO.setDimension(StatisticsConstants.DIMENSION_TIME_UNIT); + statisticsDTO.setDimensionValue(StatisticsConstants.DIMENSION_VALUE_TIME_UNIT_DAY); + statisticsDAO.upsertStatistics(dslContext, statisticsDTO); + } + + @Override + public void genStatisticsByDay(LocalDateTime dateTime) { + super.genStatisticsByDay(dateTime); + String dayTimeStr = getDayTimeStr(dateTime); + String todayDateStr = getDayTimeStr(LocalDateTime.now()); + // 更新全业务全局汇总数据 + // 比统计日期晚的统计数据都需要更新 + String timeStr = dayTimeStr; + int maxCount = 3650; + int count = 0; + while (timeStr.compareTo(todayDateStr) <= 0 && count < maxCount) { + log.debug("update total TaskStatistics of {}, count={}", timeStr, count); + updateTotalFailedTaskStatistics(timeStr); + timeStr = DateUtils.getNextDateStr(timeStr); + count += 1; + } + } +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/RollingTaskPerAppStatisticsTask.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/RollingTaskPerAppStatisticsTask.java new file mode 100644 index 0000000000..8083b47f65 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/RollingTaskPerAppStatisticsTask.java @@ -0,0 +1,130 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.task.statistics.task.impl.app.per; + +import com.tencent.bk.job.analysis.api.consts.StatisticsConstants; +import com.tencent.bk.job.analysis.api.dto.StatisticsDTO; +import com.tencent.bk.job.analysis.dao.StatisticsDAO; +import com.tencent.bk.job.analysis.service.BasicServiceManager; +import com.tencent.bk.job.analysis.task.statistics.anotation.StatisticsTask; +import com.tencent.bk.job.analysis.task.statistics.task.ExecuteBasePerAppStatisticsTask; +import com.tencent.bk.job.execute.api.inner.ServiceMetricsResource; +import com.tencent.bk.job.manage.model.inner.resp.ServiceApplicationDTO; +import org.jooq.DSLContext; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Service; + +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +@StatisticsTask +@Service +public class RollingTaskPerAppStatisticsTask extends ExecuteBasePerAppStatisticsTask { + + @Autowired + public RollingTaskPerAppStatisticsTask(ServiceMetricsResource executeMetricsResource, + BasicServiceManager basicServiceManager, + StatisticsDAO statisticsDAO, + @Qualifier("job-analysis-dsl-context") DSLContext dslContext) { + super(executeMetricsResource, basicServiceManager, statisticsDAO, dslContext); + } + + private StatisticsDTO getTaskTypeBaseStatisticsDTO(ServiceApplicationDTO app, String timeTag) { + StatisticsDTO statisticsDTO = new StatisticsDTO(); + statisticsDTO.setAppId(app.getId()); + statisticsDTO.setCreateTime(System.currentTimeMillis()); + statisticsDTO.setDate(timeTag); + statisticsDTO.setResource(StatisticsConstants.RESOURCE_ROLLING_TASK);//滚动执行过的任务 + statisticsDTO.setDimension(StatisticsConstants.DIMENSION_TASK_TYPE);//作业类型 + return statisticsDTO; + } + + @Override + public List getStatisticsFrom(ServiceApplicationDTO app, Long fromTime, Long toTime, + String timeTag) { + List statisticsDTOList = new ArrayList<>(); + // 1.普通作业 + StatisticsDTO statisticsDTO = getTaskTypeBaseStatisticsDTO(app, timeTag); + statisticsDTO.setDimensionValue(StatisticsConstants.DIMENSION_VALUE_TASK_TYPE_EXECUTE_TASK); + addExecuteStatisticsDTO(statisticsDTO, statisticsDTOList); + // 2.快速执行脚本 + statisticsDTO = getTaskTypeBaseStatisticsDTO(app, timeTag); + statisticsDTO.setDimensionValue(StatisticsConstants.DIMENSION_VALUE_TASK_TYPE_FAST_EXECUTE_SCRIPT); + addExecuteStatisticsDTO(statisticsDTO, statisticsDTOList); + // 3.快速分发文件 + statisticsDTO = getTaskTypeBaseStatisticsDTO(app, timeTag); + statisticsDTO.setDimensionValue(StatisticsConstants.DIMENSION_VALUE_TASK_TYPE_FAST_PUSH_FILE); + addExecuteStatisticsDTO(statisticsDTO, statisticsDTOList); + return statisticsDTOList; + } + + /** + * 聚合多个业务的任务类型数据 + * + * @param dayTimeStr + */ + public void aggregateAllAppTaskTypeStatistics(String dayTimeStr) { + aggregateAllAppTaskTypeStatistics(dayTimeStr, StatisticsConstants.DIMENSION_VALUE_TASK_TYPE_EXECUTE_TASK); + aggregateAllAppTaskTypeStatistics(dayTimeStr, + StatisticsConstants.DIMENSION_VALUE_TASK_TYPE_FAST_EXECUTE_SCRIPT); + aggregateAllAppTaskTypeStatistics(dayTimeStr, StatisticsConstants.DIMENSION_VALUE_TASK_TYPE_FAST_PUSH_FILE); + } + + public void aggregateAllAppTaskTypeStatistics(String dayTimeStr, String timeConsumingDimensionValue) { + Long totalValue = statisticsDAO.getTotalValueOfStatisticsList(null, + Collections.singletonList(StatisticsConstants.DEFAULT_APP_ID), StatisticsConstants.RESOURCE_ROLLING_TASK + , StatisticsConstants.DIMENSION_TASK_TYPE, timeConsumingDimensionValue, dayTimeStr); + StatisticsDTO statisticsDTO = new StatisticsDTO(); + statisticsDTO.setAppId(StatisticsConstants.DEFAULT_APP_ID); + statisticsDTO.setCreateTime(System.currentTimeMillis()); + statisticsDTO.setDate(dayTimeStr); + statisticsDTO.setValue(totalValue.toString()); + statisticsDTO.setResource(StatisticsConstants.RESOURCE_ONE_DAY_ROLLING_TASK_OF_ALL_APP); + statisticsDTO.setDimension(StatisticsConstants.DIMENSION_TASK_TYPE); + statisticsDTO.setDimensionValue(timeConsumingDimensionValue); + statisticsDAO.upsertStatistics(dslContext, statisticsDTO); + } + + @Override + public void genStatisticsByDay(LocalDateTime dateTime) { + super.genStatisticsByDay(dateTime); + // 聚合多个业务的数据 + String dayTimeStr = getDayTimeStr(dateTime); + aggregateAllAppTaskTypeStatistics(dayTimeStr); + } + + @Override + public boolean isDataComplete(String targetDateStr) { + boolean executedTaskByTaskTypeDataExists = statisticsDAO.existsStatistics(null, null, + StatisticsConstants.RESOURCE_ROLLING_TASK, StatisticsConstants.DIMENSION_TASK_TYPE, null, targetDateStr); + boolean allAppExecutedTaskByTaskTypeDataExists = statisticsDAO.existsStatistics(null, null, + StatisticsConstants.RESOURCE_ONE_DAY_ROLLING_TASK_OF_ALL_APP, StatisticsConstants.DIMENSION_TASK_TYPE, + null, targetDateStr); + return executedTaskByTaskTypeDataExists && allAppExecutedTaskByTaskTypeDataExists; + } +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/ScriptCiteInfoPerAppStatisticsTask.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/ScriptCiteInfoPerAppStatisticsTask.java index 0598240975..e67082be99 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/ScriptCiteInfoPerAppStatisticsTask.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/ScriptCiteInfoPerAppStatisticsTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,17 +24,18 @@ package com.tencent.bk.job.analysis.task.statistics.task.impl.app.per; -import com.tencent.bk.job.analysis.client.ManageMetricsClient; +import com.tencent.bk.job.analysis.api.consts.StatisticsConstants; +import com.tencent.bk.job.analysis.api.dto.StatisticsDTO; import com.tencent.bk.job.analysis.dao.StatisticsDAO; import com.tencent.bk.job.analysis.service.BasicServiceManager; import com.tencent.bk.job.analysis.task.statistics.anotation.StatisticsTask; import com.tencent.bk.job.analysis.task.statistics.task.BasePerAppStatisticsTask; import com.tencent.bk.job.common.model.InternalResponse; -import com.tencent.bk.job.common.statistics.consts.StatisticsConstants; -import com.tencent.bk.job.common.statistics.model.dto.StatisticsDTO; +import com.tencent.bk.job.manage.api.inner.ServiceMetricsResource; import com.tencent.bk.job.manage.model.inner.resp.ServiceApplicationDTO; import lombok.extern.slf4j.Slf4j; import org.jooq.DSLContext; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; import java.util.ArrayList; @@ -49,12 +50,14 @@ @Service public class ScriptCiteInfoPerAppStatisticsTask extends BasePerAppStatisticsTask { - private final ManageMetricsClient manageMetricsClient; + private final ServiceMetricsResource manageMetricsResource; - protected ScriptCiteInfoPerAppStatisticsTask(BasicServiceManager basicServiceManager, StatisticsDAO statisticsDAO - , DSLContext dslContext, ManageMetricsClient manageMetricsClient) { + protected ScriptCiteInfoPerAppStatisticsTask(BasicServiceManager basicServiceManager, + StatisticsDAO statisticsDAO, + @Qualifier("job-analysis-dsl-context") DSLContext dslContext, + ServiceMetricsResource manageMetricsResource) { super(basicServiceManager, statisticsDAO, dslContext); - this.manageMetricsClient = manageMetricsClient; + this.manageMetricsResource = manageMetricsResource; } private StatisticsDTO genScriptCountStatisticsDTO(String dateStr, Long appId, String value) { @@ -98,7 +101,7 @@ private StatisticsDTO genCitedScriptStepCountDTO(String dateStr, Long appId, Str public List calcAppScriptCiteInfo(String dateStr, Long appId) { List statisticsDTOList = new ArrayList<>(); // 1.统计脚本总数 - InternalResponse resp = manageMetricsClient.countScripts( + InternalResponse resp = manageMetricsResource.countScripts( appId, null, null @@ -110,7 +113,7 @@ public List calcAppScriptCiteInfo(String dateStr, Long appId) { Integer scriptCount = resp.getData(); statisticsDTOList.add(genScriptCountStatisticsDTO(dateStr, appId, scriptCount.toString())); // 2.统计被引用的脚本总数 - resp = manageMetricsClient.countCiteScripts(appId); + resp = manageMetricsResource.countCiteScripts(appId); if (resp == null || !resp.isSuccess()) { log.warn("Fail to call remote countCiteScripts, resp:{}", resp); return statisticsDTOList; @@ -118,7 +121,7 @@ public List calcAppScriptCiteInfo(String dateStr, Long appId) { Integer citedScriptCount = resp.getData(); statisticsDTOList.add(genCitedScriptCountDTO(dateStr, appId, citedScriptCount.toString())); // 3.统计引用脚本的步骤总数 - resp = manageMetricsClient.countCiteScriptSteps(appId); + resp = manageMetricsResource.countCiteScriptSteps(appId); if (resp == null || !resp.isSuccess()) { log.warn("Fail to call remote countCiteScriptSteps, resp:{}", resp); return statisticsDTOList; diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/ScriptPerAppStatisticsTask.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/ScriptPerAppStatisticsTask.java index 0acbb946d3..9d7e05cdd0 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/ScriptPerAppStatisticsTask.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/ScriptPerAppStatisticsTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,20 +24,21 @@ package com.tencent.bk.job.analysis.task.statistics.task.impl.app.per; -import com.tencent.bk.job.analysis.client.ManageMetricsClient; +import com.tencent.bk.job.analysis.api.consts.StatisticsConstants; +import com.tencent.bk.job.analysis.api.dto.StatisticsDTO; import com.tencent.bk.job.analysis.consts.TotalMetricEnum; import com.tencent.bk.job.analysis.dao.StatisticsDAO; import com.tencent.bk.job.analysis.service.BasicServiceManager; import com.tencent.bk.job.analysis.task.statistics.anotation.StatisticsTask; import com.tencent.bk.job.analysis.task.statistics.task.BasePerAppStatisticsTask; import com.tencent.bk.job.common.model.InternalResponse; -import com.tencent.bk.job.common.statistics.consts.StatisticsConstants; -import com.tencent.bk.job.common.statistics.model.dto.StatisticsDTO; -import com.tencent.bk.job.manage.common.consts.JobResourceStatusEnum; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.JobResourceStatusEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; +import com.tencent.bk.job.manage.api.inner.ServiceMetricsResource; import com.tencent.bk.job.manage.model.inner.resp.ServiceApplicationDTO; import lombok.extern.slf4j.Slf4j; import org.jooq.DSLContext; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; import java.util.ArrayList; @@ -52,12 +53,14 @@ @Service public class ScriptPerAppStatisticsTask extends BasePerAppStatisticsTask { - private final ManageMetricsClient manageMetricsClient; + private final ServiceMetricsResource manageMetricsClient; - protected ScriptPerAppStatisticsTask(BasicServiceManager basicServiceManager, StatisticsDAO statisticsDAO, - DSLContext dslContext, ManageMetricsClient manageMetricsClient) { + protected ScriptPerAppStatisticsTask(BasicServiceManager basicServiceManager, + StatisticsDAO statisticsDAO, + @Qualifier("job-analysis-dsl-context") DSLContext dslContext, + ServiceMetricsResource manageMetricsResource) { super(basicServiceManager, statisticsDAO, dslContext); - this.manageMetricsClient = manageMetricsClient; + this.manageMetricsClient = manageMetricsResource; } private StatisticsDTO genScriptTotalDTO(String dateStr, Long appId, String value) { diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/TagPerAppStatisticsTask.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/TagPerAppStatisticsTask.java index 914bc3f312..cb41e76888 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/TagPerAppStatisticsTask.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/TagPerAppStatisticsTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,20 +24,21 @@ package com.tencent.bk.job.analysis.task.statistics.task.impl.app.per; -import com.tencent.bk.job.analysis.client.ManageMetricsClient; -import com.tencent.bk.job.analysis.client.TagClient; +import com.tencent.bk.job.analysis.api.consts.StatisticsConstants; +import com.tencent.bk.job.analysis.api.dto.StatisticsDTO; import com.tencent.bk.job.analysis.dao.StatisticsDAO; import com.tencent.bk.job.analysis.service.BasicServiceManager; import com.tencent.bk.job.analysis.task.statistics.anotation.StatisticsTask; import com.tencent.bk.job.analysis.task.statistics.task.BasePerAppStatisticsTask; import com.tencent.bk.job.common.model.InternalResponse; -import com.tencent.bk.job.common.statistics.consts.StatisticsConstants; -import com.tencent.bk.job.common.statistics.model.dto.StatisticsDTO; import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.manage.api.inner.ServiceMetricsResource; +import com.tencent.bk.job.manage.api.inner.ServiceTagResource; import com.tencent.bk.job.manage.model.inner.ServiceTagDTO; import com.tencent.bk.job.manage.model.inner.resp.ServiceApplicationDTO; import lombok.extern.slf4j.Slf4j; import org.jooq.DSLContext; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; import java.util.ArrayList; @@ -54,15 +55,17 @@ @Service public class TagPerAppStatisticsTask extends BasePerAppStatisticsTask { - private final ManageMetricsClient manageMetricsClient; - private final TagClient tagClient; + private final ServiceMetricsResource manageMetricsResource; + private final ServiceTagResource tagResource; - protected TagPerAppStatisticsTask(BasicServiceManager basicServiceManager, StatisticsDAO statisticsDAO, - DSLContext dslContext, ManageMetricsClient manageMetricsClient, - TagClient tagClient) { + protected TagPerAppStatisticsTask(BasicServiceManager basicServiceManager, + StatisticsDAO statisticsDAO, + @Qualifier("job-analysis-dsl-context") DSLContext dslContext, + ServiceMetricsResource manageMetricsResource, + ServiceTagResource tagResource) { super(basicServiceManager, statisticsDAO, dslContext); - this.manageMetricsClient = manageMetricsClient; - this.tagClient = tagClient; + this.manageMetricsResource = manageMetricsResource; + this.tagResource = tagResource; } private StatisticsDTO genTagDistributionStatisticsDTO(String dateStr, Long appId, String value) { @@ -83,7 +86,7 @@ public List calcAppTagDistributionStatistics(String dateStr, Long Map tagCitedCountMap = new HashMap<>(); for (ServiceTagDTO serviceTagDTO : tagList) { Long tagId = serviceTagDTO.getId(); - InternalResponse resp = manageMetricsClient.tagCitedCount(appId, tagId); + InternalResponse resp = manageMetricsResource.tagCitedCount(appId, tagId); if (resp == null || !resp.isSuccess()) { log.warn("Fail to call remote tagCitedCount, resp:{}", resp); continue; @@ -106,7 +109,7 @@ public List calcAppTagDistributionStatistics(String dateStr, Long public List getStatisticsFrom(ServiceApplicationDTO app, Long fromTime, Long toTime, String timeTag) { // 获取公共标签 - InternalResponse> resp = tagClient.listPublicTags(); + InternalResponse> resp = tagResource.listPublicTags(); if (resp == null || !resp.isSuccess()) { log.warn("Fail to call remote listPublicTags, resp:{}", resp); return Collections.emptyList(); @@ -114,7 +117,7 @@ public List getStatisticsFrom(ServiceApplicationDTO app, Long fro List publicTagList = resp.getData(); log.debug("publicTagList={}", publicTagList); Long appId = app.getId(); - resp = tagClient.listTags(appId); + resp = tagResource.listTags(appId); if (resp == null || !resp.isSuccess()) { log.warn("Fail to call remote listTags, resp:{}", resp); return Collections.emptyList(); diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/TaskPlanPerAppStatisticsTask.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/TaskPlanPerAppStatisticsTask.java index 699d1fd72b..9daf89e219 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/TaskPlanPerAppStatisticsTask.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/TaskPlanPerAppStatisticsTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,18 +24,19 @@ package com.tencent.bk.job.analysis.task.statistics.task.impl.app.per; -import com.tencent.bk.job.analysis.client.ManageMetricsClient; +import com.tencent.bk.job.analysis.api.consts.StatisticsConstants; +import com.tencent.bk.job.analysis.api.dto.StatisticsDTO; import com.tencent.bk.job.analysis.consts.TotalMetricEnum; import com.tencent.bk.job.analysis.dao.StatisticsDAO; import com.tencent.bk.job.analysis.service.BasicServiceManager; import com.tencent.bk.job.analysis.task.statistics.anotation.StatisticsTask; import com.tencent.bk.job.analysis.task.statistics.task.BasePerAppStatisticsTask; import com.tencent.bk.job.common.model.InternalResponse; -import com.tencent.bk.job.common.statistics.consts.StatisticsConstants; -import com.tencent.bk.job.common.statistics.model.dto.StatisticsDTO; +import com.tencent.bk.job.manage.api.inner.ServiceMetricsResource; import com.tencent.bk.job.manage.model.inner.resp.ServiceApplicationDTO; import lombok.extern.slf4j.Slf4j; import org.jooq.DSLContext; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; import java.util.ArrayList; @@ -49,12 +50,14 @@ @Service public class TaskPlanPerAppStatisticsTask extends BasePerAppStatisticsTask { - private final ManageMetricsClient manageMetricsClient; + private final ServiceMetricsResource manageMetricsResource; - protected TaskPlanPerAppStatisticsTask(BasicServiceManager basicServiceManager, StatisticsDAO statisticsDAO, - DSLContext dslContext, ManageMetricsClient manageMetricsClient) { + protected TaskPlanPerAppStatisticsTask(BasicServiceManager basicServiceManager, + StatisticsDAO statisticsDAO, + @Qualifier("job-analysis-dsl-context") DSLContext dslContext, + ServiceMetricsResource manageMetricsResource) { super(basicServiceManager, statisticsDAO, dslContext); - this.manageMetricsClient = manageMetricsClient; + this.manageMetricsResource = manageMetricsResource; } private StatisticsDTO genTaskPlanTotalStatisticsDTO(String dateStr, Long appId, String value) { @@ -73,7 +76,7 @@ private StatisticsDTO genTaskPlanTotalStatisticsDTO(String dateStr, Long appId, public List calcAppTaskPlanTotalStatistics(String dateStr, Long appId) { List statisticsDTOList = new ArrayList<>(); - InternalResponse resp = manageMetricsClient.countTaskPlans(appId); + InternalResponse resp = manageMetricsResource.countTaskPlans(appId); if (resp == null || !resp.isSuccess()) { log.warn("Fail to call remote countTaskPlans, resp:{}", resp); return statisticsDTOList; diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/TaskTemplatePerAppStatisticsTask.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/TaskTemplatePerAppStatisticsTask.java index 6206a75514..e47c44ddeb 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/TaskTemplatePerAppStatisticsTask.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/TaskTemplatePerAppStatisticsTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,18 +24,19 @@ package com.tencent.bk.job.analysis.task.statistics.task.impl.app.per; -import com.tencent.bk.job.analysis.client.ManageMetricsClient; +import com.tencent.bk.job.analysis.api.consts.StatisticsConstants; +import com.tencent.bk.job.analysis.api.dto.StatisticsDTO; import com.tencent.bk.job.analysis.consts.TotalMetricEnum; import com.tencent.bk.job.analysis.dao.StatisticsDAO; import com.tencent.bk.job.analysis.service.BasicServiceManager; import com.tencent.bk.job.analysis.task.statistics.anotation.StatisticsTask; import com.tencent.bk.job.analysis.task.statistics.task.BasePerAppStatisticsTask; import com.tencent.bk.job.common.model.InternalResponse; -import com.tencent.bk.job.common.statistics.consts.StatisticsConstants; -import com.tencent.bk.job.common.statistics.model.dto.StatisticsDTO; +import com.tencent.bk.job.manage.api.inner.ServiceMetricsResource; import com.tencent.bk.job.manage.model.inner.resp.ServiceApplicationDTO; import lombok.extern.slf4j.Slf4j; import org.jooq.DSLContext; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; import java.util.ArrayList; @@ -50,16 +51,16 @@ @Service public class TaskTemplatePerAppStatisticsTask extends BasePerAppStatisticsTask { - private final ManageMetricsClient manageMetricsClient; + private final ServiceMetricsResource manageMetricsResource; protected TaskTemplatePerAppStatisticsTask( BasicServiceManager basicServiceManager, StatisticsDAO statisticsDAO, - DSLContext dslContext, - ManageMetricsClient manageMetricsClient + @Qualifier("job-analysis-dsl-context") DSLContext dslContext, + ServiceMetricsResource manageMetricsResource ) { super(basicServiceManager, statisticsDAO, dslContext); - this.manageMetricsClient = manageMetricsClient; + this.manageMetricsResource = manageMetricsResource; } private StatisticsDTO genTaskTemplateTotalStatisticsDTO(String dateStr, Long appId, String value) { @@ -76,7 +77,7 @@ private StatisticsDTO genTaskTemplateTotalStatisticsDTO(String dateStr, Long app public List calcAppTaskTemplateTotalStatistics(String dateStr, Long appId) { List statisticsDTOList = new ArrayList<>(); - InternalResponse resp = manageMetricsClient.countTemplates(appId); + InternalResponse resp = manageMetricsResource.countTemplates(appId); if (resp == null || !resp.isSuccess()) { log.error("Fail to call remote countTemplates, resp:{}", resp); return Collections.emptyList(); diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/TaskTemplateStepPerAppStatisticsTask.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/TaskTemplateStepPerAppStatisticsTask.java index fc9abd5a35..8961f55e78 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/TaskTemplateStepPerAppStatisticsTask.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/app/per/TaskTemplateStepPerAppStatisticsTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,21 +24,22 @@ package com.tencent.bk.job.analysis.task.statistics.task.impl.app.per; -import com.tencent.bk.job.analysis.client.ManageMetricsClient; +import com.tencent.bk.job.analysis.api.consts.StatisticsConstants; +import com.tencent.bk.job.analysis.api.dto.StatisticsDTO; import com.tencent.bk.job.analysis.consts.StepTypeEnum; import com.tencent.bk.job.analysis.dao.StatisticsDAO; import com.tencent.bk.job.analysis.service.BasicServiceManager; import com.tencent.bk.job.analysis.task.statistics.anotation.StatisticsTask; import com.tencent.bk.job.analysis.task.statistics.task.BasePerAppStatisticsTask; import com.tencent.bk.job.common.model.InternalResponse; -import com.tencent.bk.job.common.statistics.consts.StatisticsConstants; -import com.tencent.bk.job.common.statistics.model.dto.StatisticsDTO; -import com.tencent.bk.job.manage.common.consts.task.TaskFileTypeEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskScriptSourceEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskStepTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskFileTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskScriptSourceEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskStepTypeEnum; +import com.tencent.bk.job.manage.api.inner.ServiceMetricsResource; import com.tencent.bk.job.manage.model.inner.resp.ServiceApplicationDTO; import lombok.extern.slf4j.Slf4j; import org.jooq.DSLContext; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; import java.util.ArrayList; @@ -52,13 +53,14 @@ @Service public class TaskTemplateStepPerAppStatisticsTask extends BasePerAppStatisticsTask { - private final ManageMetricsClient manageMetricsClient; + private final ServiceMetricsResource manageMetricsResource; protected TaskTemplateStepPerAppStatisticsTask(BasicServiceManager basicServiceManager, - StatisticsDAO statisticsDAO, DSLContext dslContext, - ManageMetricsClient manageMetricsClient) { + StatisticsDAO statisticsDAO, + @Qualifier("job-analysis-dsl-context") DSLContext dslContext, + ServiceMetricsResource manageMetricsResource) { super(basicServiceManager, statisticsDAO, dslContext); - this.manageMetricsClient = manageMetricsClient; + this.manageMetricsResource = manageMetricsResource; } private StatisticsDTO genTaskTplStepTypeStatistics(String dateStr, Long appId, String value, @@ -76,7 +78,7 @@ private StatisticsDTO genTaskTplStepTypeStatistics(String dateStr, Long appId, S public List calcAppTaskTemplateStepTypeStatistics(String dateStr, Long appId) { List statisticsDTOList = new ArrayList<>(); // 1.手工录入脚本的步骤统计 - InternalResponse resp = manageMetricsClient.countTemplateSteps(appId, TaskStepTypeEnum.SCRIPT, + InternalResponse resp = manageMetricsResource.countTemplateSteps(appId, TaskStepTypeEnum.SCRIPT, TaskScriptSourceEnum.LOCAL, null); if (resp == null || !resp.isSuccess()) { log.warn("Fail to call remote countTemplateSteps, resp:{}", resp); @@ -86,14 +88,14 @@ public List calcAppTaskTemplateStepTypeStatistics(String dateStr, statisticsDTOList.add(genTaskTplStepTypeStatistics(dateStr, appId, localScriptStepCount.toString(), StatisticsConstants.DIMENSION_VALUE_STEP_TYPE_PREFIX + StepTypeEnum.SCRIPT_MANUAL.name())); // 2.引用脚本的步骤统计 - resp = manageMetricsClient.countTemplateSteps(appId, TaskStepTypeEnum.SCRIPT, TaskScriptSourceEnum.CITING, + resp = manageMetricsResource.countTemplateSteps(appId, TaskStepTypeEnum.SCRIPT, TaskScriptSourceEnum.CITING, null); if (resp == null || !resp.isSuccess()) { log.warn("Fail to call remote countTemplateSteps, resp:{}", resp); return statisticsDTOList; } Integer citedAppScriptStepCount = resp.getData(); - resp = manageMetricsClient.countTemplateSteps(appId, TaskStepTypeEnum.SCRIPT, TaskScriptSourceEnum.PUBLIC, + resp = manageMetricsResource.countTemplateSteps(appId, TaskStepTypeEnum.SCRIPT, TaskScriptSourceEnum.PUBLIC, null); if (resp == null || !resp.isSuccess()) { log.warn("Fail to call remote countTemplateSteps, resp:{}", resp); @@ -105,7 +107,7 @@ public List calcAppTaskTemplateStepTypeStatistics(String dateStr, Integer.toString(citedScriptStepCount), StatisticsConstants.DIMENSION_VALUE_STEP_TYPE_PREFIX + StepTypeEnum.SCRIPT_REF.name())); // 3.分发本地文件的步骤统计 - resp = manageMetricsClient.countTemplateSteps( + resp = manageMetricsResource.countTemplateSteps( appId, TaskStepTypeEnum.FILE, null, @@ -119,7 +121,7 @@ public List calcAppTaskTemplateStepTypeStatistics(String dateStr, statisticsDTOList.add(genTaskTplStepTypeStatistics(dateStr, appId, localFileStepCount.toString(), StatisticsConstants.DIMENSION_VALUE_STEP_TYPE_PREFIX + StepTypeEnum.FILE_LOCAL.name())); // 4.分发服务器文件的步骤统计 - resp = manageMetricsClient.countTemplateSteps( + resp = manageMetricsResource.countTemplateSteps( appId, TaskStepTypeEnum.FILE, null, @@ -133,7 +135,7 @@ public List calcAppTaskTemplateStepTypeStatistics(String dateStr, statisticsDTOList.add(genTaskTplStepTypeStatistics(dateStr, appId, serverFileStepCount.toString(), StatisticsConstants.DIMENSION_VALUE_STEP_TYPE_PREFIX + StepTypeEnum.FILE_SERVER.name())); // 5.人工审核的步骤统计 - resp = manageMetricsClient.countTemplateSteps(appId, TaskStepTypeEnum.APPROVAL, null, null); + resp = manageMetricsResource.countTemplateSteps(appId, TaskStepTypeEnum.APPROVAL, null, null); if (resp == null || !resp.isSuccess()) { log.warn("Fail to call remote countTemplateSteps, resp:{}", resp); return statisticsDTOList; diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/host/HostStatisticsTask.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/host/HostStatisticsTask.java index 963e17c105..0a951b9c24 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/host/HostStatisticsTask.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/task/statistics/task/impl/host/HostStatisticsTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,19 +24,26 @@ package com.tencent.bk.job.analysis.task.statistics.task.impl.host; -import com.tencent.bk.job.analysis.client.ManageMetricsClient; +import com.tencent.bk.job.analysis.api.consts.StatisticsConstants; +import com.tencent.bk.job.analysis.api.dto.StatisticsDTO; import com.tencent.bk.job.analysis.dao.StatisticsDAO; import com.tencent.bk.job.analysis.service.BasicServiceManager; import com.tencent.bk.job.analysis.task.statistics.anotation.StatisticsTask; import com.tencent.bk.job.analysis.task.statistics.task.BaseStatisticsTask; +import com.tencent.bk.job.common.cc.sdk.IBizCmdbClient; import com.tencent.bk.job.common.model.InternalResponse; -import com.tencent.bk.job.common.statistics.consts.StatisticsConstants; -import com.tencent.bk.job.common.statistics.model.dto.StatisticsDTO; +import com.tencent.bk.job.manage.api.inner.ServiceMetricsResource; import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; import org.jooq.DSLContext; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; import java.time.LocalDateTime; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; /** * 统计主机操作系统类型分布情况 @@ -46,12 +53,18 @@ @Service public class HostStatisticsTask extends BaseStatisticsTask { - private final ManageMetricsClient manageMetricsClient; + private final ServiceMetricsResource manageMetricsResource; + private final IBizCmdbClient bizCmdbClient; - protected HostStatisticsTask(BasicServiceManager basicServiceManager, StatisticsDAO statisticsDAO, - DSLContext dslContext, ManageMetricsClient manageMetricsClient) { + @Autowired + protected HostStatisticsTask(BasicServiceManager basicServiceManager, + StatisticsDAO statisticsDAO, + @Qualifier("job-analysis-dsl-context") DSLContext dslContext, + ServiceMetricsResource manageMetricsResource, + IBizCmdbClient bizCmdbClient) { super(basicServiceManager, statisticsDAO, dslContext); - this.manageMetricsClient = manageMetricsClient; + this.manageMetricsResource = manageMetricsResource; + this.bizCmdbClient = bizCmdbClient; } private StatisticsDTO genStatisticsDTO(String dateStr, String value, String dimensionValue) { @@ -65,16 +78,8 @@ private StatisticsDTO genStatisticsDTO(String dateStr, String value, String dime return statisticsDTO; } - private StatisticsDTO genLinuxStatisticsDTO(String dateStr, String value) { - return genStatisticsDTO(dateStr, value, StatisticsConstants.DIMENSION_VALUE_HOST_SYSTEM_TYPE_LINUX); - } - - private StatisticsDTO genWindowsStatisticsDTO(String dateStr, String value) { - return genStatisticsDTO(dateStr, value, StatisticsConstants.DIMENSION_VALUE_HOST_SYSTEM_TYPE_WINDOWS); - } - - private StatisticsDTO genAixStatisticsDTO(String dateStr, String value) { - return genStatisticsDTO(dateStr, value, StatisticsConstants.DIMENSION_VALUE_HOST_SYSTEM_TYPE_AIX); + private StatisticsDTO genHostOsTypeStatisticsDTO(String osTypeName, String dateStr, String value) { + return genStatisticsDTO(dateStr, value, osTypeName); } private StatisticsDTO genOthersStatisticsDTO(String dateStr, String value) { @@ -82,38 +87,63 @@ private StatisticsDTO genOthersStatisticsDTO(String dateStr, String value) { } public void calcAndSaveHostStatistics(String dateStr) { - // Linux - InternalResponse resp = manageMetricsClient.countHostsByOsType("1"); - if (resp == null || !resp.isSuccess()) { - log.warn("Fail to call remote countHostsByOsType, resp:{}", resp); - return; - } - Long linuxCount = resp.getData(); - statisticsDAO.upsertStatistics(dslContext, genLinuxStatisticsDTO(dateStr, linuxCount.toString())); - // Windows - resp = manageMetricsClient.countHostsByOsType("2"); - if (resp == null || !resp.isSuccess()) { - log.warn("Fail to call remote countHostsByOsType, resp:{}", resp); - return; + Set allOsTypeNameSet = getAllOsTypeNameSet(); + + InternalResponse> resp = manageMetricsResource.groupHostByOsType(); + Map osTypeCountMap = resp.getData(); + // 统计所有类型的主机总量 + int totalHostCount = 0; + for (Map.Entry e : osTypeCountMap.entrySet()) { + Integer count = e.getValue(); + totalHostCount += count; } - Long windowsCount = resp.getData(); - statisticsDAO.upsertStatistics(dslContext, genWindowsStatisticsDTO(dateStr, windowsCount.toString())); - // Aix - resp = manageMetricsClient.countHostsByOsType("3"); - if (resp == null || !resp.isSuccess()) { - log.warn("Fail to call remote countHostsByOsType, resp:{}", resp); - return; + Map osTypeIdNameMap = bizCmdbClient.getOsTypeIdNameMap(); + int knownOsTypeHostCount = 0; + for (Map.Entry entry : osTypeIdNameMap.entrySet()) { + String id = entry.getKey(); + String name = entry.getValue().toUpperCase(); + // 只为Job关注的系统类型主机生成统计数据 + if (!allOsTypeNameSet.contains(name)) { + continue; + } + + allOsTypeNameSet.remove(name); + Integer hostCount = 0; + if (osTypeCountMap.containsKey(id)) { + hostCount = osTypeCountMap.get(id); + } + knownOsTypeHostCount += hostCount; + statisticsDAO.upsertStatistics( + dslContext, + genHostOsTypeStatisticsDTO(name, dateStr, hostCount.toString()) + ); + log.debug("calcAndSaveHostStatistics: id={},name={},hostCount={}", id, name, hostCount); } - Long aixCount = resp.getData(); - statisticsDAO.upsertStatistics(dslContext, genAixStatisticsDTO(dateStr, aixCount.toString())); + // CMDB中没有的类型统计值默认置为0 + allOsTypeNameSet.forEach(osTypeName -> { + log.debug("calcAndSaveHostStatistics: {} count=0", osTypeName); + statisticsDAO.upsertStatistics( + dslContext, + genStatisticsDTO(dateStr, "0", osTypeName.toUpperCase()) + ); + }); // Others - resp = manageMetricsClient.countHostsByOsType(null); - if (resp == null || !resp.isSuccess()) { - log.warn("Fail to call remote countHostsByOsType, resp:{}", resp); - return; - } - Long othersCount = resp.getData() - linuxCount - windowsCount - aixCount; - statisticsDAO.upsertStatistics(dslContext, genOthersStatisticsDTO(dateStr, othersCount.toString())); + int othersCount = totalHostCount - knownOsTypeHostCount; + log.debug("calcAndSaveHostStatistics: othersCount={}", othersCount); + statisticsDAO.upsertStatistics(dslContext, genOthersStatisticsDTO(dateStr, Long.toString(othersCount))); + } + + @NotNull + private static Set getAllOsTypeNameSet() { + Set allOsTypeNameSet = new HashSet<>(); + allOsTypeNameSet.add(StatisticsConstants.DIMENSION_VALUE_HOST_SYSTEM_TYPE_LINUX); + allOsTypeNameSet.add(StatisticsConstants.DIMENSION_VALUE_HOST_SYSTEM_TYPE_WINDOWS); + allOsTypeNameSet.add(StatisticsConstants.DIMENSION_VALUE_HOST_SYSTEM_TYPE_AIX); + allOsTypeNameSet.add(StatisticsConstants.DIMENSION_VALUE_HOST_SYSTEM_TYPE_UNIX); + allOsTypeNameSet.add(StatisticsConstants.DIMENSION_VALUE_HOST_SYSTEM_TYPE_SOLARIS); + allOsTypeNameSet.add(StatisticsConstants.DIMENSION_VALUE_HOST_SYSTEM_TYPE_FREEBSD); + allOsTypeNameSet.add(StatisticsConstants.DIMENSION_VALUE_HOST_SYSTEM_TYPE_MACOS); + return allOsTypeNameSet; } @Override diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/util/ai/AIAnswerUtil.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/util/ai/AIAnswerUtil.java new file mode 100644 index 0000000000..2dee5bf5b7 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/util/ai/AIAnswerUtil.java @@ -0,0 +1,104 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.util.ai; + +import com.tencent.bk.job.analysis.consts.AIConsts; +import com.tencent.bk.job.analysis.model.web.resp.AIAnswer; +import com.tencent.bk.job.common.model.Response; +import com.tencent.bk.job.common.tracing.util.TraceUtil; +import com.tencent.bk.job.common.util.StringUtil; +import com.tencent.bk.job.common.util.json.JsonUtils; +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; +import java.io.OutputStream; +import java.nio.charset.StandardCharsets; + +/** + * 处理AI回答内容、报错信息的工具类 + */ +@Slf4j +public class AIAnswerUtil { + + /** + * 获取限长的AI回答报错信息 + * + * @param errorMessage AI回答报错信息 + * @return 处理后的AI回答报错信息 + */ + public static String getLimitedErrorMessage(String errorMessage) { + return getLimitedString(errorMessage, AIConsts.MAX_LENGTH_AI_ANSWER_ERROR_MESSAGE, "errorMessage"); + } + + /** + * 获取限长的AI回答 + * + * @param aiAnswer AI回答 + * @return 处理后的AI回答 + */ + public static String getLimitedAIAnswer(String aiAnswer) { + return getLimitedString(aiAnswer, AIConsts.MAX_LENGTH_AI_ANSWER, "aiAnswer"); + } + + /** + * 获取限长的字符串 + * + * @param rawString 原始字符串 + * @param maxLength 最大长度 + * @param stringDesc 原始字符串描述，用于日志打印 + * @return 处理后的AI回答 + */ + private static String getLimitedString(String rawString, int maxLength, String stringDesc) { + if (rawString == null) { + return null; + } + if (rawString.length() > maxLength) { + log.info( + "{} is too long({}), truncated to {}", + stringDesc, + rawString.length(), + maxLength + ); + return StringUtil.substring(rawString, maxLength); + } + return rawString; + } + + /** + * 设置RequestId、序列化AIAnswer内容并写入到输出流中 + * + * @param outputStream 输出流 + * @param respBody 响应数据 + * @throws IOException IO异常 + */ + public static void setRequestIdAndWriteResp(OutputStream outputStream, + Response respBody) throws IOException { + String traceId = TraceUtil.getTraceIdFromCurrentSpan(); + respBody.setRequestId(traceId); + String message = JsonUtils.toJson(respBody) + "\n"; + outputStream.write(message.getBytes(StandardCharsets.UTF_8)); + outputStream.flush(); + } +} diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/util/calc/AbstractMomYoyCalculator.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/util/calc/AbstractMomYoyCalculator.java index 362c962c8a..6144dffad0 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/util/calc/AbstractMomYoyCalculator.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/util/calc/AbstractMomYoyCalculator.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,69 +24,106 @@ package com.tencent.bk.job.analysis.util.calc; +import com.tencent.bk.job.analysis.api.dto.StatisticsDTO; +import com.tencent.bk.job.analysis.consts.DataTrendEnum; import com.tencent.bk.job.analysis.model.web.CommonStatisticWithRateVO; -import com.tencent.bk.job.common.statistics.model.dto.StatisticsDTO; +import lombok.AllArgsConstructor; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; +/** + * 抽象的同环比计算器 + */ @Slf4j public abstract class AbstractMomYoyCalculator { - StatisticsDTO statisticsDTO; - StatisticsDTO momStatisticsDTO; - StatisticsDTO yoyStatisticsDTO; + // 当前统计数据 + private final StatisticsDTO statisticsDTO; + // 环比上一周期的统计数据 + private final StatisticsDTO momStatisticsDTO; + // 同比上一周期的统计数据 + private final StatisticsDTO yoyStatisticsDTO; - public AbstractMomYoyCalculator(StatisticsDTO statisticsDTO, StatisticsDTO momStatisticsDTO, + public AbstractMomYoyCalculator(StatisticsDTO statisticsDTO, + StatisticsDTO momStatisticsDTO, StatisticsDTO yoyStatisticsDTO) { this.statisticsDTO = statisticsDTO; this.momStatisticsDTO = momStatisticsDTO; this.yoyStatisticsDTO = yoyStatisticsDTO; } - abstract Long getCountFromStatisticValue(String value); + /** + * 从序列化的存储数据中解析统计量数值 + * + * @param serializedData 序列化的存储数据 + * @return 统计量数值 + */ + protected abstract Long getCountFromSerializedData(String serializedData); - public CommonStatisticWithRateVO getResult() { + /** + * 根据几个时间点的统计数据计算同环比数据 + * + * @return 同环比数据 + */ + public CommonStatisticWithRateVO calc() { if (statisticsDTO == null) { return null; } - Long count = getCountFromStatisticValue(statisticsDTO.getValue()); + Long count = getCountFromSerializedData(statisticsDTO.getValue()); CommonStatisticWithRateVO commonStatisticWithRateVO = new CommonStatisticWithRateVO(); commonStatisticWithRateVO.setCount(count); if (momStatisticsDTO != null) { - Long momCount = getCountFromStatisticValue(momStatisticsDTO.getValue()); // 环比计算 - Long momValue = count - momCount; - Float momRate = 1f; - if (momCount > 0) { - momRate = (count - momCount) / (float) momCount; - } - long momTrend = 0; - if (momValue > 0) { - momTrend = 1; - } else if (momValue < 0) { - momTrend = -1; - } - commonStatisticWithRateVO.setMomValue(momValue); - commonStatisticWithRateVO.setMomRate(momRate); - commonStatisticWithRateVO.setMomTrend(momTrend); + Increment increment = calcIncrement(count, momStatisticsDTO); + commonStatisticWithRateVO.setMomValue(increment.getValue()); + commonStatisticWithRateVO.setMomRate(increment.getRate()); + commonStatisticWithRateVO.setMomTrend(increment.getTrend().getValue()); } if (yoyStatisticsDTO != null) { - Long yoyCount = getCountFromStatisticValue(yoyStatisticsDTO.getValue()); // 同比计算 - Long yoyValue = count - yoyCount; - Float yoyRate = 1f; - if (yoyCount > 0) { - yoyRate = (count - yoyCount) / (float) yoyCount; - } - long yoyTrend = 0; - if (yoyValue > 0) { - yoyTrend = 1; - } else if (yoyValue < 0) { - yoyTrend = -1; - } - - commonStatisticWithRateVO.setYoyValue(yoyValue); - commonStatisticWithRateVO.setYoyRate(yoyRate); - commonStatisticWithRateVO.setYoyTrend(yoyTrend); + Increment increment = calcIncrement(count, yoyStatisticsDTO); + commonStatisticWithRateVO.setYoyValue(increment.getValue()); + commonStatisticWithRateVO.setYoyRate(increment.getRate()); + commonStatisticWithRateVO.setYoyTrend(increment.getTrend().getValue()); } return commonStatisticWithRateVO; } + + /** + * 根据当前统计值与上一时间点的统计数据计算数据增量 + * + * @param count 当前统计值 + * @param previousStatisticsDTO 上一时间点统计数据 + * @return 数据增量 + */ + private Increment calcIncrement(Long count, StatisticsDTO previousStatisticsDTO) { + Long previousCount = getCountFromSerializedData(previousStatisticsDTO.getValue()); + // 增量计算 + long value = count - previousCount; + // 从无到有的初始增长率认定为1 + float rate = 1f; + if (previousCount > 0) { + rate = (count - previousCount) / (float) previousCount; + } + DataTrendEnum trend = DataTrendEnum.NOT_CHANGE; + if (value > 0) { + trend = DataTrendEnum.UP; + } else if (value < 0) { + trend = DataTrendEnum.DOWN; + } + return new Increment(value, rate, trend); + } + + /** + * 数据增量 + */ + @AllArgsConstructor + @Getter + static class Increment { + // 增量值 + private final long value; + // 增量比率 + private final float rate; + // 增量趋势 + private final DataTrendEnum trend; + } } diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/util/calc/AppMomYoyCalculator.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/util/calc/AppMomYoyCalculator.java index 6a6d924129..9027d2434b 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/util/calc/AppMomYoyCalculator.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/util/calc/AppMomYoyCalculator.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,7 +25,7 @@ package com.tencent.bk.job.analysis.util.calc; import com.fasterxml.jackson.core.type.TypeReference; -import com.tencent.bk.job.common.statistics.model.dto.StatisticsDTO; +import com.tencent.bk.job.analysis.api.dto.StatisticsDTO; import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.manage.model.inner.resp.ServiceApplicationDTO; @@ -41,11 +41,17 @@ public AppMomYoyCalculator(StatisticsDTO statisticsDTO, StatisticsDTO momStatist super(statisticsDTO, momStatisticsDTO, yoyStatisticsDTO); } - Long getCountFromStatisticValue(String value) { + /** + * 从序列化的存储数据中解析出业务数量 + * + * @param serializedData 序列化的存储数据 + * @return 业务数量 + */ + protected Long getCountFromSerializedData(String serializedData) { // 解析统计量 - List applicationDTOList = JsonUtils.fromJson(value, + List applicationDTOList = JsonUtils.fromJson(serializedData, new TypeReference>() { - }); + }); return (long) applicationDTOList.size(); } } diff --git a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/util/calc/SimpleMomYoyCalculator.java b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/util/calc/SimpleMomYoyCalculator.java index 4a1caa0252..6e46e6aa57 100644 --- a/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/util/calc/SimpleMomYoyCalculator.java +++ b/src/backend/job-analysis/service-job-analysis/src/main/java/com/tencent/bk/job/analysis/util/calc/SimpleMomYoyCalculator.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,20 +24,27 @@ package com.tencent.bk.job.analysis.util.calc; -import com.tencent.bk.job.common.statistics.model.dto.StatisticsDTO; +import com.tencent.bk.job.analysis.api.dto.StatisticsDTO; /** * 解析字段中直接存数字的统计数据 */ public class SimpleMomYoyCalculator extends AbstractMomYoyCalculator { - public SimpleMomYoyCalculator(StatisticsDTO statisticsDTO, StatisticsDTO momStatisticsDTO, + public SimpleMomYoyCalculator(StatisticsDTO statisticsDTO, + StatisticsDTO momStatisticsDTO, StatisticsDTO yoyStatisticsDTO) { super(statisticsDTO, momStatisticsDTO, yoyStatisticsDTO); } - Long getCountFromStatisticValue(String value) { + /** + * 从序列化的存储数据中解析出统计值数字 + * + * @param serializedData 序列化的存储数据 + * @return 统计值数字 + */ + protected Long getCountFromSerializedData(String serializedData) { // 解析统计量 - return Long.parseLong(value); + return Long.parseLong(serializedData); } } diff --git a/src/backend/job-analysis/service-job-analysis/src/main/resources/i18n/message.properties b/src/backend/job-analysis/service-job-analysis/src/main/resources/i18n/message.properties new file mode 100644 index 0000000000..31ad9affc7 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/resources/i18n/message.properties @@ -0,0 +1,43 @@ +# +# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. +# +# Copyright (C) 2021 Tencent. All rights reserved. +# +# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. +# +# License for BK-JOB蓝鲸智云作业平台: +# -------------------------------------------------------------------- +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF +# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +# 离线分析任务部分 +job.analysis.analysistask.result.ItemLocation.description.CommaSeparatedTplIdAndPlanId=逗号分隔的作业模板ID与执行方案ID +job.analysis.analysistask.result.ItemLocation.description.TemplateId=作业模板ID +job.analysis.analysistask.result.ItemLocation.description.TimerTaskName=定时任务名称 +job.analysis.analysistask.result.BadTplPlanInfo.typeName.TaskPlan=作业执行方案 +job.analysis.analysistask.result.BadTplPlanInfo.typeName.Template=作业模板 +job.analysis.analysistask.result.AbnormalTargetPlanInfo.stepName.scope.Step=步骤 +job.analysis.analysistask.result.AbnormalTargetPlanInfo.description.AbnormalTarget=执行目标异常 + +# AI分析部分 +job.analysis.ai.languageSpecifySystemMessage=当前对话环境为中文环境，请使用中文回答。 +job.analysis.ai.languageSpecifyAIReplyMessage=好的。 +job.analysis.ai.greetingMessage=欢迎使用作业平台！我是负责协助你的AI助手（小鲸），请问有什么可以帮助你的吗？ +job.analysis.ai.emptyLogTaskAnswerMessage=因执行结果的返回日志为空，小鲸无法准确帮你分析具体错误的原因和处理建议！但已知作业平台出现“日志返回为空、且返回码为非0”的情况下，大概率是脚本的执行逻辑导致的，因为JOB的返回码是取自目标机器上执行的结果；如果是“日志为空、且返回码也是空”的情况下，可能是因为目标机器未安装GSE Agent导致，可以留意执行结果的“分组标签”上的描述信息！如果还是未能解决问题可以联系 [BK助手]({BK_JOB_AI_TEMPLATE_VAR{bk_helper_link}}) 人工咨询获取帮助。 +job.analysis.ai.notFailTaskAnswerMessage=只有状态为“执行失败”的任务才需要进行分析，当前任务无需分析 +job.analysis.ai.fileTaskErrorSource.noError=任务未失败 +job.analysis.ai.fileTaskErrorSource.sourceFileUploadError=源文件上传出错导致的任务失败 +job.analysis.ai.fileTaskErrorSource.downloadError=目标执行对象下载文件出错导致的任务失败 +job.analysis.ai.fileTaskErrorSource.uploadAndDownloadError=源文件上传与目标执行对象下载文件均出错导致的任务失败 diff --git a/src/backend/job-analysis/service-job-analysis/src/main/resources/i18n/message_en.properties b/src/backend/job-analysis/service-job-analysis/src/main/resources/i18n/message_en.properties new file mode 100644 index 0000000000..09b12f0202 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/resources/i18n/message_en.properties @@ -0,0 +1,43 @@ +# +# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. +# +# Copyright (C) 2021 Tencent. All rights reserved. +# +# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. +# +# License for BK-JOB蓝鲸智云作业平台: +# -------------------------------------------------------------------- +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF +# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +# 离线分析任务部分 +job.analysis.analysistask.result.ItemLocation.description.CommaSeparatedTplIdAndPlanId=Job template ID and Plan ID separated by comma +job.analysis.analysistask.result.ItemLocation.description.TemplateId=TemplateId +job.analysis.analysistask.result.ItemLocation.description.TimerTaskName=Cron Job Name +job.analysis.analysistask.result.BadTplPlanInfo.typeName.TaskPlan=TaskPlan +job.analysis.analysistask.result.BadTplPlanInfo.typeName.Template=Template +job.analysis.analysistask.result.AbnormalTargetPlanInfo.stepName.scope.Step=Steps +job.analysis.analysistask.result.AbnormalTargetPlanInfo.description.AbnormalTarget=Abnormal hosts + +# AI分析部分 +job.analysis.ai.languageSpecifySystemMessage=Conversation occurs in an English environment now, please answer in English. +job.analysis.ai.languageSpecifyAIReplyMessage=OK. +job.analysis.ai.greetingMessage=Welcome to JOB! I'm your AI-assistant, is there anything i can help you with ? +job.analysis.ai.emptyLogTaskAnswerMessage=Due to the absence of return logs from the execution results, I cannot accurately analyze the specific cause of the error and provide suggestions for resolution. However, it is known that when the JOB system shows "log returns empty, and the return code is non-zero", it is highly likely due to the execution logic of the script, as the JOB return code is derived from the execution result on the target machine. If the situation is "logs are empty, and the return code is also empty", it may be because the target machine does not have the GSE Agent installed. Please pay attention to the description information on the "group label" of the execution result. If the problem still cannot be resolved, you can contact [BK-Assistant]({BK_JOB_AI_TEMPLATE_VAR{bk_helper_link}}) for manual consultation and assistance. +job.analysis.ai.notFailTaskAnswerMessage=Task is not in fail status, do not need to analyze +job.analysis.ai.fileTaskErrorSource.noError=No error +job.analysis.ai.fileTaskErrorSource.sourceFileUploadError=Source execute object upload file error +job.analysis.ai.fileTaskErrorSource.downloadError=Target execute object download file error +job.analysis.ai.fileTaskErrorSource.uploadAndDownloadError=Source execute object upload file error and Target execute object download file error diff --git a/src/backend/job-analysis/service-job-analysis/src/main/resources/i18n/message_en_US.properties b/src/backend/job-analysis/service-job-analysis/src/main/resources/i18n/message_en_US.properties new file mode 100644 index 0000000000..09b12f0202 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/resources/i18n/message_en_US.properties @@ -0,0 +1,43 @@ +# +# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. +# +# Copyright (C) 2021 Tencent. All rights reserved. +# +# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. +# +# License for BK-JOB蓝鲸智云作业平台: +# -------------------------------------------------------------------- +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF +# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +# 离线分析任务部分 +job.analysis.analysistask.result.ItemLocation.description.CommaSeparatedTplIdAndPlanId=Job template ID and Plan ID separated by comma +job.analysis.analysistask.result.ItemLocation.description.TemplateId=TemplateId +job.analysis.analysistask.result.ItemLocation.description.TimerTaskName=Cron Job Name +job.analysis.analysistask.result.BadTplPlanInfo.typeName.TaskPlan=TaskPlan +job.analysis.analysistask.result.BadTplPlanInfo.typeName.Template=Template +job.analysis.analysistask.result.AbnormalTargetPlanInfo.stepName.scope.Step=Steps +job.analysis.analysistask.result.AbnormalTargetPlanInfo.description.AbnormalTarget=Abnormal hosts + +# AI分析部分 +job.analysis.ai.languageSpecifySystemMessage=Conversation occurs in an English environment now, please answer in English. +job.analysis.ai.languageSpecifyAIReplyMessage=OK. +job.analysis.ai.greetingMessage=Welcome to JOB! I'm your AI-assistant, is there anything i can help you with ? +job.analysis.ai.emptyLogTaskAnswerMessage=Due to the absence of return logs from the execution results, I cannot accurately analyze the specific cause of the error and provide suggestions for resolution. However, it is known that when the JOB system shows "log returns empty, and the return code is non-zero", it is highly likely due to the execution logic of the script, as the JOB return code is derived from the execution result on the target machine. If the situation is "logs are empty, and the return code is also empty", it may be because the target machine does not have the GSE Agent installed. Please pay attention to the description information on the "group label" of the execution result. If the problem still cannot be resolved, you can contact [BK-Assistant]({BK_JOB_AI_TEMPLATE_VAR{bk_helper_link}}) for manual consultation and assistance. +job.analysis.ai.notFailTaskAnswerMessage=Task is not in fail status, do not need to analyze +job.analysis.ai.fileTaskErrorSource.noError=No error +job.analysis.ai.fileTaskErrorSource.sourceFileUploadError=Source execute object upload file error +job.analysis.ai.fileTaskErrorSource.downloadError=Target execute object download file error +job.analysis.ai.fileTaskErrorSource.uploadAndDownloadError=Source execute object upload file error and Target execute object download file error diff --git a/src/backend/job-analysis/service-job-analysis/src/main/resources/i18n/message_zh.properties b/src/backend/job-analysis/service-job-analysis/src/main/resources/i18n/message_zh.properties new file mode 100644 index 0000000000..31ad9affc7 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/resources/i18n/message_zh.properties @@ -0,0 +1,43 @@ +# +# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. +# +# Copyright (C) 2021 Tencent. All rights reserved. +# +# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. +# +# License for BK-JOB蓝鲸智云作业平台: +# -------------------------------------------------------------------- +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF +# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +# 离线分析任务部分 +job.analysis.analysistask.result.ItemLocation.description.CommaSeparatedTplIdAndPlanId=逗号分隔的作业模板ID与执行方案ID +job.analysis.analysistask.result.ItemLocation.description.TemplateId=作业模板ID +job.analysis.analysistask.result.ItemLocation.description.TimerTaskName=定时任务名称 +job.analysis.analysistask.result.BadTplPlanInfo.typeName.TaskPlan=作业执行方案 +job.analysis.analysistask.result.BadTplPlanInfo.typeName.Template=作业模板 +job.analysis.analysistask.result.AbnormalTargetPlanInfo.stepName.scope.Step=步骤 +job.analysis.analysistask.result.AbnormalTargetPlanInfo.description.AbnormalTarget=执行目标异常 + +# AI分析部分 +job.analysis.ai.languageSpecifySystemMessage=当前对话环境为中文环境，请使用中文回答。 +job.analysis.ai.languageSpecifyAIReplyMessage=好的。 +job.analysis.ai.greetingMessage=欢迎使用作业平台！我是负责协助你的AI助手（小鲸），请问有什么可以帮助你的吗？ +job.analysis.ai.emptyLogTaskAnswerMessage=因执行结果的返回日志为空，小鲸无法准确帮你分析具体错误的原因和处理建议！但已知作业平台出现“日志返回为空、且返回码为非0”的情况下，大概率是脚本的执行逻辑导致的，因为JOB的返回码是取自目标机器上执行的结果；如果是“日志为空、且返回码也是空”的情况下，可能是因为目标机器未安装GSE Agent导致，可以留意执行结果的“分组标签”上的描述信息！如果还是未能解决问题可以联系 [BK助手]({BK_JOB_AI_TEMPLATE_VAR{bk_helper_link}}) 人工咨询获取帮助。 +job.analysis.ai.notFailTaskAnswerMessage=只有状态为“执行失败”的任务才需要进行分析，当前任务无需分析 +job.analysis.ai.fileTaskErrorSource.noError=任务未失败 +job.analysis.ai.fileTaskErrorSource.sourceFileUploadError=源文件上传出错导致的任务失败 +job.analysis.ai.fileTaskErrorSource.downloadError=目标执行对象下载文件出错导致的任务失败 +job.analysis.ai.fileTaskErrorSource.uploadAndDownloadError=源文件上传与目标执行对象下载文件均出错导致的任务失败 diff --git a/src/backend/job-analysis/service-job-analysis/src/main/resources/i18n/message_zh_CN.properties b/src/backend/job-analysis/service-job-analysis/src/main/resources/i18n/message_zh_CN.properties new file mode 100644 index 0000000000..31ad9affc7 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/main/resources/i18n/message_zh_CN.properties @@ -0,0 +1,43 @@ +# +# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. +# +# Copyright (C) 2021 Tencent. All rights reserved. +# +# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. +# +# License for BK-JOB蓝鲸智云作业平台: +# -------------------------------------------------------------------- +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF +# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +# 离线分析任务部分 +job.analysis.analysistask.result.ItemLocation.description.CommaSeparatedTplIdAndPlanId=逗号分隔的作业模板ID与执行方案ID +job.analysis.analysistask.result.ItemLocation.description.TemplateId=作业模板ID +job.analysis.analysistask.result.ItemLocation.description.TimerTaskName=定时任务名称 +job.analysis.analysistask.result.BadTplPlanInfo.typeName.TaskPlan=作业执行方案 +job.analysis.analysistask.result.BadTplPlanInfo.typeName.Template=作业模板 +job.analysis.analysistask.result.AbnormalTargetPlanInfo.stepName.scope.Step=步骤 +job.analysis.analysistask.result.AbnormalTargetPlanInfo.description.AbnormalTarget=执行目标异常 + +# AI分析部分 +job.analysis.ai.languageSpecifySystemMessage=当前对话环境为中文环境，请使用中文回答。 +job.analysis.ai.languageSpecifyAIReplyMessage=好的。 +job.analysis.ai.greetingMessage=欢迎使用作业平台！我是负责协助你的AI助手（小鲸），请问有什么可以帮助你的吗？ +job.analysis.ai.emptyLogTaskAnswerMessage=因执行结果的返回日志为空，小鲸无法准确帮你分析具体错误的原因和处理建议！但已知作业平台出现“日志返回为空、且返回码为非0”的情况下，大概率是脚本的执行逻辑导致的，因为JOB的返回码是取自目标机器上执行的结果；如果是“日志为空、且返回码也是空”的情况下，可能是因为目标机器未安装GSE Agent导致，可以留意执行结果的“分组标签”上的描述信息！如果还是未能解决问题可以联系 [BK助手]({BK_JOB_AI_TEMPLATE_VAR{bk_helper_link}}) 人工咨询获取帮助。 +job.analysis.ai.notFailTaskAnswerMessage=只有状态为“执行失败”的任务才需要进行分析，当前任务无需分析 +job.analysis.ai.fileTaskErrorSource.noError=任务未失败 +job.analysis.ai.fileTaskErrorSource.sourceFileUploadError=源文件上传出错导致的任务失败 +job.analysis.ai.fileTaskErrorSource.downloadError=目标执行对象下载文件出错导致的任务失败 +job.analysis.ai.fileTaskErrorSource.uploadAndDownloadError=源文件上传与目标执行对象下载文件均出错导致的任务失败 diff --git a/src/backend/job-analysis/service-job-analysis/src/test/java/com/tencent/bk/job/analysis/util/ai/AIAnswerUtilTest.java b/src/backend/job-analysis/service-job-analysis/src/test/java/com/tencent/bk/job/analysis/util/ai/AIAnswerUtilTest.java new file mode 100644 index 0000000000..e6ce72c5b6 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/test/java/com/tencent/bk/job/analysis/util/ai/AIAnswerUtilTest.java @@ -0,0 +1,80 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.util.ai; + +import com.tencent.bk.job.analysis.consts.AIConsts; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * 单元测试 - 处理AI回答内容、报错信息的工具类 + */ +public class AIAnswerUtilTest { + + @Test + public void testGetLimitedErrorMessage() { + assertThat(AIAnswerUtil.getLimitedErrorMessage(null)).isNull(); + String rawMessage = ""; + String limitedErrorMessage = AIAnswerUtil.getLimitedErrorMessage(rawMessage); + assertThat(limitedErrorMessage).isNotNull(); + assertThat(limitedErrorMessage).isEqualTo(rawMessage); + rawMessage = "123456"; + limitedErrorMessage = AIAnswerUtil.getLimitedErrorMessage(rawMessage); + assertThat(limitedErrorMessage).isEqualTo(rawMessage); + + StringBuilder stringBuilder = new StringBuilder(); + for (int i = 0; i < 52; i++) { + stringBuilder.append("1234567890"); + } + rawMessage = stringBuilder.toString(); + limitedErrorMessage = AIAnswerUtil.getLimitedErrorMessage(rawMessage); + assertThat(limitedErrorMessage.length()).isEqualTo(AIConsts.MAX_LENGTH_AI_ANSWER_ERROR_MESSAGE); + assertThat(limitedErrorMessage).endsWith("12"); + } + + @Test + public void testGetLimitedAIAnswer() { + assertThat(AIAnswerUtil.getLimitedAIAnswer(null)).isNull(); + String rawAIAnswer = ""; + String limitedAIAnswer = AIAnswerUtil.getLimitedAIAnswer(rawAIAnswer); + assertThat(limitedAIAnswer).isNotNull(); + assertThat(limitedAIAnswer).isEqualTo(rawAIAnswer); + rawAIAnswer = "123456"; + limitedAIAnswer = AIAnswerUtil.getLimitedAIAnswer(rawAIAnswer); + assertThat(limitedAIAnswer).isEqualTo(rawAIAnswer); + + StringBuilder stringBuilder = new StringBuilder(); + for (int i = 0; i < 1500; i++) { + stringBuilder.append("1234567890"); + } + stringBuilder.append("ABC"); + rawAIAnswer = stringBuilder.toString(); + limitedAIAnswer = AIAnswerUtil.getLimitedAIAnswer(rawAIAnswer); + assertThat(limitedAIAnswer.length()).isEqualTo(AIConsts.MAX_LENGTH_AI_ANSWER); + assertThat(limitedAIAnswer).endsWith("1234567890"); + } + +} diff --git a/src/backend/job-analysis/service-job-analysis/src/test/java/com/tencent/bk/job/analysis/util/calc/AppMomYoyCalculatorTest.java b/src/backend/job-analysis/service-job-analysis/src/test/java/com/tencent/bk/job/analysis/util/calc/AppMomYoyCalculatorTest.java new file mode 100644 index 0000000000..7df54fe718 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/test/java/com/tencent/bk/job/analysis/util/calc/AppMomYoyCalculatorTest.java @@ -0,0 +1,98 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.util.calc; + +import com.tencent.bk.job.analysis.api.dto.StatisticsDTO; +import com.tencent.bk.job.analysis.consts.DataTrendEnum; +import com.tencent.bk.job.analysis.model.web.CommonStatisticWithRateVO; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * 单元测试-解析存储JSON串的业务数据的同环比计算器 + */ +public class AppMomYoyCalculatorTest { + + @Test + public void testCalc() { + // 从0上升 + StatisticsDTO statisticsDTO = new StatisticsDTO( + "[{\"name\": \"app1\"},{\"name\": \"app2\"}," + + "{\"name\": \"app3\"},{\"name\": \"app4\"}]" + ); + StatisticsDTO momStatisticsDTO = new StatisticsDTO("[]"); + StatisticsDTO yoyStatisticsDTO = new StatisticsDTO("[]"); + AppMomYoyCalculator appMomYoyCalculator = new AppMomYoyCalculator( + statisticsDTO, + momStatisticsDTO, + yoyStatisticsDTO + ); + CommonStatisticWithRateVO commonStatisticWithRateVO = appMomYoyCalculator.calc(); + assertThat(commonStatisticWithRateVO.getCount()).isEqualTo(4L); + assertThat(commonStatisticWithRateVO.getMomValue()).isEqualTo(4L); + assertThat(commonStatisticWithRateVO.getMomRate()).isEqualTo(1.0f); + assertThat(commonStatisticWithRateVO.getMomTrend()).isEqualTo(DataTrendEnum.UP.getValue()); + assertThat(commonStatisticWithRateVO.getYoyValue()).isEqualTo(4L); + assertThat(commonStatisticWithRateVO.getYoyRate()).isEqualTo(1.0f); + assertThat(commonStatisticWithRateVO.getYoyTrend()).isEqualTo(DataTrendEnum.UP.getValue()); + // 非0上升 + momStatisticsDTO = new StatisticsDTO("[{\"name\": \"app1\"},{\"name\": \"app2\"}]"); + yoyStatisticsDTO = new StatisticsDTO("[{\"name\": \"app1\"}]"); + appMomYoyCalculator = new AppMomYoyCalculator( + statisticsDTO, + momStatisticsDTO, + yoyStatisticsDTO + ); + commonStatisticWithRateVO = appMomYoyCalculator.calc(); + assertThat(commonStatisticWithRateVO.getCount()).isEqualTo(4L); + assertThat(commonStatisticWithRateVO.getMomValue()).isEqualTo(2L); + assertThat(commonStatisticWithRateVO.getMomRate()).isEqualTo(1.0f); + assertThat(commonStatisticWithRateVO.getMomTrend()).isEqualTo(DataTrendEnum.UP.getValue()); + assertThat(commonStatisticWithRateVO.getYoyValue()).isEqualTo(3L); + assertThat(commonStatisticWithRateVO.getYoyRate()).isEqualTo(3.0f); + assertThat(commonStatisticWithRateVO.getYoyTrend()).isEqualTo(DataTrendEnum.UP.getValue()); + // 下降 + statisticsDTO = new StatisticsDTO("[{\"name\": \"app1\"}]"); + momStatisticsDTO = new StatisticsDTO("[{\"name\": \"app1\"},{\"name\": \"app2\"}]"); + yoyStatisticsDTO = new StatisticsDTO( + "[{\"name\": \"app1\"},{\"name\": \"app2\"}," + + "{\"name\": \"app3\"},{\"name\": \"app4\"}]" + ); + appMomYoyCalculator = new AppMomYoyCalculator( + statisticsDTO, + momStatisticsDTO, + yoyStatisticsDTO + ); + commonStatisticWithRateVO = appMomYoyCalculator.calc(); + assertThat(commonStatisticWithRateVO.getCount()).isEqualTo(1L); + assertThat(commonStatisticWithRateVO.getMomValue()).isEqualTo(-1L); + assertThat(commonStatisticWithRateVO.getMomRate()).isEqualTo(-0.5f); + assertThat(commonStatisticWithRateVO.getMomTrend()).isEqualTo(DataTrendEnum.DOWN.getValue()); + assertThat(commonStatisticWithRateVO.getYoyValue()).isEqualTo(-3L); + assertThat(commonStatisticWithRateVO.getYoyRate()).isEqualTo(-0.75f); + assertThat(commonStatisticWithRateVO.getYoyTrend()).isEqualTo(DataTrendEnum.DOWN.getValue()); + } +} diff --git a/src/backend/job-analysis/service-job-analysis/src/test/java/com/tencent/bk/job/analysis/util/calc/SimpleMomYoyCalculatorTest.java b/src/backend/job-analysis/service-job-analysis/src/test/java/com/tencent/bk/job/analysis/util/calc/SimpleMomYoyCalculatorTest.java new file mode 100644 index 0000000000..8e4f7bc2b8 --- /dev/null +++ b/src/backend/job-analysis/service-job-analysis/src/test/java/com/tencent/bk/job/analysis/util/calc/SimpleMomYoyCalculatorTest.java @@ -0,0 +1,92 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.analysis.util.calc; + +import com.tencent.bk.job.analysis.api.dto.StatisticsDTO; +import com.tencent.bk.job.analysis.consts.DataTrendEnum; +import com.tencent.bk.job.analysis.model.web.CommonStatisticWithRateVO; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * 单元测试-简单同环比计算器 + */ +public class SimpleMomYoyCalculatorTest { + + @Test + public void testCalc() { + // 从0上升 + StatisticsDTO statisticsDTO = new StatisticsDTO("150"); + StatisticsDTO momStatisticsDTO = new StatisticsDTO("0"); + StatisticsDTO yoyStatisticsDTO = new StatisticsDTO("0"); + SimpleMomYoyCalculator simpleMomYoyCalculator = new SimpleMomYoyCalculator( + statisticsDTO, + momStatisticsDTO, + yoyStatisticsDTO + ); + CommonStatisticWithRateVO commonStatisticWithRateVO = simpleMomYoyCalculator.calc(); + assertThat(commonStatisticWithRateVO.getCount()).isEqualTo(150L); + assertThat(commonStatisticWithRateVO.getMomValue()).isEqualTo(150L); + assertThat(commonStatisticWithRateVO.getMomRate()).isEqualTo(1.0f); + assertThat(commonStatisticWithRateVO.getMomTrend()).isEqualTo(DataTrendEnum.UP.getValue()); + assertThat(commonStatisticWithRateVO.getYoyValue()).isEqualTo(150L); + assertThat(commonStatisticWithRateVO.getYoyRate()).isEqualTo(1.0f); + assertThat(commonStatisticWithRateVO.getYoyTrend()).isEqualTo(DataTrendEnum.UP.getValue()); + // 非0上升 + momStatisticsDTO = new StatisticsDTO("100"); + yoyStatisticsDTO = new StatisticsDTO("50"); + simpleMomYoyCalculator = new SimpleMomYoyCalculator( + statisticsDTO, + momStatisticsDTO, + yoyStatisticsDTO + ); + commonStatisticWithRateVO = simpleMomYoyCalculator.calc(); + assertThat(commonStatisticWithRateVO.getCount()).isEqualTo(150L); + assertThat(commonStatisticWithRateVO.getMomValue()).isEqualTo(50L); + assertThat(commonStatisticWithRateVO.getMomRate()).isEqualTo(0.5f); + assertThat(commonStatisticWithRateVO.getMomTrend()).isEqualTo(DataTrendEnum.UP.getValue()); + assertThat(commonStatisticWithRateVO.getYoyValue()).isEqualTo(100L); + assertThat(commonStatisticWithRateVO.getYoyRate()).isEqualTo(2.0f); + assertThat(commonStatisticWithRateVO.getYoyTrend()).isEqualTo(DataTrendEnum.UP.getValue()); + // 下降 + statisticsDTO = new StatisticsDTO("50"); + momStatisticsDTO = new StatisticsDTO("100"); + yoyStatisticsDTO = new StatisticsDTO("200"); + simpleMomYoyCalculator = new SimpleMomYoyCalculator( + statisticsDTO, + momStatisticsDTO, + yoyStatisticsDTO + ); + commonStatisticWithRateVO = simpleMomYoyCalculator.calc(); + assertThat(commonStatisticWithRateVO.getCount()).isEqualTo(50L); + assertThat(commonStatisticWithRateVO.getMomValue()).isEqualTo(-50L); + assertThat(commonStatisticWithRateVO.getMomRate()).isEqualTo(-0.5f); + assertThat(commonStatisticWithRateVO.getMomTrend()).isEqualTo(DataTrendEnum.DOWN.getValue()); + assertThat(commonStatisticWithRateVO.getYoyValue()).isEqualTo(-150L); + assertThat(commonStatisticWithRateVO.getYoyRate()).isEqualTo(-0.75f); + assertThat(commonStatisticWithRateVO.getYoyTrend()).isEqualTo(DataTrendEnum.DOWN.getValue()); + } +} diff --git a/src/backend/job-assemble/build.gradle b/src/backend/job-assemble/build.gradle new file mode 100644 index 0000000000..5bab374b51 --- /dev/null +++ b/src/backend/job-assemble/build.gradle @@ -0,0 +1,64 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ +version = "1.0.0" +if (System.getProperty("jobAssembleVersion")) { + version = System.getProperty("jobAssembleVersion") +} else if (System.getProperty("bkjobVersion")) { + version = System.getProperty("bkjobVersion") +} +println "jobAssembleVersion: ${version}" +apply plugin: 'org.springframework.boot' +apply plugin: 'io.spring.dependency-management' +dependencies { + implementation project(":job-manage:service-job-manage") + implementation project(":job-execute:service-job-execute") + implementation project(":job-crontab:service-job-crontab") + implementation project(":job-logsvr:service-job-logsvr") + implementation project(":job-file-gateway:service-job-file-gateway") + implementation project(":job-backup:service-job-backup") + implementation project(":job-analysis:service-job-analysis") + implementation 'org.springframework.boot:spring-boot-starter-jdbc' + implementation 'org.springframework.cloud:spring-cloud-starter-bootstrap' + runtimeOnly 'com.mysql:mysql-connector-j' + + testImplementation 'org.springframework.boot:spring-boot-starter-test' + testImplementation 'org.junit.jupiter:junit-jupiter' + testImplementation("com.h2database:h2") + testImplementation 'com.github.kstyrc:embedded-redis' +} +springBoot { + getMainClass().set("com.tencent.bk.job.assemble.JobAssembleBootApplication") + buildInfo() +} + +task copyToLatestJar(type: Copy) { + group = "local" + from('build/libs') + include "job-assemble-${version}.jar" + destinationDir file('build/libs/') + rename "job-assemble-${version}.jar", "job-assemble.jar" +} +copyToLatestJar.dependsOn assemble + +apply from: "$rootDir/task_job_package.gradle" diff --git a/src/backend/job-assemble/src/main/java/com/tencent/bk/job/assemble/JobAssembleBootApplication.java b/src/backend/job-assemble/src/main/java/com/tencent/bk/job/assemble/JobAssembleBootApplication.java new file mode 100644 index 0000000000..293adf08b0 --- /dev/null +++ b/src/backend/job-assemble/src/main/java/com/tencent/bk/job/assemble/JobAssembleBootApplication.java @@ -0,0 +1,60 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.assemble; + +import com.tencent.bk.job.common.service.boot.JobBootApplication; +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.availability.ApplicationAvailabilityAutoConfiguration; +import org.springframework.boot.autoconfigure.jooq.JooqAutoConfiguration; +import org.springframework.cache.annotation.EnableCaching; +import org.springframework.cloud.openfeign.EnableFeignClients; +import org.springframework.scheduling.annotation.EnableScheduling; + +@JobBootApplication( + scanBasePackages = + { + "com.tencent.bk.job.assemble", + "com.tencent.bk.job.manage", + "com.tencent.bk.job.execute", + "com.tencent.bk.job.crontab", + "com.tencent.bk.job.logsvr", + "com.tencent.bk.job.analysis", + "com.tencent.bk.job.file_gateway", + "com.tencent.bk.job.backup", + "com.tencent.bk.job.common.config" + }, + exclude = {JooqAutoConfiguration.class, ApplicationAvailabilityAutoConfiguration.class}, + excludeName = {"org.springframework.cloud.kubernetes.client.discovery.KubernetesDiscoveryClientAutoConfiguration"} +) +@EnableCaching +@EnableScheduling +@EnableFeignClients(basePackages = "com.tencent.bk.job") +public class JobAssembleBootApplication { + + public static void main(String[] args) { + SpringApplication.run(JobAssembleBootApplication.class, args); + } + +} diff --git a/src/backend/job-assemble/src/main/java/com/tencent/bk/job/assemble/config/JobAssembleConfiguration.java b/src/backend/job-assemble/src/main/java/com/tencent/bk/job/assemble/config/JobAssembleConfiguration.java new file mode 100644 index 0000000000..15500bc037 --- /dev/null +++ b/src/backend/job-assemble/src/main/java/com/tencent/bk/job/assemble/config/JobAssembleConfiguration.java @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.assemble.config; + +import com.tencent.bk.job.common.service.AppScopeMappingService; +import com.tencent.bk.job.common.web.interceptor.AppResourceScopeInterceptor; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class JobAssembleConfiguration { + @Bean + public AppResourceScopeInterceptor appResourceScopeInterceptor(AppScopeMappingService appScopeMappingService) { + return new AppResourceScopeInterceptor(appScopeMappingService); + } +} diff --git a/src/backend/job-assemble/src/main/resources/application.yml b/src/backend/job-assemble/src/main/resources/application.yml new file mode 100644 index 0000000000..7ac13cc0fc --- /dev/null +++ b/src/backend/job-assemble/src/main/resources/application.yml @@ -0,0 +1,63 @@ +spring: + profiles: + active: prod + mvc: + pathmatch: + matching-strategy: ant_path_matcher + +management: + health: + livenessstate: + enabled: true + readinessstate: + enabled: true + # 使用spring cloud stream rabbitmq, 禁用掉springboot的rabbitmq health检查 + rabbit: + enabled: false + endpoints: + web: + exposure: + include: health,configprops,env,beans,conditions,loggers,metrics,mappings,prometheus,scheduledtasks,info,refresh,busrefresh,bindings + base-path: /actuator + enabled-by-default: false + endpoint: + health: + enabled: true + show-details: when_authorized + probes: + enabled: true + configprops: + enabled: true + env: + enabled: true + beans: + enabled: true + conditions: + enabled: true + loggers: + enabled: true + metrics: + enabled: true + mappings: + enabled: true + prometheus: + enabled: true + scheduledtasks: + enabled: true + info: + enabled: true + refresh: + enabled: true + busrefresh: + enabled: true + bindings: + enabled: true + +server: + port: 19800 + servlet: + encoding: + charset: UTF-8 + enabled: true + force: true + shutdown: graceful diff --git a/src/backend/job-assemble/src/main/resources/bootstrap.yml b/src/backend/job-assemble/src/main/resources/bootstrap.yml new file mode 100644 index 0000000000..cfd3aa485b --- /dev/null +++ b/src/backend/job-assemble/src/main/resources/bootstrap.yml @@ -0,0 +1,15 @@ +spring: + application: + name: job-assemble + cloud: + kubernetes: + config: + name: ${spring.application.name} + namespace: ${KUBERNETES_NAMESPACE} + sources: + - namespace: ${KUBERNETES_NAMESPACE} + name: ${JOB_APPLICATION_CONFIGMAP_NAME} + - namespace: ${KUBERNETES_NAMESPACE} + name: ${JOB_COMMON_CONFIGMAP_NAME} + - namespace: ${KUBERNETES_NAMESPACE} + name: ${JOB_COMMON_REFRESHABLE_CONFIGMAP_NAME} diff --git a/src/backend/job-assemble/src/main/resources/logback-spring.xml b/src/backend/job-assemble/src/main/resources/logback-spring.xml new file mode 100644 index 0000000000..5c1e4dbb06 --- /dev/null +++ b/src/backend/job-assemble/src/main/resources/logback-spring.xml @@ -0,0 +1,85 @@ + + + logback + + + + + + + + + + + + + ${SCHEDULE_LOG_FILE} + + ${SCHEDULE_LOG_FILE}-%d{yyyyMMdd}.log.%i + 1GB + ${LOG_KEEP_DAYS} + 24GB + ${CLEAN_HISTORY_ON_START} + + + ${FILE_LOG_PATTERN} + UTF-8 + + + + + ${TASK_MONITOR_LOG_FILE} + + ${TASK_MONITOR_LOG_FILE}-%d{yyyy-MM-dd}.log.%i + 1GB + ${LOG_KEEP_DAYS} + 10GB + ${CLEAN_HISTORY_ON_START} + + + ${LOG_PATTERN} + UTF-8 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/backend/job-assemble/src/test/java/com/tencent/bk/job/assemble/BootIntegrationTest.java b/src/backend/job-assemble/src/test/java/com/tencent/bk/job/assemble/BootIntegrationTest.java new file mode 100644 index 0000000000..c3af890d95 --- /dev/null +++ b/src/backend/job-assemble/src/test/java/com/tencent/bk/job/assemble/BootIntegrationTest.java @@ -0,0 +1,66 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.assemble; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.jdbc.SqlConfig; +import org.springframework.test.context.junit.jupiter.SpringExtension; +import redis.embedded.RedisServer; + +import java.io.IOException; + +@ExtendWith(SpringExtension.class) +@SpringBootTest +@ActiveProfiles("test") +@TestPropertySource(locations = "classpath:test.properties") +@SqlConfig(encoding = "utf-8") +public class BootIntegrationTest { + private static RedisServer redisServer; + + @BeforeAll + public static void init() throws IOException { + redisServer = RedisServer.builder() + .port(6379) + .setting("maxmemory 128M") //maxheap 128M + .build(); + redisServer.start(); + } + + @AfterAll + public static void tearDown() { + redisServer.stop(); + } + +// @Test +// @DisplayName("测试 job-assemble 启动") +// public void bootTest() { +// // do nothing +// } +} diff --git a/src/backend/job-assemble/src/test/resources/application-test.yml b/src/backend/job-assemble/src/test/resources/application-test.yml new file mode 100644 index 0000000000..6af4d8bdea --- /dev/null +++ b/src/backend/job-assemble/src/test/resources/application-test.yml @@ -0,0 +1,163 @@ +## Spring 配置 ## +spring: + mvc: + pathmatch: + matching-strategy: ant_path_matcher + datasource: + job-manage: + driver-class-name: org.h2.Driver + type: com.zaxxer.hikari.HikariDataSource + sql-script-encoding: utf-8 + jdbc-url: ${job.manage.db.url} + username: ${job.manage.db.username} + password: ${job.manage.db.password} + maximum-pool-size: 20 + minimum-idle: 5 + idle-timeout: 600000 + poolName: "job-manage" + validationTimeout: 5000 + job-execute: + driver-class-name: org.h2.Driver + type: com.zaxxer.hikari.HikariDataSource + sql-script-encoding: utf-8 + jdbc-url: ${job.execute.db.url} + username: ${job.execute.db.username} + password: ${job.execute.db.password} + maximum-pool-size: 20 + minimum-idle: 5 + idle-timeout: 600000 + poolName: "job-execute" + validationTimeout: 5000 + job-crontab: + driver-class-name: org.h2.Driver + type: com.zaxxer.hikari.HikariDataSource + sql-script-encoding: utf-8 + jdbc-url: ${job.crontab.db.url} + username: ${job.crontab.db.username} + password: ${job.crontab.db.password} + maximum-pool-size: 20 + minimum-idle: 5 + idle-timeout: 600000 + poolName: "job-crontab" + validationTimeout: 5000 + job-file-gateway: + driver-class-name: org.h2.Driver + type: com.zaxxer.hikari.HikariDataSource + sql-script-encoding: utf-8 + jdbc-url: ${job.file_gateway.db.url} + username: ${job.file_gateway.db.username} + password: ${job.file_gateway.db.password} + maximum-pool-size: 20 + minimum-idle: 5 + idle-timeout: 600000 + poolName: "job-file-gateway" + validationTimeout: 5000 + job-analysis: + driver-class-name: org.h2.Driver + type: com.zaxxer.hikari.HikariDataSource + sql-script-encoding: utf-8 + jdbc-url: ${job.analysis.db.url} + username: ${job.analysis.db.username} + password: ${job.analysis.db.password} + maximum-pool-size: 20 + minimum-idle: 5 + idle-timeout: 600000 + poolName: "job-analysis" + validationTimeout: 5000 + job-backup: + driver-class-name: org.h2.Driver + type: com.zaxxer.hikari.HikariDataSource + sql-script-encoding: utf-8 + jdbc-url: ${job.backup.db.url} + username: ${job.backup.db.username} + password: ${job.backup.db.password} + maximum-pool-size: 20 + minimum-idle: 5 + idle-timeout: 600000 + poolName: "job-backup" + validationTimeout: 5000 + redis: + host: 127.0.0.1 + port: 6379 + database: 0 + data: + mongodb: + uri: mongodb://127.0.0.1:27027/job_log + quartz: + auto-startup: false +job: + security: + service: + public-key-base64: LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FROEFNSUlCQ2dLQ0FRRUFyTXN3emh3QkVUTktQOThTcElna0VjZThPNVlQWm5aUDVHVW1ZUVhhSHdHWmg5UHJwNDFseS9xbWk5enEwTlRCODRyaXhSZHZyU1c0cCtCUjk4TVRzRmpYaUlRYVBxRTdRUitpdDdoT1VQNVJMQW9KcWlvL3NlZlRIdlZxVW1GcitYU3NST05rek5selRVaVcvMFRNY0cwNVdIS3hORno5YTB2aXo1T01PeE5lUzhqS1ZiRStIMHlQS0JhR1JHMEdZNTZTS1BLbXh1RHk5a2tyVUtGSXZycldSZkh2bzVsK0xsT3IyQmVGT3FHcEhmeW1FaXlkbFRxWHNlbGlsYTRxMUoxbnBZRDYwY3JIY1ZRV3d2Ri9CZUkrZytxVkF5YzNaWmhYeEhoNWNMcmZsK3pqMktsUEFEZVIvZktubHFqKzFBQm4yaFRoVWJ2WjRCdXhTdWxOa1FJREFRQUIKLS0tLS1FTkQgUFVCTElDIEtFWS0tLS0t + private-key-base64: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV1d0lCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktVd2dnU2hBZ0VBQW9JQkFRQ3N5ekRPSEFFUk0wby8zeEtraUNRUng3dzdsZzltZGsva1pTWmhCZG9mQVptSDArdW5qV1hMK3FhTDNPclExTUh6aXVMRkYyK3RKYmluNEZIM3d4T3dXTmVJaEJvK29UdEJINkszdUU1US9sRXNDZ21xS2oreDU5TWU5V3BTWVd2NWRLeEU0MlRNMlhOTlNKYi9STXh3YlRsWWNyRTBYUDFyUytMUGs0dzdFMTVMeU1wVnNUNGZUSThvRm9aRWJRWmpucElvOHFiRzRQTDJTU3RRb1VpK3V0WkY4ZStqbVg0dVU2dllGNFU2b2FrZC9LWVNMSjJWT3BleDZXS1ZyaXJVbldlbGdQclJ5c2R4VkJiQzhYOEY0ajZENnBVREp6ZGxtRmZFZUhsd3V0K1g3T1BZcVU4QU41SDk4cWVXcVA3VUFHZmFGT0ZSdTluZ0c3Rks2VTJSQWdNQkFBRUNnZ0VBZlkvZGREaUhVZjlxV2dWNG41VEtXUjFDSEM2TUhGR205bm5HTE1TNjNzTWoxTDdiUkZNTWdQQXY2L3hwcWJ5Zy81K1I3OVI2ODJBdzBkVEEyNENuSXdNOEE5aXkxWVlGNytuVUxSckIxanNuc3cvTVBCb2RWT0wzMkV4UHFYTmZxZEx0aDlXUm53WDRzbFVvWEhDYStxdHorMkRFZ2g5OGkyYUdkMzVwYlZjT2p0czlWc3FtZE5uQTdLNFVQWkd4aVFheG1rLzBOVEVNY2RqMUl4bWNlZjNqL1RUVkFiR05IdWVtYTJKeUdZdk52N09BZUN3VGJSdW5TNW1PSmptRHlFSmp4dndaVWZwanZoM0w0SkZzMGRJUm5wZ2VtTkY3NytqUmFTTy9WUVlqc1FLelB0SnEwcHFneHUwd0RoQUVBeXpjYWI2NW8wSm1hZzNJUjZNQzdRS0JnUUR1MThHNFRVaTlSVk8yZS9YdWtDSUNiOVZISHlMeVFIUDh4NDhBVFZmVENtSVBEMUduTWp5TlFmSnhsUlNma3UyUjJIM1IyRC81YUNpNUYwTi9USG51OWVqYlMvS1NoYjFxMFl5TWF1TDkxQlZjMkEyeDZlRzRPNFFpT2daT0gzaE0raUVSREJxOExNQ1dILzVtYVJDeDhVRmRQTnpHMUtXVVNwTEJXM3l2ZXdLQmdRQzVOTS9yN09Wc3F6OGlDUjBXa2JramN2eWU4MkJNcGFsZVkyRHNWQVMwRHNMNlMvZlAzY2VUdEJQVmdJL244Sk16b1ZNZE5OK3htbXM4Wi9rMGM5OGlLTzRUeE0zN2daYkIwUE8rWGRyeUZkRFhTRDZXTm1heWpYUG5MUndBZmRwbFlWdFNqVGxWMWJzWWIySnhXNXY2RUVSaVlLYnNnRGNpWTA5ZnYva0RZd0ovUnVmR212QldwOW50QUQvTXd3WUhFcklnbnZ5ZlkwdS9JMHdiSi92T0Z0aitRM3BJdzFvbW44ajBNTVFSVzA1RE9Ra01VVC9odmlrdDdjVDZSTkJ2WW9HZW4zdnNoNU1zcUltTk1DS0xRTSsxaDlxY05qVTR6WGpkd1V0NGs3akQvaFpEdXN2ZEpBQWxMR3hUR2hRVzRMeStxdTltbTZDRmg0Q2NFTXNZT1FLQmdCa0pEWTRyd0x1V3pucndWbkZGazR3NkwvSGdua1RlSSszeGF4bW5KZGVkSXpnb2FzQTdqNDNreXNXU2ttRmRBTmo2MXJ4YytGd09ycTVFczhnbVhhUkZlaUZMK2pna1JWZS93aU1hbHMwZ3E5RmVINis0Zk50b0NhSFhWREN3MG9QZnpETjFpYW1Uc2EzVTZLN0FIbTZJSW1aV0FGcDUzbm9VaUpjOTRSQlhBb0dCQUxMcmQrbnljNzVEbjZ0ZWdVT0JQRy9HR3FYWkZGdmd6emJRRnR0SmR0NTlMZ0NBanUxMjJMRExsRjZreTB6Q0JsK0h6Q0kyUXJXVEp4cXUxSTZvbndaZDlOMllKVlNWQlFOV285NTJUamRkOHcvSkExYlNqK3JpTU1WQ0txT1BNRHNiOENEaGlEbVM3RlBEM0trTVlmcUxJOVhNRTVvNVUvNFJzcEZUZWozRQotLS0tLUVORCBQUklWQVRFIEtFWS0tLS0t +management: + health: + livenessstate: + enabled: true + readinessstate: + enabled: true + # 使用spring cloud stream rabbitmq, 禁用掉springboot的rabbitmq health检查 + rabbit: + enabled: false + endpoints: + web: + exposure: + include: health,configprops,env,beans,conditions,loggers,metrics,mappings,prometheus,scheduledtasks,info,refresh,busrefresh,bindings + base-path: /actuator + enabled-by-default: false + endpoint: + health: + enabled: true + show-details: when_authorized + probes: + enabled: true + configprops: + enabled: true + env: + enabled: true + beans: + enabled: true + conditions: + enabled: true + loggers: + enabled: true + metrics: + enabled: true + mappings: + enabled: true + prometheus: + enabled: true + scheduledtasks: + enabled: true + info: + enabled: true + refresh: + enabled: true + busrefresh: + enabled: true + bindings: + enabled: true +app: + code: ${app.code} + secret: ${app.secret} +bk-api-gateway: + gse: + url: gse.apigw.com + bkNotice: + url: bk-notice.apigw.com + cmdb: + url: cmdb.apigw.com +esb: + service: + url: esb.service +server: + port: 19800 + servlet: + encoding: + charset: UTF-8 + enabled: true + force: true + shutdown: graceful + +deploy: + mode: lite diff --git a/src/backend/job-assemble/src/test/resources/application.yml b/src/backend/job-assemble/src/test/resources/application.yml new file mode 100644 index 0000000000..cb172f4b14 --- /dev/null +++ b/src/backend/job-assemble/src/test/resources/application.yml @@ -0,0 +1,13 @@ +spring: + application: + name: job-assemble + cloud: + consul: + enabled: false + config: + discovery: + enabled: false + enabled: false + kubernetes: + discovery: + enabled: false diff --git a/src/backend/job-assemble/src/test/resources/bootstrap.yml b/src/backend/job-assemble/src/test/resources/bootstrap.yml new file mode 100644 index 0000000000..3f164b6356 --- /dev/null +++ b/src/backend/job-assemble/src/test/resources/bootstrap.yml @@ -0,0 +1,8 @@ +spring: + application: + name: job-assemble + profiles: + active: test + cloud: + kubernetes: + enabled: false diff --git a/src/backend/job-assemble/src/test/resources/init_job_analysis_schema.sql b/src/backend/job-assemble/src/test/resources/init_job_analysis_schema.sql new file mode 100644 index 0000000000..c76ef98897 --- /dev/null +++ b/src/backend/job-assemble/src/test/resources/init_job_analysis_schema.sql @@ -0,0 +1,25 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +CREATE SCHEMA IF NOT EXISTS job_analysis; diff --git a/src/backend/job-assemble/src/test/resources/init_job_backup_schema.sql b/src/backend/job-assemble/src/test/resources/init_job_backup_schema.sql new file mode 100644 index 0000000000..f596adc426 --- /dev/null +++ b/src/backend/job-assemble/src/test/resources/init_job_backup_schema.sql @@ -0,0 +1,25 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +CREATE SCHEMA IF NOT EXISTS job_backup; diff --git a/src/backend/job-assemble/src/test/resources/init_job_crontab_schema.sql b/src/backend/job-assemble/src/test/resources/init_job_crontab_schema.sql new file mode 100644 index 0000000000..285ea40058 --- /dev/null +++ b/src/backend/job-assemble/src/test/resources/init_job_crontab_schema.sql @@ -0,0 +1,25 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +CREATE SCHEMA IF NOT EXISTS job_crontab; diff --git a/src/backend/job-assemble/src/test/resources/init_job_execute_schema.sql b/src/backend/job-assemble/src/test/resources/init_job_execute_schema.sql new file mode 100644 index 0000000000..5852d1f601 --- /dev/null +++ b/src/backend/job-assemble/src/test/resources/init_job_execute_schema.sql @@ -0,0 +1,25 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +CREATE SCHEMA IF NOT EXISTS job_execute; diff --git a/src/backend/job-assemble/src/test/resources/init_job_file_gateway_schema.sql b/src/backend/job-assemble/src/test/resources/init_job_file_gateway_schema.sql new file mode 100644 index 0000000000..b594b995b9 --- /dev/null +++ b/src/backend/job-assemble/src/test/resources/init_job_file_gateway_schema.sql @@ -0,0 +1,25 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +CREATE SCHEMA IF NOT EXISTS job_file_gateway; diff --git a/src/backend/job-assemble/src/test/resources/init_job_manage_schema.sql b/src/backend/job-assemble/src/test/resources/init_job_manage_schema.sql new file mode 100644 index 0000000000..06c1fd0ffc --- /dev/null +++ b/src/backend/job-assemble/src/test/resources/init_job_manage_schema.sql @@ -0,0 +1,55 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +CREATE SCHEMA IF NOT EXISTS job_manage; + +USE job_manage; +CREATE TABLE IF NOT EXISTS `global_setting` +( + `key` varchar(255) NOT NULL, + `value` text NULL, + `decription` varchar(255) NULL DEFAULT NULL +) ENGINE = InnoDB; + +CREATE TABLE `application` +( + `app_id` BIGINT(20) UNSIGNED NOT NULL, + `row_create_time` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, + `row_update_time` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + `app_name` VARCHAR(128) DEFAULT NULL, + `maintainers` VARCHAR(8192) DEFAULT NULL, + `bk_supplier_account` VARCHAR(128) NOT NULL DEFAULT '0', + `app_type` TINYINT(4) NOT NULL DEFAULT '1', + `sub_app_ids` TEXT DEFAULT NULL, + `timezone` VARCHAR(128) DEFAULT 'Asia/Shanghai', + `bk_operate_dept_id` BIGINT(20) NULL, + `language` VARCHAR(20) DEFAULT NULL, + `bk_scope_type` VARCHAR(32) DEFAULT '', + `bk_scope_id` VARCHAR(32) DEFAULT '', + `is_deleted` TINYINT(1) UNSIGNED NOT NULL DEFAULT '0', + `attrs` TEXT DEFAULT NULL, + PRIMARY KEY (`app_id`), + KEY (`app_type`) +) ENGINE = INNODB + DEFAULT CHARSET = UTF8MB4; diff --git a/src/backend/job-assemble/src/test/resources/logback-spring.xml b/src/backend/job-assemble/src/test/resources/logback-spring.xml new file mode 100644 index 0000000000..43bc651352 --- /dev/null +++ b/src/backend/job-assemble/src/test/resources/logback-spring.xml @@ -0,0 +1,25 @@ + + + logback + + + + + + + + + + + + ${CONSOLE_LOG_PATTERN} + utf8 + + + + + + + + + diff --git a/src/backend/job-assemble/src/test/resources/test.properties b/src/backend/job-assemble/src/test/resources/test.properties new file mode 100644 index 0000000000..a60b342780 --- /dev/null +++ b/src/backend/job-assemble/src/test/resources/test.properties @@ -0,0 +1,25 @@ +job.execute.db.url=jdbc:h2:mem:job-execute;MODE=MYSQL;INIT=RUNSCRIPT FROM 'classpath:init_job_execute_schema.sql' +job.execute.db.username=job +job.execute.db.password=job_db_password +job.manage.db.url=jdbc:h2:mem:job-manage;MODE=MYSQL;INIT=RUNSCRIPT FROM 'classpath:init_job_manage_schema.sql' +job.manage.db.username=job +job.manage.db.password=job_db_password +job.crontab.db.url=jdbc:h2:mem:job-crontab;MODE=MYSQL;INIT=RUNSCRIPT FROM 'classpath:init_job_crontab_schema.sql' +job.crontab.db.username=job +job.crontab.db.password=job_db_password +job.backup.db.url=jdbc:h2:mem:job-backup;MODE=MYSQL;INIT=RUNSCRIPT FROM 'classpath:init_job_backup_schema.sql' +job.backup.db.username=job +job.backup.db.password=job_db_password +job.file_gateway.db.url=jdbc:h2:mem:job-file-gateway;MODE=MYSQL;INIT=RUNSCRIPT FROM 'classpath:init_job_file_gateway_schema.sql' +job.file_gateway.db.username=job +job.file_gateway.db.password=job_db_password +job.analysis.db.url=jdbc:h2:mem:job-analysis;MODE=MYSQL;INIT=RUNSCRIPT FROM 'classpath:init_job_analysis_schema.sql' +job.analysis.db.username=job +job.analysis.db.password=job_db_password +app.code=job +app.secret=job +iam.system-id=bk_job +iam.base-url=http://bkiam.service.consul:9080/ +job.encrypt.password=test +job.encrypt.scenarioAlgorithms.scriptSensitiveParam=None +job.encrypt.scenarioAlgorithms.cipherVariable=None diff --git a/src/backend/job-backup/api-job-backup/build.gradle b/src/backend/job-backup/api-job-backup/build.gradle index 7f9894e771..5ecc06566d 100644 --- a/src/backend/job-backup/api-job-backup/build.gradle +++ b/src/backend/job-backup/api-job-backup/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/api/inner/ServiceArchiveResource.java b/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/api/inner/ServiceArchiveResource.java deleted file mode 100644 index dc3035f09a..0000000000 --- a/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/api/inner/ServiceArchiveResource.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.api.inner; - -import com.tencent.bk.job.backup.model.inner.ServiceArchiveDBRequest; -import com.tencent.bk.job.common.annotation.InternalAPI; -import com.tencent.bk.job.common.model.InternalResponse; -import io.swagger.annotations.Api; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - -@Api(tags = {"DB_Archive"}) -@RequestMapping("/service/archiveDB") -@RestController -@InternalAPI -public interface ServiceArchiveResource { - @PostMapping - InternalResponse archive(@RequestBody ServiceArchiveDBRequest request); -} diff --git a/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/api/web/WebBackupResource.java b/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/api/web/WebBackupResource.java index 375b87d975..ccdad34475 100644 --- a/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/api/web/WebBackupResource.java +++ b/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/api/web/WebBackupResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -37,6 +37,7 @@ import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; import org.springframework.http.ResponseEntity; +import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; @@ -73,6 +74,7 @@ Response startExport( @PathVariable(value = "scopeId") String scopeId, @ApiParam(value = "作业导出请求", required = true) + @Validated @RequestBody ExportRequest exportRequest ); diff --git a/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/model/inner/ServiceArchiveDBRequest.java b/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/model/inner/ServiceArchiveDBRequest.java index 30ea747ce4..12cd0b996e 100644 --- a/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/model/inner/ServiceArchiveDBRequest.java +++ b/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/model/inner/ServiceArchiveDBRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,12 +26,64 @@ import lombok.Data; +import java.util.Map; + @Data public class ServiceArchiveDBRequest { - private boolean archiveEnabled; + /** + * 归档模式 + * + */ + private String mode; + + /** + * DB数据保留天数 + */ + private int keepDays = 30; + + /** + * 归档数据读取时每次读取的数据量（单个表），服务内存受限时可适当降低该值 + */ + private int readIdStepSize = 1000; + + /** + * 归档数据写入归档库时每次写入的数据量（单个表），服务内存受限时可适当降低该值 + */ + private int batchInsertRowSize = 1000; + + /** + * 每次执行删除的最大行数 + */ + private int deleteRowLimit = 1000; + + /** + * 每批次从 db 表中读取的记录数量 + */ + private int readRowLimit = 10000; + + private Map tableConfigs; + + @Data + public static class TableConfig { + /** + * 归档数据读取时每次读取的数据量（单个表），服务内存受限时可适当降低该值 + */ + private Integer readIdStepSize; + + /** + * 归档数据写入归档库时每次写入的数据量（单个表），服务内存受限时可适当降低该值 + */ + private Integer batchInsertRowSize; - private int dataKeepDays; + /** + * 每次执行删除的最大行数 + */ + private Integer deleteRowLimit; - private boolean deleteEnabled; + /** + * 每批次从 db 表中读取的记录数量 + */ + private int readRowLimit; + } } diff --git a/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/model/req/CheckPasswordRequest.java b/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/model/req/CheckPasswordRequest.java index 03c4de9b04..861fbd9031 100644 --- a/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/model/req/CheckPasswordRequest.java +++ b/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/model/req/CheckPasswordRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/model/req/ExportRequest.java b/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/model/req/ExportRequest.java index 9553e528ae..ec616ab9b5 100644 --- a/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/model/req/ExportRequest.java +++ b/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/model/req/ExportRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,6 +26,7 @@ import com.tencent.bk.job.backup.model.web.BackupTemplateInfoVO; import com.tencent.bk.job.common.util.JobContextUtil; +import com.tencent.bk.job.common.validation.ValidPureFileName; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; @@ -45,6 +46,7 @@ public class ExportRequest { * 压缩包名 */ @ApiModelProperty(value = "压缩包名", required = true) + @ValidPureFileName private String packageName; /** diff --git a/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/model/req/ImportRequest.java b/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/model/req/ImportRequest.java index 8d79809bd6..4411d8007b 100644 --- a/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/model/req/ImportRequest.java +++ b/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/model/req/ImportRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/model/web/BackupJobInfoVO.java b/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/model/web/BackupJobInfoVO.java index 5c9b342fd5..a4ca454193 100644 --- a/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/model/web/BackupJobInfoVO.java +++ b/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/model/web/BackupJobInfoVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/model/web/BackupTemplateInfoVO.java b/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/model/web/BackupTemplateInfoVO.java index c146cbc0ca..9a4f2faa76 100644 --- a/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/model/web/BackupTemplateInfoVO.java +++ b/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/model/web/BackupTemplateInfoVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/model/web/ExportInfoVO.java b/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/model/web/ExportInfoVO.java index c8b688e2dc..210410c1ac 100644 --- a/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/model/web/ExportInfoVO.java +++ b/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/model/web/ExportInfoVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/model/web/IdNameInfoVO.java b/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/model/web/IdNameInfoVO.java index b5f21d2fc9..e65280b9d6 100644 --- a/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/model/web/IdNameInfoVO.java +++ b/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/model/web/IdNameInfoVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/model/web/ImportInfoVO.java b/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/model/web/ImportInfoVO.java index 2e6f864172..88ef246276 100644 --- a/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/model/web/ImportInfoVO.java +++ b/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/model/web/ImportInfoVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/model/web/LogEntityVO.java b/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/model/web/LogEntityVO.java index 3e69089533..05957402ae 100644 --- a/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/model/web/LogEntityVO.java +++ b/src/backend/job-backup/api-job-backup/src/main/java/com/tencent/bk/job/backup/model/web/LogEntityVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-backup/boot-job-backup/build.gradle b/src/backend/job-backup/boot-job-backup/build.gradle index e237392639..a7fb88809c 100644 --- a/src/backend/job-backup/boot-job-backup/build.gradle +++ b/src/backend/job-backup/boot-job-backup/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -30,10 +30,9 @@ dependencies { api project(":commons:common-i18n") implementation 'org.springframework.boot:spring-boot-starter-jdbc' implementation 'org.springframework.cloud:spring-cloud-starter-bootstrap' - implementation 'org.springframework.cloud:spring-cloud-starter-bus-amqp' implementation 'org.springframework:spring-webmvc' implementation(group: 'org.springframework.boot', name: 'spring-boot-starter-data-redis') - runtimeOnly('mysql:mysql-connector-java') + runtimeOnly('com.mysql:mysql-connector-j') testImplementation("com.h2database:h2") } @@ -47,8 +46,7 @@ task renameArtifacts(type: Copy) { destinationDir file('build/libs/') rename "boot-job-backup-${version}.jar", "job-backup-${version}.jar" } -renameArtifacts.dependsOn jar -assemble.dependsOn renameArtifacts +renameArtifacts.dependsOn assemble task copyToLatestJar(type: Copy) { group = "local" @@ -59,3 +57,4 @@ task copyToLatestJar(type: Copy) { } copyToLatestJar.dependsOn assemble apply from: "$rootDir/task_job_package.gradle" +copyToRelease.dependsOn renameArtifacts diff --git a/src/backend/job-backup/boot-job-backup/src/main/java/com/tencent/bk/job/backup/JobBackupBootApplication.java b/src/backend/job-backup/boot-job-backup/src/main/java/com/tencent/bk/job/backup/JobBackupBootApplication.java index 55d0de443d..545e423f10 100644 --- a/src/backend/job-backup/boot-job-backup/src/main/java/com/tencent/bk/job/backup/JobBackupBootApplication.java +++ b/src/backend/job-backup/boot-job-backup/src/main/java/com/tencent/bk/job/backup/JobBackupBootApplication.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,25 +24,26 @@ package com.tencent.bk.job.backup; -import com.tencent.bk.job.common.config.FeatureToggleConfig; +import com.tencent.bk.job.common.service.boot.JobBootApplication; import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration; -import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.boot.autoconfigure.availability.ApplicationAvailabilityAutoConfiguration; +import org.springframework.boot.autoconfigure.jooq.JooqAutoConfiguration; import org.springframework.cache.annotation.EnableCaching; import org.springframework.cloud.openfeign.EnableFeignClients; -import org.springframework.context.annotation.DependsOn; import org.springframework.scheduling.annotation.EnableScheduling; -/** - * @since 21/7/2020 10:55 - */ -@SpringBootApplication(scanBasePackages = "com.tencent.bk.job", exclude = {RedisAutoConfiguration.class}) +@JobBootApplication( + scanBasePackages = "com.tencent.bk.job.backup", + exclude = {JooqAutoConfiguration.class, ApplicationAvailabilityAutoConfiguration.class}, + excludeName = {"org.springframework.cloud.kubernetes.client.discovery.KubernetesDiscoveryClientAutoConfiguration"}) @EnableCaching -@EnableFeignClients +@EnableFeignClients( + basePackages = { + "com.tencent.bk.job.manage.api", + "com.tencent.bk.job.execute.api" + } +) @EnableScheduling -@EnableConfigurationProperties({FeatureToggleConfig.class}) -@DependsOn("applicationContextRegister") public class JobBackupBootApplication { public static void main(String[] args) { SpringApplication.run(JobBackupBootApplication.class, args); diff --git a/src/backend/job-backup/boot-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchivistAutoConfig.java b/src/backend/job-backup/boot-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchivistAutoConfig.java deleted file mode 100644 index 58a51398f1..0000000000 --- a/src/backend/job-backup/boot-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchivistAutoConfig.java +++ /dev/null @@ -1,259 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.config; - -import com.tencent.bk.job.backup.archive.JobExecuteArchiveManage; -import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.impl.ExecuteArchiveDAOImpl; -import com.tencent.bk.job.backup.dao.impl.FileSourceTaskRecordDAO; -import com.tencent.bk.job.backup.dao.impl.GseFileAgentTaskRecordDAO; -import com.tencent.bk.job.backup.dao.impl.GseScriptAgentTaskRecordDAO; -import com.tencent.bk.job.backup.dao.impl.GseTaskIpLogRecordDAO; -import com.tencent.bk.job.backup.dao.impl.GseTaskLogRecordDAO; -import com.tencent.bk.job.backup.dao.impl.GseTaskRecordDAO; -import com.tencent.bk.job.backup.dao.impl.OperationLogRecordDAO; -import com.tencent.bk.job.backup.dao.impl.RollingConfigRecordDAO; -import com.tencent.bk.job.backup.dao.impl.StepInstanceConfirmRecordDAO; -import com.tencent.bk.job.backup.dao.impl.StepInstanceFileRecordDAO; -import com.tencent.bk.job.backup.dao.impl.StepInstanceRecordDAO; -import com.tencent.bk.job.backup.dao.impl.StepInstanceRollingTaskRecordDAO; -import com.tencent.bk.job.backup.dao.impl.StepInstanceScriptRecordDAO; -import com.tencent.bk.job.backup.dao.impl.StepInstanceVariableRecordDAO; -import com.tencent.bk.job.backup.dao.impl.TaskInstanceHostRecordDAO; -import com.tencent.bk.job.backup.dao.impl.TaskInstanceRecordDAO; -import com.tencent.bk.job.backup.dao.impl.TaskInstanceVariableRecordDAO; -import com.tencent.bk.job.backup.service.ArchiveProgressService; -import lombok.extern.slf4j.Slf4j; -import org.jooq.DSLContext; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.scheduling.annotation.EnableScheduling; - -import java.util.concurrent.ExecutorService; - -@Configuration -@EnableScheduling -@Slf4j -public class ArchivistAutoConfig { - - @ConditionalOnExpression("${job.execute.archive.enabled:false} || ${job.execute.archive.delete.enabled:false}") - public static class ExecuteDaoAutoConfig { - - @Bean(name = "taskInstanceRecordDAO") - public TaskInstanceRecordDAO taskInstanceRecordDAO(@Qualifier("job-execute-dsl-context") DSLContext context, - ArchiveConfig archiveConfig) { - log.info("Init TaskInstanceRecordDAO"); - return new TaskInstanceRecordDAO(context, archiveConfig); - } - - @Bean(name = "stepInstanceRecordDAO") - public StepInstanceRecordDAO stepInstanceRecordDAO(@Qualifier("job-execute-dsl-context") DSLContext context, - ArchiveConfig archiveConfig) { - log.info("Init StepInstanceRecordDAO"); - return new StepInstanceRecordDAO(context, archiveConfig); - } - - @Bean(name = "stepInstanceScriptRecordDAO") - public StepInstanceScriptRecordDAO stepInstanceScriptRecordDAO( - @Qualifier("job-execute-dsl-context") DSLContext context, - ArchiveConfig archiveConfig) { - log.info("Init StepInstanceScriptRecordDAO"); - return new StepInstanceScriptRecordDAO(context, archiveConfig); - } - - @Bean(name = "stepInstanceFileRecordDAO") - public StepInstanceFileRecordDAO stepInstanceFileRecordDAO( - @Qualifier("job-execute-dsl-context") DSLContext context, - ArchiveConfig archiveConfig) { - log.info("Init StepInstanceFileRecordDAO"); - return new StepInstanceFileRecordDAO(context, archiveConfig); - } - - @Bean(name = "stepInstanceConfirmRecordDAO") - public StepInstanceConfirmRecordDAO stepInstanceConfirmRecordDAO( - @Qualifier("job-execute-dsl-context") DSLContext context, - ArchiveConfig archiveConfig) { - log.info("Init StepInstanceConfirmRecordDAO"); - return new StepInstanceConfirmRecordDAO(context, archiveConfig); - } - - @Bean(name = "stepInstanceVariableRecordDAO") - public StepInstanceVariableRecordDAO stepInstanceVariableRecordDAO( - @Qualifier("job-execute-dsl-context") DSLContext context, - ArchiveConfig archiveConfig) { - log.info("Init StepInstanceVariableRecordDAO"); - return new StepInstanceVariableRecordDAO(context, archiveConfig); - } - - @Bean(name = "taskInstanceVariableRecordDAO") - public TaskInstanceVariableRecordDAO taskInstanceVariableRecordDAO( - @Qualifier("job-execute-dsl-context") DSLContext context, - ArchiveConfig archiveConfig) { - log.info("Init TaskInstanceVariableRecordDAO"); - return new TaskInstanceVariableRecordDAO(context, archiveConfig); - } - - @Bean(name = "operationLogRecordDAO") - public OperationLogRecordDAO operationLogRecordDAO( - @Qualifier("job-execute-dsl-context") DSLContext context, - ArchiveConfig archiveConfig) { - log.info("Init OperationLogRecordDAO"); - return new OperationLogRecordDAO(context, archiveConfig); - } - - @Bean(name = "gseTaskLogRecordDAO") - public GseTaskLogRecordDAO gseTaskLogRecordDAO( - @Qualifier("job-execute-dsl-context") DSLContext context, - ArchiveConfig archiveConfig) { - log.info("Init GseTaskLogRecordDAO"); - return new GseTaskLogRecordDAO(context, archiveConfig); - } - - @Bean(name = "gseTaskIpLogRecordDAO") - public GseTaskIpLogRecordDAO gseTaskIpLogRecordDAO( - @Qualifier("job-execute-dsl-context") DSLContext context, - ArchiveConfig archiveConfig) { - log.info("Init GseTaskIpLogRecordDAO"); - return new GseTaskIpLogRecordDAO(context, archiveConfig); - } - - @Bean(name = "fileSourceTaskRecordDAO") - public FileSourceTaskRecordDAO fileSourceTaskRecordDAO( - @Qualifier("job-execute-dsl-context") DSLContext context, - ArchiveConfig archiveConfig) { - log.info("Init FileSourceTaskRecordDAO"); - return new FileSourceTaskRecordDAO(context, archiveConfig); - } - - @Bean(name = "gseTaskRecordDAO") - public GseTaskRecordDAO gseTaskRecordDAO( - @Qualifier("job-execute-dsl-context") DSLContext context, - ArchiveConfig archiveConfig) { - log.info("Init GseTaskRecordDAO"); - return new GseTaskRecordDAO(context, archiveConfig); - } - - @Bean(name = "gseScriptAgentTaskRecordDAO") - public GseScriptAgentTaskRecordDAO gseScriptAgentTaskRecordDAO( - @Qualifier("job-execute-dsl-context") DSLContext context, - ArchiveConfig archiveConfig) { - log.info("Init GseScriptAgentTaskRecordDAO"); - return new GseScriptAgentTaskRecordDAO(context, archiveConfig); - } - - @Bean(name = "gseFileAgentTaskRecordDAO") - public GseFileAgentTaskRecordDAO gseFileAgentTaskRecordDAO( - @Qualifier("job-execute-dsl-context") DSLContext context, - ArchiveConfig archiveConfig) { - log.info("Init GseFileAgentTaskRecordDAO"); - return new GseFileAgentTaskRecordDAO(context, archiveConfig); - } - - @Bean(name = "stepInstanceRollingTaskRecordDAO") - public StepInstanceRollingTaskRecordDAO stepInstanceRollingTaskRecordDAO( - @Qualifier("job-execute-dsl-context") DSLContext context, - ArchiveConfig archiveConfig) { - log.info("Init StepInstanceRollingTaskRecordDAO"); - return new StepInstanceRollingTaskRecordDAO(context, archiveConfig); - } - - @Bean(name = "rollingConfigRecordDAO") - public RollingConfigRecordDAO rollingConfigRecordDAO( - @Qualifier("job-execute-dsl-context") DSLContext context, - ArchiveConfig archiveConfig) { - log.info("Init RollingConfigRecordDAO"); - return new RollingConfigRecordDAO(context, archiveConfig); - } - - @Bean(name = "taskInstanceHostRecordDAO") - public TaskInstanceHostRecordDAO taskInstanceHostRecordDAO( - @Qualifier("job-execute-dsl-context") DSLContext context, - ArchiveConfig archiveConfig) { - log.info("Init TaskInstanceHostRecordDAO"); - return new TaskInstanceHostRecordDAO(context, archiveConfig); - } - - } - - @Bean(name = "execute-archive-dao") - @ConditionalOnExpression("${job.execute.archive.enabled:false}") - public ExecuteArchiveDAO executeArchiveDAO(@Qualifier("job-execute-archive-dsl-context") DSLContext context) { - log.info("Init ExecuteArchiveDAO"); - return new ExecuteArchiveDAOImpl(context); - } - - @Bean - @ConditionalOnExpression("${job.execute.archive.enabled:false} || ${job.execute.archive.delete.enabled:false}") - public JobExecuteArchiveManage jobExecuteArchiveManage( - @Autowired(required = false) TaskInstanceRecordDAO taskInstanceRecordDAO, - @Autowired(required = false) StepInstanceRecordDAO stepInstanceRecordDAO, - @Autowired(required = false) StepInstanceScriptRecordDAO stepInstanceScriptRecordDAO, - @Autowired(required = false) StepInstanceFileRecordDAO stepInstanceFileRecordDAO, - @Autowired(required = false) StepInstanceConfirmRecordDAO stepInstanceConfirmRecordDAO, - @Autowired(required = false) StepInstanceVariableRecordDAO stepInstanceVariableRecordDAO, - @Autowired(required = false) TaskInstanceVariableRecordDAO taskInstanceVariableRecordDAO, - @Autowired(required = false) OperationLogRecordDAO operationLogRecordDAO, - @Autowired(required = false) GseTaskLogRecordDAO gseTaskLogRecordDAO, - @Autowired(required = false) GseTaskIpLogRecordDAO gseTaskIpLogRecordDAO, - @Autowired(required = false) FileSourceTaskRecordDAO fileSourceTaskRecordDAO, - @Autowired(required = false) GseTaskRecordDAO gseTaskRecordDAO, - @Autowired(required = false) GseScriptAgentTaskRecordDAO gseScriptAgentTaskRecordDAO, - @Autowired(required = false) GseFileAgentTaskRecordDAO gseFileAgentTaskRecordDAO, - @Autowired(required = false) StepInstanceRollingTaskRecordDAO stepInstanceRollingTaskRecordDAO, - @Autowired(required = false) RollingConfigRecordDAO rollingConfigRecordDAO, - @Autowired(required = false) TaskInstanceHostRecordDAO taskInstanceHostRecordDAO, - @Autowired(required = false) ExecuteArchiveDAO executeArchiveDAO, - ArchiveProgressService archiveProgressService, - @Qualifier("archiveExecutor") ExecutorService archiveExecutor, - ArchiveConfig archiveConfig) { - - log.info("Init JobExecuteArchiveManage"); - return new JobExecuteArchiveManage( - taskInstanceRecordDAO, - stepInstanceRecordDAO, - stepInstanceScriptRecordDAO, - stepInstanceFileRecordDAO, - stepInstanceConfirmRecordDAO, - stepInstanceVariableRecordDAO, - taskInstanceVariableRecordDAO, - operationLogRecordDAO, - gseTaskLogRecordDAO, - gseTaskIpLogRecordDAO, - fileSourceTaskRecordDAO, - gseTaskRecordDAO, - gseScriptAgentTaskRecordDAO, - gseFileAgentTaskRecordDAO, - stepInstanceRollingTaskRecordDAO, - rollingConfigRecordDAO, - taskInstanceHostRecordDAO, - executeArchiveDAO, - archiveProgressService, - archiveConfig, - archiveExecutor); - } -} diff --git a/src/backend/job-backup/boot-job-backup/src/main/java/com/tencent/bk/job/backup/config/BackupDbConfig.java b/src/backend/job-backup/boot-job-backup/src/main/java/com/tencent/bk/job/backup/config/BackupDbConfig.java deleted file mode 100644 index ac29a291f6..0000000000 --- a/src/backend/job-backup/boot-job-backup/src/main/java/com/tencent/bk/job/backup/config/BackupDbConfig.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.config; - -import org.jooq.ConnectionProvider; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; -import org.jooq.impl.DataSourceConnectionProvider; -import org.jooq.impl.DefaultConfiguration; -import org.jooq.impl.DefaultDSLContext; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.boot.jdbc.DataSourceBuilder; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.DependsOn; -import org.springframework.context.annotation.Primary; -import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.jdbc.datasource.DataSourceTransactionManager; -import org.springframework.transaction.annotation.EnableTransactionManagement; - -import javax.sql.DataSource; - -/** - * @date 2019/09/19 - */ -@Configuration -@EnableTransactionManagement -public class BackupDbConfig { - @Qualifier("job-backup-data-source") - @Primary - @Bean(name = "job-backup-data-source") - @ConfigurationProperties(prefix = "spring.datasource.job-backup") - public DataSource dataSource() { - return DataSourceBuilder.create().build(); - } - - @Qualifier("transactionManager") - @Bean(name = "transactionManager") - @DependsOn("job-backup-data-source") - @Primary - public DataSourceTransactionManager transactionManager(@Qualifier("job-backup-data-source") DataSource dataSource) { - return new DataSourceTransactionManager(dataSource); - } - - @Qualifier("job-backup-jdbc-template") - @Bean(name = "job-backup-jdbc-template") - public JdbcTemplate jdbcTemplate(@Qualifier("job-backup-data-source") DataSource dataSource) { - return new JdbcTemplate(dataSource); - } - - @Qualifier("job-backup-dsl-context") - @Bean(name = "job-backup-dsl-context") - public DSLContext dslContext(@Qualifier("job-backup-jooq-conf") org.jooq.Configuration configuration) { - return new DefaultDSLContext(configuration); - } - - @Qualifier("job-backup-jooq-conf") - @Bean(name = "job-backup-jooq-conf") - public org.jooq.Configuration - jooqConf(@Qualifier("job-backup-conn-provider") ConnectionProvider connectionProvider) { - return new DefaultConfiguration().derive(connectionProvider).derive(SQLDialect.MYSQL); - } - - @Qualifier("job-backup-conn-provider") - @Bean(name = "job-backup-conn-provider") - public ConnectionProvider connectionProvider(@Qualifier("job-backup-data-source") DataSource dataSource) { - return new DataSourceConnectionProvider(dataSource); - } - -} diff --git a/src/backend/job-backup/boot-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecuteDbConfig.java b/src/backend/job-backup/boot-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecuteDbConfig.java deleted file mode 100644 index 677da873b4..0000000000 --- a/src/backend/job-backup/boot-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecuteDbConfig.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.config; - -import org.jooq.ConnectionProvider; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; -import org.jooq.impl.DataSourceConnectionProvider; -import org.jooq.impl.DefaultConfiguration; -import org.jooq.impl.DefaultDSLContext; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.boot.jdbc.DataSourceBuilder; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -import javax.sql.DataSource; - -/** - * @date 2019/09/19 - */ -@Configuration -public class ExecuteDbConfig { - - @Qualifier("job-execute-source") - @Bean(name = "job-execute-source") - @ConfigurationProperties(prefix = "spring.datasource.job-execute-db") - @ConditionalOnProperty("spring.datasource.job-execute-db.jdbc-url") - public DataSource executeDataSource() { - return DataSourceBuilder.create().build(); - } - - @Qualifier("job-execute-dsl-context") - @Bean(name = "job-execute-dsl-context") - @ConditionalOnProperty("spring.datasource.job-execute-db.jdbc-url") - public DSLContext executeDslContext(@Qualifier("job-execute-jooq-conf") org.jooq.Configuration configuration) { - return new DefaultDSLContext(configuration); - } - - @Qualifier("job-execute-jooq-conf") - @Bean(name = "job-execute-jooq-conf") - @ConditionalOnProperty("spring.datasource.job-execute-db.jdbc-url") - public org.jooq.Configuration - executeJooqConf(@Qualifier("job-execute-conn-provider") ConnectionProvider connectionProvider) { - return new DefaultConfiguration().derive(connectionProvider).derive(SQLDialect.MYSQL); - } - - @Qualifier("job-execute-conn-provider") - @Bean(name = "job-execute-conn-provider") - @ConditionalOnProperty("spring.datasource.job-execute-db.jdbc-url") - public ConnectionProvider executeConnectionProvider(@Qualifier("job-execute-source") DataSource dataSource) { - return new DataSourceConnectionProvider(dataSource); - } - - - @Qualifier("job-execute-archive-source") - @Bean(name = "job-execute-archive-source") - @ConfigurationProperties(prefix = "spring.datasource.job-execute-archive") - @ConditionalOnProperty("spring.datasource.job-execute-archive.jdbc-url") - public DataSource executeArchiveDataSource() { - return DataSourceBuilder.create().build(); - } - - @Qualifier("job-execute-archive-dsl-context") - @Bean(name = "job-execute-archive-dsl-context") - @ConditionalOnProperty("spring.datasource.job-execute-archive.jdbc-url") - public DSLContext executeArchiveDslContext( - @Qualifier("job-execute-archive-jooq-conf") org.jooq.Configuration configuration) { - return new DefaultDSLContext(configuration); - } - - @Qualifier("job-execute-archive-jooq-conf") - @Bean(name = "job-execute-archive-jooq-conf") - @ConditionalOnProperty("spring.datasource.job-execute-archive.jdbc-url") - public org.jooq.Configuration - executeArchiveJooqConf(@Qualifier("job-execute-archive-conn-provider") ConnectionProvider connectionProvider) { - return new DefaultConfiguration().derive(connectionProvider).derive(SQLDialect.MYSQL); - } - - @Qualifier("job-execute-archive-conn-provider") - @Bean(name = "job-execute-archive-conn-provider") - @ConditionalOnProperty("spring.datasource.job-execute-archive.jdbc-url") - public ConnectionProvider executeArchiveConnectionProvider( - @Qualifier("job-execute-archive-source") DataSource dataSource) { - return new DataSourceConnectionProvider(dataSource); - } - -} diff --git a/src/backend/job-backup/boot-job-backup/src/main/java/com/tencent/bk/job/backup/config/InterceptorConfiguration.java b/src/backend/job-backup/boot-job-backup/src/main/java/com/tencent/bk/job/backup/config/InterceptorConfiguration.java deleted file mode 100644 index 139fcf811e..0000000000 --- a/src/backend/job-backup/boot-job-backup/src/main/java/com/tencent/bk/job/backup/config/InterceptorConfiguration.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.config; - -import com.tencent.bk.job.common.iam.interceptor.AuthAppInterceptor; -import com.tencent.bk.job.common.web.interceptor.JobCommonInterceptor; -import com.tencent.bk.job.common.web.interceptor.ServiceSecurityInterceptor; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Configuration; -import org.springframework.web.servlet.config.annotation.InterceptorRegistry; -import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; - -/** - * @since 6/11/2019 10:38 - */ -@Configuration -public class InterceptorConfiguration implements WebMvcConfigurer { - - private final JobCommonInterceptor jobCommonInterceptor; - private final AuthAppInterceptor authAppInterceptor; - private final ServiceSecurityInterceptor serviceSecurityInterceptor; - - @Autowired - public InterceptorConfiguration( - JobCommonInterceptor jobCommonInterceptor, - AuthAppInterceptor authAppInterceptor, - ServiceSecurityInterceptor serviceSecurityInterceptor - ) { - this.jobCommonInterceptor = jobCommonInterceptor; - this.authAppInterceptor = authAppInterceptor; - this.serviceSecurityInterceptor = serviceSecurityInterceptor; - } - - @Override - public void addInterceptors(InterceptorRegistry registry) { - // 注册拦截器 - registry.addInterceptor(serviceSecurityInterceptor).addPathPatterns("/**").order(0); - registry.addInterceptor(jobCommonInterceptor).addPathPatterns("/**").order(10); - registry.addInterceptor(authAppInterceptor).addPathPatterns("/web/**", "/esb/api/**").order(30); - } -} diff --git a/src/backend/job-backup/boot-job-backup/src/main/java/com/tencent/bk/job/backup/config/JobBackupConfiguration.java b/src/backend/job-backup/boot-job-backup/src/main/java/com/tencent/bk/job/backup/config/JobBackupConfiguration.java new file mode 100644 index 0000000000..f6ca9d900a --- /dev/null +++ b/src/backend/job-backup/boot-job-backup/src/main/java/com/tencent/bk/job/backup/config/JobBackupConfiguration.java @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.config; + +import com.tencent.bk.job.common.service.AppScopeMappingService; +import com.tencent.bk.job.common.web.interceptor.AppResourceScopeInterceptor; +import com.tencent.bk.job.manage.AppScopeMappingServiceImpl; +import com.tencent.bk.job.manage.api.inner.ServiceApplicationResource; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class JobBackupConfiguration { + @Bean + AppScopeMappingService appScopeMappingService(ServiceApplicationResource applicationResource) { + return new AppScopeMappingServiceImpl(applicationResource); + } + + @Bean + public AppResourceScopeInterceptor appResourceScopeInterceptor(AppScopeMappingService appScopeMappingService) { + return new AppResourceScopeInterceptor(appScopeMappingService); + } +} diff --git a/src/backend/job-backup/boot-job-backup/src/main/java/com/tencent/bk/job/backup/config/SwaggerConfig.java b/src/backend/job-backup/boot-job-backup/src/main/java/com/tencent/bk/job/backup/config/SwaggerConfig.java index 056ad1fafc..d0dd618cee 100644 --- a/src/backend/job-backup/boot-job-backup/src/main/java/com/tencent/bk/job/backup/config/SwaggerConfig.java +++ b/src/backend/job-backup/boot-job-backup/src/main/java/com/tencent/bk/job/backup/config/SwaggerConfig.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,38 +24,40 @@ package com.tencent.bk.job.backup.config; +import com.tencent.bk.job.common.service.config.JobCommonConfig; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Profile; import springfox.documentation.builders.PathSelectors; import springfox.documentation.builders.RequestHandlerSelectors; +import springfox.documentation.oas.annotations.EnableOpenApi; import springfox.documentation.spi.DocumentationType; import springfox.documentation.spring.web.plugins.Docket; -import springfox.documentation.swagger2.annotations.EnableSwagger2; import java.util.Arrays; import java.util.HashSet; /** - * @date 2019/09/19 + * Swagger 配置 */ @Configuration -@EnableSwagger2 -@Profile({"dev", "test", "local"}) +@EnableOpenApi +@Profile({"dev", "local"}) public class SwaggerConfig { - private final BkConfig bkConfig; + private final JobCommonConfig jobCommonConfig; @Autowired - public SwaggerConfig(BkConfig bkConfig) { - this.bkConfig = bkConfig; + public SwaggerConfig(JobCommonConfig jobCommonConfig) { + this.jobCommonConfig = jobCommonConfig; } @Bean public Docket api() { - return new Docket(DocumentationType.SWAGGER_2) - .host(bkConfig.getSwaggerUrl() + "/job-backup") + return new Docket(DocumentationType.OAS_30) + .host(jobCommonConfig.getSwaggerUrl()) + .pathMapping("job-backup") .protocols(new HashSet<>(Arrays.asList("http", "https"))).select() .apis(RequestHandlerSelectors.basePackage("com.tencent.bk.job.backup.api")) .paths(PathSelectors.any()) diff --git a/src/backend/job-backup/boot-job-backup/src/main/resources/application.yml b/src/backend/job-backup/boot-job-backup/src/main/resources/application.yml index c678b72aad..73fcbd6a6d 100644 --- a/src/backend/job-backup/boot-job-backup/src/main/resources/application.yml +++ b/src/backend/job-backup/boot-job-backup/src/main/resources/application.yml @@ -1,19 +1,22 @@ spring: profiles: active: prod -ribbon: - ServerListRefreshInterval: 3000 - + mvc: + pathmatch: + matching-strategy: ant_path_matcher management: health: livenessstate: enabled: true readinessstate: enabled: true + # 使用spring cloud stream rabbitmq, 禁用掉springboot的rabbitmq health检查 + rabbit: + enabled: false endpoints: web: exposure: - include: health,configprops,env,beans,conditions,loggers,metrics,mappings,prometheus,scheduledtasks,info,refresh + include: health,configprops,env,beans,conditions,loggers,metrics,mappings,prometheus,scheduledtasks,info,refresh,busrefresh,bindings base-path: /actuator enabled-by-default: false endpoint: @@ -44,6 +47,8 @@ management: enabled: true refresh: enabled: true + busrefresh: + enabled: true server: port: 19808 shutdown: graceful diff --git a/src/backend/job-backup/boot-job-backup/src/main/resources/bootstrap.yml b/src/backend/job-backup/boot-job-backup/src/main/resources/bootstrap.yml index b134b23ecd..ac23b42d2c 100644 --- a/src/backend/job-backup/boot-job-backup/src/main/resources/bootstrap.yml +++ b/src/backend/job-backup/boot-job-backup/src/main/resources/bootstrap.yml @@ -11,3 +11,5 @@ spring: name: ${JOB_APPLICATION_CONFIGMAP_NAME} - namespace: ${KUBERNETES_NAMESPACE} name: ${JOB_COMMON_CONFIGMAP_NAME} + - namespace: ${KUBERNETES_NAMESPACE} + name: ${JOB_COMMON_REFRESHABLE_CONFIGMAP_NAME} diff --git a/src/backend/job-backup/boot-job-backup/src/main/resources/i18n/message.properties b/src/backend/job-backup/boot-job-backup/src/main/resources/i18n/message.properties deleted file mode 100644 index 17e3eb5c38..0000000000 --- a/src/backend/job-backup/boot-job-backup/src/main/resources/i18n/message.properties +++ /dev/null @@ -1,72 +0,0 @@ -# -# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. -# -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. -# -# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. -# -# License for BK-JOB蓝鲸智云作业平台: -# -------------------------------------------------------------------- -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF -# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - -job.backup.startExport=开始执行作业导出任务... -job.backup.processFinished=处理完成! -job.backup.processLocalFile=正在处理作业中的 "本地文件"... -job.backup.noLocalFile=导出的作业中不存在 "本地文件", 自动跳过 -job.backup.processScript=正在获取作业中的 "引用脚本" 相关信息... -job.backup.noScript=导出的作业中没有 "引用脚本", 自动跳过 -job.backup.processCipherText=正在处理作业中的 "密文" 类型变量值... -job.backup.saveNull=保存为空值 -job.backup.saveReal=保存真实值 -job.backup.noCipherText=导出的作业中没有使用 "密文" 变量，自动跳过 -job.backup.processTemplatePlanFinished=作业模板与执行方案信息处理完成! -job.backup.startPackage=开始打包... -job.backup.packageFinished=打包完成! -job.backup.startEncrypting=正在为压缩包进行加密处理... -job.backup.encryptingFinished=加密完成! -job.backup.skipEncrypting=文件加密选项为[不加密]，自动跳过 -job.backup.exportFinished=作业导出任务完成! [文件有效期:%s] -job.backup.forever=永久 -job.backup.day=天 -job.backup.template=模板 -job.backup.plan=执行方案 -job.backup.processUploadFile=开始检测上传文件... -job.backup.detectFileType=正在检测文件类型是否正确... -job.backup.correctFileType=检测通过! -job.backup.wrongFileType=检测不通过! (仅允许 .jobexport 为后缀的文件) -job.backup.extractFileData=正在解析文件的数据格式... -job.backup.extractSuccess=解析成功! -job.backup.fileEncrypted=文件包已被加密处理，请输入对应密码 -job.backup.correctPassword=密码校验成功! -job.backup.wrongPassword=密码校验失败! -job.backup.extractFailed=解析失败! 请确认该文件包是否正确来自作业平台导出. -job.backup.startImport=开始处理导入请求... -job.backup.importSetting=按照 "导入设置": -job.backup.idAutoIncrement=1. 作业原始 ID 不保留, 将以自增的方式导入 -job.backup.idKeepOnDuplicateIncrement=1. 保留作业原始 ID, 发现冲突时将以自增的方式导入 -job.backup.idKeepOnDuplicateSkip=1. 保留作业原始 ID, 发现冲突时即不进行导入 -job.backup.nameDuplicateSuffix=2. 若出现同名的作业模板, 将自动添加后缀 %s -job.backup.idAutoIncrementSuffix=ID 不保留, 将以自增的方式导入 -job.backup.idDuplicateIncrementSuffix=ID 已存在, 改为自增 ID 进行导入 -job.backup.idDuplicateSkipSuffix=ID 已存在, 取消导入该%s -job.backup.idKeepSuffix=ID 不存在, 将以原始 ID 进行导入 -job.backup.startImportTemplate=准备导入作业模板: [%d] %s -job.backup.templateNameChange=模板名称已存在! 自动更改为 %s -job.backup.importTemplateSuccess=模板导入成功! -job.backup.startImportPlan=>>>>> 准备导入作业执行方案: %s -job.backup.importPlanSuccess=执行方案导入成功! -job.backup.importFailed=作业导入出现异常, 请稍后重试. -job.backup.importFinished=作业导入任务已全部结束! diff --git a/src/backend/job-backup/boot-job-backup/src/main/resources/i18n/message_en.properties b/src/backend/job-backup/boot-job-backup/src/main/resources/i18n/message_en.properties deleted file mode 100644 index 48ff22ec2d..0000000000 --- a/src/backend/job-backup/boot-job-backup/src/main/resources/i18n/message_en.properties +++ /dev/null @@ -1,72 +0,0 @@ -# -# Tencent is pleased to support the open source community by making BK-JOB����������ҵƽ̨ available. -# -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. -# -# BK-JOB����������ҵƽ̨ is licensed under the MIT License. -# -# License for BK-JOB����������ҵƽ̨: -# -------------------------------------------------------------------- -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF -# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - -job.backup.startExport=Starting job export process... -job.backup.processFinished=Done! -job.backup.processLocalFile=Saving "Local Files" from exported jobs... -job.backup.noLocalFile="Local file" is not found in exported jobs -job.backup.processScript=Saving "Related Scripts" from exported jobs... -job.backup.noScript="Related Script" is not found in exported jobs -job.backup.processCipherText=Processing on "Ciphertext" variable's value... -job.backup.saveNull=Replace with empty value -job.backup.saveReal=Using original value -job.backup.noCipherText="Ciphertext" variable is not found in exported jobs -job.backup.processTemplatePlanFinished=Job data information processing complete! -job.backup.startPackage=Start packing... -job.backup.packageFinished=Done! -job.backup.startEncrypting=Encrypting the package... -job.backup.encryptingFinished=Encrypt complete! -job.backup.skipEncrypting=File encrypt option is [No] -job.backup.exportFinished=Job export complete! [File Valid of days:%s] -job.backup.forever=Permanently -job.backup.day=Day -job.backup.template=Template -job.backup.plan=Plan -job.backup.processUploadFile=Detecting uploaded file... -job.backup.detectFileType=Checking file type... -job.backup.correctFileType=PASSED! -job.backup.wrongFileType=FAILED! (Only allowed .jobexport) -job.backup.extractFileData=Parsing data format of the file... -job.backup.extractSuccess=CORRECT! -job.backup.fileEncrypted=The file has been encrypted, please enter the password -job.backup.correctPassword=Password verified! -job.backup.wrongPassword=Wrong Password! -job.backup.extractFailed=INCORRECT! Please make sure the file is correctly exported from Job. -job.backup.startImport=Starting Job import process... -job.backup.importSetting=Following Import "Settings": -job.backup.idAutoIncrement=1. Do not retain Job ID, import by auto increment -job.backup.idKeepOnDuplicateIncrement=1. Retain Job ID, auto increment if ID already exists -job.backup.idKeepOnDuplicateSkip=1. Retain Job ID, skip it if ID already exists -job.backup.nameDuplicateSuffix=2. Add suffix %s if Job name is duplicate -job.backup.idAutoIncrementSuffix=Do not retain the ID, import by auto increment -job.backup.idDuplicateIncrementSuffix=ID already exists, will import by auto increment -job.backup.idDuplicateSkipSuffix=ID already exists, skip to next Job %s -job.backup.idKeepSuffix=ID not exists, will import directly -job.backup.startImportTemplate=Prepare to import Job Template: [%d] %s -job.backup.templateNameChange=Template name already exists! Rename with %s -job.backup.importTemplateSuccess=Job Template Import Successful! -job.backup.startImportPlan=>>>>> Prepare to import Job Plan: %s -job.backup.importPlanSuccess=Job Plan Import Successful! -job.backup.importFailed=Service Error! Please try again later. -job.backup.importFinished=Job import complete! diff --git a/src/backend/job-backup/boot-job-backup/src/main/resources/i18n/message_en_US.properties b/src/backend/job-backup/boot-job-backup/src/main/resources/i18n/message_en_US.properties deleted file mode 100644 index 48ff22ec2d..0000000000 --- a/src/backend/job-backup/boot-job-backup/src/main/resources/i18n/message_en_US.properties +++ /dev/null @@ -1,72 +0,0 @@ -# -# Tencent is pleased to support the open source community by making BK-JOB����������ҵƽ̨ available. -# -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. -# -# BK-JOB����������ҵƽ̨ is licensed under the MIT License. -# -# License for BK-JOB����������ҵƽ̨: -# -------------------------------------------------------------------- -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF -# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - -job.backup.startExport=Starting job export process... -job.backup.processFinished=Done! -job.backup.processLocalFile=Saving "Local Files" from exported jobs... -job.backup.noLocalFile="Local file" is not found in exported jobs -job.backup.processScript=Saving "Related Scripts" from exported jobs... -job.backup.noScript="Related Script" is not found in exported jobs -job.backup.processCipherText=Processing on "Ciphertext" variable's value... -job.backup.saveNull=Replace with empty value -job.backup.saveReal=Using original value -job.backup.noCipherText="Ciphertext" variable is not found in exported jobs -job.backup.processTemplatePlanFinished=Job data information processing complete! -job.backup.startPackage=Start packing... -job.backup.packageFinished=Done! -job.backup.startEncrypting=Encrypting the package... -job.backup.encryptingFinished=Encrypt complete! -job.backup.skipEncrypting=File encrypt option is [No] -job.backup.exportFinished=Job export complete! [File Valid of days:%s] -job.backup.forever=Permanently -job.backup.day=Day -job.backup.template=Template -job.backup.plan=Plan -job.backup.processUploadFile=Detecting uploaded file... -job.backup.detectFileType=Checking file type... -job.backup.correctFileType=PASSED! -job.backup.wrongFileType=FAILED! (Only allowed .jobexport) -job.backup.extractFileData=Parsing data format of the file... -job.backup.extractSuccess=CORRECT! -job.backup.fileEncrypted=The file has been encrypted, please enter the password -job.backup.correctPassword=Password verified! -job.backup.wrongPassword=Wrong Password! -job.backup.extractFailed=INCORRECT! Please make sure the file is correctly exported from Job. -job.backup.startImport=Starting Job import process... -job.backup.importSetting=Following Import "Settings": -job.backup.idAutoIncrement=1. Do not retain Job ID, import by auto increment -job.backup.idKeepOnDuplicateIncrement=1. Retain Job ID, auto increment if ID already exists -job.backup.idKeepOnDuplicateSkip=1. Retain Job ID, skip it if ID already exists -job.backup.nameDuplicateSuffix=2. Add suffix %s if Job name is duplicate -job.backup.idAutoIncrementSuffix=Do not retain the ID, import by auto increment -job.backup.idDuplicateIncrementSuffix=ID already exists, will import by auto increment -job.backup.idDuplicateSkipSuffix=ID already exists, skip to next Job %s -job.backup.idKeepSuffix=ID not exists, will import directly -job.backup.startImportTemplate=Prepare to import Job Template: [%d] %s -job.backup.templateNameChange=Template name already exists! Rename with %s -job.backup.importTemplateSuccess=Job Template Import Successful! -job.backup.startImportPlan=>>>>> Prepare to import Job Plan: %s -job.backup.importPlanSuccess=Job Plan Import Successful! -job.backup.importFailed=Service Error! Please try again later. -job.backup.importFinished=Job import complete! diff --git a/src/backend/job-backup/boot-job-backup/src/main/resources/i18n/message_zh.properties b/src/backend/job-backup/boot-job-backup/src/main/resources/i18n/message_zh.properties deleted file mode 100644 index 656e078db1..0000000000 --- a/src/backend/job-backup/boot-job-backup/src/main/resources/i18n/message_zh.properties +++ /dev/null @@ -1,72 +0,0 @@ -# -# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. -# -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. -# -# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. -# -# License for BK-JOB蓝鲸智云作业平台: -# -------------------------------------------------------------------- -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF -# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - -job.backup.startExport=开始执行作业导出任务... -job.backup.processFinished=处理完成! -job.backup.processLocalFile=正在处理作业中的 "本地文件"... -job.backup.noLocalFile=导出的作业中不存在 "本地文件", 自动跳过 -job.backup.processScript=正在获取作业中的 "引用脚本" 相关信息... -job.backup.noScript=导出的作业中没有 "引用脚本", 自动跳过 -job.backup.processCipherText=正在处理作业中的 "密文" 类型变量值... -job.backup.saveNull=保存为空值 -job.backup.saveReal=保存真实值 -job.backup.noCipherText=导出的作业中没有使用 "密文" 变量，自动跳过 -job.backup.processTemplatePlanFinished=作业模板与执行方案信息处理完成! -job.backup.startPackage=开始打包... -job.backup.packageFinished=打包完成! -job.backup.startEncrypting=正在为压缩包进行加密处理... -job.backup.encryptingFinished=加密完成! -job.backup.skipEncrypting=文件加密选项为[不加密]，自动跳过 -job.backup.exportFinished=作业导出任务完成! [文件有效期:%s] -job.backup.forever=永久 -job.backup.day=天 -job.backup.template=模板 -job.backup.plan=执行方案 -job.backup.processUploadFile=开始检测上传文件... -job.backup.detectFileType=正在检测文件类型是否正确... -job.backup.correctFileType=检测通过! -job.backup.wrongFileType=检测不通过! (仅允许 .jobexport 为后缀的文件) -job.backup.extractFileData=正在解析文件的数据格式... -job.backup.extractSuccess=解析成功! -job.backup.fileEncrypted=文件包已被加密处理，请输入对应密码 -job.backup.correctPassword=密码校验成功! -job.backup.wrongPassword=密码校验失败! -job.backup.extractFailed=解析失败! 请确认该文件包是否正确来自作业平台导出. -job.backup.startImport=开始处理导入请求... -job.backup.importSetting=按照 "导入设置": -job.backup.idAutoIncrement=1. 作业原始ID不保留, 将以自增的方式导入 -job.backup.idKeepOnDuplicateIncrement=1. 保留作业原始ID, 发现冲突时将以自增的方式导入 -job.backup.idKeepOnDuplicateSkip=1. 保留作业原始ID, 发现冲突时即不进行导入 -job.backup.nameDuplicateSuffix=2. 若出现同名的作业模板, 将自动添加后缀 %s -job.backup.idAutoIncrementSuffix=ID不保留, 将以自增的方式导入 -job.backup.idDuplicateIncrementSuffix=ID已存在, 改为自增ID进行导入 -job.backup.idDuplicateSkipSuffix=ID已存在, 取消导入该%s -job.backup.idKeepSuffix=ID不存在, 将以原始ID进行导入 -job.backup.startImportTemplate=准备导入作业模板: [%d] %s -job.backup.templateNameChange=模板名称已存在! 自动更改为 %s -job.backup.importTemplateSuccess=模板导入成功! -job.backup.startImportPlan=>>>>> 准备导入作业执行方案: %s -job.backup.importPlanSuccess=执行方案导入成功! -job.backup.importFailed=作业导入出现异常, 请稍后重试. -job.backup.importFinished=作业导入任务已全部结束! diff --git a/src/backend/job-backup/boot-job-backup/src/main/resources/i18n/message_zh_CN.properties b/src/backend/job-backup/boot-job-backup/src/main/resources/i18n/message_zh_CN.properties deleted file mode 100644 index 656e078db1..0000000000 --- a/src/backend/job-backup/boot-job-backup/src/main/resources/i18n/message_zh_CN.properties +++ /dev/null @@ -1,72 +0,0 @@ -# -# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. -# -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. -# -# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. -# -# License for BK-JOB蓝鲸智云作业平台: -# -------------------------------------------------------------------- -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF -# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - -job.backup.startExport=开始执行作业导出任务... -job.backup.processFinished=处理完成! -job.backup.processLocalFile=正在处理作业中的 "本地文件"... -job.backup.noLocalFile=导出的作业中不存在 "本地文件", 自动跳过 -job.backup.processScript=正在获取作业中的 "引用脚本" 相关信息... -job.backup.noScript=导出的作业中没有 "引用脚本", 自动跳过 -job.backup.processCipherText=正在处理作业中的 "密文" 类型变量值... -job.backup.saveNull=保存为空值 -job.backup.saveReal=保存真实值 -job.backup.noCipherText=导出的作业中没有使用 "密文" 变量，自动跳过 -job.backup.processTemplatePlanFinished=作业模板与执行方案信息处理完成! -job.backup.startPackage=开始打包... -job.backup.packageFinished=打包完成! -job.backup.startEncrypting=正在为压缩包进行加密处理... -job.backup.encryptingFinished=加密完成! -job.backup.skipEncrypting=文件加密选项为[不加密]，自动跳过 -job.backup.exportFinished=作业导出任务完成! [文件有效期:%s] -job.backup.forever=永久 -job.backup.day=天 -job.backup.template=模板 -job.backup.plan=执行方案 -job.backup.processUploadFile=开始检测上传文件... -job.backup.detectFileType=正在检测文件类型是否正确... -job.backup.correctFileType=检测通过! -job.backup.wrongFileType=检测不通过! (仅允许 .jobexport 为后缀的文件) -job.backup.extractFileData=正在解析文件的数据格式... -job.backup.extractSuccess=解析成功! -job.backup.fileEncrypted=文件包已被加密处理，请输入对应密码 -job.backup.correctPassword=密码校验成功! -job.backup.wrongPassword=密码校验失败! -job.backup.extractFailed=解析失败! 请确认该文件包是否正确来自作业平台导出. -job.backup.startImport=开始处理导入请求... -job.backup.importSetting=按照 "导入设置": -job.backup.idAutoIncrement=1. 作业原始ID不保留, 将以自增的方式导入 -job.backup.idKeepOnDuplicateIncrement=1. 保留作业原始ID, 发现冲突时将以自增的方式导入 -job.backup.idKeepOnDuplicateSkip=1. 保留作业原始ID, 发现冲突时即不进行导入 -job.backup.nameDuplicateSuffix=2. 若出现同名的作业模板, 将自动添加后缀 %s -job.backup.idAutoIncrementSuffix=ID不保留, 将以自增的方式导入 -job.backup.idDuplicateIncrementSuffix=ID已存在, 改为自增ID进行导入 -job.backup.idDuplicateSkipSuffix=ID已存在, 取消导入该%s -job.backup.idKeepSuffix=ID不存在, 将以原始ID进行导入 -job.backup.startImportTemplate=准备导入作业模板: [%d] %s -job.backup.templateNameChange=模板名称已存在! 自动更改为 %s -job.backup.importTemplateSuccess=模板导入成功! -job.backup.startImportPlan=>>>>> 准备导入作业执行方案: %s -job.backup.importPlanSuccess=执行方案导入成功! -job.backup.importFailed=作业导入出现异常, 请稍后重试. -job.backup.importFinished=作业导入任务已全部结束! diff --git a/src/backend/job-backup/boot-job-backup/src/main/resources/logback-spring.xml b/src/backend/job-backup/boot-job-backup/src/main/resources/logback-spring.xml new file mode 100644 index 0000000000..17d6fb4428 --- /dev/null +++ b/src/backend/job-backup/boot-job-backup/src/main/resources/logback-spring.xml @@ -0,0 +1,51 @@ + + + logback + + + + + + + + + + + + ${ARCHIVE_ERROR_LOG_FILE} + + ${ARCHIVE_ERROR_LOG_FILE}-%d{yyyy-MM-dd}.log.%i + 1GB + ${LOG_KEEP_DAYS} + 20GB + ${CLEAN_HISTORY_ON_START} + + + ${FILE_LOG_PATTERN} + UTF-8 + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/backend/job-backup/boot-job-backup/src/main/resources/logback/logback-app-props.xml b/src/backend/job-backup/boot-job-backup/src/main/resources/logback/logback-app-props.xml deleted file mode 100644 index 9791b58f20..0000000000 --- a/src/backend/job-backup/boot-job-backup/src/main/resources/logback/logback-app-props.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - diff --git a/src/backend/job-backup/boot-job-backup/src/main/resources/logback/logback-app.xml b/src/backend/job-backup/boot-job-backup/src/main/resources/logback/logback-app.xml deleted file mode 100644 index 864649003b..0000000000 --- a/src/backend/job-backup/boot-job-backup/src/main/resources/logback/logback-app.xml +++ /dev/null @@ -1,40 +0,0 @@ - - - - - ${ARCHIVE_ERROR_LOG_FILE} - - ${ARCHIVE_ERROR_LOG_FILE}-%d{yyyy-MM-dd}.log.%i - 1GB - 168 - 20GB - - - ${FILE_LOG_PATTERN} - UTF-8 - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/backend/job-backup/boot-job-backup/src/test/java/com/tencent/bk/job/backup/BootIntegrationTest.java b/src/backend/job-backup/boot-job-backup/src/test/java/com/tencent/bk/job/backup/BootIntegrationTest.java new file mode 100644 index 0000000000..86e7beb5af --- /dev/null +++ b/src/backend/job-backup/boot-job-backup/src/test/java/com/tencent/bk/job/backup/BootIntegrationTest.java @@ -0,0 +1,49 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.jdbc.SqlConfig; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +@ExtendWith(SpringExtension.class) +@SpringBootTest +@ActiveProfiles("test") +@TestPropertySource(locations = "classpath:test.properties") +@SqlConfig(encoding = "utf-8") +public class BootIntegrationTest { + + + @Test + @DisplayName("测试 job-backup 启动") + public void bootTest() { + // do nothing + } +} diff --git a/src/backend/job-backup/boot-job-backup/src/test/java/com/tencent/bk/job/backup/dao/execute/ArchiveProgressDAOImplIntegrationTest.java b/src/backend/job-backup/boot-job-backup/src/test/java/com/tencent/bk/job/backup/dao/execute/ArchiveProgressDAOImplIntegrationTest.java deleted file mode 100644 index ad8e954731..0000000000 --- a/src/backend/job-backup/boot-job-backup/src/test/java/com/tencent/bk/job/backup/dao/execute/ArchiveProgressDAOImplIntegrationTest.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.dao.execute; - -import org.junit.jupiter.api.Test; - -//@ExtendWith(SpringExtension.class) -//@SpringBootTest -//@ActiveProfiles("test") -//@TestPropertySource(locations = "classpath:test.properties") -//@SqlConfig(encoding = "utf-8") -//@Sql({"/init_archive_progress_data.sql"}) -class ArchiveProgressDAOImplIntegrationTest { - -// @Autowired -// private ArchiveProgressDAO archiveProgressDAO; - - @Test - void queryArchiveProgress() { -// ArchiveProgressDTO archiveProgress = archiveProgressDAO.queryArchiveProgress("task_instance"); -// assertThat(archiveProgress.getTableName()).isEqualTo("task_instance"); -// assertThat(archiveProgress.getProgress().getLastArchivedId()).isEqualTo(1000L); -// assertThat(archiveProgress.getLastModifyTime()).isEqualTo(1621166442000L); - } - - @Test - void saveArchiveProgress() { - } -} diff --git a/src/backend/job-backup/boot-job-backup/src/test/resources/application-test.yml b/src/backend/job-backup/boot-job-backup/src/test/resources/application-test.yml index 978de9b5bf..e5d9dbcf4a 100644 --- a/src/backend/job-backup/boot-job-backup/src/test/resources/application-test.yml +++ b/src/backend/job-backup/boot-job-backup/src/test/resources/application-test.yml @@ -1,10 +1,18 @@ spring: + mvc: + pathmatch: + matching-strategy: ant_path_matcher cloud: refresh: extra-refreshable: javax.sql.DataSource + config: + enabled: false + kubernetes: + discovery: + enabled: false datasource: job-backup: - driver-class-name: com.mysql.cj.jdbc.Driver + driver-class-name: org.h2.Driver type: com.zaxxer.hikari.HikariDataSource jdbc-url: ${job.backup.db.url} username: ${job.backup.db.username} @@ -14,8 +22,8 @@ spring: idle-timeout: 600000 poolName: "job-backup" validationTimeout: 5000 - job-execute-db: - driver-class-name: com.mysql.cj.jdbc.Driver + job-execute: + driver-class-name: org.h2.Driver type: com.zaxxer.hikari.HikariDataSource jdbc-url: ${job.execute.db.dr.url:} username: ${job.execute.db.dr.username:} @@ -23,10 +31,10 @@ spring: maximum-pool-size: 10 minimum-idle: 2 idle-timeout: 6000 - poolName: "job-execute-db" + poolName: "job-execute" validationTimeout: 5000 job-execute-archive: - driver-class-name: com.mysql.cj.jdbc.Driver + driver-class-name: org.h2.Driver type: com.zaxxer.hikari.HikariDataSource jdbc-url: ${job.execute.db.archive.url:} username: ${job.execute.db.archive.username:} @@ -41,3 +49,17 @@ job: service: public-key-base64: LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FROEFNSUlCQ2dLQ0FRRUFyTXN3emh3QkVUTktQOThTcElna0VjZThPNVlQWm5aUDVHVW1ZUVhhSHdHWmg5UHJwNDFseS9xbWk5enEwTlRCODRyaXhSZHZyU1c0cCtCUjk4TVRzRmpYaUlRYVBxRTdRUitpdDdoT1VQNVJMQW9KcWlvL3NlZlRIdlZxVW1GcitYU3NST05rek5selRVaVcvMFRNY0cwNVdIS3hORno5YTB2aXo1T01PeE5lUzhqS1ZiRStIMHlQS0JhR1JHMEdZNTZTS1BLbXh1RHk5a2tyVUtGSXZycldSZkh2bzVsK0xsT3IyQmVGT3FHcEhmeW1FaXlkbFRxWHNlbGlsYTRxMUoxbnBZRDYwY3JIY1ZRV3d2Ri9CZUkrZytxVkF5YzNaWmhYeEhoNWNMcmZsK3pqMktsUEFEZVIvZktubHFqKzFBQm4yaFRoVWJ2WjRCdXhTdWxOa1FJREFRQUIKLS0tLS1FTkQgUFVCTElDIEtFWS0tLS0t private-key-base64: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV1d0lCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktVd2dnU2hBZ0VBQW9JQkFRQ3N5ekRPSEFFUk0wby8zeEtraUNRUng3dzdsZzltZGsva1pTWmhCZG9mQVptSDArdW5qV1hMK3FhTDNPclExTUh6aXVMRkYyK3RKYmluNEZIM3d4T3dXTmVJaEJvK29UdEJINkszdUU1US9sRXNDZ21xS2oreDU5TWU5V3BTWVd2NWRLeEU0MlRNMlhOTlNKYi9STXh3YlRsWWNyRTBYUDFyUytMUGs0dzdFMTVMeU1wVnNUNGZUSThvRm9aRWJRWmpucElvOHFiRzRQTDJTU3RRb1VpK3V0WkY4ZStqbVg0dVU2dllGNFU2b2FrZC9LWVNMSjJWT3BleDZXS1ZyaXJVbldlbGdQclJ5c2R4VkJiQzhYOEY0ajZENnBVREp6ZGxtRmZFZUhsd3V0K1g3T1BZcVU4QU41SDk4cWVXcVA3VUFHZmFGT0ZSdTluZ0c3Rks2VTJSQWdNQkFBRUNnZ0VBZlkvZGREaUhVZjlxV2dWNG41VEtXUjFDSEM2TUhGR205bm5HTE1TNjNzTWoxTDdiUkZNTWdQQXY2L3hwcWJ5Zy81K1I3OVI2ODJBdzBkVEEyNENuSXdNOEE5aXkxWVlGNytuVUxSckIxanNuc3cvTVBCb2RWT0wzMkV4UHFYTmZxZEx0aDlXUm53WDRzbFVvWEhDYStxdHorMkRFZ2g5OGkyYUdkMzVwYlZjT2p0czlWc3FtZE5uQTdLNFVQWkd4aVFheG1rLzBOVEVNY2RqMUl4bWNlZjNqL1RUVkFiR05IdWVtYTJKeUdZdk52N09BZUN3VGJSdW5TNW1PSmptRHlFSmp4dndaVWZwanZoM0w0SkZzMGRJUm5wZ2VtTkY3NytqUmFTTy9WUVlqc1FLelB0SnEwcHFneHUwd0RoQUVBeXpjYWI2NW8wSm1hZzNJUjZNQzdRS0JnUUR1MThHNFRVaTlSVk8yZS9YdWtDSUNiOVZISHlMeVFIUDh4NDhBVFZmVENtSVBEMUduTWp5TlFmSnhsUlNma3UyUjJIM1IyRC81YUNpNUYwTi9USG51OWVqYlMvS1NoYjFxMFl5TWF1TDkxQlZjMkEyeDZlRzRPNFFpT2daT0gzaE0raUVSREJxOExNQ1dILzVtYVJDeDhVRmRQTnpHMUtXVVNwTEJXM3l2ZXdLQmdRQzVOTS9yN09Wc3F6OGlDUjBXa2JramN2eWU4MkJNcGFsZVkyRHNWQVMwRHNMNlMvZlAzY2VUdEJQVmdJL244Sk16b1ZNZE5OK3htbXM4Wi9rMGM5OGlLTzRUeE0zN2daYkIwUE8rWGRyeUZkRFhTRDZXTm1heWpYUG5MUndBZmRwbFlWdFNqVGxWMWJzWWIySnhXNXY2RUVSaVlLYnNnRGNpWTA5ZnYva0RZd0ovUnVmR212QldwOW50QUQvTXd3WUhFcklnbnZ5ZlkwdS9JMHdiSi92T0Z0aitRM3BJdzFvbW44ajBNTVFSVzA1RE9Ra01VVC9odmlrdDdjVDZSTkJ2WW9HZW4zdnNoNU1zcUltTk1DS0xRTSsxaDlxY05qVTR6WGpkd1V0NGs3akQvaFpEdXN2ZEpBQWxMR3hUR2hRVzRMeStxdTltbTZDRmg0Q2NFTXNZT1FLQmdCa0pEWTRyd0x1V3pucndWbkZGazR3NkwvSGdua1RlSSszeGF4bW5KZGVkSXpnb2FzQTdqNDNreXNXU2ttRmRBTmo2MXJ4YytGd09ycTVFczhnbVhhUkZlaUZMK2pna1JWZS93aU1hbHMwZ3E5RmVINis0Zk50b0NhSFhWREN3MG9QZnpETjFpYW1Uc2EzVTZLN0FIbTZJSW1aV0FGcDUzbm9VaUpjOTRSQlhBb0dCQUxMcmQrbnljNzVEbjZ0ZWdVT0JQRy9HR3FYWkZGdmd6emJRRnR0SmR0NTlMZ0NBanUxMjJMRExsRjZreTB6Q0JsK0h6Q0kyUXJXVEp4cXUxSTZvbndaZDlOMllKVlNWQlFOV285NTJUamRkOHcvSkExYlNqK3JpTU1WQ0txT1BNRHNiOENEaGlEbVM3RlBEM0trTVlmcUxJOVhNRTVvNVUvNFJzcEZUZWozRQotLS0tLUVORCBQUklWQVRFIEtFWS0tLS0t + backup: + archive: + execute: + enabled: false + mariadb: + dataSourceMode: standalone +bk-api-gateway: + gse: + url: gse.apigw.com + cmdb: + url: cmdb.apigw.com +esb: + service: + url: esb.service diff --git a/src/backend/job-backup/boot-job-backup/src/test/resources/bootstrap.yml b/src/backend/job-backup/boot-job-backup/src/test/resources/bootstrap.yml index a3c2520b1d..83fc50e136 100644 --- a/src/backend/job-backup/boot-job-backup/src/test/resources/bootstrap.yml +++ b/src/backend/job-backup/boot-job-backup/src/test/resources/bootstrap.yml @@ -1,3 +1,6 @@ spring: application: name: job-backup + cloud: + kubernetes: + enabled: false diff --git a/src/backend/job-backup/boot-job-backup/src/test/resources/init_archive_progress_data.sql b/src/backend/job-backup/boot-job-backup/src/test/resources/init_archive_progress_data.sql deleted file mode 100644 index 398ff483b2..0000000000 --- a/src/backend/job-backup/boot-job-backup/src/test/resources/init_archive_progress_data.sql +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -truncate table db_archive_progress; - -insert into job_backup.db_archive_progress (table_name,progress,last_modify_time) values ('task_instance','{\"lastArchivedId\":1000}',1621166442000); -insert into job_backup.db_archive_progress (table_name,progress,last_modify_time) values ('step_instance','{\"lastArchivedId\":2000}',1621166442000); -insert into job_backup.db_archive_progress (table_name,progress,last_modify_time) values ('gse_task_log','{\"lastArchivedId\":2000}',1621166442000); -insert into job_backup.db_archive_progress (table_name,progress,last_modify_time) values ('gse_task_ip_log','{\"lastArchivedId\":2000}',1621166442000); -insert into job_backup.db_archive_progress (table_name,progress,last_modify_time) values ('task_instance_variable','{\"lastArchivedId\":1000}',1621166442000); diff --git a/src/backend/job-backup/boot-job-backup/src/test/resources/init_schema.sql b/src/backend/job-backup/boot-job-backup/src/test/resources/init_schema.sql index 9d09d2c136..75e800baa5 100644 --- a/src/backend/job-backup/boot-job-backup/src/test/resources/init_schema.sql +++ b/src/backend/job-backup/boot-job-backup/src/test/resources/init_schema.sql @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -23,52 +23,31 @@ */ SET NAMES utf8mb4; -CREATE SCHEMA IF NOT EXISTS job_execute; CREATE SCHEMA IF NOT EXISTS job_backup; +USE job_backup; -CREATE TABLE IF NOT EXISTS `job_execute.task_instance` -( - `id` bigint(20) NOT NULL AUTO_INCREMENT, - `app_id` bigint(20) NOT NULL, - `task_id` bigint(20) NOT NULL, - `task_template_id` bigint(20) NOT NULL, - `name` varchar(512) NOT NULL, - `type` tinyint(4) NOT NULL, - `operator` varchar(128) NOT NULL, - `status` tinyint(4) NOT NULL DEFAULT '0', - `current_step_id` bigint(20) NOT NULL DEFAULT '0', - `startup_mode` tinyint(4) NOT NULL, - `total_time` bigint(20) DEFAULT NULL, - `callback_url` varchar(1024) DEFAULT NULL, - `is_debug_task` tinyint(4) NOT NULL DEFAULT '0', - `cron_task_id` bigint(20) NOT NULL DEFAULT '0', - `create_time` bigint(20) DEFAULT NULL, - `start_time` bigint(20) DEFAULT NULL, - `end_time` bigint(20) DEFAULT NULL, - `app_code` varchar(128) DEFAULT NULL, - `row_create_time` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP, - `row_update_time` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`), - KEY (`app_id`), - KEY (`operator`), - KEY (`task_id`), - KEY (`status`), - KEY (`create_time`) -) ENGINE = InnoDB - AUTO_INCREMENT = 1000000 - DEFAULT CHARSET = utf8mb4; - - USE `job_backup`; - -SET NAMES utf8mb4; - -CREATE TABLE `job_backup.db_archive_progress` ( +CREATE TABLE IF NOT EXISTS `archive_task` ( `row_create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, `row_update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - `table_name` varchar(256) NOT NULL, - `progress` text, - `last_modify_time` BIGINT(20) NOT NULL DEFAULT '0', - PRIMARY KEY (`table_name`) + `id` bigint(20) NOT NULL AUTO_INCREMENT, + `task_type` tinyint(2) DEFAULT NULL, + `data_node` varchar(128) NOT NULL, + `db_node` varchar(64) NOT NULL, + `day` int(8) DEFAULT NULL, + `hour` tinyint(2) DEFAULT NULL, + `from_timestamp` bigint(20) NOT NULL, + `to_timestamp` bigint(20) NOT NULL, + `process` varchar(256) DEFAULT NULL, + `status` tinyint(2) NOT NULL DEFAULT '0', + `create_time` bigint(20) NOT NULL, + `last_update_time` bigint(20) NOT NULL, + `task_start_time` bigint(20) DEFAULT NULL, + `task_end_time` bigint(20) DEFAULT NULL, + `task_cost` bigint(20) DEFAULT NULL, + `detail` text, + PRIMARY KEY (`id`), + UNIQUE KEY (`task_type`,`data_node`,`day`,`hour`), + KEY (`task_type`,`status`,`db_node`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; diff --git a/src/backend/job-backup/boot-job-backup/src/test/resources/logback-spring.xml b/src/backend/job-backup/boot-job-backup/src/test/resources/logback-spring.xml index b76b072601..e31db6870e 100644 --- a/src/backend/job-backup/boot-job-backup/src/test/resources/logback-spring.xml +++ b/src/backend/job-backup/boot-job-backup/src/test/resources/logback-spring.xml @@ -2,7 +2,7 @@ - + diff --git a/src/backend/job-backup/boot-job-backup/src/test/resources/test.properties b/src/backend/job-backup/boot-job-backup/src/test/resources/test.properties index 67dd196b2a..dd072144a2 100644 --- a/src/backend/job-backup/boot-job-backup/src/test/resources/test.properties +++ b/src/backend/job-backup/boot-job-backup/src/test/resources/test.properties @@ -1,28 +1,4 @@ -# -# Tencent is pleased to support the open source community by making BK-JOB����������ҵƽ̨ available. -# -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. -# -# BK-JOB����������ҵƽ̨ is licensed under the MIT License. -# -# License for BK-JOB����������ҵƽ̨: -# -------------------------------------------------------------------- -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF -# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - -job.backup.db.url=jdbc:h2:mem:job-backup;MODE=MYSQL;INIT=RUNSCRIPT FROM 'classpath:init_schema.sql' +job.backup.db.url=jdbc:h2:mem:job-execute;MODE=MYSQL;INIT=RUNSCRIPT FROM 'classpath:init_schema.sql' job.backup.db.username=job job.backup.db.password=job_db_password job.execute.db.dr.url=jdbc:h2:mem:job-execute;MODE=MYSQL;INIT=RUNSCRIPT FROM 'classpath:init_schema.sql' diff --git a/src/backend/job-backup/build.gradle b/src/backend/job-backup/build.gradle index 2111bda806..dc6c9028c4 100644 --- a/src/backend/job-backup/build.gradle +++ b/src/backend/job-backup/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-backup/model-job-backup/build.gradle b/src/backend/job-backup/model-job-backup/build.gradle index 4e297ad7b9..8006176375 100644 --- a/src/backend/job-backup/model-job-backup/build.gradle +++ b/src/backend/job-backup/model-job-backup/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,4 +27,4 @@ dependencies { implementation "org.jooq:jooq-codegen" implementation "org.jooq:jooq-meta" } -apply from: "task_gen_jooq.gradle" +apply from: "$rootDir/task_gen_jooq.gradle" diff --git a/src/backend/job-backup/model-job-backup/task_gen_jooq.gradle b/src/backend/job-backup/model-job-backup/task_gen_jooq.gradle deleted file mode 100644 index 0ed6aefc8a..0000000000 --- a/src/backend/job-backup/model-job-backup/task_gen_jooq.gradle +++ /dev/null @@ -1,96 +0,0 @@ -apply plugin: 'nu.studer.jooq' - -dependencies { - api "org.jooq:jooq" - jooqRuntime "mysql:mysql-connector-java" -} - -def nameArr = name.split('-') -def moduleName = '' -def simpleModuleName = '' -for (int i = 1; i < nameArr.size(); i++) { - moduleName += nameArr[i] + "_" - if (i > 1) { - simpleModuleName += nameArr[i] + "_" - } -} -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -//去除末尾下划线 -moduleName = moduleName.substring(0, moduleName.length() - 1) -simpleModuleName = simpleModuleName.substring(0, simpleModuleName.length() - 1) -println("moduleName=" + moduleName) -println("simpleModuleName=" + simpleModuleName) -def databaseName = moduleName -def targetPackageName = "com.tencent.bk.job." + simpleModuleName + ".model" - -jooq { - genenrate(sourceSets.main) { - - jdbc { - driver = 'com.mysql.cj.jdbc.Driver' - def mysqlURL = System.getProperty("mysqlURL") - def mysqlUser = System.getProperty("mysqlUser") - def mysqlPasswd = System.getProperty("mysqlPasswd") - - if (mysqlURL == null) { - mysqlURL = System.getenv("mysqlURL") - mysqlUser = System.getenv("mysqlUser") - mysqlPasswd = System.getenv("mysqlPasswd") - } - - if (mysqlURL == null) { - println "use default mysql database." - mysqlURL = DB_HOST - mysqlUser = DB_USERNAME - mysqlPasswd = DB_PASSWORD - } - - url = "jdbc:mysql://${mysqlURL}/${databaseName}?useSSL=false&serverTimezone=UTC" - user = mysqlUser - password = mysqlPasswd - } - - generator { - name = 'org.jooq.codegen.DefaultGenerator' - database { - name = 'org.jooq.meta.mysql.MySQLDatabase' - inputSchema = "${databaseName}" - } - - generate { - relations = false - deprecated = false - fluentSetters = true - generatedAnnotation = false - javaTimeTypes = true - } - - target { - packageName = "${targetPackageName}" - } - } - } -} diff --git a/src/backend/job-backup/service-job-backup/build.gradle b/src/backend/job-backup/service-job-backup/build.gradle index d14fc476c5..353b8fb9f9 100644 --- a/src/backend/job-backup/service-job-backup/build.gradle +++ b/src/backend/job-backup/service-job-backup/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,16 +28,22 @@ dependencies { api project(":job-backup:model-job-backup") api project(":job-manage:api-job-manage") api project(":job-execute:model-job-execute") + api project(":job-logsvr:api-job-logsvr") api project(":commons:artifactory-sdk") api project(":commons:common-web") api project(":commons:common-security") api project(":commons:common-redis") api project(":commons:common") + api project(":commons:common-crypto") + api project(":commons:common-mysql-sharding") + api project(":commons:common-mongodb") implementation("org.springframework.boot:spring-boot-starter-web") implementation("org.springframework.boot:spring-boot-starter-jdbc") implementation("org.springframework.boot:spring-boot-starter-jooq") + implementation("org.springframework.boot:spring-boot-starter-data-mongodb") implementation "org.apache.commons:commons-collections4" + implementation "commons-io:commons-io" implementation "ch.qos.logback:logback-core" implementation "ch.qos.logback:logback-classic" implementation "org.slf4j:slf4j-api" diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/api/inner/impl/ServiceArchiveResourceImpl.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/api/inner/impl/ServiceArchiveResourceImpl.java deleted file mode 100644 index 2b78021fe0..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/api/inner/impl/ServiceArchiveResourceImpl.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.api.inner.impl; - -import com.tencent.bk.job.backup.api.inner.ServiceArchiveResource; -import com.tencent.bk.job.backup.archive.JobExecuteArchiveManage; -import com.tencent.bk.job.backup.config.ArchiveConfig; -import com.tencent.bk.job.backup.model.inner.ServiceArchiveDBRequest; -import com.tencent.bk.job.common.model.InternalResponse; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.RestController; - -@RestController -@Slf4j -public class ServiceArchiveResourceImpl implements ServiceArchiveResource { - - private final JobExecuteArchiveManage jobExecuteArchiveManage; - - public ServiceArchiveResourceImpl(@Autowired(required = false) JobExecuteArchiveManage jobExecuteArchiveManage) { - this.jobExecuteArchiveManage = jobExecuteArchiveManage; - } - - @Override - public InternalResponse archive(ServiceArchiveDBRequest request) { - log.info("Begin archive db, request: {}", request); - ArchiveConfig archiveConfig = new ArchiveConfig(); - archiveConfig.setDataKeepDays(request.getDataKeepDays()); - archiveConfig.setDeleteEnabled(request.isDeleteEnabled()); - archiveConfig.setArchiveEnabled(request.isArchiveEnabled()); - jobExecuteArchiveManage.archive(archiveConfig); - return InternalResponse.buildSuccessResp(null); - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/api/web/impl/WebBackupResourceImpl.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/api/web/impl/WebBackupResourceImpl.java index b6a50b78b2..e1dec2382c 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/api/web/impl/WebBackupResourceImpl.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/api/web/impl/WebBackupResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -54,6 +54,7 @@ import com.tencent.bk.job.common.model.Response; import com.tencent.bk.job.common.model.dto.AppResourceScope; import com.tencent.bk.job.common.redis.util.LockUtils; +import com.tencent.bk.job.common.util.FilePathUtils; import com.tencent.bk.job.common.util.JobContextUtil; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; @@ -134,7 +135,7 @@ public Response startExport(String username, } else { exportJobInfoDTO.setExpireTime(0L); } - exportJobInfoDTO.setTemplateInfo(exportRequest.getTemplateInfo().parallelStream() + exportJobInfoDTO.setTemplateInfo(exportRequest.getTemplateInfo().stream() .map(BackupTemplateInfoDTO::fromVO).collect(Collectors.toList())); String id = exportJobService.startExport(exportJobInfoDTO); @@ -221,7 +222,7 @@ public ResponseEntity getExportFile(String username, String jobId) { ExportJobInfoDTO exportInfo = exportJobService.getExportInfo(appResourceScope.getAppId(), jobId); if (exportInfo != null) { - if (BackupJobStatusEnum.SUCCESS.equals(exportInfo.getStatus())) { + if (BackupJobStatusEnum.ALL_SUCCESS.equals(exportInfo.getStatus())) { Pair fileInfoPair; switch (backupStorageConfig.getStorageBackend()) { case JobConstants.FILE_STORAGE_BACKEND_ARTIFACTORY: @@ -260,7 +261,7 @@ public Response completeExport(String username, String jobId) { ExportJobInfoDTO exportInfo = exportJobService.getExportInfo(appResourceScope.getAppId(), jobId); if (exportInfo != null) { - if (BackupJobStatusEnum.SUCCESS.equals(exportInfo.getStatus())) { + if (BackupJobStatusEnum.ALL_SUCCESS.equals(exportInfo.getStatus())) { exportInfo.setStatus(BackupJobStatusEnum.FINISHED); exportInfo.setFileName(null); return Response.buildSuccessResp(exportJobService.updateExportJob(exportInfo)); @@ -296,7 +297,7 @@ public Response getImportFileInfo(String username, throw new InternalException("No File", ErrorCode.INTERNAL_ERROR); } Long appId = appResourceScope.getAppId(); - String originalFileName = uploadFile.getOriginalFilename(); + String originalFileName = FilePathUtils.getPureFileName(uploadFile.getOriginalFilename()); if (originalFileName != null && originalFileName.endsWith(Constant.JOB_EXPORT_FILE_SUFFIX)) { String id = UUID.randomUUID().toString(); String fileName = importJobService.saveFile(username, appId, id, uploadFile); @@ -417,11 +418,11 @@ public Response getCurrentJob(String username, BackupJobInfoVO backupJobInfo = new BackupJobInfoVO(); if (CollectionUtils.isNotEmpty(exportJobInfoList)) { backupJobInfo.setExportJob( - exportJobInfoList.parallelStream().map(ExportJobInfoDTO::toVO).collect(Collectors.toList())); + exportJobInfoList.stream().map(ExportJobInfoDTO::toVO).collect(Collectors.toList())); } if (CollectionUtils.isNotEmpty(importJobInfoList)) { backupJobInfo.setImportJob( - importJobInfoList.parallelStream().map(ImportJobInfoDTO::toVO).collect(Collectors.toList())); + importJobInfoList.stream().map(ImportJobInfoDTO::toVO).collect(Collectors.toList())); } return Response.buildSuccessResp(backupJobInfo); } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/api/web/impl/WebVersionResourceImpl.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/api/web/impl/WebVersionResourceImpl.java deleted file mode 100644 index 9134aea636..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/api/web/impl/WebVersionResourceImpl.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.api.web.impl; - -import com.tencent.bk.job.common.web.controller.WebVersionResource; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.actuate.autoconfigure.endpoint.condition.ConditionalOnAvailableEndpoint; -import org.springframework.boot.actuate.info.InfoEndpoint; -import org.springframework.web.bind.annotation.RestController; - -import java.util.Map; - -/** - * @since 17/7/2020 13:47 - */ -@Slf4j -@RestController -@ConditionalOnAvailableEndpoint(endpoint = InfoEndpoint.class) -public class WebVersionResourceImpl implements WebVersionResource { - - private final InfoEndpoint infoEndpoint; - - @Autowired - public WebVersionResourceImpl(InfoEndpoint infoEndpoint) { - this.infoEndpoint = infoEndpoint; - } - - @Override - public Map getVersion() { - return infoEndpoint.info(); - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbnormalArchiveTaskReScheduler.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbnormalArchiveTaskReScheduler.java new file mode 100644 index 0000000000..9264f88f43 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbnormalArchiveTaskReScheduler.java @@ -0,0 +1,158 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +import com.tencent.bk.job.backup.archive.model.ArchiveTaskInfo; +import com.tencent.bk.job.backup.archive.service.ArchiveTaskService; +import com.tencent.bk.job.backup.archive.util.lock.FailedArchiveTaskRescheduleLock; +import com.tencent.bk.job.backup.constant.ArchiveTaskStatusEnum; +import com.tencent.bk.job.backup.constant.ArchiveTaskTypeEnum; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; + +import java.util.List; + +/** + * 异常归档任务重调度 + */ +@Slf4j +public class AbnormalArchiveTaskReScheduler { + + /** + * 归档任务超时时间，用于判定归档任务是否调度异常 + */ + private static final long TIMEOUT_MILLS = 86400000L; + + private final ArchiveTaskService archiveTaskService; + + private final FailedArchiveTaskRescheduleLock failedArchiveTaskRescheduleLock; + + + public AbnormalArchiveTaskReScheduler(ArchiveTaskService archiveTaskService, + FailedArchiveTaskRescheduleLock failedArchiveTaskRescheduleLock) { + this.archiveTaskService = archiveTaskService; + this.failedArchiveTaskRescheduleLock = failedArchiveTaskRescheduleLock; + } + + /** + * 重新调度异常任务（执行失败、超时未结束） + */ + public void rescheduleFailedArchiveTasks() { + boolean locked = false; + try { + log.info("Abnormal archive task reSchedule start ..."); + locked = failedArchiveTaskRescheduleLock.lock(); + if (!locked) { + log.info("Get failed archive reSchedule lock fail"); + return; + } + // 处理失败的任务 + reScheduleFailedTasks(); + // 处理超时未结束的任务 + reScheduleTimeoutTasks(); + // 试运行的任务 + reScheduleDryRunTasks(); + } finally { + if (locked) { + failedArchiveTaskRescheduleLock.unlock(); + } + log.info("Abnormal archive task reSchedule end"); + } + } + + private void reScheduleFailedTasks() { + int readLimit = 100; + List failedTasks; + do { + failedTasks = + archiveTaskService.listTasks(ArchiveTaskStatusEnum.FAIL, readLimit); + + if (CollectionUtils.isEmpty(failedTasks)) { + return; + } + // 设置为 pending 状态，会被重新调度 + failedTasks.forEach(failTask -> { + log.info("Found fail archive task, and set archive task status to pending, taskId : {}", + failTask.buildTaskUniqueId()); + archiveTaskService.updateArchiveTaskStatus( + failTask.getTaskType(), + failTask.getDbDataNode(), + failTask.getDay(), + failTask.getHour(), + ArchiveTaskStatusEnum.PENDING + ); + }); + } while (failedTasks.size() == readLimit); + } + + private void reScheduleTimeoutTasks() { + List runningTasks = + archiveTaskService.listRunningTasks(ArchiveTaskTypeEnum.JOB_INSTANCE); + if (CollectionUtils.isEmpty(runningTasks)) { + return; + } + long currentTime = System.currentTimeMillis(); + runningTasks.forEach(runningTask -> { + // 如果归档任务没有正常结束，通过当前时间减去任务最后修改时间计算执行时长，判断是否超过合理的执行时长 + if (currentTime - runningTask.getLastUpdateTime() > TIMEOUT_MILLS) { + log.info("Found timeout archive task, and set archive task status to pending. taskId: {}", + runningTask.buildTaskUniqueId()); + // 设置为 pending 状态，会被重新调度 + archiveTaskService.updateArchiveTaskStatus( + runningTask.getTaskType(), + runningTask.getDbDataNode(), + runningTask.getDay(), + runningTask.getHour(), + ArchiveTaskStatusEnum.PENDING + ); + } + }); + } + + private void reScheduleDryRunTasks() { + int readLimit = 100; + List dryRunTasks; + do { + dryRunTasks = + archiveTaskService.listTasks(ArchiveTaskStatusEnum.DRYRUN, readLimit); + + if (CollectionUtils.isEmpty(dryRunTasks)) { + return; + } + // 设置为pending状态，会被重新调度 + dryRunTasks.forEach(taskInfo -> { + log.info("Found dry run archive task, and set archive task status to pending, taskId : {}", + taskInfo.buildTaskUniqueId()); + archiveTaskService.updateArchiveTaskStatus( + taskInfo.getTaskType(), + taskInfo.getDbDataNode(), + taskInfo.getDay(), + taskInfo.getHour(), + ArchiveTaskStatusEnum.PENDING + ); + }); + } while (dryRunTasks.size() == readLimit); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractArchiveTaskScheduler.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractArchiveTaskScheduler.java new file mode 100644 index 0000000000..a73a2fc8a1 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractArchiveTaskScheduler.java @@ -0,0 +1,357 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +import com.tencent.bk.job.backup.archive.model.ArchiveTaskInfo; +import com.tencent.bk.job.backup.archive.service.ArchiveTaskService; +import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; +import com.tencent.bk.job.common.util.ThreadUtils; +import com.tencent.bk.job.common.util.json.JsonUtils; +import lombok.extern.slf4j.Slf4j; +import org.springframework.cloud.sleuth.Tracer; +import org.springframework.context.SmartLifecycle; +import org.springframework.util.StopWatch; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; + +/** + * 通用归档任务调度抽象基类 + */ +@Slf4j +public abstract class AbstractArchiveTaskScheduler implements SmartLifecycle { + + protected final ArchiveTaskService archiveTaskService; + protected final ExecutorService archiveTaskStopExecutor; + protected final ArchiveErrorTaskCounter archiveErrorTaskCounter; + protected final Tracer tracer; + + private final Map scheduledTasks = new ConcurrentHashMap<>(); + private final Object lifecycleMonitor = new Object(); + /** + * 调度器线程挂起 object monitor + */ + private final Object schedulerHangMonitor = new Object(); + + /** + * 作业执行日志归档任务调度组件是否处于活动状态 + */ + private volatile boolean active; + /** + * 是否正在进行任务调度中 + */ + private volatile boolean scheduling = false; + + public AbstractArchiveTaskScheduler(ArchiveTaskService archiveTaskService, + ExecutorService archiveTaskStopExecutor, + ArchiveErrorTaskCounter archiveErrorTaskCounter, + Tracer tracer) { + this.archiveTaskService = archiveTaskService; + this.archiveTaskStopExecutor = archiveTaskStopExecutor; + this.archiveErrorTaskCounter = archiveErrorTaskCounter; + this.tracer = tracer; + } + + /** + * Spring Bean 生命周期管理-启动 + */ + @Override + public void start() { + if (isRunning()) { + return; + } + synchronized (lifecycleMonitor) { + this.active = true; + } + } + + /** + * Spring Bean 生命周期管理-停止 + */ + @Override + public void stop() { + log.info("JobInstanceArchiveTaskScheduler stopping."); + synchronized (this.lifecycleMonitor) { + if (!isActive()) { + log.info("Shutdown ignored - JobInstanceArchiveTaskScheduler is not active already"); + return; + } + this.active = false; + } + synchronized (schedulerHangMonitor) { + schedulerHangMonitor.notify(); + log.info("Try notify scheduler when stopping"); + } + stopTasksGraceful(); + log.info("JobInstanceArchiveTaskScheduler stop successfully!"); + } + + @Override + public boolean isRunning() { + synchronized (this.lifecycleMonitor) { + return (this.active); + } + } + + /** + * 判断是否处于激活状态 + */ + private boolean isActive() { + synchronized (this.lifecycleMonitor) { + return this.active; + } + } + + /** + * 判断是否正在进行任务调度中 + */ + private boolean isScheduling() { + synchronized (this.lifecycleMonitor) { + return this.scheduling; + } + } + + public void schedule() { + try { + if (isScheduling()) { + log.info("{} is working", getRuntimeClassName()); + return; + } + this.scheduling = true; + + while (true) { + if (!isActive()) { + log.info("{} is not active, skip", getRuntimeClassName()); + return; + } + StopWatch watch = new StopWatch("archive-task-schedule-" + getRuntimeClassName()); + boolean locked = false; + try { + // 1.获取归档任务调度锁 + locked = acquireScheduleLock(); + if (!locked) { + log.info("{} get lock fail, wait 1s", getRuntimeClassName()); + ThreadUtils.sleep(1000L); + continue; + } + + // 2.获取待调度的任务信息(按照DB节点计数)，如果所有任务都已经被调度完成，退出本次任务调度 + watch.start("countScheduleTasks"); + Map scheduleTasksGroupByDb = countScheduleTasksGroupByDb(); + if (scheduleTasksGroupByDb.isEmpty()) { + log.info("{} no archive task need scheduling! Exit", + getRuntimeClassName()); + return; + } + watch.stop(); + log.info("{} count archive task group by db, result: {}", + getRuntimeClassName(), scheduleTasksGroupByDb); + + // 3.获取正在执行中的任务列表 + watch.start("queryRunningTasks"); + List runningTasks = listRunningTasks(); + watch.stop(); + + // 4. 是否需要等待 + if (shouldWait(watch, runningTasks, scheduleTasksGroupByDb)) { + // 休眠1分钟，等待并行任务减少 + log.info("{} running archive task count exceed concurrent limit : {}, " + + "wait 60s", getRuntimeClassName(), getTaskMaxConcurrent()); + // 释放锁 + releaseScheduleLock(); + locked = false; + synchronized (schedulerHangMonitor) { + schedulerHangMonitor.wait(1000 * 60L); + } + continue; + } + + // 5.获取优先级最高的归档任务 + watch.start("getNextTask"); + ArchiveTaskInfo next = getNextTask(scheduleTasksGroupByDb, runningTasks); + watch.stop(); + + // 6. 启动归档任务 + watch.start("startTask"); + startArchiveTask(next); + watch.stop(); + + if (watch.getTotalTimeMillis() > 1000) { + log.info("{} scheduling slow for {}, times: {}", + getRuntimeClassName(), watch.prettyPrint(), watch.getTotalTimeMillis()); + } + } finally { + if (locked) { + releaseScheduleLock(); + } + if (watch.isRunning()) { + watch.stop(); + } + if (watch.getTotalTimeMillis() > 1000) { + log.info("{} schedule archive task slow, cost statistics: {}", + getRuntimeClassName(), watch.prettyPrint()); + } + } + } + } catch (Throwable t) { + log.error("{} schedule caught exception", getRuntimeClassName(), t); + } finally { + this.scheduling = false; + } + } + + private void startArchiveTask(ArchiveTaskInfo archiveTaskInfo) { + log.info("Start {}, taskId: {}, taskInfo: {}", getRuntimeClassName(), + archiveTaskInfo.buildTaskUniqueId(), JsonUtils.toJson(archiveTaskInfo)); + T archiveTask = createArchiveTask(archiveTaskInfo); + + // 注册任务完成回调函数 + archiveTask.registerDoneCallback(() -> { + synchronized (lifecycleMonitor) { + scheduledTasks.remove(archiveTask.getTaskId()); + } + }); + synchronized (lifecycleMonitor) { + if (!isActive()) { + log.info("{} is not active, skip", getRuntimeClassName()); + return; + } + scheduledTasks.put(archiveTask.getTaskId(), archiveTask); + } + ArchiveTaskWorker worker = new ArchiveTaskWorker(archiveTask, tracer); + worker.start(); + log.info("Started data archive task {} for {}", + archiveTaskInfo.buildTaskUniqueId(), getRuntimeClassName()); + } + + private void stopTasksGraceful() { + log.info("Stop archive tasks graceful - start"); + long start = System.currentTimeMillis(); + TaskCountDownLatch taskCountDownLatch = null; + synchronized (lifecycleMonitor) { + if (!this.scheduledTasks.isEmpty()) { + log.info("Stop archive tasks, size: {}, tasks: {}", scheduledTasks.size(), scheduledTasks); + taskCountDownLatch = new TaskCountDownLatch(scheduledTasks.keySet()); + } + for (JobHistoricalDataArchiveTask task : scheduledTasks.values()) { + log.info("Submit stop archive task to executor, taskId: {}", task.getTaskId()); + archiveTaskStopExecutor.execute(new StopTask(task, taskCountDownLatch)); + } + } + try { + if (taskCountDownLatch != null) { + // 等待任务结束，最多等待 30s(等待时间太长进程会被k8s kill掉) + boolean isAllTaskStopped = taskCountDownLatch.waitingForAllTasksDone(30); + if (!isAllTaskStopped) { + for (JobHistoricalDataArchiveTask task : scheduledTasks.values()) { + task.forceStopAtOnce(); + } + } + } + } catch (Throwable e) { + log.error("Stop archive tasks caught exception", e); + } + long end = System.currentTimeMillis(); + log.info("Stop archive tasks graceful - end, cost: {}", end - start); + } + + private static final class StopTask implements Runnable { + private final JobHistoricalDataArchiveTask task; + private final TaskCountDownLatch taskCountDownLatch; + + StopTask(JobHistoricalDataArchiveTask task, TaskCountDownLatch taskCountDownLatch) { + this.task = task; + this.taskCountDownLatch = taskCountDownLatch; + } + + @Override + public void run() { + try { + log.info("[{}] Run stop task begin", task.getTaskId()); + task.stop(() -> taskCountDownLatch.decrement(task.getTaskId())); + } catch (Throwable e) { + String errorMsg = "Stop archive task caught exception, task: " + task.getTaskId(); + log.warn(errorMsg, e); + } finally { + log.info("[{}] Run stop task end", task.getTaskId()); + } + } + } + + /** + * 是否需要等待，比如达到了最大归档任务数 + */ + protected boolean shouldWait(StopWatch watch, + List runningTasks, + Map scheduleTasksGroupByDb) { + int taskConcurrent = getTaskMaxConcurrent(); + return runningTasks.size() >= taskConcurrent; + } + + /** + * 获取运行时的类名 + */ + protected String getRuntimeClassName() { + return this.getClass().getSimpleName(); + } + + /** + * 获取调度锁 + */ + protected abstract boolean acquireScheduleLock(); + + /** + * 释放调度锁 + */ + protected abstract void releaseScheduleLock(); + + /** + * 获取待调度的任务信息(按照DB节点计数) + */ + protected abstract Map countScheduleTasksGroupByDb(); + + /** + * 获取正在运行的归档任务列表 + */ + protected abstract List listRunningTasks(); + + /** + * 获取任务的最大并发数 + */ + protected abstract Integer getTaskMaxConcurrent(); + + /** + * 获取优先级最高的归档任务 + */ + protected abstract ArchiveTaskInfo getNextTask(Map scheduleTasksGroupByDb, + List running); + + /** + * 创建归档任务对象 + */ + protected abstract T createArchiveTask(ArchiveTaskInfo archiveTaskInfo); +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractArchivist.java deleted file mode 100644 index 64c99339c6..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractArchivist.java +++ /dev/null @@ -1,336 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive; - -import com.tencent.bk.job.backup.config.ArchiveConfig; -import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.ExecuteRecordDAO; -import com.tencent.bk.job.backup.model.dto.ArchiveProgressDTO; -import com.tencent.bk.job.backup.model.dto.ArchiveSummary; -import com.tencent.bk.job.backup.service.ArchiveProgressService; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; -import org.jooq.TableRecord; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.CountDownLatch; - -/** - * 表归档基础实现 - * - * @param 表记录 - */ -@Data -@Slf4j -public abstract class AbstractArchivist> { - protected ExecuteRecordDAO executeRecordDAO; - protected ExecuteArchiveDAO executeArchiveDAO; - protected ArchiveProgressService archiveProgressService; - /** - * 读取DB 步长 - */ - protected int readIdStepSize = 1_000; - /** - * 写入归档数据，单批次最小行数 - */ - protected int batchInsertRow = 1_000; - /** - * 删除数据ID增加步长 - */ - protected int deleteIdStepSize = 10_000; - protected String tableName; - private ArchiveSummary archiveSummary; - - public AbstractArchivist(ExecuteRecordDAO executeRecordDAO, - ExecuteArchiveDAO executeArchiveDAO, - ArchiveProgressService archiveProgressService) { - this.executeRecordDAO = executeRecordDAO; - this.executeArchiveDAO = executeArchiveDAO; - this.archiveProgressService = archiveProgressService; - this.tableName = executeRecordDAO.getTable().getName().toLowerCase(); - } - - public void archive(ArchiveConfig archiveConfig, Long maxNeedArchiveId, CountDownLatch countDownLatch) { - boolean isAcquireLock = false; - try { - archiveSummary = new ArchiveSummary(); - archiveSummary.setTableName(tableName); - if (!ArchiveTaskLock.getInstance().lock(tableName)) { - archiveSummary.setSkip(true); - isAcquireLock = false; - return; - } else { - isAcquireLock = true; - } - archiveSummary.setSkip(false); - log.info("Start archive and delete, tableName: {}, archiveConfig: {}", tableName, archiveConfig); - - boolean archiveSuccess; - if (archiveConfig.isArchiveEnabled()) { - archiveSuccess = archive(maxNeedArchiveId); - } else { - log.info("Archive is not enabled, skip archive {}!", tableName); - archiveSuccess = true; - } - - if (archiveSuccess && archiveConfig.isDeleteEnabled()) { - delete(maxNeedArchiveId, archiveConfig); - } - } catch (Throwable e) { - log.error("Error while archiving {}", tableName, e); - } finally { - archiveSummary.setArchiveEnabled(archiveConfig.isArchiveEnabled()); - archiveSummary.setDeleteEnabled(archiveConfig.isDeleteEnabled()); - storeArchiveSummary(); - if (isAcquireLock) { - ArchiveTaskLock.getInstance().unlock(tableName); - } - countDownLatch.countDown(); - } - } - - private boolean archive(Long maxNeedArchiveId) { - long lastArchivedId = 0; - long archivedRows = 0; - long readRows = 0; - long startTime = System.currentTimeMillis(); - long minNeedArchiveId = 0; - long archiveCost = 0; - try { - lastArchivedId = getLastArchivedId(); - minNeedArchiveId = lastArchivedId; - log.info("Archive {} start, minNeedArchiveId: {}, maxNeedArchiveId:{}", - tableName, minNeedArchiveId, maxNeedArchiveId); - - long start = lastArchivedId; - long stop = start; - List recordList = new ArrayList<>(readIdStepSize); - - if (maxNeedArchiveId <= lastArchivedId) { - log.info("LastArchivedId {} is greater than or equal to maxNeedArchiveId {}, skip archive {}!", - lastArchivedId, maxNeedArchiveId, tableName); - } - - while (maxNeedArchiveId > start) { - stop = start + readIdStepSize; - if (stop > maxNeedArchiveId) { - stop = maxNeedArchiveId; - } - // 选取start records = listRecord(start, stop); - readRows += records.size(); - log.info("Read {} rows from {}", records.size(), tableName); - - if (CollectionUtils.isEmpty(records)) { - if (stop >= maxNeedArchiveId) { - log.info("Read {} finished!", tableName); - break; - } else { - if (log.isDebugEnabled()) { - log.debug("Fetching {} found data hole at {} {}", tableName, start, readIdStepSize); - } - } - } else { - recordList.addAll(records); - } - - start = stop; - if (recordList.size() >= batchInsertRow) { - int insertRows = insertAndReset(stop, recordList); - lastArchivedId = stop; - archivedRows += insertRows; - updateArchiveProgress(lastArchivedId); - } - } - if (CollectionUtils.isNotEmpty(recordList)) { - int insertRows = insertAndReset(stop, recordList); - lastArchivedId = stop; - archivedRows += insertRows; - updateArchiveProgress(lastArchivedId); - } - - // 即使没有需要归档的数据，仍然需要更新归档进度 - if (readRows == 0) { - lastArchivedId = stop; - updateArchiveProgress(lastArchivedId); - } - - archiveCost = System.currentTimeMillis() - startTime; - log.info("Archive {} finished, minNeedArchiveId: {}, maxNeedArchiveId: {}, lastArchivedId: {}, readRows: " + - "{}, archivedRows: {}, cost: {}ms", - tableName, minNeedArchiveId, maxNeedArchiveId, lastArchivedId, readRows, - archivedRows, archiveCost); - if (readRows != archivedRows) { - log.warn("Archive row are unexpected, table: {}, expected: {}, actual: {}", tableName, readRows, - archivedRows); - } - return true; - } catch (Throwable e) { - log.error("Error while archiving {}", tableName, e); - return false; - } finally { - archiveSummary.setArchiveIdStart(minNeedArchiveId); - archiveSummary.setArchiveIdEnd(maxNeedArchiveId); - archiveSummary.setNeedArchiveRecordSize(readRows); - archiveSummary.setArchivedRecordSize(archivedRows); - archiveSummary.setLastArchivedId(lastArchivedId); - archiveSummary.setArchiveCost(archiveCost); - } - } - - /** - * 获取上一次已归档完成的最后一个数据Id，后续用大于该Id选取范围数据 - */ - private long getLastArchivedId() { - ArchiveProgressDTO archiveProgress = archiveProgressService.queryArchiveProgress(tableName); - if (archiveProgress == null || archiveProgress.getLastArchivedId() == null - || archiveProgress.getLastArchivedId() <= 0L) { - long lastArchivedId = getFirstInstanceId() - 1; - log.info("Archive {} for the first time! lastArchivedId: {}", tableName, lastArchivedId); - return lastArchivedId; - } - return archiveProgress.getLastArchivedId(); - } - - private long getLastDeletedId() { - ArchiveProgressDTO archiveProgress = archiveProgressService.queryArchiveProgress(tableName); - if (archiveProgress == null || archiveProgress.getLastDeletedId() == null - || archiveProgress.getLastDeletedId() <= 0L) { - long lastDeletedId = getFirstInstanceId() - 1; - log.info("Delete {} for the first time! lastDeletedId: {}", tableName, lastDeletedId); - return lastDeletedId; - } - return archiveProgress.getLastDeletedId(); - } - - private void updateArchiveProgress(long lastArchivedId) { - ArchiveProgressDTO archiveProgress = new ArchiveProgressDTO(); - archiveProgress.setTableName(tableName); - archiveProgress.setLastArchiveTime(System.currentTimeMillis()); - archiveProgress.setLastArchivedId(lastArchivedId); - archiveProgressService.saveArchiveProgress(archiveProgress); - } - - private void updateDeleteProgress(long lastDeletedId) { - ArchiveProgressDTO archiveProgress = new ArchiveProgressDTO(); - archiveProgress.setTableName(tableName); - archiveProgress.setLastDeleteTime(System.currentTimeMillis()); - archiveProgress.setLastDeletedId(lastDeletedId); - archiveProgressService.saveDeleteProgress(archiveProgress); - } - - private void delete(Long maxNeedArchiveId, ArchiveConfig archiveConfig) { - long startTime = System.currentTimeMillis(); - - Long maxNeedDeleteId; - if (archiveConfig.isArchiveEnabled()) { - // 如果开启归档，那么删除的最大ID必须等于已经归档的ID，保证数据在删除之前已经被正确归档 - maxNeedDeleteId = getLastArchivedId(); - } else { - maxNeedDeleteId = maxNeedArchiveId; - } - - long lastDeletedId = getLastDeletedId(); - long minNeedDeleteId = lastDeletedId; - long deletedRows = 0; - - try { - log.info("Delete {} start|{}|{}", tableName, minNeedDeleteId, maxNeedDeleteId); - if (maxNeedDeleteId <= lastDeletedId) { - log.info("LastDeletedId {} is greater than or equal to maxNeedDeleteId {}, skip delete {}!", - lastDeletedId, maxNeedDeleteId, tableName); - return; - } - - long start = minNeedDeleteId; - while (maxNeedDeleteId > start) { - long batchDeleteStartTime = System.currentTimeMillis(); - long stop = start + deleteIdStepSize; - if (stop > maxNeedDeleteId) { - stop = maxNeedDeleteId; - } - int deleteCount = deleteRecord(start, stop); - lastDeletedId = stop; - deletedRows += deleteCount; - start += deleteIdStepSize; - log.info("Delete {}, lastDeletedId: {}, delete rows: {}, cost: {}ms", tableName, - lastDeletedId, deleteCount, System.currentTimeMillis() - batchDeleteStartTime); - updateDeleteProgress(lastDeletedId); - } - log.info("Delete {} finished, minNeedDeleteId: {}, maxNeedDeleteId: {}, lastDeletedId: {}, deletedRows: " + - "{}, cost: {}ms", - tableName, minNeedDeleteId, maxNeedDeleteId, lastDeletedId, deletedRows, - System.currentTimeMillis() - startTime); - } catch (Throwable e) { - log.error("Error while deleting {}", tableName, e); - } finally { - archiveSummary.setDeleteCost(System.currentTimeMillis() - startTime); - archiveSummary.setDeleteIdStart(minNeedDeleteId); - archiveSummary.setDeleteIdEnd(maxNeedDeleteId); - archiveSummary.setLastDeletedId(lastDeletedId); - archiveSummary.setDeleteRecordSize(deletedRows); - } - } - - private int insertAndReset(long lastArchivedInstanceId, List recordList) throws IOException { - if (CollectionUtils.isEmpty(recordList)) { - return 0; - } - int recordSize = recordList.size(); - int insertRows = batchInsert(recordList); - recordList.clear(); - boolean insertError = false; - if (insertRows != recordSize) { - insertError = true; - } - log.info("Batch insert {}, lastArchivedInstanceId: {}, maxBatchSize: {}, insertError: {}, expected insert " + - "rows: {}, actual insert rows: {}", - tableName, lastArchivedInstanceId, batchInsertRow, insertError, recordSize, insertRows); - return insertRows; - } - - private void storeArchiveSummary() { - ArchiveSummaryHolder.getInstance().addArchiveSummary(this.archiveSummary); - } - - private List listRecord(Long start, Long stop) { - return executeRecordDAO.listRecords(start, stop); - } - - private int batchInsert(List recordList) throws IOException { - return executeArchiveDAO.batchInsert(recordList, 1000); - } - - private int deleteRecord(Long start, Long stop) { - return executeRecordDAO.deleteRecords(start, stop); - } - - private long getFirstInstanceId() { - return executeRecordDAO.getFirstArchiveId(); - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractHistoricalDataArchiveTask.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractHistoricalDataArchiveTask.java new file mode 100644 index 0000000000..9573a12da8 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractHistoricalDataArchiveTask.java @@ -0,0 +1,318 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +import com.tencent.bk.job.backup.archive.model.ArchiveTaskContext; +import com.tencent.bk.job.backup.archive.model.ArchiveTaskExecutionDetail; +import com.tencent.bk.job.backup.archive.model.ArchiveTaskInfo; +import com.tencent.bk.job.backup.archive.model.IdBasedArchiveProcess; +import com.tencent.bk.job.backup.archive.service.ArchiveTaskService; +import com.tencent.bk.job.backup.constant.ArchiveTaskStatusEnum; +import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; +import com.tencent.bk.job.common.util.StringUtil; +import com.tencent.bk.job.common.util.json.JsonUtils; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.helpers.MessageFormatter; + +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * 作业执行历史归档任务的抽象基类，定义了归档任务的公共方法、生命周期 + * 子类必须实现{@link #backupAndDelete()}, {@link #acquireLock()}和{@link #unlock()} 方法 + */ +@Slf4j +public abstract class AbstractHistoricalDataArchiveTask implements JobHistoricalDataArchiveTask{ + protected final ArchiveErrorTaskCounter archiveErrorTaskCounter; + protected final ArchiveTaskService archiveTaskService; + protected final String taskId; + protected ArchiveTaskInfo archiveTaskInfo; + + /** + * 归档进度 + */ + protected final IdBasedArchiveProcess progress; + + protected boolean isAcquireLock; + /** + * 任务终止标识 + */ + protected volatile boolean stopFlag = false; + /** + * 任务是否已停止 + */ + protected volatile boolean isStopped = false; + /** + * 同步锁 + */ + private final Object stopMonitor = new Object(); + protected ArchiveTaskStopCallback stopCallback = null; + private volatile ArchiveTaskDoneCallback archiveTaskDoneCallback; + /** + * 任务标识-是否已被任务调度强制终止 + */ + protected final AtomicBoolean forceStoppedByScheduler = new AtomicBoolean(false); + /** + * 当前归档线程 + */ + protected ArchiveTaskWorker archiveTaskWorker; + + + protected AbstractHistoricalDataArchiveTask(ArchiveErrorTaskCounter archiveErrorTaskCounter, + ArchiveTaskInfo archiveTaskInfo, + ArchiveTaskService archiveTaskService) { + this.archiveErrorTaskCounter = archiveErrorTaskCounter; + this.archiveTaskInfo = archiveTaskInfo; + this.archiveTaskService = archiveTaskService; + this.taskId = archiveTaskInfo.buildTaskUniqueId(); + this.progress = archiveTaskInfo.getProcess(); + } + + @Override + public void execute() { + archive(); + } + + protected void archive() { + try { + // 设置归档任务上下文 + ArchiveTaskContextHolder.set(new ArchiveTaskContext(archiveTaskInfo)); + // 获取分布式锁 + if (!acquireLock()) { + return; + } + log.info("{} [{}] Start archive task", getRuntimeClassName(), taskId); + // 更新任务信息 - 启动完成 + updateStartedExecuteInfo(); + // 归档 + backupAndDelete(); + } catch (Throwable e) { + String msg = MessageFormatter.format( + "{} [{}] Error while execute archive task", + getRuntimeClassName(), + taskId + ).getMessage(); + log.error(msg, e); + archiveErrorTaskCounter.increment(); + + // 更新归档任务状态 + setArchiveTaskExecutionDetail(null, null, e.getMessage()); + updateCompletedExecuteInfo(ArchiveTaskStatusEnum.FAIL, null); + } finally { + if (this.isAcquireLock) { + unlock(); + } + log.info( + "{} [{}] Archive finished, result: {}", + getRuntimeClassName(), + taskId, + JsonUtils.toJson(archiveTaskInfo) + ); + if (archiveTaskDoneCallback != null) { + archiveTaskDoneCallback.callback(); + } + if (checkStopFlag()) { + stopTask(); + } + ArchiveTaskContextHolder.unset(); + } + } + + @Override + public void stop(ArchiveTaskStopCallback callback) { + synchronized (this) { + log.info("{} [{}] Set stop flag to true", getRuntimeClassName(), taskId); + this.stopFlag = true; + this.stopCallback = callback; + } + } + + protected boolean checkStopFlag() { + synchronized (stopMonitor) { + return stopFlag; + } + } + + protected void stopTask() { + log.info("{} [{}] Try to stop archive task", getRuntimeClassName(), taskId); + synchronized (stopMonitor) { + if (!isStopped) { + isStopped = true; + // 更新归档任务状态为暂停，用于后续调度 + updateArchiveTaskSuspended(); + if (stopCallback != null) { + stopCallback.callback(); + } + log.info("{} [{}] Stop archive task successfully", getRuntimeClassName(), taskId); + } else { + log.info("{} [{}] Archive task is already stopped", getRuntimeClassName(), taskId); + } + } + } + + private void updateArchiveTaskSuspended() { + archiveTaskInfo.setStatus(ArchiveTaskStatusEnum.SUSPENDED); + archiveTaskService.updateArchiveTaskStatus( + archiveTaskInfo.getTaskType(), + archiveTaskInfo.getDbDataNode(), + archiveTaskInfo.getDay(), + archiveTaskInfo.getHour(), + ArchiveTaskStatusEnum.SUSPENDED + ); + log.info("{} [{}] Set archive task status suspended", getRuntimeClassName(), taskId); + } + + @Override + public void forceStopAtOnce() { + log.info("{} Force stop archive task at once. taskId: {}", getRuntimeClassName(), taskId); + forceStoppedByScheduler.set(true); + // 更新归档任务状态为“暂停” + updateArchiveTaskSuspended(); + // 打断当前线程，退出执行 + archiveTaskWorker.interrupt(); + } + + @Override + public void registerDoneCallback(ArchiveTaskDoneCallback archiveTaskDoneCallback) { + this.archiveTaskDoneCallback = archiveTaskDoneCallback; + } + + @Override + public String getTaskId() { + return this.taskId; + } + + @Override + public void initArchiveTaskWorker(ArchiveTaskWorker archiveTaskWorker) { + this.archiveTaskWorker = archiveTaskWorker; + } + + protected void updateStartedExecuteInfo() { + if (!checkUpdateEnabled()) { + return; + } + Long startTime = System.currentTimeMillis(); + archiveTaskInfo.setTaskStartTime(startTime); + archiveTaskService.updateStartedExecuteInfo( + archiveTaskInfo.getTaskType(), + archiveTaskInfo.getDbDataNode(), + archiveTaskInfo.getDay(), + archiveTaskInfo.getHour(), + startTime + ); + // 如果该任务是失败任务重新调度的，需要重置执行详情错误信息 + resetIfIsReScheduleAbnormalTask(); + } + + private void resetIfIsReScheduleAbnormalTask() { + if (archiveTaskInfo.getDetail() != null + && StringUtils.isNotEmpty(archiveTaskInfo.getDetail().getErrorMsg())) { + archiveTaskInfo.getDetail().setErrorMsg(null); + archiveTaskService.updateExecutionDetail( + archiveTaskInfo.getTaskType(), + archiveTaskInfo.getDbDataNode(), + archiveTaskInfo.getDay(), + archiveTaskInfo.getHour(), + archiveTaskInfo.getDetail() + ); + } + } + + protected void updateCompletedExecuteInfo(ArchiveTaskStatusEnum status, + IdBasedArchiveProcess process) { + archiveTaskInfo.setStatus(status); + if (process != null) { + archiveTaskInfo.setProcess(process); + } + archiveTaskInfo.setTaskEndTime(System.currentTimeMillis()); + archiveTaskInfo.setTaskCost(archiveTaskInfo.getTaskEndTime() - archiveTaskInfo.getTaskStartTime()); + + if (!checkUpdateEnabled()) { + return; + } + + log.info("{} [{}] Update archive task completed execute info, status: {}, process: {}", + getRuntimeClassName(), taskId, status, process); + archiveTaskService.updateCompletedExecuteInfo( + archiveTaskInfo.getTaskType(), + archiveTaskInfo.getDbDataNode(), + archiveTaskInfo.getDay(), + archiveTaskInfo.getHour(), + archiveTaskInfo.getStatus(), + archiveTaskInfo.getProcess(), + archiveTaskInfo.getTaskEndTime(), + archiveTaskInfo.getTaskCost(), + archiveTaskInfo.getDetail() + ); + } + + protected void setArchiveTaskExecutionDetail(Long archiveRecordSize, + Long costTime, + String errorMsg) { + ArchiveTaskExecutionDetail executionDetail = archiveTaskInfo.getOrInitExecutionDetail(); + if (archiveRecordSize != null) { + executionDetail.setArchivedRecordSize(archiveRecordSize); + } + if (costTime != null) { + executionDetail.setCostTime(costTime); + } + if (StringUtils.isNotEmpty(errorMsg)) { + executionDetail.setErrorMsg(StringUtil.substring(errorMsg, 10240)); + } + } + + protected boolean checkUpdateEnabled() { + if (forceStoppedByScheduler.get()) { + // 如果已经被归档任务调度强制终止了，就不能再去更新 db，引起数据不一致 + log.info("{} [{}] Archive task is force stopped by scheduler, do not update archive task again", + getRuntimeClassName(), taskId); + return false; + } else { + return true; + } + } + + /** + * 获取运行时的类名 + */ + protected String getRuntimeClassName() { + return this.getClass().getSimpleName(); + } + + /** + * 归档操作 + */ + protected abstract void backupAndDelete(); + + /** + * 获取锁 + */ + protected abstract boolean acquireLock(); + + /** + * 释放锁 + */ + protected abstract void unlock(); +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobInstanceArchiveTask.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobInstanceArchiveTask.java new file mode 100644 index 0000000000..7625b40c86 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobInstanceArchiveTask.java @@ -0,0 +1,282 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.impl.AbstractJobInstanceMainHotRecordDAO; +import com.tencent.bk.job.backup.archive.model.ArchiveTaskInfo; +import com.tencent.bk.job.backup.archive.model.BackupResult; +import com.tencent.bk.job.backup.archive.model.DeleteResult; +import com.tencent.bk.job.backup.archive.model.IdBasedArchiveProcess; +import com.tencent.bk.job.backup.archive.model.TablesBackupResult; +import com.tencent.bk.job.backup.archive.model.TablesDeleteResult; +import com.tencent.bk.job.backup.archive.service.ArchiveTaskService; +import com.tencent.bk.job.backup.archive.util.lock.ArchiveTaskExecuteLock; +import com.tencent.bk.job.backup.config.ArchiveProperties; +import com.tencent.bk.job.backup.constant.ArchiveModeEnum; +import com.tencent.bk.job.backup.constant.ArchiveTaskStatusEnum; +import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.jooq.TableRecord; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * 作业实例数据归档任务基础实现 + * + * @param 表记录 + */ +@Slf4j +public abstract class AbstractJobInstanceArchiveTask> + extends AbstractHistoricalDataArchiveTask { + + /** + * 作业实例主表 DAO + */ + protected AbstractJobInstanceMainHotRecordDAO jobInstanceMainRecordDAO; + /** + * 冷 DB DAO + */ + protected JobInstanceColdDAO jobInstanceColdDAO; + + private final ArchiveTaskExecuteLock archiveTaskExecuteLock; + protected final ArchiveTablePropsStorage archiveTablePropsStorage; + private final ArchiveProperties archiveProperties; + + public AbstractJobInstanceArchiveTask(AbstractJobInstanceMainHotRecordDAO jobInstanceMainRecordDAO, + JobInstanceColdDAO jobInstanceColdDAO, + ArchiveProperties archiveProperties, + ArchiveTaskExecuteLock archiveTaskExecuteLock, + ArchiveErrorTaskCounter archiveErrorTaskCounter, + ArchiveTaskInfo archiveTaskInfo, + ArchiveTaskService archiveTaskService, + ArchiveTablePropsStorage archiveTablePropsStorage) { + super(archiveErrorTaskCounter, archiveTaskInfo, archiveTaskService); + this.jobInstanceMainRecordDAO = jobInstanceMainRecordDAO; + this.jobInstanceColdDAO = jobInstanceColdDAO; + this.archiveTaskExecuteLock = archiveTaskExecuteLock; + this.archiveTablePropsStorage = archiveTablePropsStorage; + this.archiveProperties = archiveProperties; + } + + @Override + public void backupAndDelete() { + boolean backupEnabled = isBackupEnable(); + boolean deleteEnabled = isDeleteEnable(); + // 获取主表对应的 readRowLimit + int readLimit = archiveTablePropsStorage.getReadRowLimit(jobInstanceMainRecordDAO.getTable().getName()); + long archivedJobInstanceCount = 0; + + long startTime = System.currentTimeMillis(); + log.info("[{}] Archive task mode: {}, backupEnabled: {}, deleteEnabled: {}", + taskId, archiveProperties.getMode(), backupEnabled, deleteEnabled); + try { + List jobInstanceRecords; + do { + // 检查任务终止标识 + if (checkStopFlag()) { + stopTask(); + } + + jobInstanceRecords = readJobInstanceRecords(readLimit); + if (CollectionUtils.isEmpty(jobInstanceRecords)) { + long archiveCost = System.currentTimeMillis() - startTime; + setArchiveTaskExecutionDetail(archivedJobInstanceCount, archiveCost, null); + updateCompletedExecuteInfo(ArchiveTaskStatusEnum.SUCCESS, null); + return; + } + archivedJobInstanceCount += jobInstanceRecords.size(); + + List jobInstanceIds = + jobInstanceRecords.stream().map(this::extractJobInstanceId).collect(Collectors.toList()); + + TablesBackupResult tablesBackupResult = null; + TablesDeleteResult tablesDeleteResult = null; + // 写入数据到冷 db + if (backupEnabled) { + long backupStartTime = System.currentTimeMillis(); + tablesBackupResult = backupJobInstanceToColdDb(jobInstanceRecords); + log.info("[{}] Backup to cold db, jobInstanceRecordSize: {}, cost: {}", + taskId, jobInstanceRecords.size(), System.currentTimeMillis() - backupStartTime); + } + // 从热 db 删除数据 + if (deleteEnabled) { + long deleteStartTime = System.currentTimeMillis(); + tablesDeleteResult = deleteJobInstanceHotData(jobInstanceIds); + log.info("[{}] Delete hot db, jobInstanceRecordSize: {}, cost: {}", + taskId, jobInstanceRecords.size(), System.currentTimeMillis() - deleteStartTime); + } + + checkBackupDeleteDataQuantity(tablesBackupResult, tablesDeleteResult); + + // 更新归档进度 + T lastRecord = jobInstanceRecords.get(jobInstanceRecords.size() - 1); + Long lastJobInstanceId = extractJobInstanceId(lastRecord); + IdBasedArchiveProcess progress = new IdBasedArchiveProcess(lastJobInstanceId); + boolean isFinished = jobInstanceRecords.size() < readLimit; + if (isFinished) { + // 更新任务结束信息 + long archiveCost = System.currentTimeMillis() - startTime; + setArchiveTaskExecutionDetail(archivedJobInstanceCount, archiveCost, null); + updateCompletedExecuteInfo(ArchiveTaskStatusEnum.SUCCESS, progress); + } else { + // 更新任务运行信息 + updateRunningExecuteInfo(progress); + } + } while (jobInstanceRecords.size() == readLimit); + } finally { + long archiveCost = System.currentTimeMillis() - startTime; + setArchiveTaskExecutionDetail(archivedJobInstanceCount, archiveCost, null); + } + } + + /** + * 检查备份与删除的数据数量 + */ + private void checkBackupDeleteDataQuantity(TablesBackupResult tablesBackupResult, + TablesDeleteResult tablesDeleteResult) { + if (tablesBackupResult == null || tablesDeleteResult == null) { + // 无需比较 + return; + } + if (!tablesBackupResult.getTables().keySet().equals(tablesDeleteResult.getTables().keySet())) { + log.error("Backup tables are not equals delete tables, backupTables: {}, deleteTables: {}", + tablesBackupResult.getTables().keySet(), tablesDeleteResult.getTables().keySet()); + throw new ArchiveException("Backup and delete table count not match"); + } + boolean isBackupDeleteRowsMatch = true; + for (Map.Entry entry : tablesBackupResult.getTables().entrySet()) { + String tableName = entry.getKey(); + BackupResult backupResult = entry.getValue(); + DeleteResult deleteResult = tablesDeleteResult.getTables().get(tableName); + if (backupResult == BackupResult.NON_OP_BACKUP_RESULT) { + // 无需归档的表，无需比较 + continue; + } + if (backupResult.getBackupRows() != deleteResult.getDeletedRows()) { + log.error("Backup rows and delete row not match, table: {}, backupRows: {}, deleteRows: {}", + tableName, backupResult.getBackupRows(), deleteResult.getDeletedRows()); + isBackupDeleteRowsMatch = false; + } + } + if (!isBackupDeleteRowsMatch) { + throw new ArchiveException("Backup and delete row count not match"); + } + } + + private List readJobInstanceRecords(int readLimit) { + List jobInstanceRecords; + long readStartTime = System.currentTimeMillis(); + + Long fromTaskInstanceId = progress != null ? progress.getId() : null; + jobInstanceRecords = jobInstanceMainRecordDAO.readSortedJobInstanceFromHotDB( + archiveTaskInfo.getFromTimestamp(), + archiveTaskInfo.getToTimestamp(), + fromTaskInstanceId, + readLimit); + long cost = System.currentTimeMillis() - readStartTime; + log.info("[{}] Read sorted job instance from hot db, fromJobCreateTime: {}, toJobCreatTime: {}, " + + "fromJobInstanceId: {}, recordSize: {}, cost: {} ms", + taskId, + archiveTaskInfo.getFromTimestamp(), + archiveTaskInfo.getToTimestamp(), + fromTaskInstanceId, + jobInstanceRecords.size(), + cost + ); + if (cost > 1000L) { + log.info("[{}] SlowQuery-ReadJobInstanceRecords, cost: {}ms", taskId, cost); + } + return jobInstanceRecords; + } + + /** + * 备份作业实例数据到冷存储 + * + * @param jobInstances 作业实例列表 + */ + protected abstract TablesBackupResult backupJobInstanceToColdDb(List jobInstances); + + /** + * 删除作业实例热数据 + * + * @param jobInstanceIds 作业实例 ID 列表 + */ + protected abstract TablesDeleteResult deleteJobInstanceHotData(List jobInstanceIds); + + @Override + public boolean acquireLock() { + this.isAcquireLock = archiveTaskExecuteLock.lock(taskId); + if (!isAcquireLock) { + log.info("{} [{}] Acquire archive task lock fail", getRuntimeClassName(), taskId); + } + return isAcquireLock; + } + + @Override + protected void unlock() { + archiveTaskExecuteLock.unlock(taskId); + } + + private void updateRunningExecuteInfo(IdBasedArchiveProcess process) { + archiveTaskInfo.setProcess(process); + + if (!checkUpdateEnabled()) { + return; + } + + archiveTaskService.updateRunningExecuteInfo( + archiveTaskInfo.getTaskType(), + archiveTaskInfo.getDbDataNode(), + archiveTaskInfo.getDay(), + archiveTaskInfo.getHour(), + process + ); + } + + /** + * 从作业实例记录中提取作业实例 ID + * + * @param record 作业实例记录 + */ + protected Long extractJobInstanceId(T record) { + return record.get(jobInstanceMainRecordDAO.getJobInstanceIdField()); + } + + protected boolean isBackupEnable() { + ArchiveModeEnum archiveMode = ArchiveModeEnum.valOf(archiveProperties.getMode()); + return archiveProperties.isEnabled() + && (ArchiveModeEnum.BACKUP_THEN_DELETE == archiveMode || ArchiveModeEnum.BACKUP_ONLY == archiveMode); + } + + protected boolean isDeleteEnable() { + ArchiveModeEnum archiveMode = ArchiveModeEnum.valOf(archiveProperties.getMode()); + return archiveProperties.isEnabled() + && (ArchiveModeEnum.BACKUP_THEN_DELETE == archiveMode || ArchiveModeEnum.DELETE_ONLY == archiveMode); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobLogArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobLogArchiver.java new file mode 100644 index 0000000000..1f0de0453c --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/AbstractJobLogArchiver.java @@ -0,0 +1,132 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +import com.tencent.bk.job.backup.archive.model.ArchiveTaskInfo; +import com.tencent.bk.job.backup.archive.model.BackupResult; +import com.tencent.bk.job.backup.archive.model.DeleteResult; +import com.tencent.bk.job.backup.archive.service.ArchiveTaskService; +import com.tencent.bk.job.backup.config.JobLogArchiveProperties; +import com.tencent.bk.job.backup.constant.ArchiveModeEnum; +import com.tencent.bk.job.backup.constant.ArchiveTaskStatusEnum; +import com.tencent.bk.job.backup.constant.ArchiveTaskTypeEnum; +import com.tencent.bk.job.logsvr.consts.LogTypeEnum; +import com.tencent.bk.job.logsvr.util.CollectionNameUtil; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.mongodb.core.MongoTemplate; + +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.util.List; +import java.util.stream.Collectors; + +/** + * 执行日志归档基础实现 + */ +@Slf4j +public abstract class AbstractJobLogArchiver implements JobLogArchiver { + private final JobLogArchiveProperties archiveProperties; + private final ArchiveTaskService archiveTaskService; + private final LogTypeEnum logTypeEnum; + private final MongoTemplate mongoTemplate; + private static final String ARCHIVE_DAY_FORMATTER = "yyyyMMdd"; + private static final String COLLECTION_NAME_DATE_FORMATTER = "yyyy_MM_dd"; + private static final DateTimeFormatter INPUT_DATE_FORMATTER = DateTimeFormatter.ofPattern(ARCHIVE_DAY_FORMATTER); + private static final DateTimeFormatter OUTPUT_DATE_FORMATTER = + DateTimeFormatter.ofPattern(COLLECTION_NAME_DATE_FORMATTER); + + public AbstractJobLogArchiver(MongoTemplate mongoTemplate, + ArchiveTaskService archiveTaskService, + JobLogArchiveProperties archiveProperties, + LogTypeEnum logTypeEnum) { + this.mongoTemplate = mongoTemplate; + this.archiveTaskService = archiveTaskService; + this.archiveProperties = archiveProperties; + this.logTypeEnum = logTypeEnum; + } + + @Override + public DeleteResult deleteRecords(Integer archiveDay) { + // 作业执行日志按集合删除，当前仅支持删除 + if (!isDeleteEnable()) { + log.info("Delete job execute log is disabled, skip delete, mode={}", + archiveProperties.getMode()); + return DeleteResult.NON_OP_DELETE_RESULT; + } + String collectionName = getCollectionName(archiveDay); + if (archiveProperties.isDryRun()) { + log.info("Dry-run mode is enabled, skipping the actual operation of drop the collection [{}]", + collectionName); + return DeleteResult.NON_OP_DELETE_RESULT; + } + + List archiveTaskInfoList = archiveTaskService.listTasks(ArchiveTaskTypeEnum.JOB_INSTANCE, + archiveDay); + if (archiveTaskInfoList.isEmpty()) { + log.warn("Job instance has not been deleted yet, " + + "execution log will not be found in the details of the job execution history. " + + "date={}, dropCollection={}", archiveDay, collectionName); + } + List unfinishedTaskList = archiveTaskInfoList.stream() + .filter(taskInfo -> taskInfo.getStatus().getStatus() != ArchiveTaskStatusEnum.SUCCESS.getStatus()) + .collect(Collectors.toList()); + if (!unfinishedTaskList.isEmpty()) { + log.warn("Job instance archive task is not fully completed, " + + "execution log will not be found in the details of the job execution history. " + + "date={}, dropCollection={}", archiveDay, collectionName); + } + long deleteStartTime = System.currentTimeMillis(); + log.info("Drop [{}] collection", collectionName); + mongoTemplate.dropCollection(collectionName); + long deleteCost = System.currentTimeMillis() - deleteStartTime; + ArchiveTaskContextHolder.get().accumulateTableDelete( + collectionName, + 0, + deleteCost + ); + return new DeleteResult(-1, deleteCost); + } + + @Override + public BackupResult backupRecords(Integer archiveDay) { + // 暂不支持执行日志备份 + log.info("[{}] Log archiving is not supported", archiveDay); + return BackupResult.NON_OP_BACKUP_RESULT; + } + + private String getCollectionName(Integer day) { + String prefix = CollectionNameUtil.buildCollectionNamePrefix(logTypeEnum); + String formattedDate = LocalDate.parse( + String.valueOf(day), + INPUT_DATE_FORMATTER + ).format(OUTPUT_DATE_FORMATTER); + return prefix + formattedDate; + } + + private boolean isDeleteEnable() { + return archiveProperties.isEnabled() + && ArchiveModeEnum.DELETE_ONLY == ArchiveModeEnum.valOf(archiveProperties.getMode()); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveCronJobs.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveCronJobs.java new file mode 100644 index 0000000000..73e6e5da1d --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveCronJobs.java @@ -0,0 +1,124 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +import com.tencent.bk.job.backup.config.ArchiveProperties; +import com.tencent.bk.job.backup.config.JobLogArchiveProperties; +import lombok.extern.slf4j.Slf4j; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.scheduling.annotation.Scheduled; + +/** + * 归档定时任务 + */ +@EnableScheduling +@Slf4j +public class ArchiveCronJobs { + + private final JobInstanceArchiveTaskGenerator jobInstanceArchiveTaskGenerator; + + private final JobLogArchiveTaskGenerator jobLogArchiveTaskGenerator; + + private final JobInstanceArchiveTaskScheduler jobInstanceArchiveTaskScheduler; + + private final JobLogArchiveTaskScheduler jobLogArchiveTaskScheduler; + + private final ArchiveProperties archiveProperties; + + private final JobLogArchiveProperties jobLogArchiveProperties; + + private final AbnormalArchiveTaskReScheduler abnormalArchiveTaskReScheduler; + + public ArchiveCronJobs(JobInstanceArchiveTaskGenerator jobInstanceArchiveTaskGenerator, + JobLogArchiveTaskGenerator jobLogArchiveTaskGenerator, + JobInstanceArchiveTaskScheduler jobInstanceArchiveTaskScheduler, + JobLogArchiveTaskScheduler jobLogArchiveTaskScheduler, + ArchiveProperties archiveProperties, + JobLogArchiveProperties jobLogArchiveProperties, + AbnormalArchiveTaskReScheduler abnormalArchiveTaskReScheduler) { + this.jobInstanceArchiveTaskGenerator = jobInstanceArchiveTaskGenerator; + this.jobLogArchiveTaskGenerator = jobLogArchiveTaskGenerator; + this.jobInstanceArchiveTaskScheduler = jobInstanceArchiveTaskScheduler; + this.jobLogArchiveTaskScheduler = jobLogArchiveTaskScheduler; + this.archiveProperties = archiveProperties; + this.jobLogArchiveProperties = jobLogArchiveProperties; + this.abnormalArchiveTaskReScheduler = abnormalArchiveTaskReScheduler; + } + + /** + * 定时创建归档任务,每小时 0 分钟 触发一次（正常情况一天触发一次即可；为了保障异常情况下任务有一定频率的重试机会） + */ + @Scheduled(cron = "0 0 * * * *") + public void generateArchiveTask() { + if (archiveProperties.isEnabled()) { + log.info("Generate historical data archive task start..."); + jobInstanceArchiveTaskGenerator.generate(); + log.info("Generate historical data archive task done"); + } + + if (jobLogArchiveProperties.isEnabled()) { + log.info("Generate historical log archive task start..."); + jobLogArchiveTaskGenerator.generate(); + log.info("Generate historical log archive task done"); + } + } + + /** + * 定时调度并执行历史数据归档任务，默认每小时第 1 分钟触发一次 + */ + @Scheduled(cron = "${job.backup.archive.execute.cron: 0 1 * * * *}") + public void runHistoricalDataArchive() { + if (archiveProperties.isEnabled()) { + log.info("Schedule and execute historical data archive task start..."); + jobInstanceArchiveTaskScheduler.schedule(); + log.info("Schedule and execute historical data archive task done"); + } + } + + /** + * 定时调度并执行执行日志归档任务，默认每小时第 1 分钟触发一次 + */ + @Scheduled(cron = "${job.backup.archive.execute-log.cron: 0 1 * * * *}") + public void runJobLogArchive() { + if (jobLogArchiveProperties.isEnabled()) { + log.info("Schedule and execute historical log archive task start..."); + jobLogArchiveTaskScheduler.schedule(); + log.info("Schedule and execute historical log archive task done"); + } + } + + /** + * 失败归档任务重调度，每小时触发一次 + */ + @Scheduled(cron = "0 59 * * * *") + public void scheduleFailedTasks() { + if (!archiveProperties.isEnabled() && !jobLogArchiveProperties.isEnabled()) { + return; + } + log.info("ReSchedule fail/timout/dryrun archive task start..."); + abnormalArchiveTaskReScheduler.rescheduleFailedArchiveTasks(); + log.info("ReSchedule fail/timeout/dryrun archive task done"); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveDbNodePriorityEvaluator.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveDbNodePriorityEvaluator.java new file mode 100644 index 0000000000..50fc32b803 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveDbNodePriorityEvaluator.java @@ -0,0 +1,120 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +import com.tencent.bk.job.backup.archive.model.ArchiveTaskInfo; +import lombok.Data; +import org.apache.commons.collections4.CollectionUtils; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * 归档任务调度,DB 节点优先级计算，保证归档任务均衡分布在每个 db 节点上 + */ +public class ArchiveDbNodePriorityEvaluator { + + /** + * 归档任务调度优先级计算 + * + * @param runningTasks 正在执行中的归档任务 + * @param scheduleTaskCountGroupByDb 待调度归档任务计数（根据 db) + * @return DbNodeTasksInfo + */ + public static DbNodeTasksInfo evaluateHighestPriorityDbNode(List runningTasks, + Map scheduleTaskCountGroupByDb) { + Map runningTaskCountGroupByDb = groupRunningTaskCountByDb(runningTasks); + + List dbNodeTasksInfos = new ArrayList<>(); + scheduleTaskCountGroupByDb.keySet().forEach(dbNodeId -> + dbNodeTasksInfos.add(new DbNodeTasksInfo( + dbNodeId, + runningTaskCountGroupByDb.get(dbNodeId), + scheduleTaskCountGroupByDb.get(dbNodeId) + ))); + // 优先级排序 + sortDbNodeTaskCounters(dbNodeTasksInfos); + return dbNodeTasksInfos.get(0); + } + + @Data + public static class DbNodeTasksInfo { + private String dbNodeId; + private int runningTaskCount; + private int scheduleTaskCount; + + public DbNodeTasksInfo(String dbNodeId, Integer runningTaskCount, Integer scheduleTaskCount) { + this.dbNodeId = dbNodeId; + this.runningTaskCount = runningTaskCount == null ? 0 : runningTaskCount; + this.scheduleTaskCount = scheduleTaskCount == null ? 0 : scheduleTaskCount; + } + } + + private static Map groupRunningTaskCountByDb(List runningTasks) { + Map dbTaskCountMap = new HashMap<>(); + if (CollectionUtils.isEmpty(runningTasks)) { + return dbTaskCountMap; + } + + runningTasks.forEach(task -> { + String dbNodeId = task.getDbDataNode().toDbNodeId(); + dbTaskCountMap.compute(dbNodeId, (k, v) -> { + if (v == null) { + v = 1; + } else { + v++; + } + return v; + }); + }); + return dbTaskCountMap; + } + + private static void sortDbNodeTaskCounters(List dbNodeTasksInfos) { + dbNodeTasksInfos.sort((node1, node2) -> { + // 先比较每个 db 上正在运行的任务数量，任务数量越少优先级越高 + int node1RunningTaskCount = node1.getRunningTaskCount(); + int node2RunningTaskCount = node2.getRunningTaskCount(); + if (node1RunningTaskCount < node2RunningTaskCount) { + return -1; + } else if (node1RunningTaskCount > node2RunningTaskCount) { + return 1; + } else { + // 再比较每个 db 上的待调度任务的数量，待调度任务数量越多优先级越高 + int node1ScheduleTaskCount = node1.getScheduleTaskCount(); + int node2ScheduleTaskCount = node2.getScheduleTaskCount(); + if (node1ScheduleTaskCount < node2ScheduleTaskCount) { + return 1; + } else if (node1ScheduleTaskCount > node2ScheduleTaskCount) { + return -1; + } else { + return 0; + } + } + }); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveException.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveException.java new file mode 100644 index 0000000000..67205ed3bd --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveException.java @@ -0,0 +1,42 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +import com.tencent.bk.job.common.exception.InternalException; + +/** + * 数据归档异常 + */ +public class ArchiveException extends InternalException { + + public ArchiveException(String message) { + super(message, (Throwable) null); + } + + public ArchiveException(String message, Throwable throwable) { + super(message, throwable); + } + +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveSummaryHolder.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveSummaryHolder.java deleted file mode 100644 index dbb563bb7d..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveSummaryHolder.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive; - -import com.tencent.bk.job.backup.model.dto.ArchiveSummary; -import com.tencent.bk.job.common.util.date.DateUtils; -import com.tencent.bk.job.common.util.json.JsonUtils; -import lombok.extern.slf4j.Slf4j; - -import java.time.temporal.ChronoUnit; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -@Slf4j -public class ArchiveSummaryHolder { - private Map summaryMap = new ConcurrentHashMap<>(); - private Long endTimeInMills; - - private ArchiveSummaryHolder() { - } - - public static ArchiveSummaryHolder getInstance() { - return Inner.instance; - } - - public void init(Long endTimeInMills) { - this.summaryMap.clear(); - this.endTimeInMills = endTimeInMills; - } - - public void addArchiveSummary(ArchiveSummary summary) { - if (summary == null) { - return; - } - summary.setArchiveEndDate(DateUtils.formatUnixTimestamp(endTimeInMills, ChronoUnit.MILLIS)); - summaryMap.put(summary.getTableName(), summary); - } - - public void print() { - log.info("Archive summary : {}", JsonUtils.toJson(summaryMap.values())); - } - - private static class Inner { - private static final ArchiveSummaryHolder instance = new ArchiveSummaryHolder(); - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTablePropsStorage.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTablePropsStorage.java new file mode 100644 index 0000000000..657a0ad47d --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTablePropsStorage.java @@ -0,0 +1,95 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +import com.tencent.bk.job.backup.archive.model.ArchiveTableProps; +import com.tencent.bk.job.backup.config.ArchiveProperties; +import com.tencent.bk.job.common.util.json.JsonUtils; +import lombok.extern.slf4j.Slf4j; + +import java.util.HashMap; +import java.util.Map; + +/** + * 归档配置存储 + */ +@Slf4j +public class ArchiveTablePropsStorage { + private final ArchiveProperties archiveProperties; + private final Map tablePropsMap = new HashMap<>(); + + public ArchiveTablePropsStorage(ArchiveProperties archiveProperties) { + this.archiveProperties = archiveProperties; + if (archiveProperties.getTableConfigs() != null && !archiveProperties.getTableConfigs().isEmpty()) { + archiveProperties.getTableConfigs().forEach( + (tableName, tableConfig) -> + tablePropsMap.put(tableName, new ArchiveTableProps( + tableConfig.getReadRowLimit(), + tableConfig.getBatchInsertRowSize(), + tableConfig.getDeleteRowLimit() + ))); + } + log.info("Init archive table properties, tableProps: {}", JsonUtils.toJson(tablePropsMap)); + } + + /** + * 从 热 DB 表中读取归档数据，每次读取的记录数量限制 + * + * @param tableName 表名 + */ + public int getReadRowLimit(String tableName) { + ArchiveTableProps props = tablePropsMap.get(tableName); + if (props == null || props.getReadRowLimit() == null) { + return archiveProperties.getReadRowLimit(); + } + return props.getReadRowLimit(); + } + + /** + * 写入归档数据到冷 DB，单批次最小行数 + * + * @param tableName 表名 + */ + public int getBatchInsertRowSize(String tableName) { + ArchiveTableProps props = tablePropsMap.get(tableName); + if (props == null || props.getBatchInsertRowSize() == null) { + return archiveProperties.getBatchInsertRowSize(); + } + return props.getBatchInsertRowSize(); + } + + /** + * 从热 DB 删除数据，每次删除的最大行数 + * + * @param tableName 表名 + */ + public int getDeleteLimitRowCount(String tableName) { + ArchiveTableProps props = tablePropsMap.get(tableName); + if (props == null || props.getDeleteLimitRowCount() == null) { + return archiveProperties.getDeleteRowLimit(); + } + return props.getDeleteLimitRowCount(); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskContextHolder.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskContextHolder.java new file mode 100644 index 0000000000..74902be74d --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskContextHolder.java @@ -0,0 +1,49 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +import com.tencent.bk.job.backup.archive.model.ArchiveTaskContext; + +public class ArchiveTaskContextHolder { + + private static final ThreadLocal HOLDER = new ThreadLocal<>(); + + public static void set(ArchiveTaskContext archiveTaskContext) { + HOLDER.set(archiveTaskContext); + } + + public static void unset() { + HOLDER.remove(); + } + + public static ArchiveTaskContext get() { + return HOLDER.get(); + } + + public static String getArchiveTaskId() { + ArchiveTaskContext context = HOLDER.get(); + return context != null ? context.getArchiveTaskInfo().buildTaskUniqueId() : null; + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskDoneCallback.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskDoneCallback.java new file mode 100644 index 0000000000..003ae31168 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskDoneCallback.java @@ -0,0 +1,32 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +/** + * 归档任务执行完成回调 + */ +public interface ArchiveTaskDoneCallback { + void callback(); +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskLock.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskLock.java deleted file mode 100644 index b0f64a8a8f..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskLock.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive; - -import com.tencent.bk.job.common.redis.util.LockUtils; -import com.tencent.bk.job.common.util.ThreadUtils; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; - -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; - -@Slf4j -public class ArchiveTaskLock { - private final String ARCHIVE_LOCK_KEY_PREFIX = "JOB_EXECUTE_LOG_ARCHIVE_LOCK"; - private final Long LOCK_TIME = 24 * 3600 * 1000L; - /** - * 最小获取锁间隔时间；为了保证在分布式系统中多个节点都能均匀获取到任务，会优先让空闲的节点获取到任务 - */ - private final long MIN_ACQUIRE_LOCK_INTERVAL_MS = 1000L; - private volatile long lastAcquireLockTimeMS = 0L; - private Map locks = new ConcurrentHashMap<>(); - - private ArchiveTaskLock() { - } - - public static ArchiveTaskLock getInstance() { - return ArchiveTaskLock.Inner.instance; - } - - public synchronized boolean lock(String tableName) { - while (System.currentTimeMillis() - lastAcquireLockTimeMS < MIN_ACQUIRE_LOCK_INTERVAL_MS) { - ThreadUtils.sleep(10L); - } - String lockRequestId = UUID.randomUUID().toString(); - String archiveLockKey = ARCHIVE_LOCK_KEY_PREFIX + "_" + tableName; - if (!LockUtils.tryGetDistributedLock(archiveLockKey, lockRequestId, LOCK_TIME)) { - log.info("Acquire archive task lock failed! tableName: {}", tableName); - return false; - } else { - log.info("Acquire archive task lock successfully! tableName: {}", tableName); - this.lastAcquireLockTimeMS = System.currentTimeMillis(); - this.locks.put(tableName, lockRequestId); - return true; - } - - } - - public synchronized void unlock(String tableName, String lockRequestId) { - String archiveLockKey = ARCHIVE_LOCK_KEY_PREFIX + "_" + tableName; - LockUtils.releaseDistributedLock(archiveLockKey, lockRequestId); - this.locks.remove(archiveLockKey); - } - - public synchronized void unlock(String tableName) { - String lockRequestId = this.locks.get(tableName); - if (StringUtils.isNotEmpty(lockRequestId)) { - String archiveLockKey = ARCHIVE_LOCK_KEY_PREFIX + "_" + tableName; - LockUtils.releaseDistributedLock(archiveLockKey, lockRequestId); - this.locks.remove(archiveLockKey); - } - } - - public void unlockAll() { - this.locks.forEach(this::unlock); - this.locks.clear(); - } - - private static class Inner { - private static final ArchiveTaskLock instance = new ArchiveTaskLock(); - } - -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskStopCallback.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskStopCallback.java new file mode 100644 index 0000000000..34e2c4aa12 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskStopCallback.java @@ -0,0 +1,32 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +/** + * 归档任务被终止回调 + */ +public interface ArchiveTaskStopCallback { + void callback(); +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskWorker.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskWorker.java new file mode 100644 index 0000000000..0ed51e9d56 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/ArchiveTaskWorker.java @@ -0,0 +1,61 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.cloud.sleuth.Span; +import org.springframework.cloud.sleuth.Tracer; + +/** + * 归档任务执行线程 + */ +@Slf4j +public class ArchiveTaskWorker extends Thread { + + private final JobHistoricalDataArchiveTask archiveTask; + private final Tracer tracer; + + public ArchiveTaskWorker(JobHistoricalDataArchiveTask archiveTask, Tracer tracer) { + this.tracer = tracer; + this.archiveTask = archiveTask; + this.archiveTask.initArchiveTaskWorker(this); + this.setName("ArchiveWorker-" + archiveTask.getTaskId()); + } + + @Override + public void run() { + Span span = tracer.nextSpan().name("archive-task"); + + try (Tracer.SpanInScope ignored = tracer.withSpan(span.start())) { + archiveTask.execute(); + } catch (Throwable e) { + span.error(e); + log.error("Caught archive task run exception", e); + } finally { + span.end(); + } + } + +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobExecuteArchiveManage.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobExecuteArchiveManage.java deleted file mode 100644 index 3bb6244e69..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobExecuteArchiveManage.java +++ /dev/null @@ -1,334 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive; - -import com.tencent.bk.job.backup.archive.impl.FileSourceTaskLogArchivist; -import com.tencent.bk.job.backup.archive.impl.GseFileAgentTaskArchivist; -import com.tencent.bk.job.backup.archive.impl.GseScriptAgentTaskArchivist; -import com.tencent.bk.job.backup.archive.impl.GseTaskArchivist; -import com.tencent.bk.job.backup.archive.impl.GseTaskIpLogArchivist; -import com.tencent.bk.job.backup.archive.impl.GseTaskLogArchivist; -import com.tencent.bk.job.backup.archive.impl.OperationLogArchivist; -import com.tencent.bk.job.backup.archive.impl.RollingConfigArchivist; -import com.tencent.bk.job.backup.archive.impl.StepInstanceArchivist; -import com.tencent.bk.job.backup.archive.impl.StepInstanceConfirmArchivist; -import com.tencent.bk.job.backup.archive.impl.StepInstanceFileArchivist; -import com.tencent.bk.job.backup.archive.impl.StepInstanceRollingTaskArchivist; -import com.tencent.bk.job.backup.archive.impl.StepInstanceScriptArchivist; -import com.tencent.bk.job.backup.archive.impl.StepInstanceVariableArchivist; -import com.tencent.bk.job.backup.archive.impl.TaskInstanceArchivist; -import com.tencent.bk.job.backup.archive.impl.TaskInstanceHostArchivist; -import com.tencent.bk.job.backup.archive.impl.TaskInstanceVariableArchivist; -import com.tencent.bk.job.backup.config.ArchiveConfig; -import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.impl.FileSourceTaskRecordDAO; -import com.tencent.bk.job.backup.dao.impl.GseFileAgentTaskRecordDAO; -import com.tencent.bk.job.backup.dao.impl.GseScriptAgentTaskRecordDAO; -import com.tencent.bk.job.backup.dao.impl.GseTaskIpLogRecordDAO; -import com.tencent.bk.job.backup.dao.impl.GseTaskLogRecordDAO; -import com.tencent.bk.job.backup.dao.impl.GseTaskRecordDAO; -import com.tencent.bk.job.backup.dao.impl.OperationLogRecordDAO; -import com.tencent.bk.job.backup.dao.impl.RollingConfigRecordDAO; -import com.tencent.bk.job.backup.dao.impl.StepInstanceConfirmRecordDAO; -import com.tencent.bk.job.backup.dao.impl.StepInstanceFileRecordDAO; -import com.tencent.bk.job.backup.dao.impl.StepInstanceRecordDAO; -import com.tencent.bk.job.backup.dao.impl.StepInstanceRollingTaskRecordDAO; -import com.tencent.bk.job.backup.dao.impl.StepInstanceScriptRecordDAO; -import com.tencent.bk.job.backup.dao.impl.StepInstanceVariableRecordDAO; -import com.tencent.bk.job.backup.dao.impl.TaskInstanceHostRecordDAO; -import com.tencent.bk.job.backup.dao.impl.TaskInstanceRecordDAO; -import com.tencent.bk.job.backup.dao.impl.TaskInstanceVariableRecordDAO; -import com.tencent.bk.job.backup.model.dto.ArchiveProgressDTO; -import com.tencent.bk.job.backup.service.ArchiveProgressService; -import lombok.extern.slf4j.Slf4j; -import org.joda.time.DateTime; -import org.springframework.context.SmartLifecycle; -import org.springframework.scheduling.annotation.Scheduled; - -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; - -@Slf4j -public class JobExecuteArchiveManage implements SmartLifecycle { - - private final ArchiveConfig archiveConfig; - - private final ExecutorService archiveExecutor; - - private final FileSourceTaskLogArchivist fileSourceTaskLogArchivist; - private final StepInstanceArchivist stepInstanceArchivist; - private final StepInstanceConfirmArchivist stepInstanceConfirmArchivist; - private final StepInstanceFileArchivist stepInstanceFileArchivist; - private final StepInstanceScriptArchivist stepInstanceScriptArchivist; - private final StepInstanceVariableArchivist stepInstanceVariableArchivist; - private final GseTaskLogArchivist gseTaskLogArchivist; - private final GseTaskIpLogArchivist gseTaskIpLogArchivist; - private final TaskInstanceArchivist taskInstanceArchivist; - private final TaskInstanceVariableArchivist taskInstanceVariableArchivist; - private final OperationLogArchivist operationLogArchivist; - private final GseTaskArchivist gseTaskArchivist; - private final GseScriptAgentTaskArchivist gseScriptAgentTaskArchivist; - private final GseFileAgentTaskArchivist gseFileAgentTaskArchivist; - private final StepInstanceRollingTaskArchivist stepInstanceRollingTaskArchivist; - private final RollingConfigArchivist rollingConfigArchivist; - private final TaskInstanceHostArchivist taskInstanceHostArchivist; - private final ArchiveProgressService archiveProgressService; - - - /** - * whether this component is currently running(Spring Lifecycle isRunning method) - */ - private volatile boolean running = false; - - public JobExecuteArchiveManage(TaskInstanceRecordDAO taskInstanceRecordDAO, - StepInstanceRecordDAO stepInstanceRecordDAO, - StepInstanceScriptRecordDAO stepInstanceScriptRecordDAO, - StepInstanceFileRecordDAO stepInstanceFileRecordDAO, - StepInstanceConfirmRecordDAO stepInstanceConfirmRecordDAO, - StepInstanceVariableRecordDAO stepInstanceVariableRecordDAO, - TaskInstanceVariableRecordDAO taskInstanceVariableRecordDAO, - OperationLogRecordDAO operationLogRecordDAO, - GseTaskLogRecordDAO gseTaskLogRecordDAO, - GseTaskIpLogRecordDAO gseTaskIpLogRecordDAO, - FileSourceTaskRecordDAO fileSourceTaskRecordDAO, - GseTaskRecordDAO gseTaskRecordDAO, - GseScriptAgentTaskRecordDAO gseScriptAgentTaskRecordDAO, - GseFileAgentTaskRecordDAO gseFileAgentTaskRecordDAO, - StepInstanceRollingTaskRecordDAO stepInstanceRollingTaskRecordDAO, - RollingConfigRecordDAO rollingConfigRecordDAO, - TaskInstanceHostRecordDAO taskInstanceHostRecordDAO, - ExecuteArchiveDAO executeArchiveDAO, - ArchiveProgressService archiveProgressService, - ArchiveConfig archiveConfig, - ExecutorService archiveExecutor) { - log.info("Init JobExecuteArchiveManage! archiveConfig: {}", archiveConfig); - this.archiveConfig = archiveConfig; - this.archiveProgressService = archiveProgressService; - this.fileSourceTaskLogArchivist = new FileSourceTaskLogArchivist(fileSourceTaskRecordDAO, executeArchiveDAO, - archiveProgressService); - this.stepInstanceArchivist = new StepInstanceArchivist(stepInstanceRecordDAO, executeArchiveDAO, - archiveProgressService); - this.stepInstanceConfirmArchivist = new StepInstanceConfirmArchivist(stepInstanceConfirmRecordDAO, - executeArchiveDAO, archiveProgressService); - this.stepInstanceFileArchivist = new StepInstanceFileArchivist(stepInstanceFileRecordDAO, executeArchiveDAO, - archiveProgressService); - this.stepInstanceScriptArchivist = new StepInstanceScriptArchivist(stepInstanceScriptRecordDAO, - executeArchiveDAO, archiveProgressService); - this.stepInstanceVariableArchivist = new StepInstanceVariableArchivist(stepInstanceVariableRecordDAO, - executeArchiveDAO, archiveProgressService); - this.gseTaskLogArchivist = new GseTaskLogArchivist(gseTaskLogRecordDAO, executeArchiveDAO, - archiveProgressService); - this.gseTaskIpLogArchivist = new GseTaskIpLogArchivist(gseTaskIpLogRecordDAO, executeArchiveDAO, - archiveProgressService); - this.taskInstanceArchivist = new TaskInstanceArchivist(taskInstanceRecordDAO, executeArchiveDAO, - archiveProgressService); - this.taskInstanceVariableArchivist = new TaskInstanceVariableArchivist(taskInstanceVariableRecordDAO, - executeArchiveDAO, archiveProgressService); - this.operationLogArchivist = new OperationLogArchivist(operationLogRecordDAO, executeArchiveDAO, - archiveProgressService); - this.gseTaskArchivist = new GseTaskArchivist(gseTaskRecordDAO, executeArchiveDAO, archiveProgressService); - this.gseScriptAgentTaskArchivist = new GseScriptAgentTaskArchivist(gseScriptAgentTaskRecordDAO, - executeArchiveDAO, archiveProgressService); - this.gseFileAgentTaskArchivist = new GseFileAgentTaskArchivist(gseFileAgentTaskRecordDAO, - executeArchiveDAO, archiveProgressService); - this.stepInstanceRollingTaskArchivist = new StepInstanceRollingTaskArchivist(stepInstanceRollingTaskRecordDAO, - executeArchiveDAO, archiveProgressService); - this.rollingConfigArchivist = new RollingConfigArchivist(rollingConfigRecordDAO, executeArchiveDAO, - archiveProgressService); - this.taskInstanceHostArchivist = new TaskInstanceHostArchivist(taskInstanceHostRecordDAO, executeArchiveDAO, - archiveProgressService); - this.archiveExecutor = archiveExecutor; - } - - @Scheduled(cron = "${job.execute.archive.cron:0 0 4 * * *}") - public void cronArchive() { - archive(archiveConfig); - } - - public void archive(ArchiveConfig archiveConfig) { - try { - new ArchiveThread(archiveConfig).start(); - } catch (Throwable e) { - log.error("Error while archive job_execute data", e); - } - } - - @Override - public void start() { - this.running = true; - } - - @Override - public void stop() { - log.info("Stop JobExecuteArchiveManage!"); - ArchiveTaskLock.getInstance().unlockAll(); - log.info("Release all archive locks when stop!"); - this.running = false; - } - - @Override - public boolean isRunning() { - return this.running; - } - - - class ArchiveThread extends Thread { - private final ArchiveConfig archiveConfig; - - - ArchiveThread(ArchiveConfig archiveConfig) { - this.archiveConfig = archiveConfig; - this.setName("Data-Archive-Thread"); - } - - @Override - public void run() { - try { - log.info("Job Execute archive task begin|{}", System.currentTimeMillis()); - log.info("Archive days : {}", archiveConfig.getDataKeepDays()); - if (archiveConfig.isArchiveEnabled() || archiveConfig.isDeleteEnabled()) { - doArchive(getEndTime(archiveConfig.getDataKeepDays())); - } else { - log.info("Archive and delete tasks are disabled, skip archive"); - } - log.info("Job Execute archive task finished|{}", System.currentTimeMillis()); - } catch (InterruptedException e) { - log.warn("Thread interrupted!"); - } - } - - private Long getEndTime(int archiveDays) { - DateTime now = DateTime.now(); - // 置为前一天天 24:00:00 - long todayMaxMills = now.minusMillis(now.getMillisOfDay()).getMillis(); - - //减掉当前xx天后 - long archiveMills = archiveDays * 24 * 3600 * 1000L; - return todayMaxMills - archiveMills; - } - - - private void doArchive(Long endTime) throws InterruptedException { - try { - log.info("Start job execute archive before {} at {}", endTime, System.currentTimeMillis()); - - long maxNeedArchiveTaskInstanceId = computeMaxNeedArchiveTaskInstanceId(endTime); - long maxNeedArchiveStepInstanceId = computeMaxNeedArchiveStepInstanceId(maxNeedArchiveTaskInstanceId); - - log.info("Compute archive instance id range, maxNeedArchiveTaskInstanceId: {}, " + - "maxNeedArchiveStepInstanceId: {}", maxNeedArchiveTaskInstanceId, maxNeedArchiveStepInstanceId); - - ArchiveSummaryHolder.getInstance().init(endTime); - archive(maxNeedArchiveTaskInstanceId, maxNeedArchiveStepInstanceId); - ArchiveSummaryHolder.getInstance().print(); - - log.info("Job execute archive before {} success at {}", endTime, System.currentTimeMillis()); - } catch (InterruptedException e) { - throw e; - } catch (Throwable e) { - log.error("Error while do archive!|{}", endTime, e); - } - } - - public long computeMaxNeedArchiveTaskInstanceId(Long endTime) { - ArchiveProgressDTO archiveProgress = - archiveProgressService.queryArchiveProgress(taskInstanceArchivist.getTableName()); - long lastArchivedId = archiveProgress != null ? archiveProgress.getLastArchivedId() : 0L; - long maxId = taskInstanceArchivist.getMaxId(endTime); - return Math.max(lastArchivedId, maxId); - } - - public long computeMaxNeedArchiveStepInstanceId(Long taskInstanceId) { - ArchiveProgressDTO archiveProgress = - archiveProgressService.queryArchiveProgress(stepInstanceArchivist.getTableName()); - long lastArchivedId = archiveProgress != null ? archiveProgress.getLastArchivedId() : 0L; - long maxId = stepInstanceArchivist.getMaxId(taskInstanceId); - return Math.max(lastArchivedId, maxId); - } - - private void archive(long maxNeedArchiveTaskInstanceId, long maxNeedArchiveStepInstanceId) - throws InterruptedException { - CountDownLatch countDownLatch = new CountDownLatch(16); - log.info("Submitting archive task..."); - - // task_instance - archiveExecutor.execute(() -> taskInstanceArchivist.archive(archiveConfig, - maxNeedArchiveTaskInstanceId, countDownLatch)); - // step_instance - archiveExecutor.execute(() -> stepInstanceArchivist.archive(archiveConfig, - maxNeedArchiveStepInstanceId, countDownLatch)); - // step_instance_confirm - archiveExecutor.execute(() -> stepInstanceConfirmArchivist.archive(archiveConfig, - maxNeedArchiveStepInstanceId, countDownLatch)); - // step_instance_file - archiveExecutor.execute(() -> stepInstanceFileArchivist.archive(archiveConfig, - maxNeedArchiveStepInstanceId, countDownLatch)); - // step_instance_script - archiveExecutor.execute(() -> stepInstanceScriptArchivist.archive(archiveConfig, - maxNeedArchiveStepInstanceId, countDownLatch)); - // gse_task_log - archiveExecutor.execute(() -> gseTaskLogArchivist.archive(archiveConfig, - maxNeedArchiveStepInstanceId, countDownLatch)); - // file_source_task 非正式发布功能，暂时不开启 -// ARCHIVE_THREAD_POOL_EXECUTOR.execute(() -> fileSourceTaskLogArchivist.archive(archiveConfig, -// maxNeedArchiveStepInstanceId, countDownLatch)); - // gse_task_ip_log - archiveExecutor.execute(() -> gseTaskIpLogArchivist.archive(archiveConfig, - maxNeedArchiveStepInstanceId, countDownLatch)); - // task_instance_variable - archiveExecutor.execute(() -> taskInstanceVariableArchivist.archive(archiveConfig, - maxNeedArchiveTaskInstanceId, countDownLatch)); - // step_instance_variable - archiveExecutor.execute(() -> stepInstanceVariableArchivist.archive(archiveConfig, - maxNeedArchiveStepInstanceId, countDownLatch)); - // operation_log - archiveExecutor.execute(() -> operationLogArchivist.archive(archiveConfig, - maxNeedArchiveTaskInstanceId, countDownLatch)); - // gse_task - archiveExecutor.execute(() -> gseTaskArchivist.archive(archiveConfig, - maxNeedArchiveStepInstanceId, countDownLatch)); - // gse_script_agent_task - archiveExecutor.execute(() -> gseScriptAgentTaskArchivist.archive(archiveConfig, - maxNeedArchiveStepInstanceId, countDownLatch)); - // gse_file_agent_task - archiveExecutor.execute(() -> gseFileAgentTaskArchivist.archive(archiveConfig, - maxNeedArchiveStepInstanceId, countDownLatch)); - // step_instance_rolling_task - archiveExecutor.execute(() -> stepInstanceRollingTaskArchivist.archive(archiveConfig, - maxNeedArchiveStepInstanceId, countDownLatch)); - // rolling_config - archiveExecutor.execute(() -> rollingConfigArchivist.archive(archiveConfig, - maxNeedArchiveTaskInstanceId, countDownLatch)); - // task_instance_host - archiveExecutor.execute(() -> taskInstanceHostArchivist.archive(archiveConfig, - maxNeedArchiveTaskInstanceId, countDownLatch)); - - log.info("Archive task submitted. Waiting for complete..."); - countDownLatch.await(); - - log.info("Archive task execute completed."); - } - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobHistoricalDataArchiveTask.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobHistoricalDataArchiveTask.java new file mode 100644 index 0000000000..75ecf9d579 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobHistoricalDataArchiveTask.java @@ -0,0 +1,68 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +/** + * 作业执行历史归档任务 + */ +public interface JobHistoricalDataArchiveTask { + + /** + * 执行任务 + */ + void execute(); + + /** + * 优雅终止任务 + * + * @param stopCallback 回调 + */ + void stop(ArchiveTaskStopCallback stopCallback); + + /** + * 强制终止 + */ + void forceStopAtOnce(); + + /** + * 注册任务完成回调函数 + * + * @param archiveTaskDoneCallback 回调函数 + */ + void registerDoneCallback(ArchiveTaskDoneCallback archiveTaskDoneCallback); + + /** + * 获取任务 ID + */ + String getTaskId(); + + /** + * 设置归档任务的 worker 信息 + * + * @param archiveTaskWorker 归档任务执行线程 + */ + void initArchiveTaskWorker(ArchiveTaskWorker archiveTaskWorker); + +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java new file mode 100644 index 0000000000..91e10ad443 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskGenerator.java @@ -0,0 +1,215 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +import com.tencent.bk.job.backup.archive.dao.impl.JobInstanceHotRecordDAO; +import com.tencent.bk.job.backup.archive.model.ArchiveTaskInfo; +import com.tencent.bk.job.backup.archive.model.DbDataNode; +import com.tencent.bk.job.backup.archive.service.ArchiveTaskService; +import com.tencent.bk.job.backup.archive.util.ArchiveDateTimeUtil; +import com.tencent.bk.job.backup.archive.util.lock.JobInstanceArchiveTaskGenerateLock; +import com.tencent.bk.job.backup.config.ArchiveProperties; +import com.tencent.bk.job.backup.constant.ArchiveTaskStatusEnum; +import com.tencent.bk.job.backup.constant.ArchiveTaskTypeEnum; +import com.tencent.bk.job.backup.constant.DbDataNodeTypeEnum; +import com.tencent.bk.job.common.mysql.dynamic.ds.DataSourceMode; +import com.tencent.bk.job.common.util.json.JsonUtils; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; + +import java.time.LocalDateTime; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.List; + +/** + * 生成作业实例归档任务 + */ +@Slf4j +public class JobInstanceArchiveTaskGenerator { + + private final ArchiveTaskService archiveTaskService; + + private final JobInstanceHotRecordDAO taskInstanceRecordDAO; + + private final ArchiveProperties archiveProperties; + + private final JobInstanceArchiveTaskGenerateLock archiveTaskGenerateLock; + + /** + * 归档数据时间范围计算所依据的时区 + */ + private final ZoneId archiveZoneId; + + + public JobInstanceArchiveTaskGenerator(ArchiveTaskService archiveTaskService, + JobInstanceHotRecordDAO taskInstanceRecordDAO, + ArchiveProperties archiveProperties, + JobInstanceArchiveTaskGenerateLock archiveTaskGenerateLock) { + this.archiveTaskService = archiveTaskService; + this.taskInstanceRecordDAO = taskInstanceRecordDAO; + this.archiveProperties = archiveProperties; + this.archiveTaskGenerateLock = archiveTaskGenerateLock; + archiveZoneId = ArchiveDateTimeUtil.getArchiveBasedTimeZone(archiveProperties.getTimeZone()); + } + + + public void generate() { + boolean locked = false; + try { + locked = archiveTaskGenerateLock.lock(); + if (!locked) { + return; + } + + if (taskInstanceRecordDAO.isTableEmpty()) { + log.info("Job instance table is empty and does not require processing"); + return; + } + List archiveTaskList = new ArrayList<>(); + + log.info("Compute archive task generate startDateTime and endDateTime"); + // 归档任务创建范围-起始时间 + LocalDateTime archiveStartDateTime = computeArchiveStartDateTime(); + // 归档任务创建范围-结束时间 + LocalDateTime archiveEndDateTime = + ArchiveDateTimeUtil.computeArchiveEndTime(archiveProperties.getKeepDays(), archiveZoneId); + if (archiveEndDateTime.isBefore(archiveStartDateTime) || archiveEndDateTime.equals(archiveStartDateTime)) { + log.info("Archive endTime is before startTime, does not require generating archive task." + + " startTime: {}, endTime: {}", + archiveStartDateTime, archiveEndDateTime); + return; + } + + log.info("Generate job instance archive tasks between {} and {}", + archiveStartDateTime, archiveEndDateTime); + // 创建归档任务。每个基础归档任务定义为：一个数据节点（db+表）+ 日期 + 小时 + while (archiveStartDateTime.isBefore(archiveEndDateTime)) { + log.info("Generate archive task for datetime : {}", archiveStartDateTime); + // 水平分库分表 + if (isHorizontalShardingEnabled()) { + // 作业实例数据归档任务,现版本暂不支持 + archiveTaskList.addAll(buildArchiveTasksForShardingDataNodes(ArchiveTaskTypeEnum.JOB_INSTANCE, + archiveStartDateTime, + archiveProperties.getTasks().getJobInstance().getShardingDataNodes())); + } else { + // 单db + DbDataNode dbDataNode = DbDataNode.standaloneDbDataNode(); + ArchiveTaskInfo archiveTaskInfo = + buildArchiveTask(ArchiveTaskTypeEnum.JOB_INSTANCE, archiveStartDateTime, dbDataNode); + archiveTaskList.add(archiveTaskInfo); + log.info("Add JobInstanceArchiveTaskInfo: {}", JsonUtils.toJson(archiveTaskInfo)); + } + + archiveStartDateTime = archiveStartDateTime.plusHours(1L); + } + + if (CollectionUtils.isNotEmpty(archiveTaskList)) { + archiveTaskService.saveArchiveTasks(archiveTaskList); + log.info("Generate archive tasks : {}", JsonUtils.toJson(archiveTaskList)); + } else { + log.info("No new archive tasks are generated"); + } + } catch (Throwable e) { + log.error("Generate archive task caught exception", e); + } finally { + if (locked) { + archiveTaskGenerateLock.unlock(); + } + } + + } + + private List buildArchiveTasksForShardingDataNodes( + ArchiveTaskTypeEnum archiveTaskType, + LocalDateTime startDateTime, + List shardingDataNodes) { + List tasks = new ArrayList<>(); + + // 任务：dataNode + day + hour + shardingDataNodes.forEach(dataNode -> { + int dbNodeCount = dataNode.getDbCount(); + int tableNodeCount = dataNode.getTableCount(); + String dataSource = dataNode.getDataSourceName(); + for (int dbNodeIndex = 0; dbNodeIndex < dbNodeCount; dbNodeIndex++) { + for (int tableNodeIndex = 0; tableNodeIndex < tableNodeCount; tableNodeIndex++) { + DbDataNode dbDataNode = new DbDataNode(DbDataNodeTypeEnum.SHARDING, dataSource, + dbNodeIndex, tableNodeIndex); + // 作业实例数据归档任务 + ArchiveTaskInfo archiveTaskInfo = + buildArchiveTask(archiveTaskType, startDateTime, dbDataNode); + tasks.add(archiveTaskInfo); + log.info("Add JobInstanceArchiveTaskInfo: {}", JsonUtils.toJson(archiveTaskInfo)); + } + } + }); + + return tasks; + } + + private ArchiveTaskInfo buildArchiveTask(ArchiveTaskTypeEnum archiveTaskType, + LocalDateTime startDateTime, + DbDataNode dbDataNode) { + ArchiveTaskInfo archiveTask = new ArchiveTaskInfo(); + int day = ArchiveDateTimeUtil.computeDay(startDateTime); + int hour = ArchiveDateTimeUtil.computeHour(startDateTime); + archiveTask.setDay(day); + archiveTask.setHour(hour); + long fromTimestamp = ArchiveDateTimeUtil.toTimestampMillsAtZone(startDateTime, archiveZoneId); + archiveTask.setFromTimestamp(fromTimestamp); + archiveTask.setToTimestamp(fromTimestamp + 1000 * 3600L); + archiveTask.setTaskType(archiveTaskType); + archiveTask.setDbDataNode(dbDataNode); + archiveTask.setStatus(ArchiveTaskStatusEnum.PENDING); + return archiveTask; + } + + + private LocalDateTime computeArchiveStartDateTime() { + LocalDateTime startDateTime; + ArchiveTaskInfo latestArchiveTask = + archiveTaskService.getLatestArchiveTask(ArchiveTaskTypeEnum.JOB_INSTANCE); + if (latestArchiveTask == null) { + // 从表数据中的 job_create_time 计算归档任务开始时间 + log.info("Latest archive task is empty, try compute from table min job create time"); + Long minJobCreateTimeMills = taskInstanceRecordDAO.getMinJobInstanceCreateTime(); + log.info("Min job create time in db is : {}", minJobCreateTimeMills); + startDateTime = ArchiveDateTimeUtil.toHourlyRoundDown( + ArchiveDateTimeUtil.unixTimestampMillToZoneDateTime(minJobCreateTimeMills, archiveZoneId)); + } else { + // 根据最新的归档任务计算开始 + log.info("Compute archive from latest generated archive task: {}", JsonUtils.toJson(latestArchiveTask)); + startDateTime = ArchiveDateTimeUtil.unixTimestampMillToZoneDateTime( + latestArchiveTask.getToTimestamp(), archiveZoneId); + } + + return startDateTime; + } + + private boolean isHorizontalShardingEnabled() { + return archiveProperties.getTasks().getJobInstance().getDataSourceMode() + .equals(DataSourceMode.Constants.HORIZONTAL_SHARDING); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java new file mode 100644 index 0000000000..5e04d40ec5 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceArchiveTaskScheduler.java @@ -0,0 +1,141 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.impl.JobInstanceHotRecordDAO; +import com.tencent.bk.job.backup.archive.model.ArchiveTaskInfo; +import com.tencent.bk.job.backup.archive.service.ArchiveTaskService; +import com.tencent.bk.job.backup.archive.util.lock.ArchiveTaskExecuteLock; +import com.tencent.bk.job.backup.archive.util.lock.JobInstanceArchiveTaskScheduleLock; +import com.tencent.bk.job.backup.config.ArchiveProperties; +import com.tencent.bk.job.backup.constant.ArchiveTaskTypeEnum; +import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; +import lombok.extern.slf4j.Slf4j; +import org.springframework.cloud.sleuth.Tracer; +import org.springframework.util.StopWatch; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; + +/** + * 作业执行历史归档任务调度 + */ +@Slf4j +public class JobInstanceArchiveTaskScheduler extends AbstractArchiveTaskScheduler { + + private final JobInstanceHotRecordDAO taskInstanceRecordDAO; + private final JobInstanceSubTableArchivers jobInstanceSubTableArchivers; + private final JobInstanceColdDAO jobInstanceColdDAO; + private final ArchiveTaskExecuteLock archiveTaskExecuteLock; + private final JobInstanceArchiveTaskScheduleLock scheduleLock; + private final ArchiveTablePropsStorage archiveTablePropsStorage; + private final ArchiveProperties archiveProperties; + + public JobInstanceArchiveTaskScheduler(ArchiveTaskService archiveTaskService, + JobInstanceHotRecordDAO taskInstanceRecordDAO, + ArchiveProperties archiveProperties, + JobInstanceArchiveTaskScheduleLock scheduleLock, + JobInstanceSubTableArchivers jobInstanceSubTableArchivers, + JobInstanceColdDAO jobInstanceColdDAO, + ArchiveTaskExecuteLock archiveTaskExecuteLock, + ArchiveErrorTaskCounter archiveErrorTaskCounter, + ArchiveTablePropsStorage archiveTablePropsStorage, + Tracer tracer, + ExecutorService archiveTaskStopExecutor) { + super(archiveTaskService, archiveTaskStopExecutor, archiveErrorTaskCounter, tracer); + this.taskInstanceRecordDAO = taskInstanceRecordDAO; + this.scheduleLock = scheduleLock; + this.jobInstanceSubTableArchivers = jobInstanceSubTableArchivers; + this.jobInstanceColdDAO = jobInstanceColdDAO; + this.archiveTaskExecuteLock = archiveTaskExecuteLock; + this.archiveTablePropsStorage = archiveTablePropsStorage; + this.archiveProperties = archiveProperties; + } + + @Override + protected boolean acquireScheduleLock() { + return scheduleLock.lock(); + } + + @Override + protected void releaseScheduleLock() { + scheduleLock.unlock(); + } + + @Override + protected Map countScheduleTasksGroupByDb() { + return archiveTaskService.countScheduleTasksGroupByDb(ArchiveTaskTypeEnum.JOB_INSTANCE); + } + + @Override + protected List listRunningTasks() { + return archiveTaskService.listRunningTasks(ArchiveTaskTypeEnum.JOB_INSTANCE); + } + + @Override + protected boolean shouldWait(StopWatch watch, + List runningTasks, + Map scheduleTasksGroupByDb) { + // 对待调度的任务进行优先级排序，保证同一个 db 上的归档任务数量尽可能均衡，避免出现db 热点 + watch.start("evaluateTaskPriority"); + ArchiveDbNodePriorityEvaluator.DbNodeTasksInfo highestPriorityDbNodeTasksInfo = + ArchiveDbNodePriorityEvaluator.evaluateHighestPriorityDbNode(runningTasks, + scheduleTasksGroupByDb); + watch.stop(); + return highestPriorityDbNodeTasksInfo.getRunningTaskCount() >= getTaskMaxConcurrent(); + } + + @Override + protected ArchiveTaskInfo getNextTask(Map scheduleTasksGroupByDb, + List running) { + // 根据优先级评估 + ArchiveDbNodePriorityEvaluator.DbNodeTasksInfo info = + ArchiveDbNodePriorityEvaluator.evaluateHighestPriorityDbNode(running, scheduleTasksGroupByDb); + return archiveTaskService.getFirstScheduleArchiveTaskByDb(ArchiveTaskTypeEnum.JOB_INSTANCE, + info.getDbNodeId()); + } + + @Override + protected Integer getTaskMaxConcurrent() { + return archiveProperties.getTasks().getJobInstance().getConcurrent(); + } + + @Override + protected JobInstanceMainDataArchiveTask createArchiveTask(ArchiveTaskInfo archiveTaskInfo) { + return new JobInstanceMainDataArchiveTask( + taskInstanceRecordDAO, + jobInstanceSubTableArchivers, + jobInstanceColdDAO, + archiveProperties, + archiveTaskExecuteLock, + super.archiveErrorTaskCounter, + archiveTaskInfo, + super.archiveTaskService, + archiveTablePropsStorage + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceMainDataArchiveTask.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceMainDataArchiveTask.java new file mode 100644 index 0000000000..55abc34000 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceMainDataArchiveTask.java @@ -0,0 +1,140 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.impl.JobInstanceHotRecordDAO; +import com.tencent.bk.job.backup.archive.model.BackupResult; +import com.tencent.bk.job.backup.archive.model.DeleteResult; +import com.tencent.bk.job.backup.archive.model.ArchiveTaskInfo; +import com.tencent.bk.job.backup.archive.model.TablesBackupResult; +import com.tencent.bk.job.backup.archive.model.TablesDeleteResult; +import com.tencent.bk.job.backup.archive.service.ArchiveTaskService; +import com.tencent.bk.job.backup.archive.util.lock.ArchiveTaskExecuteLock; +import com.tencent.bk.job.backup.config.ArchiveProperties; +import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; +import com.tencent.bk.job.execute.model.tables.records.TaskInstanceRecord; +import lombok.extern.slf4j.Slf4j; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * 作业实例数据归档 + **/ +@Slf4j +public class JobInstanceMainDataArchiveTask extends AbstractJobInstanceArchiveTask { + + private final JobInstanceSubTableArchivers jobInstanceSubTableArchivers; + + public JobInstanceMainDataArchiveTask(JobInstanceHotRecordDAO jobInstanceHotRecordDAO, + JobInstanceSubTableArchivers jobInstanceSubTableArchivers, + JobInstanceColdDAO jobInstanceColdDAO, + ArchiveProperties archiveProperties, + ArchiveTaskExecuteLock archiveTaskExecuteLock, + ArchiveErrorTaskCounter archiveErrorTaskCounter, + ArchiveTaskInfo archiveTask, + ArchiveTaskService archiveTaskService, + ArchiveTablePropsStorage archiveTablePropsStorage) { + super( + jobInstanceHotRecordDAO, + jobInstanceColdDAO, + archiveProperties, + archiveTaskExecuteLock, + archiveErrorTaskCounter, + archiveTask, + archiveTaskService, + archiveTablePropsStorage + ); + this.jobInstanceSubTableArchivers = jobInstanceSubTableArchivers; + } + + + @Override + protected TablesBackupResult backupJobInstanceToColdDb(List jobInstanceRecords) { + TablesBackupResult tablesBackupResult = new TablesBackupResult(); + + List jobInstanceIds = + jobInstanceRecords.stream().map(this::extractJobInstanceId).collect(Collectors.toList()); + // 备份主表数据 + BackupResult primaryTableBackupResult = + backupPrimaryTableRecord(jobInstanceRecords); + tablesBackupResult.add(jobInstanceMainRecordDAO.getTable().getName(), primaryTableBackupResult); + + // 备份子表数据 + jobInstanceSubTableArchivers.getAll().forEach(tableArchiver -> { + BackupResult backupResult = tableArchiver.backupRecords(jobInstanceIds); + tablesBackupResult.add(tableArchiver.getTableName(), backupResult); + }); + return tablesBackupResult; + } + + private BackupResult backupPrimaryTableRecord(List jobInstanceRecords) { + // 备份主表数据 + long startTime = System.currentTimeMillis(); + long backupRows = jobInstanceColdDAO.batchInsert(jobInstanceRecords, + archiveTablePropsStorage.getBatchInsertRowSize(jobInstanceMainRecordDAO.getTable().getName())); + long cost = System.currentTimeMillis() - startTime; + ArchiveTaskContextHolder.get().accumulateTableBackup( + jobInstanceMainRecordDAO.getTable().getName(), + backupRows, + cost + ); + return new BackupResult(backupRows, cost); + } + + @Override + protected TablesDeleteResult deleteJobInstanceHotData(List jobInstanceIds) { + TablesDeleteResult tablesDeleteResult = new TablesDeleteResult(); + long startTime = System.currentTimeMillis(); + // 先删除子表数据 + jobInstanceSubTableArchivers.getAll().forEach(tableArchiver -> { + DeleteResult deleteResult = tableArchiver.deleteRecords(jobInstanceIds); + tablesDeleteResult.add(tableArchiver.getTableName(), deleteResult); + }); + // 删除主表数据 + DeleteResult primaryTableDeleteResult = deletePrimaryTableRecord(jobInstanceIds); + tablesDeleteResult.add(jobInstanceMainRecordDAO.getTable().getName(), primaryTableDeleteResult); + + log.info("Delete {}, taskInstanceIdSize: {}, cost: {}ms", "task_instance", + jobInstanceIds.size(), System.currentTimeMillis() - startTime); + + return tablesDeleteResult; + } + + private DeleteResult deletePrimaryTableRecord(List jobInstanceIds) { + long startTime = System.currentTimeMillis(); + String tableName = jobInstanceMainRecordDAO.getTable().getName(); + int deleteRows = jobInstanceMainRecordDAO.deleteRecords(jobInstanceIds, + archiveTablePropsStorage.getDeleteLimitRowCount(tableName)); + long cost = System.currentTimeMillis() - startTime; + ArchiveTaskContextHolder.get().accumulateTableDelete( + tableName, + deleteRows, + cost + ); + return new DeleteResult(deleteRows, cost); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceSubTableArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceSubTableArchiver.java new file mode 100644 index 0000000000..c99b0cde13 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceSubTableArchiver.java @@ -0,0 +1,56 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +import com.tencent.bk.job.backup.archive.model.BackupResult; +import com.tencent.bk.job.backup.archive.model.DeleteResult; + +import java.util.List; + +/** + * 作业实例子表归档 + */ +public interface JobInstanceSubTableArchiver { + + /** + * 备份作业实例数据到冷存储 + * + * @param jobInstanceIds 作业实例 ID 列表 + */ + BackupResult backupRecords(List jobInstanceIds); + + /** + * 删除作业实例热数据 + * + * @param jobInstanceIds 作业实例 ID 列表 + */ + DeleteResult deleteRecords(List jobInstanceIds); + + /** + * 获取被归档的表名 + * @return 表名 + */ + String getTableName(); +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceSubTableArchivers.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceSubTableArchivers.java new file mode 100644 index 0000000000..6ed844bd4e --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobInstanceSubTableArchivers.java @@ -0,0 +1,88 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +import com.tencent.bk.job.backup.archive.impl.FileSourceTaskLogArchiver; +import com.tencent.bk.job.backup.archive.impl.GseFileAgentTaskArchiver; +import com.tencent.bk.job.backup.archive.impl.GseFileExecuteObjTaskArchiver; +import com.tencent.bk.job.backup.archive.impl.GseScriptAgentTaskArchiver; +import com.tencent.bk.job.backup.archive.impl.GseScriptExecuteObjTaskArchiver; +import com.tencent.bk.job.backup.archive.impl.GseTaskArchiver; +import com.tencent.bk.job.backup.archive.impl.OperationLogArchiver; +import com.tencent.bk.job.backup.archive.impl.RollingConfigArchiver; +import com.tencent.bk.job.backup.archive.impl.StepInstanceArchiver; +import com.tencent.bk.job.backup.archive.impl.StepInstanceConfirmArchiver; +import com.tencent.bk.job.backup.archive.impl.StepInstanceFileArchiver; +import com.tencent.bk.job.backup.archive.impl.StepInstanceRollingTaskArchiver; +import com.tencent.bk.job.backup.archive.impl.StepInstanceScriptArchiver; +import com.tencent.bk.job.backup.archive.impl.StepInstanceVariableArchiver; +import com.tencent.bk.job.backup.archive.impl.TaskInstanceHostArchiver; +import com.tencent.bk.job.backup.archive.impl.TaskInstanceVariableArchiver; + +import java.util.ArrayList; +import java.util.List; + +public class JobInstanceSubTableArchivers { + + private final List subTableArchivers = new ArrayList<>(); + + public JobInstanceSubTableArchivers(FileSourceTaskLogArchiver fileSourceTaskLogArchiver, + GseFileAgentTaskArchiver gseFileAgentTaskArchiver, + GseFileExecuteObjTaskArchiver gseFileExecuteObjTaskArchiver, + GseScriptAgentTaskArchiver gseScriptAgentTaskArchiver, + GseScriptExecuteObjTaskArchiver gseScriptExecuteObjTaskArchiver, + GseTaskArchiver gseTaskArchiver, + OperationLogArchiver operationLogArchiver, + RollingConfigArchiver rollingConfigArchiver, + StepInstanceArchiver stepInstanceArchiver, + StepInstanceConfirmArchiver stepInstanceConfirmArchiver, + StepInstanceFileArchiver stepInstanceFileArchiver, + StepInstanceScriptArchiver stepInstanceScriptArchiver, + StepInstanceRollingTaskArchiver stepInstanceRollingTaskArchiver, + StepInstanceVariableArchiver stepInstanceVariableArchiver, + TaskInstanceHostArchiver taskInstanceHostArchiver, + TaskInstanceVariableArchiver taskInstanceVariableArchiver) { + this.subTableArchivers.add(fileSourceTaskLogArchiver); + this.subTableArchivers.add(gseFileAgentTaskArchiver); + this.subTableArchivers.add(gseFileExecuteObjTaskArchiver); + this.subTableArchivers.add(gseScriptAgentTaskArchiver); + this.subTableArchivers.add(gseScriptExecuteObjTaskArchiver); + this.subTableArchivers.add(gseTaskArchiver); + this.subTableArchivers.add(operationLogArchiver); + this.subTableArchivers.add(rollingConfigArchiver); + this.subTableArchivers.add(stepInstanceArchiver); + this.subTableArchivers.add(stepInstanceConfirmArchiver); + this.subTableArchivers.add(stepInstanceFileArchiver); + this.subTableArchivers.add(stepInstanceScriptArchiver); + this.subTableArchivers.add(stepInstanceRollingTaskArchiver); + this.subTableArchivers.add(stepInstanceVariableArchiver); + this.subTableArchivers.add(taskInstanceHostArchiver); + this.subTableArchivers.add(taskInstanceVariableArchiver); + } + + public List getAll() { + return this.subTableArchivers; + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobLogArchiveTask.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobLogArchiveTask.java new file mode 100644 index 0000000000..17bd5ee6bf --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobLogArchiveTask.java @@ -0,0 +1,86 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +import com.tencent.bk.job.backup.archive.model.ArchiveTaskInfo; +import com.tencent.bk.job.backup.archive.service.ArchiveTaskService; +import com.tencent.bk.job.backup.archive.util.lock.ArchiveLogTaskExecuteLock; +import com.tencent.bk.job.backup.config.JobLogArchiveProperties; +import com.tencent.bk.job.backup.constant.ArchiveTaskStatusEnum; +import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; +import lombok.extern.slf4j.Slf4j; + +/** + * 作业执行日志归档任务实现 + */ +@Slf4j +public class JobLogArchiveTask extends AbstractHistoricalDataArchiveTask { + private final ArchiveLogTaskExecuteLock archiveLogTaskExecuteLock; + private final JobLogArchivers jobLogArchivers; + private final JobLogArchiveProperties archiveProperties; + + public JobLogArchiveTask(JobLogArchiveProperties archiveProperties, + ArchiveLogTaskExecuteLock archiveLogTaskExecuteLock, + ArchiveErrorTaskCounter archiveErrorTaskCounter, + ArchiveTaskInfo archiveTaskInfo, + ArchiveTaskService archiveTaskService, + JobLogArchivers jobLogArchivers) { + super(archiveErrorTaskCounter, archiveTaskInfo, archiveTaskService); + this.archiveLogTaskExecuteLock = archiveLogTaskExecuteLock; + this.jobLogArchivers = jobLogArchivers; + this.archiveProperties = archiveProperties; + } + + @Override + public void backupAndDelete() { + long startTime = System.currentTimeMillis(); + log.info("[{}] archive log task mode: {}", taskId, archiveProperties.getMode()); + jobLogArchivers.getAll().forEach(archiver -> { + archiver.backupRecords(archiveTaskInfo.getDay()); + archiver.deleteRecords(archiveTaskInfo.getDay()); + }); + long archiveCost = System.currentTimeMillis() - startTime; + if (archiveProperties.isDryRun()) { + updateCompletedExecuteInfo(ArchiveTaskStatusEnum.DRYRUN, null); + return; + } + setArchiveTaskExecutionDetail(null, archiveCost, null); + updateCompletedExecuteInfo(ArchiveTaskStatusEnum.SUCCESS, null); + } + + @Override + public boolean acquireLock() { + this.isAcquireLock = archiveLogTaskExecuteLock.lock(taskId); + if (!isAcquireLock) { + log.info("{} [{}] Acquire archive log task lock fail", getRuntimeClassName(), taskId); + } + return isAcquireLock; + } + + @Override + protected void unlock() { + archiveLogTaskExecuteLock.unlock(taskId); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobLogArchiveTaskGenerator.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobLogArchiveTaskGenerator.java new file mode 100644 index 0000000000..8b58a6d1c3 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobLogArchiveTaskGenerator.java @@ -0,0 +1,191 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +import com.tencent.bk.job.backup.archive.model.ArchiveTaskInfo; +import com.tencent.bk.job.backup.archive.model.DbDataNode; +import com.tencent.bk.job.backup.archive.service.ArchiveTaskService; +import com.tencent.bk.job.backup.archive.util.ArchiveDateTimeUtil; +import com.tencent.bk.job.backup.archive.util.lock.JobLogArchiveTaskGenerateLock; +import com.tencent.bk.job.backup.config.JobLogArchiveProperties; +import com.tencent.bk.job.backup.constant.ArchiveTaskStatusEnum; +import com.tencent.bk.job.backup.constant.ArchiveTaskTypeEnum; +import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.logsvr.util.CollectionNameUtil; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.springframework.data.mongodb.core.MongoTemplate; + +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.ZoneId; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * 生成作业执行日志归档任务 + */ +@Slf4j +public class JobLogArchiveTaskGenerator { + + private final ArchiveTaskService archiveTaskService; + + private final JobLogArchiveProperties archiveProperties; + + private final JobLogArchiveTaskGenerateLock jobLogArchiveTaskGenerateLock; + + private final MongoTemplate mongoTemplate; + + /** + * 归档数据时间范围计算所依据的时区 + */ + private final ZoneId archiveZoneId; + + private static final DateTimeFormatter DATE_FORMAT_YYYY_MM_DD = DateTimeFormatter.ofPattern("yyyy_MM_dd"); + + + public JobLogArchiveTaskGenerator(ArchiveTaskService archiveTaskService, + JobLogArchiveProperties archiveProperties, + JobLogArchiveTaskGenerateLock jobLogArchiveTaskGenerateLock, + MongoTemplate mongoTemplate) { + this.archiveTaskService = archiveTaskService; + this.archiveProperties = archiveProperties; + this.jobLogArchiveTaskGenerateLock = jobLogArchiveTaskGenerateLock; + this.mongoTemplate = mongoTemplate; + archiveZoneId = ArchiveDateTimeUtil.getArchiveBasedTimeZone(archiveProperties.getTimeZone()); + } + + public void generate() { + boolean locked = false; + try { + locked = jobLogArchiveTaskGenerateLock.lock(); + if (!locked) { + return; + } + + // 根据集合名称找出最早的一个 + Set allCollections = mongoTemplate.getCollectionNames(); + Optional earliestCollection = findEarliestCollection(allCollections); + if (!earliestCollection.isPresent()) { + log.info("No collection to be archived was found，allCollections={}", allCollections); + return; + } + + log.info("Compute archive log task generate startDateTime and endDateTime"); + LocalDateTime archiveStartDateTime = computeDateTimeByCollectionName(earliestCollection.get()); + LocalDateTime archiveEndDateTime = + ArchiveDateTimeUtil.computeArchiveEndTime(archiveProperties.getKeepDays(), + archiveZoneId); + if (archiveEndDateTime.isBefore(archiveStartDateTime) + || archiveEndDateTime.equals(archiveStartDateTime)) { + log.info("Archive endTime is before startTime, do not require to set up archive log task." + + " startTime: {}, endTime: {}", + archiveStartDateTime, archiveEndDateTime); + return; + } + + log.info("Generate job execute log archive tasks between {} and {}", + archiveStartDateTime, archiveEndDateTime); + List archiveTaskList = new ArrayList<>(); + // 按天创建日志归档任务 + while (archiveStartDateTime.isBefore(archiveEndDateTime)) { + log.info("Generate archive log task for datetime : {}", archiveStartDateTime); + DbDataNode dbDataNode = DbDataNode.standaloneMongoDbDataNode(); + ArchiveTaskInfo archiveTaskInfo = + buildArchiveTask(ArchiveTaskTypeEnum.JOB_EXECUTE_LOG, archiveStartDateTime, dbDataNode); + archiveTaskList.add(archiveTaskInfo); + log.info("Add JobInstanceArchiveTaskInfo: {}", JsonUtils.toJson(archiveTaskInfo)); + archiveStartDateTime = archiveStartDateTime.plusDays(1); + } + + if (CollectionUtils.isNotEmpty(archiveTaskList)) { + // 获取已有的归档任务列表，如果任务已存在就不用重新创建了 + List existedArchiveTaskList = + archiveTaskService.listTasksSinceDay(ArchiveTaskTypeEnum.JOB_EXECUTE_LOG, + archiveTaskList.get(0).getDay()); + if (CollectionUtils.isNotEmpty(existedArchiveTaskList)) { + Set existedIds = existedArchiveTaskList.stream() + .map(ArchiveTaskInfo::buildTaskUniqueId) + .collect(Collectors.toSet()); + archiveTaskList.removeIf(task -> existedIds.contains(task.buildTaskUniqueId())); + } + archiveTaskService.saveArchiveTasks(archiveTaskList); + log.info("Generate archive log tasks : {}", JsonUtils.toJson(archiveTaskList)); + } else { + log.info("No new archive log tasks are generated"); + } + } catch (Throwable e) { + log.error("Generate archive log task caught exception", e); + } finally { + if (locked) { + jobLogArchiveTaskGenerateLock.unlock(); + } + } + } + + /** + * 找到最早的集合名。 + */ + private Optional findEarliestCollection(Set allCollections) { + if (allCollections.isEmpty()) return Optional.empty(); + return allCollections.stream() + .filter(name -> CollectionNameUtil.collectionNameToDateStr(name) != null) + .min(Comparator.comparing(name -> + LocalDate.parse(Objects.requireNonNull(CollectionNameUtil.collectionNameToDateStr(name)), + DATE_FORMAT_YYYY_MM_DD) + )); + } + + private ArchiveTaskInfo buildArchiveTask(ArchiveTaskTypeEnum archiveTaskType, + LocalDateTime startDateTime, + DbDataNode dbDataNode) { + ArchiveTaskInfo archiveTask = new ArchiveTaskInfo(); + int day = ArchiveDateTimeUtil.computeDay(startDateTime); + archiveTask.setDay(day); + // 执行日志按天清理，小时不填 + archiveTask.setHour(-1); + long fromTimestamp = startDateTime.with(LocalTime.MIN).atZone(archiveZoneId).toInstant().toEpochMilli(); + long toTimestamp = startDateTime.with(LocalTime.MAX).atZone(archiveZoneId).toInstant().toEpochMilli(); + archiveTask.setFromTimestamp(fromTimestamp); + archiveTask.setToTimestamp(toTimestamp); + archiveTask.setTaskType(archiveTaskType); + archiveTask.setDbDataNode(dbDataNode); + archiveTask.setStatus(ArchiveTaskStatusEnum.PENDING); + return archiveTask; + } + + private LocalDateTime computeDateTimeByCollectionName(String name) { + LocalDate date = LocalDate.parse(Objects.requireNonNull(CollectionNameUtil.collectionNameToDateStr(name)), + DATE_FORMAT_YYYY_MM_DD); + return date.atStartOfDay(); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobLogArchiveTaskScheduler.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobLogArchiveTaskScheduler.java new file mode 100644 index 0000000000..da4ff9ce7a --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobLogArchiveTaskScheduler.java @@ -0,0 +1,108 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +import com.tencent.bk.job.backup.archive.model.ArchiveTaskInfo; +import com.tencent.bk.job.backup.archive.service.ArchiveTaskService; +import com.tencent.bk.job.backup.archive.util.lock.ArchiveLogTaskExecuteLock; +import com.tencent.bk.job.backup.archive.util.lock.JobLogArchiveTaskScheduleLock; +import com.tencent.bk.job.backup.config.JobLogArchiveProperties; +import com.tencent.bk.job.backup.constant.ArchiveTaskTypeEnum; +import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; +import lombok.extern.slf4j.Slf4j; +import org.springframework.cloud.sleuth.Tracer; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.ExecutorService; + +/** + * 作业执行日志归档任务调度 + */ +@Slf4j +public class JobLogArchiveTaskScheduler extends AbstractArchiveTaskScheduler { + + private final JobLogArchiveTaskScheduleLock scheduleLock; + private final JobLogArchivers jobLogArchivers; + private final ArchiveLogTaskExecuteLock archiveLogTaskExecuteLock; + private final JobLogArchiveProperties archiveProperties; + + public JobLogArchiveTaskScheduler(ArchiveTaskService archiveTaskService, + JobLogArchiveProperties archiveProperties, + JobLogArchiveTaskScheduleLock scheduleLock, + JobLogArchivers jobLogArchivers, + ArchiveLogTaskExecuteLock archiveLogTaskExecuteLock, + ArchiveErrorTaskCounter archiveErrorTaskCounter, + Tracer tracer, + ExecutorService archiveTaskStopExecutor) { + super(archiveTaskService, archiveTaskStopExecutor, archiveErrorTaskCounter, tracer); + this.scheduleLock = scheduleLock; + this.jobLogArchivers = jobLogArchivers; + this.archiveLogTaskExecuteLock = archiveLogTaskExecuteLock; + this.archiveProperties = archiveProperties; + } + + @Override + protected boolean acquireScheduleLock() { + return scheduleLock.lock(); + } + + @Override + protected void releaseScheduleLock() { + scheduleLock.unlock(); + } + + @Override + protected Map countScheduleTasksGroupByDb() { + return archiveTaskService.countScheduleTasksGroupByDb(ArchiveTaskTypeEnum.JOB_EXECUTE_LOG); + } + + @Override + protected List listRunningTasks() { + return archiveTaskService.listRunningTasks(ArchiveTaskTypeEnum.JOB_EXECUTE_LOG); + } + + @Override + protected ArchiveTaskInfo getNextTask(Map group, List running) { + return archiveTaskService.getFirstScheduleArchiveTask(ArchiveTaskTypeEnum.JOB_EXECUTE_LOG); + } + + @Override + protected Integer getTaskMaxConcurrent() { + return archiveProperties.getConcurrent(); + } + + @Override + protected JobLogArchiveTask createArchiveTask(ArchiveTaskInfo archiveTaskInfo) { + return new JobLogArchiveTask( + archiveProperties, + archiveLogTaskExecuteLock, + super.archiveErrorTaskCounter, + archiveTaskInfo, + super.archiveTaskService, + jobLogArchivers + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobLogArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobLogArchiver.java new file mode 100644 index 0000000000..76ea07e3aa --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobLogArchiver.java @@ -0,0 +1,48 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +import com.tencent.bk.job.backup.archive.model.BackupResult; +import com.tencent.bk.job.backup.archive.model.DeleteResult; + +/** + * 作业执行日志归档 + */ +public interface JobLogArchiver { + + /** + * 备份作业执行日志 + * + * @param archiveDay 归档数据所在天 + */ + BackupResult backupRecords(Integer archiveDay); + + /** + * 删除作业执行日志 + * + * @param archiveDay 归档数据所在天 + */ + DeleteResult deleteRecords(Integer archiveDay); +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobLogArchivers.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobLogArchivers.java new file mode 100644 index 0000000000..e94537a8ca --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/JobLogArchivers.java @@ -0,0 +1,46 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +import com.tencent.bk.job.backup.archive.impl.JobFileLogArchiver; +import com.tencent.bk.job.backup.archive.impl.JobScriptLogArchiver; + +import java.util.ArrayList; +import java.util.List; + +public class JobLogArchivers { + + private final List archivers = new ArrayList<>(); + + public JobLogArchivers(JobFileLogArchiver jobFileLogArchiver, + JobScriptLogArchiver jobScriptLogArchiver) { + this.archivers.add(jobFileLogArchiver); + this.archivers.add(jobScriptLogArchiver); + } + + public List getAll() { + return this.archivers; + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/TaskCountDownLatch.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/TaskCountDownLatch.java new file mode 100644 index 0000000000..a107b145d6 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/TaskCountDownLatch.java @@ -0,0 +1,109 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +import com.tencent.bk.job.common.util.ThreadUtils; +import lombok.extern.slf4j.Slf4j; + +import javax.annotation.PreDestroy; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +/** + * 任务 CountDownLatch 组件，用于等待所有任务完成后再退出 + */ +@Slf4j +public class TaskCountDownLatch { + private final Object taskMonitor = new Object(); + private final CountDownLatch latch; + private final Set taskIds = new HashSet<>(); + private volatile boolean monitorInitial = false; + private volatile boolean isAllTaskDone = false; + + public TaskCountDownLatch(Set taskIds) { + this.taskIds.addAll(taskIds); + this.latch = new CountDownLatch(taskIds.size()); + startMonitor(); + } + + public void decrement(String taskId) { + synchronized (taskMonitor) { + if (this.latch != null) { + if (taskIds.remove(taskId)) { + this.latch.countDown(); + log.info("Task is stopped, remove from counter! taskId: {}", taskId); + } else { + log.warn("Unexpected stopped taskId: {}", taskId); + } + } + } + } + + public boolean waitingForAllTasksDone(long timeoutSeconds) { + try { + log.info("Waiting for all tasks done! total: {}", latch.getCount()); + isAllTaskDone = latch.await(timeoutSeconds, TimeUnit.SECONDS); + if (isAllTaskDone) { + log.info("All tasks have been completed"); + } else { + log.info("Some tasks did not end within the timeout period, timeout: {}, notCompletedTaskIds: {}", + timeoutSeconds, taskIds); + } + return isAllTaskDone; + } catch (InterruptedException e) { + log.warn("Task count down latch wait interrupted", e); + return false; + } + } + + private synchronized void startMonitor() { + if (!monitorInitial) { + log.info("Start TaskCountDownLatch monitor ..."); + Thread monitorThread = new Thread("TaskCountDownLatchMonitor") { + @Override + public void run() { + while (!isAllTaskDone) { + log.info("Waiting for tasks stopped! taskSize: {}, tasks: {}", taskIds.size(), taskIds); + ThreadUtils.sleep(2000L); + } + } + }; + monitorThread.start(); + monitorInitial = true; + } + } + + @PreDestroy + private void destroy() { + if (isAllTaskDone) { + log.info("All task are stopped!"); + } else { + log.warn("Tasks are not completely stopped! remain task size: {}, remain tasks: {}", taskIds.size(), + taskIds); + } + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/ArchiveTaskDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/ArchiveTaskDAO.java new file mode 100644 index 0000000000..b26816c017 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/ArchiveTaskDAO.java @@ -0,0 +1,175 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao; + +import com.tencent.bk.job.backup.archive.model.ArchiveTaskExecutionDetail; +import com.tencent.bk.job.backup.archive.model.DbDataNode; +import com.tencent.bk.job.backup.archive.model.ArchiveTaskInfo; +import com.tencent.bk.job.backup.archive.model.IdBasedArchiveProcess; +import com.tencent.bk.job.backup.constant.ArchiveTaskStatusEnum; +import com.tencent.bk.job.backup.constant.ArchiveTaskTypeEnum; + +import java.util.List; +import java.util.Map; + +/** + * 归档任务 DAO + */ +public interface ArchiveTaskDAO { + + /** + * 获取最新的归档任务 + * + * @param taskType 归档任务类型 + */ + ArchiveTaskInfo getLatestArchiveTask(ArchiveTaskTypeEnum taskType); + + void saveArchiveTask(ArchiveTaskInfo archiveTaskInfo); + + List listRunningTasks(ArchiveTaskTypeEnum taskType); + + /** + * 获取归档任务 + * + * @param taskType 查询条件 - 任务类型 + * @param status 查询条件 - 任务状态 + * @param limit 查询条件 - 查询最大数量 + * @return 归档任务列表 + */ + List listTasks(ArchiveTaskTypeEnum taskType, + ArchiveTaskStatusEnum status, + int limit); + + /** + * 获取归档任务 + * + * @param status 查询条件 - 任务状态 + * @param limit 查询条件 - 查询最大数量 + * @return 归档任务列表 + */ + List listTasks(ArchiveTaskStatusEnum status, + int limit); + + /** + * 返回根据 db 分组的归档任务数量 + * + * @param taskType 归档任务类型 + * @return key: db 名称; value: 任务数量 + */ + Map countScheduleTasksGroupByDb(ArchiveTaskTypeEnum taskType); + + + /** + * 更新归档任务执行信息 - 启动后 + * + * @param taskType 任务类型 + * @param dataNode 数据节点 + * @param day 归档数据所在天 + * @param hour 归档数据所在小时 + * @param startTime 任务开始时间 + */ + void updateStartedExecuteInfo(ArchiveTaskTypeEnum taskType, + DbDataNode dataNode, + Integer day, + Integer hour, + Long startTime); + + /** + * 更新归档任务执行信息 - 运行中 + * + * @param taskType 任务类型 + * @param dataNode 数据节点 + * @param day 归档数据所在天 + * @param hour 归档数据所在小时 + * @param process 进度 + */ + void updateRunningExecuteInfo(ArchiveTaskTypeEnum taskType, + DbDataNode dataNode, + Integer day, + Integer hour, + IdBasedArchiveProcess process); + + /** + * 更新归档任务执行信息 - 结束 + * + * @param taskType 任务类型 + * @param dataNode 数据节点 + * @param day 归档数据所在天 + * @param hour 归档数据所在小时 + * @param status 任务状态 + * @param process 进度 + * @param endTime 结束时间 + * @param cost 任务耗时 + * @param detail 执行详情 + */ + void updateCompletedExecuteInfo(ArchiveTaskTypeEnum taskType, + DbDataNode dataNode, + Integer day, + Integer hour, + ArchiveTaskStatusEnum status, + IdBasedArchiveProcess process, + Long endTime, + Long cost, + ArchiveTaskExecutionDetail detail); + + ArchiveTaskInfo getFirstScheduleArchiveTaskByDb(ArchiveTaskTypeEnum taskType, String dbNodeId); + + ArchiveTaskInfo getFirstScheduleArchiveTask(ArchiveTaskTypeEnum taskType); + + void updateArchiveTaskStatus(ArchiveTaskTypeEnum taskType, + DbDataNode dataNode, + Integer day, + Integer hour, + ArchiveTaskStatusEnum status); + + Map countTaskByStatus(ArchiveTaskTypeEnum taskType, + List statusList); + + void updateExecutionDetail(ArchiveTaskTypeEnum taskType, + DbDataNode dataNode, + Integer day, + Integer hour, + ArchiveTaskExecutionDetail detail); + + /** + * 获取归档任务 + * + * @param taskType 查询条件 - 任务类型 + * @param day 查询条件 - 归档数据所在天 + * @return 归档任务列表 + */ + List listTasks(ArchiveTaskTypeEnum taskType, + int day); + + /** + * 获取从某天开始的归档任务 + * + * @param taskType 查询条件 - 任务类型 + * @param day 查询条件 - 归档数据所在天 + * @return 归档任务列表 + */ + List listTasksSinceDay(ArchiveTaskTypeEnum taskType, + int day); +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/ExecuteArchiveDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/ExecuteArchiveDAO.java similarity index 93% rename from src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/ExecuteArchiveDAO.java rename to src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/ExecuteArchiveDAO.java index 60fcd5cf09..9b9d33a60c 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/ExecuteArchiveDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/ExecuteArchiveDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.backup.dao; +package com.tencent.bk.job.backup.archive.dao; import org.jooq.TableRecord; diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/ExecuteRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/ExecuteRecordDAO.java new file mode 100644 index 0000000000..42dfff561b --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/ExecuteRecordDAO.java @@ -0,0 +1,81 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao; + +import org.jooq.Record; +import org.jooq.Table; +import org.jooq.TableField; + +import java.util.List; + +/** + * job-execute 微服务的表数据DAO + * + * @param 表记录 + */ +public interface ExecuteRecordDAO { + + /** + * 获取表 + * + * @return 表 + */ + Table getTable(); + + /** + * 获取用于查询归档记录的ID字段 + * + * @return ID字段 + */ + TableField getArchiveIdField(); + + /** + * 根据起始/结束ID获取表记录 + * + * @param start 起始ID(exclude) + * @param end 结束ID(include) + * @param offset 记录偏移数 + * @param limit 获取的记录数量 + * @return 表记录 + */ + List listRecords(Long start, Long end, Long offset, Long limit); + + /** + * 根据起始/结束ID删除表记录 + * + * @param start 起始ID(exclude) + * @param end 结束ID(include) + * @param maxLimitedDeleteRows 批量删除每批次limit + * @return 删除的记录数量 + */ + int deleteRecords(Long start, Long end, long maxLimitedDeleteRows); + + /** + * 获取表中最小归档ID + * + * @return id值。如果表中没有数据，那么返回 null + */ + Long getMinArchiveId(); +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/JobInstanceColdDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/JobInstanceColdDAO.java new file mode 100644 index 0000000000..23cf053a02 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/JobInstanceColdDAO.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao; + +import com.tencent.bk.job.backup.archive.ArchiveException; +import org.jooq.TableRecord; + +import java.util.List; + +/** + * 作业实例存储冷数据 DAO + */ +public interface JobInstanceColdDAO { + /** + * 批量写入数据到冷 db + * + * @param recordList 记录列表 + * @param bulkSize db 批量插入的最大记录数量 + * @return 写入的数据行数 + */ + Integer batchInsert(List> recordList, int bulkSize) throws ArchiveException; +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/JobInstanceHotRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/JobInstanceHotRecordDAO.java new file mode 100644 index 0000000000..64a735e924 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/JobInstanceHotRecordDAO.java @@ -0,0 +1,91 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao; + +import com.tencent.bk.job.backup.archive.dao.resultset.RecordResultSet; +import org.jooq.Record; +import org.jooq.Table; +import org.jooq.TableField; + +import java.util.Collection; +import java.util.List; + +/** + * 作业实例热数据 DAO + * + * @param 表记录 + */ +public interface JobInstanceHotRecordDAO { + + /** + * 获取表 + * + * @return 表 + */ + Table getTable(); + + /** + * 获取表中作业实例 ID 对应的字段 + * + * @return ID字段 + */ + TableField getJobInstanceIdField(); + + /** + * 获取表中作业实例创建时间对应的字段（仅主表存在） + * + * @return ID字段 + */ + default TableField getJobInstanceCreateTimeField() { + return null; + } + + /** + * 根据作业实例 ID 列表获取表记录 + * + * @param jobInstanceIds 作业实例 ID 列表 + * @param readRowLimit 获取的记录数量 + * @return 表记录 + */ + List listRecords(Collection jobInstanceIds, long readRowLimit); + + /** + * ResultSet 方式查询 + * + * @param jobInstanceIds 作业实例 ID 列表 + * @param readRowLimit 每次查询获取的记录数量上限 + * @return ResultSet + */ + RecordResultSet executeQuery(Collection jobInstanceIds, long readRowLimit); + + /** + * 根据起始/结束ID删除表记录 + * + * @param jobInstanceIds 作业实例 ID 列表 + * @param maxLimitedDeleteRows 批量删除每批次limit + * @return 删除的记录数量 + */ + int deleteRecords(Collection jobInstanceIds, long maxLimitedDeleteRows); +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/ds/JobExecuteVerticalShardingDSLContextProvider.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/ds/JobExecuteVerticalShardingDSLContextProvider.java new file mode 100644 index 0000000000..f8271f7d6a --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/ds/JobExecuteVerticalShardingDSLContextProvider.java @@ -0,0 +1,75 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.ds; + +import com.tencent.bk.job.common.mysql.dynamic.ds.VerticalShardingDSLContextProvider; +import lombok.extern.slf4j.Slf4j; +import org.jooq.DSLContext; + +import java.util.HashMap; +import java.util.Map; + +/** + * job-execute 垂直分库 + */ +@Slf4j +public class JobExecuteVerticalShardingDSLContextProvider extends VerticalShardingDSLContextProvider { + + /** + * 垂直分库，固定分成 3 个分库，每个分库与表的关系也是固定的。 + * + * @param dslContextA 分库 a + * @param dslContextB 分库 b + * @param dslContextC 分库 c + */ + public JobExecuteVerticalShardingDSLContextProvider(DSLContext dslContextA, + DSLContext dslContextB, + DSLContext dslContextC) { + super(); + Map tableNameAndDslContextMap = new HashMap<>(); + tableNameAndDslContextMap.put("task_instance", dslContextA); + tableNameAndDslContextMap.put("task_instance_host", dslContextA); + tableNameAndDslContextMap.put("gse_script_agent_task", dslContextB); + tableNameAndDslContextMap.put("gse_script_execute_obj_task", dslContextB); + tableNameAndDslContextMap.put("dangerous_record", dslContextC); + tableNameAndDslContextMap.put("file_source_task_log", dslContextC); + tableNameAndDslContextMap.put("gse_file_agent_task", dslContextC); + tableNameAndDslContextMap.put("gse_file_execute_obj_task", dslContextC); + tableNameAndDslContextMap.put("gse_task", dslContextC); + tableNameAndDslContextMap.put("gse_task_ip_log", dslContextC); + tableNameAndDslContextMap.put("gse_task_log", dslContextC); + tableNameAndDslContextMap.put("operation_log", dslContextC); + tableNameAndDslContextMap.put("rolling_config", dslContextC); + tableNameAndDslContextMap.put("statistics", dslContextC); + tableNameAndDslContextMap.put("step_instance", dslContextC); + tableNameAndDslContextMap.put("step_instance_confirm", dslContextC); + tableNameAndDslContextMap.put("step_instance_file", dslContextC); + tableNameAndDslContextMap.put("step_instance_rolling_task", dslContextC); + tableNameAndDslContextMap.put("step_instance_script", dslContextC); + tableNameAndDslContextMap.put("step_instance_variable", dslContextC); + tableNameAndDslContextMap.put("task_instance_variable", dslContextC); + super.initTableNameAndDslContextMap(tableNameAndDslContextMap); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/AbstractJobInstanceHotRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/AbstractJobInstanceHotRecordDAO.java new file mode 100644 index 0000000000..6e86dbda62 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/AbstractJobInstanceHotRecordDAO.java @@ -0,0 +1,121 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.impl; + +import com.tencent.bk.job.backup.archive.dao.JobInstanceHotRecordDAO; +import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; +import org.apache.commons.collections4.CollectionUtils; +import org.jooq.Condition; +import org.jooq.DSLContext; +import org.jooq.OrderField; +import org.jooq.Record; +import org.jooq.Result; +import org.jooq.SelectConditionStep; +import org.jooq.Table; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * 作业实例热数据查询 DAO 基础抽象实现 + * + * @param 表记录 + */ +public abstract class AbstractJobInstanceHotRecordDAO implements JobInstanceHotRecordDAO { + + protected final DSLContextProvider dslContextProvider; + + protected final Table table; + + public AbstractJobInstanceHotRecordDAO(DSLContextProvider dslContextProvider, Table table) { + this.dslContextProvider = dslContextProvider; + this.table = table; + } + + @Override + public List listRecords(Collection jobInstanceIds, long readRowLimit) { + return query(table, buildBasicConditions(jobInstanceIds), readRowLimit); + } + + public List listRecordsByConditions(List conditions, long readRowLimit) { + return query(table, conditions, readRowLimit); + } + + @Override + public int deleteRecords(Collection jobInstanceIds, long maxLimitedDeleteRows) { + return deleteWithLimit(table, buildBasicConditions(jobInstanceIds), maxLimitedDeleteRows); + } + + public List buildBasicConditions(Collection jobInstanceIds) { + List conditions = new ArrayList<>(); + conditions.add(getJobInstanceIdField().in(jobInstanceIds)); + return conditions; + } + + private int deleteWithLimit(Table table, List conditions, long maxLimitedDeleteRows) { + int totalDeleteRows = 0; + while (true) { + int deletedRows = dsl() + .delete(table) + .where(conditions) + .limit(maxLimitedDeleteRows) + .execute(); + totalDeleteRows += deletedRows; + if (deletedRows < maxLimitedDeleteRows) { + break; + } + } + return totalDeleteRows; + } + + protected List query(Table table, + List conditions, + long readRowLimit) { + SelectConditionStep selectConditionStep = dsl() + .select() + .from(table) + .where(conditions); + + Result result; + if (CollectionUtils.isNotEmpty(getListRecordsOrderFields())) { + result = selectConditionStep.orderBy(getListRecordsOrderFields()).limit(0, readRowLimit).fetch(); + } else { + result = selectConditionStep.limit(0, readRowLimit).fetch(); + } + return result.into(getTable()); + + } + + public Table getTable() { + return this.table; + } + + protected abstract Collection> getListRecordsOrderFields(); + + protected DSLContext dsl() { + return this.dslContextProvider.get(table.getName()); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/AbstractJobInstanceMainHotRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/AbstractJobInstanceMainHotRecordDAO.java new file mode 100644 index 0000000000..a6eacd404c --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/AbstractJobInstanceMainHotRecordDAO.java @@ -0,0 +1,115 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.impl; + +import com.tencent.bk.job.backup.archive.dao.resultset.JobInstanceRecordResultSetFactory; +import com.tencent.bk.job.backup.archive.dao.resultset.RecordResultSet; +import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; +import org.jooq.Record; +import org.jooq.Record1; +import org.jooq.Result; +import org.jooq.SelectConditionStep; +import org.jooq.Table; + +import java.util.Collection; +import java.util.List; + +import static org.jooq.impl.DSL.min; + +/** + * 作业实例 - 主表 - 热数据查询 DAO 基础抽象实现 + * + * @param + */ +public abstract class AbstractJobInstanceMainHotRecordDAO extends AbstractJobInstanceHotRecordDAO { + + public AbstractJobInstanceMainHotRecordDAO(DSLContextProvider dslContextProvider, Table table) { + super(dslContextProvider, table); + } + + /** + * 获取表中作业实例的最早创建时间 + */ + public Long getMinJobInstanceCreateTime() { + Record1 record = + dsl().select(min(getJobInstanceCreateTimeField())) + .from(getTable()) + .fetchOne(); + if (record != null) { + Long minJobCreateTime = (Long) record.get(0); + if (minJobCreateTime != null) { + return minJobCreateTime; + } + } + return Long.MAX_VALUE; + } + + /** + * 是否为空表 + */ + public boolean isTableEmpty() { + return !dsl().fetchExists(getTable()); + } + + /** + * 按时间范围+作业实例 ID，顺序读取表记录 + * + * @param fromTimestamp 开始时间(include) + * @param endTimestamp 开始时间(exclude) + * @param fromJobInstanceId 起始作业实例 ID + * @param limit 读取最大行数 + * @return 记录 + */ + public List readSortedJobInstanceFromHotDB(Long fromTimestamp, + Long endTimestamp, + Long fromJobInstanceId, + int limit) { + SelectConditionStep selectConditionStep = + dsl().select() + .from(getTable()) + .where(getJobInstanceCreateTimeField().greaterOrEqual(fromTimestamp)) + .and(getJobInstanceCreateTimeField().lessThan(endTimestamp)); + if (fromJobInstanceId != null) { + selectConditionStep.and(getJobInstanceIdField().greaterThan(fromJobInstanceId)); + } + Result result = selectConditionStep.orderBy( + getJobInstanceIdField().asc() + ) + .limit(limit) + .fetch(); + return result.into(getTable()); + } + + + @Override + public RecordResultSet executeQuery(Collection jobInstanceIds, + long readRowLimit) { + return JobInstanceRecordResultSetFactory.createOneQueryResultSet( + this, + jobInstanceIds, + readRowLimit + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ArchiveTaskDAOImpl.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ArchiveTaskDAOImpl.java new file mode 100644 index 0000000000..022ba0afae --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/ArchiveTaskDAOImpl.java @@ -0,0 +1,387 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.impl; + +import com.tencent.bk.job.backup.archive.dao.ArchiveTaskDAO; +import com.tencent.bk.job.backup.archive.model.ArchiveTaskExecutionDetail; +import com.tencent.bk.job.backup.archive.model.ArchiveTaskInfo; +import com.tencent.bk.job.backup.archive.model.DbDataNode; +import com.tencent.bk.job.backup.archive.model.IdBasedArchiveProcess; +import com.tencent.bk.job.backup.constant.ArchiveTaskStatusEnum; +import com.tencent.bk.job.backup.constant.ArchiveTaskTypeEnum; +import com.tencent.bk.job.backup.model.tables.ArchiveTask; +import com.tencent.bk.job.common.mysql.jooq.JooqDataTypeUtil; +import com.tencent.bk.job.common.util.json.JsonUtils; +import org.apache.commons.lang3.StringUtils; +import org.jooq.DSLContext; +import org.jooq.Record; +import org.jooq.Record2; +import org.jooq.Result; +import org.jooq.TableField; +import org.jooq.impl.DSL; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Repository; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Repository +public class ArchiveTaskDAOImpl implements ArchiveTaskDAO { + + private final DSLContext ctx; + private static final ArchiveTask T = ArchiveTask.ARCHIVE_TASK; + + private static final TableField[] ALL_FIELDS = { + T.TASK_TYPE, + T.DATA_NODE, + T.DB_NODE, + T.DAY, + T.HOUR, + T.FROM_TIMESTAMP, + T.TO_TIMESTAMP, + T.PROCESS, + T.STATUS, + T.CREATE_TIME, + T.LAST_UPDATE_TIME, + T.TASK_START_TIME, + T.TASK_END_TIME, + T.TASK_COST, + T.DETAIL + }; + + @Autowired + public ArchiveTaskDAOImpl(@Qualifier("job-backup-dsl-context") DSLContext ctx) { + this.ctx = ctx; + } + + @Override + public ArchiveTaskInfo getLatestArchiveTask(ArchiveTaskTypeEnum taskType) { + Record record = ctx.select(ALL_FIELDS) + .from(T) + .where(T.TASK_TYPE.eq(JooqDataTypeUtil.toByte(taskType.getType()))) + .orderBy(T.DAY.desc(), T.HOUR.desc()) + .limit(1) + .fetchOne(); + + return extract(record); + } + + private ArchiveTaskInfo extract(Record record) { + if (record == null) { + return null; + } + ArchiveTaskInfo archiveTask = new ArchiveTaskInfo(); + archiveTask.setTaskType(ArchiveTaskTypeEnum.valOf(record.get(T.TASK_TYPE))); + archiveTask.setDbDataNode(DbDataNode.fromDataNodeId(record.get(T.DATA_NODE))); + archiveTask.setDay(record.get(T.DAY)); + archiveTask.setHour(JooqDataTypeUtil.toInteger(record.get(T.HOUR))); + archiveTask.setFromTimestamp(record.get(T.FROM_TIMESTAMP)); + archiveTask.setToTimestamp(record.get(T.TO_TIMESTAMP)); + archiveTask.setProcess(IdBasedArchiveProcess.fromPersistentProcess(record.get(T.PROCESS))); + archiveTask.setStatus(ArchiveTaskStatusEnum.valOf(record.get(T.STATUS))); + archiveTask.setCreateTime(record.get(T.CREATE_TIME)); + archiveTask.setLastUpdateTime(record.get(T.LAST_UPDATE_TIME)); + archiveTask.setTaskStartTime(record.get(T.TASK_START_TIME)); + archiveTask.setTaskEndTime(record.get(T.TASK_END_TIME)); + archiveTask.setTaskCost(record.get(T.TASK_COST)); + String detail = record.get(T.DETAIL); + if (StringUtils.isNotBlank(detail)) { + archiveTask.setDetail(JsonUtils.fromJson(detail, ArchiveTaskExecutionDetail.class)); + } + return archiveTask; + } + + @Override + public void saveArchiveTask(ArchiveTaskInfo archiveTaskInfo) { + long createTime = System.currentTimeMillis(); + ctx.insertInto( + T, + T.TASK_TYPE, + T.DATA_NODE, + T.DB_NODE, + T.DAY, + T.HOUR, + T.FROM_TIMESTAMP, + T.TO_TIMESTAMP, + T.PROCESS, + T.STATUS, + T.CREATE_TIME, + T.LAST_UPDATE_TIME, + T.TASK_START_TIME, + T.TASK_END_TIME, + T.TASK_COST, + T.DETAIL) + .values( + JooqDataTypeUtil.toByte(archiveTaskInfo.getTaskType().getType()), + archiveTaskInfo.getDbDataNode().toDataNodeId(), + archiveTaskInfo.getDbDataNode().toDbNodeId(), + archiveTaskInfo.getDay(), + JooqDataTypeUtil.toByte(archiveTaskInfo.getHour()), + archiveTaskInfo.getFromTimestamp(), + archiveTaskInfo.getToTimestamp(), + archiveTaskInfo.getProcess() != null ? + archiveTaskInfo.getProcess().toPersistentProcess() : null, + JooqDataTypeUtil.toByte(archiveTaskInfo.getStatus().getStatus()), + createTime, + createTime, + null, + null, + null, + null + ) + .execute(); + } + + @Override + public List listRunningTasks(ArchiveTaskTypeEnum taskType) { + Result result = ctx.select(ALL_FIELDS) + .from(T) + .where(T.STATUS.eq(JooqDataTypeUtil.toByte(ArchiveTaskStatusEnum.RUNNING.getStatus()))) + .and(T.TASK_TYPE.eq(JooqDataTypeUtil.toByte(taskType.getType()))) + .fetch(); + + List tasks = new ArrayList<>(result.size()); + result.forEach(record -> tasks.add(extract(record))); + return tasks; + } + + @Override + public List listTasks(ArchiveTaskTypeEnum taskType, + ArchiveTaskStatusEnum status, + int limit) { + Result result = ctx.select(ALL_FIELDS) + .from(T) + .where(T.STATUS.eq(JooqDataTypeUtil.toByte(status.getStatus()))) + .and(T.TASK_TYPE.eq(JooqDataTypeUtil.toByte(taskType.getType()))) + .limit(limit) + .fetch(); + + List tasks = new ArrayList<>(result.size()); + result.forEach(record -> tasks.add(extract(record))); + return tasks; + } + + @Override + public List listTasks(ArchiveTaskStatusEnum status, + int limit) { + Result result = ctx.select(ALL_FIELDS) + .from(T) + .where(T.STATUS.eq(JooqDataTypeUtil.toByte(status.getStatus()))) + .limit(limit) + .fetch(); + + List tasks = new ArrayList<>(result.size()); + result.forEach(record -> tasks.add(extract(record))); + return tasks; + } + + @Override + public Map countScheduleTasksGroupByDb(ArchiveTaskTypeEnum taskType) { + Result> result = ctx.select(T.DB_NODE, DSL.count().as("task_count")) + .from(T) + .where(T.STATUS.in( + JooqDataTypeUtil.toByte(ArchiveTaskStatusEnum.PENDING.getStatus()), + JooqDataTypeUtil.toByte(ArchiveTaskStatusEnum.SUSPENDED.getStatus()))) + .and(T.TASK_TYPE.eq(JooqDataTypeUtil.toByte(taskType.getType()))) + .groupBy(T.DB_NODE) + .fetch(); + + Map dbAndTaskCount = new HashMap<>(); + result.forEach(record -> dbAndTaskCount.put(record.get(T.DB_NODE), (Integer) record.get("task_count"))); + return dbAndTaskCount; + } + + + @Override + public void updateStartedExecuteInfo(ArchiveTaskTypeEnum taskType, + DbDataNode dataNode, + Integer day, + Integer hour, + Long startTime) { + ctx.update(T) + .set(T.LAST_UPDATE_TIME, System.currentTimeMillis()) + .set(T.STATUS, JooqDataTypeUtil.toByte(ArchiveTaskStatusEnum.RUNNING.getStatus())) + .set(T.TASK_START_TIME, startTime) + .where(T.TASK_TYPE.eq(JooqDataTypeUtil.toByte(taskType.getType()))) + .and(T.DATA_NODE.eq(dataNode.toDataNodeId())) + .and(T.DAY.eq(day)) + .and(T.HOUR.eq(hour.byteValue())) + .execute(); + } + + @Override + public void updateRunningExecuteInfo(ArchiveTaskTypeEnum taskType, + DbDataNode dataNode, + Integer day, + Integer hour, + IdBasedArchiveProcess process) { + ctx.update(T) + .set(T.LAST_UPDATE_TIME, System.currentTimeMillis()) + .set(T.PROCESS, process.toPersistentProcess()) + .where(T.TASK_TYPE.eq(JooqDataTypeUtil.toByte(taskType.getType()))) + .and(T.DATA_NODE.eq(dataNode.toDataNodeId())) + .and(T.DAY.eq(day)) + .and(T.HOUR.eq(hour.byteValue())) + .execute(); + } + + @Override + public void updateCompletedExecuteInfo(ArchiveTaskTypeEnum taskType, + DbDataNode dataNode, + Integer day, + Integer hour, + ArchiveTaskStatusEnum status, + IdBasedArchiveProcess process, + Long endTime, + Long cost, + ArchiveTaskExecutionDetail detail) { + ctx.update(T) + .set(T.LAST_UPDATE_TIME, System.currentTimeMillis()) + .set(T.STATUS, JooqDataTypeUtil.toByte(status.getStatus())) + .set(T.PROCESS, process != null ? process.toPersistentProcess() : null) + .set(T.TASK_END_TIME, endTime) + .set(T.TASK_COST, cost) + .set(T.DETAIL, detail != null ? JsonUtils.toJson(detail) : null) + .where(T.TASK_TYPE.eq(JooqDataTypeUtil.toByte(taskType.getType()))) + .and(T.DATA_NODE.eq(dataNode.toDataNodeId())) + .and(T.DAY.eq(day)) + .and(T.HOUR.eq(hour.byteValue())) + .execute(); + } + + @Override + public ArchiveTaskInfo getFirstScheduleArchiveTaskByDb(ArchiveTaskTypeEnum taskType, String dbNodeId) { + Record record = ctx.select(ALL_FIELDS) + .from(T) + .where(T.STATUS.in( + JooqDataTypeUtil.toByte(ArchiveTaskStatusEnum.PENDING.getStatus()), + JooqDataTypeUtil.toByte(ArchiveTaskStatusEnum.SUSPENDED.getStatus()))) + .and(T.TASK_TYPE.eq(JooqDataTypeUtil.toByte(taskType.getType()))) + .and(T.DB_NODE.eq(dbNodeId)) + .orderBy(T.DAY.asc(), T.HOUR.asc(), T.DATA_NODE) + .limit(1) + .fetchOne(); + + return extract(record); + } + + @Override + public ArchiveTaskInfo getFirstScheduleArchiveTask(ArchiveTaskTypeEnum taskType) { + Record record = ctx.select(ALL_FIELDS) + .from(T) + .where(T.STATUS.in( + JooqDataTypeUtil.toByte(ArchiveTaskStatusEnum.PENDING.getStatus()), + JooqDataTypeUtil.toByte(ArchiveTaskStatusEnum.SUSPENDED.getStatus()))) + .and(T.TASK_TYPE.eq(JooqDataTypeUtil.toByte(taskType.getType()))) + .orderBy(T.DAY.asc()) + .limit(1) + .fetchOne(); + + return extract(record); + } + + @Override + public void updateArchiveTaskStatus(ArchiveTaskTypeEnum taskType, + DbDataNode dataNode, + Integer day, + Integer hour, + ArchiveTaskStatusEnum status) { + ctx.update(T) + .set(T.STATUS, JooqDataTypeUtil.toByte(status.getStatus())) + .where(T.TASK_TYPE.eq(JooqDataTypeUtil.toByte(taskType.getType()))) + .and(T.DATA_NODE.eq(dataNode.toDataNodeId())) + .and(T.DAY.eq(day)) + .and(T.HOUR.eq(hour.byteValue())) + .execute(); + } + + @Override + public Map countTaskByStatus(ArchiveTaskTypeEnum taskType, + List statusList) { + Result> result = ctx.select(T.STATUS, DSL.count().as("task_count")) + .from(T) + .where(T.TASK_TYPE.eq(JooqDataTypeUtil.toByte(taskType.getType()))) + .and(T.STATUS.in( + statusList.stream() + .map(statusEnum -> JooqDataTypeUtil.toByte(statusEnum.getStatus())) + .collect(Collectors.toList())) + ) + .groupBy(T.STATUS) + .fetch(); + Map taskCountGroupByStatus = new HashMap<>(); + result.forEach(record -> taskCountGroupByStatus.put( + ArchiveTaskStatusEnum.valOf(record.get(T.STATUS).intValue()), + (Integer) record.get("task_count")) + ); + return taskCountGroupByStatus; + } + + @Override + public void updateExecutionDetail(ArchiveTaskTypeEnum taskType, + DbDataNode dataNode, + Integer day, + Integer hour, + ArchiveTaskExecutionDetail detail) { + ctx.update(T) + .set(T.DETAIL, detail != null ? JsonUtils.toJson(detail) : null) + .where(T.TASK_TYPE.eq(JooqDataTypeUtil.toByte(taskType.getType()))) + .and(T.DATA_NODE.eq(dataNode.toDataNodeId())) + .and(T.DAY.eq(day)) + .and(T.HOUR.eq(hour.byteValue())) + .execute(); + } + + @Override + public List listTasks(ArchiveTaskTypeEnum taskType, + int day) { + Result result = ctx.select(ALL_FIELDS) + .from(T) + .where(T.TASK_TYPE.eq(JooqDataTypeUtil.toByte(taskType.getType()))) + .and(T.DAY.eq(day)) + .fetch(); + + List tasks = new ArrayList<>(result.size()); + result.forEach(record -> tasks.add(extract(record))); + return tasks; + } + + @Override + public List listTasksSinceDay(ArchiveTaskTypeEnum taskType, + int day) { + Result result = ctx.select(ALL_FIELDS) + .from(T) + .where(T.TASK_TYPE.eq(JooqDataTypeUtil.toByte(taskType.getType()))) + .and(T.DAY.greaterOrEqual(day)) + .fetch(); + + List tasks = new ArrayList<>(result.size()); + result.forEach(record -> tasks.add(extract(record))); + return tasks; + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/FileSourceTaskLogRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/FileSourceTaskLogRecordDAO.java new file mode 100644 index 0000000000..4190371766 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/FileSourceTaskLogRecordDAO.java @@ -0,0 +1,82 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.impl; + + +import com.tencent.bk.job.backup.archive.dao.resultset.JobInstanceRecordResultSetFactory; +import com.tencent.bk.job.backup.archive.dao.resultset.RecordResultSet; +import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; +import com.tencent.bk.job.execute.model.tables.FileSourceTaskLog; +import com.tencent.bk.job.execute.model.tables.records.FileSourceTaskLogRecord; +import org.jooq.Condition; +import org.jooq.OrderField; +import org.jooq.TableField; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * file_source_task_log DAO + */ +public class FileSourceTaskLogRecordDAO extends AbstractJobInstanceHotRecordDAO { + + private static final FileSourceTaskLog TABLE = FileSourceTaskLog.FILE_SOURCE_TASK_LOG; + private static final List> ORDER_FIELDS = new ArrayList<>(); + + static { + ORDER_FIELDS.add(FileSourceTaskLog.FILE_SOURCE_TASK_LOG.ID.asc()); + } + + public FileSourceTaskLogRecordDAO(DSLContextProvider dslContextProvider) { + super(dslContextProvider, TABLE); + } + + @Override + protected Collection> getListRecordsOrderFields() { + return ORDER_FIELDS; + } + + @Override + public TableField getJobInstanceIdField() { + return TABLE.TASK_INSTANCE_ID; + } + + @Override + public RecordResultSet executeQuery(Collection jobInstanceIds, long readRowLimit) { + return JobInstanceRecordResultSetFactory.createMultiQueryResultSet( + this, + jobInstanceIds, + readRowLimit, + lastRecord -> { + List conditions = new ArrayList<>(); + conditions.add(TABLE.ID.gt(lastRecord.getId())); + return conditions; + } + ); + } + + +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseFileAgentTaskRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseFileAgentTaskRecordDAO.java new file mode 100644 index 0000000000..933057a026 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseFileAgentTaskRecordDAO.java @@ -0,0 +1,80 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.impl; + +import com.tencent.bk.job.backup.archive.dao.resultset.JobInstanceRecordResultSetFactory; +import com.tencent.bk.job.backup.archive.dao.resultset.RecordResultSet; +import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; +import com.tencent.bk.job.execute.model.tables.GseFileAgentTask; +import com.tencent.bk.job.execute.model.tables.records.GseFileAgentTaskRecord; +import org.jooq.Condition; +import org.jooq.OrderField; +import org.jooq.TableField; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * gse_file_agent_task DAO + */ +public class GseFileAgentTaskRecordDAO extends AbstractJobInstanceHotRecordDAO { + + private static final GseFileAgentTask TABLE = GseFileAgentTask.GSE_FILE_AGENT_TASK; + private static final List> ORDER_FIELDS = new ArrayList<>(); + + static { + ORDER_FIELDS.add(GseFileAgentTask.GSE_FILE_AGENT_TASK.ID.asc()); + } + + public GseFileAgentTaskRecordDAO(DSLContextProvider dslContextProvider) { + super(dslContextProvider, TABLE); + } + + @Override + public TableField getJobInstanceIdField() { + return TABLE.TASK_INSTANCE_ID; + } + + @Override + protected Collection> getListRecordsOrderFields() { + return ORDER_FIELDS; + } + + @Override + public RecordResultSet executeQuery(Collection jobInstanceIds, + long readRowLimit) { + return JobInstanceRecordResultSetFactory.createMultiQueryResultSet( + this, + jobInstanceIds, + readRowLimit, + lastRecord -> { + List conditions = new ArrayList<>(); + conditions.add(TABLE.ID.gt(lastRecord.getId())); + return conditions; + } + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseFileExecuteObjTaskRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseFileExecuteObjTaskRecordDAO.java new file mode 100644 index 0000000000..e00601e122 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseFileExecuteObjTaskRecordDAO.java @@ -0,0 +1,80 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.impl; + +import com.tencent.bk.job.backup.archive.dao.resultset.JobInstanceRecordResultSetFactory; +import com.tencent.bk.job.backup.archive.dao.resultset.RecordResultSet; +import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; +import com.tencent.bk.job.execute.model.tables.GseFileExecuteObjTask; +import com.tencent.bk.job.execute.model.tables.GseScriptExecuteObjTask; +import com.tencent.bk.job.execute.model.tables.records.GseFileExecuteObjTaskRecord; +import org.jooq.Condition; +import org.jooq.OrderField; +import org.jooq.TableField; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * gse_file_execute_obj_task DAO + */ +public class GseFileExecuteObjTaskRecordDAO extends AbstractJobInstanceHotRecordDAO { + + private static final GseFileExecuteObjTask TABLE = GseFileExecuteObjTask.GSE_FILE_EXECUTE_OBJ_TASK; + private static final List> ORDER_FIELDS = new ArrayList<>(); + + static { + ORDER_FIELDS.add(GseFileExecuteObjTask.GSE_FILE_EXECUTE_OBJ_TASK.ID.asc()); + } + + public GseFileExecuteObjTaskRecordDAO(DSLContextProvider dslContextProvider) { + super(dslContextProvider, TABLE); + } + + @Override + protected Collection> getListRecordsOrderFields() { + return ORDER_FIELDS; + } + + @Override + public TableField getJobInstanceIdField() { + return TABLE.TASK_INSTANCE_ID; + } + + public RecordResultSet executeQuery(Collection jobInstanceIds, + long readRowLimit) { + return JobInstanceRecordResultSetFactory.createMultiQueryResultSet( + this, + jobInstanceIds, + readRowLimit, + lastRecord -> { + List conditions = new ArrayList<>(); + conditions.add(TABLE.ID.gt(lastRecord.getId())); + return conditions; + } + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseScriptAgentTaskRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseScriptAgentTaskRecordDAO.java new file mode 100644 index 0000000000..a0b8d4d45d --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseScriptAgentTaskRecordDAO.java @@ -0,0 +1,80 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.impl; + +import com.tencent.bk.job.backup.archive.dao.resultset.JobInstanceRecordResultSetFactory; +import com.tencent.bk.job.backup.archive.dao.resultset.RecordResultSet; +import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; +import com.tencent.bk.job.execute.model.tables.GseScriptAgentTask; +import com.tencent.bk.job.execute.model.tables.records.GseScriptAgentTaskRecord; +import org.jooq.Condition; +import org.jooq.OrderField; +import org.jooq.TableField; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * gse_script_agent_task DAO + */ +public class GseScriptAgentTaskRecordDAO extends AbstractJobInstanceHotRecordDAO { + + private static final GseScriptAgentTask TABLE = GseScriptAgentTask.GSE_SCRIPT_AGENT_TASK; + private static final List> ORDER_FIELDS = new ArrayList<>(); + + static { + ORDER_FIELDS.add(GseScriptAgentTask.GSE_SCRIPT_AGENT_TASK.ID.asc()); + } + + public GseScriptAgentTaskRecordDAO(DSLContextProvider dslContextProvider) { + super(dslContextProvider, TABLE); + } + + @Override + public TableField getJobInstanceIdField() { + return TABLE.TASK_INSTANCE_ID; + } + + @Override + protected Collection> getListRecordsOrderFields() { + return ORDER_FIELDS; + } + + @Override + public RecordResultSet executeQuery(Collection jobInstanceIds, + long readRowLimit) { + return JobInstanceRecordResultSetFactory.createMultiQueryResultSet( + this, + jobInstanceIds, + readRowLimit, + lastRecord -> { + List conditions = new ArrayList<>(); + conditions.add(TABLE.ID.gt(lastRecord.getId())); + return conditions; + } + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseScriptExecuteObjTaskRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseScriptExecuteObjTaskRecordDAO.java new file mode 100644 index 0000000000..6c4c3c31cb --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseScriptExecuteObjTaskRecordDAO.java @@ -0,0 +1,80 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.impl; + +import com.tencent.bk.job.backup.archive.dao.resultset.JobInstanceRecordResultSetFactory; +import com.tencent.bk.job.backup.archive.dao.resultset.RecordResultSet; +import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; +import com.tencent.bk.job.execute.model.tables.GseScriptExecuteObjTask; +import com.tencent.bk.job.execute.model.tables.records.GseScriptExecuteObjTaskRecord; +import org.jooq.Condition; +import org.jooq.OrderField; +import org.jooq.TableField; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * gse_script_execute_obj_task DAO + */ +public class GseScriptExecuteObjTaskRecordDAO extends AbstractJobInstanceHotRecordDAO { + + private static final GseScriptExecuteObjTask TABLE = GseScriptExecuteObjTask.GSE_SCRIPT_EXECUTE_OBJ_TASK; + private static final List> ORDER_FIELDS = new ArrayList<>(); + + static { + ORDER_FIELDS.add(GseScriptExecuteObjTask.GSE_SCRIPT_EXECUTE_OBJ_TASK.ID.asc()); + } + + public GseScriptExecuteObjTaskRecordDAO(DSLContextProvider dslContextProvider) { + super(dslContextProvider, TABLE); + } + + @Override + public TableField getJobInstanceIdField() { + return TABLE.TASK_INSTANCE_ID; + } + + @Override + protected Collection> getListRecordsOrderFields() { + return ORDER_FIELDS; + } + + @Override + public RecordResultSet executeQuery(Collection jobInstanceIds, + long readRowLimit) { + return JobInstanceRecordResultSetFactory.createMultiQueryResultSet( + this, + jobInstanceIds, + readRowLimit, + lastRecord -> { + List conditions = new ArrayList<>(); + conditions.add(TABLE.ID.gt(lastRecord.getId())); + return conditions; + } + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseTaskRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseTaskRecordDAO.java new file mode 100644 index 0000000000..d694d57ec5 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/GseTaskRecordDAO.java @@ -0,0 +1,77 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.impl; + +import com.tencent.bk.job.backup.archive.dao.resultset.JobInstanceRecordResultSetFactory; +import com.tencent.bk.job.backup.archive.dao.resultset.RecordResultSet; +import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; +import com.tencent.bk.job.execute.model.tables.GseTask; +import com.tencent.bk.job.execute.model.tables.records.GseTaskRecord; +import org.jooq.Condition; +import org.jooq.OrderField; +import org.jooq.TableField; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +public class GseTaskRecordDAO extends AbstractJobInstanceHotRecordDAO { + + private static final GseTask TABLE = GseTask.GSE_TASK; + private static final List> ORDER_FIELDS = new ArrayList<>(); + + static { + ORDER_FIELDS.add(GseTask.GSE_TASK.ID.asc()); + } + + public GseTaskRecordDAO(DSLContextProvider dslContextProvider) { + super(dslContextProvider, TABLE); + } + + @Override + protected Collection> getListRecordsOrderFields() { + return ORDER_FIELDS; + } + + @Override + public TableField getJobInstanceIdField() { + return TABLE.TASK_INSTANCE_ID; + } + + @Override + public RecordResultSet executeQuery(Collection jobInstanceIds, + long readRowLimit) { + return JobInstanceRecordResultSetFactory.createMultiQueryResultSet( + this, + jobInstanceIds, + readRowLimit, + lastRecord -> { + List conditions = new ArrayList<>(); + conditions.add(TABLE.ID.gt(lastRecord.getId())); + return conditions; + } + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/JobInstanceColdDAOImpl.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/JobInstanceColdDAOImpl.java new file mode 100644 index 0000000000..5ff6fb923f --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/JobInstanceColdDAOImpl.java @@ -0,0 +1,157 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.impl; + +import com.tencent.bk.job.backup.archive.ArchiveException; +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.jooq.DSLContext; +import org.jooq.Loader; +import org.jooq.LoaderError; +import org.jooq.TableRecord; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.List; + + +@Slf4j +public class JobInstanceColdDAOImpl implements JobInstanceColdDAO { + + private static final Logger ARCHIVE_FAILED_LOGGER = LoggerFactory.getLogger("ArchiveFailedLogger"); + + private final DSLContext context; + + public JobInstanceColdDAOImpl(DSLContext coldDbDslContext) { + log.info("Init JobInstanceColdDAOImpl."); + this.context = coldDbDslContext; + } + + @Override + public Integer batchInsert(List> recordList, int bulkSize) throws ArchiveException { + long start = System.currentTimeMillis(); + int successInsertedRecords = 0; + String table = recordList.get(0).getTable().getName(); + boolean success = true; + try { + Loader loader = + context.loadInto(recordList.get(0).getTable()) + // 由于这里是批量写入，jooq 不允许使用 onDuplicateKeyIgnore/onDuplicateKeyUpdate. + // 否则会报错"Cannot apply bulk loading with onDuplicateKey flags" + // 所以这里暂时使用 onDuplicateKeyError 错误处理方式，等后续流程进一步判断是否是主键冲突错误 + // issue 参考：https://github.com/jOOQ/jOOQ/issues/12740 + .onDuplicateKeyError() + .bulkAfter(bulkSize) + .loadRecords(recordList) + .fieldsCorresponding() + .execute(); + successInsertedRecords = loader.stored(); + String bulkInsertResult = successInsertedRecords == recordList.size() ? "success" : "fail"; + log.info( + "InsertBulk: Load {} data|result|{}|executed|{}|processed|{}|stored|{}|ignored|{}|errors|{}", + table, + bulkInsertResult, + loader.executed(), + loader.processed(), + loader.stored(), + loader.ignored(), + loader.errors().size() + ); + if (CollectionUtils.isNotEmpty(loader.errors())) { + for (LoaderError error : loader.errors()) { + if (isDuplicateError(error)) { + log.info("LoadDataDuplicateError. table: {}, exception: {}, row: {}", + table, error.exception().getMessage(), error.row()); + } else { + ARCHIVE_FAILED_LOGGER.error("Error while load {} data, exception: {}， error row: {}", table, + error.exception().getMessage(), error.row()); + } + } + if (hasDuplicateError(loader.errors())) { + // 如果存在主键冲突的数据，尝试每一条记录单独插入，就可以使用 onDuplicateKeyIgnore 错误处理方式 + successInsertedRecords = insertSingle(recordList); + } + } + } catch (IOException e) { + String errorMsg = String.format("Error while loading %s data!", table); + log.error(errorMsg, e); + success = false; + throw new ArchiveException(errorMsg, e); + } finally { + log.info("Load data to {} done! success: {}, total: {}, inserted: {}, cost: {}ms", table, success, + recordList.size(), successInsertedRecords, System.currentTimeMillis() - start); + } + + return successInsertedRecords; + } + + private boolean hasDuplicateError(List errors) { + // 通过 mysql 执行的错误消息判断是否是由于数据唯一性冲突引起的 + return errors.stream().anyMatch(this::isDuplicateError); + } + + private boolean isDuplicateError(LoaderError error) { + return error.exception().getMessage() != null + && error.exception().getMessage().contains("Duplicate entry"); + } + + private int insertSingle(List> recordList) throws IOException { + int successInsertedRecords; + String table = recordList.get(0).getTable().getName(); + try { + Loader loader = + context.loadInto(recordList.get(0).getTable()) + .onDuplicateKeyIgnore() + .loadRecords(recordList) + .fieldsCorresponding() + .execute(); + successInsertedRecords = loader.stored() + loader.ignored(); + String bulkInsertResult = successInsertedRecords == recordList.size() ? "success" : "fail"; + log.info( + "InsertSingle: Load {} data|result|{}|executed|{}|processed|{}|stored|{}|ignored|{}|errors|{}", + table, + bulkInsertResult, + loader.executed(), + loader.processed(), + loader.stored(), + loader.ignored(), + loader.errors().size() + ); + if (CollectionUtils.isNotEmpty(loader.errors())) { + for (LoaderError error : loader.errors()) { + ARCHIVE_FAILED_LOGGER.error("InsertSingle: Error while load {} data, exception: {}, error row: {}", + table, error.row(), error.exception().getMessage()); + } + } + } catch (IOException e) { + String errorMsg = String.format("Error while loading %s data!", table); + log.error(errorMsg, e); + throw e; + } + return successInsertedRecords; + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/JobInstanceHotRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/JobInstanceHotRecordDAO.java new file mode 100644 index 0000000000..1d44038849 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/JobInstanceHotRecordDAO.java @@ -0,0 +1,68 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.impl; + +import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; +import com.tencent.bk.job.execute.model.tables.TaskInstance; +import com.tencent.bk.job.execute.model.tables.records.TaskInstanceRecord; +import org.jooq.OrderField; +import org.jooq.TableField; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * task_instance DAO + */ +public class JobInstanceHotRecordDAO extends AbstractJobInstanceMainHotRecordDAO { + + private static final TaskInstance TABLE = TaskInstance.TASK_INSTANCE; + + private static final List> ORDER_FIELDS = new ArrayList<>(); + + static { + ORDER_FIELDS.add(TaskInstance.TASK_INSTANCE.ID.asc()); + } + + public JobInstanceHotRecordDAO(DSLContextProvider dslContextProvider) { + super(dslContextProvider, TABLE); + } + + @Override + public TableField getJobInstanceIdField() { + return TABLE.ID; + } + + @Override + public TableField getJobInstanceCreateTimeField() { + return TABLE.CREATE_TIME; + } + + protected Collection> getListRecordsOrderFields() { + return ORDER_FIELDS; + } + +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/OperationLogRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/OperationLogRecordDAO.java new file mode 100644 index 0000000000..ebe136355c --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/OperationLogRecordDAO.java @@ -0,0 +1,81 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.impl; + +import com.tencent.bk.job.backup.archive.dao.resultset.JobInstanceRecordResultSetFactory; +import com.tencent.bk.job.backup.archive.dao.resultset.RecordResultSet; +import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; +import com.tencent.bk.job.execute.model.tables.OperationLog; +import com.tencent.bk.job.execute.model.tables.records.OperationLogRecord; +import org.jooq.Condition; +import org.jooq.OrderField; +import org.jooq.TableField; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * operation_log DAO + */ +public class OperationLogRecordDAO extends AbstractJobInstanceHotRecordDAO { + + private static final OperationLog TABLE = OperationLog.OPERATION_LOG; + + private static final List> ORDER_FIELDS = new ArrayList<>(); + + static { + ORDER_FIELDS.add(OperationLog.OPERATION_LOG.ID.asc()); + } + + public OperationLogRecordDAO(DSLContextProvider dslContextProvider) { + super(dslContextProvider, TABLE); + } + + @Override + protected Collection> getListRecordsOrderFields() { + return ORDER_FIELDS; + } + + @Override + public TableField getJobInstanceIdField() { + return TABLE.TASK_INSTANCE_ID; + } + + @Override + public RecordResultSet executeQuery(Collection jobInstanceIds, + long readRowLimit) { + return JobInstanceRecordResultSetFactory.createMultiQueryResultSet( + this, + jobInstanceIds, + readRowLimit, + lastRecord -> { + List conditions = new ArrayList<>(); + conditions.add(TABLE.ID.gt(lastRecord.getId())); + return conditions; + } + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/RollingConfigRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/RollingConfigRecordDAO.java new file mode 100644 index 0000000000..29b924765f --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/RollingConfigRecordDAO.java @@ -0,0 +1,74 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.impl; + +import com.tencent.bk.job.backup.archive.dao.resultset.JobInstanceRecordResultSetFactory; +import com.tencent.bk.job.backup.archive.dao.resultset.RecordResultSet; +import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; +import com.tencent.bk.job.execute.model.tables.RollingConfig; +import com.tencent.bk.job.execute.model.tables.records.RollingConfigRecord; +import org.jooq.OrderField; +import org.jooq.TableField; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * rolling_config DAO + */ +public class RollingConfigRecordDAO extends AbstractJobInstanceHotRecordDAO { + + private static final RollingConfig TABLE = RollingConfig.ROLLING_CONFIG; + + private static final List> ORDER_FIELDS = new ArrayList<>(); + + static { + ORDER_FIELDS.add(RollingConfig.ROLLING_CONFIG.TASK_INSTANCE_ID.asc()); + } + + public RollingConfigRecordDAO(DSLContextProvider dslContextProvider) { + super(dslContextProvider, TABLE); + } + + @Override + protected Collection> getListRecordsOrderFields() { + return ORDER_FIELDS; + } + + @Override + public TableField getJobInstanceIdField() { + return TABLE.TASK_INSTANCE_ID; + } + + public RecordResultSet executeQuery(Collection jobInstanceIds, + long readRowLimit) { + return JobInstanceRecordResultSetFactory.createOneQueryResultSet( + this, + jobInstanceIds, + readRowLimit + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceConfirmRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceConfirmRecordDAO.java new file mode 100644 index 0000000000..46a34062d2 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceConfirmRecordDAO.java @@ -0,0 +1,82 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.impl; + +import com.tencent.bk.job.backup.archive.dao.resultset.JobInstanceRecordResultSetFactory; +import com.tencent.bk.job.backup.archive.dao.resultset.RecordResultSet; +import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; +import com.tencent.bk.job.execute.model.tables.StepInstanceConfirm; +import com.tencent.bk.job.execute.model.tables.records.StepInstanceConfirmRecord; +import org.jooq.Condition; +import org.jooq.OrderField; +import org.jooq.TableField; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * step_instance_confirm DAO + */ +public class StepInstanceConfirmRecordDAO extends AbstractJobInstanceHotRecordDAO { + + + private static final StepInstanceConfirm TABLE = StepInstanceConfirm.STEP_INSTANCE_CONFIRM; + + private static final List> ORDER_FIELDS = new ArrayList<>(); + + static { + ORDER_FIELDS.add(StepInstanceConfirm.STEP_INSTANCE_CONFIRM.STEP_INSTANCE_ID.asc()); + } + + + public StepInstanceConfirmRecordDAO(DSLContextProvider dslContextProvider) { + super(dslContextProvider, TABLE); + } + + @Override + public TableField getJobInstanceIdField() { + return TABLE.TASK_INSTANCE_ID; + } + + protected Collection> getListRecordsOrderFields() { + return ORDER_FIELDS; + } + + @Override + public RecordResultSet executeQuery(Collection jobInstanceIds, + long readRowLimit) { + return JobInstanceRecordResultSetFactory.createMultiQueryResultSet( + this, + jobInstanceIds, + readRowLimit, + lastRecord -> { + List conditions = new ArrayList<>(); + conditions.add(TABLE.STEP_INSTANCE_ID.gt(lastRecord.getStepInstanceId())); + return conditions; + } + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceFileRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceFileRecordDAO.java new file mode 100644 index 0000000000..c99cde1e03 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceFileRecordDAO.java @@ -0,0 +1,82 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.impl; + +import com.tencent.bk.job.backup.archive.dao.resultset.JobInstanceRecordResultSetFactory; +import com.tencent.bk.job.backup.archive.dao.resultset.RecordResultSet; +import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; +import com.tencent.bk.job.execute.model.tables.StepInstanceFile; +import com.tencent.bk.job.execute.model.tables.records.StepInstanceFileRecord; +import org.jooq.Condition; +import org.jooq.OrderField; +import org.jooq.TableField; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * step_instance_file DAO + */ +public class StepInstanceFileRecordDAO extends AbstractJobInstanceHotRecordDAO { + + private static final StepInstanceFile TABLE = StepInstanceFile.STEP_INSTANCE_FILE; + + private static final List> ORDER_FIELDS = new ArrayList<>(); + + static { + ORDER_FIELDS.add(StepInstanceFile.STEP_INSTANCE_FILE.STEP_INSTANCE_ID.asc()); + } + + + public StepInstanceFileRecordDAO(DSLContextProvider dslContextProvider) { + super(dslContextProvider, TABLE); + } + + + @Override + public TableField getJobInstanceIdField() { + return TABLE.TASK_INSTANCE_ID; + } + + protected Collection> getListRecordsOrderFields() { + return ORDER_FIELDS; + } + + @Override + public RecordResultSet executeQuery(Collection jobInstanceIds, + long readRowLimit) { + return JobInstanceRecordResultSetFactory.createMultiQueryResultSet( + this, + jobInstanceIds, + readRowLimit, + lastRecord -> { + List conditions = new ArrayList<>(); + conditions.add(TABLE.STEP_INSTANCE_ID.gt(lastRecord.getStepInstanceId())); + return conditions; + } + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceRecordDAO.java new file mode 100644 index 0000000000..2c062dc18e --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceRecordDAO.java @@ -0,0 +1,81 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.impl; + +import com.tencent.bk.job.backup.archive.dao.resultset.JobInstanceRecordResultSetFactory; +import com.tencent.bk.job.backup.archive.dao.resultset.RecordResultSet; +import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; +import com.tencent.bk.job.execute.model.tables.StepInstance; +import com.tencent.bk.job.execute.model.tables.records.StepInstanceRecord; +import org.jooq.Condition; +import org.jooq.OrderField; +import org.jooq.TableField; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * step_instance DAO + */ +public class StepInstanceRecordDAO extends AbstractJobInstanceHotRecordDAO { + + private static final StepInstance TABLE = StepInstance.STEP_INSTANCE; + + private static final List> ORDER_FIELDS = new ArrayList<>(); + + static { + ORDER_FIELDS.add(StepInstance.STEP_INSTANCE.ID.asc()); + } + + public StepInstanceRecordDAO(DSLContextProvider dslContextProvider) { + super(dslContextProvider, TABLE); + } + + + @Override + public TableField getJobInstanceIdField() { + return TABLE.TASK_INSTANCE_ID; + } + + protected Collection> getListRecordsOrderFields() { + return ORDER_FIELDS; + } + + @Override + public RecordResultSet executeQuery(Collection jobInstanceIds, + long readRowLimit) { + return JobInstanceRecordResultSetFactory.createMultiQueryResultSet( + this, + jobInstanceIds, + readRowLimit, + lastRecord -> { + List conditions = new ArrayList<>(); + conditions.add(TABLE.ID.gt(lastRecord.getId())); + return conditions; + } + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceRollingTaskRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceRollingTaskRecordDAO.java new file mode 100644 index 0000000000..0a409e9058 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceRollingTaskRecordDAO.java @@ -0,0 +1,81 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.impl; + +import com.tencent.bk.job.backup.archive.dao.resultset.JobInstanceRecordResultSetFactory; +import com.tencent.bk.job.backup.archive.dao.resultset.RecordResultSet; +import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; +import com.tencent.bk.job.execute.model.tables.StepInstanceRollingTask; +import com.tencent.bk.job.execute.model.tables.records.StepInstanceRollingTaskRecord; +import org.jooq.Condition; +import org.jooq.OrderField; +import org.jooq.TableField; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * step_instance_rolling_task DAO + */ +public class StepInstanceRollingTaskRecordDAO extends AbstractJobInstanceHotRecordDAO { + + private static final StepInstanceRollingTask TABLE = StepInstanceRollingTask.STEP_INSTANCE_ROLLING_TASK; + private static final List> ORDER_FIELDS = new ArrayList<>(); + + static { + ORDER_FIELDS.add(StepInstanceRollingTask.STEP_INSTANCE_ROLLING_TASK.ID.asc()); + } + + public StepInstanceRollingTaskRecordDAO(DSLContextProvider dslContextProvider) { + super(dslContextProvider, TABLE); + } + + + @Override + public TableField getJobInstanceIdField() { + return TABLE.TASK_INSTANCE_ID; + } + + protected Collection> getListRecordsOrderFields() { + return ORDER_FIELDS; + } + + @Override + public RecordResultSet executeQuery(Collection jobInstanceIds, + long readRowLimit) { + return JobInstanceRecordResultSetFactory.createMultiQueryResultSet( + this, + jobInstanceIds, + readRowLimit, + lastRecord -> { + List conditions = new ArrayList<>(); + conditions.add(TABLE.ID.gt(lastRecord.getId())); + return conditions; + } + ); + } + +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceScriptRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceScriptRecordDAO.java new file mode 100644 index 0000000000..1a3390982d --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceScriptRecordDAO.java @@ -0,0 +1,82 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.impl; + +import com.tencent.bk.job.backup.archive.dao.resultset.JobInstanceRecordResultSetFactory; +import com.tencent.bk.job.backup.archive.dao.resultset.RecordResultSet; +import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; +import com.tencent.bk.job.execute.model.tables.StepInstanceScript; +import com.tencent.bk.job.execute.model.tables.records.StepInstanceScriptRecord; +import org.jooq.Condition; +import org.jooq.OrderField; +import org.jooq.TableField; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * step_instance_script DAO + */ +public class StepInstanceScriptRecordDAO extends AbstractJobInstanceHotRecordDAO { + + private static final StepInstanceScript TABLE = StepInstanceScript.STEP_INSTANCE_SCRIPT; + + private static final List> ORDER_FIELDS = new ArrayList<>(); + + static { + ORDER_FIELDS.add(StepInstanceScript.STEP_INSTANCE_SCRIPT.STEP_INSTANCE_ID.asc()); + } + + + public StepInstanceScriptRecordDAO(DSLContextProvider dslContextProvider) { + super(dslContextProvider, TABLE); + } + + + @Override + public TableField getJobInstanceIdField() { + return TABLE.TASK_INSTANCE_ID; + } + + protected Collection> getListRecordsOrderFields() { + return ORDER_FIELDS; + } + + @Override + public RecordResultSet executeQuery(Collection jobInstanceIds, + long readRowLimit) { + return JobInstanceRecordResultSetFactory.createMultiQueryResultSet( + this, + jobInstanceIds, + readRowLimit, + lastRecord -> { + List conditions = new ArrayList<>(); + conditions.add(TABLE.STEP_INSTANCE_ID.gt(lastRecord.getStepInstanceId())); + return conditions; + } + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceVariableRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceVariableRecordDAO.java new file mode 100644 index 0000000000..26cc5f6e7a --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/StepInstanceVariableRecordDAO.java @@ -0,0 +1,80 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.impl; + +import com.tencent.bk.job.backup.archive.dao.resultset.JobInstanceRecordResultSetFactory; +import com.tencent.bk.job.backup.archive.dao.resultset.RecordResultSet; +import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; +import com.tencent.bk.job.execute.model.tables.StepInstanceVariable; +import com.tencent.bk.job.execute.model.tables.records.StepInstanceVariableRecord; +import org.jooq.Condition; +import org.jooq.OrderField; +import org.jooq.TableField; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * step_instance_variable DAO + */ +public class StepInstanceVariableRecordDAO extends AbstractJobInstanceHotRecordDAO { + + private static final StepInstanceVariable TABLE = StepInstanceVariable.STEP_INSTANCE_VARIABLE; + private static final List> ORDER_FIELDS = new ArrayList<>(); + + static { + ORDER_FIELDS.add(StepInstanceVariable.STEP_INSTANCE_VARIABLE.ID.asc()); + } + + public StepInstanceVariableRecordDAO(DSLContextProvider dslContextProvider) { + super(dslContextProvider, TABLE); + } + + + @Override + public TableField getJobInstanceIdField() { + return TABLE.TASK_INSTANCE_ID; + } + + protected Collection> getListRecordsOrderFields() { + return ORDER_FIELDS; + } + + @Override + public RecordResultSet executeQuery(Collection jobInstanceIds, + long readRowLimit) { + return JobInstanceRecordResultSetFactory.createMultiQueryResultSet( + this, + jobInstanceIds, + readRowLimit, + lastRecord -> { + List conditions = new ArrayList<>(); + conditions.add(TABLE.ID.gt(lastRecord.getId())); + return conditions; + } + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceHostRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceHostRecordDAO.java new file mode 100644 index 0000000000..6ae66f4f8f --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceHostRecordDAO.java @@ -0,0 +1,78 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.impl; + +import com.tencent.bk.job.backup.archive.dao.resultset.JobInstanceRecordResultSetFactory; +import com.tencent.bk.job.backup.archive.dao.resultset.RecordResultSet; +import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; +import com.tencent.bk.job.execute.model.tables.TaskInstanceHost; +import com.tencent.bk.job.execute.model.tables.records.TaskInstanceHostRecord; +import org.jooq.Condition; +import org.jooq.OrderField; +import org.jooq.TableField; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +public class TaskInstanceHostRecordDAO extends AbstractJobInstanceHotRecordDAO { + + private static final TaskInstanceHost TABLE = TaskInstanceHost.TASK_INSTANCE_HOST; + + private static final List> ORDER_FIELDS = new ArrayList<>(); + + static { + ORDER_FIELDS.add(TaskInstanceHost.TASK_INSTANCE_HOST.HOST_ID.asc()); + } + + public TaskInstanceHostRecordDAO(DSLContextProvider dslContextProvider) { + super(dslContextProvider, TABLE); + } + + + @Override + public TableField getJobInstanceIdField() { + return TABLE.TASK_INSTANCE_ID; + } + + protected Collection> getListRecordsOrderFields() { + return ORDER_FIELDS; + } + + @Override + public RecordResultSet executeQuery(Collection jobInstanceIds, + long readRowLimit) { + return JobInstanceRecordResultSetFactory.createMultiQueryResultSet( + this, + jobInstanceIds, + readRowLimit, + lastRecord -> { + List conditions = new ArrayList<>(); + conditions.add(TABLE.HOST_ID.gt(lastRecord.getHostId())); + return conditions; + } + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceVariableRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceVariableRecordDAO.java new file mode 100644 index 0000000000..18bd5456b0 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/impl/TaskInstanceVariableRecordDAO.java @@ -0,0 +1,80 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.impl; + +import com.tencent.bk.job.backup.archive.dao.resultset.JobInstanceRecordResultSetFactory; +import com.tencent.bk.job.backup.archive.dao.resultset.RecordResultSet; +import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; +import com.tencent.bk.job.execute.model.tables.TaskInstanceVariable; +import com.tencent.bk.job.execute.model.tables.records.TaskInstanceVariableRecord; +import org.jooq.Condition; +import org.jooq.OrderField; +import org.jooq.TableField; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +/** + * task_instance_variable DAO + */ +public class TaskInstanceVariableRecordDAO extends AbstractJobInstanceHotRecordDAO { + + private static final TaskInstanceVariable TABLE = TaskInstanceVariable.TASK_INSTANCE_VARIABLE; + + private static final List> ORDER_FIELDS = new ArrayList<>(); + + static { + ORDER_FIELDS.add(TaskInstanceVariable.TASK_INSTANCE_VARIABLE.ID.asc()); + } + + public TaskInstanceVariableRecordDAO(DSLContextProvider dslContextProvider) { + super(dslContextProvider, TABLE); + } + + @Override + public TableField getJobInstanceIdField() { + return TABLE.TASK_INSTANCE_ID; + } + + protected Collection> getListRecordsOrderFields() { + return ORDER_FIELDS; + } + + @Override + public RecordResultSet executeQuery(Collection jobInstanceIds, + long readRowLimit) { + return JobInstanceRecordResultSetFactory.createMultiQueryResultSet( + this, + jobInstanceIds, + readRowLimit, + lastRecord -> { + List conditions = new ArrayList<>(); + conditions.add(TABLE.ID.gt(lastRecord.getId())); + return conditions; + } + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/resultset/AbstractJobInstanceRecordResultSet.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/resultset/AbstractJobInstanceRecordResultSet.java new file mode 100644 index 0000000000..b6997f48e6 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/resultset/AbstractJobInstanceRecordResultSet.java @@ -0,0 +1,109 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.resultset; + +import com.tencent.bk.job.backup.archive.dao.impl.AbstractJobInstanceHotRecordDAO; +import org.apache.commons.collections4.CollectionUtils; +import org.jooq.Condition; +import org.jooq.Record; + +import java.util.Collection; +import java.util.List; + +/** + * 作业实例数据查询结果集 + * + * @param + */ +abstract class AbstractJobInstanceRecordResultSet implements RecordResultSet { + private final AbstractJobInstanceHotRecordDAO jobInstanceHotRecordDAO; + private final Collection jobInstanceIds; + private final long readRowLimit; + /** + * 当前查询的记录列表 + */ + private List records; + /** + * 当前查询的最后一条记录 + */ + private T lastRecord; + private boolean hasNext = true; + + + public AbstractJobInstanceRecordResultSet(AbstractJobInstanceHotRecordDAO jobInstanceHotRecordDAO, + Collection jobInstanceIds, + long readRowLimit) { + this.jobInstanceHotRecordDAO = jobInstanceHotRecordDAO; + this.jobInstanceIds = jobInstanceIds; + this.readRowLimit = readRowLimit; + } + + @Override + public boolean next() { + if (!hasNext) { + return false; + } + if (lastRecord == null) { + // 首次查询 + records = jobInstanceHotRecordDAO.listRecords(jobInstanceIds, readRowLimit); + } else { + // 非首次查询，需要加入数据查询偏移条件 + List offsetConditions = buildOffsetQueryConditions(lastRecord); + if (CollectionUtils.isEmpty(offsetConditions)) { + // 如果构造的数据查询偏移条件为空，不再继续查询 + hasNext = false; + records = null; + lastRecord = null; + return false; + } + List conditions = jobInstanceHotRecordDAO.buildBasicConditions(jobInstanceIds); + conditions.addAll(offsetConditions); + records = jobInstanceHotRecordDAO.listRecordsByConditions(conditions, readRowLimit); + } + + if (CollectionUtils.isEmpty(records)) { + hasNext = false; + lastRecord = null; + return false; + } else { + lastRecord = records.get(records.size() - 1); + hasNext = records.size() >= readRowLimit; + return true; + } + } + + /** + * 构造数据查询偏移条件 + * + * @param lastRecord 当前最后一条记录 + * @return 数据查询偏移条件 + */ + protected abstract List buildOffsetQueryConditions(T lastRecord); + + @Override + public List getRecords() { + return records; + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/resultset/JobInstanceRecordResultSetFactory.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/resultset/JobInstanceRecordResultSetFactory.java new file mode 100644 index 0000000000..9eb8471638 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/resultset/JobInstanceRecordResultSetFactory.java @@ -0,0 +1,102 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.resultset; + +import com.tencent.bk.job.backup.archive.dao.impl.AbstractJobInstanceHotRecordDAO; +import org.jooq.Condition; +import org.jooq.Record; + +import java.util.Collection; +import java.util.List; +import java.util.function.Function; + +public class JobInstanceRecordResultSetFactory { + + /** + * 创建一个多次分批查询 + * + * @param jobInstanceHotRecordDAO dao + * @param jobInstanceIds 作业实例 ID 列表 + * @param readRowLimit 单次查询最大读取行数限制 + * @param offsetConditionBuilder 查询偏移条件构造 + * @param 记录 + * @return ResultSet + */ + public static DefaultJobInstanceRecordResultSet createMultiQueryResultSet( + AbstractJobInstanceHotRecordDAO jobInstanceHotRecordDAO, + Collection jobInstanceIds, + long readRowLimit, + Function> offsetConditionBuilder) { + return new DefaultJobInstanceRecordResultSet<>( + jobInstanceHotRecordDAO, + jobInstanceIds, + readRowLimit, + offsetConditionBuilder + ); + } + + /** + * 创建一个单次全量查询 + * + * @param jobInstanceHotRecordDAO dao + * @param jobInstanceIds 作业实例 ID 列表 + * @param readRowLimit 单次查询最大读取行数限制 + * @param 记录 + * @return ResultSet + */ + public static DefaultJobInstanceRecordResultSet createOneQueryResultSet( + AbstractJobInstanceHotRecordDAO jobInstanceHotRecordDAO, + Collection jobInstanceIds, + long readRowLimit) { + return new DefaultJobInstanceRecordResultSet<>( + jobInstanceHotRecordDAO, + jobInstanceIds, + readRowLimit, + lastRecord -> null + ); + } + + public static class DefaultJobInstanceRecordResultSet + extends AbstractJobInstanceRecordResultSet { + + Function> offsetConditionBuilder; + + public DefaultJobInstanceRecordResultSet( + AbstractJobInstanceHotRecordDAO jobInstanceHotRecordDAO, + Collection jobInstanceIds, + long readRowLimit, + Function> offsetConditionBuilder) { + super(jobInstanceHotRecordDAO, jobInstanceIds, readRowLimit); + this.offsetConditionBuilder = offsetConditionBuilder; + } + + @Override + protected List buildOffsetQueryConditions(T lastRecord) { + return offsetConditionBuilder.apply(lastRecord); + } + } + + +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/resultset/RecordResultSet.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/resultset/RecordResultSet.java new file mode 100644 index 0000000000..be3aab312a --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/dao/resultset/RecordResultSet.java @@ -0,0 +1,46 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.dao.resultset; + +import org.jooq.Record; + +import java.util.List; + +/** + * 表查询结果 + * + * @param + */ +public interface RecordResultSet { + /** + * 将查询游标指向下一次查询 + */ + boolean next(); + + /** + * 获取下一次查询的表记录 + */ + List getRecords(); +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/AbstractJobInstanceSubTableArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/AbstractJobInstanceSubTableArchiver.java new file mode 100644 index 0000000000..c2c3a938c5 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/AbstractJobInstanceSubTableArchiver.java @@ -0,0 +1,124 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.impl; + +import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; +import com.tencent.bk.job.backup.archive.ArchiveTaskContextHolder; +import com.tencent.bk.job.backup.archive.JobInstanceSubTableArchiver; +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.impl.AbstractJobInstanceHotRecordDAO; +import com.tencent.bk.job.backup.archive.dao.resultset.RecordResultSet; +import com.tencent.bk.job.backup.archive.model.BackupResult; +import com.tencent.bk.job.backup.archive.model.DeleteResult; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.jooq.TableRecord; + +import java.util.List; + +@Slf4j +public class AbstractJobInstanceSubTableArchiver implements JobInstanceSubTableArchiver { + + private final JobInstanceColdDAO jobInstanceColdDAO; + + private final AbstractJobInstanceHotRecordDAO> jobInstanceHotRecordDAO; + + private final ArchiveTablePropsStorage archiveTablePropsStorage; + + protected String tableName; + + public AbstractJobInstanceSubTableArchiver( + JobInstanceColdDAO jobInstanceColdDAO, + AbstractJobInstanceHotRecordDAO> jobInstanceHotRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage + ) { + this.jobInstanceColdDAO = jobInstanceColdDAO; + this.jobInstanceHotRecordDAO = jobInstanceHotRecordDAO; + this.archiveTablePropsStorage = archiveTablePropsStorage; + this.tableName = jobInstanceHotRecordDAO.getTable().getName(); + } + + @Override + public BackupResult backupRecords(List jobInstanceIds) { + long startTime = System.currentTimeMillis(); + long backupRows = 0; + + RecordResultSet> recordResultSet = + jobInstanceHotRecordDAO.executeQuery(jobInstanceIds, + archiveTablePropsStorage.getReadRowLimit(tableName)); + // 数据偏移量 + int offset = 0; + long readStartTime = System.currentTimeMillis(); + while (recordResultSet.next()) { + List> records = recordResultSet.getRecords(); + long readEndTime = System.currentTimeMillis(); + int rowSize = CollectionUtils.isEmpty(records) ? 0 : records.size(); + long readCost = readEndTime - readStartTime; + log.info("[{}] Read {}, offset[{}-{}], readRows: {}, cost: {}ms", + ArchiveTaskContextHolder.getArchiveTaskId(), + tableName, + offset + 1, + offset + rowSize, + rowSize, + readCost + ); + if (readCost > 1000L) { + log.info("[{}] SlowQuery-ReadBackupRecords, table: {}, cost: {}ms", + ArchiveTaskContextHolder.getArchiveTaskId(), tableName, readCost); + } + if (CollectionUtils.isNotEmpty(records)) { + jobInstanceColdDAO.batchInsert(records, + archiveTablePropsStorage.getBatchInsertRowSize(tableName)); + backupRows += records.size(); + } + + readStartTime = System.currentTimeMillis(); + offset += rowSize; + } + + long costTime = System.currentTimeMillis() - startTime; + ArchiveTaskContextHolder.get().accumulateTableBackup(tableName, backupRows, costTime); + return new BackupResult(backupRows, costTime); + } + + @Override + public DeleteResult deleteRecords(List jobInstanceIds) { + long startTime = System.currentTimeMillis(); + int deleteRows = jobInstanceHotRecordDAO.deleteRecords(jobInstanceIds, + archiveTablePropsStorage.getDeleteLimitRowCount(tableName)); + log.info("[{}] Delete {}, deletedRows: {}, cost: {}ms", + ArchiveTaskContextHolder.getArchiveTaskId(), tableName, deleteRows, + System.currentTimeMillis() - startTime); + + long costTime = System.currentTimeMillis() - startTime; + ArchiveTaskContextHolder.get().accumulateTableDelete(tableName, deleteRows, costTime); + return new DeleteResult(deleteRows, costTime); + } + + @Override + public String getTableName() { + return tableName; + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/FileSourceTaskLogArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/FileSourceTaskLogArchiver.java new file mode 100644 index 0000000000..542fdc6f0c --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/FileSourceTaskLogArchiver.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.impl; + +import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.impl.FileSourceTaskLogRecordDAO; + + +public class FileSourceTaskLogArchiver extends AbstractJobInstanceSubTableArchiver { + + public FileSourceTaskLogArchiver( + JobInstanceColdDAO jobInstanceColdDAO, + FileSourceTaskLogRecordDAO jobInstanceHotRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage) { + super( + jobInstanceColdDAO, + jobInstanceHotRecordDAO, + archiveTablePropsStorage + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/FileSourceTaskLogArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/FileSourceTaskLogArchivist.java deleted file mode 100644 index d4a468bb2d..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/FileSourceTaskLogArchivist.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive.impl; - -import com.tencent.bk.job.backup.archive.AbstractArchivist; -import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.impl.FileSourceTaskRecordDAO; -import com.tencent.bk.job.backup.service.ArchiveProgressService; -import org.jooq.generated.tables.records.FileSourceTaskLogRecord; - -/** - * file_source_task_log 表归档 - */ -public class FileSourceTaskLogArchivist extends AbstractArchivist { - - public FileSourceTaskLogArchivist(FileSourceTaskRecordDAO executeRecordDAO, - ExecuteArchiveDAO executeArchiveDAO, - ArchiveProgressService archiveProgressService) { - super(executeRecordDAO, executeArchiveDAO, archiveProgressService); - this.deleteIdStepSize = 100_000; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseFileAgentTaskArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseFileAgentTaskArchiver.java new file mode 100644 index 0000000000..044b6ac4f0 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseFileAgentTaskArchiver.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.impl; + +import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.impl.GseFileAgentTaskRecordDAO; + + +public class GseFileAgentTaskArchiver extends AbstractJobInstanceSubTableArchiver { + + public GseFileAgentTaskArchiver( + JobInstanceColdDAO jobInstanceColdDAO, + GseFileAgentTaskRecordDAO jobInstanceHotRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage) { + super( + jobInstanceColdDAO, + jobInstanceHotRecordDAO, + archiveTablePropsStorage + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseFileAgentTaskArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseFileAgentTaskArchivist.java deleted file mode 100644 index 117abf3eba..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseFileAgentTaskArchivist.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive.impl; - -import com.tencent.bk.job.backup.archive.AbstractArchivist; -import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.impl.GseFileAgentTaskRecordDAO; -import com.tencent.bk.job.backup.service.ArchiveProgressService; -import org.jooq.generated.tables.records.GseFileAgentTaskRecord; - -/** - * gse_file_agent_task 表归档 - */ -public class GseFileAgentTaskArchivist extends AbstractArchivist { - - public GseFileAgentTaskArchivist(GseFileAgentTaskRecordDAO executeRecordDAO, - ExecuteArchiveDAO executeArchiveDAO, - ArchiveProgressService archiveProgressService) { - super(executeRecordDAO, executeArchiveDAO, archiveProgressService); - this.deleteIdStepSize = 1_000; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseFileExecuteObjTaskArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseFileExecuteObjTaskArchiver.java new file mode 100644 index 0000000000..d7216806b5 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseFileExecuteObjTaskArchiver.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.impl; + +import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.impl.GseFileExecuteObjTaskRecordDAO; + + +public class GseFileExecuteObjTaskArchiver extends AbstractJobInstanceSubTableArchiver { + + public GseFileExecuteObjTaskArchiver( + JobInstanceColdDAO jobInstanceColdDAO, + GseFileExecuteObjTaskRecordDAO jobInstanceHotRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage) { + super( + jobInstanceColdDAO, + jobInstanceHotRecordDAO, + archiveTablePropsStorage + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseScriptAgentTaskArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseScriptAgentTaskArchiver.java new file mode 100644 index 0000000000..56e10da34e --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseScriptAgentTaskArchiver.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.impl; + +import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.impl.GseScriptAgentTaskRecordDAO; + + +public class GseScriptAgentTaskArchiver extends AbstractJobInstanceSubTableArchiver { + + public GseScriptAgentTaskArchiver( + JobInstanceColdDAO jobInstanceColdDAO, + GseScriptAgentTaskRecordDAO jobInstanceHotRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage) { + super( + jobInstanceColdDAO, + jobInstanceHotRecordDAO, + archiveTablePropsStorage + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseScriptAgentTaskArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseScriptAgentTaskArchivist.java deleted file mode 100644 index 4580b37ba1..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseScriptAgentTaskArchivist.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive.impl; - -import com.tencent.bk.job.backup.archive.AbstractArchivist; -import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.impl.GseScriptAgentTaskRecordDAO; -import com.tencent.bk.job.backup.service.ArchiveProgressService; -import org.jooq.generated.tables.records.GseScriptAgentTaskRecord; - -/** - * gse_script_agent_task 表归档 - */ -public class GseScriptAgentTaskArchivist extends AbstractArchivist { - - public GseScriptAgentTaskArchivist(GseScriptAgentTaskRecordDAO executeRecordDAO, - ExecuteArchiveDAO executeArchiveDAO, - ArchiveProgressService archiveProgressService) { - super(executeRecordDAO, executeArchiveDAO, archiveProgressService); - this.deleteIdStepSize = 1_000; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseScriptExecuteObjTaskArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseScriptExecuteObjTaskArchiver.java new file mode 100644 index 0000000000..af2f6bee2e --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseScriptExecuteObjTaskArchiver.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.impl; + +import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.impl.GseScriptExecuteObjTaskRecordDAO; + + +public class GseScriptExecuteObjTaskArchiver extends AbstractJobInstanceSubTableArchiver { + + public GseScriptExecuteObjTaskArchiver( + JobInstanceColdDAO jobInstanceColdDAO, + GseScriptExecuteObjTaskRecordDAO jobInstanceHotRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage) { + super( + jobInstanceColdDAO, + jobInstanceHotRecordDAO, + archiveTablePropsStorage + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseTaskArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseTaskArchiver.java new file mode 100644 index 0000000000..57dbabd69d --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseTaskArchiver.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.impl; + +import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.impl.GseTaskRecordDAO; + + +public class GseTaskArchiver extends AbstractJobInstanceSubTableArchiver { + + public GseTaskArchiver( + JobInstanceColdDAO jobInstanceColdDAO, + GseTaskRecordDAO jobInstanceHotRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage) { + super( + jobInstanceColdDAO, + jobInstanceHotRecordDAO, + archiveTablePropsStorage + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseTaskArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseTaskArchivist.java deleted file mode 100644 index ece9ec48a5..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseTaskArchivist.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive.impl; - -import com.tencent.bk.job.backup.archive.AbstractArchivist; -import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.impl.GseTaskRecordDAO; -import com.tencent.bk.job.backup.service.ArchiveProgressService; -import org.jooq.generated.tables.records.GseTaskRecord; - -/** - * gse_task 表归档 - */ -public class GseTaskArchivist extends AbstractArchivist { - - public GseTaskArchivist(GseTaskRecordDAO executeRecordDAO, - ExecuteArchiveDAO executeArchiveDAO, - ArchiveProgressService archiveProgressService) { - super(executeRecordDAO, executeArchiveDAO, archiveProgressService); - this.deleteIdStepSize = 10_000; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseTaskIpLogArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseTaskIpLogArchivist.java deleted file mode 100644 index f53428c3c7..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseTaskIpLogArchivist.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive.impl; - -import com.tencent.bk.job.backup.archive.AbstractArchivist; -import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.impl.GseTaskIpLogRecordDAO; -import com.tencent.bk.job.backup.service.ArchiveProgressService; -import org.jooq.generated.tables.records.GseTaskIpLogRecord; - -/** - * gse_task_ip_log 表归档 - */ -public class GseTaskIpLogArchivist extends AbstractArchivist { - - public GseTaskIpLogArchivist(GseTaskIpLogRecordDAO executeRecordDAO, - ExecuteArchiveDAO executeArchiveDAO, - ArchiveProgressService archiveProgressService) { - super(executeRecordDAO, executeArchiveDAO, archiveProgressService); - this.deleteIdStepSize = 1_000; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseTaskLogArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseTaskLogArchivist.java deleted file mode 100644 index 85bfc6fd86..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/GseTaskLogArchivist.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive.impl; - -import com.tencent.bk.job.backup.archive.AbstractArchivist; -import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.impl.GseTaskLogRecordDAO; -import com.tencent.bk.job.backup.service.ArchiveProgressService; -import org.jooq.generated.tables.records.GseTaskLogRecord; - -/** - * gse_task_log 表归档 - */ -public class GseTaskLogArchivist extends AbstractArchivist { - - public GseTaskLogArchivist(GseTaskLogRecordDAO executeRecordDAO, - ExecuteArchiveDAO executeArchiveDAO, - ArchiveProgressService archiveProgressService) { - super(executeRecordDAO, executeArchiveDAO, archiveProgressService); - this.deleteIdStepSize = 10_000; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/JobFileLogArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/JobFileLogArchiver.java new file mode 100644 index 0000000000..df8bca38f7 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/JobFileLogArchiver.java @@ -0,0 +1,48 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.impl; + +import com.tencent.bk.job.backup.archive.AbstractJobLogArchiver; +import com.tencent.bk.job.backup.archive.service.ArchiveTaskService; +import com.tencent.bk.job.backup.config.JobLogArchiveProperties; +import com.tencent.bk.job.logsvr.consts.LogTypeEnum; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.mongodb.core.MongoTemplate; + +/** + * mongodb 文件日志归档 + */ +@Slf4j +public class JobFileLogArchiver extends AbstractJobLogArchiver { + + public JobFileLogArchiver(MongoTemplate mongoTemplate, + ArchiveTaskService archiveTaskService, + JobLogArchiveProperties archiveProperties) { + super(mongoTemplate, + archiveTaskService, + archiveProperties, + LogTypeEnum.FILE); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/JobScriptLogArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/JobScriptLogArchiver.java new file mode 100644 index 0000000000..523ba180a4 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/JobScriptLogArchiver.java @@ -0,0 +1,48 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.impl; + +import com.tencent.bk.job.backup.archive.AbstractJobLogArchiver; +import com.tencent.bk.job.backup.archive.service.ArchiveTaskService; +import com.tencent.bk.job.backup.config.JobLogArchiveProperties; +import com.tencent.bk.job.logsvr.consts.LogTypeEnum; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.mongodb.core.MongoTemplate; + +/** + * mongodb 脚本日志归档 + */ +@Slf4j +public class JobScriptLogArchiver extends AbstractJobLogArchiver { + + public JobScriptLogArchiver(MongoTemplate mongoTemplate, + ArchiveTaskService archiveTaskService, + JobLogArchiveProperties archiveProperties) { + super(mongoTemplate, + archiveTaskService, + archiveProperties, + LogTypeEnum.SCRIPT); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/OperationLogArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/OperationLogArchiver.java new file mode 100644 index 0000000000..b61703153a --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/OperationLogArchiver.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.impl; + +import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.impl.OperationLogRecordDAO; + + +public class OperationLogArchiver extends AbstractJobInstanceSubTableArchiver { + + public OperationLogArchiver( + JobInstanceColdDAO jobInstanceColdDAO, + OperationLogRecordDAO jobInstanceHotRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage) { + super( + jobInstanceColdDAO, + jobInstanceHotRecordDAO, + archiveTablePropsStorage + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/OperationLogArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/OperationLogArchivist.java deleted file mode 100644 index 2619917fe6..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/OperationLogArchivist.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive.impl; - -import com.tencent.bk.job.backup.archive.AbstractArchivist; -import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.impl.OperationLogRecordDAO; -import com.tencent.bk.job.backup.service.ArchiveProgressService; -import org.jooq.generated.tables.records.OperationLogRecord; - -/** - * operation_log 表归档 - */ -public class OperationLogArchivist extends AbstractArchivist { - - public OperationLogArchivist(OperationLogRecordDAO executeRecordDAO, - ExecuteArchiveDAO executeArchiveDAO, - ArchiveProgressService archiveProgressService) { - super(executeRecordDAO, executeArchiveDAO, archiveProgressService); - this.deleteIdStepSize = 10_000; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/RollingConfigArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/RollingConfigArchiver.java new file mode 100644 index 0000000000..a4a16ef4cb --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/RollingConfigArchiver.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.impl; + +import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.impl.RollingConfigRecordDAO; + + +public class RollingConfigArchiver extends AbstractJobInstanceSubTableArchiver { + + public RollingConfigArchiver( + JobInstanceColdDAO jobInstanceColdDAO, + RollingConfigRecordDAO jobInstanceHotRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage) { + super( + jobInstanceColdDAO, + jobInstanceHotRecordDAO, + archiveTablePropsStorage + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/RollingConfigArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/RollingConfigArchivist.java deleted file mode 100644 index c0f94ee9be..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/RollingConfigArchivist.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive.impl; - -import com.tencent.bk.job.backup.archive.AbstractArchivist; -import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.impl.RollingConfigRecordDAO; -import com.tencent.bk.job.backup.service.ArchiveProgressService; -import org.jooq.generated.tables.records.RollingConfigRecord; - -/** - * rolling_config 表归档 - */ -public class RollingConfigArchivist extends AbstractArchivist { - - public RollingConfigArchivist(RollingConfigRecordDAO executeRecordDAO, - ExecuteArchiveDAO executeArchiveDAO, - ArchiveProgressService archiveProgressService) { - super(executeRecordDAO, executeArchiveDAO, archiveProgressService); - this.deleteIdStepSize = 100_000; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceArchiver.java new file mode 100644 index 0000000000..ee404d25f0 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceArchiver.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.impl; + +import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.impl.StepInstanceRecordDAO; + + +public class StepInstanceArchiver extends AbstractJobInstanceSubTableArchiver { + + public StepInstanceArchiver( + JobInstanceColdDAO jobInstanceColdDAO, + StepInstanceRecordDAO jobInstanceHotRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage) { + super( + jobInstanceColdDAO, + jobInstanceHotRecordDAO, + archiveTablePropsStorage + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceArchivist.java deleted file mode 100644 index 1af5ceaaaf..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceArchivist.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive.impl; - -import com.tencent.bk.job.backup.archive.AbstractArchivist; -import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.impl.StepInstanceRecordDAO; -import com.tencent.bk.job.backup.service.ArchiveProgressService; -import org.jooq.generated.tables.records.StepInstanceRecord; - -/** - * step_instance 表归档 - */ -public class StepInstanceArchivist extends AbstractArchivist { - - public StepInstanceArchivist(StepInstanceRecordDAO executeRecordDAO, - ExecuteArchiveDAO executeArchiveDAO, - ArchiveProgressService archiveProgressService) { - super(executeRecordDAO, executeArchiveDAO, archiveProgressService); - this.deleteIdStepSize = 10_000; - } - - /** - * 获取作业实例ID范围内的步骤实例ID最大值 - * - * @param taskInstanceId 作业实例ID - * @return 步骤实例ID 最大值 - */ - public Long getMaxId(Long taskInstanceId) { - StepInstanceRecordDAO stepInstanceRecordDAO = (StepInstanceRecordDAO) executeRecordDAO; - return stepInstanceRecordDAO.getMaxId(taskInstanceId); - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceConfirmArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceConfirmArchiver.java new file mode 100644 index 0000000000..0a8f0c0f51 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceConfirmArchiver.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.impl; + +import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.impl.StepInstanceConfirmRecordDAO; + + +public class StepInstanceConfirmArchiver extends AbstractJobInstanceSubTableArchiver { + + public StepInstanceConfirmArchiver( + JobInstanceColdDAO jobInstanceColdDAO, + StepInstanceConfirmRecordDAO jobInstanceHotRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage) { + super( + jobInstanceColdDAO, + jobInstanceHotRecordDAO, + archiveTablePropsStorage + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceConfirmArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceConfirmArchivist.java deleted file mode 100644 index e144c72d45..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceConfirmArchivist.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive.impl; - -import com.tencent.bk.job.backup.archive.AbstractArchivist; -import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.impl.StepInstanceConfirmRecordDAO; -import com.tencent.bk.job.backup.service.ArchiveProgressService; -import org.jooq.generated.tables.records.StepInstanceConfirmRecord; - -/** - * step_instance_confirm 表归档 - */ -public class StepInstanceConfirmArchivist extends AbstractArchivist { - - public StepInstanceConfirmArchivist(StepInstanceConfirmRecordDAO executeRecordDAO, - ExecuteArchiveDAO executeArchiveDAO, - ArchiveProgressService archiveProgressService) { - super(executeRecordDAO, executeArchiveDAO, archiveProgressService); - this.deleteIdStepSize = 100_000; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceFileArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceFileArchiver.java new file mode 100644 index 0000000000..4e6b7fc933 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceFileArchiver.java @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.impl; + +import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.impl.StepInstanceFileRecordDAO; + + +public class StepInstanceFileArchiver extends AbstractJobInstanceSubTableArchiver { + + public StepInstanceFileArchiver( + JobInstanceColdDAO jobInstanceColdDAO, + StepInstanceFileRecordDAO jobInstanceHotRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage) { + super( + jobInstanceColdDAO, + jobInstanceHotRecordDAO, + archiveTablePropsStorage + ); + } + +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceFileArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceFileArchivist.java deleted file mode 100644 index e98f51bda1..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceFileArchivist.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive.impl; - -import com.tencent.bk.job.backup.archive.AbstractArchivist; -import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.impl.StepInstanceFileRecordDAO; -import com.tencent.bk.job.backup.service.ArchiveProgressService; -import org.jooq.generated.tables.records.StepInstanceFileRecord; - -/** - * step_instance_file 表归档 - */ -public class StepInstanceFileArchivist extends AbstractArchivist { - - public StepInstanceFileArchivist(StepInstanceFileRecordDAO executeRecordDAO, - ExecuteArchiveDAO executeArchiveDAO, - ArchiveProgressService archiveProgressService) { - super(executeRecordDAO, executeArchiveDAO, archiveProgressService); - this.deleteIdStepSize = 10_000; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceRollingTaskArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceRollingTaskArchiver.java new file mode 100644 index 0000000000..4b871ce9fb --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceRollingTaskArchiver.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.impl; + +import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.impl.StepInstanceRollingTaskRecordDAO; + + +public class StepInstanceRollingTaskArchiver extends AbstractJobInstanceSubTableArchiver { + + public StepInstanceRollingTaskArchiver( + JobInstanceColdDAO jobInstanceColdDAO, + StepInstanceRollingTaskRecordDAO jobInstanceHotRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage) { + super( + jobInstanceColdDAO, + jobInstanceHotRecordDAO, + archiveTablePropsStorage + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceRollingTaskArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceRollingTaskArchivist.java deleted file mode 100644 index 8b19a8941a..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceRollingTaskArchivist.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive.impl; - -import com.tencent.bk.job.backup.archive.AbstractArchivist; -import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.impl.StepInstanceRollingTaskRecordDAO; -import com.tencent.bk.job.backup.service.ArchiveProgressService; -import org.jooq.generated.tables.records.StepInstanceRollingTaskRecord; - -/** - * step_instance_rolling_task 表归档 - */ -public class StepInstanceRollingTaskArchivist extends AbstractArchivist { - - public StepInstanceRollingTaskArchivist(StepInstanceRollingTaskRecordDAO executeRecordDAO, - ExecuteArchiveDAO executeArchiveDAO, - ArchiveProgressService archiveProgressService) { - super(executeRecordDAO, executeArchiveDAO, archiveProgressService); - this.deleteIdStepSize = 100_000; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceScriptArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceScriptArchiver.java new file mode 100644 index 0000000000..f1f5326fa2 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceScriptArchiver.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.impl; + +import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.impl.StepInstanceScriptRecordDAO; + + +public class StepInstanceScriptArchiver extends AbstractJobInstanceSubTableArchiver { + + public StepInstanceScriptArchiver( + JobInstanceColdDAO jobInstanceColdDAO, + StepInstanceScriptRecordDAO jobInstanceHotRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage) { + super( + jobInstanceColdDAO, + jobInstanceHotRecordDAO, + archiveTablePropsStorage + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceScriptArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceScriptArchivist.java deleted file mode 100644 index c9374ae363..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceScriptArchivist.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive.impl; - -import com.tencent.bk.job.backup.archive.AbstractArchivist; -import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.impl.StepInstanceScriptRecordDAO; -import com.tencent.bk.job.backup.service.ArchiveProgressService; -import org.jooq.generated.tables.records.StepInstanceScriptRecord; - -/** - * step_instance_script 表归档 - */ -public class StepInstanceScriptArchivist extends AbstractArchivist { - - public StepInstanceScriptArchivist(StepInstanceScriptRecordDAO executeRecordDAO, - ExecuteArchiveDAO executeArchiveDAO, - ArchiveProgressService archiveProgressService) { - super(executeRecordDAO, executeArchiveDAO, archiveProgressService); - this.deleteIdStepSize = 10_000; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceVariableArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceVariableArchiver.java new file mode 100644 index 0000000000..4ac0a3b5c6 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceVariableArchiver.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.impl; + +import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.impl.StepInstanceVariableRecordDAO; + + +public class StepInstanceVariableArchiver extends AbstractJobInstanceSubTableArchiver { + + public StepInstanceVariableArchiver( + JobInstanceColdDAO jobInstanceColdDAO, + StepInstanceVariableRecordDAO jobInstanceHotRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage) { + super( + jobInstanceColdDAO, + jobInstanceHotRecordDAO, + archiveTablePropsStorage + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceVariableArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceVariableArchivist.java deleted file mode 100644 index 892adb58ef..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/StepInstanceVariableArchivist.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive.impl; - -import com.tencent.bk.job.backup.archive.AbstractArchivist; -import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.impl.StepInstanceVariableRecordDAO; -import com.tencent.bk.job.backup.service.ArchiveProgressService; -import org.jooq.generated.tables.records.StepInstanceVariableRecord; - -/** - * step_instance_variable 表归档 - */ -public class StepInstanceVariableArchivist extends AbstractArchivist { - - public StepInstanceVariableArchivist(StepInstanceVariableRecordDAO executeRecordDAO, - ExecuteArchiveDAO executeArchiveDAO, - ArchiveProgressService archiveProgressService) { - super(executeRecordDAO, executeArchiveDAO, archiveProgressService); - this.deleteIdStepSize = 100_000; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceArchivist.java deleted file mode 100644 index dff75b38ec..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceArchivist.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive.impl; - -import com.tencent.bk.job.backup.archive.AbstractArchivist; -import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.impl.TaskInstanceRecordDAO; -import com.tencent.bk.job.backup.service.ArchiveProgressService; -import org.jooq.generated.tables.records.TaskInstanceRecord; - -/** - * task_instance 表归档 - */ -public class TaskInstanceArchivist extends AbstractArchivist { - - public TaskInstanceArchivist(TaskInstanceRecordDAO executeRecordDAO, - ExecuteArchiveDAO executeArchiveDAO, - ArchiveProgressService archiveProgressService) { - super(executeRecordDAO, executeArchiveDAO, archiveProgressService); - this.deleteIdStepSize = 10_000; - } - - public Long getMaxId(Long endTime) { - TaskInstanceRecordDAO taskInstanceRecordDAO = (TaskInstanceRecordDAO) executeRecordDAO; - return taskInstanceRecordDAO.getMaxId(endTime); - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceHostArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceHostArchiver.java new file mode 100644 index 0000000000..de68b69ad3 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceHostArchiver.java @@ -0,0 +1,55 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.impl; + +import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.impl.TaskInstanceHostRecordDAO; +import com.tencent.bk.job.backup.archive.model.BackupResult; + +import java.util.List; + +import static com.tencent.bk.job.backup.archive.model.BackupResult.NON_OP_BACKUP_RESULT; + + +public class TaskInstanceHostArchiver extends AbstractJobInstanceSubTableArchiver { + + public TaskInstanceHostArchiver( + JobInstanceColdDAO jobInstanceColdDAO, + TaskInstanceHostRecordDAO jobInstanceHotRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage) { + super( + jobInstanceColdDAO, + jobInstanceHotRecordDAO, + archiveTablePropsStorage + ); + } + + @Override + public BackupResult backupRecords(List jobInstanceIds) { + // task_instance_host 表的数据暂时无需备份到冷存储 + return NON_OP_BACKUP_RESULT; + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceHostArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceHostArchivist.java deleted file mode 100644 index d8a9e51fef..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceHostArchivist.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive.impl; - -import com.tencent.bk.job.backup.archive.AbstractArchivist; -import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.impl.TaskInstanceHostRecordDAO; -import com.tencent.bk.job.backup.service.ArchiveProgressService; -import org.jooq.generated.tables.records.TaskInstanceHostRecord; - -/** - * gse_task 表归档 - */ -public class TaskInstanceHostArchivist extends AbstractArchivist { - - public TaskInstanceHostArchivist(TaskInstanceHostRecordDAO executeRecordDAO, - ExecuteArchiveDAO executeArchiveDAO, - ArchiveProgressService archiveProgressService) { - super(executeRecordDAO, executeArchiveDAO, archiveProgressService); - this.deleteIdStepSize = 10_000; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceVariableArchiver.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceVariableArchiver.java new file mode 100644 index 0000000000..cf142ff155 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceVariableArchiver.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.impl; + +import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.impl.TaskInstanceVariableRecordDAO; + + +public class TaskInstanceVariableArchiver extends AbstractJobInstanceSubTableArchiver { + + public TaskInstanceVariableArchiver( + JobInstanceColdDAO jobInstanceColdDAO, + TaskInstanceVariableRecordDAO jobInstanceHotRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage) { + super( + jobInstanceColdDAO, + jobInstanceHotRecordDAO, + archiveTablePropsStorage + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceVariableArchivist.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceVariableArchivist.java deleted file mode 100644 index 76ea71bec2..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/impl/TaskInstanceVariableArchivist.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.archive.impl; - -import com.tencent.bk.job.backup.archive.AbstractArchivist; -import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import com.tencent.bk.job.backup.dao.impl.TaskInstanceVariableRecordDAO; -import com.tencent.bk.job.backup.service.ArchiveProgressService; -import org.jooq.generated.tables.records.TaskInstanceVariableRecord; - -/** - * task_instance_variable 表归档 - */ -public class TaskInstanceVariableArchivist extends AbstractArchivist { - - public TaskInstanceVariableArchivist(TaskInstanceVariableRecordDAO executeRecordDAO, - ExecuteArchiveDAO executeArchiveDAO, - ArchiveProgressService archiveProgressService) { - super(executeRecordDAO, executeArchiveDAO, archiveProgressService); - this.deleteIdStepSize = 100_000; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/metrics/ArchiveTasksGauge.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/metrics/ArchiveTasksGauge.java new file mode 100644 index 0000000000..8a5458b987 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/metrics/ArchiveTasksGauge.java @@ -0,0 +1,100 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.metrics; + +import com.tencent.bk.job.backup.archive.service.ArchiveTaskService; +import com.tencent.bk.job.backup.constant.ArchiveTaskStatusEnum; +import com.tencent.bk.job.backup.constant.ArchiveTaskTypeEnum; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tags; +import org.springframework.scheduling.annotation.Scheduled; + +import java.util.Arrays; +import java.util.List; +import java.util.Map; + + +public class ArchiveTasksGauge { + + /** + * 监控指标 - 归档任务数量 + */ + private static final String METRIC_NAME_FAILED_ARCHIVE_TASK = "job.archive.task"; + + /** + * tag - 归档任务类型 + */ + private static final String TAG_TASK_TYPE = "task_type"; + + /** + * tag - 归档任务状态 + */ + private static final String TAG_STATUS = "status"; + + /** + * 需要监控的状态 + */ + private static final List MONITOR_STATUS_LIST = Arrays.asList( + ArchiveTaskStatusEnum.PENDING, + ArchiveTaskStatusEnum.SUSPENDED, + ArchiveTaskStatusEnum.RUNNING, + ArchiveTaskStatusEnum.FAIL + ); + + private volatile Map archiveTaskCountByStatus; + + private final ArchiveTaskService archiveTaskService; + + public ArchiveTasksGauge(MeterRegistry meterRegistry, + ArchiveTaskService archiveTaskService) { + this.archiveTaskService = archiveTaskService; + this.archiveTaskCountByStatus = archiveTaskService.countTaskByStatus( + ArchiveTaskTypeEnum.JOB_INSTANCE, MONITOR_STATUS_LIST); + for (ArchiveTaskStatusEnum status : MONITOR_STATUS_LIST) { + meterRegistry.gauge( + METRIC_NAME_FAILED_ARCHIVE_TASK, + Tags.of(TAG_TASK_TYPE, ArchiveTaskTypeEnum.JOB_INSTANCE.name(), + TAG_STATUS, status.name()), + this, + (ArchiveTasksGauge statObject) -> statObject.countByStatus(status)); + } + } + + private long countByStatus(ArchiveTaskStatusEnum status) { + Integer count = archiveTaskCountByStatus.get(status); + return count == null ? 0 : count; + } + + + /** + * 触发指标查询 + */ + @Scheduled(cron = "0 0/2 * * * *") + public void loadMetrics() { + this.archiveTaskCountByStatus = archiveTaskService.countTaskByStatus( + ArchiveTaskTypeEnum.JOB_INSTANCE, MONITOR_STATUS_LIST); + } + +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/ArchiveTableDetail.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/ArchiveTableDetail.java new file mode 100644 index 0000000000..672c9fe052 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/ArchiveTableDetail.java @@ -0,0 +1,79 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.model; + +import lombok.Data; + +/** + * 归档表执行性情 + */ +@Data +public class ArchiveTableDetail { + /** + * 写入归档冷 DB 的记录行数 + */ + private long backupRows; + /** + * 从热 DB 删除的记录行数 + */ + private long deleteRows; + /** + * 备份到冷 DB 耗时(毫秒) + */ + private long backupCostTime; + /** + * 删除热数据耗时(毫秒) + */ + private long deleteCostTime; + /** + * 总耗时(毫秒) + */ + private long costTime; + + /** + * 累加统计数据 - 备份 + * + * @param backupRows 已备份行数 + * @param costTime 耗时 + */ + public void accumulateBackup(long backupRows, long costTime) { + this.backupRows += backupRows; + this.backupCostTime += costTime; + this.costTime += costTime; + } + + /** + * 累加统计数据 - 删除 + * + * @param deleteRows 已删除行数 + * @param costTime 耗时 + */ + public void accumulateDelete(long deleteRows, long costTime) { + this.deleteRows += deleteRows; + this.deleteCostTime += costTime; + this.costTime += costTime; + } + +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/ArchiveTableProps.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/ArchiveTableProps.java new file mode 100644 index 0000000000..ac55a2d79a --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/ArchiveTableProps.java @@ -0,0 +1,49 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.model; + +import lombok.Data; + +@Data +public class ArchiveTableProps { + /** + * 从 热 DB 表中读取归档数据，每次读取的记录数量限制 + */ + private Integer readRowLimit; + /** + * 写入归档数据到冷 DB，单批次最小行数 + */ + private Integer batchInsertRowSize; + /** + * 从热 DB 删除数据，每次删除的最大行数 + */ + private Integer deleteLimitRowCount; + + public ArchiveTableProps(Integer readRowLimit, Integer batchInsertRowSize, Integer deleteLimitRowCount) { + this.readRowLimit = readRowLimit; + this.batchInsertRowSize = batchInsertRowSize; + this.deleteLimitRowCount = deleteLimitRowCount; + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/ArchiveTaskContext.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/ArchiveTaskContext.java new file mode 100644 index 0000000000..743f25024b --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/ArchiveTaskContext.java @@ -0,0 +1,48 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.model; + +import lombok.Data; + +@Data +public class ArchiveTaskContext { + + private ArchiveTaskInfo archiveTaskInfo; + + + public ArchiveTaskContext(ArchiveTaskInfo archiveTaskInfo) { + this.archiveTaskInfo = archiveTaskInfo; + } + + public void accumulateTableBackup(String tableName, long backupRows, long costTime) { + archiveTaskInfo.getOrInitExecutionDetail() + .accumulateTableBackup(tableName, backupRows, costTime); + } + + public void accumulateTableDelete(String tableName, long deleteRows, long costTime) { + archiveTaskInfo.getOrInitExecutionDetail() + .accumulateTableDelete(tableName, deleteRows, costTime); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/ArchiveTaskExecutionDetail.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/ArchiveTaskExecutionDetail.java new file mode 100644 index 0000000000..e1c16ee1f8 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/ArchiveTaskExecutionDetail.java @@ -0,0 +1,79 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.model; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.tencent.bk.job.common.annotation.PersistenceObject; +import lombok.Data; +import lombok.ToString; +import net.minidev.json.annotate.JsonIgnore; + +import java.util.HashMap; +import java.util.Map; + +/** + * 作业实例归档任务执行详情;通过 json 反序列化存储到 MySQL 中 + */ +@Data +@ToString +@PersistenceObject +@JsonInclude(JsonInclude.Include.NON_EMPTY) +public class ArchiveTaskExecutionDetail { + /** + * 归档任务耗时（毫秒） + */ + private long costTime; + /** + * 已归档的记录数量（主表） + */ + private long archivedRecordSize; + /** + * 执行错误信息 + */ + private String errorMsg; + + private Map tables = new HashMap<>(); + + public void accumulateTableBackup(String tableName, long backupRows, long costTime) { + ArchiveTableDetail archiveTableDetail = getOrInitArchiveTableDetail(tableName); + archiveTableDetail.accumulateBackup(backupRows, costTime); + } + + public void accumulateTableDelete(String tableName, long deleteRows, long costTime) { + ArchiveTableDetail archiveTableDetail = getOrInitArchiveTableDetail(tableName); + archiveTableDetail.accumulateDelete(deleteRows, costTime); + } + + @JsonIgnore + private ArchiveTableDetail getOrInitArchiveTableDetail(String tableName) { + ArchiveTableDetail archiveTableDetail = tables.get(tableName); + if (archiveTableDetail == null) { + archiveTableDetail = new ArchiveTableDetail(); + tables.put(tableName, archiveTableDetail); + } + return archiveTableDetail; + } + +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/ArchiveTaskInfo.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/ArchiveTaskInfo.java new file mode 100644 index 0000000000..988d16b818 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/ArchiveTaskInfo.java @@ -0,0 +1,106 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.model; + +import com.tencent.bk.job.backup.constant.ArchiveTaskStatusEnum; +import com.tencent.bk.job.backup.constant.ArchiveTaskTypeEnum; +import lombok.Data; +import lombok.ToString; + +/** + * 归档任务信息 + */ +@Data +@ToString +public class ArchiveTaskInfo { + /** + * 归档任务类型 + */ + private ArchiveTaskTypeEnum taskType; + /** + * 归档数据节点信息 + */ + private DbDataNode dbDataNode; + + /** + * 归档数据所在天.比如 20240806 + */ + private Integer day; + /** + * 归档数据所在小时。 0-23 + */ + private Integer hour; + /** + * 归档数据时间范围-from timestamp + */ + private Long fromTimestamp; + /** + * 归档数据时间范围-to timestamp + */ + private Long toTimestamp; + /** + * 归档进度 + */ + private IdBasedArchiveProcess process; + /** + * 归档任务状态 + */ + private ArchiveTaskStatusEnum status; + /** + * 归档任务创建时间 + */ + private Long createTime; + /** + * 归档任务最后更新时间 + */ + private Long lastUpdateTime; + /** + * 归档任务启动时间 + */ + private Long taskStartTime; + /** + * 归档任务结束时间 + */ + private Long taskEndTime; + /** + * 归档任务耗时，单位毫秒 + */ + private Long taskCost; + /** + * 归档任务运行详情 + */ + private ArchiveTaskExecutionDetail detail; + + public String buildTaskUniqueId() { + return taskType.getType() + ":" + day + ":" + hour + ":" + dbDataNode.toDataNodeId(); + } + + public ArchiveTaskExecutionDetail getOrInitExecutionDetail() { + if (detail == null) { + detail = new ArchiveTaskExecutionDetail(); + } + return detail; + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/BackupResult.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/BackupResult.java new file mode 100644 index 0000000000..f23d5a41ae --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/BackupResult.java @@ -0,0 +1,53 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.model; + +import lombok.Data; + +/** + * 归档-备份结果 + */ +@Data +public class BackupResult { + + /** + * 未执行备份操作返回的特殊BackupResult + */ + public static final BackupResult NON_OP_BACKUP_RESULT = new BackupResult(-1, -1); + + /** + * 备份成功的记录数量 + */ + private long backupRows; + /** + * 总耗时 + */ + private long cost; + + public BackupResult(long backupRows, long cost) { + this.backupRows = backupRows; + this.cost = cost; + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/DbDataNode.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/DbDataNode.java new file mode 100644 index 0000000000..66eb528121 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/DbDataNode.java @@ -0,0 +1,123 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.model; + +import com.tencent.bk.job.backup.constant.DbDataNodeTypeEnum; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * db 数据节点信息 + */ +@Data +@NoArgsConstructor +public class DbDataNode { + /** + * db 数据节点类型 + */ + private DbDataNodeTypeEnum type; + /** + * 分库分表数据源 + */ + private String dataSource; + /** + * 分库分表 db 实例位置索引，从 0 开始 + */ + private Integer dbIndex; + + /** + * 分库分表表位置索引，从 0 开始 + */ + private Integer tableIndex; + + public static final String STANDALONE_DS_NAME = "ds_standalone"; + + public DbDataNode(DbDataNodeTypeEnum type, String dataSource, Integer dbIndex, Integer tableIndex) { + this.type = type; + this.dataSource = dataSource; + this.dbIndex = dbIndex; + this.tableIndex = tableIndex; + } + + public String toDataNodeId() { + switch (type) { + case STANDALONE: + case SINGLE_MONGODB: + return type.getValue() + ":" + STANDALONE_DS_NAME; + case SHARDING: + return type.getValue() + ":" + dataSource + ":" + dbIndex + ":" + tableIndex; + default: + throw new IllegalArgumentException("Invalid DbDataNodeTypeEnum"); + } + } + + public String toDbNodeId() { + switch (type) { + case STANDALONE: + case SINGLE_MONGODB: + return STANDALONE_DS_NAME; + case SHARDING: + return dataSource + ":" + dbIndex; + default: + throw new IllegalArgumentException("Invalid DbDataNodeTypeEnum"); + } + } + + public static DbDataNode fromDataNodeId(String dataNodeId) { + String[] dataNodeParts = dataNodeId.split(":"); + DbDataNodeTypeEnum dbDataNodeType = DbDataNodeTypeEnum.valOf(Integer.parseInt(dataNodeParts[0])); + switch (dbDataNodeType) { + case STANDALONE: + case SINGLE_MONGODB: + return new DbDataNode(dbDataNodeType, STANDALONE_DS_NAME, null, null); + case SHARDING: + return new DbDataNode( + dbDataNodeType, + dataNodeParts[1], + Integer.parseInt(dataNodeParts[2]), + Integer.parseInt(dataNodeParts[3]) + ); + default: + throw new IllegalArgumentException("Invalid DbDataNodeId"); + } + } + + public static DbDataNode standaloneDbDataNode() { + return new DbDataNode(DbDataNodeTypeEnum.STANDALONE, STANDALONE_DS_NAME, null, null); + } + + public static DbDataNode shardingDbDataNode(String dataSource, Integer dbIndex, Integer tableIndex) { + return new DbDataNode(DbDataNodeTypeEnum.SHARDING, dataSource, dbIndex, tableIndex); + } + + public static DbDataNode standaloneMongoDbDataNode() { + return new DbDataNode(DbDataNodeTypeEnum.SINGLE_MONGODB, STANDALONE_DS_NAME, null, null); + } + + @Override + public DbDataNode clone() { + return new DbDataNode(type, dataSource, dbIndex, tableIndex); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/DeleteResult.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/DeleteResult.java new file mode 100644 index 0000000000..e5b830dfd4 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/DeleteResult.java @@ -0,0 +1,53 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.model; + +import lombok.Data; + +/** + * 归档-删除结果 + */ +@Data +public class DeleteResult { + + /** + * 删除成功的记录数量 + */ + private long deletedRows; + /** + * 删除耗时 + */ + private long deleteCost; + + public DeleteResult(long deletedRows, long deleteCost) { + this.deletedRows = deletedRows; + this.deleteCost = deleteCost; + } + + /** + * 未执行删除操作返回的特殊DeleteResult + */ + public static final DeleteResult NON_OP_DELETE_RESULT = new DeleteResult(-1, -1); +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/IdBasedArchiveProcess.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/IdBasedArchiveProcess.java new file mode 100644 index 0000000000..fc3900993f --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/IdBasedArchiveProcess.java @@ -0,0 +1,58 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.model; + +import lombok.Data; +import lombok.NoArgsConstructor; +import org.apache.commons.lang3.StringUtils; + +/** + * 基于自增ID的归档进度 + */ +@Data +@NoArgsConstructor +public class IdBasedArchiveProcess { + private Long id; + + public IdBasedArchiveProcess(Long id) { + this.id = id; + } + + @Override + public IdBasedArchiveProcess clone() { + return new IdBasedArchiveProcess(id); + } + + public String toPersistentProcess() { + return String.valueOf(id); + } + + public static IdBasedArchiveProcess fromPersistentProcess(String process) { + if (StringUtils.isEmpty(process)) { + return null; + } + return new IdBasedArchiveProcess(Long.parseLong(process)); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/TablesBackupResult.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/TablesBackupResult.java new file mode 100644 index 0000000000..1255c74870 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/TablesBackupResult.java @@ -0,0 +1,17 @@ +package com.tencent.bk.job.backup.archive.model; + +import lombok.Data; + +import java.util.HashMap; +import java.util.Map; + +@Data +public class TablesBackupResult { + + private Map tables = new HashMap<>(); + + public void add(String tableName, BackupResult backupResult) { + this.tables.put(tableName, backupResult); + } + +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/TablesDeleteResult.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/TablesDeleteResult.java new file mode 100644 index 0000000000..807e599bc2 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/model/TablesDeleteResult.java @@ -0,0 +1,16 @@ +package com.tencent.bk.job.backup.archive.model; + +import lombok.Data; + +import java.util.HashMap; +import java.util.Map; + +@Data +public class TablesDeleteResult { + + private Map tables = new HashMap<>(); + + public void add(String tableName, DeleteResult deleteResult) { + this.tables.put(tableName, deleteResult); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/service/ArchiveTaskService.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/service/ArchiveTaskService.java new file mode 100644 index 0000000000..40d97d5f26 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/service/ArchiveTaskService.java @@ -0,0 +1,271 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.service; + +import com.tencent.bk.job.backup.archive.dao.ArchiveTaskDAO; +import com.tencent.bk.job.backup.archive.model.ArchiveTaskExecutionDetail; +import com.tencent.bk.job.backup.archive.model.DbDataNode; +import com.tencent.bk.job.backup.archive.model.ArchiveTaskInfo; +import com.tencent.bk.job.backup.archive.model.IdBasedArchiveProcess; +import com.tencent.bk.job.backup.constant.ArchiveTaskStatusEnum; +import com.tencent.bk.job.backup.constant.ArchiveTaskTypeEnum; +import com.tencent.bk.job.common.mysql.JobTransactional; +import org.apache.commons.collections4.CollectionUtils; +import org.springframework.stereotype.Service; + +import java.util.Collection; +import java.util.List; +import java.util.Map; + +@Service +public class ArchiveTaskService { + + private final ArchiveTaskDAO archiveTaskDAO; + + public ArchiveTaskService(ArchiveTaskDAO archiveTaskDAO) { + this.archiveTaskDAO = archiveTaskDAO; + } + + /** + * 获取最新创建的归档任务 + * + * @param taskType 归档任务类型 + */ + public ArchiveTaskInfo getLatestArchiveTask(ArchiveTaskTypeEnum taskType) { + return archiveTaskDAO.getLatestArchiveTask(taskType); + } + + public void saveArchiveTask(ArchiveTaskInfo archiveTaskInfo) { + archiveTaskDAO.saveArchiveTask(archiveTaskInfo); + } + + @JobTransactional(transactionManager = "jobBackupTransactionManager") + public void saveArchiveTasks(Collection archiveTaskList) { + if (CollectionUtils.isNotEmpty(archiveTaskList)) { + archiveTaskList.forEach(archiveTaskDAO::saveArchiveTask); + } + } + + public List listRunningTasks(ArchiveTaskTypeEnum taskType) { + return archiveTaskDAO.listRunningTasks(taskType); + } + + /** + * 获取归档任务 + * + * @param taskType 查询条件 - 任务类型 + * @param status 查询条件 - 任务状态 + * @param limit 查询条件 - 查询最大数量 + * @return 归档任务列表 + */ + public List listTasks(ArchiveTaskTypeEnum taskType, + ArchiveTaskStatusEnum status, + int limit) { + return archiveTaskDAO.listTasks(taskType, status, limit); + } + + /** + * 获取归档任务 + * + * @param status 查询条件 - 任务状态 + * @param limit 查询条件 - 查询最大数量 + * @return 归档任务列表 + */ + public List listTasks(ArchiveTaskStatusEnum status, + int limit) { + return archiveTaskDAO.listTasks(status, limit); + } + + /** + * 返回根据 db 分组的归档任务数量 + * + * @param taskType 归档任务类型 + * @return key: db 名称; value: 任务数量 + */ + public Map countScheduleTasksGroupByDb(ArchiveTaskTypeEnum taskType) { + return archiveTaskDAO.countScheduleTasksGroupByDb(taskType); + } + + + /** + * 更新归档任务执行信息 - 启动后 + * + * @param taskType 任务类型 + * @param dataNode 数据节点 + * @param day 归档数据所在天 + * @param hour 归档数据所在小时 + * @param startTime 任务开始时间 + */ + public void updateStartedExecuteInfo(ArchiveTaskTypeEnum taskType, + DbDataNode dataNode, + Integer day, + Integer hour, + Long startTime) { + archiveTaskDAO.updateStartedExecuteInfo( + taskType, + dataNode, + day, + hour, + startTime + ); + + } + + /** + * 更新归档任务执行信息 - 运行中 + * + * @param taskType 任务类型 + * @param dataNode 数据节点 + * @param day 归档数据所在天 + * @param hour 归档数据所在小时 + * @param process 进度 + */ + public void updateRunningExecuteInfo(ArchiveTaskTypeEnum taskType, + DbDataNode dataNode, + Integer day, + Integer hour, + IdBasedArchiveProcess process) { + archiveTaskDAO.updateRunningExecuteInfo( + taskType, + dataNode, + day, + hour, + process + ); + + } + + /** + * 更新归档任务执行信息 - 结束 + * + * @param taskType 任务类型 + * @param dataNode 数据节点 + * @param day 归档数据所在天 + * @param hour 归档数据所在小时 + * @param status 任务状态 + * @param process 进度 + * @param endTime 结束时间 + * @param cost 任务耗时 + * @param detail 执行详情 + */ + public void updateCompletedExecuteInfo(ArchiveTaskTypeEnum taskType, + DbDataNode dataNode, + Integer day, + Integer hour, + ArchiveTaskStatusEnum status, + IdBasedArchiveProcess process, + Long endTime, + Long cost, + ArchiveTaskExecutionDetail detail) { + archiveTaskDAO.updateCompletedExecuteInfo( + taskType, + dataNode, + day, + hour, + status, + process, + endTime, + cost, + detail + ); + } + + public ArchiveTaskInfo getFirstScheduleArchiveTaskByDb(ArchiveTaskTypeEnum taskType, String dbNodeId) { + return archiveTaskDAO.getFirstScheduleArchiveTaskByDb(taskType, dbNodeId); + } + + public ArchiveTaskInfo getFirstScheduleArchiveTask(ArchiveTaskTypeEnum taskType) { + return archiveTaskDAO.getFirstScheduleArchiveTask(taskType); + } + + /** + * 更新归档任务执行状态 + * + * @param taskType 任务类型 + * @param dataNode 数据节点 + * @param day 归档数据所在天 + * @param hour 归档数据所在小时 + * @param status 任务状态 + */ + public void updateArchiveTaskStatus(ArchiveTaskTypeEnum taskType, + DbDataNode dataNode, + Integer day, + Integer hour, + ArchiveTaskStatusEnum status) { + archiveTaskDAO.updateArchiveTaskStatus(taskType, dataNode, day, hour, status); + } + + /** + * 按照任务类型和状态，统计归档任务数量 + * + * @param taskType 任务类型 + * @param statusList 状态列表 + * @return 数量 + */ + public Map countTaskByStatus(ArchiveTaskTypeEnum taskType, + List statusList) { + return archiveTaskDAO.countTaskByStatus(taskType, statusList); + } + + /** + * 更新归档任务执行详情 + * + * @param taskType 任务类型 + * @param dataNode 数据节点 + * @param day 归档数据所在天 + * @param hour 归档数据所在小时 + * @param detail 执行详情 + */ + public void updateExecutionDetail(ArchiveTaskTypeEnum taskType, + DbDataNode dataNode, + Integer day, + Integer hour, + ArchiveTaskExecutionDetail detail) { + archiveTaskDAO.updateExecutionDetail(taskType, dataNode, day, hour, detail); + } + + /** + * 获取归档任务 + * + * @param taskType 查询条件 - 任务类型 + * @param day 查询条件 - 归档数据所在天 + * @return 归档任务列表 + */ + public List listTasks(ArchiveTaskTypeEnum taskType, + Integer day) { + return archiveTaskDAO.listTasks(taskType, day); + } + + /** + * 获取从某天开始的归档任务 + * + * @param taskType 查询条件 - 任务类型 + * @param day 查询条件 - 归档数据所在天 + * @return 归档任务列表 + */ + public List listTasksSinceDay(ArchiveTaskTypeEnum taskType, Integer day) { + return archiveTaskDAO.listTasksSinceDay(taskType, day); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/ArchiveDateTimeUtil.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/ArchiveDateTimeUtil.java new file mode 100644 index 0000000000..f6c9fb3a25 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/ArchiveDateTimeUtil.java @@ -0,0 +1,108 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.util; + +import com.tencent.bk.job.common.util.date.DateUtils; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +import java.time.DateTimeException; +import java.time.Instant; +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.ZoneId; + +/** + * 归档日期计算工具类 + */ +@Slf4j +public class ArchiveDateTimeUtil { + /** + * 计算天（日期），返回 yyyyMMdd 格式的数字 + * + * @param dateTime 日期 + * @return 天（日期） + */ + public static int computeDay(LocalDateTime dateTime) { + return Integer.parseInt(DateUtils.formatLocalDateTime(dateTime, "yyyyMMdd")); + } + + /** + * 计算小时 + * + * @param dateTime 日期 + * @return 小时 + */ + public static int computeHour(LocalDateTime dateTime) { + return dateTime.getHour(); + } + + public static LocalDateTime computeStartOfDayBeforeDays(LocalDateTime now, int beforeDays) { + // 当前日期的最早时间 + LocalDateTime startOfDay = now.toLocalDate().atStartOfDay(); + return startOfDay.minusDays(beforeDays); + } + + public static LocalDateTime unixTimestampMillToZoneDateTime(long unixTimestampMill, ZoneId zoneId) { + // 创建一个 Instant 对象，表示从 1970-01-01T00:00:00Z 开始的指定毫秒数 + Instant instant = Instant.ofEpochMilli(unixTimestampMill); + // 将 Instant 对象转换为 对应时区的 LocalDateTime 对象 + return LocalDateTime.ofInstant(instant, zoneId); + } + + public static LocalDateTime toHourlyRoundDown(LocalDateTime localDateTime) { + return localDateTime.withMinute(0).withSecond(0).withNano(0); + } + + public static long toTimestampMillsAtZone(LocalDateTime localDateTime, ZoneId zoneId) { + OffsetDateTime offsetDateTime = + localDateTime.atOffset(zoneId.getRules().getOffset(localDateTime)); + return 1000 * offsetDateTime.toEpochSecond(); + } + + /** + * 根据配置的时区计算所依据时区 + * + * @param timeZone 时区 + * @return 时区 + */ + public static ZoneId getArchiveBasedTimeZone(String timeZone) throws DateTimeException { + ZoneId zoneId; + if (StringUtils.isBlank(timeZone)) { + zoneId = ZoneId.systemDefault(); + log.info("Use system zone as archive base time zone, zoneId: {}", zoneId); + return zoneId; + } + zoneId = ZoneId.of(timeZone); + log.info("Use configured zone as archive base time zone, zoneId: {}", zoneId); + return zoneId; + } + + public static LocalDateTime computeArchiveEndTime(int archiveDays, ZoneId zoneId) { + log.info("Compute archive task generate end time before {} days", archiveDays); + LocalDateTime now = LocalDateTime.now(zoneId); + return ArchiveDateTimeUtil.computeStartOfDayBeforeDays(now, archiveDays); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/ServiceNodeInfo.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/ServiceNodeInfo.java new file mode 100644 index 0000000000..192f4456d0 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/ServiceNodeInfo.java @@ -0,0 +1,31 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.util; + +import com.tencent.bk.job.common.util.ip.IpUtils; + +public class ServiceNodeInfo { + public static final String NODE_IP = IpUtils.getFirstMachineIP(); +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/ArchiveLogTaskExecuteLock.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/ArchiveLogTaskExecuteLock.java new file mode 100644 index 0000000000..47a528ed27 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/ArchiveLogTaskExecuteLock.java @@ -0,0 +1,58 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.util.lock; + +import com.tencent.bk.job.common.redis.util.HeartBeatRedisLockConfig; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.redis.core.StringRedisTemplate; + +/** + * 归档执行日志任务执行分布式锁 + */ +@Slf4j +public class ArchiveLogTaskExecuteLock { + + private final HeartBeatRedisLocks locks; + + public ArchiveLogTaskExecuteLock(StringRedisTemplate redisTemplate) { + locks = new HeartBeatRedisLocks( + "archive:log:task:execute", + redisTemplate, + new HeartBeatRedisLockConfig( + "RedisKeyHeartBeatThread-archive:task:log:execute", + 600 * 1000L, // 10min 超时时间 + 60 * 1000L // 1min 续期一次 + ) + ); + } + + public boolean lock(String taskId) { + return locks.lock(taskId); + } + + public void unlock(String taskId) { + locks.unlock(taskId); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/ArchiveTaskExecuteLock.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/ArchiveTaskExecuteLock.java new file mode 100644 index 0000000000..ca452333ac --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/ArchiveTaskExecuteLock.java @@ -0,0 +1,58 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.util.lock; + +import com.tencent.bk.job.common.redis.util.HeartBeatRedisLockConfig; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.redis.core.StringRedisTemplate; + +/** + * 归档任务执行分布式锁 + */ +@Slf4j +public class ArchiveTaskExecuteLock { + + private final HeartBeatRedisLocks locks; + + public ArchiveTaskExecuteLock(StringRedisTemplate redisTemplate) { + locks = new HeartBeatRedisLocks( + "archive:task:execute", + redisTemplate, + new HeartBeatRedisLockConfig( + "RedisKeyHeartBeatThread-archive:task:execute", + 3600 * 1000L, // 1h 超时时间 + 600 * 1000L // 10min 续期一次 + ) + ); + } + + public boolean lock(String taskId) { + return locks.lock(taskId); + } + + public void unlock(String taskId) { + locks.unlock(taskId); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/FailedArchiveTaskRescheduleLock.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/FailedArchiveTaskRescheduleLock.java new file mode 100644 index 0000000000..dec1d42cf6 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/FailedArchiveTaskRescheduleLock.java @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.util.lock; + +import com.tencent.bk.job.common.redis.util.HeartBeatRedisLockConfig; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.redis.core.StringRedisTemplate; + +/** + * 失败/超时归档任务重调度分布式锁 + */ +@Slf4j +public class FailedArchiveTaskRescheduleLock extends PreemptiveDistributeLock { + + public FailedArchiveTaskRescheduleLock(StringRedisTemplate redisTemplate) { + super(redisTemplate, + "fail:archive:task:reschedule", + new HeartBeatRedisLockConfig( + "RedisKeyHeartBeatThread-fail:archive:task:reschedule", + 300 * 1000L, // 5min 超时时间 + 60 * 1000L // 1min 续期一次 + )); + } + +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/FairDistributeLock.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/FairDistributeLock.java new file mode 100644 index 0000000000..8254b67c1e --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/FairDistributeLock.java @@ -0,0 +1,89 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.util.lock; + +import com.tencent.bk.job.common.redis.util.LockUtils; +import com.tencent.bk.job.common.util.ThreadUtils; +import lombok.extern.slf4j.Slf4j; + +/** + * 分布式锁（公平锁，非饥饿抢占方式) + */ +@Slf4j +public class FairDistributeLock { + /** + * 锁名称 + */ + private final String lockName; + + private final String lockKey; + /** + * 锁时间（毫秒) + */ + private final Long lockMills; + /** + * 最小获取锁间隔时间 + */ + private final long MIN_ACQUIRE_LOCK_INTERVAL_MS = 1000L; + + private volatile long lastAcquireLockTimeMS = 0L; + + private volatile String lockRequestId = null; + + public FairDistributeLock(String lockName, String lockKey, long lockMills) { + this.lockName = lockName; + this.lockKey = lockKey; + this.lockMills = lockMills; + } + + public synchronized boolean lock() { + while (System.currentTimeMillis() - lastAcquireLockTimeMS < MIN_ACQUIRE_LOCK_INTERVAL_MS) { + // 为了保证在分布式系统中多个节点都能均匀获取到任务，需要最小获取锁间隔时间，让其他服务节点优先获取任务锁 + ThreadUtils.sleep(100L); + } + String lockRequestId = LockUtil.generateLockRequestId(); + if (!LockUtils.tryGetDistributedLock(lockKey, lockRequestId, lockMills)) { + log.info("[{}] Acquire lock failed!", lockName); + return false; + } else { + log.info("[{}] Acquire lock successfully!", lockName); + this.lastAcquireLockTimeMS = System.currentTimeMillis(); + this.lockRequestId = lockRequestId; + return true; + } + } + + + public synchronized boolean unlock() { + boolean success = LockUtils.releaseDistributedLock(lockKey, lockRequestId); + if (success) { + log.info("[{}] Release lock successfully", lockName); + this.lockRequestId = null; + } else { + log.warn("[{}] Release lock fail", lockName); + } + return success; + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/HeartBeatRedisLocks.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/HeartBeatRedisLocks.java new file mode 100644 index 0000000000..40fdcada47 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/HeartBeatRedisLocks.java @@ -0,0 +1,101 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.util.lock; + +import com.tencent.bk.job.common.redis.util.HeartBeatRedisLock; +import com.tencent.bk.job.common.redis.util.HeartBeatRedisLockConfig; +import com.tencent.bk.job.common.redis.util.LockResult; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.redis.core.StringRedisTemplate; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +/** + * 分布式锁(自动续期) + */ +@Slf4j +public class HeartBeatRedisLocks { + /** + * 锁 key 前缀 + */ + private final String lockKeyPrefix; + /** + * Key: lockKey ; Value: 分布式锁 + */ + private final Map locks = new ConcurrentHashMap<>(); + + + private final StringRedisTemplate redisTemplate; + + private final HeartBeatRedisLockConfig heartBeatRedisLockConfig; + + public HeartBeatRedisLocks(String lockKeyPrefix, + StringRedisTemplate redisTemplate, + HeartBeatRedisLockConfig heartBeatRedisLockConfig) { + this.lockKeyPrefix = lockKeyPrefix; + this.redisTemplate = redisTemplate; + if (heartBeatRedisLockConfig == null) { + this.heartBeatRedisLockConfig = HeartBeatRedisLockConfig.getDefault(); + } else { + this.heartBeatRedisLockConfig = heartBeatRedisLockConfig; + } + } + + public synchronized boolean lock(String lockKey) { + String lockRequestId = LockUtil.generateLockRequestId(); + String actualLockKey = buildActualLockKey(lockKey); + HeartBeatRedisLock redisLock = + new HeartBeatRedisLock(redisTemplate, actualLockKey, lockRequestId, heartBeatRedisLockConfig); + + LockResult lockResult = redisLock.lock(); + if (!lockResult.isLockGotten()) { + log.warn("Lock is held by another process: {}", lockResult.getLockValue()); + return false; + } + + this.locks.put(lockKey, lockResult); + return true; + } + + private String buildActualLockKey(String lockKey) { + return lockKeyPrefix + ":" + lockKey; + } + + + public synchronized void unlock(String lockKey) { + LockResult lockResult = this.locks.get(lockKey); + if (lockResult == null) { + log.warn("RedisLock is not found, lockKey: {}", lockKey); + return; + } + try { + lockResult.tryToRelease(); + } finally { + locks.remove(lockKey); + } + } + +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/JobInstanceArchiveTaskGenerateLock.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/JobInstanceArchiveTaskGenerateLock.java new file mode 100644 index 0000000000..4416428444 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/JobInstanceArchiveTaskGenerateLock.java @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.util.lock; + +import com.tencent.bk.job.common.redis.util.HeartBeatRedisLockConfig; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.redis.core.StringRedisTemplate; + +/** + * 归档作业执行历史任务创建分布式锁 + */ +@Slf4j +public class JobInstanceArchiveTaskGenerateLock extends PreemptiveDistributeLock { + + + public JobInstanceArchiveTaskGenerateLock(StringRedisTemplate redisTemplate) { + super(redisTemplate, + "job:instance:archive:task:generate", + new HeartBeatRedisLockConfig( + "RedisKeyHeartBeatThread-job:instance:archive:task:generate", + 60 * 1000L, // 60s 超时时间 + 10 * 1000L // 10s 续期一次 + )); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/JobInstanceArchiveTaskScheduleLock.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/JobInstanceArchiveTaskScheduleLock.java new file mode 100644 index 0000000000..d38682311b --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/JobInstanceArchiveTaskScheduleLock.java @@ -0,0 +1,42 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.util.lock; + +import lombok.extern.slf4j.Slf4j; + +/** + * 作业执行历史归档任务调度分布式锁 + */ +@Slf4j +public class JobInstanceArchiveTaskScheduleLock extends FairDistributeLock { + + public JobInstanceArchiveTaskScheduleLock() { + super( + "job:instance:archive:task:schedule:lock", + "job:instance:archive:task:schedule", + 60 * 1000L + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/JobLogArchiveTaskGenerateLock.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/JobLogArchiveTaskGenerateLock.java new file mode 100644 index 0000000000..cc57e92400 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/JobLogArchiveTaskGenerateLock.java @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.util.lock; + +import com.tencent.bk.job.common.redis.util.HeartBeatRedisLockConfig; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.redis.core.StringRedisTemplate; + +/** + * 归档执行日志任务创建分布式锁 + */ +@Slf4j +public class JobLogArchiveTaskGenerateLock extends PreemptiveDistributeLock { + + + public JobLogArchiveTaskGenerateLock(StringRedisTemplate redisTemplate) { + super(redisTemplate, + "job:execute:log:archive:task:generate", + new HeartBeatRedisLockConfig( + "RedisKeyHeartBeatThread-job:execute:log:archive:task:generate", + 60 * 1000L, // 60s 超时时间 + 10 * 1000L // 10s 续期一次 + )); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/JobLogArchiveTaskScheduleLock.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/JobLogArchiveTaskScheduleLock.java new file mode 100644 index 0000000000..018330daac --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/JobLogArchiveTaskScheduleLock.java @@ -0,0 +1,42 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.util.lock; + +import lombok.extern.slf4j.Slf4j; + +/** + * 作业执行日志归档任务调度分布式锁 + */ +@Slf4j +public class JobLogArchiveTaskScheduleLock extends FairDistributeLock { + + public JobLogArchiveTaskScheduleLock() { + super( + "job:execute:log:archive:task:schedule:lock", + "job:execute:log:archive:task:schedule", + 60 * 1000L + ); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/LockUtil.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/LockUtil.java new file mode 100644 index 0000000000..06c16a2bc4 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/LockUtil.java @@ -0,0 +1,43 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.util.lock; + +import com.tencent.bk.job.backup.archive.util.ServiceNodeInfo; + +import java.util.UUID; + +/** + * 分布式锁工具类 + */ +public class LockUtil { + /** + * 生成随机的分布式锁 requestId + * + */ + public static String generateLockRequestId() { + return ServiceNodeInfo.NODE_IP + "_" + UUID.randomUUID(); + } + +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/PreemptiveDistributeLock.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/PreemptiveDistributeLock.java new file mode 100644 index 0000000000..ef4a025ac7 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/archive/util/lock/PreemptiveDistributeLock.java @@ -0,0 +1,84 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.util.lock; + +import com.tencent.bk.job.common.redis.util.HeartBeatRedisLock; +import com.tencent.bk.job.common.redis.util.HeartBeatRedisLockConfig; +import com.tencent.bk.job.common.redis.util.LockResult; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.redis.core.StringRedisTemplate; + +/** + * 抢占锁 + */ +@Slf4j +public class PreemptiveDistributeLock { + + private final StringRedisTemplate redisTemplate; + + private final String redisLockKey; + + private final HeartBeatRedisLockConfig heartBeatRedisLockConfig; + + private volatile LockResult lockResult = null; + + public PreemptiveDistributeLock(StringRedisTemplate redisTemplate, + String redisLockKey, + HeartBeatRedisLockConfig heartBeatRedisLockConfig) { + this.redisTemplate = redisTemplate; + this.redisLockKey = redisLockKey; + this.heartBeatRedisLockConfig = heartBeatRedisLockConfig; + } + + public synchronized boolean lock() { + if (this.lockResult != null) { + log.warn("[{}] Lock is held by another process: {}", redisLockKey, lockResult.getLockValue()); + return false; + } + + String lockRequestId = LockUtil.generateLockRequestId(); + HeartBeatRedisLock redisLock = new HeartBeatRedisLock( + redisTemplate, redisLockKey, lockRequestId, heartBeatRedisLockConfig); + LockResult lockResult = redisLock.lock(); + if (!lockResult.isLockGotten()) { + return false; + } + + this.lockResult = lockResult; + return true; + } + + public synchronized void unlock() { + if (this.lockResult == null) { + log.warn("[{}] Lock is not found", redisLockKey); + return; + } + try { + lockResult.tryToRelease(); + } finally { + this.lockResult = null; + } + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/auth/ResourceNameQueryServiceImpl.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/auth/ResourceNameQueryServiceImpl.java index d356640b45..2a4d4a30d7 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/auth/ResourceNameQueryServiceImpl.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/auth/ResourceNameQueryServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,38 +24,38 @@ package com.tencent.bk.job.backup.auth; -import com.tencent.bk.job.backup.client.ServiceApplicationResourceClient; import com.tencent.bk.job.common.iam.constant.ResourceTypeEnum; import com.tencent.bk.job.common.iam.service.AppAuthService; import com.tencent.bk.job.common.iam.service.AuthService; import com.tencent.bk.job.common.iam.service.ResourceNameQueryService; import com.tencent.bk.job.common.iam.util.IamUtil; import com.tencent.bk.job.common.service.AppScopeMappingService; +import com.tencent.bk.job.manage.api.inner.ServiceApplicationResource; import com.tencent.bk.job.manage.model.inner.resp.ServiceApplicationDTO; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @Slf4j -@Service("ResourceNameQueryService") +@Service("jobBackupResourceNameQueryService") public class ResourceNameQueryServiceImpl implements ResourceNameQueryService { - private final ServiceApplicationResourceClient applicationResourceClient; + private final ServiceApplicationResource applicationResource; private final AppScopeMappingService appScopeMappingService; @Autowired public ResourceNameQueryServiceImpl(AuthService authService, AppAuthService appAuthService, - ServiceApplicationResourceClient applicationResourceClient, + ServiceApplicationResource applicationResource, AppScopeMappingService appScopeMappingService) { - this.applicationResourceClient = applicationResourceClient; + this.applicationResource = applicationResource; this.appScopeMappingService = appScopeMappingService; authService.setResourceNameQueryService(this); appAuthService.setResourceNameQueryService(this); } private String getAppName(long appId) { - ServiceApplicationDTO serviceApplicationDTO = applicationResourceClient.queryAppById(appId); + ServiceApplicationDTO serviceApplicationDTO = applicationResource.queryAppById(appId); if (serviceApplicationDTO == null) return null; return serviceApplicationDTO.getName(); } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/client/ServiceAccountResourceClient.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/client/ServiceAccountResourceClient.java deleted file mode 100644 index a54a2ca8c5..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/client/ServiceAccountResourceClient.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.client; - -import com.tencent.bk.job.manage.api.inner.ServiceAccountResource; -import org.springframework.cloud.openfeign.FeignClient; - -/** - * 账号资源后台接口远程调用客户端 - * - * @since 24/11/2020 20:57 - */ -@FeignClient(value = "job-manage", contextId = "job-manage-service-account") -public interface ServiceAccountResourceClient extends ServiceAccountResource { -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/client/ServiceApplicationResourceClient.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/client/ServiceApplicationResourceClient.java deleted file mode 100644 index eeabf62449..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/client/ServiceApplicationResourceClient.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.client; - -import com.tencent.bk.job.manage.api.inner.ServiceApplicationResource; -import org.springframework.cloud.openfeign.FeignClient; - -/** - * 业务资源后台接口远程调用客户端 - */ -@FeignClient(value = "job-manage", contextId = "job-manage-app") -public interface ServiceApplicationResourceClient extends ServiceApplicationResource { -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/client/ServiceBackupTmpResourceClient.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/client/ServiceBackupTmpResourceClient.java deleted file mode 100644 index 20cde85882..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/client/ServiceBackupTmpResourceClient.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.client; - -import com.tencent.bk.job.manage.api.inner.ServiceBackupTmpResource; -import org.springframework.cloud.openfeign.FeignClient; - - -@FeignClient(value = "job-manage", contextId = "job-manage-backup-tmp") -public interface ServiceBackupTmpResourceClient extends ServiceBackupTmpResource { -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/client/ServicePlanResourceClient.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/client/ServicePlanResourceClient.java deleted file mode 100644 index 0be5e05a98..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/client/ServicePlanResourceClient.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.client; - -import com.tencent.bk.job.manage.api.inner.ServiceTaskPlanResource; -import org.springframework.cloud.openfeign.FeignClient; - -/** - * 执行方案资源后台接口远程调用客户端 - * - */ -@FeignClient(value = "job-manage", contextId = "job-manage-service-plan") -public interface ServicePlanResourceClient extends ServiceTaskPlanResource { -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/client/ServiceScriptResourceClient.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/client/ServiceScriptResourceClient.java deleted file mode 100644 index fe94f84b74..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/client/ServiceScriptResourceClient.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.client; - -import com.tencent.bk.job.manage.api.inner.ServiceScriptResource; -import org.springframework.cloud.openfeign.FeignClient; - -/** - * 脚本资源后台接口远程调用客户端 - * - * @since 6/8/2020 22:05 - */ -@FeignClient(value = "job-manage", contextId = "job-manage-service-script") -public interface ServiceScriptResourceClient extends ServiceScriptResource { -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/client/ServiceTemplateResourceClient.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/client/ServiceTemplateResourceClient.java deleted file mode 100644 index 9f2347bafd..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/client/ServiceTemplateResourceClient.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.client; - -import com.tencent.bk.job.manage.api.inner.ServiceTaskTemplateResource; -import org.springframework.cloud.openfeign.FeignClient; - -/** - * 作业模版资源后台接口远程调用客户端 - */ -@FeignClient(value = "job-manage", contextId = "job-manage-service-template") -public interface ServiceTemplateResourceClient extends ServiceTaskTemplateResource { -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/client/WebAccountResourceClient.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/client/WebAccountResourceClient.java deleted file mode 100644 index 37378d9c59..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/client/WebAccountResourceClient.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.client; - -import com.tencent.bk.job.manage.api.web.WebAppAccountResource; -import org.springframework.cloud.openfeign.FeignClient; - -/** - * 账号资源 Web 接口远程调用客户端 - * - * @since 25/11/2020 17:11 - */ -@FeignClient(value = "job-manage", contextId = "job-manage-web-account") -public interface WebAccountResourceClient extends WebAppAccountResource { -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/client/WebPlanResourceClient.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/client/WebPlanResourceClient.java deleted file mode 100644 index b8f9324c98..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/client/WebPlanResourceClient.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.client; - -import com.tencent.bk.job.manage.api.web.WebTaskPlanResource; -import org.springframework.cloud.openfeign.FeignClient; - -/** - * 执行方案资源 Web 接口远程调用客户端 - */ -@FeignClient(value = "job-manage", contextId = "job-manage-web-plan") -public interface WebPlanResourceClient extends WebTaskPlanResource { -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/client/WebScriptResourceClient.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/client/WebScriptResourceClient.java deleted file mode 100644 index e5fa43a237..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/client/WebScriptResourceClient.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.client; - -import com.tencent.bk.job.manage.api.web.WebScriptResource; -import org.springframework.cloud.openfeign.FeignClient; - -/** - * 脚本资源 Web 接口远程调用客户端 - * - * @since 6/8/2020 22:04 - */ -@FeignClient(value = "job-manage", contextId = "job-manage-web-script") -public interface WebScriptResourceClient extends WebScriptResource { -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/client/WebTemplateResourceClient.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/client/WebTemplateResourceClient.java deleted file mode 100644 index 4ea11bd31f..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/client/WebTemplateResourceClient.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.client; - -import com.tencent.bk.job.manage.api.web.WebTaskTemplateResource; -import org.springframework.cloud.openfeign.FeignClient; - -/** - * 作业模版资源 Web 接口远程调用客户端 - */ -@FeignClient(value = "job-manage", contextId = "job-manage-web-template") -public interface WebTemplateResourceClient extends WebTaskTemplateResource { -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfig.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfig.java deleted file mode 100644 index eb84bcd9f5..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfig.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.config; - -import lombok.Getter; -import lombok.Setter; -import lombok.ToString; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Configuration; - -@Getter -@Setter -@ToString -@Configuration -public class ArchiveConfig { - @Value("${job.execute.archive.enabled:false}") - private boolean archiveEnabled; - - /** - * 归档多少天前的 Days - */ - @Value("${job.execute.archive.data.keep_days:30}") - private int dataKeepDays; - - @Value("${job.execute.archive.db:mysql}") - private String archiveDB; - - @Value("${job.execute.archive.delete.enabled:false}") - private boolean deleteEnabled; - - @Value("${job.execute.archive.delete.limit-row-count:5000}") - private Integer deleteLimitRowCount; -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java new file mode 100644 index 0000000000..0866f0741e --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveConfiguration.java @@ -0,0 +1,342 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.config; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import com.tencent.bk.job.backup.archive.AbnormalArchiveTaskReScheduler; +import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; +import com.tencent.bk.job.backup.archive.JobLogArchiveTaskGenerator; +import com.tencent.bk.job.backup.archive.JobLogArchiveTaskScheduler; +import com.tencent.bk.job.backup.archive.JobLogArchivers; +import com.tencent.bk.job.backup.archive.ArchiveCronJobs; +import com.tencent.bk.job.backup.archive.JobInstanceArchiveTaskGenerator; +import com.tencent.bk.job.backup.archive.JobInstanceArchiveTaskScheduler; +import com.tencent.bk.job.backup.archive.JobInstanceSubTableArchivers; +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.impl.JobInstanceHotRecordDAO; +import com.tencent.bk.job.backup.archive.metrics.ArchiveTasksGauge; +import com.tencent.bk.job.backup.archive.service.ArchiveTaskService; +import com.tencent.bk.job.backup.archive.util.lock.ArchiveLogTaskExecuteLock; +import com.tencent.bk.job.backup.archive.util.lock.ArchiveTaskExecuteLock; +import com.tencent.bk.job.backup.archive.util.lock.FailedArchiveTaskRescheduleLock; +import com.tencent.bk.job.backup.archive.util.lock.JobLogArchiveTaskGenerateLock; +import com.tencent.bk.job.backup.archive.util.lock.JobLogArchiveTaskScheduleLock; +import com.tencent.bk.job.backup.archive.util.lock.JobInstanceArchiveTaskGenerateLock; +import com.tencent.bk.job.backup.archive.util.lock.JobInstanceArchiveTaskScheduleLock; +import com.tencent.bk.job.backup.metrics.ArchiveErrorTaskCounter; +import com.tencent.bk.job.common.WatchableThreadPoolExecutor; +import io.micrometer.core.instrument.MeterRegistry; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.cloud.sleuth.Tracer; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Condition; +import org.springframework.context.annotation.ConditionContext; +import org.springframework.context.annotation.Conditional; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Import; +import org.springframework.core.env.Environment; +import org.springframework.core.type.AnnotatedTypeMetadata; +import org.springframework.data.mongodb.core.MongoTemplate; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.lang.Nullable; +import org.springframework.scheduling.annotation.EnableScheduling; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +/** + * job-execute 模块数据归档配置 + */ +@Configuration +@EnableScheduling +@Slf4j +@EnableConfigurationProperties({ArchiveProperties.class, JobLogArchiveProperties.class}) +@Import({ExecuteHotDbConfiguration.class, ExecuteColdDbConfiguration.class, ExecuteMongoDBConfiguration.class}) +public class ArchiveConfiguration { + + @Bean + @ConditionalOnHistoricDataArchiveEnabled + public ArchiveTaskExecuteLock archiveTaskLock(StringRedisTemplate redisTemplate) { + log.info("Init ArchiveTaskExecuteLock"); + return new ArchiveTaskExecuteLock(redisTemplate); + } + + @Bean + @ConditionalOnExecuteLogArchiveEnabled + public ArchiveLogTaskExecuteLock archiveLogTaskLock(StringRedisTemplate redisTemplate) { + log.info("Init ArchiveLogTaskExecuteLock"); + return new ArchiveLogTaskExecuteLock(redisTemplate); + } + + @Bean + @ConditionalOnHistoricDataArchiveEnabled + public JobInstanceArchiveTaskGenerateLock jobInstanceArchiveTaskGenerateLock(StringRedisTemplate redisTemplate) { + log.info("Init JobInstanceArchiveTaskGenerateLock"); + return new JobInstanceArchiveTaskGenerateLock(redisTemplate); + } + + @Bean + @ConditionalOnAnyArchiveEnabled + public FailedArchiveTaskRescheduleLock failedArchiveTaskRescheduleLock(StringRedisTemplate redisTemplate) { + log.info("Init FailedArchiveTaskRescheduleLock"); + return new FailedArchiveTaskRescheduleLock(redisTemplate); + } + + @Bean + @ConditionalOnExecuteLogArchiveEnabled + public JobLogArchiveTaskGenerateLock jobLogArchiveTaskGenerateLock( + StringRedisTemplate redisTemplate) { + log.info("Init JobLogArchiveTaskGenerateLock"); + return new JobLogArchiveTaskGenerateLock(redisTemplate); + } + + @Bean + @ConditionalOnHistoricDataArchiveEnabled + public JobInstanceArchiveTaskGenerator jobInstanceArchiveTaskGenerator( + ArchiveTaskService archiveTaskService, + JobInstanceHotRecordDAO taskInstanceRecordDAO, + ArchiveProperties archiveProperties, + JobInstanceArchiveTaskGenerateLock jobInstanceArchiveTaskGenerateLock) { + + log.info("Init JobInstanceArchiveTaskGenerator"); + return new JobInstanceArchiveTaskGenerator( + archiveTaskService, + taskInstanceRecordDAO, + archiveProperties, + jobInstanceArchiveTaskGenerateLock + ); + } + + @Bean + @ConditionalOnExecuteLogArchiveEnabled + public JobLogArchiveTaskGenerator jobLogArchiveTaskGenerator( + ArchiveTaskService archiveTaskService, + JobLogArchiveProperties archiveProperties, + JobLogArchiveTaskGenerateLock jobLogArchiveTaskGenerateLock, + @Nullable MongoTemplate mongoTemplate) { + + log.info("Init JobLogArchiveTaskGenerator"); + return new JobLogArchiveTaskGenerator( + archiveTaskService, + archiveProperties, + jobLogArchiveTaskGenerateLock, + mongoTemplate + ); + } + + @Bean("archiveTaskStopExecutor") + @ConditionalOnHistoricDataArchiveEnabled + public ThreadPoolExecutor archiveTaskStopExecutor(MeterRegistry meterRegistry) { + return new WatchableThreadPoolExecutor( + meterRegistry, + "archiveTaskStopExecutor", + 5, + 20, + 120L, + TimeUnit.SECONDS, + new LinkedBlockingQueue<>(), + new ThreadFactoryBuilder().setNameFormat("archive-task-stop-thread-pool-%d").build() + ); + } + + @Bean("jobLogArchiveTaskStopExecutor") + @ConditionalOnExecuteLogArchiveEnabled + public ThreadPoolExecutor jobLogArchiveTaskStopExecutor(MeterRegistry meterRegistry) { + return new WatchableThreadPoolExecutor( + meterRegistry, + "jobLogArchiveTaskStopExecutor", + 5, + 20, + 120L, + TimeUnit.SECONDS, + new LinkedBlockingQueue<>(), + new ThreadFactoryBuilder().setNameFormat("execute-log-archive-task-stop-thread-pool-%d").build() + ); + } + + @Bean + @ConditionalOnHistoricDataArchiveEnabled + public JobInstanceArchiveTaskScheduler jobInstanceArchiveTaskScheduler( + ArchiveTaskService archiveTaskService, + JobInstanceHotRecordDAO taskInstanceRecordDAO, + ArchiveProperties archiveProperties, + JobInstanceArchiveTaskScheduleLock jobInstanceArchiveTaskScheduleLock, + JobInstanceSubTableArchivers jobInstanceSubTableArchivers, + ObjectProvider jobInstanceColdDAOObjectProvider, + ArchiveTaskExecuteLock archiveTaskExecuteLock, + ArchiveErrorTaskCounter archiveErrorTaskCounter, + ArchiveTablePropsStorage archiveTablePropsStorage, + Tracer tracer, + @Qualifier("archiveTaskStopExecutor") ThreadPoolExecutor archiveTaskStopExecutor) { + + log.info("Init JobInstanceArchiveTaskScheduler"); + return new JobInstanceArchiveTaskScheduler( + archiveTaskService, + taskInstanceRecordDAO, + archiveProperties, + jobInstanceArchiveTaskScheduleLock, + jobInstanceSubTableArchivers, + jobInstanceColdDAOObjectProvider.getIfAvailable(), + archiveTaskExecuteLock, + archiveErrorTaskCounter, + archiveTablePropsStorage, + tracer, + archiveTaskStopExecutor + ); + } + + @Bean + @ConditionalOnExecuteLogArchiveEnabled + public JobLogArchiveTaskScheduler jobLogArchiveTaskScheduler( + ArchiveTaskService archiveTaskService, + JobLogArchiveProperties archiveProperties, + JobLogArchiveTaskScheduleLock jobLogArchiveTaskScheduleLock, + JobLogArchivers jobLogArchivers, + ArchiveLogTaskExecuteLock archiveLogTaskExecuteLock, + ArchiveErrorTaskCounter archiveErrorTaskCounter, + Tracer tracer, + @Qualifier("jobLogArchiveTaskStopExecutor") ThreadPoolExecutor archiveTaskStopExecutor) { + log.info("Init JobLogArchiveTaskScheduler"); + return new JobLogArchiveTaskScheduler( + archiveTaskService, + archiveProperties, + jobLogArchiveTaskScheduleLock, + jobLogArchivers, + archiveLogTaskExecuteLock, + archiveErrorTaskCounter, + tracer, + archiveTaskStopExecutor + ); + } + + @Bean + @ConditionalOnAnyArchiveEnabled + public ArchiveCronJobs archiveCronJobs( + @Nullable JobInstanceArchiveTaskGenerator jobInstanceArchiveTaskGenerator, + @Nullable JobLogArchiveTaskGenerator jobLogArchiveTaskGenerator, + @Nullable JobInstanceArchiveTaskScheduler jobInstanceArchiveTaskScheduler, + @Nullable JobLogArchiveTaskScheduler jobLogArchiveTaskScheduler, + ArchiveProperties archiveProperties, + JobLogArchiveProperties jobLogArchiveProperties, + AbnormalArchiveTaskReScheduler abnormalArchiveTaskReScheduler) { + log.info("Init ArchiveCronJobs"); + return new ArchiveCronJobs( + jobInstanceArchiveTaskGenerator, + jobLogArchiveTaskGenerator, + jobInstanceArchiveTaskScheduler, + jobLogArchiveTaskScheduler, + archiveProperties, + jobLogArchiveProperties, + abnormalArchiveTaskReScheduler + ); + } + + @Bean + @ConditionalOnHistoricDataArchiveEnabled + public JobInstanceArchiveTaskScheduleLock jobInstanceArchiveTaskScheduleLock() { + log.info("Init JobInstanceArchiveTaskScheduleLock"); + return new JobInstanceArchiveTaskScheduleLock(); + } + + @Bean + @ConditionalOnExecuteLogArchiveEnabled + public JobLogArchiveTaskScheduleLock jobLogArchiveTaskScheduleLock() { + log.info("Init JobLogArchiveTaskScheduleLock"); + return new JobLogArchiveTaskScheduleLock(); + } + + @Bean + @ConditionalOnAnyArchiveEnabled + public AbnormalArchiveTaskReScheduler failArchiveTaskReScheduler( + ArchiveTaskService archiveTaskService, + FailedArchiveTaskRescheduleLock failedArchiveTaskRescheduleLock) { + log.info("Init FailArchiveTaskReScheduler"); + return new AbnormalArchiveTaskReScheduler(archiveTaskService, failedArchiveTaskRescheduleLock); + } + + @Bean + @ConditionalOnHistoricDataArchiveEnabled + public ArchiveTasksGauge archiveTasksGauge(MeterRegistry meterRegistry, + ArchiveTaskService archiveTaskService) { + return new ArchiveTasksGauge(meterRegistry, archiveTaskService); + } + + @Bean + @ConditionalOnHistoricDataArchiveEnabled + public ArchiveTablePropsStorage archiveTablePropsStorage(ArchiveProperties archiveProperties) { + log.info("Init ArchiveTablePropsStorage"); + return new ArchiveTablePropsStorage(archiveProperties); + } + + /** + * 条件注解：控制历史数据归档相关bean的注入 + */ + @Target({ElementType.TYPE, ElementType.METHOD}) + @Retention(RetentionPolicy.RUNTIME) + @ConditionalOnProperty( + value = "job.backup.archive.execute.enabled", + havingValue = "true" + ) + public @interface ConditionalOnHistoricDataArchiveEnabled { + } + + /** + * 条件注解：控制执行日志归档相关bean的注入 + */ + @Target({ElementType.TYPE, ElementType.METHOD}) + @Retention(RetentionPolicy.RUNTIME) + @ConditionalOnProperty( + value = "job.backup.archive.execute-log.enabled", + havingValue = "true" + ) + public @interface ConditionalOnExecuteLogArchiveEnabled { + } + + /** + * 条件注解：控制归档任务公共bean的注入 + */ + @Target({ElementType.METHOD, ElementType.TYPE}) + @Retention(RetentionPolicy.RUNTIME) + @Conditional(AnyArchiveEnabledCondition.class) + public @interface ConditionalOnAnyArchiveEnabled { + } + static class AnyArchiveEnabledCondition implements Condition { + @Override + public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) { + Environment env = context.getEnvironment(); + return env.getProperty("job.backup.archive.execute.enabled", Boolean.class, false) + || env.getProperty("job.backup.archive.execute-log.enabled", Boolean.class, false); + } + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveProperties.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveProperties.java new file mode 100644 index 0000000000..6d2b563cb5 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArchiveProperties.java @@ -0,0 +1,154 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.config; + +import com.tencent.bk.job.backup.constant.ArchiveModeEnum; +import lombok.Data; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.springframework.boot.context.properties.ConfigurationProperties; + +import java.util.List; +import java.util.Map; + +@Getter +@Setter +@ToString +@ConfigurationProperties(prefix = "job.backup.archive.execute") +public class ArchiveProperties { + /** + * 是否启用 DB 归档 + */ + private boolean enabled; + + /** + * 归档模式 + * + * @see ArchiveModeEnum + */ + private String mode; + + /** + * 触发时间 CRON 表达式 + */ + private String cron; + + /** + * DB数据保留天数 + */ + private int keepDays = 30; + + /** + * 归档数据时间范围计算所依据的时区，如果不指定默认为系统时区 + */ + private String timeZone; + + /** + * 归档数据写入归档库时每次写入的数据量（单个表），服务内存受限时可适当降低该值 + */ + private int batchInsertRowSize = 1000; + + /** + * 每次执行删除的最大行数 + */ + private int deleteRowLimit = 1000; + + /** + * 每批次从 db 表中读取的记录数量 + */ + private int readRowLimit = 1000; + + private Map tableConfigs; + + /** + * 归档任务配置 + */ + private ArchiveTasksConfig tasks; + + @Data + public static class TableConfig { + /** + * 归档数据写入归档库时每次写入的数据量（单个表），服务内存受限时可适当降低该值 + */ + private Integer batchInsertRowSize; + + /** + * 每次执行删除的最大行数 + */ + private Integer deleteRowLimit; + + /** + * 每批次从 db 表中读取的记录数量 + */ + private Integer readRowLimit; + } + + @Data + public static class ArchiveTasksConfig { + /** + * 归档任务配置 + */ + private ArchiveTaskConfig jobInstance; + } + + @Data + public static class ArchiveTaskConfig { + /** + * 数据源模式 + * + * @see com.tencent.bk.job.common.mysql.dynamic.ds.DataSourceMode + */ + private String dataSourceMode; + + /** + * 要归档的分库分表数据节点配置 + */ + private List shardingDataNodes; + /** + * 归档任务并行执行数量 + */ + private Integer concurrent = 6; + } + + + @Data + public static class ShardingDataNode { + /** + * 数据源名称 + */ + private String dataSourceName; + /** + * 分库数量 + */ + private Integer dbCount; + /** + * 分表数量 + */ + private Integer tableCount; + } + + +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArtifactoryConfig.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArtifactoryConfig.java index 2f620a9959..13644c34a3 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArtifactoryConfig.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ArtifactoryConfig.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/BackupStorageConfig.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/BackupStorageConfig.java index 92ba1634fb..e99eb66de5 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/BackupStorageConfig.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/BackupStorageConfig.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/BkConfig.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/BkConfig.java deleted file mode 100644 index ec9d73ea45..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/BkConfig.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.config; - -import lombok.Getter; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Configuration; - -@Getter -@Configuration -public class BkConfig { - - @Value("${job.web.url:}") - private String jobWebUrl; - - @Value("${swagger.url:swagger.job.com}") - private String swaggerUrl; - -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/DbConfig.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/DbConfig.java new file mode 100644 index 0000000000..fb3ad3f18f --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/DbConfig.java @@ -0,0 +1,100 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.config; + +import com.tencent.bk.job.common.mysql.util.JooqConfigurationUtil; +import org.jooq.ConnectionProvider; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; +import org.jooq.impl.DataSourceConnectionProvider; +import org.jooq.impl.DefaultConfiguration; +import org.jooq.impl.DefaultDSLContext; +import org.jooq.impl.DefaultExecuteListenerProvider; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.jdbc.DataSourceBuilder; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.DependsOn; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.DataSourceTransactionManager; +import org.springframework.jdbc.datasource.TransactionAwareDataSourceProxy; +import org.springframework.transaction.annotation.EnableTransactionManagement; + +import javax.sql.DataSource; + +@Configuration(value = "jobBackupDbConfig") +@EnableTransactionManagement +public class DbConfig { + @Qualifier("job-backup-data-source") + @Bean(name = "job-backup-data-source") + @ConfigurationProperties(prefix = "spring.datasource.job-backup") + public DataSource dataSource() { + return DataSourceBuilder.create().build(); + } + + @Qualifier("jobBackupTransactionManager") + @Bean(name = "jobBackupTransactionManager") + @DependsOn("job-backup-data-source") + public DataSourceTransactionManager transactionManager(@Qualifier("job-backup-data-source") DataSource dataSource) { + return new DataSourceTransactionManager(dataSource); + } + + @Qualifier("job-backup-jdbc-template") + @Bean(name = "job-backup-jdbc-template") + public JdbcTemplate jdbcTemplate(@Qualifier("job-backup-data-source") DataSource dataSource) { + return new JdbcTemplate(dataSource); + } + + @Qualifier("job-backup-dsl-context") + @Bean(name = "job-backup-dsl-context") + public DSLContext dslContext(@Qualifier("job-backup-jooq-conf") org.jooq.Configuration configuration) { + return new DefaultDSLContext(configuration); + } + + @Qualifier("job-backup-jooq-conf") + @Bean(name = "job-backup-jooq-conf") + public org.jooq.Configuration + jooqConf(@Qualifier("job-backup-conn-provider") ConnectionProvider connectionProvider, + DefaultExecuteListenerProvider jooqExecuteListenerProvider + ) { + return JooqConfigurationUtil.getConfiguration(connectionProvider, jooqExecuteListenerProvider); + } + + @Qualifier("job-backup-conn-provider") + @Bean(name = "job-backup-conn-provider") + public ConnectionProvider connectionProvider( + @Qualifier("jobBackupTransactionAwareDataSource") DataSource dataSource) { + return new DataSourceConnectionProvider(dataSource); + } + + @Qualifier("jobBackupTransactionAwareDataSource") + @Bean(name = "jobBackupTransactionAwareDataSource") + public TransactionAwareDataSourceProxy + transactionAwareDataSourceProxy(@Qualifier("job-backup-data-source") DataSource dataSource) { + return new TransactionAwareDataSourceProxy(dataSource); + } + +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecuteColdDbConfiguration.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecuteColdDbConfiguration.java new file mode 100644 index 0000000000..f700beafd7 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecuteColdDbConfiguration.java @@ -0,0 +1,130 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.config; + +import com.tencent.bk.job.backup.archive.dao.impl.JobInstanceColdDAOImpl; +import com.tencent.bk.job.backup.constant.ArchiveModeEnum; +import com.tencent.bk.job.common.mysql.util.JooqConfigurationUtil; +import lombok.extern.slf4j.Slf4j; +import org.jooq.ConnectionProvider; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; +import org.jooq.impl.DataSourceConnectionProvider; +import org.jooq.impl.DefaultConfiguration; +import org.jooq.impl.DefaultDSLContext; +import org.jooq.impl.DefaultExecuteListenerProvider; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.condition.AllNestedConditions; +import org.springframework.boot.autoconfigure.condition.AnyNestedCondition; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.jdbc.DataSourceBuilder; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Conditional; +import org.springframework.context.annotation.Configuration; + +import javax.sql.DataSource; + +/** + * 归档-冷 DB 配置 + */ +@Configuration(value = "executeColdDbConfiguration") +@Conditional(ExecuteColdDbConfiguration.JobExecuteColdDbInitCondition.class) +@Slf4j +public class ExecuteColdDbConfiguration { + + @Qualifier("job-execute-archive-cold-source") + @Bean(name = "job-execute-archive-cold-source") + @ConfigurationProperties(prefix = "spring.datasource.job-execute-archive") + public DataSource executeArchiveDataSource() { + log.info("Init job-execute-archive cold datasource"); + return DataSourceBuilder.create().build(); + } + + @Qualifier("job-execute-archive-cold-dsl-context") + @Bean(name = "job-execute-archive-cold-dsl-context") + public DSLContext executeArchiveDslContext( + @Qualifier("job-execute-archive-cold-jooq-conf") org.jooq.Configuration configuration) { + return new DefaultDSLContext(configuration); + } + + @Qualifier("job-execute-archive-cold-jooq-conf") + @Bean(name = "job-execute-archive-cold-jooq-conf") + public org.jooq.Configuration + executeArchiveJooqConf(@Qualifier("job-execute-archive-cold-conn-provider") ConnectionProvider connectionProvider, + DefaultExecuteListenerProvider jooqExecuteListenerProvider + ) { + return JooqConfigurationUtil.getConfiguration(connectionProvider, jooqExecuteListenerProvider); + } + + @Qualifier("job-execute-archive-cold-conn-provider") + @Bean(name = "job-execute-archive-cold-conn-provider") + public ConnectionProvider executeArchiveConnectionProvider( + @Qualifier("job-execute-archive-cold-source") DataSource dataSource) { + return new DataSourceConnectionProvider(dataSource); + } + + @Bean(name = "execute-archive-dao") + public JobInstanceColdDAOImpl jobInstanceColdDAO( + @Qualifier("job-execute-archive-cold-dsl-context") DSLContext context) { + log.info("Init JobInstanceColdDAO"); + return new JobInstanceColdDAOImpl(context); + } + + static class JobExecuteColdDbInitCondition extends AllNestedConditions { + public JobExecuteColdDbInitCondition() { + super(ConfigurationPhase.PARSE_CONFIGURATION); + } + + @ConditionalOnProperty(value = "job.backup.archive.execute.enabled", havingValue = "true") + static class ArchiveEnableCondition { + + } + + @Conditional(JobExecuteBackupCondition.class) + static class BackupCondition { + + } + + static class JobExecuteBackupCondition extends AnyNestedCondition { + public JobExecuteBackupCondition() { + super(ConfigurationPhase.PARSE_CONFIGURATION); + } + + @ConditionalOnProperty(value = "job.backup.archive.execute.mode", + havingValue = ArchiveModeEnum.Constants.BACKUP_THEN_DELETE) + static class ArchiveModeBackupThenDeleteCondition { + + } + + @ConditionalOnProperty(value = "job.backup.archive.execute.mode", + havingValue = ArchiveModeEnum.Constants.BACKUP_ONLY) + static class ArchiveModeBackupOnlyCondition { + + } + } + } + +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecuteHotDbConfiguration.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecuteHotDbConfiguration.java new file mode 100644 index 0000000000..fb00f8ed69 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecuteHotDbConfiguration.java @@ -0,0 +1,691 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.config; + +import com.tencent.bk.job.backup.archive.ArchiveTablePropsStorage; +import com.tencent.bk.job.backup.archive.JobInstanceSubTableArchivers; +import com.tencent.bk.job.backup.archive.dao.JobInstanceColdDAO; +import com.tencent.bk.job.backup.archive.dao.ds.JobExecuteVerticalShardingDSLContextProvider; +import com.tencent.bk.job.backup.archive.dao.impl.FileSourceTaskLogRecordDAO; +import com.tencent.bk.job.backup.archive.dao.impl.GseFileAgentTaskRecordDAO; +import com.tencent.bk.job.backup.archive.dao.impl.GseFileExecuteObjTaskRecordDAO; +import com.tencent.bk.job.backup.archive.dao.impl.GseScriptAgentTaskRecordDAO; +import com.tencent.bk.job.backup.archive.dao.impl.GseScriptExecuteObjTaskRecordDAO; +import com.tencent.bk.job.backup.archive.dao.impl.GseTaskRecordDAO; +import com.tencent.bk.job.backup.archive.dao.impl.JobInstanceHotRecordDAO; +import com.tencent.bk.job.backup.archive.dao.impl.OperationLogRecordDAO; +import com.tencent.bk.job.backup.archive.dao.impl.RollingConfigRecordDAO; +import com.tencent.bk.job.backup.archive.dao.impl.StepInstanceConfirmRecordDAO; +import com.tencent.bk.job.backup.archive.dao.impl.StepInstanceFileRecordDAO; +import com.tencent.bk.job.backup.archive.dao.impl.StepInstanceRecordDAO; +import com.tencent.bk.job.backup.archive.dao.impl.StepInstanceRollingTaskRecordDAO; +import com.tencent.bk.job.backup.archive.dao.impl.StepInstanceScriptRecordDAO; +import com.tencent.bk.job.backup.archive.dao.impl.StepInstanceVariableRecordDAO; +import com.tencent.bk.job.backup.archive.dao.impl.TaskInstanceHostRecordDAO; +import com.tencent.bk.job.backup.archive.dao.impl.TaskInstanceVariableRecordDAO; +import com.tencent.bk.job.backup.archive.impl.FileSourceTaskLogArchiver; +import com.tencent.bk.job.backup.archive.impl.GseFileAgentTaskArchiver; +import com.tencent.bk.job.backup.archive.impl.GseFileExecuteObjTaskArchiver; +import com.tencent.bk.job.backup.archive.impl.GseScriptAgentTaskArchiver; +import com.tencent.bk.job.backup.archive.impl.GseScriptExecuteObjTaskArchiver; +import com.tencent.bk.job.backup.archive.impl.GseTaskArchiver; +import com.tencent.bk.job.backup.archive.impl.OperationLogArchiver; +import com.tencent.bk.job.backup.archive.impl.RollingConfigArchiver; +import com.tencent.bk.job.backup.archive.impl.StepInstanceArchiver; +import com.tencent.bk.job.backup.archive.impl.StepInstanceConfirmArchiver; +import com.tencent.bk.job.backup.archive.impl.StepInstanceFileArchiver; +import com.tencent.bk.job.backup.archive.impl.StepInstanceRollingTaskArchiver; +import com.tencent.bk.job.backup.archive.impl.StepInstanceScriptArchiver; +import com.tencent.bk.job.backup.archive.impl.StepInstanceVariableArchiver; +import com.tencent.bk.job.backup.archive.impl.TaskInstanceHostArchiver; +import com.tencent.bk.job.backup.archive.impl.TaskInstanceVariableArchiver; +import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; +import com.tencent.bk.job.common.mysql.dynamic.ds.DataSourceMode; +import com.tencent.bk.job.common.mysql.dynamic.ds.StandaloneDSLContextProvider; +import com.tencent.bk.job.common.mysql.dynamic.ds.VerticalShardingDSLContextProvider; +import com.tencent.bk.job.common.mysql.util.JooqConfigurationUtil; +import lombok.extern.slf4j.Slf4j; +import org.jooq.ConnectionProvider; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; +import org.jooq.impl.DataSourceConnectionProvider; +import org.jooq.impl.DefaultConfiguration; +import org.jooq.impl.DefaultDSLContext; +import org.jooq.impl.DefaultExecuteListenerProvider; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.jdbc.DataSourceBuilder; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.DependsOn; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.DataSourceTransactionManager; +import org.springframework.jdbc.datasource.TransactionAwareDataSourceProxy; + +import javax.sql.DataSource; + +/** + * 归档-job-execute 热 DB 配置 + */ +@Configuration("executeHotDbConfiguration") +@ConditionalOnExpression("${job.backup.archive.execute.enabled:false}") +@Slf4j +public class ExecuteHotDbConfiguration { + + @ConditionalOnProperty(value = "job.backup.archive.execute.mariadb.dataSourceMode", + havingValue = DataSourceMode.Constants.STANDALONE, matchIfMissing = false) + protected static class JobExecuteStandaloneDslContextConfiguration { + @Qualifier("job-execute-archive-hot-data-source") + @Bean(name = "job-execute-archive-hot-data-source") + @ConfigurationProperties(prefix = "spring.datasource.job-execute") + public DataSource dataSource() { + return DataSourceBuilder.create().build(); + } + + @Qualifier("job-execute-archive-hot-transaction-manager") + @Bean(name = "job-execute-archive-hot-transaction-manager") + @DependsOn("job-execute-archive-hot-data-source") + public DataSourceTransactionManager transactionManager( + @Qualifier("job-execute-archive-hot-data-source") DataSource dataSource) { + return new DataSourceTransactionManager(dataSource); + } + + @Qualifier("job-execute-archive-hot-jdbc-template") + @Bean(name = "job-execute-archive-hot-jdbc-template") + public JdbcTemplate jdbcTemplate(@Qualifier("job-execute-archive-hot-data-source") DataSource dataSource) { + return new JdbcTemplate(dataSource); + } + + @Qualifier("job-execute-archive-hot-dsl-context") + @Bean(name = "job-execute-archive-hot-dsl-context") + public DSLContext dslContext( + @Qualifier("job-execute-archive-hot-jooq-conf") org.jooq.Configuration configuration) { + log.info("Init DSLContext job-execute-archive-hot-standalone"); + return new DefaultDSLContext(configuration); + } + + @Qualifier("job-execute-archive-hot-jooq-conf") + @Bean(name = "job-execute-archive-hot-jooq-conf") + public org.jooq.Configuration jooqConf( + @Qualifier("job-execute-archive-hot-conn-provider") ConnectionProvider connectionProvider, + DefaultExecuteListenerProvider jooqExecuteListenerProvider + ) { + return JooqConfigurationUtil.getConfiguration(connectionProvider, jooqExecuteListenerProvider); + } + + @Qualifier("job-execute-archive-hot-conn-provider") + @Bean(name = "job-execute-archive-hot-conn-provider") + public ConnectionProvider connectionProvider( + @Qualifier("job-execute-archive-hot-transaction-aware-data-source") DataSource dataSource) { + return new DataSourceConnectionProvider(dataSource); + } + + @Qualifier("job-execute-archive-hot-transaction-aware-data-source") + @Bean(name = "job-execute-archive-hot-transaction-aware-data-source") + public TransactionAwareDataSourceProxy + transactionAwareDataSourceProxy(@Qualifier("job-execute-archive-hot-data-source") DataSource dataSource) { + return new TransactionAwareDataSourceProxy(dataSource); + } + + @Qualifier("job-execute-archive-hot-dsl-context-provider") + @Bean(name = "job-execute-archive-hot-dsl-context-provider") + public StandaloneDSLContextProvider standaloneDSLContextProvider( + @Qualifier("job-execute-archive-hot-dsl-context") DSLContext dslContext + ) { + log.info("Init StandaloneDSLContextProvider"); + return new StandaloneDSLContextProvider(dslContext); + } + } + + @ConditionalOnProperty(value = "job.backup.archive.execute.mariadb.dataSourceMode", + havingValue = DataSourceMode.Constants.VERTICAL_SHARDING, matchIfMissing = false) + protected static class VerticalDslContextConfiguration { + // 配置垂直分片数据源-a + @Qualifier("job-execute-archive-hot-data-source-a") + @Bean(name = "job-execute-archive-hot-data-source-a") + @ConfigurationProperties(prefix = "spring.datasource.job-execute-archive-hot-vertical-a") + public DataSource dataSourceA() { + return DataSourceBuilder.create().build(); + } + + @Qualifier("job-execute-archive-hot-transaction-manager-a") + @Bean(name = "job-execute-archive-hot-transaction-manager-a") + @DependsOn("job-execute-archive-hot-data-source-a") + public DataSourceTransactionManager transactionManagerA( + @Qualifier("job-execute-archive-hot-data-source-a") DataSource dataSource) { + return new DataSourceTransactionManager(dataSource); + } + + @Qualifier("job-execute-archive-hot-jdbc-template-a") + @Bean(name = "job-execute-archive-hot-jdbc-template-a") + public JdbcTemplate jdbcTemplateA(@Qualifier("job-execute-archive-hot-data-source-a") DataSource dataSource) { + return new JdbcTemplate(dataSource); + } + + @Qualifier("job-execute-archive-hot-dsl-context-a") + @Bean(name = "job-execute-archive-hot-dsl-context-a") + public DSLContext dslContextA( + @Qualifier("job-execute-archive-hot-jooq-conf-a") org.jooq.Configuration configuration) { + log.info("Init DSLContext job-execute-archive-hot-vertical-a"); + return new DefaultDSLContext(configuration); + } + + @Qualifier("job-execute-archive-hot-jooq-conf-a") + @Bean(name = "job-execute-archive-hot-jooq-conf-a") + public org.jooq.Configuration jooqConfA( + @Qualifier("job-execute-archive-hot-conn-provider-a") ConnectionProvider connectionProvider) { + return new DefaultConfiguration().derive(connectionProvider).derive(SQLDialect.MYSQL); + } + + @Qualifier("job-execute-archive-hot-conn-provider-a") + @Bean(name = "job-execute-archive-hot-conn-provider-a") + public ConnectionProvider connectionProviderA( + @Qualifier("job-execute-archive-hot-transaction-aware-data-source-a") DataSource dataSource) { + return new DataSourceConnectionProvider(dataSource); + } + + @Qualifier("job-execute-archive-hot-transaction-aware-data-source-a") + @Bean(name = "job-execute-archive-hot-transaction-aware-data-source-a") + public TransactionAwareDataSourceProxy + transactionAwareDataSourceProxyA(@Qualifier("job-execute-archive-hot-data-source-a") DataSource dataSource) { + return new TransactionAwareDataSourceProxy(dataSource); + } + + // 配置垂直分片数据源-b + @Qualifier("job-execute-archive-hot-data-source-b") + @Bean(name = "job-execute-archive-hot-data-source-b") + @ConfigurationProperties(prefix = "spring.datasource.job-execute-archive-hot-vertical-b") + public DataSource dataSourceB() { + return DataSourceBuilder.create().build(); + } + + @Qualifier("job-execute-archive-hot-transaction-manager-b") + @Bean(name = "job-execute-archive-hot-transaction-manager-b") + @DependsOn("job-execute-archive-hot-data-source-b") + public DataSourceTransactionManager transactionManagerB( + @Qualifier("job-execute-archive-hot-data-source-b") DataSource dataSource) { + return new DataSourceTransactionManager(dataSource); + } + + @Qualifier("job-execute-archive-hot-jdbc-template-b") + @Bean(name = "job-execute-archive-hot-jdbc-template-b") + public JdbcTemplate jdbcTemplateB(@Qualifier("job-execute-archive-hot-data-source-b") DataSource dataSource) { + return new JdbcTemplate(dataSource); + } + + @Qualifier("job-execute-archive-hot-dsl-context-b") + @Bean(name = "job-execute-archive-hot-dsl-context-b") + public DSLContext dslContextB( + @Qualifier("job-execute-archive-hot-jooq-conf-b") org.jooq.Configuration configuration) { + log.info("Init DSLContext job-execute-archive-hot-vertical-b"); + return new DefaultDSLContext(configuration); + } + + @Qualifier("job-execute-archive-hot-jooq-conf-b") + @Bean(name = "job-execute-archive-hot-jooq-conf-b") + public org.jooq.Configuration jooqConfB( + @Qualifier("job-execute-archive-hot-conn-provider-b") ConnectionProvider connectionProvider) { + return new DefaultConfiguration().derive(connectionProvider).derive(SQLDialect.MYSQL); + } + + @Qualifier("job-execute-archive-hot-conn-provider-b") + @Bean(name = "job-execute-archive-hot-conn-provider-b") + public ConnectionProvider connectionProviderB( + @Qualifier("job-execute-archive-hot-transaction-aware-data-source-b") DataSource dataSource) { + return new DataSourceConnectionProvider(dataSource); + } + + @Qualifier("job-execute-archive-hot-transaction-aware-data-source-b") + @Bean(name = "job-execute-archive-hot-transaction-aware-data-source-b") + public TransactionAwareDataSourceProxy + transactionAwareDataSourceProxyB(@Qualifier("job-execute-archive-hot-data-source-b") DataSource dataSource) { + return new TransactionAwareDataSourceProxy(dataSource); + } + + // 配置垂直分片数据源-c + @Qualifier("job-execute-archive-hot-data-source-c") + @Bean(name = "job-execute-archive-hot-data-source-c") + @ConfigurationProperties(prefix = "spring.datasource.job-execute-archive-hot-vertical-c") + public DataSource dataSourceC() { + return DataSourceBuilder.create().build(); + } + + @Qualifier("job-execute-archive-hot-transaction-manager-c") + @Bean(name = "job-execute-archive-hot-transaction-manager-c") + @DependsOn("job-execute-archive-hot-data-source-c") + public DataSourceTransactionManager transactionManagerC( + @Qualifier("job-execute-archive-hot-data-source-c") DataSource dataSource) { + return new DataSourceTransactionManager(dataSource); + } + + @Qualifier("job-execute-archive-hot-jdbc-template-c") + @Bean(name = "job-execute-archive-hot-jdbc-template-c") + public JdbcTemplate jdbcTemplateC(@Qualifier("job-execute-archive-hot-data-source-c") DataSource dataSource) { + return new JdbcTemplate(dataSource); + } + + @Qualifier("job-execute-archive-hot-dsl-context-c") + @Bean(name = "job-execute-archive-hot-dsl-context-c") + public DSLContext dslContextC( + @Qualifier("job-execute-archive-hot-jooq-conf-c") org.jooq.Configuration configuration) { + log.info("Init DSLContext job-execute-archive-hot-vertical-c"); + return new DefaultDSLContext(configuration); + } + + @Qualifier("job-execute-archive-hot-jooq-conf-c") + @Bean(name = "job-execute-archive-hot-jooq-conf-c") + public org.jooq.Configuration jooqConfC( + @Qualifier("job-execute-archive-hot-conn-provider-c") ConnectionProvider connectionProvider) { + return new DefaultConfiguration().derive(connectionProvider).derive(SQLDialect.MYSQL); + } + + @Qualifier("job-execute-archive-hot-conn-provider-c") + @Bean(name = "job-execute-archive-hot-conn-provider-c") + public ConnectionProvider connectionProviderC( + @Qualifier("job-execute-archive-hot-transaction-aware-data-source-c") DataSource dataSource) { + return new DataSourceConnectionProvider(dataSource); + } + + @Qualifier("job-execute-archive-hot-transaction-aware-data-source-c") + @Bean(name = "job-execute-archive-hot-transaction-aware-data-source-c") + public TransactionAwareDataSourceProxy + transactionAwareDataSourceProxyC(@Qualifier("job-execute-archive-hot-data-source-c") DataSource dataSource) { + return new TransactionAwareDataSourceProxy(dataSource); + } + + @Qualifier("job-execute-archive-hot-dsl-context-provider") + @Bean(name = "job-execute-archive-hot-dsl-context-provider") + public VerticalShardingDSLContextProvider verticalShardingDSLContextProvider( + @Qualifier("job-execute-archive-hot-dsl-context-a") DSLContext dslContextA, + @Qualifier("job-execute-archive-hot-dsl-context-b") DSLContext dslContextB, + @Qualifier("job-execute-archive-hot-dsl-context-c") DSLContext dslContextC + ) { + log.info("Init VerticalShardingDSLContextProvider"); + return new JobExecuteVerticalShardingDSLContextProvider( + dslContextA, + dslContextB, + dslContextC + ); + } + } + + /** + * job-execute hot DB DAO 配置 + */ + protected static class ExecuteHotDaoConfiguration { + + @Bean(name = "taskInstanceRecordDAO") + public JobInstanceHotRecordDAO taskInstanceRecordDAO( + @Qualifier("job-execute-archive-hot-dsl-context-provider") DSLContextProvider dslContextProvider) { + log.info("Init TaskInstanceRecordDAO"); + return new JobInstanceHotRecordDAO(dslContextProvider); + } + + @Bean(name = "stepInstanceRecordDAO") + public StepInstanceRecordDAO stepInstanceRecordDAO( + @Qualifier("job-execute-archive-hot-dsl-context-provider") DSLContextProvider dslContextProvider) { + log.info("Init StepInstanceRecordDAO"); + return new StepInstanceRecordDAO(dslContextProvider); + } + + @Bean + public StepInstanceArchiver stepInstanceArchiver( + ObjectProvider jobInstanceColdDAOObjectProvider, + StepInstanceRecordDAO stepInstanceRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage + ) { + return new StepInstanceArchiver( + jobInstanceColdDAOObjectProvider.getIfAvailable(), + stepInstanceRecordDAO, + archiveTablePropsStorage); + } + + @Bean(name = "stepInstanceScriptRecordDAO") + public StepInstanceScriptRecordDAO stepInstanceScriptRecordDAO( + @Qualifier("job-execute-archive-hot-dsl-context-provider") DSLContextProvider dslContextProvider) { + log.info("Init StepInstanceScriptRecordDAO"); + return new StepInstanceScriptRecordDAO(dslContextProvider); + } + + @Bean + public StepInstanceScriptArchiver stepInstanceScriptArchiver( + ObjectProvider jobInstanceColdDAOObjectProvider, + StepInstanceScriptRecordDAO stepInstanceScriptRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage + ) { + return new StepInstanceScriptArchiver( + jobInstanceColdDAOObjectProvider.getIfAvailable(), + stepInstanceScriptRecordDAO, + archiveTablePropsStorage); + } + + @Bean(name = "stepInstanceFileRecordDAO") + public StepInstanceFileRecordDAO stepInstanceFileRecordDAO( + @Qualifier("job-execute-archive-hot-dsl-context-provider") DSLContextProvider dslContextProvider) { + log.info("Init StepInstanceFileRecordDAO"); + return new StepInstanceFileRecordDAO(dslContextProvider); + } + + @Bean + public StepInstanceFileArchiver stepInstanceFileArchiver( + ObjectProvider jobInstanceColdDAOObjectProvider, + StepInstanceFileRecordDAO stepInstanceFileRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage + ) { + return new StepInstanceFileArchiver( + jobInstanceColdDAOObjectProvider.getIfAvailable(), + stepInstanceFileRecordDAO, + archiveTablePropsStorage); + } + + @Bean(name = "stepInstanceConfirmRecordDAO") + public StepInstanceConfirmRecordDAO stepInstanceConfirmRecordDAO( + @Qualifier("job-execute-archive-hot-dsl-context-provider") DSLContextProvider dslContextProvider) { + log.info("Init StepInstanceConfirmRecordDAO"); + return new StepInstanceConfirmRecordDAO(dslContextProvider); + } + + @Bean + public StepInstanceConfirmArchiver stepInstanceConfirmArchiver( + ObjectProvider jobInstanceColdDAOObjectProvider, + StepInstanceConfirmRecordDAO stepInstanceConfirmRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage + ) { + return new StepInstanceConfirmArchiver( + jobInstanceColdDAOObjectProvider.getIfAvailable(), + stepInstanceConfirmRecordDAO, + archiveTablePropsStorage); + } + + @Bean(name = "stepInstanceVariableRecordDAO") + public StepInstanceVariableRecordDAO stepInstanceVariableRecordDAO( + @Qualifier("job-execute-archive-hot-dsl-context-provider") DSLContextProvider dslContextProvider) { + log.info("Init StepInstanceVariableRecordDAO"); + return new StepInstanceVariableRecordDAO(dslContextProvider); + } + + @Bean + public StepInstanceVariableArchiver stepInstanceVariableArchiver( + ObjectProvider jobInstanceColdDAOObjectProvider, + StepInstanceVariableRecordDAO stepInstanceVariableRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage + ) { + return new StepInstanceVariableArchiver( + jobInstanceColdDAOObjectProvider.getIfAvailable(), + stepInstanceVariableRecordDAO, + archiveTablePropsStorage); + } + + @Bean(name = "taskInstanceVariableRecordDAO") + public TaskInstanceVariableRecordDAO taskInstanceVariableRecordDAO( + @Qualifier("job-execute-archive-hot-dsl-context-provider") DSLContextProvider dslContextProvider) { + log.info("Init TaskInstanceVariableRecordDAO"); + return new TaskInstanceVariableRecordDAO(dslContextProvider); + } + + @Bean + public TaskInstanceVariableArchiver taskInstanceVariableArchiver( + ObjectProvider jobInstanceColdDAOObjectProvider, + TaskInstanceVariableRecordDAO taskInstanceVariableRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage + ) { + return new TaskInstanceVariableArchiver( + jobInstanceColdDAOObjectProvider.getIfAvailable(), + taskInstanceVariableRecordDAO, + archiveTablePropsStorage); + } + + @Bean(name = "operationLogRecordDAO") + public OperationLogRecordDAO operationLogRecordDAO( + @Qualifier("job-execute-archive-hot-dsl-context-provider") DSLContextProvider dslContextProvider) { + log.info("Init OperationLogRecordDAO"); + return new OperationLogRecordDAO(dslContextProvider); + } + + @Bean + public OperationLogArchiver operationLogArchiver( + ObjectProvider jobInstanceColdDAOObjectProvider, + OperationLogRecordDAO operationLogRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage + ) { + return new OperationLogArchiver( + jobInstanceColdDAOObjectProvider.getIfAvailable(), + operationLogRecordDAO, + archiveTablePropsStorage); + } + + @Bean(name = "fileSourceTaskLogRecordDAO") + public FileSourceTaskLogRecordDAO fileSourceTaskLogRecordDAO( + @Qualifier("job-execute-archive-hot-dsl-context-provider") DSLContextProvider dslContextProvider) { + log.info("Init FileSourceTaskRecordDAO"); + return new FileSourceTaskLogRecordDAO(dslContextProvider); + } + + @Bean + public FileSourceTaskLogArchiver fileSourceTaskLogArchiver( + ObjectProvider jobInstanceColdDAOObjectProvider, + FileSourceTaskLogRecordDAO fileSourceTaskLogRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage + ) { + return new FileSourceTaskLogArchiver( + jobInstanceColdDAOObjectProvider.getIfAvailable(), + fileSourceTaskLogRecordDAO, + archiveTablePropsStorage); + } + + @Bean(name = "gseTaskRecordDAO") + public GseTaskRecordDAO gseTaskRecordDAO( + @Qualifier("job-execute-archive-hot-dsl-context-provider") DSLContextProvider dslContextProvider) { + log.info("Init GseTaskRecordDAO"); + return new GseTaskRecordDAO(dslContextProvider); + } + + @Bean + public GseTaskArchiver gseTaskArchiver( + ObjectProvider jobInstanceColdDAOObjectProvider, + GseTaskRecordDAO gseTaskRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage + ) { + return new GseTaskArchiver( + jobInstanceColdDAOObjectProvider.getIfAvailable(), + gseTaskRecordDAO, + archiveTablePropsStorage); + } + + @Bean(name = "gseScriptAgentTaskRecordDAO") + public GseScriptAgentTaskRecordDAO gseScriptAgentTaskRecordDAO( + @Qualifier("job-execute-archive-hot-dsl-context-provider") DSLContextProvider dslContextProvider) { + log.info("Init GseScriptAgentTaskRecordDAO"); + return new GseScriptAgentTaskRecordDAO(dslContextProvider); + } + + @Bean + public GseScriptAgentTaskArchiver gseScriptAgentTaskArchiver( + ObjectProvider jobInstanceColdDAOObjectProvider, + GseScriptAgentTaskRecordDAO gseScriptAgentTaskRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage + ) { + return new GseScriptAgentTaskArchiver( + jobInstanceColdDAOObjectProvider.getIfAvailable(), + gseScriptAgentTaskRecordDAO, + archiveTablePropsStorage); + } + + @Bean(name = "gseFileAgentTaskRecordDAO") + public GseFileAgentTaskRecordDAO gseFileAgentTaskRecordDAO( + @Qualifier("job-execute-archive-hot-dsl-context-provider") DSLContextProvider dslContextProvider) { + log.info("Init GseFileAgentTaskRecordDAO"); + return new GseFileAgentTaskRecordDAO(dslContextProvider); + } + + @Bean + public GseFileAgentTaskArchiver gseFileAgentTaskArchiver( + ObjectProvider jobInstanceColdDAOObjectProvider, + GseFileAgentTaskRecordDAO gseFileAgentTaskRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage + ) { + return new GseFileAgentTaskArchiver( + jobInstanceColdDAOObjectProvider.getIfAvailable(), + gseFileAgentTaskRecordDAO, + archiveTablePropsStorage); + } + + @Bean(name = "gseScriptExecuteObjTaskRecordDAO") + public GseScriptExecuteObjTaskRecordDAO gseScriptExecuteObjTaskRecordDAO( + @Qualifier("job-execute-archive-hot-dsl-context-provider") DSLContextProvider dslContextProvider) { + log.info("Init GseScriptExecuteObjTaskRecordDAO"); + return new GseScriptExecuteObjTaskRecordDAO(dslContextProvider); + } + + @Bean + public GseScriptExecuteObjTaskArchiver gseScriptExecuteObjTaskArchiver( + ObjectProvider jobInstanceColdDAOObjectProvider, + GseScriptExecuteObjTaskRecordDAO gseScriptExecuteObjTaskRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage + ) { + return new GseScriptExecuteObjTaskArchiver( + jobInstanceColdDAOObjectProvider.getIfAvailable(), + gseScriptExecuteObjTaskRecordDAO, + archiveTablePropsStorage); + } + + @Bean(name = "gseFileExecuteObjTaskRecordDAO") + public GseFileExecuteObjTaskRecordDAO gseFileExecuteObjTaskRecordDAO( + @Qualifier("job-execute-archive-hot-dsl-context-provider") DSLContextProvider dslContextProvider) { + log.info("Init GseFileExecuteObjTaskRecordDAO"); + return new GseFileExecuteObjTaskRecordDAO(dslContextProvider); + } + + @Bean + public GseFileExecuteObjTaskArchiver gseFileExecuteObjTaskArchiver( + ObjectProvider jobInstanceColdDAOObjectProvider, + GseFileExecuteObjTaskRecordDAO gseFileExecuteObjTaskRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage + ) { + return new GseFileExecuteObjTaskArchiver( + jobInstanceColdDAOObjectProvider.getIfAvailable(), + gseFileExecuteObjTaskRecordDAO, + archiveTablePropsStorage); + } + + @Bean(name = "stepInstanceRollingTaskRecordDAO") + public StepInstanceRollingTaskRecordDAO stepInstanceRollingTaskRecordDAO( + @Qualifier("job-execute-archive-hot-dsl-context-provider") DSLContextProvider dslContextProvider) { + log.info("Init StepInstanceRollingTaskRecordDAO"); + return new StepInstanceRollingTaskRecordDAO(dslContextProvider); + } + + @Bean + public StepInstanceRollingTaskArchiver stepInstanceRollingTaskArchiver( + ObjectProvider jobInstanceColdDAOObjectProvider, + StepInstanceRollingTaskRecordDAO stepInstanceRollingTaskRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage + ) { + return new StepInstanceRollingTaskArchiver( + jobInstanceColdDAOObjectProvider.getIfAvailable(), + stepInstanceRollingTaskRecordDAO, + archiveTablePropsStorage); + } + + @Bean(name = "rollingConfigRecordDAO") + public RollingConfigRecordDAO rollingConfigRecordDAO( + @Qualifier("job-execute-archive-hot-dsl-context-provider") DSLContextProvider dslContextProvider) { + log.info("Init RollingConfigRecordDAO"); + return new RollingConfigRecordDAO(dslContextProvider); + } + + @Bean + public RollingConfigArchiver rollingConfigArchiver( + ObjectProvider jobInstanceColdDAOObjectProvider, + RollingConfigRecordDAO rollingConfigRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage + ) { + return new RollingConfigArchiver( + jobInstanceColdDAOObjectProvider.getIfAvailable(), + rollingConfigRecordDAO, + archiveTablePropsStorage); + } + + @Bean(name = "taskInstanceHostRecordDAO") + public TaskInstanceHostRecordDAO taskInstanceHostRecordDAO( + @Qualifier("job-execute-archive-hot-dsl-context-provider") DSLContextProvider dslContextProvider) { + log.info("Init TaskInstanceHostRecordDAO"); + return new TaskInstanceHostRecordDAO(dslContextProvider); + } + + @Bean + public TaskInstanceHostArchiver taskInstanceHostArchiver( + ObjectProvider jobInstanceColdDAOObjectProvider, + TaskInstanceHostRecordDAO taskInstanceHostRecordDAO, + ArchiveTablePropsStorage archiveTablePropsStorage + ) { + return new TaskInstanceHostArchiver( + jobInstanceColdDAOObjectProvider.getIfAvailable(), + taskInstanceHostRecordDAO, + archiveTablePropsStorage); + } + + @Bean + public JobInstanceSubTableArchivers jobInstanceSubTableArchivers( + FileSourceTaskLogArchiver fileSourceTaskLogArchiver, + GseFileAgentTaskArchiver gseFileAgentTaskArchiver, + GseFileExecuteObjTaskArchiver gseFileExecuteObjTaskArchiver, + GseScriptAgentTaskArchiver gseScriptAgentTaskArchiver, + GseScriptExecuteObjTaskArchiver gseScriptExecuteObjTaskArchiver, + GseTaskArchiver gseTaskArchiver, + OperationLogArchiver operationLogArchiver, + RollingConfigArchiver rollingConfigArchiver, + StepInstanceArchiver stepInstanceArchiver, + StepInstanceConfirmArchiver stepInstanceConfirmArchiver, + StepInstanceFileArchiver stepInstanceFileArchiver, + StepInstanceScriptArchiver stepInstanceScriptArchiver, + StepInstanceRollingTaskArchiver stepInstanceRollingTaskArchiver, + StepInstanceVariableArchiver stepInstanceVariableArchiver, + TaskInstanceHostArchiver taskInstanceHostArchiver, + TaskInstanceVariableArchiver taskInstanceVariableArchiver) { + return new JobInstanceSubTableArchivers( + fileSourceTaskLogArchiver, + gseFileAgentTaskArchiver, + gseFileExecuteObjTaskArchiver, + gseScriptAgentTaskArchiver, + gseScriptExecuteObjTaskArchiver, + gseTaskArchiver, + operationLogArchiver, + rollingConfigArchiver, + stepInstanceArchiver, + stepInstanceConfirmArchiver, + stepInstanceFileArchiver, + stepInstanceScriptArchiver, + stepInstanceRollingTaskArchiver, + stepInstanceVariableArchiver, + taskInstanceHostArchiver, + taskInstanceVariableArchiver); + } + + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecuteMongoDBConfiguration.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecuteMongoDBConfiguration.java new file mode 100644 index 0000000000..30af19d0c8 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecuteMongoDBConfiguration.java @@ -0,0 +1,93 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.config; + +import com.mongodb.ConnectionString; +import com.mongodb.MongoClientSettings; +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import com.tencent.bk.job.backup.archive.JobLogArchivers; +import com.tencent.bk.job.backup.archive.impl.JobFileLogArchiver; +import com.tencent.bk.job.backup.archive.impl.JobScriptLogArchiver; +import com.tencent.bk.job.backup.archive.service.ArchiveTaskService; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.boot.autoconfigure.mongo.MongoClientSettingsBuilderCustomizer; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.mongodb.core.MongoTemplate; + +import java.util.List; + +@Configuration("executeMongoDBConfiguration") +@ConditionalOnExpression("${job.backup.archive.execute-log.enabled:false}") +public class ExecuteMongoDBConfiguration { + + @Value("${spring.datasource.job-execute-mongodb.uri:}") + private String mongoUri; + + @Value("${spring.datasource.job-execute-mongodb.database:joblog}") + private String database; + + @Bean + public MongoClient mongoClient(List clientSettingsCustomizers) { + MongoClientSettings.Builder builder = MongoClientSettings.builder(); + clientSettingsCustomizers.forEach(c -> c.customize(builder)); + MongoClientSettings settings = builder + .applyConnectionString(new ConnectionString(mongoUri)) + .build(); + return MongoClients.create(settings); + } + + @Bean + public MongoTemplate mongoTemplate(MongoClient mongoClient) { + return new MongoTemplate(mongoClient, database); + } + + @Bean + public JobScriptLogArchiver jobScriptLogArchiver(MongoTemplate mongoTemplate, + ArchiveTaskService archiveTaskService, + JobLogArchiveProperties archiveProperties) { + return new JobScriptLogArchiver(mongoTemplate, + archiveTaskService, + archiveProperties); + } + + @Bean + public JobFileLogArchiver jobFileLogArchiver(MongoTemplate mongoTemplate, + ArchiveTaskService archiveTaskService, + JobLogArchiveProperties archiveProperties) { + return new JobFileLogArchiver(mongoTemplate, + archiveTaskService, + archiveProperties); + } + + @Bean + public JobLogArchivers jobLogArchivers(JobFileLogArchiver jobFileLogArchiver, + JobScriptLogArchiver jobScriptLogArchiver) { + return new JobLogArchivers(jobFileLogArchiver, + jobScriptLogArchiver); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecutorConfiguration.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecutorConfiguration.java deleted file mode 100644 index 79565daf87..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/ExecutorConfiguration.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.config; - -import com.google.common.util.concurrent.ThreadFactoryBuilder; -import lombok.extern.slf4j.Slf4j; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; - -@Slf4j -@Configuration -public class ExecutorConfiguration { - - @Bean("archiveExecutor") - public ThreadPoolExecutor archiveExecutor() { - return new ThreadPoolExecutor( - 20, - 20, - 0L, - TimeUnit.MILLISECONDS, - new LinkedBlockingQueue<>(20), - new ThreadFactoryBuilder().setNameFormat("archive-thread-pool-%d").build(), - new ThreadPoolExecutor.AbortPolicy() - ); - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/JobBackupAutoConfiguration.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/JobBackupAutoConfiguration.java index c18d214066..f99a23c179 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/JobBackupAutoConfiguration.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/JobBackupAutoConfiguration.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,11 +24,7 @@ package com.tencent.bk.job.backup.config; -import com.tencent.bk.job.backup.client.ServiceApplicationResourceClient; import com.tencent.bk.job.common.artifactory.sdk.ArtifactoryClient; -import com.tencent.bk.job.common.esb.metrics.EsbApiTimedAspect; -import com.tencent.bk.job.common.service.AppScopeMappingService; -import com.tencent.bk.job.manage.AppScopeMappingServiceImpl; import io.micrometer.core.instrument.MeterRegistry; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; @@ -36,10 +32,6 @@ @Configuration public class JobBackupAutoConfiguration { - @Bean - public EsbApiTimedAspect esbApiTimedAspect(@Autowired MeterRegistry meterRegistry) { - return new EsbApiTimedAspect(meterRegistry); - } @Bean public ArtifactoryClient artifactoryClient(@Autowired ArtifactoryConfig artifactoryConfig, @@ -51,9 +43,4 @@ public ArtifactoryClient artifactoryClient(@Autowired ArtifactoryConfig artifact meterRegistry ); } - - @Bean - AppScopeMappingService appScopeMappingService(ServiceApplicationResourceClient applicationResource) { - return new AppScopeMappingServiceImpl(applicationResource); - } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/JobLogArchiveProperties.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/JobLogArchiveProperties.java new file mode 100644 index 0000000000..77dcc671e3 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/JobLogArchiveProperties.java @@ -0,0 +1,77 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.config; + +import com.tencent.bk.job.backup.constant.ArchiveModeEnum; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import org.springframework.boot.context.properties.ConfigurationProperties; + +/** + * 作业执行日志归档配置 + */ +@Getter +@Setter +@ToString +@ConfigurationProperties(prefix = "job.backup.archive.execute-log") +public class JobLogArchiveProperties { + /** + * 是否启用执行日志归档 + */ + private boolean enabled = false; + + /** + * 是否试运行 + */ + private boolean dryRun = true; + + /** + * 执行日志归档模式 + * + * @see ArchiveModeEnum + */ + private String mode; + + /** + * 归档任务触发时间 + */ + private String cron; + + /** + * 归档数据时间范围计算所依据的时区，如果不指定默认为系统时区 + */ + private String timeZone; + + /** + * 执行日志保留天数 + */ + private int keepDays = 360; + + /** + * 执行日志归档任务并行数量 + */ + private Integer concurrent = 6; +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/LocalFileConfigForBackup.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/LocalFileConfigForBackup.java index a75040279e..553fa245a3 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/LocalFileConfigForBackup.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/LocalFileConfigForBackup.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/NfsStorageSystemConfig.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/NfsStorageSystemConfig.java index 777e30e587..ff8dfedd26 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/NfsStorageSystemConfig.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/NfsStorageSystemConfig.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/SpringScheduleConfig.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/SpringScheduleConfig.java new file mode 100644 index 0000000000..5476b6887e --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/config/SpringScheduleConfig.java @@ -0,0 +1,20 @@ +package com.tencent.bk.job.backup.config; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.annotation.SchedulingConfigurer; +import org.springframework.scheduling.config.ScheduledTaskRegistrar; + +import java.util.concurrent.ScheduledThreadPoolExecutor; + +@Configuration +@Slf4j +public class SpringScheduleConfig implements SchedulingConfigurer { + public void configureTasks(ScheduledTaskRegistrar taskRegistrar) { + log.info("Configure spring cron task scheduler"); + //设定一个长度5的定时任务线程池 + taskRegistrar.setScheduler(new ScheduledThreadPoolExecutor(5, (r, executor) -> log.error( + "ScheduledThreadPoolExecutor rejected a runnable"))); + } + +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/ArchiveDBEnum.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/ArchiveDBEnum.java index 76084e1f89..c8164edcee 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/ArchiveDBEnum.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/ArchiveDBEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/ArchiveModeEnum.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/ArchiveModeEnum.java new file mode 100644 index 0000000000..693e43f44f --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/ArchiveModeEnum.java @@ -0,0 +1,68 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.constant; + +/** + * 归档模式 + */ +public enum ArchiveModeEnum { + /** + * 仅删除原始数据 + */ + DELETE_ONLY(Constants.DELETE_ONLY), + /** + * 备份数据后再删除 + */ + BACKUP_THEN_DELETE(Constants.BACKUP_THEN_DELETE), + /** + * 仅备份数据,不删除原始数据。该模式为开发/测试场景下使用，暂不对外开放 + */ + BACKUP_ONLY(Constants.BACKUP_ONLY); + + ArchiveModeEnum(String mode) { + this.mode = mode; + } + + public interface Constants { + String DELETE_ONLY = "deleteOnly"; + String BACKUP_THEN_DELETE = "backupThenDelete"; + String BACKUP_ONLY = "backupOnly"; + } + + private final String mode; + + public static ArchiveModeEnum valOf(String mode) { + for (ArchiveModeEnum value : values()) { + if (value.mode.equalsIgnoreCase(mode)) { + return value; + } + } + throw new IllegalArgumentException("No ArchiveModeEnum constant: " + mode); + } + + public String getMode() { + return mode; + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/ArchiveTaskStatusEnum.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/ArchiveTaskStatusEnum.java new file mode 100644 index 0000000000..fe696fbfe7 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/ArchiveTaskStatusEnum.java @@ -0,0 +1,73 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.constant; + +import lombok.Getter; + +/** + * 归档任务状态 + */ +@Getter +public enum ArchiveTaskStatusEnum { + /** + * 待执行 + */ + PENDING(0), + /** + * 暂停 + */ + SUSPENDED(1), + /** + * 运行中 + */ + RUNNING(2), + /** + * 失败 + */ + FAIL(3), + /** + * 成功 + */ + SUCCESS(4), + /** + * 试运行 + */ + DRYRUN(5); + + private final int status; + + ArchiveTaskStatusEnum(int status) { + this.status = status; + } + + public static ArchiveTaskStatusEnum valOf(int status) { + for (ArchiveTaskStatusEnum taskStatus : values()) { + if (taskStatus.getStatus() == status) { + return taskStatus; + } + } + throw new IllegalArgumentException("No ArchiveTaskStatusEnum constant: " + status); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/ArchiveTaskTypeEnum.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/ArchiveTaskTypeEnum.java new file mode 100644 index 0000000000..bb86f615a6 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/ArchiveTaskTypeEnum.java @@ -0,0 +1,58 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.constant; + +import lombok.Getter; + +@Getter +public enum ArchiveTaskTypeEnum { + /** + * 作业实例数据归档 + */ + JOB_INSTANCE(1), + /** + * 作业实例按业务冗余数据归档 + */ + JOB_INSTANCE_APP(2), + /** + * 作业执行日志归档 + */ + JOB_EXECUTE_LOG(3); + + private final int type; + + ArchiveTaskTypeEnum(int type) { + this.type = type; + } + + public static ArchiveTaskTypeEnum valOf(int type) { + for (ArchiveTaskTypeEnum taskType : values()) { + if (taskType.getType() == type) { + return taskType; + } + } + throw new IllegalArgumentException("No ArchiveTaskTypeEnum constant: " + type); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/BackupJobStatusEnum.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/BackupJobStatusEnum.java index 91eae0ac74..6f9dcf139b 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/BackupJobStatusEnum.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/BackupJobStatusEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -66,14 +66,14 @@ public enum BackupJobStatusEnum { PROCESSING(5), /** - * 处理成功 + * 全部处理成功 */ - SUCCESS(6), + ALL_SUCCESS(6), /** - * 处理失败 + * 全部处理失败 */ - FAILED(7), + ALL_FAILED(7), /** * 已取消 @@ -83,7 +83,12 @@ public enum BackupJobStatusEnum { /** * 已完成 */ - FINISHED(9); + FINISHED(9), + + /** + * 部分成功/失败 + */ + PARTIAL_FAILED(10); @JsonValue private final Integer status; diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/Constant.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/Constant.java index 89f1d25b09..6f7a441f68 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/Constant.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/Constant.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/DbDataNodeTypeEnum.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/DbDataNodeTypeEnum.java new file mode 100644 index 0000000000..4f5c850bb8 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/DbDataNodeTypeEnum.java @@ -0,0 +1,62 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.constant; + +import lombok.Getter; + +/** + * DB 数据节点类型 + */ +@Getter +public enum DbDataNodeTypeEnum { + /** + * 单MySql节点 + */ + STANDALONE(0), + /** + * MySql分库分表节点 + */ + SHARDING(1), + + /** + * 单MongoDB节点 + */ + SINGLE_MONGODB(2); + + private final int value; + + DbDataNodeTypeEnum(int value) { + this.value = value; + } + + public static DbDataNodeTypeEnum valOf(int type) { + for (DbDataNodeTypeEnum dataNodeType : values()) { + if (dataNodeType.getValue() == type) { + return dataNodeType; + } + } + throw new IllegalArgumentException("No DbDataNodeTypeEnum constant: " + type); + } +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/DuplicateIdHandlerEnum.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/DuplicateIdHandlerEnum.java index 5be1a7c94f..6be9eaa011 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/DuplicateIdHandlerEnum.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/DuplicateIdHandlerEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/LogEntityTypeEnum.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/LogEntityTypeEnum.java index 9cfb60da42..c3be722eb9 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/LogEntityTypeEnum.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/LogEntityTypeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/LogMessage.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/LogMessage.java index de62d1297d..7068cbe0ec 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/LogMessage.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/LogMessage.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -55,11 +55,15 @@ public class LogMessage { public static final String CORRECT_FILE_TYPE = "job.backup.correctFileType"; public static final String WRONG_FILE_TYPE = "job.backup.wrongFileType"; public static final String EXTRACT_FILE_DATA = "job.backup.extractFileData"; + // 导入文件已过期！ + public static final String IMPORT_FILE_EXPIRED = "job.backup.importFileExpired"; public static final String EXTRACT_SUCCESS = "job.backup.extractSuccess"; public static final String FILE_ENCRYPTED = "job.backup.fileEncrypted"; public static final String CORRECT_PASSWORD = "job.backup.correctPassword"; public static final String WRONG_PASSWORD = "job.backup.wrongPassword"; public static final String EXTRACT_FAILED = "job.backup.extractFailed"; + // 未找到待导入文件 + public static final String CANNOT_FIND_IMPORT_FILE = "job.backup.cannotFindImportFile"; public static final String START_IMPORT = "job.backup.startImport"; public static final String IMPORT_SETTING = "job.backup.importSetting"; public static final String ID_AUTO_INCREMENT = "job.backup.idAutoIncrement"; @@ -67,19 +71,35 @@ public class LogMessage { public static final String ID_KEEP_ON_DUPLICATE_SKIP = "job.backup.idKeepOnDuplicateSkip"; public static final String NAME_DUPLICATE_SUFFIX = "job.backup.nameDuplicateSuffix"; + // ID不保留, 将以自增的方式导入 public static final String ID_AUTO_INCREMENT_SUFFIX = "job.backup.idAutoIncrementSuffix"; + // ID 已存在, 改为自增 ID 进行导入 public static final String ID_DUPLICATE_INCREMENT_SUFFIX = "job.backup.idDuplicateIncrementSuffix"; + // ID 已存在, 取消导入该%s public static final String ID_DUPLICATE_SKIP_SUFFIX = "job.backup.idDuplicateSkipSuffix"; + // ID不存在, 将以原始ID进行导入 public static final String ID_KEEP_SUFFIX = "job.backup.idKeepSuffix"; + // 准备导入作业模板: [%d] %s public static final String START_IMPORT_TEMPLATE = "job.backup.startImportTemplate"; + // 模板名称已存在! 自动更改为 %s public static final String TEMPLATE_NAME_CHANGE = "job.backup.templateNameChange"; + // 模板导入成功! public static final String IMPORT_TEMPLATE_SUCCESS = "job.backup.importTemplateSuccess"; + // 作业模板名称超长，跳过当前模版和该模版下的所有执行方案 + public static final String TEMPLATE_NAME_TOO_LONG_SKIP = "job.backup.templateNameTooLongSkip"; + // >>>>> 准备导入作业执行方案: %s public static final String START_IMPORT_PLAN = "job.backup.startImportPlan"; public static final String IMPORT_PLAN_SUCCESS = "job.backup.importPlanSuccess"; public static final String IMPORT_FAILED = "job.backup.importFailed"; public static final String IMPORT_FINISHED = "job.backup.importFinished"; + + // [%s]账号[%s]不存在，请手动添加，添加完成后修改作业对应的字段 + public static final String IMPORT_ACCOUNT_NOT_EXIST = "job.backup.importAccountNotExist"; + + // 作业[%s]步骤[%s]的账号无效 + public static final String IMPORT_STEP_ACCOUNT_INVALID = "job.backup.importStepAccountInvalid"; } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/LogTypeEnum.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/LogTypeEnum.java index 3c43bd18ba..aeec7b5720 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/LogTypeEnum.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/LogTypeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -44,8 +44,7 @@ public enum LogTypeEnum { /** * 导入日志 */ - IMPORT(2), - ; + IMPORT(2); @JsonValue private final Integer type; diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/SecretHandlerEnum.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/SecretHandlerEnum.java index 42abfb34f0..e67e423846 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/SecretHandlerEnum.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/constant/SecretHandlerEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/crypto/BackupFileCryptoService.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/crypto/BackupFileCryptoService.java new file mode 100644 index 0000000000..62d6dd6917 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/crypto/BackupFileCryptoService.java @@ -0,0 +1,82 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.crypto; + +import com.tencent.bk.job.common.crypto.CryptoScenarioEnum; +import com.tencent.bk.job.common.crypto.JobCryptorNames; +import com.tencent.bk.job.common.crypto.SymmetricCryptoService; +import com.tencent.bk.sdk.crypto.exception.CryptoException; +import com.tencent.bk.sdk.crypto.util.CryptorMetaUtil; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; + +/** + * 导入导出文件相关加解密服务 + */ +@Slf4j +@Service +public class BackupFileCryptoService { + + private final SymmetricCryptoService symmetricCryptoService; + + @Autowired + public BackupFileCryptoService(SymmetricCryptoService symmetricCryptoService) { + this.symmetricCryptoService = symmetricCryptoService; + } + + public void encryptBackupFile(String password, File inFile, File outFile) { + try (FileInputStream in = new FileInputStream(inFile); FileOutputStream out = new FileOutputStream(outFile)) { + symmetricCryptoService.encrypt(password, in, out, CryptoScenarioEnum.BACKUP_FILE); + } catch (Exception e) { + throw new CryptoException("Fail to encrypt backupFile", e); + } + } + + public void decryptBackupFile(String password, File inFile, File outFile) { + try { + FileInputStream in = new FileInputStream(inFile); + log.debug("in.available={}", in.available()); + BufferedInputStream bis = new BufferedInputStream(in); + String algorithm = CryptorMetaUtil.getCryptorNameFromCipherStream(bis); + if (StringUtils.isBlank(algorithm)) { + algorithm = JobCryptorNames.AES_CBC; + } + try (FileOutputStream out = new FileOutputStream(outFile)) { + log.debug("Use {} to decryptBackupFile, bis.available={}", algorithm, bis.available()); + symmetricCryptoService.decrypt(password, bis, out, algorithm); + } + } catch (Exception e) { + throw new CryptoException("Fail to decrypt backupFile", e); + } + } + +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/crypto/ExportJobPasswordCryptoService.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/crypto/ExportJobPasswordCryptoService.java new file mode 100644 index 0000000000..56ad957a18 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/crypto/ExportJobPasswordCryptoService.java @@ -0,0 +1,79 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.crypto; + +import com.tencent.bk.job.common.crypto.CryptoScenarioEnum; +import com.tencent.bk.job.common.crypto.SymmetricCryptoService; +import com.tencent.bk.sdk.crypto.cryptor.consts.CryptorNames; +import com.tencent.bk.sdk.crypto.util.CryptorMetaUtil; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +/** + * 作业导出任务密码相关加解密服务 + */ +@Slf4j +@Service +public class ExportJobPasswordCryptoService { + + private final SymmetricCryptoService symmetricCryptoService; + + @Autowired + public ExportJobPasswordCryptoService(SymmetricCryptoService symmetricCryptoService) { + this.symmetricCryptoService = symmetricCryptoService; + } + + public String getExportJobPasswordEncryptAlgorithmByCipher(String cipher) { + if (StringUtils.isEmpty(cipher)) { + return CryptorNames.NONE; + } + String algorithm = CryptorMetaUtil.getCryptorNameFromCipher(cipher); + if (algorithm != null) { + return algorithm; + } + return CryptorNames.NONE; + } + + public String encryptExportJobPassword(String exportJobPassword) { + if (StringUtils.isEmpty(exportJobPassword)) { + return exportJobPassword; + } + return symmetricCryptoService.encryptToBase64Str(exportJobPassword, CryptoScenarioEnum.EXPORT_JOB_PASSWORD); + } + + public String decryptExportJobPassword(String encryptedExportJobPassword) { + if (StringUtils.isEmpty(encryptedExportJobPassword)) { + return encryptedExportJobPassword; + } + String algorithm = getExportJobPasswordEncryptAlgorithmByCipher(encryptedExportJobPassword); + if (StringUtils.isBlank(algorithm)) { + return encryptedExportJobPassword; + } + return symmetricCryptoService.decrypt(encryptedExportJobPassword, algorithm); + } + +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/ArchiveProgressDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/ArchiveProgressDAO.java deleted file mode 100644 index 625309b8d4..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/ArchiveProgressDAO.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.dao; - -import com.tencent.bk.job.backup.model.dto.ArchiveProgressDTO; - -public interface ArchiveProgressDAO { - ArchiveProgressDTO queryArchiveProgress(String table); - - void saveArchiveProgress(ArchiveProgressDTO archiveProgress); - - void saveDeleteProgress(ArchiveProgressDTO archiveProgress); -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/ExecuteRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/ExecuteRecordDAO.java deleted file mode 100644 index 2b13ac02c8..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/ExecuteRecordDAO.java +++ /dev/null @@ -1,54 +0,0 @@ -package com.tencent.bk.job.backup.dao; - -import org.jooq.Record; -import org.jooq.Table; -import org.jooq.TableField; - -import java.util.List; - -/** - * job-execute 微服务的表数据DAO - * - * @param 表记录 - */ -public interface ExecuteRecordDAO { - - /** - * 获取表 - * - * @return 表 - */ - Table getTable(); - - /** - * 获取用于查询归档记录的ID字段 - * - * @return ID字段 - */ - TableField getArchiveIdField(); - - /** - * 根据起始/结束ID获取表记录 - * - * @param start 起始ID - * @param end 结束ID - * @return 表记录 - */ - List listRecords(Long start, Long end); - - /** - * 根据起始/结束ID删除表记录 - * - * @param start 起始ID - * @param end 结束ID - * @return 删除的记录数量 - */ - int deleteRecords(Long start, Long end); - - /** - * 获取表中第一条数据的ID(按照ID升序) - * - * @return id值 - */ - Long getFirstArchiveId(); -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/ExportJobDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/ExportJobDAO.java index f616ab2a92..3806750bf1 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/ExportJobDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/ExportJobDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/ImportJobDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/ImportJobDAO.java index 35d7839cd7..45ccce4473 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/ImportJobDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/ImportJobDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/LogDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/LogDAO.java index 44c72af14c..3a9dc02524 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/LogDAO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/LogDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/AbstractExecuteRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/AbstractExecuteRecordDAO.java deleted file mode 100644 index e4bf2ec29d..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/AbstractExecuteRecordDAO.java +++ /dev/null @@ -1,75 +0,0 @@ -package com.tencent.bk.job.backup.dao.impl; - -import com.tencent.bk.job.backup.config.ArchiveConfig; -import com.tencent.bk.job.backup.dao.ExecuteRecordDAO; -import org.jooq.Condition; -import org.jooq.DSLContext; -import org.jooq.Record; -import org.jooq.Record1; -import org.jooq.Result; -import org.jooq.Table; - -import java.util.ArrayList; -import java.util.List; - -import static org.jooq.impl.DSL.min; - -public abstract class AbstractExecuteRecordDAO implements ExecuteRecordDAO { - - protected final DSLContext context; - protected final ArchiveConfig archiveConfig; - - public AbstractExecuteRecordDAO(DSLContext context, ArchiveConfig archiveConfig) { - this.context = context; - this.archiveConfig = archiveConfig; - } - - @Override - public List listRecords(Long start, Long end) { - Result result = query(getTable(), buildConditions(start, end)); - return result.into(getTable()); - } - - @Override - public int deleteRecords(Long start, Long end) { - return deleteWithLimit(getTable(), buildConditions(start, end)); - } - - private List buildConditions(Long start, Long end) { - List conditions = new ArrayList<>(); - conditions.add(getArchiveIdField().greaterThan(start)); - conditions.add(getArchiveIdField().lessOrEqual(end)); - return conditions; - } - - private int deleteWithLimit(Table table, List conditions) { - int totalDeleteRows = 0; - int maxLimitedDeleteRows = archiveConfig.getDeleteLimitRowCount(); - while (true) { - int deletedRows = context.delete(table).where(conditions).limit(maxLimitedDeleteRows).execute(); - totalDeleteRows += deletedRows; - if (deletedRows < maxLimitedDeleteRows) { - break; - } - } - return totalDeleteRows; - } - - private Result query(Table table, List conditions) { - return context.select() - .from(table) - .where(conditions) - .fetch(); - } - - @Override - public Long getFirstArchiveId() { - Record1 firstArchiveIdRecord = context.select(min(getArchiveIdField())).from(getTable()).fetchOne(); - if (firstArchiveIdRecord != null && firstArchiveIdRecord.get(0) != null) { - return (Long) firstArchiveIdRecord.get(0); - } - return 0L; - } - - public abstract Table getTable(); -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/ArchiveProgressDAOImpl.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/ArchiveProgressDAOImpl.java deleted file mode 100644 index e9cd6a9fa6..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/ArchiveProgressDAOImpl.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.dao.impl; - -import com.tencent.bk.job.backup.dao.ArchiveProgressDAO; -import com.tencent.bk.job.backup.model.dto.ArchiveProgressDTO; -import com.tencent.bk.job.backup.model.tables.ArchiveProgress; -import org.jooq.DSLContext; -import org.jooq.Record; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.stereotype.Repository; - -@Repository -public class ArchiveProgressDAOImpl implements ArchiveProgressDAO { - - private final DSLContext ctx; - private final ArchiveProgress T = ArchiveProgress.ARCHIVE_PROGRESS; - - @Autowired - public ArchiveProgressDAOImpl(@Qualifier("job-backup-dsl-context") DSLContext ctx) { - this.ctx = ctx; - } - - @Override - public ArchiveProgressDTO queryArchiveProgress(String table) { - Record record = ctx.select(T.TABLE_NAME, T.LAST_ARCHIVED_ID, T.LAST_ARCHIVE_TIME, - T.LAST_DELETED_ID, T.LAST_DELETE_TIME) - .from(T) - .where(T.TABLE_NAME.eq(table)) - .fetchOne(); - - return extract(record); - } - - private ArchiveProgressDTO extract(Record record) { - if (record == null) { - return null; - } - ArchiveProgressDTO archiveProgress = new ArchiveProgressDTO(); - archiveProgress.setTableName(record.get(T.TABLE_NAME)); - archiveProgress.setLastArchivedId(record.get(T.LAST_ARCHIVED_ID)); - archiveProgress.setLastArchiveTime(record.get(T.LAST_ARCHIVE_TIME)); - archiveProgress.setLastDeletedId(record.get(T.LAST_DELETED_ID)); - archiveProgress.setLastDeleteTime(record.get(T.LAST_DELETE_TIME)); - return archiveProgress; - } - - @Override - public void saveArchiveProgress(ArchiveProgressDTO archiveProgress) { - ctx.insertInto(T, T.TABLE_NAME, T.LAST_ARCHIVED_ID, T.LAST_ARCHIVE_TIME) - .values(archiveProgress.getTableName(), archiveProgress.getLastArchivedId(), - archiveProgress.getLastArchiveTime()) - .onDuplicateKeyUpdate() - .set(T.LAST_ARCHIVED_ID, archiveProgress.getLastArchivedId()) - .set(T.LAST_ARCHIVE_TIME, archiveProgress.getLastArchiveTime()) - .execute(); - } - - @Override - public void saveDeleteProgress(ArchiveProgressDTO archiveProgress) { - ctx.insertInto(T, T.TABLE_NAME, T.LAST_DELETED_ID, T.LAST_DELETE_TIME) - .values(archiveProgress.getTableName(), archiveProgress.getLastDeletedId(), - archiveProgress.getLastDeleteTime()) - .onDuplicateKeyUpdate() - .set(T.LAST_DELETED_ID, archiveProgress.getLastDeletedId()) - .set(T.LAST_DELETE_TIME, archiveProgress.getLastDeleteTime()) - .execute(); - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/ExecuteArchiveDAOImpl.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/ExecuteArchiveDAOImpl.java deleted file mode 100644 index ce350a0ed8..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/ExecuteArchiveDAOImpl.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.dao.impl; - -import com.tencent.bk.job.backup.dao.ExecuteArchiveDAO; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; -import org.jooq.DSLContext; -import org.jooq.Loader; -import org.jooq.LoaderError; -import org.jooq.TableRecord; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.util.List; - - -@Slf4j -public class ExecuteArchiveDAOImpl implements ExecuteArchiveDAO { - - private static final Logger ARCHIVE_FAILED_LOGGER = LoggerFactory.getLogger("ArchiveFailedLogger"); - - private final DSLContext context; - - public ExecuteArchiveDAOImpl(DSLContext context) { - log.info("Init ExecuteArchiveDAO."); - this.context = context; - } - - @Override - public Integer batchInsert(List> recordList, int bulkSize) throws IOException { - long start = System.currentTimeMillis(); - int executeResult = 0; - String table = recordList.get(0).getTable().getName(); - boolean success = true; - try { - Loader loader = - context.loadInto(recordList.get(0).getTable()) - .onErrorIgnore() - .bulkAfter(bulkSize) - .loadRecords(recordList) - .fieldsCorresponding() - .execute(); - executeResult = loader.stored(); - log.info("Load {} data result|executed|{}|processed|{}|stored|{}|ignored|{}|errors|{}", table, - loader.executed(), loader.processed(), loader.stored(), loader.ignored(), loader.errors().size()); - if (CollectionUtils.isNotEmpty(loader.errors())) { - for (LoaderError error : loader.errors()) { - ARCHIVE_FAILED_LOGGER.error("Error while load {} data, error row: {}, exception: {}", table, - error.row(), error.exception().getMessage()); - } - } - } catch (IOException e) { - String errorMsg = String.format("Error while batch loading %s data!", table); - log.error(errorMsg, e); - success = false; - throw e; - } finally { - log.info("Batch insert to {} done! success: {}, total: {}, inserted: {}, cost: {}ms", table, success, - recordList.size(), executeResult, System.currentTimeMillis() - start); - } - - return executeResult; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/ExportJobDAOImpl.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/ExportJobDAOImpl.java index dec835ffd7..d836187617 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/ExportJobDAOImpl.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/ExportJobDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,15 +24,23 @@ package com.tencent.bk.job.backup.dao.impl; +import com.fasterxml.jackson.core.type.TypeReference; import com.tencent.bk.job.backup.constant.BackupJobStatusEnum; +import com.tencent.bk.job.backup.constant.SecretHandlerEnum; +import com.tencent.bk.job.backup.crypto.ExportJobPasswordCryptoService; import com.tencent.bk.job.backup.dao.ExportJobDAO; +import com.tencent.bk.job.backup.model.dto.BackupTemplateInfoDTO; import com.tencent.bk.job.backup.model.dto.ExportJobInfoDTO; import com.tencent.bk.job.backup.model.tables.ExportJob; import com.tencent.bk.job.backup.model.tables.records.ExportJobRecord; -import com.tencent.bk.job.backup.util.DbRecordMapper; import com.tencent.bk.job.common.util.json.JsonMapper; +import com.tencent.bk.job.common.util.json.JsonUtils; import lombok.extern.slf4j.Slf4j; -import org.jooq.*; +import org.jooq.Condition; +import org.jooq.DSLContext; +import org.jooq.Record13; +import org.jooq.Result; +import org.jooq.UpdateSetMoreStep; import org.jooq.types.UByte; import org.jooq.types.ULong; import org.springframework.beans.factory.annotation.Autowired; @@ -42,6 +50,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Locale; /** * @since 28/7/2020 12:26 @@ -52,26 +61,48 @@ public class ExportJobDAOImpl implements ExportJobDAO { private static final ExportJob TABLE = ExportJob.EXPORT_JOB; private final DSLContext context; + private final ExportJobPasswordCryptoService exportJobPasswordCryptoService; @Autowired - public ExportJobDAOImpl(@Qualifier("job-backup-dsl-context") DSLContext context) { + public ExportJobDAOImpl(@Qualifier("job-backup-dsl-context") DSLContext context, + ExportJobPasswordCryptoService exportJobPasswordCryptoService) { this.context = context; + this.exportJobPasswordCryptoService = exportJobPasswordCryptoService; } @Override public String insertExportJob(ExportJobInfoDTO exportJobInfo) { - if (1 == context.insertInto(TABLE) - .columns(TABLE.ID, TABLE.APP_ID, TABLE.CREATOR, TABLE.CREATE_TIME, TABLE.UPDATE_TIME, TABLE.STATUS, - TABLE.PASSWORD, TABLE.PACKAGE_NAME, TABLE.SECRET_HANDLER, TABLE.EXPIRE_TIME, TABLE.TEMPLATE_PLAN_INFO, - TABLE.FILE_NAME, TABLE.LOCALE) - .values(exportJobInfo.getId(), ULong.valueOf(exportJobInfo.getAppId()), exportJobInfo.getCreator(), - ULong.valueOf(exportJobInfo.getCreateTime()), ULong.valueOf(exportJobInfo.getUpdateTime()), - UByte.valueOf(exportJobInfo.getStatus().getStatus()), exportJobInfo.getPassword(), - exportJobInfo.getPackageName(), UByte.valueOf(exportJobInfo.getSecretHandler().getType()), + int affectedNum = context.insertInto(TABLE) + .columns( + TABLE.ID, + TABLE.APP_ID, + TABLE.CREATOR, + TABLE.CREATE_TIME, + TABLE.UPDATE_TIME, + TABLE.STATUS, + TABLE.PASSWORD, + TABLE.PACKAGE_NAME, + TABLE.SECRET_HANDLER, + TABLE.EXPIRE_TIME, + TABLE.TEMPLATE_PLAN_INFO, + TABLE.FILE_NAME, + TABLE.LOCALE + ).values( + exportJobInfo.getId(), + ULong.valueOf(exportJobInfo.getAppId()), + exportJobInfo.getCreator(), + ULong.valueOf(exportJobInfo.getCreateTime()), + ULong.valueOf(exportJobInfo.getUpdateTime()), + UByte.valueOf(exportJobInfo.getStatus().getStatus()), + exportJobPasswordCryptoService.encryptExportJobPassword(exportJobInfo.getPassword()), + exportJobInfo.getPackageName(), + UByte.valueOf(exportJobInfo.getSecretHandler().getType()), ULong.valueOf(exportJobInfo.getExpireTime()), JsonMapper.nonEmptyMapper().toJson(exportJobInfo.getTemplateInfo()), - exportJobInfo.getFileName(), LocaleContextHolder.getLocale().toLanguageTag()) - .execute()) { + exportJobInfo.getFileName(), + LocaleContextHolder.getLocale().toLanguageTag() + ).execute(); + if (1 == affectedNum) { return exportJobInfo.getId(); } else { return null; @@ -85,13 +116,24 @@ public ExportJobInfoDTO getExportJobById(Long appId, String jobId) { if (appId > 0) { conditions.add(TABLE.APP_ID.equal(ULong.valueOf(appId))); } - Record13 record = context.select(TABLE.ID, TABLE.APP_ID, TABLE.CREATOR, TABLE.CREATE_TIME, - TABLE.UPDATE_TIME, TABLE.STATUS, TABLE.PASSWORD, TABLE.PACKAGE_NAME, TABLE.SECRET_HANDLER, - TABLE.EXPIRE_TIME, TABLE.TEMPLATE_PLAN_INFO, TABLE.FILE_NAME, TABLE.LOCALE) - .from(TABLE).where(conditions).fetchOne(); - return DbRecordMapper.convertRecordToExportJobInfo(record); + String, String> record = + context.select( + TABLE.ID, + TABLE.APP_ID, + TABLE.CREATOR, + TABLE.CREATE_TIME, + TABLE.UPDATE_TIME, + TABLE.STATUS, + TABLE.PASSWORD, + TABLE.PACKAGE_NAME, + TABLE.SECRET_HANDLER, + TABLE.EXPIRE_TIME, + TABLE.TEMPLATE_PLAN_INFO, + TABLE.FILE_NAME, + TABLE.LOCALE + ).from(TABLE).where(conditions).fetchOne(); + return convertRecordToExportJobInfo(record); } @Override @@ -99,20 +141,32 @@ public List getExportJobByUser(Long appId, String username) { List conditions = new ArrayList<>(); conditions.add(TABLE.CREATOR.equal(username)); conditions.add(TABLE.APP_ID.equal(ULong.valueOf(appId))); - conditions.add(TABLE.STATUS.in(UByte.valueOf(BackupJobStatusEnum.SUBMIT.getStatus()), + conditions.add(TABLE.STATUS.in( + UByte.valueOf(BackupJobStatusEnum.SUBMIT.getStatus()), UByte.valueOf(BackupJobStatusEnum.PROCESSING.getStatus()), - UByte.valueOf(BackupJobStatusEnum.SUCCESS.getStatus()))); + UByte.valueOf(BackupJobStatusEnum.ALL_SUCCESS.getStatus()) + )); Result< Record13> result = context.select(TABLE.ID, TABLE.APP_ID, TABLE.CREATOR, TABLE.CREATE_TIME, - TABLE.UPDATE_TIME, TABLE.STATUS, TABLE.PASSWORD, TABLE.PACKAGE_NAME, TABLE.SECRET_HANDLER, - TABLE.EXPIRE_TIME, TABLE.TEMPLATE_PLAN_INFO, TABLE.FILE_NAME, TABLE.LOCALE) - .from(TABLE).where(conditions).fetch(); + String>> result = + context.select( + TABLE.ID, + TABLE.APP_ID, + TABLE.CREATOR, + TABLE.CREATE_TIME, + TABLE.UPDATE_TIME, + TABLE.STATUS, + TABLE.PASSWORD, + TABLE.PACKAGE_NAME, + TABLE.SECRET_HANDLER, + TABLE.EXPIRE_TIME, + TABLE.TEMPLATE_PLAN_INFO, + TABLE.FILE_NAME, + TABLE.LOCALE + ).from(TABLE).where(conditions).fetch(); List exportJobInfoList = new ArrayList<>(); - if (result != null) { - result.forEach(record -> exportJobInfoList.add(DbRecordMapper.convertRecordToExportJobInfo(record))); - } + result.forEach(record -> exportJobInfoList.add(convertRecordToExportJobInfo(record))); return exportJobInfoList; } @@ -126,7 +180,10 @@ public boolean updateExportJob(ExportJobInfoDTO exportInfo) { UpdateSetMoreStep updateStep = context.update(TABLE).set(TABLE.UPDATE_TIME, ULong.valueOf(System.currentTimeMillis())); - updateStep = updateStep.set(TABLE.PASSWORD, exportInfo.getPassword()); + updateStep = updateStep.set( + TABLE.PASSWORD, + exportJobPasswordCryptoService.encryptExportJobPassword(exportInfo.getPassword()) + ); if (exportInfo.getStatus() != null) { updateStep = updateStep.set(TABLE.STATUS, UByte.valueOf(exportInfo.getStatus().getStatus())); @@ -147,13 +204,23 @@ public List listOldExportJob() { Result< Record13> result = - context.select(TABLE.ID, TABLE.APP_ID, TABLE.CREATOR, TABLE.CREATE_TIME, TABLE.UPDATE_TIME, - TABLE.STATUS, TABLE.PASSWORD, TABLE.PACKAGE_NAME, TABLE.SECRET_HANDLER, TABLE.EXPIRE_TIME, - TABLE.TEMPLATE_PLAN_INFO, TABLE.FILE_NAME, TABLE.LOCALE).from(TABLE).where(conditions).fetch(); + context.select( + TABLE.ID, + TABLE.APP_ID, + TABLE.CREATOR, + TABLE.CREATE_TIME, + TABLE.UPDATE_TIME, + TABLE.STATUS, + TABLE.PASSWORD, + TABLE.PACKAGE_NAME, + TABLE.SECRET_HANDLER, + TABLE.EXPIRE_TIME, + TABLE.TEMPLATE_PLAN_INFO, + TABLE.FILE_NAME, + TABLE.LOCALE + ).from(TABLE).where(conditions).fetch(); List exportJobInfoList = new ArrayList<>(); - if (result != null) { - result.forEach(record -> exportJobInfoList.add(DbRecordMapper.convertRecordToExportJobInfo(record))); - } + result.forEach(record -> exportJobInfoList.add(convertRecordToExportJobInfo(record))); return exportJobInfoList; } @@ -165,4 +232,33 @@ public boolean setCleanMark(Long appId, String jobId) { return 1 == context.update(TABLE).set(TABLE.IS_CLEANED, UByte.valueOf(1)).where(conditions).execute(); } + public ExportJobInfoDTO convertRecordToExportJobInfo( + Record13 record) { + if (record == null) { + return null; + } + ExportJob table = ExportJob.EXPORT_JOB; + ExportJobInfoDTO exportJobInfo = new ExportJobInfoDTO(); + exportJobInfo.setId(record.get(table.ID)); + exportJobInfo.setAppId(record.get(table.APP_ID).longValue()); + exportJobInfo.setCreator(record.get(table.CREATOR)); + exportJobInfo.setCreateTime(record.get(table.CREATE_TIME).longValue()); + exportJobInfo.setUpdateTime(record.get(table.UPDATE_TIME).longValue()); + exportJobInfo.setStatus(BackupJobStatusEnum.valueOf(record.get(table.STATUS).intValue())); + + // 导出密码解密 + String encryptedPassword = record.get(table.PASSWORD); + exportJobInfo.setPassword(exportJobPasswordCryptoService.decryptExportJobPassword(encryptedPassword)); + + exportJobInfo.setPackageName(record.get(table.PACKAGE_NAME)); + exportJobInfo.setSecretHandler(SecretHandlerEnum.valueOf(record.get(table.SECRET_HANDLER).intValue())); + exportJobInfo.setExpireTime(record.get(table.EXPIRE_TIME).longValue()); + exportJobInfo.setTemplateInfo(JsonUtils.fromJson(record.get(table.TEMPLATE_PLAN_INFO), + new TypeReference>() { + })); + exportJobInfo.setFileName(record.get(table.FILE_NAME)); + exportJobInfo.setLocale(Locale.forLanguageTag(record.get(table.LOCALE))); + return exportJobInfo; + } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/FileSourceTaskRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/FileSourceTaskRecordDAO.java deleted file mode 100644 index 33dc2c3a24..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/FileSourceTaskRecordDAO.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.tencent.bk.job.backup.dao.impl; - -import com.tencent.bk.job.backup.config.ArchiveConfig; -import org.jooq.DSLContext; -import org.jooq.Table; -import org.jooq.TableField; -import org.jooq.generated.tables.FileSourceTaskLog; -import org.jooq.generated.tables.records.FileSourceTaskLogRecord; - -/** - * file_source_task DAO - */ -public class FileSourceTaskRecordDAO extends AbstractExecuteRecordDAO { - - private static final FileSourceTaskLog TABLE = FileSourceTaskLog.FILE_SOURCE_TASK_LOG; - - public FileSourceTaskRecordDAO(DSLContext context, ArchiveConfig archiveConfig) { - super(context, archiveConfig); - } - - @Override - public Table getTable() { - return TABLE; - } - - @Override - public TableField getArchiveIdField() { - return TABLE.STEP_INSTANCE_ID; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseFileAgentTaskRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseFileAgentTaskRecordDAO.java deleted file mode 100644 index b41d333963..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseFileAgentTaskRecordDAO.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.tencent.bk.job.backup.dao.impl; - -import com.tencent.bk.job.backup.config.ArchiveConfig; -import org.jooq.DSLContext; -import org.jooq.Table; -import org.jooq.TableField; -import org.jooq.generated.tables.GseFileAgentTask; -import org.jooq.generated.tables.records.GseFileAgentTaskRecord; - -/** - * gse_file_agent_task DAO - */ -public class GseFileAgentTaskRecordDAO extends AbstractExecuteRecordDAO { - - private static final GseFileAgentTask TABLE = GseFileAgentTask.GSE_FILE_AGENT_TASK; - - public GseFileAgentTaskRecordDAO(DSLContext context, ArchiveConfig archiveConfig) { - super(context, archiveConfig); - } - - @Override - public Table getTable() { - return TABLE; - } - - @Override - public TableField getArchiveIdField() { - return TABLE.STEP_INSTANCE_ID; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseScriptAgentTaskRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseScriptAgentTaskRecordDAO.java deleted file mode 100644 index bcbb40850b..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseScriptAgentTaskRecordDAO.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.tencent.bk.job.backup.dao.impl; - -import com.tencent.bk.job.backup.config.ArchiveConfig; -import org.jooq.DSLContext; -import org.jooq.Table; -import org.jooq.TableField; -import org.jooq.generated.tables.GseScriptAgentTask; -import org.jooq.generated.tables.records.GseScriptAgentTaskRecord; - -/** - * gse_script_agent_task DAO - */ -public class GseScriptAgentTaskRecordDAO extends AbstractExecuteRecordDAO { - - private static final GseScriptAgentTask TABLE = GseScriptAgentTask.GSE_SCRIPT_AGENT_TASK; - - public GseScriptAgentTaskRecordDAO(DSLContext context, ArchiveConfig archiveConfig) { - super(context, archiveConfig); - } - - @Override - public Table getTable() { - return TABLE; - } - - @Override - public TableField getArchiveIdField() { - return TABLE.STEP_INSTANCE_ID; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseTaskIpLogRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseTaskIpLogRecordDAO.java deleted file mode 100644 index eccbe21a04..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseTaskIpLogRecordDAO.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.tencent.bk.job.backup.dao.impl; - -import com.tencent.bk.job.backup.config.ArchiveConfig; -import org.jooq.DSLContext; -import org.jooq.Table; -import org.jooq.TableField; -import org.jooq.generated.tables.GseTaskIpLog; -import org.jooq.generated.tables.records.GseTaskIpLogRecord; - -/** - * gse_task_ip_log DAO - */ -public class GseTaskIpLogRecordDAO extends AbstractExecuteRecordDAO { - - private static final GseTaskIpLog TABLE = GseTaskIpLog.GSE_TASK_IP_LOG; - - public GseTaskIpLogRecordDAO(DSLContext context, ArchiveConfig archiveConfig) { - super(context, archiveConfig); - } - - @Override - public Table getTable() { - return TABLE; - } - - @Override - public TableField getArchiveIdField() { - return TABLE.STEP_INSTANCE_ID; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseTaskLogRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseTaskLogRecordDAO.java deleted file mode 100644 index 348c9531b6..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseTaskLogRecordDAO.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.tencent.bk.job.backup.dao.impl; - -import com.tencent.bk.job.backup.config.ArchiveConfig; -import org.jooq.DSLContext; -import org.jooq.Table; -import org.jooq.TableField; -import org.jooq.generated.tables.GseTaskLog; -import org.jooq.generated.tables.records.GseTaskLogRecord; - -/** - * gse_task_log DAO - */ -public class GseTaskLogRecordDAO extends AbstractExecuteRecordDAO { - - private static final GseTaskLog TABLE = GseTaskLog.GSE_TASK_LOG; - - public GseTaskLogRecordDAO(DSLContext context, ArchiveConfig archiveConfig) { - super(context, archiveConfig); - } - - @Override - public Table getTable() { - return TABLE; - } - - @Override - public TableField getArchiveIdField() { - return TABLE.STEP_INSTANCE_ID; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseTaskRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseTaskRecordDAO.java deleted file mode 100644 index 854921c598..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/GseTaskRecordDAO.java +++ /dev/null @@ -1,27 +0,0 @@ -package com.tencent.bk.job.backup.dao.impl; - -import com.tencent.bk.job.backup.config.ArchiveConfig; -import org.jooq.DSLContext; -import org.jooq.Table; -import org.jooq.TableField; -import org.jooq.generated.tables.GseTask; -import org.jooq.generated.tables.records.GseTaskRecord; - -public class GseTaskRecordDAO extends AbstractExecuteRecordDAO { - - private static final GseTask TABLE = GseTask.GSE_TASK; - - public GseTaskRecordDAO(DSLContext context, ArchiveConfig archiveConfig) { - super(context, archiveConfig); - } - - @Override - public Table getTable() { - return TABLE; - } - - @Override - public TableField getArchiveIdField() { - return TABLE.STEP_INSTANCE_ID; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/ImportJobDAOImpl.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/ImportJobDAOImpl.java index f3dbdb2f00..eca78dd87e 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/ImportJobDAOImpl.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/ImportJobDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -33,7 +33,11 @@ import com.tencent.bk.job.common.util.json.JsonMapper; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; -import org.jooq.*; +import org.jooq.Condition; +import org.jooq.DSLContext; +import org.jooq.Record13; +import org.jooq.Result; +import org.jooq.UpdateSetMoreStep; import org.jooq.types.UByte; import org.jooq.types.ULong; import org.springframework.beans.factory.annotation.Autowired; @@ -111,9 +115,7 @@ public List getImportJobByUser(Long appId, String username) { TABLE.DUPLICATE_SUFFIX, TABLE.DUPLICATE_ID_HANDLER, TABLE.ID_NAME_INFO, TABLE.LOCALE) .from(TABLE).where(conditions).fetch(); List importJobInfoList = new ArrayList<>(); - if (result != null) { - result.forEach(record -> importJobInfoList.add(DbRecordMapper.convertRecordToImportJobInfo(record))); - } + result.forEach(record -> importJobInfoList.add(DbRecordMapper.convertRecordToImportJobInfo(record))); return importJobInfoList; } @@ -123,7 +125,7 @@ public boolean updateImportJobById(ImportJobInfoDTO importJobInfo) { conditions.add(TABLE.ID.equal(importJobInfo.getId())); conditions.add(TABLE.APP_ID.equal(ULong.valueOf(importJobInfo.getAppId()))); conditions.add(TABLE.CREATOR.equal(importJobInfo.getCreator())); - conditions.add(TABLE.STATUS.notIn(UByte.valueOf(BackupJobStatusEnum.FAILED.getStatus()), + conditions.add(TABLE.STATUS.notIn(UByte.valueOf(BackupJobStatusEnum.ALL_FAILED.getStatus()), UByte.valueOf(BackupJobStatusEnum.CANCEL.getStatus()))); UpdateSetMoreStep updateStep = @@ -170,9 +172,7 @@ public List listOldImportJob() { TABLE.STATUS, TABLE.EXPORT_ID, TABLE.FILE_NAME, TABLE.TEMPLATE_PLAN_INFO, TABLE.DUPLICATE_SUFFIX, TABLE.DUPLICATE_ID_HANDLER, TABLE.ID_NAME_INFO, TABLE.LOCALE).from(TABLE).where(conditions).fetch(); List importJobInfoList = new ArrayList<>(); - if (result != null) { - result.forEach(record -> importJobInfoList.add(DbRecordMapper.convertRecordToImportJobInfo(record))); - } + result.forEach(record -> importJobInfoList.add(DbRecordMapper.convertRecordToImportJobInfo(record))); return importJobInfoList; } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/LogDAOImpl.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/LogDAOImpl.java index ec24e7c455..3abdb255fc 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/LogDAOImpl.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/LogDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/OperationLogRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/OperationLogRecordDAO.java deleted file mode 100644 index 0fa7ea537e..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/OperationLogRecordDAO.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.tencent.bk.job.backup.dao.impl; - -import com.tencent.bk.job.backup.config.ArchiveConfig; -import org.jooq.DSLContext; -import org.jooq.Table; -import org.jooq.TableField; -import org.jooq.generated.tables.OperationLog; -import org.jooq.generated.tables.records.OperationLogRecord; - -/** - * operation_log DAO - */ -public class OperationLogRecordDAO extends AbstractExecuteRecordDAO { - - private static final OperationLog TABLE = OperationLog.OPERATION_LOG; - - public OperationLogRecordDAO(DSLContext context, ArchiveConfig archiveConfig) { - super(context, archiveConfig); - } - - @Override - public Table getTable() { - return TABLE; - } - - @Override - public TableField getArchiveIdField() { - return TABLE.TASK_INSTANCE_ID; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/RollingConfigRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/RollingConfigRecordDAO.java deleted file mode 100644 index 9f2243e41e..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/RollingConfigRecordDAO.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.tencent.bk.job.backup.dao.impl; - -import com.tencent.bk.job.backup.config.ArchiveConfig; -import org.jooq.DSLContext; -import org.jooq.Table; -import org.jooq.TableField; -import org.jooq.generated.tables.RollingConfig; -import org.jooq.generated.tables.records.RollingConfigRecord; - -/** - * rolling_config DAO - */ -public class RollingConfigRecordDAO extends AbstractExecuteRecordDAO { - - private static final RollingConfig TABLE = RollingConfig.ROLLING_CONFIG; - - public RollingConfigRecordDAO(DSLContext context, ArchiveConfig archiveConfig) { - super(context, archiveConfig); - } - - @Override - public Table getTable() { - return TABLE; - } - - @Override - public TableField getArchiveIdField() { - return TABLE.TASK_INSTANCE_ID; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceConfirmRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceConfirmRecordDAO.java deleted file mode 100644 index 4789097fc6..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceConfirmRecordDAO.java +++ /dev/null @@ -1,31 +0,0 @@ -package com.tencent.bk.job.backup.dao.impl; - -import com.tencent.bk.job.backup.config.ArchiveConfig; -import org.jooq.DSLContext; -import org.jooq.Table; -import org.jooq.TableField; -import org.jooq.generated.tables.StepInstanceConfirm; -import org.jooq.generated.tables.records.StepInstanceConfirmRecord; - -/** - * step_instance_confirm DAO - */ -public class StepInstanceConfirmRecordDAO extends AbstractExecuteRecordDAO { - - - private static final StepInstanceConfirm TABLE = StepInstanceConfirm.STEP_INSTANCE_CONFIRM; - - public StepInstanceConfirmRecordDAO(DSLContext context, ArchiveConfig archiveConfig) { - super(context, archiveConfig); - } - - @Override - public Table getTable() { - return TABLE; - } - - @Override - public TableField getArchiveIdField() { - return TABLE.STEP_INSTANCE_ID; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceFileRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceFileRecordDAO.java deleted file mode 100644 index 2693aafe2c..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceFileRecordDAO.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.tencent.bk.job.backup.dao.impl; - -import com.tencent.bk.job.backup.config.ArchiveConfig; -import org.jooq.DSLContext; -import org.jooq.Table; -import org.jooq.TableField; -import org.jooq.generated.tables.StepInstanceFile; -import org.jooq.generated.tables.records.StepInstanceFileRecord; - -/** - * step_instance_file DAO - */ -public class StepInstanceFileRecordDAO extends AbstractExecuteRecordDAO { - - private static final StepInstanceFile TABLE = StepInstanceFile.STEP_INSTANCE_FILE; - - public StepInstanceFileRecordDAO(DSLContext context, ArchiveConfig archiveConfig) { - super(context, archiveConfig); - } - - @Override - public Table getTable() { - return TABLE; - } - - @Override - public TableField getArchiveIdField() { - return TABLE.STEP_INSTANCE_ID; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceRecordDAO.java deleted file mode 100644 index 212cd45a19..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceRecordDAO.java +++ /dev/null @@ -1,54 +0,0 @@ -package com.tencent.bk.job.backup.dao.impl; - -import com.tencent.bk.job.backup.config.ArchiveConfig; -import org.jooq.DSLContext; -import org.jooq.Record1; -import org.jooq.Table; -import org.jooq.TableField; -import org.jooq.generated.tables.StepInstance; -import org.jooq.generated.tables.records.StepInstanceRecord; - -import static org.jooq.impl.DSL.max; - -/** - * step_instance DAO - */ -public class StepInstanceRecordDAO extends AbstractExecuteRecordDAO { - - private static final StepInstance TABLE = StepInstance.STEP_INSTANCE; - - public StepInstanceRecordDAO(DSLContext context, ArchiveConfig archiveConfig) { - super(context, archiveConfig); - } - - @Override - public Table getTable() { - return TABLE; - } - - /** - * 获取作业实例ID范围内的步骤实例ID最大值 - * - * @param taskInstanceId 作业实例ID - * @return 步骤实例ID 最大值 - */ - public Long getMaxId(Long taskInstanceId) { - Record1 record = - context.select(max(TABLE.ID)) - .from(TABLE) - .where(TABLE.TASK_INSTANCE_ID.lessOrEqual(taskInstanceId)) - .fetchOne(); - if (record != null) { - Long maxId = (Long) record.get(0); - if (maxId != null) { - return maxId; - } - } - return 0L; - } - - @Override - public TableField getArchiveIdField() { - return TABLE.ID; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceRollingTaskRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceRollingTaskRecordDAO.java deleted file mode 100644 index 9a3e3b1406..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceRollingTaskRecordDAO.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.tencent.bk.job.backup.dao.impl; - -import com.tencent.bk.job.backup.config.ArchiveConfig; -import org.jooq.DSLContext; -import org.jooq.Table; -import org.jooq.TableField; -import org.jooq.generated.tables.StepInstanceRollingTask; -import org.jooq.generated.tables.records.StepInstanceRollingTaskRecord; - -/** - * step_instance_rolling_task DAO - */ -public class StepInstanceRollingTaskRecordDAO extends AbstractExecuteRecordDAO { - - private static final StepInstanceRollingTask TABLE = StepInstanceRollingTask.STEP_INSTANCE_ROLLING_TASK; - - public StepInstanceRollingTaskRecordDAO(DSLContext context, ArchiveConfig archiveConfig) { - super(context, archiveConfig); - } - - @Override - public Table getTable() { - return TABLE; - } - - @Override - public TableField getArchiveIdField() { - return TABLE.STEP_INSTANCE_ID; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceScriptRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceScriptRecordDAO.java deleted file mode 100644 index 27d73c4e97..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceScriptRecordDAO.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.tencent.bk.job.backup.dao.impl; - -import com.tencent.bk.job.backup.config.ArchiveConfig; -import org.jooq.DSLContext; -import org.jooq.Table; -import org.jooq.TableField; -import org.jooq.generated.tables.StepInstanceScript; -import org.jooq.generated.tables.records.StepInstanceScriptRecord; - -/** - * step_instance_script DAO - */ -public class StepInstanceScriptRecordDAO extends AbstractExecuteRecordDAO { - - private static final StepInstanceScript TABLE = StepInstanceScript.STEP_INSTANCE_SCRIPT; - - public StepInstanceScriptRecordDAO(DSLContext context, ArchiveConfig archiveConfig) { - super(context, archiveConfig); - } - - @Override - public Table getTable() { - return TABLE; - } - - @Override - public TableField getArchiveIdField() { - return TABLE.STEP_INSTANCE_ID; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceVariableRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceVariableRecordDAO.java deleted file mode 100644 index d863edf502..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/StepInstanceVariableRecordDAO.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.tencent.bk.job.backup.dao.impl; - -import com.tencent.bk.job.backup.config.ArchiveConfig; -import org.jooq.DSLContext; -import org.jooq.Table; -import org.jooq.TableField; -import org.jooq.generated.tables.StepInstanceVariable; -import org.jooq.generated.tables.records.StepInstanceVariableRecord; - -/** - * step_instance_variable DAO - */ -public class StepInstanceVariableRecordDAO extends AbstractExecuteRecordDAO { - - private static final StepInstanceVariable TABLE = StepInstanceVariable.STEP_INSTANCE_VARIABLE; - - public StepInstanceVariableRecordDAO(DSLContext context, ArchiveConfig archiveConfig) { - super(context, archiveConfig); - } - - @Override - public Table getTable() { - return TABLE; - } - - @Override - public TableField getArchiveIdField() { - return TABLE.STEP_INSTANCE_ID; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/TaskInstanceHostRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/TaskInstanceHostRecordDAO.java deleted file mode 100644 index a5bae27460..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/TaskInstanceHostRecordDAO.java +++ /dev/null @@ -1,27 +0,0 @@ -package com.tencent.bk.job.backup.dao.impl; - -import com.tencent.bk.job.backup.config.ArchiveConfig; -import org.jooq.DSLContext; -import org.jooq.Table; -import org.jooq.TableField; -import org.jooq.generated.tables.TaskInstanceHost; -import org.jooq.generated.tables.records.TaskInstanceHostRecord; - -public class TaskInstanceHostRecordDAO extends AbstractExecuteRecordDAO { - - private static final TaskInstanceHost TABLE = TaskInstanceHost.TASK_INSTANCE_HOST; - - public TaskInstanceHostRecordDAO(DSLContext context, ArchiveConfig archiveConfig) { - super(context, archiveConfig); - } - - @Override - public Table getTable() { - return TABLE; - } - - @Override - public TableField getArchiveIdField() { - return TABLE.TASK_INSTANCE_ID; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/TaskInstanceRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/TaskInstanceRecordDAO.java deleted file mode 100644 index 67c5155049..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/TaskInstanceRecordDAO.java +++ /dev/null @@ -1,50 +0,0 @@ -package com.tencent.bk.job.backup.dao.impl; - -import com.tencent.bk.job.backup.config.ArchiveConfig; -import org.jooq.DSLContext; -import org.jooq.Record1; -import org.jooq.Table; -import org.jooq.TableField; -import org.jooq.generated.tables.TaskInstance; -import org.jooq.generated.tables.records.TaskInstanceRecord; - -import static org.jooq.impl.DSL.max; - -/** - * task_instance DAO - */ -public class TaskInstanceRecordDAO extends AbstractExecuteRecordDAO { - - private static final TaskInstance TABLE = TaskInstance.TASK_INSTANCE; - - public TaskInstanceRecordDAO(DSLContext context, ArchiveConfig archiveConfig) { - super(context, archiveConfig); - } - - @Override - public Table getTable() { - return TABLE; - } - - @Override - public TableField getArchiveIdField() { - return TABLE.ID; - } - - public Long getMaxId(Long endTime) { - Record1 record = - context.select(max(TABLE.ID)) - .from(TABLE) - .where(TABLE.CREATE_TIME.lessOrEqual(endTime)) - .fetchOne(); - if (record != null) { - Long maxId = (Long) record.get(0); - if (maxId != null) { - return maxId; - } - } - return 0L; - } - - -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/TaskInstanceVariableRecordDAO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/TaskInstanceVariableRecordDAO.java deleted file mode 100644 index 491a610254..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/dao/impl/TaskInstanceVariableRecordDAO.java +++ /dev/null @@ -1,30 +0,0 @@ -package com.tencent.bk.job.backup.dao.impl; - -import com.tencent.bk.job.backup.config.ArchiveConfig; -import org.jooq.DSLContext; -import org.jooq.Table; -import org.jooq.TableField; -import org.jooq.generated.tables.TaskInstanceVariable; -import org.jooq.generated.tables.records.TaskInstanceVariableRecord; - -/** - * task_instance_variable DAO - */ -public class TaskInstanceVariableRecordDAO extends AbstractExecuteRecordDAO { - - private static final TaskInstanceVariable TABLE = TaskInstanceVariable.TASK_INSTANCE_VARIABLE; - - public TaskInstanceVariableRecordDAO(DSLContext context, ArchiveConfig archiveConfig) { - super(context, archiveConfig); - } - - @Override - public Table getTable() { - return TABLE; - } - - @Override - public TableField getArchiveIdField() { - return TABLE.TASK_INSTANCE_ID; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/executor/ExportJobExecutor.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/executor/ExportJobExecutor.java index 8dc8db40f0..45ce81bc1c 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/executor/ExportJobExecutor.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/executor/ExportJobExecutor.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -31,6 +31,7 @@ import com.tencent.bk.job.backup.constant.Constant; import com.tencent.bk.job.backup.constant.LogMessage; import com.tencent.bk.job.backup.constant.SecretHandlerEnum; +import com.tencent.bk.job.backup.crypto.BackupFileCryptoService; import com.tencent.bk.job.backup.model.dto.BackupTemplateInfoDTO; import com.tencent.bk.job.backup.model.dto.ExportJobInfoDTO; import com.tencent.bk.job.backup.model.dto.JobBackupInfoDTO; @@ -42,19 +43,20 @@ import com.tencent.bk.job.backup.service.TaskPlanService; import com.tencent.bk.job.backup.service.TaskTemplateService; import com.tencent.bk.job.common.artifactory.sdk.ArtifactoryClient; +import com.tencent.bk.job.common.constant.AccountCategoryEnum; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.constant.JobConstants; import com.tencent.bk.job.common.constant.TaskVariableTypeEnum; import com.tencent.bk.job.common.exception.InternalException; import com.tencent.bk.job.common.i18n.service.MessageI18nService; import com.tencent.bk.job.common.util.Base64Util; -import com.tencent.bk.job.common.util.FileUtil; -import com.tencent.bk.job.common.util.crypto.AESUtils; +import com.tencent.bk.job.common.util.file.FileUtil; import com.tencent.bk.job.common.util.file.ZipUtil; import com.tencent.bk.job.common.util.json.JsonMapper; -import com.tencent.bk.job.manage.common.consts.task.TaskFileTypeEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskScriptSourceEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskStepTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.account.AccountTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskFileTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskScriptSourceEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskStepTypeEnum; import com.tencent.bk.job.manage.model.inner.ServiceAccountDTO; import com.tencent.bk.job.manage.model.inner.ServiceScriptDTO; import com.tencent.bk.job.manage.model.inner.ServiceTaskVariableDTO; @@ -71,6 +73,7 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; +import org.slf4j.helpers.MessageFormatter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.i18n.LocaleContextHolder; import org.springframework.stereotype.Service; @@ -110,16 +113,22 @@ public class ExportJobExecutor { private final ArtifactoryConfig artifactoryConfig; private final BackupStorageConfig backupStorageConfig; private final LocalFileConfigForBackup localFileConfig; + private final BackupFileCryptoService backupFileCryptoService; @Autowired - public ExportJobExecutor(ExportJobService exportJobService, TaskTemplateService taskTemplateService, - TaskPlanService taskPlanService, ScriptService scriptService, - AccountService accountService, LogService logService, - StorageService storageService, MessageI18nService i18nService, + public ExportJobExecutor(ExportJobService exportJobService, + TaskTemplateService taskTemplateService, + TaskPlanService taskPlanService, + ScriptService scriptService, + AccountService accountService, + LogService logService, + StorageService storageService, + MessageI18nService i18nService, ArtifactoryClient artifactoryClient, ArtifactoryConfig artifactoryConfig, BackupStorageConfig backupStorageConfig, - LocalFileConfigForBackup localFileConfig) { + LocalFileConfigForBackup localFileConfig, + BackupFileCryptoService backupFileCryptoService) { this.exportJobService = exportJobService; this.taskTemplateService = taskTemplateService; this.taskPlanService = taskPlanService; @@ -132,6 +141,7 @@ public ExportJobExecutor(ExportJobService exportJobService, TaskTemplateService this.artifactoryConfig = artifactoryConfig; this.backupStorageConfig = backupStorageConfig; this.localFileConfig = localFileConfig; + this.backupFileCryptoService = backupFileCryptoService; File storageDirectory = new File(storageService.getStoragePath().concat(JOB_EXPORT_FILE_PREFIX)); checkDirectory(storageDirectory); @@ -189,14 +199,14 @@ private void processExportJob(String jobId) { processTemplatePlanDetail(exportInfo, jobBackupInfo); try { // 2.处理账号 - processAccount(exportInfo, jobBackupInfo); + processAccount(jobBackupInfo); } catch (Exception e) { log.error("Error while processing account!", e); logService.addExportLog(exportInfo.getAppId(), exportInfo.getId(), "Process account failed! Please try again!" ); exportInfo.setPassword(null); - exportInfo.setStatus(BackupJobStatusEnum.FAILED); + exportInfo.setStatus(BackupJobStatusEnum.ALL_FAILED); exportJobService.updateExportJob(exportInfo); } // 3.处理本地文件 @@ -234,7 +244,7 @@ private void processExportJob(String jobId) { } exportInfo.setPassword(null); - exportInfo.setStatus(BackupJobStatusEnum.SUCCESS); + exportInfo.setStatus(BackupJobStatusEnum.ALL_SUCCESS); exportInfo.setFileName(fileName); exportJobService.updateExportJob(exportInfo); @@ -251,7 +261,7 @@ private void processExportJob(String jobId) { } } - private void processAccount(ExportJobInfoDTO exportInfo, JobBackupInfoDTO jobBackupInfo) { + private void processAccount(JobBackupInfoDTO jobBackupInfo) { List accountList = new ArrayList<>(); Set accountIdSet = new HashSet<>(); for (TaskTemplateVO taskTemplate : jobBackupInfo.getTemplateDetailInfoMap().values()) { @@ -265,13 +275,35 @@ private void processAccount(ExportJobInfoDTO exportInfo, JobBackupInfoDTO jobBac } if (CollectionUtils.isNotEmpty(accountIdSet)) { - accountIdSet.forEach(accountId -> accountList.add( - accountService.getAccountAliasById(accountId))); + accountIdSet.forEach(accountId -> { + ServiceAccountDTO accountDTO = accountService.getAccountAliasById(accountId); + clearSensitiveInfo(accountDTO); + accountList.add(accountDTO); + }); } jobBackupInfo.setAccountList(accountList); } + private void clearSensitiveInfo(ServiceAccountDTO accountDTO) { + if (accountDTO == null) { + return; + } + + if (AccountTypeEnum.WINDOW.getType().equals(accountDTO.getType())) { + accountDTO.setPassword(null); + } + + if (AccountCategoryEnum.DB.getValue().equals(accountDTO.getCategory())) { + accountDTO.setDbPassword(null); + ServiceAccountDTO dbSystemAccount = accountDTO.getDbSystemAccount(); + if (dbSystemAccount != null && + AccountTypeEnum.WINDOW.getType().equals(dbSystemAccount.getType())) { + dbSystemAccount.setPassword(null); + } + } + } + private void extractAccount(List stepList, Set accountIdSet) { for (TaskStepVO taskStep : stepList) { switch (taskStep.getType()) { @@ -293,7 +325,6 @@ private void extractAccount(List stepList, Set accountIdSet) { } break; default: - continue; } } } @@ -305,11 +336,11 @@ private String encryptFile(ExportJobInfoDTO exportInfo, File zipFile) { i18nService.getI18n(LogMessage.START_ENCRYPTING)); File finalFileTmp = new File(zipFile.getPath().concat(".enc.tmp")); try { - AESUtils.encrypt(zipFile, finalFileTmp, exportInfo.getPassword()); + backupFileCryptoService.encryptBackupFile(exportInfo.getPassword(), zipFile, finalFileTmp); FileUtils.deleteQuietly(zipFile); } catch (Exception e) { log.error("Error while processing export job! Encrypt failed!", e); - exportInfo.setStatus(BackupJobStatusEnum.FAILED); + exportInfo.setStatus(BackupJobStatusEnum.ALL_FAILED); exportJobService.updateExportJob(exportInfo); return null; } @@ -321,7 +352,7 @@ private String encryptFile(ExportJobInfoDTO exportInfo, File zipFile) { IOUtils.copy(in, out); } catch (IOException e) { log.error("Error while processing export job! Generate final file failed!", e); - exportInfo.setStatus(BackupJobStatusEnum.FAILED); + exportInfo.setStatus(BackupJobStatusEnum.ALL_FAILED); exportJobService.updateExportJob(exportInfo); return null; } @@ -386,7 +417,7 @@ private void processTemplatePlanDetail(ExportJobInfoDTO exportInfo, JobBackupInf exportInfo.getAppId(), backupTemplateInfo.getId()); if (CollectionUtils.isNotEmpty(taskPlanList)) { backupTemplateInfo.setPlanId( - taskPlanList.parallelStream().map(TaskPlanVO::getId).collect(Collectors.toList())); + taskPlanList.stream().map(TaskPlanVO::getId).collect(Collectors.toList())); } } for (TaskPlanVO taskPlan : taskPlanService.getTaskPlanByIdList(exportInfo.getCreator(), @@ -463,9 +494,18 @@ private void processVariableValue(ExportJobInfoDTO exportInfo, JobBackupInfoDTO if (SecretHandlerEnum.SAVE_NULL == exportInfo.getSecretHandler()) { logService.addExportLog(exportInfo.getAppId(), exportInfo.getId(), i18nService.getI18n(LogMessage.PROCESS_FINISHED) + i18nService.getI18n(LogMessage.SAVE_NULL)); + for (TaskTemplateVO taskTemplate : jobBackupInfo.getTemplateDetailInfoMap().values()) { + setBlankValueForCipherVariables(taskTemplate.getVariableList()); + } + + if (MapUtils.isNotEmpty(jobBackupInfo.getPlanDetailInfoMap())) { + for (TaskPlanVO taskPlan : jobBackupInfo.getPlanDetailInfoMap().values()) { + setBlankValueForCipherVariables(taskPlan.getVariableList()); + } + } return; } - + // SecretHandlerEnum.SAVE_REAL，保存真实值 for (TaskTemplateVO taskTemplate : jobBackupInfo.getTemplateDetailInfoMap().values()) { extractVariableRealValue(exportInfo, taskTemplate.getId(), null, taskTemplate.getVariableList()); } @@ -481,6 +521,22 @@ private void processVariableValue(ExportJobInfoDTO exportInfo, JobBackupInfoDTO i18nService.getI18n(LogMessage.PROCESS_FINISHED) + i18nService.getI18n(LogMessage.SAVE_REAL)); } + private void setBlankValueForCipherVariables(List variableList) { + if (CollectionUtils.isEmpty(variableList)) { + return; + } + + List needProcessVariableList = variableList.stream() + .filter(taskVariableVO -> taskVariableVO.getType().equals(TaskVariableTypeEnum.CIPHER.getType())) + .collect(Collectors.toList()); + + if (CollectionUtils.isEmpty(needProcessVariableList)) { + return; + } + + needProcessVariableList.forEach(cipherVariable -> cipherVariable.setDefaultValue("")); + } + private void extractVariableRealValue(ExportJobInfoDTO exportInfo, Long templateId, Long planId, List variableList) { if (CollectionUtils.isEmpty(variableList)) { @@ -488,9 +544,10 @@ private void extractVariableRealValue(ExportJobInfoDTO exportInfo, Long template } Map needProcessVariableList = variableList.parallelStream() + TaskVariableVO> needProcessVariableList = variableList.stream() .filter(taskVariableVO -> taskVariableVO.getType().equals(TaskVariableTypeEnum.CIPHER.getType())) - .collect(Collectors.toMap(TaskVariableVO::getId, taskVariableVO -> taskVariableVO)); + .collect(Collectors.toMap(TaskVariableVO::getId, taskVariableVO -> taskVariableVO, + (oldValue, newValue) -> newValue)); if (MapUtils.isEmpty(needProcessVariableList)) { return; @@ -598,6 +655,7 @@ private void checkDirectory(File directory) { } } + @SuppressWarnings("InfiniteLoopStatement") class ExportJobExecutorThread extends Thread { @Override public void run() { @@ -612,7 +670,11 @@ public void run() { } catch (InterruptedException e) { Thread.currentThread().interrupt(); } catch (Exception e) { - log.error("{}|Error while processing export job!", uuid, e); + String msg = MessageFormatter.format( + "{}|Error while processing export job!", + uuid + ).getMessage(); + log.error(msg, e); } } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/executor/ImportJobExecutor.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/executor/ImportJobExecutor.java index 967cb1d402..b901871686 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/executor/ImportJobExecutor.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/executor/ImportJobExecutor.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -44,18 +44,18 @@ import com.tencent.bk.job.backup.service.TaskPlanService; import com.tencent.bk.job.backup.service.TaskTemplateService; import com.tencent.bk.job.common.artifactory.sdk.ArtifactoryClient; +import com.tencent.bk.job.common.constant.AccountCategoryEnum; import com.tencent.bk.job.common.constant.ErrorCode; -import com.tencent.bk.job.common.exception.InternalException; import com.tencent.bk.job.common.exception.ServiceException; import com.tencent.bk.job.common.i18n.service.MessageI18nService; -import com.tencent.bk.job.common.util.FileUtil; +import com.tencent.bk.job.common.util.file.FileUtil; import com.tencent.bk.job.common.util.file.PathUtil; import com.tencent.bk.job.common.util.file.ZipUtil; import com.tencent.bk.job.common.util.json.JsonMapper; import com.tencent.bk.job.common.util.json.JsonUtils; -import com.tencent.bk.job.manage.common.consts.account.AccountCategoryEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskScriptSourceEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskStepTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.account.AccountTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskScriptSourceEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskStepTypeEnum; import com.tencent.bk.job.manage.model.inner.ServiceAccountDTO; import com.tencent.bk.job.manage.model.inner.ServiceIdNameCheckDTO; import com.tencent.bk.job.manage.model.inner.ServiceScriptDTO; @@ -73,6 +73,8 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.http.client.methods.HttpRequestBase; +import org.slf4j.helpers.FormattingTuple; +import org.slf4j.helpers.MessageFormatter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.i18n.LocaleContextHolder; import org.springframework.stereotype.Service; @@ -83,8 +85,10 @@ import java.io.InputStream; import java.nio.charset.StandardCharsets; import java.util.HashMap; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.LinkedBlockingQueue; @@ -96,7 +100,6 @@ @Service public class ImportJobExecutor { private static final LinkedBlockingQueue IMPORT_JOB_QUEUE = new LinkedBlockingQueue<>(100); - private static final String JOB_IMPORT_FILE_PREFIX = "import" + File.separatorChar; private static final String LOG_HR = "************************************************************"; private final ImportJobService importJobService; private final TaskTemplateService taskTemplateService; @@ -218,7 +221,7 @@ private void processImportJob(String jobId) { } catch (InterruptedException e) { String msg = "Fail to write artifactory file to local"; log.warn(msg, e); - importJobService.markJobFailed(importJob, msg); + importJobService.markJobAllFailed(importJob, msg); return; } // 解压 @@ -236,118 +239,217 @@ private void processImportJob(String jobId) { } } else { log.warn("Import file not found"); - importJobService.markJobFailed(importJob, i18nService.getI18n(LogMessage.EXTRACT_FAILED)); + importJobService.markJobAllFailed(importJob, i18nService.getI18n(LogMessage.EXTRACT_FAILED)); } } else { - importJobService.markJobFailed(importJob, "未找到待导入文件"); + importJobService.markJobAllFailed(importJob, i18nService.getI18n(LogMessage.CANNOT_FIND_IMPORT_FILE)); } } } private boolean processImportFile(File file, ImportJobInfoDTO importJob) { - if (file.getName().endsWith(".json")) { - JobBackupInfoDTO jobBackupInfo = readJobBackupInfoFromFile(file); - if (jobBackupInfo != null) { - try { - processAccount(importJob, jobBackupInfo); - List templateInfo = importJob.getTemplateInfo(); - if (CollectionUtils.isNotEmpty(templateInfo)) { - for (BackupTemplateInfoDTO backupTemplateInfo : templateInfo) { - long templateId = backupTemplateInfo.getId(); - TaskTemplateVO oldTemplate = JsonUtils.fromJson( - JsonMapper.getAllOutPutMapper() - .toJson(jobBackupInfo.getTemplateDetailInfoMap().get(templateId)), - new TypeReference() { - }); - TaskTemplateVO newTemplate = - processTemplate(importJob, jobBackupInfo, templateId); - if (newTemplate != null) { - TemplateIdMapDTO templateIdMap = - processTemplateIdMap(oldTemplate, newTemplate); - if (CollectionUtils.isNotEmpty(backupTemplateInfo.getPlanId())) { - for (Long planId : backupTemplateInfo.getPlanId()) { - processPlan(importJob, jobBackupInfo, templateIdMap, - newTemplate.getId(), planId); - } - } - } - } - } - importJob.setStatus(BackupJobStatusEnum.SUCCESS); - importJobService.updateImportJob(importJob); - logService.addImportLog(importJob.getAppId(), importJob.getId(), LOG_HR); + if (!file.getName().endsWith(".json")) { + return false; + } + JobBackupInfoDTO jobBackupInfo = readJobBackupInfoFromFile(file); + if (jobBackupInfo == null) { + importJobService.markJobAllFailed(importJob, + i18nService.getI18n(LogMessage.EXTRACT_FAILED)); + return false; + } + try { + return importJobBackupInfo(importJob, jobBackupInfo); + } catch (Exception e) { + log.error("Error while process import job!|{}|{}", importJob.getAppId(), + importJob.getId(), e); + if (e instanceof ServiceException) { + if (ErrorCode.PERMISSION_DENIED == ((ServiceException) e).getErrorCode()) { logService.addImportLog(importJob.getAppId(), importJob.getId(), - i18nService.getI18n(LogMessage.IMPORT_FINISHED), LogEntityTypeEnum.FINISHED); - return true; - } catch (Exception e) { - log.error("Error while process import job!|{}|{}", importJob.getAppId(), - importJob.getId(), e); - if (e instanceof ServiceException) { - if (ErrorCode.PERMISSION_DENIED == ((ServiceException) e).getErrorCode()) { - logService.addImportLog(importJob.getAppId(), importJob.getId(), - i18nService.getI18n(String.valueOf(((ServiceException) e).getErrorCode())), - LogEntityTypeEnum.ERROR); - } - } - importJobService.markJobFailed(importJob, - i18nService.getI18n(LogMessage.IMPORT_FAILED)); + i18nService.getI18n(String.valueOf(((ServiceException) e).getErrorCode())), + LogEntityTypeEnum.ERROR); } - } else { - importJobService.markJobFailed(importJob, - i18nService.getI18n(LogMessage.EXTRACT_FAILED)); } + importJobService.markJobAllFailed(importJob, + i18nService.getI18n(LogMessage.IMPORT_FAILED)); } return false; } + private boolean importJobBackupInfo(ImportJobInfoDTO importJob, JobBackupInfoDTO jobBackupInfo) { + processAccount(importJob, jobBackupInfo); + List templateInfo = importJob.getTemplateInfo(); + if (CollectionUtils.isEmpty(templateInfo)) { + setImportJobResult(importJob, BackupJobStatusEnum.ALL_SUCCESS); + return true; + } + int totalTemplateNum = templateInfo.size(); + int failedTemplateNum = 0; + for (BackupTemplateInfoDTO backupTemplateInfo : templateInfo) { + boolean success = processBackupTemplateInfo( + backupTemplateInfo, + importJob, + jobBackupInfo + ); + if (!success) { + failedTemplateNum += 1; + } + } + if (failedTemplateNum == 0) { + setImportJobResult(importJob, BackupJobStatusEnum.ALL_SUCCESS); + return true; + } else if (failedTemplateNum < totalTemplateNum) { + setImportJobResult(importJob, BackupJobStatusEnum.PARTIAL_FAILED); + return true; + } else { + setImportJobResult(importJob, BackupJobStatusEnum.ALL_FAILED); + return false; + } + } + + /** + * 导入单个作业模板及对应的执行方案 + * + * @param backupTemplateInfo 作业模板信息 + * @param importJob 导入任务信息 + * @param jobBackupInfo 整体导入数据信息 + * @return 是否导入成功 + */ + private boolean processBackupTemplateInfo(BackupTemplateInfoDTO backupTemplateInfo, + ImportJobInfoDTO importJob, + JobBackupInfoDTO jobBackupInfo) { + long templateId = backupTemplateInfo.getId(); + TaskTemplateVO oldTemplate = JsonUtils.fromJson( + JsonMapper.getAllOutPutMapper().toJson(jobBackupInfo.getTemplateDetailInfoMap().get(templateId)), + new TypeReference() { + } + ); + Pair pair = processTemplate(importJob, jobBackupInfo, templateId); + Boolean successImportedOrSkipped = pair.getLeft(); + TaskTemplateVO newTemplate = pair.getRight(); + if (!successImportedOrSkipped) { + return false; + } + if (newTemplate == null) { + return true; + } + TemplateIdMapDTO templateIdMap = processTemplateIdMap(oldTemplate, newTemplate); + if (CollectionUtils.isEmpty(backupTemplateInfo.getPlanId())) { + return true; + } + int failedPlanNum = 0; + for (Long planId : backupTemplateInfo.getPlanId()) { + try { + boolean success = processPlan(importJob, jobBackupInfo, templateIdMap, newTemplate.getId(), planId); + failedPlanNum += success ? 0 : 1; + } catch (Exception e) { + failedPlanNum += 1; + FormattingTuple msg = MessageFormatter.arrayFormat( + "Fail to import plan {} of template (id={},name={})", + new Object[]{ + planId, + newTemplate.getId(), + newTemplate.getName() + } + ); + log.warn(msg.getMessage(), e); + } + } + if (failedPlanNum > 0) { + log.warn( + "Fail to import {}/{} plan of template (id={},name={})", + failedPlanNum, + backupTemplateInfo.getPlanId().size(), + newTemplate.getId(), + newTemplate.getName() + ); + return false; + } + return true; + } + + private void setImportJobResult(ImportJobInfoDTO importJob, BackupJobStatusEnum status) { + importJob.setStatus(status); + importJobService.updateImportJob(importJob); + logService.addImportLog(importJob.getAppId(), importJob.getId(), LOG_HR); + logService.addImportLog(importJob.getAppId(), importJob.getId(), + i18nService.getI18n(LogMessage.IMPORT_FINISHED), LogEntityTypeEnum.FINISHED); + } + private void processAccount(ImportJobInfoDTO importJob, JobBackupInfoDTO jobBackupInfo) { Map finalAccountIdMap = new HashMap<>(); if (CollectionUtils.isNotEmpty(jobBackupInfo.getAccountList())) { List appAccountList = accountService.listAccountByAppId(importJob.getCreator(), importJob.getAppId()); - Map appAccountIdMap = new ConcurrentHashMap<>(); - appAccountList.parallelStream().forEach(account -> appAccountIdMap.put(account.getAlias(), - account.getId())); - + Map> categoryAliasToAccountIdMap = new ConcurrentHashMap<>(); + appAccountList.forEach(account -> categoryAliasToAccountIdMap + .computeIfAbsent(account.getCategory(), k -> new ConcurrentHashMap<>()) + .put(account.getAlias(), account.getId()) + ); + Set noExistAccountSet = new LinkedHashSet<>(); for (ServiceAccountDTO account : jobBackupInfo.getAccountList()) { if (AccountCategoryEnum.DB.getValue().equals(account.getCategory())) { // DB account process related system account first - doProcessAccount(importJob, finalAccountIdMap, appAccountIdMap, account.getDbSystemAccount()); + doProcessAccount(importJob, finalAccountIdMap, categoryAliasToAccountIdMap, + account.getDbSystemAccount(), noExistAccountSet); if (finalAccountIdMap.get(account.getDbSystemAccount().getId()) == null) { - log.error("Error while find or create db account!|{}|{}|{}|{}", importJob.getCreator(), - account.getAppId(), account.getAlias(), account.getDbSystemAccount().getAlias()); - logService.addImportLog(importJob.getAppId(), importJob.getId(), - "Find or create db account " + account.getAlias() + - "|" + account.getDbSystemAccount().getAlias() + " " + "failed!", - LogEntityTypeEnum.ERROR); - throw new InternalException("Find or create account failed!", ErrorCode.INTERNAL_ERROR); + log.warn("The system account associated with the database was not found!|{}|{}|{}|{}", + importJob.getCreator(), account.getAppId(), account.getAlias(), + account.getDbSystemAccount().getAlias()); + } else { + account.getDbSystemAccount().setId(finalAccountIdMap.get(account.getDbSystemAccount().getId())); } - account.getDbSystemAccount().setId(finalAccountIdMap.get(account.getDbSystemAccount().getId())); } - doProcessAccount(importJob, finalAccountIdMap, appAccountIdMap, account); + doProcessAccount(importJob, finalAccountIdMap, categoryAliasToAccountIdMap, account, + noExistAccountSet); + } + if (!noExistAccountSet.isEmpty()) { + logService.addImportLog(importJob.getAppId(), importJob.getId(), LOG_HR); + noExistAccountSet.forEach(account -> logService.addImportLog( + importJob.getAppId(), + importJob.getId(), + String.format( + i18nService.getI18n(LogMessage.IMPORT_ACCOUNT_NOT_EXIST), + AccountTypeEnum.valueOf(account.getType()).getName(), + account.getAlias() + ) + )); } } importJob.setAccountIdMap(finalAccountIdMap); } - private void doProcessAccount(ImportJobInfoDTO importJob, Map finalAccountIdMap, - Map appAccountIdMap, ServiceAccountDTO account) { - if (appAccountIdMap.get(account.getAlias()) != null) { - finalAccountIdMap.put(account.getId(), appAccountIdMap.get(account.getAlias())); + private void doProcessAccount(ImportJobInfoDTO importJob, + Map finalAccountIdMap, + Map> categoryAliasToAccountIdMap, + ServiceAccountDTO account, + Set noExistAccountSet) { + Map accountAliasMap = categoryAliasToAccountIdMap.get(account.getCategory()); + if (accountAliasMap != null && accountAliasMap.get(account.getAlias()) != null) { + finalAccountIdMap.put(account.getId(), accountAliasMap.get(account.getAlias())); } else if (finalAccountIdMap.get(account.getId()) == null) { + saveAccount(importJob, account, finalAccountIdMap, noExistAccountSet); + } else { + log.debug("Already create account|{}|{}", account.getAppId(), account.getAlias()); + } + } + + private void saveAccount(ImportJobInfoDTO importJob, + ServiceAccountDTO account, + Map finalAccountIdMap, + Set noExistAccountSet) { + // 导入作业只支持自动创建不需要填写密码的账号 + if (AccountTypeEnum.LINUX.getType().equals(account.getType())) { Long newAccountId = accountService.saveAccount(importJob.getCreator(), importJob.getAppId(), account); if (newAccountId != null && newAccountId > 0) { finalAccountIdMap.put(account.getId(), newAccountId); } else { - log.error("Error while find or create account!|{}|{}|{}", importJob.getCreator(), - account.getAppId(), account.getAlias()); - logService.addImportLog(importJob.getAppId(), importJob.getId(), - "Find or create account " + account.getAlias() + - " " + "failed!", LogEntityTypeEnum.ERROR); - throw new InternalException("Find or create account failed!", ErrorCode.INTERNAL_ERROR); + log.error("Failed to create account!|{}|{}|{}|{}", importJob.getCreator(), account.getAppId(), + account.getAlias(), AccountTypeEnum.valueOf(account.getType()).getName()); } } else { - log.debug("Already create account|{}|{}", account.getAppId(), account.getAlias()); + log.warn("[appId={},alias={}] account does not exist, it needs to be added manually", + account.getAppId(), account.getAlias()); + noExistAccountSet.add(account); } } @@ -386,37 +488,102 @@ private TemplateIdMapDTO processTemplateIdMap(TaskTemplateVO oldTemplate, TaskTe return templateIdMap; } - private void processPlan(ImportJobInfoDTO importJob, JobBackupInfoDTO jobBackupInfo, TemplateIdMapDTO templateIdMap, - Long templateId, Long planId) { - TaskPlanVO planInfo = jobBackupInfo.getPlanDetailInfoMap().get(planId); + private void addStartImportLog(ImportJobInfoDTO importJob, Long planId) { + logService.addImportLog( + importJob.getAppId(), + importJob.getId(), + String.format( + i18nService.getI18n(LogMessage.START_IMPORT_PLAN), + importJob.getIdNameInfo().getPlanNameMap().get(planId) + ) + ); + } - logService.addImportLog(importJob.getAppId(), importJob.getId(), - String.format(i18nService.getI18n(LogMessage.START_IMPORT_PLAN), - importJob.getIdNameInfo().getPlanNameMap().get(planId))); - ServiceIdNameCheckDTO idNameCheckResult = - taskPlanService.checkIdAndName(importJob.getAppId(), templateId, planId, planInfo.getName()); + private void addIdAutoIncrementImportLog(ImportJobInfoDTO importJob) { + logService.addImportLog( + importJob.getAppId(), + importJob.getId(), + i18nService.getI18n(LogMessage.ID_AUTO_INCREMENT_SUFFIX) + ); + } + + private void addIdKeepImportLog(ImportJobInfoDTO importJob) { + logService.addImportLog( + importJob.getAppId(), + importJob.getId(), + i18nService.getI18n(LogMessage.ID_KEEP_SUFFIX) + ); + } + + private void addIdDuplicateSkipImportLog(ImportJobInfoDTO importJob) { + logService.addImportLog( + importJob.getAppId(), + importJob.getId(), + String.format( + i18nService.getI18n(LogMessage.ID_DUPLICATE_SKIP_SUFFIX), + i18nService.getI18n(LogMessage.PLAN) + ) + ); + } + + private void addIdDuplicateIncrementLog(ImportJobInfoDTO importJob) { + logService.addImportLog( + importJob.getAppId(), + importJob.getId(), + i18nService.getI18n(LogMessage.ID_DUPLICATE_INCREMENT_SUFFIX) + ); + } + + /** + * @param importJob 导入任务信息 + * @param templateId 作业模板ID + * @param planId 执行方案ID + * @param planInfo 执行方案内容 + * @return 是否需要跳过该执行方案 + */ + private boolean processIdAndName(ImportJobInfoDTO importJob, + Long templateId, + Long planId, + TaskPlanVO planInfo + ) { + ServiceIdNameCheckDTO idNameCheckResult = taskPlanService.checkIdAndName( + importJob.getAppId(), + templateId, + planId, + planInfo.getName() + ); if (DuplicateIdHandlerEnum.AUTO_INCREMENT.equals(importJob.getDuplicateIdHandler())) { - logService.addImportLog(importJob.getAppId(), importJob.getId(), - i18nService.getI18n(LogMessage.ID_AUTO_INCREMENT_SUFFIX)); + addIdAutoIncrementImportLog(importJob); planInfo.setId(0L); } else { if (idNameCheckResult != null && idNameCheckResult.getIdCheckResult() == 1) { - logService.addImportLog(importJob.getAppId(), importJob.getId(), - i18nService.getI18n(LogMessage.ID_KEEP_SUFFIX)); + addIdKeepImportLog(importJob); } else { if (DuplicateIdHandlerEnum.ON_DUPLICATE_SKIP == importJob.getDuplicateIdHandler()) { - logService.addImportLog(importJob.getAppId(), importJob.getId(), - String.format(i18nService.getI18n(LogMessage.ID_DUPLICATE_SKIP_SUFFIX), - i18nService.getI18n(LogMessage.PLAN))); - return; + addIdDuplicateSkipImportLog(importJob); + return true; } else if (DuplicateIdHandlerEnum.ON_DUPLICATE_INCREMENT == importJob.getDuplicateIdHandler()) { - logService.addImportLog(importJob.getAppId(), importJob.getId(), - i18nService.getI18n(LogMessage.ID_DUPLICATE_INCREMENT_SUFFIX)); + addIdDuplicateIncrementLog(importJob); planInfo.setId(0L); } } } + return false; + } + + private boolean processPlan(ImportJobInfoDTO importJob, + JobBackupInfoDTO jobBackupInfo, + TemplateIdMapDTO templateIdMap, + Long templateId, + Long planId) { + TaskPlanVO planInfo = jobBackupInfo.getPlanDetailInfoMap().get(planId); + + addStartImportLog(importJob, planId); + boolean needToSkip = processIdAndName(importJob, templateId, planId, planInfo); + if (needToSkip) { + return true; + } if (planInfo.getVersion().equals(templateIdMap.getOldVersion())) { planInfo.setVersion(templateIdMap.getNewVersion()); @@ -442,27 +609,56 @@ private void processPlan(ImportJobInfoDTO importJob, JobBackupInfoDTO jobBackupI if (MapUtils.isNotEmpty(linkScriptContentMap)) { fixScript(importJob, linkScriptContentMap, planInfo.getStepList()); } - fixAccount(importJob.getAccountIdMap(), planInfo.getStepList()); + fixAccount(importJob, planInfo.getStepList(), planInfo.getName()); - Long resultPlanId = - taskPlanService.savePlan(importJob.getCreator(), importJob.getAppId(), templateId, planInfo); + Long resultPlanId = taskPlanService.savePlan( + importJob.getCreator(), + importJob.getAppId(), + templateId, planInfo + ); if (resultPlanId != null && resultPlanId > 0) { - logService.addImportLog(importJob.getAppId(), importJob.getId(), - i18nService.getI18n(LogMessage.IMPORT_PLAN_SUCCESS), - LogEntityTypeEnum.PLAN, templateId, resultPlanId); + addImportPlanSuccessLog(importJob, templateId, resultPlanId); if (MapUtils.isNotEmpty(jobBackupInfo.getPlanFileList())) { - processFile(importJob, jobBackupInfo, jobBackupInfo.getPlanFileList().get(planId)); + processFile(importJob, jobBackupInfo.getPlanFileList().get(planId)); } + return true; } else { - logService.addImportLog(importJob.getAppId(), importJob.getId(), - i18nService.getI18n(LogMessage.IMPORT_FAILED), - LogEntityTypeEnum.ERROR); + addImportFailedLog(importJob); + return false; } } - private TaskTemplateVO processTemplate(ImportJobInfoDTO importJob, JobBackupInfoDTO jobBackupInfo, - long templateId) { + private void addImportPlanSuccessLog(ImportJobInfoDTO importJob, Long templateId, Long resultPlanId) { + logService.addImportLog( + importJob.getAppId(), + importJob.getId(), + i18nService.getI18n(LogMessage.IMPORT_PLAN_SUCCESS), + LogEntityTypeEnum.PLAN, + templateId, + resultPlanId + ); + } + + private void addImportFailedLog(ImportJobInfoDTO importJob) { + logService.addImportLog( + importJob.getAppId(), + importJob.getId(), + i18nService.getI18n(LogMessage.IMPORT_FAILED), + LogEntityTypeEnum.ERROR + ); + } + + /** + * 导入作业模板 + * + * @param importJob 导入任务信息 + * @param jobBackupInfo 导入作业信息 + * @param templateId 作业模板ID + * @return Pair<是否成功导入或跳过, 成功导入后得到的作业模板> + */ + private Pair processTemplate(ImportJobInfoDTO importJob, JobBackupInfoDTO jobBackupInfo, + long templateId) { TaskTemplateVO templateInfo = jobBackupInfo.getTemplateDetailInfoMap().get(templateId); logService.addImportLog(importJob.getAppId(), importJob.getId(), LOG_HR); @@ -471,8 +667,12 @@ private TaskTemplateVO processTemplate(ImportJobInfoDTO importJob, JobBackupInfo importJob.getIdNameInfo().getTemplateNameMap().get(templateId))); if (templateInfo.getName().length() > 60) { - logService.addImportLog(importJob.getAppId(), importJob.getId(), "作业模板名称超长，跳过当前模版和该模版下的所有执行方案"); - return null; + logService.addImportLog( + importJob.getAppId(), + importJob.getId(), + i18nService.getI18n(LogMessage.TEMPLATE_NAME_TOO_LONG_SKIP) + ); + return Pair.of(false, null); } ServiceIdNameCheckDTO idNameCheckResult = @@ -491,7 +691,7 @@ private TaskTemplateVO processTemplate(ImportJobInfoDTO importJob, JobBackupInfo logService.addImportLog(importJob.getAppId(), importJob.getId(), String.format(i18nService.getI18n(LogMessage.ID_DUPLICATE_SKIP_SUFFIX), i18nService.getI18n(LogMessage.TEMPLATE))); - return null; + return Pair.of(true, null); } else if (DuplicateIdHandlerEnum.ON_DUPLICATE_INCREMENT == importJob.getDuplicateIdHandler()) { logService.addImportLog(importJob.getAppId(), importJob.getId(), i18nService.getI18n(LogMessage.ID_DUPLICATE_INCREMENT_SUFFIX)); @@ -509,8 +709,12 @@ private TaskTemplateVO processTemplate(ImportJobInfoDTO importJob, JobBackupInfo String.format(i18nService.getI18n(LogMessage.TEMPLATE_NAME_CHANGE), templateInfo.getName())); if (templateInfo.getName().length() > 60) { - logService.addImportLog(importJob.getAppId(), importJob.getId(), "作业模板名称超长，跳过当前模版和该模版下的所有执行方案"); - return null; + logService.addImportLog( + importJob.getAppId(), + importJob.getId(), + i18nService.getI18n(LogMessage.TEMPLATE_NAME_TOO_LONG_SKIP) + ); + return Pair.of(false, null); } } @@ -518,7 +722,7 @@ private TaskTemplateVO processTemplate(ImportJobInfoDTO importJob, JobBackupInfo if (MapUtils.isNotEmpty(linkScriptContentMap)) { fixScript(importJob, linkScriptContentMap, templateInfo.getStepList()); } - fixAccount(importJob.getAccountIdMap(), templateInfo.getStepList()); + fixAccount(importJob, templateInfo.getStepList(), templateInfo.getName()); Long resultTemplateId = taskTemplateService.saveTemplate(importJob.getCreator(), importJob.getAppId(), templateInfo); @@ -529,19 +733,25 @@ private TaskTemplateVO processTemplate(ImportJobInfoDTO importJob, JobBackupInfo LogEntityTypeEnum.TEMPLATE, resultTemplateId, 0); if (MapUtils.isNotEmpty(jobBackupInfo.getTemplateFileList())) { - processFile(importJob, jobBackupInfo, jobBackupInfo.getTemplateFileList().get(templateId)); + processFile(importJob, jobBackupInfo.getTemplateFileList().get(templateId)); } } else { logService.addImportLog(importJob.getAppId(), importJob.getId(), i18nService.getI18n(LogMessage.IMPORT_FAILED), LogEntityTypeEnum.ERROR); - return null; + return Pair.of(false, null); } - return taskTemplateService.getTemplateById(importJob.getCreator(), importJob.getAppId(), resultTemplateId); + return Pair.of(true, taskTemplateService.getTemplateById( + importJob.getCreator(), + importJob.getAppId(), + resultTemplateId) + ); } - private void fixAccount(Map accountIdMap, List stepList) { + private void fixAccount(ImportJobInfoDTO importJob, List stepList, String jobName) { + Map accountIdMap = importJob.getAccountIdMap(); if (CollectionUtils.isNotEmpty(stepList)) { + Set withoutAccountStepSet = new LinkedHashSet<>(); for (TaskStepVO taskStep : stepList) { switch (taskStep.getType()) { case 1: @@ -554,6 +764,7 @@ private void fixAccount(Map accountIdMap, List stepList) } else { log.warn("Error while fix old account {}|{}|{}|{}", scriptStepInfo.getAccount(), taskStep.getId(), taskStep.getName(), taskStep.getType()); + withoutAccountStepSet.add(taskStep.getName()); } } else { log.warn("Empty script step info|{}|{}|{}", taskStep.getId(), taskStep.getName(), @@ -571,6 +782,7 @@ private void fixAccount(Map accountIdMap, List stepList) } else { log.warn("Error while fix old account {}|{}|{}|{}", fileDestination.getAccount(), taskStep.getId(), taskStep.getName(), taskStep.getType()); + withoutAccountStepSet.add(taskStep.getName()); } } else { log.warn("Empty file destination|{}|{}|{}", taskStep.getId(), taskStep.getName(), @@ -591,6 +803,7 @@ private void fixAccount(Map accountIdMap, List stepList) log.warn("Error while fix old account {}|{}|{}|{}", fileSourceInfo.getAccount(), taskStep.getId(), taskStep.getName(), taskStep.getType()); + withoutAccountStepSet.add(taskStep.getName()); } } else { log.warn("Missing account in file source info|{}|{}|{}", @@ -623,10 +836,22 @@ private void fixAccount(Map accountIdMap, List stepList) taskStep.getType()); } } + + if (!withoutAccountStepSet.isEmpty()) { + withoutAccountStepSet.forEach(stepName -> logService.addImportLog( + importJob.getAppId(), + importJob.getId(), + String.format( + i18nService.getI18n(LogMessage.IMPORT_STEP_ACCOUNT_INVALID), + jobName, + stepName + ) + )); + } } } - private void processFile(ImportJobInfoDTO importJob, JobBackupInfoDTO jobBackupInfo, List fileList) { + private void processFile(ImportJobInfoDTO importJob, List fileList) { if (CollectionUtils.isEmpty(fileList)) { return; } @@ -699,7 +924,9 @@ private void getUsableName(Long appId, TaskTemplateVO templateInfo) { templateInfo.setName(templateName); } + @SuppressWarnings("InfiniteLoopStatement") class ImportJobExecutorThread extends Thread { + @Override public void run() { this.setName("Import-Job-Executor-Thread"); @@ -713,7 +940,11 @@ public void run() { } catch (InterruptedException e) { Thread.currentThread().interrupt(); } catch (Exception e) { - log.error("{}|Error while processing import job!", uuid, e); + String msg = MessageFormatter.format( + "{}|Error while processing import job!", + uuid + ).getMessage(); + log.error(msg, e); } } } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/metrics/ArchiveErrorTaskCounter.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/metrics/ArchiveErrorTaskCounter.java new file mode 100644 index 0000000000..35b6e8e1de --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/metrics/ArchiveErrorTaskCounter.java @@ -0,0 +1,49 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.metrics; + +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.MeterRegistry; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +@Component +public class ArchiveErrorTaskCounter { + + private final Counter archiveErrorTaskCounter; + + @Autowired + public ArchiveErrorTaskCounter(MeterRegistry meterRegistry) { + this.archiveErrorTaskCounter = meterRegistry.counter(MetricConstants.ARCHIVE_ERROR_TASK_TOTAL); + } + + /** + * 计数+1 + */ + public void increment() { + this.archiveErrorTaskCounter.increment(); + } + +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/metrics/MetricConstants.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/metrics/MetricConstants.java new file mode 100644 index 0000000000..fcd4879282 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/metrics/MetricConstants.java @@ -0,0 +1,34 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.metrics; + +public class MetricConstants { + /** + * Job 执行历史归档异常任务总数 + */ + public static final String ARCHIVE_ERROR_TASK_TOTAL = "job.history.archive.error.task.total"; + + +} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/ArchiveProgressDTO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/ArchiveProgressDTO.java deleted file mode 100644 index 9a1c8499e7..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/ArchiveProgressDTO.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.model.dto; - -import lombok.Data; -import lombok.ToString; - -@Data -@ToString -public class ArchiveProgressDTO { - private String tableName; - /** - * 最后归档ID - */ - private Long lastArchivedId; - - /** - * 最后删除ID - */ - private Long lastDeletedId; - private Long lastArchiveTime; - private Long lastDeleteTime; -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/ArchiveSummary.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/ArchiveSummary.java deleted file mode 100644 index 05d1d3b353..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/ArchiveSummary.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.model.dto; - -import com.fasterxml.jackson.annotation.JsonInclude; -import lombok.Data; -import lombok.NoArgsConstructor; - -@Data -@JsonInclude(JsonInclude.Include.NON_EMPTY) -@NoArgsConstructor -public class ArchiveSummary { - private String tableName; - private String archiveEndDate; - private boolean skip; - - private boolean archiveEnabled; - private Long archiveIdStart; - private Long archiveIdEnd; - private Long lastArchivedId; - private Long needArchiveRecordSize; - private Long archivedRecordSize; - private Long archiveCost; - - private boolean deleteEnabled; - private Long deleteIdStart; - private Long deleteIdEnd; - private Long lastDeletedId; - private Long deleteRecordSize; - private Long deleteCost; - - public ArchiveSummary(String tableName) { - this.tableName = tableName; - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/BackupTemplateInfoDTO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/BackupTemplateInfoDTO.java index fbbc795d99..306732571d 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/BackupTemplateInfoDTO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/BackupTemplateInfoDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/ExportJobInfoDTO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/ExportJobInfoDTO.java index ef9273a2ed..9bfa867ad9 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/ExportJobInfoDTO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/ExportJobInfoDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/IdNameInfoDTO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/IdNameInfoDTO.java index d53b0a31f3..e7796d5bc5 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/IdNameInfoDTO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/IdNameInfoDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/ImportJobInfoDTO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/ImportJobInfoDTO.java index e914f98e15..9759d4d20d 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/ImportJobInfoDTO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/ImportJobInfoDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/JobBackupInfoDTO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/JobBackupInfoDTO.java index f19dc06fed..26191410c9 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/JobBackupInfoDTO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/JobBackupInfoDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/LogEntityDTO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/LogEntityDTO.java index 5c70cd22d3..61dee3d872 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/LogEntityDTO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/LogEntityDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/TemplateIdMapDTO.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/TemplateIdMapDTO.java index 628c81862f..cc7d7dbb4e 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/TemplateIdMapDTO.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/model/dto/TemplateIdMapDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/runner/InitArtifactoryDataRunner.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/runner/InitArtifactoryDataRunner.java index a7a4f5146f..1c3f1bfd92 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/runner/InitArtifactoryDataRunner.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/runner/InitArtifactoryDataRunner.java @@ -10,7 +10,7 @@ import org.springframework.stereotype.Component; @Slf4j -@Component +@Component("jobBackupInitArtifactoryDataRunner") public class InitArtifactoryDataRunner implements CommandLineRunner { private final ArtifactoryConfig artifactoryConfig; diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/AccountService.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/AccountService.java index 9cdd68b4f5..35407526c2 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/AccountService.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/AccountService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/ArchiveProgressService.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/ArchiveProgressService.java deleted file mode 100644 index 522ad9c72c..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/ArchiveProgressService.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.service; - -import com.tencent.bk.job.backup.model.dto.ArchiveProgressDTO; - -public interface ArchiveProgressService { - ArchiveProgressDTO queryArchiveProgress(String table); - - void saveArchiveProgress(ArchiveProgressDTO archiveProgress); - - void saveDeleteProgress(ArchiveProgressDTO deleteProgress); -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/ExportJobService.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/ExportJobService.java index 864f9d2943..3cb73bcec6 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/ExportJobService.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/ExportJobService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/ImportJobService.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/ImportJobService.java index cb4ca4fd6a..53f7ee7f6a 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/ImportJobService.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/ImportJobService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -106,9 +106,8 @@ public interface ImportJobService { * 更新导入任务信息 * * @param importJob 导入任务信息 - * @return 更新是否成功 */ - Boolean updateImportJob(ImportJobInfoDTO importJob); + void updateImportJob(ImportJobInfoDTO importJob); /** * 标记任务为失败 @@ -116,7 +115,7 @@ public interface ImportJobService { * @param importJob 导入任务信息 * @param message 提示信息 */ - void markJobFailed(ImportJobInfoDTO importJob, String message); + void markJobAllFailed(ImportJobInfoDTO importJob, String message); /** * 清理导入文件 diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/LogService.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/LogService.java index a33c43b7f9..c6c36eca9f 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/LogService.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/LogService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/ScriptService.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/ScriptService.java index 763596a8b8..e1269c44ae 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/ScriptService.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/ScriptService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/StorageService.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/StorageService.java index 034b92ce47..449a6e5be5 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/StorageService.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/StorageService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/TaskPlanService.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/TaskPlanService.java index a6211e8d37..82a9475706 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/TaskPlanService.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/TaskPlanService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/TaskTemplateService.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/TaskTemplateService.java index 2bc318c261..d567a2dbec 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/TaskTemplateService.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/TaskTemplateService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/impl/AccountServiceImpl.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/impl/AccountServiceImpl.java index c6db4652c0..33930d022b 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/impl/AccountServiceImpl.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/impl/AccountServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,12 +24,12 @@ package com.tencent.bk.job.backup.service.impl; -import com.tencent.bk.job.backup.client.ServiceAccountResourceClient; import com.tencent.bk.job.backup.service.AccountService; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.exception.InternalException; import com.tencent.bk.job.common.model.InternalResponse; import com.tencent.bk.job.common.model.Response; +import com.tencent.bk.job.manage.api.inner.ServiceAccountResource; import com.tencent.bk.job.manage.model.inner.ServiceAccountDTO; import com.tencent.bk.job.manage.model.web.request.AccountCreateUpdateReq; import lombok.extern.slf4j.Slf4j; @@ -41,22 +41,19 @@ import java.util.Collections; import java.util.List; -/** - * @since 24/11/2020 21:08 - */ @Slf4j -@Service +@Service("jobBackupAccountService") public class AccountServiceImpl implements AccountService { - private final ServiceAccountResourceClient serviceAccountResourceClient; + private final ServiceAccountResource accountResource; @Autowired - public AccountServiceImpl(ServiceAccountResourceClient serviceAccountResourceClient) { - this.serviceAccountResourceClient = serviceAccountResourceClient; + public AccountServiceImpl(ServiceAccountResource accountResource) { + this.accountResource = accountResource; } @Override public ServiceAccountDTO getAccountAliasById(Long id) { - InternalResponse accountResp = serviceAccountResourceClient.getAccountByAccountId(id); + InternalResponse accountResp = accountResource.getAccountByAccountId(id); if (accountResp != null) { if (0 == accountResp.getCode()) { ServiceAccountDTO account = accountResp.getData(); @@ -81,7 +78,7 @@ public ServiceAccountDTO getAccountAliasById(Long id) { @Override public List listAccountByAppId(String username, Long appId) { - Response> appAccountListResp = serviceAccountResourceClient.listAccounts(appId, null); + Response> appAccountListResp = accountResource.listAccounts(appId, null); if (appAccountListResp != null) { if (0 == appAccountListResp.getCode()) { List accountList = appAccountListResp.getData(); @@ -117,7 +114,7 @@ public Long saveAccount(String username, Long appId, ServiceAccountDTO account) accountCreateUpdateReq.setDbPassword(account.getDbPassword()); accountCreateUpdateReq.setDbPort(account.getDbPort()); - Response saveAccountResp = serviceAccountResourceClient.saveAccount(username, + Response saveAccountResp = accountResource.saveAccount(username, appId, accountCreateUpdateReq); Integer errorCode = -1; diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/impl/ArchiveProgressServiceImpl.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/impl/ArchiveProgressServiceImpl.java deleted file mode 100644 index 74f02bd166..0000000000 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/impl/ArchiveProgressServiceImpl.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.backup.service.impl; - -import com.tencent.bk.job.backup.dao.ArchiveProgressDAO; -import com.tencent.bk.job.backup.model.dto.ArchiveProgressDTO; -import com.tencent.bk.job.backup.service.ArchiveProgressService; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -@Slf4j -@Service -public class ArchiveProgressServiceImpl implements ArchiveProgressService { - private final ArchiveProgressDAO archiveProgressDAO; - - @Autowired - public ArchiveProgressServiceImpl(ArchiveProgressDAO archiveProgressDAO) { - this.archiveProgressDAO = archiveProgressDAO; - } - - - @Override - public ArchiveProgressDTO queryArchiveProgress(String table) { - return archiveProgressDAO.queryArchiveProgress(table); - } - - @Override - public void saveArchiveProgress(ArchiveProgressDTO archiveProgress) { - if (archiveProgress.getLastArchiveTime() == null) { - archiveProgress.setLastArchiveTime(System.currentTimeMillis()); - } - archiveProgressDAO.saveArchiveProgress(archiveProgress); - } - - @Override - public void saveDeleteProgress(ArchiveProgressDTO deleteProgress) { - if (deleteProgress.getLastDeleteTime() == null) { - deleteProgress.setLastDeleteTime(System.currentTimeMillis()); - } - archiveProgressDAO.saveDeleteProgress(deleteProgress); - } -} diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/impl/ExportJobServiceImpl.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/impl/ExportJobServiceImpl.java index 2556cf9cf1..21222a9d19 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/impl/ExportJobServiceImpl.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/impl/ExportJobServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/impl/ImportJobServiceImpl.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/impl/ImportJobServiceImpl.java index 3da43a973e..4a2f268261 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/impl/ImportJobServiceImpl.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/impl/ImportJobServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -29,6 +29,7 @@ import com.tencent.bk.job.backup.constant.DuplicateIdHandlerEnum; import com.tencent.bk.job.backup.constant.LogEntityTypeEnum; import com.tencent.bk.job.backup.constant.LogMessage; +import com.tencent.bk.job.backup.crypto.BackupFileCryptoService; import com.tencent.bk.job.backup.dao.ImportJobDAO; import com.tencent.bk.job.backup.executor.ImportJobExecutor; import com.tencent.bk.job.backup.model.dto.IdNameInfoDTO; @@ -40,7 +41,6 @@ import com.tencent.bk.job.common.i18n.service.MessageI18nService; import com.tencent.bk.job.common.redis.util.LockUtils; import com.tencent.bk.job.common.util.JobContextUtil; -import com.tencent.bk.job.common.util.crypto.AESUtils; import com.tencent.bk.job.common.util.file.ZipUtil; import com.tencent.bk.job.manage.model.web.vo.task.TaskPlanVO; import com.tencent.bk.job.manage.model.web.vo.task.TaskTemplateVO; @@ -77,14 +77,19 @@ public class ImportJobServiceImpl implements ImportJobService { private final StorageService storageService; private final LogService logService; private final MessageI18nService i18nService; + private final BackupFileCryptoService backupFileCryptoService; @Autowired - public ImportJobServiceImpl(ImportJobDAO importJobDAO, StorageService storageService, LogService logService, - MessageI18nService i18nService) { + public ImportJobServiceImpl(ImportJobDAO importJobDAO, + StorageService storageService, + LogService logService, + MessageI18nService i18nService, + BackupFileCryptoService backupFileCryptoService) { this.importJobDAO = importJobDAO; this.storageService = storageService; this.logService = logService; this.i18nService = i18nService; + this.backupFileCryptoService = backupFileCryptoService; } @Override @@ -134,74 +139,78 @@ public Boolean startImport(ImportJobInfoDTO importJobInfo) { @Override public Boolean parseFile(String username, Long appId, String jobId) { ImportJobInfoDTO importJob = importJobDAO.getImportJobById(appId, jobId); - if (importJob != null) { - logService.addImportLog(appId, jobId, i18nService.getI18n(LogMessage.PROCESS_UPLOAD_FILE)); - File uploadFile = storageService.getFile(importJob.getFileName()); - if (uploadFile != null && uploadFile.exists()) { - logService.addImportLog(appId, jobId, i18nService.getI18n(LogMessage.DETECT_FILE_TYPE)); - if (!uploadFile.getName().endsWith(Constant.JOB_EXPORT_FILE_SUFFIX) - && !uploadFile.getName().endsWith( - Constant.JOB_EXPORT_FILE_SUFFIX.concat(Constant.JOB_IMPORT_DECRYPT_SUFFIX))) { - markJobFailed(importJob, i18nService.getI18n(LogMessage.WRONG_FILE_TYPE)); - return false; + if (importJob == null) { + return false; + } + logService.addImportLog(appId, jobId, i18nService.getI18n(LogMessage.PROCESS_UPLOAD_FILE)); + File uploadFile = storageService.getFile(importJob.getFileName()); + if (uploadFile == null || !uploadFile.exists()) { + return false; + } + logService.addImportLog(appId, jobId, i18nService.getI18n(LogMessage.DETECT_FILE_TYPE)); + if (!uploadFile.getName().endsWith(Constant.JOB_EXPORT_FILE_SUFFIX) + && !uploadFile.getName().endsWith( + Constant.JOB_EXPORT_FILE_SUFFIX.concat(Constant.JOB_IMPORT_DECRYPT_SUFFIX))) { + markJobAllFailed(importJob, i18nService.getI18n(LogMessage.WRONG_FILE_TYPE)); + return false; + } + logService.addImportLog(appId, jobId, i18nService.getI18n(LogMessage.CORRECT_FILE_TYPE)); + ZipFile zipFile = null; + try { + zipFile = new ZipFile(uploadFile); + List fileList = ZipUtil.unzip(uploadFile); + logService.addImportLog(appId, jobId, i18nService.getI18n(LogMessage.EXTRACT_FILE_DATA)); + boolean success = false; + for (File file : fileList) { + if (!file.getName().endsWith(".json")) { + continue; } - logService.addImportLog(appId, jobId, i18nService.getI18n(LogMessage.CORRECT_FILE_TYPE)); - ZipFile zipFile = null; + JobBackupInfoDTO jobBackupInfo = ImportJobExecutor.readJobBackupInfoFromFile(file); + if (jobBackupInfo == null) { + continue; + } + if (jobBackupInfo.getExpireTime() != 0 + && jobBackupInfo.getExpireTime() <= System.currentTimeMillis()) { + markJobAllFailed(importJob, i18nService.getI18n(LogMessage.IMPORT_FILE_EXPIRED)); + } + importJob.setExportId(jobBackupInfo.getId()); + importJob.setStatus(BackupJobStatusEnum.PARSE_SUCCESS); + importJob.setTemplateInfo(jobBackupInfo.getTemplateInfo()); + importJob.setIdNameInfo(extractIdNameInfo(jobBackupInfo)); + importJobDAO.updateImportJobById(importJob); + logService.addImportLog(appId, jobId, i18nService.getI18n(LogMessage.EXTRACT_SUCCESS)); + success = true; + break; + } + if (!success) { + log.warn("Parse import file fail"); + markJobAllFailed(importJob, i18nService.getI18n(LogMessage.EXTRACT_FAILED)); + } else { + return true; + } + } catch (IOException | RuntimeException e) { + log.error("Error while unzip upload file", e); + if (detectAndRemoveMagic(uploadFile)) { + importJob.setStatus(BackupJobStatusEnum.NEED_PASSWORD); + importJobDAO.updateImportJobById(importJob); + logService.addImportLog(appId, jobId, i18nService.getI18n(LogMessage.FILE_ENCRYPTED), + LogEntityTypeEnum.REQUEST_PASSWORD); + } else { + markJobAllFailed(importJob, i18nService.getI18n(LogMessage.EXTRACT_FAILED)); + } + } finally { + if (zipFile != null) { try { - zipFile = new ZipFile(uploadFile); - List fileList = ZipUtil.unzip(uploadFile); - logService.addImportLog(appId, jobId, i18nService.getI18n(LogMessage.EXTRACT_FILE_DATA)); - boolean success = false; - for (File file : fileList) { - if (file.getName().endsWith(".json")) { - JobBackupInfoDTO jobBackupInfo = ImportJobExecutor.readJobBackupInfoFromFile(file); - if (jobBackupInfo != null) { - if (jobBackupInfo.getExpireTime() != 0 - && jobBackupInfo.getExpireTime() <= System.currentTimeMillis()) { - markJobFailed(importJob, "作业已过期！"); - } - importJob.setExportId(jobBackupInfo.getId()); - importJob.setStatus(BackupJobStatusEnum.PARSE_SUCCESS); - importJob.setTemplateInfo(jobBackupInfo.getTemplateInfo()); - importJob.setIdNameInfo(extractIdNameInfo(jobBackupInfo)); - importJobDAO.updateImportJobById(importJob); - logService.addImportLog(appId, jobId, i18nService.getI18n(LogMessage.EXTRACT_SUCCESS)); - success = true; - break; - } - } - } - if (!success) { - log.warn("Parse import file fail"); - markJobFailed(importJob, i18nService.getI18n(LogMessage.EXTRACT_FAILED)); - } else { - return true; - } - } catch (IOException | RuntimeException e) { - log.error("Error while unzip upload file", e); - if (detectAndRemoveMagic(appId, jobId, uploadFile)) { - importJob.setStatus(BackupJobStatusEnum.NEED_PASSWORD); - importJobDAO.updateImportJobById(importJob); - logService.addImportLog(appId, jobId, i18nService.getI18n(LogMessage.FILE_ENCRYPTED), - LogEntityTypeEnum.REQUEST_PASSWORD); - } else { - markJobFailed(importJob, i18nService.getI18n(LogMessage.EXTRACT_FAILED)); - } - } finally { - if (zipFile != null) { - try { - zipFile.close(); - } catch (IOException e) { - log.warn("Error when close", e); - } - } + zipFile.close(); + } catch (IOException e) { + log.warn("Error when close", e); } } } return false; } - private boolean detectAndRemoveMagic(Long appId, String jobId, File uploadFile) { + private boolean detectAndRemoveMagic(File uploadFile) { boolean success = false; File tmpFile = new File(uploadFile.getPath().concat(".tmp")); try (FileInputStream in = new FileInputStream(uploadFile); FileOutputStream out = @@ -280,7 +289,7 @@ public Boolean checkPassword(String username, Long appId, String jobId, String p new File(parentPath.concat(File.separatorChar + uploadFile.getName() + Constant.JOB_IMPORT_DECRYPT_SUFFIX)); try { - AESUtils.decrypt(uploadFile, decryptedFile, password); + backupFileCryptoService.decryptBackupFile(password, uploadFile, decryptedFile); logService.addImportLog(appId, jobId, i18nService.getI18n(LogMessage.CORRECT_PASSWORD)); FileUtils.deleteQuietly(uploadFile); importInfo.setFileName(importInfo.getFileName().concat(Constant.JOB_IMPORT_DECRYPT_SUFFIX)); @@ -288,6 +297,7 @@ public Boolean checkPassword(String username, Long appId, String jobId, String p parseFile(username, appId, jobId); return true; } catch (Exception e) { + log.warn("Fail to decrypt backupFile", e); logService.addImportLog(appId, jobId, i18nService.getI18n(LogMessage.WRONG_PASSWORD), LogEntityTypeEnum.RETRY_PASSWORD); importInfo.setStatus(BackupJobStatusEnum.WRONG_PASSWORD); @@ -300,13 +310,13 @@ public Boolean checkPassword(String username, Long appId, String jobId, String p } @Override - public Boolean updateImportJob(ImportJobInfoDTO importJob) { - return importJobDAO.updateImportJobById(importJob); + public void updateImportJob(ImportJobInfoDTO importJob) { + importJobDAO.updateImportJobById(importJob); } @Override - public void markJobFailed(ImportJobInfoDTO importJob, String message) { - importJob.setStatus(BackupJobStatusEnum.FAILED); + public void markJobAllFailed(ImportJobInfoDTO importJob, String message) { + importJob.setStatus(BackupJobStatusEnum.ALL_FAILED); importJobDAO.updateImportJobById(importJob); logService.addImportLog(importJob.getAppId(), importJob.getId(), message, LogEntityTypeEnum.ERROR); } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/impl/LogServiceImpl.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/impl/LogServiceImpl.java index 9651195272..98dfb8f612 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/impl/LogServiceImpl.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/impl/LogServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -35,11 +35,8 @@ import java.util.List; -/** - * @since 29/7/2020 11:10 - */ @Slf4j -@Service +@Service("importAndExportLogService") public class LogServiceImpl implements LogService { private final LogDAO logDAO; diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/impl/NfsStorageServiceImpl.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/impl/NfsStorageServiceImpl.java index 0afc5e2ada..3f28c88624 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/impl/NfsStorageServiceImpl.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/impl/NfsStorageServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,6 +28,8 @@ import com.tencent.bk.job.backup.service.StorageService; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.exception.InternalException; +import com.tencent.bk.job.common.exception.InvalidParamException; +import com.tencent.bk.job.common.util.FilePathUtils; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -69,7 +71,15 @@ public String store(String id, String prefix, MultipartFile file) { } else { prefix = File.separator; } - String fileName = prefix + id + File.separatorChar + file.getOriginalFilename(); + String originalFileName = file.getOriginalFilename(); + if (StringUtils.isBlank(originalFileName)) { + throw new InvalidParamException( + ErrorCode.ILLEGAL_PARAM_WITH_REASON, + new String[]{"file", "filename cannot be blank"} + ); + } + String fileName = prefix + id + File.separatorChar + + FilePathUtils.parseDirAndFileName(originalFileName).getRight(); String fullFileName = storagePath.concat(fileName); File theFile = new File(fullFileName); diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/impl/ScriptServiceImpl.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/impl/ScriptServiceImpl.java index 2c8f30c272..55e98b8205 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/impl/ScriptServiceImpl.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/impl/ScriptServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,39 +24,45 @@ package com.tencent.bk.job.backup.service.impl; -import com.tencent.bk.job.backup.client.ServiceScriptResourceClient; import com.tencent.bk.job.backup.service.ScriptService; import com.tencent.bk.job.common.model.InternalResponse; +import com.tencent.bk.job.manage.api.inner.ServiceScriptResource; import com.tencent.bk.job.manage.model.inner.ServiceScriptDTO; import lombok.extern.slf4j.Slf4j; +import org.slf4j.helpers.MessageFormatter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -/** - * @since 6/8/2020 22:01 - */ @Slf4j -@Service +@Service("jobBackupScriptService") public class ScriptServiceImpl implements ScriptService { - private final ServiceScriptResourceClient serviceScriptResourceClient; + private final ServiceScriptResource scriptResource; @Autowired - public ScriptServiceImpl(ServiceScriptResourceClient serviceScriptResourceClient) { - this.serviceScriptResourceClient = serviceScriptResourceClient; + public ScriptServiceImpl(ServiceScriptResource scriptResource) { + this.scriptResource = scriptResource; } @Override public ServiceScriptDTO getScriptInfoById(String username, Long appId, Long scriptVersionId) { try { InternalResponse scriptByVersionIdResponse = - serviceScriptResourceClient.getScriptByAppIdAndScriptVersionId(username, appId, scriptVersionId); + scriptResource.getScriptByAppIdAndScriptVersionId(username, appId, scriptVersionId); if (scriptByVersionIdResponse != null) { if (scriptByVersionIdResponse.getCode() == 0) { return scriptByVersionIdResponse.getData(); } } } catch (Exception e) { - log.error("Error while getting script info by id!|{}|{}|{}", username, appId, scriptVersionId, e); + String msg = MessageFormatter.arrayFormat( + "Error while getting script info by id!|{}|{}|{}", + new String[]{ + username, + String.valueOf(appId), + String.valueOf(scriptVersionId) + } + ).getMessage(); + log.error(msg, e); } return null; } diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/impl/TaskPlanServiceImpl.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/impl/TaskPlanServiceImpl.java index 8c285952ec..f4effab4a9 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/impl/TaskPlanServiceImpl.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/impl/TaskPlanServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,16 +24,17 @@ package com.tencent.bk.job.backup.service.impl; -import com.tencent.bk.job.backup.client.ServiceBackupTmpResourceClient; -import com.tencent.bk.job.backup.client.ServicePlanResourceClient; import com.tencent.bk.job.backup.service.TaskPlanService; import com.tencent.bk.job.common.model.InternalResponse; import com.tencent.bk.job.common.model.Response; +import com.tencent.bk.job.manage.api.inner.ServiceBackupTmpResource; +import com.tencent.bk.job.manage.api.inner.ServiceTaskPlanResource; import com.tencent.bk.job.manage.model.inner.ServiceIdNameCheckDTO; import com.tencent.bk.job.manage.model.inner.ServiceTaskVariableDTO; import com.tencent.bk.job.manage.model.web.vo.task.TaskPlanVO; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; +import org.slf4j.helpers.MessageFormatter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -42,20 +43,17 @@ import java.util.List; import java.util.stream.Collectors; -/** - * @since 29/7/2020 17:46 - */ @Slf4j -@Service +@Service("jobBackupTaskPlanService") public class TaskPlanServiceImpl implements TaskPlanService { - private final ServiceBackupTmpResourceClient serviceBackupTmpResourceClient; - private final ServicePlanResourceClient servicePlanResourceClient; + private final ServiceBackupTmpResource backupTmpResource; + private final ServiceTaskPlanResource taskPlanResource; @Autowired - public TaskPlanServiceImpl(ServiceBackupTmpResourceClient serviceBackupTmpResourceClient, - ServicePlanResourceClient servicePlanResourceClient) { - this.serviceBackupTmpResourceClient = serviceBackupTmpResourceClient; - this.servicePlanResourceClient = servicePlanResourceClient; + public TaskPlanServiceImpl(ServiceBackupTmpResource backupTmpResource, + ServiceTaskPlanResource taskPlanResource) { + this.backupTmpResource = backupTmpResource; + this.taskPlanResource = taskPlanResource; } @Override @@ -72,7 +70,7 @@ public List getTaskPlanByIdList(String username, Long appId, Long te } log.debug("Fetching plan {}/{}/{} using {}", appId, templateId, planId, username); Response planByIdResponse = - serviceBackupTmpResourceClient.getPlanById(username, appId, templateId, planId); + backupTmpResource.getPlanById(username, appId, templateId, planId); if (planByIdResponse != null) { if (0 == planByIdResponse.getCode()) { taskPlanList.add(planByIdResponse.getData()); @@ -84,7 +82,16 @@ public List getTaskPlanByIdList(String username, Long appId, Long te } } } catch (Exception e) { - log.error("Error while getting plan info!|{}|{}|{}|{}", username, appId, templateId, planIdList, e); + String msg = MessageFormatter.arrayFormat( + "Error while getting plan info!|{}|{}|{}|{}", + new String[]{ + username, + String.valueOf(appId), + String.valueOf(templateId), + String.valueOf(planIdList) + } + ).getMessage(); + log.error(msg, e); } return taskPlanList; @@ -94,7 +101,7 @@ public List getTaskPlanByIdList(String username, Long appId, Long te public List listPlans(String username, Long appId, Long templateId) { try { Response> planListResponse = - serviceBackupTmpResourceClient.listPlans(username, appId, templateId); + backupTmpResource.listPlans(username, appId, templateId); if (planListResponse != null) { if (0 == planListResponse.getCode()) { log.debug("Fetching plan list of {}/{} finished.", appId, templateId); @@ -105,7 +112,15 @@ public List listPlans(String username, Long appId, Long templateId) } } } catch (Exception e) { - log.error("Error while list plan info by template id!|{}|{}|{}", username, appId, templateId, e); + String msg = MessageFormatter.arrayFormat( + "Error while list plan info by template id!|{}|{}|{}", + new String[]{ + username, + String.valueOf(appId), + String.valueOf(templateId) + } + ).getMessage(); + log.error(msg, e); } return null; } @@ -114,7 +129,7 @@ public List listPlans(String username, Long appId, Long templateId) public ServiceIdNameCheckDTO checkIdAndName(Long appId, Long templateId, Long planId, String name) { try { InternalResponse idNameCheckResponse = - servicePlanResourceClient.checkIdAndName(appId, templateId, planId, name); + taskPlanResource.checkIdAndName(appId, templateId, planId, name); if (idNameCheckResponse != null) { if (0 == idNameCheckResponse.getCode()) { return idNameCheckResponse.getData(); @@ -143,7 +158,7 @@ public Long savePlan(String username, Long appId, Long templateId, TaskPlanVO pl planInfo.setVariableList(planInfo.getVariableList()); InternalResponse savePlanResult = - servicePlanResourceClient.savePlanForImport(username, appId, templateId, null, planInfo); + taskPlanResource.savePlanForImport(username, appId, templateId, null, planInfo); if (savePlanResult != null) { if (0 == savePlanResult.getCode()) { @@ -154,7 +169,16 @@ public Long savePlan(String username, Long appId, Long templateId, TaskPlanVO pl } } } catch (Exception e) { - log.error("Error while save plan!|{}|{}|{}|{}", username, appId, templateId, planInfo, e); + String msg = MessageFormatter.arrayFormat( + "Error while save plan!|{}|{}|{}|{}", + new String[]{ + username, + String.valueOf(appId), + String.valueOf(templateId), + String.valueOf(planInfo) + } + ).getMessage(); + log.error(msg, e); } return null; @@ -164,7 +188,7 @@ public Long savePlan(String username, Long appId, Long templateId, TaskPlanVO pl public List getPlanVariable(String username, Long appId, Long templateId, Long planId) { try { InternalResponse> planVariableResponse = - servicePlanResourceClient.getPlanVariable(username, appId, templateId, planId); + taskPlanResource.getPlanVariable(username, appId, templateId, planId); if (planVariableResponse != null) { if (0 == planVariableResponse.getCode()) { return planVariableResponse.getData(); diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/impl/TaskTemplateServiceImpl.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/impl/TaskTemplateServiceImpl.java index c029ad01f2..890d26555e 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/impl/TaskTemplateServiceImpl.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/service/impl/TaskTemplateServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,18 +24,19 @@ package com.tencent.bk.job.backup.service.impl; -import com.tencent.bk.job.backup.client.ServiceBackupTmpResourceClient; -import com.tencent.bk.job.backup.client.ServiceTemplateResourceClient; import com.tencent.bk.job.backup.service.TaskTemplateService; import com.tencent.bk.job.common.constant.JobConstants; import com.tencent.bk.job.common.model.InternalResponse; import com.tencent.bk.job.common.model.Response; +import com.tencent.bk.job.manage.api.inner.ServiceBackupTmpResource; +import com.tencent.bk.job.manage.api.inner.ServiceTaskTemplateResource; import com.tencent.bk.job.manage.model.inner.ServiceIdNameCheckDTO; import com.tencent.bk.job.manage.model.inner.ServiceTaskVariableDTO; import com.tencent.bk.job.manage.model.web.request.TaskTemplateCreateUpdateReq; import com.tencent.bk.job.manage.model.web.vo.task.TaskTemplateVO; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; +import org.slf4j.helpers.MessageFormatter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -43,27 +44,25 @@ import java.util.List; import java.util.stream.Collectors; -/** - * @since 29/7/2020 17:46 - */ + @Slf4j -@Service +@Service("jobBackupTaskTemplateServiceImpl") public class TaskTemplateServiceImpl implements TaskTemplateService { - private final ServiceTemplateResourceClient serviceTemplateResourceClient; - private final ServiceBackupTmpResourceClient serviceBackupTmpResourceClient; + private final ServiceTaskTemplateResource templateResource; + private final ServiceBackupTmpResource backupTmpResource; @Autowired - public TaskTemplateServiceImpl(ServiceTemplateResourceClient serviceTemplateResourceClient, - ServiceBackupTmpResourceClient serviceBackupTmpResourceClient) { - this.serviceTemplateResourceClient = serviceTemplateResourceClient; - this.serviceBackupTmpResourceClient = serviceBackupTmpResourceClient; + public TaskTemplateServiceImpl(ServiceTaskTemplateResource templateResource, + ServiceBackupTmpResource backupTmpResource) { + this.templateResource = templateResource; + this.backupTmpResource = backupTmpResource; } @Override public TaskTemplateVO getTemplateById(String username, Long appId, Long id) { try { Response templateByIdResponse = - serviceBackupTmpResourceClient.getTemplateById(username, appId, id); + backupTmpResource.getTemplateById(username, appId, id); if (templateByIdResponse != null) { if (0 == templateByIdResponse.getCode()) { return templateByIdResponse.getData(); @@ -72,7 +71,15 @@ public TaskTemplateVO getTemplateById(String username, Long appId, Long id) { } } } catch (Exception e) { - log.error("Error while getting template info!|{}|{}|{}", username, appId, id, e); + String msg = MessageFormatter.arrayFormat( + "Error while getting template info!|{}|{}|{}", + new String[]{ + username, + String.valueOf(appId), + String.valueOf(id) + } + ).getMessage(); + log.error(msg, e); } return null; } @@ -81,7 +88,7 @@ public TaskTemplateVO getTemplateById(String username, Long appId, Long id) { public ServiceIdNameCheckDTO checkIdAndName(Long appId, long id, String name) { try { InternalResponse idNameCheckResponse = - serviceTemplateResourceClient.checkIdAndName(appId, id, name); + templateResource.checkIdAndName(appId, id, name); if (idNameCheckResponse != null) { if (0 == idNameCheckResponse.getCode()) { return idNameCheckResponse.getData(); @@ -90,7 +97,15 @@ public ServiceIdNameCheckDTO checkIdAndName(Long appId, long id, String name) { } } } catch (Exception e) { - log.error("Error while check id and name!|{}|{}|{}", appId, id, name, e); + String msg = MessageFormatter.arrayFormat( + "Error while check id and name!|{}|{}|{}", + new String[]{ + String.valueOf(appId), + String.valueOf(id), + name + } + ).getMessage(); + log.error(msg, e); } return null; } @@ -118,7 +133,7 @@ public Long saveTemplate(String username, Long appId, TaskTemplateVO taskTemplat } templateCreateUpdateReq.setTags(taskTemplate.getTags()); - InternalResponse saveTemplateResult = serviceTemplateResourceClient.saveTemplateForMigration( + InternalResponse saveTemplateResult = templateResource.saveTemplateForMigration( username, appId, taskTemplate.getId(), @@ -138,7 +153,15 @@ public Long saveTemplate(String username, Long appId, TaskTemplateVO taskTemplat } } } catch (Exception e) { - log.error("Error while trying to save template!|{}|{}|{}", username, appId, taskTemplate, e); + String msg = MessageFormatter.arrayFormat( + "Error while trying to save template!|{}|{}|{}", + new String[]{ + username, + String.valueOf(appId), + String.valueOf(taskTemplate) + } + ).getMessage(); + log.error(msg, e); } return null; } @@ -147,7 +170,7 @@ public Long saveTemplate(String username, Long appId, TaskTemplateVO taskTemplat public List getTemplateVariable(String username, Long appId, Long templateId) { try { InternalResponse> templateVariableResponse = - serviceTemplateResourceClient.getTemplateVariable(username, appId, templateId); + templateResource.getTemplateVariable(username, appId, templateId); if (templateVariableResponse != null) { if (0 == templateVariableResponse.getCode()) { return templateVariableResponse.getData(); diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/task/ScheduledTask.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/task/ScheduledTask.java index be989e42df..6137b21e53 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/task/ScheduledTask.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/task/ScheduledTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/util/DbRecordMapper.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/util/DbRecordMapper.java index 848e11fc2d..c006eff822 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/util/DbRecordMapper.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/util/DbRecordMapper.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,9 +28,10 @@ import com.tencent.bk.job.backup.constant.BackupJobStatusEnum; import com.tencent.bk.job.backup.constant.DuplicateIdHandlerEnum; import com.tencent.bk.job.backup.constant.LogEntityTypeEnum; -import com.tencent.bk.job.backup.constant.SecretHandlerEnum; -import com.tencent.bk.job.backup.model.dto.*; -import com.tencent.bk.job.backup.model.tables.ExportJob; +import com.tencent.bk.job.backup.model.dto.BackupTemplateInfoDTO; +import com.tencent.bk.job.backup.model.dto.IdNameInfoDTO; +import com.tencent.bk.job.backup.model.dto.ImportJobInfoDTO; +import com.tencent.bk.job.backup.model.dto.LogEntityDTO; import com.tencent.bk.job.backup.model.tables.ImportJob; import com.tencent.bk.job.common.util.json.JsonUtils; import org.jooq.Record13; @@ -46,32 +47,6 @@ */ public class DbRecordMapper { - public static ExportJobInfoDTO convertRecordToExportJobInfo( - Record13 record) { - if (record == null) { - return null; - } - ExportJob table = ExportJob.EXPORT_JOB; - ExportJobInfoDTO exportJobInfo = new ExportJobInfoDTO(); - exportJobInfo.setId(record.get(table.ID)); - exportJobInfo.setAppId(record.get(table.APP_ID).longValue()); - exportJobInfo.setCreator(record.get(table.CREATOR)); - exportJobInfo.setCreateTime(record.get(table.CREATE_TIME).longValue()); - exportJobInfo.setUpdateTime(record.get(table.UPDATE_TIME).longValue()); - exportJobInfo.setStatus(BackupJobStatusEnum.valueOf(record.get(table.STATUS).intValue())); - exportJobInfo.setPassword(record.get(table.PASSWORD)); - exportJobInfo.setPackageName(record.get(table.PACKAGE_NAME)); - exportJobInfo.setSecretHandler(SecretHandlerEnum.valueOf(record.get(table.SECRET_HANDLER).intValue())); - exportJobInfo.setExpireTime(record.get(table.EXPIRE_TIME).longValue()); - exportJobInfo.setTemplateInfo(JsonUtils.fromJson(record.get(table.TEMPLATE_PLAN_INFO), - new TypeReference>() { - })); - exportJobInfo.setFileName(record.get(table.FILE_NAME)); - exportJobInfo.setLocale(Locale.forLanguageTag(record.get(table.LOCALE))); - return exportJobInfo; - } - public static ImportJobInfoDTO convertRecordToImportJobInfo( Record13 record) { diff --git a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/util/DbUtils.java b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/util/DbUtils.java index 50f6b815cf..8b7df0c5a1 100644 --- a/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/util/DbUtils.java +++ b/src/backend/job-backup/service-job-backup/src/main/java/com/tencent/bk/job/backup/util/DbUtils.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-backup/service-job-backup/src/main/resources/i18n/message.properties b/src/backend/job-backup/service-job-backup/src/main/resources/i18n/message.properties new file mode 100644 index 0000000000..2700f4d620 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/resources/i18n/message.properties @@ -0,0 +1,77 @@ +# +# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. +# +# Copyright (C) 2021 Tencent. All rights reserved. +# +# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. +# +# License for BK-JOB蓝鲸智云作业平台: +# -------------------------------------------------------------------- +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF +# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +job.backup.startExport=开始执行作业导出任务... +job.backup.processFinished=处理完成! +job.backup.processLocalFile=正在处理作业中的 "本地文件"... +job.backup.noLocalFile=导出的作业中不存在 "本地文件", 自动跳过 +job.backup.processScript=正在获取作业中的 "引用脚本" 相关信息... +job.backup.noScript=导出的作业中没有 "引用脚本", 自动跳过 +job.backup.processCipherText=正在处理作业中的 "密文" 类型变量值... +job.backup.saveNull=保存为空值 +job.backup.saveReal=保存真实值 +job.backup.noCipherText=导出的作业中没有使用 "密文" 变量，自动跳过 +job.backup.processTemplatePlanFinished=作业模板与执行方案信息处理完成! +job.backup.startPackage=开始打包... +job.backup.packageFinished=打包完成! +job.backup.startEncrypting=正在为压缩包进行加密处理... +job.backup.encryptingFinished=加密完成! +job.backup.skipEncrypting=文件加密选项为[不加密]，自动跳过 +job.backup.exportFinished=作业导出任务完成! [文件有效期：%s] +job.backup.forever=永久 +job.backup.day=天 +job.backup.template=模板 +job.backup.plan=执行方案 +job.backup.processUploadFile=开始检测上传文件... +job.backup.detectFileType=正在检测文件类型是否正确... +job.backup.correctFileType=检测通过! +job.backup.wrongFileType=检测不通过! (仅允许 .jobexport 为后缀的文件) +job.backup.extractFileData=正在解析文件的数据格式... +job.backup.importFileExpired=导入文件已过期！ +job.backup.extractSuccess=解析成功! +job.backup.fileEncrypted=文件包已被加密处理，请输入对应密码 +job.backup.correctPassword=密码校验成功! +job.backup.wrongPassword=密码校验失败! +job.backup.extractFailed=解析失败! 请确认该文件包是否正确来自作业平台导出. +job.backup.cannotFindImportFile=未找到待导入文件 +job.backup.startImport=开始处理导入请求... +job.backup.importSetting=按照 "导入设置"： +job.backup.idAutoIncrement=1. 作业原始ID不保留, 将以自增的方式导入 +job.backup.idKeepOnDuplicateIncrement=1. 保留作业原始ID, 发现冲突时将以自增的方式导入 +job.backup.idKeepOnDuplicateSkip=1. 保留作业原始ID, 发现冲突时即不进行导入 +job.backup.nameDuplicateSuffix=2. 若出现同名的作业模板, 将自动添加后缀 %s +job.backup.idAutoIncrementSuffix=ID 不保留, 将以自增的方式导入 +job.backup.idDuplicateIncrementSuffix=ID 已存在, 改为自增ID进行导入 +job.backup.idDuplicateSkipSuffix=ID 已存在, 取消导入该%s +job.backup.idKeepSuffix=ID 不存在, 将以原始ID进行导入 +job.backup.startImportTemplate=准备导入作业模板： [%d] %s +job.backup.templateNameChange=模板名称已存在! 自动更改为 %s +job.backup.importTemplateSuccess=模板导入成功! +job.backup.templateNameTooLongSkip=作业模板名称超长，跳过当前模版和该模版下的所有执行方案 +job.backup.startImportPlan=>>>>> 准备导入作业执行方案： %s +job.backup.importPlanSuccess=执行方案导入成功! +job.backup.importFailed=作业导入出现异常, 请稍后重试. +job.backup.importFinished=作业导入任务已全部结束! +job.backup.importAccountNotExist=[%s]账号[%s]不存在，请手动添加，添加完成后修改作业对应的字段 +job.backup.importStepAccountInvalid=作业[%s]步骤[%s]的账号无效 diff --git a/src/backend/job-backup/service-job-backup/src/main/resources/i18n/message_en.properties b/src/backend/job-backup/service-job-backup/src/main/resources/i18n/message_en.properties new file mode 100644 index 0000000000..b0f87dc3c1 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/resources/i18n/message_en.properties @@ -0,0 +1,77 @@ +# +# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. +# +# Copyright (C) 2021 Tencent. All rights reserved. +# +# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. +# +# License for BK-JOB蓝鲸智云作业平台: +# -------------------------------------------------------------------- +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF +# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +job.backup.startExport=Starting job export process... +job.backup.processFinished=Done! +job.backup.processLocalFile=Saving "Local Files" from exported jobs... +job.backup.noLocalFile="Local file" is not found in exported jobs +job.backup.processScript=Saving "Related Scripts" from exported jobs... +job.backup.noScript="Related Script" is not found in exported jobs +job.backup.processCipherText=Processing on "Ciphertext" variable's value... +job.backup.saveNull=Replace with empty value +job.backup.saveReal=Using original value +job.backup.noCipherText="Ciphertext" variable is not found in exported jobs +job.backup.processTemplatePlanFinished=Job data information processing complete! +job.backup.startPackage=Start packing... +job.backup.packageFinished=Done! +job.backup.startEncrypting=Encrypting the package... +job.backup.encryptingFinished=Encrypt complete! +job.backup.skipEncrypting=File encrypt option is [No] +job.backup.exportFinished=Job export complete! [File Valid of days:%s] +job.backup.forever=Permanently +job.backup.day=Day +job.backup.template=Template +job.backup.plan=Plan +job.backup.processUploadFile=Detecting uploaded file... +job.backup.detectFileType=Checking file type... +job.backup.correctFileType=PASSED! +job.backup.wrongFileType=FAILED! (Only allowed .jobexport) +job.backup.extractFileData=Parsing data format of the file... +job.backup.importFileExpired=Import file expired! +job.backup.extractSuccess=CORRECT! +job.backup.fileEncrypted=The file has been encrypted, please enter the password +job.backup.correctPassword=Password verified! +job.backup.wrongPassword=Wrong Password! +job.backup.extractFailed=INCORRECT! Please make sure the file is correctly exported from Job. +job.backup.cannotFindImportFile=Cannot find import file +job.backup.startImport=Starting Job import process... +job.backup.importSetting=Following "Import Settings": +job.backup.idAutoIncrement=1. Do not retain Job ID, import by auto increment +job.backup.idKeepOnDuplicateIncrement=1. Retain Job ID, auto increment if ID already exists +job.backup.idKeepOnDuplicateSkip=1. Retain Job ID, skip it if ID already exists +job.backup.nameDuplicateSuffix=2. Add suffix %s if Job name is duplicate +job.backup.idAutoIncrementSuffix=Do not retain the ID, import by auto increment +job.backup.idDuplicateIncrementSuffix=ID already exists, will import by auto increment +job.backup.idDuplicateSkipSuffix=ID already exists, skip to next Job %s +job.backup.idKeepSuffix=ID not exists, will import directly +job.backup.startImportTemplate=Prepare to import Job Template: [%d] %s +job.backup.templateNameChange=Template name already exists! Rename with %s +job.backup.importTemplateSuccess=Job Template Import Successful! +job.backup.templateNameTooLongSkip=Template name too long, skip this template and plans +job.backup.startImportPlan=>>>>> Prepare to import Job Plan: %s +job.backup.importPlanSuccess=Job Plan Import Successful! +job.backup.importFailed=Service Error! Please try again later. +job.backup.importFinished=Job import complete! +job.backup.importAccountNotExist=[%s] account [%s] does not exist.Please manually add it and modify the fields corresponding to the job +job.backup.importStepAccountInvalid=Job [%s] step [%s] account is invalid diff --git a/src/backend/job-backup/service-job-backup/src/main/resources/i18n/message_en_US.properties b/src/backend/job-backup/service-job-backup/src/main/resources/i18n/message_en_US.properties new file mode 100644 index 0000000000..b0f87dc3c1 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/resources/i18n/message_en_US.properties @@ -0,0 +1,77 @@ +# +# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. +# +# Copyright (C) 2021 Tencent. All rights reserved. +# +# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. +# +# License for BK-JOB蓝鲸智云作业平台: +# -------------------------------------------------------------------- +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF +# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +job.backup.startExport=Starting job export process... +job.backup.processFinished=Done! +job.backup.processLocalFile=Saving "Local Files" from exported jobs... +job.backup.noLocalFile="Local file" is not found in exported jobs +job.backup.processScript=Saving "Related Scripts" from exported jobs... +job.backup.noScript="Related Script" is not found in exported jobs +job.backup.processCipherText=Processing on "Ciphertext" variable's value... +job.backup.saveNull=Replace with empty value +job.backup.saveReal=Using original value +job.backup.noCipherText="Ciphertext" variable is not found in exported jobs +job.backup.processTemplatePlanFinished=Job data information processing complete! +job.backup.startPackage=Start packing... +job.backup.packageFinished=Done! +job.backup.startEncrypting=Encrypting the package... +job.backup.encryptingFinished=Encrypt complete! +job.backup.skipEncrypting=File encrypt option is [No] +job.backup.exportFinished=Job export complete! [File Valid of days:%s] +job.backup.forever=Permanently +job.backup.day=Day +job.backup.template=Template +job.backup.plan=Plan +job.backup.processUploadFile=Detecting uploaded file... +job.backup.detectFileType=Checking file type... +job.backup.correctFileType=PASSED! +job.backup.wrongFileType=FAILED! (Only allowed .jobexport) +job.backup.extractFileData=Parsing data format of the file... +job.backup.importFileExpired=Import file expired! +job.backup.extractSuccess=CORRECT! +job.backup.fileEncrypted=The file has been encrypted, please enter the password +job.backup.correctPassword=Password verified! +job.backup.wrongPassword=Wrong Password! +job.backup.extractFailed=INCORRECT! Please make sure the file is correctly exported from Job. +job.backup.cannotFindImportFile=Cannot find import file +job.backup.startImport=Starting Job import process... +job.backup.importSetting=Following "Import Settings": +job.backup.idAutoIncrement=1. Do not retain Job ID, import by auto increment +job.backup.idKeepOnDuplicateIncrement=1. Retain Job ID, auto increment if ID already exists +job.backup.idKeepOnDuplicateSkip=1. Retain Job ID, skip it if ID already exists +job.backup.nameDuplicateSuffix=2. Add suffix %s if Job name is duplicate +job.backup.idAutoIncrementSuffix=Do not retain the ID, import by auto increment +job.backup.idDuplicateIncrementSuffix=ID already exists, will import by auto increment +job.backup.idDuplicateSkipSuffix=ID already exists, skip to next Job %s +job.backup.idKeepSuffix=ID not exists, will import directly +job.backup.startImportTemplate=Prepare to import Job Template: [%d] %s +job.backup.templateNameChange=Template name already exists! Rename with %s +job.backup.importTemplateSuccess=Job Template Import Successful! +job.backup.templateNameTooLongSkip=Template name too long, skip this template and plans +job.backup.startImportPlan=>>>>> Prepare to import Job Plan: %s +job.backup.importPlanSuccess=Job Plan Import Successful! +job.backup.importFailed=Service Error! Please try again later. +job.backup.importFinished=Job import complete! +job.backup.importAccountNotExist=[%s] account [%s] does not exist.Please manually add it and modify the fields corresponding to the job +job.backup.importStepAccountInvalid=Job [%s] step [%s] account is invalid diff --git a/src/backend/job-backup/service-job-backup/src/main/resources/i18n/message_zh.properties b/src/backend/job-backup/service-job-backup/src/main/resources/i18n/message_zh.properties new file mode 100644 index 0000000000..2700f4d620 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/resources/i18n/message_zh.properties @@ -0,0 +1,77 @@ +# +# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. +# +# Copyright (C) 2021 Tencent. All rights reserved. +# +# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. +# +# License for BK-JOB蓝鲸智云作业平台: +# -------------------------------------------------------------------- +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF +# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +job.backup.startExport=开始执行作业导出任务... +job.backup.processFinished=处理完成! +job.backup.processLocalFile=正在处理作业中的 "本地文件"... +job.backup.noLocalFile=导出的作业中不存在 "本地文件", 自动跳过 +job.backup.processScript=正在获取作业中的 "引用脚本" 相关信息... +job.backup.noScript=导出的作业中没有 "引用脚本", 自动跳过 +job.backup.processCipherText=正在处理作业中的 "密文" 类型变量值... +job.backup.saveNull=保存为空值 +job.backup.saveReal=保存真实值 +job.backup.noCipherText=导出的作业中没有使用 "密文" 变量，自动跳过 +job.backup.processTemplatePlanFinished=作业模板与执行方案信息处理完成! +job.backup.startPackage=开始打包... +job.backup.packageFinished=打包完成! +job.backup.startEncrypting=正在为压缩包进行加密处理... +job.backup.encryptingFinished=加密完成! +job.backup.skipEncrypting=文件加密选项为[不加密]，自动跳过 +job.backup.exportFinished=作业导出任务完成! [文件有效期：%s] +job.backup.forever=永久 +job.backup.day=天 +job.backup.template=模板 +job.backup.plan=执行方案 +job.backup.processUploadFile=开始检测上传文件... +job.backup.detectFileType=正在检测文件类型是否正确... +job.backup.correctFileType=检测通过! +job.backup.wrongFileType=检测不通过! (仅允许 .jobexport 为后缀的文件) +job.backup.extractFileData=正在解析文件的数据格式... +job.backup.importFileExpired=导入文件已过期！ +job.backup.extractSuccess=解析成功! +job.backup.fileEncrypted=文件包已被加密处理，请输入对应密码 +job.backup.correctPassword=密码校验成功! +job.backup.wrongPassword=密码校验失败! +job.backup.extractFailed=解析失败! 请确认该文件包是否正确来自作业平台导出. +job.backup.cannotFindImportFile=未找到待导入文件 +job.backup.startImport=开始处理导入请求... +job.backup.importSetting=按照 "导入设置"： +job.backup.idAutoIncrement=1. 作业原始ID不保留, 将以自增的方式导入 +job.backup.idKeepOnDuplicateIncrement=1. 保留作业原始ID, 发现冲突时将以自增的方式导入 +job.backup.idKeepOnDuplicateSkip=1. 保留作业原始ID, 发现冲突时即不进行导入 +job.backup.nameDuplicateSuffix=2. 若出现同名的作业模板, 将自动添加后缀 %s +job.backup.idAutoIncrementSuffix=ID 不保留, 将以自增的方式导入 +job.backup.idDuplicateIncrementSuffix=ID 已存在, 改为自增ID进行导入 +job.backup.idDuplicateSkipSuffix=ID 已存在, 取消导入该%s +job.backup.idKeepSuffix=ID 不存在, 将以原始ID进行导入 +job.backup.startImportTemplate=准备导入作业模板： [%d] %s +job.backup.templateNameChange=模板名称已存在! 自动更改为 %s +job.backup.importTemplateSuccess=模板导入成功! +job.backup.templateNameTooLongSkip=作业模板名称超长，跳过当前模版和该模版下的所有执行方案 +job.backup.startImportPlan=>>>>> 准备导入作业执行方案： %s +job.backup.importPlanSuccess=执行方案导入成功! +job.backup.importFailed=作业导入出现异常, 请稍后重试. +job.backup.importFinished=作业导入任务已全部结束! +job.backup.importAccountNotExist=[%s]账号[%s]不存在，请手动添加，添加完成后修改作业对应的字段 +job.backup.importStepAccountInvalid=作业[%s]步骤[%s]的账号无效 diff --git a/src/backend/job-backup/service-job-backup/src/main/resources/i18n/message_zh_CN.properties b/src/backend/job-backup/service-job-backup/src/main/resources/i18n/message_zh_CN.properties new file mode 100644 index 0000000000..2700f4d620 --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/main/resources/i18n/message_zh_CN.properties @@ -0,0 +1,77 @@ +# +# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. +# +# Copyright (C) 2021 Tencent. All rights reserved. +# +# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. +# +# License for BK-JOB蓝鲸智云作业平台: +# -------------------------------------------------------------------- +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF +# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +job.backup.startExport=开始执行作业导出任务... +job.backup.processFinished=处理完成! +job.backup.processLocalFile=正在处理作业中的 "本地文件"... +job.backup.noLocalFile=导出的作业中不存在 "本地文件", 自动跳过 +job.backup.processScript=正在获取作业中的 "引用脚本" 相关信息... +job.backup.noScript=导出的作业中没有 "引用脚本", 自动跳过 +job.backup.processCipherText=正在处理作业中的 "密文" 类型变量值... +job.backup.saveNull=保存为空值 +job.backup.saveReal=保存真实值 +job.backup.noCipherText=导出的作业中没有使用 "密文" 变量，自动跳过 +job.backup.processTemplatePlanFinished=作业模板与执行方案信息处理完成! +job.backup.startPackage=开始打包... +job.backup.packageFinished=打包完成! +job.backup.startEncrypting=正在为压缩包进行加密处理... +job.backup.encryptingFinished=加密完成! +job.backup.skipEncrypting=文件加密选项为[不加密]，自动跳过 +job.backup.exportFinished=作业导出任务完成! [文件有效期：%s] +job.backup.forever=永久 +job.backup.day=天 +job.backup.template=模板 +job.backup.plan=执行方案 +job.backup.processUploadFile=开始检测上传文件... +job.backup.detectFileType=正在检测文件类型是否正确... +job.backup.correctFileType=检测通过! +job.backup.wrongFileType=检测不通过! (仅允许 .jobexport 为后缀的文件) +job.backup.extractFileData=正在解析文件的数据格式... +job.backup.importFileExpired=导入文件已过期！ +job.backup.extractSuccess=解析成功! +job.backup.fileEncrypted=文件包已被加密处理，请输入对应密码 +job.backup.correctPassword=密码校验成功! +job.backup.wrongPassword=密码校验失败! +job.backup.extractFailed=解析失败! 请确认该文件包是否正确来自作业平台导出. +job.backup.cannotFindImportFile=未找到待导入文件 +job.backup.startImport=开始处理导入请求... +job.backup.importSetting=按照 "导入设置"： +job.backup.idAutoIncrement=1. 作业原始ID不保留, 将以自增的方式导入 +job.backup.idKeepOnDuplicateIncrement=1. 保留作业原始ID, 发现冲突时将以自增的方式导入 +job.backup.idKeepOnDuplicateSkip=1. 保留作业原始ID, 发现冲突时即不进行导入 +job.backup.nameDuplicateSuffix=2. 若出现同名的作业模板, 将自动添加后缀 %s +job.backup.idAutoIncrementSuffix=ID 不保留, 将以自增的方式导入 +job.backup.idDuplicateIncrementSuffix=ID 已存在, 改为自增ID进行导入 +job.backup.idDuplicateSkipSuffix=ID 已存在, 取消导入该%s +job.backup.idKeepSuffix=ID 不存在, 将以原始ID进行导入 +job.backup.startImportTemplate=准备导入作业模板： [%d] %s +job.backup.templateNameChange=模板名称已存在! 自动更改为 %s +job.backup.importTemplateSuccess=模板导入成功! +job.backup.templateNameTooLongSkip=作业模板名称超长，跳过当前模版和该模版下的所有执行方案 +job.backup.startImportPlan=>>>>> 准备导入作业执行方案： %s +job.backup.importPlanSuccess=执行方案导入成功! +job.backup.importFailed=作业导入出现异常, 请稍后重试. +job.backup.importFinished=作业导入任务已全部结束! +job.backup.importAccountNotExist=[%s]账号[%s]不存在，请手动添加，添加完成后修改作业对应的字段 +job.backup.importStepAccountInvalid=作业[%s]步骤[%s]的账号无效 diff --git a/src/backend/job-backup/service-job-backup/src/test/java/com/tencent/bk/job/backup/archive/ArchiveDbNodePriorityEvaluatorTest.java b/src/backend/job-backup/service-job-backup/src/test/java/com/tencent/bk/job/backup/archive/ArchiveDbNodePriorityEvaluatorTest.java new file mode 100644 index 0000000000..5ae684418f --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/test/java/com/tencent/bk/job/backup/archive/ArchiveDbNodePriorityEvaluatorTest.java @@ -0,0 +1,100 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive; + +import com.tencent.bk.job.backup.archive.model.DbDataNode; +import com.tencent.bk.job.backup.archive.model.ArchiveTaskInfo; +import com.tencent.bk.job.backup.constant.ArchiveTaskTypeEnum; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; + +class ArchiveDbNodePriorityEvaluatorTest { + + @Test + void testStandaloneDb() { + + List runningTasks = new ArrayList<>(); + runningTasks.add(genTask(DbDataNode.standaloneDbDataNode(), 20240808, 2)); + runningTasks.add(genTask(DbDataNode.standaloneDbDataNode(), 20240808, 2)); + runningTasks.add(genTask(DbDataNode.standaloneDbDataNode(), 20240808, 3)); + + Map scheduleTaskCountGroupByDb = new HashMap<>(); + scheduleTaskCountGroupByDb.put(DbDataNode.STANDALONE_DS_NAME, 2); + + ArchiveDbNodePriorityEvaluator.DbNodeTasksInfo dbNodeTasksInfo = + ArchiveDbNodePriorityEvaluator.evaluateHighestPriorityDbNode(runningTasks, scheduleTaskCountGroupByDb); + assertThat(dbNodeTasksInfo).isNotNull(); + assertThat(dbNodeTasksInfo.getDbNodeId()).isEqualTo(DbDataNode.STANDALONE_DS_NAME); + assertThat(dbNodeTasksInfo.getRunningTaskCount()).isEqualTo(3); + assertThat(dbNodeTasksInfo.getScheduleTaskCount()).isEqualTo(2); + } + + @Test + void testShardingDb() { + + List runningTasks = new ArrayList<>(); + Map scheduleTaskCountGroupByDb = new HashMap<>(); + + runningTasks.add(genTask(DbDataNode.shardingDbDataNode("ds", 0, 1), 20240808, 1)); + runningTasks.add(genTask(DbDataNode.shardingDbDataNode("ds", 1, 0), 20240808, 1)); + runningTasks.add(genTask(DbDataNode.shardingDbDataNode("ds", 1, 1), 20240808, 1)); + scheduleTaskCountGroupByDb.put("ds:0", 2); + scheduleTaskCountGroupByDb.put("ds:1", 1); + + ArchiveDbNodePriorityEvaluator.DbNodeTasksInfo dbNodeTasksInfo = + ArchiveDbNodePriorityEvaluator.evaluateHighestPriorityDbNode(runningTasks, scheduleTaskCountGroupByDb); + assertThat(dbNodeTasksInfo).isNotNull(); + assertThat(dbNodeTasksInfo.getDbNodeId()).isEqualTo("ds:0"); + + runningTasks.clear(); + scheduleTaskCountGroupByDb.clear(); + + runningTasks.add(genTask(DbDataNode.shardingDbDataNode("ds", 0, 1), 20240808, 1)); + runningTasks.add(genTask(DbDataNode.shardingDbDataNode("ds", 1, 1), 20240808, 1)); + scheduleTaskCountGroupByDb.put("ds:0", 2); + scheduleTaskCountGroupByDb.put("ds:1", 1); + + dbNodeTasksInfo = ArchiveDbNodePriorityEvaluator.evaluateHighestPriorityDbNode( + runningTasks, scheduleTaskCountGroupByDb); + assertThat(dbNodeTasksInfo).isNotNull(); + assertThat(dbNodeTasksInfo.getDbNodeId()).isEqualTo("ds:0"); + + } + + private ArchiveTaskInfo genTask(DbDataNode dataNode, Integer day, Integer hour) { + ArchiveTaskInfo task = new ArchiveTaskInfo(); + task.setTaskType(ArchiveTaskTypeEnum.JOB_INSTANCE); + task.setDbDataNode(dataNode); + task.setDay(day); + task.setHour(hour); + return task; + } +} diff --git a/src/backend/job-backup/service-job-backup/src/test/java/com/tencent/bk/job/backup/archive/util/ArchiveDateTimeUtilTest.java b/src/backend/job-backup/service-job-backup/src/test/java/com/tencent/bk/job/backup/archive/util/ArchiveDateTimeUtilTest.java new file mode 100644 index 0000000000..46dfacaf1a --- /dev/null +++ b/src/backend/job-backup/service-job-backup/src/test/java/com/tencent/bk/job/backup/archive/util/ArchiveDateTimeUtilTest.java @@ -0,0 +1,87 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.backup.archive.util; + +import org.junit.jupiter.api.Test; + +import java.time.LocalDateTime; +import java.time.ZoneId; + +import static org.assertj.core.api.AssertionsForClassTypes.assertThat; + +class ArchiveDateTimeUtilTest { + + @Test + void computeDay() { + LocalDateTime dateTime = LocalDateTime.of(2024, 11, 11, 1, 11, 11); + assertThat(ArchiveDateTimeUtil.computeDay(dateTime)).isEqualTo(20241111); + + dateTime = LocalDateTime.of(2024, 8, 12, 0, 0, 0); + assertThat(ArchiveDateTimeUtil.computeDay(dateTime)).isEqualTo(20240812); + } + + @Test + void computeHour() { + LocalDateTime dateTime = LocalDateTime.of(2024, 11, 11, 1, 11, 11); + assertThat(ArchiveDateTimeUtil.computeHour(dateTime)).isEqualTo(1); + + dateTime = LocalDateTime.of(2024, 8, 12, 0, 0, 0); + assertThat(ArchiveDateTimeUtil.computeHour(dateTime)).isEqualTo(0); + + dateTime = LocalDateTime.of(2024, 8, 12, 0, 0, 1); + assertThat(ArchiveDateTimeUtil.computeHour(dateTime)).isEqualTo(0); + } + + @Test + void computeStartOfDayBeforeDays() { + LocalDateTime dateTime = LocalDateTime.of(2024, 11, 11, 1, 11, 11); + LocalDateTime result = ArchiveDateTimeUtil.computeStartOfDayBeforeDays(dateTime, 1); + assertThat(result).isEqualTo(LocalDateTime.of(2024, 11, 10, 0, 0, 0)); + + result = ArchiveDateTimeUtil.computeStartOfDayBeforeDays(dateTime, 30); + assertThat(result).isEqualTo(LocalDateTime.of(2024, 10, 12, 0, 0, 0)); + } + + @Test + void unixTimestampMillToLocalDateTime() { + LocalDateTime dateTime = ArchiveDateTimeUtil.unixTimestampMillToZoneDateTime( + 1732072271000L, ZoneId.of("GMT+8")); + assertThat(dateTime).isEqualTo(LocalDateTime.of(2024, 11, 20, 11, 11, 11)); + } + + @Test + void toHourlyRoundDown() { + LocalDateTime dateTime = + ArchiveDateTimeUtil.toHourlyRoundDown(LocalDateTime.of(2024, 11, 20, 11, 1, 1)); + assertThat(dateTime).isEqualTo(LocalDateTime.of(2024, 11, 20, 11, 0, 0)); + } + + @Test + void toTimestampMillsAtZone() { + LocalDateTime dateTime = LocalDateTime.of(2024, 11, 11, 1, 11, 11); + assertThat(ArchiveDateTimeUtil.toTimestampMillsAtZone(dateTime, ZoneId.of("GMT+8"))) + .isEqualTo(1731258671000L); + } +} diff --git a/src/backend/job-config/build.gradle b/src/backend/job-config/build.gradle index be727a6647..5ac6ad32e6 100644 --- a/src/backend/job-config/build.gradle +++ b/src/backend/job-config/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -34,10 +34,9 @@ ext { version "${jobConfigVersion}" dependencies { api project(':commons:common') - api project(":commons:common-i18n") + api project(":commons:common-log") + api project(":commons:common-task") api project(":commons:common-consul") - implementation 'io.springfox:springfox-swagger2' - implementation 'io.springfox:springfox-swagger-ui' implementation("org.springframework.boot:spring-boot-autoconfigure") implementation 'org.springframework.boot:spring-boot-starter-web' implementation 'org.springframework.boot:spring-boot-starter-actuator' @@ -45,6 +44,8 @@ dependencies { implementation 'org.springframework.cloud:spring-cloud-starter-bootstrap' implementation 'org.springframework.cloud:spring-cloud-config-server' implementation 'org.springframework.cloud:spring-cloud-starter-sleuth' + implementation 'org.springframework.cloud:spring-cloud-starter-bus-amqp' + implementation 'org.springframework.cloud:spring-cloud-starter-stream-rabbit' implementation 'com.fasterxml.jackson.core:jackson-core' implementation 'com.fasterxml.jackson.core:jackson-databind' implementation 'com.fasterxml.jackson.core:jackson-annotations' @@ -59,4 +60,9 @@ springBoot { getMainClass().set("com.tencent.bk.job.config.JobConfigBootApplication") buildInfo() } + +configurations { + all*.exclude module : 'springfox-boot-starter' +} + apply from: "$rootDir/task_job_package.gradle" diff --git a/src/backend/job-config/src/main/java/com/tencent/bk/job/config/JobConfigBootApplication.java b/src/backend/job-config/src/main/java/com/tencent/bk/job/config/JobConfigBootApplication.java index 5c145a8df8..037b0ef71c 100644 --- a/src/backend/job-config/src/main/java/com/tencent/bk/job/config/JobConfigBootApplication.java +++ b/src/backend/job-config/src/main/java/com/tencent/bk/job/config/JobConfigBootApplication.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,12 +26,11 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.availability.ApplicationAvailabilityAutoConfiguration; import org.springframework.cloud.config.server.EnableConfigServer; -/** - * @date 2019/09/20 - */ -@SpringBootApplication(scanBasePackages = "com.tencent.bk.job") +@SpringBootApplication(scanBasePackages = "com.tencent.bk.job.config", + exclude = {ApplicationAvailabilityAutoConfiguration.class}) @EnableConfigServer public class JobConfigBootApplication { diff --git a/src/backend/job-config/src/main/java/com/tencent/bk/job/config/config/WebSecurityConfiguration.java b/src/backend/job-config/src/main/java/com/tencent/bk/job/config/config/WebSecurityConfiguration.java index 5c90aece5f..20a7412465 100644 --- a/src/backend/job-config/src/main/java/com/tencent/bk/job/config/config/WebSecurityConfiguration.java +++ b/src/backend/job-config/src/main/java/com/tencent/bk/job/config/config/WebSecurityConfiguration.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-config/src/main/resources/application.yml b/src/backend/job-config/src/main/resources/application.yml index 203b841b1d..20fa12a788 100644 --- a/src/backend/job-config/src/main/resources/application.yml +++ b/src/backend/job-config/src/main/resources/application.yml @@ -3,16 +3,23 @@ spring: name: job-config profiles: active: prod,native + mvc: + pathmatch: + matching-strategy: ant_path_matcher cloud: config: server: native: - search-locations: file:///${BK_JOB_CONFIG_DIR}/job-common,file:///${BK_JOB_CONFIG_DIR}/job-gateway,file:///${BK_JOB_CONFIG_DIR}/job-manage,file:///${BK_JOB_CONFIG_DIR}/job-execute,file:///${BK_JOB_CONFIG_DIR}/job-logsvr,file:///${BK_JOB_CONFIG_DIR}/job-crontab,file:///${BK_JOB_CONFIG_DIR}/job-backup,file:///${BK_JOB_CONFIG_DIR}/job-file-gateway,file:///${BK_JOB_CONFIG_DIR}/job-ticket,file:///${BK_JOB_CONFIG_DIR}/job-analysis + search-locations: file:///${BK_JOB_CONFIG_DIR}/job-common,file:///${BK_JOB_CONFIG_DIR}/job-gateway,file:///${BK_JOB_CONFIG_DIR}/job-manage,file:///${BK_JOB_CONFIG_DIR}/job-execute,file:///${BK_JOB_CONFIG_DIR}/job-logsvr,file:///${BK_JOB_CONFIG_DIR}/job-crontab,file:///${BK_JOB_CONFIG_DIR}/job-backup,file:///${BK_JOB_CONFIG_DIR}/job-file-gateway,file:///${BK_JOB_CONFIG_DIR}/job-ticket,file:///${BK_JOB_CONFIG_DIR}/job-analysis,file:///${BK_JOB_CONFIG_DIR}/job-assemble management: + health: + # 使用spring cloud stream rabbitmq, 禁用掉springboot的rabbitmq health检查 + rabbit: + enabled: false endpoints: web: exposure: - include: health,configprops,env,beans,conditions,loggers,metrics,mappings,prometheus,scheduledtasks,info + include: health,configprops,env,beans,conditions,loggers,metrics,mappings,prometheus,scheduledtasks,info,refresh,busrefresh,bindings base-path: /actuator enabled-by-default: false endpoint: @@ -41,6 +48,10 @@ management: enabled: true info: enabled: true + refresh: + enabled: true + busrefresh: + enabled: true server: port: 19801 shutdown: graceful diff --git a/src/backend/job-config/src/main/resources/logback-spring.xml b/src/backend/job-config/src/main/resources/logback-spring.xml new file mode 100644 index 0000000000..369b4fa6b8 --- /dev/null +++ b/src/backend/job-config/src/main/resources/logback-spring.xml @@ -0,0 +1,93 @@ + + + logback + + + + + + + + + + + + + + + + + ${LOG_KEEP_HOURS} + + + + + + ${APP_LOG_FILE} + + ${APP_LOG_FILE}-%d{yyyyMMdd_HH}.log.%i + 1GB + ${LOG_KEEP_HOURS} + 100GB + ${CLEAN_HISTORY_ON_START} + + + ${FILE_LOG_PATTERN} + UTF-8 + + + + ${ERROR_LOG_FILE} + + ${ERROR_LOG_FILE}-%d{yyyy-MM-dd}.log.%i + 1GB + ${LOG_KEEP_DAYS} + 20GB + ${CLEAN_HISTORY_ON_START} + + + ${FILE_LOG_PATTERN} + UTF-8 + + + ERROR + ACCEPT + DENY + + + + + + + ${CONSOLE_LOG_PATTERN} + utf8 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/backend/job-config/src/main/resources/logback/logback-app-props.xml b/src/backend/job-config/src/main/resources/logback/logback-app-props.xml deleted file mode 100644 index 69bd3ec4b5..0000000000 --- a/src/backend/job-config/src/main/resources/logback/logback-app-props.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - diff --git a/src/backend/job-crontab/api-job-crontab/build.gradle b/src/backend/job-crontab/api-job-crontab/build.gradle index 5f932fffe3..c53792a3b9 100644 --- a/src/backend/job-crontab/api-job-crontab/build.gradle +++ b/src/backend/job-crontab/api-job-crontab/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/esb/EsbCronJobResource.java b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/esb/EsbCronJobResource.java index d1d6240850..ec8d6226b2 100644 --- a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/esb/EsbCronJobResource.java +++ b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/esb/EsbCronJobResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,6 +25,7 @@ package com.tencent.bk.job.crontab.api.esb; import com.tencent.bk.job.common.annotation.EsbAPI; +import com.tencent.bk.job.common.constant.JobCommonHeaders; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.crontab.model.esb.request.EsbGetCronListRequest; import com.tencent.bk.job.crontab.model.esb.request.EsbSaveCronRequest; @@ -33,6 +34,7 @@ import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @@ -48,6 +50,8 @@ public interface EsbCronJobResource { @PostMapping("/get_cron_list") EsbResp> getCronList( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbGetCronListRequest request @@ -55,6 +59,8 @@ EsbResp> getCronList( @PostMapping(value = "/update_cron_status") EsbResp updateCronStatus( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbUpdateCronStatusRequest request @@ -62,6 +68,8 @@ EsbResp updateCronStatus( @PostMapping(value = "/save_cron") EsbResp saveCron( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbSaveCronRequest request diff --git a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/esb/v3/EsbCronJobV3Resource.java b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/esb/v3/EsbCronJobV3Resource.java index bdeb3d90bd..b34113640f 100644 --- a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/esb/v3/EsbCronJobV3Resource.java +++ b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/esb/v3/EsbCronJobV3Resource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,6 +28,7 @@ import com.tencent.bk.job.common.constant.JobCommonHeaders; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.common.esb.model.job.v3.EsbPageDataV3; +import com.tencent.bk.job.crontab.model.esb.v3.request.EsbDeleteCronV3Request; import com.tencent.bk.job.crontab.model.esb.v3.request.EsbGetCronDetailV3Request; import com.tencent.bk.job.crontab.model.esb.v3.request.EsbGetCronListV3Request; import com.tencent.bk.job.crontab.model.esb.v3.request.EsbSaveCronV3Request; @@ -85,6 +86,8 @@ EsbResp getCronDetail(@RequestHeader(value = JobCommonHeaders. */ @PostMapping("/get_cron_list") EsbResp> getCronListUsingPost( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbGetCronListV3Request request @@ -98,6 +101,8 @@ EsbResp> getCronListUsingPost( */ @PostMapping("/get_cron_detail") EsbResp getCronDetailUsingPost( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbGetCronDetailV3Request request @@ -111,6 +116,8 @@ EsbResp getCronDetailUsingPost( */ @PostMapping(value = "/update_cron_status") EsbResp updateCronStatus( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbUpdateCronStatusV3Request request @@ -124,8 +131,22 @@ EsbResp updateCronStatus( */ @PostMapping(value = "/save_cron") EsbResp saveCron( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbSaveCronV3Request request ); + + /** + * 删除定时任务 + */ + @PostMapping(value = "/delete_cron") + EsbResp deleteCron( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, + @RequestBody + @Validated + EsbDeleteCronV3Request request + ); } diff --git a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/iam/IamCronCallbackResource.java b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/iam/IamCronCallbackResource.java index 010f062f73..3a2f0cd58c 100644 --- a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/iam/IamCronCallbackResource.java +++ b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/iam/IamCronCallbackResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/inner/ServiceCronJobResource.java b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/inner/ServiceCronJobResource.java index 597094a53e..299a6d4ee6 100644 --- a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/inner/ServiceCronJobResource.java +++ b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/inner/ServiceCronJobResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -30,6 +30,7 @@ import com.tencent.bk.job.crontab.model.CronJobVO; import com.tencent.bk.job.crontab.model.inner.ServiceCronJobDTO; import com.tencent.bk.job.crontab.model.inner.request.InternalUpdateCronStatusRequest; +import com.tentent.bk.job.common.api.feign.annotation.SmartFeignClient; import io.swagger.annotations.Api; import io.swagger.annotations.ApiParam; import org.springframework.web.bind.annotation.GetMapping; @@ -38,19 +39,16 @@ import org.springframework.web.bind.annotation.PutMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestHeader; -import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; import java.util.List; import java.util.Map; /** - * @since 20/2/2020 19:54 + * 定时任务 Resource API */ @Api(tags = {"Cron_Job"}) -@RequestMapping("/service/app/{appId}/cron/job") -@RestController +@SmartFeignClient(value = "job-crontab", contextId = "cronJobResource") @InternalAPI public interface ServiceCronJobResource { @@ -61,7 +59,7 @@ public interface ServiceCronJobResource { * @param enable 定时任务状态 * @return 定时任务列表 */ - @GetMapping("/") + @GetMapping("/service/app/{appId}/cron/job") InternalResponse> listCronJobs( @ApiParam(value = "业务 ID", required = true, example = "2") @PathVariable("appId") Long appId, @ApiParam(value = "是否开启", required = false, example = "true") @RequestParam("enable") Boolean enable @@ -76,7 +74,7 @@ InternalResponse> listCronJobs( * @param cronJobCreateUpdateReq 定时任务新建、更新请求 * @return 定时任务 ID */ - @PutMapping("/{cronJobId}") + @PutMapping("/service/app/{appId}/cron/job/{cronJobId}") InternalResponse saveCronJob( @ApiParam(value = "用户名，网关自动传入") @RequestHeader("username") String username, @ApiParam(value = "业务 ID", required = true, example = "2") @PathVariable("appId") Long appId, @@ -90,10 +88,10 @@ InternalResponse saveCronJob( * * @param appId 业务 ID * @param cronJobId 定时任务 ID - * @param status 定时任务状态 + * @param request 更新请求 * @return 是否更新成功 */ - @PostMapping("/{cronJobId}/status") + @PostMapping("/service/app/{appId}/cron/job/{cronJobId}/status") InternalResponse updateCronJobStatus( @ApiParam(value = "业务 ID", required = true, example = "2") @PathVariable("appId") @@ -113,7 +111,7 @@ InternalResponse updateCronJobStatus( * @param planIdList 执行方案 ID 列表 * @return 执行方案与定时任务列表对应表 */ - @GetMapping("/plan") + @GetMapping("/service/app/{appId}/cron/job/plan") InternalResponse>> batchListCronJobByPlanIds( @ApiParam(value = "业务 ID", required = true, example = "2") @PathVariable("appId") Long appId, @ApiParam(value = "执行方案 ID 列表", required = true) @RequestParam(value = "planId") List planIdList @@ -131,7 +129,7 @@ InternalResponse>> batchListCronJobByPlanIds( * @param cronJobCreateUpdateReq 定时任务创建请求 * @return 定时任务 ID */ - @PutMapping("/{cronJobId}/saveCronJobWithId") + @PutMapping("/service/app/{appId}/cron/job/{cronJobId}/saveCronJobWithId") InternalResponse saveCronJobWithId( @ApiParam(value = "用户名，网关自动传入") @RequestHeader("username") String username, @ApiParam(value = "业务 ID", required = true, example = "2") @PathVariable("appId") Long appId, diff --git a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/inner/ServiceCronMetricsResource.java b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/inner/ServiceCronMetricsResource.java index 28a1b1ab93..c64fc14845 100644 --- a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/inner/ServiceCronMetricsResource.java +++ b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/inner/ServiceCronMetricsResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,22 +26,20 @@ import com.tencent.bk.job.common.annotation.EsbAPI; import com.tencent.bk.job.common.model.InternalResponse; +import com.tentent.bk.job.common.api.feign.annotation.SmartFeignClient; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; @Api(tags = {"job-crontab:Service:Metrics"}) -@RequestMapping("/service/metrics") -@RestController +@SmartFeignClient(value = "job-crontab", contextId = "cronMetricResource") @EsbAPI public interface ServiceCronMetricsResource { @ApiOperation(value = "定时任务总量", produces = "application/json") - @GetMapping("/count") + @GetMapping("/service/metrics/count") InternalResponse countCronJob( @ApiParam(value = "业务Id") @RequestParam(value = "appId", required = false) diff --git a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/inner/ServiceInnerCronJobResource.java b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/inner/ServiceInnerCronJobResource.java index 5c63c98c58..c4606b3aad 100644 --- a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/inner/ServiceInnerCronJobResource.java +++ b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/inner/ServiceInnerCronJobResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,14 +28,13 @@ import com.tencent.bk.job.common.model.InternalResponse; import com.tencent.bk.job.crontab.model.inner.ServiceInnerCronJobInfoDTO; import com.tencent.bk.job.crontab.model.inner.request.ServiceAddInnerCronJobRequestDTO; +import com.tentent.bk.job.common.api.feign.annotation.SmartFeignClient; import io.swagger.annotations.Api; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PutMapping; import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; import java.util.List; @@ -43,8 +42,7 @@ * @since 18/2/2020 15:11 */ @Api(tags = {"Inner_Cron_Job"}) -@RequestMapping("/service/inner/cron/job") -@RestController +@SmartFeignClient(value = "job-crontab", contextId = "innerCronJobResource") @EsbAPI public interface ServiceInnerCronJobResource { @@ -56,7 +54,7 @@ public interface ServiceInnerCronJobResource { * @param request 作业详情 * @return 是否创建成功 */ - @PutMapping("/{systemId}/{jobKey}") + @PutMapping("/service/inner/cron/job/{systemId}/{jobKey}") InternalResponse addNewCronJob( @PathVariable("systemId") String systemId, @PathVariable("jobKey") String jobKey, @RequestBody ServiceAddInnerCronJobRequestDTO request @@ -69,7 +67,7 @@ InternalResponse addNewCronJob( * @param jobKey 任务 Key * @return 定时任务详情 */ - @GetMapping("/{systemId}/{jobKey}") + @GetMapping("/service/inner/cron/job/{systemId}/{jobKey}") InternalResponse getCronJobInfoByKey( @PathVariable("systemId") String systemId, @PathVariable("jobKey") String jobKey @@ -82,7 +80,7 @@ InternalResponse getCronJobInfoByKey( * @param jobKey 任务 Key * @return 删除是否成功 */ - @DeleteMapping("/{systemId}/{jobKey}") + @DeleteMapping("/service/inner/cron/job/{systemId}/{jobKey}") InternalResponse deleteCronJob( @PathVariable("systemId") String systemId, @PathVariable("jobKey") String jobKey @@ -94,7 +92,7 @@ InternalResponse deleteCronJob( * @param systemId 系统 ID * @return 定时任务列表 */ - @GetMapping("/{systemId}") + @GetMapping("/service/inner/cron/job/{systemId}") InternalResponse> listCronJobs(@PathVariable("systemId") String systemId); } diff --git a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/web/WebCronJobResource.java b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/web/WebCronJobResource.java index 13ade2a1c8..fbd940c18a 100644 --- a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/web/WebCronJobResource.java +++ b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/web/WebCronJobResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -174,10 +174,41 @@ Response getCronJobById( @ApiParam(value = "资源范围ID", required = true) @PathVariable(value = "scopeId") String scopeId, - @ApiParam(value = "定时任务 ID", required = true) @PathVariable(value = "cronJobId") + @ApiParam(value = "定时任务 ID", required = true) + @PathVariable(value = "cronJobId") Long cronJobId ); + /** + * 更新定时任务 + * + * @param username 用户名 + * @param appResourceScope 资源范围；Spring拦截器注入 + * @param scopeType 资源范围类型 + * @param scopeId 资源范围ID + * @param cronJobCreateUpdateReq 定时任务信息 + * @return 定时任务 + */ + @ApiOperation(value = "创建定时任务", produces = "application/json") + @PostMapping + Response createCronJob( + @ApiParam(value = "用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId, + @ApiParam(value = "创建的定时任务对象", name = "cronJobCreateUpdateReq", required = true) + @RequestBody + CronJobCreateUpdateReq cronJobCreateUpdateReq + ); + /** * 更新定时任务 * @@ -187,11 +218,11 @@ Response getCronJobById( * @param scopeId 资源范围ID * @param cronJobId 定时任务 ID * @param cronJobCreateUpdateReq 定时任务信息 - * @return 定时任务 ID + * @return 定时任务 */ @ApiOperation(value = "更新定时任务", produces = "application/json") @PutMapping("/{cronJobId}") - Response saveCronJob( + Response updateCronJob( @ApiParam(value = "用户名，网关自动传入") @RequestHeader("username") String username, diff --git a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/web/WebPermissionResource.java b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/web/WebPermissionResource.java index c574c5dd9b..bf78dac5cb 100644 --- a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/web/WebPermissionResource.java +++ b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/web/WebPermissionResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -40,7 +40,7 @@ import org.springframework.web.bind.annotation.RestController; @Api(tags = {"job-crontab:web:Permission"}) -@RequestMapping("/web/permission/") +@RequestMapping("/web/permission/crontab") @RestController @WebAPI public interface WebPermissionResource { diff --git a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/common/constants/CronStatusEnum.java b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/common/constants/CronStatusEnum.java index 675b70c39f..2bde7bd663 100644 --- a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/common/constants/CronStatusEnum.java +++ b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/common/constants/CronStatusEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/BatchUpdateCronJobReq.java b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/BatchUpdateCronJobReq.java index 2f1a940a96..7ef2bca7d3 100644 --- a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/BatchUpdateCronJobReq.java +++ b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/BatchUpdateCronJobReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/CronJobCreateUpdateReq.java b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/CronJobCreateUpdateReq.java index c55f3a4047..ed2918f27b 100644 --- a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/CronJobCreateUpdateReq.java +++ b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/CronJobCreateUpdateReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,7 @@ package com.tencent.bk.job.crontab.model; +import com.tencent.bk.job.common.model.dto.notify.StatusNotifyChannel; import com.tencent.bk.job.common.model.vo.UserRoleInfoVO; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; @@ -43,10 +44,8 @@ public class CronJobCreateUpdateReq { /** * 定时任务 ID - *

- * 创建时填 0 */ - @ApiModelProperty(value = "任务 ID，创建填 0", required = true) + @ApiModelProperty(value = "定时任务 ID", hidden = true) private Long id; /** @@ -138,6 +137,27 @@ public class CronJobCreateUpdateReq { @ApiModelProperty("通知渠道，若 notifyOffset > 0，不可为空") private List notifyChannel = Collections.emptyList(); + /** + * 通知方式（1-继承业务, 2-自定义） + * @see com.tencent.bk.job.common.constant.CronJobNotifyType + */ + @ApiModelProperty("通知方式（1:继承业务/2:自定义），默认继承业务") + private Integer notifyType = 1; + + /** + * 自定义通知，当notifyType为CUSTOM时生效 + * 通知对象 + */ + @ApiModelProperty("自定义通知，当notifyType为CUSTOM时生效，通知对象") + private UserRoleInfoVO customNotifyUser; + + /** + * 自定义通知，当notifyType为CUSTOM时生效 + * 执行状态与对应通知渠道列表 + */ + @ApiModelProperty("自定义通知，当notifyType为CUSTOM时生效，执行状态与对应通知渠道列表") + private List customNotifyChannel = Collections.emptyList(); + /** * 周期执行的结束时间 */ diff --git a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/CronJobLaunchHistoryVO.java b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/CronJobLaunchHistoryVO.java index e6a00de6c1..036f2282ca 100644 --- a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/CronJobLaunchHistoryVO.java +++ b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/CronJobLaunchHistoryVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/CronJobVO.java b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/CronJobVO.java index e0e24f85ad..5f68865c11 100644 --- a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/CronJobVO.java +++ b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/CronJobVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,6 +25,7 @@ package com.tencent.bk.job.crontab.model; import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.tencent.bk.job.common.model.dto.notify.CustomNotifyVO; import com.tencent.bk.job.common.model.vo.UserRoleInfoVO; import com.tencent.bk.job.common.util.json.LongTimestampSerializer; import io.swagger.annotations.ApiModel; @@ -194,6 +195,16 @@ public class CronJobVO { @ApiModelProperty("通知渠道") private List notifyChannel; + /** + * 通知方式（1-继承业务, 2-自定义） + * @see com.tencent.bk.job.common.constant.CronJobNotifyType + */ + @ApiModelProperty("通知方式（1-继承业务, 2-自定义）") + private Integer notifyType; + + @ApiModelProperty("自定义通知配置") + private CustomNotifyVO cronJobCustomNotifyVO; + /** * 周期执行的结束时间 */ diff --git a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/CronJobVariableVO.java b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/CronJobVariableVO.java index dde21b79c8..f59032b035 100644 --- a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/CronJobVariableVO.java +++ b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/CronJobVariableVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/OperationPermissionReq.java b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/OperationPermissionReq.java index 72d90dca6b..e289190daa 100644 --- a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/OperationPermissionReq.java +++ b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/OperationPermissionReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/esb/request/EsbGetCronListRequest.java b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/esb/request/EsbGetCronListRequest.java index b64646d708..21fc1c875b 100644 --- a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/esb/request/EsbGetCronListRequest.java +++ b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/esb/request/EsbGetCronListRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/esb/request/EsbSaveCronRequest.java b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/esb/request/EsbSaveCronRequest.java index 8db79e8fee..d9d35f7f4f 100644 --- a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/esb/request/EsbSaveCronRequest.java +++ b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/esb/request/EsbSaveCronRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/esb/request/EsbUpdateCronStatusRequest.java b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/esb/request/EsbUpdateCronStatusRequest.java index fc6c1dec96..4eb0b8df56 100644 --- a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/esb/request/EsbUpdateCronStatusRequest.java +++ b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/esb/request/EsbUpdateCronStatusRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/esb/response/EsbCronInfoResponse.java b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/esb/response/EsbCronInfoResponse.java index 75661161e2..123bac8669 100644 --- a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/esb/response/EsbCronInfoResponse.java +++ b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/esb/response/EsbCronInfoResponse.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,11 +27,13 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.tencent.bk.job.common.esb.model.EsbAppScopeDTO; import lombok.Data; +import lombok.EqualsAndHashCode; /** * @since 26/2/2020 20:58 */ @Data +@EqualsAndHashCode(callSuper = true) public class EsbCronInfoResponse extends EsbAppScopeDTO { /** diff --git a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/esb/v3/request/EsbDeleteCronV3Request.java b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/esb/v3/request/EsbDeleteCronV3Request.java new file mode 100644 index 0000000000..67816e7a7e --- /dev/null +++ b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/esb/v3/request/EsbDeleteCronV3Request.java @@ -0,0 +1,46 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.crontab.model.esb.v3.request; + +import com.tencent.bk.job.common.esb.model.EsbAppScopeReq; +import io.swagger.annotations.ApiModel; +import lombok.Data; +import lombok.EqualsAndHashCode; + +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; + +@Data +@EqualsAndHashCode(callSuper = true) +@ApiModel("删除定时任务请求报文") +public class EsbDeleteCronV3Request extends EsbAppScopeReq { + + /** + * 定时作业 ID + */ + @NotNull(message = "{validation.constraints.CronId_null.message}") + @Min(value = 1L, message = "{validation.constraints.InvalidCronId.message}") + private Long id; +} diff --git a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/esb/v3/request/EsbGetCronDetailV3Request.java b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/esb/v3/request/EsbGetCronDetailV3Request.java index 1c91e3ee15..fba3cc8598 100644 --- a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/esb/v3/request/EsbGetCronDetailV3Request.java +++ b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/esb/v3/request/EsbGetCronDetailV3Request.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/esb/v3/request/EsbGetCronListV3Request.java b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/esb/v3/request/EsbGetCronListV3Request.java index 7a7328fe08..af3f6dd1b7 100644 --- a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/esb/v3/request/EsbGetCronListV3Request.java +++ b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/esb/v3/request/EsbGetCronListV3Request.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/esb/v3/request/EsbSaveCronV3Request.java b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/esb/v3/request/EsbSaveCronV3Request.java index 2ab2d6d55d..a6f42d9bbe 100644 --- a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/esb/v3/request/EsbSaveCronV3Request.java +++ b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/esb/v3/request/EsbSaveCronV3Request.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/esb/v3/request/EsbUpdateCronStatusV3Request.java b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/esb/v3/request/EsbUpdateCronStatusV3Request.java index 3f7f05cc57..d917353c9e 100644 --- a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/esb/v3/request/EsbUpdateCronStatusV3Request.java +++ b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/esb/v3/request/EsbUpdateCronStatusV3Request.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/esb/v3/response/EsbCronInfoV3DTO.java b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/esb/v3/response/EsbCronInfoV3DTO.java index 6bce2dfc85..17202de319 100644 --- a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/esb/v3/response/EsbCronInfoV3DTO.java +++ b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/esb/v3/response/EsbCronInfoV3DTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,6 +25,7 @@ package com.tencent.bk.job.crontab.model.esb.v3.response; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; import com.tencent.bk.job.common.esb.model.EsbAppScopeDTO; import com.tencent.bk.job.common.esb.model.job.v3.EsbGlobalVarV3DTO; import lombok.Getter; @@ -45,56 +46,66 @@ public class EsbCronInfoV3DTO extends EsbAppScopeDTO { * 作业模板 ID */ @JsonProperty("job_plan_id") + @JsonPropertyDescription("Job plan id") private Long planId; /** * 定时作业ID */ + @JsonPropertyDescription("Cron job id") private Long id; /** * 定时作业名称 */ + @JsonPropertyDescription("Cron job name") private String name; /** * 定时作业状态：1.已启动、2.已暂停 */ + @JsonPropertyDescription("Cron job status, 1 - Started, 2 - Stopped") private Integer status; /** * 定时任务的 cron 表达式 */ @JsonProperty("expression") + @JsonPropertyDescription("Cron job expression") private String cronExpression; /** * 定时任务的变量信息 */ @JsonProperty("global_var_list") + @JsonPropertyDescription("Global variables") private List globalVarList; /** * 作业创建人帐号 */ + @JsonPropertyDescription("Creator") private String creator; /** * 创建时间，毫秒时间戳 */ @JsonProperty("create_time") + @JsonPropertyDescription("Create time") private Long createTime; /** * 作业修改人帐号 */ @JsonProperty("last_modify_user") + @JsonPropertyDescription("Last modify user") private String lastModifyUser; /** * 最后修改时间，毫秒时间戳 */ @JsonProperty("last_modify_time") + @JsonPropertyDescription("Last modify time") private Long lastModifyTime; } diff --git a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/inner/ServerDTO.java b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/inner/ServerDTO.java index 532e91ad39..87bd2a8c8e 100644 --- a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/inner/ServerDTO.java +++ b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/inner/ServerDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,12 +24,17 @@ package com.tencent.bk.job.crontab.model.inner; -import com.tencent.bk.job.common.esb.model.job.EsbCmdbTopoNodeDTO; +import com.tencent.bk.job.common.annotation.PersistenceObject; import com.tencent.bk.job.common.esb.model.job.EsbIpDTO; -import com.tencent.bk.job.common.esb.model.job.v3.EsbDynamicGroupDTO; import com.tencent.bk.job.common.esb.model.job.v3.EsbServerV3DTO; import com.tencent.bk.job.common.model.dto.CmdbTopoNodeDTO; import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.common.model.openapi.v3.EsbCmdbTopoNodeDTO; +import com.tencent.bk.job.common.model.openapi.v3.EsbDynamicGroupDTO; +import com.tencent.bk.job.common.model.vo.DynamicGroupIdWithMeta; +import com.tencent.bk.job.common.model.vo.HostInfoVO; +import com.tencent.bk.job.common.model.vo.TargetNodeVO; +import com.tencent.bk.job.common.model.vo.TaskExecuteObjectsInfoVO; import com.tencent.bk.job.common.model.vo.TaskHostNodeVO; import com.tencent.bk.job.common.model.vo.TaskTargetVO; import com.tencent.bk.job.execute.model.inner.ServiceTargetServers; @@ -43,9 +48,10 @@ import java.util.List; import java.util.stream.Collectors; +@PersistenceObject @ApiModel("目标服务器，四个不可同时为空") @Data -public class ServerDTO { +public class ServerDTO implements Cloneable { /** * 全局变量名 *

@@ -78,18 +84,31 @@ public static TaskTargetVO toTargetVO(ServerDTO server) { } TaskTargetVO taskTarget = new TaskTargetVO(); taskTarget.setVariable(server.getVariable()); + TaskExecuteObjectsInfoVO taskExecuteObjectsInfoVO = new TaskExecuteObjectsInfoVO(); TaskHostNodeVO taskHostNode = new TaskHostNodeVO(); + taskTarget.setExecuteObjectsInfo(taskExecuteObjectsInfoVO); + taskTarget.setHostNodeInfo(taskHostNode); + + // 聚合通过hostId与IP指定的主机信息 + List hostInfoVOList = new ArrayList<>(); if (CollectionUtils.isNotEmpty(server.getIps())) { - taskHostNode.setIpList(server.getIps().parallelStream().map(HostDTO::toVO).collect(Collectors.toList())); + hostInfoVOList.addAll(server.getIps().stream().map(HostDTO::toHostInfoVO).collect(Collectors.toList())); + } + if (!hostInfoVOList.isEmpty()) { + taskExecuteObjectsInfoVO.setHostList(hostInfoVOList); + taskHostNode.setHostList(hostInfoVOList); } if (CollectionUtils.isNotEmpty(server.getDynamicGroupIds())) { - taskHostNode.setDynamicGroupList(server.getDynamicGroupIds()); + taskExecuteObjectsInfoVO.setDynamicGroupList( + server.getDynamicGroupIds().stream().map(DynamicGroupIdWithMeta::new).collect(Collectors.toList())); + taskHostNode.setDynamicGroupIdList(server.getDynamicGroupIds()); } if (CollectionUtils.isNotEmpty(server.getTopoNodes())) { - taskHostNode.setTopoNodeList(server.getTopoNodes().parallelStream() - .map(CmdbTopoNodeDTO::toVO).collect(Collectors.toList())); + List nodeList = server.getTopoNodes().stream() + .map(CmdbTopoNodeDTO::toVO).collect(Collectors.toList()); + taskExecuteObjectsInfoVO.setNodeList(nodeList); + taskHostNode.setNodeList(nodeList); } - taskTarget.setHostNodeInfo(taskHostNode); return taskTarget; } @@ -99,17 +118,18 @@ public static ServerDTO fromTargetVO(TaskTargetVO taskTarget) { } ServerDTO server = new ServerDTO(); server.setVariable(taskTarget.getVariable()); - if (taskTarget.getHostNodeInfo() != null) { - TaskHostNodeVO hostNodeInfo = taskTarget.getHostNodeInfo(); - if (CollectionUtils.isNotEmpty(hostNodeInfo.getIpList())) { - server.setIps(hostNodeInfo.getIpList().parallelStream() - .map(HostDTO::fromVO).collect(Collectors.toList())); + if (taskTarget.getExecuteObjectsInfoCompatibly() != null) { + TaskExecuteObjectsInfoVO taskExecuteObjectsInfoVO = taskTarget.getExecuteObjectsInfoCompatibly(); + if (CollectionUtils.isNotEmpty(taskExecuteObjectsInfoVO.getHostList())) { + server.setIps(taskExecuteObjectsInfoVO.getHostList().stream() + .map(HostDTO::fromHostInfoVO).collect(Collectors.toList())); } - if (CollectionUtils.isNotEmpty(hostNodeInfo.getDynamicGroupList())) { - server.setDynamicGroupIds(hostNodeInfo.getDynamicGroupList()); + if (CollectionUtils.isNotEmpty(taskExecuteObjectsInfoVO.getDynamicGroupList())) { + server.setDynamicGroupIds(taskExecuteObjectsInfoVO.getDynamicGroupList().stream() + .map(DynamicGroupIdWithMeta::getId).collect(Collectors.toList())); } - if (CollectionUtils.isNotEmpty(hostNodeInfo.getTopoNodeList())) { - server.setTopoNodes(hostNodeInfo.getTopoNodeList().parallelStream() + if (CollectionUtils.isNotEmpty(taskExecuteObjectsInfoVO.getNodeList())) { + server.setTopoNodes(taskExecuteObjectsInfoVO.getNodeList().stream() .map(CmdbTopoNodeDTO::fromVO).collect(Collectors.toList())); } } @@ -132,11 +152,18 @@ public static ServerDTO fromEsbServerV3(EsbServerV3DTO server) { server.getDynamicGroups().forEach(group -> dynamicGroupIds.add(group.getId())); serverDTO.setDynamicGroupIds(dynamicGroupIds); } + List hosts = new ArrayList<>(); if (CollectionUtils.isNotEmpty(server.getIps())) { List staticIpList = new ArrayList<>(); server.getIps().forEach(ip -> staticIpList.add(new HostDTO(ip.getBkCloudId(), ip.getIp()))); - serverDTO.setIps(staticIpList); + hosts.addAll(staticIpList); } + if (CollectionUtils.isNotEmpty(server.getHostIds())) { + List hostIdHostList = new ArrayList<>(); + server.getHostIds().forEach(hostId -> hostIdHostList.add(HostDTO.fromHostId(hostId))); + hosts.addAll(hostIdHostList); + } + serverDTO.setIps(hosts); return serverDTO; } @@ -147,17 +174,17 @@ public static EsbServerV3DTO toEsbServerV3(ServerDTO server) { EsbServerV3DTO esbServer = new EsbServerV3DTO(); esbServer.setVariable(server.getVariable()); if (CollectionUtils.isNotEmpty(server.getIps())) { - esbServer.setIps(server.getIps().parallelStream().map(EsbIpDTO::fromHost).collect(Collectors.toList())); + esbServer.setIps(server.getIps().stream().map(EsbIpDTO::fromHost).collect(Collectors.toList())); } if (CollectionUtils.isNotEmpty(server.getDynamicGroupIds())) { - esbServer.setDynamicGroups(server.getDynamicGroupIds().parallelStream().map(id -> { + esbServer.setDynamicGroups(server.getDynamicGroupIds().stream().map(id -> { EsbDynamicGroupDTO esbDynamicGroup = new EsbDynamicGroupDTO(); esbDynamicGroup.setId(id); return esbDynamicGroup; }).collect(Collectors.toList())); } if (CollectionUtils.isNotEmpty(server.getTopoNodes())) { - esbServer.setTopoNodes(server.getTopoNodes().parallelStream() + esbServer.setTopoNodes(server.getTopoNodes().stream() .map(EsbCmdbTopoNodeDTO::fromCmdbTopoNode).collect(Collectors.toList())); } return esbServer; @@ -169,9 +196,11 @@ public static ServiceTargetServers toServiceServer(ServerDTO server) { } ServiceTargetServers serviceServer = new ServiceTargetServers(); serviceServer.setVariable(server.getVariable()); + List hosts = new ArrayList<>(); if (CollectionUtils.isNotEmpty(server.getIps())) { - serviceServer.setIps(server.getIps()); + hosts.addAll(server.getIps()); } + serviceServer.setIps(hosts); serviceServer.setDynamicGroupIds(server.getDynamicGroupIds()); if (CollectionUtils.isNotEmpty(server.getTopoNodes())) { serviceServer.setTopoNodes(server.getTopoNodes()); @@ -197,4 +226,38 @@ public void standardizeDynamicGroupId() { this.dynamicGroupIds = standardDynamicGroupIdList; } } + + @SuppressWarnings("MethodDoesntCallSuperMethod") + @Override + public ServerDTO clone() { + ServerDTO serverDTO = new ServerDTO(); + serverDTO.setVariable(variable); + if (null != ips) { + List cloneIps = new ArrayList<>(ips.size()); + for (HostDTO ip : ips) { + if (ip != null) { + cloneIps.add(ip.clone()); + } else { + cloneIps.add(null); + } + } + serverDTO.setIps(cloneIps); + } + if (null != dynamicGroupIds) { + List cloneDynamicGroupIds = new ArrayList<>(dynamicGroupIds); + serverDTO.setDynamicGroupIds(cloneDynamicGroupIds); + } + if (null != topoNodes) { + List cloneTopoNodes = new ArrayList<>(topoNodes.size()); + for (CmdbTopoNodeDTO topoNode : topoNodes) { + if (topoNode != null) { + cloneTopoNodes.add(topoNode.clone()); + } else { + cloneTopoNodes.add(null); + } + } + serverDTO.setTopoNodes(cloneTopoNodes); + } + return serverDTO; + } } diff --git a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/inner/ServiceCronJobDTO.java b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/inner/ServiceCronJobDTO.java index a9a13f39fc..7efa57e3d7 100644 --- a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/inner/ServiceCronJobDTO.java +++ b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/inner/ServiceCronJobDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,14 +24,13 @@ package com.tencent.bk.job.crontab.model.inner; +import com.tencent.bk.job.common.model.dto.notify.CustomNotifyDTO; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -import java.util.List; - @NoArgsConstructor @AllArgsConstructor @Data @@ -65,9 +64,6 @@ public class ServiceCronJobDTO { @ApiModelProperty("单次执行的指定执行时间戳") private Long executeTime; - @ApiModelProperty("变量信息") - private List variableValue; - @ApiModelProperty("上次执行结果 0 - 未执行 1 - 成功 2 - 失败") private Integer lastExecuteStatus; @@ -79,4 +75,17 @@ public class ServiceCronJobDTO { @ApiModelProperty("最后修改时间戳") private Long lastModifyTime; + + /** + * 通知方式（1-继承业务, 2-自定义） + * @see com.tencent.bk.job.common.constant.CronJobNotifyType + */ + @ApiModelProperty("通知方式（1-继承业务, 2-自定义）") + private Integer notifyType = 1; + + /** + * 自定义通知配置 + */ + @ApiModelProperty("自定义通知配置") + private CustomNotifyDTO customCronJobNotifyDTO = new CustomNotifyDTO(); } diff --git a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/inner/ServiceCronJobVariableDTO.java b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/inner/ServiceCronJobVariableDTO.java index 858d3a0953..eae79c4098 100644 --- a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/inner/ServiceCronJobVariableDTO.java +++ b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/inner/ServiceCronJobVariableDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/inner/ServiceInnerCronJobInfoDTO.java b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/inner/ServiceInnerCronJobInfoDTO.java index ae458befcd..24d02641dc 100644 --- a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/inner/ServiceInnerCronJobInfoDTO.java +++ b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/inner/ServiceInnerCronJobInfoDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/inner/request/InternalUpdateCronStatusRequest.java b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/inner/request/InternalUpdateCronStatusRequest.java index d38d209753..119a11b84b 100644 --- a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/inner/request/InternalUpdateCronStatusRequest.java +++ b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/inner/request/InternalUpdateCronStatusRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/inner/request/ServiceAddInnerCronJobRequestDTO.java b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/inner/request/ServiceAddInnerCronJobRequestDTO.java index 8c86324f9b..ad5341b9e3 100644 --- a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/inner/request/ServiceAddInnerCronJobRequestDTO.java +++ b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/inner/request/ServiceAddInnerCronJobRequestDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/validation/provider/EsbSaveCronV3RequestSequenceProvider.java b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/validation/provider/EsbSaveCronV3RequestSequenceProvider.java index bda75c3e1e..02b412a3ea 100644 --- a/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/validation/provider/EsbSaveCronV3RequestSequenceProvider.java +++ b/src/backend/job-crontab/api-job-crontab/src/main/java/com/tencent/bk/job/crontab/validation/provider/EsbSaveCronV3RequestSequenceProvider.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/boot-job-crontab/build.gradle b/src/backend/job-crontab/boot-job-crontab/build.gradle index e7e295bd34..66bfdd6e74 100644 --- a/src/backend/job-crontab/boot-job-crontab/build.gradle +++ b/src/backend/job-crontab/boot-job-crontab/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -32,7 +32,7 @@ dependencies { implementation(group: 'org.springframework.boot', name: 'spring-boot-starter-data-redis') implementation 'org.springframework.cloud:spring-cloud-starter-bootstrap' implementation 'org.springframework:spring-webmvc' - runtimeOnly 'mysql:mysql-connector-java' + runtimeOnly 'com.mysql:mysql-connector-j' testImplementation("com.h2database:h2") } @@ -46,8 +46,7 @@ task renameArtifacts(type: Copy) { destinationDir file('build/libs/') rename "boot-job-crontab-${version}.jar", "job-crontab-${version}.jar" } -renameArtifacts.dependsOn jar -assemble.dependsOn renameArtifacts +renameArtifacts.dependsOn assemble task copyToLatestJar(type: Copy) { group = "local" @@ -58,3 +57,4 @@ task copyToLatestJar(type: Copy) { } copyToLatestJar.dependsOn assemble apply from: "$rootDir/task_job_package.gradle" +copyToRelease.dependsOn renameArtifacts diff --git a/src/backend/job-crontab/boot-job-crontab/src/main/java/com/tencent/bk/job/crontab/JobCrontabBootApplication.java b/src/backend/job-crontab/boot-job-crontab/src/main/java/com/tencent/bk/job/crontab/JobCrontabBootApplication.java index c2320cfa93..8648022405 100644 --- a/src/backend/job-crontab/boot-job-crontab/src/main/java/com/tencent/bk/job/crontab/JobCrontabBootApplication.java +++ b/src/backend/job-crontab/boot-job-crontab/src/main/java/com/tencent/bk/job/crontab/JobCrontabBootApplication.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,20 +24,18 @@ package com.tencent.bk.job.crontab; -import com.tencent.bk.job.common.config.FeatureToggleConfig; -import com.tencent.bk.job.crontab.config.QuartzProperties; +import com.tencent.bk.job.common.service.boot.JobBootApplication; import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration; -import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.boot.autoconfigure.availability.ApplicationAvailabilityAutoConfiguration; +import org.springframework.boot.autoconfigure.jooq.JooqAutoConfiguration; import org.springframework.cloud.openfeign.EnableFeignClients; -/** - * @date 2019/09/20 - */ -@SpringBootApplication(scanBasePackages = "com.tencent.bk.job", exclude = {RedisAutoConfiguration.class}) -@EnableFeignClients -@EnableConfigurationProperties({FeatureToggleConfig.class, QuartzProperties.class}) +@JobBootApplication( + scanBasePackages = { + "com.tencent.bk.job.crontab"}, + exclude = {JooqAutoConfiguration.class, ApplicationAvailabilityAutoConfiguration.class}, + excludeName = {"org.springframework.cloud.kubernetes.client.discovery.KubernetesDiscoveryClientAutoConfiguration"}) +@EnableFeignClients(basePackages = {"com.tencent.bk.job.manage.api", "com.tencent.bk.job.execute.api"}) public class JobCrontabBootApplication { public static void main(String[] args) { diff --git a/src/backend/job-crontab/boot-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/DbConfig.java b/src/backend/job-crontab/boot-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/DbConfig.java deleted file mode 100644 index 7b5a9ce227..0000000000 --- a/src/backend/job-crontab/boot-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/DbConfig.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.crontab.config; - -import org.jooq.ConnectionProvider; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; -import org.jooq.impl.DataSourceConnectionProvider; -import org.jooq.impl.DefaultConfiguration; -import org.jooq.impl.DefaultDSLContext; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.boot.jdbc.DataSourceBuilder; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.DependsOn; -import org.springframework.context.annotation.Primary; -import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.jdbc.datasource.DataSourceTransactionManager; -import org.springframework.jdbc.datasource.TransactionAwareDataSourceProxy; -import org.springframework.transaction.annotation.EnableTransactionManagement; - -import javax.sql.DataSource; - -/** - * @date 2019/09/19 - */ -@Configuration -@EnableTransactionManagement -public class DbConfig { - @Qualifier("job-crontab-data-source") - @Primary - @Bean(name = "job-crontab-data-source") - @ConfigurationProperties(prefix = "spring.datasource.job-crontab") - public DataSource dataSource() { - return DataSourceBuilder.create().build(); - } - - @Qualifier("transactionManager") - @Bean(name = "transactionManager") - @DependsOn("job-crontab-data-source") - @Primary - public DataSourceTransactionManager - transactionManager(@Qualifier("job-crontab-data-source") DataSource dataSource) { - return new DataSourceTransactionManager(dataSource); - } - - @Qualifier("job-crontab-jdbc-template") - @Bean(name = "job-crontab-jdbc-template") - public JdbcTemplate jdbcTemplate(@Qualifier("job-crontab-data-source") DataSource dataSource) { - return new JdbcTemplate(dataSource); - } - - @Qualifier("job-crontab-dsl-context") - @Bean(name = "job-crontab-dsl-context") - public DSLContext dslContext(@Qualifier("job-crontab-jooq-conf") org.jooq.Configuration configuration) { - return new DefaultDSLContext(configuration); - } - - @Qualifier("job-crontab-jooq-conf") - @Bean(name = "job-crontab-jooq-conf") - public org.jooq.Configuration - jooqConf(@Qualifier("job-crontab-conn-provider") ConnectionProvider connectionProvider) { - return new DefaultConfiguration().derive(connectionProvider).derive(SQLDialect.MYSQL); - } - - @Qualifier("job-crontab-conn-provider") - @Bean(name = "job-crontab-conn-provider") - public ConnectionProvider connectionProvider( - @Qualifier("transactionAwareDataSource") TransactionAwareDataSourceProxy transactionAwareDataSource) { - return new DataSourceConnectionProvider(transactionAwareDataSource); - } - - @Qualifier("transactionAwareDataSource") - @Bean(name = "transactionAwareDataSource") - public TransactionAwareDataSourceProxy - transactionAwareDataSourceProxy(@Qualifier("job-crontab-data-source") DataSource dataSource) { - return new TransactionAwareDataSourceProxy(dataSource); - } - -} diff --git a/src/backend/job-crontab/boot-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/FilterConfig.java b/src/backend/job-crontab/boot-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/FilterConfig.java deleted file mode 100644 index 3dcd1283fd..0000000000 --- a/src/backend/job-crontab/boot-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/FilterConfig.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.crontab.config; - -import com.tencent.bk.job.common.web.filter.RepeatableReadWriteServletRequestResponseFilter; -import org.springframework.boot.web.servlet.FilterRegistrationBean; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Configuration -public class FilterConfig { - @Bean - public FilterRegistrationBean repeatableRSRRFilterRegister() { - FilterRegistrationBean registration = - new FilterRegistrationBean<>(); - registration.setFilter(repeatableRRRFilter()); - registration.addUrlPatterns("/esb/api/*"); - registration.setName("repeatableReadRequestResponseFilter"); - registration.setOrder(0); - return registration; - } - - @Bean(name = "repeatableReadRequestResponseFilter") - public RepeatableReadWriteServletRequestResponseFilter repeatableRRRFilter() { - return new RepeatableReadWriteServletRequestResponseFilter(); - } -} diff --git a/src/backend/job-crontab/boot-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/InterceptorConfiguration.java b/src/backend/job-crontab/boot-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/InterceptorConfiguration.java deleted file mode 100644 index 6b2bfd2ff7..0000000000 --- a/src/backend/job-crontab/boot-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/InterceptorConfiguration.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.crontab.config; - -import com.tencent.bk.job.common.iam.interceptor.AuthAppInterceptor; -import com.tencent.bk.job.common.iam.interceptor.JobIamInterceptor; -import com.tencent.bk.job.common.web.interceptor.EsbApiLogInterceptor; -import com.tencent.bk.job.common.web.interceptor.EsbReqRewriteInterceptor; -import com.tencent.bk.job.common.web.interceptor.JobCommonInterceptor; -import com.tencent.bk.job.common.web.interceptor.ServiceSecurityInterceptor; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Configuration; -import org.springframework.web.servlet.config.annotation.InterceptorRegistry; -import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; - -/** - * @since 6/11/2019 10:38 - */ -@Configuration -public class InterceptorConfiguration implements WebMvcConfigurer { - - private final JobCommonInterceptor jobCommonInterceptor; - private final AuthAppInterceptor authAppInterceptor; - private final EsbApiLogInterceptor esbApiLogInterceptor; - private final ServiceSecurityInterceptor serviceSecurityInterceptor; - private final JobIamInterceptor iamInterceptor; - private final EsbReqRewriteInterceptor esbReqRewriteInterceptor; - - @Autowired - public InterceptorConfiguration( - JobCommonInterceptor jobCommonInterceptor, - AuthAppInterceptor authAppInterceptor, - EsbApiLogInterceptor esbApiLogInterceptor, - ServiceSecurityInterceptor serviceSecurityInterceptor, - JobIamInterceptor iamInterceptor, - EsbReqRewriteInterceptor esbReqRewriteInterceptor - ) { - this.jobCommonInterceptor = jobCommonInterceptor; - this.authAppInterceptor = authAppInterceptor; - this.esbApiLogInterceptor = esbApiLogInterceptor; - this.serviceSecurityInterceptor = serviceSecurityInterceptor; - this.iamInterceptor = iamInterceptor; - this.esbReqRewriteInterceptor = esbReqRewriteInterceptor; - } - - @Override - public void addInterceptors(InterceptorRegistry registry) { - // 注册拦截器 - registry.addInterceptor(serviceSecurityInterceptor).addPathPatterns("/**").order(0); - registry.addInterceptor(jobCommonInterceptor).addPathPatterns("/**").order(10); - registry.addInterceptor(iamInterceptor).addPathPatterns("/iam/api/v1/resources/**").order(30); - registry.addInterceptor(esbApiLogInterceptor).addPathPatterns("/esb/api/**").order(40); - registry.addInterceptor(esbReqRewriteInterceptor).addPathPatterns("/esb/api/**").order(50); - registry.addInterceptor(authAppInterceptor).addPathPatterns("/web/**", "/esb/api/**").order(60); - } - -} diff --git a/src/backend/job-crontab/boot-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/JobCrontabConfiguration.java b/src/backend/job-crontab/boot-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/JobCrontabConfiguration.java new file mode 100644 index 0000000000..7fd1c61db2 --- /dev/null +++ b/src/backend/job-crontab/boot-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/JobCrontabConfiguration.java @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.crontab.config; + +import com.tencent.bk.job.common.service.AppScopeMappingService; +import com.tencent.bk.job.common.web.interceptor.AppResourceScopeInterceptor; +import com.tencent.bk.job.manage.AppScopeMappingServiceImpl; +import com.tencent.bk.job.manage.api.inner.ServiceApplicationResource; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class JobCrontabConfiguration { + @Bean + AppScopeMappingService appScopeMappingService(ServiceApplicationResource applicationResource) { + return new AppScopeMappingServiceImpl(applicationResource); + } + + @Bean + public AppResourceScopeInterceptor appResourceScopeInterceptor(AppScopeMappingService appScopeMappingService) { + return new AppResourceScopeInterceptor(appScopeMappingService); + } +} diff --git a/src/backend/job-crontab/boot-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/RestTemplateConfig.java b/src/backend/job-crontab/boot-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/RestTemplateConfig.java deleted file mode 100644 index f8cfadb49d..0000000000 --- a/src/backend/job-crontab/boot-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/RestTemplateConfig.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.crontab.config; - -import org.springframework.cloud.client.loadbalancer.LoadBalanced; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.web.client.RestTemplate; - -/** - * @since 24/2/2020 14:51 - */ -@Configuration -public class RestTemplateConfig { - @Bean - @LoadBalanced - public RestTemplate loadBalancedRestTemplate() { - return new RestTemplate(); - } -} diff --git a/src/backend/job-crontab/boot-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/SwaggerConfig.java b/src/backend/job-crontab/boot-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/SwaggerConfig.java index 5d9853663a..54db090eba 100644 --- a/src/backend/job-crontab/boot-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/SwaggerConfig.java +++ b/src/backend/job-crontab/boot-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/SwaggerConfig.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,38 +24,40 @@ package com.tencent.bk.job.crontab.config; +import com.tencent.bk.job.common.service.config.JobCommonConfig; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Profile; import springfox.documentation.builders.PathSelectors; import springfox.documentation.builders.RequestHandlerSelectors; +import springfox.documentation.oas.annotations.EnableOpenApi; import springfox.documentation.spi.DocumentationType; import springfox.documentation.spring.web.plugins.Docket; -import springfox.documentation.swagger2.annotations.EnableSwagger2; import java.util.Arrays; import java.util.HashSet; /** - * @date 2019/09/19 + * Swagger 配置 */ @Configuration -@EnableSwagger2 -@Profile({"dev", "test", "local"}) +@EnableOpenApi +@Profile({"dev", "local"}) public class SwaggerConfig { - private final BkConfig bkConfig; + private final JobCommonConfig jobCommonConfig; @Autowired - public SwaggerConfig(BkConfig bkConfig) { - this.bkConfig = bkConfig; + public SwaggerConfig(JobCommonConfig jobCommonConfig) { + this.jobCommonConfig = jobCommonConfig; } @Bean public Docket api() { - return new Docket(DocumentationType.SWAGGER_2) - .host(bkConfig.getSwaggerUrl() + "/job-crontab") + return new Docket(DocumentationType.OAS_30) + .host(jobCommonConfig.getSwaggerUrl()) + .pathMapping("job-crontab") .protocols(new HashSet<>(Arrays.asList("http", "https"))).select() .apis(RequestHandlerSelectors.basePackage("com.tencent.bk.job.crontab.api")) .paths(PathSelectors.any()) diff --git a/src/backend/job-crontab/boot-job-crontab/src/main/resources/application.yml b/src/backend/job-crontab/boot-job-crontab/src/main/resources/application.yml index 033cf5770c..4853e55f97 100644 --- a/src/backend/job-crontab/boot-job-crontab/src/main/resources/application.yml +++ b/src/backend/job-crontab/boot-job-crontab/src/main/resources/application.yml @@ -1,20 +1,22 @@ spring: profiles: active: prod - -ribbon: - ServerListRefreshInterval: 3000 - + mvc: + pathmatch: + matching-strategy: ant_path_matcher management: health: livenessstate: enabled: true readinessstate: enabled: true + # 使用spring cloud stream rabbitmq, 禁用掉springboot的rabbitmq health检查 + rabbit: + enabled: false endpoints: web: exposure: - include: health,configprops,env,beans,conditions,loggers,metrics,mappings,prometheus,scheduledtasks,info,refresh + include: health,configprops,env,beans,conditions,loggers,metrics,mappings,prometheus,scheduledtasks,info,refresh,busrefresh,bindings base-path: /actuator enabled-by-default: false endpoint: @@ -45,6 +47,8 @@ management: enabled: true refresh: enabled: true + busrefresh: + enabled: true server: port: 19805 shutdown: graceful diff --git a/src/backend/job-crontab/boot-job-crontab/src/main/resources/bootstrap.yml b/src/backend/job-crontab/boot-job-crontab/src/main/resources/bootstrap.yml index 50878cc860..e51364aa48 100644 --- a/src/backend/job-crontab/boot-job-crontab/src/main/resources/bootstrap.yml +++ b/src/backend/job-crontab/boot-job-crontab/src/main/resources/bootstrap.yml @@ -11,3 +11,5 @@ spring: name: ${JOB_APPLICATION_CONFIGMAP_NAME} - namespace: ${KUBERNETES_NAMESPACE} name: ${JOB_COMMON_CONFIGMAP_NAME} + - namespace: ${KUBERNETES_NAMESPACE} + name: ${JOB_COMMON_REFRESHABLE_CONFIGMAP_NAME} diff --git a/src/backend/job-crontab/boot-job-crontab/src/main/resources/logback-spring.xml b/src/backend/job-crontab/boot-job-crontab/src/main/resources/logback-spring.xml new file mode 100644 index 0000000000..fe53ed6f03 --- /dev/null +++ b/src/backend/job-crontab/boot-job-crontab/src/main/resources/logback-spring.xml @@ -0,0 +1,13 @@ + + + logback + + + + + + + + + + diff --git a/src/backend/job-crontab/boot-job-crontab/src/main/resources/logback/logback-app-props.xml b/src/backend/job-crontab/boot-job-crontab/src/main/resources/logback/logback-app-props.xml deleted file mode 100644 index 61489b7fef..0000000000 --- a/src/backend/job-crontab/boot-job-crontab/src/main/resources/logback/logback-app-props.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - diff --git a/src/backend/job-crontab/boot-job-crontab/src/main/resources/quartz.properties b/src/backend/job-crontab/boot-job-crontab/src/main/resources/quartz.properties deleted file mode 100644 index 69f8891c3b..0000000000 --- a/src/backend/job-crontab/boot-job-crontab/src/main/resources/quartz.properties +++ /dev/null @@ -1,93 +0,0 @@ -# -# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. -# -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. -# -# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. -# -# License for BK-JOB蓝鲸智云作业平台: -# -------------------------------------------------------------------- -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF -# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - -#============================================================== -#Configure Main Scheduler Properties -#============================================================== -#调度器实例名称 -org.quartz.scheduler.instanceName=Job_Crontab_Quartz_Scheduler -#调度器实例编号自动生成 -org.quartz.scheduler.instanceId=AUTO -org.quartz.scheduler.batchTriggerAcquisitionMaxCount=50 -#============================================================== -#Configure ThreadPool -#============================================================== -#线程池实现类 -org.quartz.threadPool.class=org.quartz.simpl.SimpleThreadPool -#执行最大并发线程数量 -org.quartz.threadPool.threadCount=50 -#线程优先级 -#org.quartz.threadPool.threadPriority = 5 -#配置为守护线程，设置后任务将不会执行 -#org.quartz.threadPool.makeThreadsDaemons = true -#配置是否启动自动加载数据库内的定时任务，默认true -#org.quartz.threadPool.threadsInheritContextClassLoaderOfInitializingThread = true -#============================================================== -#Configure JobStore -#============================================================== -#持久化方式配置 -org.quartz.jobStore.class=org.quartz.impl.jdbcjobstore.JobStoreTX -#持久化方式配置数据驱动，MySQL数据库 -org.quartz.jobStore.driverDelegateClass=org.quartz.impl.jdbcjobstore.StdJDBCDelegate -#quartz相关数据表前缀名 -org.quartz.jobStore.tablePrefix=QRTZ_ -#开启分布式部署 -org.quartz.jobStore.isClustered=true -#分布式节点有效性检查时间间隔，单位：毫秒 -org.quartz.jobStore.clusterCheckinInterval=20000 -#配置是否使用 -org.quartz.jobStore.useProperties=true -org.quartz.jobStore.selectWithLockSQL=SELECT * FROM {0}LOCKS WHERE LOCK_NAME \= ? FOR UPDATE -org.quartz.jobStore.dataSource=myDS -#============================================================== -#Configure DataSource -#============================================================== -org.quartz.dataSource.myDS.driver=io.opentelemetry.instrumentation.jdbc.OpenTelemetryDriver -org.quartz.dataSource.myDS.URL=${job.crontab.db.url} -org.quartz.dataSource.myDS.user=${job.crontab.db.user} -org.quartz.dataSource.myDS.password=${job.crontab.db.password} -org.quartz.dataSource.myDS.maxConnections=30 -# Enable HikariCP DB conn Pool -org.quartz.dataSource.myDS.provider=hikaricp -# Custom hikaricp config properties. You can add more based on their docs: -# https://github.com/brettwooldridge/HikariCP#configuration-knobs-baby -org.quartz.dataSource.myDS.maximumPoolSize=10 -org.quartz.dataSource.myDS.connectionTestQuery=SELECT 1 -org.quartz.dataSource.myDS.validationTimeout=5000 -org.quartz.dataSource.myDS.idleTimeout=0 -#============================================================== -#============================================================== -# Configure quartz db -#============================================================== -#Skip Check Update -#update:true -#not update:false -#============================================================== -org.quartz.scheduler.skipUpdateCheck=true -#============================================================================ -# Configure Plugins -#============================================================================ -org.quartz.plugin.triggHistory.class=org.quartz.plugins.history.LoggingJobHistoryPlugin -org.quartz.plugin.shutdownhook.class=org.quartz.plugins.management.ShutdownHookPlugin -org.quartz.plugin.shutdownhook.cleanShutdown=true diff --git a/src/backend/job-crontab/boot-job-crontab/src/test/java/com/tencent/bk/job/crontab/dao/impl/CronJobDAOImplIntegrationTest.java b/src/backend/job-crontab/boot-job-crontab/src/test/java/com/tencent/bk/job/crontab/dao/impl/CronJobDAOImplIntegrationTest.java index 3433fc3561..2b8e99ea98 100644 --- a/src/backend/job-crontab/boot-job-crontab/src/test/java/com/tencent/bk/job/crontab/dao/impl/CronJobDAOImplIntegrationTest.java +++ b/src/backend/job-crontab/boot-job-crontab/src/test/java/com/tencent/bk/job/crontab/dao/impl/CronJobDAOImplIntegrationTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -49,7 +49,13 @@ import java.text.ParseException; import java.time.LocalDateTime; import java.time.ZoneOffset; -import java.util.*; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Random; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatExceptionOfType; @@ -353,7 +359,8 @@ void updateCronJobById() { @Test void giveCronJobIdReturnDeleteSuccess() { - assertThat(cronJobDAO.getCronJobById(CRON_JOB_1.getAppId(), CRON_JOB_1.getId())).isEqualTo(CRON_JOB_1); + CronJobInfoDTO cronJobInfoDTO = cronJobDAO.getCronJobById(CRON_JOB_1.getAppId(), CRON_JOB_1.getId()); + assertThat(cronJobInfoDTO).isEqualTo(CRON_JOB_1); assertThat(cronJobDAO.deleteCronJobById(CRON_JOB_1.getAppId(), CRON_JOB_1.getId())).isTrue(); assertThat(cronJobDAO.deleteCronJobById(CRON_JOB_1.getAppId(), CRON_JOB_1.getId())).isFalse(); assertThat(cronJobDAO.getCronJobById(CRON_JOB_1.getAppId(), CRON_JOB_1.getId())).isNull(); @@ -395,7 +402,8 @@ void insertCronJobReturnCorrectId() { CRON_JOB_1.setNotifyUser(userRoleInfo); CRON_JOB_1.setNotifyChannel(Arrays.asList(UUID.randomUUID().toString(), UUID.randomUUID().toString())); CRON_JOB_1.setId(cronJobDAO.insertCronJob(CRON_JOB_1)); - assertThat(cronJobDAO.getCronJobById(CRON_JOB_1.getAppId(), CRON_JOB_1.getId())).isEqualTo(CRON_JOB_1); + CronJobInfoDTO cronJobInfoDTO = cronJobDAO.getCronJobById(CRON_JOB_1.getAppId(), CRON_JOB_1.getId()); + assertThat(cronJobInfoDTO).isEqualTo(CRON_JOB_1); } @Test @@ -409,30 +417,33 @@ void listCronJobByCondition() { baseSearchCondition.setOrderField("last_modify_user"); baseSearchCondition.setCreator(CRON_JOB_1.getCreator()); PageData cronJobInfoPageData = - cronJobDAO.listPageCronJobsByCondition(cronJobCondition, baseSearchCondition); + cronJobDAO.listPageCronJobsWithoutVarsByCondition(cronJobCondition, baseSearchCondition); assertThat(cronJobInfoPageData.getStart()).isEqualTo(0); assertThat(cronJobInfoPageData.getPageSize()).isEqualTo(2); assertThat(cronJobInfoPageData.getTotal()).isEqualTo(4); - assertThat(cronJobInfoPageData.getData()).contains(CRON_JOB_5); - assertThat(cronJobInfoPageData.getData()).contains(CRON_JOB_1); + Set ids = cronJobInfoPageData.getData().stream().map(CronJobInfoDTO::getId).collect(Collectors.toSet()); + assertThat(ids).contains(CRON_JOB_5.getId()); + assertThat(ids).contains(CRON_JOB_1.getId()); baseSearchCondition.setStart(2); - cronJobInfoPageData = cronJobDAO.listPageCronJobsByCondition(cronJobCondition, baseSearchCondition); + cronJobInfoPageData = cronJobDAO.listPageCronJobsWithoutVarsByCondition(cronJobCondition, baseSearchCondition); assertThat(cronJobInfoPageData.getStart()).isEqualTo(2); assertThat(cronJobInfoPageData.getPageSize()).isEqualTo(2); assertThat(cronJobInfoPageData.getTotal()).isEqualTo(4); - assertThat(cronJobInfoPageData.getData()).contains(CRON_JOB_9); - assertThat(cronJobInfoPageData.getData()).contains(CRON_JOB_3); + ids = cronJobInfoPageData.getData().stream().map(CronJobInfoDTO::getId).collect(Collectors.toSet()); + assertThat(ids).contains(CRON_JOB_9.getId()); + assertThat(ids).contains(CRON_JOB_3.getId()); System.out.println(cronJobInfoPageData.getData()); baseSearchCondition.setStart(0); baseSearchCondition.setOrder(1); - cronJobInfoPageData = cronJobDAO.listPageCronJobsByCondition(cronJobCondition, baseSearchCondition); + cronJobInfoPageData = cronJobDAO.listPageCronJobsWithoutVarsByCondition(cronJobCondition, baseSearchCondition); assertThat(cronJobInfoPageData.getStart()).isEqualTo(0); assertThat(cronJobInfoPageData.getPageSize()).isEqualTo(2); assertThat(cronJobInfoPageData.getTotal()).isEqualTo(4); - assertThat(cronJobInfoPageData.getData()).contains(CRON_JOB_9); - assertThat(cronJobInfoPageData.getData()).contains(CRON_JOB_3); + ids = cronJobInfoPageData.getData().stream().map(CronJobInfoDTO::getId).collect(Collectors.toSet()); + assertThat(ids).contains(CRON_JOB_9.getId()); + assertThat(ids).contains(CRON_JOB_3.getId()); System.out.println(cronJobInfoPageData.getData()); baseSearchCondition.setOrder(0); } diff --git a/src/backend/job-crontab/boot-job-crontab/src/test/java/com/tencent/bk/job/crontab/package-info.java b/src/backend/job-crontab/boot-job-crontab/src/test/java/com/tencent/bk/job/crontab/package-info.java index cd0bf3fc47..3bf0c67874 100644 --- a/src/backend/job-crontab/boot-job-crontab/src/test/java/com/tencent/bk/job/crontab/package-info.java +++ b/src/backend/job-crontab/boot-job-crontab/src/test/java/com/tencent/bk/job/crontab/package-info.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/boot-job-crontab/src/test/resources/application-test.yml b/src/backend/job-crontab/boot-job-crontab/src/test/resources/application-test.yml index 627f6fd1a9..cdaa254a0c 100644 --- a/src/backend/job-crontab/boot-job-crontab/src/test/resources/application-test.yml +++ b/src/backend/job-crontab/boot-job-crontab/src/test/resources/application-test.yml @@ -1,4 +1,7 @@ spring: + mvc: + pathmatch: + matching-strategy: ant_path_matcher http: encoding: charset: UTF-8 @@ -32,6 +35,9 @@ spring: cloud: config: enabled: false + kubernetes: + discovery: + enabled: false quartz: thread-pool: core-pool-size: 3 @@ -45,14 +51,22 @@ job: service: public-key-base64: LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FROEFNSUlCQ2dLQ0FRRUFyTXN3emh3QkVUTktQOThTcElna0VjZThPNVlQWm5aUDVHVW1ZUVhhSHdHWmg5UHJwNDFseS9xbWk5enEwTlRCODRyaXhSZHZyU1c0cCtCUjk4TVRzRmpYaUlRYVBxRTdRUitpdDdoT1VQNVJMQW9KcWlvL3NlZlRIdlZxVW1GcitYU3NST05rek5selRVaVcvMFRNY0cwNVdIS3hORno5YTB2aXo1T01PeE5lUzhqS1ZiRStIMHlQS0JhR1JHMEdZNTZTS1BLbXh1RHk5a2tyVUtGSXZycldSZkh2bzVsK0xsT3IyQmVGT3FHcEhmeW1FaXlkbFRxWHNlbGlsYTRxMUoxbnBZRDYwY3JIY1ZRV3d2Ri9CZUkrZytxVkF5YzNaWmhYeEhoNWNMcmZsK3pqMktsUEFEZVIvZktubHFqKzFBQm4yaFRoVWJ2WjRCdXhTdWxOa1FJREFRQUIKLS0tLS1FTkQgUFVCTElDIEtFWS0tLS0t private-key-base64: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV1d0lCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktVd2dnU2hBZ0VBQW9JQkFRQ3N5ekRPSEFFUk0wby8zeEtraUNRUng3dzdsZzltZGsva1pTWmhCZG9mQVptSDArdW5qV1hMK3FhTDNPclExTUh6aXVMRkYyK3RKYmluNEZIM3d4T3dXTmVJaEJvK29UdEJINkszdUU1US9sRXNDZ21xS2oreDU5TWU5V3BTWVd2NWRLeEU0MlRNMlhOTlNKYi9STXh3YlRsWWNyRTBYUDFyUytMUGs0dzdFMTVMeU1wVnNUNGZUSThvRm9aRWJRWmpucElvOHFiRzRQTDJTU3RRb1VpK3V0WkY4ZStqbVg0dVU2dllGNFU2b2FrZC9LWVNMSjJWT3BleDZXS1ZyaXJVbldlbGdQclJ5c2R4VkJiQzhYOEY0ajZENnBVREp6ZGxtRmZFZUhsd3V0K1g3T1BZcVU4QU41SDk4cWVXcVA3VUFHZmFGT0ZSdTluZ0c3Rks2VTJSQWdNQkFBRUNnZ0VBZlkvZGREaUhVZjlxV2dWNG41VEtXUjFDSEM2TUhGR205bm5HTE1TNjNzTWoxTDdiUkZNTWdQQXY2L3hwcWJ5Zy81K1I3OVI2ODJBdzBkVEEyNENuSXdNOEE5aXkxWVlGNytuVUxSckIxanNuc3cvTVBCb2RWT0wzMkV4UHFYTmZxZEx0aDlXUm53WDRzbFVvWEhDYStxdHorMkRFZ2g5OGkyYUdkMzVwYlZjT2p0czlWc3FtZE5uQTdLNFVQWkd4aVFheG1rLzBOVEVNY2RqMUl4bWNlZjNqL1RUVkFiR05IdWVtYTJKeUdZdk52N09BZUN3VGJSdW5TNW1PSmptRHlFSmp4dndaVWZwanZoM0w0SkZzMGRJUm5wZ2VtTkY3NytqUmFTTy9WUVlqc1FLelB0SnEwcHFneHUwd0RoQUVBeXpjYWI2NW8wSm1hZzNJUjZNQzdRS0JnUUR1MThHNFRVaTlSVk8yZS9YdWtDSUNiOVZISHlMeVFIUDh4NDhBVFZmVENtSVBEMUduTWp5TlFmSnhsUlNma3UyUjJIM1IyRC81YUNpNUYwTi9USG51OWVqYlMvS1NoYjFxMFl5TWF1TDkxQlZjMkEyeDZlRzRPNFFpT2daT0gzaE0raUVSREJxOExNQ1dILzVtYVJDeDhVRmRQTnpHMUtXVVNwTEJXM3l2ZXdLQmdRQzVOTS9yN09Wc3F6OGlDUjBXa2JramN2eWU4MkJNcGFsZVkyRHNWQVMwRHNMNlMvZlAzY2VUdEJQVmdJL244Sk16b1ZNZE5OK3htbXM4Wi9rMGM5OGlLTzRUeE0zN2daYkIwUE8rWGRyeUZkRFhTRDZXTm1heWpYUG5MUndBZmRwbFlWdFNqVGxWMWJzWWIySnhXNXY2RUVSaVlLYnNnRGNpWTA5ZnYva0RZd0ovUnVmR212QldwOW50QUQvTXd3WUhFcklnbnZ5ZlkwdS9JMHdiSi92T0Z0aitRM3BJdzFvbW44ajBNTVFSVzA1RE9Ra01VVC9odmlrdDdjVDZSTkJ2WW9HZW4zdnNoNU1zcUltTk1DS0xRTSsxaDlxY05qVTR6WGpkd1V0NGs3akQvaFpEdXN2ZEpBQWxMR3hUR2hRVzRMeStxdTltbTZDRmg0Q2NFTXNZT1FLQmdCa0pEWTRyd0x1V3pucndWbkZGazR3NkwvSGdua1RlSSszeGF4bW5KZGVkSXpnb2FzQTdqNDNreXNXU2ttRmRBTmo2MXJ4YytGd09ycTVFczhnbVhhUkZlaUZMK2pna1JWZS93aU1hbHMwZ3E5RmVINis0Zk50b0NhSFhWREN3MG9QZnpETjFpYW1Uc2EzVTZLN0FIbTZJSW1aV0FGcDUzbm9VaUpjOTRSQlhBb0dCQUxMcmQrbnljNzVEbjZ0ZWdVT0JQRy9HR3FYWkZGdmd6emJRRnR0SmR0NTlMZ0NBanUxMjJMRExsRjZreTB6Q0JsK0h6Q0kyUXJXVEp4cXUxSTZvbndaZDlOMllKVlNWQlFOV285NTJUamRkOHcvSkExYlNqK3JpTU1WQ0txT1BNRHNiOENEaGlEbVM3RlBEM0trTVlmcUxJOVhNRTVvNVUvNFJzcEZUZWozRQotLS0tLUVORCBQUklWQVRFIEtFWS0tLS0t - -notification-policy: - failed: - #连续启动失败通知策略：默认 从第一次失败开始，连续失败每5次通知第一次 begin = 1；frequency = 5；totalTimes = -1 - start: - #通知开始：默认 1 - 从第一次失败开始通知 - begin: 1 - #通知频率：默认 5 - 每5次通知， 1 - 每次通知 - frequency: 5 - #通知次数：默认 -1 - 不限制通知次数， 0 - 不通知， 1 - 只通知1次， - totalTimes: -1 + crontab: + notification-policy: + failed: + #连续启动失败通知策略：默认 从第一次失败开始，连续失败每5次通知第一次 begin = 1；frequency = 5；totalTimes = -1 + start: + #通知开始：默认 1 - 从第一次失败开始通知 + begin: 1 + #通知频率：默认 5 - 每5次通知， 1 - 每次通知 + frequency: 5 + #通知次数：默认 -1 - 不限制通知次数， 0 - 不通知， 1 - 只通知1次， + totalTimes: -1 +bk-api-gateway: + gse: + url: gse.apigw.com + cmdb: + url: cmdb.apigw.com +esb: + service: + url: esb.service diff --git a/src/backend/job-crontab/boot-job-crontab/src/test/resources/bootstrap.yml b/src/backend/job-crontab/boot-job-crontab/src/test/resources/bootstrap.yml index 3e814c6211..c59c9985cc 100644 --- a/src/backend/job-crontab/boot-job-crontab/src/test/resources/bootstrap.yml +++ b/src/backend/job-crontab/boot-job-crontab/src/test/resources/bootstrap.yml @@ -2,4 +2,7 @@ spring: application: name: job-crontab profiles: - active: test \ No newline at end of file + active: test + cloud: + kubernetes: + enabled: false diff --git a/src/backend/job-crontab/boot-job-crontab/src/test/resources/init_cron_job_data.sql b/src/backend/job-crontab/boot-job-crontab/src/test/resources/init_cron_job_data.sql index 64dae34e1c..c8134a0306 100644 --- a/src/backend/job-crontab/boot-job-crontab/src/test/resources/init_cron_job_data.sql +++ b/src/backend/job-crontab/boot-job-crontab/src/test/resources/init_cron_job_data.sql @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -29,32 +29,32 @@ INSERT INTO `cron_job` (id, app_id, name, creator, task_template_id, task_plan_i create_time, last_modify_user, last_modify_time, end_time, notify_offset, notify_user, notify_channel) VALUES (1, 2, 'cron_job_1', 'userC', 1, 100, null, null, '* * * * *', null, - '[{"id":null,"name":"a","type":3,"value":"b","server":null},{"id":null,"name":"b","type":4,"value":"c","server":null}]', + '[{"id":null,"name":"a","type":3,"value":"b","cipherEncryptAlgorithm":"None","server":null},{"id":null,"name":"b","type":4,"value":"c","cipherEncryptAlgorithm":"None","server":null}]', null, 1, 0, '1546272000', 'userT', '1546272000', 0, 600, '{"userList":["userC", "userJ"], "roleList":["JOB_ROLE_1", "JOB_ROLE_2"]}', '["wechat", "email"]'), (2, 2, 'cron_job_2', 'userT', 2, 200, null, null, null, '1546272000', - '[{"id":null,"name":"a","type":3,"value":"b","server":null},{"id":null,"name":"b","type":4,"value":"c","server":null}]', + '[{"id":null,"name":"a","type":3,"value":"b","cipherEncryptAlgorithm":"None","server":null},{"id":null,"name":"b","type":4,"value":"c","cipherEncryptAlgorithm":"None","server":null}]', 1, 0, 1, '1546272000', 'userT', '1546272000', 0, 600, '{"userList":["userT", "userJ"], "roleList":["JOB_ROLE_3", "JOB_ROLE_4"]}', '["email"]'), (3, 2, 'cron_job_3', 'userC', 3, 300, null, null, '* * * * *', null, - '[{"id":null,"name":"a","type":3,"value":"b","server":null},{"id":null,"name":"b","type":4,"value":"c","server":null}]', + '[{"id":null,"name":"a","type":3,"value":"b","cipherEncryptAlgorithm":"None","server":null},{"id":null,"name":"b","type":4,"value":"c","cipherEncryptAlgorithm":"None","server":null}]', 0, 1, 0, '1546272000', 'userC', '1546272000', 1577808000, 0, null, null), (4, 2, 'cron_job_4', 'userT', 4, 400, null, null, null, '1546272000', - '[{"id":null,"name":"a","type":3,"value":"b","server":null},{"id":null,"name":"b","type":4,"value":"c","server":null}]', + '[{"id":null,"name":"a","type":3,"value":"b","cipherEncryptAlgorithm":"None","server":null},{"id":null,"name":"b","type":4,"value":"c","cipherEncryptAlgorithm":"None","server":null}]', null, 1, 0, '1546272000', 'userC', '1546272000', 0, 0, null, null), (5, 2, 'cron_job_5', 'userC', 5, 500, null, null, '* * * * *', null, - '[{"id":null,"name":"a","type":3,"value":"b","server":null},{"id":null,"name":"b","type":4,"value":"c","server":null}]', + '[{"id":null,"name":"a","type":3,"value":"b","cipherEncryptAlgorithm":"None","server":null},{"id":null,"name":"b","type":4,"value":"c","cipherEncryptAlgorithm":"None","server":null}]', 1, 1, 0, '1546272000', 'userT', '1546272000', 0, 0, null, null), (6, 2, 'cron_job_6', 'userT', null, null, 'aaaa', 1, null, '1546272000', - '[{"id":null,"name":"a","type":3,"value":"b","server":null},{"id":null,"name":"b","type":4,"value":"c","server":null}]', + '[{"id":null,"name":"a","type":3,"value":"b","cipherEncryptAlgorithm":"None","server":null},{"id":null,"name":"b","type":4,"value":"c","cipherEncryptAlgorithm":"None","server":null}]', 0, 1, 0, '1546272000', 'userT', '1546272000', 0, 0, null, null), (7, 2, 'cron_job_7', 'userC', null, null, 'bbbb', 2, '* * * * *', null, - '[{"id":null,"name":"a","type":3,"value":"b","server":null},{"id":null,"name":"b","type":4,"value":"c","server":null}]', + '[{"id":null,"name":"a","type":3,"value":"b","cipherEncryptAlgorithm":"None","server":null},{"id":null,"name":"b","type":4,"value":"c","cipherEncryptAlgorithm":"None","server":null}]', null, 0, 1, '1546272000', 'userC', '1546272000', 0, 0, null, null), (8, 2, 'cron_job_8', 'userT', null, null, 'cccc', 3, null, '1546272000', - '[{"id":null,"name":"a","type":3,"value":"b","server":null},{"id":null,"name":"b","type":4,"value":"c","server":null}]', + '[{"id":null,"name":"a","type":3,"value":"b","cipherEncryptAlgorithm":"None","server":null},{"id":null,"name":"b","type":4,"value":"c","cipherEncryptAlgorithm":"None","server":null}]', 1, 1, 0, '1546272000', 'userC', '1546272000', 0, 0, null, null), (9, 2, 'cron_job_9', 'userC', null, null, 'vvvv', 4, '* * * * *', null, - '[{"id":null,"name":"a","type":3,"value":"b","server":null},{"id":null,"name":"b","type":4,"value":"c","server":null}]', + '[{"id":null,"name":"a","type":3,"value":"b","cipherEncryptAlgorithm":"None","server":null},{"id":null,"name":"b","type":4,"value":"c","cipherEncryptAlgorithm":"None","server":null}]', 0, 1, 0, '1546272000', 'userC', '1546272000', 0, 0, null, null) ; diff --git a/src/backend/job-crontab/boot-job-crontab/src/test/resources/init_schema.sql b/src/backend/job-crontab/boot-job-crontab/src/test/resources/init_schema.sql index 8a5cf3b1eb..189fa51456 100644 --- a/src/backend/job-crontab/boot-job-crontab/src/test/resources/init_schema.sql +++ b/src/backend/job-crontab/boot-job-crontab/src/test/resources/init_schema.sql @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/boot-job-crontab/src/test/resources/test.properties b/src/backend/job-crontab/boot-job-crontab/src/test/resources/test.properties index 095ed914fe..feac2227ad 100644 --- a/src/backend/job-crontab/boot-job-crontab/src/test/resources/test.properties +++ b/src/backend/job-crontab/boot-job-crontab/src/test/resources/test.properties @@ -1,11 +1,11 @@ # -# Tencent is pleased to support the open source community by making BK-JOB����������ҵƽ̨ available. +# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. # -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +# Copyright (C) 2021 Tencent. All rights reserved. # -# BK-JOB����������ҵƽ̨ is licensed under the MIT License. +# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. # -# License for BK-JOB����������ҵƽ̨: +# License for BK-JOB蓝鲸智云作业平台: # -------------------------------------------------------------------- # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated # documentation files (the "Software"), to deal in the Software without restriction, including without limitation diff --git a/src/backend/job-crontab/build.gradle b/src/backend/job-crontab/build.gradle index af956a0a3c..738a554461 100644 --- a/src/backend/job-crontab/build.gradle +++ b/src/backend/job-crontab/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -39,6 +39,7 @@ subprojects { compileOnly 'org.projectlombok:lombok' annotationProcessor 'org.projectlombok:lombok' implementation(group: 'org.apache.commons', name: 'commons-pool2') + implementation "io.github.openfeign:feign-httpclient" testImplementation 'org.springframework.boot:spring-boot-starter-test' testImplementation 'org.junit.jupiter:junit-jupiter' } diff --git a/src/backend/job-crontab/model-job-crontab/build.gradle b/src/backend/job-crontab/model-job-crontab/build.gradle index 53cd347e49..8006176375 100644 --- a/src/backend/job-crontab/model-job-crontab/build.gradle +++ b/src/backend/job-crontab/model-job-crontab/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/service-job-crontab/build.gradle b/src/backend/job-crontab/service-job-crontab/build.gradle index c56cd6de1e..5fe45c4a41 100644 --- a/src/backend/job-crontab/service-job-crontab/build.gradle +++ b/src/backend/job-crontab/service-job-crontab/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,9 +24,11 @@ apply plugin: 'io.spring.dependency-management' dependencies { + api project(":commons:common-crypto") api project(":commons:common-redis") api project(":commons:common-web") api project(':commons:common-security') + api project(':commons:common-mysql') api project(":job-execute:api-job-execute") api project(":job-manage:api-job-manage") api project(":job-crontab:api-job-crontab") diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/esb/impl/EsbCronJobResourceImpl.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/esb/impl/EsbCronJobResourceImpl.java index e4cf603829..259c03da98 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/esb/impl/EsbCronJobResourceImpl.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/esb/impl/EsbCronJobResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,17 +24,20 @@ package com.tencent.bk.job.crontab.api.esb.impl; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.audit.annotations.AuditRequestBody; +import com.tencent.bk.audit.context.AuditContext; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.esb.metrics.EsbApiTimed; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.common.exception.InternalException; import com.tencent.bk.job.common.exception.InvalidParamException; +import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; import com.tencent.bk.job.common.iam.model.AuthResult; import com.tencent.bk.job.common.metrics.CommonMetricNames; import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; -import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.crontab.api.common.CronCheckUtil; import com.tencent.bk.job.crontab.api.esb.EsbCronJobResource; @@ -67,21 +70,19 @@ public class EsbCronJobResourceImpl implements EsbCronJobResource { private final CronJobService cronJobService; private final CronAuthService cronAuthService; - private final AppScopeMappingService appScopeMappingService; @Autowired public EsbCronJobResourceImpl(CronJobService cronJobService, - CronAuthService cronAuthService, - AppScopeMappingService appScopeMappingService) { + CronAuthService cronAuthService) { this.cronJobService = cronJobService; this.cronAuthService = cronAuthService; - this.appScopeMappingService = appScopeMappingService; } @Override @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v2_get_cron_list"}) - public EsbResp> getCronList(EsbGetCronListRequest request) { - request.fillAppResourceScope(appScopeMappingService); + public EsbResp> getCronList(String username, + String appCode, + EsbGetCronListRequest request) { if (request.validate()) { if (request.getId() != null && request.getId() > 0) { CronJobInfoDTO cronJobInfoById = cronJobService.getCronJobInfoById(request.getAppId(), request.getId()); @@ -119,8 +120,8 @@ public EsbResp> getCronList(EsbGetCronListRequest requ request.getLastModifyTimeEnd(), "yyyy-MM-dd", ChronoUnit.SECONDS, ZoneId.systemDefault())); } PageData cronJobInfoPageData = - cronJobService.listPageCronJobInfos(cronJobCondition, baseSearchCondition); - return EsbResp.buildSuccessResp(cronJobInfoPageData.getData().parallelStream() + cronJobService.listPageCronJobInfosWithoutVars(cronJobCondition, baseSearchCondition); + return EsbResp.buildSuccessResp(cronJobInfoPageData.getData().stream() .map(CronJobInfoDTO::toEsbCronInfo).collect(Collectors.toList())); } } @@ -129,14 +130,15 @@ public EsbResp> getCronList(EsbGetCronListRequest requ @Override @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v2_update_cron_status"}) - public EsbResp updateCronStatus(EsbUpdateCronStatusRequest request) { - request.fillAppResourceScope(appScopeMappingService); - String username = request.getUserName(); + @AuditEntry(actionId = ActionId.MANAGE_CRON) + public EsbResp updateCronStatus(String username, + String appCode, + @AuditRequestBody EsbUpdateCronStatusRequest request) { Long appId = request.getAppId(); if (request.validate()) { AuthResult authResult = cronAuthService.authManageCron( - request.getUserName(), request.getAppResourceScope(), request.getId(), null + username, request.getAppResourceScope(), request.getId(), null ); if (!authResult.isPass()) { throw new PermissionDeniedException(authResult); @@ -176,44 +178,40 @@ private void checkRequest(EsbSaveCronRequest request) { @Override @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v2_save_cron"}) - public EsbResp saveCron(EsbSaveCronRequest request) { - request.fillAppResourceScope(appScopeMappingService); + @AuditEntry + public EsbResp saveCron(String username, + String appCode, + @AuditRequestBody EsbSaveCronRequest request) { + boolean isUpdate = request.getId() != null && request.getId() > 0; + // 判断审计操作 + AuditContext.current().updateActionId(isUpdate ? ActionId.MANAGE_CRON : ActionId.CREATE_CRON); + CronJobInfoDTO cronJobInfo = new CronJobInfoDTO(); EsbCronInfoResponse esbCronInfoResponse = new EsbCronInfoResponse(); esbCronInfoResponse.setId(0L); checkRequest(request); - AuthResult authResult; - if (request.getId() != null && request.getId() > 0) { - authResult = cronAuthService.authManageCron( - request.getUserName(), request.getAppResourceScope(), request.getId(), null - ); - } else { - authResult = cronAuthService.authCreateCron(request.getUserName(), request.getAppResourceScope()); - } - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } + cronJobInfo.setId(request.getId()); cronJobInfo.setAppId(request.getAppId()); cronJobInfo.setName(request.getName()); cronJobInfo.setTaskPlanId(request.getPlanId()); cronJobInfo.setCronExpression(CronExpressionUtil.fixExpressionForQuartz(request.getCronExpression())); - if (cronJobInfo.getId() == null || cronJobInfo.getId() == 0) { - cronJobInfo.setCreator(request.getUserName()); + if (!isUpdate) { + cronJobInfo.setCreator(username); cronJobInfo.setDelete(false); } cronJobInfo.setEnable(false); - cronJobInfo.setLastModifyUser(request.getUserName()); + cronJobInfo.setLastModifyUser(username); cronJobInfo.setLastModifyTime(DateUtils.currentTimeSeconds()); - Long cronId; - try { - cronId = cronJobService.saveCronJobInfo(cronJobInfo); - } catch (TaskExecuteAuthFailedException e) { - throw new PermissionDeniedException(e.getAuthResult()); + CronJobInfoDTO result; + if (isUpdate) { + result = cronJobService.updateCronJobInfo(username, cronJobInfo); + } else { + result = cronJobService.createCronJobInfo(username, cronJobInfo); } - if (cronId > 0) { - esbCronInfoResponse = CronJobInfoDTO.toEsbCronInfo(cronJobService.getCronJobInfoById(cronId)); + if (result.getId() > 0) { + esbCronInfoResponse = CronJobInfoDTO.toEsbCronInfo(cronJobService.getCronJobInfoById(result.getId())); return EsbResp.buildSuccessResp(esbCronInfoResponse); } else { throw new InternalException(ErrorCode.UPDATE_CRON_JOB_FAILED); diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/esb/v3/impl/EsbCronJobV3ResourceImpl.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/esb/v3/impl/EsbCronJobV3ResourceImpl.java index 4af680c4b9..2faf4ec6b3 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/esb/v3/impl/EsbCronJobV3ResourceImpl.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/esb/v3/impl/EsbCronJobV3ResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,9 @@ package com.tencent.bk.job.crontab.api.esb.v3.impl; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.audit.annotations.AuditRequestBody; +import com.tencent.bk.audit.context.AuditContext; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.constant.TaskVariableTypeEnum; import com.tencent.bk.job.common.esb.metrics.EsbApiTimed; @@ -32,6 +35,7 @@ import com.tencent.bk.job.common.esb.model.job.v3.EsbPageDataV3; import com.tencent.bk.job.common.exception.InternalException; import com.tencent.bk.job.common.exception.InvalidParamException; +import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; import com.tencent.bk.job.common.iam.model.AuthResult; import com.tencent.bk.job.common.metrics.CommonMetricNames; @@ -44,11 +48,11 @@ import com.tencent.bk.job.crontab.api.common.CronCheckUtil; import com.tencent.bk.job.crontab.api.esb.v3.EsbCronJobV3Resource; import com.tencent.bk.job.crontab.auth.CronAuthService; -import com.tencent.bk.job.crontab.client.ServiceTaskPlanResourceClient; import com.tencent.bk.job.crontab.common.constants.CronStatusEnum; import com.tencent.bk.job.crontab.exception.TaskExecuteAuthFailedException; import com.tencent.bk.job.crontab.model.dto.CronJobInfoDTO; import com.tencent.bk.job.crontab.model.dto.CronJobVariableDTO; +import com.tencent.bk.job.crontab.model.esb.v3.request.EsbDeleteCronV3Request; import com.tencent.bk.job.crontab.model.esb.v3.request.EsbGetCronDetailV3Request; import com.tencent.bk.job.crontab.model.esb.v3.request.EsbGetCronListV3Request; import com.tencent.bk.job.crontab.model.esb.v3.request.EsbSaveCronV3Request; @@ -57,6 +61,7 @@ import com.tencent.bk.job.crontab.model.inner.ServerDTO; import com.tencent.bk.job.crontab.service.CronJobService; import com.tencent.bk.job.crontab.util.CronExpressionUtil; +import com.tencent.bk.job.manage.api.inner.ServiceTaskPlanResource; import com.tencent.bk.job.manage.model.inner.ServiceTaskVariableDTO; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -76,13 +81,13 @@ public class EsbCronJobV3ResourceImpl implements EsbCronJobV3Resource { private final CronJobService cronJobService; private final CronAuthService cronAuthService; - private final ServiceTaskPlanResourceClient taskPlanResource; + private final ServiceTaskPlanResource taskPlanResource; private final AppScopeMappingService appScopeMappingService; @Autowired public EsbCronJobV3ResourceImpl(CronJobService cronJobService, CronAuthService cronAuthService, - ServiceTaskPlanResourceClient taskPlanResource, + ServiceTaskPlanResource taskPlanResource, AppScopeMappingService appScopeMappingService) { this.cronJobService = cronJobService; this.cronAuthService = cronAuthService; @@ -108,8 +113,6 @@ public EsbResp> getCronList(String username, Integer start, Integer length) { EsbGetCronListV3Request request = new EsbGetCronListV3Request(); - request.setUserName(username); - request.setAppCode(appCode); request.setBizId(bizId); request.setScopeType(scopeType); request.setScopeId(scopeId); @@ -124,7 +127,8 @@ public EsbResp> getCronList(String username, request.setLastModifyTimeStart(lastModifyTimeStart); request.setStart(start); request.setLength(length); - return getCronListUsingPost(request); + request.fillAppResourceScope(appScopeMappingService); + return getCronListUsingPost(username, appCode, request); } @Override @@ -135,19 +139,19 @@ public EsbResp getCronDetail(String username, String scopeId, Long id) { EsbGetCronDetailV3Request request = new EsbGetCronDetailV3Request(); - request.setUserName(username); - request.setAppCode(appCode); request.setBizId(bizId); request.setScopeType(scopeType); request.setScopeId(scopeId); request.setId(id); - return getCronDetailUsingPost(request); + request.fillAppResourceScope(appScopeMappingService); + return getCronDetailUsingPost(username, appCode, request); } @Override @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_get_cron_list"}) - public EsbResp> getCronListUsingPost(EsbGetCronListV3Request request) { - request.fillAppResourceScope(appScopeMappingService); + public EsbResp> getCronListUsingPost(String username, + String appCode, + EsbGetCronListV3Request request) { if (request.validate()) { if (request.getId() != null && request.getId() > 0) { CronJobInfoDTO cronJobInfoById = cronJobService.getCronJobInfoById(request.getAppId(), request.getId()); @@ -187,9 +191,9 @@ public EsbResp> getCronListUsingPost(EsbGetCronL baseSearchCondition.setLastModifyTimeStart(request.getLastModifyTimeStart()); baseSearchCondition.setLastModifyTimeEnd(request.getLastModifyTimeEnd()); } - PageData cronJobInfoPageData = cronJobService.listPageCronJobInfos(cronJobCondition, + PageData cronJobInfoPageData = cronJobService.listPageCronJobInfosWithoutVars(cronJobCondition, baseSearchCondition); - List cronInfoV3ResponseData = cronJobInfoPageData.getData().parallelStream() + List cronInfoV3ResponseData = cronJobInfoPageData.getData().stream() .peek(cronJobInfoDTO -> cronJobInfoDTO.setVariableValue(null)) .map(CronJobInfoDTO::toEsbCronInfoV3).collect(Collectors.toList()); EsbPageDataV3 esbPageDataV3 = new EsbPageDataV3<>(); @@ -205,13 +209,14 @@ public EsbResp> getCronListUsingPost(EsbGetCronL @Override @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_update_cron_status"}) - public EsbResp updateCronStatus(EsbUpdateCronStatusV3Request request) { - request.fillAppResourceScope(appScopeMappingService); - String username = request.getUserName(); + @AuditEntry(actionId = ActionId.MANAGE_CRON) + public EsbResp updateCronStatus(String username, + String appCode, + @AuditRequestBody EsbUpdateCronStatusV3Request request) { Long appId = request.getAppId(); request.validate(); AuthResult authResult = cronAuthService.authManageCron( - request.getUserName(), + username, new AppResourceScope(request.getScopeType(), request.getScopeId(), request.getAppId()), request.getId(), null @@ -271,7 +276,7 @@ private void checkAndFillGlobalVar(Long planId, esbGlobalVarV3DTO.setName(taskVariableDTO.getName()); esbGlobalVarV3DTO.setType(taskVariableDTO.getType()); } - cronJobInfo.setVariableValue(globalVarV3DTOList.parallelStream().map(globalVarV3DTO -> { + cronJobInfo.setVariableValue(globalVarV3DTOList.stream().map(globalVarV3DTO -> { CronJobVariableDTO cronJobVariableDTO = new CronJobVariableDTO(); cronJobVariableDTO.setId(globalVarV3DTO.getId()); cronJobVariableDTO.setName(globalVarV3DTO.getName()); @@ -284,32 +289,22 @@ private void checkAndFillGlobalVar(Long planId, @Override @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_save_cron"}) - public EsbResp saveCron(EsbSaveCronV3Request request) { - request.fillAppResourceScope(appScopeMappingService); + @AuditEntry + public EsbResp saveCron(String username, + String appCode, + @AuditRequestBody EsbSaveCronV3Request request) { + boolean isUpdate = request.getId() != null && request.getId() > 0; + // 判断审计操作 + AuditContext.current().updateActionId(isUpdate ? ActionId.MANAGE_CRON : ActionId.CREATE_CRON); + CronJobInfoDTO cronJobInfo = new CronJobInfoDTO(); EsbCronInfoV3DTO esbCronInfoV3DTO = new EsbCronInfoV3DTO(); esbCronInfoV3DTO.setId(0L); checkRequest(request); Long appId = request.getAppId(); - AuthResult authResult; - if (request.getId() != null && request.getId() > 0) { - authResult = cronAuthService.authManageCron( - request.getUserName(), - request.getAppResourceScope(), - request.getId(), - null - ); - } else { - authResult = cronAuthService.authCreateCron( - request.getUserName(), - request.getAppResourceScope() - ); - } - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } + cronJobInfo.setId(request.getId()); - cronJobInfo.setAppId(request.getAppId()); + cronJobInfo.setAppId(appId); cronJobInfo.setName(request.getName()); cronJobInfo.setTaskPlanId(request.getPlanId()); cronJobInfo.setCronExpression(CronExpressionUtil.fixExpressionForQuartz(request.getCronExpression())); @@ -319,26 +314,39 @@ public EsbResp saveCron(EsbSaveCronV3Request request) { checkAndFillGlobalVar(request.getPlanId(), globalVarV3DTOList, cronJobInfo); } if (cronJobInfo.getId() == null || cronJobInfo.getId() == 0) { - cronJobInfo.setCreator(request.getUserName()); + cronJobInfo.setCreator(username); cronJobInfo.setDelete(false); } cronJobInfo.setEnable(false); - cronJobInfo.setLastModifyUser(request.getUserName()); + cronJobInfo.setLastModifyUser(username); cronJobInfo.setLastModifyTime(DateUtils.currentTimeSeconds()); - Long cronId; - try { - cronId = cronJobService.saveCronJobInfo(cronJobInfo); - } catch (TaskExecuteAuthFailedException e) { - throw new PermissionDeniedException(e.getAuthResult()); + CronJobInfoDTO result; + if (isUpdate) { + result = cronJobService.updateCronJobInfo(username, cronJobInfo); + } else { + result = cronJobService.createCronJobInfo(username, cronJobInfo); } - if (cronId > 0) { - esbCronInfoV3DTO = CronJobInfoDTO.toEsbCronInfoV3Response(cronJobService.getCronJobInfoById(cronId)); + if (result.getId() > 0) { + esbCronInfoV3DTO = + CronJobInfoDTO.toEsbCronInfoV3Response(cronJobService.getCronJobInfoById(result.getId())); return EsbResp.buildSuccessResp(esbCronInfoV3DTO); } else { throw new InternalException(ErrorCode.UPDATE_CRON_JOB_FAILED); } } + @Override + @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_delete_cron"}) + @AuditEntry + public EsbResp deleteCron(String username, + String appCode, + @AuditRequestBody EsbDeleteCronV3Request request) { + if (cronJobService.deleteCronJobInfo(username, request.getAppId(), request.getId())) { + return EsbResp.buildSuccessResp(null); + } + return EsbResp.buildCommonFailResp(ErrorCode.DELETE_CRON_FAILED); + } + private void checkRequest(EsbSaveCronV3Request request) { // 定时任务表达式有效性校验 if (StringUtils.isNotBlank(request.getCronExpression())) { @@ -347,8 +355,9 @@ private void checkRequest(EsbSaveCronV3Request request) { } @Override - public EsbResp getCronDetailUsingPost(EsbGetCronDetailV3Request request) { - request.fillAppResourceScope(appScopeMappingService); + public EsbResp getCronDetailUsingPost(String username, + String appCode, + EsbGetCronDetailV3Request request) { if (request.validate()) { CronJobInfoDTO cronJobInfoById = cronJobService.getCronJobInfoById(request.getAppId(), request.getId()); return EsbResp.buildSuccessResp(CronJobInfoDTO.toEsbCronInfoV3(cronJobInfoById)); diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/iam/impl/IamCronCallbackResourceImpl.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/iam/impl/IamCronCallbackResourceImpl.java index 835a1e6373..4111af46f8 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/iam/impl/IamCronCallbackResourceImpl.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/iam/impl/IamCronCallbackResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,7 @@ package com.tencent.bk.job.crontab.api.iam.impl; +import com.tencent.bk.audit.utils.json.JsonSchemaUtils; import com.tencent.bk.job.common.iam.constant.ResourceTypeId; import com.tencent.bk.job.common.iam.service.BaseIamCallbackService; import com.tencent.bk.job.common.iam.util.IamRespUtil; @@ -33,17 +34,20 @@ import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.crontab.api.iam.IamCronCallbackResource; import com.tencent.bk.job.crontab.model.dto.CronJobInfoDTO; +import com.tencent.bk.job.crontab.model.esb.v3.response.EsbCronInfoV3DTO; import com.tencent.bk.job.crontab.service.CronJobService; import com.tencent.bk.sdk.iam.dto.PathInfoDTO; import com.tencent.bk.sdk.iam.dto.callback.request.CallbackRequestDTO; import com.tencent.bk.sdk.iam.dto.callback.request.IamSearchCondition; import com.tencent.bk.sdk.iam.dto.callback.response.CallbackBaseResponseDTO; import com.tencent.bk.sdk.iam.dto.callback.response.FetchInstanceInfoResponseDTO; +import com.tencent.bk.sdk.iam.dto.callback.response.FetchResourceTypeSchemaResponseDTO; import com.tencent.bk.sdk.iam.dto.callback.response.InstanceInfoDTO; import com.tencent.bk.sdk.iam.dto.callback.response.ListInstanceResponseDTO; import com.tencent.bk.sdk.iam.dto.callback.response.SearchInstanceResponseDTO; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Pair; +import org.slf4j.helpers.MessageFormatter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RestController; @@ -96,7 +100,7 @@ protected SearchInstanceResponseDTO searchInstanceResp( cronJobQuery.setName(callbackRequest.getFilter().getKeyword()); PageData cronJobInfoPageData = - cronJobService.listPageCronJobInfos(cronJobQuery, baseSearchCondition); + cronJobService.listPageCronJobInfosWithoutVars(cronJobQuery, baseSearchCondition); return IamRespUtil.getSearchInstanceRespFromPageData(cronJobInfoPageData, this::convert); } @@ -110,7 +114,7 @@ protected ListInstanceResponseDTO listInstanceResp( BaseSearchCondition baseSearchCondition = basicQueryCond.getRight(); PageData cronJobInfoPageData = - cronJobService.listPageCronJobInfos(cronJobQuery, baseSearchCondition); + cronJobService.listPageCronJobInfosWithoutVars(cronJobQuery, baseSearchCondition); return IamRespUtil.getListInstanceRespFromPageData(cronJobInfoPageData, this::convert); } @@ -145,7 +149,11 @@ protected CallbackBaseResponseDTO fetchInstanceResp( long id = Long.parseLong(instanceId); cronJobIdList.add(id); } catch (NumberFormatException e) { - log.error("Parse object id failed!|{}", instanceId, e); + String msg = MessageFormatter.format( + "Parse object id failed!|{}", + instanceId + ).getMessage(); + log.error(msg, e); } } Map cronJobInfoMap = cronJobService.getCronJobInfoMapByIds(cronJobIdList); @@ -177,4 +185,12 @@ protected CallbackBaseResponseDTO fetchInstanceResp( public CallbackBaseResponseDTO callback(CallbackRequestDTO callbackRequest) { return baseCallback(callbackRequest); } + + @Override + protected FetchResourceTypeSchemaResponseDTO fetchResourceTypeSchemaResp( + CallbackRequestDTO callbackRequest) { + FetchResourceTypeSchemaResponseDTO resp = new FetchResourceTypeSchemaResponseDTO(); + resp.setData(JsonSchemaUtils.generateJsonSchema(EsbCronInfoV3DTO.class)); + return resp; + } } diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/inner/impl/ServiceCronJobResourceImpl.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/inner/impl/ServiceCronJobResourceImpl.java index 036d8b0b79..2e0e6362de 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/inner/impl/ServiceCronJobResourceImpl.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/inner/impl/ServiceCronJobResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,7 +28,7 @@ import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.InternalResponse; -import com.tencent.bk.job.common.util.ClassUtil; +import com.tencent.bk.job.common.mysql.JobTransactional; import com.tencent.bk.job.common.util.JobContextUtil; import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.common.util.json.JsonUtils; @@ -41,7 +41,6 @@ import com.tencent.bk.job.crontab.model.inner.request.InternalUpdateCronStatusRequest; import com.tencent.bk.job.crontab.service.CronJobService; import lombok.extern.slf4j.Slf4j; -import lombok.val; import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -59,7 +58,7 @@ @RestController public class ServiceCronJobResourceImpl implements ServiceCronJobResource { - private CronJobService cronJobService; + private final CronJobService cronJobService; @Autowired public ServiceCronJobResourceImpl(CronJobService cronJobService) { @@ -77,9 +76,10 @@ public InternalResponse> listCronJobs(Long appId, Boolea BaseSearchCondition baseSearchCondition = new BaseSearchCondition(); baseSearchCondition.setStart(0); baseSearchCondition.setLength(Integer.MAX_VALUE); - val cronJobDTOList = cronJobService.listPageCronJobInfos(cronJobInfoDTO, baseSearchCondition).getData(); - val resultData = cronJobDTOList.stream() - .map(it -> ClassUtil.copyAttrsBetweenClasses(it, ServiceCronJobDTO.class)).collect(Collectors.toList()); + List cronJobDTOList = cronJobService.listPageCronJobInfosWithoutVars(cronJobInfoDTO, + baseSearchCondition).getData(); + List resultData = cronJobDTOList.stream() + .map(CronJobInfoDTO::toServiceCronJobDTO).collect(Collectors.toList()); if (log.isDebugEnabled()) { log.debug("cronJobDTOList.size={}", cronJobDTOList.size()); log.info("resultData=" + JsonUtils.toJson(resultData)); @@ -116,6 +116,7 @@ public InternalResponse>> batchListCronJobByPlanIds(Lo } @Override + @JobTransactional(transactionManager = "jobCrontabTransactionManager") public InternalResponse saveCronJobWithId(String username, Long appId, Long cronJobId, Long createTime, Long lastModifyTime, String lastModifyUser, CronJobCreateUpdateReq cronJobCreateUpdateReq) { diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/inner/impl/ServiceCronMetricsResourceImpl.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/inner/impl/ServiceCronMetricsResourceImpl.java index 0e782b18d4..b777f01b21 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/inner/impl/ServiceCronMetricsResourceImpl.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/inner/impl/ServiceCronMetricsResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/inner/impl/ServiceInnerCronJobResourceImpl.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/inner/impl/ServiceInnerCronJobResourceImpl.java index 84c6b5ceea..bf91ae8126 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/inner/impl/ServiceInnerCronJobResourceImpl.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/inner/impl/ServiceInnerCronJobResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/migration/impl/MigrationResource.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/migration/impl/MigrationResource.java new file mode 100644 index 0000000000..c110b3e32d --- /dev/null +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/migration/impl/MigrationResource.java @@ -0,0 +1,90 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.crontab.api.migration.impl; + +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.model.Response; +import com.tencent.bk.job.common.model.dto.ResourceScope; +import com.tencent.bk.job.common.service.AppScopeMappingService; +import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.crontab.migration.AddHostIdForCronVariableMigrationTask; +import com.tencent.bk.job.manage.model.migration.AddHostIdMigrationReq; +import com.tencent.bk.job.manage.model.migration.AddHostIdResult; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +/** + * 微服务升级 + */ +@RequestMapping("/crontab/migration") +@Slf4j +@RestController("jobCrontabMigrationResource") +public class MigrationResource { + private final AddHostIdForCronVariableMigrationTask addHostIdForCronVariableMigrationTask; + private final AppScopeMappingService appScopeMappingService; + + @Autowired + public MigrationResource(AddHostIdForCronVariableMigrationTask addHostIdForCronVariableMigrationTask, + AppScopeMappingService appScopeMappingService) { + this.addHostIdForCronVariableMigrationTask = addHostIdForCronVariableMigrationTask; + this.appScopeMappingService = appScopeMappingService; + } + + /** + * 定时任务变量包含的主机数据，在原来的云区域+ip的基础上，填充hostID属性 + */ + @PostMapping("/action/addHostIdMigrationTask") + public Response addHostIdMigrationTask(@RequestBody AddHostIdMigrationReq req) { + List results = new ArrayList<>(); + List scopeList = req.getScopeList(); + List appIdList = null; + if (scopeList != null) { + Map scopeAppIdMap = appScopeMappingService.getAppIdByScopeList(scopeList); + if (scopeAppIdMap.size() == scopeList.size()) { + appIdList = new ArrayList<>(scopeAppIdMap.values()); + } else { + scopeList.removeIf(scopeAppIdMap::containsKey); + return Response.buildCommonFailResp(ErrorCode.MIGRATION_FAIL, new String[]{ + "AddHostIdMigrationTask", + "Cannot find appId by scope:" + scopeList + } + ); + } + } + results.add(addHostIdForCronVariableMigrationTask.execute(appIdList, req.isDryRun())); + boolean success = results.stream().allMatch(AddHostIdResult::isSuccess); + return success ? Response.buildSuccessResp(JsonUtils.toJson(results)) : + Response.buildCommonFailResp(ErrorCode.MIGRATION_FAIL, new String[]{"AddHostIdMigrationTask", + JsonUtils.toJson(results)}); + } +} diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/web/impl/WebCronJobResourceImpl.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/web/impl/WebCronJobResourceImpl.java index 3e89544fba..c1d666a4b4 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/web/impl/WebCronJobResourceImpl.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/web/impl/WebCronJobResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,10 +25,11 @@ package com.tencent.bk.job.crontab.api.web.impl; import com.google.common.base.CaseFormat; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.audit.annotations.AuditRequestBody; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.exception.InvalidParamException; -import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; -import com.tencent.bk.job.common.iam.model.AuthResult; +import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; import com.tencent.bk.job.common.model.Response; @@ -43,7 +44,6 @@ import com.tencent.bk.job.crontab.api.web.WebCronJobResource; import com.tencent.bk.job.crontab.auth.CronAuthService; import com.tencent.bk.job.crontab.constant.ExecuteStatusEnum; -import com.tencent.bk.job.crontab.exception.TaskExecuteAuthFailedException; import com.tencent.bk.job.crontab.model.BatchUpdateCronJobReq; import com.tencent.bk.job.crontab.model.CronJobCreateUpdateReq; import com.tencent.bk.job.crontab.model.CronJobLaunchHistoryVO; @@ -51,6 +51,7 @@ import com.tencent.bk.job.crontab.model.dto.CronJobHistoryDTO; import com.tencent.bk.job.crontab.model.dto.CronJobInfoDTO; import com.tencent.bk.job.crontab.model.dto.CronJobLaunchResultStatistics; +import com.tencent.bk.job.crontab.service.CronJobExecuteResultService; import com.tencent.bk.job.crontab.service.CronJobHistoryService; import com.tencent.bk.job.crontab.service.CronJobService; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; @@ -60,6 +61,7 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.StringUtils; +import org.slf4j.helpers.MessageFormatter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RestController; @@ -77,14 +79,17 @@ public class WebCronJobResourceImpl implements WebCronJobResource { private final CronJobService cronJobService; private final CronJobHistoryService cronJobHistoryService; private final CronAuthService cronAuthService; + private final CronJobExecuteResultService cronJobExecuteResultService; @Autowired public WebCronJobResourceImpl(CronJobService cronJobService, CronJobHistoryService cronJobHistoryService, - CronAuthService cronAuthService) { + CronAuthService cronAuthService, + CronJobExecuteResultService cronJobExecuteResultService) { this.cronJobService = cronJobService; this.cronJobHistoryService = cronJobHistoryService; this.cronAuthService = cronAuthService; + this.cronJobExecuteResultService = cronJobExecuteResultService; } @Override @@ -125,7 +130,7 @@ public Response> listCronJobs(String username, } baseSearchCondition.setOrder(order); PageData cronJobInfoPageData = - cronJobService.listPageCronJobInfos(cronJobCondition, baseSearchCondition); + cronJobService.listPageCronJobInfosWithoutVars(cronJobCondition, baseSearchCondition); List resultCronJobs = new ArrayList<>(); cronJobInfoPageData.getData().forEach(cronJobInfo -> resultCronJobs.add(CronJobInfoDTO.toVO(cronJobInfo))); @@ -175,7 +180,7 @@ private void processCronJobPermission(AppResourceScope appResourceScope, PageDat private void processCronJobPermission(AppResourceScope appResourceScope, List cronJobList) { List cronJobIdList = new ArrayList<>(); cronJobList.forEach(cronJob -> cronJobIdList.add(cronJob.getId())); - List allowedCronJob = cronAuthService.batchAuthManageCron( + List allowedCronJob = cronAuthService.getPermissionAllowedCronIds( JobContextUtil.getUsername(), appResourceScope, cronJobIdList); cronJobList.forEach(cronJob -> { cronJob.setCanManage(allowedCronJob.contains(cronJob.getId())); @@ -193,11 +198,15 @@ private void processCronExecuteHistory(Long appId, List resultCronJob Map cronJobLaunchHistory; try { List cronJobIdList = - resultCronJobs.parallelStream().map(CronJobVO::getId).collect(Collectors.toList()); - cronJobExecuteHistory = cronJobService.getCronJobExecuteHistory(appId, cronJobIdList); + resultCronJobs.stream().map(CronJobVO::getId).collect(Collectors.toList()); + cronJobExecuteHistory = cronJobExecuteResultService.getCronJobExecuteHistory(appId, cronJobIdList); cronJobLaunchHistory = cronJobHistoryService.getCronTaskLaunchResultStatistics(appId, cronJobIdList); } catch (Exception e) { - log.error("Error while processing cron history!|{}", appId, e); + String msg = MessageFormatter.format( + "Error while processing cron history!|{}", + appId + ).getMessage(); + log.error(msg, e); return; } if (MapUtils.isEmpty(cronJobExecuteHistory) && MapUtils.isEmpty(cronJobLaunchHistory)) { @@ -320,48 +329,62 @@ private void fillStatisticInfoWithLaunchHistory(CronJobVO resultCronJob, } @Override + @AuditEntry(actionId = ActionId.MANAGE_CRON) public Response getCronJobById(String username, AppResourceScope appResourceScope, String scopeType, String scopeId, Long cronJobId) { Long appId = appResourceScope.getAppId(); + CronJobInfoDTO cronJob = cronJobService.getCronJobInfoById(username, appId, cronJobId); + CronJobVO cronJobVO = CronJobInfoDTO.toVO(cronJob); + return Response.buildSuccessResp(cronJobVO); + } - CronJobVO cronJobVO = CronJobInfoDTO.toVO(cronJobService.getCronJobInfoById(appId, cronJobId)); + @Override + @AuditEntry(actionId = ActionId.CREATE_CRON) + public Response createCronJob(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + @AuditRequestBody CronJobCreateUpdateReq cronJobCreateUpdateReq) { - AuthResult authResult = cronAuthService.authManageCron(username, - appResourceScope, cronJobId, cronJobVO.getName()); - if (authResult.isPass()) { - return Response.buildSuccessResp(cronJobVO); + Long appId = appResourceScope.getAppId(); + checkCronName(cronJobCreateUpdateReq); + CronJobInfoDTO cronJobInfoDTO = CronJobInfoDTO.fromReq(username, appId, cronJobCreateUpdateReq); + if (cronJobInfoDTO.validate()) { + CronJobInfoDTO createdCronJob = cronJobService.createCronJobInfo(username, cronJobInfoDTO); + return Response.buildSuccessResp(CronJobInfoDTO.toVO(createdCronJob)); } else { - throw new PermissionDeniedException(authResult); + log.warn("Validate cron job failed!|{}", JobContextUtil.getDebugMessage()); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } } @Override - public Response saveCronJob(String username, - AppResourceScope appResourceScope, - String scopeType, - String scopeId, - Long cronJobId, - CronJobCreateUpdateReq cronJobCreateUpdateReq) { + @AuditEntry(actionId = ActionId.MANAGE_CRON) + public Response updateCronJob(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + Long cronJobId, + @AuditRequestBody CronJobCreateUpdateReq cronJobCreateUpdateReq) { Long appId = appResourceScope.getAppId(); + cronJobCreateUpdateReq.setId(cronJobId); - if (cronJobId > 0) { - cronJobCreateUpdateReq.setId(cronJobId); - AuthResult authResult = cronAuthService.authManageCron(username, - appResourceScope, cronJobId, null); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } + checkCronName(cronJobCreateUpdateReq); + CronJobInfoDTO cronJobInfoDTO = CronJobInfoDTO.fromReq(username, appId, cronJobCreateUpdateReq); + if (cronJobInfoDTO.validate()) { + CronJobInfoDTO updatedCronJob = cronJobService.updateCronJobInfo(username, cronJobInfoDTO); + return Response.buildSuccessResp(CronJobInfoDTO.toVO(updatedCronJob)); } else { - AuthResult authResult = cronAuthService.authCreateCron(username, - appResourceScope); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } + log.warn("Validate cron job failed!|{}", JobContextUtil.getDebugMessage()); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } + } + + private void checkCronName(CronJobCreateUpdateReq cronJobCreateUpdateReq) { try { StringCheckHelper stringCheckHelper = new StringCheckHelper(new TrimChecker(), new NotEmptyChecker(), new IlegalCharChecker(), new MaxLengthChecker(60)); @@ -370,38 +393,21 @@ public Response saveCronJob(String username, log.warn("Cron Job Name is invalid:", e); throw new InvalidParamException(e, ErrorCode.ILLEGAL_PARAM); } - CronJobInfoDTO cronJobInfoDTO = CronJobInfoDTO.fromReq(username, appId, cronJobCreateUpdateReq); - if (cronJobInfoDTO.validate()) { - try { - Long finalCronJobId = cronJobService.saveCronJobInfo(cronJobInfoDTO); - return Response.buildSuccessResp(finalCronJobId); - } catch (TaskExecuteAuthFailedException e) { - throw new PermissionDeniedException(e.getAuthResult()); - } - } else { - log.warn("Validate cron job failed!|{}", JobContextUtil.getDebugMessage()); - throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); - } } @Override + @AuditEntry(actionId = ActionId.MANAGE_CRON) public Response deleteCronJob(String username, AppResourceScope appResourceScope, String scopeType, String scopeId, Long cronJobId) { - Long appId = appResourceScope.getAppId(); - - AuthResult authResult = cronAuthService.authManageCron(username, - appResourceScope, cronJobId, null); - if (authResult.isPass()) { - return Response.buildSuccessResp(cronJobService.deleteCronJobInfo(appId, cronJobId)); - } else { - throw new PermissionDeniedException(authResult); - } + return Response.buildSuccessResp(cronJobService.deleteCronJobInfo(username, + appResourceScope.getAppId(), cronJobId)); } @Override + @AuditEntry(actionId = ActionId.MANAGE_CRON) public Response changeCronJobEnableStatus(String username, AppResourceScope appResourceScope, String scopeType, @@ -409,19 +415,8 @@ public Response changeCronJobEnableStatus(String username, Long cronJobId, Boolean enable) { Long appId = appResourceScope.getAppId(); - - AuthResult authResult = cronAuthService.authManageCron(username, - new AppResourceScope(appId), cronJobId, null); - if (authResult.isPass()) { - try { - return Response - .buildSuccessResp(cronJobService.changeCronJobEnableStatus(username, appId, cronJobId, enable)); - } catch (TaskExecuteAuthFailedException e) { - throw new PermissionDeniedException(e.getAuthResult()); - } - } else { - throw new PermissionDeniedException(authResult); - } + return Response.buildSuccessResp( + cronJobService.changeCronJobEnableStatus(username, appId, cronJobId, enable)); } @Override @@ -436,23 +431,14 @@ public Response checkCronJobName(String username, } @Override + @AuditEntry(actionId = ActionId.MANAGE_CRON) public Response batchUpdateCronJob(String username, AppResourceScope appResourceScope, String scopeType, String scopeId, - BatchUpdateCronJobReq batchUpdateCronJobReq) { - Long appId = appResourceScope.getAppId(); - - List cronJobInstanceList = new ArrayList<>(); - batchUpdateCronJobReq.getCronJobInfoList() - .forEach(cronJobCreateUpdateReq -> cronJobInstanceList.add(cronJobCreateUpdateReq.getId())); - List allowed = - cronAuthService.batchAuthManageCron(username, appResourceScope, cronJobInstanceList); - if (allowed.size() == cronJobInstanceList.size()) { - return Response.buildSuccessResp(cronJobService.batchUpdateCronJob(appId, batchUpdateCronJobReq)); - } else { - return Response.buildCommonFailResp(ErrorCode.BK_PERMISSION_DENIED); - } + @AuditRequestBody BatchUpdateCronJobReq batchUpdateCronJobReq) { + return Response.buildSuccessResp(cronJobService.batchUpdateCronJob(username, appResourceScope.getAppId(), + batchUpdateCronJobReq)); } @Override @@ -462,11 +448,10 @@ public Response> getCronJobListByPlanId(String username, String scopeId, Long planId) { Long appId = appResourceScope.getAppId(); - List cronJobInfoList = cronJobService.listCronJobByPlanId(appId, planId); if (CollectionUtils.isNotEmpty(cronJobInfoList)) { List cronJobList = - cronJobInfoList.parallelStream().map(CronJobInfoDTO::toBasicVO).collect(Collectors.toList()); + cronJobInfoList.stream().map(CronJobInfoDTO::toBasicVO).collect(Collectors.toList()); processCronJobPermission(appResourceScope, cronJobList); return Response.buildSuccessResp(cronJobList); } else { @@ -488,7 +473,7 @@ public Response>> getCronJobListByPlanIdList(String us for (Map.Entry> cronJobInfoListEntity : cronJobInfoMap.entrySet()) { List cronJobInfoList = cronJobInfoListEntity.getValue(); List cronJobList = - cronJobInfoList.parallelStream().map(CronJobInfoDTO::toBasicVO).collect(Collectors.toList()); + cronJobInfoList.stream().map(CronJobInfoDTO::toBasicVO).collect(Collectors.toList()); processCronJobPermission(appResourceScope, cronJobList); cronJobMap.put(cronJobInfoListEntity.getKey(), cronJobList); } diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/web/impl/WebPermissionResourceImpl.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/web/impl/WebPermissionResourceImpl.java index 2f9ad19fd3..265d3cc8ef 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/web/impl/WebPermissionResourceImpl.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/web/impl/WebPermissionResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -37,7 +37,7 @@ import org.springframework.web.bind.annotation.RestController; @Slf4j -@RestController +@RestController("jobCrontabWebPermissionResourceImpl") public class WebPermissionResourceImpl implements WebPermissionResource { private final WebAuthService webAuthService; private final CronAuthService cronAuthService; diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/web/impl/WebVersionResourceImpl.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/web/impl/WebVersionResourceImpl.java deleted file mode 100644 index 3eb5812573..0000000000 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/api/web/impl/WebVersionResourceImpl.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.crontab.api.web.impl; - -import com.tencent.bk.job.common.web.controller.WebVersionResource; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.actuate.autoconfigure.endpoint.condition.ConditionalOnAvailableEndpoint; -import org.springframework.boot.actuate.info.InfoEndpoint; -import org.springframework.web.bind.annotation.RestController; - -import java.util.Map; - -/** - * @since 17/7/2020 13:47 - */ -@Slf4j -@RestController -@ConditionalOnAvailableEndpoint(endpoint = InfoEndpoint.class) -public class WebVersionResourceImpl implements WebVersionResource { - - private final InfoEndpoint infoEndpoint; - - @Autowired - public WebVersionResourceImpl(InfoEndpoint infoEndpoint) { - this.infoEndpoint = infoEndpoint; - } - - @Override - public Map getVersion() { - return infoEndpoint.info(); - } -} diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/auth/CronAuthService.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/auth/CronAuthService.java index 81ebe18d1c..36be471bfa 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/auth/CronAuthService.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/auth/CronAuthService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,7 @@ package com.tencent.bk.job.crontab.auth; +import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; import com.tencent.bk.job.common.iam.model.AuthResult; import com.tencent.bk.job.common.model.dto.AppResourceScope; @@ -64,9 +65,21 @@ AuthResult authManageCron(String username, * @param cronIdList 定时任务ID列表 * @return 有权限的定时任务ID */ - List batchAuthManageCron(String username, - AppResourceScope appResourceScope, - List cronIdList); + List getPermissionAllowedCronIds(String username, + AppResourceScope appResourceScope, + List cronIdList); + + /** + * 定时任务管理批量鉴权 + * + * @param username 用户账号 + * @param appResourceScope 资源范围 + * @param cronIdList 定时任务ID列表 + * @throws PermissionDeniedException 鉴权未通过 + */ + void batchAuthManageCron(String username, + AppResourceScope appResourceScope, + List cronIdList) throws PermissionDeniedException; /** * 注册定时任务实例 diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/auth/impl/CronAuthServiceImpl.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/auth/impl/CronAuthServiceImpl.java index b3bafc7844..f646ca81b7 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/auth/impl/CronAuthServiceImpl.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/auth/impl/CronAuthServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,12 +27,15 @@ import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.iam.constant.ResourceTypeEnum; import com.tencent.bk.job.common.iam.constant.ResourceTypeId; +import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; import com.tencent.bk.job.common.iam.model.AuthResult; +import com.tencent.bk.job.common.iam.model.PermissionResource; import com.tencent.bk.job.common.iam.service.AppAuthService; import com.tencent.bk.job.common.iam.service.AuthService; import com.tencent.bk.job.common.iam.util.IamUtil; import com.tencent.bk.job.common.model.dto.AppResourceScope; import com.tencent.bk.job.crontab.auth.CronAuthService; +import com.tencent.bk.sdk.iam.constants.SystemId; import com.tencent.bk.sdk.iam.dto.PathInfoDTO; import com.tencent.bk.sdk.iam.util.PathBuilder; import org.springframework.beans.factory.annotation.Autowired; @@ -84,14 +87,38 @@ public AuthResult authManageCron(String username, } @Override - public List batchAuthManageCron(String username, - AppResourceScope appResourceScope, - List cronIdList) { + public List getPermissionAllowedCronIds(String username, + AppResourceScope appResourceScope, + List cronIdList) { List allowedIdList = appAuthService.batchAuth(username, ActionId.MANAGE_CRON, appResourceScope, ResourceTypeEnum.CRON, - cronIdList.parallelStream().map(Objects::toString).collect(Collectors.toList())); - return allowedIdList.parallelStream().map(Long::valueOf).collect(Collectors.toList()); + cronIdList.stream().map(Objects::toString).collect(Collectors.toList())); + return allowedIdList.stream().map(Long::valueOf).collect(Collectors.toList()); + } + + public void batchAuthManageCron(String username, + AppResourceScope appResourceScope, + List cronIdList) throws PermissionDeniedException { + + List allowedIdList = appAuthService.batchAuth(username, ActionId.MANAGE_CRON, + appResourceScope, ResourceTypeEnum.CRON, + cronIdList.stream().map(Objects::toString).collect(Collectors.toList())); + if (cronIdList.size() == allowedIdList.size()) { + return; + } + AuthResult authResult = AuthResult.fail(); + for (Long cronId : cronIdList) { + String cronIdStr = String.valueOf(cronId); + if (!allowedIdList.contains(cronIdStr)) { + PermissionResource resource = new PermissionResource(); + resource.setSystemId(SystemId.JOB); + resource.setResourceId(cronIdStr); + resource.setResourceType(ResourceTypeEnum.CRON); + authResult.addRequiredPermission(ActionId.MANAGE_CRON, resource); + } + } + throw new PermissionDeniedException(authResult); } @Override diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/client/ServiceApplicationResourceClient.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/client/ServiceApplicationResourceClient.java deleted file mode 100644 index 0a39f9213c..0000000000 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/client/ServiceApplicationResourceClient.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.crontab.client; - -import com.tencent.bk.job.manage.api.inner.ServiceApplicationResource; -import org.springframework.cloud.openfeign.FeignClient; - -/** - * 业务服务远程调用客户端 - */ -@FeignClient(value = "job-manage", contextId = "job-manage-app") -public interface ServiceApplicationResourceClient extends ServiceApplicationResource { -} diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/client/ServiceExecuteTaskResourceClient.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/client/ServiceExecuteTaskResourceClient.java deleted file mode 100644 index 744f1c9c5f..0000000000 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/client/ServiceExecuteTaskResourceClient.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.crontab.client; - -import com.tencent.bk.job.execute.api.inner.ServiceExecuteTaskResource; -import org.springframework.cloud.openfeign.FeignClient; - -/** - * 作业执行内部接口远程调用客户端 - * - * @since 17/2/2020 19:54 - */ -@FeignClient(value = "job-execute", contextId = "execute-task") -public interface ServiceExecuteTaskResourceClient extends ServiceExecuteTaskResource { -} diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/client/ServiceNotificationResourceClient.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/client/ServiceNotificationResourceClient.java deleted file mode 100644 index c6c8efdcdb..0000000000 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/client/ServiceNotificationResourceClient.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.crontab.client; - -import com.tencent.bk.job.manage.api.inner.ServiceNotificationResource; -import org.springframework.cloud.openfeign.FeignClient; - -/** - * 消息通知服务内部接口远程调用客户端 - */ -@FeignClient(value = "job-manage", contextId = "notification") -public interface ServiceNotificationResourceClient extends ServiceNotificationResource { -} diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/client/ServiceTaskExecuteResultResourceClient.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/client/ServiceTaskExecuteResultResourceClient.java deleted file mode 100644 index a6eb3f6ed3..0000000000 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/client/ServiceTaskExecuteResultResourceClient.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.crontab.client; - -import com.tencent.bk.job.execute.api.inner.ServiceTaskExecuteResultResource; -import org.springframework.cloud.openfeign.FeignClient; - -/** - * 执行结果内部接口远程调用客户端 - * - * @since 2/3/2020 20:59 - */ -@FeignClient(value = "job-execute", contextId = "execute-result") -public interface ServiceTaskExecuteResultResourceClient extends ServiceTaskExecuteResultResource { -} diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/client/ServiceTaskPlanResourceClient.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/client/ServiceTaskPlanResourceClient.java deleted file mode 100644 index e1650a171d..0000000000 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/client/ServiceTaskPlanResourceClient.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.crontab.client; - -import com.tencent.bk.job.manage.api.inner.ServiceTaskPlanResource; -import org.springframework.cloud.openfeign.FeignClient; - -/** - * 执行方案内部接口远程调用客户端 - * - * @since 27/4/2020 12:12 - */ -@FeignClient(value = "job-manage", contextId = "job-manage-plan") -public interface ServiceTaskPlanResourceClient extends ServiceTaskPlanResource { -} diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/BkConfig.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/BkConfig.java deleted file mode 100644 index 158c0ac857..0000000000 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/BkConfig.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.crontab.config; - -import lombok.Getter; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Configuration; - -@Getter -@Configuration -public class BkConfig { - - @Value("${swagger.url:swagger.job.com}") - private String swaggerUrl; - -} diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/DbConfig.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/DbConfig.java new file mode 100644 index 0000000000..0852ac74d3 --- /dev/null +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/DbConfig.java @@ -0,0 +1,101 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.crontab.config; + +import com.tencent.bk.job.common.mysql.util.JooqConfigurationUtil; +import org.jooq.ConnectionProvider; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; +import org.jooq.impl.DataSourceConnectionProvider; +import org.jooq.impl.DefaultConfiguration; +import org.jooq.impl.DefaultDSLContext; +import org.jooq.impl.DefaultExecuteListenerProvider; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.jdbc.DataSourceBuilder; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.DependsOn; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.DataSourceTransactionManager; +import org.springframework.jdbc.datasource.TransactionAwareDataSourceProxy; +import org.springframework.transaction.annotation.EnableTransactionManagement; + +import javax.sql.DataSource; + +@Configuration(value = "jobCrontabDbConfig") +@EnableTransactionManagement +public class DbConfig { + @Qualifier("job-crontab-data-source") + @Bean(name = "job-crontab-data-source") + @ConfigurationProperties(prefix = "spring.datasource.job-crontab") + public DataSource dataSource() { + return DataSourceBuilder.create().build(); + } + + @Qualifier("jobCrontabTransactionManager") + @Bean(name = "jobCrontabTransactionManager") + @DependsOn("job-crontab-data-source") + public DataSourceTransactionManager + transactionManager(@Qualifier("job-crontab-data-source") DataSource dataSource) { + return new DataSourceTransactionManager(dataSource); + } + + @Qualifier("job-crontab-jdbc-template") + @Bean(name = "job-crontab-jdbc-template") + public JdbcTemplate jdbcTemplate(@Qualifier("job-crontab-data-source") DataSource dataSource) { + return new JdbcTemplate(dataSource); + } + + @Qualifier("job-crontab-dsl-context") + @Bean(name = "job-crontab-dsl-context") + public DSLContext dslContext(@Qualifier("job-crontab-jooq-conf") org.jooq.Configuration configuration) { + return new DefaultDSLContext(configuration); + } + + @Qualifier("job-crontab-jooq-conf") + @Bean(name = "job-crontab-jooq-conf") + public org.jooq.Configuration + jooqConf(@Qualifier("job-crontab-conn-provider") ConnectionProvider connectionProvider, + DefaultExecuteListenerProvider jooqExecuteListenerProvider + ) { + return JooqConfigurationUtil.getConfiguration(connectionProvider, jooqExecuteListenerProvider); + } + + @Qualifier("job-crontab-conn-provider") + @Bean(name = "job-crontab-conn-provider") + public ConnectionProvider connectionProvider( + @Qualifier("jobCrontabTransactionAwareDataSource") TransactionAwareDataSourceProxy transactionAwareDataSource) { + return new DataSourceConnectionProvider(transactionAwareDataSource); + } + + @Qualifier("jobCrontabTransactionAwareDataSource") + @Bean(name = "jobCrontabTransactionAwareDataSource") + public TransactionAwareDataSourceProxy + transactionAwareDataSourceProxy(@Qualifier("job-crontab-data-source") DataSource dataSource) { + return new TransactionAwareDataSourceProxy(dataSource); + } + +} diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/ExecutorConfiguration.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/ExecutorConfiguration.java index 233b727444..120d9c9e39 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/ExecutorConfiguration.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/ExecutorConfiguration.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,27 +24,31 @@ package com.tencent.bk.job.crontab.config; +import com.tencent.bk.job.common.WatchableThreadPoolExecutor; +import io.micrometer.core.instrument.MeterRegistry; import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; + +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; @Slf4j -@Configuration +@Configuration(value = "jobCrontabExecutorConfig") public class ExecutorConfiguration { - @Bean - public ThreadPoolTaskExecutor quartzTaskExecutor(QuartzProperties quartzProperties) { - ThreadPoolTaskExecutor threadPoolTaskExecutor = new ThreadPoolTaskExecutor(); - threadPoolTaskExecutor.setCorePoolSize(quartzProperties.getThreadPool().getCorePoolSize()); - threadPoolTaskExecutor.setMaxPoolSize(quartzProperties.getThreadPool().getMaxPoolSize()); - threadPoolTaskExecutor.setQueueCapacity(quartzProperties.getThreadPool().getQueueCapacity()); - threadPoolTaskExecutor.setKeepAliveSeconds(quartzProperties.getThreadPool().getKeepAliveSeconds()); - threadPoolTaskExecutor.setWaitForTasksToCompleteOnShutdown( - quartzProperties.getThreadPool().isWaitForTasksToCompleteOnShutdown()); - threadPoolTaskExecutor.setAwaitTerminationSeconds( - quartzProperties.getThreadPool().getAwaitTerminationSeconds()); - - return threadPoolTaskExecutor; + @Bean("crontabInitRunnerExecutor") + public ThreadPoolExecutor crontabInitRunnerExecutor(MeterRegistry meterRegistry) { + return new WatchableThreadPoolExecutor( + meterRegistry, + "initRunnerExecutor", + 0, + 5, + 1, + TimeUnit.SECONDS, + new LinkedBlockingQueue<>() + ); } + } diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/FeignConfig.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/FeignConfig.java new file mode 100644 index 0000000000..6ceb43d7d5 --- /dev/null +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/FeignConfig.java @@ -0,0 +1,112 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.crontab.config; + +import com.tencent.bk.job.common.util.http.JobHttpClientConnectionManagerFactory; +import feign.Client; +import feign.httpclient.ApacheHttpClient; +import lombok.extern.slf4j.Slf4j; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.conn.HttpClientConnectionManager; +import org.apache.http.conn.socket.LayeredConnectionSocketFactory; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.conn.ssl.TrustSelfSignedStrategy; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.impl.client.StandardHttpRequestRetryHandler; +import org.apache.http.ssl.SSLContexts; +import org.springframework.cloud.client.loadbalancer.LoadBalancedRetryFactory; +import org.springframework.cloud.client.loadbalancer.LoadBalancerClient; +import org.springframework.cloud.loadbalancer.support.LoadBalancerClientFactory; +import org.springframework.cloud.openfeign.loadbalancer.RetryableFeignBlockingLoadBalancerClient; +import org.springframework.cloud.openfeign.support.FeignHttpClientProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import java.security.KeyManagementException; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; +import java.util.concurrent.TimeUnit; + +@Slf4j +@Configuration +public class FeignConfig { + + @Bean + public HttpClientConnectionManager feignClientConnectionManager(FeignHttpClientProperties httpClientProperties) { + LayeredConnectionSocketFactory sslSocketFactory = null; + try { + sslSocketFactory = new SSLConnectionSocketFactory( + SSLContexts.custom() + .loadTrustMaterial(null, new TrustSelfSignedStrategy()) + .build() + ); + } catch (NoSuchAlgorithmException | KeyManagementException | KeyStoreException e) { + log.error("Fail to create SSLConnectionSocketFactory", e); + } + return JobHttpClientConnectionManagerFactory.createWatchableConnectionManager( + httpClientProperties.getMaxConnectionsPerRoute(), + httpClientProperties.getMaxConnections(), + // 服务端Keep-Alive timeout为60s，此处45s+4s(默认10s)<60s即可 + 45, + TimeUnit.SECONDS, + sslSocketFactory + ); + } + + public ApacheHttpClient getApacheHttpClient(FeignHttpClientProperties httpClientProperties, + HttpClientConnectionManager feignClientConnectionManager) { + RequestConfig defaultRequestConfig = RequestConfig.custom() + .setConnectTimeout(httpClientProperties.getConnectionTimeout()) + .setRedirectsEnabled(httpClientProperties.isFollowRedirects()).build(); + HttpClientBuilder httpClientBuilder = HttpClientBuilder.create() + .setDefaultRequestConfig(defaultRequestConfig) + .evictExpiredConnections() + .evictIdleConnections(4000, TimeUnit.MILLISECONDS) + .disableAutomaticRetries() + .disableAuthCaching() + .disableCookieManagement(); + httpClientBuilder.setRetryHandler(new StandardHttpRequestRetryHandler()); + CloseableHttpClient httpClient; + httpClientBuilder.setConnectionManager(feignClientConnectionManager); + httpClient = httpClientBuilder.build(); + return new ApacheHttpClient(httpClient); + } + + @Bean + public Client feignClient(FeignHttpClientProperties httpClientProperties, + HttpClientConnectionManager feignClientConnectionManager, + LoadBalancerClient loadBalancerClient, + LoadBalancedRetryFactory loadBalancedRetryFactory, + LoadBalancerClientFactory loadBalancerClientFactory) { + ApacheHttpClient delegate = getApacheHttpClient(httpClientProperties, feignClientConnectionManager); + return new RetryableFeignBlockingLoadBalancerClient( + delegate, + loadBalancerClient, + loadBalancedRetryFactory, + loadBalancerClientFactory + ); + } +} diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/JobCrontabAutoConfiguration.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/JobCrontabAutoConfiguration.java deleted file mode 100644 index f8546152eb..0000000000 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/JobCrontabAutoConfiguration.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.crontab.config; - -import com.tencent.bk.job.common.esb.metrics.EsbApiTimedAspect; -import com.tencent.bk.job.common.service.AppScopeMappingService; -import com.tencent.bk.job.crontab.client.ServiceApplicationResourceClient; -import com.tencent.bk.job.manage.AppScopeMappingServiceImpl; -import io.micrometer.core.instrument.MeterRegistry; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Configuration -public class JobCrontabAutoConfiguration { - @Bean - public EsbApiTimedAspect esbApiTimedAspect(@Autowired MeterRegistry meterRegistry) { - return new EsbApiTimedAspect(meterRegistry); - } - - @Bean - AppScopeMappingService appScopeMappingService(ServiceApplicationResourceClient applicationResource) { - return new AppScopeMappingServiceImpl(applicationResource); - } -} diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/JobFunctionConfiguration.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/JobFunctionConfiguration.java new file mode 100644 index 0000000000..5ca164f502 --- /dev/null +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/JobFunctionConfiguration.java @@ -0,0 +1,50 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.crontab.config; + +import com.tencent.bk.job.crontab.listener.CrontabEventListener; +import com.tencent.bk.job.crontab.listener.event.CrontabEvent; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import java.util.function.Consumer; + +/** + * spring cloud function 定义 + *

+ * 注意：方法名与配置文件中的spring.cloud.function.definition对应，修改需要注意！！！ + */ +@Configuration +@Slf4j +public class JobFunctionConfiguration { + @Bean + public Consumer handleCrontabFanoutEvent(@Autowired CrontabEventListener crontabEventListener) { + log.info("Init handleCrontabFanoutEvent consumer"); + return crontabEventListener::handleEvent; + } + +} diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/QuartzConfig.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/QuartzConfig.java index f0b5dfe803..5aa760337a 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/QuartzConfig.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/QuartzConfig.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,71 +24,28 @@ package com.tencent.bk.job.crontab.config; -import com.tencent.bk.job.crontab.timer.AutowiredSpringBeanJobFactory; -import org.quartz.Scheduler; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.quartz.SchedulerFactoryBeanCustomizer; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Profile; -import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; -import org.springframework.scheduling.quartz.SchedulerFactoryBean; -import org.springframework.transaction.PlatformTransactionManager; import javax.sql.DataSource; -import java.util.Map; -import java.util.Properties; /** - * @since 14/1/2020 16:57 + * Quartz 设置 */ @Configuration -@Profile({"prod", "dev"}) +@Profile("!test") public class QuartzConfig { - private final PlatformTransactionManager transactionManager; - private final QuartzProperties quartzProperties; - private final DataSource dataSource; - private final AutowiredSpringBeanJobFactory autowiredSpringBeanJobFactory; - - @Autowired - public QuartzConfig(PlatformTransactionManager transactionManager, QuartzProperties quartzProperties, - @Qualifier("job-crontab-data-source") DataSource dataSource, - AutowiredSpringBeanJobFactory autowiredSpringBeanJobFactory) { - this.transactionManager = transactionManager; - this.quartzProperties = quartzProperties; - this.dataSource = dataSource; - this.autowiredSpringBeanJobFactory = autowiredSpringBeanJobFactory; - } - @Bean - public SchedulerFactoryBean schedulerFactoryBean(ThreadPoolTaskExecutor quartzTaskExecutor) { - SchedulerFactoryBean schedulerFactoryBean = new SchedulerFactoryBean(); - schedulerFactoryBean.setConfigLocation(quartzProperties.getScheduler().getConfigLocation()); - // 此处设置数据源之后，会覆盖quartz.properties中的myDS数据源 - schedulerFactoryBean.setDataSource(dataSource); - schedulerFactoryBean.setJobFactory(autowiredSpringBeanJobFactory); - schedulerFactoryBean.setSchedulerName(quartzProperties.getScheduler().getSchedulerName()); - schedulerFactoryBean.setTaskExecutor(quartzTaskExecutor); - schedulerFactoryBean.setTransactionManager(transactionManager); - schedulerFactoryBean.setApplicationContextSchedulerContextKey( - quartzProperties.getScheduler().getApplicationContextSchedulerContextKey()); - schedulerFactoryBean.setOverwriteExistingJobs(quartzProperties.getScheduler().isOverwriteExistingJobs()); - schedulerFactoryBean.setAutoStartup(quartzProperties.getScheduler().isAutoStartup()); - schedulerFactoryBean.setStartupDelay(quartzProperties.getScheduler().getStartupDelay()); - schedulerFactoryBean.setQuartzProperties(asProperties(quartzProperties.getProperties())); - - return schedulerFactoryBean; - } - - @Bean - public Scheduler scheduler(SchedulerFactoryBean schedulerFactoryBean) { - return schedulerFactoryBean.getObject(); - } - - private Properties asProperties(Map source) { - Properties properties = new Properties(); - properties.putAll(source); - return properties; + public SchedulerFactoryBeanCustomizer schedulerFactoryBeanCustomizer( + @Qualifier("job-crontab-data-source") DataSource dataSource) { + return schedulerFactoryBean -> { + schedulerFactoryBean.setOverwriteExistingJobs(true); + schedulerFactoryBean.setStartupDelay(10); + schedulerFactoryBean.setDataSource(dataSource); + }; } } diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/QuartzProperties.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/QuartzProperties.java deleted file mode 100644 index 5102339ff0..0000000000 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/QuartzProperties.java +++ /dev/null @@ -1,95 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.crontab.config; - -import lombok.Data; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.core.io.Resource; - -import java.util.HashMap; -import java.util.Map; - -/** - * @since 1.0 - */ -@Data -@ConfigurationProperties(prefix = "spring.quartz") -public class QuartzProperties { - - /** - * Additional Quartz Scheduler properties. - */ - private final Map properties = new HashMap<>(); - private ThreadPool threadPool = new ThreadPool(); - private Scheduler scheduler = new Scheduler(); - - public QuartzProperties() { - - } - - public Map getProperties() { - return this.properties; - } - - @Data - public static class ThreadPool { - private String threadNamePrefix; - - private int threadPriority = 5; - - private boolean daemon = false; - - private String threadGroupName; - - private int corePoolSize = 10; - - private int maxPoolSize = Integer.MAX_VALUE; - - private int keepAliveSeconds = 60; - - private int queueCapacity = Integer.MAX_VALUE; - - private boolean allowCoreThreadTimeOut = false; - - private boolean waitForTasksToCompleteOnShutdown = false; - - private int awaitTerminationSeconds = 0; - } - - @Data - public static class Scheduler { - private Resource configLocation; - - private String schedulerName; - - private String applicationContextSchedulerContextKey = "applicationContext"; - - private boolean overwriteExistingJobs = true; - - private boolean autoStartup = true; - - private int startupDelay = 5; - } -} diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/ScheduleConfig.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/ScheduleConfig.java new file mode 100644 index 0000000000..f396ca5e39 --- /dev/null +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/config/ScheduleConfig.java @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.crontab.config; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.annotation.SchedulingConfigurer; +import org.springframework.scheduling.config.ScheduledTaskRegistrar; + +import java.util.concurrent.ScheduledThreadPoolExecutor; + +//设定一个长度5的定时任务线程池 +@Slf4j +@Configuration +public class ScheduleConfig implements SchedulingConfigurer { + @Override + public void configureTasks(ScheduledTaskRegistrar taskRegistrar) { + taskRegistrar.setScheduler( + new ScheduledThreadPoolExecutor( + 5, (r, executor) -> + log.error("ScheduledThreadPoolExecutor rejected a runnable") + ) + ); + } +} diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/constant/CronConstants.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/constant/CronConstants.java index 7878a65d4b..025f14c987 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/constant/CronConstants.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/constant/CronConstants.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/constant/CrontabActionEnum.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/constant/CrontabActionEnum.java new file mode 100644 index 0000000000..23056e563c --- /dev/null +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/constant/CrontabActionEnum.java @@ -0,0 +1,58 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.crontab.constant; + +/** + * 定时任务相关操作 + */ +public enum CrontabActionEnum { + /** + * 添加定时任务 + */ + ADD_CRON(1), + /** + * 删除定时任务 + */ + DELETE_CRON(2); + + private final int value; + + CrontabActionEnum(int val) { + this.value = val; + } + + public static CrontabActionEnum valueOf(int value) { + for (CrontabActionEnum crontabAction : values()) { + if (crontabAction.getValue() == value) { + return crontabAction; + } + } + throw new IllegalArgumentException("No CrontabActionEnum constant: " + value); + } + + public int getValue() { + return value; + } +} diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/constant/ExecuteStatusEnum.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/constant/ExecuteStatusEnum.java index 19e2268a02..e042a01a3d 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/constant/ExecuteStatusEnum.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/constant/ExecuteStatusEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/dao/CronJobDAO.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/dao/CronJobDAO.java index 96f9da587a..dc81e4e524 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/dao/CronJobDAO.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/dao/CronJobDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,7 +26,9 @@ import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; +import com.tencent.bk.job.crontab.model.dto.CronJobBasicInfoDTO; import com.tencent.bk.job.crontab.model.dto.CronJobInfoDTO; +import com.tencent.bk.job.crontab.model.dto.CronJobWithVarsDTO; import java.util.List; @@ -35,15 +37,26 @@ */ public interface CronJobDAO { + // 查询 + /** - * 根据业务 ID 列表批量查询定时任务信息 + * 查询含有主机变量信息一批定时任务基础信息 + * + * @param appIdList 限定的appId范围 + * @param start 起始值 + * @param limit 最大记录数 + */ + List listBasicCronJobWithHostVars(List appIdList, int start, int limit); + + /** + * 根据业务 ID 列表批量查询定时任务信息（不含变量） * * @param cronJobCondition 查询参数 * @param baseSearchCondition 分页信息 * @return 分页的定时任务信息列表 */ - PageData listPageCronJobsByCondition(CronJobInfoDTO cronJobCondition, - BaseSearchCondition baseSearchCondition); + PageData listPageCronJobsWithoutVarsByCondition(CronJobInfoDTO cronJobCondition, + BaseSearchCondition baseSearchCondition); /** * 根据定时任务 ID 查询定时任务信息 @@ -59,7 +72,7 @@ PageData listPageCronJobsByCondition(CronJobInfoDTO cronJobCondi * @param cronJobIdList 定时任务 IDs * @return 定时任务信息 */ - List getCronJobByIds(List cronJobIdList); + List listCronJobByIds(List cronJobIdList); /** * 根据定时任务 ID 查询定时任务信息 @@ -79,39 +92,6 @@ PageData listPageCronJobsByCondition(CronJobInfoDTO cronJobCondi */ CronJobInfoDTO getCronJobErrorById(long appId, long cronJobId); - /** - * 新增定时任务信息 - * - * @param cronJob 定时任务信息 - * @return 新增定时任务信息的 ID - */ - long insertCronJob(CronJobInfoDTO cronJob); - - /** - * 根据 ID 和定时任务 ID 更新定时任务 - * - * @param cronJob 定时任务信息 - * @return 是否更新成功 - */ - boolean updateCronJobById(CronJobInfoDTO cronJob); - - /** - * 根据 ID 和定时任务 ID 更新定时任务错误信息 - * - * @param cronJobErrorInfo 定时任务错误信息 - * @return 是否更新成功 - */ - boolean updateCronJobErrorById(CronJobInfoDTO cronJobErrorInfo); - - /** - * 根据定时任务 ID 删除定时任务 - * - * @param appId 业务 ID - * @param cronJobId 定时任务 ID - * @return 是否删除成功 - */ - boolean deleteCronJobById(long appId, long cronJobId); - /** * 检查定时任务名称是否可用 * @@ -131,14 +111,6 @@ PageData listPageCronJobsByCondition(CronJobInfoDTO cronJobCondi */ List listCronJobByPlanId(long appId, long planId); - /** - * 保留 ID 插入定时任务 - * - * @param cronJob 定时任务信息 - * @return 是否插入成功 - */ - boolean insertCronJobWithId(CronJobInfoDTO cronJob); - /** * 根据定时任务 ID 查询定时任务名称 * @@ -165,4 +137,71 @@ PageData listPageCronJobsByCondition(CronJobInfoDTO cronJobCondi boolean isExistAnyAppCronJob(Long appId); Integer countCronJob(Long appId, Boolean active, Boolean cron); + + List listEnabledCronBasicInfoForUpdate(int start, int limit); + + // 新增 + + /** + * 新增定时任务信息 + * + * @param cronJob 定时任务信息 + * @return 新增定时任务信息的 ID + */ + long insertCronJob(CronJobInfoDTO cronJob); + + /** + * 保留 ID 插入定时任务 + * + * @param cronJob 定时任务信息 + * @return 是否插入成功 + */ + boolean insertCronJobWithId(CronJobInfoDTO cronJob); + + + // 修改 + + /** + * 根据 ID 和定时任务 ID 更新定时任务 + * + * @param cronJob 定时任务信息 + * @return 是否更新成功 + */ + boolean updateCronJobById(CronJobInfoDTO cronJob); + + /** + * 根据 ID 和定时任务 ID 更新定时任务错误信息 + * + * @param cronJobErrorInfo 定时任务错误信息 + * @return 是否更新成功 + */ + boolean updateCronJobErrorById(CronJobInfoDTO cronJobErrorInfo); + + /** + * 根据ID更新定时任务的变量值 + * + * @param id 定时任务ID + * @param cronJobWithVarsDTO 含变量的定时任务数据 + * @return 受影响行数 + */ + int updateVariableById(Long id, CronJobWithVarsDTO cronJobWithVarsDTO); + + // 删除 + + /** + * 根据定时任务 ID 删除定时任务 + * + * @param appId 业务 ID + * @param cronJobId 定时任务 ID + * @return 是否删除成功 + */ + boolean deleteCronJobById(long appId, long cronJobId); + + /** + * 查询定时任务id + * + * @param cronJobInfoDTO 定时任务信息 + * @return 定时任务id列表 + */ + List listCronJobIds(CronJobInfoDTO cronJobInfoDTO); } diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/dao/CronJobHistoryDAO.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/dao/CronJobHistoryDAO.java index 0b0c21195c..ce64eae2cb 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/dao/CronJobHistoryDAO.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/dao/CronJobHistoryDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/dao/InnerCronJobHistoryDAO.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/dao/InnerCronJobHistoryDAO.java index f61c216b9d..9e1ea49669 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/dao/InnerCronJobHistoryDAO.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/dao/InnerCronJobHistoryDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/dao/impl/CronJobDAOImpl.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/dao/impl/CronJobDAOImpl.java index 8ead16d1e8..2f135dbe2a 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/dao/impl/CronJobDAOImpl.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/dao/impl/CronJobDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -30,24 +30,31 @@ import com.tencent.bk.job.common.model.dto.UserRoleInfoDTO; import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.crontab.dao.CronJobDAO; +import com.tencent.bk.job.crontab.model.dto.CronJobBasicInfoDTO; import com.tencent.bk.job.crontab.model.dto.CronJobInfoDTO; import com.tencent.bk.job.crontab.model.dto.CronJobVariableDTO; +import com.tencent.bk.job.crontab.model.dto.CronJobWithVarsDTO; +import com.tencent.bk.job.crontab.model.tables.CronJob; +import com.tencent.bk.job.crontab.model.tables.records.CronJobRecord; import com.tencent.bk.job.crontab.util.DbRecordMapper; import com.tencent.bk.job.crontab.util.DbUtils; import lombok.extern.slf4j.Slf4j; +import lombok.val; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.jooq.Condition; import org.jooq.DSLContext; import org.jooq.OrderField; +import org.jooq.Record; import org.jooq.Record1; import org.jooq.Record21; +import org.jooq.Record3; +import org.jooq.Record4; import org.jooq.Record5; import org.jooq.Result; import org.jooq.TableField; import org.jooq.UpdateSetMoreStep; -import org.jooq.generated.tables.CronJob; -import org.jooq.generated.tables.records.CronJobRecord; +import org.jooq.conf.ParamType; import org.jooq.types.UByte; import org.jooq.types.UInteger; import org.jooq.types.ULong; @@ -87,8 +94,32 @@ private static OrderField buildOrderField(TableField field, } @Override - public PageData listPageCronJobsByCondition(CronJobInfoDTO cronJobCondition, - BaseSearchCondition baseSearchCondition) { + public List listBasicCronJobWithHostVars(List appIdList, int start, int limit) { + List conditions = new ArrayList<>(); + conditions.add(TABLE.VARIABLE_VALUE.like("%ips\":[{%")); + if (appIdList != null) { + conditions.add(TABLE.APP_ID.in(appIdList)); + } + Result> records = context + .select(TABLE.ID, TABLE.APP_ID, TABLE.NAME, TABLE.VARIABLE_VALUE) + .from(TABLE) + .where(conditions) + .orderBy(TABLE.ID) + .limit(start, limit) + .fetch(); + return records.map(record -> { + CronJobWithVarsDTO cronJobWithVarsDTO = new CronJobWithVarsDTO(); + cronJobWithVarsDTO.setId(record.get(TABLE.ID).longValue()); + cronJobWithVarsDTO.setAppId(record.get(TABLE.APP_ID).longValue()); + cronJobWithVarsDTO.setName(record.get(TABLE.NAME)); + cronJobWithVarsDTO.decryptAndSetVariableValue(record.get(TABLE.VARIABLE_VALUE)); + return cronJobWithVarsDTO; + }); + } + + @Override + public PageData listPageCronJobsWithoutVarsByCondition(CronJobInfoDTO cronJobCondition, + BaseSearchCondition baseSearchCondition) { List conditions = buildConditionList(cronJobCondition, baseSearchCondition); long templateCount = getPageCronJobCount(conditions); List> orderFields = new ArrayList<>(); @@ -113,17 +144,35 @@ public PageData listPageCronJobsByCondition(CronJobInfoDTO cronJ int start = baseSearchCondition.getStartOrDefault(0); int length = baseSearchCondition.getLengthOrDefault(10); - Result> records = - context - .select(TABLE.ID, TABLE.APP_ID, TABLE.NAME, TABLE.CREATOR, TABLE.TASK_TEMPLATE_ID, - TABLE.TASK_PLAN_ID, TABLE.SCRIPT_ID, TABLE.SCRIPT_VERSION_ID, TABLE.CRON_EXPRESSION, - TABLE.EXECUTE_TIME, TABLE.VARIABLE_VALUE, TABLE.LAST_EXECUTE_STATUS, TABLE.IS_ENABLE, - TABLE.IS_DELETED, TABLE.CREATE_TIME, TABLE.LAST_MODIFY_USER, TABLE.LAST_MODIFY_TIME, - TABLE.END_TIME, TABLE.NOTIFY_OFFSET, TABLE.NOTIFY_USER, TABLE.NOTIFY_CHANNEL) - .from(TABLE).where(conditions).orderBy(orderFields).limit(start, length).fetch(); + Result records = + context.select( + TABLE.ID, + TABLE.APP_ID, + TABLE.NAME, + TABLE.CREATOR, + TABLE.TASK_TEMPLATE_ID, + TABLE.TASK_PLAN_ID, + TABLE.SCRIPT_ID, + TABLE.SCRIPT_VERSION_ID, + TABLE.CRON_EXPRESSION, + TABLE.EXECUTE_TIME, + TABLE.LAST_EXECUTE_STATUS, + TABLE.IS_ENABLE, + TABLE.IS_DELETED, + TABLE.CREATE_TIME, + TABLE.LAST_MODIFY_USER, + TABLE.LAST_MODIFY_TIME, + TABLE.END_TIME, + TABLE.NOTIFY_OFFSET, + TABLE.NOTIFY_USER, + TABLE.NOTIFY_CHANNEL + ).from(TABLE) + .where(conditions) + .orderBy(orderFields) + .limit(start, length) + .fetch(); List cronJobInfoList = new ArrayList<>(); - if (records != null && records.size() >= 1) { + if (!records.isEmpty()) { records.map(record -> cronJobInfoList.add(convertToCronJobDTO(record))); } PageData cronJobInfoPageData = new PageData<>(); @@ -169,7 +218,9 @@ private List buildConditionList(CronJobInfoDTO cronJobCondition, } private long getPageCronJobCount(List conditions) { - return context.selectCount().from(TABLE).where(conditions).fetchOne(0, Long.class); + Long count = context.selectCount().from(TABLE).where(conditions).fetchOne(0, Long.class); + assert count != null; + return count; } @Override @@ -181,9 +232,9 @@ public CronJobInfoDTO getCronJobById(long cronJobId) { } @Override - public List getCronJobByIds(List cronJobIdList) { + public List listCronJobByIds(List cronJobIdList) { List conditions = new ArrayList<>(); - conditions.add(TABLE.ID.in(cronJobIdList.parallelStream().map(ULong::valueOf).collect(Collectors.toList()))); + conditions.add(TABLE.ID.in(cronJobIdList.stream().map(ULong::valueOf).collect(Collectors.toList()))); conditions.add(TABLE.IS_DELETED.equal(UByte.valueOf(0))); return fetchData(conditions); } @@ -247,7 +298,7 @@ public long insertCronJob(CronJobInfoDTO cronJob) { DbUtils.getJooqLongValue(cronJob.getTaskTemplateId()), DbUtils.getJooqLongValue(cronJob.getTaskPlanId()), cronJob.getScriptId(), DbUtils.getJooqLongValue(cronJob.getScriptVersionId()), cronJob.getCronExpression(), - DbUtils.getJooqLongValue(cronJob.getExecuteTime()), JsonUtils.toJson(cronJob.getVariableValue()), + DbUtils.getJooqLongValue(cronJob.getExecuteTime()), cronJob.getEncryptedVariableValue(), lastExecuteStatus, DbUtils.getBooleanValue(cronJob.getEnable()), DbUtils.getBooleanValue(cronJob.getDelete()), ULong.valueOf(cronJob.getCreateTime()), cronJob.getLastModifyUser(), ULong.valueOf(cronJob.getLastModifyTime()), @@ -304,7 +355,7 @@ public boolean updateCronJobById(CronJobInfoDTO cronJob) { updateStep.setNull(TABLE.CRON_EXPRESSION); } if (CollectionUtils.isNotEmpty(cronJob.getVariableValue())) { - updateStep = updateStep.set(TABLE.VARIABLE_VALUE, JsonUtils.toJson(cronJob.getVariableValue())); + updateStep = updateStep.set(TABLE.VARIABLE_VALUE, cronJob.getEncryptedVariableValue()); } if (cronJob.getLastExecuteStatus() != null) { updateStep = updateStep.set(TABLE.LAST_EXECUTE_STATUS, UByte.valueOf(cronJob.getLastExecuteStatus())); @@ -345,23 +396,39 @@ public boolean updateCronJobErrorById(CronJobInfoDTO cronJobErrorInfo) { conditions.add(TABLE.IS_DELETED.equal(UByte.valueOf(0))); UpdateSetMoreStep updateStep = - context.update(TABLE).set(TABLE.LAST_EXECUTE_STATUS, UByte.valueOf(cronJobErrorInfo.getLastExecuteStatus())); + context.update(TABLE).set(TABLE.LAST_EXECUTE_STATUS, + UByte.valueOf(cronJobErrorInfo.getLastExecuteStatus())); if (cronJobErrorInfo.getLastExecuteErrorCode() != null) { - updateStep = updateStep.set(TABLE.LAST_EXECUTE_ERROR_CODE, ULong.valueOf(cronJobErrorInfo.getLastExecuteErrorCode())); + updateStep = updateStep.set(TABLE.LAST_EXECUTE_ERROR_CODE, + ULong.valueOf(cronJobErrorInfo.getLastExecuteErrorCode())); } if (cronJobErrorInfo.getLastExecuteErrorCount() != null) { - updateStep = updateStep.set(TABLE.LAST_EXECUTE_ERROR_COUNT, UInteger.valueOf(cronJobErrorInfo.getLastExecuteErrorCount())); + updateStep = updateStep.set(TABLE.LAST_EXECUTE_ERROR_COUNT, + UInteger.valueOf(cronJobErrorInfo.getLastExecuteErrorCount())); } return 1 == updateStep.where(conditions).limit(1).execute(); } + @Override + public int updateVariableById(Long id, CronJobWithVarsDTO cronJobWithVarsDTO) { + return context + .update(TABLE) + .set(TABLE.VARIABLE_VALUE, cronJobWithVarsDTO.getEncryptedVariableValue()) + .where(TABLE.ID.eq(ULong.valueOf(id))) + .execute(); + } + @Override public boolean deleteCronJobById(long appId, long cronJobId) { List conditions = new ArrayList<>(); conditions.add(TABLE.ID.equal(ULong.valueOf(cronJobId))); conditions.add(TABLE.APP_ID.equal(ULong.valueOf(appId))); conditions.add(TABLE.IS_DELETED.equal(UByte.valueOf(0))); - return context.update(TABLE).set(TABLE.IS_DELETED, UByte.valueOf(1)).where(conditions).limit(1).execute() == 1; + return context.update(TABLE) + .set(TABLE.IS_DELETED, UByte.valueOf(1)) + .where(conditions) + .limit(1) + .execute() == 1; } @Override @@ -373,7 +440,9 @@ public boolean checkCronJobName(long appId, long cronJobId, String name) { conditions.add(TABLE.ID.notEqual(ULong.valueOf(cronJobId))); } conditions.add(TABLE.NAME.equal(name)); - return context.selectCount().from(TABLE).where(conditions).fetchOne(0, Integer.class) == 0; + Integer count = context.selectCount().from(TABLE).where(conditions).fetchOne(0, Integer.class); + assert count != null; + return count == 0; } @Override @@ -399,7 +468,7 @@ public boolean insertCronJobWithId(CronJobInfoDTO cronJob) { cronJob.getCreator(), DbUtils.getJooqLongValue(cronJob.getTaskTemplateId()), DbUtils.getJooqLongValue(cronJob.getTaskPlanId()), cronJob.getScriptId(), DbUtils.getJooqLongValue(cronJob.getScriptVersionId()), cronJob.getCronExpression(), - DbUtils.getJooqLongValue(cronJob.getExecuteTime()), JsonUtils.toJson(cronJob.getVariableValue()), + DbUtils.getJooqLongValue(cronJob.getExecuteTime()), cronJob.getEncryptedVariableValue(), lastExecuteStatus, DbUtils.getBooleanValue(cronJob.getEnable()), DbUtils.getBooleanValue(cronJob.getDelete()), ULong.valueOf(cronJob.getCreateTime()), cronJob.getLastModifyUser(), ULong.valueOf(cronJob.getLastModifyTime()), @@ -427,7 +496,7 @@ public List listCronJobByIds(long appId, List cronJobIdLis List conditions = new ArrayList<>(); conditions.add(TABLE.APP_ID.eq(DbRecordMapper.getJooqLongValue(appId))); conditions.add(TABLE.IS_DELETED.equal(UByte.valueOf(0))); - conditions.add(TABLE.ID.in(cronJobIdList.parallelStream().map(ULong::valueOf).collect(Collectors.toList()))); + conditions.add(TABLE.ID.in(cronJobIdList.stream().map(ULong::valueOf).collect(Collectors.toList()))); return fetchData(conditions); } @@ -475,12 +544,50 @@ public Integer countCronJob(Long appId, Boolean active, Boolean cron) { conditions.add(TABLE.EXECUTE_TIME.isNotNull().and(TABLE.CRON_EXPRESSION.isNull())); } } - return context.selectCount().from(TABLE) - .where(conditions).fetchOne().value1(); + Record1 record = context.selectCount().from(TABLE) + .where(conditions).fetchOne(); + assert record != null; + return record.value1(); + } + + @Override + public List listEnabledCronBasicInfoForUpdate(int start, int limit) { + List conditions = new ArrayList<>(); + conditions.add(TABLE.IS_DELETED.eq(UByte.valueOf(0))); + conditions.add(TABLE.IS_ENABLE.eq(UByte.valueOf(1))); + val query = context + .select(TABLE.ID, TABLE.APP_ID, TABLE.NAME) + .from(TABLE) + .where(conditions) + .orderBy(TABLE.ID) + .limit(start, limit) + .forUpdate(); + if (log.isDebugEnabled()) { + log.debug( + "start={},limit={},SQL={}", + start, + limit, + query.getSQL(ParamType.INLINED) + ); + } + Result> records = query.fetch(); + return records.map(record -> { + CronJobBasicInfoDTO cronJobBasicInfoDTO = new CronJobBasicInfoDTO(); + cronJobBasicInfoDTO.setId(record.get(TABLE.ID).longValue()); + cronJobBasicInfoDTO.setAppId(record.get(TABLE.APP_ID).longValue()); + cronJobBasicInfoDTO.setName(record.get(TABLE.NAME)); + return cronJobBasicInfoDTO; + }); + } + + @Override + public List listCronJobIds(CronJobInfoDTO cronJobInfoDTO) { + List conditions = buildConditionList(cronJobInfoDTO, new BaseSearchCondition()); + List cronJobIds = context.select(TABLE.ID).from(TABLE).where(conditions).fetch(TABLE.ID); + return cronJobIds.stream().map(ULong::longValue).collect(Collectors.toList()); } - private CronJobInfoDTO convertToCronJobDTO(Record21 record) { + private CronJobInfoDTO convertToCronJobDTO(Record record) { if (record == null) { return null; } @@ -501,9 +608,9 @@ private CronJobInfoDTO convertToCronJobDTO(Record21>() { - })); + if (record.indexOf(TABLE.VARIABLE_VALUE) != -1) { + cronJobInfoDTO.decryptAndSetVariableValue(record.get(TABLE.VARIABLE_VALUE)); + } standardizeDynamicGroupId(cronJobInfoDTO.getVariableValue()); cronJobInfoDTO.setLastExecuteStatus(record.get(TABLE.LAST_EXECUTE_STATUS).intValue()); cronJobInfoDTO.setEnable(record.get(TABLE.IS_ENABLE).intValue() == 1); diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/dao/impl/CronJobHistoryDAOImpl.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/dao/impl/CronJobHistoryDAOImpl.java index 0357a8353b..fb64e7807c 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/dao/impl/CronJobHistoryDAOImpl.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/dao/impl/CronJobHistoryDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,14 +26,19 @@ import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; +import com.tencent.bk.job.common.util.StringUtil; import com.tencent.bk.job.crontab.constant.ExecuteStatusEnum; import com.tencent.bk.job.crontab.dao.CronJobHistoryDAO; import com.tencent.bk.job.crontab.model.dto.CronJobHistoryDTO; +import com.tencent.bk.job.crontab.model.tables.CronJobHistory; +import com.tencent.bk.job.crontab.model.tables.records.CronJobHistoryRecord; import com.tencent.bk.job.crontab.util.DbRecordMapper; import lombok.extern.slf4j.Slf4j; -import org.jooq.*; -import org.jooq.generated.tables.CronJobHistory; -import org.jooq.generated.tables.records.CronJobHistoryRecord; +import org.jooq.Condition; +import org.jooq.DSLContext; +import org.jooq.Record10; +import org.jooq.Result; +import org.jooq.SelectSeekStep1; import org.jooq.types.UByte; import org.jooq.types.ULong; import org.springframework.beans.factory.annotation.Qualifier; @@ -95,8 +100,12 @@ public boolean updateStatusByIdAndTime(long appId, long cronJobId, long schedule public boolean fillErrorInfo(long historyId, long errorCode, String errorMsg) { List conditions = new ArrayList<>(); conditions.add(TABLE.ID.eq(ULong.valueOf(historyId))); - return 1 == context.update(TABLE).set(TABLE.ERROR_CODE, ULong.valueOf(errorCode)).set(TABLE.ERROR_MESSAGE, - errorMsg).where(conditions).limit(1).execute(); + return 1 == context.update(TABLE) + .set(TABLE.ERROR_CODE, ULong.valueOf(errorCode)) + .set(TABLE.ERROR_MESSAGE, StringUtil.substring(errorMsg, 255)) + .where(conditions) + .limit(1) + .execute(); } @Override @@ -115,8 +124,8 @@ public PageData listPageHistoryByCondition(CronJobHistoryDTO int length = baseSearchCondition.getLengthOrDefault(10); Result> result = context.select(TABLE.ID, TABLE.APP_ID, TABLE.CRON_JOB_ID, TABLE.STATUS, TABLE.SCHEDULED_TIME, - TABLE.START_TIME, TABLE.FINISH_TIME, TABLE.EXECUTOR, TABLE.ERROR_CODE, TABLE.ERROR_MESSAGE) - .from(TABLE).where(conditions).orderBy(TABLE.START_TIME.desc()).limit(start, length).fetch(); + TABLE.START_TIME, TABLE.FINISH_TIME, TABLE.EXECUTOR, TABLE.ERROR_CODE, TABLE.ERROR_MESSAGE) + .from(TABLE).where(conditions).orderBy(TABLE.START_TIME.desc()).limit(start, length).fetch(); List cronJobHistoryList = new ArrayList<>(); diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/dao/impl/InnerCronJobHistoryDAOImpl.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/dao/impl/InnerCronJobHistoryDAOImpl.java index 90bf755a4c..27b7b907ed 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/dao/impl/InnerCronJobHistoryDAOImpl.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/dao/impl/InnerCronJobHistoryDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,13 +27,13 @@ import com.tencent.bk.job.crontab.constant.ExecuteStatusEnum; import com.tencent.bk.job.crontab.dao.InnerCronJobHistoryDAO; import com.tencent.bk.job.crontab.model.dto.InnerCronJobHistoryDTO; +import com.tencent.bk.job.crontab.model.tables.InnerCronJobHistory; +import com.tencent.bk.job.crontab.model.tables.records.InnerCronJobHistoryRecord; import com.tencent.bk.job.crontab.util.DbRecordMapper; import lombok.extern.slf4j.Slf4j; import org.jooq.Condition; import org.jooq.DSLContext; import org.jooq.Record7; -import org.jooq.generated.tables.InnerCronJobHistory; -import org.jooq.generated.tables.records.InnerCronJobHistoryRecord; import org.jooq.types.UByte; import org.jooq.types.ULong; import org.springframework.beans.factory.annotation.Autowired; diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/exception/TaskExecuteAuthFailedException.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/exception/TaskExecuteAuthFailedException.java index 5fc102bd12..953a90a7ae 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/exception/TaskExecuteAuthFailedException.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/exception/TaskExecuteAuthFailedException.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/listener/CrontabEventListener.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/listener/CrontabEventListener.java new file mode 100644 index 0000000000..eef99c1f09 --- /dev/null +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/listener/CrontabEventListener.java @@ -0,0 +1,122 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.crontab.listener; + +import com.tencent.bk.job.crontab.constant.CrontabActionEnum; +import com.tencent.bk.job.crontab.listener.event.CrontabEvent; +import com.tencent.bk.job.crontab.model.dto.CronJobInfoDTO; +import com.tencent.bk.job.crontab.service.CronJobService; +import com.tencent.bk.job.crontab.service.QuartzService; +import lombok.extern.slf4j.Slf4j; +import org.slf4j.helpers.MessageFormatter; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +/** + * 定时任务事件处理 + */ +@Component("crontabEventListener") +@Slf4j +public class CrontabEventListener { + + private final CronJobService cronJobService; + private final QuartzService quartzService; + + @Autowired + public CrontabEventListener(CronJobService cronJobService, QuartzService quartzService) { + this.cronJobService = cronJobService; + this.quartzService = quartzService; + } + + + /** + * 处理定时任务相关的事件 + * + * @param crontabEvent 定时任务相关的事件 + */ + public void handleEvent(CrontabEvent crontabEvent) { + log.info("Handle crontab event, event: {}, duration: {}ms", crontabEvent, crontabEvent.duration()); + long appId = crontabEvent.getAppId(); + long cronJobId = crontabEvent.getCronJobId(); + CrontabActionEnum action = CrontabActionEnum.valueOf(crontabEvent.getAction()); + try { + switch (action) { + case ADD_CRON: + CronJobInfoDTO cronJobInfoDTO = cronJobService.getCronJobInfoById(cronJobId); + refreshCronJobInQuartz(cronJobInfoDTO); + break; + case DELETE_CRON: + deleteCronJobFromQuartz(appId, cronJobId); + break; + default: + log.error("Invalid crontabEvent action: {}", action); + } + } catch (Throwable e) { + String errorMsg = MessageFormatter.format( + "Handle crontab event error, appId={}, cronJobId={}", + appId, + cronJobId + ).getMessage(); + log.error(errorMsg, e); + } + } + + private void refreshCronJobInQuartz(CronJobInfoDTO cronJobInfoDTO) { + if (cronJobInfoDTO == null) { + return; + } + if (cronJobInfoDTO.getEnable()) { + // 开启定时任务 + boolean result = cronJobService.checkAndAddJobToQuartz(cronJobInfoDTO.getAppId(), cronJobInfoDTO.getId()); + log.info( + "add cronJob({},{}) to quartz, result={}", + cronJobInfoDTO.getAppId(), + cronJobInfoDTO.getId(), + result + ); + } else { + // 关闭定时任务 + boolean result = quartzService.deleteJobFromQuartz(cronJobInfoDTO.getAppId(), cronJobInfoDTO.getId()); + log.info( + "delete cronJob({},{}) from quartz, result={}", + cronJobInfoDTO.getAppId(), + cronJobInfoDTO.getId(), + result + ); + } + } + + private void deleteCronJobFromQuartz(long appId, long cronJobId) { + // 删除定时任务 + boolean result = quartzService.deleteJobFromQuartz(appId, cronJobId); + log.info( + "delete cronJob({},{}) from quartz, result={}", + appId, + cronJobId, + result + ); + } + +} diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/listener/event/CrontabEvent.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/listener/event/CrontabEvent.java new file mode 100644 index 0000000000..aa079707b8 --- /dev/null +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/listener/event/CrontabEvent.java @@ -0,0 +1,101 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.crontab.listener.event; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.tencent.bk.job.common.event.Event; +import com.tencent.bk.job.crontab.constant.CrontabActionEnum; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import java.time.LocalDateTime; +import java.util.StringJoiner; + +/** + * 定时任务事件 + */ +@Getter +@Setter +@NoArgsConstructor +@JsonInclude(JsonInclude.Include.NON_NULL) +public class CrontabEvent extends Event { + /** + * 定时任务操作 + * + * @see CrontabActionEnum + */ + private int action; + /** + * Job业务ID + */ + private Long appId; + /** + * 定时任务ID + */ + private Long cronJobId; + + /** + * 构造添加定时任务事件 + * + * @param appId Job业务ID + * @param cronJobId 定时任务ID + * @return 事件 + */ + public static CrontabEvent addCron(long appId, long cronJobId) { + CrontabEvent crontabEvent = new CrontabEvent(); + crontabEvent.setAppId(appId); + crontabEvent.setCronJobId(cronJobId); + crontabEvent.setAction(CrontabActionEnum.ADD_CRON.getValue()); + crontabEvent.setTime(LocalDateTime.now()); + return crontabEvent; + } + + /** + * 构造删除定时任务事件 + * + * @param appId Job业务ID + * @param cronJobId 定时任务ID + * @return 事件 + */ + public static CrontabEvent deleteCron(long appId, long cronJobId) { + CrontabEvent crontabEvent = new CrontabEvent(); + crontabEvent.setAppId(appId); + crontabEvent.setCronJobId(cronJobId); + crontabEvent.setAction(CrontabActionEnum.DELETE_CRON.getValue()); + crontabEvent.setTime(LocalDateTime.now()); + return crontabEvent; + } + + @Override + public String toString() { + return new StringJoiner(", ", CrontabEvent.class.getSimpleName() + "[", "]") + .add("action=" + action) + .add("appId=" + appId) + .add("cronJobId=" + cronJobId) + .add("time=" + time) + .toString(); + } +} diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/metrics/CronMetricsConstants.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/metrics/CronMetricsConstants.java index c1711f3807..83a26582fb 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/metrics/CronMetricsConstants.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/metrics/CronMetricsConstants.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,19 +26,19 @@ public class CronMetricsConstants { /** - * Quartz工作线程池线程数 + * 定时任务调度延迟指标 */ - public static final String NAME_CRON_QUARTZ_TASK_EXECUTOR_POOL_SIZE = "cron.quartz.task.executor.pool.size"; + public static final String NAME_JOB_CRON_SCHEDULE_DELAY = "job.cron.schedule.delay"; /** - * Quartz工作线程池队列大小 + * 定时任务执行延迟指标 */ - public static final String NAME_CRON_QUARTZ_TASK_EXECUTOR_QUEUE_SIZE = "cron.quartz.task.executor.queue.size"; + public static final String NAME_JOB_CRON_EXECUTE_DELAY = "job.cron.execute.delay"; /** - * 定时任务调度延迟指标 + * 定时任务执行耗时指标 */ - public static final String NAME_JOB_CRON_SCHEDULE_DELAY = "job.cron.schedule.delay"; - - public static final String TAG_KEY_MODULE = "module"; - - public static final String TAG_VALUE_MODULE_CRON = "cron"; + public static final String NAME_JOB_CRON_TIME_CONSUMING = "job.cron.time.consuming"; + /** + * 标签：定时任务是否真正执行（抢到分布式锁） + */ + public static final String TAG_KEY_JOB_CRON_EXECUTE_ACTUALLY = "execute_actually"; } diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/metrics/MeasureServiceImpl.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/metrics/MeasureServiceImpl.java deleted file mode 100644 index 3b41cddb97..0000000000 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/metrics/MeasureServiceImpl.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.crontab.metrics; - -import io.micrometer.core.instrument.MeterRegistry; -import io.micrometer.core.instrument.Tag; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; -import org.springframework.stereotype.Service; - -import java.util.Collections; - -@Slf4j -@Service -public class MeasureServiceImpl { - - @Autowired - public MeasureServiceImpl(MeterRegistry meterRegistry, ThreadPoolTaskExecutor quartzTaskExecutor) { - // 同步线程池监控：Agent状态 - meterRegistry.gauge( - CronMetricsConstants.NAME_CRON_QUARTZ_TASK_EXECUTOR_POOL_SIZE, - Collections.singletonList(Tag.of(CronMetricsConstants.TAG_KEY_MODULE, - CronMetricsConstants.TAG_VALUE_MODULE_CRON)), - quartzTaskExecutor, - taskExecutor -> taskExecutor.getThreadPoolExecutor().getPoolSize() - ); - meterRegistry.gauge( - CronMetricsConstants.NAME_CRON_QUARTZ_TASK_EXECUTOR_QUEUE_SIZE, - Collections.singletonList(Tag.of(CronMetricsConstants.TAG_KEY_MODULE, - CronMetricsConstants.TAG_VALUE_MODULE_CRON)), - quartzTaskExecutor, - taskExecutor -> taskExecutor.getThreadPoolExecutor().getQueue().size() - ); - } -} diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/metrics/ScheduleMeasureService.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/metrics/ScheduleMeasureService.java index 75fade433d..668f8a6363 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/metrics/ScheduleMeasureService.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/metrics/ScheduleMeasureService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -55,6 +55,76 @@ public ScheduleMeasureService(MeterRegistry registry) { this.registry = registry; } + /** + * 记录定时任务执行过程耗时 + * + * @param name 定时任务名称 + * @param context 定时任务上下文 + * @param timeConsumingMills 定时任务执行耗时 + * @param tags 额外的标签 + */ + public void recordCronTimeConsuming(String name, + JobExecutionContext context, + long timeConsumingMills, + Tag... tags) { + try { + recordCronTimeConsumingIndeed(name, context, timeConsumingMills, tags); + } catch (Exception e) { + log.warn("Fail to recordCronTimeConsuming", e); + } + } + + private void recordCronTimeConsumingIndeed(String name, + JobExecutionContext context, + long timeConsumingMills, + Tag... tags) { + Tags finalTags = parseKeyNameAndAppIdTags(name, context); + finalTags = finalTags.and(tags); + record( + CronMetricsConstants.NAME_JOB_CRON_TIME_CONSUMING, + "cron execute time consuming", + timeConsumingMills, + finalTags, + Duration.ofMillis(10), + Duration.ofSeconds(30) + ); + } + + /** + * 记录用户侧感知到的定时任务实际执行时的延迟 + * + * @param name 定时任务名称 + * @param context 定时任务上下文 + * @param executeDelayMills 定时任务实际执行时的延迟 + */ + public void recordCronExecuteDelay(String name, JobExecutionContext context, long executeDelayMills) { + try { + recordCronExecuteDelayIndeed(name, context, executeDelayMills); + } catch (Exception e) { + log.warn("Fail to recordCronExecuteDelay", e); + } + } + + private void recordCronExecuteDelayIndeed(String name, + JobExecutionContext context, + long executeDelayMills) { + Tags finalTags = parseKeyNameAndAppIdTags(name, context); + record( + CronMetricsConstants.NAME_JOB_CRON_EXECUTE_DELAY, + "cron execute delay", + executeDelayMills, + finalTags, + Duration.ofSeconds(1), + Duration.ofSeconds(90) + ); + } + + /** + * 记录定时任务调度延迟 + * + * @param name 定时任务名称 + * @param context 定时任务上下文 + */ public void recordCronScheduleDelay(String name, JobExecutionContext context) { try { recordCronScheduleDelayIndeed(name, context); @@ -63,11 +133,23 @@ public void recordCronScheduleDelay(String name, JobExecutionContext context) { } } - public void recordCronScheduleDelayIndeed(String name, JobExecutionContext context) { + private void recordCronScheduleDelayIndeed(String name, JobExecutionContext context) { + Tags tags = parseKeyNameAndAppIdTags(name, context); + record( + CronMetricsConstants.NAME_JOB_CRON_SCHEDULE_DELAY, + "cron schedule delay", + context.getFireTime().getTime() - context.getScheduledFireTime().getTime(), + tags, + Duration.ofSeconds(1), + Duration.ofSeconds(60) + ); + } + + private Tags parseKeyNameAndAppIdTags(String name, JobExecutionContext context) { Tag tag = Tag.of(CommonMetricTags.KEY_NAME, name); Tags tags = Tags.of(tag); tags = tags.and(parseTagsFromJobExecutionContext(context)); - record(context.getFireTime().getTime() - context.getScheduledFireTime().getTime(), tags); + return tags; } private Tags parseTagsFromJobExecutionContext(JobExecutionContext context) { @@ -79,14 +161,19 @@ private Tags parseTagsFromJobExecutionContext(JobExecutionContext context) { } } - private void record(long delayMillis, Tags tags) { - Timer.builder(CronMetricsConstants.NAME_JOB_CRON_SCHEDULE_DELAY) - .description("cron schedule delay") + private void record(String metricName, + String description, + long timeMillis, + Tags tags, + Duration minimumExpectedValue, + Duration maximumExpectedValue) { + Timer.builder(metricName) + .description(description) .tags(tags) .publishPercentileHistogram(true) - .minimumExpectedValue(Duration.ofMillis(10)) - .maximumExpectedValue(Duration.ofSeconds(30)) + .minimumExpectedValue(minimumExpectedValue) + .maximumExpectedValue(maximumExpectedValue) .register(registry) - .record(delayMillis, TimeUnit.MILLISECONDS); + .record(timeMillis, TimeUnit.MILLISECONDS); } } diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/migration/AddHostIdForCronVariableMigrationTask.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/migration/AddHostIdForCronVariableMigrationTask.java new file mode 100644 index 0000000000..dd91b7ffaa --- /dev/null +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/migration/AddHostIdForCronVariableMigrationTask.java @@ -0,0 +1,214 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.crontab.migration; + +import com.tencent.bk.job.common.constant.TaskVariableTypeEnum; +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.crontab.dao.CronJobDAO; +import com.tencent.bk.job.crontab.model.dto.CronJobVariableDTO; +import com.tencent.bk.job.crontab.model.dto.CronJobWithVarsDTO; +import com.tencent.bk.job.crontab.model.inner.ServerDTO; +import com.tencent.bk.job.crontab.service.HostService; +import com.tencent.bk.job.manage.model.migration.AddHostIdResult; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.util.StopWatch; + +import java.util.ArrayList; +import java.util.List; + +/** + * 对定时任务变量数据中不存在hostId的数据使用CMDB中的完整数据补全hostId + */ +@Slf4j +@Service +public class AddHostIdForCronVariableMigrationTask { + + private final CronJobDAO cronJobDAO; + private final HostService hostService; + + @Autowired + public AddHostIdForCronVariableMigrationTask(CronJobDAO cronJobDAO, + HostService hostService) { + this.cronJobDAO = cronJobDAO; + this.hostService = hostService; + } + + private String getTaskName() { + return "migrate_cron_job"; + } + + private AddHostIdResult getInitAddHostIdResult() { + AddHostIdResult result = new AddHostIdResult(getTaskName()); + result.setTotalRecords(0); + result.setSuccessRecords(0); + result.setSuccess(true); + return result; + } + + public AddHostIdResult execute(List appIdList, boolean isDryRun) { + StopWatch watch = new StopWatch(getTaskName()); + try { + return updateHostIdForCronVariables(appIdList, isDryRun, watch); + } catch (Throwable t) { + log.warn("Fail to updateHostIdForCronVariables", t); + AddHostIdResult result = getInitAddHostIdResult(); + result.setSuccess(false); + return result; + } finally { + if (watch.isRunning()) { + watch.stop(); + } + log.info("AddHostIdForCronVariableMigrationTask finished, time consuming:{}", watch.prettyPrint()); + } + } + + /** + * 判断定时任务的变量中是否存在hostId + * + * @param cronJobDTO 定时任务信息 + * @return 是否存在hostId + */ + private boolean existsHostId(CronJobWithVarsDTO cronJobDTO) { + if (cronJobDTO == null) { + return false; + } + List variableValue = cronJobDTO.getVariableValue(); + if (CollectionUtils.isEmpty(variableValue)) { + return false; + } + for (CronJobVariableDTO cronJobVariableDTO : variableValue) { + if (TaskVariableTypeEnum.HOST_LIST != cronJobVariableDTO.getType()) { + continue; + } + ServerDTO server = cronJobVariableDTO.getServer(); + if (server == null || CollectionUtils.isEmpty(server.getIps())) { + continue; + } + List hosts = server.getIps(); + for (HostDTO host : hosts) { + if (host.getHostId() != null) { + return true; + } + } + } + return false; + } + + private AddHostIdResult updateHostIdForCronVariables(List appIdList, boolean isDryRun, StopWatch watch) { + AddHostIdResult result = getInitAddHostIdResult(); + int start = 0; + int batchSize = 100; + List cronJobList; + int totalNullHostIdRecordsCount = 0; + int totalUpdatedCount = 0; + do { + // 1.查询含有主机变量的定时任务 + watch.start("listBasicCronJobWithHostVars_" + start + "_" + (start + batchSize)); + cronJobList = cronJobDAO.listBasicCronJobWithHostVars(appIdList, start, batchSize); + watch.stop(); + if (CollectionUtils.isEmpty(cronJobList)) { + continue; + } + totalNullHostIdRecordsCount += cronJobList.size(); + int count = 0; + + // 2.查出变量中主机的hostId并填充 + watch.start("addHostIdForCronVariables_" + start + "_" + (start + batchSize)); + int filledHostCount = tryToAddHostIdForCronVarHosts(cronJobList); + watch.stop(); + + // 3.更新已填充hostId的变量到DB + watch.start("updateVariableById_" + start + "_" + (start + batchSize)); + log.debug("filled {} hosts with hostId for {} cronJobs", filledHostCount, cronJobList.size()); + for (CronJobWithVarsDTO cronJobWithVarsDTO : cronJobList) { + if (!existsHostId(cronJobWithVarsDTO)) { + continue; + } + if (isDryRun) { + log.info( + "[DryRun]set variable_value={} for cronJob(id={})", + cronJobWithVarsDTO.getEncryptedVariableValue(), + cronJobWithVarsDTO.getId() + ); + count += 1; + } else { + count += cronJobDAO.updateVariableById(cronJobWithVarsDTO.getId(), cronJobWithVarsDTO); + } + } + watch.stop(); + + // 4.统计信息记录、打印 + start += batchSize; + totalUpdatedCount += count; + log.info((isDryRun ? "[DryRun]" : "") + "{}/{} cronJobs have been added hostId", count, cronJobList.size()); + } while (!CollectionUtils.isEmpty(cronJobList)); + log.info((isDryRun ? "[DryRun]" : "") + "{} cronJobs have been added hostId in total", totalUpdatedCount); + result.setTotalRecords(totalNullHostIdRecordsCount); + result.setSuccessRecords(totalUpdatedCount); + result.setSuccess(true); + return result; + } + + /** + * 尝试为定时任务变量中的主机填充hostId，忽略找不到hostId的主机（可能已在CMDB中被移除） + * + * @param cronJobList 定时任务列表 + * @return 实际填充了hostId的主机数量 + */ + private int tryToAddHostIdForCronVarHosts(List cronJobList) { + List hostList = new ArrayList<>(); + // 1.收集所有主机信息 + for (CronJobWithVarsDTO cronJobDTO : cronJobList) { + List variableValue = cronJobDTO.getVariableValue(); + if (CollectionUtils.isEmpty(variableValue)) { + continue; + } + for (CronJobVariableDTO cronJobVariableDTO : variableValue) { + if (TaskVariableTypeEnum.HOST_LIST != cronJobVariableDTO.getType()) { + continue; + } + ServerDTO server = cronJobVariableDTO.getServer(); + if (server == null || CollectionUtils.isEmpty(server.getIps())) { + continue; + } + for (HostDTO host : server.getIps()) { + // 已存在hostId的主机不再填充 + if (host.getHostId() == null) { + hostList.add(host); + } + } + } + } + // 2.批量查询hostId并填充 + if (CollectionUtils.isNotEmpty(hostList)) { + return hostService.fillHosts(hostList); + } else { + return 0; + } + } +} diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/AddJobToQuartzResult.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/AddJobToQuartzResult.java new file mode 100644 index 0000000000..e4c5a1a398 --- /dev/null +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/AddJobToQuartzResult.java @@ -0,0 +1,97 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.crontab.model.dto; + +import lombok.Data; + +/** + * 添加任务到Quartz的结果 + */ +@Data +public class AddJobToQuartzResult { + /** + * 定时任务基本信息 + */ + private CronJobBasicInfoDTO cronJobBasicInfo; + /** + * 是否成功 + */ + private boolean success; + /** + * 提示信息 + */ + private String message; + /** + * 异常信息 + */ + private Exception exception; + + /** + * 构造失败结果 + * + * @param cronJobBasicInfo 定时任务基本信息 + * @param message 提示信息 + * @return 失败结果 + */ + public static AddJobToQuartzResult failResult(CronJobBasicInfoDTO cronJobBasicInfo, String message) { + AddJobToQuartzResult result = new AddJobToQuartzResult(); + result.setCronJobBasicInfo(cronJobBasicInfo); + result.setSuccess(false); + result.setMessage(message); + return result; + } + + /** + * 构造失败结果 + * + * @param cronJobBasicInfo 定时任务基本信息 + * @param message 提示信息 + * @param exception 异常信息 + * @return 失败结果 + */ + public static AddJobToQuartzResult failResult(CronJobBasicInfoDTO cronJobBasicInfo, + String message, + Exception exception) { + AddJobToQuartzResult result = new AddJobToQuartzResult(); + result.setCronJobBasicInfo(cronJobBasicInfo); + result.setSuccess(false); + result.setMessage(message); + result.setException(exception); + return result; + } + + /** + * 构造成功结果 + * + * @param cronJobBasicInfo 定时任务基本信息 + * @return 成功结果 + */ + public static AddJobToQuartzResult successResult(CronJobBasicInfoDTO cronJobBasicInfo) { + AddJobToQuartzResult result = new AddJobToQuartzResult(); + result.setCronJobBasicInfo(cronJobBasicInfo); + result.setSuccess(true); + return result; + } +} diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/BatchAddResult.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/BatchAddResult.java new file mode 100644 index 0000000000..ef2045efe1 --- /dev/null +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/BatchAddResult.java @@ -0,0 +1,141 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.crontab.model.dto; + +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.springframework.util.CollectionUtils; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * 定时任务批量添加到Quartz的结果 + */ +@Slf4j +@Data +public class BatchAddResult { + /** + * 添加结果Map，key为定时任务ID，value为添加结果 + */ + private Map resultMap; + /** + * 添加成功的任务数量 + */ + private int successNum = 0; + /** + * 添加失败的任务数量 + */ + private int failNum = 0; + + /** + * 添加一个结果数据至批量结果 + * + * @param result 单个结果数据 + */ + public void addResult(AddJobToQuartzResult result) { + if (result == null) { + return; + } + if (resultMap == null) { + resultMap = new HashMap<>(); + } + CronJobBasicInfoDTO cronJobBasicInfo = result.getCronJobBasicInfo(); + if (cronJobBasicInfo == null) { + log.info("cronJobBasicInfo is null, ignore"); + return; + } + resultMap.put(cronJobBasicInfo.getId(), result); + if (result.isSuccess()) { + successNum++; + } else { + failNum++; + } + } + + /** + * 合并批量结果 + * + * @param batchAddResult 待合并的批量结果 + */ + public void merge(BatchAddResult batchAddResult) { + if (batchAddResult == null) { + return; + } + if (CollectionUtils.isEmpty(batchAddResult.resultMap)) { + return; + } + if (resultMap == null) { + resultMap = new HashMap<>(batchAddResult.resultMap); + } else { + resultMap.putAll(batchAddResult.resultMap); + } + successNum += batchAddResult.successNum; + failNum += batchAddResult.failNum; + } + + /** + * 获取批量添加失败的添加结果列表 + * + * @return 批量添加失败的添加结果列表 + */ + public List getFailedResultList() { + if (failNum == 0) { + return Collections.emptyList(); + } + List failedResultList = new ArrayList<>(); + for (Map.Entry entry : resultMap.entrySet()) { + if (!entry.getValue().isSuccess()) { + failedResultList.add(entry.getValue()); + } + } + return failedResultList; + } + + /** + * 获取批量添加的总数 + * + * @return 批量添加总数 + */ + public int getTotalNum() { + return successNum + failNum; + } + + /** + * 获取批量添加失败的比率 + * + * @return 批量添加失败的比率 + */ + public float getFailRate() { + int totalNum = successNum + failNum; + if (totalNum == 0) { + return 0; + } + return 1.0f * failNum / totalNum; + } +} diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/CronJobBasicInfoDTO.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/CronJobBasicInfoDTO.java new file mode 100644 index 0000000000..c968cfd59a --- /dev/null +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/CronJobBasicInfoDTO.java @@ -0,0 +1,51 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.crontab.model.dto; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + +@Data +@EqualsAndHashCode +@NoArgsConstructor +@AllArgsConstructor +public class CronJobBasicInfoDTO { + /** + * 定时任务 ID + */ + private Long id; + + /** + * 业务 ID + */ + private Long appId; + + /** + * 定时任务名称 + */ + private String name; +} diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/CronJobHistoryDTO.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/CronJobHistoryDTO.java index 28c501cbd6..0c66032df3 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/CronJobHistoryDTO.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/CronJobHistoryDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/CronJobInfoDTO.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/CronJobInfoDTO.java index a98360038c..48e439f747 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/CronJobInfoDTO.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/CronJobInfoDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,7 @@ package com.tencent.bk.job.crontab.model.dto; +import com.tencent.bk.job.common.constant.CronJobNotifyType; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.esb.util.EsbDTOAppScopeMappingHelper; import com.tencent.bk.job.common.exception.InternalException; @@ -35,11 +36,13 @@ import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.crontab.model.CronJobCreateUpdateReq; import com.tencent.bk.job.crontab.model.CronJobVO; +import com.tencent.bk.job.common.model.dto.notify.CustomNotifyDTO; import com.tencent.bk.job.crontab.model.esb.response.EsbCronInfoResponse; import com.tencent.bk.job.crontab.model.esb.v3.response.EsbCronInfoV3DTO; +import com.tencent.bk.job.crontab.model.inner.ServiceCronJobDTO; import com.tencent.bk.job.crontab.service.TaskPlanService; import com.tencent.bk.job.crontab.util.CronExpressionUtil; -import com.tencent.bk.job.manage.common.consts.notify.NotifyConsts; +import com.tencent.bk.job.manage.api.common.constants.notify.NotifyConsts; import com.tencent.bk.job.manage.model.inner.ServiceTaskPlanDTO; import com.tencent.bk.job.manage.model.inner.ServiceTemplateNotificationDTO; import lombok.AllArgsConstructor; @@ -57,16 +60,17 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.stream.Collectors; /** * @since 23/12/2019 21:03 */ @Data -@EqualsAndHashCode +@EqualsAndHashCode(callSuper = true) @NoArgsConstructor @AllArgsConstructor -public class CronJobInfoDTO { +public class CronJobInfoDTO extends EncryptEnableVariables { /** * 定时任务 ID */ @@ -117,11 +121,6 @@ public class CronJobInfoDTO { */ private Long executeTime; - /** - * 变量值列表 - */ - private List variableValue; - /** * 上次执行状态 */ @@ -177,6 +176,17 @@ public class CronJobInfoDTO { */ private List notifyChannel = Collections.emptyList(); + /** + * 通知方式（1-继承业务, 2-自定义） + * @see com.tencent.bk.job.common.constant.CronJobNotifyType + */ + private Integer notifyType = 1; + + /** + * 自定义通知配置 + */ + private CustomNotifyDTO customCronJobNotifyDTO = new CustomNotifyDTO(); + /** * 周期执行结束时间 */ @@ -200,17 +210,17 @@ public static CronJobVO toVO(CronJobInfoDTO cronJobInfo) { cronJobVO.setTaskPlanId(cronJobInfo.getTaskPlanId()); cronJobVO.setScriptId(cronJobInfo.getScriptId()); cronJobVO.setScriptVersionId(cronJobInfo.getScriptVersionId()); + cronJobVO.setNotifyType(cronJobInfo.getNotifyType()); + cronJobVO.setCronJobCustomNotifyVO(CustomNotifyDTO.toVO(cronJobInfo.getCustomCronJobNotifyDTO())); if (StringUtils.isNotBlank(cronJobInfo.getCronExpression())) { cronJobVO.setCronExpression(CronExpressionUtil.fixExpressionForUser(cronJobInfo.getCronExpression())); } else { cronJobVO.setCronExpression(null); } cronJobVO.setExecuteTime(cronJobInfo.getExecuteTime()); - if (CollectionUtils.isNotEmpty(cronJobInfo.getVariableValue())) { - cronJobVO.setVariableValue(cronJobInfo.getVariableValue().parallelStream().map(CronJobVariableDTO::toVO) + if (cronJobInfo.getVariableValue() != null) { + cronJobVO.setVariableValue(cronJobInfo.getVariableValue().stream().map(CronJobVariableDTO::toVO) .collect(Collectors.toList())); - } else { - cronJobVO.setVariableValue(Collections.emptyList()); } cronJobVO.setLastExecuteStatus(cronJobInfo.getLastExecuteStatus()); cronJobVO.setLastExecuteErrorCode(cronJobInfo.getLastExecuteErrorCode()); @@ -241,7 +251,7 @@ public static CronJobVO toBasicVO(CronJobInfoDTO cronJobInfo) { cronJobVO.setTaskPlanId(cronJobInfo.getTaskPlanId()); cronJobVO.setEnable(cronJobInfo.getEnable()); if (CollectionUtils.isNotEmpty(cronJobInfo.getVariableValue())) { - cronJobVO.setVariableValue(cronJobInfo.getVariableValue().parallelStream().map(CronJobVariableDTO::toVO) + cronJobVO.setVariableValue(cronJobInfo.getVariableValue().stream().map(CronJobVariableDTO::toVO) .collect(Collectors.toList())); } else { cronJobVO.setVariableValue(Collections.emptyList()); @@ -275,7 +285,7 @@ public static CronJobInfoDTO fromReq(String username, Long appId, CronJobCreateU cronJobInfo.setExecuteTime(cronJobCreateUpdateReq.getExecuteTime()); } if (CollectionUtils.isNotEmpty(cronJobCreateUpdateReq.getVariableValue())) { - cronJobInfo.setVariableValue(cronJobCreateUpdateReq.getVariableValue().parallelStream() + cronJobInfo.setVariableValue(cronJobCreateUpdateReq.getVariableValue().stream() .map(CronJobVariableDTO::fromVO).collect(Collectors.toList())); } else { cronJobInfo.setVariableValue(null); @@ -289,10 +299,24 @@ public static CronJobInfoDTO fromReq(String username, Long appId, CronJobCreateU } cronJobInfo.setNotifyUser(UserRoleInfoDTO.fromVO(cronJobCreateUpdateReq.getNotifyUser())); cronJobInfo.setNotifyChannel(cronJobCreateUpdateReq.getNotifyChannel()); + cronJobInfo.setNotifyType(cronJobCreateUpdateReq.getNotifyType()); + if (cronJobInfo.hasCustomNotifyPolicy()) { + cronJobInfo.setCustomCronJobNotifyDTO(extractCustomNotifyDTOFromReq(cronJobCreateUpdateReq)); + } cronJobInfo.setEndTime(cronJobCreateUpdateReq.getEndTime()); return cronJobInfo; } + private static CustomNotifyDTO extractCustomNotifyDTOFromReq(CronJobCreateUpdateReq cronJobCreateUpdateReq) { + CustomNotifyDTO cronJobNotifyDTO = new CustomNotifyDTO(); + if (cronJobCreateUpdateReq.getCustomNotifyUser() != null) { + cronJobNotifyDTO.setRoleList(cronJobCreateUpdateReq.getCustomNotifyUser().getRoleList()); + cronJobNotifyDTO.setExtraObserverList(cronJobCreateUpdateReq.getCustomNotifyUser().getUserList()); + } + cronJobNotifyDTO.setCustomNotifyChannel(cronJobCreateUpdateReq.getCustomNotifyChannel()); + return cronJobNotifyDTO; + } + public static EsbCronInfoResponse toEsbCronInfo(CronJobInfoDTO cronJobInfoDTO) { if (cronJobInfoDTO == null) { return null; @@ -380,6 +404,10 @@ private static void addTaskUrlToVarMap(Map variableMap, CronJobI variableMap.put("cron_uri", cronUri); } + public boolean hasCustomNotifyPolicy() { + return Objects.equals(this.getNotifyType(), CronJobNotifyType.CUSTOM.getType()); + } + public static ServiceTemplateNotificationDTO buildNotifyInfo(CronJobInfoDTO cronJobInfo) { ServiceTemplateNotificationDTO notifyInfo = new ServiceTemplateNotificationDTO(); notifyInfo.setTriggerUser(cronJobInfo.getLastModifyUser()); @@ -398,7 +426,7 @@ public static ServiceTemplateNotificationDTO buildNotifyInfo(CronJobInfoDTO cron // 结束前通知 notifyInfo.setTemplateCode(NotifyConsts.NOTIFY_TEMPLATE_CODE_BEFORE_CRON_JOB_END); variableMap.put("cron_type", "周期执行"); - String cronRuleStr = cronJobInfo.getCronExpression().substring(2).replace("?", "*"); + String cronRuleStr = CronExpressionUtil.fixExpressionForUser(cronJobInfo.getCronExpression()); variableMap.put("cron_rule", cronRuleStr); triggerTime = cronJobInfo.getEndTime() - cronJobInfo.getNotifyOffset(); variableMap.put("task.cron.repeat_freq", "周期执行"); @@ -449,7 +477,7 @@ public static ServiceTemplateNotificationDTO buildFailedNotifyInfo(CronJobInfoDT if (StringUtils.isNotBlank(cronJobInfo.getCronExpression())) { // 结束前通知 variableMap.put("task.cron.type", "周期执行"); - String cronRuleStr = cronJobInfo.getCronExpression().substring(2).replace("?", "*"); + String cronRuleStr = CronExpressionUtil.fixExpressionForUser(cronJobInfo.getCronExpression()); variableMap.put("task.cron.rule", cronRuleStr); variableMap.put("task.cron.repeat_freq", "周期执行"); variableMap.put("task.cron.time_set", cronRuleStr); @@ -487,14 +515,14 @@ public static EsbCronInfoV3DTO toEsbCronInfoV3(CronJobInfoDTO cronJobInfoDTO) { esbCronInfoResponse.setStatus(cronJobInfoDTO.getEnable() ? 1 : 0); if (StringUtils.isNotBlank(cronJobInfoDTO.getCronExpression())) { esbCronInfoResponse.setCronExpression( - cronJobInfoDTO.getCronExpression().substring(2).replace("?", "*")); + CronExpressionUtil.fixExpressionForUser(cronJobInfoDTO.getCronExpression())); } esbCronInfoResponse.setCreator(cronJobInfoDTO.getCreator()); esbCronInfoResponse.setCreateTime(cronJobInfoDTO.getCreateTime()); esbCronInfoResponse.setLastModifyUser(cronJobInfoDTO.getLastModifyUser()); esbCronInfoResponse.setLastModifyTime(cronJobInfoDTO.getLastModifyTime()); if (CollectionUtils.isNotEmpty(cronJobInfoDTO.getVariableValue())) { - esbCronInfoResponse.setGlobalVarList(cronJobInfoDTO.getVariableValue().parallelStream() + esbCronInfoResponse.setGlobalVarList(cronJobInfoDTO.getVariableValue().stream() .map(CronJobVariableDTO::toEsbGlobalVarV3).collect(Collectors.toList())); } return esbCronInfoResponse; @@ -576,4 +604,33 @@ private boolean validateCronExpression() { } return true; } + + public ServiceCronJobDTO toServiceCronJobDTO() { + ServiceCronJobDTO cronJob = new ServiceCronJobDTO(); + cronJob.setId(id); + cronJob.setAppId(appId); + cronJob.setName(name); + cronJob.setCreator(creator); + cronJob.setLastModifyTime(lastModifyTime); + cronJob.setTaskPlanId(taskPlanId); + cronJob.setScriptId(scriptId); + cronJob.setScriptVersionId(scriptVersionId); + cronJob.setCronExpression(cronExpression); + cronJob.setExecuteTime(executeTime); + cronJob.setLastExecuteStatus(lastExecuteStatus); + cronJob.setEnable(enable); + cronJob.setLastModifyUser(lastModifyUser); + cronJob.setLastModifyTime(lastModifyTime); + cronJob.setNotifyType(notifyType); + cronJob.setCustomCronJobNotifyDTO(customCronJobNotifyDTO); + return cronJob; + } + + public CronJobBasicInfoDTO toBasicInfoDTO() { + CronJobBasicInfoDTO cronJob = new CronJobBasicInfoDTO(); + cronJob.setId(id); + cronJob.setAppId(appId); + cronJob.setName(name); + return cronJob; + } } diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/CronJobLaunchResultStatistics.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/CronJobLaunchResultStatistics.java index 73e1558e08..ce1bbda765 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/CronJobLaunchResultStatistics.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/CronJobLaunchResultStatistics.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/CronJobVariableDTO.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/CronJobVariableDTO.java index 3afb3aab29..e3b0617cca 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/CronJobVariableDTO.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/CronJobVariableDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,7 @@ package com.tencent.bk.job.crontab.model.dto; +import com.tencent.bk.job.common.annotation.PersistenceObject; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.constant.TaskVariableTypeEnum; import com.tencent.bk.job.common.esb.model.job.v3.EsbGlobalVarV3DTO; @@ -39,11 +40,12 @@ /** * @since 3/10/2019 17:14 */ +@PersistenceObject @Data @EqualsAndHashCode @NoArgsConstructor @AllArgsConstructor -public class CronJobVariableDTO { +public class CronJobVariableDTO implements Cloneable { /** * 变量 ID */ @@ -142,4 +144,18 @@ public static EsbGlobalVarV3DTO toEsbGlobalVarV3(CronJobVariableDTO cronJobVaria esbGlobalVar.setServer(ServerDTO.toEsbServerV3(cronJobVariableDTO.getServer())); return esbGlobalVar; } + + @SuppressWarnings("MethodDoesntCallSuperMethod") + @Override + public CronJobVariableDTO clone() { + CronJobVariableDTO cronJobVariableDTO = new CronJobVariableDTO(); + cronJobVariableDTO.setId(id); + cronJobVariableDTO.setType(type); + cronJobVariableDTO.setName(name); + cronJobVariableDTO.setValue(value); + if (server != null) { + cronJobVariableDTO.setServer(server.clone()); + } + return cronJobVariableDTO; + } } diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/CronJobWithVarsDTO.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/CronJobWithVarsDTO.java new file mode 100644 index 0000000000..9263a1d69a --- /dev/null +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/CronJobWithVarsDTO.java @@ -0,0 +1,62 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.crontab.model.dto; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + +import java.util.List; + +/** + * 含有基本信息与变量信息的定时任务实体 + */ +@Data +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor +@AllArgsConstructor +public class CronJobWithVarsDTO extends EncryptEnableVariables { + /** + * 定时任务 ID + */ + private Long id; + + /** + * 业务 ID + */ + private Long appId; + + /** + * 定时任务名称 + */ + private String name; + + /** + * 变量值列表 + */ + private List variableValue; + +} diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/EncryptEnableVariables.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/EncryptEnableVariables.java new file mode 100644 index 0000000000..4f385680f9 --- /dev/null +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/EncryptEnableVariables.java @@ -0,0 +1,96 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.crontab.model.dto; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.tencent.bk.job.common.crypto.scenario.CipherVariableCryptoService; +import com.tencent.bk.job.common.util.ApplicationContextRegister; +import com.tencent.bk.job.common.util.json.JsonUtils; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; + +import java.util.ArrayList; +import java.util.List; + +/** + * 可加密变量 + */ +@Data +@EqualsAndHashCode +@NoArgsConstructor +@AllArgsConstructor +public class EncryptEnableVariables { + + /** + * 变量值列表 + */ + protected List variableValue; + + public String getEncryptedVariableValue() { + CipherVariableCryptoService cipherVariableCryptoService = + ApplicationContextRegister.getBean(CipherVariableCryptoService.class); + if (CollectionUtils.isEmpty(this.variableValue)) { + return JsonUtils.toJson(this.variableValue); + } + List cloneVariableList = new ArrayList<>(this.variableValue.size()); + for (CronJobVariableDTO cronJobVariableDTO : this.variableValue) { + CronJobVariableDTO cloneCronJobVariable = cronJobVariableDTO.clone(); + String encryptedValue = cipherVariableCryptoService.encryptTaskVariableIfNeeded( + cloneCronJobVariable.getType(), + cloneCronJobVariable.getValue() + ); + cloneCronJobVariable.setValue(encryptedValue); + cloneVariableList.add(cloneCronJobVariable); + } + return JsonUtils.toJson(cloneVariableList); + } + + public void decryptAndSetVariableValue(String encryptedVariableValue) { + CipherVariableCryptoService cipherVariableCryptoService = + ApplicationContextRegister.getBean(CipherVariableCryptoService.class); + if (StringUtils.isBlank(encryptedVariableValue)) { + this.variableValue = new ArrayList<>(); + } + this.variableValue = JsonUtils.fromJson( + encryptedVariableValue, + new TypeReference>() { + } + ); + if (CollectionUtils.isEmpty(this.variableValue)) { + return; + } + for (CronJobVariableDTO cronJobVariableDTO : this.variableValue) { + String decryptedValue = cipherVariableCryptoService.decryptTaskVariableIfNeeded( + cronJobVariableDTO.getType(), + cronJobVariableDTO.getValue() + ); + cronJobVariableDTO.setValue(decryptedValue); + } + } +} diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/InnerCronJobHistoryDTO.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/InnerCronJobHistoryDTO.java index b4128bafe2..e0f9be89a1 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/InnerCronJobHistoryDTO.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/InnerCronJobHistoryDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/InnerCronJobInfoDTO.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/InnerCronJobInfoDTO.java index f5c7c87b18..2da1ab7805 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/InnerCronJobInfoDTO.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/InnerCronJobInfoDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/NeedScheduleCronInfo.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/NeedScheduleCronInfo.java new file mode 100644 index 0000000000..a4f9b7ef07 --- /dev/null +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/NeedScheduleCronInfo.java @@ -0,0 +1,52 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.crontab.model.dto; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + +import java.util.Collections; +import java.util.List; + +@Data +@EqualsAndHashCode +@NoArgsConstructor +@AllArgsConstructor +public class NeedScheduleCronInfo { + /** + * 需要添加到调度引擎的定时任务ID列表 + */ + private List needAddCronIdList; + /** + * 需要从调度引擎删除的定时任务ID列表 + */ + private List needDeleteCronIdList; + + public static NeedScheduleCronInfo emptyInstance() { + return new NeedScheduleCronInfo(Collections.emptyList(), Collections.emptyList()); + } +} diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/QuartzJobInfoDTO.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/QuartzJobInfoDTO.java index b463e539d5..8871e56dc8 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/QuartzJobInfoDTO.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/model/dto/QuartzJobInfoDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/mq/CrontabMQEventDispatcher.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/mq/CrontabMQEventDispatcher.java new file mode 100644 index 0000000000..bbdd7ac141 --- /dev/null +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/mq/CrontabMQEventDispatcher.java @@ -0,0 +1,55 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.crontab.mq; + +import com.tencent.bk.job.crontab.listener.event.CrontabEvent; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cloud.stream.function.StreamBridge; +import org.springframework.stereotype.Component; + +@Component +@Slf4j +public class CrontabMQEventDispatcher { + + private final StreamBridge streamBridge; + + @Autowired + public CrontabMQEventDispatcher(StreamBridge streamBridge) { + this.streamBridge = streamBridge; + } + + /** + * 广播定时任务事件 + * + * @param crontabEvent 定时任务事件 + */ + public void broadCastCrontabEvent(CrontabEvent crontabEvent) { + log.info("Begin to broadcast crontab event, event: {}", crontabEvent); + String crontabFanoutOutput = "crontabFanout-out-0"; + streamBridge.send(crontabFanoutOutput, crontabEvent); + log.info("Broadcast crontab event successfully, event: {}", crontabEvent); + } +} diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/runner/LoadCronJobRunner.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/runner/LoadCronJobRunner.java new file mode 100644 index 0000000000..377c02676d --- /dev/null +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/runner/LoadCronJobRunner.java @@ -0,0 +1,73 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.crontab.runner; + +import com.tencent.bk.job.crontab.service.CronJobLoadingService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.CommandLineRunner; +import org.springframework.stereotype.Component; + +import javax.annotation.PreDestroy; +import java.util.concurrent.Future; +import java.util.concurrent.ThreadPoolExecutor; + +/** + * 进程启动时立即将DB中的定时任务加载到Quartz + */ +@Slf4j +@Component +public class LoadCronJobRunner implements CommandLineRunner { + + private final CronJobLoadingService cronJobLoadingService; + private final ThreadPoolExecutor crontabInitRunnerExecutor; + + private Future loadCronJobFuture; + + @Autowired + public LoadCronJobRunner(CronJobLoadingService cronJobLoadingService, + @Qualifier("crontabInitRunnerExecutor") ThreadPoolExecutor crontabInitRunnerExecutor) { + this.cronJobLoadingService = cronJobLoadingService; + this.crontabInitRunnerExecutor = crontabInitRunnerExecutor; + } + + @Override + public void run(String... args) { + loadCronJobFuture = crontabInitRunnerExecutor.submit(() -> { + log.info("loadCronToQuartzOnStartup"); + cronJobLoadingService.loadAllCronJob(); + }); + } + + @PreDestroy + public void destroy() { + log.info("destroy LoadCronJobRunner"); + if (loadCronJobFuture != null) { + boolean result = loadCronJobFuture.cancel(true); + log.info("loadCronJobFuture cancel result:{}", result); + } + } +} diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/BatchCronJobService.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/BatchCronJobService.java new file mode 100644 index 0000000000..2bd55e3f76 --- /dev/null +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/BatchCronJobService.java @@ -0,0 +1,55 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.crontab.service; + +import com.tencent.bk.job.crontab.model.BatchUpdateCronJobReq; +import com.tencent.bk.job.crontab.model.dto.BatchAddResult; +import com.tencent.bk.job.crontab.model.dto.CronJobBasicInfoDTO; +import com.tencent.bk.job.crontab.model.dto.NeedScheduleCronInfo; + +import java.util.List; + +public interface BatchCronJobService { + + /** + * 批量添加定时任务到Quartz + * + * @param cronJobBasicInfoList 定时任务列表 + * @return 批量添加结果 + */ + BatchAddResult batchAddJobToQuartz(List cronJobBasicInfoList); + + /** + * 批量更新定时任务 + * + * @param username 用户名 + * @param appId Job业务ID + * @param batchUpdateCronJobReq 批量更新请求 + * @return 更新结果数据 + */ + NeedScheduleCronInfo batchUpdateCronJob(String username, + Long appId, + BatchUpdateCronJobReq batchUpdateCronJobReq); +} diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/CronJobBatchLoadService.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/CronJobBatchLoadService.java new file mode 100644 index 0000000000..692df4c83a --- /dev/null +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/CronJobBatchLoadService.java @@ -0,0 +1,62 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.crontab.service; + +import com.tencent.bk.job.crontab.model.dto.CronJobBasicInfoDTO; +import lombok.Data; + +import java.util.List; + +public interface CronJobBatchLoadService { + + /** + * 将DB中的定时任务批量加载至Quartz + * + * @param start 起始位置 + * @param limit 一批定时任务的数量 + * @return 加载结果数据 + */ + CronLoadResult batchLoadCronToQuartz(int start, int limit) throws InterruptedException; + + @Data + class CronLoadResult { + /** + * 从DB中获取到的定时任务数量 + */ + int fetchNum; + /** + * 加载成功的定时任务数量 + */ + int successNum; + /** + * 加载失败的定时任务数量 + */ + int failedNum; + /** + * 加载失败的定时任务列表 + */ + List failedCronList; + } +} diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/CronJobExecuteResultService.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/CronJobExecuteResultService.java new file mode 100644 index 0000000000..cc0418ff84 --- /dev/null +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/CronJobExecuteResultService.java @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.crontab.service; + +import com.tencent.bk.job.execute.model.inner.ServiceCronTaskExecuteResultStatistics; + +import java.util.List; +import java.util.Map; + +/** + * 定时任务执行结果 Service + */ +public interface CronJobExecuteResultService { + /** + * ESB 后端调用 URL + */ + /** + * 根据定时任务 ID 批量执行历史 + * + * @param appId 业务 ID + * @param cronIdList 定时任务 ID 列表 + * @return 定时任务 ID 与执行历史对应表 + */ + Map getCronJobExecuteHistory(Long appId, List cronIdList); +} diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/CronJobHistoryService.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/CronJobHistoryService.java index 33fbf74258..c7826d3755 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/CronJobHistoryService.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/CronJobHistoryService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/CronJobLoadingService.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/CronJobLoadingService.java new file mode 100644 index 0000000000..2e454dddb9 --- /dev/null +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/CronJobLoadingService.java @@ -0,0 +1,29 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.crontab.service; + +public interface CronJobLoadingService { + void loadAllCronJob(); +} diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/CronJobService.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/CronJobService.java index ccc33bb0c5..48a85c67e6 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/CronJobService.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/CronJobService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,13 +24,14 @@ package com.tencent.bk.job.crontab.service; +import com.tencent.bk.job.common.exception.ServiceException; import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; import com.tencent.bk.job.crontab.model.BatchUpdateCronJobReq; +import com.tencent.bk.job.crontab.model.dto.CronJobBasicInfoDTO; import com.tencent.bk.job.crontab.model.dto.CronJobInfoDTO; import com.tencent.bk.job.crontab.model.inner.ServiceInnerCronJobInfoDTO; import com.tencent.bk.job.crontab.model.inner.request.ServiceAddInnerCronJobRequestDTO; -import com.tencent.bk.job.execute.model.inner.ServiceCronTaskExecuteResultStatistics; import java.util.List; import java.util.Map; @@ -40,14 +41,14 @@ */ public interface CronJobService { /** - * 分页查询定时任务列表 + * 分页查询定时任务列表（不含变量） * * @param cronJobCondition 查询条件 * @param baseSearchCondition 搜索参数 * @return 分页后的定时任务列表 */ - PageData listPageCronJobInfos(CronJobInfoDTO cronJobCondition, - BaseSearchCondition baseSearchCondition); + PageData listPageCronJobInfosWithoutVars(CronJobInfoDTO cronJobCondition, + BaseSearchCondition baseSearchCondition); /** * 根据 ID 查询定时任务信息 @@ -58,12 +59,12 @@ PageData listPageCronJobInfos(CronJobInfoDTO cronJobCondition, CronJobInfoDTO getCronJobInfoById(Long cronJobId); /** - * 根据 IDs 查询定时任务信息，按照传入的id顺序返回结果 + * 根据 ID 查询完整的定时任务详情 * - * @param cronJobIdList 定时任务 IDs - * @return 定时任务信息 + * @param cronJobId 定时任务 ID + * @return 带有自定义消息通知配置定时任务信息 */ - List getOrderedCronJobInfoByIds(List cronJobIdList); + public CronJobInfoDTO getIntegralCronJobInfoById(Long cronJobId); /** * 根据 IDs 查询定时任务信息，返回Map @@ -82,6 +83,16 @@ PageData listPageCronJobInfos(CronJobInfoDTO cronJobCondition, */ CronJobInfoDTO getCronJobInfoById(Long appId, Long cronJobId); + /** + * 根据 ID 查询定时任务信息 + * + * @param username 用户账号 + * @param appId 业务 ID + * @param cronJobId 定时任务 ID + * @return 定时任务信息 + */ + CronJobInfoDTO getCronJobInfoById(String username, Long appId, Long cronJobId); + /** * 根据 ID 查询定时任务错误信息，上次执行状态，错误码，错误次数 * @@ -100,21 +111,32 @@ PageData listPageCronJobInfos(CronJobInfoDTO cronJobCondition, boolean updateCronJobErrorById(CronJobInfoDTO cronJobErrorInfo); /** - * 新增、保存定时任务信息 + * 创建定时任务 * - * @param cronJobInfo 待新增、保存的定时任务信息 - * @return 定时任务 ID + * @param username 用户账号 + * @param cronJobInfo 定时任务信息 + * @return 定时任务 */ - Long saveCronJobInfo(CronJobInfoDTO cronJobInfo); + CronJobInfoDTO createCronJobInfo(String username, CronJobInfoDTO cronJobInfo); + + /** + * 更新定时任务信息 + * + * @param username 用户账号 + * @param cronJobInfo 定时任务信息 + * @return 定时任务 + */ + CronJobInfoDTO updateCronJobInfo(String username, CronJobInfoDTO cronJobInfo); /** * 删除定时任务 * + * @param username 用户账号 * @param appId 业务 ID * @param cronJobId 定时任务 ID * @return 是否删除成功 */ - Boolean deleteCronJobInfo(Long appId, Long cronJobId); + Boolean deleteCronJobInfo(String username, Long appId, Long cronJobId); /** * 启用、禁用定时任务 @@ -191,25 +213,17 @@ PageData listPageCronJobInfos(CronJobInfoDTO cronJobCondition, */ Boolean deleteInnerCronJob(String systemId, String jobKey); - /** - * 根据定时任务 ID 批量执行历史 - * - * @param appId 业务 ID - * @param cronIdList 定时任务 ID 列表 - * @return 定时任务 ID 与执行历史对应表 - */ - Map getCronJobExecuteHistory(Long appId, List cronIdList); - /** * 批量更新定时任务信息 *

* 只更新 变量 和 启用 字段 * + * @param username 用户账号 * @param appId 业务 ID * @param batchUpdateCronJobReq 批量更新请求 * @return 是否更新成功 */ - Boolean batchUpdateCronJob(Long appId, BatchUpdateCronJobReq batchUpdateCronJobReq); + Boolean batchUpdateCronJob(String username, Long appId, BatchUpdateCronJobReq batchUpdateCronJobReq); /** * 带 ID 新建定时任务 @@ -245,4 +259,13 @@ PageData listPageCronJobInfos(CronJobInfoDTO cronJobCondition, boolean isExistAnyAppCronJob(Long appId); Integer countCronJob(Long appId, Boolean active, Boolean cron); + + boolean checkAndAddJobToQuartz(long appId, long cronJobId) throws ServiceException; + + List listEnabledCronBasicInfoForUpdate(int start, int limit); + + /** + * 通过业务id禁用定时任务 + */ + boolean disableCronJobByAppId(Long appId); } diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/CustomNotifyPolicyService.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/CustomNotifyPolicyService.java new file mode 100644 index 0000000000..23ece48342 --- /dev/null +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/CustomNotifyPolicyService.java @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.crontab.service; + + +import com.tencent.bk.job.common.model.dto.notify.CustomNotifyDTO; +import com.tencent.bk.job.crontab.model.dto.CronJobInfoDTO; + +public interface CustomNotifyPolicyService { + + void createOrUpdateCronJobCustomNotifyPolicy(Long cronJobId, CronJobInfoDTO cronJobInfoDTO); + + void deleteCronJobCustomNotifyPolicy(Long appId, Long cronJobId); + + CustomNotifyDTO getCronJobCustomNotifyPolicyById(Long appId, Long cronJobId); +} diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/ExecuteTaskService.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/ExecuteTaskService.java index d3b402dc6e..daaac69be6 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/ExecuteTaskService.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/ExecuteTaskService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -66,7 +66,7 @@ InternalResponse executeTask( * @throws TaskExecuteAuthFailedException 鉴权失败抛出异常 */ void authExecuteTask( - long appId, long taskId, long cronTaskId, String cronName, + long appId, long taskId, Long cronTaskId, String cronName, List variableList, String operator ) throws TaskExecuteAuthFailedException; } diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/HostService.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/HostService.java new file mode 100644 index 0000000000..e1f04f368c --- /dev/null +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/HostService.java @@ -0,0 +1,41 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.crontab.service; + +import com.tencent.bk.job.common.model.dto.HostDTO; + +import java.util.List; + +public interface HostService { + + /** + * 通过hostId/IP查询主机信息并填充 + * + * @param hostList 主机列表 + * @return 实际填充的主机数量 + */ + int fillHosts(List hostList); + +} diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/InnerJobHistoryService.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/InnerJobHistoryService.java index 0f60261755..8b6834ad00 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/InnerJobHistoryService.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/InnerJobHistoryService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/NotifyService.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/NotifyService.java index 3ccb693b52..0335e0007d 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/NotifyService.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/NotifyService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/QuartzService.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/QuartzService.java new file mode 100644 index 0000000000..eb898c0fd9 --- /dev/null +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/QuartzService.java @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.crontab.service; + +import com.tencent.bk.job.crontab.model.dto.CronJobInfoDTO; + +public interface QuartzService { + /** + * 尝试将定时任务添加到Quartz中，若失败则删除 + * + * @param cronJobInfo 定时任务信息 + */ + void tryToAddJobToQuartz(CronJobInfoDTO cronJobInfo); + + /** + * 从Quartz中删除定时任务 + * + * @param appId Job业务ID + * @param cronJobId 定时任务ID + * @return 是否删除成功 + */ + boolean deleteJobFromQuartz(long appId, long cronJobId); +} diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/TaskExecuteResultService.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/TaskExecuteResultService.java index 50906c3274..fd90ca18e4 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/TaskExecuteResultService.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/TaskExecuteResultService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/TaskPlanService.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/TaskPlanService.java index 896f41b095..2c45b7a420 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/TaskPlanService.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/TaskPlanService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/BatchCronJobServiceImpl.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/BatchCronJobServiceImpl.java new file mode 100644 index 0000000000..83ce64f803 --- /dev/null +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/BatchCronJobServiceImpl.java @@ -0,0 +1,244 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.crontab.service.impl; + +import com.tencent.bk.audit.context.ActionAuditContext; +import com.tencent.bk.job.common.model.dto.AppResourceScope; +import com.tencent.bk.job.common.mysql.JobTransactional; +import com.tencent.bk.job.crontab.auth.CronAuthService; +import com.tencent.bk.job.crontab.dao.CronJobDAO; +import com.tencent.bk.job.crontab.exception.TaskExecuteAuthFailedException; +import com.tencent.bk.job.crontab.model.BatchUpdateCronJobReq; +import com.tencent.bk.job.crontab.model.CronJobCreateUpdateReq; +import com.tencent.bk.job.crontab.model.dto.AddJobToQuartzResult; +import com.tencent.bk.job.crontab.model.dto.BatchAddResult; +import com.tencent.bk.job.crontab.model.dto.CronJobBasicInfoDTO; +import com.tencent.bk.job.crontab.model.dto.CronJobInfoDTO; +import com.tencent.bk.job.crontab.model.dto.CronJobVariableDTO; +import com.tencent.bk.job.crontab.model.dto.NeedScheduleCronInfo; +import com.tencent.bk.job.crontab.service.BatchCronJobService; +import com.tencent.bk.job.crontab.service.ExecuteTaskService; +import com.tencent.bk.job.crontab.service.QuartzService; +import com.tencent.bk.job.execute.model.inner.ServiceTaskVariable; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.slf4j.helpers.MessageFormatter; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * 批量操作定时任务 Service 实现 + */ +@Slf4j +@Service +public class BatchCronJobServiceImpl implements BatchCronJobService { + + private final CronJobDAO cronJobDAO; + private final CronAuthService cronAuthService; + private final ExecuteTaskService executeTaskService; + private final QuartzService quartzService; + + @Autowired + public BatchCronJobServiceImpl(CronJobDAO cronJobDAO, + CronAuthService cronAuthService, + ExecuteTaskService executeTaskService, QuartzService quartzService) { + this.cronJobDAO = cronJobDAO; + this.cronAuthService = cronAuthService; + this.executeTaskService = executeTaskService; + this.quartzService = quartzService; + } + + /** + * 批量添加定时任务到Quartz + * + * @param cronJobBasicInfoList 定时任务列表 + * @return 批量添加结果 + */ + @Override + public BatchAddResult batchAddJobToQuartz(List cronJobBasicInfoList) { + List cronJobIdList = cronJobBasicInfoList.stream() + .map(CronJobBasicInfoDTO::getId) + .distinct() + .collect(Collectors.toList()); + // 1.批量获取定时任务信息 + List cronJobList = cronJobDAO.listCronJobByIds(cronJobIdList); + Map cronJobMap = cronJobList.stream() + .collect( + Collectors.toMap( + CronJobInfoDTO::getId, + cronJobInfoDTO -> cronJobInfoDTO + ) + ); + BatchAddResult finalBatchResult = new BatchAddResult(); + BatchAddResult notFoundBatchResult = recordNotFoundCronJobs( + cronJobBasicInfoList, + cronJobMap + ); + finalBatchResult.merge(notFoundBatchResult); + // 2.过滤出开启的定时任务进行触发 + BatchAddResult addToQuartzBatchResult = addEnabledCronJobToQuartz(cronJobList); + finalBatchResult.merge(addToQuartzBatchResult); + return finalBatchResult; + } + + /** + * 批量添加开启的定时任务到Quartz + * + * @param cronJobList 定时任务列表 + * @return 批量添加结果 + */ + private BatchAddResult addEnabledCronJobToQuartz(List cronJobList) { + BatchAddResult batchAddResult = new BatchAddResult(); + for (CronJobInfoDTO cronJobInfoDTO : cronJobList) { + if (!cronJobInfoDTO.getEnable()) { + batchAddResult.addResult(AddJobToQuartzResult.failResult( + cronJobInfoDTO.toBasicInfoDTO(), + String.format("CronJob(id=%s) is not enabled, ignore", cronJobInfoDTO.getId()) + )); + continue; + } + try { + quartzService.tryToAddJobToQuartz(cronJobInfoDTO); + batchAddResult.addResult(AddJobToQuartzResult.successResult(cronJobInfoDTO.toBasicInfoDTO())); + } catch (Exception e) { + String message = MessageFormatter.format( + "Add cronJob(id={}) to quartz failed", + cronJobInfoDTO.getId() + ).getMessage(); + batchAddResult.addResult( + AddJobToQuartzResult.failResult( + cronJobInfoDTO.toBasicInfoDTO(), + message, + e + ) + ); + } + } + return batchAddResult; + } + + /** + * 记录DB中不存在的定时任务 + * + * @param cronJobBasicInfoList 需要添加到Quartz的定时任务基础信息列表 + * @param cronJobMap DB中存在的定时任务信息 + * @return 批量添加失败的定时任务信息 + */ + private BatchAddResult recordNotFoundCronJobs(List cronJobBasicInfoList, + Map cronJobMap) { + BatchAddResult batchAddResult = new BatchAddResult(); + for (CronJobBasicInfoDTO cronJobBasicInfoDTO : cronJobBasicInfoList) { + Long cronJobId = cronJobBasicInfoDTO.getId(); + if (!cronJobMap.containsKey(cronJobId)) { + batchAddResult.addResult( + AddJobToQuartzResult.failResult( + cronJobBasicInfoDTO, + "Cannot find cronJob in DB by id: " + cronJobId + ) + ); + } + } + return batchAddResult; + } + + /** + * 批量更新定时任务DB数据，返回需要重新进行调度的任务信息，支持事务 + * + * @param username 用户名 + * @param appId Job业务ID + * @param batchUpdateCronJobReq 批量更新定时任务请求 + * @return 需要重新进行调度的任务信息 + */ + @JobTransactional(transactionManager = "jobCrontabTransactionManager") + public NeedScheduleCronInfo batchUpdateCronJob(String username, + Long appId, + BatchUpdateCronJobReq batchUpdateCronJobReq) { + List cronJobReqList = batchUpdateCronJobReq.getCronJobInfoList(); + if (CollectionUtils.isEmpty(cronJobReqList)) { + return NeedScheduleCronInfo.emptyInstance(); + } + List cronJobInstanceList = new ArrayList<>(); + cronJobReqList.forEach(cronJobCreateUpdateReq -> cronJobInstanceList.add(cronJobCreateUpdateReq.getId())); + cronAuthService.batchAuthManageCron(username, new AppResourceScope(appId), cronJobInstanceList); + + List needAddCronIdList = new ArrayList<>(); + List needDeleteCronIdList = new ArrayList<>(); + cronJobReqList.forEach(cronJobReq -> + updateCronJob(username, appId, cronJobReq, needAddCronIdList, needDeleteCronIdList) + ); + return new NeedScheduleCronInfo(needAddCronIdList, needDeleteCronIdList); + } + + private void updateCronJob(String username, + Long appId, + CronJobCreateUpdateReq cronJobReq, + List needAddCronIdList, + List needDeleteCronIdList) { + Long cronJobId = cronJobReq.getId(); + CronJobInfoDTO cronJobInfoFromReq = CronJobInfoDTO.fromReq(username, appId, cronJobReq); + cronJobInfoFromReq.setEnable(cronJobReq.getEnable()); + CronJobInfoDTO originCronJobInfo = cronJobDAO.getCronJobById(appId, cronJobId); + if (cronJobReq.getEnable()) { + try { + List taskVariables = null; + if (CollectionUtils.isNotEmpty(originCronJobInfo.getVariableValue())) { + taskVariables = originCronJobInfo.getVariableValue().stream() + .map(CronJobVariableDTO::toServiceTaskVariable) + .collect(Collectors.toList()); + } + executeTaskService.authExecuteTask( + appId, + originCronJobInfo.getTaskPlanId(), + cronJobId, + originCronJobInfo.getName(), + taskVariables, + username + ); + if (cronJobDAO.updateCronJobById(cronJobInfoFromReq)) { + needAddCronIdList.add(cronJobId); + } + } catch (TaskExecuteAuthFailedException e) { + log.error("Error while pre auth cron execute!", e); + throw e; + } + } else { + if (cronJobDAO.updateCronJobById(cronJobInfoFromReq)) { + needDeleteCronIdList.add(cronJobId); + } + } + CronJobInfoDTO updateCronJobInfo = cronJobDAO.getCronJobById(appId, cronJobId); + ActionAuditContext.current().addInstanceInfo( + String.valueOf(cronJobId), + cronJobReq.getName(), + originCronJobInfo, + updateCronJobInfo + ); + } +} diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/CronJobBatchLoadServiceImpl.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/CronJobBatchLoadServiceImpl.java new file mode 100644 index 0000000000..620f1ab025 --- /dev/null +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/CronJobBatchLoadServiceImpl.java @@ -0,0 +1,120 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.crontab.service.impl; + +import com.tencent.bk.job.common.mysql.JobTransactional; +import com.tencent.bk.job.crontab.model.dto.AddJobToQuartzResult; +import com.tencent.bk.job.crontab.model.dto.BatchAddResult; +import com.tencent.bk.job.crontab.model.dto.CronJobBasicInfoDTO; +import com.tencent.bk.job.crontab.service.BatchCronJobService; +import com.tencent.bk.job.crontab.service.CronJobBatchLoadService; +import com.tencent.bk.job.crontab.service.CronJobService; +import lombok.extern.slf4j.Slf4j; +import org.slf4j.helpers.MessageFormatter; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.List; + +@Slf4j +@Service +public class CronJobBatchLoadServiceImpl implements CronJobBatchLoadService { + + private final CronJobService cronJobService; + private final BatchCronJobService batchCronJobService; + + @Autowired + public CronJobBatchLoadServiceImpl(CronJobService cronJobService, BatchCronJobService batchCronJobService) { + this.cronJobService = cronJobService; + this.batchCronJobService = batchCronJobService; + } + + @Override + @JobTransactional(transactionManager = "jobCrontabTransactionManager", timeout = 30) + public CronLoadResult batchLoadCronToQuartz(int start, int limit) throws InterruptedException { + checkInterrupt(); + List cronJobBasicInfoList = cronJobService.listEnabledCronBasicInfoForUpdate(start, limit); + BatchAddResult batchAddResult = batchCronJobService.batchAddJobToQuartz(cronJobBasicInfoList); + List failedCronList = extractFailedCronList(batchAddResult); + CronLoadResult cronLoadResult = new CronLoadResult(); + cronLoadResult.setFetchNum(cronJobBasicInfoList.size()); + cronLoadResult.setSuccessNum(batchAddResult.getSuccessNum()); + cronLoadResult.setFailedNum(batchAddResult.getFailNum()); + cronLoadResult.setFailedCronList(failedCronList); + return cronLoadResult; + } + + /** + * 从批量添加定时任务结果数据中提取失败的定时任务信息 + * + * @param batchAddResult 批量添加定时任务结果 + * @return 失败的定时任务信息 + */ + private List extractFailedCronList(BatchAddResult batchAddResult) { + if (batchAddResult == null || batchAddResult.getTotalNum() == 0) { + return new ArrayList<>(); + } + List failedCronList = new ArrayList<>(); + int successNum = batchAddResult.getSuccessNum(); + int failedNum = batchAddResult.getFailNum(); + if (failedNum > 0) { + String message = MessageFormatter.format( + "batchAddJobToQuartz result: {} failed, {} success", + failedNum, + successNum + ).getMessage(); + if (batchAddResult.getFailRate() > 0.5) { + log.error(message); + } else { + log.warn(message); + } + List failedResultList = batchAddResult.getFailedResultList(); + for (AddJobToQuartzResult addJobToQuartzResult : failedResultList) { + CronJobBasicInfoDTO cronJobBasicInfo = addJobToQuartzResult.getCronJobBasicInfo(); + failedCronList.add(cronJobBasicInfo); + message = MessageFormatter.arrayFormat( + "Fail to load cronJob({},{},{}), reason={}", + new Object[]{ + cronJobBasicInfo.getAppId(), + cronJobBasicInfo.getId(), + cronJobBasicInfo.getName(), + addJobToQuartzResult.getMessage() + } + ).getMessage(); + log.warn(message, addJobToQuartzResult.getException()); + } + } else { + log.info("batchAddJobToQuartz result: All success, num={}", successNum); + } + return failedCronList; + } + + private void checkInterrupt() throws InterruptedException { + if (Thread.currentThread().isInterrupted()) { + throw new InterruptedException("batchLoadCronToQuartz thread is interrupted, exit"); + } + } +} diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/CronJobExecuteResultServiceImpl.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/CronJobExecuteResultServiceImpl.java new file mode 100644 index 0000000000..46bc15512d --- /dev/null +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/CronJobExecuteResultServiceImpl.java @@ -0,0 +1,52 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.crontab.service.impl; + +import com.tencent.bk.job.crontab.service.CronJobExecuteResultService; +import com.tencent.bk.job.crontab.service.TaskExecuteResultService; +import com.tencent.bk.job.execute.model.inner.ServiceCronTaskExecuteResultStatistics; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.List; +import java.util.Map; + +@Service +public class CronJobExecuteResultServiceImpl implements CronJobExecuteResultService { + + + private final TaskExecuteResultService taskExecuteResultService; + + @Autowired + public CronJobExecuteResultServiceImpl(TaskExecuteResultService taskExecuteResultService) { + this.taskExecuteResultService = taskExecuteResultService; + } + + @Override + public Map getCronJobExecuteHistory(Long appId, + List cronIdList) { + return taskExecuteResultService.getCronTaskExecuteResultStatistics(appId, cronIdList); + } +} diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/CronJobHistoryServiceImpl.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/CronJobHistoryServiceImpl.java index ed896911cd..f46ab97343 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/CronJobHistoryServiceImpl.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/CronJobHistoryServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -44,6 +44,7 @@ import org.quartz.CronExpression; import org.quartz.SchedulerException; import org.quartz.SimpleTrigger; +import org.slf4j.helpers.MessageFormatter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Component; @@ -81,7 +82,11 @@ private void addCleanJob() { try { new CronExpression(cleanJobCronExp); } catch (ParseException e) { - log.warn("Error while adding cron history clean job! Invalid expression|{}", cleanJobCronExp, e); + String msg = MessageFormatter.format( + "Error while adding cron history clean job! Invalid expression|{}", + cleanJobCronExp + ).getMessage(); + log.warn(msg, e); return; } diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/CronJobLoadingServiceImpl.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/CronJobLoadingServiceImpl.java new file mode 100644 index 0000000000..912204040a --- /dev/null +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/CronJobLoadingServiceImpl.java @@ -0,0 +1,112 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.crontab.service.impl; + +import com.tencent.bk.job.crontab.model.dto.CronJobBasicInfoDTO; +import com.tencent.bk.job.crontab.service.CronJobBatchLoadService; +import com.tencent.bk.job.crontab.service.CronJobLoadingService; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.quartz.Scheduler; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.List; + +@Slf4j +@Service +public class CronJobLoadingServiceImpl implements CronJobLoadingService { + + private final CronJobBatchLoadService cronJobBatchLoadService; + private final Scheduler scheduler; + private volatile boolean loadingCronToQuartz = false; + + @Autowired + public CronJobLoadingServiceImpl(CronJobBatchLoadService cronJobBatchLoadService, Scheduler scheduler) { + this.cronJobBatchLoadService = cronJobBatchLoadService; + this.scheduler = scheduler; + } + + @Override + public void loadAllCronJob() { + long start = System.currentTimeMillis(); + try { + if (loadingCronToQuartz) { + log.info("Last loading not finish, ignore"); + return; + } + loadingCronToQuartz = true; + waitUtilQuartzStarted(); + loadAllCronJobToQuartz(); + } catch (InterruptedException e) { + log.info("loadAllCronJob interrupted, application may be closing"); + } catch (Exception e) { + log.warn("Fail to loadAllCronJob", e); + } finally { + loadingCronToQuartz = false; + log.info("loadAllCronJob end, duration={}ms", System.currentTimeMillis() - start); + } + } + + private void waitUtilQuartzStarted() throws Exception { + while (!scheduler.isStarted()) { + log.info("Quartz Scheduler is not started, sleep 1s and retry"); + Thread.sleep(1000); + } + log.info("Quartz Scheduler is started now"); + } + + private void loadAllCronJobToQuartz() throws InterruptedException { + int start = 0; + int limit = 100; + int currentFetchNum; + int allFetchNum = 0; + int successNum = 0; + int failedNum = 0; + List failedCronList = new ArrayList<>(); + do { + CronJobBatchLoadService.CronLoadResult loadResult = cronJobBatchLoadService.batchLoadCronToQuartz( + start, + limit + ); + currentFetchNum = loadResult.getFetchNum(); + allFetchNum += currentFetchNum; + successNum += loadResult.getSuccessNum(); + failedNum += loadResult.getFailedNum(); + if (CollectionUtils.isNotEmpty(loadResult.getFailedCronList())) { + failedCronList.addAll(loadResult.getFailedCronList()); + } + start += limit; + } while (currentFetchNum > 0); + log.info( + "CronJobs load from db finished: fetchNum={}, successNum={}, failedNum={}, failedCronList={}", + allFetchNum, + successNum, + failedNum, + failedCronList + ); + } +} diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/CronJobServiceImpl.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/CronJobServiceImpl.java index 213acac989..daff60bb60 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/CronJobServiceImpl.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/CronJobServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,33 +24,52 @@ package com.tencent.bk.job.crontab.service.impl; +import com.tencent.bk.audit.annotations.ActionAuditRecord; +import com.tencent.bk.audit.annotations.AuditInstanceRecord; +import com.tencent.bk.audit.context.ActionAuditContext; +import com.tencent.bk.job.common.audit.constants.EventContentConstants; +import com.tencent.bk.job.common.constant.CronJobNotifyType; import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.constant.JobConstants; +import com.tencent.bk.job.common.constant.TaskVariableTypeEnum; import com.tencent.bk.job.common.exception.AlreadyExistsException; import com.tencent.bk.job.common.exception.FailedPreconditionException; import com.tencent.bk.job.common.exception.InternalException; import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.exception.NotFoundException; import com.tencent.bk.job.common.exception.ServiceException; +import com.tencent.bk.job.common.iam.constant.ActionId; +import com.tencent.bk.job.common.iam.constant.ResourceTypeId; import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; -import com.tencent.bk.job.common.redis.util.LockUtils; -import com.tencent.bk.job.common.util.JobContextUtil; +import com.tencent.bk.job.common.model.dto.AppResourceScope; +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.common.model.dto.notify.CustomNotifyDTO; +import com.tencent.bk.job.common.mysql.JobTransactional; import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.crontab.auth.CronAuthService; import com.tencent.bk.job.crontab.constant.CronConstants; import com.tencent.bk.job.crontab.dao.CronJobDAO; import com.tencent.bk.job.crontab.exception.TaskExecuteAuthFailedException; +import com.tencent.bk.job.crontab.listener.event.CrontabEvent; import com.tencent.bk.job.crontab.model.BatchUpdateCronJobReq; +import com.tencent.bk.job.crontab.model.dto.CronJobBasicInfoDTO; import com.tencent.bk.job.crontab.model.dto.CronJobInfoDTO; import com.tencent.bk.job.crontab.model.dto.CronJobVariableDTO; import com.tencent.bk.job.crontab.model.dto.InnerCronJobInfoDTO; +import com.tencent.bk.job.crontab.model.dto.NeedScheduleCronInfo; import com.tencent.bk.job.crontab.model.dto.QuartzJobInfoDTO; +import com.tencent.bk.job.crontab.model.inner.ServerDTO; import com.tencent.bk.job.crontab.model.inner.ServiceInnerCronJobInfoDTO; import com.tencent.bk.job.crontab.model.inner.request.ServiceAddInnerCronJobRequestDTO; +import com.tencent.bk.job.crontab.mq.CrontabMQEventDispatcher; +import com.tencent.bk.job.crontab.service.BatchCronJobService; import com.tencent.bk.job.crontab.service.CronJobService; +import com.tencent.bk.job.crontab.service.CustomNotifyPolicyService; import com.tencent.bk.job.crontab.service.ExecuteTaskService; -import com.tencent.bk.job.crontab.service.TaskExecuteResultService; +import com.tencent.bk.job.crontab.service.HostService; +import com.tencent.bk.job.crontab.service.QuartzService; import com.tencent.bk.job.crontab.service.TaskPlanService; import com.tencent.bk.job.crontab.timer.AbstractQuartzTaskHandler; import com.tencent.bk.job.crontab.timer.QuartzJob; @@ -58,9 +77,6 @@ import com.tencent.bk.job.crontab.timer.QuartzTrigger; import com.tencent.bk.job.crontab.timer.QuartzTriggerBuilder; import com.tencent.bk.job.crontab.timer.executor.InnerJobExecutor; -import com.tencent.bk.job.crontab.timer.executor.NotifyJobExecutor; -import com.tencent.bk.job.crontab.timer.executor.SimpleJobExecutor; -import com.tencent.bk.job.execute.model.inner.ServiceCronTaskExecuteResultStatistics; import com.tencent.bk.job.execute.model.inner.ServiceTaskVariable; import com.tencent.bk.job.manage.model.inner.ServiceTaskPlanDTO; import lombok.extern.slf4j.Slf4j; @@ -70,9 +86,9 @@ import org.quartz.JobKey; import org.quartz.SchedulerException; import org.quartz.SimpleTrigger; +import org.slf4j.helpers.MessageFormatter; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import org.springframework.transaction.annotation.Transactional; +import org.springframework.stereotype.Service; import java.sql.Date; import java.time.Instant; @@ -84,51 +100,54 @@ import java.util.Map; import java.util.stream.Collectors; +import static com.tencent.bk.job.common.audit.JobAuditAttributeNames.OPERATION; + /** - * @since 2/1/2020 12:18 + * 定时任务 Service 实现 */ @Slf4j -@Component +@Service public class CronJobServiceImpl implements CronJobService { private final CronJobDAO cronJobDAO; - private final TaskExecuteResultService taskExecuteResultService; private final AbstractQuartzTaskHandler quartzTaskHandler; + private final QuartzService quartzService; private final TaskPlanService taskPlanService; - private final CronAuthService cronAuthService; - private final ExecuteTaskService executeTaskService; + private final HostService hostService; + private final CrontabMQEventDispatcher crontabMQEventDispatcher; + private final BatchCronJobService batchCronJobService; + private final CustomNotifyPolicyService customNotifyPolicyService; @Autowired - public CronJobServiceImpl(CronJobDAO cronJobDAO, TaskExecuteResultService taskExecuteResultService, - AbstractQuartzTaskHandler quartzTaskHandler, TaskPlanService taskPlanService, - CronAuthService cronAuthService, ExecuteTaskService executeTaskService) { + public CronJobServiceImpl(CronJobDAO cronJobDAO, + AbstractQuartzTaskHandler quartzTaskHandler, + QuartzService quartzService, + TaskPlanService taskPlanService, + CronAuthService cronAuthService, + ExecuteTaskService executeTaskService, + HostService hostService, + CrontabMQEventDispatcher crontabMQEventDispatcher, + BatchCronJobServiceImpl batchCronJobService, + CustomNotifyPolicyService customNotifyPolicyService) { this.cronJobDAO = cronJobDAO; - this.taskExecuteResultService = taskExecuteResultService; this.quartzTaskHandler = quartzTaskHandler; + this.quartzService = quartzService; this.taskPlanService = taskPlanService; this.cronAuthService = cronAuthService; this.executeTaskService = executeTaskService; - } - - private static String getJobName(long appId, long cronJobId) { - return "job_" + cronJobId; - } - - private static String getJobGroup(long appId, long cronJobId) { - return "bk_app_" + appId; - } - - private static String getNotifyJobName(long appId, long cronJobId) { - return getJobName(appId, cronJobId) + "_notify"; + this.hostService = hostService; + this.crontabMQEventDispatcher = crontabMQEventDispatcher; + this.batchCronJobService = batchCronJobService; + this.customNotifyPolicyService = customNotifyPolicyService; } @Override - public PageData listPageCronJobInfos(CronJobInfoDTO cronJobCondition, - BaseSearchCondition baseSearchCondition) { - return cronJobDAO.listPageCronJobsByCondition(cronJobCondition, baseSearchCondition); + public PageData listPageCronJobInfosWithoutVars(CronJobInfoDTO cronJobCondition, + BaseSearchCondition baseSearchCondition) { + return cronJobDAO.listPageCronJobsWithoutVarsByCondition(cronJobCondition, baseSearchCondition); } @Override @@ -137,21 +156,15 @@ public CronJobInfoDTO getCronJobInfoById(Long cronJobId) { } @Override - public List getOrderedCronJobInfoByIds(List cronJobIdList) { - Map map = getCronJobInfoMapByIds(cronJobIdList); - List cronJobInfoDTOList = new ArrayList<>(); - for (Long id : cronJobIdList) { - CronJobInfoDTO cronJobInfoDTO = map.get(id); - if (cronJobInfoDTO != null) { - cronJobInfoDTOList.add(cronJobInfoDTO); - } - } - return cronJobInfoDTOList; + public CronJobInfoDTO getIntegralCronJobInfoById(Long cronJobId) { + CronJobInfoDTO cronJobInfo = getCronJobInfoById(cronJobId); + fillCronJobInfoWithCustomNotifyPolicy(cronJobId, cronJobInfo); + return cronJobInfo; } @Override public Map getCronJobInfoMapByIds(List cronJobIdList) { - List cronJobInfoDTOList = cronJobDAO.getCronJobByIds(cronJobIdList); + List cronJobInfoDTOList = cronJobDAO.listCronJobByIds(cronJobIdList); Map map = new HashMap<>(); for (CronJobInfoDTO cronJobInfoDTO : cronJobInfoDTOList) { map.put(cronJobInfoDTO.getId(), cronJobInfoDTO); @@ -164,6 +177,26 @@ public CronJobInfoDTO getCronJobInfoById(Long appId, Long cronJobId) { return cronJobDAO.getCronJobById(appId, cronJobId); } + @Override + @ActionAuditRecord( + actionId = ActionId.MANAGE_CRON, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.CRON, + instanceIds = "#cronJobId", + instanceNames = "#$?.name" + ), + content = EventContentConstants.VIEW_CRON_JOB + ) + public CronJobInfoDTO getCronJobInfoById(String username, Long appId, Long cronJobId) { + CronJobInfoDTO cronJob = getIntegralCronJobInfoById(cronJobId); + if (cronJob == null) { + throw new NotFoundException(ErrorCode.CRON_JOB_NOT_EXIST); + } + cronAuthService.authManageCron(username, + new AppResourceScope(appId), cronJobId, null).denyIfNoPermission(); + return cronJob; + } + @Override public CronJobInfoDTO getCronJobErrorInfoById(Long appId, Long cronJobId) { return cronJobDAO.getCronJobErrorById(appId, cronJobId); @@ -175,77 +208,190 @@ public boolean updateCronJobErrorById(CronJobInfoDTO cronJobErrorInfo) { } @Override - @Transactional(rollbackFor = {Exception.class, Error.class}) - public Long saveCronJobInfo(CronJobInfoDTO cronJobInfo) { + @JobTransactional(transactionManager = "jobCrontabTransactionManager") + @ActionAuditRecord( + actionId = ActionId.CREATE_CRON, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.CRON, + instanceIds = "#$?.id", + instanceNames = "#$?.name" + ), + content = EventContentConstants.CREATE_CRON_JOB + ) + public CronJobInfoDTO createCronJobInfo(String username, CronJobInfoDTO cronJobInfo) { + cronAuthService.authCreateCron(username, + new AppResourceScope(cronJobInfo.getAppId())).denyIfNoPermission(); + checkCronJobPlanOrScript(cronJobInfo); - if (cronJobInfo.getId() == null || cronJobInfo.getId() == 0) { - cronJobInfo.setCreateTime(DateUtils.currentTimeSeconds()); - cronJobInfo.setEnable(false); - Long id = cronJobDAO.insertCronJob(cronJobInfo); - cronAuthService.registerCron(id, cronJobInfo.getName(), cronJobInfo.getCreator()); - return id; + saveSnapShotForHostVaiableValue(cronJobInfo); + + // 有执行方案运行权限，才能用该执行方案创建定时任务 + authExecuteTask(cronJobInfo); + cronJobInfo.setCreateTime(DateUtils.currentTimeSeconds()); + cronJobInfo.setEnable(false); + + Long id = cronJobDAO.insertCronJob(cronJobInfo); + // 保存定时任务自定义通知策略 + saveCustomNotifyPolicyIfNeeded(id, cronJobInfo); + + cronAuthService.registerCron(id, cronJobInfo.getName(), cronJobInfo.getCreator()); + + return getIntegralCronJobInfoById(id); + } + + private void saveCustomNotifyPolicyIfNeeded(Long id, CronJobInfoDTO cronJobInfo) { + if (cronJobInfo.hasCustomNotifyPolicy()) { + log.debug("[saveCustomNotifyPolicyIfNeeded] cron task:{} has custom notify policy," + + " try to save custom notify policy", id); + customNotifyPolicyService.createOrUpdateCronJobCustomNotifyPolicy(id, cronJobInfo); } else { - checkCronJobVariableValue(cronJobInfo); - if (cronJobInfo.getEnable()) { - try { - List taskVariables = null; - if (CollectionUtils.isNotEmpty(cronJobInfo.getVariableValue())) { - taskVariables = - cronJobInfo.getVariableValue().parallelStream() - .map(CronJobVariableDTO::toServiceTaskVariable).collect(Collectors.toList()); - } - executeTaskService.authExecuteTask(cronJobInfo.getAppId(), cronJobInfo.getTaskPlanId(), - cronJobInfo.getId(), cronJobInfo.getName(), taskVariables, cronJobInfo.getLastModifyUser()); - if (cronJobDAO.updateCronJobById(cronJobInfo)) { - addJob(cronJobInfo.getAppId(), cronJobInfo.getId()); - } else { - throw new InternalException(ErrorCode.UPDATE_CRON_JOB_FAILED); - } - } catch (TaskExecuteAuthFailedException e) { - log.error("Error while pre auth cron execute!", e); - throw e; - } + log.debug("[saveCustomNotifyPolicyIfNeeded] cron task with id={} use app notify policy," + + "clean its configured policy", id); + customNotifyPolicyService.deleteCronJobCustomNotifyPolicy(cronJobInfo.getAppId(), id); + } + } + + @Override + @ActionAuditRecord( + actionId = ActionId.MANAGE_CRON, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.CRON, + instanceIds = "#cronJobInfo?.id", + instanceNames = "#$?.name" + ), + content = EventContentConstants.EDIT_CRON_JOB + ) + public CronJobInfoDTO updateCronJobInfo(String username, CronJobInfoDTO cronJobInfo) { + cronAuthService.authManageCron(username, + new AppResourceScope(cronJobInfo.getAppId()), cronJobInfo.getId(), null).denyIfNoPermission(); + + CronJobInfoDTO originCron = getIntegralCronJobInfoById(cronJobInfo.getId()); + if (originCron == null) { + throw new NotFoundException(ErrorCode.CRON_JOB_NOT_EXIST); + } + + checkCronJobPlanOrScript(cronJobInfo); + processCronJobVariableValueMask(cronJobInfo); + + if (cronJobInfo.getEnable()) { + authExecuteTask(cronJobInfo); + if (cronJobDAO.updateCronJobById(cronJobInfo)) { + informAllToAddJobToQuartz(cronJobInfo.getAppId(), cronJobInfo.getId()); } else { - if (cronJobDAO.updateCronJobById(cronJobInfo)) { - deleteJob(cronJobInfo.getAppId(), cronJobInfo.getId()); - } else { - throw new InternalException(ErrorCode.UPDATE_CRON_JOB_FAILED); - } + throw new InternalException(ErrorCode.UPDATE_CRON_JOB_FAILED); + } + } else { + if (cronJobDAO.updateCronJobById(cronJobInfo)) { + informAllToDeleteJobFromQuartz(cronJobInfo.getAppId(), cronJobInfo.getId()); + } else { + throw new InternalException(ErrorCode.UPDATE_CRON_JOB_FAILED); } - return cronJobInfo.getId(); } + + // 保存自定义定时任务级别通知策略 + saveCustomNotifyPolicyIfNeeded(cronJobInfo.getId(), cronJobInfo); + + CronJobInfoDTO updateCron = getIntegralCronJobInfoById(cronJobInfo.getId()); + + // 审计 + ActionAuditContext.current() + .setOriginInstance(CronJobInfoDTO.toEsbCronInfoV3(originCron)) + .setInstance(CronJobInfoDTO.toEsbCronInfoV3(updateCron)); + + return updateCron; } - private void checkCronJobVariableValue(CronJobInfoDTO cronJobInfo) { - if (CollectionUtils.isNotEmpty(cronJobInfo.getVariableValue())) { - List hasMaskVariableList = new ArrayList<>(); - for (CronJobVariableDTO cronJobVariable : cronJobInfo.getVariableValue()) { - if (cronJobVariable.getType().needMask()) { - if (cronJobVariable.getValue().equals(cronJobVariable.getType().getMask())) { - hasMaskVariableList.add(cronJobVariable); - } + private void fillCronJobInfoWithCustomNotifyPolicy(Long cronJobId, CronJobInfoDTO cronJobInfo) { + CustomNotifyDTO cronJobCustomNotifyPolicy = customNotifyPolicyService.getCronJobCustomNotifyPolicyById( + cronJobInfo.getAppId(), cronJobId); + if (cronJobCustomNotifyPolicy == null) { + // 定时任务继承业务通知配置 + cronJobInfo.setNotifyType(CronJobNotifyType.EXTENDS_APP.getType()); + } else { + cronJobInfo.setNotifyType(CronJobNotifyType.CUSTOM.getType()); + cronJobInfo.setCustomCronJobNotifyDTO(cronJobCustomNotifyPolicy); + } + } + + private void authExecuteTask(CronJobInfoDTO cronJobInfo) { + try { + List taskVariables = null; + if (CollectionUtils.isNotEmpty(cronJobInfo.getVariableValue())) { + taskVariables = + cronJobInfo.getVariableValue().parallelStream() + .map(CronJobVariableDTO::toServiceTaskVariable).collect(Collectors.toList()); + } + executeTaskService.authExecuteTask(cronJobInfo.getAppId(), cronJobInfo.getTaskPlanId(), + cronJobInfo.getId(), cronJobInfo.getName(), taskVariables, cronJobInfo.getLastModifyUser()); + } catch (TaskExecuteAuthFailedException e) { + log.error("Error while pre auth cron execute!", e); + throw e; + } + } + + /** + * 保存定时任务主机变量中的IP等快照信息 + * + * @param cronJobInfo 定时任务信息 + */ + private void saveSnapShotForHostVaiableValue(CronJobInfoDTO cronJobInfo) { + List variableValue = cronJobInfo.getVariableValue(); + if (CollectionUtils.isEmpty(variableValue)) { + return; + } + for (CronJobVariableDTO variable : variableValue) { + if (variable.getType() != TaskVariableTypeEnum.HOST_LIST) { + continue; + } + ServerDTO serverDTO = variable.getServer(); + if (serverDTO == null) { + continue; + } + List hostByIpList = serverDTO.getIps(); + if (CollectionUtils.isNotEmpty(hostByIpList)) { + hostService.fillHosts(hostByIpList); + } + } + } + + /** + * 更新定时任务时将使用mask指代的密文值设置为其真实值 + * + * @param cronJobInfo 定时任务信息 + */ + private void processCronJobVariableValueMask(CronJobInfoDTO cronJobInfo) { + if (CollectionUtils.isEmpty(cronJobInfo.getVariableValue())) { + return; + } + List hasMaskVariableList = new ArrayList<>(); + for (CronJobVariableDTO cronJobVariable : cronJobInfo.getVariableValue()) { + if (cronJobVariable.getType().needMask()) { + if (cronJobVariable.getValue().equals(cronJobVariable.getType().getMask())) { + hasMaskVariableList.add(cronJobVariable); } } - if (CollectionUtils.isNotEmpty(hasMaskVariableList)) { - CronJobInfoDTO originCronJob = - cronJobDAO.getCronJobById(cronJobInfo.getAppId(), cronJobInfo.getId()); - if (CollectionUtils.isNotEmpty(originCronJob.getVariableValue())) { - for (CronJobVariableDTO cronJobVariable : originCronJob.getVariableValue()) { - Iterator newCronJobVariableIterator = hasMaskVariableList.iterator(); - while (newCronJobVariableIterator.hasNext()) { - CronJobVariableDTO newCronJobVariable = newCronJobVariableIterator.next(); - if (newCronJobVariable.getId().equals(cronJobVariable.getId())) { - newCronJobVariable.setValue(cronJobVariable.getValue()); - newCronJobVariableIterator.remove(); - break; - } - } - if (CollectionUtils.isEmpty(hasMaskVariableList)) { - break; - } - } + } + if (CollectionUtils.isEmpty(hasMaskVariableList)) { + return; + } + CronJobInfoDTO originCronJob = + cronJobDAO.getCronJobById(cronJobInfo.getAppId(), cronJobInfo.getId()); + if (CollectionUtils.isEmpty(originCronJob.getVariableValue())) { + return; + } + for (CronJobVariableDTO cronJobVariable : originCronJob.getVariableValue()) { + Iterator newCronJobVariableIterator = hasMaskVariableList.iterator(); + while (newCronJobVariableIterator.hasNext()) { + CronJobVariableDTO newCronJobVariable = newCronJobVariableIterator.next(); + if (newCronJobVariable.getId().equals(cronJobVariable.getId())) { + newCronJobVariable.setValue(cronJobVariable.getValue()); + newCronJobVariableIterator.remove(); + break; } } + if (CollectionUtils.isEmpty(hasMaskVariableList)) { + break; + } } } @@ -262,17 +408,65 @@ private void checkCronJobPlanOrScript(CronJobInfoDTO cronJobInfo) { } @Override - public Boolean deleteCronJobInfo(Long appId, Long cronJobId) { + @ActionAuditRecord( + actionId = ActionId.MANAGE_CRON, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.CRON, + instanceIds = "#cronJobId" + ), + content = EventContentConstants.DELETE_CRON_JOB + ) + public Boolean deleteCronJobInfo(String username, Long appId, Long cronJobId) { + cronAuthService.authManageCron(username, + new AppResourceScope(appId), cronJobId, null).denyIfNoPermission(); + + CronJobInfoDTO cron = getIntegralCronJobInfoById(cronJobId); + if (cron == null) { + throw new NotFoundException(ErrorCode.CRON_JOB_NOT_EXIST); + } + + // 审计 + ActionAuditContext.current().setInstanceName(cron.getName()); + if (cronJobDAO.deleteCronJobById(appId, cronJobId)) { - deleteJob(appId, cronJobId); + informAllToDeleteJobFromQuartz(appId, cronJobId); + deleteCustomNotifyPolicy(appId, cron); return true; } return false; } + private void deleteCustomNotifyPolicy(Long appId, CronJobInfoDTO cronJob) { + if (cronJob.hasCustomNotifyPolicy()) { + log.debug("[deleteCustomNotifyPolicy]deleted cron task:{} has custom notify policy," + + "try to delete notify policy", cronJob.getId()); + customNotifyPolicyService.deleteCronJobCustomNotifyPolicy(appId, cronJob.getId()); + } + } + @Override - @Transactional(rollbackFor = {Exception.class, Error.class}) + @ActionAuditRecord( + actionId = ActionId.MANAGE_CRON, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.CRON, + instanceIds = "#cronJobId" + ), + content = EventContentConstants.SWITCH_CRON_JOB_STATUS + ) public Boolean changeCronJobEnableStatus(String username, Long appId, Long cronJobId, Boolean enable) { + cronAuthService.authManageCron(username, + new AppResourceScope(appId), cronJobId, null).denyIfNoPermission(); + + CronJobInfoDTO originCronJobInfo = getCronJobInfoById(appId, cronJobId); + if (originCronJobInfo == null) { + throw new NotFoundException(ErrorCode.CRON_JOB_NOT_EXIST); + } + + // 审计 + ActionAuditContext.current() + .setInstanceName(originCronJobInfo.getName()) + .addAttribute(OPERATION, enable ? "Switch on" : "Switch off"); + CronJobInfoDTO cronJobInfo = new CronJobInfoDTO(); cronJobInfo.setAppId(appId); cronJobInfo.setId(cronJobId); @@ -281,17 +475,16 @@ public Boolean changeCronJobEnableStatus(String username, Long appId, Long cronJ cronJobInfo.setLastModifyTime(DateUtils.currentTimeSeconds()); if (enable) { try { - CronJobInfoDTO originCronJobInfo = cronJobDAO.getCronJobById(appId, cronJobId); List taskVariables = null; if (CollectionUtils.isNotEmpty(originCronJobInfo.getVariableValue())) { taskVariables = - originCronJobInfo.getVariableValue().parallelStream() + originCronJobInfo.getVariableValue().stream() .map(CronJobVariableDTO::toServiceTaskVariable).collect(Collectors.toList()); } executeTaskService.authExecuteTask(appId, originCronJobInfo.getTaskPlanId(), cronJobId, originCronJobInfo.getName(), taskVariables, username); if (cronJobDAO.updateCronJobById(cronJobInfo)) { - return addJob(appId, cronJobId); + return informAllToAddJobToQuartz(appId, cronJobId); } else { return false; } @@ -301,7 +494,7 @@ public Boolean changeCronJobEnableStatus(String username, Long appId, Long cronJ } } else { if (cronJobDAO.updateCronJobById(cronJobInfo)) { - return deleteJob(appId, cronJobId); + return informAllToDeleteJobFromQuartz(appId, cronJobId); } else { return false; } @@ -310,7 +503,6 @@ public Boolean changeCronJobEnableStatus(String username, Long appId, Long cronJ } @Override - @Transactional(rollbackFor = {Exception.class, Error.class}) public Boolean disableExpiredCronJob(Long appId, Long cronJobId, String lastModifyUser, Long lastModifyTime) { CronJobInfoDTO cronJobInfo = new CronJobInfoDTO(); cronJobInfo.setAppId(appId); @@ -319,7 +511,7 @@ public Boolean disableExpiredCronJob(Long appId, Long cronJobId, String lastModi cronJobInfo.setLastModifyTime(lastModifyTime); cronJobInfo.setEnable(false); if (cronJobDAO.updateCronJobById(cronJobInfo)) { - return deleteJob(appId, cronJobId); + return informAllToDeleteJobFromQuartz(appId, cronJobId); } else { return false; } @@ -366,7 +558,7 @@ public Map> listCronJobByPlanIds(Long appId, List getCronJobExecuteHistory(Long appId, - List cronIdList) { - return taskExecuteResultService.getCronTaskExecuteResultStatistics(appId, cronIdList); - } - - @Override - @Transactional(rollbackFor = {Exception.class, Error.class}) - public Boolean batchUpdateCronJob(Long appId, BatchUpdateCronJobReq batchUpdateCronJobReq) { - if (batchUpdateCronJobReq != null) { - if (CollectionUtils.isNotEmpty(batchUpdateCronJobReq.getCronJobInfoList())) { - batchUpdateCronJobReq.getCronJobInfoList().forEach(cronJobInfo -> { - CronJobInfoDTO cronJobInfoFromReq = - CronJobInfoDTO.fromReq(JobContextUtil.getUsername(), appId, cronJobInfo); - cronJobInfoFromReq.setEnable(cronJobInfo.getEnable()); - if (cronJobInfo.getEnable()) { - try { - CronJobInfoDTO originCronJobInfo = cronJobDAO.getCronJobById(appId, cronJobInfo.getId()); - List taskVariables = null; - if (CollectionUtils.isNotEmpty(originCronJobInfo.getVariableValue())) { - taskVariables = - originCronJobInfo.getVariableValue().parallelStream() - .map(CronJobVariableDTO::toServiceTaskVariable).collect(Collectors.toList()); - } - executeTaskService.authExecuteTask(appId, originCronJobInfo.getTaskPlanId(), - cronJobInfo.getId(), originCronJobInfo.getName(), taskVariables, - JobContextUtil.getUsername()); - if (cronJobDAO.updateCronJobById(cronJobInfoFromReq)) { - addJob(appId, cronJobInfo.getId()); - } - } catch (TaskExecuteAuthFailedException e) { - log.error("Error while pre auth cron execute!", e); - throw e; - } - } else { - if (cronJobDAO.updateCronJobById(cronJobInfoFromReq)) { - deleteJob(appId, cronJobInfo.getId()); - } - } - }); - } + @ActionAuditRecord( + actionId = ActionId.MANAGE_CRON, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.CRON + ), + content = EventContentConstants.EDIT_CRON_JOB + ) + public Boolean batchUpdateCronJob(String username, + Long appId, + BatchUpdateCronJobReq batchUpdateCronJobReq) { + // 更新DB中的数据 + NeedScheduleCronInfo needScheduleCronInfo = batchCronJobService.batchUpdateCronJob( + username, + appId, + batchUpdateCronJobReq + ); + // 更新Quartz调度 + List needAddCronIdList = needScheduleCronInfo.getNeedAddCronIdList(); + List needDeleteCronIdList = needScheduleCronInfo.getNeedDeleteCronIdList(); + if (CollectionUtils.isNotEmpty(needAddCronIdList)) { + needAddCronIdList.forEach(cronId -> informAllToAddJobToQuartz(appId, cronId)); + } + if (CollectionUtils.isNotEmpty(needDeleteCronIdList)) { + needDeleteCronIdList.forEach(cronId -> informAllToDeleteJobFromQuartz(appId, cronId)); } return true; } @@ -509,7 +699,7 @@ public String getCronJobNameById(long id) { @Override public List listCronJobByIds(Long appId, List cronJobIdList) { - cronJobIdList = cronJobIdList.parallelStream().filter(id -> id != null && id > 0).collect(Collectors.toList()); + cronJobIdList = cronJobIdList.stream().filter(id -> id != null && id > 0).collect(Collectors.toList()); if (appId != null && appId > 0 && CollectionUtils.isNotEmpty(cronJobIdList)) { return cronJobDAO.listCronJobByIds(appId, cronJobIdList); } else { @@ -553,124 +743,14 @@ private ServiceInnerCronJobInfoDTO fromQuartzJob(QuartzJobInfoDTO jobInfo) { return innerCronJobInfoDTO; } - private boolean addJob(long appId, long cronJobId) throws ServiceException { - if (appId <= 0 || cronJobId <= 0) { + private boolean informAllToAddJobToQuartz(long appId, long cronJobId) throws ServiceException { + try { + crontabMQEventDispatcher.broadCastCrontabEvent(CrontabEvent.addCron(appId, cronJobId)); + return true; + } catch (Exception e) { + log.error("Fail to broadCast addCronEvent", e); return false; } - String lockKey = appId + ":" + cronJobId; - if (LockUtils.tryGetDistributedLock(lockKey, JobContextUtil.getRequestId(), 60_000)) { - try { - CronJobInfoDTO cronJobInfo = getCronJobInfoById(appId, cronJobId); - if (StringUtils.isBlank(cronJobInfo.getCronExpression()) - && cronJobInfo.getExecuteTime() < DateUtils.currentTimeSeconds()) { - throw new FailedPreconditionException(ErrorCode.CRON_JOB_TIME_PASSED); - } - checkCronRelatedPlan(cronJobInfo.getAppId(), cronJobInfo.getTaskPlanId()); - QuartzTrigger trigger = null; - if (StringUtils.isNotBlank(cronJobInfo.getCronExpression())) { - QuartzTriggerBuilder cronTriggerBuilder = - QuartzTriggerBuilder.newTrigger().ofType(QuartzTrigger.TriggerType.CRON) - .withIdentity(getJobName(appId, cronJobId), getJobGroup(appId, cronJobId)) - .withCronExpression(cronJobInfo.getCronExpression()) - .withMisfireInstruction(CronTrigger.MISFIRE_INSTRUCTION_DO_NOTHING); - if (cronJobInfo.getEndTime() > 0) { - if (cronJobInfo.getEndTime() < DateUtils.currentTimeSeconds()) { - throw new FailedPreconditionException(ErrorCode.END_TIME_OR_NOTIFY_TIME_ALREADY_PASSED); - } else { - cronTriggerBuilder = - cronTriggerBuilder.endAt(Date.from(Instant.ofEpochSecond(cronJobInfo.getEndTime()))); - } - } - trigger = cronTriggerBuilder.build(); - } else if (cronJobInfo.getExecuteTime() > DateUtils.currentTimeSeconds()) { - trigger = QuartzTriggerBuilder.newTrigger().ofType(QuartzTrigger.TriggerType.SIMPLE) - .withIdentity(getJobName(appId, cronJobId), getJobGroup(appId, cronJobId)) - .startAt(Date.from(Instant.ofEpochSecond(cronJobInfo.getExecuteTime()))).withRepeatCount(0) - .withIntervalInHours(1) - .withMisfireInstruction(SimpleTrigger.MISFIRE_INSTRUCTION_RESCHEDULE_NEXT_WITH_REMAINING_COUNT) - .build(); - } - if (trigger == null) { - throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); - } - - QuartzJob job = - QuartzJobBuilder.newJob().withIdentity(getJobName(appId, cronJobId), getJobGroup(appId, cronJobId)) - .forJob(SimpleJobExecutor.class) - .usingJobData(CronConstants.JOB_DATA_KEY_APP_ID_STR, String.valueOf(appId)) - .usingJobData(CronConstants.JOB_DATA_KEY_CRON_JOB_ID_STR, String.valueOf(cronJobId)) - .withTrigger(trigger) - .build(); - - try { - quartzTaskHandler - .deleteJob(JobKey.jobKey(getJobName(appId, cronJobId), getJobGroup(appId, cronJobId))); - quartzTaskHandler.addJob(job); - } catch (SchedulerException e) { - log.error("Error while add job to quartz!", e); - throw new InternalException("Add to quartz failed!", e, ErrorCode.INTERNAL_ERROR); - } - - if (cronJobInfo.getNotifyOffset() > 0) { - long notifyTime = 0L; - if (StringUtils.isNotBlank(cronJobInfo.getCronExpression())) { - if (cronJobInfo.getEndTime() > 0) { - notifyTime = cronJobInfo.getEndTime() - cronJobInfo.getNotifyOffset(); - } - } else { - notifyTime = cronJobInfo.getExecuteTime() - cronJobInfo.getNotifyOffset(); - } - if (notifyTime < DateUtils.currentTimeSeconds()) { - throw new FailedPreconditionException(ErrorCode.END_TIME_OR_NOTIFY_TIME_ALREADY_PASSED); - } - - QuartzTrigger notifyTrigger = QuartzTriggerBuilder.newTrigger() - .ofType(QuartzTrigger.TriggerType.SIMPLE) - .withIdentity(getNotifyJobName(appId, cronJobId), getJobGroup(appId, cronJobId)) - .startAt(Date.from(Instant.ofEpochSecond(notifyTime))).withRepeatCount(0).withIntervalInHours(1) - .withMisfireInstruction(SimpleTrigger.MISFIRE_INSTRUCTION_RESCHEDULE_NEXT_WITH_REMAINING_COUNT) - .build(); - - QuartzJob notifyJob = QuartzJobBuilder.newJob() - .withIdentity(getNotifyJobName(appId, cronJobId), getJobGroup(appId, cronJobId)) - .forJob(NotifyJobExecutor.class) - .usingJobData(CronConstants.JOB_DATA_KEY_APP_ID_STR, String.valueOf(appId)) - .usingJobData(CronConstants.JOB_DATA_KEY_CRON_JOB_ID_STR, String.valueOf(cronJobId)) - .withTrigger(notifyTrigger) - .build(); - - try { - quartzTaskHandler.deleteJob( - JobKey.jobKey(getNotifyJobName(appId, cronJobId), getJobGroup(appId, cronJobId))); - quartzTaskHandler.addJob(notifyJob); - } catch (SchedulerException e) { - log.error("Error while add job to quartz!", e); - throw new InternalException("Add to quartz failed!", e, ErrorCode.INTERNAL_ERROR); - } - } else { - try { - quartzTaskHandler.deleteJob( - JobKey.jobKey(getNotifyJobName(appId, cronJobId), getJobGroup(appId, cronJobId))); - } catch (SchedulerException e) { - log.error("Error while add job to quartz!", e); - throw new InternalException("Add to quartz failed!", e, ErrorCode.INTERNAL_ERROR); - } - } - return true; - } catch (ServiceException e) { - deleteJob(appId, cronJobId); - log.debug("Error while schedule job", e); - throw e; - } catch (Exception e) { - deleteJob(appId, cronJobId); - log.error("Unknown exception while process cron status change!", e); - throw new InternalException(ErrorCode.UPDATE_CRON_JOB_FAILED); - } finally { - LockUtils.releaseDistributedLock(lockKey, JobContextUtil.getRequestId()); - } - } else { - throw new InternalException(ErrorCode.ACQUIRE_CRON_JOB_LOCK_FAILED); - } } private void checkCronRelatedPlan(Long appId, Long taskPlanId) throws ServiceException { @@ -679,23 +759,64 @@ private void checkCronRelatedPlan(Long appId, Long taskPlanId) throws ServiceExc } } - private boolean deleteJob(long appId, long cronJobId) { + private boolean informAllToDeleteJobFromQuartz(long appId, long cronJobId) { + try { + crontabMQEventDispatcher.broadCastCrontabEvent(CrontabEvent.deleteCron(appId, cronJobId)); + return true; + } catch (Exception e) { + log.error("Fail to broadCast deleteCronEvent", e); + return false; + } + } + + @Override + public boolean checkAndAddJobToQuartz(long appId, long cronJobId) throws ServiceException { if (appId <= 0 || cronJobId <= 0) { return false; } - String lockKey = appId + ":" + cronJobId; - if (LockUtils.tryGetDistributedLock(lockKey, JobContextUtil.getRequestId(), 60_000)) { + CronJobInfoDTO cronJobInfo = getCronJobInfoById(appId, cronJobId); + if (StringUtils.isBlank(cronJobInfo.getCronExpression()) + && cronJobInfo.getExecuteTime() < DateUtils.currentTimeSeconds()) { + throw new FailedPreconditionException(ErrorCode.CRON_JOB_TIME_PASSED); + } + checkCronRelatedPlan(cronJobInfo.getAppId(), cronJobInfo.getTaskPlanId()); + quartzService.tryToAddJobToQuartz(cronJobInfo); + return true; + } + + + @Override + public List listEnabledCronBasicInfoForUpdate(int start, int limit) { + return cronJobDAO.listEnabledCronBasicInfoForUpdate(start, limit); + } + + @Override + public boolean disableCronJobByAppId(Long appId) { + CronJobInfoDTO cronJobInfoDTO = new CronJobInfoDTO(); + cronJobInfoDTO.setAppId(appId); + cronJobInfoDTO.setEnable(true); + List cronJobIdList = cronJobDAO.listCronJobIds(cronJobInfoDTO); + if (CollectionUtils.isEmpty(cronJobIdList)) { + return true; + } + List failedCronJobIds = new ArrayList<>(); + log.info("cron job will be disabled, appId:{}, cronJobIds:{}", appId, cronJobIdList); + for (Long cronJobId : cronJobIdList) { try { - quartzTaskHandler.deleteJob(JobKey.jobKey(getJobName(appId, cronJobId), getJobGroup(appId, cronJobId))); - quartzTaskHandler - .deleteJob(JobKey.jobKey(getNotifyJobName(appId, cronJobId), getJobGroup(appId, cronJobId))); - return true; - } catch (SchedulerException e) { - log.error("Error while delete job!", e); - } finally { - LockUtils.releaseDistributedLock(lockKey, JobContextUtil.getRequestId()); + Boolean disableResult = changeCronJobEnableStatus(JobConstants.DEFAULT_SYSTEM_USER_ADMIN, appId, + cronJobId, false); + log.debug("disable cron job, result:{}, appId:{}, cronId:{}", disableResult, appId, cronJobId); + if (!disableResult) { + failedCronJobIds.add(cronJobId); + } + } catch (Exception e) { + log.error("Failed to disable cron job with appId:{} and cronId:{}", appId, cronJobId, e); + failedCronJobIds.add(cronJobId); } } - return false; + if (!failedCronJobIds.isEmpty()) { + log.warn("Failed to disable cron jobs for appId:{} with cronJobIds:{}", appId, failedCronJobIds); + } + return failedCronJobIds.isEmpty(); } } diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/CustomNotifyPolicyServiceImpl.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/CustomNotifyPolicyServiceImpl.java new file mode 100644 index 0000000000..84a89d3a14 --- /dev/null +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/CustomNotifyPolicyServiceImpl.java @@ -0,0 +1,111 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.crontab.service.impl; + +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.InternalException; +import com.tencent.bk.job.common.model.InternalResponse; +import com.tencent.bk.job.common.model.dto.notify.CustomNotifyDTO; +import com.tencent.bk.job.crontab.model.dto.CronJobInfoDTO; +import com.tencent.bk.job.crontab.service.CustomNotifyPolicyService; +import com.tencent.bk.job.manage.api.common.constants.notify.ResourceTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.notify.TriggerTypeEnum; +import com.tencent.bk.job.manage.api.inner.ServiceNotificationResource; +import com.tencent.bk.job.manage.model.inner.ServiceSpecificResourceNotifyPolicyDTO; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Slf4j +@Service +public class CustomNotifyPolicyServiceImpl implements CustomNotifyPolicyService { + + private final ServiceNotificationResource notificationResource; + + @Autowired + public CustomNotifyPolicyServiceImpl( + ServiceNotificationResource notificationResource + ) { + this.notificationResource = notificationResource; + } + + @Override + public void createOrUpdateCronJobCustomNotifyPolicy(Long cronJobId, CronJobInfoDTO cronJobInfoDTO) { + log.info("Start create or update cronJob custom notify policy with cronJobId:{}", cronJobId); + + ServiceSpecificResourceNotifyPolicyDTO specificResourceNotifyPolicy = + new ServiceSpecificResourceNotifyPolicyDTO(); + specificResourceNotifyPolicy.setAppId(cronJobInfoDTO.getAppId()); + specificResourceNotifyPolicy.setTriggerType(TriggerTypeEnum.TIMER_TASK.getType()); + specificResourceNotifyPolicy.setResourceType(ResourceTypeEnum.CRON.getType()); + specificResourceNotifyPolicy.setResourceId(cronJobId); + if (cronJobInfoDTO.getCustomCronJobNotifyDTO() != null) { + specificResourceNotifyPolicy.setRoleList(cronJobInfoDTO.getCustomCronJobNotifyDTO().getRoleList()); + specificResourceNotifyPolicy.setExtraObserverList( + cronJobInfoDTO.getCustomCronJobNotifyDTO().getExtraObserverList() + ); + Map> statusChannelMap = new HashMap<>(); + cronJobInfoDTO.getCustomCronJobNotifyDTO().getCustomNotifyChannel().forEach(statusNotifyChannel -> { + statusChannelMap.put( + statusNotifyChannel.getExecuteStatus().getValue(), + statusNotifyChannel.getChannelList() + ); + }); + specificResourceNotifyPolicy.setResourceStatusChannelMap(statusChannelMap); + } + InternalResponse resp = notificationResource.createOrUpdateSpecificResourceNotifyPolicy( + cronJobInfoDTO.getLastModifyUser(), + cronJobInfoDTO.getAppId(), + specificResourceNotifyPolicy); + if (resp == null || !resp.isSuccess()) { + throw new InternalException(ErrorCode.SAVE_CRON_CUSTOM_NOTIFY_FAILED, new Object[]{cronJobId}); + } + } + + @Override + public void deleteCronJobCustomNotifyPolicy(Long appId, Long cronJobId) { + Integer cnt = notificationResource.deleteSpecificResourceNotifyPolicy( + appId, + ResourceTypeEnum.CRON.getType(), + String.valueOf(cronJobId) + ).getData(); + log.info("Deleted custom notify policy with cron task id:{}, affect resources:{}", cronJobId, cnt); + } + + @Override + public CustomNotifyDTO getCronJobCustomNotifyPolicyById(Long appId, Long cronJobId) { + return notificationResource.getSpecificResourceNotifyPolicy( + appId, + ResourceTypeEnum.CRON.getType(), + String.valueOf(cronJobId), + TriggerTypeEnum.TIMER_TASK.getType() + ).getData(); + } + +} diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/ExecuteTaskServiceImpl.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/ExecuteTaskServiceImpl.java index 897f2bb0f5..bd41235159 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/ExecuteTaskServiceImpl.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/ExecuteTaskServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,13 +28,14 @@ import com.tencent.bk.job.common.model.InternalResponse; import com.tencent.bk.job.common.model.iam.AuthResultDTO; import com.tencent.bk.job.common.util.json.JsonUtils; -import com.tencent.bk.job.crontab.client.ServiceExecuteTaskResourceClient; import com.tencent.bk.job.crontab.exception.TaskExecuteAuthFailedException; import com.tencent.bk.job.crontab.service.ExecuteTaskService; +import com.tencent.bk.job.execute.api.inner.ServiceExecuteTaskResource; import com.tencent.bk.job.execute.model.inner.ServiceTaskExecuteResult; import com.tencent.bk.job.execute.model.inner.ServiceTaskVariable; import com.tencent.bk.job.execute.model.inner.request.ServiceTaskExecuteRequest; import lombok.extern.slf4j.Slf4j; +import org.slf4j.helpers.MessageFormatter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -47,11 +48,11 @@ @Service public class ExecuteTaskServiceImpl implements ExecuteTaskService { - private ServiceExecuteTaskResourceClient serviceExecuteTaskResourceClient; + private final ServiceExecuteTaskResource executeTaskResource; @Autowired - public ExecuteTaskServiceImpl(ServiceExecuteTaskResourceClient serviceExecuteTaskResourceClient) { - this.serviceExecuteTaskResourceClient = serviceExecuteTaskResourceClient; + public ExecuteTaskServiceImpl(ServiceExecuteTaskResource executeTaskResource) { + this.executeTaskResource = executeTaskResource; } @Override @@ -74,14 +75,29 @@ public InternalResponse executeTask(long appId, long t log.debug("Sending request to executor|{}", request); } - InternalResponse taskExecuteResult = - serviceExecuteTaskResourceClient.executeTask(request); - log.info("Get execute task by cron|appId|{}|taskId|{}|cronTaskId|{}|{}|operator|{}|result|{}", appId, - taskId, cronTaskId, cronName, operator, JsonUtils.toJson(taskExecuteResult)); - return taskExecuteResult; + InternalResponse resp = + executeTaskResource.executeTask(request); + log.info( + "Get execute task by cron|appId={}|taskId={}|cronTaskId={}|cronName={}|operator={}|result={}", + appId, + taskId, + cronTaskId, + cronName, + operator, + JsonUtils.toJson(resp) + ); + return resp; } catch (Throwable e) { - log.error("Get execute task by cron caught exception|appId|{}|taskId|{}|cronTaskId|{}|{}|operator|{}", - appId, taskId, cronTaskId, cronName, operator, e); + String msg = MessageFormatter.arrayFormat( + "Get execute task by cron caught exception|appId={}|taskId={}|cronTaskId={}|cronName={}|operator={}", + new String[]{ + String.valueOf(appId), + String.valueOf(taskId), + String.valueOf(cronTaskId), + cronName, + operator + }).getMessage(); + log.error(msg, e); return null; } } @@ -90,7 +106,7 @@ public InternalResponse executeTask(long appId, long t public void authExecuteTask( long appId, long taskId, - long cronTaskId, + Long cronTaskId, String cronName, List variableList, String operator @@ -99,7 +115,6 @@ public void authExecuteTask( request.setAppId(appId); request.setOperator(operator); request.setPlanId(taskId); - request.setCronTaskId(cronTaskId); request.setTaskName(cronName); request.setTaskVariables(variableList); request.setStartupMode(3); @@ -107,7 +122,7 @@ public void authExecuteTask( log.debug("Sending auth execute request to executor|{}", request); } - InternalResponse authExecuteResult = serviceExecuteTaskResourceClient.authExecuteTask(request); + InternalResponse authExecuteResult = executeTaskResource.authExecuteTask(request); log.info("Auth execute result|appId|{}|taskId|{}|cronTaskId|{}|{}|operator|{}|result|{}", appId, taskId, cronTaskId, cronName, operator, JsonUtils.toJson(authExecuteResult)); if (authExecuteResult != null) { diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/HostServiceImpl.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/HostServiceImpl.java new file mode 100644 index 0000000000..bdeac45449 --- /dev/null +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/HostServiceImpl.java @@ -0,0 +1,135 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.crontab.service.impl; + +import com.tencent.bk.job.common.model.InternalResponse; +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.crontab.service.HostService; +import com.tencent.bk.job.manage.api.inner.ServiceHostResource; +import com.tencent.bk.job.manage.model.inner.ServiceHostDTO; +import com.tencent.bk.job.manage.model.inner.request.ServiceBatchGetHostsReq; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +@Slf4j +@Service("jobCrontabHostService") +public class HostServiceImpl implements HostService { + + private final ServiceHostResource hostResource; + + @Autowired + public HostServiceImpl(ServiceHostResource hostResource) { + this.hostResource = hostResource; + } + + private void fillHostInfo(HostDTO hostDTO, ServiceHostDTO serviceHostDTO) { + if (hostDTO == null || serviceHostDTO == null) { + return; + } + hostDTO.setHostId(serviceHostDTO.getHostId()); + hostDTO.setBkCloudId(serviceHostDTO.getCloudAreaId()); + hostDTO.setIp(serviceHostDTO.getIp()); + } + + private String buildCloudIp(ServiceHostDTO serviceHostDTO) { + Long cloudAreaId = serviceHostDTO.getCloudAreaId(); + String ip = serviceHostDTO.getIp(); + return buildCloudIp(cloudAreaId, ip); + } + + private String buildCloudIp(Long cloudAreaId, String ip) { + if (cloudAreaId != null && StringUtils.isNotBlank(ip)) { + return cloudAreaId + ":" + ip; + } + return null; + } + + @Override + public int fillHosts(List hostList) { + if (CollectionUtils.isEmpty(hostList)) { + return 0; + } + InternalResponse> resp = + hostResource.batchGetHosts(new ServiceBatchGetHostsReq(hostList)); + List serviceHostDTOList = resp.getData(); + if (CollectionUtils.isEmpty(serviceHostDTOList)) { + log.info( + "cannot find any host details by {}", + hostList.stream().map(HostDTO::getUniqueKey).collect(Collectors.toList()) + ); + return 0; + } + Map hostIdMap = new HashMap<>(); + Map cloudIpMap = new HashMap<>(); + serviceHostDTOList.forEach(serviceHostDTO -> { + hostIdMap.put(serviceHostDTO.getHostId(), serviceHostDTO); + cloudIpMap.put(buildCloudIp(serviceHostDTO), serviceHostDTO); + }); + int filledCount = 0; + Set noHostIds = new HashSet<>(); + Set noHostCloudIps = new HashSet<>(); + for (HostDTO hostDTO : hostList) { + Long hostId = hostDTO.getHostId(); + String cloudIp = buildCloudIp(hostDTO.getBkCloudId(), hostDTO.getIp()); + if (hostId != null) { + ServiceHostDTO serviceHostDTO = hostIdMap.get(hostId); + if (serviceHostDTO == null) { + noHostIds.add(hostId); + } else { + fillHostInfo(hostDTO, serviceHostDTO); + filledCount += 1; + } + } else if (StringUtils.isNotBlank(cloudIp)) { + ServiceHostDTO serviceHostDTO = cloudIpMap.get(cloudIp); + if (serviceHostDTO == null) { + noHostCloudIps.add(cloudIp); + } else { + fillHostInfo(hostDTO, serviceHostDTO); + filledCount += 1; + } + } else { + log.warn("host does not contains hostId/cloudIp:{}", hostDTO); + } + } + if (CollectionUtils.isNotEmpty(noHostIds) || CollectionUtils.isNotEmpty(noHostCloudIps)) { + log.info( + "cannot find host by cloudIps:{}, hostIds:{}, which may be removed from cmdb", + noHostCloudIps, + noHostIds + ); + } + return filledCount; + } +} diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/InnerJobHistoryServiceImpl.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/InnerJobHistoryServiceImpl.java index 426d0bf347..21b2793617 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/InnerJobHistoryServiceImpl.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/InnerJobHistoryServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/JobCronNameUtil.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/JobCronNameUtil.java new file mode 100644 index 0000000000..7b575089dd --- /dev/null +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/JobCronNameUtil.java @@ -0,0 +1,60 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.crontab.service.impl; + +/** + * Job定时任务名称工具类 + */ +public class JobCronNameUtil { + + /** + * 获取Quartz Job名称 + * + * @param cronJobId 定时任务ID + * @return Quartz Job名称 + */ + public static String getJobName(long cronJobId) { + return "job_" + cronJobId; + } + + /** + * 获取Quartz Job分组 + * + * @param appId Job业务ID + * @return Quartz Job分组 + */ + public static String getJobGroup(long appId) { + return "bk_app_" + appId; + } + + /*** + * 获取通知Job名称 + * @param cronJobId 定时任务ID + * @return 通知Job名称 + */ + public static String getNotifyJobName(long cronJobId) { + return getJobName(cronJobId) + "_notify"; + } +} diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/NotifyServiceImpl.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/NotifyServiceImpl.java index 8f8f0fae32..af70ab8da2 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/NotifyServiceImpl.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/NotifyServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,32 +25,29 @@ package com.tencent.bk.job.crontab.service.impl; import com.tencent.bk.job.common.model.InternalResponse; -import com.tencent.bk.job.crontab.client.ServiceNotificationResourceClient; import com.tencent.bk.job.crontab.model.dto.CronJobInfoDTO; import com.tencent.bk.job.crontab.service.NotifyService; +import com.tencent.bk.job.manage.api.inner.ServiceNotificationResource; import com.tencent.bk.job.manage.model.inner.ServiceTemplateNotificationDTO; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -/** - * @since 29/4/2020 15:52 - */ @Slf4j -@Service +@Service("jobCrontabNotifyService") public class NotifyServiceImpl implements NotifyService { - private final ServiceNotificationResourceClient notificationClient; + private final ServiceNotificationResource notificationResource; @Autowired - public NotifyServiceImpl(ServiceNotificationResourceClient notificationClient) { - this.notificationClient = notificationClient; + public NotifyServiceImpl(ServiceNotificationResource notificationResource) { + this.notificationResource = notificationResource; } @Override public Integer sendCronJobNotification(CronJobInfoDTO cronJobInfo) { InternalResponse sendNotifyResponse = - notificationClient.sendTemplateNotification(CronJobInfoDTO.buildNotifyInfo(cronJobInfo)); + notificationResource.sendTemplateNotification(CronJobInfoDTO.buildNotifyInfo(cronJobInfo)); if (sendNotifyResponse != null) { if (sendNotifyResponse.isSuccess()) { if (sendNotifyResponse.getData() != null) { @@ -82,7 +79,7 @@ public Integer sendCronJobFailedNotification(Integer errorCode, String errorMess } InternalResponse sendNotifyResponse = - notificationClient.sendTemplateNotification(templateNotificationRequest); + notificationResource.sendTemplateNotification(templateNotificationRequest); if (sendNotifyResponse != null) { if (sendNotifyResponse.isSuccess()) { if (sendNotifyResponse.getData() != null) { diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/QuartzServiceImpl.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/QuartzServiceImpl.java new file mode 100644 index 0000000000..3fde005369 --- /dev/null +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/QuartzServiceImpl.java @@ -0,0 +1,222 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.crontab.service.impl; + +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.FailedPreconditionException; +import com.tencent.bk.job.common.exception.InternalException; +import com.tencent.bk.job.common.exception.ServiceException; +import com.tencent.bk.job.common.util.TimeUtil; +import com.tencent.bk.job.common.util.date.DateUtils; +import com.tencent.bk.job.crontab.constant.CronConstants; +import com.tencent.bk.job.crontab.model.dto.CronJobInfoDTO; +import com.tencent.bk.job.crontab.service.QuartzService; +import com.tencent.bk.job.crontab.timer.AbstractQuartzTaskHandler; +import com.tencent.bk.job.crontab.timer.QuartzJob; +import com.tencent.bk.job.crontab.timer.QuartzJobBuilder; +import com.tencent.bk.job.crontab.timer.QuartzTrigger; +import com.tencent.bk.job.crontab.timer.QuartzTriggerBuilder; +import com.tencent.bk.job.crontab.timer.executor.NotifyJobExecutor; +import com.tencent.bk.job.crontab.timer.executor.SimpleJobExecutor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.quartz.CronTrigger; +import org.quartz.JobKey; +import org.quartz.SchedulerException; +import org.quartz.SimpleTrigger; +import org.slf4j.helpers.MessageFormatter; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.sql.Date; +import java.time.Instant; + +@Slf4j +@Service +public class QuartzServiceImpl implements QuartzService { + + private final AbstractQuartzTaskHandler quartzTaskHandler; + + @Autowired + public QuartzServiceImpl(AbstractQuartzTaskHandler quartzTaskHandler) { + this.quartzTaskHandler = quartzTaskHandler; + } + + @Override + public void tryToAddJobToQuartz(CronJobInfoDTO cronJobInfo) { + try { + addJobToQuartz(cronJobInfo); + } catch (ServiceException e) { + deleteJobFromQuartz(cronJobInfo.getAppId(), cronJobInfo.getId()); + throw e; + } catch (Exception e) { + deleteJobFromQuartz(cronJobInfo.getAppId(), cronJobInfo.getId()); + throw new InternalException(e, ErrorCode.INTERNAL_ERROR); + } + } + + /** + * 从Quartz引擎删除定时任务（含相关的通知任务） + * + * @param appId Job业务ID + * @param cronJobId 定时任务ID + * @return 是否删除成功 + */ + @Override + public boolean deleteJobFromQuartz(long appId, long cronJobId) { + if (appId <= 0 || cronJobId <= 0) { + return false; + } + String jobName = JobCronNameUtil.getJobName(cronJobId); + String jobGroup = JobCronNameUtil.getJobGroup(appId); + String notifyJobName = JobCronNameUtil.getNotifyJobName(cronJobId); + try { + quartzTaskHandler.deleteJob(JobKey.jobKey(jobName, jobGroup)); + quartzTaskHandler.deleteJob(JobKey.jobKey(notifyJobName, jobGroup)); + return true; + } catch (SchedulerException e) { + log.error("Error while delete job!", e); + } + return false; + } + + /** + * 将定时任务添加至Quartz引擎（含相关的前置通知任务） + * + * @param cronJobInfo 定时任务信息 + * @throws SchedulerException Quartz调度异常 + */ + private void addJobToQuartz(CronJobInfoDTO cronJobInfo) throws SchedulerException { + Long appId = cronJobInfo.getAppId(); + Long cronJobId = cronJobInfo.getId(); + String jobName = JobCronNameUtil.getJobName(cronJobId); + String jobGroup = JobCronNameUtil.getJobGroup(appId); + QuartzTrigger trigger = buildTrigger(cronJobInfo); + + QuartzJob job = QuartzJobBuilder.newJob() + .withIdentity(jobName, jobGroup) + .forJob(SimpleJobExecutor.class) + .usingJobData(CronConstants.JOB_DATA_KEY_APP_ID_STR, String.valueOf(appId)) + .usingJobData(CronConstants.JOB_DATA_KEY_CRON_JOB_ID_STR, String.valueOf(cronJobId)) + .withTrigger(trigger) + .build(); + + quartzTaskHandler.deleteJob(JobKey.jobKey(jobName, jobGroup)); + quartzTaskHandler.addJob(job); + + addNotifyJobIfNeed(cronJobInfo); + } + + /** + * 构建定时任务Quartz触发器 + * + * @param cronJobInfo 定时任务信息 + * @return 定时任务Quartz触发器 + */ + private QuartzTrigger buildTrigger(CronJobInfoDTO cronJobInfo) { + QuartzTrigger trigger; + String jobName = JobCronNameUtil.getJobName(cronJobInfo.getId()); + String jobGroup = JobCronNameUtil.getJobGroup(cronJobInfo.getAppId()); + if (StringUtils.isNotBlank(cronJobInfo.getCronExpression())) { + // 根据cron表达式执行的定时任务 + QuartzTriggerBuilder cronTriggerBuilder = QuartzTriggerBuilder.newTrigger() + .ofType(QuartzTrigger.TriggerType.CRON) + .withIdentity(jobName, jobGroup) + .withCronExpression(cronJobInfo.getCronExpression()) + .withMisfireInstruction(CronTrigger.MISFIRE_INSTRUCTION_DO_NOTHING); + if (cronJobInfo.getEndTime() > 0) { + if (cronJobInfo.getEndTime() < DateUtils.currentTimeSeconds()) { + throw new FailedPreconditionException(ErrorCode.END_TIME_OR_NOTIFY_TIME_ALREADY_PASSED); + } else { + cronTriggerBuilder = cronTriggerBuilder. + endAt(Date.from(Instant.ofEpochSecond(cronJobInfo.getEndTime()))); + } + } + trigger = cronTriggerBuilder.build(); + } else if (cronJobInfo.getExecuteTime() > DateUtils.currentTimeSeconds()) { + // 只执行一次的定时任务 + trigger = QuartzTriggerBuilder.newTrigger().ofType(QuartzTrigger.TriggerType.SIMPLE) + .withIdentity(jobName, jobGroup) + .startAt(Date.from(Instant.ofEpochSecond(cronJobInfo.getExecuteTime()))).withRepeatCount(0) + .withIntervalInHours(1) + .withMisfireInstruction(SimpleTrigger.MISFIRE_INSTRUCTION_RESCHEDULE_NEXT_WITH_REMAINING_COUNT) + .build(); + } else { + // 只执行一次但是执行时间已经过期的定时任务 + String message = MessageFormatter.format( + "Cron job executionTime({}) already passed", + TimeUtil.formatTime(cronJobInfo.getExecuteTime()) + ).getMessage(); + throw new FailedPreconditionException(message, ErrorCode.CRON_JOB_TIME_PASSED); + } + return trigger; + } + + /** + * 按需添加通知任务至Quartz引擎 + * + * @param cronJobInfo 定时任务信息 + * @throws SchedulerException Quartz调度异常 + */ + private void addNotifyJobIfNeed(CronJobInfoDTO cronJobInfo) throws SchedulerException { + Long appId = cronJobInfo.getAppId(); + Long cronJobId = cronJobInfo.getId(); + String notifyJobName = JobCronNameUtil.getNotifyJobName(cronJobId); + String jobGroup = JobCronNameUtil.getJobGroup(appId); + if (cronJobInfo.getNotifyOffset() > 0) { + long notifyTime = 0L; + if (StringUtils.isNotBlank(cronJobInfo.getCronExpression())) { + if (cronJobInfo.getEndTime() > 0) { + notifyTime = cronJobInfo.getEndTime() - cronJobInfo.getNotifyOffset(); + } + } else { + notifyTime = cronJobInfo.getExecuteTime() - cronJobInfo.getNotifyOffset(); + } + if (notifyTime < DateUtils.currentTimeSeconds()) { + throw new FailedPreconditionException(ErrorCode.END_TIME_OR_NOTIFY_TIME_ALREADY_PASSED); + } + QuartzTrigger notifyTrigger = QuartzTriggerBuilder.newTrigger() + .ofType(QuartzTrigger.TriggerType.SIMPLE) + .withIdentity(notifyJobName, jobGroup) + .startAt(Date.from(Instant.ofEpochSecond(notifyTime))).withRepeatCount(0).withIntervalInHours(1) + .withMisfireInstruction(SimpleTrigger.MISFIRE_INSTRUCTION_RESCHEDULE_NEXT_WITH_REMAINING_COUNT) + .build(); + + QuartzJob notifyJob = QuartzJobBuilder.newJob() + .withIdentity(notifyJobName, jobGroup) + .forJob(NotifyJobExecutor.class) + .usingJobData(CronConstants.JOB_DATA_KEY_APP_ID_STR, String.valueOf(appId)) + .usingJobData(CronConstants.JOB_DATA_KEY_CRON_JOB_ID_STR, String.valueOf(cronJobId)) + .withTrigger(notifyTrigger) + .build(); + + quartzTaskHandler.deleteJob(JobKey.jobKey(notifyJobName, jobGroup)); + quartzTaskHandler.addJob(notifyJob); + } else { + quartzTaskHandler.deleteJob(JobKey.jobKey(notifyJobName, jobGroup)); + } + } + +} diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/ResourceNameQueryServiceImpl.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/ResourceNameQueryServiceImpl.java index 09c8168707..8666dc5e1e 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/ResourceNameQueryServiceImpl.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/ResourceNameQueryServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -30,40 +30,37 @@ import com.tencent.bk.job.common.iam.service.ResourceNameQueryService; import com.tencent.bk.job.common.iam.util.IamUtil; import com.tencent.bk.job.common.service.AppScopeMappingService; -import com.tencent.bk.job.crontab.client.ServiceApplicationResourceClient; import com.tencent.bk.job.crontab.service.CronJobService; +import com.tencent.bk.job.manage.api.inner.ServiceApplicationResource; import com.tencent.bk.job.manage.model.inner.resp.ServiceApplicationDTO; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -/** - * @since 18/6/2020 15:46 - */ @Slf4j -@Service("ResourceNameQueryService") +@Service("jobCrontabResourceNameQueryService") public class ResourceNameQueryServiceImpl implements ResourceNameQueryService { private final CronJobService cronJobService; - private final ServiceApplicationResourceClient applicationClient; + private final ServiceApplicationResource applicationResource; private final AppScopeMappingService appScopeMappingService; @Autowired public ResourceNameQueryServiceImpl(CronJobService cronJobService, - ServiceApplicationResourceClient applicationClient, + ServiceApplicationResource applicationResource, AuthService authService, AppAuthService appAuthService, AppScopeMappingService appScopeMappingService) { this.cronJobService = cronJobService; - this.applicationClient = applicationClient; + this.applicationResource = applicationResource; this.appScopeMappingService = appScopeMappingService; authService.setResourceNameQueryService(this); appAuthService.setResourceNameQueryService(this); } private String getAppName(Long appId) { - ServiceApplicationDTO applicationInfo = applicationClient.queryAppById(appId); + ServiceApplicationDTO applicationInfo = applicationResource.queryAppById(appId); if (applicationInfo != null) { if (StringUtils.isNotBlank(applicationInfo.getName())) { return applicationInfo.getName(); diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/TaskExecuteResultServiceImpl.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/TaskExecuteResultServiceImpl.java index 09bc2e8c4a..054439f9b7 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/TaskExecuteResultServiceImpl.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/TaskExecuteResultServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,12 +26,13 @@ import com.tencent.bk.job.common.model.InternalResponse; import com.tencent.bk.job.common.util.json.JsonUtils; -import com.tencent.bk.job.crontab.client.ServiceTaskExecuteResultResourceClient; import com.tencent.bk.job.crontab.service.TaskExecuteResultService; +import com.tencent.bk.job.execute.api.inner.ServiceTaskExecuteResultResource; import com.tencent.bk.job.execute.model.inner.ServiceCronTaskExecuteResultStatistics; import com.tencent.bk.job.execute.model.inner.request.ServiceGetCronTaskExecuteStatisticsRequest; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; +import org.slf4j.helpers.MessageFormatter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -39,19 +40,19 @@ import java.util.Map; /** - * @since 2/3/2020 22:24 + * 作业执行结果 Service */ @Slf4j @Service public class TaskExecuteResultServiceImpl implements TaskExecuteResultService { - private ServiceTaskExecuteResultResourceClient serviceTaskExecuteResultResourceClient; + private final ServiceTaskExecuteResultResource serviceTaskExecuteResultResource; @Autowired public TaskExecuteResultServiceImpl( - ServiceTaskExecuteResultResourceClient serviceTaskExecuteResultResourceClient + ServiceTaskExecuteResultResource serviceTaskExecuteResultResource ) { - this.serviceTaskExecuteResultResourceClient = serviceTaskExecuteResultResourceClient; + this.serviceTaskExecuteResultResource = serviceTaskExecuteResultResource; } @Override @@ -73,7 +74,7 @@ public Map getCronTaskExecuteResul } InternalResponse> cronTaskExecuteResultStatResp = - serviceTaskExecuteResultResourceClient.getCronTaskExecuteResultStatistics( + serviceTaskExecuteResultResource.getCronTaskExecuteResultStatistics( getCronTaskExecuteStatisticsRequest ); @@ -90,7 +91,14 @@ public Map getCronTaskExecuteResul return null; } } catch (Exception e) { - log.error("Get cron execute result failed!|{}|{}", appId, cronIdList, e); + String msg = MessageFormatter.arrayFormat( + "Get cron execute result failed!|{}|{}", + new String[]{ + String.valueOf(appId), + cronIdList.toString() + } + ).getMessage(); + log.error(msg, e); return null; } } diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/TaskPlanServiceImpl.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/TaskPlanServiceImpl.java index c5e12ffb3e..6d165064b1 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/TaskPlanServiceImpl.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/service/impl/TaskPlanServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,25 +25,22 @@ package com.tencent.bk.job.crontab.service.impl; import com.tencent.bk.job.common.model.InternalResponse; -import com.tencent.bk.job.crontab.client.ServiceTaskPlanResourceClient; import com.tencent.bk.job.crontab.service.TaskPlanService; +import com.tencent.bk.job.manage.api.inner.ServiceTaskPlanResource; import com.tencent.bk.job.manage.model.inner.ServiceTaskPlanDTO; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -/** - * @since 27/4/2020 12:13 - */ @Slf4j -@Service +@Service("jobCrontabTaskPlanService") public class TaskPlanServiceImpl implements TaskPlanService { - private final ServiceTaskPlanResourceClient serviceTaskPlanResourceClient; + private final ServiceTaskPlanResource taskPlanResource; @Autowired - public TaskPlanServiceImpl(ServiceTaskPlanResourceClient serviceTaskPlanResourceClient) { - this.serviceTaskPlanResourceClient = serviceTaskPlanResourceClient; + public TaskPlanServiceImpl(ServiceTaskPlanResource taskPlanResource) { + this.taskPlanResource = taskPlanResource; } @Override @@ -52,7 +49,7 @@ public ServiceTaskPlanDTO getPlanBasicInfoById(long appId, long planId) { return null; } InternalResponse planByIdResponse = - serviceTaskPlanResourceClient.getPlanBasicInfoById(appId, planId); + taskPlanResource.getPlanBasicInfoById(appId, planId); if (planByIdResponse.isSuccess()) { return planByIdResponse.getData(); } diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/task/DisableCronJobOfArchivedScopeTask.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/task/DisableCronJobOfArchivedScopeTask.java new file mode 100644 index 0000000000..8303075e91 --- /dev/null +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/task/DisableCronJobOfArchivedScopeTask.java @@ -0,0 +1,138 @@ +/* + * + * * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * * + * * Copyright (C) 2021 Tencent. All rights reserved. + * * + * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * * + * * License for BK-JOB蓝鲸智云作业平台: + * * -------------------------------------------------------------------- + * * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * * + * * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * * the Software. + * * + * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * * IN THE SOFTWARE. + * + */ + +package com.tencent.bk.job.crontab.task; + +import com.tencent.bk.job.common.cc.sdk.IBizCmdbClient; +import com.tencent.bk.job.common.cc.sdk.IBizSetCmdbClient; +import com.tencent.bk.job.common.redis.util.LockUtils; +import com.tencent.bk.job.common.redis.util.RedisKeyHeartBeatThread; +import com.tencent.bk.job.common.util.ip.IpUtils; +import com.tencent.bk.job.crontab.service.CronJobService; +import com.tencent.bk.job.manage.api.inner.ServiceApplicationResource; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.stereotype.Component; + +import java.util.ArrayList; +import java.util.List; + +/** + * 业务(集)被归档了，禁用其关联的定时任务 + */ +@Slf4j +@Component +public class DisableCronJobOfArchivedScopeTask { + private static final String DISABLE_CRON_JOB_TASK_RUNNING_MACHINE = "disable:appId-not-exist:cron"; + + private final RedisTemplate redisTemplate; + private final IBizCmdbClient bizCmdbClient; + private final IBizSetCmdbClient bizSetCmdbClient; + private final CronJobService cronJobService; + private final ServiceApplicationResource serviceApplicationResource; + + @Value("${job.crontab.autoDisableCronOfArchivedScope.enabled:true}") + private Boolean enabled; + + public DisableCronJobOfArchivedScopeTask(RedisTemplate redisTemplate, + IBizCmdbClient bizCmdbClient, + IBizSetCmdbClient bizSetCmdbClient, + CronJobService cronJobService, + ServiceApplicationResource serviceApplicationResource){ + this.redisTemplate = redisTemplate; + this.bizCmdbClient = bizCmdbClient; + this.bizSetCmdbClient = bizSetCmdbClient; + this.cronJobService = cronJobService; + this.serviceApplicationResource = serviceApplicationResource; + } + + public boolean execute() { + if (!enabled) { + log.info("disableCronJobOfArchivedScopeTask not enabled, skip, you can enable it in config file"); + return false; + } + log.info("disableCronJobOfArchivedScopeTask arranged"); + String machineIp = IpUtils.getFirstMachineIP(); + boolean lockGotten = LockUtils.tryGetDistributedLock( + DISABLE_CRON_JOB_TASK_RUNNING_MACHINE, + machineIp, + 5000 + ); + if (!lockGotten) { + String runningMachine = redisTemplate.opsForValue().get(DISABLE_CRON_JOB_TASK_RUNNING_MACHINE); + if (StringUtils.isNotBlank(runningMachine)) { + //已有线程在跑 + log.warn("disable cron job thread already running on {}", runningMachine); + } else { + log.info("disable cron job lock not gotten, return"); + } + return false; + } + + // 开一个心跳子线程，维护当前机器的状态 + RedisKeyHeartBeatThread disableCronJobRedisKeyHeartBeatThread = new RedisKeyHeartBeatThread( + redisTemplate, + DISABLE_CRON_JOB_TASK_RUNNING_MACHINE, + machineIp, + 5000L, + 4000L + ); + disableCronJobRedisKeyHeartBeatThread.setName("disableCronJobRedisKeyHeartBeatThread"); + disableCronJobRedisKeyHeartBeatThread.start(); + try { + disableCronJobOfArchivedScope(); + return true; + } catch (Throwable t) { + log.warn("Fail to disableCronJob", t); + return false; + } finally { + disableCronJobRedisKeyHeartBeatThread.stopAtOnce(); + } + } + + /** + * 禁用条件：1. 业务在作业平台中是软删除状态， 2. 查询配置平台业务接口，业务不在返回列表中 + */ + private void disableCronJobOfArchivedScope() { + List archivedAppIds = serviceApplicationResource.listAllAppIdOfArchivedScope().getData(); + log.info("finally find archived appIds={}", archivedAppIds); + List failedAppIds = new ArrayList<>(); + for (Long appId : archivedAppIds) { + boolean result = cronJobService.disableCronJobByAppId(appId); + if (!result) { + failedAppIds.add(appId); + } + } + if (failedAppIds.isEmpty()) { + log.info("all cron jobs with archived apps have been successfully disabled."); + } else { + log.warn("failed to disable cron jobs for the following archived appIds: {}}", failedAppIds); + } + } + +} diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/task/ScheduledTasks.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/task/ScheduledTasks.java new file mode 100644 index 0000000000..3c24ac9566 --- /dev/null +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/task/ScheduledTasks.java @@ -0,0 +1,74 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.crontab.task; + +import com.tencent.bk.job.crontab.service.CronJobLoadingService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +@Slf4j +@Component("jobCrontabScheduledTasks") +@EnableScheduling +public class ScheduledTasks { + + private final CronJobLoadingService cronJobLoadingService; + + private final DisableCronJobOfArchivedScopeTask disableCronJobOfArchivedScopeTask; + + @Autowired + public ScheduledTasks(CronJobLoadingService cronJobLoadingService, + DisableCronJobOfArchivedScopeTask disableCronJobOfArchivedScopeTask) { + this.cronJobLoadingService = cronJobLoadingService; + this.disableCronJobOfArchivedScopeTask = disableCronJobOfArchivedScopeTask; + } + + /** + * 每天早上9:30更新一次定时任务数据到Quartz内存 + */ + @Scheduled(cron = "0 30 9 * * *") + public void loadCronToQuartzPeriodically() { + log.info("loadCronToQuartzPeriodically"); + cronJobLoadingService.loadAllCronJob(); + } + + /** + * 每上午10点把已归档业务(集)的定时任务禁用 + */ + @Scheduled(cron = "0 0 10 * * ?") + public void disableCronJobOfArchivedScopeTask() { + log.info(Thread.currentThread().getId() + ":disableCronJobOfArchivedScopeTask start"); + long start = System.currentTimeMillis(); + try { + disableCronJobOfArchivedScopeTask.execute(); + } catch (Exception e) { + log.error("disableCronJobOfArchivedScopeTask fail", e); + } finally { + log.info("disableCronJobOfArchivedScopeTask end, duration={}ms", System.currentTimeMillis() - start); + } + } +} diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/AbstractQuartzJobBean.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/AbstractQuartzJobBean.java index 5af9abce60..a95781c4be 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/AbstractQuartzJobBean.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/AbstractQuartzJobBean.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,11 +26,16 @@ import com.tencent.bk.job.common.redis.util.LockUtils; import com.tencent.bk.job.common.util.JobContextUtil; +import com.tencent.bk.job.common.util.TimeUtil; +import com.tencent.bk.job.crontab.metrics.CronMetricsConstants; import com.tencent.bk.job.crontab.metrics.ScheduleMeasureService; +import io.micrometer.core.instrument.Tag; import lombok.extern.slf4j.Slf4j; +import org.jetbrains.annotations.NotNull; import org.quartz.JobExecutionContext; import org.quartz.JobExecutionException; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cloud.sleuth.ScopedSpan; import org.springframework.cloud.sleuth.Tracer; import org.springframework.scheduling.quartz.QuartzJobBean; @@ -38,6 +43,7 @@ import java.time.ZoneOffset; import java.time.format.DateTimeFormatter; +@SuppressWarnings("SpringJavaAutowiredMembersInspection") @Slf4j public abstract class AbstractQuartzJobBean extends QuartzJobBean { @@ -45,7 +51,7 @@ public abstract class AbstractQuartzJobBean extends QuartzJobBean { DateTimeFormatter.ofPattern("yyyyMMddHHmmssX").withZone(ZoneOffset.UTC); @Autowired - ScheduleMeasureService scheduleMeasureService; + public ScheduleMeasureService scheduleMeasureService; @Autowired Tracer tracer; @@ -65,30 +71,63 @@ protected static String getLockKey(JobExecutionContext context) { public abstract String name(); @Override - protected void executeInternal(JobExecutionContext context) { + protected void executeInternal(@NotNull JobExecutionContext context) { scheduleMeasureService.recordCronScheduleDelay(name(), context); - JobContextUtil.setRequestId(tracer.currentSpan().context().traceId()); + ScopedSpan span = tracer.startScopedSpan("executeCronJob"); + JobContextUtil.setRequestId(span.context().traceId()); String executeId = JobContextUtil.getRequestId(); + long startTimeMills = System.currentTimeMillis(); + boolean redisLockGotten = false; try { if (log.isDebugEnabled()) { log.debug("{}|Job {} key {} start execute ...", executeId, name(), getLockKey(context)); } - if (LockUtils.tryGetDistributedLock(getLockKey(context), executeId, 1000L)) { + redisLockGotten = LockUtils.tryGetDistributedLock(getLockKey(context), executeId, 90000L); + if (redisLockGotten) { executeInternalInternal(context); } else { - log.warn("{}|Job {} key {} execute aborted. Acquire lock failed!", executeId, name(), - getLockKey(context)); + if (log.isDebugEnabled()) { + log.debug( + "{}|Job {} key {} execute aborted. Acquire lock failed!", + executeId, + name(), + getLockKey(context) + ); + } } - if (log.isDebugEnabled()) { log.debug("{}|Job {} key {} execute finished.", executeId, name(), getLockKey(context)); } } catch (JobExecutionException e) { log.error("fail to executeInternal", e); } finally { - LockUtils.releaseDistributedLock(getLockKey(context), executeId); + recordCronTimeConsuming(context, redisLockGotten, startTimeMills); + span.end(); } } + private void recordCronTimeConsuming(JobExecutionContext context, + boolean redisLockGotten, + long startTimeMills) { + long timeConsumingMills = System.currentTimeMillis() - startTimeMills; + String timeFormat = "yyyy-MM-dd HH:mm:ss.SSS"; + log.info( + "CronJob finished: {}, " + + "redisLockGotten={}, scheduledFireTime={}, fireTime={}, fireDelay={}ms, executeDuration={}ms", + getLockKey(context), + redisLockGotten, + TimeUtil.formatTime(context.getScheduledFireTime().getTime(), timeFormat), + TimeUtil.formatTime(context.getFireTime().getTime(), timeFormat), + context.getFireTime().getTime() - context.getScheduledFireTime().getTime(), + timeConsumingMills + ); + scheduleMeasureService.recordCronTimeConsuming( + name(), + context, + timeConsumingMills, + Tag.of(CronMetricsConstants.TAG_KEY_JOB_CRON_EXECUTE_ACTUALLY, String.valueOf(redisLockGotten)) + ); + } + protected abstract void executeInternalInternal(JobExecutionContext context) throws JobExecutionException; } diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/AbstractQuartzTaskHandler.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/AbstractQuartzTaskHandler.java index 614a4efd1e..aeee773682 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/AbstractQuartzTaskHandler.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/AbstractQuartzTaskHandler.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/AutowiredSpringBeanJobFactory.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/AutowiredSpringBeanJobFactory.java index 7cdf40735e..30c2d20167 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/AutowiredSpringBeanJobFactory.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/AutowiredSpringBeanJobFactory.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/ListenerRegister.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/ListenerRegister.java index aef4d169eb..ae90bbdd5f 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/ListenerRegister.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/ListenerRegister.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/NotificationPolicy.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/NotificationPolicy.java index 2fdea3b3b6..aca754f97e 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/NotificationPolicy.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/NotificationPolicy.java @@ -14,10 +14,10 @@ @ToString @Component public class NotificationPolicy { - @Value("${notification-policy.failed.start.begin:1}") + @Value("${job.crontab.notification-policy.failed.start.begin:1}") private Integer begin; - @Value("${notification-policy.failed.start.frequency:5}") + @Value("${job.crontab.notification-policy.failed.start.frequency:5}") private Integer frequency; - @Value("${notification-policy.failed.start.total:-1}") + @Value("${job.crontab.notification-policy.failed.start.total:-1}") private Integer totalTimes; } diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/QuartzJob.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/QuartzJob.java index b9911a90cb..bce1a2cd17 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/QuartzJob.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/QuartzJob.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/QuartzJobBuilder.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/QuartzJobBuilder.java index 1ea3ccc8f6..1d66491531 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/QuartzJobBuilder.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/QuartzJobBuilder.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/QuartzTrigger.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/QuartzTrigger.java index df29e3950f..afe6280d78 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/QuartzTrigger.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/QuartzTrigger.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/QuartzTriggerBuilder.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/QuartzTriggerBuilder.java index bcbc38d4ef..d20924e712 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/QuartzTriggerBuilder.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/QuartzTriggerBuilder.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/executor/CronHistoryCleanJobExecutor.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/executor/CronHistoryCleanJobExecutor.java index e2e82ca2b0..9f19009be1 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/executor/CronHistoryCleanJobExecutor.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/executor/CronHistoryCleanJobExecutor.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/executor/InnerJobExecutor.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/executor/InnerJobExecutor.java index c51be1bc19..06bc90a705 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/executor/InnerJobExecutor.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/executor/InnerJobExecutor.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -35,8 +35,13 @@ import org.apache.commons.lang3.StringUtils; import org.quartz.JobExecutionContext; import org.quartz.JobExecutionException; +import org.slf4j.helpers.MessageFormatter; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.http.*; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; import org.springframework.web.client.RestClientException; import org.springframework.web.client.RestTemplate; @@ -119,7 +124,15 @@ protected void executeInternalInternal(JobExecutionContext context) throws JobEx ExecuteStatusEnum.FAIL); } } catch (RestClientException e) { - log.error("Execute task failed!|{}|{}|{}", systemId, jobKey, scheduledFireTime, e); + String msg = MessageFormatter.arrayFormat( + "Execute task failed!|{}|{}|{}", + new String[]{ + systemId, + jobKey, + String.valueOf(scheduledFireTime) + } + ).getMessage(); + log.error(msg, e); cronJobHistoryService.updateStatusByIdAndTime(systemId, jobKey, scheduledFireTime, ExecuteStatusEnum.FAIL); } } diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/executor/NotifyJobExecutor.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/executor/NotifyJobExecutor.java index bf69fda15b..abeb879b23 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/executor/NotifyJobExecutor.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/executor/NotifyJobExecutor.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/executor/SimpleJobExecutor.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/executor/SimpleJobExecutor.java index d6416d011d..1ade4efd32 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/executor/SimpleJobExecutor.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/executor/SimpleJobExecutor.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,7 @@ package com.tencent.bk.job.crontab.timer.executor; +import com.tencent.bk.job.common.exception.ServiceException; import com.tencent.bk.job.common.model.InternalResponse; import com.tencent.bk.job.crontab.constant.ExecuteStatusEnum; import com.tencent.bk.job.crontab.constant.NotificationPolicyEnum; @@ -38,11 +39,11 @@ import com.tencent.bk.job.crontab.timer.NotificationPolicy; import com.tencent.bk.job.execute.model.inner.ServiceTaskExecuteResult; import com.tencent.bk.job.execute.model.inner.ServiceTaskVariable; +import com.tencent.bk.job.manage.api.inner.ServiceApplicationResource; import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.quartz.JobExecutionContext; -import org.quartz.JobExecutionException; import org.springframework.beans.factory.annotation.Autowired; import java.util.List; @@ -53,6 +54,7 @@ * * @since 16/2/2020 15:38 */ +@SuppressWarnings("SpringJavaAutowiredMembersInspection") @Slf4j @Setter public class SimpleJobExecutor extends AbstractQuartzJobBean { @@ -72,6 +74,8 @@ public class SimpleJobExecutor extends AbstractQuartzJobBean { @Autowired NotificationPolicy notificationPolicy; + @Autowired + ServiceApplicationResource applicationResource; /** * 业务 ID 字符串 @@ -89,7 +93,7 @@ public String name() { } @Override - protected void executeInternalInternal(JobExecutionContext context) throws JobExecutionException { + protected void executeInternalInternal(JobExecutionContext context) { // Parse basic info if (log.isDebugEnabled()) { log.debug("Execute task|{}|{}", appIdStr, cronJobIdStr); @@ -98,6 +102,18 @@ protected void executeInternalInternal(JobExecutionContext context) throws JobEx long cronJobId = Long.parseLong(cronJobIdStr); long scheduledFireTime = getScheduledFireTime(context).toEpochMilli(); + // 判断业务/业务集的存在性，如果不存在不执行定时任务 + try { + if (!applicationResource.existsAppById(appId).getData()) { + log.warn("appId not exists, cron not execute! appId:{}, cronId:{}", appId, cronJobId); + return; + } + } catch (ServiceException e) { + log.error("Error(biz or biz set not exists) occurred while querying app by id," + + "cron not execute! appId:{}, cronId:{}",appId, cronJobId); + return; + } + CronJobHistoryDTO cronJobHistory = cronJobHistoryService.getHistoryByIdAndTime(appId, cronJobId, scheduledFireTime); if (cronJobHistory != null) { @@ -114,49 +130,95 @@ protected void executeInternalInternal(JobExecutionContext context) throws JobEx if (log.isDebugEnabled()) { log.debug("Get cronjob info return|{}", cronJobInfo); } - - CronJobInfoDTO cronJobErrorInfo = cronJobService.getCronJobErrorInfoById(appId, cronJobId); - if (log.isDebugEnabled()) { - log.debug("Get cronjob Error info return|{}", cronJobErrorInfo); + if (!cronJobInfo.getEnable()) { + log.error("cronJob {} scheduled unexpectedly, do not execute", cronJobInfo); + return; } List variables = cronJobInfo.getVariableValue(); List taskVariables = null; if (CollectionUtils.isNotEmpty(variables)) { taskVariables = - variables.parallelStream().map(CronJobVariableDTO::toServiceTaskVariable).collect(Collectors.toList()); + variables.stream().map(CronJobVariableDTO::toServiceTaskVariable).collect(Collectors.toList()); } - boolean executeFailed = false; - Integer errorCode = null; - String errorMessage = null; cronJobHistoryService.fillExecutor(historyId, cronJobInfo.getLastModifyUser()); - InternalResponse executeResult = executeTaskService.executeTask(appId, + InternalResponse executeResultResp = executeTaskService.executeTask( + appId, cronJobInfo.getTaskPlanId(), - cronJobInfo.getId(), cronJobInfo.getName(), taskVariables, cronJobInfo.getLastModifyUser()); + cronJobInfo.getId(), + cronJobInfo.getName(), + taskVariables, + cronJobInfo.getLastModifyUser() + ); if (log.isDebugEnabled()) { - log.debug("Execute result|{}", executeResult); + log.debug("Execute result|{}", executeResultResp); + } + recordCronExecuteDelay(context, executeResultResp); + updateCronJobHistoryAndErrorInfo( + context, + appId, + cronJobId, + scheduledFireTime, + cronJobInfo, + historyId, + executeResultResp + ); + } + + private void recordCronExecuteDelay(JobExecutionContext context, + InternalResponse executeResultResp) { + if (executeResultResp == null) { + return; + } + ServiceTaskExecuteResult taskExecuteResult = executeResultResp.getData(); + if (taskExecuteResult == null) { + return; + } + Long createTime = taskExecuteResult.getCreateTime(); + if (createTime == null) { + return; } - if (executeResult != null && executeResult.getData() != null - && executeResult.getData().getTaskInstanceId() > 0) { + long executeDelayMills = System.currentTimeMillis() - createTime; + scheduleMeasureService.recordCronExecuteDelay(name(), context, executeDelayMills); + } + + private void updateCronJobHistoryAndErrorInfo(JobExecutionContext context, + long appId, + long cronJobId, + long scheduledFireTime, + CronJobInfoDTO cronJobInfo, + long historyId, + InternalResponse executeResultResp) { + + boolean executeFailed = false; + Integer errorCode = null; + String errorMessage = null; + if (executeResultResp != null && executeResultResp.getData() != null + && executeResultResp.getData().getTaskInstanceId() > 0) { if (log.isDebugEnabled()) { - log.debug("Execute success! Task instance id {}", executeResult.getData().getTaskInstanceId()); + log.debug("Execute success! Task instance id {}", executeResultResp.getData().getTaskInstanceId()); } cronJobHistoryService.updateStatusByIdAndTime(appId, cronJobId, scheduledFireTime, ExecuteStatusEnum.RUNNING); } else { - log.error("Execute task failed!|{}|{}|{}|{}", appId, cronJobId, scheduledFireTime, executeResult); + log.warn("Execute task failed!|{}|{}|{}|{}", appId, cronJobId, scheduledFireTime, executeResultResp); cronJobHistoryService.updateStatusByIdAndTime(appId, cronJobId, scheduledFireTime, ExecuteStatusEnum.FAIL); executeFailed = true; - if (executeResult != null) { - errorCode = executeResult.getCode(); - errorMessage = executeResult.getErrorMsg(); + if (executeResultResp != null) { + errorCode = executeResultResp.getCode(); + errorMessage = executeResultResp.getErrorMsg(); if (errorCode != null) { cronJobHistoryService.fillErrorInfo(historyId, errorCode.longValue(), errorMessage); } } } + CronJobInfoDTO cronJobErrorInfo = cronJobService.getCronJobErrorInfoById(appId, cronJobId); + if (log.isDebugEnabled()) { + log.debug("Get cronjob Error info return|{}", cronJobErrorInfo); + } + updateCronJobErrorInfo(cronJobErrorInfo, executeFailed, errorCode); if (context.getNextFireTime() == null) { @@ -167,7 +229,8 @@ protected void executeInternalInternal(JobExecutionContext context) throws JobEx if (executeFailed && isNotify(cronJobErrorInfo)) { notifyService.sendCronJobFailedNotification(errorCode, errorMessage, cronJobInfo); if (log.isDebugEnabled()) { - log.debug("Send cronjob failed notification, execute error count is {}", cronJobErrorInfo.getLastExecuteErrorCount()); + log.debug("Send cronjob failed notification, execute error count is {}", + cronJobErrorInfo.getLastExecuteErrorCount()); } } } diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/handler/DefaultQuartzTaskHandler.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/handler/DefaultQuartzTaskHandler.java index 598cc31e41..77a863df13 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/handler/DefaultQuartzTaskHandler.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/handler/DefaultQuartzTaskHandler.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,18 +26,25 @@ import com.tencent.bk.job.crontab.timer.AbstractQuartzTaskHandler; import com.tencent.bk.job.crontab.timer.QuartzJob; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; -import org.quartz.*; +import org.quartz.JobDetail; +import org.quartz.JobKey; +import org.quartz.Scheduler; +import org.quartz.SchedulerException; +import org.quartz.Trigger; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; import org.springframework.util.Assert; import java.util.List; import java.util.Set; +import java.util.stream.Collectors; /** * QuartzTaskHandler 的默认实现 **/ +@Slf4j @Component public class DefaultQuartzTaskHandler extends AbstractQuartzTaskHandler { @@ -60,6 +67,11 @@ public void addJob(QuartzJob quartzJob) throws SchedulerException { Set triggers = createTriggers(quartzJob); + if (!scheduler.isStarted()) { + log.info("scheduler is not started, ignore add job {}!", quartzJob.getKey().getName()); + return; + } + if (CollectionUtils.isEmpty(triggers)) { this.scheduler.addJob(jobDetail, false); } else { @@ -84,6 +96,11 @@ public void deleteJob(JobKey jobKey) throws SchedulerException { Assert.notNull(jobKey, "jobKey cannot be empty!"); Assert.notNull(jobKey.getName(), "jobKey name cannot be empty!"); + if (!scheduler.isStarted()) { + log.info("scheduler is not started, ignore delete job {}!", jobKey.getName()); + return; + } + this.scheduler.deleteJob(jobKey); } @@ -91,6 +108,15 @@ public void deleteJob(JobKey jobKey) throws SchedulerException { public void deleteJob(List jobKeys) throws SchedulerException { Assert.notNull(jobKeys, "jobKeys cannot be empty!"); + if (!scheduler.isStarted()) { + log.info( + "scheduler is not started, ignore delete {} job keys: {}", + jobKeys.size(), + jobKeys.stream().map(JobKey::getName).collect(Collectors.toList()) + ); + return; + } + this.scheduler.deleteJobs(jobKeys); } @@ -99,6 +125,10 @@ public void deleteJob(List jobKeys) throws SchedulerException { */ @Override public void pauseAll() throws SchedulerException { + if (!scheduler.isStarted()) { + log.info("scheduler is not started, ignore pauseAll!"); + return; + } this.scheduler.pauseAll(); } } diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/listener/AbstractJobListener.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/listener/AbstractJobListener.java index b527643464..a4cc74054f 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/listener/AbstractJobListener.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/listener/AbstractJobListener.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/listener/AbstractSchedulerListener.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/listener/AbstractSchedulerListener.java index 4e136b7997..1ca51012ca 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/listener/AbstractSchedulerListener.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/listener/AbstractSchedulerListener.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/listener/AbstractTriggerListener.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/listener/AbstractTriggerListener.java index 16402516f6..6674a9e924 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/listener/AbstractTriggerListener.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/listener/AbstractTriggerListener.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/listener/impl/JobListener.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/listener/impl/JobListener.java index 12e96a9147..297f007684 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/listener/impl/JobListener.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/listener/impl/JobListener.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -33,7 +33,7 @@ import java.time.LocalDateTime; @Slf4j -@Component +@Component("cronJobListener") public class JobListener extends AbstractJobListener { @Override diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/listener/impl/SchedulerListener.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/listener/impl/SchedulerListener.java index e8e8e4c840..737d1a65d7 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/listener/impl/SchedulerListener.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/listener/impl/SchedulerListener.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/listener/impl/TriggerListener.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/listener/impl/TriggerListener.java index f396871c03..a99381cb8f 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/listener/impl/TriggerListener.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/timer/listener/impl/TriggerListener.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/util/CronExpressionUtil.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/util/CronExpressionUtil.java index 5aa7fe3219..99b45ef06f 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/util/CronExpressionUtil.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/util/CronExpressionUtil.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/util/DbRecordMapper.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/util/DbRecordMapper.java index fc65f6a38b..f69a7a3cde 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/util/DbRecordMapper.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/util/DbRecordMapper.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,10 +27,10 @@ import com.tencent.bk.job.crontab.constant.ExecuteStatusEnum; import com.tencent.bk.job.crontab.model.dto.CronJobHistoryDTO; import com.tencent.bk.job.crontab.model.dto.InnerCronJobHistoryDTO; +import com.tencent.bk.job.crontab.model.tables.CronJobHistory; +import com.tencent.bk.job.crontab.model.tables.InnerCronJobHistory; import org.jooq.Record10; import org.jooq.Record7; -import org.jooq.generated.tables.CronJobHistory; -import org.jooq.generated.tables.InnerCronJobHistory; import org.jooq.types.UByte; import org.jooq.types.ULong; diff --git a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/util/DbUtils.java b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/util/DbUtils.java index 6842aabc8a..b4ec0e2957 100644 --- a/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/util/DbUtils.java +++ b/src/backend/job-crontab/service-job-crontab/src/main/java/com/tencent/bk/job/crontab/util/DbUtils.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/build.gradle b/src/backend/job-execute/api-job-execute/build.gradle index a9c4f5b36a..588f900ccd 100644 --- a/src/backend/job-execute/api-job-execute/build.gradle +++ b/src/backend/job-execute/api-job-execute/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,7 +27,8 @@ dependencies { api project(':commons:common-iam') api project(':commons:common-i18n') api project(':commons:esb-sdk') - api project(':commons:common-statistics') + api project(':job-analysis:api-common-job-analysis') + api project(':job-manage:api-common-job-manage') api(project(":commons:common-api")) implementation 'com.fasterxml.jackson.core:jackson-core' implementation 'com.fasterxml.jackson.core:jackson-databind' diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/gse/GseGetProcResultResource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/gse/GseGetProcResultResource.java deleted file mode 100644 index 8400fd1e03..0000000000 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/gse/GseGetProcResultResource.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.api.esb.gse; - -import com.tencent.bk.job.common.annotation.EsbAPI; -import com.tencent.bk.job.common.esb.model.EsbResp; -import com.tencent.bk.job.execute.model.esb.gse.req.EsbGseGetProcResultRequest; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - -import java.util.Map; - -/** - * GSE接口-操作服务器上的进程结果查询 - */ -@RequestMapping(value = {"/esb/api/v2", "/esb/api/v3"}) -@RestController -@EsbAPI -public interface GseGetProcResultResource { - - @PostMapping("/get_proc_result") - EsbResp> gseGetProcResult(@RequestBody EsbGseGetProcResultRequest request); - - -} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/gse/GseManageProcessResource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/gse/GseManageProcessResource.java deleted file mode 100644 index 5c038f2165..0000000000 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/gse/GseManageProcessResource.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.api.esb.gse; - -import com.tencent.bk.job.common.annotation.EsbAPI; -import com.tencent.bk.job.common.esb.model.EsbResp; -import com.tencent.bk.job.execute.model.esb.gse.EsbGseTaskResultDTO; -import com.tencent.bk.job.execute.model.esb.gse.req.EsbGseManageProcRequest; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - -/** - * GSE接口-进程管理接口-- 注册托管/取消注册托管 - */ -@RequestMapping(value = {"/esb/api/v2", "/esb/api/v3"}) -@RestController -@EsbAPI -public interface GseManageProcessResource { - - @PostMapping("/manage_proc") - EsbResp gseManageProc(@RequestBody EsbGseManageProcRequest request); - - -} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/gse/GseOperateProcessResource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/gse/GseOperateProcessResource.java deleted file mode 100644 index 73245bd980..0000000000 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/gse/GseOperateProcessResource.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.api.esb.gse; - -import com.tencent.bk.job.common.annotation.EsbAPI; -import com.tencent.bk.job.common.esb.model.EsbResp; -import com.tencent.bk.job.execute.model.esb.gse.EsbGseTaskResultDTO; -import com.tencent.bk.job.execute.model.esb.gse.req.EsbGseOperateProcessRequest; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - -/** - * GSE接口-GSE进程操作-V1版本 - */ -@RequestMapping(value = {"/esb/api/v2", "/esb/api/v3"}) -@RestController -@EsbAPI -public interface GseOperateProcessResource { - - @PostMapping("/operate_proc") - EsbResp gseOperateProcess(@RequestBody EsbGseOperateProcessRequest request); - - -} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/gse/GseOperateProcessV2Resource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/gse/GseOperateProcessV2Resource.java deleted file mode 100644 index 05d3c071e9..0000000000 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/gse/GseOperateProcessV2Resource.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.api.esb.gse; - -import com.tencent.bk.job.common.annotation.EsbAPI; -import com.tencent.bk.job.common.esb.model.EsbResp; -import com.tencent.bk.job.execute.model.esb.gse.EsbGseTaskResultDTO; -import com.tencent.bk.job.execute.model.esb.gse.req.EsbGseOperateProcessRequest; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - -/** - * GSE接口-GSE进程操作-V2版本 - */ -@RequestMapping(value = {"/esb/api/v2", "/esb/api/v3"}) -@RestController -@EsbAPI -public interface GseOperateProcessV2Resource { - - @PostMapping("/operate_process") - EsbResp gseOperateProcessV2(@RequestBody EsbGseOperateProcessRequest request); - - -} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/EsbExecuteTaskResource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/EsbExecuteTaskResource.java index 3d7edb6f89..7ab9f425d6 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/EsbExecuteTaskResource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/EsbExecuteTaskResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,12 +25,14 @@ package com.tencent.bk.job.execute.api.esb.v2; import com.tencent.bk.job.common.annotation.EsbAPI; +import com.tencent.bk.job.common.constant.JobCommonHeaders; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.execute.model.esb.v2.EsbJobExecuteDTO; import com.tencent.bk.job.execute.model.esb.v2.request.EsbExecuteJobRequest; import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @@ -44,6 +46,8 @@ public interface EsbExecuteTaskResource { @PostMapping("/execute_job") EsbResp executeJob( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbExecuteJobRequest request diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/EsbFastExecuteSQLResource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/EsbFastExecuteSQLResource.java index 6ab20737fc..d78b2ee56f 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/EsbFastExecuteSQLResource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/EsbFastExecuteSQLResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,12 +25,14 @@ package com.tencent.bk.job.execute.api.esb.v2; import com.tencent.bk.job.common.annotation.EsbAPI; +import com.tencent.bk.job.common.constant.JobCommonHeaders; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.execute.model.esb.v2.EsbJobExecuteDTO; import com.tencent.bk.job.execute.model.esb.v2.request.EsbFastExecuteSQLRequest; import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @@ -44,6 +46,8 @@ public interface EsbFastExecuteSQLResource { @PostMapping("/fast_execute_sql") EsbResp fastExecuteSQL( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbFastExecuteSQLRequest request diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/EsbFastExecuteScriptResource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/EsbFastExecuteScriptResource.java index 5070f10884..4186af5da7 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/EsbFastExecuteScriptResource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/EsbFastExecuteScriptResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,12 +25,14 @@ package com.tencent.bk.job.execute.api.esb.v2; import com.tencent.bk.job.common.annotation.EsbAPI; +import com.tencent.bk.job.common.constant.JobCommonHeaders; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.execute.model.esb.v2.EsbJobExecuteDTO; import com.tencent.bk.job.execute.model.esb.v2.request.EsbFastExecuteScriptRequest; import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @@ -44,6 +46,8 @@ public interface EsbFastExecuteScriptResource { @PostMapping("/fast_execute_script") EsbResp fastExecuteScript( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbFastExecuteScriptRequest request diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/EsbFastPushFileResource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/EsbFastPushFileResource.java index 8f4b8b91ab..fb83808838 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/EsbFastPushFileResource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/EsbFastPushFileResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,12 +25,14 @@ package com.tencent.bk.job.execute.api.esb.v2; import com.tencent.bk.job.common.annotation.EsbAPI; +import com.tencent.bk.job.common.constant.JobCommonHeaders; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.execute.model.esb.v2.EsbJobExecuteDTO; import com.tencent.bk.job.execute.model.esb.v2.request.EsbFastPushFileRequest; import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @@ -44,6 +46,8 @@ public interface EsbFastPushFileResource { @PostMapping("/fast_push_file") EsbResp fastPushFile( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbFastPushFileRequest request diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/EsbGetJobInstanceGlobalVarValueResource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/EsbGetJobInstanceGlobalVarValueResource.java index feb4167361..cf86c62d31 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/EsbGetJobInstanceGlobalVarValueResource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/EsbGetJobInstanceGlobalVarValueResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,12 +25,14 @@ package com.tencent.bk.job.execute.api.esb.v2; import com.tencent.bk.job.common.annotation.EsbAPI; +import com.tencent.bk.job.common.constant.JobCommonHeaders; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.execute.model.esb.v2.EsbTaskInstanceGlobalVarValueDTO; import com.tencent.bk.job.execute.model.esb.v2.request.EsbGetJobInstanceGlobalVarValueRequest; import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @@ -44,6 +46,8 @@ public interface EsbGetJobInstanceGlobalVarValueResource { @PostMapping("/get_job_instance_global_var_value") EsbResp getJobInstanceGlobalVarValue( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbGetJobInstanceGlobalVarValueRequest request diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/EsbGetJobInstanceLogResource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/EsbGetJobInstanceLogResource.java index 7e547eaa1a..b9310ef3ce 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/EsbGetJobInstanceLogResource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/EsbGetJobInstanceLogResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -50,6 +50,8 @@ public interface EsbGetJobInstanceLogResource { @PostMapping("/get_job_instance_log") EsbResp> getJobInstanceLogUsingPost( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbGetJobInstanceLogRequest request @@ -57,8 +59,8 @@ EsbResp> getJobInstanceLogUsingPost( @GetMapping("/get_job_instance_log") EsbResp> getJobInstanceLog( - @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestParam(value = "bk_biz_id", required = false) Long bizId, @RequestParam(value = "bk_scope_type", required = false) String scopeType, @RequestParam(value = "bk_scope_id", required = false) String scopeId, diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/EsbGetJobInstanceStatusResource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/EsbGetJobInstanceStatusResource.java index 0398709cfe..1529fcba1c 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/EsbGetJobInstanceStatusResource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/EsbGetJobInstanceStatusResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -48,6 +48,8 @@ public interface EsbGetJobInstanceStatusResource { @PostMapping("/get_job_instance_status") EsbResp getJobInstanceStatusUsingPost( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbGetJobInstanceStatusRequest request @@ -55,8 +57,8 @@ EsbResp getJobInstanceStatusUsingPost( @GetMapping("/get_job_instance_status") EsbResp getJobInstanceStatus( - @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestParam(value = "bk_biz_id", required = false) Long bizId, @RequestParam(value = "bk_scope_type", required = false) String scopeType, @RequestParam(value = "bk_scope_id", required = false) String scopeId, diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/EsbGetStepInstanceStatusResource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/EsbGetStepInstanceStatusResource.java index 5d5ed6d8eb..f603aff4a2 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/EsbGetStepInstanceStatusResource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/EsbGetStepInstanceStatusResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,12 +25,14 @@ package com.tencent.bk.job.execute.api.esb.v2; import com.tencent.bk.job.common.annotation.EsbAPI; +import com.tencent.bk.job.common.constant.JobCommonHeaders; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.execute.model.esb.v2.EsbStepInstanceStatusDTO; import com.tencent.bk.job.execute.model.esb.v2.request.EsbGetStepInstanceStatusRequest; import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @@ -44,6 +46,8 @@ public interface EsbGetStepInstanceStatusResource { @PostMapping("/get_step_instance_status") EsbResp getJobStepInstanceStatus( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbGetStepInstanceStatusRequest request diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/EsbOperateJobInstanceResource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/EsbOperateJobInstanceResource.java index 8c30a31686..9aedc005a3 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/EsbOperateJobInstanceResource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/EsbOperateJobInstanceResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,12 +25,14 @@ package com.tencent.bk.job.execute.api.esb.v2; import com.tencent.bk.job.common.annotation.EsbAPI; +import com.tencent.bk.job.common.constant.JobCommonHeaders; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.execute.model.esb.v2.EsbJobExecuteDTO; import com.tencent.bk.job.execute.model.esb.v2.request.EsbOperateJobInstanceRequest; import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @@ -44,6 +46,8 @@ public interface EsbOperateJobInstanceResource { @PostMapping("/operate_job_instance") EsbResp operateJobInstance( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbOperateJobInstanceRequest request ); diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/EsbOperateStepInstanceResource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/EsbOperateStepInstanceResource.java index e4f0c15de6..aee99d7fc9 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/EsbOperateStepInstanceResource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/EsbOperateStepInstanceResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,12 +25,14 @@ package com.tencent.bk.job.execute.api.esb.v2; import com.tencent.bk.job.common.annotation.EsbAPI; +import com.tencent.bk.job.common.constant.JobCommonHeaders; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.execute.model.esb.v2.EsbJobExecuteDTO; import com.tencent.bk.job.execute.model.esb.v2.request.EsbOperateStepInstanceRequest; import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @@ -44,6 +46,8 @@ public interface EsbOperateStepInstanceResource { @PostMapping("/operate_step_instance") EsbResp operateStepInstance( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbOperateStepInstanceRequest request diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/EsbPushConfigFileResource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/EsbPushConfigFileResource.java index ba65e777a6..ab19808926 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/EsbPushConfigFileResource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/EsbPushConfigFileResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,12 +25,14 @@ package com.tencent.bk.job.execute.api.esb.v2; import com.tencent.bk.job.common.annotation.EsbAPI; +import com.tencent.bk.job.common.constant.JobCommonHeaders; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.execute.model.esb.v2.EsbJobExecuteDTO; import com.tencent.bk.job.execute.model.esb.v2.request.EsbPushConfigFileRequest; import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @@ -47,6 +49,8 @@ public interface EsbPushConfigFileResource { */ @PostMapping("/push_config_file") EsbResp pushConfigFile( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbPushConfigFileRequest request diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBatchGetJobInstanceIpLogV3Resource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBatchGetJobInstanceIpLogV3Resource.java index 3e0b205d71..1b6f69fc84 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBatchGetJobInstanceIpLogV3Resource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBatchGetJobInstanceIpLogV3Resource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,12 +25,14 @@ package com.tencent.bk.job.execute.api.esb.v3; import com.tencent.bk.job.common.annotation.EsbAPI; +import com.tencent.bk.job.common.constant.JobCommonHeaders; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.execute.model.esb.v3.EsbIpLogsV3DTO; import com.tencent.bk.job.execute.model.esb.v3.request.EsbBatchGetJobInstanceIpLogV3Request; import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @@ -44,6 +46,8 @@ public interface EsbBatchGetJobInstanceIpLogV3Resource { @PostMapping("/batch_get_job_instance_ip_log") EsbResp batchGetJobInstanceIpLogs( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbBatchGetJobInstanceIpLogV3Request request diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBkCIPluginBatchGetJobInstanceExecuteObjectLogV3Resource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBkCIPluginBatchGetJobInstanceExecuteObjectLogV3Resource.java new file mode 100644 index 0000000000..dcc2f050de --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBkCIPluginBatchGetJobInstanceExecuteObjectLogV3Resource.java @@ -0,0 +1,56 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.api.esb.v3; + +import com.tencent.bk.job.common.annotation.EsbAPI; +import com.tencent.bk.job.common.constant.JobCommonHeaders; +import com.tencent.bk.job.common.esb.model.EsbResp; +import com.tencent.bk.job.execute.model.esb.v3.bkci.plugin.EsbBkCIPluginBatchGetJobInstanceExecuteObjectLogRequest; +import com.tencent.bk.job.execute.model.esb.v3.bkci.plugin.EsbExecuteObjectLogsDTO; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +/** + * 根据执行对象列表批量查询执行日志(蓝盾作业执行插件专用，非正式公开 API) + */ +@RequestMapping("/esb/api/v3") +@RestController +@EsbAPI +public interface EsbBkCIPluginBatchGetJobInstanceExecuteObjectLogV3Resource { + + @PostMapping("/bkci_plugin_batch_get_job_instance_execute_object_log") + EsbResp batchGetJobInstanceExecuteObjectLogs( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, + @RequestBody + @Validated + EsbBkCIPluginBatchGetJobInstanceExecuteObjectLogRequest request + ); + +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBkCIPluginFastExecuteScriptV3Resource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBkCIPluginFastExecuteScriptV3Resource.java new file mode 100644 index 0000000000..cf770edf61 --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBkCIPluginFastExecuteScriptV3Resource.java @@ -0,0 +1,56 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.api.esb.v3; + +import com.tencent.bk.job.common.annotation.EsbAPI; +import com.tencent.bk.job.common.constant.JobCommonHeaders; +import com.tencent.bk.job.common.esb.model.EsbResp; +import com.tencent.bk.job.execute.model.esb.v3.EsbJobExecuteV3DTO; +import com.tencent.bk.job.execute.model.esb.v3.bkci.plugin.EsbBkCIPluginFastExecuteScriptRequest; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +/** + * 快速执行脚本(蓝盾作业执行插件专用，非正式公开 API) + */ +@RequestMapping("/esb/api/v3") +@RestController +@EsbAPI +public interface EsbBkCIPluginFastExecuteScriptV3Resource { + + @PostMapping("/bkci_plugin_fast_execute_script") + EsbResp fastExecuteScript( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, + @RequestBody + @Validated + EsbBkCIPluginFastExecuteScriptRequest request + ); + +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBkCIPluginFastTransferFileV3Resource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBkCIPluginFastTransferFileV3Resource.java new file mode 100644 index 0000000000..785231f5a2 --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBkCIPluginFastTransferFileV3Resource.java @@ -0,0 +1,53 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.api.esb.v3; + +import com.tencent.bk.job.common.annotation.EsbAPI; +import com.tencent.bk.job.common.constant.JobCommonHeaders; +import com.tencent.bk.job.common.esb.model.EsbResp; +import com.tencent.bk.job.execute.model.esb.v3.EsbJobExecuteV3DTO; +import com.tencent.bk.job.execute.model.esb.v3.bkci.plugin.EsbBkCIPluginFastTransferFileV3Request; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@RequestMapping("/esb/api/v3") +@RestController +@EsbAPI +public interface EsbBkCIPluginFastTransferFileV3Resource { + + @PostMapping("/bkci_plugin_fast_transfer_file") + EsbResp bkciPluginFastTransferFile( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, + @RequestBody + @Validated + EsbBkCIPluginFastTransferFileV3Request request + ); + +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBkCIPluginGetJobInstanceStatusV3Resource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBkCIPluginGetJobInstanceStatusV3Resource.java new file mode 100644 index 0000000000..9c4d0bc738 --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBkCIPluginGetJobInstanceStatusV3Resource.java @@ -0,0 +1,55 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.api.esb.v3; + +import com.tencent.bk.job.common.annotation.EsbAPI; +import com.tencent.bk.job.common.constant.JobCommonHeaders; +import com.tencent.bk.job.common.esb.model.EsbResp; +import com.tencent.bk.job.execute.model.esb.v3.bkci.plugin.EsbBkCIPluginGetJobInstanceStatusRequest; +import com.tencent.bk.job.execute.model.esb.v3.bkci.plugin.EsbJobInstanceStatusDTO; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +/** + * 根据作业实例 ID 查询作业执行状态(蓝盾作业执行插件专用，非正式公开 API) + */ +@RequestMapping("/esb/api/v3") +@RestController +@EsbAPI +public interface EsbBkCIPluginGetJobInstanceStatusV3Resource { + + @PostMapping("/bkci_plugin_get_job_instance_status") + EsbResp getJobInstanceStatus( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, + @RequestBody + @Validated + EsbBkCIPluginGetJobInstanceStatusRequest request + ); +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbExecuteJobPlanV3Resource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbExecuteJobPlanV3Resource.java index 7b391aa383..b497d8b7ca 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbExecuteJobPlanV3Resource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbExecuteJobPlanV3Resource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,12 +25,14 @@ package com.tencent.bk.job.execute.api.esb.v3; import com.tencent.bk.job.common.annotation.EsbAPI; +import com.tencent.bk.job.common.constant.JobCommonHeaders; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.execute.model.esb.v3.EsbJobExecuteV3DTO; import com.tencent.bk.job.execute.model.esb.v3.request.EsbExecuteJobV3Request; import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @@ -44,6 +46,8 @@ public interface EsbExecuteJobPlanV3Resource { @PostMapping("/execute_job_plan") EsbResp executeJobPlan( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbExecuteJobV3Request request diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastExecuteSQLV3Resource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastExecuteSQLV3Resource.java index f49a402b96..ec746b599f 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastExecuteSQLV3Resource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastExecuteSQLV3Resource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,12 +25,14 @@ package com.tencent.bk.job.execute.api.esb.v3; import com.tencent.bk.job.common.annotation.EsbAPI; +import com.tencent.bk.job.common.constant.JobCommonHeaders; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.execute.model.esb.v3.EsbJobExecuteV3DTO; import com.tencent.bk.job.execute.model.esb.v3.request.EsbFastExecuteSQLV3Request; import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @@ -44,6 +46,8 @@ public interface EsbFastExecuteSQLV3Resource { @PostMapping("/fast_execute_sql") EsbResp fastExecuteSQL( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbFastExecuteSQLV3Request request diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastExecuteScriptV3Resource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastExecuteScriptV3Resource.java index f34a0f895c..6071423a98 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastExecuteScriptV3Resource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastExecuteScriptV3Resource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,12 +25,14 @@ package com.tencent.bk.job.execute.api.esb.v3; import com.tencent.bk.job.common.annotation.EsbAPI; +import com.tencent.bk.job.common.constant.JobCommonHeaders; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.execute.model.esb.v3.EsbJobExecuteV3DTO; import com.tencent.bk.job.execute.model.esb.v3.request.EsbFastExecuteScriptV3Request; import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @@ -44,6 +46,8 @@ public interface EsbFastExecuteScriptV3Resource { @PostMapping("/fast_execute_script") EsbResp fastExecuteScript( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbFastExecuteScriptV3Request request diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastTransferFileV3Resource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastTransferFileV3Resource.java index ed68a15d55..f913e83949 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastTransferFileV3Resource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastTransferFileV3Resource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,12 +25,14 @@ package com.tencent.bk.job.execute.api.esb.v3; import com.tencent.bk.job.common.annotation.EsbAPI; +import com.tencent.bk.job.common.constant.JobCommonHeaders; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.execute.model.esb.v3.EsbJobExecuteV3DTO; import com.tencent.bk.job.execute.model.esb.v3.request.EsbFastTransferFileV3Request; import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @@ -44,6 +46,8 @@ public interface EsbFastTransferFileV3Resource { @PostMapping("/fast_transfer_file") EsbResp fastTransferFile( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbFastTransferFileV3Request request diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceGlobalVarValueV3Resource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceGlobalVarValueV3Resource.java index c422c921c3..6149ec758f 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceGlobalVarValueV3Resource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceGlobalVarValueV3Resource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -49,6 +49,8 @@ public interface EsbGetJobInstanceGlobalVarValueV3Resource { @PostMapping("/get_job_instance_global_var_value") EsbResp getJobInstanceGlobalVarValueUsingPost( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbGetJobInstanceGlobalVarValueV3Request request diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceIpLogV3Resource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceIpLogV3Resource.java index 712f95fa0f..711a36587a 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceIpLogV3Resource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceIpLogV3Resource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -48,6 +48,8 @@ public interface EsbGetJobInstanceIpLogV3Resource { @PostMapping("/get_job_instance_ip_log") EsbResp getJobInstanceIpLogUsingPost( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbGetJobInstanceIpLogV3Request request @@ -62,7 +64,8 @@ EsbResp getJobInstanceIpLog( @RequestParam(value = "bk_scope_id", required = false) String scopeId, @RequestParam(value = "job_instance_id") Long taskInstanceId, @RequestParam(value = "step_instance_id") Long stepInstanceId, - @RequestParam(value = "bk_cloud_id") Long cloudAreaId, - @RequestParam(value = "ip") String ip); + @RequestParam(value = "bk_host_id", required = false) Long hostId, + @RequestParam(value = "bk_cloud_id", required = false) Long cloudAreaId, + @RequestParam(value = "ip", required = false) String ip); } diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceListV3Resource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceListV3Resource.java index 113095f895..b2ce7803db 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceListV3Resource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceListV3Resource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -49,6 +49,8 @@ public interface EsbGetJobInstanceListV3Resource { @PostMapping("/get_job_instance_list") EsbResp> getJobInstanceListUsingPost( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbGetJobInstanceListV3Request request @@ -70,6 +72,7 @@ EsbResp> getJobInstanceList( @RequestParam(value = "type", required = false) Integer taskType, @RequestParam(value = "status", required = false) Integer taskStatus, @RequestParam(value = "ip", required = false) String ip, + @RequestParam(value = "job_cron_id", required = false) Long cronId, @RequestParam(value = "start", required = false) Integer start, @RequestParam(value = "length", required = false) Integer length); diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceStatusV3Resource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceStatusV3Resource.java index 223b6bbd7f..c986784e12 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceStatusV3Resource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceStatusV3Resource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -48,6 +48,8 @@ public interface EsbGetJobInstanceStatusV3Resource { @PostMapping("/get_job_instance_status") EsbResp getJobInstanceStatusUsingPost( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbGetJobInstanceStatusV3Request request diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetStepInstanceDetailV3Resource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetStepInstanceDetailV3Resource.java new file mode 100644 index 0000000000..90072eadb5 --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetStepInstanceDetailV3Resource.java @@ -0,0 +1,65 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.api.esb.v3; + +import com.tencent.bk.job.common.annotation.EsbAPI; +import com.tencent.bk.job.common.constant.JobCommonHeaders; +import com.tencent.bk.job.common.esb.model.EsbResp; +import com.tencent.bk.job.common.esb.model.job.v3.resp.EsbStepV3DTO; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; + +/** + * 根据步骤实例 ID 查询步骤执行详情API-V3 + */ +@Validated +@RequestMapping("/esb/api/v3") +@RestController +@EsbAPI +public interface EsbGetStepInstanceDetailV3Resource { + + @GetMapping("/get_step_instance_detail") + EsbResp getStepInstanceDetail( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, + @RequestParam(value = "bk_scope_type") String scopeType, + @RequestParam(value = "bk_scope_id") String scopeId, + @RequestParam(value = "job_instance_id") + @NotNull(message = "{validation.constraints.InvalidJobInstanceId.message}") + @Min(message = "{validation.constraints.InvalidJobInstanceId.message}", value = 1L) + Long taskInstanceId, + @RequestParam(value = "step_instance_id") + @NotNull(message = "{validation.constraints.InvalidStepInstanceId.message}") + @Min(message = "{validation.constraints.InvalidStepInstanceId.message}", value = 1L) + Long stepInstanceId); + +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetStepInstanceStatusV3Resource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetStepInstanceStatusV3Resource.java new file mode 100644 index 0000000000..4ba87a1d96 --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetStepInstanceStatusV3Resource.java @@ -0,0 +1,70 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.api.esb.v3; + +import com.tencent.bk.job.common.annotation.EsbAPI; +import com.tencent.bk.job.common.constant.JobCommonHeaders; +import com.tencent.bk.job.common.esb.model.EsbResp; +import com.tencent.bk.job.execute.model.esb.v3.EsbStepInstanceStatusV3DTO; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; + +/** + * 根据步骤实例 ID 查询步骤执行状态API-V3 + */ +@Validated +@RequestMapping("/esb/api/v3") +@RestController +@EsbAPI +public interface EsbGetStepInstanceStatusV3Resource { + + @GetMapping("/get_step_instance_status") + EsbResp getStepInstanceStatus( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, + @RequestParam(value = "bk_scope_type") String scopeType, + @RequestParam(value = "bk_scope_id") String scopeId, + @NotNull(message = "{validation.constraints.InvalidJobInstanceId.message}") + @Min(message = "{validation.constraints.InvalidJobInstanceId.message}", value = 1L) + @RequestParam(value = "job_instance_id") Long taskInstanceId, + @NotNull(message = "{validation.constraints.InvalidStepInstanceId.message}") + @Min(message = "{validation.constraints.InvalidStepInstanceId.message}", value = 1L) + @RequestParam(value = "step_instance_id") Long stepInstanceId, + @RequestParam(value = "execute_count", required = false) Integer executeCount, + @RequestParam(value = "batch", required = false) Integer batch, + @RequestParam(value = "max_host_num_per_group", required = false) Integer maxHostNumPerGroup, + @RequestParam(value = "keyword", required = false) String keyword, + @RequestParam(value = "search_ip", required = false) String searchIp, + @RequestParam(value = "status", required = false) Integer status, + @RequestParam(value = "tag", required = false) String tag); + +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbOperateJobInstanceV3Resource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbOperateJobInstanceV3Resource.java index 6a56502a8f..11a6028dd2 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbOperateJobInstanceV3Resource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbOperateJobInstanceV3Resource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,12 +25,14 @@ package com.tencent.bk.job.execute.api.esb.v3; import com.tencent.bk.job.common.annotation.EsbAPI; +import com.tencent.bk.job.common.constant.JobCommonHeaders; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.execute.model.esb.v3.EsbJobExecuteV3DTO; import com.tencent.bk.job.execute.model.esb.v3.request.EsbOperateJobInstanceV3Request; import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @@ -44,6 +46,8 @@ public interface EsbOperateJobInstanceV3Resource { @PostMapping("/operate_job_instance") EsbResp operateJobInstance( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbOperateJobInstanceV3Request request diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbOperateStepInstanceV3Resource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbOperateStepInstanceV3Resource.java index a886c724bb..e3c2874487 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbOperateStepInstanceV3Resource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbOperateStepInstanceV3Resource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,12 +25,14 @@ package com.tencent.bk.job.execute.api.esb.v3; import com.tencent.bk.job.common.annotation.EsbAPI; +import com.tencent.bk.job.common.constant.JobCommonHeaders; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.execute.model.esb.v3.EsbJobExecuteV3DTO; import com.tencent.bk.job.execute.model.esb.v3.request.EsbOperateStepInstanceV3Request; import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @@ -44,6 +46,8 @@ public interface EsbOperateStepInstanceV3Resource { @PostMapping("/operate_step_instance") EsbResp operateStepInstance( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbOperateStepInstanceV3Request request diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbPushConfigFileV3Resource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbPushConfigFileV3Resource.java index f09112e654..a549ba3909 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbPushConfigFileV3Resource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbPushConfigFileV3Resource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,12 +25,14 @@ package com.tencent.bk.job.execute.api.esb.v3; import com.tencent.bk.job.common.annotation.EsbAPI; +import com.tencent.bk.job.common.constant.JobCommonHeaders; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.execute.model.esb.v3.EsbJobExecuteV3DTO; import com.tencent.bk.job.execute.model.esb.v3.request.EsbPushConfigFileV3Request; import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @@ -47,6 +49,8 @@ public interface EsbPushConfigFileV3Resource { */ @PostMapping("/push_config_file") EsbResp pushConfigFile( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbPushConfigFileV3Request request diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbQueryToolsV3Resource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbQueryToolsV3Resource.java new file mode 100644 index 0000000000..2205621a6d --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbQueryToolsV3Resource.java @@ -0,0 +1,63 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.api.esb.v3; + +import com.tencent.bk.job.common.annotation.EsbAPI; +import com.tencent.bk.job.common.constant.JobCommonHeaders; +import com.tencent.bk.job.common.esb.model.EsbResp; +import com.tencent.bk.job.execute.model.esb.v3.EsbTaskLinkV3DTO; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; + +@RequestMapping("/esb/api/v3") +@RestController +@EsbAPI +public interface EsbQueryToolsV3Resource { + + @GetMapping("/query_gse_task") + EsbResp> queryGSETaskByStep( + @RequestHeader(value = JobCommonHeaders.USERNAME) + String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) + String appCode, + @RequestParam(value = "step_instance_id") + Long stepInstanceId + ); + + @GetMapping("/query_job_instance") + EsbResp queryJobInstanceByGseTask( + @RequestHeader(value = JobCommonHeaders.USERNAME) + String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) + String appCode, + @RequestParam(value = "gse_task_id") + String gseTaskId + ); +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceExecuteTaskResource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceExecuteTaskResource.java index 6380c6791a..073d71ab99 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceExecuteTaskResource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceExecuteTaskResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -29,11 +29,10 @@ import com.tencent.bk.job.common.model.iam.AuthResultDTO; import com.tencent.bk.job.execute.model.inner.ServiceTaskExecuteResult; import com.tencent.bk.job.execute.model.inner.request.ServiceTaskExecuteRequest; +import com.tentent.bk.job.common.api.feign.annotation.SmartFeignClient; import io.swagger.annotations.Api; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; /** * 作业执行API-服务内部调用 @@ -41,13 +40,12 @@ * @date 2019/09/18 */ @Api(tags = {"Task_Execute"}) -@RequestMapping("/service/execution") -@RestController +@SmartFeignClient(value = "job-execute", contextId = "executeTaskResource") @InternalAPI public interface ServiceExecuteTaskResource { - @PostMapping("/task-execution/task") + @PostMapping("/service/execution/task-execution/task") InternalResponse executeTask(@RequestBody ServiceTaskExecuteRequest request); - @PostMapping("/task-execution/task/auth") + @PostMapping("/service/execution/task-execution/task/auth") InternalResponse authExecuteTask(@RequestBody ServiceTaskExecuteRequest request); } diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceMetricsResource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceMetricsResource.java index 5ddc965c77..195e5fcf8d 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceMetricsResource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceMetricsResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,29 +24,23 @@ package com.tencent.bk.job.execute.api.inner; +import com.tencent.bk.job.analysis.api.dto.StatisticsDTO; import com.tencent.bk.job.common.annotation.InternalAPI; import com.tencent.bk.job.common.model.InternalResponse; -import com.tencent.bk.job.common.statistics.model.dto.StatisticsDTO; -import com.tencent.bk.job.execute.common.constants.RunStatusEnum; -import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; -import com.tencent.bk.job.execute.common.constants.TaskStartupModeEnum; -import com.tencent.bk.job.execute.common.constants.TaskTypeEnum; import com.tencent.bk.job.execute.model.inner.request.ServiceTriggerStatisticsRequest; +import com.tentent.bk.job.common.api.feign.annotation.SmartFeignClient; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; import java.util.List; @Api(tags = {"job-execute:service:Metrics"}) -@RequestMapping("/service/metrics") -@RestController +@SmartFeignClient(value = "job-execute", contextId = "executeMetricsResource") @InternalAPI public interface ServiceMetricsResource { @@ -55,7 +49,7 @@ public interface ServiceMetricsResource { InternalResponse> getJoinedAppIdList(); @ApiOperation(value = "是否有执行记录", produces = "application/json") - @GetMapping("/app/hasExecuteHistory") + @GetMapping("/service/metrics/app/hasExecuteHistory") InternalResponse hasExecuteHistory( @ApiParam(value = "业务Id", required = false) @RequestParam(value = "appId", required = false) Long appId, @@ -67,77 +61,8 @@ InternalResponse hasExecuteHistory( @RequestParam(value = "toTime", required = false) Long toTime ); - /** - * 大数据量下容易导致慢查询影响全局，非特殊情况不使用 - */ - @Deprecated - @ApiOperation(value = "快速文件分发统计", produces = "application/json") - @GetMapping("/fastPushFile/count") - InternalResponse countFastPushFile( - @ApiParam(value = "业务Id", required = false) - @RequestParam(value = "appId", required = false) Long appId, - @ApiParam(value = "传输模式", required = false) - @RequestParam(value = "transferMode", required = false) Integer transferMode, - @ApiParam(value = "文件源是否为本地文件", required = false) - @RequestParam(value = "localUpload", required = false) Boolean localUpload, - @ApiParam(value = "步骤状态", required = false) - @RequestParam(value = "runStatus", required = false) RunStatusEnum runStatus, - @ApiParam(value = "统计的起始时间", required = false) - @RequestParam(value = "fromTime", required = false) Long fromTime, - @ApiParam(value = "统计的截止时间", required = false) - @RequestParam(value = "toTime", required = false) Long toTime - ); - - /** - * 大数据量下容易导致慢查询影响全局，非特殊情况不使用 - */ - @Deprecated - @ApiOperation(value = "步骤执行统计", produces = "application/json") - @GetMapping("/stepInstances/count") - InternalResponse countStepInstances( - @ApiParam(value = "业务Id", required = false) - @RequestParam(value = "appId", required = false) Long appId, - @ApiParam(value = "步骤对应的StepId列表", required = false) - @RequestParam(value = "stepIdList", required = false) List stepIdList, - @ApiParam(value = "步骤类型", required = false) - @RequestParam(value = "stepExecuteType", required = false) StepExecuteTypeEnum stepExecuteType, - @ApiParam(value = "脚本类型", required = false) - @RequestParam(value = "scriptType", required = false) Integer scriptType, - @ApiParam(value = "步骤状态", required = false) - @RequestParam(value = "runStatus", required = false) RunStatusEnum runStatus, - @ApiParam(value = "统计的起始时间", required = false) - @RequestParam(value = "fromTime", required = false) Long fromTime, - @ApiParam(value = "统计的截止时间", required = false) - @RequestParam(value = "toTime", required = false) Long toTime - ); - - /** - * 大数据量下容易导致慢查询影响全局，非特殊情况不使用 - */ - @Deprecated - @ApiOperation(value = "任务（含快速/作业）执行统计", produces = "application/json") - @GetMapping("/taskInstances/count") - InternalResponse countTaskInstances( - @ApiParam(value = "业务Id", required = false) - @RequestParam(value = "appId", required = false) Long appId, - @ApiParam(value = "最小执行用时(单位：秒)", required = false) - @RequestParam(value = "minTotalTime", required = false) Long minTotalTime, - @ApiParam(value = "最大执行用时(单位：秒)", required = false) - @RequestParam(value = "maxTotalTime", required = false) Long maxTotalTime, - @ApiParam(value = "触发方式", required = false) - @RequestParam(value = "taskStartupMode", required = false) TaskStartupModeEnum taskStartupMode, - @ApiParam(value = "任务类型", required = false) - @RequestParam(value = "taskType", required = false) TaskTypeEnum taskType, - @ApiParam(value = "任务状态列表", required = false) - @RequestParam(value = "runStatusList", required = false) List runStatusList, - @ApiParam(value = "统计的起始时间(ms)", required = false) - @RequestParam(value = "fromTime", required = false) Long fromTime, - @ApiParam(value = "统计的截止时间(ms)", required = false) - @RequestParam(value = "toTime", required = false) Long toTime - ); - @ApiOperation(value = "获取统计数据", produces = "application/json") - @GetMapping("/statistics") + @GetMapping("/service/metrics/statistics") InternalResponse getStatistics( @ApiParam(value = "业务Id", required = true) @RequestParam(value = "appId", required = true) Long appId, @@ -151,26 +76,10 @@ InternalResponse getStatistics( @RequestParam(value = "dateStr", required = true) String dateStr ); - @ApiOperation(value = "获取统计数据", produces = "application/json") - @GetMapping("/statistics/list") - InternalResponse> listStatistics( - @ApiParam(value = "业务Id", required = false) - @RequestParam(value = "appId", required = false) Long appId, - @ApiParam(value = "资源类型", required = false) - @RequestParam(value = "resource", required = false) String resource, - @ApiParam(value = "资源维度", required = false) - @RequestParam(value = "dimension", required = false) String dimension, - @ApiParam(value = "资源维度取值", required = false) - @RequestParam(value = "dimensionValue", required = false) String dimensionValue, - @ApiParam(value = "统计日期(yyyy-MM-dd)", required = false) - @RequestParam(value = "dateStr", required = false) String dateStr - ); - @ApiOperation(value = "触发指定时间的数据统计", produces = "application/json") - @PostMapping("/statistics/trigger") + @PostMapping("/service/metrics/statistics/trigger") InternalResponse triggerStatistics( @ApiParam(value = "统计日期(yyyy-MM-dd)", required = false) @RequestBody ServiceTriggerStatisticsRequest request ); - } diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceResourceQuotaResource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceResourceQuotaResource.java new file mode 100644 index 0000000000..f387edb327 --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceResourceQuotaResource.java @@ -0,0 +1,54 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.api.inner; + +import com.tencent.bk.job.common.annotation.InternalAPI; +import com.tencent.bk.job.common.model.InternalResponse; +import com.tencent.bk.job.common.resource.quota.ResourceQuotaLimit; +import com.tencent.bk.job.execute.model.inner.RunningJobQuotaUsage; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.util.Map; + +/** + * 正在执行中的作业资源配额相关 API + */ +@Api(tags = {"job-execute:service:RunningJobResourceQuota"}) +@RestController +@InternalAPI +@RequestMapping("/service/resourceQuota") +public interface ServiceResourceQuotaResource { + @ApiOperation(value = "获取配额限制配置", produces = "application/json") + @GetMapping("/config") + InternalResponse> getResourceQuotaConfig(); + + @ApiOperation(value = "获取正在执行中的作业配额使用情况", produces = "application/json") + @GetMapping("/runningJob/quotaUsage") + InternalResponse getRunningJobQuotaUsage(); +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceStepInstanceResource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceStepInstanceResource.java new file mode 100644 index 0000000000..f750eff3ed --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceStepInstanceResource.java @@ -0,0 +1,57 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.api.inner; + +import com.tencent.bk.job.common.annotation.InternalAPI; +import com.tencent.bk.job.common.model.InternalResponse; +import com.tencent.bk.job.execute.model.inner.ServiceStepInstanceDTO; +import com.tentent.bk.job.common.api.feign.annotation.SmartFeignClient; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiParam; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestHeader; + +/** + * 步骤实例API-服务内部调用 + */ +@Api(tags = {"StepInstance"}) +@SmartFeignClient(value = "job-execute", contextId = "stepInstanceResource") +@InternalAPI +public interface ServiceStepInstanceResource { + @GetMapping("/service/app/{appId}/taskInstance/{taskInstanceId}/stepInstance/{stepInstanceId}") + InternalResponse getStepInstance( + @RequestHeader("username") + String username, + @ApiParam(value = "作业平台业务ID", required = true) + @PathVariable(value = "appId") + Long appId, + @ApiParam(value = "作业实例ID", name = "taskInstanceId", required = true) + @PathVariable("taskInstanceId") + Long taskInstanceId, + @ApiParam(value = "步骤实例ID", name = "stepInstanceId", required = true) + @PathVariable("stepInstanceId") + Long stepInstanceId); +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceTaskEvictPolicyResource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceTaskEvictPolicyResource.java index 5a2ace9ae6..4147b773f8 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceTaskEvictPolicyResource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceTaskEvictPolicyResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -33,17 +33,15 @@ import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PutMapping; import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @Api(tags = {"job-execute:service:TaskEvictPolicys"}) -@RequestMapping("/service/task-evict-policys") @RestController @InternalAPI public interface ServiceTaskEvictPolicyResource { @ApiOperation(value = "获取当前使用的任务驱逐策略", produces = "application/json") - @GetMapping("/current") + @GetMapping("/service/task-evict-policys/current") InternalResponse getCurrentPolicy(); @ApiOperation(value = "设置任务驱逐策略，参考值：{\"@type\":\"ComposedTaskEvictPolicy\",\"operator\":\"OR\"," + @@ -51,11 +49,11 @@ public interface ServiceTaskEvictPolicyResource { "{\"@type\":\"AppIdTaskEvictPolicy\",\"appIdsToEvict\":[2,3]},{\"@type\":\"AppCodeTaskEvictPolicy\"," + "\"appCodesToEvict\":[\"appCode1\",\"appCode2\"]}]}", produces = "application/json") - @PutMapping("/") + @PutMapping("/service/task-evict-policys") InternalResponse setPolicy(@RequestBody ComposedTaskEvictPolicyDTO policyDTO); @ApiOperation(value = "清除所有任务驱逐策略", produces = "application/json") - @DeleteMapping("/clear") + @DeleteMapping("/service/task-evict-policys/clear") InternalResponse clearPolicy(); } diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceTaskExecuteResultResource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceTaskExecuteResultResource.java index 9c048b5a0c..188a0538c6 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceTaskExecuteResultResource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceTaskExecuteResultResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -30,6 +30,7 @@ import com.tencent.bk.job.execute.model.inner.ServiceCronTaskExecuteResultStatistics; import com.tencent.bk.job.execute.model.inner.ServiceTaskInstanceDTO; import com.tencent.bk.job.execute.model.inner.request.ServiceGetCronTaskExecuteStatisticsRequest; +import com.tentent.bk.job.common.api.feign.annotation.SmartFeignClient; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; @@ -37,9 +38,7 @@ import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; import java.util.Map; @@ -47,8 +46,7 @@ * 作业执行结果API-服务内部调用 */ @Api(tags = {"job-execute:service:Task_Execution_Result"}) -@RequestMapping("/service/execution") -@RestController +@SmartFeignClient(value = "job-execute", contextId = "taskExecuteResultResource") @InternalAPI public interface ServiceTaskExecuteResultResource { /** @@ -56,12 +54,12 @@ public interface ServiceTaskExecuteResultResource { * @return Map<定时任务ID, 统计信息> */ @ApiOperation(value = "获取定时作业执行结果统计", produces = "application/json") - @PostMapping("/task-execution-history/execute-result-statistics/cron") + @PostMapping("/service/execution/task-execution-history/execute-result-statistics/cron") InternalResponse> getCronTaskExecuteResultStatistics( @ApiParam("获取定时作业执行结果统计") @RequestBody ServiceGetCronTaskExecuteStatisticsRequest request); @ApiOperation(value = "获取作业执行历史列表", produces = "application/json") - @GetMapping("/app/{appId}/task-execution-history/list") + @GetMapping("/service/execution/app/{appId}/task-execution-history/list") InternalResponse> getTaskExecuteResult( @ApiParam(value = "业务ID", required = true, example = "1") @PathVariable("appId") Long appId, @ApiParam(value = "任务名称", name = "taskName", required = false) @RequestParam(value = "taskName", diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/op/SwitchableTopoPathResource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/op/SwitchableTopoPathResource.java new file mode 100644 index 0000000000..7719ff8137 --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/op/SwitchableTopoPathResource.java @@ -0,0 +1,52 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.api.op; + +import com.tencent.bk.job.common.model.Response; +import com.tencent.bk.job.execute.model.op.SwitchStatusReq; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; +import java.util.Map; + + +@Api(tags = {"job-execute:OP:主机拓扑路径服务操作接口"}) +@RequestMapping("/op/topoPath") +@RestController +public interface SwitchableTopoPathResource { + + @ApiOperation(value = "根据hostId获取主机拓扑路径信息", produces = "application/json") + @PostMapping("/batchGet") + Response>> getTopoPathByHostIds(@RequestBody List hostIdList); + + @ApiOperation(value = "切换主机拓扑路径服务状态", produces = "application/json") + @PostMapping("/switch") + Response switchStatus(@RequestBody SwitchStatusReq req); +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebDangerousRecordResource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebDangerousRecordResource.java index 68280518cc..995bcad3c2 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebDangerousRecordResource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebDangerousRecordResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebExecuteTaskResource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebExecuteTaskResource.java index 41bbb25ebe..b418a0816b 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebExecuteTaskResource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebExecuteTaskResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,9 @@ package com.tencent.bk.job.execute.api.web; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; import com.tencent.bk.job.common.annotation.WebAPI; +import com.tencent.bk.job.common.constant.CompatibleType; import com.tencent.bk.job.common.model.Response; import com.tencent.bk.job.common.model.dto.AppResourceScope; import com.tencent.bk.job.execute.model.web.request.RedoTaskRequest; @@ -141,7 +143,33 @@ Response fastPushFile( @ApiOperation(value = "执行作业步骤操作", produces = "application/json") @PostMapping(value = {"/do-step-operation/stepInstanceId/{stepInstanceId}"}) + @Deprecated + @CompatibleImplementation(name = "dao_add_task_instance_id", deprecatedVersion = "3.11.x", + type = CompatibleType.DEPLOY, explain = "发布完成后可以删除") Response doStepOperation( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId, + @ApiParam(value = "步骤实例ID", required = true, example = "1") + @PathVariable("stepInstanceId") + Long stepInstanceId, + @ApiParam(value = "步骤实例操作请求报文", name = "operation", required = true) + @RequestBody + WebStepOperation operation + ); + + @ApiOperation(value = "执行作业步骤操作", produces = "application/json") + @PostMapping(value = {"/taskInstance/{taskInstanceId}/stepInstance/{stepInstanceId}/operate"}) + Response doStepOperationV2( @ApiParam("用户名，网关自动传入") @RequestHeader("username") String username, @@ -154,6 +182,9 @@ Response doStepOperation( @ApiParam(value = "资源范围ID", required = true) @PathVariable(value = "scopeId") String scopeId, + @ApiParam(value = "作业实例ID", required = true, example = "1") + @PathVariable("taskInstanceId") + Long taskInstanceId, @ApiParam(value = "步骤实例ID", required = true, example = "1") @PathVariable("stepInstanceId") Long stepInstanceId, diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebPermissionResource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebPermissionResource.java index 28f559df1d..3609c619fa 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebPermissionResource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebPermissionResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -40,7 +40,7 @@ import org.springframework.web.bind.annotation.RestController; @Api(tags = {"job-execute:web:Permission"}) -@RequestMapping("/web/permission/") +@RequestMapping("/web/permission/execute") @RestController @WebAPI public interface WebPermissionResource { @@ -66,7 +66,8 @@ Response checkOperationPermission( * 检查操作权限 * * @param username 用户名 - * @param appId 业务ID + * @param scopeType 资源范围类型 + * @param scopeId 资源范围ID * @param operation 操作ID * @param resourceId 资源ID * @param returnPermissionDetail 是否返回详细的权限信息 @@ -84,19 +85,7 @@ Response checkOperationPermission( @ApiParam(value = "资源范围ID", required = true) @RequestParam(value = "scopeId") String scopeId, - @ApiParam("操作ID,取值为: [script/create,script/view," + - "script/edit,script/delete,script/execute," + - "script/clone],[job_template/create,job_template/view," + - "job_template/edit,job_template/delete," + - "job_template/clone,job_template/debug]," + - "[job_plan/create,job_plan/view,job_plan/edit," + - "job_plan/delete,job_plan/execute,job_plan/sync]," + - "[account/create,account/view,account/edit," + - "account/delete],[public_script/create," + - "public_script/view,public_script/edit," + - "public_script/delete,public_script/execute]," + - "[whitelist/create,whitelist/view,whitelist/edit," + - "whitelist/delete],[tag/create,tag/edit,tag/delete]") + @ApiParam("操作ID,取值为: [task_instance/view,task_instance/redo]") @RequestParam(value = "operation") String operation, @ApiParam(value = "资源ID,比如作业ID,定时任务ID;对于部分不需要资源ID的操作(新建),不需要传参") diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebSearchToolsResource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebSearchToolsResource.java new file mode 100644 index 0000000000..b022dc6a88 --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebSearchToolsResource.java @@ -0,0 +1,70 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.api.web; + +import com.tencent.bk.job.common.annotation.WebAPI; +import com.tencent.bk.job.common.model.Response; +import com.tencent.bk.job.execute.model.web.vo.TaskLinkVO; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; + +/** + * 自助查询API-前端调用 + */ +@Api(tags = {"job-execute:web:Search_Tools"}) +@RequestMapping("/web/tools") +@RestController +@WebAPI +public interface WebSearchToolsResource { + @ApiOperation(value = "查询任务链接", produces = "application/json") + @GetMapping(value = {"/queryJobInstance/gseTaskIds/{gseTaskId}"}) + Response getTaskLink( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiParam(value = "GSE任务ID", required = true) + @PathVariable(value = "gseTaskId") + String gseTaskId + ); + + @ApiOperation(value = "根据作业步骤ID获取任务链接", produces = "application/json") + @GetMapping(value = {"/queryJobInstance/stepInstanceIds/{stepInstanceId}"}) + Response> getTaskLinkByStepId( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiParam(value = "任务步骤ID", required = true) + @PathVariable(value = "stepInstanceId") + Long stepInstanceId + ); +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskExecutionResultResource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskExecutionResultResource.java index 35948b606c..bdc21f89b3 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskExecutionResultResource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskExecutionResultResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,16 +24,21 @@ package com.tencent.bk.job.execute.api.web; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; import com.tencent.bk.job.common.annotation.WebAPI; +import com.tencent.bk.job.common.constant.CompatibleType; import com.tencent.bk.job.common.model.PageData; import com.tencent.bk.job.common.model.Response; import com.tencent.bk.job.common.model.dto.AppResourceScope; import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.execute.common.constants.TaskTotalTimeTypeEnum; +import com.tencent.bk.job.execute.model.web.vo.ExecuteObjectFileLogVO; +import com.tencent.bk.job.execute.model.web.vo.ExecuteObjectScriptLogVO; +import com.tencent.bk.job.execute.model.web.vo.ExecuteObjectVO; import com.tencent.bk.job.execute.model.web.vo.ExecuteVariableVO; -import com.tencent.bk.job.execute.model.web.vo.FileDistributionDetailVO; -import com.tencent.bk.job.execute.model.web.vo.IpFileLogContentVO; +import com.tencent.bk.job.execute.model.web.vo.FileDistributionDetailV2VO; import com.tencent.bk.job.execute.model.web.vo.IpScriptLogContentVO; +import com.tencent.bk.job.execute.model.web.vo.StepExecutionDetailV2VO; import com.tencent.bk.job.execute.model.web.vo.StepExecutionDetailVO; import com.tencent.bk.job.execute.model.web.vo.StepExecutionRecordVO; import com.tencent.bk.job.execute.model.web.vo.TaskExecuteResultVO; @@ -52,6 +57,7 @@ import org.springframework.web.bind.annotation.RestController; import springfox.documentation.annotations.ApiIgnore; +import javax.ws.rs.QueryParam; import java.util.List; /** @@ -104,12 +110,15 @@ Response> getTaskHistoryList( @ApiParam(value = "耗时类型", name = "totalTimeType") @RequestParam(value = "totalTimeType", required = false) TaskTotalTimeTypeEnum totalTimeType, - @ApiParam(value = "分页-开始") + @ApiParam(value = "分页-开始，默认值：0") @RequestParam(value = "start", required = false) Integer start, - @ApiParam(value = "分页-每页大小") + @ApiParam(value = "分页-每页大小，默认值：10") @RequestParam(value = "pageSize", required = false) Integer pageSize, + @ApiParam(value = "分页-是否计算总数；默认值：true。计算总数可能会影响 API 性能, 必要场景才可使用") + @RequestParam(value = "countPageTotal", required = false) + Boolean countPageTotal, @ApiParam(value = "定时任务ID") @RequestParam(value = "cronTaskId", required = false) Long cronTaskId, @@ -140,11 +149,14 @@ Response getTaskExecutionResult( Long taskInstanceId ); - @ApiOperation(value = "获取作业步骤执行信息", produces = "application/json") + @ApiOperation(value = "获取作业步骤执行信息（废弃）", produces = "application/json") @GetMapping(value = { "/step-execution-result/{stepInstanceId}/{executeCount}", "/step-execution-result/{stepInstanceId}" }) + @Deprecated + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.DEPLOY, + explain = "发布完成后可以删除") Response getStepExecutionResult( @ApiParam("用户名，网关自动传入") @RequestHeader("username") @@ -188,9 +200,11 @@ Response getStepExecutionResult( Integer order ); - @ApiOperation(value = "获取快速作业的步骤执行信息", produces = "application/json") - @GetMapping(value = {"/step-execution-result/taskInstanceId/{taskInstanceId}"}) - Response getFastTaskStepExecutionResult( + @ApiOperation(value = "获取作业步骤执行信息", produces = "application/json") + @GetMapping(value = { + "/taskInstance/{taskInstanceId}/stepInstance/{stepInstanceId}/stepExecutionResult" + }) + Response getStepExecutionResult( @ApiParam("用户名，网关自动传入") @RequestHeader("username") String username, @@ -203,9 +217,15 @@ Response getFastTaskStepExecutionResult( @ApiParam(value = "资源范围ID", required = true) @PathVariable(value = "scopeId") String scopeId, - @ApiParam(value = "任务实例ID", name = "taskInstanceId", required = true) - @PathVariable("taskInstanceId") Long taskInstanceId, - @ApiParam(value = "滚动执行批次", name = "batch") + @ApiParam(value = "作业实例ID", name = "taskInstanceId", required = true) + @PathVariable("taskInstanceId") + Long taskInstanceId, + @ApiParam(value = "步骤实例ID", name = "stepInstanceId", required = true) + @PathVariable("stepInstanceId") + Long stepInstanceId, + @ApiParam(value = "执行次数,如果不传表示获取最近一次的执行结果", name = "executeCount") + @RequestParam(value = "executeCount", required = false) Integer executeCount, + @ApiParam(value = "滚动执行批次。如果不传表示返回最新批次，传入0表示返回全部批次", name = "batch") @RequestParam(value = "batch", required = false) Integer batch, @ApiParam(value = "任务执行结果", name = "resultType") @RequestParam(value = "resultType", required = false) @@ -213,20 +233,29 @@ Response getFastTaskStepExecutionResult( @ApiParam(value = "用户脚本输出的结果分组tag", name = "tag") @RequestParam(value = "tag", required = false) String tag, - @ApiParam(value = "结果分组下返回的ip最大数", name = "maxIpsPerResultGroup") - @RequestParam(value = "maxIpsPerResultGroup", required = false) - Integer maxIpsPerResultGroup, + @ApiParam(value = "结果分组下返回的最大任务数", name = "maxExecuteObjectPerResultGroup") + @RequestParam(value = "maxTasksPerResultGroup", required = false) + Integer maxTasksPerResultGroup, + @ApiParam(value = "日志搜索关键词", name = "keyword", required = true) + @RequestParam(value = "keyword", required = false) + String keyword, + @ApiParam(value = "过滤ip,支持模糊匹配", name = "searchIp") + @RequestParam(value = "searchIp", required = false) + String searchIp, @ApiParam(value = "排序字段，当前支持totalTime|cloudAreaId|exitCode", name = "orderField") @RequestParam(value = "orderField", required = false) String orderField, - @ApiParam(value = "排序顺序,0:降序;1:升序", name = "searchIp") + @ApiParam(value = "排序顺序,0:降序;1:升序", name = "order") @RequestParam(value = "order", required = false) Integer order ); - @ApiOperation(value = "获取IP对应的脚本日志内容", produces = "application/json") - @GetMapping(value = {"/step-execution-result/log-content/{stepInstanceId}/{executeCount}/{ip}"}) - Response getScriptLogContentByIp( + @ApiOperation(value = "获取主机对应的脚本日志内容（废弃）", produces = "application/json") + @GetMapping(value = {"/step-execution-result/log-content/{stepInstanceId}/{executeCount}/host/{hostId}"}) + @Deprecated + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.DEPLOY, + explain = "发布完成后可以删除") + Response getScriptLogContentByHost( @ApiParam("用户名，网关自动传入") @RequestHeader("username") String username, @@ -245,15 +274,16 @@ Response getScriptLogContentByIp( @ApiParam(value = "执行次数，首次传0", name = "executeCount", required = true) @PathVariable("executeCount") Integer executeCount, - @ApiParam(value = "IP，格式为云区域ID:IP,比如1:10.10.10.10", name = "ip", required = true) - @PathVariable("ip") String ip, + @ApiParam(value = "主机ID,优先级比ip参数高", name = "hostId") + @PathVariable(value = "hostId", required = false) Long hostId, @ApiParam(value = "滚动批次，非滚动步骤不需要传入", name = "batch") @RequestParam(value = "batch", required = false) Integer batch ); - @ApiOperation(value = "获取文件分发步骤IP对应的日志", produces = "application/json") - @GetMapping(value = {"/step-execution-result/log-content/file/{stepInstanceId}/{executeCount}/{ip}"}) - Response getFileLogContentByIp( + @ApiOperation(value = "获取执行对象对应的脚本日志内容", produces = "application/json") + @GetMapping(value = {"/taskInstance/{taskInstanceId}/stepInstance/{stepInstanceId}/executeObject" + + "/{executeObjectType}/{executeObjectResourceId}/scriptLog"}) + Response getScriptLogContentByExecuteObject( @ApiParam("用户名，网关自动传入") @RequestHeader("username") String username, @@ -266,26 +296,68 @@ Response getFileLogContentByIp( @ApiParam(value = "资源范围ID", required = true) @PathVariable(value = "scopeId") String scopeId, + @ApiParam(value = "作业实例ID", name = "taskInstanceId", required = true) + @PathVariable("taskInstanceId") + Long taskInstanceId, @ApiParam(value = "步骤实例ID", name = "stepInstanceId", required = true) @PathVariable("stepInstanceId") Long stepInstanceId, - @ApiParam(value = "执行次数，首次传0", name = "executeCount", required = true) - @PathVariable("executeCount") + @ApiParam(value = "执行对象类型", name = "executeObjectType", required = true) + @PathVariable(value = "executeObjectType") + Integer executeObjectType, + @ApiParam(value = "执行对象资源 ID", name = "executeObjectType", required = true) + @PathVariable(value = "executeObjectResourceId") + Long executeObjectResourceId, + @ApiParam(value = "执行次数,如果不传表示获取最近一次的执行结果", name = "executeCount") + @RequestParam(value = "executeCount", required = false) Integer executeCount, - @ApiParam(value = "IP，格式为云区域ID:IP,比如1:10.10.10.10", name = "ip", required = true) - @PathVariable("ip") - String ip, - @ApiParam(value = "文件任务上传下载标识,upload-上传,download-下载", name = "mode", required = true) - @RequestParam(value = "mode") - String mode, @ApiParam(value = "滚动批次，非滚动步骤不需要传入", name = "batch") @RequestParam(value = "batch", required = false) Integer batch ); + @ApiOperation(value = "获取文件分发步骤执行对象对应的日志", produces = "application/json") + @GetMapping(value = {"/taskInstance/{taskInstanceId}/stepInstance/{stepInstanceId}/executeObject" + + "/{executeObjectType}/{executeObjectResourceId}/fileLog"}) + Response getFileLogContentByExecuteObject( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId, + @ApiParam(value = "作业实例ID", name = "taskInstanceId", required = true) + @PathVariable("taskInstanceId") + Long taskInstanceId, + @ApiParam(value = "步骤实例ID", name = "stepInstanceId", required = true) + @PathVariable("stepInstanceId") + Long stepInstanceId, + @ApiParam(value = "执行对象类型", name = "executeObjectType", required = true) + @PathVariable(value = "executeObjectType") + Integer executeObjectType, + @ApiParam(value = "执行对象资源 ID", name = "executeObjectType", required = true) + @PathVariable(value = "executeObjectResourceId") + Long executeObjectResourceId, + @ApiParam(value = "执行次数,如果不传表示获取最近一次的执行结果", name = "executeCount") + @RequestParam(value = "executeCount", required = false) + Integer executeCount, + @ApiParam(value = "滚动批次，非滚动步骤不需要传入", name = "batch") + @RequestParam(value = "batch", required = false) + Integer batch, + @ApiParam(value = "文件任务上传下载标识,0-上传,1-下载", name = "mode", required = true) + @RequestParam(value = "mode") + Integer mode + ); + @ApiOperation(value = "获取文件分发步骤文件任务ID对应的执行日志", produces = "application/json") - @PostMapping(value = {"/step-execution-result/log-content/file/{stepInstanceId}/{executeCount}/query-by-ids"}) - Response> getFileLogContentByFileTaskIds( + @PostMapping(value = {"/taskInstance/{taskInstanceId}/stepInstance/{stepInstanceId}/fileLog/queryByIds"}) + Response> getFileLogContentByFileTaskIds( @ApiParam("用户名，网关自动传入") @RequestHeader("username") String username, @@ -298,11 +370,14 @@ Response> getFileLogContentByFileTaskIds( @ApiParam(value = "资源范围ID", required = true) @PathVariable(value = "scopeId") String scopeId, + @ApiParam(value = "作业实例ID", name = "taskInstanceId", required = true) + @PathVariable("taskInstanceId") + Long taskInstanceId, @ApiParam(value = "步骤实例ID", name = "stepInstanceId", required = true) @PathVariable("stepInstanceId") Long stepInstanceId, - @ApiParam(value = "执行次数，首次传0", name = "executeCount", required = true) - @PathVariable("executeCount") + @ApiParam(value = "执行次数,如果不传表示获取最近一次的执行结果", name = "executeCount") + @RequestParam(value = "executeCount", required = false) Integer executeCount, @ApiParam(value = "滚动批次，非滚动步骤不需要传入", name = "batch") @RequestParam(value = "batch", required = false) @@ -312,9 +387,12 @@ Response> getFileLogContentByFileTaskIds( List taskIds ); - @ApiOperation(value = "获取执行步骤-主机对应的变量列表", produces = "application/json") - @GetMapping(value = {"/step-execution-result/variable/{stepInstanceId}/{ip}"}) - Response> getStepVariableByIp( + @ApiOperation(value = "获取执行步骤-主机对应的变量列表（废弃）", produces = "application/json") + @GetMapping(value = {"/step-execution-result/step/{stepInstanceId}/variables"}) + @Deprecated + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.DEPLOY, + explain = "发布完成后可以删除") + Response> getStepVariableByHost( @ApiParam("用户名，网关自动传入") @RequestHeader("username") String username, @@ -330,13 +408,49 @@ Response> getStepVariableByIp( @ApiParam(value = "步骤实例ID", name = "stepInstanceId", required = true) @PathVariable("stepInstanceId") Long stepInstanceId, - @ApiParam(value = "ip", name = "ip", required = true) - @PathVariable("ip") + @ApiParam(value = "hostId", name = "主机ID") + @QueryParam(value = "hostId") + Long hostId, + @ApiParam(value = "ip", name = "云区域ID:IPv4，为了兼容历史数据的查询保留；如果返回的任务中包含ip，那么需要传入") + @QueryParam(value = "ip") String ip ); - @ApiOperation(value = "获取执行结果分组下的主机列表", produces = "application/json") + @ApiOperation(value = "获取执行步骤-执行对象对应的变量列表", produces = "application/json") + @GetMapping(value = {"/taskInstance/{taskInstanceId}/stepInstance/{stepInstanceId}/executeObject" + + "/{executeObjectType}/{executeObjectResourceId}/variables"}) + Response> getStepVariableByExecuteObject( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId, + @ApiParam(value = "作业实例ID", name = "taskInstanceId", required = true) + @PathVariable("taskInstanceId") + Long taskInstanceId, + @ApiParam(value = "步骤实例ID", name = "stepInstanceId", required = true) + @PathVariable("stepInstanceId") + Long stepInstanceId, + @ApiParam(value = "执行对象类型", name = "executeObjectType", required = true) + @PathVariable(value = "executeObjectType") + Integer executeObjectType, + @ApiParam(value = "执行对象资源 ID", name = "executeObjectType", required = true) + @PathVariable(value = "executeObjectResourceId") + Long executeObjectResourceId + ); + + @ApiOperation(value = "获取执行结果分组下的主机列表（废弃）", produces = "application/json") @GetMapping(value = {"/step-execution-result/hosts/{stepInstanceId}/{executeCount}"}) + @Deprecated + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.DEPLOY, + explain = "发布完成后可以删除") Response> getHostsByResultType( @ApiParam("用户名，网关自动传入") @RequestHeader("username") @@ -368,8 +482,47 @@ Response> getHostsByResultType( String keyword ); + @ApiOperation(value = "获取执行结果分组下的执行对象列表", produces = "application/json") + @GetMapping(value = {"/taskInstance/{taskInstanceId}/stepInstance/{stepInstanceId}/{executeCount}/executeObjects"}) + Response> getExecuteObjectsByResultType( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId, + @ApiParam(value = "作业实例ID", name = "taskInstanceId", required = true) + @PathVariable("taskInstanceId") + Long taskInstanceId, + @ApiParam(value = "步骤实例ID", name = "stepInstanceId", required = true) + @PathVariable("stepInstanceId") + Long stepInstanceId, + @ApiParam(value = "执行次数", name = "executeCount", required = true) + @PathVariable("executeCount") Integer executeCount, + @ApiParam(value = "滚动执行批次，该步骤为滚动步骤时并且用户指定了批次的场景下需要传入该参数", name = "batch") + @RequestParam(value = "batch", required = false) Integer batch, + @ApiParam(value = "任务执行结果", name = "resultType", required = true) + @RequestParam(value = "resultType") + Integer resultType, + @ApiParam(value = "用户脚本输出的结果分组tag", name = "tag") + @RequestParam(value = "tag", required = false) + String tag, + @ApiParam(value = "关键字", name = "keyword") + @RequestParam(value = "keyword", required = false) + String keyword + ); + @ApiOperation(value = "获取步骤执行历史", produces = "application/json") @GetMapping(value = {"/step-execution-history/{stepInstanceId}"}) + @Deprecated + @CompatibleImplementation(name = "dao_add_task_instance_id", deprecatedVersion = "3.11.x", + type = CompatibleType.DEPLOY, explain = "发布完成后可以删除") Response> listStepExecutionHistory( @ApiParam("用户名，网关自动传入") @RequestHeader("username") @@ -391,4 +544,31 @@ Response> listStepExecutionHistory( Integer batch ); + + @ApiOperation(value = "获取步骤执行历史", produces = "application/json") + @GetMapping(value = {"/taskInstance/{taskInstanceId}/stepInstance/{stepInstanceId}/stepExecutionHistory"}) + Response> listStepExecutionHistoryV2( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId, + @ApiParam(value = "作业实例ID", name = "taskInstanceId", required = true) + @PathVariable("taskInstanceId") + Long taskInstanceId, + @ApiParam(value = "步骤实例ID", name = "stepInstanceId", required = true) + @PathVariable("stepInstanceId") + Long stepInstanceId, + @ApiParam(value = "滚动批次，非滚动步骤不需要传入", name = "batch") + @RequestParam(value = "batch", required = false) + Integer batch + ); + } diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskInstanceResource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskInstanceResource.java index e20490a731..e04e7722f1 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskInstanceResource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskInstanceResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,9 @@ package com.tencent.bk.job.execute.api.web; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; import com.tencent.bk.job.common.annotation.WebAPI; +import com.tencent.bk.job.common.constant.CompatibleType; import com.tencent.bk.job.common.model.Response; import com.tencent.bk.job.common.model.dto.AppResourceScope; import com.tencent.bk.job.execute.model.web.vo.ExecuteStepVO; @@ -53,24 +55,51 @@ @RestController @WebAPI public interface WebTaskInstanceResource { + @Deprecated + @CompatibleImplementation(name = "dao_add_task_instance_id", deprecatedVersion = "3.11.x", + type = CompatibleType.DEPLOY, explain = "发布完成后可以删除") @ApiOperation(value = "获取作业步骤实例详情", produces = "application/json") @GetMapping(value = {"/scope/{scopeType}/{scopeId}/task-instance/step_instance/{stepInstanceId}"}) Response getStepInstanceDetail( @ApiParam("用户名，网关自动传入") @RequestHeader("username") - String username, + String username, @ApiIgnore @RequestAttribute(value = "appResourceScope") - AppResourceScope appResourceScope, + AppResourceScope appResourceScope, @ApiParam(value = "资源范围类型", required = true) @PathVariable(value = "scopeType") - String scopeType, + String scopeType, @ApiParam(value = "资源范围ID", required = true) @PathVariable(value = "scopeId") - String scopeId, + String scopeId, @ApiParam(value = "步骤实例ID", name = "stepInstanceId", required = true) @PathVariable("stepInstanceId") - Long stepInstanceId + Long stepInstanceId + ); + + @ApiOperation(value = "获取作业步骤实例详情", produces = "application/json") + @GetMapping(value = { + "/scope/{scopeType}/{scopeId}/taskInstance/{taskInstanceId}/stepInstance/{stepInstanceId}/detail"}) + Response getStepInstanceDetailV2( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId, + @ApiParam(value = "作业实例ID", name = "taskInstanceId", required = true) + @PathVariable("taskInstanceId") + Long taskInstanceId, + @ApiParam(value = "步骤实例ID", name = "stepInstanceId", required = true) + @PathVariable("stepInstanceId") + Long stepInstanceId ); @ApiOperation(value = "获取作业实例全局参数", produces = "application/json") @@ -78,19 +107,19 @@ Response getStepInstanceDetail( Response> getTaskInstanceVariables( @ApiParam("用户名，网关自动传入") @RequestHeader("username") - String username, + String username, @ApiIgnore @RequestAttribute(value = "appResourceScope") - AppResourceScope appResourceScope, + AppResourceScope appResourceScope, @ApiParam(value = "资源范围类型", required = true) @PathVariable(value = "scopeType") - String scopeType, + String scopeType, @ApiParam(value = "资源范围ID", required = true) @PathVariable(value = "scopeId") - String scopeId, + String scopeId, @ApiParam(value = "作业实例ID", name = "taskInstanceId", required = true) @PathVariable("taskInstanceId") - Long taskInstanceId + Long taskInstanceId ); @ApiOperation(value = "获取作业操作日志", produces = "application/json") @@ -98,19 +127,19 @@ Response> getTaskInstanceVariables( Response> getTaskInstanceOperationLog( @ApiParam("用户名，网关自动传入") @RequestHeader("username") - String username, + String username, @ApiIgnore @RequestAttribute(value = "appResourceScope") - AppResourceScope appResourceScope, + AppResourceScope appResourceScope, @ApiParam(value = "资源范围类型", required = true) @PathVariable(value = "scopeType") - String scopeType, + String scopeType, @ApiParam(value = "资源范围ID", required = true) @PathVariable(value = "scopeId") - String scopeId, + String scopeId, @ApiParam(value = "作业实例ID", name = "taskInstanceId", required = true) @PathVariable("taskInstanceId") - Long taskInstanceId + Long taskInstanceId ); @ApiOperation(value = "获取作业实例详情,包括步骤列表和全局变量列表", produces = "application/json") @@ -118,19 +147,19 @@ Response> getTaskInstanceOperationLog( Response getTaskInstanceDetail( @ApiParam("用户名，网关自动传入") @RequestHeader("username") - String username, + String username, @ApiIgnore @RequestAttribute(value = "appResourceScope") - AppResourceScope appResourceScope, + AppResourceScope appResourceScope, @ApiParam(value = "资源范围类型", required = true) @PathVariable(value = "scopeType") - String scopeType, + String scopeType, @ApiParam(value = "资源范围ID", required = true) @PathVariable(value = "scopeId") - String scopeId, + String scopeId, @ApiParam(value = "作业实例ID", name = "taskInstanceId", required = true) @PathVariable("taskInstanceId") - Long taskInstanceId + Long taskInstanceId ); @ApiOperation(value = "获取作业实例基本信息", produces = "application/json") @@ -140,6 +169,6 @@ Response getTaskInstanceBasic( @RequestHeader("username") String username, @ApiParam(value = "作业实例ID", name = "taskInstanceId", required = true) @PathVariable("taskInstanceId") - Long taskInstanceId + Long taskInstanceId ); } diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskLogResource.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskLogResource.java index 6645b97289..4cc97740f4 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskLogResource.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/WebTaskLogResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -48,13 +48,13 @@ * 作业执行日志API-前端调用 */ @Api(tags = {"job-execute:web:Task_Execution_Log"}) -@RequestMapping("/web/execution") +@RequestMapping("/web/execution/scope/{scopeType}/{scopeId}") @RestController @WebAPI public interface WebTaskLogResource { @ApiOperation(value = "请求下载执行日志文件", produces = "application/json") - @GetMapping("/scope/{scopeType}/{scopeId}/step-execution-result/{stepInstanceId}/log-file") + @GetMapping("/taskInstance/{taskInstanceId}/stepInstance/{stepInstanceId}/requestDownloadLogFile") Response requestDownloadLogFile( @ApiParam("用户名，网关自动传入") @RequestHeader("username") @@ -68,22 +68,25 @@ Response requestDownloadLogFile( @ApiParam(value = "资源范围ID", required = true) @PathVariable(value = "scopeId") String scopeId, + @ApiParam(value = "作业实例 ID", name = "taskInstanceId", required = true) + @PathVariable("taskInstanceId") + Long taskInstanceId, @ApiParam(value = "步骤实例 ID", name = "stepInstanceId", required = true) @PathVariable("stepInstanceId") Long stepInstanceId, - @ApiParam(value = "hostId", name = "hostId") - @RequestParam(value = "hostId", required = false) - Long hostId, - @ApiParam(value = "ip", name = "ip") - @RequestParam(value = "ip", required = false) - String cloudIp, + @ApiParam(value = "执行对象类型", name = "executeObjectType") + @RequestParam(value = "executeObjectType", required = false) + Integer executeObjectType, + @ApiParam(value = "执行对象资源 ID", name = "executeObjectResourceId") + @RequestParam(value = "executeObjectResourceId", required = false) + Long executeObjectResourceId, @ApiParam(value = "重新打包", name = "repackage") @RequestParam(value = "repackage", required = false) Boolean repackage ); @ApiOperation(value = "下载执行日志文件", produces = "application/json") - @GetMapping("/scope/{scopeType}/{scopeId}/step-execution-result/{stepInstanceId}/log-file/download") + @GetMapping("/taskInstance/{taskInstanceId}/stepInstance/{stepInstanceId}/downloadLogFile") ResponseEntity downloadLogFile( HttpServletResponse response, @ApiParam("用户名，网关自动传入") @@ -98,14 +101,17 @@ ResponseEntity downloadLogFile( @ApiParam(value = "资源范围ID", required = true) @PathVariable(value = "scopeId") String scopeId, + @ApiParam(value = "作业实例 ID", name = "taskInstanceId", required = true) + @PathVariable("taskInstanceId") + Long taskInstanceId, @ApiParam(value = "步骤实例 ID", name = "stepInstanceId", required = true) @PathVariable("stepInstanceId") Long stepInstanceId, - @ApiParam(value = "hostId", name = "hostId") - @RequestParam(value = "hostId", required = false) - Long hostId, - @ApiParam(value = "ip", name = "ip") - @RequestParam(value = "ip", required = false) - String cloudIp + @ApiParam(value = "执行对象类型", name = "executeObjectType") + @RequestParam(value = "executeObjectType", required = false) + Integer executeObjectType, + @ApiParam(value = "执行对象资源 ID", name = "executeObjectResourceId") + @RequestParam(value = "executeObjectResourceId", required = false) + Long executeObjectResourceId ); } diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/AppStatisticTypeEnum.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/AppStatisticTypeEnum.java index 708532ff22..4e33d86d25 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/AppStatisticTypeEnum.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/AppStatisticTypeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/FileDistStatusEnum.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/FileDistStatusEnum.java index dd0ce93780..7b3943cf60 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/FileDistStatusEnum.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/FileDistStatusEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,12 +25,15 @@ package com.tencent.bk.job.execute.common.constants; /** - * 文件分发状态 + * 单个文件分发对（源执行对象，源文件路径，目标执行对象，目标文件路径）的文件分发状态，体现在详情页文件分发日志中 */ public enum FileDistStatusEnum { - PULLING(0, "Pulling from third file source"), WAITING(1, "Waiting"), UPLOADING(2, "Uploading"), DOWNLOADING(3, - "Downloading"), - FINISHED(4, "Finished"), FAILED(5, "Failed"); + PULLING(0, "Pulling from third file source"), + WAITING(1, "Waiting"), + UPLOADING(2, "Uploading"), + DOWNLOADING(3, "Downloading"), + FINISHED(4, "Finished"), + FAILED(5, "Failed"); private final Integer value; private final String name; @@ -42,14 +45,14 @@ public enum FileDistStatusEnum { public static FileDistStatusEnum getFileDistStatus(Integer status) { if (status == null) { - return null; + throw new IllegalArgumentException("Null FileDistStatusEnum constant"); } for (FileDistStatusEnum fileDistStatus : values()) { if (fileDistStatus.getValue().equals(status)) { return fileDistStatus; } } - return null; + throw new IllegalArgumentException("No FileDistStatusEnum constant: " + status); } public Integer getValue() { @@ -59,4 +62,12 @@ public Integer getValue() { public String getName() { return name; } + + public static boolean isFinishedStatus(FileDistStatusEnum status) { + return status == FINISHED || status == FAILED; + } + + public static boolean isFinishedStatus(Integer statusValue) { + return statusValue.equals(FINISHED.value) || statusValue.equals(FAILED.value); + } } diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/FileTransferModeEnum.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/FileTransferModeEnum.java index 89b1482f2e..90576a0e3d 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/FileTransferModeEnum.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/FileTransferModeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/HostSystemTypeEnum.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/HostSystemTypeEnum.java index cad69ff6a5..a5fdcddf9b 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/HostSystemTypeEnum.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/HostSystemTypeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/RunStatusEnum.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/RunStatusEnum.java index 954aa2ee5e..3cf024c2f7 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/RunStatusEnum.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/RunStatusEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,7 +28,7 @@ import java.util.Set; /** - * 作业执行状态 + * 任务与步骤的执行状态 */ public enum RunStatusEnum { /** diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/StepExecuteTypeEnum.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/StepExecuteTypeEnum.java index 5b7992bf85..3d1bea8cad 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/StepExecuteTypeEnum.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/StepExecuteTypeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -41,7 +41,16 @@ public enum StepExecuteTypeEnum { this.name = name; } - public static StepExecuteTypeEnum valueOf(int type) { + public static StepExecuteTypeEnum valOf(int type) { + for (StepExecuteTypeEnum stepType : values()) { + if (stepType.getValue() == type) { + return stepType; + } + } + throw new IllegalArgumentException("StepExecuteTypeEnum:" + type); + } + + public static StepExecuteTypeEnum valOf(byte type) { for (StepExecuteTypeEnum stepType : values()) { if (stepType.getValue() == type) { return stepType; diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/StepRunModeEnum.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/StepRunModeEnum.java index 582beb124d..7190425f13 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/StepRunModeEnum.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/StepRunModeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/TaskStartupModeEnum.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/TaskStartupModeEnum.java index e06c2461ca..55d511c226 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/TaskStartupModeEnum.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/TaskStartupModeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,7 +28,7 @@ * 作业启动方式 */ public enum TaskStartupModeEnum { - NORMAL(1, "页面执行"), API(2, "API调用"), CRON(3, "定时执行"); + WEB(1, "web"), API(2, "api"), CRON(3, "cron"); private final int value; private final String name; @@ -56,7 +56,7 @@ public String getName() { } public String getI18nKey() { - if (this == NORMAL) { + if (this == WEB) { return "task.startup.mode.normal"; } else if (this == API) { return "task.startup.mode.api"; diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/TaskTotalTimeTypeEnum.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/TaskTotalTimeTypeEnum.java index 04ae4d6dc4..60d61f6cde 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/TaskTotalTimeTypeEnum.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/TaskTotalTimeTypeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/TaskTypeEnum.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/TaskTypeEnum.java index 87dca97a2f..704293d0d6 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/TaskTypeEnum.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/common/constants/TaskTypeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/gse/EsbGseTaskResultDTO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/gse/EsbGseTaskResultDTO.java index ee517c6986..b469f0f093 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/gse/EsbGseTaskResultDTO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/gse/EsbGseTaskResultDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/gse/EsbProcessInfoDTO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/gse/EsbProcessInfoDTO.java index 9b5f864396..11cb004839 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/gse/EsbProcessInfoDTO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/gse/EsbProcessInfoDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/gse/req/EsbGseGetProcResultRequest.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/gse/req/EsbGseGetProcResultRequest.java index 14109966b1..cc3d103ac5 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/gse/req/EsbGseGetProcResultRequest.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/gse/req/EsbGseGetProcResultRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/gse/req/EsbGseManageProcRequest.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/gse/req/EsbGseManageProcRequest.java index e17ea4a266..7247342295 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/gse/req/EsbGseManageProcRequest.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/gse/req/EsbGseManageProcRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/gse/req/EsbGseOperateProcessRequest.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/gse/req/EsbGseOperateProcessRequest.java index 6c53b0973e..07f998703b 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/gse/req/EsbGseOperateProcessRequest.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/gse/req/EsbGseOperateProcessRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/EsbIpStatusDTO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/EsbIpStatusDTO.java index acba2f6a1a..e8b03fe111 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/EsbIpStatusDTO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/EsbIpStatusDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/EsbJobExecuteDTO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/EsbJobExecuteDTO.java index 0b2ddfa319..8ebb2263f1 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/EsbJobExecuteDTO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/EsbJobExecuteDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/EsbJobInstanceStatusDTO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/EsbJobInstanceStatusDTO.java index 855a3491a6..608700c83c 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/EsbJobInstanceStatusDTO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/EsbJobInstanceStatusDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/EsbStepInstanceResultAndLog.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/EsbStepInstanceResultAndLog.java index 66d1490d5c..23763af6ff 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/EsbStepInstanceResultAndLog.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/EsbStepInstanceResultAndLog.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/EsbStepInstanceStatusDTO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/EsbStepInstanceStatusDTO.java index e26dd7e25d..1781d72a22 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/EsbStepInstanceStatusDTO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/EsbStepInstanceStatusDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/EsbTaskInstanceGlobalVarValueDTO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/EsbTaskInstanceGlobalVarValueDTO.java index fe1524dd4c..6ebad6c760 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/EsbTaskInstanceGlobalVarValueDTO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/EsbTaskInstanceGlobalVarValueDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbExecuteJobRequest.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbExecuteJobRequest.java index 5f102b584a..eaf26994c3 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbExecuteJobRequest.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbExecuteJobRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbFastExecuteSQLRequest.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbFastExecuteSQLRequest.java index 285490e7d9..748ea6528e 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbFastExecuteSQLRequest.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbFastExecuteSQLRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -61,16 +61,16 @@ public class EsbFastExecuteSQLRequest extends EsbAppScopeReq { private Long dbAccountId; /** - * 脚本ID + * 脚本版本ID */ @JsonProperty("script_id") - private Long scriptId; + private Long scriptVersionId; /** * 执行超时时间,单位秒 */ @JsonProperty("script_timeout") - @Range(min = JobConstants.MIN_JOB_TIMEOUT_SECONDS, max= JobConstants.MAX_JOB_TIMEOUT_SECONDS, + @Range(min = JobConstants.MIN_JOB_TIMEOUT_SECONDS, max = JobConstants.MAX_JOB_TIMEOUT_SECONDS, message = "{validation.constraints.InvalidJobTimeout_outOfRange.message}") private Integer timeout; diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbFastExecuteScriptRequest.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbFastExecuteScriptRequest.java index dd71dc149b..20be2fcd01 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbFastExecuteScriptRequest.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbFastExecuteScriptRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,9 +26,11 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.tencent.bk.job.common.constant.JobConstants; +import com.tencent.bk.job.common.constant.MySQLTextDataType; import com.tencent.bk.job.common.esb.model.EsbAppScopeReq; import com.tencent.bk.job.common.esb.model.job.EsbIpDTO; import com.tencent.bk.job.common.esb.model.job.EsbServerDTO; +import com.tencent.bk.job.common.validation.NotExceedMySQLTextFieldLength; import lombok.Getter; import lombok.Setter; import org.hibernate.validator.constraints.Range; @@ -52,6 +54,11 @@ public class EsbFastExecuteScriptRequest extends EsbAppScopeReq { * "脚本内容，BASE64编码 */ @JsonProperty("script_content") + @NotExceedMySQLTextFieldLength( + fieldName = "script_content", + fieldType = MySQLTextDataType.MEDIUMTEXT, + base64 = true + ) private String content; /** @@ -69,13 +76,18 @@ public class EsbFastExecuteScriptRequest extends EsbAppScopeReq { * 脚本参数， BASE64编码 */ @JsonProperty("script_param") + @NotExceedMySQLTextFieldLength( + fieldName = "script_param", + fieldType = MySQLTextDataType.TEXT, + base64 = true + ) private String scriptParam; /** - * 脚本ID + * 脚本版本ID */ @JsonProperty("script_id") - private Long scriptId; + private Long scriptVersionId; /** * 是否敏感参数 @@ -87,7 +99,7 @@ public class EsbFastExecuteScriptRequest extends EsbAppScopeReq { * 执行超时时间,单位秒 */ @JsonProperty("script_timeout") - @Range(min = JobConstants.MIN_JOB_TIMEOUT_SECONDS, max= JobConstants.MAX_JOB_TIMEOUT_SECONDS, + @Range(min = JobConstants.MIN_JOB_TIMEOUT_SECONDS, max = JobConstants.MAX_JOB_TIMEOUT_SECONDS, message = "{validation.constraints.InvalidJobTimeout_outOfRange.message}") private Integer timeout; diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbFastPushFileRequest.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbFastPushFileRequest.java index 1b43a4c159..42d9009599 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbFastPushFileRequest.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbFastPushFileRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbGetJobInstanceGlobalVarValueRequest.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbGetJobInstanceGlobalVarValueRequest.java index 3eb0610222..1af68b979d 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbGetJobInstanceGlobalVarValueRequest.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbGetJobInstanceGlobalVarValueRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbGetJobInstanceLogRequest.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbGetJobInstanceLogRequest.java index 11f61e8129..ce83a8db47 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbGetJobInstanceLogRequest.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbGetJobInstanceLogRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbGetJobInstanceStatusRequest.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbGetJobInstanceStatusRequest.java index bd5a26f8da..f0948daae4 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbGetJobInstanceStatusRequest.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbGetJobInstanceStatusRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbGetStepInstanceStatusRequest.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbGetStepInstanceStatusRequest.java index 59da68c8c7..a22cf7c42b 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbGetStepInstanceStatusRequest.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbGetStepInstanceStatusRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbOperateJobInstanceRequest.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbOperateJobInstanceRequest.java index c47c8a9d23..f306d913a2 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbOperateJobInstanceRequest.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbOperateJobInstanceRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbOperateStepInstanceRequest.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbOperateStepInstanceRequest.java index 8463872800..07dd9e03f8 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbOperateStepInstanceRequest.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbOperateStepInstanceRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbPlatformExecuteScriptRequest.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbPlatformExecuteScriptRequest.java index e2c3e773c6..fa57a56b45 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbPlatformExecuteScriptRequest.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbPlatformExecuteScriptRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbPushConfigFileRequest.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbPushConfigFileRequest.java index 3b272952b3..e3198b26b9 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbPushConfigFileRequest.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v2/request/EsbPushConfigFileRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbFileIpLogV3DTO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbFileIpLogV3DTO.java index cc652a708e..6aba42eda1 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbFileIpLogV3DTO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbFileIpLogV3DTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -35,6 +35,12 @@ @Data public class EsbFileIpLogV3DTO { + /** + * 主机ID + */ + @JsonProperty("bk_host_id") + private Long hostId; + /** * 云区域ID */ diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbFileLogV3DTO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbFileLogV3DTO.java index 7f146d81f0..c55f6254c6 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbFileLogV3DTO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbFileLogV3DTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -55,6 +55,15 @@ public class EsbFileLogV3DTO { @JsonProperty("log_content") private String logContent; + @JsonProperty("size") + private String size; + + @JsonProperty("speed") + private String speed; + + @JsonProperty("process") + private String process; + } diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbIpLogListV3DTO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbIpLogListV3DTO.java index 43e68d1d80..a217bb914c 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbIpLogListV3DTO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbIpLogListV3DTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbIpLogV3DTO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbIpLogV3DTO.java index 94286c752c..65d1eda83c 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbIpLogV3DTO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbIpLogV3DTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -39,6 +39,13 @@ public class EsbIpLogV3DTO { */ @JsonProperty("log_type") private Integer logType; + + /** + * 主机ID + */ + @JsonProperty("bk_host_id") + private Long hostId; + /** * 云区域ID */ diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbIpLogsV3DTO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbIpLogsV3DTO.java index e50e81eb28..ac9b298331 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbIpLogsV3DTO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbIpLogsV3DTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -56,7 +56,7 @@ public class EsbIpLogsV3DTO { * 脚本任务日志 */ @JsonProperty("script_task_logs") - private List scriptTaskLogs; + private List scriptTaskLogs; /** * 文件任务日志 diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbJobExecuteV3DTO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbJobExecuteV3DTO.java index b85ff1c86c..e97a3c967d 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbJobExecuteV3DTO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbJobExecuteV3DTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbJobInstanceGlobalVarValueV3DTO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbJobInstanceGlobalVarValueV3DTO.java index f26f462442..cc461ae24b 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbJobInstanceGlobalVarValueV3DTO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbJobInstanceGlobalVarValueV3DTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbJobInstanceStatusV3DTO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbJobInstanceStatusV3DTO.java index c8846e536c..49704acee7 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbJobInstanceStatusV3DTO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbJobInstanceStatusV3DTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -141,6 +141,9 @@ public static class StepInst { @Setter @Getter public static class IpResult { + @JsonProperty("bk_host_id") + private Long hostId; + private String ip; @JsonProperty("bk_cloud_id") private Long cloudAreaId; diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbRollingConfigDTO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbRollingConfigDTO.java index 3894a9b396..03610a511b 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbRollingConfigDTO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbRollingConfigDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbScriptHostLogV3DTO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbScriptHostLogV3DTO.java new file mode 100644 index 0000000000..ed5043f241 --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbScriptHostLogV3DTO.java @@ -0,0 +1,65 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model.esb.v3; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +/** + * 主机对应的脚本执行日志 + */ +@Data +public class EsbScriptHostLogV3DTO { + + /** + * 主机ID + */ + @JsonProperty("host_id") + private Long hostId; + + /** + * 云区域ID + */ + @JsonProperty("bk_cloud_id") + private Long cloudAreaId; + + /** + * ipv4 + */ + @JsonProperty("ip") + private String ip; + + /** + * ipv6 + */ + @JsonProperty("ipv6") + private String ipv6; + + /** + * 脚本任务日志内容 + */ + @JsonProperty("log_content") + private String logContent; +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbScriptIpLogV3DTO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbScriptIpLogV3DTO.java deleted file mode 100644 index 624cf6d6f3..0000000000 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbScriptIpLogV3DTO.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.model.esb.v3; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -/** - * IP对应的脚本执行日志 - */ -@Data -public class EsbScriptIpLogV3DTO { - - /** - * 云区域ID - */ - @JsonProperty("bk_cloud_id") - private Long cloudAreaId; - - @JsonProperty("ip") - private String ip; - - /** - * 脚本任务日志内容 - */ - @JsonProperty("log_content") - private String logContent; -} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbStepInstanceDetailV3DTO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbStepInstanceDetailV3DTO.java new file mode 100644 index 0000000000..6ed0103bed --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbStepInstanceDetailV3DTO.java @@ -0,0 +1,290 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model.esb.v3; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.esb.model.job.v3.EsbServerV3DTO; +import lombok.Data; +import lombok.Getter; +import lombok.Setter; + +import java.util.List; + +@Data +public class EsbStepInstanceDetailV3DTO { + + /** + * 步骤实例ID + */ + private Long id; + + /** + * 步骤类型：1-脚本，2-文件，3-人工确认 + */ + @JsonProperty("type") + private Integer type; + /** + * 步骤名称 + */ + @JsonProperty("name") + private String name; + + /** + * 脚本步骤信息 + */ + @JsonProperty("script_step_info") + @JsonInclude(JsonInclude.Include.NON_NULL) + private ScriptStepInfo scriptStepInfo; + + /** + * 文件步骤信息 + */ + @JsonProperty("file_step_info") + @JsonInclude(JsonInclude.Include.NON_NULL) + private FileStepInfo fileStepInfo; + + /** + * 审批步骤信息 + */ + @JsonProperty("approval_step_info") + @JsonInclude(JsonInclude.Include.NON_NULL) + private ApprovalStepInfo approvalStepInfo; + + @Setter + @Getter + public static class ScriptStepInfo { + + /** + * 脚本类型：1-本地脚本，2-引用业务脚本，3-引用公共脚本 + */ + @JsonProperty("script_source") + private Integer scriptSource; + + /** + * 脚本 ID + */ + @JsonProperty("script_id") + private String scriptId; + + /** + * 脚本版本 ID + */ + @JsonProperty("script_version_id") + private Long scriptVersionId; + + /** + * 脚本内容 + */ + private String content; + + /** + * 脚本语言：1-shell，2-bat，3-perl，4-python，5-powershell，6-sql + */ + @JsonProperty("script_language") + private Integer scriptLanguage; + + /** + * 脚本参数 + */ + @JsonProperty("script_param") + private String scriptParam; + + /** + * 脚本超时时间，单位为秒 + */ + private Integer timeout; + + /** + * 执行账号ID + */ + @JsonProperty("account_id") + private Long accountId; + + /** + * 执行账号名称 + */ + @JsonProperty("account_name") + private String accountName; + + /** + * 执行目标机器 + */ + @JsonProperty("execute_target") + private EsbServerV3DTO executeTarget; + + /** + * 参数是否为敏感参数：0-不敏感，1-敏感 + */ + @JsonProperty("secure_param") + private Integer secureParam; + + /** + * 是否忽略错误：0-不忽略，1-忽略 + */ + @JsonProperty("ignore_error") + private Integer ignoreError; + } + + @Setter + @Getter + public static class FileStepInfo { + + /** + * 源文件列表 + */ + @JsonProperty("file_source_list") + private List fileSourceList; + /** + * 目标信息 + */ + @JsonProperty("file_destination") + private FileDestination fileDestination; + + /** + * 超时，单位为秒 + */ + private Integer timeout; + + /** + * 上传文件限速，单位为MB/s，没有值表示不限速 + */ + @JsonProperty("upload_speed_limit") + private Integer uploadSpeedLimit; + + /** + * 下载文件限速，单位为MB/s，没有值表示不限速 + */ + @JsonProperty("download_speed_limit") + private Integer downloadSpeedLimit; + + /** + * 传输模式： 1-严谨模式，2-强制模式，3-安全模式 + */ + @JsonProperty("transfer_mode") + private Integer transferMode; + + /** + * 是否忽略错误：0-不忽略，1-忽略 + */ + @JsonProperty("ignore_error") + private Integer ignoreError; + } + + @Setter + @Getter + public static class FileSource { + + /** + * 文件类型：1-服务器文件，2-本地文件，3-文件源文件 + */ + @JsonProperty("file_type") + private Integer fileType; + + /** + * 文件路径列表 + */ + @JsonProperty("file_location") + private List fileLocation; + + /** + * 文件Hash值，仅本地文件该字段有值 + */ + @JsonProperty("file_hash") + private String fileHash; + + /** + * 文件大小，单位为字节，仅本地文件该字段有值 + */ + @JsonProperty("file_size") + private Integer fileSize; + + /** + * 源文件所在机器 + */ + private EsbServerV3DTO host; + + /** + * 执行账号ID + */ + @JsonProperty("account_id") + private Long accountId; + + /** + * 执行账号名称 + */ + @JsonProperty("account_name") + private String accountName; + + /** + * 第三方文件源ID + */ + @JsonProperty("file_source_id") + private Long fileSourceId; + + /** + * 第三方文件源名称 + */ + @JsonProperty("file_source_name") + private String fileSourceName; + } + + @Setter + @Getter + public static class FileDestination { + /** + * 目标路径 + */ + private String path; + + /** + * 执行账号ID + */ + @JsonProperty("account_id") + private Long accountId; + + /** + * 执行账号名称 + */ + @JsonProperty("account_name") + private String accountName; + + /** + * 分发目标机器 + */ + @JsonProperty("target_server") + private EsbServerV3DTO targetServer; + } + + @Setter + @Getter + public static class ApprovalStepInfo { + /** + * 确认消息 + */ + @JsonProperty("approval_message") + private String approvalMessage; + } +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbStepInstanceStatusV3DTO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbStepInstanceStatusV3DTO.java new file mode 100644 index 0000000000..6ceb89d2ae --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbStepInstanceStatusV3DTO.java @@ -0,0 +1,153 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model.esb.v3; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import lombok.Getter; +import lombok.Setter; + +import java.util.List; + +@Data +public class EsbStepInstanceStatusV3DTO { + + /** + * 步骤实例id + */ + @JsonProperty("step_instance_id") + private Long id; + + /** + * 执行次数 + */ + @JsonProperty("execute_count") + private Integer executeCount; + /** + * 名称 + */ + @JsonProperty("name") + private String name; + + /** + * 步骤类型：1.脚本步骤; 2.文件步骤; 4.SQL步骤 + */ + @JsonProperty("type") + private Integer type; + /** + * 状态： 1.未执行、2.正在执行、3.执行完成且成功、4.执行失败 + */ + @JsonProperty("status") + private Integer status; + /** + * 创建时间 + */ + @JsonProperty("create_time") + private Long createTime; + /** + * 开始时间 + */ + @JsonProperty("start_time") + private Long startTime; + /** + * 结束时间 + */ + @JsonProperty("end_time") + private Long endTime; + /** + * 总耗时，单位：毫秒 + */ + @JsonProperty("total_time") + private Long totalTime; + + @JsonProperty("step_result_group_list") + @JsonInclude(JsonInclude.Include.NON_EMPTY) + private List stepResultGroupList; + + @Setter + @Getter + public static class StepResultGroup { + @JsonProperty("result_type") + private Integer resultType; + + @JsonProperty("result_type_desc") + private String resultTypeDesc; + + private String tag; + + @JsonProperty("host_size") + private Integer hostSize; + + @JsonProperty("host_result_list") + private List hostResultList; + } + + @Setter + @Getter + public static class HostResult { + @JsonProperty("bk_host_id") + private Long hostId; + + private String ip; + + private String ipv6; + + @JsonProperty("bk_agent_id") + private String agentId; + + @JsonProperty("bk_cloud_id") + private Long cloudAreaId; + + @JsonProperty("bk_cloud_name") + private String cloudAreaName; + + private Integer status; + + @JsonProperty("status_desc") + private String statusDesc; + + private String tag; + + @JsonProperty("exit_code") + private Integer exitCode; + + /** + * 开始时间 + */ + @JsonProperty("start_time") + private Long startTime; + /** + * 结束时间 + */ + @JsonProperty("end_time") + private Long endTime; + /** + * 总耗时，单位：毫秒 + */ + @JsonProperty("total_time") + private Long totalTime; + } +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbTaskInstanceV3DTO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbTaskInstanceV3DTO.java index 36a04f9e5b..b937a9f068 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbTaskInstanceV3DTO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbTaskInstanceV3DTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,6 +25,7 @@ package com.tencent.bk.job.execute.model.esb.v3; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; import com.tencent.bk.job.common.esb.model.EsbAppScopeDTO; import lombok.Getter; import lombok.Setter; @@ -38,69 +39,81 @@ public class EsbTaskInstanceV3DTO extends EsbAppScopeDTO { * id */ @JsonProperty("job_instance_id") + @JsonPropertyDescription("Job instance id") private Long id; /** * 执行方案id */ @JsonProperty("job_plan_id") + @JsonPropertyDescription("Job plan id") private Long taskId; /** * 作业模板id */ @JsonProperty("job_template_id") + @JsonPropertyDescription("Job template id") private Long templateId; /** * 名称 */ + @JsonPropertyDescription("Job instance name") private String name; /** * 执行人 */ + @JsonPropertyDescription("Operator") private String operator; /** * 启动方式，1-页面执行、2-API调用、3-定时执行 */ @JsonProperty("launch_mode") + @JsonPropertyDescription("Launch mode") private Integer startupMode; /** * 任务状态。1 - 等待执行，2 - 正在执行，3 - 执行成功，4 - 执行失败，7 - 等待确认，10 - 强制终止中，11 - 强制终止成功，13 - 确认终止 */ @JsonProperty("status") + @JsonPropertyDescription("Job instance status") private Integer status; /** * 开始时间 */ @JsonProperty("start_time") + @JsonPropertyDescription("Job instance start time") private Long startTime; /** * 开始时间 */ @JsonProperty("end_time") + @JsonPropertyDescription("Job instance end time") private Long endTime; /** * 总耗时，单位：毫秒 */ @JsonProperty("total_time") + @JsonPropertyDescription("Job instance execution cost") private Long totalTime; /** * 创建时间 */ @JsonProperty("create_time") + @JsonPropertyDescription("Job instance create time") private Long createTime; /** * 任务类型,0-作业执行,1-脚本执行,2-文件分发 */ @JsonProperty("type") + @JsonPropertyDescription("Job instance type") private Integer type; } diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbTaskLinkV3DTO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbTaskLinkV3DTO.java new file mode 100644 index 0000000000..dfc8f2c6fe --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/EsbTaskLinkV3DTO.java @@ -0,0 +1,71 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model.esb.v3; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.esb.model.EsbAppScopeDTO; +import lombok.Data; + +import java.util.List; + +@Data +public class EsbTaskLinkV3DTO extends EsbAppScopeDTO { + + /** + * 任务ID + */ + @JsonProperty("job_instance_id") + private Long jobInstanceId; + + /** + * 步骤ID + */ + @JsonProperty("step_instance_id") + private Long stepInstanceId; + + /** + * 执行次数 + */ + @JsonProperty("retry_count") + private Integer retryCount; + + /** + * 批次 + */ + @JsonProperty("batch") + private Integer batch; + + /** + * gse任务ID + */ + @JsonProperty("gse_task_id") + private String gseTaskId; + + /** + * web访问链接 + */ + @JsonProperty("link") + private List link; +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/bkci/plugin/EsbBkCIPluginBatchGetJobInstanceExecuteObjectLogRequest.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/bkci/plugin/EsbBkCIPluginBatchGetJobInstanceExecuteObjectLogRequest.java new file mode 100644 index 0000000000..faf74e1bde --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/bkci/plugin/EsbBkCIPluginBatchGetJobInstanceExecuteObjectLogRequest.java @@ -0,0 +1,71 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model.esb.v3.bkci.plugin; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.esb.model.EsbAppScopeReq; +import com.tencent.bk.job.common.model.openapi.v4.OpenApiExecuteObjectDTO; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +import javax.validation.Valid; +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; +import javax.validation.constraints.Size; +import java.util.List; + +/** + * 根据执行对象列表批量查询执行日志请求 + */ +@Getter +@Setter +@ToString +public class EsbBkCIPluginBatchGetJobInstanceExecuteObjectLogRequest extends EsbAppScopeReq { + + /** + * 作业执行实例 ID + */ + @JsonProperty("job_instance_id") + @NotNull(message = "{validation.constraints.InvalidJobInstanceId.message}") + @Min(value = 1L, message = "{validation.constraints.InvalidJobInstanceId.message}") + private Long taskInstanceId; + + /** + * 作业步骤实例ID + */ + @JsonProperty("step_instance_id") + @NotNull(message = "{validation.constraints.InvalidStepInstanceId.message}") + @Min(value = 1L, message = "{validation.constraints.InvalidStepInstanceId.message}") + private Long stepInstanceId; + + /** + * 执行对象列表 + */ + @JsonProperty("execute_object_list") + @Valid + @Size(max = 500, min = 1, message = "{validation.constraints.queryExecuteObjects_outOfRange.message}") + private List executeObjects; +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/bkci/plugin/EsbBkCIPluginFastExecuteScriptRequest.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/bkci/plugin/EsbBkCIPluginFastExecuteScriptRequest.java new file mode 100644 index 0000000000..610683853f --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/bkci/plugin/EsbBkCIPluginFastExecuteScriptRequest.java @@ -0,0 +1,169 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model.esb.v3.bkci.plugin; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.constant.JobConstants; +import com.tencent.bk.job.common.constant.MySQLTextDataType; +import com.tencent.bk.job.common.esb.model.EsbAppScopeReq; +import com.tencent.bk.job.common.model.openapi.v4.OpenApiExecuteTargetDTO; +import com.tencent.bk.job.common.validation.CheckEnum; +import com.tencent.bk.job.common.validation.EndWith; +import com.tencent.bk.job.common.validation.MaxLength; +import com.tencent.bk.job.common.validation.NotExceedMySQLTextFieldLength; +import com.tencent.bk.job.common.validation.ValidationGroups; +import com.tencent.bk.job.execute.model.esb.v3.EsbRollingConfigDTO; +import com.tencent.bk.job.execute.model.esb.v3.bkci.plugin.validator.EsbBkCIPluginFastExecuteScriptRequestGroupSequenceProvider; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; +import lombok.Getter; +import lombok.Setter; +import org.hibernate.validator.constraints.Length; +import org.hibernate.validator.constraints.Range; +import org.hibernate.validator.group.GroupSequenceProvider; + +import javax.validation.Valid; +import javax.validation.constraints.NotNull; + +/** + * 脚本执行请求 + */ +@Getter +@Setter +@GroupSequenceProvider(EsbBkCIPluginFastExecuteScriptRequestGroupSequenceProvider.class) +public class EsbBkCIPluginFastExecuteScriptRequest extends EsbAppScopeReq { + + /** + * 脚本执行任务名称 + */ + @JsonProperty("task_name") + @Length(max = 512, message = "{validation.constraints.TaskName_outOfLength.message}") + private String name; + + /** + * 脚本内容，BASE64编码 + */ + @JsonProperty("script_content") + @NotExceedMySQLTextFieldLength( + fieldName = "script_content", + fieldType = MySQLTextDataType.MEDIUMTEXT, + base64 = true + ) + private String content; + + /** + * 执行账号别名 + */ + @JsonProperty("account_alias") + @NotNull(message = "{validation.constraints.AccountAlias_empty.message}", + groups = ValidationGroups.Account.AccountAlias.class) + private String accountAlias; + + /** + * 执行账号别名 + */ + @JsonProperty("account_id") + @NotNull(message = "{validation.constraints.AccountId_empty.message}", + groups = ValidationGroups.Account.AccountId.class) + private Long accountId; + + /** + * 脚本类型，1：shell，2：bat，3：perl，4：python，5：powershell + */ + @JsonProperty("script_language") + @CheckEnum(message = "{validation.constraints.ScriptType_illegal.message}", enumClass = ScriptTypeEnum.class, + groups = ValidationGroups.Script.ScriptContent.class) + private Integer scriptLanguage; + + /** + * 脚本参数， BASE64编码 + */ + @JsonProperty("script_param") + @NotExceedMySQLTextFieldLength( + fieldName = "script_param", + fieldType = MySQLTextDataType.TEXT, + base64 = true + ) + private String scriptParam; + + /** + * 自定义Windows解释器路径 + */ + @EndWith(fieldName = "windows_interpreter", value = ".exe") + @MaxLength(value = 260, + message = "{validation.constraints.WindowsInterpreterExceedMaxLength.message}") + @JsonProperty("windows_interpreter") + private String windowsInterpreter; + + /** + * 脚本ID + */ + @JsonProperty("script_id") + private String scriptId; + + /** + * 脚本版本ID + */ + @JsonProperty("script_version_id") + private Long scriptVersionId; + + /** + * 是否敏感参数 + */ + @JsonProperty("is_param_sensitive") + private boolean isParamSensitive; + + /** + * 执行超时时间,单位秒 + */ + @JsonProperty("timeout") + @Range(min = JobConstants.MIN_JOB_TIMEOUT_SECONDS, max = JobConstants.MAX_JOB_TIMEOUT_SECONDS, + message = "{validation.constraints.InvalidJobTimeout_outOfRange.message}") + private Integer timeout; + + @JsonProperty("execute_target") + @Valid + private OpenApiExecuteTargetDTO executeTarget; + + /** + * 任务执行完成之后回调 + */ + @JsonProperty("callback") + private EsbCallbackDTO callback; + + /** + * 滚动配置 + */ + @JsonProperty("rolling_config") + private EsbRollingConfigDTO rollingConfig; + + /** + * 获取去除首尾空格后的windowsInterpreter + * + * @return Trim后的windowsInterpreter + */ + public String getTrimmedWindowsInterpreter() { + return windowsInterpreter != null ? windowsInterpreter.trim() : null; + } +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/bkci/plugin/EsbBkCIPluginFastTransferFileV3Request.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/bkci/plugin/EsbBkCIPluginFastTransferFileV3Request.java new file mode 100644 index 0000000000..3fab9afdb5 --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/bkci/plugin/EsbBkCIPluginFastTransferFileV3Request.java @@ -0,0 +1,127 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model.esb.v3.bkci.plugin; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.constant.JobConstants; +import com.tencent.bk.job.common.esb.model.EsbAppScopeReq; +import com.tencent.bk.job.common.esb.model.job.v3.EsbFileSourceV3DTO; +import com.tencent.bk.job.common.model.openapi.v4.OpenApiExecuteTargetDTO; +import com.tencent.bk.job.execute.model.esb.v3.EsbRollingConfigDTO; +import lombok.Getter; +import lombok.Setter; +import org.hibernate.validator.constraints.Range; + +import java.util.List; + + +@Setter +@Getter +public class EsbBkCIPluginFastTransferFileV3Request extends EsbAppScopeReq { + + /** + * 文件分发任务名称 + */ + @JsonProperty("task_name") + private String name; + /** + * 源文件 + */ + @JsonProperty("file_source_list") + private List fileSources; + + /** + * 目标路径 + */ + @JsonProperty("file_target_path") + private String targetPath; + + /** + * 目标文件名 + */ + @JsonProperty("file_target_name") + private String targetName; + + /** + * 目标服务器账户别名 + */ + @JsonProperty("account_alias") + private String accountAlias; + + /** + * 目标服务器账号ID + */ + @JsonProperty("account_id") + private Long accountId; + + @JsonProperty("execute_object") + private OpenApiExecuteTargetDTO executeObject; + + /** + * 任务执行完成之后回调URL + */ + @JsonProperty("callback_url") + private String callbackUrl; + + /** + * 下载限速，单位MB + */ + @JsonProperty("download_speed_limit") + private Integer downloadSpeedLimit; + + /** + * 上传限速，单位MB + */ + @JsonProperty("upload_speed_limit") + private Integer uploadSpeedLimit; + + /** + * 超时时间，单位秒 + */ + @JsonProperty("timeout") + @Range(min = JobConstants.MIN_JOB_TIMEOUT_SECONDS, max = JobConstants.MAX_JOB_TIMEOUT_SECONDS, + message = "{validation.constraints.InvalidJobTimeout_outOfRange.message}") + private Integer timeout; + + /** + * 传输模式。1-严谨模式，2-强制模式 + */ + @JsonProperty("transfer_mode") + private Integer transferMode; + + /** + * 滚动配置 + */ + @JsonProperty("rolling_config") + private EsbRollingConfigDTO rollingConfig; + + @JsonIgnore + public EsbBkCIPluginFastTransferFileV3Request trimTargetPath() { + targetPath = targetPath == null ? null : targetPath.trim(); + return this; + } + +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/bkci/plugin/EsbBkCIPluginGetJobInstanceStatusRequest.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/bkci/plugin/EsbBkCIPluginGetJobInstanceStatusRequest.java new file mode 100644 index 0000000000..e4fd8af005 --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/bkci/plugin/EsbBkCIPluginGetJobInstanceStatusRequest.java @@ -0,0 +1,55 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model.esb.v3.bkci.plugin; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.esb.model.EsbAppScopeReq; +import lombok.Getter; +import lombok.Setter; + +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; + +/** + * 根据作业实例 ID 查询作业执行状态请求 + */ +@Getter +@Setter +public class EsbBkCIPluginGetJobInstanceStatusRequest extends EsbAppScopeReq { + + /** + * 作业执行实例 ID + */ + @JsonProperty("job_instance_id") + @NotNull(message = "{validation.constraints.InvalidJobInstanceId.message}") + @Min(value = 1L, message = "{validation.constraints.InvalidJobInstanceId.message}") + private Long taskInstanceId; + + /** + * 是否返回每个ip上的任务详情 + */ + @JsonProperty("include_execute_object_task_result") + private boolean includeExecuteObjectTaskResult; +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/bkci/plugin/EsbCallbackDTO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/bkci/plugin/EsbCallbackDTO.java new file mode 100644 index 0000000000..6feec6953d --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/bkci/plugin/EsbCallbackDTO.java @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model.esb.v3.bkci.plugin; + +import lombok.Data; + +/** + * 作业执行回调 + */ +@Data +public class EsbCallbackDTO { + /** + * 回调地址 + */ + private String url; +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/bkci/plugin/EsbExecuteObjectLogsDTO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/bkci/plugin/EsbExecuteObjectLogsDTO.java new file mode 100644 index 0000000000..459900b01f --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/bkci/plugin/EsbExecuteObjectLogsDTO.java @@ -0,0 +1,66 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model.esb.v3.bkci.plugin; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +import java.util.List; + +/** + * 作业执行日志（批量） + */ +@Data +public class EsbExecuteObjectLogsDTO { + /** + * 作业实例ID + */ + @JsonProperty("job_instance_id") + private Long taskInstanceId; + + /** + * 步骤实例 ID + */ + @JsonProperty("step_instance_id") + private Long stepInstanceId; + + /** + * 日志类型 + */ + @JsonProperty("log_type") + private Integer logType; + + /** + * 脚本任务日志 + */ + @JsonProperty("script_execute_object_task_logs") + private List scriptTaskLogs; + + /** + * 文件任务日志 + */ + @JsonProperty("file_execute_object_task_logs") + private List fileTaskLogs; +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/bkci/plugin/EsbFileAtomicTaskLogDTO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/bkci/plugin/EsbFileAtomicTaskLogDTO.java new file mode 100644 index 0000000000..2e6483e33d --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/bkci/plugin/EsbFileAtomicTaskLogDTO.java @@ -0,0 +1,99 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model.esb.v3.bkci.plugin; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.model.openapi.v4.OpenApiExecuteObjectDTO; +import lombok.Data; + +/** + * 文件分发原子任务，指一个文件从一个源执行对象分发到一个目标执行对象的任务 + */ +@Data +public class EsbFileAtomicTaskLogDTO { + + /** + * 分发模式 + */ + @JsonProperty("mode") + private Integer mode; + + /** + * 源文件所在执行对象 + */ + @JsonProperty("src_execute_object") + private OpenApiExecuteObjectDTO srcExecuteObject; + + /** + * 源文件路径 + */ + @JsonProperty("src_path") + private String srcPath; + + /** + * 文件分发目标执行对象 + */ + @JsonProperty("dest_execute_object") + private OpenApiExecuteObjectDTO destExecuteObject; + + /** + * 目标路径 + */ + @JsonProperty("dest_path") + private String destPath; + + /** + * 任务状态 + */ + @JsonProperty("status") + private Integer status; + + /** + * 分发日志 + */ + @JsonProperty("log_content") + private String logContent; + + /** + * 文件大小 + */ + @JsonProperty("size") + private String size; + + /** + * 分发速度 + */ + @JsonProperty("speed") + private String speed; + + /** + * 分发进度 + */ + @JsonProperty("process") + private String process; + +} + + diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/bkci/plugin/EsbFileExecuteObjectLogDTO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/bkci/plugin/EsbFileExecuteObjectLogDTO.java new file mode 100644 index 0000000000..215004e3f3 --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/bkci/plugin/EsbFileExecuteObjectLogDTO.java @@ -0,0 +1,50 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model.esb.v3.bkci.plugin; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.model.openapi.v4.OpenApiExecuteObjectDTO; +import lombok.Data; + +import java.util.List; + +/** + * 执行对象对应的文件分发日志 + */ +@Data +public class EsbFileExecuteObjectLogDTO { + + /** + * 执行对象 + */ + @JsonProperty("execute_object") + private OpenApiExecuteObjectDTO executeObject; + + /** + * 文件任务日志 + */ + @JsonProperty("file_atomic_task_logs") + private List fileAtomicTaskLogs; +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/bkci/plugin/EsbJobInstanceStatusDTO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/bkci/plugin/EsbJobInstanceStatusDTO.java new file mode 100644 index 0000000000..c41eeea1b7 --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/bkci/plugin/EsbJobInstanceStatusDTO.java @@ -0,0 +1,188 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model.esb.v3.bkci.plugin; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.esb.model.EsbAppScopeDTO; +import com.tencent.bk.job.common.model.openapi.v4.OpenApiExecuteObjectDTO; +import lombok.Data; +import lombok.Getter; +import lombok.Setter; + +import java.util.List; + +@Data +public class EsbJobInstanceStatusDTO { + + @JsonProperty("finished") + private boolean finished; + + @JsonProperty("job_instance") + private JobInstance jobInstance; + + @JsonProperty("step_instance_list") + private List stepResults; + + @Setter + @Getter + public static class JobInstance extends EsbAppScopeDTO { + @JsonProperty("job_instance_id") + private Long id; + /** + * 状态： 1.未执行、2.正在执行、3.执行完成且成功、4.执行失败 + */ + @JsonProperty("status") + private Integer status; + /** + * 作业实例名称 + */ + @JsonProperty("name") + private String name; + /** + * 脚本创建时间 + */ + @JsonProperty("create_time") + private Long createTime; + /** + * 开始时间 + */ + @JsonProperty("start_time") + private Long startTime; + /** + * 结束时间 + */ + @JsonProperty("end_time") + private Long endTime; + /** + * 总耗时，单位：秒 + */ + @JsonProperty("total_time") + private Long totalTime; + } + + /** + * 步骤实例状态 + */ + @Setter + @Getter + public static class StepResult { + /** + * id + */ + @JsonProperty("step_instance_id") + private Long id; + /** + * 名称 + */ + @JsonProperty("name") + private String name; + /** + * 状态： 1.未执行、2.正在执行、3.执行完成且成功、4.执行失败 + */ + @JsonProperty("status") + private Integer status; + + /** + * 步骤类型：1.脚本步骤; 2.文件步骤; 3.人工确认 + */ + @JsonProperty("type") + private Integer type; + + /** + * 执行次数 + */ + @JsonProperty("execute_count") + private Integer executeCount; + + + /** + * 开始时间 + */ + @JsonProperty("start_time") + private Long startTime; + + /** + * 结束时间 + */ + @JsonProperty("end_time") + private Long endTime; + + /** + * 总耗时，单位：毫秒 + */ + @JsonProperty("total_time") + private Long totalTime; + + /** + * 创建时间 + */ + @JsonProperty("create_time") + private Long createTime; + + @JsonProperty("execute_object_result_list") + @JsonInclude(JsonInclude.Include.NON_EMPTY) + private List executeObjectResults; + } + + /** + * 执行对象任务结果 + */ + @Setter + @Getter + public static class ExecuteObjectResult { + + @JsonProperty("execute_object") + private OpenApiExecuteObjectDTO executeObject; + + private Integer status; + + private String tag; + + @JsonProperty("exit_code") + private Integer exitCode; + + @JsonProperty("error_code") + private Integer errorCode; + + /** + * 开始时间 + */ + @JsonProperty("start_time") + private Long startTime; + + /** + * 结束时间 + */ + @JsonProperty("end_time") + private Long endTime; + + /** + * 总耗时，单位：毫秒 + */ + @JsonProperty("total_time") + private Long totalTime; + } +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/bkci/plugin/EsbScriptExecuteObjectLogDTO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/bkci/plugin/EsbScriptExecuteObjectLogDTO.java new file mode 100644 index 0000000000..21a7b81893 --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/bkci/plugin/EsbScriptExecuteObjectLogDTO.java @@ -0,0 +1,48 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model.esb.v3.bkci.plugin; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.model.openapi.v4.OpenApiExecuteObjectDTO; +import lombok.Data; + +/** + * 执行对象对应的脚本执行日志 + */ +@Data +public class EsbScriptExecuteObjectLogDTO { + + /** + * 执行对象 + */ + @JsonProperty("execute_object") + private OpenApiExecuteObjectDTO executeObject; + + /** + * 脚本任务日志内容 + */ + @JsonProperty("log_content") + private String logContent; +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/bkci/plugin/validator/EsbBkCIPluginFastExecuteScriptRequestGroupSequenceProvider.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/bkci/plugin/validator/EsbBkCIPluginFastExecuteScriptRequestGroupSequenceProvider.java new file mode 100644 index 0000000000..44ea53e36d --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/bkci/plugin/validator/EsbBkCIPluginFastExecuteScriptRequestGroupSequenceProvider.java @@ -0,0 +1,69 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model.esb.v3.bkci.plugin.validator; + +import com.tencent.bk.job.common.validation.ValidationGroups; +import com.tencent.bk.job.execute.model.esb.v3.bkci.plugin.EsbBkCIPluginFastExecuteScriptRequest; +import lombok.extern.slf4j.Slf4j; +import org.hibernate.validator.spi.group.DefaultGroupSequenceProvider; + +import java.util.ArrayList; +import java.util.List; + +/** + * 主机联合校验 + */ +@Slf4j +public class EsbBkCIPluginFastExecuteScriptRequestGroupSequenceProvider + implements DefaultGroupSequenceProvider { + + @Override + public List> getValidationGroups(EsbBkCIPluginFastExecuteScriptRequest request) { + List> validationGroups = new ArrayList<>(); + validationGroups.add(EsbBkCIPluginFastExecuteScriptRequest.class); + if (request != null) { + // 优先级 accountId > accountAlias + if (request.getAccountId() != null) { + validationGroups.add(ValidationGroups.Account.AccountId.class); + } else if (request.getAccountAlias() != null) { + validationGroups.add(ValidationGroups.Account.AccountAlias.class); + } else { + validationGroups.add(ValidationGroups.Account.AccountId.class); + } + + // 脚本优先级 scriptVersionId > scriptId > scriptContent + if (request.getScriptVersionId() != null) { + validationGroups.add(ValidationGroups.Script.ScriptVersionId.class); + } else if (request.getScriptId() != null) { + validationGroups.add(ValidationGroups.Script.ScriptId.class); + } else if (request.getContent() != null) { + validationGroups.add(ValidationGroups.Script.ScriptContent.class); + } else { + validationGroups.add(ValidationGroups.Script.ScriptVersionId.class); + } + } + return validationGroups; + } +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbBatchGetJobInstanceIpLogV3Request.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbBatchGetJobInstanceIpLogV3Request.java index 7b3cbdfa42..5e6d6f520d 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbBatchGetJobInstanceIpLogV3Request.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbBatchGetJobInstanceIpLogV3Request.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,11 +26,14 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.tencent.bk.job.common.esb.model.EsbAppScopeReq; -import com.tencent.bk.job.common.esb.model.job.EsbIpDTO; +import com.tencent.bk.job.common.model.openapi.v4.OpenApiHostDTO; import lombok.Getter; import lombok.Setter; import lombok.ToString; +import javax.validation.Valid; +import javax.validation.constraints.Min; +import javax.validation.constraints.NotNull; import java.util.List; @Getter @@ -42,17 +45,28 @@ public class EsbBatchGetJobInstanceIpLogV3Request extends EsbAppScopeReq { * 作业执行实例 ID */ @JsonProperty("job_instance_id") + @NotNull(message = "{validation.constraints.InvalidJobInstanceId.message}") + @Min(value = 1L, message = "{validation.constraints.InvalidJobInstanceId.message}") private Long taskInstanceId; /** * 作业步骤实例ID */ @JsonProperty("step_instance_id") + @NotNull(message = "{validation.constraints.InvalidStepInstanceId.message}") + @Min(value = 1L, message = "{validation.constraints.InvalidStepInstanceId.message}") private Long stepInstanceId; /** * 目标服务器IP列表 */ @JsonProperty("ip_list") - private List ipList; + @Valid + private List ipList; + + /** + * 目标主机ID列表 + */ + @JsonProperty("host_id_list") + private List hostIdList; } diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbExecuteJobV3Request.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbExecuteJobV3Request.java index 373fe258e7..a6c8e651e4 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbExecuteJobV3Request.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbExecuteJobV3Request.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -55,6 +55,12 @@ public class EsbExecuteJobV3Request extends EsbAppScopeReq { @JsonProperty("callback_url") private String callbackUrl; + /** + * 是否启动任务 + */ + @JsonProperty("start_task") + private Boolean startTask = true; + public void trimIps() { if (globalVars != null && globalVars.size() > 0) { globalVars.forEach(globalVar -> { diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbFastExecuteSQLV3Request.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbFastExecuteSQLV3Request.java index bb3fad5198..994dee79ae 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbFastExecuteSQLV3Request.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbFastExecuteSQLV3Request.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -88,6 +88,12 @@ public class EsbFastExecuteSQLV3Request extends EsbAppScopeReq { @JsonProperty("callback_url") private String callbackUrl; + /** + * 是否启动任务 + */ + @JsonProperty("start_task") + private Boolean startTask = true; + public void trimIps() { if (this.targetServer != null) { trimIps(this.targetServer.getIps()); diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbFastExecuteScriptV3Request.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbFastExecuteScriptV3Request.java index bb51a1cbb9..e7a7eedfb5 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbFastExecuteScriptV3Request.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbFastExecuteScriptV3Request.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,9 +26,13 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.tencent.bk.job.common.constant.JobConstants; +import com.tencent.bk.job.common.constant.MySQLTextDataType; import com.tencent.bk.job.common.esb.model.EsbAppScopeReq; import com.tencent.bk.job.common.esb.model.job.EsbIpDTO; import com.tencent.bk.job.common.esb.model.job.v3.EsbServerV3DTO; +import com.tencent.bk.job.common.validation.EndWith; +import com.tencent.bk.job.common.validation.MaxLength; +import com.tencent.bk.job.common.validation.NotExceedMySQLTextFieldLength; import com.tencent.bk.job.execute.model.esb.v3.EsbRollingConfigDTO; import lombok.Getter; import lombok.Setter; @@ -54,6 +58,11 @@ public class EsbFastExecuteScriptV3Request extends EsbAppScopeReq { * "脚本内容，BASE64编码 */ @JsonProperty("script_content") + @NotExceedMySQLTextFieldLength( + fieldName = "script_content", + fieldType = MySQLTextDataType.MEDIUMTEXT, + base64 = true + ) private String content; /** @@ -78,8 +87,22 @@ public class EsbFastExecuteScriptV3Request extends EsbAppScopeReq { * 脚本参数， BASE64编码 */ @JsonProperty("script_param") + @NotExceedMySQLTextFieldLength( + fieldName = "script_param", + fieldType = MySQLTextDataType.TEXT, + base64 = true + ) private String scriptParam; + /** + * 自定义Windows解释器路径 + */ + @EndWith(fieldName = "windows_interpreter", value = ".exe") + @MaxLength(value = 260, + message = "{validation.constraints.WindowsInterpreterExceedMaxLength.message}") + @JsonProperty("windows_interpreter") + private String windowsInterpreter; + /** * 脚本ID */ @@ -102,7 +125,7 @@ public class EsbFastExecuteScriptV3Request extends EsbAppScopeReq { * 执行超时时间,单位秒 */ @JsonProperty("timeout") - @Range(min = JobConstants.MIN_JOB_TIMEOUT_SECONDS, max= JobConstants.MAX_JOB_TIMEOUT_SECONDS, + @Range(min = JobConstants.MIN_JOB_TIMEOUT_SECONDS, max = JobConstants.MAX_JOB_TIMEOUT_SECONDS, message = "{validation.constraints.InvalidJobTimeout_outOfRange.message}") private Integer timeout; @@ -122,6 +145,12 @@ public class EsbFastExecuteScriptV3Request extends EsbAppScopeReq { @JsonProperty("rolling_config") private EsbRollingConfigDTO rollingConfig; + /** + * 是否启动任务 + */ + @JsonProperty("start_task") + private Boolean startTask = true; + public void trimIps() { if (this.targetServer != null) { trimIps(this.targetServer.getIps()); @@ -136,4 +165,12 @@ private void trimIps(List ips) { } } + /** + * 获取去除首尾空格后的windowsInterpreter + * + * @return Trim后的windowsInterpreter + */ + public String getTrimmedWindowsInterpreter() { + return windowsInterpreter != null ? windowsInterpreter.trim() : null; + } } diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbFastTransferFileV3Request.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbFastTransferFileV3Request.java index 1f80c9aa1b..6f357ef74e 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbFastTransferFileV3Request.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbFastTransferFileV3Request.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,7 @@ package com.tencent.bk.job.execute.model.esb.v3.request; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.tencent.bk.job.common.constant.JobConstants; import com.tencent.bk.job.common.esb.model.EsbAppScopeReq; @@ -120,6 +121,12 @@ public class EsbFastTransferFileV3Request extends EsbAppScopeReq { @JsonProperty("rolling_config") private EsbRollingConfigDTO rollingConfig; + /** + * 是否启动任务 + */ + @JsonProperty("start_task") + private Boolean startTask = true; + public void trimIps() { if (this.targetServer != null) { trimIps(this.targetServer.getIps()); @@ -140,6 +147,11 @@ private void trimIps(List ips) { }); } } + + @JsonIgnore + public String getTrimmedTargetPath(){ + return targetPath == null ? null : targetPath.trim(); + } } diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbGetJobInstanceGlobalVarValueV3Request.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbGetJobInstanceGlobalVarValueV3Request.java index 9af262916f..9e748db617 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbGetJobInstanceGlobalVarValueV3Request.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbGetJobInstanceGlobalVarValueV3Request.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbGetJobInstanceIpLogV3Request.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbGetJobInstanceIpLogV3Request.java index df5f1b6b1f..d6ce90ae0e 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbGetJobInstanceIpLogV3Request.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbGetJobInstanceIpLogV3Request.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -52,4 +52,7 @@ public class EsbGetJobInstanceIpLogV3Request extends EsbAppScopeReq { private Long cloudAreaId; private String ip; + + @JsonProperty("bk_host_id") + private Long hostId; } diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbGetJobInstanceListV3Request.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbGetJobInstanceListV3Request.java index 4ee9065f7c..62d2970073 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbGetJobInstanceListV3Request.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbGetJobInstanceListV3Request.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbGetJobInstanceStatusV3Request.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbGetJobInstanceStatusV3Request.java index 2cf4ce7cd9..7ff1133a8a 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbGetJobInstanceStatusV3Request.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbGetJobInstanceStatusV3Request.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbOperateJobInstanceV3Request.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbOperateJobInstanceV3Request.java index 39dde5dff3..5f13290b72 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbOperateJobInstanceV3Request.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbOperateJobInstanceV3Request.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbOperateStepInstanceV3Request.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbOperateStepInstanceV3Request.java index 8290594194..11cd85a072 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbOperateStepInstanceV3Request.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbOperateStepInstanceV3Request.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbPushConfigFileV3Request.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbPushConfigFileV3Request.java index 0c4bf5ca15..cd96239e6e 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbPushConfigFileV3Request.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/v3/request/EsbPushConfigFileV3Request.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/AppCodeTaskEvictPolicyDTO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/AppCodeTaskEvictPolicyDTO.java index 551a28f6a8..711d58c709 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/AppCodeTaskEvictPolicyDTO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/AppCodeTaskEvictPolicyDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/AppIdTaskEvictPolicyDTO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/AppIdTaskEvictPolicyDTO.java index 5e707c4884..ac9fdf4ff0 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/AppIdTaskEvictPolicyDTO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/AppIdTaskEvictPolicyDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ComposedTaskEvictPolicyDTO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ComposedTaskEvictPolicyDTO.java index e17c25ab18..ebb2c068c4 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ComposedTaskEvictPolicyDTO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ComposedTaskEvictPolicyDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/CronTaskExecuteResult.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/CronTaskExecuteResult.java index 06012a539e..f10d7d6918 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/CronTaskExecuteResult.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/CronTaskExecuteResult.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/RunningJobQuotaUsage.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/RunningJobQuotaUsage.java new file mode 100644 index 0000000000..5264bfee28 --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/RunningJobQuotaUsage.java @@ -0,0 +1,54 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model.inner; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.util.Map; + +/** + * 正在执行中的作业配额使用情况 + */ +@ApiModel("正在执行中的作业配额使用情况") +@Data +public class RunningJobQuotaUsage { + /** + * 作业总量 + */ + @ApiModelProperty("作业总量") + private Long total; + /** + * 按应用维度统计的作业数量 + */ + @ApiModelProperty("按应用维度统计的作业数量") + private Map appCount; + /** + * 按资源管理空间统计的作业数量 + */ + @ApiModelProperty("按资源管理空间统计的作业数量") + private Map resourceScopeCount; +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ServiceCronTaskExecuteResultStatistics.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ServiceCronTaskExecuteResultStatistics.java index 7ddc4e7c83..53300efb81 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ServiceCronTaskExecuteResultStatistics.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ServiceCronTaskExecuteResultStatistics.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ServiceExecuteObject.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ServiceExecuteObject.java new file mode 100644 index 0000000000..badb799297 --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ServiceExecuteObject.java @@ -0,0 +1,76 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model.inner; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.tencent.bk.job.common.annotation.PersistenceObject; +import com.tencent.bk.job.common.constant.ExecuteObjectTypeEnum; +import com.tencent.bk.job.common.model.dto.Container; +import com.tencent.bk.job.common.model.dto.HostDTO; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import lombok.extern.slf4j.Slf4j; + +/** + * 作业执行对象通用模型 + */ +@Setter +@Getter +@NoArgsConstructor +@JsonInclude(JsonInclude.Include.NON_NULL) +@ToString +@PersistenceObject +@Slf4j +public class ServiceExecuteObject { + /** + * 执行对象 ID + */ + private String id; + + /** + * 执行对象类型 + * + * @see ExecuteObjectTypeEnum + */ + private int type; + + /** + * 执行对象资源 ID（主机 ID/容器 ID) + */ + private Long resourceId; + + /** + * 容器 + */ + private Container container; + + /** + * 主机 + */ + private HostDTO host; + +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ServiceExecuteTargetDTO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ServiceExecuteTargetDTO.java new file mode 100644 index 0000000000..01a70d1920 --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ServiceExecuteTargetDTO.java @@ -0,0 +1,52 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model.inner; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.tencent.bk.job.common.annotation.PersistenceObject; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; + +import java.util.List; + +/** + * 任务执行目标 DTO + */ +@Data +@PersistenceObject +@JsonInclude(JsonInclude.Include.NON_EMPTY) +@Slf4j +public class ServiceExecuteTargetDTO { + /** + * 如果执行目标是通过全局变量-主机列表定义的，variable 表示变量 name + */ + private String variable; + + /** + * 执行对象列表(所有主机+容器） + */ + private List executeObjects; + +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ServiceFileDetailDTO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ServiceFileDetailDTO.java new file mode 100644 index 0000000000..9dc5b2643c --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ServiceFileDetailDTO.java @@ -0,0 +1,40 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model.inner; + +import com.fasterxml.jackson.annotation.JsonInclude; +import lombok.Data; +import lombok.NoArgsConstructor; + + +@Data +@NoArgsConstructor +@JsonInclude(JsonInclude.Include.NON_NULL) +public class ServiceFileDetailDTO { + /** + * 文件路径 + */ + private String filePath; +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ServiceFileSourceDTO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ServiceFileSourceDTO.java new file mode 100644 index 0000000000..6fb8709810 --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ServiceFileSourceDTO.java @@ -0,0 +1,59 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model.inner; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.annotation.PersistenceObject; +import lombok.Data; + +import java.util.List; + +/** + * 文件源 + */ +@Data +@JsonInclude(JsonInclude.Include.NON_EMPTY) +@PersistenceObject +public class ServiceFileSourceDTO { + + /** + * 文件类型 + */ + @JsonProperty("fileType") + private Integer fileType; + /** + * 文件列表 + */ + @JsonProperty("files") + private List files; + /** + * 文件源服务器 + */ + @JsonProperty("servers") + private ServiceExecuteTargetDTO servers; + + +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ServiceFileStepInstanceDTO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ServiceFileStepInstanceDTO.java new file mode 100644 index 0000000000..5c7ebe89b9 --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ServiceFileStepInstanceDTO.java @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model.inner; + +import io.swagger.annotations.ApiModel; +import lombok.Data; + +import java.util.List; + +@ApiModel("文件步骤实例") +@Data +public class ServiceFileStepInstanceDTO { + + /** + * 文件传输的源文件 + */ + private List fileSourceList; + + /** + * 执行目标 + */ + protected ServiceExecuteTargetDTO targetExecuteObjects; +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ServiceScriptStepInstanceDTO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ServiceScriptStepInstanceDTO.java new file mode 100644 index 0000000000..1e6af31d9f --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ServiceScriptStepInstanceDTO.java @@ -0,0 +1,57 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model.inner; + +import io.swagger.annotations.ApiModel; +import lombok.Data; + +@ApiModel("脚本步骤实例") +@Data +public class ServiceScriptStepInstanceDTO { + /** + * 步骤实例ID + */ + private Long stepInstanceId; + + /** + * 脚本类型:1(shell脚本)、2(bat脚本)、3(perl脚本)、4(python脚本)、5(powershell脚本)、6(SQL脚本) + */ + private Integer scriptType; + + /** + * 脚本内容 + */ + private String scriptContent; + + /** + * 脚本参数 + */ + private String scriptParam; + + /** + * 脚本参数是否为敏感参数 + */ + private boolean secureParam; +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ServiceStepInstanceDTO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ServiceStepInstanceDTO.java new file mode 100644 index 0000000000..9a2353e172 --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ServiceStepInstanceDTO.java @@ -0,0 +1,67 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model.inner; + +import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +@ApiModel("步骤实例") +@Data +public class ServiceStepInstanceDTO { + + @ApiModelProperty("步骤实例ID") + private Long id; + + @ApiModelProperty("步骤实例名称") + private String name; + + @ApiModelProperty("步骤类型") + private Integer executeType; + + @ApiModelProperty("步骤状态") + private Integer status; + + @ApiModelProperty("创建时间") + private Long createTime; + + @ApiModelProperty("脚本任务步骤") + private ServiceScriptStepInstanceDTO scriptStepInstance; + + @ApiModelProperty("文件任务步骤") + private ServiceFileStepInstanceDTO fileStepInstance; + + public boolean isScriptStep() { + StepExecuteTypeEnum stepExecuteTypeEnum = StepExecuteTypeEnum.valOf(executeType); + return stepExecuteTypeEnum == StepExecuteTypeEnum.EXECUTE_SCRIPT + || stepExecuteTypeEnum == StepExecuteTypeEnum.EXECUTE_SQL; + } + + public boolean isFileStep() { + StepExecuteTypeEnum stepExecuteTypeEnum = StepExecuteTypeEnum.valOf(executeType); + return stepExecuteTypeEnum == StepExecuteTypeEnum.SEND_FILE; + } +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ServiceTargetServers.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ServiceTargetServers.java index 36bc5d89b3..83a4708117 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ServiceTargetServers.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ServiceTargetServers.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ServiceTaskExecuteResult.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ServiceTaskExecuteResult.java index 12dcc459e4..03b039f794 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ServiceTaskExecuteResult.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ServiceTaskExecuteResult.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -30,4 +30,5 @@ public class ServiceTaskExecuteResult { private long taskInstanceId; private String name; + private Long createTime; } diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ServiceTaskInstanceDTO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ServiceTaskInstanceDTO.java index f03c548acc..b060c26608 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ServiceTaskInstanceDTO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ServiceTaskInstanceDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ServiceTaskVariable.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ServiceTaskVariable.java index fbd4608a5e..5affa49069 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ServiceTaskVariable.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/ServiceTaskVariable.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/TaskEvictPolicyDTO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/TaskEvictPolicyDTO.java index 42e3f06435..5befd6959e 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/TaskEvictPolicyDTO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/TaskEvictPolicyDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/TaskInstanceIdEvictPolicyDTO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/TaskInstanceIdEvictPolicyDTO.java index 111a5d6b92..a4e581a01b 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/TaskInstanceIdEvictPolicyDTO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/TaskInstanceIdEvictPolicyDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/request/ServiceGetCronTaskExecuteStatisticsRequest.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/request/ServiceGetCronTaskExecuteStatisticsRequest.java index d943841a08..0ffb35edfb 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/request/ServiceGetCronTaskExecuteStatisticsRequest.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/request/ServiceGetCronTaskExecuteStatisticsRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/request/ServiceTaskExecuteRequest.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/request/ServiceTaskExecuteRequest.java index 2dce69ee5f..e876cc05d9 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/request/ServiceTaskExecuteRequest.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/request/ServiceTaskExecuteRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/request/ServiceTriggerStatisticsRequest.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/request/ServiceTriggerStatisticsRequest.java index 78d555371e..2ee360a9b0 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/request/ServiceTriggerStatisticsRequest.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/inner/request/ServiceTriggerStatisticsRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/op/SwitchStatusReq.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/op/SwitchStatusReq.java new file mode 100644 index 0000000000..e30995daee --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/op/SwitchStatusReq.java @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model.op; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +@Data +@ApiModel("切换状态请求请求报文") +public class SwitchStatusReq { + + @ApiModelProperty(value = "目标状态，是否开启", required = true) + private boolean status; + +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/request/OperationPermissionReq.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/request/OperationPermissionReq.java index 2d82f5c935..2bf8cca5bc 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/request/OperationPermissionReq.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/request/OperationPermissionReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/request/RedoTaskRequest.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/request/RedoTaskRequest.java index 4cae128d88..37b5f354a8 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/request/RedoTaskRequest.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/request/RedoTaskRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/request/WebFastExecuteScriptRequest.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/request/WebFastExecuteScriptRequest.java index e0309d2dbc..119e40373c 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/request/WebFastExecuteScriptRequest.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/request/WebFastExecuteScriptRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,8 +25,14 @@ package com.tencent.bk.job.execute.model.web.request; import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.CompatibleType; import com.tencent.bk.job.common.constant.JobConstants; -import com.tencent.bk.job.execute.model.web.vo.ExecuteTargetVO; +import com.tencent.bk.job.common.constant.MySQLTextDataType; +import com.tencent.bk.job.common.model.vo.TaskTargetVO; +import com.tencent.bk.job.common.validation.EndWith; +import com.tencent.bk.job.common.validation.MaxLength; +import com.tencent.bk.job.common.validation.NotExceedMySQLTextFieldLength; import com.tencent.bk.job.execute.model.web.vo.RollingConfigVO; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; @@ -50,6 +56,11 @@ public class WebFastExecuteScriptRequest { * 脚本内容 */ @ApiModelProperty(value = "脚本内容，BASE64编码，当手动录入的时候使用此参数") + @NotExceedMySQLTextFieldLength( + fieldName = "scriptContent", + fieldType = MySQLTextDataType.MEDIUMTEXT, + base64 = true + ) private String content; @ApiModelProperty(value = "脚本ID,当引用脚本的时候传该参数") @@ -80,21 +91,49 @@ public class WebFastExecuteScriptRequest { * 脚本参数 */ @ApiModelProperty(value = "脚本参数") + @NotExceedMySQLTextFieldLength( + fieldName = "scriptParam", + fieldType = MySQLTextDataType.TEXT, + base64 = false + ) private String scriptParam; + /** + * 自定义Windows解释器路径 + */ + @ApiModelProperty(value = "自定义Windows解释器路径，对Linux机器不生效，对SQL脚本不生效") + @EndWith(fieldName = "windowsInterpreter", value = ".exe", + message = "{validation.constraints.WinInterpreterInvalidSuffix.message}") + @MaxLength(value = 260, + message = "{validation.constraints.WindowsInterpreterExceedMaxLength.message}") + private String windowsInterpreter; + /** * 执行超时时间 */ @ApiModelProperty(value = "执行超时时间，单位秒", required = true) @NotNull(message = "{validation.constraints.InvalidJobTimeout_empty.message}") - @Range(min = JobConstants.MIN_JOB_TIMEOUT_SECONDS, max= JobConstants.MAX_JOB_TIMEOUT_SECONDS, - message = "{validation.constraints.InvalidJobTimeout_outOfRange.message}") + @Range( + min = JobConstants.MIN_JOB_TIMEOUT_SECONDS, + max = JobConstants.MAX_JOB_TIMEOUT_SECONDS, + message = "{validation.constraints.InvalidJobTimeout_outOfRange.message}" + ) private Integer timeout; /** - * 目标服务器 + * 目标执行对象 */ - private ExecuteTargetVO targetServers; + @Deprecated + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.DEPLOY, + explain = "使用 taskTarget 参数替换。发布完成后可以删除") + @ApiModelProperty(hidden = true) + private TaskTargetVO targetServers; + + /** + * 目标执行对象 + */ + @ApiModelProperty(value = "执行目标", required = true) + private TaskTargetVO taskTarget; /** * 是否敏感参数 0-否，1-是 @@ -115,4 +154,18 @@ public class WebFastExecuteScriptRequest { @ApiModelProperty(value = "是否启用滚动执行") private boolean rollingEnabled; + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.DEPLOY, + explain = "发布完成后可以删除") + public TaskTargetVO getTaskTarget() { + return taskTarget != null ? taskTarget : targetServers; + } + + /** + * 获取去除首尾空格后的windowsInterpreter + * + * @return Trim后的windowsInterpreter + */ + public String getTrimmedWindowsInterpreter() { + return windowsInterpreter != null ? windowsInterpreter.trim() : null; + } } diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/request/WebFastPushFileRequest.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/request/WebFastPushFileRequest.java index 5a7d8f5a44..c9211d5610 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/request/WebFastPushFileRequest.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/request/WebFastPushFileRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/request/WebStepOperation.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/request/WebStepOperation.java index a4bb87a0c7..330c1a09da 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/request/WebStepOperation.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/request/WebStepOperation.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/request/WebTaskExecuteRequest.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/request/WebTaskExecuteRequest.java index 16edbfd9e8..b124768b80 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/request/WebTaskExecuteRequest.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/request/WebTaskExecuteRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/AgentTaskExecutionVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/AgentTaskExecutionVO.java index 638a3eeaee..e926e363b1 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/AgentTaskExecutionVO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/AgentTaskExecutionVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,6 +25,8 @@ package com.tencent.bk.job.execute.model.web.vo; import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.CompatibleType; import com.tencent.bk.job.common.util.json.DecimalFormatJsonSerializer; import com.tencent.bk.job.common.util.json.LongTimestampSerializer; import io.swagger.annotations.ApiModel; @@ -33,38 +35,58 @@ @ApiModel("Agent任务执行信息") @Data +@Deprecated +@CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.DEPLOY, + explain = "使用 ExecuteObjectTaskVO 替换。发布完成后可以删除") public class AgentTaskExecutionVO { @ApiModelProperty("执行次数") private Integer retryCount; + @ApiModelProperty("滚动批次") private Integer batch; - @ApiModelProperty("主机ID") + + @ApiModelProperty("Agent对应的主机ID") private Long hostId; - @ApiModelProperty("Agent绑定的ip，包含云区域") - private String ip; - @ApiModelProperty("Agent ip显示名称，展示给用户使用该ip") - private String displayIp; + + @ApiModelProperty("Agent ID") + private String agentId; + + @ApiModelProperty("Agent ipv4") + private String ipv4; + + @ApiModelProperty("Agent ipv6") + private String ipv6; + @ApiModelProperty("Agent任务执行状态") private Integer status; + @ApiModelProperty("Agent任务执行状态描述") private String statusDesc; + @ApiModelProperty("开始时间") @JsonSerialize(using = LongTimestampSerializer.class) private Long startTime; + @ApiModelProperty("结束时间") @JsonSerialize(using = LongTimestampSerializer.class) private Long endTime; + @ApiModelProperty("耗时") @JsonSerialize(using = DecimalFormatJsonSerializer.class) private Long totalTime; + @ApiModelProperty("脚本返回码") private Integer exitCode; + @ApiModelProperty("脚本错误码") private Integer errorCode; + @ApiModelProperty("脚本执行输出") private String tag; + @ApiModelProperty("云区域ID") private Long cloudAreaId; + @ApiModelProperty("云区域名称") private String cloudAreaName; diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/DangerousRecordVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/DangerousRecordVO.java index 7103d55a3b..5499ee5406 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/DangerousRecordVO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/DangerousRecordVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteApprovalStepVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteApprovalStepVO.java index eae3c6d361..fa348f7e58 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteApprovalStepVO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteApprovalStepVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteCloudAreaInfoVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteCloudAreaInfoVO.java index 19a06db09e..8f8dabe714 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteCloudAreaInfoVO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteCloudAreaInfoVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteFileDestinationInfoVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteFileDestinationInfoVO.java index c5ea6ebea1..b127f615dc 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteFileDestinationInfoVO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteFileDestinationInfoVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,9 @@ package com.tencent.bk.job.execute.model.web.vo; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.model.vo.TaskTargetVO; import io.swagger.annotations.ApiModelProperty; import lombok.Data; @@ -41,5 +43,10 @@ public class ExecuteFileDestinationInfoVO { private String accountName; @ApiModelProperty("目标机器列表") - private ExecuteTargetVO server; + private TaskTargetVO server; + + @JsonIgnore + public String getTrimmedPath() { + return path == null ? null : path.trim(); + } } diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteFileSourceInfoVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteFileSourceInfoVO.java index 15fbfa051c..691c35a7ff 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteFileSourceInfoVO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteFileSourceInfoVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,10 @@ package com.tencent.bk.job.execute.model.web.vo; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.model.vo.TaskTargetVO; +import com.tencent.bk.job.common.util.ListUtil; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; @@ -48,7 +51,7 @@ public class ExecuteFileSourceInfoVO { private String fileSize; @ApiModelProperty(value = "主机列表") - private ExecuteTargetVO host; + private TaskTargetVO host; @ApiModelProperty(value = "主机账号") @JsonProperty("account") @@ -59,4 +62,9 @@ public class ExecuteFileSourceInfoVO { @ApiModelProperty(value = "文件源ID") private Integer fileSourceId; + + @JsonIgnore + public List getTrimmedFileLocation() { + return ListUtil.trimStringList(fileLocation); + } } diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteFileStepVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteFileStepVO.java index af400c62e0..845d9d445a 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteFileStepVO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteFileStepVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,19 +25,12 @@ package com.tencent.bk.job.execute.model.web.vo; import com.fasterxml.jackson.annotation.JsonProperty; -import com.tencent.bk.job.common.constant.DuplicateHandlerEnum; -import com.tencent.bk.job.common.constant.NotExistPathHandlerEnum; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; import java.util.List; -import static com.tencent.bk.job.common.constant.DuplicateHandlerEnum.GROUP_BY_IP; -import static com.tencent.bk.job.common.constant.DuplicateHandlerEnum.OVERWRITE; -import static com.tencent.bk.job.common.constant.NotExistPathHandlerEnum.CREATE_DIR; -import static com.tencent.bk.job.common.constant.NotExistPathHandlerEnum.STEP_FAIL; - @Data @ApiModel("步骤文件信息") public class ExecuteFileStepVO { @@ -69,33 +62,4 @@ public class ExecuteFileStepVO { @ApiModelProperty("忽略错误 0 - 不忽略 1 - 忽略") private Integer ignoreError; - - public static Integer getTransferMode(DuplicateHandlerEnum duplicateHandlerEnum, - NotExistPathHandlerEnum notExistPathHandlerEnum) { - if (duplicateHandlerEnum == null) { - // 默认覆盖 - duplicateHandlerEnum = OVERWRITE; - } - if (notExistPathHandlerEnum == null) { - // 默认直接创建 - notExistPathHandlerEnum = CREATE_DIR; - } - if (OVERWRITE == duplicateHandlerEnum && STEP_FAIL == notExistPathHandlerEnum) { - return 1; - } else if (OVERWRITE == duplicateHandlerEnum && CREATE_DIR == notExistPathHandlerEnum) { - return 2; - } else if (GROUP_BY_IP == duplicateHandlerEnum && CREATE_DIR == notExistPathHandlerEnum) { - return 3; - } else { - return 1; - } - } - - public Integer getNotExistPathHandler() { - if (transferMode == 1) { - return STEP_FAIL.getValue(); - } else { - return CREATE_DIR.getValue(); - } - } } diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteHostVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteHostVO.java deleted file mode 100644 index 40971a4053..0000000000 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteHostVO.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.model.web.vo; - -import com.fasterxml.jackson.annotation.JsonInclude; -import io.swagger.annotations.ApiModel; -import io.swagger.annotations.ApiModelProperty; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -@NoArgsConstructor -@AllArgsConstructor -@ApiModel("主机信息") -@Data -@JsonInclude(JsonInclude.Include.NON_NULL) -public class ExecuteHostVO { - - @ApiModelProperty("主机 IP") - private String ip; - - @ApiModelProperty("描述") - private String ipDesc; - - @ApiModelProperty("agent 状态 0-异常 1-正常") - private Integer alive; - - @ApiModelProperty("云区域信息") - private ExecuteCloudAreaInfoVO cloudAreaInfo; -} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteObjectFileLogVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteObjectFileLogVO.java new file mode 100644 index 0000000000..823a0b466d --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteObjectFileLogVO.java @@ -0,0 +1,42 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model.web.vo; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.util.List; + +@ApiModel("文件任务-执行对象执行日志") +@Data +public class ExecuteObjectFileLogVO { + @ApiModelProperty(name = "isIncludingLogContent", value = "是否包含日志内容") + private boolean includingLogContent; + @ApiModelProperty("文件分发执行详情") + private List fileDistributionDetails; + @ApiModelProperty(name = "finished", value = "日志是否拉取完成") + private boolean finished; +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteObjectScriptLogVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteObjectScriptLogVO.java new file mode 100644 index 0000000000..d1b3028bb7 --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteObjectScriptLogVO.java @@ -0,0 +1,40 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model.web.vo; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +@ApiModel("脚本任务-执行对象执行日志") +@Data +public class ExecuteObjectScriptLogVO { + @ApiModelProperty("执行对象") + ExecuteObjectVO executeObject; + @ApiModelProperty("日志内容") + private String logContent; + @ApiModelProperty(name = "finished", value = "日志是否拉取完成") + private boolean finished; +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteObjectTaskVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteObjectTaskVO.java new file mode 100644 index 0000000000..944fa6d21c --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteObjectTaskVO.java @@ -0,0 +1,73 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model.web.vo; + +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.tencent.bk.job.common.util.json.DecimalFormatJsonSerializer; +import com.tencent.bk.job.common.util.json.LongTimestampSerializer; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +@ApiModel("执行对象任务执行信息") +@Data +public class ExecuteObjectTaskVO { + @ApiModelProperty("执行次数") + private Integer executeCount; + + @ApiModelProperty("滚动批次") + private Integer batch; + + @ApiModelProperty("执行对象类型") + private ExecuteObjectVO executeObject; + + @ApiModelProperty("执行对象任务执行状态") + private Integer status; + + @ApiModelProperty("执行对象任务执行状态描述") + private String statusDesc; + + @ApiModelProperty("开始时间") + @JsonSerialize(using = LongTimestampSerializer.class) + private Long startTime; + + @ApiModelProperty("结束时间") + @JsonSerialize(using = LongTimestampSerializer.class) + private Long endTime; + + @ApiModelProperty("耗时") + @JsonSerialize(using = DecimalFormatJsonSerializer.class) + private Long totalTime; + + @ApiModelProperty("脚本返回码") + private Integer exitCode; + + @ApiModelProperty("脚本错误码") + private Integer errorCode; + + @ApiModelProperty("脚本执行输出") + private String tag; + +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteObjectVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteObjectVO.java new file mode 100644 index 0000000000..8664cf47ab --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteObjectVO.java @@ -0,0 +1,77 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model.web.vo; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.tencent.bk.job.common.constant.ExecuteObjectTypeEnum; +import com.tencent.bk.job.common.model.vo.ContainerVO; +import com.tencent.bk.job.common.model.vo.HostInfoVO; +import io.swagger.annotations.ApiModel; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +/** + * 作业执行对象 VO + */ +@Setter +@Getter +@NoArgsConstructor +@JsonInclude(JsonInclude.Include.NON_NULL) +@ApiModel("作业执行对象") +public class ExecuteObjectVO { + + /** + * 执行对象类型 + * + * @see ExecuteObjectTypeEnum + */ + private ExecuteObjectTypeEnum type; + + /** + * 执行对象资源实例 ID（比如 主机/容器在 cmdb 对应的资源ID) + */ + private Long executeObjectResourceId; + + /** + * 容器 + */ + private ContainerVO container; + + /** + * 主机 + */ + private HostInfoVO host; + + @JsonCreator(mode = JsonCreator.Mode.DELEGATING) + public static ExecuteObjectTypeEnum fromExecuteObjectTypeValue(int type) { + return ExecuteObjectTypeEnum.valOf(type); + } + + public static String buildExecuteObjectId(Integer executeObjectType, Long executeObjectResoruceId) { + return executeObjectType + ":" + executeObjectResoruceId; + } +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteScriptStepVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteScriptStepVO.java index 793c056e47..6472ef92a2 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteScriptStepVO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteScriptStepVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,6 +25,7 @@ package com.tencent.bk.job.execute.model.web.vo; import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.model.vo.TaskTargetVO; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; @@ -51,6 +52,9 @@ public class ExecuteScriptStepVO { @ApiModelProperty("脚本参数") private String scriptParam; + @ApiModelProperty("自定义Windows解释器路径") + private String windowsInterpreter; + @ApiModelProperty("脚本超时时间") private Integer timeout; @@ -62,7 +66,7 @@ public class ExecuteScriptStepVO { private String accountName; @ApiModelProperty("执行目标") - private ExecuteTargetVO executeTarget; + private TaskTargetVO executeTarget; @ApiModelProperty("敏感参数") private Integer secureParam; diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteServersVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteServersVO.java deleted file mode 100644 index d72f2c2d77..0000000000 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteServersVO.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.model.web.vo; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import io.swagger.annotations.ApiModel; -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; - -import java.util.List; - -@Data -@ApiModel("目标服务器信息") -@JsonInclude(JsonInclude.Include.NON_NULL) -public class ExecuteServersVO { - - @ApiModelProperty("静态 IP 列表") - @JsonProperty("ipList") - private List ipList; - - @ApiModelProperty("拓扑节点") - @JsonProperty("topoNodeList") - private List topoNodeList; - - @ApiModelProperty("动态分组 ID") - @JsonProperty("dynamicGroupList") - private List dynamicGroupList; -} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteStepVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteStepVO.java index f5b973ef4a..f7e98519d5 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteStepVO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteStepVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteTargetVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteTargetVO.java deleted file mode 100644 index 0fe7f4034b..0000000000 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteTargetVO.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.model.web.vo; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import io.swagger.annotations.ApiModel; -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; - -@Data -@ApiModel("执行目标信息") -@JsonInclude(JsonInclude.Include.NON_NULL) -public class ExecuteTargetVO { - - @ApiModelProperty(value = "全局变量名") - private String variable; - - @ApiModelProperty(value = "主机节点列表") - @JsonProperty("hostNodeInfo") - private ExecuteServersVO hostNodeInfo; -} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteTopoNodeVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteTopoNodeVO.java deleted file mode 100644 index 43b905e139..0000000000 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteTopoNodeVO.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.model.web.vo; - -import com.fasterxml.jackson.annotation.JsonInclude; -import io.swagger.annotations.ApiModel; -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; - -@Data -@ApiModel("目标拓扑节点信息") -@JsonInclude(JsonInclude.Include.NON_NULL) -public class ExecuteTopoNodeVO { - @ApiModelProperty(value = "节点 ID，对应拓扑树节点中的instanceId", required = true) - private Long id; - - @ApiModelProperty(value = "节点类型 biz-业务 set-集群 module-模块 xxx-用户自定义节点类型，对应拓扑树节点中的objectId", required = true) - private String type; -} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteVariableVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteVariableVO.java index 3fdf92c542..96a4799ed7 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteVariableVO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecuteVariableVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,7 @@ package com.tencent.bk.job.execute.model.web.vo; +import com.tencent.bk.job.common.model.vo.TaskTargetVO; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; @@ -45,7 +46,7 @@ public class ExecuteVariableVO { private String value; @ApiModelProperty(value = "主机变量值，当变量类型为主机列表的时有效") - private ExecuteTargetVO targetValue; + private TaskTargetVO targetValue; @ApiModelProperty(value = "赋值可变 0-不可变 1-可变") private Integer changeable; diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecutionResultGroupV2VO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecutionResultGroupV2VO.java new file mode 100644 index 0000000000..de427e0ade --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecutionResultGroupV2VO.java @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model.web.vo; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.util.List; + +@Data +@ApiModel("执行结果分组") +public class ExecutionResultGroupV2VO { + @ApiModelProperty("执行结果") + private Integer resultType; + @ApiModelProperty("执行结果描述") + private String resultTypeDesc; + @ApiModelProperty("用户通过job_success/job_fail自定义的结果分类tag") + private String tag; + @ApiModelProperty("执行对象对应的任务执行情况") + private List tasks; + @ApiModelProperty("结果分组下的执行对象任务数") + private int taskSize; + +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecutionResultGroupVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecutionResultGroupVO.java index 93c04a096b..b99aa209bc 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecutionResultGroupVO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ExecutionResultGroupVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,8 @@ package com.tencent.bk.job.execute.model.web.vo; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.CompatibleType; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; @@ -31,7 +33,10 @@ import java.util.List; @Data -@ApiModel("执行结果分组") +@ApiModel("执行结果分组-废弃") +@Deprecated +@CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.DEPLOY, + explain = "使用 ExecutionResultGroupV2VO 参数替换。发布完成后可以删除") public class ExecutionResultGroupVO { @ApiModelProperty("执行结果") private Integer resultType; @@ -39,9 +44,9 @@ public class ExecutionResultGroupVO { private String resultTypeDesc; @ApiModelProperty("用户通过job_success/job_fail自定义的结果分类tag") private String tag; - @ApiModelProperty("IP对应的任务执行情况") + @ApiModelProperty("执行对象对应的任务执行情况") private List agentTaskExecutionDetail; - @ApiModelProperty("结果分组下的agent任务数") + @ApiModelProperty("结果分组下的执行对象任务数") private int agentTaskSize; } diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/FileDistributionDetailV2VO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/FileDistributionDetailV2VO.java new file mode 100644 index 0000000000..1799e930a6 --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/FileDistributionDetailV2VO.java @@ -0,0 +1,88 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model.web.vo; + +import com.fasterxml.jackson.annotation.JsonInclude; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +@ApiModel("文件分发执行详情") +@Data +@JsonInclude(JsonInclude.Include.NON_NULL) +public class FileDistributionDetailV2VO implements Comparable { + @ApiModelProperty(name = "taskId", value = "文件任务ID,用于检索单个文件分发的结果") + private String taskId; + @ApiModelProperty(name = "srcExecuteObject", value = "文件源执行对象") + private ExecuteObjectVO srcExecuteObject; + @ApiModelProperty(name = "destExecuteObject", value = "目标执行对象") + private ExecuteObjectVO destExecuteObject; + @ApiModelProperty("文件名称") + private String fileName; + @ApiModelProperty("文件大小") + private String fileSize; + @ApiModelProperty("状态,0-Pulling,1-Waiting,2-Uploading,3-Downloading,4-Finished,5-Failed") + private Integer status; + @ApiModelProperty("状态描述") + private String statusDesc; + @ApiModelProperty("速率") + private String speed; + @ApiModelProperty("进度") + private String progress; + @ApiModelProperty("文件任务上传下载标识,0-上传,1-下载") + private Integer mode; + @ApiModelProperty("日志内容") + private String logContent; + + @Override + public int compareTo(FileDistributionDetailV2VO other) { + if (other == null) { + return 1; + } + // 从文件源拉取文件的详情日志放在最前面 + if (this.status == 0 && other.status > 0) { + return -1; + } + int compareFileNameResult = compareString(this.fileName, other.getFileName()); + if (compareFileNameResult != 0) { + return compareFileNameResult; + } + return this.srcExecuteObject.getExecuteObjectResourceId() + .compareTo(other.getSrcExecuteObject().getExecuteObjectResourceId()); + } + + + private int compareString(String a, String b) { + if (a == null && b == null) { + return 0; + } else if (a != null && b == null) { + return 1; + } else if (a == null) { + return -1; + } else { + return a.compareTo(b); + } + } +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/FileDistributionDetailVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/FileDistributionDetailVO.java index 4d88aafc16..763fc6352b 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/FileDistributionDetailVO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/FileDistributionDetailVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,6 +25,8 @@ package com.tencent.bk.job.execute.model.web.vo; import com.fasterxml.jackson.annotation.JsonInclude; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.CompatibleType; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; @@ -32,13 +34,20 @@ @ApiModel("文件分发执行详情") @Data @JsonInclude(JsonInclude.Include.NON_NULL) -public class FileDistributionDetailVO implements Comparable { +@Deprecated +@CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.DEPLOY, + explain = "使用 FileDistributionDetailV2VO 替换。发布完成后可以删除") +public class FileDistributionDetailVO implements Comparable { @ApiModelProperty(name = "taskId", value = "文件任务ID,用于检索单个文件分发的结果") private String taskId; - @ApiModelProperty(name = "destIp", value = "下载目标IP") + @ApiModelProperty(name = "destIp", value = "下载目标IPv4") private String destIp; - @ApiModelProperty(name = "srcIp", value = "上传源IP") + @ApiModelProperty(name = "destIpv6", value = "下载目标IPv6") + private String destIpv6; + @ApiModelProperty(name = "srcIp", value = "上传源IPv4") private String srcIp; + @ApiModelProperty(name = "srcIpv6", value = "上传源IPv6") + private String srcIpv6; @ApiModelProperty("文件名称") private String fileName; @ApiModelProperty("文件大小") @@ -57,11 +66,10 @@ public class FileDistributionDetailVO implements Comparable { private String logContent; @Override - public int compareTo(Object o) { - if (o == null) { + public int compareTo(FileDistributionDetailVO other) { + if (other == null) { return 1; } - FileDistributionDetailVO other = (FileDistributionDetailVO) o; // 从文件源拉取文件的详情日志放在最前面 if (this.status == 0 && other.status > 0) { return -1; @@ -70,9 +78,11 @@ public int compareTo(Object o) { if (compareFileNameResult != 0) { return compareFileNameResult; } - return compareString(this.srcIp, other.getSrcIp()); + return compareString(this.srcIp != null ? this.srcIp : this.srcIpv6, + other.getSrcIp() != null ? other.getSrcIp() : other.getSrcIpv6()); } + private int compareString(String a, String b) { if (a == null && b == null) { return 0; diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/FileLogContentVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/FileLogContentVO.java deleted file mode 100644 index 815145714b..0000000000 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/FileLogContentVO.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.model.web.vo; - -import io.swagger.annotations.ApiModel; -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; - -@ApiModel("文件任务执行日志") -@Data -public class FileLogContentVO { - @ApiModelProperty("ip") - private String displayIp; - @ApiModelProperty("文件名称") - private String fileName; - @ApiModelProperty("日志内容") - private String logContent; -} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/IpFileLogContentVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/IpFileLogContentVO.java index ad5ff6fffd..bc708c93e4 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/IpFileLogContentVO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/IpFileLogContentVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,8 @@ package com.tencent.bk.job.execute.model.web.vo; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.CompatibleType; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; @@ -32,6 +34,9 @@ @ApiModel("文件任务Agent执行日志") @Data +@Deprecated +@CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.DEPLOY, + explain = "使用 ExecuteObjectFileLogVO 替换。发布完成后可以删除") public class IpFileLogContentVO { @ApiModelProperty(name = "isIncludingLogContent", value = "是否包含日志内容") private boolean includingLogContent; diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/IpScriptLogContentVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/IpScriptLogContentVO.java index efe40f423b..fcb41c39e6 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/IpScriptLogContentVO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/IpScriptLogContentVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,12 +24,17 @@ package com.tencent.bk.job.execute.model.web.vo; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.CompatibleType; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; @ApiModel("Agent作业执行日志") @Data +@Deprecated +@CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.DEPLOY, + explain = "使用 ExecuteObjectScriptLogVO 参数替换。发布完成后可以删除") public class IpScriptLogContentVO { @ApiModelProperty("ip") private String displayIp; diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/LogExportJobInfoVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/LogExportJobInfoVO.java index c8e1c544ac..e5dd5d99fc 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/LogExportJobInfoVO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/LogExportJobInfoVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/RollingBatchTaskVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/RollingBatchTaskVO.java index ac286cd5ce..5d59896199 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/RollingBatchTaskVO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/RollingBatchTaskVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/RollingConfigVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/RollingConfigVO.java index fd688f95cc..13566aead8 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/RollingConfigVO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/RollingConfigVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/RollingStepBatchTaskVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/RollingStepBatchTaskVO.java index 3eb5e44730..35de10b90b 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/RollingStepBatchTaskVO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/RollingStepBatchTaskVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/RollingTaskVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/RollingTaskVO.java index c6265d2c7d..4b5d43bfeb 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/RollingTaskVO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/RollingTaskVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ScriptCheckResultItemVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ScriptCheckResultItemVO.java index 51cb8caca8..d7cc0b388a 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ScriptCheckResultItemVO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/ScriptCheckResultItemVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/StepExecuteVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/StepExecuteVO.java index da96ca8f2d..78b2821d28 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/StepExecuteVO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/StepExecuteVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/StepExecutionDetailV2VO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/StepExecutionDetailV2VO.java new file mode 100644 index 0000000000..7cddeabb96 --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/StepExecutionDetailV2VO.java @@ -0,0 +1,86 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model.web.vo; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.tencent.bk.job.common.constant.ExecuteObjectTypeEnum; +import com.tencent.bk.job.common.util.json.DecimalFormatJsonSerializer; +import com.tencent.bk.job.common.util.json.LongTimestampSerializer; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.util.List; + +@ApiModel("步骤执行详情") +@Data +public class StepExecutionDetailV2VO { + @ApiModelProperty("步骤实例ID") + private Long stepInstanceId; + @ApiModelProperty("执行次数,默认为0") + private Integer executeCount; + @ApiModelProperty("步骤执行是否结束") + private boolean finished; + @ApiModelProperty("步骤名称") + private String name; + @ApiModelProperty("开始时间") + @JsonSerialize(using = LongTimestampSerializer.class) + private Long startTime; + @ApiModelProperty("结束时间") + @JsonSerialize(using = LongTimestampSerializer.class) + private Long endTime; + @ApiModelProperty("总耗时") + @JsonSerialize(using = DecimalFormatJsonSerializer.class) + private Long totalTime; + @ApiModelProperty("步骤状态,1-等待执行，2-正在执行，3-执行成功，4-执行失败，5-跳过，6-忽略错误，7-等待用户，8-手动结束，9-状态异常" + + "，10-强制终止中，11-强制终止成功，12-强制终止失败，13-确认终止，14-被丢弃，15-滚动等待") + private Integer status; + @ApiModelProperty("步骤状态描述") + private String statusDesc; + @ApiModelProperty("任务执行结果分组") + private List resultGroups; + @ApiModelProperty("是否是作业中最后一个步骤") + private Boolean isLastStep; + @ApiModelProperty("步骤类型，1-脚本，2-文件，3-人工确认") + private Integer type; + /** + * 步骤执行模式 + * + * @see com.tencent.bk.job.execute.common.constants.StepRunModeEnum + */ + @ApiModelProperty("步骤执行模式。1-单次全量执行(非滚动步骤)；2-滚动全量执行(滚动步骤)；3-滚动分批执行(滚动步骤)") + private Integer runMode; + @ApiModelProperty("步骤包含的滚动任务;如果非滚动步骤，那么该值为空") + private List rollingTasks; + + @ApiModelProperty("步骤使用的执行对象类型，1-主机，2-容器") + private ExecuteObjectTypeEnum executeObjectType; + + @JsonCreator(mode = JsonCreator.Mode.DELEGATING) + public static ExecuteObjectTypeEnum fromExecuteObjectTypeValue(int type) { + return ExecuteObjectTypeEnum.valOf(type); + } +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/StepExecutionDetailVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/StepExecutionDetailVO.java index c4cb22a13c..1f80bf1012 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/StepExecutionDetailVO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/StepExecutionDetailVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,6 +25,8 @@ package com.tencent.bk.job.execute.model.web.vo; import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.CompatibleType; import com.tencent.bk.job.common.util.json.DecimalFormatJsonSerializer; import com.tencent.bk.job.common.util.json.LongTimestampSerializer; import io.swagger.annotations.ApiModel; @@ -33,8 +35,11 @@ import java.util.List; -@ApiModel("步骤执行详情") +@ApiModel("步骤执行详情-废弃") @Data +@Deprecated +@CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.DEPLOY, + explain = "使用 StepExecuteDetailV2VO 参数替换。发布完成后可以删除") public class StepExecutionDetailVO { @ApiModelProperty("步骤实例ID") private Long stepInstanceId; diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/StepExecutionRecordVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/StepExecutionRecordVO.java index bc2cf4b3cf..57012b9c36 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/StepExecutionRecordVO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/StepExecutionRecordVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,6 +25,8 @@ package com.tencent.bk.job.execute.model.web.vo; import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.CompatibleType; import com.tencent.bk.job.common.util.json.LongTimestampSerializer; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; @@ -33,12 +35,21 @@ @ApiModel("步骤执行记录") @Data public class StepExecutionRecordVO { + + @ApiModelProperty(value = "作业实例ID") + private Long taskInstanceId; + @ApiModelProperty(value = "步骤实例ID") private Long stepInstanceId; - @ApiModelProperty(value = "重试次数") + @ApiModelProperty(value = "重试次数", hidden = true) + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.DEPLOY, + explain = "使用 executeCount 参数替换。发布完成后可以删除") private Integer retryCount; + @ApiModelProperty(value = "执行次数") + private Integer executeCount; + @ApiModelProperty(value = "执行记录创建时间") @JsonSerialize(using = LongTimestampSerializer.class) private Long createTime; diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/StepExecutionVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/StepExecutionVO.java index 23aa3c9283..016425e579 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/StepExecutionVO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/StepExecutionVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,6 +26,8 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.CompatibleType; import com.tencent.bk.job.common.util.json.DecimalFormatJsonSerializer; import com.tencent.bk.job.common.util.json.LongTimestampSerializer; import io.swagger.annotations.ApiModel; @@ -40,8 +42,14 @@ public class StepExecutionVO { @ApiModelProperty("步骤实例ID") private Long stepInstanceId; - @ApiModelProperty("执行次数,默认为0") + + @ApiModelProperty(value = "重试次数", hidden = true) + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.DEPLOY, + explain = "使用 executeCount 参数替换。发布完成后可以删除") private Integer retryCount; + + @ApiModelProperty("执行次数,默认为0") + private Integer executeCount; @ApiModelProperty("步骤名称") private String name; @ApiModelProperty("步骤类型,1-脚本，2-文件，3-人工确认") diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/StepOperationVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/StepOperationVO.java index 379bc56450..cb11066ce0 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/StepOperationVO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/StepOperationVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,6 +25,8 @@ package com.tencent.bk.job.execute.model.web.vo; import com.fasterxml.jackson.annotation.JsonInclude; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.CompatibleType; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; @@ -38,11 +40,19 @@ public class StepOperationVO { @ApiModelProperty("步骤实例ID") private Long stepInstanceId; - @ApiModelProperty("执行次数") + + @ApiModelProperty(value = "重试次数", hidden = true) + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.DEPLOY, + explain = "使用 executeCount 参数替换。发布完成后可以删除") private Integer retryCount; - public StepOperationVO(Long stepInstanceId, Integer retryCount) { + + @ApiModelProperty("执行次数") + private Integer executeCount; + + public StepOperationVO(Long stepInstanceId, Integer executeCount) { this.stepInstanceId = stepInstanceId; - this.retryCount = retryCount; + this.retryCount = executeCount; + this.executeCount = executeCount; } } diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/TaskExecuteResultVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/TaskExecuteResultVO.java index 897f85c511..eca6cbaed5 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/TaskExecuteResultVO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/TaskExecuteResultVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/TaskExecuteVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/TaskExecuteVO.java index 1b8e19b9c0..aff60d5bbd 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/TaskExecuteVO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/TaskExecuteVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/TaskExecutionVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/TaskExecutionVO.java index 268fad9eb0..b3acf00d76 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/TaskExecutionVO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/TaskExecutionVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/TaskInstanceDetailVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/TaskInstanceDetailVO.java index 9c2d22143b..0f5ca18491 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/TaskInstanceDetailVO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/TaskInstanceDetailVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/TaskInstanceVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/TaskInstanceVO.java index e76cf26cd3..d36b872a7b 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/TaskInstanceVO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/TaskInstanceVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/TaskLinkVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/TaskLinkVO.java new file mode 100644 index 0000000000..6e7c4ba24f --- /dev/null +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/TaskLinkVO.java @@ -0,0 +1,89 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model.web.vo; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.util.List; + +@ApiModel("任务链接") +@Data +public class TaskLinkVO { + /** + * 资源范围类型 + */ + @ApiModelProperty(value = "资源范围类型", allowableValues = "biz-业务,biz_set-业务集") + private String scopeType; + + /** + * 资源范围ID + */ + @ApiModelProperty("资源范围ID") + private String scopeId; + + /** + * 业务ID + */ + @ApiModelProperty("业务ID") + private Long appId; + + /** + * 任务ID + */ + @ApiModelProperty("任务ID") + private Long jobInstanceId; + + /** + * 步骤ID + */ + @ApiModelProperty("步骤ID") + private Long stepInstanceId; + + /** + * 执行次数 + */ + @ApiModelProperty("重试次数") + private Integer retryCount; + + /** + * 批次 + */ + @ApiModelProperty("批次") + private Integer batch; + + /** + * gse任务ID + */ + @ApiModelProperty("gse任务ID") + private String gseTaskId; + + /** + * web访问链接 + */ + @ApiModelProperty("web访问链接") + private List link; +} diff --git a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/TaskOperationLogVO.java b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/TaskOperationLogVO.java index 513f54392f..7eeb3811fb 100644 --- a/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/TaskOperationLogVO.java +++ b/src/backend/job-execute/api-job-execute/src/main/java/com/tencent/bk/job/execute/model/web/vo/TaskOperationLogVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/boot-job-execute/build.gradle b/src/backend/job-execute/boot-job-execute/build.gradle index b5b9c42ec0..500ca1cb9e 100644 --- a/src/backend/job-execute/boot-job-execute/build.gradle +++ b/src/backend/job-execute/boot-job-execute/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -21,7 +21,6 @@ * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. */ - apply plugin: 'org.springframework.boot' apply plugin: 'io.spring.dependency-management' dependencies { @@ -29,8 +28,7 @@ dependencies { api project(":commons:common-i18n") implementation 'org.springframework.boot:spring-boot-starter-jdbc' implementation 'org.springframework.cloud:spring-cloud-starter-bootstrap' - implementation 'org.springframework.cloud:spring-cloud-starter-stream-rabbit' - runtimeOnly 'mysql:mysql-connector-java' + runtimeOnly 'com.mysql:mysql-connector-j' testImplementation("com.h2database:h2") } @@ -42,10 +40,22 @@ task renameArtifacts(type: Copy) { from('build/libs') include "boot-job-execute-${version}.jar" destinationDir file('build/libs/') + println("renameArtifacts, version=" + version) rename "boot-job-execute-${version}.jar", "job-execute-${version}.jar" + doLast{ + println("After rename:") + def libsDir = file("build/libs") + if (libsDir.exists() && libsDir.isDirectory()) { + println "Files in $libsDir:" + libsDir.eachFile { file -> + println file.name + } + } else { + println "$libsDir does not exist or is not a directory." + } + } } -renameArtifacts.dependsOn jar -assemble.dependsOn renameArtifacts +renameArtifacts.dependsOn assemble task copyToLatestJar(type: Copy) { group = "local" @@ -57,3 +67,4 @@ task copyToLatestJar(type: Copy) { copyToLatestJar.dependsOn assemble apply from: "$rootDir/task_job_package.gradle" +copyToRelease.dependsOn renameArtifacts diff --git a/src/backend/job-execute/boot-job-execute/src/main/java/com/tencent/bk/job/execute/JobConfig.java b/src/backend/job-execute/boot-job-execute/src/main/java/com/tencent/bk/job/execute/JobConfig.java deleted file mode 100644 index b6c0b71052..0000000000 --- a/src/backend/job-execute/boot-job-execute/src/main/java/com/tencent/bk/job/execute/JobConfig.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute; - -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Configuration; - -@Configuration -public class JobConfig { - @Value("${password:password}") - private String password; -} diff --git a/src/backend/job-execute/boot-job-execute/src/main/java/com/tencent/bk/job/execute/JobExecuteBootApplication.java b/src/backend/job-execute/boot-job-execute/src/main/java/com/tencent/bk/job/execute/JobExecuteBootApplication.java index 065e9f81f5..a21aa3ac95 100644 --- a/src/backend/job-execute/boot-job-execute/src/main/java/com/tencent/bk/job/execute/JobExecuteBootApplication.java +++ b/src/backend/job-execute/boot-job-execute/src/main/java/com/tencent/bk/job/execute/JobExecuteBootApplication.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,19 +24,25 @@ package com.tencent.bk.job.execute; -import com.tencent.bk.job.common.config.FeatureToggleConfig; +import com.tencent.bk.job.common.service.boot.JobBootApplication; +import com.tencent.devops.leaf.plugin.annotation.EnableLeafServer; import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration; -import org.springframework.boot.context.properties.EnableConfigurationProperties; -import org.springframework.cloud.client.discovery.EnableDiscoveryClient; +import org.springframework.boot.autoconfigure.availability.ApplicationAvailabilityAutoConfiguration; +import org.springframework.boot.autoconfigure.jooq.JooqAutoConfiguration; import org.springframework.cloud.openfeign.EnableFeignClients; - -@SpringBootApplication(scanBasePackages = "com.tencent.bk.job", exclude = {RedisAutoConfiguration.class}) -@EnableDiscoveryClient -@EnableFeignClients -@EnableConfigurationProperties({FeatureToggleConfig.class}) +@JobBootApplication( + scanBasePackages = "com.tencent.bk.job.execute", + exclude = {JooqAutoConfiguration.class, ApplicationAvailabilityAutoConfiguration.class}, + excludeName = {"org.springframework.cloud.kubernetes.client.discovery.KubernetesDiscoveryClientAutoConfiguration"}) +@EnableFeignClients( + basePackages = { + "com.tencent.bk.job.manage.api", + "com.tencent.bk.job.logsvr.api", + "com.tencent.bk.job.file_gateway.api" + } +) +@EnableLeafServer public class JobExecuteBootApplication { public static void main(String[] args) { diff --git a/src/backend/job-execute/boot-job-execute/src/main/java/com/tencent/bk/job/execute/config/DbConfig.java b/src/backend/job-execute/boot-job-execute/src/main/java/com/tencent/bk/job/execute/config/DbConfig.java deleted file mode 100644 index c0736d2338..0000000000 --- a/src/backend/job-execute/boot-job-execute/src/main/java/com/tencent/bk/job/execute/config/DbConfig.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.config; - -import org.jooq.ConnectionProvider; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; -import org.jooq.impl.DataSourceConnectionProvider; -import org.jooq.impl.DefaultConfiguration; -import org.jooq.impl.DefaultDSLContext; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.boot.jdbc.DataSourceBuilder; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.DependsOn; -import org.springframework.context.annotation.Primary; -import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.jdbc.datasource.DataSourceTransactionManager; - -import javax.sql.DataSource; - -/** - * @date 2019/09/19 - */ -@Configuration -public class DbConfig { - - @Qualifier("job-execute-data-source") - @Primary - @Bean(name = "job-execute-data-source") - @ConfigurationProperties(prefix = "spring.datasource.job-execute") - public DataSource dataSource() { - return DataSourceBuilder.create().build(); - } - - @Qualifier("transactionManager") - @Bean(name = "transactionManager") - @DependsOn("job-execute-data-source") - @Primary - public DataSourceTransactionManager transactionManager( - @Qualifier("job-execute-data-source") DataSource dataSource) { - return new DataSourceTransactionManager(dataSource); - } - - @Qualifier("job-execute-jdbc-template") - @Bean(name = "job-execute-jdbc-template") - public JdbcTemplate jdbcTemplate(@Qualifier("job-execute-data-source") DataSource dataSource) { - return new JdbcTemplate(dataSource); - } - - @Qualifier("job-execute-dsl-context") - @Bean(name = "job-execute-dsl-context") - public DSLContext dslContext(@Qualifier("job-execute-jooq-conf") org.jooq.Configuration configuration) { - return new DefaultDSLContext(configuration); - } - - @Qualifier("job-execute-jooq-conf") - @Bean(name = "job-execute-jooq-conf") - public org.jooq.Configuration jooqConf( - @Qualifier("job-execute-conn-provider") ConnectionProvider connectionProvider) { - return new DefaultConfiguration().derive(connectionProvider).derive(SQLDialect.MYSQL); - } - - @Qualifier("job-execute-conn-provider") - @Bean(name = "job-execute-conn-provider") - public ConnectionProvider connectionProvider(@Qualifier("job-execute-data-source") DataSource dataSource) { - return new DataSourceConnectionProvider(dataSource); - } - -} diff --git a/src/backend/job-execute/boot-job-execute/src/main/java/com/tencent/bk/job/execute/config/FilterConfig.java b/src/backend/job-execute/boot-job-execute/src/main/java/com/tencent/bk/job/execute/config/FilterConfig.java deleted file mode 100644 index d5f85336d9..0000000000 --- a/src/backend/job-execute/boot-job-execute/src/main/java/com/tencent/bk/job/execute/config/FilterConfig.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.config; - -import com.tencent.bk.job.common.web.filter.RepeatableReadWriteServletRequestResponseFilter; -import org.springframework.boot.web.servlet.FilterRegistrationBean; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Configuration -public class FilterConfig { - @Bean - public FilterRegistrationBean repeatableRSRRFilterRegister() { - FilterRegistrationBean registration = - new FilterRegistrationBean<>(); - registration.setFilter(repeatableRRRFilter()); - registration.addUrlPatterns("/esb/api/*"); - registration.setName("repeatableReadRequestResponseFilter"); - registration.setOrder(0); - return registration; - } - - @Bean(name = "repeatableReadRequestResponseFilter") - public RepeatableReadWriteServletRequestResponseFilter repeatableRRRFilter() { - return new RepeatableReadWriteServletRequestResponseFilter(); - } -} diff --git a/src/backend/job-execute/boot-job-execute/src/main/java/com/tencent/bk/job/execute/config/InterceptorConfiguration.java b/src/backend/job-execute/boot-job-execute/src/main/java/com/tencent/bk/job/execute/config/InterceptorConfiguration.java deleted file mode 100644 index 75bc9426fc..0000000000 --- a/src/backend/job-execute/boot-job-execute/src/main/java/com/tencent/bk/job/execute/config/InterceptorConfiguration.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.config; - -import com.tencent.bk.job.common.iam.interceptor.AuthAppInterceptor; -import com.tencent.bk.job.common.web.interceptor.CustomTimedMetricsInterceptor; -import com.tencent.bk.job.common.web.interceptor.EsbApiLogInterceptor; -import com.tencent.bk.job.common.web.interceptor.EsbReqRewriteInterceptor; -import com.tencent.bk.job.common.web.interceptor.JobCommonInterceptor; -import com.tencent.bk.job.common.web.interceptor.ServiceSecurityInterceptor; -import com.tencent.bk.job.execute.common.interceptor.UriPermissionInterceptor; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Configuration; -import org.springframework.web.servlet.config.annotation.InterceptorRegistry; -import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; - -@Configuration -public class InterceptorConfiguration implements WebMvcConfigurer { - - private final JobCommonInterceptor jobCommonInterceptor; - private final CustomTimedMetricsInterceptor customTimedMetricsInterceptor; - private final UriPermissionInterceptor uriPermissionInterceptor; - private final AuthAppInterceptor authAppInterceptor; - private final EsbApiLogInterceptor esbApiLogInterceptor; - private final ServiceSecurityInterceptor serviceSecurityInterceptor; - private final EsbReqRewriteInterceptor esbReqRewriteInterceptor; - - @Autowired - public InterceptorConfiguration( - JobCommonInterceptor jobCommonInterceptor, - CustomTimedMetricsInterceptor customTimedMetricsInterceptor, - AuthAppInterceptor authAppInterceptor, - UriPermissionInterceptor uriPermissionInterceptor, EsbApiLogInterceptor esbApiLogInterceptor, - ServiceSecurityInterceptor serviceSecurityInterceptor, - EsbReqRewriteInterceptor esbReqRewriteInterceptor - ) { - this.jobCommonInterceptor = jobCommonInterceptor; - this.customTimedMetricsInterceptor = customTimedMetricsInterceptor; - this.uriPermissionInterceptor = uriPermissionInterceptor; - this.authAppInterceptor = authAppInterceptor; - this.esbApiLogInterceptor = esbApiLogInterceptor; - this.serviceSecurityInterceptor = serviceSecurityInterceptor; - this.esbReqRewriteInterceptor = esbReqRewriteInterceptor; - } - - @Override - public void addInterceptors(InterceptorRegistry registry) { - // 注册拦截器 - // 最高优先级：初始化JobContext - registry.addInterceptor(jobCommonInterceptor).addPathPatterns("/**").order(0); - registry.addInterceptor(customTimedMetricsInterceptor).addPathPatterns("/**").order(10); - registry.addInterceptor(serviceSecurityInterceptor).addPathPatterns("/**").order(20); - registry.addInterceptor(uriPermissionInterceptor) - .addPathPatterns( - uriPermissionInterceptor.getControlUriPatterns() - ).order(30); - registry.addInterceptor(esbApiLogInterceptor).addPathPatterns("/esb/api/**").order(40); - registry.addInterceptor(esbReqRewriteInterceptor).addPathPatterns("/esb/api/**").order(50); - registry.addInterceptor(authAppInterceptor).addPathPatterns("/web/**", "/esb/api/**").order(60); - } -} diff --git a/src/backend/job-execute/boot-job-execute/src/main/java/com/tencent/bk/job/execute/config/JobExecuteConfiguration.java b/src/backend/job-execute/boot-job-execute/src/main/java/com/tencent/bk/job/execute/config/JobExecuteConfiguration.java new file mode 100644 index 0000000000..8777f4f4e3 --- /dev/null +++ b/src/backend/job-execute/boot-job-execute/src/main/java/com/tencent/bk/job/execute/config/JobExecuteConfiguration.java @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.config; + +import com.tencent.bk.job.common.service.AppScopeMappingService; +import com.tencent.bk.job.common.web.interceptor.AppResourceScopeInterceptor; +import com.tencent.bk.job.manage.AppScopeMappingServiceImpl; +import com.tencent.bk.job.manage.api.inner.ServiceApplicationResource; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class JobExecuteConfiguration { + @Bean + AppScopeMappingService appScopeMappingService(ServiceApplicationResource applicationResource) { + return new AppScopeMappingServiceImpl(applicationResource); + } + + @Bean + public AppResourceScopeInterceptor appResourceScopeInterceptor(AppScopeMappingService appScopeMappingService) { + return new AppResourceScopeInterceptor(appScopeMappingService); + } +} diff --git a/src/backend/job-execute/boot-job-execute/src/main/java/com/tencent/bk/job/execute/config/SwaggerConfig.java b/src/backend/job-execute/boot-job-execute/src/main/java/com/tencent/bk/job/execute/config/SwaggerConfig.java index 4491f24a56..0ec9e61887 100644 --- a/src/backend/job-execute/boot-job-execute/src/main/java/com/tencent/bk/job/execute/config/SwaggerConfig.java +++ b/src/backend/job-execute/boot-job-execute/src/main/java/com/tencent/bk/job/execute/config/SwaggerConfig.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,38 +24,40 @@ package com.tencent.bk.job.execute.config; +import com.tencent.bk.job.common.service.config.JobCommonConfig; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Profile; import springfox.documentation.builders.PathSelectors; import springfox.documentation.builders.RequestHandlerSelectors; +import springfox.documentation.oas.annotations.EnableOpenApi; import springfox.documentation.spi.DocumentationType; import springfox.documentation.spring.web.plugins.Docket; -import springfox.documentation.swagger2.annotations.EnableSwagger2; import java.util.Arrays; import java.util.HashSet; /** - * @date 2019/09/19 + * Swagger 配置 */ @Configuration -@EnableSwagger2 -@Profile({"dev", "test", "local"}) +@EnableOpenApi +@Profile({"dev", "local"}) public class SwaggerConfig { - private final JobExecuteConfig config; + private final JobCommonConfig jobCommonConfig; @Autowired - public SwaggerConfig(JobExecuteConfig config) { - this.config = config; + public SwaggerConfig(JobCommonConfig jobCommonConfig) { + this.jobCommonConfig = jobCommonConfig; } @Bean public Docket api() { - return new Docket(DocumentationType.SWAGGER_2) - .host(config.getSwaggerUrl() + "/job-execute") + return new Docket(DocumentationType.OAS_30) + .host(jobCommonConfig.getSwaggerUrl()) + .pathMapping("/job-execute") .protocols(new HashSet<>(Arrays.asList("http", "https"))) .select() .apis(RequestHandlerSelectors.basePackage("com.tencent.bk.job.execute.api")) diff --git a/src/backend/job-execute/boot-job-execute/src/main/resources/application.yml b/src/backend/job-execute/boot-job-execute/src/main/resources/application.yml index 2a2849f7ab..cc954dac64 100644 --- a/src/backend/job-execute/boot-job-execute/src/main/resources/application.yml +++ b/src/backend/job-execute/boot-job-execute/src/main/resources/application.yml @@ -1,20 +1,22 @@ spring: profiles: active: prod - -ribbon: - ServerListRefreshInterval: 3000 - + mvc: + pathmatch: + matching-strategy: ant_path_matcher management: health: livenessstate: enabled: true readinessstate: enabled: true + # 使用spring cloud stream rabbitmq, 禁用掉springboot的rabbitmq health检查 + rabbit: + enabled: false endpoints: web: exposure: - include: health,configprops,env,beans,conditions,loggers,metrics,mappings,prometheus,scheduledtasks,info,refresh + include: health,configprops,env,beans,conditions,loggers,metrics,mappings,prometheus,scheduledtasks,info,refresh,busrefresh,bindings base-path: /actuator enabled-by-default: false endpoint: @@ -45,6 +47,10 @@ management: enabled: true refresh: enabled: true + busrefresh: + enabled: true + bindings: + enabled: true server: port: 19804 diff --git a/src/backend/job-execute/boot-job-execute/src/main/resources/bootstrap.yml b/src/backend/job-execute/boot-job-execute/src/main/resources/bootstrap.yml index 58da1565a1..2eff535bed 100644 --- a/src/backend/job-execute/boot-job-execute/src/main/resources/bootstrap.yml +++ b/src/backend/job-execute/boot-job-execute/src/main/resources/bootstrap.yml @@ -11,3 +11,5 @@ spring: name: ${JOB_APPLICATION_CONFIGMAP_NAME} - namespace: ${KUBERNETES_NAMESPACE} name: ${JOB_COMMON_CONFIGMAP_NAME} + - namespace: ${KUBERNETES_NAMESPACE} + name: ${JOB_COMMON_REFRESHABLE_CONFIGMAP_NAME} diff --git a/src/backend/job-execute/boot-job-execute/src/main/resources/i18n/message.properties b/src/backend/job-execute/boot-job-execute/src/main/resources/i18n/message.properties deleted file mode 100644 index e9c0fbcef3..0000000000 --- a/src/backend/job-execute/boot-job-execute/src/main/resources/i18n/message.properties +++ /dev/null @@ -1,124 +0,0 @@ -# -# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. -# -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. -# -# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. -# -# License for BK-JOB蓝鲸智云作业平台: -# -------------------------------------------------------------------- -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF -# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - -task.startup.mode.normal=页面执行 -task.startup.mode.api=API调用 -task.startup.mode.cron=定时执行 - -task.instance.type.normal=作业执行 -task.instance.type.script=脚本执行 -task.instance.type.file=文件分发 - -task.run.status.blank=等待执行 -task.run.status.running=正在执行 -task.run.status.success=执行成功 -task.run.status.fail=执行失败 -task.run.status.skipped=跳过 -task.run.status.ignore_error=忽略错误 -task.run.status.waiting_user=等待确认 -task.run.status.terminated=手动结束 -task.run.status.abnormal_state=状态异常 -task.run.status.stopping=强制终止中 -task.run.status.stop_success=强制终止成功 -task.run.status.confirm_terminated=确认终止 -task.run.status.abandoned=被丢弃 -task.run.status.rolling_waiting=滚动等待 - -agent.task.status.unknown_error=未知错误 -agent.task.status.agent_error=Agent异常 -agent.task.status.last_success=上次已成功 -agent.task.status.waiting=等待执行 -agent.task.status.running=正在执行 -agent.task.status.success=执行成功 -agent.task.status.failed=任务失败 -agent.task.status.submit_failed=任务下发失败 -agent.task.status.task_timeout=任务超时 -agent.task.status.log_error=任务日志错误 -agent.task.status.gse_script_timeout=脚本日志超时 -agent.task.status.gse_file_timeout=文件日志超时 -agent.task.status.script_failed=脚本执行失败 -agent.task.status.script_timeout=脚本执行超时 -agent.task.status.script_terminate=脚本执行被终止 -agent.task.status.script_not_zero_exit_code=脚本返回码非零 -agent.task.status.copyfile_failed=文件传输失败 -agent.task.status.copyfile_source_file_not_exist=源文件不存在 -agent.task.status.file_error_unclassified=文件任务未分类错误 -agent.task.status.gse_timeout=执行任务超时 -agent.task.status.gse_agent_error=Agent异常 -agent.task.status.gse_user_error=用户名不存在 -agent.task.status.gse_user_pwd_error=用户密码不正确 -agent.task.status.gse_file_error=文件获取失败 -agent.task.status.gse_file_size_exceed=文件超出限制 -agent.task.status.gse_file_task_error=文件传输错误 -agent.task.status.gse_task_error=任务执行出错 -agent.task.status.gse_task_terminate_success=任务强制终止成功 -agent.task.status.gse_task_terminate_failed=任务强制终止失败 -agent.task.status.host_not_exist=无效主机 -agent.task.status.unknown=未知 - -user.operation.retry_step_fail=失败重试 -user.operation.ignore_error=忽略错误 -user.operation.skip_step=手动跳过 -user.operation.terminate_job=强制终止 -user.operation.retry_step_all=全部重试 -user.operation.start=启动作业 -user.operation.confirm_continue=人工确认 -user.operation.confirm_terminate=人工确认 -user.operation.confirm_restart=人工确认 -user.operation.next_step=进入下一步 - -user.operation.detail.start.web=用户从页面点击启动 -user.operation.detail.start.cron=由定时任务调度启动 -user.operation.detail.start.api=由 {0} 调用 API 启动 -user.operation.detail.confirm.terminate=用户已确认中止任务，理由：{0} -user.operation.detail.confirm.continue=用户已确认继续执行，理由：{0} -user.operation.detail.confirm.restart=用户重新发起确认 -user.operation.detail.retry_step_all=点击查看重试前的执行详情 -user.operation.detail.retry_step_fail=点击查看重试前的执行详情 -user.operation.detail.ignore_error=已忽略执行失败的目标，任务继续执行 -user.operation.detail.next_step=用户已确认直接进入下一步 -user.operation.detail.terminate_job=强制终止 - -task.variable.type.name.string=字符串 -task.variable.type.name.namespace=命名空间 -task.variable.type.name.host_list=主机列表 -task.variable.type.name.cipher=密文 -task.variable.type.name.associative_array=关联数组 -task.variable.type.name.index_array=索引数组 - -task.step.type.name.script=执行脚本 -task.step.type.name.file=分发文件 -task.step.type.name.manual_confirm=人工确认 - -task.type.name.fast_execute_script=快速执行脚本 -task.type.name.fast_execute_sql=快速执行SQL -task.type.name.fast_push_file=快速分发文件 -task.type.name.job=执行方案 - -resource.topo_node.name=节点 -resource.host.name=主机 -resource.dynamic_group.name=动态分组 -resource.biz_set.name=业务集 - -script.check.result.detect.dangerous.script=高危脚本 diff --git a/src/backend/job-execute/boot-job-execute/src/main/resources/i18n/message_en.properties b/src/backend/job-execute/boot-job-execute/src/main/resources/i18n/message_en.properties deleted file mode 100644 index c640e2fc6c..0000000000 --- a/src/backend/job-execute/boot-job-execute/src/main/resources/i18n/message_en.properties +++ /dev/null @@ -1,124 +0,0 @@ -# -# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. -# -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. -# -# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. -# -# License for BK-JOB蓝鲸智云作业平台: -# -------------------------------------------------------------------- -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF -# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - -task.startup.mode.normal=Web UI -task.startup.mode.api=API -task.startup.mode.cron=Cron - -task.instance.type.normal=Job Execution -task.instance.type.script=Script Execution -task.instance.type.file=File Transfer - -task.run.status.blank=Pending -task.run.status.running=Running -task.run.status.success=Success -task.run.status.fail=Failed -task.run.status.skipped=Skipped -task.run.status.ignore_error=Ignore Error -task.run.status.waiting_user=Waiting -task.run.status.terminated=Terminated -task.run.status.abnormal_state=Abnormal -task.run.status.stopping=Terminating -task.run.status.stop_success=Terminate Success -task.run.status.confirm_terminated=Termination Confirmed -task.run.status.abandoned=Abandoned -task.run.status.rolling_waiting=Rolling Waiting - -agent.task.status.unknown_error=Unknown Error -agent.task.status.agent_error=Abnormal Agent -agent.task.status.last_success=Successful Last Time -agent.task.status.waiting=Waiting -agent.task.status.running=Running -agent.task.status.success=Success -agent.task.status.failed=Failed -agent.task.status.submit_failed=Task send failed -agent.task.status.task_timeout=Task Timeout -agent.task.status.log_error=Error in Log -agent.task.status.gse_script_timeout=Script Log Timeout -agent.task.status.gse_file_timeout=File Log Timeout -agent.task.status.script_failed=Script Execution Failed -agent.task.status.script_timeout=Script Execution Timeout -agent.task.status.script_terminate=Script Execution Terminated -agent.task.status.script_not_zero_exit_code=Non-zero Exit Code -agent.task.status.copyfile_failed=File Transmission Failed -agent.task.status.copyfile_source_file_not_exist=Source File Not Exists -agent.task.status.file_error_unclassified=Unclassified File Failure -agent.task.status.gse_timeout=File Transmission Timeout -agent.task.status.gse_agent_error=Abnormal Agent -agent.task.status.gse_user_error=User not exist -agent.task.status.gse_user_pwd_error=User password incorrect -agent.task.status.gse_file_error=File pull failed -agent.task.status.gse_file_size_exceed=File size exceeds limit -agent.task.status.gse_file_task_error=File Transmission Error -agent.task.status.gse_task_error=Execution Error -agent.task.status.gse_task_terminate_success=Force Terminated -agent.task.status.gse_task_terminate_failed=Force Terminate Failed -agent.task.status.host_not_exist=Invalid host -agent.task.status.unknown=Unknown - -user.operation.retry_step_fail=Failed-host Retry -user.operation.ignore_error=Ignore Error -user.operation.skip_step=Skip -user.operation.terminate_job=Force Terminate -user.operation.retry_step_all=Retry All -user.operation.start=Start -user.operation.confirm_continue=Confirm -user.operation.confirm_terminate=Confirm -user.operation.confirm_restart=Confirm -user.operation.next_step=Next step - -user.operation.detail.start.web=User start it by Web UI. -user.operation.detail.start.cron=Started by Cron Service. -user.operation.detail.start.api=Started by {0} call API. -user.operation.detail.confirm.terminate=User has confirmed to refute the task, reason: {0}. -user.operation.detail.confirm.continue=User has confirmed to continue execution, reason: {0}. -user.operation.detail.confirm.restart=User re-initiates confirmation. -user.operation.detail.retry_step_all=Click here to view the execution details before retrying. -user.operation.detail.retry_step_fail=Click here to view the execution details before retrying. -user.operation.detail.ignore_error=Ignored Failed-host, and continues. -user.operation.detail.next_step=User has confirmed to go directly to the next step. -user.operation.detail.terminate_job=Task has been force terminated. - -task.variable.type.name.string=String -task.variable.type.name.namespace=Namespace -task.variable.type.name.host_list=Host -task.variable.type.name.cipher=Cipher -task.variable.type.name.associative_array=Associative array -task.variable.type.name.index_array=Index array - -task.step.type.name.script=Script Execution -task.step.type.name.file=File Transfer -task.step.type.name.manual_confirm=User Confirmation - -task.type.name.fast_execute_script=Script Execution -task.type.name.fast_execute_sql=SQL Script -task.type.name.fast_push_file=File Transfer -task.type.name.job=Job - -resource.topo_node.name=node -resource.host.name=host -resource.dynamic_group.name=group -resource.biz_set.name=business set - -script.check.result.detect.dangerous.script=Dangerous script diff --git a/src/backend/job-execute/boot-job-execute/src/main/resources/i18n/message_en_US.properties b/src/backend/job-execute/boot-job-execute/src/main/resources/i18n/message_en_US.properties deleted file mode 100644 index f82a53a08b..0000000000 --- a/src/backend/job-execute/boot-job-execute/src/main/resources/i18n/message_en_US.properties +++ /dev/null @@ -1,124 +0,0 @@ -# -# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. -# -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. -# -# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. -# -# License for BK-JOB蓝鲸智云作业平台: -# -------------------------------------------------------------------- -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF -# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - -task.startup.mode.normal=Web UI -task.startup.mode.api=API -task.startup.mode.cron=Cron - -task.instance.type.normal=Job Execution -task.instance.type.script=Script Execution -task.instance.type.file=File Transfer - -task.run.status.blank=Pending -task.run.status.running=Running -task.run.status.success=Success -task.run.status.fail=Failed -task.run.status.skipped=Skipped -task.run.status.ignore_error=Ignore Error -task.run.status.waiting_user=Waiting -task.run.status.terminated=Terminated -task.run.status.abnormal_state=Abnormal -task.run.status.stopping=Terminating -task.run.status.stop_success=Terminate Success -task.run.status.confirm_terminated=Termination Confirmed -task.run.status.abandoned=Abandoned -task.run.status.rolling_waiting=Rolling Waiting - -agent.task.status.unknown_error=Unknown Error -agent.task.status.agent_error=Abnormal Agent -agent.task.status.last_success=Successful Last Time -agent.task.status.waiting=Waiting -agent.task.status.running=Running -agent.task.status.success=Success -agent.task.status.failed=Failed -agent.task.status.submit_failed=Task send failed -agent.task.status.task_timeout=Task Timeout -agent.task.status.log_error=Error in Log -agent.task.status.gse_script_timeout=Script Log Timeout -agent.task.status.gse_file_timeout=File Log Timeout -agent.task.status.script_failed=Script Execution Failed -agent.task.status.script_timeout=Script Execution Timeout -agent.task.status.script_terminate=Script Execution Terminated -agent.task.status.script_not_zero_exit_code=Non-zero Exit Code -agent.task.status.copyfile_failed=File Transmission Failed -agent.task.status.copyfile_source_file_not_exist=Source File Not Exists -agent.task.status.file_error_unclassified=Unclassified File Failure -agent.task.status.gse_timeout=File Transmission Timeout -agent.task.status.gse_agent_error=Abnormal Agent -agent.task.status.gse_user_error=User not exist -agent.task.status.gse_user_pwd_error=User password incorrect -agent.task.status.gse_file_error=File pull failed -agent.task.status.gse_file_size_exceed=File size exceeds limit -agent.task.status.gse_file_task_error=File Transmission Error -agent.task.status.gse_task_error=Execution Error -agent.task.status.gse_task_terminate_success=Force Terminated -agent.task.status.gse_task_terminate_failed=Force Terminate Failed -agent.task.status.host_not_exist=Invalid host -agent.task.status.unknown=Unknown - -user.operation.retry_step_fail=Failed-host Retry -user.operation.ignore_error=Ignore Error -user.operation.skip_step=Skip -user.operation.terminate_job=Force Terminate -user.operation.retry_step_all=Retry All -user.operation.start=Start -user.operation.confirm_continue=Confirm -user.operation.confirm_terminate=Confirm -user.operation.confirm_restart=Confirm -user.operation.next_step=Next step - -user.operation.detail.start.web=User start it by Web UI. -user.operation.detail.start.cron=Started by Cron Service. -user.operation.detail.start.api=Started by {0} call API. -user.operation.detail.confirm.terminate=User has confirmed to refute the task, reason: {}. -user.operation.detail.confirm.continue=User has confirmed to continue execution, reason: {}. -user.operation.detail.confirm.restart=User re-initiates confirmation. -user.operation.detail.retry_step_all=Click here to view the execution details before retrying. -user.operation.detail.retry_step_fail=Click here to view the execution details before retrying. -user.operation.detail.ignore_error=Ignored Failed-host, and continues. -user.operation.detail.next_step=User has confirmed to go directly to the next step. -user.operation.detail.terminate_job=Task has been force terminated. - -task.variable.type.name.string=String -task.variable.type.name.namespace=Namespace -task.variable.type.name.host_list=Host -task.variable.type.name.cipher=Cipher -task.variable.type.name.associative_array=Associative array -task.variable.type.name.index_array=Index array - -task.step.type.name.script=Script Execution -task.step.type.name.file=File Transfer -task.step.type.name.manual_confirm=User Confirmation - -task.type.name.fast_execute_script=Script Execution -task.type.name.fast_execute_sql=SQL Script -task.type.name.fast_push_file=File Transfer -task.type.name.job=Job - -resource.topo_node.name=node -resource.host.name=host -resource.dynamic_group.name=group -resource.biz_set.name=business set - -script.check.result.detect.dangerous.script=Dangerous script diff --git a/src/backend/job-execute/boot-job-execute/src/main/resources/i18n/message_zh.properties b/src/backend/job-execute/boot-job-execute/src/main/resources/i18n/message_zh.properties deleted file mode 100644 index e9c0fbcef3..0000000000 --- a/src/backend/job-execute/boot-job-execute/src/main/resources/i18n/message_zh.properties +++ /dev/null @@ -1,124 +0,0 @@ -# -# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. -# -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. -# -# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. -# -# License for BK-JOB蓝鲸智云作业平台: -# -------------------------------------------------------------------- -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF -# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - -task.startup.mode.normal=页面执行 -task.startup.mode.api=API调用 -task.startup.mode.cron=定时执行 - -task.instance.type.normal=作业执行 -task.instance.type.script=脚本执行 -task.instance.type.file=文件分发 - -task.run.status.blank=等待执行 -task.run.status.running=正在执行 -task.run.status.success=执行成功 -task.run.status.fail=执行失败 -task.run.status.skipped=跳过 -task.run.status.ignore_error=忽略错误 -task.run.status.waiting_user=等待确认 -task.run.status.terminated=手动结束 -task.run.status.abnormal_state=状态异常 -task.run.status.stopping=强制终止中 -task.run.status.stop_success=强制终止成功 -task.run.status.confirm_terminated=确认终止 -task.run.status.abandoned=被丢弃 -task.run.status.rolling_waiting=滚动等待 - -agent.task.status.unknown_error=未知错误 -agent.task.status.agent_error=Agent异常 -agent.task.status.last_success=上次已成功 -agent.task.status.waiting=等待执行 -agent.task.status.running=正在执行 -agent.task.status.success=执行成功 -agent.task.status.failed=任务失败 -agent.task.status.submit_failed=任务下发失败 -agent.task.status.task_timeout=任务超时 -agent.task.status.log_error=任务日志错误 -agent.task.status.gse_script_timeout=脚本日志超时 -agent.task.status.gse_file_timeout=文件日志超时 -agent.task.status.script_failed=脚本执行失败 -agent.task.status.script_timeout=脚本执行超时 -agent.task.status.script_terminate=脚本执行被终止 -agent.task.status.script_not_zero_exit_code=脚本返回码非零 -agent.task.status.copyfile_failed=文件传输失败 -agent.task.status.copyfile_source_file_not_exist=源文件不存在 -agent.task.status.file_error_unclassified=文件任务未分类错误 -agent.task.status.gse_timeout=执行任务超时 -agent.task.status.gse_agent_error=Agent异常 -agent.task.status.gse_user_error=用户名不存在 -agent.task.status.gse_user_pwd_error=用户密码不正确 -agent.task.status.gse_file_error=文件获取失败 -agent.task.status.gse_file_size_exceed=文件超出限制 -agent.task.status.gse_file_task_error=文件传输错误 -agent.task.status.gse_task_error=任务执行出错 -agent.task.status.gse_task_terminate_success=任务强制终止成功 -agent.task.status.gse_task_terminate_failed=任务强制终止失败 -agent.task.status.host_not_exist=无效主机 -agent.task.status.unknown=未知 - -user.operation.retry_step_fail=失败重试 -user.operation.ignore_error=忽略错误 -user.operation.skip_step=手动跳过 -user.operation.terminate_job=强制终止 -user.operation.retry_step_all=全部重试 -user.operation.start=启动作业 -user.operation.confirm_continue=人工确认 -user.operation.confirm_terminate=人工确认 -user.operation.confirm_restart=人工确认 -user.operation.next_step=进入下一步 - -user.operation.detail.start.web=用户从页面点击启动 -user.operation.detail.start.cron=由定时任务调度启动 -user.operation.detail.start.api=由 {0} 调用 API 启动 -user.operation.detail.confirm.terminate=用户已确认中止任务，理由：{0} -user.operation.detail.confirm.continue=用户已确认继续执行，理由：{0} -user.operation.detail.confirm.restart=用户重新发起确认 -user.operation.detail.retry_step_all=点击查看重试前的执行详情 -user.operation.detail.retry_step_fail=点击查看重试前的执行详情 -user.operation.detail.ignore_error=已忽略执行失败的目标，任务继续执行 -user.operation.detail.next_step=用户已确认直接进入下一步 -user.operation.detail.terminate_job=强制终止 - -task.variable.type.name.string=字符串 -task.variable.type.name.namespace=命名空间 -task.variable.type.name.host_list=主机列表 -task.variable.type.name.cipher=密文 -task.variable.type.name.associative_array=关联数组 -task.variable.type.name.index_array=索引数组 - -task.step.type.name.script=执行脚本 -task.step.type.name.file=分发文件 -task.step.type.name.manual_confirm=人工确认 - -task.type.name.fast_execute_script=快速执行脚本 -task.type.name.fast_execute_sql=快速执行SQL -task.type.name.fast_push_file=快速分发文件 -task.type.name.job=执行方案 - -resource.topo_node.name=节点 -resource.host.name=主机 -resource.dynamic_group.name=动态分组 -resource.biz_set.name=业务集 - -script.check.result.detect.dangerous.script=高危脚本 diff --git a/src/backend/job-execute/boot-job-execute/src/main/resources/i18n/message_zh_CN.properties b/src/backend/job-execute/boot-job-execute/src/main/resources/i18n/message_zh_CN.properties deleted file mode 100644 index e9c0fbcef3..0000000000 --- a/src/backend/job-execute/boot-job-execute/src/main/resources/i18n/message_zh_CN.properties +++ /dev/null @@ -1,124 +0,0 @@ -# -# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. -# -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. -# -# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. -# -# License for BK-JOB蓝鲸智云作业平台: -# -------------------------------------------------------------------- -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF -# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - -task.startup.mode.normal=页面执行 -task.startup.mode.api=API调用 -task.startup.mode.cron=定时执行 - -task.instance.type.normal=作业执行 -task.instance.type.script=脚本执行 -task.instance.type.file=文件分发 - -task.run.status.blank=等待执行 -task.run.status.running=正在执行 -task.run.status.success=执行成功 -task.run.status.fail=执行失败 -task.run.status.skipped=跳过 -task.run.status.ignore_error=忽略错误 -task.run.status.waiting_user=等待确认 -task.run.status.terminated=手动结束 -task.run.status.abnormal_state=状态异常 -task.run.status.stopping=强制终止中 -task.run.status.stop_success=强制终止成功 -task.run.status.confirm_terminated=确认终止 -task.run.status.abandoned=被丢弃 -task.run.status.rolling_waiting=滚动等待 - -agent.task.status.unknown_error=未知错误 -agent.task.status.agent_error=Agent异常 -agent.task.status.last_success=上次已成功 -agent.task.status.waiting=等待执行 -agent.task.status.running=正在执行 -agent.task.status.success=执行成功 -agent.task.status.failed=任务失败 -agent.task.status.submit_failed=任务下发失败 -agent.task.status.task_timeout=任务超时 -agent.task.status.log_error=任务日志错误 -agent.task.status.gse_script_timeout=脚本日志超时 -agent.task.status.gse_file_timeout=文件日志超时 -agent.task.status.script_failed=脚本执行失败 -agent.task.status.script_timeout=脚本执行超时 -agent.task.status.script_terminate=脚本执行被终止 -agent.task.status.script_not_zero_exit_code=脚本返回码非零 -agent.task.status.copyfile_failed=文件传输失败 -agent.task.status.copyfile_source_file_not_exist=源文件不存在 -agent.task.status.file_error_unclassified=文件任务未分类错误 -agent.task.status.gse_timeout=执行任务超时 -agent.task.status.gse_agent_error=Agent异常 -agent.task.status.gse_user_error=用户名不存在 -agent.task.status.gse_user_pwd_error=用户密码不正确 -agent.task.status.gse_file_error=文件获取失败 -agent.task.status.gse_file_size_exceed=文件超出限制 -agent.task.status.gse_file_task_error=文件传输错误 -agent.task.status.gse_task_error=任务执行出错 -agent.task.status.gse_task_terminate_success=任务强制终止成功 -agent.task.status.gse_task_terminate_failed=任务强制终止失败 -agent.task.status.host_not_exist=无效主机 -agent.task.status.unknown=未知 - -user.operation.retry_step_fail=失败重试 -user.operation.ignore_error=忽略错误 -user.operation.skip_step=手动跳过 -user.operation.terminate_job=强制终止 -user.operation.retry_step_all=全部重试 -user.operation.start=启动作业 -user.operation.confirm_continue=人工确认 -user.operation.confirm_terminate=人工确认 -user.operation.confirm_restart=人工确认 -user.operation.next_step=进入下一步 - -user.operation.detail.start.web=用户从页面点击启动 -user.operation.detail.start.cron=由定时任务调度启动 -user.operation.detail.start.api=由 {0} 调用 API 启动 -user.operation.detail.confirm.terminate=用户已确认中止任务，理由：{0} -user.operation.detail.confirm.continue=用户已确认继续执行，理由：{0} -user.operation.detail.confirm.restart=用户重新发起确认 -user.operation.detail.retry_step_all=点击查看重试前的执行详情 -user.operation.detail.retry_step_fail=点击查看重试前的执行详情 -user.operation.detail.ignore_error=已忽略执行失败的目标，任务继续执行 -user.operation.detail.next_step=用户已确认直接进入下一步 -user.operation.detail.terminate_job=强制终止 - -task.variable.type.name.string=字符串 -task.variable.type.name.namespace=命名空间 -task.variable.type.name.host_list=主机列表 -task.variable.type.name.cipher=密文 -task.variable.type.name.associative_array=关联数组 -task.variable.type.name.index_array=索引数组 - -task.step.type.name.script=执行脚本 -task.step.type.name.file=分发文件 -task.step.type.name.manual_confirm=人工确认 - -task.type.name.fast_execute_script=快速执行脚本 -task.type.name.fast_execute_sql=快速执行SQL -task.type.name.fast_push_file=快速分发文件 -task.type.name.job=执行方案 - -resource.topo_node.name=节点 -resource.host.name=主机 -resource.dynamic_group.name=动态分组 -resource.biz_set.name=业务集 - -script.check.result.detect.dangerous.script=高危脚本 diff --git a/src/backend/job-execute/boot-job-execute/src/main/resources/logback-spring.xml b/src/backend/job-execute/boot-job-execute/src/main/resources/logback-spring.xml new file mode 100644 index 0000000000..4b6595f631 --- /dev/null +++ b/src/backend/job-execute/boot-job-execute/src/main/resources/logback-spring.xml @@ -0,0 +1,85 @@ + + + logback + + + + + + + + + + + + + ${SCHEDULE_LOG_FILE} + + ${SCHEDULE_LOG_FILE}-%d{yyyyMMdd}.log.%i + 1GB + ${LOG_KEEP_DAYS} + 24GB + ${CLEAN_HISTORY_ON_START} + + + ${FILE_LOG_PATTERN} + UTF-8 + + + + + ${TASK_MONITOR_LOG_FILE} + + ${TASK_MONITOR_LOG_FILE}-%d{yyyy-MM-dd}.log.%i + 1GB + ${LOG_KEEP_DAYS} + 10GB + ${CLEAN_HISTORY_ON_START} + + + ${FILE_LOG_PATTERN} + UTF-8 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/backend/job-execute/boot-job-execute/src/main/resources/logback/logback-app-props.xml b/src/backend/job-execute/boot-job-execute/src/main/resources/logback/logback-app-props.xml deleted file mode 100644 index 88a771523e..0000000000 --- a/src/backend/job-execute/boot-job-execute/src/main/resources/logback/logback-app-props.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - diff --git a/src/backend/job-execute/boot-job-execute/src/main/resources/logback/logback-app.xml b/src/backend/job-execute/boot-job-execute/src/main/resources/logback/logback-app.xml deleted file mode 100644 index aad9a3d1de..0000000000 --- a/src/backend/job-execute/boot-job-execute/src/main/resources/logback/logback-app.xml +++ /dev/null @@ -1,73 +0,0 @@ - - - - - - ${SCHEDULE_LOG_FILE} - - ${SCHEDULE_LOG_FILE}-%d{yyyyMMdd}.log.%i - 1GB - 24 - 24GB - - - ${FILE_LOG_PATTERN} - UTF-8 - - - - - ${TASK_MONITOR_LOG_FILE} - - ${TASK_MONITOR_LOG_FILE}-%d{yyyy-MM-dd}.log.%i - 1GB - 3 - 10GB - - - ${LOG_PATTERN} - UTF-8 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/DangerousRecordDAOImplIntegrationTest.java b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/DangerousRecordDAOImplIntegrationTest.java index ac1974d2dc..e68eeb6bd7 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/DangerousRecordDAOImplIntegrationTest.java +++ b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/DangerousRecordDAOImplIntegrationTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -31,8 +31,8 @@ import com.tencent.bk.job.execute.model.DangerousRecordDTO; import com.tencent.bk.job.execute.model.ScriptCheckItemDTO; import com.tencent.bk.job.execute.model.ScriptCheckResultDTO; -import com.tencent.bk.job.manage.common.consts.RuleMatchHandleActionEnum; -import com.tencent.bk.job.manage.common.consts.script.ScriptCheckErrorLevelEnum; +import com.tencent.bk.job.manage.api.common.constants.RuleMatchHandleActionEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptCheckErrorLevelEnum; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; diff --git a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImplIntegrationTest.java b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImplIntegrationTest.java index c9fd43c502..706f8f639b 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImplIntegrationTest.java +++ b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImplIntegrationTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,9 +25,9 @@ package com.tencent.bk.job.execute.dao.impl; import com.tencent.bk.job.execute.dao.FileAgentTaskDAO; -import com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum; -import com.tencent.bk.job.execute.model.AgentTaskDTO; -import com.tencent.bk.job.execute.model.AgentTaskResultGroupBaseDTO; +import com.tencent.bk.job.execute.engine.consts.ExecuteObjectTaskStatusEnum; +import com.tencent.bk.job.execute.model.ExecuteObjectTask; +import com.tencent.bk.job.execute.model.ResultGroupBaseDTO; import com.tencent.bk.job.logsvr.consts.FileTaskModeEnum; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -59,12 +59,15 @@ public class FileAgentTaskDAOImplIntegrationTest { @DisplayName("根据主机ID获取Agent任务") public void testGetAgentTaskByHostId() { long hostId = 101L; + long taskInstanceId = 1L; long stepInstanceId = 1L; int executeCount = 0; int batch = 1; FileTaskModeEnum mode = FileTaskModeEnum.UPLOAD; - AgentTaskDTO agentTask = fileAgentTaskDAO.getAgentTaskByHostId(stepInstanceId, executeCount, batch, mode, hostId); + ExecuteObjectTask agentTask = fileAgentTaskDAO.getAgentTaskByHostId(taskInstanceId, + stepInstanceId, executeCount, batch, mode, hostId); + assertThat(agentTask.getTaskInstanceId()).isEqualTo(taskInstanceId); assertThat(agentTask.getStepInstanceId()).isEqualTo(stepInstanceId); assertThat(agentTask.getExecuteCount()).isEqualTo(executeCount); assertThat(agentTask.getBatch()).isEqualTo(batch); @@ -72,7 +75,7 @@ public void testGetAgentTaskByHostId() { assertThat(agentTask.getHostId()).isEqualTo(hostId); assertThat(agentTask.getAgentId()).isEqualTo("0:127.0.0.1"); assertThat(agentTask.getGseTaskId()).isEqualTo(1L); - assertThat(agentTask.getStatus()).isEqualTo(AgentTaskStatusEnum.SUCCESS); + assertThat(agentTask.getStatus()).isEqualTo(ExecuteObjectTaskStatusEnum.SUCCESS); assertThat(agentTask.getStartTime()).isEqualTo(1565767148000L); assertThat(agentTask.getEndTime()).isEqualTo(1565767149000L); assertThat(agentTask.getTotalTime()).isEqualTo(1000L); @@ -82,8 +85,9 @@ public void testGetAgentTaskByHostId() { @Test @DisplayName("批量新增Agent任务") public void testBatchSaveAgentTasks() { - List agentTaskList = new ArrayList<>(); - AgentTaskDTO agentTask1 = new AgentTaskDTO(); + List agentTaskList = new ArrayList<>(); + ExecuteObjectTask agentTask1 = new ExecuteObjectTask(); + agentTask1.setTaskInstanceId(100L); agentTask1.setStepInstanceId(100L); agentTask1.setExecuteCount(1); agentTask1.setActualExecuteCount(1); @@ -96,10 +100,11 @@ public void testBatchSaveAgentTasks() { agentTask1.setEndTime(1572858335000L); agentTask1.setTotalTime(1000L); agentTask1.setErrorCode(99); - agentTask1.setStatus(AgentTaskStatusEnum.AGENT_ERROR); + agentTask1.setStatus(ExecuteObjectTaskStatusEnum.AGENT_ERROR); agentTaskList.add(agentTask1); - AgentTaskDTO agentTask2 = new AgentTaskDTO(); + ExecuteObjectTask agentTask2 = new ExecuteObjectTask(); + agentTask2.setTaskInstanceId(100L); agentTask2.setStepInstanceId(100L); agentTask2.setExecuteCount(1); agentTask2.setActualExecuteCount(1); @@ -113,13 +118,14 @@ public void testBatchSaveAgentTasks() { agentTask2.setEndTime(1572858331000L); agentTask2.setTotalTime(1000L); agentTask2.setErrorCode(88); - agentTask2.setStatus(AgentTaskStatusEnum.HOST_NOT_EXIST); + agentTask2.setStatus(ExecuteObjectTaskStatusEnum.INVALID_EXECUTE_OBJECT); agentTaskList.add(agentTask2); fileAgentTaskDAO.batchSaveAgentTasks(agentTaskList); - AgentTaskDTO agentTask1Return = fileAgentTaskDAO.getAgentTaskByHostId(100L, 1, 1, + ExecuteObjectTask agentTask1Return = fileAgentTaskDAO.getAgentTaskByHostId(100L, 100L, 1, 1, FileTaskModeEnum.UPLOAD, 101L); + assertThat(agentTask1Return.getTaskInstanceId()).isEqualTo(100L); assertThat(agentTask1Return.getStepInstanceId()).isEqualTo(100L); assertThat(agentTask1Return.getExecuteCount()).isEqualTo(1L); assertThat(agentTask1Return.getActualExecuteCount()).isEqualTo(1L); @@ -132,11 +138,12 @@ public void testBatchSaveAgentTasks() { assertThat(agentTask1Return.getEndTime()).isEqualTo(1572858335000L); assertThat(agentTask1Return.getTotalTime()).isEqualTo(1000L); assertThat(agentTask1Return.getErrorCode()).isEqualTo(99); - assertThat(agentTask1Return.getStatus()).isEqualTo(AgentTaskStatusEnum.AGENT_ERROR); + assertThat(agentTask1Return.getStatus()).isEqualTo(ExecuteObjectTaskStatusEnum.AGENT_ERROR); - AgentTaskDTO agentTask2Return = fileAgentTaskDAO.getAgentTaskByHostId(100L, 1, 1, + ExecuteObjectTask agentTask2Return = fileAgentTaskDAO.getAgentTaskByHostId(100L, 100L, 1, 1, FileTaskModeEnum.DOWNLOAD, 102L); + assertThat(agentTask2Return.getTaskInstanceId()).isEqualTo(100L); assertThat(agentTask2Return.getStepInstanceId()).isEqualTo(100L); assertThat(agentTask2Return.getExecuteCount()).isEqualTo(1L); assertThat(agentTask2Return.getActualExecuteCount()).isEqualTo(1L); @@ -149,14 +156,15 @@ public void testBatchSaveAgentTasks() { assertThat(agentTask2Return.getEndTime()).isEqualTo(1572858331000L); assertThat(agentTask2Return.getTotalTime()).isEqualTo(1000L); assertThat(agentTask2Return.getErrorCode()).isEqualTo(88); - assertThat(agentTask2Return.getStatus()).isEqualTo(AgentTaskStatusEnum.HOST_NOT_EXIST); + assertThat(agentTask2Return.getStatus()).isEqualTo(ExecuteObjectTaskStatusEnum.INVALID_EXECUTE_OBJECT); } @Test @DisplayName("批量更新Agent任务") public void testBatchUpdateAgentTasks() { - List agentTaskList = new ArrayList<>(); - AgentTaskDTO agentTask1 = new AgentTaskDTO(); + List agentTaskList = new ArrayList<>(); + ExecuteObjectTask agentTask1 = new ExecuteObjectTask(); + agentTask1.setTaskInstanceId(1L); agentTask1.setStepInstanceId(1L); agentTask1.setExecuteCount(0); agentTask1.setBatch(2); @@ -168,10 +176,11 @@ public void testBatchUpdateAgentTasks() { agentTask1.setEndTime(1572858335000L); agentTask1.setTotalTime(1000L); agentTask1.setErrorCode(99); - agentTask1.setStatus(AgentTaskStatusEnum.AGENT_ERROR); + agentTask1.setStatus(ExecuteObjectTaskStatusEnum.AGENT_ERROR); agentTaskList.add(agentTask1); - AgentTaskDTO agentTask2 = new AgentTaskDTO(); + ExecuteObjectTask agentTask2 = new ExecuteObjectTask(); + agentTask2.setTaskInstanceId(1L); agentTask2.setStepInstanceId(1L); agentTask2.setExecuteCount(0); agentTask2.setBatch(2); @@ -184,13 +193,14 @@ public void testBatchUpdateAgentTasks() { agentTask2.setEndTime(1572858331000L); agentTask2.setTotalTime(1000L); agentTask2.setErrorCode(88); - agentTask2.setStatus(AgentTaskStatusEnum.HOST_NOT_EXIST); + agentTask2.setStatus(ExecuteObjectTaskStatusEnum.INVALID_EXECUTE_OBJECT); agentTaskList.add(agentTask2); fileAgentTaskDAO.batchUpdateAgentTasks(agentTaskList); - AgentTaskDTO agentTask1Return = fileAgentTaskDAO.getAgentTaskByHostId(1L, 0, 2, + ExecuteObjectTask agentTask1Return = fileAgentTaskDAO.getAgentTaskByHostId(1L, 1L, 0, 2, FileTaskModeEnum.UPLOAD, 101L); + assertThat(agentTask1Return.getTaskInstanceId()).isEqualTo(1L); assertThat(agentTask1Return.getStepInstanceId()).isEqualTo(1L); assertThat(agentTask1Return.getExecuteCount()).isEqualTo(0L); assertThat(agentTask1Return.getBatch()).isEqualTo(2); @@ -201,11 +211,12 @@ public void testBatchUpdateAgentTasks() { assertThat(agentTask1Return.getEndTime()).isEqualTo(1572858335000L); assertThat(agentTask1Return.getTotalTime()).isEqualTo(1000L); assertThat(agentTask1Return.getErrorCode()).isEqualTo(99); - assertThat(agentTask1Return.getStatus()).isEqualTo(AgentTaskStatusEnum.AGENT_ERROR); + assertThat(agentTask1Return.getStatus()).isEqualTo(ExecuteObjectTaskStatusEnum.AGENT_ERROR); - AgentTaskDTO agentTask2Return = fileAgentTaskDAO.getAgentTaskByHostId(1L, 0, 2, + ExecuteObjectTask agentTask2Return = fileAgentTaskDAO.getAgentTaskByHostId(1L, 1L, 0, 2, FileTaskModeEnum.DOWNLOAD, 103L); + assertThat(agentTask2Return.getTaskInstanceId()).isEqualTo(1L); assertThat(agentTask2Return.getStepInstanceId()).isEqualTo(1L); assertThat(agentTask2Return.getExecuteCount()).isEqualTo(0L); assertThat(agentTask2Return.getBatch()).isEqualTo(2); @@ -216,46 +227,46 @@ public void testBatchUpdateAgentTasks() { assertThat(agentTask2Return.getEndTime()).isEqualTo(1572858331000L); assertThat(agentTask2Return.getTotalTime()).isEqualTo(1000L); assertThat(agentTask2Return.getErrorCode()).isEqualTo(88); - assertThat(agentTask2Return.getStatus()).isEqualTo(AgentTaskStatusEnum.HOST_NOT_EXIST); + assertThat(agentTask2Return.getStatus()).isEqualTo(ExecuteObjectTaskStatusEnum.INVALID_EXECUTE_OBJECT); } @Test @DisplayName("Agent任务结果分组") public void listResultGroups() { - List resultGroups = fileAgentTaskDAO.listResultGroups(1L, 0, null); + List resultGroups = fileAgentTaskDAO.listResultGroups(1L, 1L, 0, null); assertThat(resultGroups.size()).isEqualTo(2); assertThat(resultGroups).extracting("status").containsOnly(9, 11); - AgentTaskResultGroupBaseDTO resultGroup = resultGroups.get(0); + ResultGroupBaseDTO resultGroup = resultGroups.get(0); if (resultGroup.getStatus().equals(9)) { - assertThat(resultGroup.getTotalAgentTasks()).isEqualTo(2); + assertThat(resultGroup.getTotal()).isEqualTo(2); } if (resultGroup.getStatus().equals(11)) { - assertThat(resultGroup.getTotalAgentTasks()).isEqualTo(1); + assertThat(resultGroup.getTotal()).isEqualTo(1); } // 根据滚动执行批次查询 - resultGroups = fileAgentTaskDAO.listResultGroups(1L, 0, 2); + resultGroups = fileAgentTaskDAO.listResultGroups(1L, 1L, 0, 2); assertThat(resultGroups.size()).isEqualTo(2); assertThat(resultGroups).extracting("status").containsOnly(9, 11); resultGroup = resultGroups.get(0); if (resultGroup.getStatus().equals(9)) { - assertThat(resultGroup.getTotalAgentTasks()).isEqualTo(1); + assertThat(resultGroup.getTotal()).isEqualTo(1); } if (resultGroup.getStatus().equals(11)) { - assertThat(resultGroup.getTotalAgentTasks()).isEqualTo(1); + assertThat(resultGroup.getTotal()).isEqualTo(1); } } @Test public void testListAgentTaskByResultGroup() { - List agentTasks = fileAgentTaskDAO.listAgentTaskByResultGroup(1L, 0, 2, 9); + List agentTasks = fileAgentTaskDAO.listAgentTaskByResultGroup(1L, 1L, 0, 2, 9); assertThat(agentTasks.size()).isEqualTo(1); assertThat(agentTasks.get(0).getStepInstanceId()).isEqualTo(1L); assertThat(agentTasks.get(0).getExecuteCount()).isEqualTo(0); assertThat(agentTasks.get(0).getBatch()).isEqualTo(2); - assertThat(agentTasks.get(0).getStatus()).isEqualTo(AgentTaskStatusEnum.SUCCESS); + assertThat(agentTasks.get(0).getStatus()).isEqualTo(ExecuteObjectTaskStatusEnum.SUCCESS); assertThat(agentTasks).extracting("hostId").containsOnly(103L); } diff --git a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/FileExecuteObjectTaskDAOImplIntegrationTest.java b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/FileExecuteObjectTaskDAOImplIntegrationTest.java new file mode 100644 index 0000000000..bf9fafb3bf --- /dev/null +++ b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/FileExecuteObjectTaskDAOImplIntegrationTest.java @@ -0,0 +1,272 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.dao.impl; + +import com.tencent.bk.job.common.constant.ExecuteObjectTypeEnum; +import com.tencent.bk.job.execute.dao.FileExecuteObjectTaskDAO; +import com.tencent.bk.job.execute.engine.consts.ExecuteObjectTaskStatusEnum; +import com.tencent.bk.job.execute.model.ExecuteObjectTask; +import com.tencent.bk.job.execute.model.ResultGroupBaseDTO; +import com.tencent.bk.job.logsvr.consts.FileTaskModeEnum; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.jdbc.Sql; +import org.springframework.test.context.jdbc.SqlConfig; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import java.util.ArrayList; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +@ExtendWith(SpringExtension.class) +@SpringBootTest +@ActiveProfiles("test") +@TestPropertySource(locations = "classpath:test.properties") +@SqlConfig(encoding = "utf-8") +@Sql({"/init_gse_file_execute_obj_task_data.sql"}) +public class FileExecuteObjectTaskDAOImplIntegrationTest { + @Autowired + private FileExecuteObjectTaskDAO fileExecuteObjectTaskDAO; + + @Test + @DisplayName("根据执行对象ID获取任务") + public void testGetTaskByExecuteObjectId() { + String executeObjectId = "1:101"; + long taskInstanceId = 1L; + long stepInstanceId = 1L; + int executeCount = 0; + int batch = 1; + FileTaskModeEnum mode = FileTaskModeEnum.UPLOAD; + ExecuteObjectTask executeObjectTask = fileExecuteObjectTaskDAO.getTaskByExecuteObjectId( + taskInstanceId, stepInstanceId, executeCount, batch, mode, executeObjectId); + + assertThat(executeObjectTask.getTaskInstanceId()).isEqualTo(taskInstanceId); + assertThat(executeObjectTask.getStepInstanceId()).isEqualTo(stepInstanceId); + assertThat(executeObjectTask.getExecuteCount()).isEqualTo(executeCount); + assertThat(executeObjectTask.getBatch()).isEqualTo(batch); + assertThat(executeObjectTask.getFileTaskMode()).isEqualTo(FileTaskModeEnum.UPLOAD); + assertThat(executeObjectTask.getExecuteObjectId()).isEqualTo(executeObjectId); + assertThat(executeObjectTask.getExecuteObjectType()).isEqualTo(ExecuteObjectTypeEnum.HOST); + assertThat(executeObjectTask.getGseTaskId()).isEqualTo(1L); + assertThat(executeObjectTask.getStatus()).isEqualTo(ExecuteObjectTaskStatusEnum.SUCCESS); + assertThat(executeObjectTask.getStartTime()).isEqualTo(1565767148000L); + assertThat(executeObjectTask.getEndTime()).isEqualTo(1565767149000L); + assertThat(executeObjectTask.getTotalTime()).isEqualTo(1000L); + assertThat(executeObjectTask.getErrorCode()).isEqualTo(0); + } + + @Test + @DisplayName("批量新增任务") + public void testBatchSaveTasks() { + List executeObjectTaskList = new ArrayList<>(); + ExecuteObjectTask executeObjectTask1 = new ExecuteObjectTask(); + executeObjectTask1.setTaskInstanceId(100L); + executeObjectTask1.setStepInstanceId(100L); + executeObjectTask1.setExecuteCount(1); + executeObjectTask1.setActualExecuteCount(1); + executeObjectTask1.setBatch(1); + executeObjectTask1.setFileTaskMode(FileTaskModeEnum.UPLOAD); + executeObjectTask1.setExecuteObjectId("1:101"); + executeObjectTask1.setExecuteObjectType(ExecuteObjectTypeEnum.HOST); + executeObjectTask1.setGseTaskId(1000L); + executeObjectTask1.setStartTime(1572858334000L); + executeObjectTask1.setEndTime(1572858335000L); + executeObjectTask1.setTotalTime(1000L); + executeObjectTask1.setErrorCode(99); + executeObjectTask1.setStatus(ExecuteObjectTaskStatusEnum.AGENT_ERROR); + executeObjectTaskList.add(executeObjectTask1); + + ExecuteObjectTask executeObjectTask2 = new ExecuteObjectTask(); + executeObjectTask2.setTaskInstanceId(100L); + executeObjectTask2.setStepInstanceId(100L); + executeObjectTask2.setExecuteCount(1); + executeObjectTask2.setActualExecuteCount(1); + executeObjectTask2.setBatch(1); + executeObjectTask2.setFileTaskMode(FileTaskModeEnum.DOWNLOAD); + executeObjectTask2.setExecuteObjectId("1:102"); + executeObjectTask2.setExecuteObjectType(ExecuteObjectTypeEnum.HOST); + executeObjectTask2.setGseTaskId(1001L); + executeObjectTask2.setErrorCode(88); + executeObjectTask2.setStartTime(1572858330000L); + executeObjectTask2.setEndTime(1572858331000L); + executeObjectTask2.setTotalTime(1000L); + executeObjectTask2.setErrorCode(88); + executeObjectTask2.setStatus(ExecuteObjectTaskStatusEnum.INVALID_EXECUTE_OBJECT); + executeObjectTaskList.add(executeObjectTask2); + + fileExecuteObjectTaskDAO.batchSaveTasks(executeObjectTaskList); + + ExecuteObjectTask executeObjectTask1Return = fileExecuteObjectTaskDAO.getTaskByExecuteObjectId(100L, 100L, 1, 1, + FileTaskModeEnum.UPLOAD, "1:101"); + assertThat(executeObjectTask1Return.getTaskInstanceId()).isEqualTo(100L); + assertThat(executeObjectTask1Return.getStepInstanceId()).isEqualTo(100L); + assertThat(executeObjectTask1Return.getExecuteCount()).isEqualTo(1L); + assertThat(executeObjectTask1Return.getActualExecuteCount()).isEqualTo(1L); + assertThat(executeObjectTask1Return.getBatch()).isEqualTo(1); + assertThat(executeObjectTask1Return.getFileTaskMode()).isEqualTo(FileTaskModeEnum.UPLOAD); + assertThat(executeObjectTask1Return.getExecuteObjectId()).isEqualTo("1:101"); + assertThat(executeObjectTask1Return.getExecuteObjectType()).isEqualTo(ExecuteObjectTypeEnum.HOST); + assertThat(executeObjectTask1Return.getGseTaskId()).isEqualTo(1000L); + assertThat(executeObjectTask1Return.getStartTime()).isEqualTo(1572858334000L); + assertThat(executeObjectTask1Return.getEndTime()).isEqualTo(1572858335000L); + assertThat(executeObjectTask1Return.getTotalTime()).isEqualTo(1000L); + assertThat(executeObjectTask1Return.getErrorCode()).isEqualTo(99); + assertThat(executeObjectTask1Return.getStatus()).isEqualTo(ExecuteObjectTaskStatusEnum.AGENT_ERROR); + + + ExecuteObjectTask executeObjectTask2Return = fileExecuteObjectTaskDAO.getTaskByExecuteObjectId(100L, 100L, 1, 1, + FileTaskModeEnum.DOWNLOAD, "1:102"); + assertThat(executeObjectTask2Return.getTaskInstanceId()).isEqualTo(100L); + assertThat(executeObjectTask2Return.getStepInstanceId()).isEqualTo(100L); + assertThat(executeObjectTask2Return.getExecuteCount()).isEqualTo(1L); + assertThat(executeObjectTask2Return.getActualExecuteCount()).isEqualTo(1L); + assertThat(executeObjectTask2Return.getBatch()).isEqualTo(1); + assertThat(executeObjectTask2Return.getFileTaskMode()).isEqualTo(FileTaskModeEnum.DOWNLOAD); + assertThat(executeObjectTask2Return.getExecuteObjectId()).isEqualTo("1:102"); + assertThat(executeObjectTask2Return.getExecuteObjectType()).isEqualTo(ExecuteObjectTypeEnum.HOST); + assertThat(executeObjectTask2Return.getGseTaskId()).isEqualTo(1001L); + assertThat(executeObjectTask2Return.getStartTime()).isEqualTo(1572858330000L); + assertThat(executeObjectTask2Return.getEndTime()).isEqualTo(1572858331000L); + assertThat(executeObjectTask2Return.getTotalTime()).isEqualTo(1000L); + assertThat(executeObjectTask2Return.getErrorCode()).isEqualTo(88); + assertThat(executeObjectTask2Return.getStatus()).isEqualTo(ExecuteObjectTaskStatusEnum.INVALID_EXECUTE_OBJECT); + } + + @Test + @DisplayName("批量更新任务") + public void testBatchUpdateAgentTasks() { + List executeObjectTaskList = new ArrayList<>(); + ExecuteObjectTask executeObjectTask1 = new ExecuteObjectTask(); + executeObjectTask1.setTaskInstanceId(1L); + executeObjectTask1.setStepInstanceId(1L); + executeObjectTask1.setExecuteCount(0); + executeObjectTask1.setBatch(2); + executeObjectTask1.setFileTaskMode(FileTaskModeEnum.UPLOAD); + executeObjectTask1.setExecuteObjectId("1:101"); + executeObjectTask1.setExecuteObjectType(ExecuteObjectTypeEnum.HOST); + executeObjectTask1.setGseTaskId(1000L); + executeObjectTask1.setStartTime(1572858334000L); + executeObjectTask1.setEndTime(1572858335000L); + executeObjectTask1.setTotalTime(1000L); + executeObjectTask1.setErrorCode(99); + executeObjectTask1.setStatus(ExecuteObjectTaskStatusEnum.AGENT_ERROR); + executeObjectTaskList.add(executeObjectTask1); + + ExecuteObjectTask executeObjectTask2 = new ExecuteObjectTask(); + executeObjectTask2.setTaskInstanceId(1L); + executeObjectTask2.setStepInstanceId(1L); + executeObjectTask2.setExecuteCount(0); + executeObjectTask2.setBatch(2); + executeObjectTask2.setFileTaskMode(FileTaskModeEnum.DOWNLOAD); + executeObjectTask2.setExecuteObjectId("1:103"); + executeObjectTask2.setExecuteObjectType(ExecuteObjectTypeEnum.HOST); + executeObjectTask2.setGseTaskId(1001L); + executeObjectTask2.setErrorCode(88); + executeObjectTask2.setStartTime(1572858330000L); + executeObjectTask2.setEndTime(1572858331000L); + executeObjectTask2.setTotalTime(1000L); + executeObjectTask2.setErrorCode(88); + executeObjectTask2.setStatus(ExecuteObjectTaskStatusEnum.INVALID_EXECUTE_OBJECT); + executeObjectTaskList.add(executeObjectTask2); + + fileExecuteObjectTaskDAO.batchUpdateTasks(executeObjectTaskList); + + ExecuteObjectTask executeObjectTask1Return = fileExecuteObjectTaskDAO.getTaskByExecuteObjectId(1L, 1L, 0, 2, + FileTaskModeEnum.UPLOAD, "1:101"); + assertThat(executeObjectTask1Return.getStepInstanceId()).isEqualTo(1L); + assertThat(executeObjectTask1Return.getExecuteCount()).isEqualTo(0L); + assertThat(executeObjectTask1Return.getBatch()).isEqualTo(2); + assertThat(executeObjectTask1Return.getExecuteObjectType()).isEqualTo(ExecuteObjectTypeEnum.HOST); + assertThat(executeObjectTask1Return.getExecuteObjectId()).isEqualTo("1:101"); + assertThat(executeObjectTask1Return.getGseTaskId()).isEqualTo(1000L); + assertThat(executeObjectTask1Return.getStartTime()).isEqualTo(1572858334000L); + assertThat(executeObjectTask1Return.getEndTime()).isEqualTo(1572858335000L); + assertThat(executeObjectTask1Return.getTotalTime()).isEqualTo(1000L); + assertThat(executeObjectTask1Return.getErrorCode()).isEqualTo(99); + assertThat(executeObjectTask1Return.getStatus()).isEqualTo(ExecuteObjectTaskStatusEnum.AGENT_ERROR); + + + ExecuteObjectTask executeObjectTask2Return = fileExecuteObjectTaskDAO.getTaskByExecuteObjectId(1L, 1L, 0, 2, + FileTaskModeEnum.DOWNLOAD, "1:103"); + assertThat(executeObjectTask2Return.getStepInstanceId()).isEqualTo(1L); + assertThat(executeObjectTask2Return.getExecuteCount()).isEqualTo(0L); + assertThat(executeObjectTask2Return.getBatch()).isEqualTo(2); + assertThat(executeObjectTask2Return.getGseTaskId()).isEqualTo(1001L); + assertThat(executeObjectTask2Return.getExecuteObjectId()).isEqualTo("1:103"); + assertThat(executeObjectTask2Return.getExecuteObjectType()).isEqualTo(ExecuteObjectTypeEnum.HOST); + assertThat(executeObjectTask2Return.getStartTime()).isEqualTo(1572858330000L); + assertThat(executeObjectTask2Return.getEndTime()).isEqualTo(1572858331000L); + assertThat(executeObjectTask2Return.getTotalTime()).isEqualTo(1000L); + assertThat(executeObjectTask2Return.getErrorCode()).isEqualTo(88); + assertThat(executeObjectTask2Return.getStatus()).isEqualTo(ExecuteObjectTaskStatusEnum.INVALID_EXECUTE_OBJECT); + } + + @Test + @DisplayName("任务结果分组") + public void listResultGroups() { + List resultGroups = fileExecuteObjectTaskDAO.listResultGroups(1L, 1L, 0, null); + + assertThat(resultGroups.size()).isEqualTo(2); + assertThat(resultGroups).extracting("status").containsOnly(9, 11); + ResultGroupBaseDTO resultGroup = resultGroups.get(0); + if (resultGroup.getStatus().equals(9)) { + assertThat(resultGroup.getTotal()).isEqualTo(2); + } + if (resultGroup.getStatus().equals(11)) { + assertThat(resultGroup.getTotal()).isEqualTo(1); + } + + // 根据滚动执行批次查询 + resultGroups = fileExecuteObjectTaskDAO.listResultGroups(1L, 1L, 0, 2); + + assertThat(resultGroups.size()).isEqualTo(2); + assertThat(resultGroups).extracting("status").containsOnly(9, 11); + resultGroup = resultGroups.get(0); + if (resultGroup.getStatus().equals(9)) { + assertThat(resultGroup.getTotal()).isEqualTo(1); + } + if (resultGroup.getStatus().equals(11)) { + assertThat(resultGroup.getTotal()).isEqualTo(1); + } + } + + @Test + public void testListAgentTaskByResultGroup() { + List executeObjectTasks = fileExecuteObjectTaskDAO.listTaskByResultGroup(1L, 1L, 0, 2, 9); + assertThat(executeObjectTasks.size()).isEqualTo(1); + assertThat(executeObjectTasks.get(0).getStepInstanceId()).isEqualTo(1L); + assertThat(executeObjectTasks.get(0).getExecuteCount()).isEqualTo(0); + assertThat(executeObjectTasks.get(0).getBatch()).isEqualTo(2); + assertThat(executeObjectTasks.get(0).getStatus()).isEqualTo(ExecuteObjectTaskStatusEnum.SUCCESS); + assertThat(executeObjectTasks).extracting("executeObjectId").containsOnly("1:103"); + } + +} diff --git a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/GseTaskDAOImplIntegrationTest.java b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/GseTaskDAOImplIntegrationTest.java index 159bea4f45..43b26d4e8b 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/GseTaskDAOImplIntegrationTest.java +++ b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/GseTaskDAOImplIntegrationTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -55,6 +55,7 @@ public class GseTaskDAOImplIntegrationTest { @DisplayName("保存GseTask") void saveGseTask() { GseTaskDTO gseTask = new GseTaskDTO(); + gseTask.setTaskInstanceId(9L); gseTask.setStepInstanceId(10L); gseTask.setExecuteCount(0); gseTask.setBatch(1); @@ -67,8 +68,9 @@ void saveGseTask() { long id = gseTaskDAO.saveGseTask(gseTask); assertThat(id).isGreaterThan(0); - GseTaskDTO savedGseTask = gseTaskDAO.getGseTask(id); + GseTaskDTO savedGseTask = gseTaskDAO.getGseTask(9L, id); + assertThat(savedGseTask.getTaskInstanceId()).isEqualTo(9L); assertThat(savedGseTask.getStepInstanceId()).isEqualTo(10L); assertThat(savedGseTask.getExecuteCount()).isEqualTo(0); assertThat(savedGseTask.getBatch()).isEqualTo(1); @@ -84,6 +86,7 @@ void saveGseTask() { void updateGseTask() { GseTaskDTO gseTask = new GseTaskDTO(); gseTask.setId(1L); + gseTask.setTaskInstanceId(1L); gseTask.setStepInstanceId(1L); gseTask.setExecuteCount(0); gseTask.setBatch(0); @@ -96,8 +99,9 @@ void updateGseTask() { boolean result = gseTaskDAO.updateGseTask(gseTask); assertThat(result).isTrue(); - GseTaskDTO savedGseTask = gseTaskDAO.getGseTask(1L); + GseTaskDTO savedGseTask = gseTaskDAO.getGseTask(1L, 1L); + assertThat(savedGseTask.getTaskInstanceId()).isEqualTo(1L); assertThat(savedGseTask.getStepInstanceId()).isEqualTo(1L); assertThat(savedGseTask.getExecuteCount()).isEqualTo(0); assertThat(savedGseTask.getBatch()).isEqualTo(0); diff --git a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/GseTaskIpLogDAOImplIntegrationTest.java b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/GseTaskIpLogDAOImplIntegrationTest.java deleted file mode 100644 index ba4f7bccfa..0000000000 --- a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/GseTaskIpLogDAOImplIntegrationTest.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.dao.impl; - -import com.tencent.bk.job.execute.dao.GseTaskIpLogDAO; -import com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum; -import com.tencent.bk.job.execute.model.AgentTaskDTO; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.test.context.SpringBootTest; -import org.springframework.test.context.ActiveProfiles; -import org.springframework.test.context.TestPropertySource; -import org.springframework.test.context.jdbc.Sql; -import org.springframework.test.context.jdbc.SqlConfig; -import org.springframework.test.context.junit.jupiter.SpringExtension; - -import java.util.List; - -import static org.assertj.core.api.Assertions.assertThat; - -@ExtendWith(SpringExtension.class) -@SpringBootTest -@ActiveProfiles("test") -@TestPropertySource(locations = "classpath:test.properties") -@SqlConfig(encoding = "utf-8") -@Sql({"/init_gse_task_ip_log_data.sql"}) -public class GseTaskIpLogDAOImplIntegrationTest { - @Autowired - private GseTaskIpLogDAO gseTaskIpLogDAO; - - @Test - public void testGetAgentTaskByIp() { - String ip = "0:127.0.0.1"; - long stepInstanceId = 1L; - int executeCount = 0; - AgentTaskDTO agentTask = gseTaskIpLogDAO.getAgentTaskByIp(stepInstanceId, executeCount, ip); - - assertThat(agentTask.getStepInstanceId()).isEqualTo(stepInstanceId); - assertThat(agentTask.getExecuteCount()).isEqualTo(executeCount); - assertThat(agentTask.getCloudIp()).isEqualTo(ip); - assertThat(agentTask.getStatus()).isEqualTo(AgentTaskStatusEnum.SUCCESS); - Long expectStartTime = 1565767148000L; - Long expectEndTime = 1565767149000L; - assertThat(agentTask.getStartTime()).isEqualTo(expectStartTime); - assertThat(agentTask.getEndTime()).isEqualTo(expectEndTime); - assertThat(agentTask.getTotalTime()).isEqualTo(1316L); - assertThat(agentTask.getErrorCode()).isEqualTo(0); - assertThat(agentTask.getExitCode()).isEqualTo(0); - assertThat(agentTask.getTag()).isEqualTo("succ"); - assertThat(agentTask.getScriptLogOffset()).isEqualTo(0); - } - - @Test - public void testGetSuccessAgentTaskCount() { - Integer count = gseTaskIpLogDAO.getSuccessAgentTaskCount(1L, 0); - assertThat(count).isEqualTo(2); - } - - @Test - public void testListAgentTaskByResultGroup() { - List agentTasks = gseTaskIpLogDAO.listAgentTaskByResultGroup(1L, 0, 9, "succ"); - assertThat(agentTasks.size()).isEqualTo(2); - assertThat(agentTasks).extracting("tag").containsOnly("succ", "succ"); - assertThat(agentTasks).extracting("stepInstanceId").containsOnly(1L, 1L); - } - -} diff --git a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/OperationLogDAOImplIntegrationTest.java b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/OperationLogDAOImplIntegrationTest.java index 4785355aea..c485437a74 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/OperationLogDAOImplIntegrationTest.java +++ b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/OperationLogDAOImplIntegrationTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/RollingConfigDAOImplIntegrationTest.java b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/RollingConfigDAOImplIntegrationTest.java index 388c6f2c1a..ba28fecd82 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/RollingConfigDAOImplIntegrationTest.java +++ b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/RollingConfigDAOImplIntegrationTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,9 +27,10 @@ import com.tencent.bk.job.common.constant.RollingModeEnum; import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.execute.dao.RollingConfigDAO; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; import com.tencent.bk.job.execute.model.RollingConfigDTO; import com.tencent.bk.job.execute.model.db.RollingConfigDetailDO; -import com.tencent.bk.job.execute.model.db.RollingHostsBatchDO; +import com.tencent.bk.job.execute.model.db.RollingExecuteObjectsBatchDO; import com.tencent.bk.job.execute.model.db.StepRollingConfigDO; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -64,36 +65,52 @@ public class RollingConfigDAOImplIntegrationTest { @DisplayName("根据ID获取滚动配置") void queryRollingConfig() { RollingConfigDTO savedTaskInstanceRollingConfig = - rollingConfigDAO.queryRollingConfigById(1L); + rollingConfigDAO.queryRollingConfigById(2L, 1L); assertThat(savedTaskInstanceRollingConfig.getId()).isEqualTo(1L); + assertThat(savedTaskInstanceRollingConfig.getTaskInstanceId()).isEqualTo(2L); assertThat(savedTaskInstanceRollingConfig.getConfigName()).isEqualTo("config1"); - assertThat(savedTaskInstanceRollingConfig.getTaskInstanceId()).isEqualTo(1L); assertThat(savedTaskInstanceRollingConfig.getConfigDetail()).isNotNull(); assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getExpr()).isEqualTo("1 10% 100%"); - assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getMode()).isEqualTo(RollingModeEnum.PAUSE_IF_FAIL.getValue()); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getMode()) + .isEqualTo(RollingModeEnum.PAUSE_IF_FAIL.getValue()); assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getName()).isEqualTo("config1"); assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getStepRollingConfigs()).hasSize(4); - Map stepRollingConfigs = savedTaskInstanceRollingConfig.getConfigDetail().getStepRollingConfigs(); + Map stepRollingConfigs = + savedTaskInstanceRollingConfig.getConfigDetail().getStepRollingConfigs(); assertThat(stepRollingConfigs.get(100L).isBatch()).isEqualTo(true); assertThat(stepRollingConfigs.get(101L).isBatch()).isEqualTo(false); assertThat(stepRollingConfigs.get(102L).isBatch()).isEqualTo(true); assertThat(stepRollingConfigs.get(103L).isBatch()).isEqualTo(true); - assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getHostsBatchList()).hasSize(3); - assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getHostsBatchList().get(0).getBatch()).isEqualTo(1); - assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getHostsBatchList().get(0).getHosts()).hasSize(1); - assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getHostsBatchList().get(0).getHosts().get(0).getBkCloudId()).isEqualTo(0L); - assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getHostsBatchList().get(0).getHosts().get(0).getIp()).isEqualTo("127.0.0.1"); - assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getHostsBatchList().get(1).getBatch()).isEqualTo(2); - assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getHostsBatchList().get(1).getHosts()).hasSize(1); - assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getHostsBatchList().get(1).getHosts().get(0).getBkCloudId()).isEqualTo(0L); - assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getHostsBatchList().get(1).getHosts().get(0).getIp()).isEqualTo("127.0.0.2"); - assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getHostsBatchList().get(2).getBatch()).isEqualTo(3); - assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getHostsBatchList().get(2).getHosts()).hasSize(2); - assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getHostsBatchList().get(2).getHosts().get(0).getBkCloudId()).isEqualTo(0L); - assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getHostsBatchList().get(2).getHosts().get(0).getIp()).isEqualTo("127.0.0.3"); - assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getHostsBatchList().get(2).getHosts().get(1).getBkCloudId()).isEqualTo(0L); - assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getHostsBatchList().get(2).getHosts().get(1).getIp()).isEqualTo("127.0.0.4"); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getExecuteObjectsBatchListCompatibly()).hasSize(3); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getExecuteObjectsBatchListCompatibly().get(0) + .getBatch()).isEqualTo(1); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getExecuteObjectsBatchListCompatibly().get(0) + .getExecuteObjectsCompatibly()).hasSize(1); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getExecuteObjectsBatchListCompatibly().get(0) + .getExecuteObjectsCompatibly().get(0).getHost().getBkCloudId()).isEqualTo(0L); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getExecuteObjectsBatchListCompatibly().get(0) + .getExecuteObjectsCompatibly().get(0).getHost().getIp()).isEqualTo("127.0.0.1"); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getExecuteObjectsBatchListCompatibly().get(1) + .getBatch()).isEqualTo(2); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getExecuteObjectsBatchListCompatibly().get(1) + .getExecuteObjectsCompatibly()).hasSize(1); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getExecuteObjectsBatchListCompatibly().get(1) + .getExecuteObjectsCompatibly().get(0).getHost().getBkCloudId()).isEqualTo(0L); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getExecuteObjectsBatchListCompatibly().get(1) + .getExecuteObjectsCompatibly().get(0).getHost().getIp()).isEqualTo("127.0.0.2"); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getExecuteObjectsBatchListCompatibly().get(2) + .getBatch()).isEqualTo(3); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getExecuteObjectsBatchListCompatibly().get(2) + .getExecuteObjectsCompatibly()).hasSize(2); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getExecuteObjectsBatchListCompatibly().get(2) + .getExecuteObjectsCompatibly().get(0).getHost().getBkCloudId()).isEqualTo(0L); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getExecuteObjectsBatchListCompatibly().get(2) + .getExecuteObjectsCompatibly().get(0).getHost().getIp()).isEqualTo("127.0.0.3"); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getExecuteObjectsBatchListCompatibly().get(2) + .getExecuteObjectsCompatibly().get(1).getHost().getBkCloudId()).isEqualTo(0L); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getExecuteObjectsBatchListCompatibly().get(2) + .getExecuteObjectsCompatibly().get(1).getHost().getIp()).isEqualTo("127.0.0.4"); } @Test @@ -118,37 +135,44 @@ void saveRollingConfig() { stepRollingConfigs.put(1002L, new StepRollingConfigDO(true)); stepRollingConfigs.put(1003L, new StepRollingConfigDO(true)); rollingConfig.setStepRollingConfigs(stepRollingConfigs); - List hostsBatchList = new ArrayList<>(); - List servers = new ArrayList<>(); - servers.add(new HostDTO(0L, "127.0.0.1")); - RollingHostsBatchDO hostBatch1 = new RollingHostsBatchDO(1, servers); - hostsBatchList.add(hostBatch1); - rollingConfig.setHostsBatchList(hostsBatchList); + List executeObjectsBatchList = new ArrayList<>(); + List executeObjects = new ArrayList<>(); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject((new HostDTO(1L, 0L, "127.0.0.1")))); + RollingExecuteObjectsBatchDO hostBatch1 = new RollingExecuteObjectsBatchDO(1, executeObjects); + executeObjectsBatchList.add(hostBatch1); + rollingConfig.setExecuteObjectsBatchList(executeObjectsBatchList); taskInstanceRollingConfig.setConfigDetail(rollingConfig); long rollingConfigId = rollingConfigDAO.saveRollingConfig(taskInstanceRollingConfig); assertThat(rollingConfigId).isGreaterThan(0); RollingConfigDTO savedTaskInstanceRollingConfig = - rollingConfigDAO.queryRollingConfigById(rollingConfigId); + rollingConfigDAO.queryRollingConfigById(10L, rollingConfigId); assertThat(savedTaskInstanceRollingConfig.getId()).isEqualTo(rollingConfigId); assertThat(savedTaskInstanceRollingConfig.getConfigName()).isEqualTo("default"); assertThat(savedTaskInstanceRollingConfig.getTaskInstanceId()).isEqualTo(10L); assertThat(savedTaskInstanceRollingConfig.getConfigDetail()).isNotNull(); assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getExpr()).isEqualTo("10%"); - assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getMode()).isEqualTo(RollingModeEnum.PAUSE_IF_FAIL.getValue()); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getMode()) + .isEqualTo(RollingModeEnum.PAUSE_IF_FAIL.getValue()); assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getName()).isEqualTo("default"); assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getStepRollingConfigs()).hasSize(4); - Map savedStepRollingConfigs = savedTaskInstanceRollingConfig.getConfigDetail().getStepRollingConfigs(); + Map savedStepRollingConfigs = + savedTaskInstanceRollingConfig.getConfigDetail().getStepRollingConfigs(); assertThat(savedStepRollingConfigs.get(1000L).isBatch()).isEqualTo(true); assertThat(savedStepRollingConfigs.get(1001L).isBatch()).isEqualTo(false); assertThat(savedStepRollingConfigs.get(1002L).isBatch()).isEqualTo(true); assertThat(savedStepRollingConfigs.get(1003L).isBatch()).isEqualTo(true); - assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getHostsBatchList()).hasSize(1); - assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getHostsBatchList().get(0).getBatch()).isEqualTo(1); - assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getHostsBatchList().get(0).getHosts().get(0).getBkCloudId()).isEqualTo(0L); - assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getHostsBatchList().get(0).getHosts().get(0).getIp()).isEqualTo("127.0.0.1"); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getExecuteObjectsBatchListCompatibly()).hasSize(1); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getExecuteObjectsBatchListCompatibly().get(0) + .getBatch()).isEqualTo(1); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getExecuteObjectsBatchListCompatibly().get(0) + .getExecuteObjectsCompatibly().get(0).getHost().getHostId()).isEqualTo(1L); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getExecuteObjectsBatchListCompatibly().get(0) + .getExecuteObjectsCompatibly().get(0).getHost().getBkCloudId()).isEqualTo(0L); + assertThat(savedTaskInstanceRollingConfig.getConfigDetail().getExecuteObjectsBatchListCompatibly().get(0) + .getExecuteObjectsCompatibly().get(0).getHost().getIp()).isEqualTo("127.0.0.1"); } } diff --git a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImplIntegrationTest.java b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImplIntegrationTest.java index 957fe52d1f..b55ff15006 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImplIntegrationTest.java +++ b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImplIntegrationTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,9 +25,9 @@ package com.tencent.bk.job.execute.dao.impl; import com.tencent.bk.job.execute.dao.ScriptAgentTaskDAO; -import com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum; -import com.tencent.bk.job.execute.model.AgentTaskDTO; -import com.tencent.bk.job.execute.model.AgentTaskResultGroupBaseDTO; +import com.tencent.bk.job.execute.engine.consts.ExecuteObjectTaskStatusEnum; +import com.tencent.bk.job.execute.model.ExecuteObjectTask; +import com.tencent.bk.job.execute.model.ResultGroupBaseDTO; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -58,18 +58,21 @@ public class ScriptAgentTaskDAOImplIntegrationTest { @DisplayName("根据主机获取Agent任务") public void testGetAgentTaskByHostId() { long hostId = 101L; + long taskInstanceId = 1L; long stepInstanceId = 1L; int executeCount = 0; int batch = 1; - AgentTaskDTO agentTask = scriptAgentTaskDAO.getAgentTaskByHostId(stepInstanceId, executeCount, batch, hostId); + ExecuteObjectTask agentTask = scriptAgentTaskDAO.getAgentTaskByHostId(taskInstanceId, + stepInstanceId, executeCount, batch, hostId); + assertThat(agentTask.getTaskInstanceId()).isEqualTo(taskInstanceId); assertThat(agentTask.getStepInstanceId()).isEqualTo(stepInstanceId); assertThat(agentTask.getExecuteCount()).isEqualTo(executeCount); assertThat(agentTask.getBatch()).isEqualTo(batch); assertThat(agentTask.getHostId()).isEqualTo(hostId); assertThat(agentTask.getAgentId()).isEqualTo("0:127.0.0.1"); assertThat(agentTask.getGseTaskId()).isEqualTo(1L); - assertThat(agentTask.getStatus()).isEqualTo(AgentTaskStatusEnum.SUCCESS); + assertThat(agentTask.getStatus()).isEqualTo(ExecuteObjectTaskStatusEnum.SUCCESS); Long expectStartTime = 1565767148000L; Long expectEndTime = 1565767149000L; assertThat(agentTask.getStartTime()).isEqualTo(expectStartTime); @@ -84,8 +87,9 @@ public void testGetAgentTaskByHostId() { @Test @DisplayName("批量新增Agent任务") public void testBatchSaveAgentTasks() { - List agentTaskList = new ArrayList<>(); - AgentTaskDTO agentTask1 = new AgentTaskDTO(); + List agentTaskList = new ArrayList<>(); + ExecuteObjectTask agentTask1 = new ExecuteObjectTask(); + agentTask1.setTaskInstanceId(100L); agentTask1.setStepInstanceId(100L); agentTask1.setExecuteCount(1); agentTask1.setActualExecuteCount(1); @@ -97,12 +101,13 @@ public void testBatchSaveAgentTasks() { agentTask1.setEndTime(1572858335000L); agentTask1.setTotalTime(1000L); agentTask1.setErrorCode(99); - agentTask1.setStatus(AgentTaskStatusEnum.AGENT_ERROR); + agentTask1.setStatus(ExecuteObjectTaskStatusEnum.AGENT_ERROR); agentTask1.setTag("aa"); agentTask1.setExitCode(1); agentTaskList.add(agentTask1); - AgentTaskDTO agentTask2 = new AgentTaskDTO(); + ExecuteObjectTask agentTask2 = new ExecuteObjectTask(); + agentTask2.setTaskInstanceId(100L); agentTask2.setStepInstanceId(100L); agentTask2.setExecuteCount(1); agentTask2.setActualExecuteCount(1); @@ -116,14 +121,15 @@ public void testBatchSaveAgentTasks() { agentTask2.setEndTime(1572858331000L); agentTask2.setTotalTime(1000L); agentTask2.setErrorCode(88); - agentTask2.setStatus(AgentTaskStatusEnum.HOST_NOT_EXIST); + agentTask2.setStatus(ExecuteObjectTaskStatusEnum.INVALID_EXECUTE_OBJECT); agentTask2.setTag("bb"); agentTask2.setExitCode(2); agentTaskList.add(agentTask2); scriptAgentTaskDAO.batchSaveAgentTasks(agentTaskList); - AgentTaskDTO agentTask1Return = scriptAgentTaskDAO.getAgentTaskByHostId(100L, 1, 1, 101L); + ExecuteObjectTask agentTask1Return = scriptAgentTaskDAO.getAgentTaskByHostId(100L, 100L, 1, 1, 101L); + assertThat(agentTask1Return.getTaskInstanceId()).isEqualTo(100L); assertThat(agentTask1Return.getStepInstanceId()).isEqualTo(100L); assertThat(agentTask1Return.getExecuteCount()).isEqualTo(1L); assertThat(agentTask1Return.getActualExecuteCount()).isEqualTo(1L); @@ -135,12 +141,13 @@ public void testBatchSaveAgentTasks() { assertThat(agentTask1Return.getEndTime()).isEqualTo(1572858335000L); assertThat(agentTask1Return.getTotalTime()).isEqualTo(1000L); assertThat(agentTask1Return.getErrorCode()).isEqualTo(99); - assertThat(agentTask1Return.getStatus()).isEqualTo(AgentTaskStatusEnum.AGENT_ERROR); + assertThat(agentTask1Return.getStatus()).isEqualTo(ExecuteObjectTaskStatusEnum.AGENT_ERROR); assertThat(agentTask1Return.getTag()).isEqualTo("aa"); assertThat(agentTask1Return.getExitCode()).isEqualTo(1); - AgentTaskDTO agentTask2Return = scriptAgentTaskDAO.getAgentTaskByHostId(100L, 1, 1, 102L); + ExecuteObjectTask agentTask2Return = scriptAgentTaskDAO.getAgentTaskByHostId(100L, 100L, 1, 1, 102L); + assertThat(agentTask2Return.getTaskInstanceId()).isEqualTo(100L); assertThat(agentTask2Return.getStepInstanceId()).isEqualTo(100L); assertThat(agentTask2Return.getExecuteCount()).isEqualTo(1L); assertThat(agentTask2Return.getActualExecuteCount()).isEqualTo(1L); @@ -152,7 +159,7 @@ public void testBatchSaveAgentTasks() { assertThat(agentTask2Return.getEndTime()).isEqualTo(1572858331000L); assertThat(agentTask2Return.getTotalTime()).isEqualTo(1000L); assertThat(agentTask2Return.getErrorCode()).isEqualTo(88); - assertThat(agentTask2Return.getStatus()).isEqualTo(AgentTaskStatusEnum.HOST_NOT_EXIST); + assertThat(agentTask2Return.getStatus()).isEqualTo(ExecuteObjectTaskStatusEnum.INVALID_EXECUTE_OBJECT); assertThat(agentTask2Return.getTag()).isEqualTo("bb"); assertThat(agentTask2Return.getExitCode()).isEqualTo(2); } @@ -160,8 +167,9 @@ public void testBatchSaveAgentTasks() { @Test @DisplayName("批量更新Agent任务") public void testBatchUpdateAgentTasks() { - List agentTaskList = new ArrayList<>(); - AgentTaskDTO agentTask1 = new AgentTaskDTO(); + List agentTaskList = new ArrayList<>(); + ExecuteObjectTask agentTask1 = new ExecuteObjectTask(); + agentTask1.setTaskInstanceId(1L); agentTask1.setStepInstanceId(1L); agentTask1.setExecuteCount(0); agentTask1.setBatch(3); @@ -172,12 +180,13 @@ public void testBatchUpdateAgentTasks() { agentTask1.setEndTime(1572858335000L); agentTask1.setTotalTime(1000L); agentTask1.setErrorCode(99); - agentTask1.setStatus(AgentTaskStatusEnum.AGENT_ERROR); + agentTask1.setStatus(ExecuteObjectTaskStatusEnum.AGENT_ERROR); agentTask1.setTag("aa"); agentTask1.setExitCode(1); agentTaskList.add(agentTask1); - AgentTaskDTO agentTask2 = new AgentTaskDTO(); + ExecuteObjectTask agentTask2 = new ExecuteObjectTask(); + agentTask2.setTaskInstanceId(1L); agentTask2.setStepInstanceId(1L); agentTask2.setExecuteCount(0); agentTask2.setBatch(3); @@ -190,14 +199,15 @@ public void testBatchUpdateAgentTasks() { agentTask2.setEndTime(1572858331000L); agentTask2.setTotalTime(1000L); agentTask2.setErrorCode(88); - agentTask2.setStatus(AgentTaskStatusEnum.HOST_NOT_EXIST); + agentTask2.setStatus(ExecuteObjectTaskStatusEnum.INVALID_EXECUTE_OBJECT); agentTask2.setTag("bb"); agentTask2.setExitCode(2); agentTaskList.add(agentTask2); scriptAgentTaskDAO.batchUpdateAgentTasks(agentTaskList); - AgentTaskDTO agentTask1Return = scriptAgentTaskDAO.getAgentTaskByHostId(1L, 0, 3, 103L); + ExecuteObjectTask agentTask1Return = scriptAgentTaskDAO.getAgentTaskByHostId(1L, 1L, 0, 3, 103L); + assertThat(agentTask1Return.getTaskInstanceId()).isEqualTo(1L); assertThat(agentTask1Return.getStepInstanceId()).isEqualTo(1L); assertThat(agentTask1Return.getExecuteCount()).isEqualTo(0L); assertThat(agentTask1Return.getBatch()).isEqualTo(3); @@ -208,12 +218,13 @@ public void testBatchUpdateAgentTasks() { assertThat(agentTask1Return.getEndTime()).isEqualTo(1572858335000L); assertThat(agentTask1Return.getTotalTime()).isEqualTo(1000L); assertThat(agentTask1Return.getErrorCode()).isEqualTo(99); - assertThat(agentTask1Return.getStatus()).isEqualTo(AgentTaskStatusEnum.AGENT_ERROR); + assertThat(agentTask1Return.getStatus()).isEqualTo(ExecuteObjectTaskStatusEnum.AGENT_ERROR); assertThat(agentTask1Return.getTag()).isEqualTo("aa"); assertThat(agentTask1Return.getExitCode()).isEqualTo(1); - AgentTaskDTO agentTask2Return = scriptAgentTaskDAO.getAgentTaskByHostId(1L, 0, 3, 104L); + ExecuteObjectTask agentTask2Return = scriptAgentTaskDAO.getAgentTaskByHostId(1L, 1L, 0, 3, 104L); + assertThat(agentTask2Return.getTaskInstanceId()).isEqualTo(1L); assertThat(agentTask2Return.getStepInstanceId()).isEqualTo(1L); assertThat(agentTask2Return.getExecuteCount()).isEqualTo(0L); assertThat(agentTask2Return.getBatch()).isEqualTo(3); @@ -224,56 +235,56 @@ public void testBatchUpdateAgentTasks() { assertThat(agentTask2Return.getEndTime()).isEqualTo(1572858331000L); assertThat(agentTask2Return.getTotalTime()).isEqualTo(1000L); assertThat(agentTask2Return.getErrorCode()).isEqualTo(88); - assertThat(agentTask2Return.getStatus()).isEqualTo(AgentTaskStatusEnum.HOST_NOT_EXIST); + assertThat(agentTask2Return.getStatus()).isEqualTo(ExecuteObjectTaskStatusEnum.INVALID_EXECUTE_OBJECT); assertThat(agentTask2Return.getTag()).isEqualTo("bb"); assertThat(agentTask2Return.getExitCode()).isEqualTo(2); } @Test public void testGetSuccessIpCount() { - Integer count = scriptAgentTaskDAO.getSuccessAgentTaskCount(1L, 0); + Integer count = scriptAgentTaskDAO.getSuccessAgentTaskCount(1L, 1L, 0); assertThat(count).isEqualTo(4); } @Test @DisplayName("Agent任务结果分组") public void listResultGroups() { - List resultGroups = scriptAgentTaskDAO.listResultGroups(1L, 0, null); + List resultGroups = scriptAgentTaskDAO.listResultGroups(1L, 1L, 0, null); assertThat(resultGroups.size()).isEqualTo(2); assertThat(resultGroups).extracting("status").containsOnly(9, 11); assertThat(resultGroups).extracting("tag").containsOnly("succ", "fail"); - AgentTaskResultGroupBaseDTO resultGroup = resultGroups.get(0); + ResultGroupBaseDTO resultGroup = resultGroups.get(0); if (resultGroup.getStatus().equals(9)) { - assertThat(resultGroup.getTotalAgentTasks()).isEqualTo(4); + assertThat(resultGroup.getTotal()).isEqualTo(4); } if (resultGroup.getStatus().equals(11)) { - assertThat(resultGroup.getTotalAgentTasks()).isEqualTo(1); + assertThat(resultGroup.getTotal()).isEqualTo(1); } // 根据滚动执行批次查询 - resultGroups = scriptAgentTaskDAO.listResultGroups(1L, 0, 3); + resultGroups = scriptAgentTaskDAO.listResultGroups(1L, 1L, 0, 3); assertThat(resultGroups.size()).isEqualTo(2); assertThat(resultGroups).extracting("status").containsOnly(9, 11); assertThat(resultGroups).extracting("tag").containsOnly("succ", "fail"); resultGroup = resultGroups.get(0); if (resultGroup.getStatus().equals(9)) { - assertThat(resultGroup.getTotalAgentTasks()).isEqualTo(2); + assertThat(resultGroup.getTotal()).isEqualTo(2); } if (resultGroup.getStatus().equals(11)) { - assertThat(resultGroup.getTotalAgentTasks()).isEqualTo(1); + assertThat(resultGroup.getTotal()).isEqualTo(1); } } @Test public void testListAgentTaskByResultGroup() { - List agentTasks = scriptAgentTaskDAO.listAgentTaskByResultGroup(1L, 0, 1, 9, "succ"); + List agentTasks = scriptAgentTaskDAO.listAgentTaskByResultGroup(1L, 1L, 0, 1, 9, "succ"); assertThat(agentTasks.size()).isEqualTo(1); assertThat(agentTasks.get(0).getStepInstanceId()).isEqualTo(1L); assertThat(agentTasks.get(0).getExecuteCount()).isEqualTo(0); assertThat(agentTasks.get(0).getBatch()).isEqualTo(1); - assertThat(agentTasks.get(0).getStatus()).isEqualTo(AgentTaskStatusEnum.SUCCESS); + assertThat(agentTasks.get(0).getStatus()).isEqualTo(ExecuteObjectTaskStatusEnum.SUCCESS); assertThat(agentTasks.get(0).getTag()).isEqualTo("succ"); } diff --git a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/ScriptExecuteObjectTaskDAOImplIntegrationTest.java b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/ScriptExecuteObjectTaskDAOImplIntegrationTest.java new file mode 100644 index 0000000000..1c3fba22dd --- /dev/null +++ b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/ScriptExecuteObjectTaskDAOImplIntegrationTest.java @@ -0,0 +1,295 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.dao.impl; + +import com.tencent.bk.job.common.constant.ExecuteObjectTypeEnum; +import com.tencent.bk.job.execute.dao.ScriptExecuteObjectTaskDAO; +import com.tencent.bk.job.execute.engine.consts.ExecuteObjectTaskStatusEnum; +import com.tencent.bk.job.execute.model.ExecuteObjectTask; +import com.tencent.bk.job.execute.model.ResultGroupBaseDTO; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.jdbc.Sql; +import org.springframework.test.context.jdbc.SqlConfig; +import org.springframework.test.context.junit.jupiter.SpringExtension; + +import java.util.ArrayList; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +@ExtendWith(SpringExtension.class) +@SpringBootTest +@ActiveProfiles("test") +@TestPropertySource(locations = "classpath:test.properties") +@SqlConfig(encoding = "utf-8") +@Sql({"/init_gse_script_execute_obj_task_data.sql"}) +public class ScriptExecuteObjectTaskDAOImplIntegrationTest { + @Autowired + private ScriptExecuteObjectTaskDAO scriptExecuteObjectTaskDAO; + + @Test + @DisplayName("根据执行对象 ID 获取任务") + public void testGetTaskByExecuteObjectId() { + String executeObjectId = "1:101"; + long stepInstanceId = 1L; + long taskInstanceId = 1L; + int executeCount = 0; + int batch = 1; + ExecuteObjectTask executeObjectTask = scriptExecuteObjectTaskDAO.getTaskByExecuteObjectId( + taskInstanceId, stepInstanceId, executeCount, batch, executeObjectId); + + assertThat(executeObjectTask.getStepInstanceId()).isEqualTo(stepInstanceId); + assertThat(executeObjectTask.getTaskInstanceId()).isEqualTo(taskInstanceId); + assertThat(executeObjectTask.getExecuteCount()).isEqualTo(executeCount); + assertThat(executeObjectTask.getBatch()).isEqualTo(batch); + assertThat(executeObjectTask.getExecuteObjectType()).isEqualTo(ExecuteObjectTypeEnum.HOST); + assertThat(executeObjectTask.getExecuteObjectId()).isEqualTo(executeObjectId); + assertThat(executeObjectTask.getGseTaskId()).isEqualTo(1L); + assertThat(executeObjectTask.getStatus()).isEqualTo(ExecuteObjectTaskStatusEnum.SUCCESS); + Long expectStartTime = 1565767148000L; + Long expectEndTime = 1565767149000L; + assertThat(executeObjectTask.getStartTime()).isEqualTo(expectStartTime); + assertThat(executeObjectTask.getEndTime()).isEqualTo(expectEndTime); + assertThat(executeObjectTask.getTotalTime()).isEqualTo(1316L); + assertThat(executeObjectTask.getErrorCode()).isEqualTo(0); + assertThat(executeObjectTask.getExitCode()).isEqualTo(0); + assertThat(executeObjectTask.getTag()).isEqualTo("succ"); + assertThat(executeObjectTask.getScriptLogOffset()).isEqualTo(0); + } + + @Test + @DisplayName("批量新增任务") + public void testBatchSaveTasks() { + List executeObjectTaskList = new ArrayList<>(); + ExecuteObjectTask executeObjectTask1 = new ExecuteObjectTask(); + executeObjectTask1.setTaskInstanceId(100L); + executeObjectTask1.setStepInstanceId(100L); + executeObjectTask1.setExecuteCount(1); + executeObjectTask1.setActualExecuteCount(1); + executeObjectTask1.setBatch(1); + executeObjectTask1.setExecuteObjectType(ExecuteObjectTypeEnum.HOST); + executeObjectTask1.setExecuteObjectId("1:101"); + executeObjectTask1.setGseTaskId(1000L); + executeObjectTask1.setStartTime(1572858334000L); + executeObjectTask1.setEndTime(1572858335000L); + executeObjectTask1.setTotalTime(1000L); + executeObjectTask1.setErrorCode(99); + executeObjectTask1.setStatus(ExecuteObjectTaskStatusEnum.AGENT_ERROR); + executeObjectTask1.setTag("aa"); + executeObjectTask1.setExitCode(1); + executeObjectTaskList.add(executeObjectTask1); + + ExecuteObjectTask executeObjectTask2 = new ExecuteObjectTask(); + executeObjectTask2.setTaskInstanceId(100L); + executeObjectTask2.setStepInstanceId(100L); + executeObjectTask2.setExecuteCount(1); + executeObjectTask2.setActualExecuteCount(1); + executeObjectTask2.setBatch(1); + executeObjectTask2.setExecuteObjectType(ExecuteObjectTypeEnum.HOST); + executeObjectTask2.setExecuteObjectId("1:102"); + executeObjectTask2.setGseTaskId(1001L); + executeObjectTask2.setErrorCode(88); + executeObjectTask2.setExitCode(1); + executeObjectTask2.setStartTime(1572858330000L); + executeObjectTask2.setEndTime(1572858331000L); + executeObjectTask2.setTotalTime(1000L); + executeObjectTask2.setErrorCode(88); + executeObjectTask2.setStatus(ExecuteObjectTaskStatusEnum.INVALID_EXECUTE_OBJECT); + executeObjectTask2.setTag("bb"); + executeObjectTask2.setExitCode(2); + executeObjectTaskList.add(executeObjectTask2); + + scriptExecuteObjectTaskDAO.batchSaveTasks(executeObjectTaskList); + + ExecuteObjectTask executeObjectTask1Return = scriptExecuteObjectTaskDAO + .getTaskByExecuteObjectId(100L, 100L, 1, 1, "1:101"); + assertThat(executeObjectTask1Return.getTaskInstanceId()).isEqualTo(100L); + assertThat(executeObjectTask1Return.getStepInstanceId()).isEqualTo(100L); + assertThat(executeObjectTask1Return.getExecuteCount()).isEqualTo(1L); + assertThat(executeObjectTask1Return.getActualExecuteCount()).isEqualTo(1L); + assertThat(executeObjectTask1Return.getBatch()).isEqualTo(1); + assertThat(executeObjectTask1Return.getExecuteObjectType()).isEqualTo(ExecuteObjectTypeEnum.HOST); + assertThat(executeObjectTask1Return.getExecuteObjectId()).isEqualTo("1:101"); + assertThat(executeObjectTask1Return.getGseTaskId()).isEqualTo(1000L); + assertThat(executeObjectTask1Return.getStartTime()).isEqualTo(1572858334000L); + assertThat(executeObjectTask1Return.getEndTime()).isEqualTo(1572858335000L); + assertThat(executeObjectTask1Return.getTotalTime()).isEqualTo(1000L); + assertThat(executeObjectTask1Return.getErrorCode()).isEqualTo(99); + assertThat(executeObjectTask1Return.getStatus()).isEqualTo(ExecuteObjectTaskStatusEnum.AGENT_ERROR); + assertThat(executeObjectTask1Return.getTag()).isEqualTo("aa"); + assertThat(executeObjectTask1Return.getExitCode()).isEqualTo(1); + + + ExecuteObjectTask executeObjectTask2Return = scriptExecuteObjectTaskDAO + .getTaskByExecuteObjectId(100L, 100L, 1, 1, "1:102"); + assertThat(executeObjectTask2Return.getStepInstanceId()).isEqualTo(100L); + assertThat(executeObjectTask2Return.getTaskInstanceId()).isEqualTo(100L); + assertThat(executeObjectTask2Return.getExecuteCount()).isEqualTo(1L); + assertThat(executeObjectTask2Return.getActualExecuteCount()).isEqualTo(1L); + assertThat(executeObjectTask2Return.getBatch()).isEqualTo(1); + assertThat(executeObjectTask2Return.getGseTaskId()).isEqualTo(1001L); + assertThat(executeObjectTask2Return.getExecuteObjectType()).isEqualTo(ExecuteObjectTypeEnum.HOST); + assertThat(executeObjectTask2Return.getExecuteObjectId()).isEqualTo("1:102"); + assertThat(executeObjectTask2Return.getStartTime()).isEqualTo(1572858330000L); + assertThat(executeObjectTask2Return.getEndTime()).isEqualTo(1572858331000L); + assertThat(executeObjectTask2Return.getTotalTime()).isEqualTo(1000L); + assertThat(executeObjectTask2Return.getErrorCode()).isEqualTo(88); + assertThat(executeObjectTask2Return.getStatus()).isEqualTo(ExecuteObjectTaskStatusEnum.INVALID_EXECUTE_OBJECT); + assertThat(executeObjectTask2Return.getTag()).isEqualTo("bb"); + assertThat(executeObjectTask2Return.getExitCode()).isEqualTo(2); + } + + @Test + @DisplayName("批量更新任务") + public void testBatchUpdateTasks() { + List executeObjectTaskList = new ArrayList<>(); + ExecuteObjectTask executeObjectTask1 = new ExecuteObjectTask(); + executeObjectTask1.setTaskInstanceId(1L); + executeObjectTask1.setStepInstanceId(1L); + executeObjectTask1.setExecuteCount(0); + executeObjectTask1.setBatch(3); + executeObjectTask1.setExecuteObjectId("1:103"); + executeObjectTask1.setExecuteObjectType(ExecuteObjectTypeEnum.HOST); + executeObjectTask1.setGseTaskId(1000L); + executeObjectTask1.setStartTime(1572858334000L); + executeObjectTask1.setEndTime(1572858335000L); + executeObjectTask1.setTotalTime(1000L); + executeObjectTask1.setErrorCode(99); + executeObjectTask1.setStatus(ExecuteObjectTaskStatusEnum.AGENT_ERROR); + executeObjectTask1.setTag("aa"); + executeObjectTask1.setExitCode(1); + executeObjectTaskList.add(executeObjectTask1); + + ExecuteObjectTask executeObjectTask2 = new ExecuteObjectTask(); + executeObjectTask2.setTaskInstanceId(1L); + executeObjectTask2.setStepInstanceId(1L); + executeObjectTask2.setExecuteCount(0); + executeObjectTask2.setBatch(3); + executeObjectTask2.setExecuteObjectId("1:104"); + executeObjectTask2.setExecuteObjectType(ExecuteObjectTypeEnum.HOST); + executeObjectTask2.setGseTaskId(1001L); + executeObjectTask2.setErrorCode(88); + executeObjectTask2.setExitCode(1); + executeObjectTask2.setStartTime(1572858330000L); + executeObjectTask2.setEndTime(1572858331000L); + executeObjectTask2.setTotalTime(1000L); + executeObjectTask2.setErrorCode(88); + executeObjectTask2.setStatus(ExecuteObjectTaskStatusEnum.INVALID_EXECUTE_OBJECT); + executeObjectTask2.setTag("bb"); + executeObjectTask2.setExitCode(2); + executeObjectTaskList.add(executeObjectTask2); + + scriptExecuteObjectTaskDAO.batchUpdateTasks(executeObjectTaskList); + + ExecuteObjectTask executeObjectTask1Return = + scriptExecuteObjectTaskDAO.getTaskByExecuteObjectId(1L, 1L, 0, 3, "1:103"); + assertThat(executeObjectTask1Return.getStepInstanceId()).isEqualTo(1L); + assertThat(executeObjectTask1Return.getExecuteCount()).isEqualTo(0L); + assertThat(executeObjectTask1Return.getBatch()).isEqualTo(3); + assertThat(executeObjectTask1Return.getExecuteObjectId()).isEqualTo("1:103"); + assertThat(executeObjectTask1Return.getExecuteObjectType()).isEqualTo(ExecuteObjectTypeEnum.HOST); + assertThat(executeObjectTask1Return.getGseTaskId()).isEqualTo(1000L); + assertThat(executeObjectTask1Return.getStartTime()).isEqualTo(1572858334000L); + assertThat(executeObjectTask1Return.getEndTime()).isEqualTo(1572858335000L); + assertThat(executeObjectTask1Return.getTotalTime()).isEqualTo(1000L); + assertThat(executeObjectTask1Return.getErrorCode()).isEqualTo(99); + assertThat(executeObjectTask1Return.getStatus()).isEqualTo(ExecuteObjectTaskStatusEnum.AGENT_ERROR); + assertThat(executeObjectTask1Return.getTag()).isEqualTo("aa"); + assertThat(executeObjectTask1Return.getExitCode()).isEqualTo(1); + + + ExecuteObjectTask executeObjectTask2Return = + scriptExecuteObjectTaskDAO.getTaskByExecuteObjectId(1L, 1L, 0, 3, "1:104"); + assertThat(executeObjectTask2Return.getStepInstanceId()).isEqualTo(1L); + assertThat(executeObjectTask2Return.getExecuteCount()).isEqualTo(0L); + assertThat(executeObjectTask2Return.getBatch()).isEqualTo(3); + assertThat(executeObjectTask2Return.getGseTaskId()).isEqualTo(1001L); + assertThat(executeObjectTask2Return.getExecuteObjectId()).isEqualTo("1:104"); + assertThat(executeObjectTask2Return.getExecuteObjectType()).isEqualTo(ExecuteObjectTypeEnum.HOST); + assertThat(executeObjectTask2Return.getStartTime()).isEqualTo(1572858330000L); + assertThat(executeObjectTask2Return.getEndTime()).isEqualTo(1572858331000L); + assertThat(executeObjectTask2Return.getTotalTime()).isEqualTo(1000L); + assertThat(executeObjectTask2Return.getErrorCode()).isEqualTo(88); + assertThat(executeObjectTask2Return.getStatus()).isEqualTo(ExecuteObjectTaskStatusEnum.INVALID_EXECUTE_OBJECT); + assertThat(executeObjectTask2Return.getTag()).isEqualTo("bb"); + assertThat(executeObjectTask2Return.getExitCode()).isEqualTo(2); + } + + @Test + public void testGetSuccessIpCount() { + Integer count = scriptExecuteObjectTaskDAO.getSuccessTaskCount(1L, 1L, 0); + assertThat(count).isEqualTo(4); + } + + @Test + @DisplayName("查询任务结果分组") + public void listResultGroups() { + List resultGroups = scriptExecuteObjectTaskDAO.listResultGroups(1L, 1L, 0, null); + + assertThat(resultGroups.size()).isEqualTo(2); + assertThat(resultGroups).extracting("status").containsOnly(9, 11); + assertThat(resultGroups).extracting("tag").containsOnly("succ", "fail"); + ResultGroupBaseDTO resultGroup = resultGroups.get(0); + if (resultGroup.getStatus().equals(9)) { + assertThat(resultGroup.getTotal()).isEqualTo(4); + } + if (resultGroup.getStatus().equals(11)) { + assertThat(resultGroup.getTotal()).isEqualTo(1); + } + + // 根据滚动执行批次查询 + resultGroups = scriptExecuteObjectTaskDAO.listResultGroups(1L, 1L, 0, 3); + + assertThat(resultGroups.size()).isEqualTo(2); + assertThat(resultGroups).extracting("status").containsOnly(9, 11); + assertThat(resultGroups).extracting("tag").containsOnly("succ", "fail"); + resultGroup = resultGroups.get(0); + if (resultGroup.getStatus().equals(9)) { + assertThat(resultGroup.getTotal()).isEqualTo(2); + } + if (resultGroup.getStatus().equals(11)) { + assertThat(resultGroup.getTotal()).isEqualTo(1); + } + } + + @Test + public void testListAgentTaskByResultGroup() { + List executeObjectTasks = scriptExecuteObjectTaskDAO.listTasksByResultGroup(1L, 1L, 0, 1, 9, + "succ"); + assertThat(executeObjectTasks.size()).isEqualTo(1); + assertThat(executeObjectTasks.get(0).getStepInstanceId()).isEqualTo(1L); + assertThat(executeObjectTasks.get(0).getExecuteCount()).isEqualTo(0); + assertThat(executeObjectTasks.get(0).getBatch()).isEqualTo(1); + assertThat(executeObjectTasks.get(0).getStatus()).isEqualTo(ExecuteObjectTaskStatusEnum.SUCCESS); + assertThat(executeObjectTasks.get(0).getTag()).isEqualTo("succ"); + } + +} diff --git a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/StepInstanceDAOImplIntegrationTest.java b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/StepInstanceDAOImplIntegrationTest.java index 785ba7156c..884b99c91b 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/StepInstanceDAOImplIntegrationTest.java +++ b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/StepInstanceDAOImplIntegrationTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -29,15 +29,15 @@ import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; import com.tencent.bk.job.execute.dao.StepInstanceDAO; import com.tencent.bk.job.execute.model.ConfirmStepInstanceDTO; +import com.tencent.bk.job.execute.model.ExecuteTargetDTO; import com.tencent.bk.job.execute.model.FileDetailDTO; import com.tencent.bk.job.execute.model.FileSourceDTO; import com.tencent.bk.job.execute.model.FileStepInstanceDTO; import com.tencent.bk.job.execute.model.ScriptStepInstanceDTO; -import com.tencent.bk.job.execute.model.ServersDTO; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskFileTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskFileTypeEnum; import org.assertj.core.util.Lists; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; @@ -67,8 +67,9 @@ public class StepInstanceDAOImplIntegrationTest { @Test public void testGetStepInstanceBase() { + long taskInstanceId = 1L; long stepInstanceId = 1L; - StepInstanceBaseDTO stepInstance = stepInstanceDAO.getStepInstanceBase(stepInstanceId); + StepInstanceBaseDTO stepInstance = stepInstanceDAO.getStepInstanceBase(taskInstanceId, stepInstanceId); assertThat(stepInstance.getId()).isEqualTo(stepInstanceId); assertThat(stepInstance.getAppId()).isEqualTo(2L); @@ -77,8 +78,7 @@ public void testGetStepInstanceBase() { assertThat(stepInstance.getStepNum()).isEqualTo(2); assertThat(stepInstance.getStepOrder()).isEqualTo(1); assertThat(stepInstance.getName()).isEqualTo("task1-step1"); - assertThat(stepInstance.getExecuteType()).isEqualTo(1); - assertThat(stepInstance.getIpList()).isEqualTo("0:127.0.0.1"); + assertThat(stepInstance.getExecuteType()).isEqualTo(StepExecuteTypeEnum.EXECUTE_SCRIPT); assertThat(stepInstance.getOperator()).isEqualTo("admin"); assertThat(stepInstance.getStatus()).isEqualTo(RunStatusEnum.SUCCESS); assertThat(stepInstance.getExecuteCount()).isEqualTo(0); @@ -86,10 +86,10 @@ public void testGetStepInstanceBase() { assertThat(stepInstance.getEndTime()).isEqualTo(1572868801000L); assertThat(stepInstance.getTotalTime()).isEqualTo(1111L); assertThat(stepInstance.getCreateTime()).isEqualTo(1572868800000L); - assertThat(stepInstance.getTargetServers()).isNotNull(); + assertThat(stepInstance.getTargetExecuteObjects()).isNotNull(); List expectedServer = new ArrayList<>(); expectedServer.add(new HostDTO(0L, "127.0.0.1")); - assertThat(stepInstance.getTargetServers().getIpList()).containsAll(expectedServer); + assertThat(stepInstance.getTargetExecuteObjects().getIpList()).containsAll(expectedServer); assertThat(stepInstance.getBatch()).isEqualTo(0); } @@ -100,12 +100,12 @@ public void testAddStepInstanceBase() { stepInstanceDTO.setName("task1-step1"); stepInstanceDTO.setTaskInstanceId(1L); stepInstanceDTO.setStepId(1L); - stepInstanceDTO.setExecuteType(StepExecuteTypeEnum.EXECUTE_SCRIPT.getValue()); - ServersDTO servers = new ServersDTO(); + stepInstanceDTO.setExecuteType(StepExecuteTypeEnum.EXECUTE_SCRIPT); + ExecuteTargetDTO executeTarget = new ExecuteTargetDTO(); List ipList = new ArrayList<>(); ipList.add(new HostDTO(0L, "127.0.0.1")); - servers.setIpList(ipList); - stepInstanceDTO.setTargetServers(servers); + executeTarget.setIpList(ipList); + stepInstanceDTO.setTargetExecuteObjects(executeTarget); stepInstanceDTO.setOperator("admin"); stepInstanceDTO.setStatus(RunStatusEnum.SUCCESS); stepInstanceDTO.setExecuteCount(0); @@ -125,12 +125,11 @@ public void testAddStepInstanceBase() { assertThat(returnStepInstance.getTaskInstanceId()).isEqualTo(1L); assertThat(returnStepInstance.getStepId()).isEqualTo(1L); assertThat(returnStepInstance.getName()).isEqualTo("task1-step1"); - assertThat(returnStepInstance.getExecuteType()).isEqualTo(StepExecuteTypeEnum.EXECUTE_SCRIPT.getValue()); - assertThat(returnStepInstance.getIpList()).isEqualTo("0:127.0.0.1"); - assertThat(returnStepInstance.getTargetServers().getIpList()).hasSize(1); + assertThat(returnStepInstance.getExecuteType()).isEqualTo(StepExecuteTypeEnum.EXECUTE_SCRIPT); + assertThat(returnStepInstance.getTargetExecuteObjects().getIpList()).hasSize(1); List expectedServer = new ArrayList<>(); expectedServer.add(new HostDTO(0L, "127.0.0.1")); - assertThat(returnStepInstance.getTargetServers().getIpList()).containsAll(expectedServer); + assertThat(returnStepInstance.getTargetExecuteObjects().getIpList()).containsAll(expectedServer); assertThat(returnStepInstance.getOperator()).isEqualTo("admin"); assertThat(returnStepInstance.getStatus()).isEqualTo(RunStatusEnum.SUCCESS); assertThat(returnStepInstance.getExecuteCount()).isEqualTo(0); @@ -156,12 +155,14 @@ public void testListStepInstanceBaseByTaskInstanceId() { @Test public void testResetStepStatus() { + long taskInstanceId = 1L; long stepInstanceId = 1L; - stepInstanceDAO.resetStepStatus(stepInstanceId); + stepInstanceDAO.resetStepStatus(taskInstanceId, stepInstanceId); - StepInstanceBaseDTO returnStepInstance = stepInstanceDAO.getStepInstanceBase(stepInstanceId); + StepInstanceBaseDTO returnStepInstance = stepInstanceDAO.getStepInstanceBase(taskInstanceId, stepInstanceId); assertThat(returnStepInstance.getId()).isEqualTo(stepInstanceId); + assertThat(returnStepInstance.getTaskInstanceId()).isEqualTo(taskInstanceId); assertThat(returnStepInstance.getStartTime()).isNull(); assertThat(returnStepInstance.getEndTime()).isNull(); assertThat(returnStepInstance.getTotalTime()).isNull(); @@ -169,90 +170,107 @@ public void testResetStepStatus() { @Test public void testAddStepRetryCount() { + long taskInstanceId = 1L; long stepInstanceId = 1L; - stepInstanceDAO.addStepExecuteCount(stepInstanceId); + stepInstanceDAO.addStepExecuteCount(taskInstanceId, stepInstanceId); - StepInstanceBaseDTO returnStepInstance = stepInstanceDAO.getStepInstanceBase(stepInstanceId); + StepInstanceBaseDTO returnStepInstance = stepInstanceDAO.getStepInstanceBase(taskInstanceId, stepInstanceId); + assertThat(returnStepInstance.getTaskInstanceId()).isEqualTo(taskInstanceId); assertThat(returnStepInstance.getId()).isEqualTo(stepInstanceId); assertThat(returnStepInstance.getExecuteCount()).isEqualTo(1); } @Test public void testUpdateStepStatus() { + long taskInstanceId = 1L; long stepInstanceId = 1L; - stepInstanceDAO.updateStepStatus(stepInstanceId, RunStatusEnum.RUNNING.getValue()); + stepInstanceDAO.updateStepStatus(taskInstanceId, stepInstanceId, RunStatusEnum.RUNNING.getValue()); StepInstanceBaseDTO returnStepInstance = stepInstanceDAO.getStepInstanceBase(stepInstanceId); + assertThat(returnStepInstance.getId()).isEqualTo(stepInstanceId); assertThat(returnStepInstance.getId()).isEqualTo(stepInstanceId); assertThat(returnStepInstance.getStatus()).isEqualTo(RunStatusEnum.RUNNING); } @Test public void testUpdateStepStartTime() { + long taskInstanceId = 3L; long stepInstanceId = 4L; long startTime = 1573041600000L; - stepInstanceDAO.updateStepStartTimeIfNull(stepInstanceId, startTime); - StepInstanceBaseDTO returnStepInstance = stepInstanceDAO.getStepInstanceBase(stepInstanceId); + stepInstanceDAO.updateStepStartTimeIfNull(taskInstanceId, stepInstanceId, startTime); + StepInstanceBaseDTO returnStepInstance = stepInstanceDAO.getStepInstanceBase(taskInstanceId, stepInstanceId); + assertThat(returnStepInstance.getTaskInstanceId()).isEqualTo(taskInstanceId); assertThat(returnStepInstance.getId()).isEqualTo(stepInstanceId); assertThat(returnStepInstance.getStartTime()).isEqualTo(startTime); stepInstanceId = 1L; + taskInstanceId = 1L; startTime = 1573041700000L; - stepInstanceDAO.updateStepStartTimeIfNull(stepInstanceId, startTime); - returnStepInstance = stepInstanceDAO.getStepInstanceBase(stepInstanceId); + stepInstanceDAO.updateStepStartTimeIfNull(taskInstanceId, stepInstanceId, startTime); + returnStepInstance = stepInstanceDAO.getStepInstanceBase(taskInstanceId, stepInstanceId); + assertThat(returnStepInstance.getTaskInstanceId()).isEqualTo(taskInstanceId); assertThat(returnStepInstance.getId()).isEqualTo(stepInstanceId); assertThat(returnStepInstance.getStartTime()).isEqualTo(1572868800000L); } @Test void testUpdateStepStartTimeIfNull() { + long taskInstanceId = 1L; long stepInstanceId = 1L; long startTime = 1573041600000L; - stepInstanceDAO.updateStepStartTime(stepInstanceId, startTime); + stepInstanceDAO.updateStepStartTime(taskInstanceId, stepInstanceId, startTime); - StepInstanceBaseDTO returnStepInstance = stepInstanceDAO.getStepInstanceBase(stepInstanceId); + StepInstanceBaseDTO returnStepInstance = stepInstanceDAO.getStepInstanceBase(taskInstanceId, stepInstanceId); + assertThat(returnStepInstance.getTaskInstanceId()).isEqualTo(taskInstanceId); assertThat(returnStepInstance.getId()).isEqualTo(stepInstanceId); assertThat(returnStepInstance.getStartTime()).isEqualTo(startTime); } @Test public void testUpdateStepEndTime() { + long taskInstanceId = 1L; long stepInstanceId = 1L; long endTime = 1573041600000L; - stepInstanceDAO.updateStepEndTime(stepInstanceId, endTime); + stepInstanceDAO.updateStepEndTime(taskInstanceId, stepInstanceId, endTime); - StepInstanceBaseDTO returnStepInstance = stepInstanceDAO.getStepInstanceBase(stepInstanceId); + StepInstanceBaseDTO returnStepInstance = stepInstanceDAO.getStepInstanceBase(taskInstanceId, stepInstanceId); + assertThat(returnStepInstance.getTaskInstanceId()).isEqualTo(taskInstanceId); assertThat(returnStepInstance.getId()).isEqualTo(stepInstanceId); assertThat(returnStepInstance.getEndTime()).isEqualTo(endTime); } @Test public void testUpdateStepTotalTime() { + long taskInstanceId = 1L; long stepInstanceId = 1L; long totalTime = 1234L; - stepInstanceDAO.updateStepTotalTime(stepInstanceId, totalTime); + stepInstanceDAO.updateStepTotalTime(taskInstanceId, stepInstanceId, totalTime); - StepInstanceBaseDTO returnStepInstance = stepInstanceDAO.getStepInstanceBase(stepInstanceId); + StepInstanceBaseDTO returnStepInstance = stepInstanceDAO.getStepInstanceBase(taskInstanceId, stepInstanceId); + assertThat(returnStepInstance.getTaskInstanceId()).isEqualTo(taskInstanceId); assertThat(returnStepInstance.getId()).isEqualTo(stepInstanceId); assertThat(returnStepInstance.getTotalTime()).isEqualTo(totalTime); } @Test public void testResetStepExecuteInfoForRetry() { + long taskInstanceId = 1L; long stepInstanceId = 1L; - stepInstanceDAO.resetStepExecuteInfoForRetry(stepInstanceId); - StepInstanceBaseDTO returnStepInstance = stepInstanceDAO.getStepInstanceBase(stepInstanceId); + stepInstanceDAO.resetStepExecuteInfoForRetry(taskInstanceId, stepInstanceId); + StepInstanceBaseDTO returnStepInstance = stepInstanceDAO.getStepInstanceBase(taskInstanceId, stepInstanceId); assertThat(returnStepInstance).isNotNull(); + assertThat(returnStepInstance.getTaskInstanceId()).isEqualTo(taskInstanceId); + assertThat(returnStepInstance.getId()).isEqualTo(stepInstanceId); assertThat(returnStepInstance.getStartTime()).isNotNull(); assertThat(returnStepInstance.getEndTime()).isNull(); assertThat(returnStepInstance.getStatus()).isEqualTo(RunStatusEnum.RUNNING); @@ -261,8 +279,12 @@ public void testResetStepExecuteInfoForRetry() { @Test public void testGetScriptStepInstance() { - ScriptStepInstanceDTO returnStepInstance = stepInstanceDAO.getScriptStepInstance(1L); - assertThat(returnStepInstance.getStepInstanceId()).isEqualTo(1L); + long taskInstanceId = 1L; + long stepInstanceId = 1L; + ScriptStepInstanceDTO returnStepInstance = stepInstanceDAO.getScriptStepInstance(taskInstanceId, + stepInstanceId); + assertThat(returnStepInstance.getTaskInstanceId()).isEqualTo(taskInstanceId); + assertThat(returnStepInstance.getStepInstanceId()).isEqualTo(stepInstanceId); assertThat(returnStepInstance.getAccount()).isEqualTo("root"); assertThat(returnStepInstance.getAccountId()).isEqualTo(1L); assertThat(returnStepInstance.getDbPass()).isEqualTo("db_password"); @@ -273,15 +295,16 @@ public void testGetScriptStepInstance() { assertThat(returnStepInstance.getScriptContent()).isEqualTo("script_content"); assertThat(returnStepInstance.getScriptParam()).isEqualTo("${var1}"); assertThat(returnStepInstance.getResolvedScriptParam()).isEqualTo("var1"); - assertThat(returnStepInstance.getScriptType()).isEqualTo(1); + assertThat(returnStepInstance.getScriptType()).isEqualTo(ScriptTypeEnum.SHELL); assertThat(returnStepInstance.getTimeout()).isEqualTo(1000); - assertThat(returnStepInstance.isSecureParam()).isEqualTo(false); + assertThat(returnStepInstance.isSecureParam()).isEqualTo(true); } @Test public void testAddScriptStepInstance() { StepInstanceDTO scriptStepInstance = new StepInstanceDTO(); scriptStepInstance.setId(100L); + scriptStepInstance.setTaskInstanceId(100L); scriptStepInstance.setAccount("root"); scriptStepInstance.setAccountId(100L); scriptStepInstance.setAccountAlias("root"); @@ -293,12 +316,13 @@ public void testAddScriptStepInstance() { scriptStepInstance.setScriptContent("script_content"); scriptStepInstance.setScriptParam("${var1} ${var2}"); scriptStepInstance.setResolvedScriptParam("var1 var2"); - scriptStepInstance.setScriptType(ScriptTypeEnum.SHELL.getValue()); + scriptStepInstance.setScriptType(ScriptTypeEnum.SHELL); scriptStepInstance.setTimeout(1000); stepInstanceDAO.addScriptStepInstance(scriptStepInstance); - ScriptStepInstanceDTO savedStepInstance = stepInstanceDAO.getScriptStepInstance(100L); + ScriptStepInstanceDTO savedStepInstance = stepInstanceDAO.getScriptStepInstance(100L, 100L); + assertThat(savedStepInstance.getTaskInstanceId()).isEqualTo(100L); assertThat(savedStepInstance.getStepInstanceId()).isEqualTo(100L); assertThat(savedStepInstance.getAccount()).isEqualTo("root"); assertThat(savedStepInstance.getAccountId()).isEqualTo(100L); @@ -310,7 +334,7 @@ public void testAddScriptStepInstance() { assertThat(savedStepInstance.getScriptContent()).isEqualTo("script_content"); assertThat(savedStepInstance.getScriptParam()).isEqualTo("${var1} ${var2}"); assertThat(savedStepInstance.getResolvedScriptParam()).isEqualTo("var1 var2"); - assertThat(savedStepInstance.getScriptType()).isEqualTo(ScriptTypeEnum.SHELL.getValue()); + assertThat(savedStepInstance.getScriptType()).isEqualTo(ScriptTypeEnum.SHELL); assertThat(savedStepInstance.getTimeout()).isEqualTo(1000); } @@ -318,6 +342,7 @@ public void testAddScriptStepInstance() { public void testAddFileStepInstance() { StepInstanceDTO fileStepInstance = new StepInstanceDTO(); fileStepInstance.setId(101L); + fileStepInstance.setTaskInstanceId(101L); fileStepInstance.setAccount("root"); fileStepInstance.setAccountId(1L); fileStepInstance.setAccountAlias("root"); @@ -335,9 +360,9 @@ public void testAddFileStepInstance() { fileSource.setAccountId(1L); fileSource.setLocalUpload(false); fileSource.setFileType(TaskFileTypeEnum.SERVER.getType()); - ServersDTO fileSourceServers = new ServersDTO(); - fileSourceServers.setIpList(Lists.newArrayList(new HostDTO(1L, "10.10.10.10"))); - fileSource.setServers(fileSourceServers); + ExecuteTargetDTO fileSourceExecuteTarget = new ExecuteTargetDTO(); + fileSourceExecuteTarget.setIpList(Lists.newArrayList(new HostDTO(1L, "10.10.10.10"))); + fileSource.setServers(fileSourceExecuteTarget); FileDetailDTO fileDetail = new FileDetailDTO(); fileDetail.setFilePath("/tmp/1.log"); fileDetail.setFileHash("hash"); @@ -348,7 +373,8 @@ public void testAddFileStepInstance() { stepInstanceDAO.addFileStepInstance(fileStepInstance); - FileStepInstanceDTO savedStepInstance = stepInstanceDAO.getFileStepInstance(101L); + FileStepInstanceDTO savedStepInstance = stepInstanceDAO.getFileStepInstance(101L, 101L); + assertThat(savedStepInstance.getTaskInstanceId()).isEqualTo(101L); assertThat(savedStepInstance.getStepInstanceId()).isEqualTo(101L); assertThat(savedStepInstance.getAccount()).isEqualTo("root"); assertThat(savedStepInstance.getAccountId()).isEqualTo(1L); @@ -367,7 +393,7 @@ public void testAddFileStepInstance() { assertThat(savedStepInstance.getFileSourceList().get(0).getServers().getIpList()).isNotEmpty(); assertThat(savedStepInstance.getFileSourceList().get(0).getServers().getIpList()) .containsOnly(new HostDTO(1L, - "10.10.10.10")); + "10.10.10.10")); assertThat(savedStepInstance.getFileSourceList().get(0).getFiles()).isNotEmpty(); assertThat(savedStepInstance.getFileSourceList().get(0).getFiles().get(0).getFilePath()) .isEqualTo("/tmp/1.log"); @@ -375,14 +401,13 @@ public void testAddFileStepInstance() { @Test public void testGetFileStepInstance() { - FileStepInstanceDTO returnStepInstance = stepInstanceDAO.getFileStepInstance(2L); + FileStepInstanceDTO returnStepInstance = stepInstanceDAO.getFileStepInstance(1L, 2L); assertThat(returnStepInstance).isNotNull(); + assertThat(returnStepInstance.getTaskInstanceId()).isEqualTo(1L); assertThat(returnStepInstance.getStepInstanceId()).isEqualTo(2L); assertThat(returnStepInstance.getFileSourceList()).isNotEmpty(); assertThat(returnStepInstance.getFileSourceList().get(0).getFiles().get(0).getFilePath()).isEqualTo("/$" + "{log_dir}/1.log"); - assertThat(returnStepInstance.getResolvedFileSourceList().get(0).getFiles().get(0).getResolvedFilePath()) - .isEqualTo("/tmp/1.log"); assertThat(returnStepInstance.getFileTargetPath()).isEqualTo("/${log_dir}/"); assertThat(returnStepInstance.getFileTargetName()).isEqualTo("2.log"); assertThat(returnStepInstance.getResolvedFileTargetPath()).isEqualTo("/tmp/"); @@ -398,9 +423,10 @@ public void testGetFileStepInstance() { @Test public void testGetConfirmStepInstance() { - ConfirmStepInstanceDTO returnStepInstance = stepInstanceDAO.getConfirmStepInstance(3L); + ConfirmStepInstanceDTO returnStepInstance = stepInstanceDAO.getConfirmStepInstance(13L, 15L); assertThat(returnStepInstance).isNotNull(); - assertThat(returnStepInstance.getStepInstanceId()).isEqualTo(3L); + assertThat(returnStepInstance.getTaskInstanceId()).isEqualTo(13L); + assertThat(returnStepInstance.getStepInstanceId()).isEqualTo(15L); assertThat(returnStepInstance.getConfirmMessage()).isEqualTo("confirm_message"); assertThat(returnStepInstance.getConfirmReason()).isEqualTo("confirm_reason"); assertThat(returnStepInstance.getConfirmRoles()).containsOnly("JOB_RESOURCE_TRIGGER_USER"); @@ -412,6 +438,7 @@ public void testGetConfirmStepInstance() { public void testAddConfirmStepInstance() { StepInstanceDTO confirmStepInstance = new StepInstanceDTO(); confirmStepInstance.setId(102L); + confirmStepInstance.setTaskInstanceId(102L); confirmStepInstance.setConfirmMessage("confirm_message"); confirmStepInstance.setConfirmUsers(Lists.newArrayList("admin")); confirmStepInstance.setConfirmRoles(Lists.newArrayList("JOB_RESOURCE_TRIGGER_USER")); @@ -419,8 +446,9 @@ public void testAddConfirmStepInstance() { stepInstanceDAO.addConfirmStepInstance(confirmStepInstance); - ConfirmStepInstanceDTO returnStepInstance = stepInstanceDAO.getConfirmStepInstance(102L); + ConfirmStepInstanceDTO returnStepInstance = stepInstanceDAO.getConfirmStepInstance(102L, 102L); assertThat(returnStepInstance).isNotNull(); + assertThat(returnStepInstance.getTaskInstanceId()).isEqualTo(102L); assertThat(returnStepInstance.getStepInstanceId()).isEqualTo(102L); assertThat(returnStepInstance.getConfirmMessage()).isEqualTo("confirm_message"); assertThat(returnStepInstance.getConfirmRoles()).containsOnly("JOB_RESOURCE_TRIGGER_USER"); @@ -431,8 +459,8 @@ public void testAddConfirmStepInstance() { @Test void testUpdateResolvedScriptParam() { - stepInstanceDAO.updateResolvedScriptParam(1L, "resolved_var"); - ScriptStepInstanceDTO updatedStepInstance = stepInstanceDAO.getScriptStepInstance(1L); + stepInstanceDAO.updateResolvedScriptParam(1L, 1L, true, "resolved_var"); + ScriptStepInstanceDTO updatedStepInstance = stepInstanceDAO.getScriptStepInstance(1L, 1L); assertThat(updatedStepInstance).isNotNull(); assertThat(updatedStepInstance.getResolvedScriptParam()).isEqualTo("resolved_var"); @@ -454,16 +482,16 @@ void testUpdateResolvedFileSource() { fileSourceDTO.setFileType(TaskFileTypeEnum.SERVER.getType()); fileSourceDTO.setAccountId(1L); fileSourceDTO.setAccount("root"); - ServersDTO servers = new ServersDTO(); + ExecuteTargetDTO executeTarget = new ExecuteTargetDTO(); List ips = new ArrayList<>(); ips.add(new HostDTO(1L, "10.10.10.10")); - servers.setIpList(ips); - fileSourceDTO.setServers(servers); + executeTarget.setIpList(ips); + fileSourceDTO.setServers(executeTarget); fileSources.add(fileSourceDTO); - stepInstanceDAO.updateResolvedSourceFile(2L, fileSources); - FileStepInstanceDTO updatedStepInstance = stepInstanceDAO.getFileStepInstance(2L); + stepInstanceDAO.updateResolvedSourceFile(1L, 2L, fileSources); + FileStepInstanceDTO updatedStepInstance = stepInstanceDAO.getFileStepInstance(1L, 2L); assertThat(updatedStepInstance).isNotNull(); assertThat(updatedStepInstance.getFileSourceList()).hasSize(1); @@ -479,8 +507,8 @@ void testUpdateResolvedFileSource() { @Test void testUpdateResolvedFileTargetPath() { - stepInstanceDAO.updateResolvedTargetPath(2L, "/data/bkee/"); - FileStepInstanceDTO updatedStepInstance = stepInstanceDAO.getFileStepInstance(2L); + stepInstanceDAO.updateResolvedTargetPath(1L, 2L, "/data/bkee/"); + FileStepInstanceDTO updatedStepInstance = stepInstanceDAO.getFileStepInstance(1L, 2L); assertThat(updatedStepInstance).isNotNull(); assertThat(updatedStepInstance.getResolvedFileTargetPath()).isEqualTo("/data/bkee/"); @@ -488,8 +516,8 @@ void testUpdateResolvedFileTargetPath() { @Test void testUpdateConfirmReason() { - stepInstanceDAO.updateConfirmReason(3L, "ok"); - ConfirmStepInstanceDTO updatedStepInstance = stepInstanceDAO.getConfirmStepInstance(3L); + stepInstanceDAO.updateConfirmReason(13L, 15L, "ok"); + ConfirmStepInstanceDTO updatedStepInstance = stepInstanceDAO.getConfirmStepInstance(13L, 15L); assertThat(updatedStepInstance).isNotNull(); assertThat(updatedStepInstance.getConfirmReason()).isEqualTo("ok"); @@ -497,8 +525,8 @@ void testUpdateConfirmReason() { @Test void testUpdateStepOperator() { - stepInstanceDAO.updateStepOperator(3L, "test"); - StepInstanceBaseDTO updatedStepInstance = stepInstanceDAO.getStepInstanceBase(3L); + stepInstanceDAO.updateStepOperator(3L, 4L, "test"); + StepInstanceBaseDTO updatedStepInstance = stepInstanceDAO.getStepInstanceBase(3L, 4L); assertThat(updatedStepInstance).isNotNull(); assertThat(updatedStepInstance.getOperator()).isEqualTo("test"); @@ -506,24 +534,25 @@ void testUpdateStepOperator() { @Test void testGetPreExecutableStepInstance() { - StepInstanceBaseDTO preStepInstance = stepInstanceDAO.getPreExecutableStepInstance(1L, 2L); + StepInstanceBaseDTO preStepInstance = stepInstanceDAO.getPreExecutableStepInstance(1L, 2); assertThat(preStepInstance).isNotNull(); + assertThat(preStepInstance.getTaskInstanceId()).isEqualTo(1L); assertThat(preStepInstance.getId()).isEqualTo(1L); } @Test void updateStepCurrentBatch() { - stepInstanceDAO.updateStepCurrentBatch(1L, 1); + stepInstanceDAO.updateStepCurrentBatch(1L, 1L, 1); - StepInstanceBaseDTO stepInstance = stepInstanceDAO.getStepInstanceBase(1L); + StepInstanceBaseDTO stepInstance = stepInstanceDAO.getStepInstanceBase(1L, 1L); assertThat(stepInstance.getBatch()).isEqualTo(1); } @Test void updateStepRollingConfigId() { - stepInstanceDAO.updateStepRollingConfigId(1L, 1000L); + stepInstanceDAO.updateStepRollingConfigId(1L, 1L, 1000L); - StepInstanceBaseDTO stepInstance = stepInstanceDAO.getStepInstanceBase(1L); + StepInstanceBaseDTO stepInstance = stepInstanceDAO.getStepInstanceBase(1L, 1L); assertThat(stepInstance.getRollingConfigId()).isEqualTo(1000L); } diff --git a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/StepInstanceRollingTaskDAOImplIntegrationTest.java b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/StepInstanceRollingTaskDAOImplIntegrationTest.java index 31378cb33d..cf774e8296 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/StepInstanceRollingTaskDAOImplIntegrationTest.java +++ b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/StepInstanceRollingTaskDAOImplIntegrationTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -56,7 +56,7 @@ public class StepInstanceRollingTaskDAOImplIntegrationTest { @Test @DisplayName("根据步骤实例ID、执行次数、滚动批次查询步骤滚动任务") void queryRollingTask() { - StepInstanceRollingTaskDTO rollingTask = stepInstanceRollingTaskDAO.queryRollingTask(1, 0, 1); + StepInstanceRollingTaskDTO rollingTask = stepInstanceRollingTaskDAO.queryRollingTask(1L, 1, 0, 1); assertThat(rollingTask).isNotNull(); assertThat(rollingTask.getStepInstanceId()).isEqualTo(1L); assertThat(rollingTask.getExecuteCount()).isEqualTo(0); @@ -70,7 +70,7 @@ void queryRollingTask() { @Test @DisplayName("根据步骤实例ID查询步骤滚动任务") void listRollingTasks() { - List rollingTasks = stepInstanceRollingTaskDAO.listRollingTasks(1L, null, null); + List rollingTasks = stepInstanceRollingTaskDAO.listRollingTasks(1L, 1L, null, null); assertThat(rollingTasks).hasSize(2); StepInstanceRollingTaskDTO rollingTask1 = rollingTasks.get(0); @@ -105,7 +105,7 @@ void saveRollingTask() { rollingTask.setStartTime(1642247802000L); stepInstanceRollingTaskDAO.saveRollingTask(rollingTask); - StepInstanceRollingTaskDTO savedRollingTask = stepInstanceRollingTaskDAO.queryRollingTask(100L, 0, 1); + StepInstanceRollingTaskDTO savedRollingTask = stepInstanceRollingTaskDAO.queryRollingTask(100L, 100L, 0, 1); assertThat(savedRollingTask).isNotNull(); assertThat(savedRollingTask.getStepInstanceId()).isEqualTo(100L); assertThat(savedRollingTask.getExecuteCount()).isEqualTo(0); @@ -121,9 +121,10 @@ void updateRollingTask() { long startTime = System.currentTimeMillis(); Long endTime = startTime + 1000L; Long totalTime = 1000L; - stepInstanceRollingTaskDAO.updateRollingTask(1L, 0, 1, RunStatusEnum.SUCCESS, startTime, endTime, totalTime); + stepInstanceRollingTaskDAO.updateRollingTask(1L, 1L, 0, 1, RunStatusEnum.SUCCESS, startTime, endTime, + totalTime); - StepInstanceRollingTaskDTO savedRollingTask = stepInstanceRollingTaskDAO.queryRollingTask(1L, 0, 1); + StepInstanceRollingTaskDTO savedRollingTask = stepInstanceRollingTaskDAO.queryRollingTask(1L, 1L, 0, 1); assertThat(savedRollingTask).isNotNull(); assertThat(savedRollingTask.getStepInstanceId()).isEqualTo(1L); diff --git a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/StepInstanceVariableDAOImplIntegrationTest.java b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/StepInstanceVariableDAOImplIntegrationTest.java index 2be1ccb31d..a615d746ef 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/StepInstanceVariableDAOImplIntegrationTest.java +++ b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/StepInstanceVariableDAOImplIntegrationTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -69,7 +69,8 @@ void testSaveVariableValues() { variableValues.setGlobalParams(globalParams); List hostParamValuesList = new ArrayList<>(); HostVariableValuesDTO hostParamValues = new HostVariableValuesDTO(); - hostParamValues.setIp("1.1.1.1"); + hostParamValues.setHostId(1L); + hostParamValues.setCloudIpv4("0:1.1.1.1"); List namespaceParamValues = new ArrayList<>(); namespaceParamValues.add(new VariableValueDTO("param11", 2, "value11")); namespaceParamValues.add(new VariableValueDTO("param12", 2, "value12")); @@ -80,7 +81,7 @@ void testSaveVariableValues() { stepInstanceVariableDAO.saveVariableValues(variableValues); StepInstanceVariableValuesDTO actual = stepInstanceVariableDAO - .getStepVariableValues(200L, 1, VariableValueTypeEnum.OUTPUT); + .getStepVariableValues(100L, 200L, 1, VariableValueTypeEnum.OUTPUT); assertThat(actual.getStepInstanceId()).isEqualTo(200L); assertThat(actual.getExecuteCount()).isEqualTo(1); @@ -91,27 +92,12 @@ void testSaveVariableValues() { assertThat(actual.getGlobalParams()).extracting("value").containsOnly("value11", "value12"); assertThat(actual.getNamespaceParams()).hasSize(1); - assertThat(actual.getNamespaceParams().get(0).getIp()).isEqualTo("1.1.1.1"); + assertThat(actual.getNamespaceParams().get(0).getCloudIpv4()).isEqualTo("0:1.1.1.1"); + assertThat(actual.getNamespaceParams().get(0).getHostId()).isEqualTo(1L); assertThat(actual.getNamespaceParams().get(0).getValues()).hasSize(2); assertThat(actual.getNamespaceParams().get(0).getValues()).extracting("name") .containsOnly("param11", "param12"); assertThat(actual.getNamespaceParams().get(0).getValues()).extracting("value") .containsOnly("value11", "value12"); } - - @Test - void testListSortedPreStepOutputVariableValues() { - List stepInstanceVariableValuesList = - stepInstanceVariableDAO.listSortedPreStepOutputVariableValues(1L, 3L); - assertThat(stepInstanceVariableValuesList).hasSize(3); - assertThat(stepInstanceVariableValuesList.get(0).getTaskInstanceId()).isEqualTo(1L); - assertThat(stepInstanceVariableValuesList.get(0).getStepInstanceId()).isEqualTo(1L); - assertThat(stepInstanceVariableValuesList.get(0).getExecuteCount()).isEqualTo(0); - assertThat(stepInstanceVariableValuesList.get(1).getTaskInstanceId()).isEqualTo(1L); - assertThat(stepInstanceVariableValuesList.get(1).getStepInstanceId()).isEqualTo(2L); - assertThat(stepInstanceVariableValuesList.get(1).getExecuteCount()).isEqualTo(0); - assertThat(stepInstanceVariableValuesList.get(2).getTaskInstanceId()).isEqualTo(1L); - assertThat(stepInstanceVariableValuesList.get(2).getStepInstanceId()).isEqualTo(2L); - assertThat(stepInstanceVariableValuesList.get(2).getExecuteCount()).isEqualTo(1); - } } diff --git a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceDAOImplIntegrationTest.java b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceDAOImplIntegrationTest.java index a56e3d68db..75bdb03504 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceDAOImplIntegrationTest.java +++ b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceDAOImplIntegrationTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -67,7 +67,7 @@ void testGetTaskInstance() { assertThat(taskInstance.getId()).isEqualTo(1L); assertThat(taskInstance.getAppId()).isEqualTo(2L); - assertThat(taskInstance.getTaskId()).isEqualTo(1L); + assertThat(taskInstance.getPlanId()).isEqualTo(1L); assertThat(taskInstance.getTaskTemplateId()).isEqualTo(1L); assertThat(taskInstance.getName()).isEqualTo("task1"); assertThat(taskInstance.getType()).isEqualTo(TaskTypeEnum.SCRIPT.getValue()); @@ -85,7 +85,7 @@ void testGetTaskInstance() { @Test void testAddTaskInstance() { TaskInstanceDTO taskInstance = new TaskInstanceDTO(); - taskInstance.setTaskId(3L); + taskInstance.setPlanId(3L); taskInstance.setTaskTemplateId(3L); taskInstance.setAppId(2L); taskInstance.setName("task3"); @@ -101,9 +101,9 @@ void testAddTaskInstance() { TaskInstanceDTO returnTaskInstance = taskInstanceDAO.getTaskInstance(taskInstanceId); - assertThat(returnTaskInstance.getId()).isEqualTo(4L); + assertThat(returnTaskInstance.getId()).isGreaterThan(0L); assertThat(returnTaskInstance.getAppId()).isEqualTo(2L); - assertThat(returnTaskInstance.getTaskId()).isEqualTo(3L); + assertThat(returnTaskInstance.getPlanId()).isEqualTo(3L); assertThat(returnTaskInstance.getTaskTemplateId()).isEqualTo(3L); assertThat(returnTaskInstance.getName()).isEqualTo("task3"); assertThat(taskInstance.getType()).isEqualTo(TaskTypeEnum.NORMAL.getValue()); @@ -117,16 +117,6 @@ void testAddTaskInstance() { assertThat(taskInstance.getAppCode()).isEqualTo("bk_monitor"); } - @Test - void testGetTaskInstanceByTaskId() { - long taskId = 1L; - - List taskInstances = taskInstanceDAO.getTaskInstanceByTaskId(taskId); - - assertThat(taskInstances).hasSize(2).extracting("id").containsOnly(1L, 2L); - - } - @Test void testUpdateTaskStatus() { long taskInstanceId = 2L; @@ -137,30 +127,6 @@ void testUpdateTaskStatus() { assertThat(taskInstanceDTO.getStatus()).isEqualTo(RunStatusEnum.ABNORMAL_STATE); } - @Test - void testUpdateTaskStartTime() { - long taskInstanceId = 2L; - long startTime = 1572955200000L; - - taskInstanceDAO.updateTaskStartTime(taskInstanceId, startTime); - - TaskInstanceDTO taskInstanceDTO = taskInstanceDAO.getTaskInstance(taskInstanceId); - assertThat(taskInstanceDTO.getId()).isEqualTo(taskInstanceId); - assertThat(taskInstanceDTO.getStartTime()).isEqualTo(startTime); - } - - @Test - void testUpdateTaskEndTime() { - long taskInstanceId = 2L; - Long endTime = 1572955200000L; - - taskInstanceDAO.updateTaskEndTime(taskInstanceId, endTime); - - TaskInstanceDTO taskInstanceDTO = taskInstanceDAO.getTaskInstance(taskInstanceId); - assertThat(taskInstanceDTO.getId()).isEqualTo(taskInstanceId); - assertThat(taskInstanceDTO.getEndTime()).isEqualTo(endTime); - } - @Test void testUpdateTaskCurrentStepId() { long taskInstanceId = 2L; @@ -188,43 +154,6 @@ void testResetTaskStatus() { assertThat(taskInstanceDTO.getCurrentStepInstanceId()).isEqualTo(0L); } - @Test - void testCleanTaskEndTime() { - long taskInstanceId = 2L; - taskInstanceDAO.cleanTaskEndTime(taskInstanceId); - - TaskInstanceDTO taskInstanceDTO = taskInstanceDAO.getTaskInstance(taskInstanceId); - - assertThat(taskInstanceDTO.getEndTime()).isNull(); - assertThat(taskInstanceDTO.getTotalTime()).isNull(); - } - - @Test - void testUpdateTaskTotalTime() { - long taskInstanceId = 2L; - long totalTime = 1612L; - taskInstanceDAO.updateTaskTotalTime(taskInstanceId, totalTime); - - TaskInstanceDTO taskInstanceDTO = taskInstanceDAO.getTaskInstance(taskInstanceId); - - assertThat(taskInstanceDTO.getId()).isEqualTo(taskInstanceId); - assertThat(taskInstanceDTO.getTotalTime()).isEqualTo(totalTime); - } - - @Test - void testAddCallbackUrl() { - String callbackUrl = "http://newbkjob.com"; - long taskInstanceId = 2L; - - taskInstanceDAO.addCallbackUrl(taskInstanceId, callbackUrl); - - TaskInstanceDTO taskInstanceDTO = taskInstanceDAO.getTaskInstance(taskInstanceId); - - assertThat(taskInstanceDTO.getId()).isEqualTo(taskInstanceId); - assertThat(taskInstanceDTO.getCallbackUrl()).isEqualTo(callbackUrl); - - } - @Test @DisplayName("测试分页查询执行实例-使用ID") void testQueryPageListById() { @@ -251,7 +180,7 @@ void testQueryPageListWithSearchCondition() { TaskInstanceQuery taskQuery = new TaskInstanceQuery(); taskQuery.setAppId(2L); taskQuery.setOperator("admin"); - taskQuery.setStartupModes(Collections.singletonList(TaskStartupModeEnum.NORMAL)); + taskQuery.setStartupModes(Collections.singletonList(TaskStartupModeEnum.WEB)); taskQuery.setTaskType(TaskTypeEnum.SCRIPT); taskQuery.setStatus(RunStatusEnum.SUCCESS); taskQuery.setStartTime(1572865200000L); @@ -325,6 +254,7 @@ void testResetTaskExecuteInfoForResume() { @Test void testSaveTaskInstanceHosts() { long taskInstanceId = 10L; + long appId = 2L; HostDTO host1 = new HostDTO(); host1.setHostId(1L); host1.setIp("127.0.0.1"); @@ -340,7 +270,6 @@ void testSaveTaskInstanceHosts() { hosts.add(host2); hosts.add(host3); - taskInstanceDAO.saveTaskInstanceHosts(taskInstanceId, hosts); + taskInstanceDAO.saveTaskInstanceHosts(appId, taskInstanceId, hosts); } - } diff --git a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceVariableDAOImplIntegrationTest.java b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceVariableDAOImplIntegrationTest.java index 10b92a0ebc..b8bef1c0f7 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceVariableDAOImplIntegrationTest.java +++ b/src/backend/job-execute/boot-job-execute/src/test/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceVariableDAOImplIntegrationTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/boot-job-execute/src/test/resources/application-test.yml b/src/backend/job-execute/boot-job-execute/src/test/resources/application-test.yml index 82a922201c..01cd5f8e32 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/resources/application-test.yml +++ b/src/backend/job-execute/boot-job-execute/src/test/resources/application-test.yml @@ -4,6 +4,9 @@ spring: charset: UTF-8 enabled: true force: true + mvc: + pathmatch: + matching-strategy: ant_path_matcher datasource: job-execute: driver-class-name: org.h2.Driver @@ -25,9 +28,29 @@ spring: cloud: config: enabled: false + kubernetes: + discovery: + enabled: false job: security: service: public-key-base64: LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FROEFNSUlCQ2dLQ0FRRUFyTXN3emh3QkVUTktQOThTcElna0VjZThPNVlQWm5aUDVHVW1ZUVhhSHdHWmg5UHJwNDFseS9xbWk5enEwTlRCODRyaXhSZHZyU1c0cCtCUjk4TVRzRmpYaUlRYVBxRTdRUitpdDdoT1VQNVJMQW9KcWlvL3NlZlRIdlZxVW1GcitYU3NST05rek5selRVaVcvMFRNY0cwNVdIS3hORno5YTB2aXo1T01PeE5lUzhqS1ZiRStIMHlQS0JhR1JHMEdZNTZTS1BLbXh1RHk5a2tyVUtGSXZycldSZkh2bzVsK0xsT3IyQmVGT3FHcEhmeW1FaXlkbFRxWHNlbGlsYTRxMUoxbnBZRDYwY3JIY1ZRV3d2Ri9CZUkrZytxVkF5YzNaWmhYeEhoNWNMcmZsK3pqMktsUEFEZVIvZktubHFqKzFBQm4yaFRoVWJ2WjRCdXhTdWxOa1FJREFRQUIKLS0tLS1FTkQgUFVCTElDIEtFWS0tLS0t private-key-base64: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV1d0lCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktVd2dnU2hBZ0VBQW9JQkFRQ3N5ekRPSEFFUk0wby8zeEtraUNRUng3dzdsZzltZGsva1pTWmhCZG9mQVptSDArdW5qV1hMK3FhTDNPclExTUh6aXVMRkYyK3RKYmluNEZIM3d4T3dXTmVJaEJvK29UdEJINkszdUU1US9sRXNDZ21xS2oreDU5TWU5V3BTWVd2NWRLeEU0MlRNMlhOTlNKYi9STXh3YlRsWWNyRTBYUDFyUytMUGs0dzdFMTVMeU1wVnNUNGZUSThvRm9aRWJRWmpucElvOHFiRzRQTDJTU3RRb1VpK3V0WkY4ZStqbVg0dVU2dllGNFU2b2FrZC9LWVNMSjJWT3BleDZXS1ZyaXJVbldlbGdQclJ5c2R4VkJiQzhYOEY0ajZENnBVREp6ZGxtRmZFZUhsd3V0K1g3T1BZcVU4QU41SDk4cWVXcVA3VUFHZmFGT0ZSdTluZ0c3Rks2VTJSQWdNQkFBRUNnZ0VBZlkvZGREaUhVZjlxV2dWNG41VEtXUjFDSEM2TUhGR205bm5HTE1TNjNzTWoxTDdiUkZNTWdQQXY2L3hwcWJ5Zy81K1I3OVI2ODJBdzBkVEEyNENuSXdNOEE5aXkxWVlGNytuVUxSckIxanNuc3cvTVBCb2RWT0wzMkV4UHFYTmZxZEx0aDlXUm53WDRzbFVvWEhDYStxdHorMkRFZ2g5OGkyYUdkMzVwYlZjT2p0czlWc3FtZE5uQTdLNFVQWkd4aVFheG1rLzBOVEVNY2RqMUl4bWNlZjNqL1RUVkFiR05IdWVtYTJKeUdZdk52N09BZUN3VGJSdW5TNW1PSmptRHlFSmp4dndaVWZwanZoM0w0SkZzMGRJUm5wZ2VtTkY3NytqUmFTTy9WUVlqc1FLelB0SnEwcHFneHUwd0RoQUVBeXpjYWI2NW8wSm1hZzNJUjZNQzdRS0JnUUR1MThHNFRVaTlSVk8yZS9YdWtDSUNiOVZISHlMeVFIUDh4NDhBVFZmVENtSVBEMUduTWp5TlFmSnhsUlNma3UyUjJIM1IyRC81YUNpNUYwTi9USG51OWVqYlMvS1NoYjFxMFl5TWF1TDkxQlZjMkEyeDZlRzRPNFFpT2daT0gzaE0raUVSREJxOExNQ1dILzVtYVJDeDhVRmRQTnpHMUtXVVNwTEJXM3l2ZXdLQmdRQzVOTS9yN09Wc3F6OGlDUjBXa2JramN2eWU4MkJNcGFsZVkyRHNWQVMwRHNMNlMvZlAzY2VUdEJQVmdJL244Sk16b1ZNZE5OK3htbXM4Wi9rMGM5OGlLTzRUeE0zN2daYkIwUE8rWGRyeUZkRFhTRDZXTm1heWpYUG5MUndBZmRwbFlWdFNqVGxWMWJzWWIySnhXNXY2RUVSaVlLYnNnRGNpWTA5ZnYva0RZd0ovUnVmR212QldwOW50QUQvTXd3WUhFcklnbnZ5ZlkwdS9JMHdiSi92T0Z0aitRM3BJdzFvbW44ajBNTVFSVzA1RE9Ra01VVC9odmlrdDdjVDZSTkJ2WW9HZW4zdnNoNU1zcUltTk1DS0xRTSsxaDlxY05qVTR6WGpkd1V0NGs3akQvaFpEdXN2ZEpBQWxMR3hUR2hRVzRMeStxdTltbTZDRmg0Q2NFTXNZT1FLQmdCa0pEWTRyd0x1V3pucndWbkZGazR3NkwvSGdua1RlSSszeGF4bW5KZGVkSXpnb2FzQTdqNDNreXNXU2ttRmRBTmo2MXJ4YytGd09ycTVFczhnbVhhUkZlaUZMK2pna1JWZS93aU1hbHMwZ3E5RmVINis0Zk50b0NhSFhWREN3MG9QZnpETjFpYW1Uc2EzVTZLN0FIbTZJSW1aV0FGcDUzbm9VaUpjOTRSQlhBb0dCQUxMcmQrbnljNzVEbjZ0ZWdVT0JQRy9HR3FYWkZGdmd6emJRRnR0SmR0NTlMZ0NBanUxMjJMRExsRjZreTB6Q0JsK0h6Q0kyUXJXVEp4cXUxSTZvbndaZDlOMllKVlNWQlFOV285NTJUamRkOHcvSkExYlNqK3JpTU1WQ0txT1BNRHNiOENEaGlEbVM3RlBEM0trTVlmcUxJOVhNRTVvNVUvNFJzcEZUZWozRQotLS0tLUVORCBQUklWQVRFIEtFWS0tLS0t +bk-api-gateway: + gse: + url: gse.apigw.com + cmdb: + url: cmdb.apigw.com +esb: + service: + url: esb.service +mysql: + standalone: + enabled : true +leaf: + enabled: false +idGen: + type: auto_increment + migration: + enabled: false diff --git a/src/backend/job-execute/boot-job-execute/src/test/resources/bootstrap.yml b/src/backend/job-execute/boot-job-execute/src/test/resources/bootstrap.yml index 68074f0042..4a536186c2 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/resources/bootstrap.yml +++ b/src/backend/job-execute/boot-job-execute/src/test/resources/bootstrap.yml @@ -1,3 +1,6 @@ spring: application: - name: job-execute \ No newline at end of file + name: job-execute + cloud: + kubernetes: + enabled: false diff --git a/src/backend/job-execute/boot-job-execute/src/test/resources/init_dangerous_record_data.sql b/src/backend/job-execute/boot-job-execute/src/test/resources/init_dangerous_record_data.sql index 20a3d9c2ca..73e150327c 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/resources/init_dangerous_record_data.sql +++ b/src/backend/job-execute/boot-job-execute/src/test/resources/init_dangerous_record_data.sql @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/boot-job-execute/src/test/resources/init_file_agent_task_data.sql b/src/backend/job-execute/boot-job-execute/src/test/resources/init_file_agent_task_data.sql index ab4d5a8a5d..83ddb19b62 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/resources/init_file_agent_task_data.sql +++ b/src/backend/job-execute/boot-job-execute/src/test/resources/init_file_agent_task_data.sql @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,7 +25,8 @@ TRUNCATE TABLE job_execute.gse_file_agent_task; INSERT INTO job_execute.gse_file_agent_task - (step_instance_id, + (task_instance_id, + step_instance_id, execute_count, batch, mode, @@ -38,10 +39,10 @@ INSERT INTO job_execute.gse_file_agent_task total_time, error_code) VALUES - (1,0,1,0,101,'0:127.0.0.1',1,9,1565767148000,1565767149000,1000,0), - (1,0,1,1,102,'0:127.0.0.2',2,9,1565767148000,1565767149000,1000,0), - (1,0,2,0,101,'0:127.0.0.1',3,9,1565767150000,1565767151000,1000,0), - (1,0,2,1,103,'0:127.0.0.3',3,9,1565767150000,1565767151000,1000,0), - (1,0,2,1,104,'0:127.0.0.4',3,11,1565767150000,1565767151000,1000,99), - (2,0,0,0,101,'0:127.0.0.1',4,9,1565767148000,1565767149000,1000,0), - (2,0,0,1,102,'0:127.0.0.2',4,9,1565767148000,1565767149000,1000,0); + (1,1,0,1,0,101,'0:127.0.0.1',1,9,1565767148000,1565767149000,1000,0), + (1,1,0,1,1,102,'0:127.0.0.2',2,9,1565767148000,1565767149000,1000,0), + (1,1,0,2,0,101,'0:127.0.0.1',3,9,1565767150000,1565767151000,1000,0), + (1,1,0,2,1,103,'0:127.0.0.3',3,9,1565767150000,1565767151000,1000,0), + (1,1,0,2,1,104,'0:127.0.0.4',3,11,1565767150000,1565767151000,1000,99), + (2,2,0,0,0,101,'0:127.0.0.1',4,9,1565767148000,1565767149000,1000,0), + (2,2,0,0,1,102,'0:127.0.0.2',4,9,1565767148000,1565767149000,1000,0); diff --git a/src/backend/job-execute/boot-job-execute/src/test/resources/init_gse_file_execute_obj_task_data.sql b/src/backend/job-execute/boot-job-execute/src/test/resources/init_gse_file_execute_obj_task_data.sql new file mode 100644 index 0000000000..fc662bc208 --- /dev/null +++ b/src/backend/job-execute/boot-job-execute/src/test/resources/init_gse_file_execute_obj_task_data.sql @@ -0,0 +1,48 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +TRUNCATE TABLE job_execute.gse_file_execute_obj_task; + +INSERT INTO job_execute.gse_file_execute_obj_task + (task_instance_id, + step_instance_id, + execute_count, + batch, + mode, + execute_obj_type, + execute_obj_id, + gse_task_id, + status, + start_time, + end_time, + total_time, + error_code) +VALUES + (1,1,0,1,0,1,'1:101',1,9,1565767148000,1565767149000,1000,0), + (1,1,0,1,1,1,'1:102',2,9,1565767148000,1565767149000,1000,0), + (1,1,0,2,0,1,'1:101',3,9,1565767150000,1565767151000,1000,0), + (1,1,0,2,1,1,'1:103',3,9,1565767150000,1565767151000,1000,0), + (1,1,0,2,1,1,'1:104',3,11,1565767150000,1565767151000,1000,99), + (2,2,0,0,0,1,'1:101',4,9,1565767148000,1565767149000,1000,0), + (2,2,0,0,1,1,'1:102',4,9,1565767148000,1565767149000,1000,0); diff --git a/src/backend/job-execute/boot-job-execute/src/test/resources/init_gse_script_execute_obj_task_data.sql b/src/backend/job-execute/boot-job-execute/src/test/resources/init_gse_script_execute_obj_task_data.sql new file mode 100644 index 0000000000..2c7d2b7ff9 --- /dev/null +++ b/src/backend/job-execute/boot-job-execute/src/test/resources/init_gse_script_execute_obj_task_data.sql @@ -0,0 +1,52 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +TRUNCATE TABLE job_execute.gse_script_execute_obj_task; + +INSERT INTO job_execute.gse_script_execute_obj_task + (task_instance_id, + step_instance_id, + execute_count, + batch, + execute_obj_type, + execute_obj_id, + gse_task_id, + status, + start_time, + end_time, + total_time, + error_code, + exit_code, + tag, + log_offset) +VALUES + (1,1,0,1,1,'1:101',1,9,1565767148000,1565767149000,1316,0,0,'succ',0), + (1,1,0,2,1,'1:102',2,9,1565767148000,1565767149000,1211,0,0,'succ',0), + (1,1,0,3,1,'1:103',3,9,1565767148000,1565767149000,1211,0,0,'succ',0), + (1,1,0,3,1,'1:104',3,9,1565767148000,1565767149000,1211,0,0,'succ',0), + (1,1,0,3,1,'1:105',3,11,1565767148000,1565767149000,1211,0,0,'fail',0), + (2,2,0,0,1,'1:101',4,9,1565767148000,1565767209000,1211,0,0,'succ',0), + (2,2,1,0,1,'1:101',4,9,1565766610000,1565767211000,1215,0,0,'succ',0), + (3,3,0,0,1,'1:101',5,9,1565766610000,1565767211000,1215,0,0,'succ',0), + (3,3,0,0,1,'1:102',5,9,1565766610000,1565767211000,1215,0,0,'succ',0); diff --git a/src/backend/job-execute/boot-job-execute/src/test/resources/init_gse_task_data.sql b/src/backend/job-execute/boot-job-execute/src/test/resources/init_gse_task_data.sql index 69c531371a..c083ba4ea7 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/resources/init_gse_task_data.sql +++ b/src/backend/job-execute/boot-job-execute/src/test/resources/init_gse_task_data.sql @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,8 +24,8 @@ TRUNCATE TABLE job_execute.gse_task; -INSERT INTO job_execute.gse_task (id,step_instance_id,execute_count,batch,status,start_time,end_time,total_time,gse_task_id) +INSERT INTO job_execute.gse_task (id,task_instance_id,step_instance_id,execute_count,batch,status,start_time,end_time,total_time,gse_task_id) VALUES -(1,1,0,0,4,1660639901000,1660639902000,1000,'GSE_TASK_1'), -(2,2,0,1,3,1660639903000,1660639904000,1000,'GSE_TASK_2'), -(3,2,0,2,4,1660639905000,1660639906000,1000,'GSE_TASK_3'); +(1,1,1,0,0,4,1660639901000,1660639902000,1000,'GSE_TASK_1'), +(2,2,2,0,1,3,1660639903000,1660639904000,1000,'GSE_TASK_2'), +(3,2,2,0,2,4,1660639905000,1660639906000,1000,'GSE_TASK_3'); diff --git a/src/backend/job-execute/boot-job-execute/src/test/resources/init_gse_task_ip_log_data.sql b/src/backend/job-execute/boot-job-execute/src/test/resources/init_gse_task_ip_log_data.sql deleted file mode 100644 index dc792f4346..0000000000 --- a/src/backend/job-execute/boot-job-execute/src/test/resources/init_gse_task_ip_log_data.sql +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -TRUNCATE TABLE job_execute.gse_task_ip_log; - -INSERT INTO job_execute.gse_task_ip_log (step_instance_id,execute_count,ip,status,start_time,end_time,total_time,error_code,exit_code,tag,log_offset,display_ip,is_target,is_source) -VALUES -(1,0,'0:127.0.0.1',9,1565767148000,1565767149000,1316,0,0,'succ',0,'127.0.0.1',1,0), -(1,0,'0:127.0.0.2',9,1565767148000,1565767149000,1211,0,0,'succ',0,'127.0.0.2',1,0), -(1,0,'0:127.0.0.3',9,1565767148000,1565767149000,1211,0,0,'succ',0,'127.0.0.3',0,1), -(2,0,'0:127.0.0.1',9,1565767148000,1565767209000,1211,0,0,'succ',0,'127.0.0.1',1,0), -(2,1,'0:127.0.0.1',9,1565766610000,1565767211000,1215,0,0,'succ',0,'127.0.0.1',1,0); diff --git a/src/backend/job-execute/boot-job-execute/src/test/resources/init_operation_log_data.sql b/src/backend/job-execute/boot-job-execute/src/test/resources/init_operation_log_data.sql index 7abcf1c1a2..91d8ddf705 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/resources/init_operation_log_data.sql +++ b/src/backend/job-execute/boot-job-execute/src/test/resources/init_operation_log_data.sql @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/boot-job-execute/src/test/resources/init_rolling_config_data.sql b/src/backend/job-execute/boot-job-execute/src/test/resources/init_rolling_config_data.sql index 049b763a5d..a4556f4784 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/resources/init_rolling_config_data.sql +++ b/src/backend/job-execute/boot-job-execute/src/test/resources/init_rolling_config_data.sql @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,5 +24,5 @@ truncate table rolling_config; insert into job_execute.rolling_config (id,task_instance_id,config_name,config) values -(1,1,'config1', '{"name":"config1","includeStepInstanceIdList":[100,101,102,103],"stepRollingConfigs": {"100": {"batch": true},"101": {"batch": false},"102": {"batch": true},"103": {"batch": true}},"mode":1,"expr":"1 10% 100%","hostsBatchList":[{"batch":1,"hosts":[{"cloudAreaId":0,"ip":"127.0.0.1"}]},{"batch":2,"hosts":[{"cloudAreaId":0,"ip":"127.0.0.2"}]},{"batch":3,"hosts":[{"cloudAreaId":0,"ip":"127.0.0.3"},{"cloudAreaId":0,"ip":"127.0.0.4"}]}]}'); +(1,2,'config1', '{"name":"config1","includeStepInstanceIdList":[100,101,102,103],"stepRollingConfigs": {"100": {"batch": true},"101": {"batch": false},"102": {"batch": true},"103": {"batch": true}},"mode":1,"expr":"1 10% 100%","executeObjectsBatchList":[{"batch":1,"executeObjects":[{"id":"1:1","type":1,"resourceId":1,"host":{"hostId":1,"cloudAreaId":0,"ip":"127.0.0.1"}}]},{"batch":2,"executeObjects":[{"id":"1:2","type":1,"resourceId":2,"host":{"hostId":2,"cloudAreaId":0,"ip":"127.0.0.2"}}]},{"batch":3,"executeObjects":[{"id":"1:3","type":1,"resourceId":3,"host":{"hostId":3,"cloudAreaId":0,"ip":"127.0.0.3"}},{"id":"1:4","type":1,"resourceId":4,"host":{"hostId":4,"cloudAreaId":0,"ip":"127.0.0.4"}}]}]}'); diff --git a/src/backend/job-execute/boot-job-execute/src/test/resources/init_schema.sql b/src/backend/job-execute/boot-job-execute/src/test/resources/init_schema.sql index babd26a3f5..04a6902b9f 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/resources/init_schema.sql +++ b/src/backend/job-execute/boot-job-execute/src/test/resources/init_schema.sql @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -90,6 +90,7 @@ CREATE TABLE IF NOT EXISTS `step_instance` CREATE TABLE IF NOT EXISTS `step_instance_script` ( `step_instance_id` bigint(20) NOT NULL, + `task_instance_id` bigint(20) NOT NULL DEFAULT 0, `script_content` mediumtext, `script_type` tinyint(4) DEFAULT NULL, `script_param` text, @@ -108,6 +109,7 @@ CREATE TABLE IF NOT EXISTS `step_instance_script` `script_id` varchar(32) DEFAULT NULL, `script_version_id` bigint(20) DEFAULT NULL, `is_secure_param` tinyint(1) DEFAULT 0, + `windows_interpreter` varchar(260) DEFAULT NULL, PRIMARY KEY (`step_instance_id`) ) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4; @@ -115,6 +117,7 @@ CREATE TABLE IF NOT EXISTS `step_instance_script` CREATE TABLE IF NOT EXISTS `step_instance_file` ( `step_instance_id` bigint(20) NOT NULL, + `task_instance_id` bigint(20) NOT NULL DEFAULT 0, `file_source` mediumtext NOT NULL, `resolved_file_source` mediumtext DEFAULT NULL, `file_target_path` varchar(512) DEFAULT NULL, @@ -136,6 +139,7 @@ CREATE TABLE IF NOT EXISTS `step_instance_file` CREATE TABLE IF NOT EXISTS `step_instance_confirm` ( `step_instance_id` bigint(20) NOT NULL, + `task_instance_id` bigint(20) NOT NULL DEFAULT 0, `confirm_message` text NOT NULL, `confirm_reason` varchar(256) DEFAULT NULL, `confirm_users` varchar(1024) DEFAULT NULL, @@ -190,6 +194,7 @@ CREATE TABLE IF NOT EXISTS `gse_task_ip_log` CREATE TABLE IF NOT EXISTS `gse_task` ( `id` bigint(20) NOT NULL AUTO_INCREMENT, + `task_instance_id` bigint(20) NOT NULL DEFAULT 0, `step_instance_id` bigint(20) NOT NULL DEFAULT '0', `execute_count` smallint(6) NOT NULL DEFAULT '0', `batch` smallint(6) NOT NULL DEFAULT '0', @@ -209,6 +214,7 @@ CREATE TABLE IF NOT EXISTS `gse_task` CREATE TABLE IF NOT EXISTS `gse_script_agent_task` ( `id` bigint(20) NOT NULL AUTO_INCREMENT PRIMARY KEY, + `task_instance_id` bigint(20) NOT NULL DEFAULT 0, `step_instance_id` bigint(20) NOT NULL, `execute_count` smallint(6) NOT NULL DEFAULT '0', `actual_execute_count` smallint(6) DEFAULT NULL, @@ -234,6 +240,7 @@ CREATE TABLE IF NOT EXISTS `gse_script_agent_task` CREATE TABLE IF NOT EXISTS `gse_file_agent_task` ( `id` bigint(20) NOT NULL AUTO_INCREMENT PRIMARY KEY, + `task_instance_id` bigint(20) NOT NULL DEFAULT 0, `step_instance_id` bigint(20) NOT NULL, `execute_count` smallint(6) NOT NULL DEFAULT '0', `actual_execute_count` smallint(6) DEFAULT NULL, @@ -345,6 +352,7 @@ CREATE TABLE IF NOT EXISTS `rolling_config` CREATE TABLE IF NOT EXISTS `step_instance_rolling_task` ( `id` bigint(20) NOT NULL AUTO_INCREMENT, + `task_instance_id` bigint(20) NOT NULL DEFAULT 0, `step_instance_id` bigint(20) NOT NULL DEFAULT '0', `batch` smallint(6) NOT NULL DEFAULT '0', `execute_count` smallint(6) NOT NULL DEFAULT '0', @@ -361,15 +369,69 @@ CREATE TABLE IF NOT EXISTS `step_instance_rolling_task` CREATE TABLE IF NOT EXISTS `task_instance_host` ( - `task_instance_id` bigint(20) NOT NULL DEFAULT '0', - `host_id` bigint(20) NOT NULL DEFAULT '0', - `ip` varchar(15) DEFAULT NULL, - `ipv6` varchar(46) DEFAULT NULL, - `row_create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, - `row_update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`task_instance_id`,`host_id`), + `task_instance_id` bigint(20) NOT NULL DEFAULT '0', + `host_id` bigint(20) NOT NULL DEFAULT '0', + `ip` varchar(15) DEFAULT NULL, + `ipv6` varchar(46) DEFAULT NULL, + `app_id` bigint(20) NOT NULL DEFAULT '0', + `row_create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `row_update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`task_instance_id`, `host_id`), KEY (`ip`), KEY (`ipv6`) -) ENGINE=InnoDB - DEFAULT CHARSET=utf8mb4; +) ENGINE = InnoDB + DEFAULT CHARSET = utf8mb4; + + +CREATE TABLE IF NOT EXISTS `gse_script_execute_obj_task` +( + `id` bigint(20) NOT NULL AUTO_INCREMENT PRIMARY KEY, + `task_instance_id` bigint(20) NOT NULL, + `step_instance_id` bigint(20) NOT NULL, + `execute_count` smallint(6) NOT NULL DEFAULT '0', + `actual_execute_count` smallint(6) DEFAULT NULL, + `batch` smallint(6) NOT NULL DEFAULT '0', + `execute_obj_type` tinyint(4) NOT NULL, + `execute_obj_id` varchar(24) NOT NULL, + `gse_task_id` bigint(20) NOT NULL DEFAULT '0', + `status` int(11) DEFAULT '1', + `start_time` bigint(20) DEFAULT NULL, + `end_time` bigint(20) DEFAULT NULL, + `total_time` bigint(20) DEFAULT NULL, + `error_code` int(11) DEFAULT '0', + `exit_code` int(11) DEFAULT NULL, + `tag` varchar(256) DEFAULT '', + `log_offset` int(11) NOT NULL DEFAULT '0', + `row_create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `row_update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + UNIQUE KEY (`step_instance_id`, `execute_count`, `batch`, `execute_obj_id`), + KEY (`task_instance_id`), + KEY (`step_instance_id`, `execute_obj_id`) +) ENGINE = InnoDB + DEFAULT CHARSET = utf8mb4; + +CREATE TABLE IF NOT EXISTS `gse_file_execute_obj_task` +( + `id` bigint(20) NOT NULL AUTO_INCREMENT PRIMARY KEY, + `task_instance_id` bigint(20) NOT NULL, + `step_instance_id` bigint(20) NOT NULL, + `execute_count` smallint(6) NOT NULL DEFAULT '0', + `actual_execute_count` smallint(6) DEFAULT NULL, + `batch` smallint(6) NOT NULL DEFAULT '0', + `execute_obj_type` tinyint(4) NOT NULL, + `execute_obj_id` varchar(24) NOT NULL, + `mode` tinyint(1) NOT NULL, + `gse_task_id` bigint(20) NOT NULL DEFAULT '0', + `status` int(11) DEFAULT '1', + `start_time` bigint(20) DEFAULT NULL, + `end_time` bigint(20) DEFAULT NULL, + `total_time` bigint(20) DEFAULT NULL, + `error_code` int(11) DEFAULT '0', + `row_create_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP, + `row_update_time` datetime NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + UNIQUE KEY (`step_instance_id`, `execute_count`, `batch`, `mode`, `execute_obj_id`), + KEY (`task_instance_id`), + KEY (`step_instance_id`, `execute_obj_id`) +) ENGINE = InnoDB + DEFAULT CHARSET = utf8mb4; diff --git a/src/backend/job-execute/boot-job-execute/src/test/resources/init_script_agent_task_data.sql b/src/backend/job-execute/boot-job-execute/src/test/resources/init_script_agent_task_data.sql index f8d2fc25d3..ad61e0f2ca 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/resources/init_script_agent_task_data.sql +++ b/src/backend/job-execute/boot-job-execute/src/test/resources/init_script_agent_task_data.sql @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,7 +25,8 @@ TRUNCATE TABLE job_execute.gse_script_agent_task; INSERT INTO job_execute.gse_script_agent_task - (step_instance_id, + (task_instance_id, + step_instance_id, execute_count, batch, host_id, @@ -40,12 +41,12 @@ INSERT INTO job_execute.gse_script_agent_task tag, log_offset) VALUES - (1,0,1,101,'0:127.0.0.1',1,9,1565767148000,1565767149000,1316,0,0,'succ',0), - (1,0,2,102,'0:127.0.0.2',2,9,1565767148000,1565767149000,1211,0,0,'succ',0), - (1,0,3,103,'0:127.0.0.3',3,9,1565767148000,1565767149000,1211,0,0,'succ',0), - (1,0,3,104,'0:127.0.0.4',3,9,1565767148000,1565767149000,1211,0,0,'succ',0), - (1,0,3,105,'0:127.0.0.5',3,11,1565767148000,1565767149000,1211,0,0,'fail',0), - (2,0,0,101,'0:127.0.0.1',4,9,1565767148000,1565767209000,1211,0,0,'succ',0), - (2,1,0,101,'0:127.0.0.1',4,9,1565766610000,1565767211000,1215,0,0,'succ',0), - (3,0,0,101,'0:127.0.0.1',5,9,1565766610000,1565767211000,1215,0,0,'succ',0), - (3,0,0,102,'0:127.0.0.2',5,9,1565766610000,1565767211000,1215,0,0,'succ',0); + (1,1,0,1,101,'0:127.0.0.1',1,9,1565767148000,1565767149000,1316,0,0,'succ',0), + (1,1,0,2,102,'0:127.0.0.2',2,9,1565767148000,1565767149000,1211,0,0,'succ',0), + (1,1,0,3,103,'0:127.0.0.3',3,9,1565767148000,1565767149000,1211,0,0,'succ',0), + (1,1,0,3,104,'0:127.0.0.4',3,9,1565767148000,1565767149000,1211,0,0,'succ',0), + (1,1,0,3,105,'0:127.0.0.5',3,11,1565767148000,1565767149000,1211,0,0,'fail',0), + (2,2,0,0,101,'0:127.0.0.1',4,9,1565767148000,1565767209000,1211,0,0,'succ',0), + (2,2,1,0,101,'0:127.0.0.1',4,9,1565766610000,1565767211000,1215,0,0,'succ',0), + (3,3,0,0,101,'0:127.0.0.1',5,9,1565766610000,1565767211000,1215,0,0,'succ',0), + (3,3,0,0,102,'0:127.0.0.2',5,9,1565766610000,1565767211000,1215,0,0,'succ',0); diff --git a/src/backend/job-execute/boot-job-execute/src/test/resources/init_step_instance_data.sql b/src/backend/job-execute/boot-job-execute/src/test/resources/init_step_instance_data.sql index 962cc07651..ea98e31782 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/resources/init_step_instance_data.sql +++ b/src/backend/job-execute/boot-job-execute/src/test/resources/init_step_instance_data.sql @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,28 +27,23 @@ truncate table step_instance_script; truncate table step_instance_file; truncate table step_instance_confirm; -insert into job_execute.step_instance (id,app_id,task_instance_id,step_id,name,type,target_servers,operator,status,execute_count,start_time,end_time,total_time, -create_time,step_num,step_order) values (1,2,1,1,'task1-step1',1,'{"ipList":[{"cloudAreaId":0,"ip":"127.0.0.1"}]}','admin',3,0,1572868800000,1572868801000,1111,1572868800000,2,1); +insert into job_execute.step_instance ( +id,app_id,task_instance_id,step_id,name,type,target_servers,operator,status,execute_count,start_time,end_time,total_time,create_time,step_num,step_order) +values + (1,2,1,1,'task1-step1',1,'{"ipList":[{"cloudAreaId":0,"ip":"127.0.0.1"}]}','admin',3,0,1572868800000,1572868801000,1111,1572868800000,2,1), + (2,2,1,2,'task1-step2',2,'{"ipList":[{"cloudAreaId":0,"ip":"127.0.0.1"}]}','admin',3,0,1572868801000,1572868802000,1112,1572868800000,2,2), + (3,2,2,-1,'fast_execute_task_name',1,'{"ipList":[{"cloudAreaId":0,"ip":"127.0.0.1"}]}','admin',3,0,1572868800000,1572868801000,1111,1572868800000,1,1), + (4,2,3,-1,'fast_execute_task_name',1,'{"ipList":[{"cloudAreaId":0,"ip":"127.0.0.1"}]}','admin',3,0,null,1572868801000,0,null,1,1); -insert into job_execute.step_instance (id,app_id,task_instance_id,step_id,name,type,target_servers,operator,status,execute_count,start_time,end_time,total_time, -create_time,step_num,step_order) values (2,2,1,2,'task1-step2',2,'{"ipList":[{"cloudAreaId":0,"ip":"127.0.0.1"}]}','admin',3,0,1572868801000,1572868802000,1112,1572868800000,2,2); +insert into job_execute.step_instance_script(step_instance_id,task_instance_id,script_content,script_type,script_param,resolved_script_param,execution_timeout,system_account_id,system_account, + db_account_id,db_type,db_account,db_password,db_port,script_source,script_id,script_version_id,is_secure_param,windows_interpreter) values (1,1,'script_content',1,'${var1}','var1',1000,1,'root',11,1,'root','ESKsXn+pF9hACG3BSYG38ZnUjQQ8bUcOylREiEnDTPU=',3306,1,NULL,NULL,1,NULL); -insert into job_execute.step_instance (id,app_id,task_instance_id,step_id,name,type,target_servers,operator,status,execute_count,start_time,end_time,total_time, -create_time,step_num,step_order) values (3,2,2,-1,'fast_execute_task_name',1,'{"ipList":[{"cloudAreaId":0,"ip":"127.0.0.1"}]}','admin',3,0,1572868800000,1572868801000,1111,1572868800000,1,1); +insert into job_execute.step_instance_file(step_instance_id,task_instance_id,file_source,resolved_file_source,file_target_path,file_target_name,resolved_file_target_path,file_upload_speed_limit,file_download_speed_limit, + file_duplicate_handle,not_exist_path_handler,execution_timeout,system_account_id,system_account) values (2,1,'[{"files":[{ "filePath":"/${log_dir}/1.log" }],"localUpload":false}]', + '[{"files":[{ "resolvedFilePath":"/tmp/1.log", "filePath":"/${log_dir}/1.log" }],"localUpload":false}]','/${log_dir}/','2.log','/tmp/', + 100,100,1,1,1000,1,'root'); -insert into job_execute.step_instance (id,app_id,task_instance_id,step_id,name,type,target_servers,operator,status,execute_count,start_time,end_time,total_time, -create_time,step_num,step_order) values (4,2,3,-1,'fast_execute_task_name',1,'{"ipList":[{"cloudAreaId":0,"ip":"127.0.0.1"}]}','admin',3,0,null,1572868801000,0,null,1,1); - - -insert into job_execute.step_instance_script(step_instance_id,script_content,script_type,script_param,resolved_script_param,execution_timeout,system_account_id,system_account, -db_account_id,db_type,db_account,db_password,db_port,script_source,script_id,script_version_id) values (1,'script_content',1,'${var1}','var1',1000,1,'root',11,1,'root','db_password',3306,1,NULL,NULL); - -insert into job_execute.step_instance_file(step_instance_id,file_source,resolved_file_source,file_target_path,file_target_name,resolved_file_target_path,file_upload_speed_limit,file_download_speed_limit, -file_duplicate_handle,not_exist_path_handler,execution_timeout,system_account_id,system_account) values (2,'[{"files":[{ "filePath":"/${log_dir}/1.log" }],"localUpload":false}]', -'[{"files":[{ "resolvedFilePath":"/tmp/1.log", "filePath":"/${log_dir}/1.log" }],"localUpload":false}]','/${log_dir}/','2.log','/tmp/', -100,100,1,1,1000,1,'root'); - -insert into job_execute.step_instance_confirm(step_instance_id,confirm_message,confirm_users,confirm_roles,notify_channels,confirm_reason) values ( -3,'confirm_message','admin,test','JOB_RESOURCE_TRIGGER_USER','weixin','confirm_reason'); +insert into job_execute.step_instance_confirm(step_instance_id,task_instance_id,confirm_message,confirm_users,confirm_roles,notify_channels,confirm_reason) values ( + 15,13,'confirm_message','admin,test','JOB_RESOURCE_TRIGGER_USER','weixin','confirm_reason'); diff --git a/src/backend/job-execute/boot-job-execute/src/test/resources/init_step_instance_rolling_task_data.sql b/src/backend/job-execute/boot-job-execute/src/test/resources/init_step_instance_rolling_task_data.sql index 780a6c7e77..eebcc2b680 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/resources/init_step_instance_rolling_task_data.sql +++ b/src/backend/job-execute/boot-job-execute/src/test/resources/init_step_instance_rolling_task_data.sql @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,7 +25,8 @@ TRUNCATE TABLE step_instance_rolling_task; INSERT INTO job_execute.step_instance_rolling_task - (step_instance_id, + (task_instance_id, + step_instance_id, execute_count, batch, status, @@ -33,9 +34,9 @@ INSERT INTO job_execute.step_instance_rolling_task end_time, total_time) VALUES - (1,0,1,1,1642247802000,1642247803000,1000), - (1,0,2,1,1642247804000,1642247805000,1000), - (2,0,1,1,1642247804000,1642247805000,1000), - (2,0,2,1,1642247805000,1642247806000,1000), - (2,0,3,1,1642247806000,1642247807000,1000); + (1,1,0,1,1,1642247802000,1642247803000,1000), + (1,1,0,2,1,1642247804000,1642247805000,1000), + (2,2,0,1,1,1642247804000,1642247805000,1000), + (2,2,0,2,1,1642247805000,1642247806000,1000), + (2,2,0,3,1,1642247806000,1642247807000,1000); diff --git a/src/backend/job-execute/boot-job-execute/src/test/resources/init_step_instance_variable_data.sql b/src/backend/job-execute/boot-job-execute/src/test/resources/init_step_instance_variable_data.sql index 4498c86dd4..98e1fce34d 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/resources/init_step_instance_variable_data.sql +++ b/src/backend/job-execute/boot-job-execute/src/test/resources/init_step_instance_variable_data.sql @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/boot-job-execute/src/test/resources/init_task_instance_data.sql b/src/backend/job-execute/boot-job-execute/src/test/resources/init_task_instance_data.sql index 3b9fb5813b..c439cf80c6 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/resources/init_task_instance_data.sql +++ b/src/backend/job-execute/boot-job-execute/src/test/resources/init_task_instance_data.sql @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/boot-job-execute/src/test/resources/init_task_instance_variable_data.sql b/src/backend/job-execute/boot-job-execute/src/test/resources/init_task_instance_variable_data.sql index c43ebf9211..89b32fa70f 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/resources/init_task_instance_variable_data.sql +++ b/src/backend/job-execute/boot-job-execute/src/test/resources/init_task_instance_variable_data.sql @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/boot-job-execute/src/test/resources/test.properties b/src/backend/job-execute/boot-job-execute/src/test/resources/test.properties index 60e2c486d7..776524f90d 100644 --- a/src/backend/job-execute/boot-job-execute/src/test/resources/test.properties +++ b/src/backend/job-execute/boot-job-execute/src/test/resources/test.properties @@ -1,11 +1,11 @@ # -# Tencent is pleased to support the open source community by making BK-JOB����������ҵƽ̨ available. +# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. # -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +# Copyright (C) 2021 Tencent. All rights reserved. # -# BK-JOB����������ҵƽ̨ is licensed under the MIT License. +# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. # -# License for BK-JOB����������ҵƽ̨: +# License for BK-JOB蓝鲸智云作业平台: # -------------------------------------------------------------------- # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated # documentation files (the "Software"), to deal in the Software without restriction, including without limitation @@ -30,3 +30,5 @@ app.secret=job iam.system-id=bk_job iam.base-url=http://bkiam.service.consul:9080/ job.encrypt.password=test +job.encrypt.scenarioAlgorithms.scriptSensitiveParam=None +job.encrypt.scenarioAlgorithms.cipherVariable=None diff --git a/src/backend/job-execute/build.gradle b/src/backend/job-execute/build.gradle index 871bb0b29f..6d287ddb87 100644 --- a/src/backend/job-execute/build.gradle +++ b/src/backend/job-execute/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/model-job-execute/build.gradle b/src/backend/job-execute/model-job-execute/build.gradle index 53cd347e49..8006176375 100644 --- a/src/backend/job-execute/model-job-execute/build.gradle +++ b/src/backend/job-execute/model-job-execute/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/build.gradle b/src/backend/job-execute/service-job-execute/build.gradle index d6598f4911..89063d1fcb 100644 --- a/src/backend/job-execute/service-job-execute/build.gradle +++ b/src/backend/job-execute/service-job-execute/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -31,10 +31,12 @@ dependencies { api project(":job-manage:api-job-manage") api project(":job-logsvr:api-job-logsvr") api project(':commons:gse-sdk') + api project(":commons:common-crypto") + api project(':commons:cmdb-sdk') api project(':commons:common-security') api project(':commons:common-redis') - api project(":commons:cmdb-sdk-ext") api project(":commons:artifactory-sdk") + api project(":commons:common-mysql-sharding") api("org.springframework.boot:spring-boot-starter-web") api("org.springframework.boot:spring-boot-starter-jdbc") api "org.springframework.boot:spring-boot-starter-jooq" @@ -53,9 +55,9 @@ dependencies { implementation 'io.micrometer:micrometer-registry-prometheus' implementation "org.hibernate.validator:hibernate-validator" // https://mvnrepository.com/artifact/org.apache.curator/curator-framework - api(group: 'org.apache.curator', name: 'curator-framework', version: '5.1.0') + api(group: 'org.apache.curator', name: 'curator-framework') // https://mvnrepository.com/artifact/org.apache.curator/curator-recipes - api(group: 'org.apache.curator', name: 'curator-recipes', version: '5.1.0') + api(group: 'org.apache.curator', name: 'curator-recipes') testImplementation 'org.springframework.boot:spring-boot-starter-test' testImplementation 'org.apache.commons:commons-lang3' diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/common/ConfigFileUtil.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/common/ConfigFileUtil.java index 44164752e5..bdf689acd0 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/common/ConfigFileUtil.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/common/ConfigFileUtil.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,8 +26,9 @@ import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.exception.InternalException; -import com.tencent.bk.job.common.util.FileUtil; +import com.tencent.bk.job.common.util.FilePathUtils; import com.tencent.bk.job.common.util.JobUUID; +import com.tencent.bk.job.common.util.file.FileUtil; import com.tencent.bk.job.execute.engine.consts.FileDirTypeConf; import com.tencent.bk.job.execute.engine.util.NFSUtils; @@ -48,9 +49,10 @@ public static String saveConfigFileToLocal(String jobStorageRootPath, String userName, String fileName, String base64Content) { + String pureFileName = FilePathUtils.getPureFileName(fileName); String uploadPath = NFSUtils.getFileDir(jobStorageRootPath, FileDirTypeConf.UPLOAD_FILE_DIR); String configFileRelativePath = JobUUID.getUUID() + File.separatorChar + - userName + File.separatorChar + fileName; + userName + File.separatorChar + pureFileName; String fullFilePath = uploadPath.concat(configFileRelativePath); if (!FileUtil.saveBase64StrToFile(fullFilePath, base64Content)) { throw new InternalException(ErrorCode.FAIL_TO_SAVE_FILE_TO_LOCAL); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/gse/impl/GseGetProcResultResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/gse/impl/GseGetProcResultResourceImpl.java deleted file mode 100644 index 94e7e211a3..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/gse/impl/GseGetProcResultResourceImpl.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.api.esb.gse.impl; - -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.collect.Maps; -import com.tencent.bk.gse.taskapi.api_map_rsp; -import com.tencent.bk.job.common.constant.ErrorCode; -import com.tencent.bk.job.common.esb.model.EsbResp; -import com.tencent.bk.job.common.exception.InternalException; -import com.tencent.bk.job.common.exception.InvalidParamException; -import com.tencent.bk.job.common.i18n.service.MessageI18nService; -import com.tencent.bk.job.common.util.json.JsonUtils; -import com.tencent.bk.job.execute.api.esb.gse.GseGetProcResultResource; -import com.tencent.bk.job.execute.gse.GseApiExecutor; -import com.tencent.bk.job.execute.model.esb.gse.req.EsbGseGetProcResultRequest; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.RestController; - -import java.util.HashMap; -import java.util.Map; - -@RestController -@Slf4j -public class GseGetProcResultResourceImpl implements GseGetProcResultResource { - private final MessageI18nService i18nService; - - @Autowired - public GseGetProcResultResourceImpl(MessageI18nService i18nService) { - this.i18nService = i18nService; - } - - @Override - public EsbResp> gseGetProcResult(EsbGseGetProcResultRequest request) { - log.info("Gse process result, request={}", request); - if (!checkRequest(request)) { - throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); - } - String gseTaskId = request.getGseTaskId(); - - api_map_rsp resp = new GseApiExecutor("").getGetProcRst(gseTaskId); - - if (resp == null) { - throw new InternalException(ErrorCode.GSE_ERROR); - } - int RESULT_RUNNING = 1000115; - - if (resp.getBk_error_code() != ErrorCode.RESULT_OK && resp.getBk_error_code() != RESULT_RUNNING) { - return EsbResp.buildCommonFailResp(resp.getBk_error_code()); - } - - Map resultData = Maps.newHashMap(); - resultData.put("status", resp.getStatus()); - - // 把GSE返回的字符串转换为json - Map jsonResult = new HashMap<>(); - if (resp.getResult() != null && !resp.getResult().isEmpty()) { - for (Map.Entry resultKV : resp.getResult().entrySet()) { - log.info("Get proc result from gse, key={},value={}", resultKV.getKey(), resultKV.getValue()); - ObjectNode procObjectNode = (ObjectNode) JsonUtils.toJsonNode(resultKV.getValue()); - if (procObjectNode != null) { - if (procObjectNode.get("content") != null - && StringUtils.isNotEmpty(procObjectNode.get("content").textValue())) { - ObjectNode contentNode = - (ObjectNode) JsonUtils.toJsonNode(procObjectNode.get("content").textValue()); - procObjectNode.replace("content", contentNode); - } - jsonResult.put(resultKV.getKey(), procObjectNode); - } - } - } - resultData.put("result", jsonResult); - - return EsbResp.buildSuccessResp(resultData); - } - - private boolean checkRequest(EsbGseGetProcResultRequest request) { - if (request.getAppId() == null || request.getAppId() <= 0) { - log.warn("AppId is empty!"); - return false; - } - if (StringUtils.isBlank(request.getGseTaskId())) { - log.warn("GseTaskid is empty!"); - return false; - } - return true; - } - - @Data - @AllArgsConstructor - private static class GseResp { - private Integer bk_error_code; - private String bk_error_msg; - private Map result; - } -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/gse/impl/GseManageProcessResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/gse/impl/GseManageProcessResourceImpl.java deleted file mode 100644 index 6b2973519d..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/gse/impl/GseManageProcessResourceImpl.java +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.api.esb.gse.impl; - -import com.tencent.bk.job.common.constant.ErrorCode; -import com.tencent.bk.job.common.esb.model.EsbResp; -import com.tencent.bk.job.common.exception.InvalidParamException; -import com.tencent.bk.job.common.i18n.service.MessageI18nService; -import com.tencent.bk.job.common.model.dto.HostDTO; -import com.tencent.bk.job.execute.api.esb.gse.GseManageProcessResource; -import com.tencent.bk.job.execute.engine.model.GseTaskResponse; -import com.tencent.bk.job.execute.gse.GseApiExecutor; -import com.tencent.bk.job.execute.gse.model.GseProcessInfoDTO; -import com.tencent.bk.job.execute.gse.model.ProcessManageTypeEnum; -import com.tencent.bk.job.execute.model.esb.gse.EsbGseTaskResultDTO; -import com.tencent.bk.job.execute.model.esb.gse.req.EsbGseManageProcRequest; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.RestController; - -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -@RestController -@Slf4j -public class GseManageProcessResourceImpl implements GseManageProcessResource { - - private final MessageI18nService i18nService; - - @Autowired - public GseManageProcessResourceImpl(MessageI18nService i18nService) { - this.i18nService = i18nService; - } - - @Override - public EsbResp gseManageProc(EsbGseManageProcRequest request) { - log.info("Gse manage process, request={}", request); - if (!checkRequest(request)) { - throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); - } - - List gseProcessInfos = new ArrayList<>(); - for (EsbGseManageProcRequest.ManageProcessInfoDTO manageProcInfo : request.getProcessInfos()) { - GseProcessInfoDTO gseProcessInfo = buildGseProcessInfoDTO(manageProcInfo); - gseProcessInfos.add(gseProcessInfo); - } - GseTaskResponse resp = new GseApiExecutor("") - .processOperateProcess(gseProcessInfos, request.getOpType()); - - if (resp == null || resp.getErrorCode() != GseTaskResponse.ERROR_CODE_SUCCESS) { - throw new InvalidParamException(ErrorCode.GSE_ERROR); - } - EsbGseTaskResultDTO result = new EsbGseTaskResultDTO(); - result.setGseTaskId(resp.getGseTaskId()); - return EsbResp.buildSuccessResp(result); - } - - private GseProcessInfoDTO buildGseProcessInfoDTO(EsbGseManageProcRequest.ManageProcessInfoDTO manageProcInfo) { - GseProcessInfoDTO gseProcessInfo = new GseProcessInfoDTO(); - String procName = manageProcInfo.getProcName(); - String contact = manageProcInfo.getProcOwner(); - gseProcessInfo.setIpList(manageProcInfo.getIpList().stream() - .map(ipDTO -> new HostDTO(ipDTO.getBkCloudId(), ipDTO.getIp())).collect(Collectors.toList())); - gseProcessInfo.setProcName(procName); - gseProcessInfo.setContact(contact); - - String account = manageProcInfo.getAccount(); - if (StringUtils.isBlank(account)) { - account = "root"; - } - gseProcessInfo.setUserName(account); - gseProcessInfo.setSetupPath(getDefaultIfNull(manageProcInfo.getSetupPath(), "")); - gseProcessInfo.setPidPath(getDefaultIfNull(manageProcInfo.getPidPath(), "")); - gseProcessInfo.setCfgPath(getDefaultIfNull(manageProcInfo.getCfgPath(), "")); - gseProcessInfo.setLogPath(getDefaultIfNull(manageProcInfo.getLogPath(), "")); - gseProcessInfo.setStartCmd(getDefaultIfNull(manageProcInfo.getStartCmd(), "")); - gseProcessInfo.setStopCmd(getDefaultIfNull(manageProcInfo.getStopCmd(), "")); - gseProcessInfo.setRestartCmd(getDefaultIfNull(manageProcInfo.getRestartCmd(), "")); - gseProcessInfo.setKillCmd(getDefaultIfNull(manageProcInfo.getKillCmd(), "")); - gseProcessInfo.setReloadCmd(getDefaultIfNull(manageProcInfo.getReloadCmd(), "")); - gseProcessInfo.setFuncID(getDefaultIfNull(manageProcInfo.getFuncId(), "")); - gseProcessInfo.setInstanceID(getDefaultIfNull(manageProcInfo.getInstanceId(), "")); - gseProcessInfo.setValueKey(getDefaultIfNull(manageProcInfo.getValueKey(), "")); - gseProcessInfo.setType(getDefaultIfNull(manageProcInfo.getType(), 0)); - Integer cpuLmt = manageProcInfo.getCpuLmt(); - if (cpuLmt == null || cpuLmt < 0 || cpuLmt > 100) { - cpuLmt = 0; - } - gseProcessInfo.setCpuLmt(cpuLmt); - Integer memLmt = manageProcInfo.getMemLmt(); - if (memLmt == null || memLmt < 0 || memLmt > 100) { - memLmt = 0; - } - gseProcessInfo.setMemLmt(memLmt); - gseProcessInfo.setCycleTime(getDefaultIfNull(manageProcInfo.getCycleTime(), 0)); - gseProcessInfo.setInstanceNum(getDefaultIfNull(manageProcInfo.getInstanceNum(), 0)); - gseProcessInfo.setStartCheckBeginTime(getDefaultIfNull(manageProcInfo.getStartCheckBeginTime(), 0)); - gseProcessInfo.setStartCheckEndTime(getDefaultIfNull(manageProcInfo.getStartCheckEndTime(), 0)); - gseProcessInfo.setOpTimeOut(getDefaultIfNull(manageProcInfo.getOpTimeout(), 0)); - - return gseProcessInfo; - } - - private int getDefaultIfNull(Integer value, int defaultValue) { - return value == null ? defaultValue : value; - } - - private String getDefaultIfNull(String value, String defaultValue) { - return value == null ? defaultValue : value; - } - - private boolean checkRequest(EsbGseManageProcRequest request) { - if (request.getAppId() == null || request.getAppId() <= 0) { - log.warn("AppId is empty!"); - return false; - } - if (request.getOpType() == null) { - log.warn("OpType is empty!"); - return false; - } - if (!isManageProcTypeValid(request.getOpType())) { - log.warn("OpType:{} is invalid!", request.getOpType()); - return false; - } - if (request.getProcessInfos() == null || request.getProcessInfos().isEmpty()) { - log.warn("ProcessInfos is empty!"); - return false; - } - for (EsbGseManageProcRequest.ManageProcessInfoDTO processInfo : request.getProcessInfos()) { - if (StringUtils.isBlank(processInfo.getSetupPath()) || StringUtils.isBlank(processInfo.getProcName()) - || StringUtils.isBlank(processInfo.getPidPath())) { - log.warn("ProcessInfo setup_path|proc_name|pid_path is empty!"); - return false; - } - if (processInfo.getIpList() == null || processInfo.getIpList().isEmpty()) { - log.warn("ProcessInfo ip_list is empty!"); - return false; - } - - } - return true; - } - - private boolean isManageProcTypeValid(Integer opType) { - for (ProcessManageTypeEnum manageType : ProcessManageTypeEnum.values()) { - if (opType.equals(manageType.getValue())) { - return true; - } - } - return false; - } -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/gse/impl/GseOperateProcessResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/gse/impl/GseOperateProcessResourceImpl.java deleted file mode 100644 index e1c4f5a45f..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/gse/impl/GseOperateProcessResourceImpl.java +++ /dev/null @@ -1,195 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.api.esb.gse.impl; - -import com.tencent.bk.job.common.constant.ErrorCode; -import com.tencent.bk.job.common.esb.model.EsbResp; -import com.tencent.bk.job.common.exception.InternalException; -import com.tencent.bk.job.common.exception.InvalidParamException; -import com.tencent.bk.job.common.i18n.service.MessageI18nService; -import com.tencent.bk.job.common.model.dto.HostDTO; -import com.tencent.bk.job.execute.api.esb.gse.GseOperateProcessResource; -import com.tencent.bk.job.execute.engine.model.GseTaskResponse; -import com.tencent.bk.job.execute.gse.GseApiExecutor; -import com.tencent.bk.job.execute.gse.model.GseProcessInfoDTO; -import com.tencent.bk.job.execute.gse.model.ProcessOperateTypeEnum; -import com.tencent.bk.job.execute.model.esb.gse.EsbGseTaskResultDTO; -import com.tencent.bk.job.execute.model.esb.gse.EsbProcessInfoDTO; -import com.tencent.bk.job.execute.model.esb.gse.req.EsbGseOperateProcessRequest; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.RestController; - -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -@RestController -@Slf4j -public class GseOperateProcessResourceImpl implements GseOperateProcessResource { - private final MessageI18nService i18nService; - - @Autowired - public GseOperateProcessResourceImpl(MessageI18nService i18nService) { - this.i18nService = i18nService; - } - - @Override - public EsbResp gseOperateProcess(EsbGseOperateProcessRequest request) { - log.info("Gse operate process, request={}", request); - if (!checkRequest(request)) { - throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); - } - - List gseProcessInfos = new ArrayList<>(); - for (EsbProcessInfoDTO processInfo : request.getProcessInfos()) { - GseProcessInfoDTO gseProcessInfo = new GseProcessInfoDTO(); - gseProcessInfo.setIpList(processInfo.getIpList().stream().map(ipDTO -> new HostDTO(ipDTO.getBkCloudId(), - ipDTO.getIp())).collect(Collectors.toList())); - String setupPath = processInfo.getSetupPath(); - String procName = processInfo.getProcName(); - String cmdTemplate = "cd ${path} && ${cmd} ${param} ${bg}"; - - String account = processInfo.getAccount(); - if (StringUtils.isBlank(account)) { - account = "root"; - } - String cmdShellExt = processInfo.getCmdShellExt();//"sh", "bat", "ps1" - if (StringUtils.isBlank(cmdShellExt)) { - cmdShellExt = "sh"; - } - String cmdPrefix = "./", bg = "&"; - if (cmdShellExt.equalsIgnoreCase("bat") || cmdShellExt.equalsIgnoreCase("ps1")) {// bat,ps1文件不带执行前缀路径./ - // 也无后台& - cmdPrefix = ""; - bg = ""; - } - gseProcessInfo.setUserName(account); - gseProcessInfo.setProcName(procName);// bk_gse_unifyTlogc - gseProcessInfo.setSetupPath(setupPath);// /usr/local/gse/gseagent/plugins/unifyTlogc/sbin - gseProcessInfo.setCfgPath("");// /usr/local/gse/gseagent/plugins/unifyTlogc/conf/ - gseProcessInfo.setLogPath("");// /usr/local/gse/gseagent/plugins/unifyTlogc/log/ - gseProcessInfo.setPidPath(processInfo.getPidPath());// /usr/local/gse/gseagent/plugins/unifyTlogc/log - // /unifyTlogc.pid - gseProcessInfo.setContact("mapleleaf"); - gseProcessInfo.setStartCmd(cmdTemplate.replace("${path}", setupPath).replace("${cmd}", cmdPrefix + "start" + - "." + cmdShellExt).replace("${param}", procName).replace("${bg}", bg)); - gseProcessInfo.setStopCmd(cmdTemplate.replace("${path}", setupPath).replace("${cmd}", - cmdPrefix + "stop." + cmdShellExt).replace("${param}", procName).replace("${bg}", bg)); - gseProcessInfo.setRestartCmd(cmdTemplate.replace("${path}", setupPath).replace("${cmd}", cmdPrefix + - "restart." + cmdShellExt).replace("${param}", procName).replace("${bg}", bg)); - gseProcessInfo.setReloadCmd(cmdTemplate.replace("${path}", setupPath).replace("${cmd}", cmdPrefix + - "reload." + cmdShellExt).replace("${param}", procName).replace("${bg}", bg)); - gseProcessInfo.setKillCmd(""); - gseProcessInfo.setFuncID(""); - gseProcessInfo.setInstanceID(""); - - String fullPath; - if (setupPath.endsWith("/") || setupPath.endsWith("\\")) { - fullPath = setupPath + procName; - } else if (setupPath.contains("\\")) { - fullPath = setupPath + '\\' + procName; - } else { - fullPath = setupPath + '/' + procName; - } - gseProcessInfo.setValueKey(fullPath); - - gseProcessInfo.setType(1);//常驻进程托管 - Integer cpuLmt = processInfo.getCpuLmt(); - if (cpuLmt == null || cpuLmt < 0 || cpuLmt > 100) { - cpuLmt = 50; - } else { - cpuLmt = processInfo.getCpuLmt(); - } - gseProcessInfo.setCpuLmt(cpuLmt); - Integer memLmt = processInfo.getMemLmt(); - if (memLmt == null || memLmt < 0 || memLmt > 100) { - memLmt = 30; - } else { - memLmt = processInfo.getMemLmt(); - } - gseProcessInfo.setMemLmt(memLmt); - gseProcessInfo.setCycleTime(0); - gseProcessInfo.setInstanceNum(1); - gseProcessInfo.setStartCheckBeginTime(5); - gseProcessInfo.setStartCheckEndTime(1); - gseProcessInfo.setOpTimeOut(10);//5-10 - - gseProcessInfos.add(gseProcessInfo); - } - GseTaskResponse resp = new GseApiExecutor("") - .processOperateProcess(gseProcessInfos, request.getOpType()); - - if (resp == null || resp.getErrorCode() != GseTaskResponse.ERROR_CODE_SUCCESS) { - throw new InternalException(ErrorCode.GSE_ERROR); - } - EsbGseTaskResultDTO result = new EsbGseTaskResultDTO(); - result.setGseTaskId(resp.getGseTaskId()); - return EsbResp.buildSuccessResp(result); - } - - private boolean checkRequest(EsbGseOperateProcessRequest request) { - if (request.getAppId() == null || request.getAppId() <= 0) { - log.warn("AppId is empty!"); - return false; - } - if (request.getOpType() == null) { - log.warn("OpType is empty!"); - return false; - } - if (!isOperateTypeValid(request.getOpType())) { - log.warn("OpType:{} is invalid!", request.getOpType()); - return false; - } - if (request.getProcessInfos() == null || request.getProcessInfos().isEmpty()) { - log.warn("ProcessInfos is empty!"); - return false; - } - for (EsbProcessInfoDTO processInfo : request.getProcessInfos()) { - if (StringUtils.isBlank(processInfo.getSetupPath()) || StringUtils.isBlank(processInfo.getProcName()) - || StringUtils.isBlank(processInfo.getPidPath())) { - log.warn("ProcessInfo setup_path|proc_name|pid_path is empty!"); - return false; - } - if (processInfo.getIpList() == null || processInfo.getIpList().isEmpty()) { - log.warn("ProcessInfo ip_list is empty!"); - return false; - } - - } - return true; - } - - private boolean isOperateTypeValid(Integer opType) { - for (ProcessOperateTypeEnum operateType : ProcessOperateTypeEnum.values()) { - if (opType.equals(operateType.getValue())) { - return true; - } - } - return false; - } - -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/gse/impl/GseOperateProcessV2ResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/gse/impl/GseOperateProcessV2ResourceImpl.java deleted file mode 100644 index 540fefcdd2..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/gse/impl/GseOperateProcessV2ResourceImpl.java +++ /dev/null @@ -1,159 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.api.esb.gse.impl; - -import com.tencent.bk.job.common.constant.ErrorCode; -import com.tencent.bk.job.common.esb.model.EsbResp; -import com.tencent.bk.job.common.exception.InternalException; -import com.tencent.bk.job.common.exception.InvalidParamException; -import com.tencent.bk.job.common.i18n.service.MessageI18nService; -import com.tencent.bk.job.common.model.dto.HostDTO; -import com.tencent.bk.job.execute.api.esb.gse.GseOperateProcessV2Resource; -import com.tencent.bk.job.execute.engine.model.GseTaskResponse; -import com.tencent.bk.job.execute.gse.GseApiExecutor; -import com.tencent.bk.job.execute.gse.model.GseProcessInfoDTO; -import com.tencent.bk.job.execute.gse.model.ProcessOperateTypeEnum; -import com.tencent.bk.job.execute.model.esb.gse.EsbGseTaskResultDTO; -import com.tencent.bk.job.execute.model.esb.gse.EsbProcessInfoDTO; -import com.tencent.bk.job.execute.model.esb.gse.req.EsbGseOperateProcessRequest; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.RestController; - -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -@RestController -@Slf4j -public class GseOperateProcessV2ResourceImpl implements GseOperateProcessV2Resource { - - private final MessageI18nService i18nService; - - @Autowired - public GseOperateProcessV2ResourceImpl(MessageI18nService i18nService) { - this.i18nService = i18nService; - } - - @Override - public EsbResp gseOperateProcessV2(EsbGseOperateProcessRequest request) { - log.info("Gse operate process v2, request={}", request); - if (!checkRequest(request)) { - throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); - } - - List gseProcessInfos = new ArrayList<>(); - for (EsbProcessInfoDTO processInfo : request.getProcessInfos()) { - GseProcessInfoDTO gseProcessInfo = new GseProcessInfoDTO(); - gseProcessInfo.setIpList(processInfo.getIpList().stream().map(ipDTO -> new HostDTO(ipDTO.getBkCloudId(), - ipDTO.getIp())).collect(Collectors.toList())); - String setupPath = processInfo.getSetupPath(); - String procName = processInfo.getProcName(); - - String account = processInfo.getAccount(); - if (StringUtils.isBlank(account)) { - account = "root"; - } - gseProcessInfo.setUserName(account); - gseProcessInfo.setProcName(procName);// bk_gse_unifyTlogc - gseProcessInfo.setSetupPath(setupPath);// /usr/local/gse/gseagent/plugins/unifyTlogc/sbin - gseProcessInfo.setCfgPath(processInfo.getCfgPath());// /usr/local/gse/gseagent/plugins/unifyTlogc/conf/ - gseProcessInfo.setLogPath(processInfo.getLogPath());// /usr/local/gse/gseagent/plugins/unifyTlogc/log/ - gseProcessInfo.setPidPath(processInfo.getPidPath());// /usr/local/gse/gseagent/plugins/unifyTlogc/log - // /unifyTlogc.pid - gseProcessInfo.setContact(processInfo.getContact()); - gseProcessInfo.setStartCmd(processInfo.getStartCmd()); - gseProcessInfo.setStopCmd(processInfo.getStopCmd()); - gseProcessInfo.setRestartCmd(processInfo.getRestartCmd()); - gseProcessInfo.setReloadCmd(processInfo.getReloadCmd()); - gseProcessInfo.setKillCmd(processInfo.getKillCmd()); - gseProcessInfo.setFuncID(processInfo.getFuncId()); - gseProcessInfo.setInstanceID(processInfo.getInstanceId()); - gseProcessInfo.setValueKey(processInfo.getValueKey()); - - gseProcessInfo.setType(processInfo.getType());//常驻进程托管 - gseProcessInfo.setCpuLmt(processInfo.getCpuLmt()); - gseProcessInfo.setMemLmt(processInfo.getMemLmt()); - gseProcessInfo.setCycleTime(processInfo.getCycleTime()); - gseProcessInfo.setInstanceNum(processInfo.getInstanceNum()); - gseProcessInfo.setStartCheckBeginTime(processInfo.getStartCheckBeginTime()); - gseProcessInfo.setStartCheckEndTime(processInfo.getStartCheckBeginTime()); - gseProcessInfo.setOpTimeOut(processInfo.getOpTimeout());//5-10 - - gseProcessInfos.add(gseProcessInfo); - } - GseTaskResponse resp = new GseApiExecutor("") - .processOperateProcess(gseProcessInfos, request.getOpType()); - - if (resp == null || resp.getErrorCode() != GseTaskResponse.ERROR_CODE_SUCCESS) { - throw new InternalException(ErrorCode.GSE_ERROR); - } - EsbGseTaskResultDTO result = new EsbGseTaskResultDTO(); - result.setGseTaskId(resp.getGseTaskId()); - return EsbResp.buildSuccessResp(result); - } - - private boolean checkRequest(EsbGseOperateProcessRequest request) { - if (request.getAppId() == null || request.getAppId() <= 0) { - log.warn("AppId is empty!"); - return false; - } - if (request.getOpType() == null) { - log.warn("OpType is empty!"); - return false; - } - if (!isOperateTypeValid(request.getOpType())) { - log.warn("OpType:{} is invalid!", request.getOpType()); - return false; - } - if (request.getProcessInfos() == null || request.getProcessInfos().isEmpty()) { - log.warn("ProcessInfos is empty!"); - return false; - } - for (EsbProcessInfoDTO processInfo : request.getProcessInfos()) { - if (StringUtils.isBlank(processInfo.getSetupPath()) || StringUtils.isBlank(processInfo.getProcName()) - || StringUtils.isBlank(processInfo.getPidPath())) { - log.warn("ProcessInfo setup_path|proc_name|pid_path is empty!"); - return false; - } - if (processInfo.getIpList() == null || processInfo.getIpList().isEmpty()) { - log.warn("ProcessInfo ip_list is empty!"); - return false; - } - - } - return true; - } - - private boolean isOperateTypeValid(Integer opType) { - for (ProcessOperateTypeEnum operateType : ProcessOperateTypeEnum.values()) { - if (opType.equals(operateType.getValue())) { - return true; - } - } - return false; - } -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbExecuteTaskResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbExecuteTaskResourceImpl.java index 53362cba60..e5c816d467 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbExecuteTaskResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbExecuteTaskResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,8 @@ package com.tencent.bk.job.execute.api.esb.v2.impl; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.audit.annotations.AuditRequestBody; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.esb.metrics.EsbApiTimed; import com.tencent.bk.job.common.esb.model.EsbResp; @@ -31,10 +33,10 @@ import com.tencent.bk.job.common.esb.model.job.EsbIpDTO; import com.tencent.bk.job.common.esb.model.job.EsbServerDTO; import com.tencent.bk.job.common.exception.InvalidParamException; +import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.metrics.CommonMetricNames; import com.tencent.bk.job.common.model.ValidateResult; import com.tencent.bk.job.common.model.dto.HostDTO; -import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.common.web.metrics.CustomTimed; import com.tencent.bk.job.execute.api.esb.v2.EsbExecuteTaskResource; @@ -43,7 +45,7 @@ import com.tencent.bk.job.execute.metrics.ExecuteMetricsConstants; import com.tencent.bk.job.execute.model.DynamicServerGroupDTO; import com.tencent.bk.job.execute.model.DynamicServerTopoNodeDTO; -import com.tencent.bk.job.execute.model.ServersDTO; +import com.tencent.bk.job.execute.model.ExecuteTargetDTO; import com.tencent.bk.job.execute.model.TaskExecuteParam; import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.model.esb.v2.EsbJobExecuteDTO; @@ -62,13 +64,10 @@ public class EsbExecuteTaskResourceImpl extends JobExecuteCommonProcessor implements EsbExecuteTaskResource { private final TaskExecuteService taskExecuteService; - private final AppScopeMappingService appScopeMappingService; @Autowired - public EsbExecuteTaskResourceImpl(TaskExecuteService taskExecuteService, - AppScopeMappingService appScopeMappingService) { + public EsbExecuteTaskResourceImpl(TaskExecuteService taskExecuteService) { this.taskExecuteService = taskExecuteService; - this.appScopeMappingService = appScopeMappingService; } @Override @@ -78,8 +77,11 @@ public EsbExecuteTaskResourceImpl(TaskExecuteService taskExecuteService, ExecuteMetricsConstants.TAG_KEY_START_MODE, ExecuteMetricsConstants.TAG_VALUE_START_MODE_API, ExecuteMetricsConstants.TAG_KEY_TASK_TYPE, ExecuteMetricsConstants.TAG_VALUE_TASK_TYPE_EXECUTE_PLAN }) - public EsbResp executeJob(EsbExecuteJobRequest request) { - request.fillAppResourceScope(appScopeMappingService); + @AuditEntry(actionId = ActionId.LAUNCH_JOB_PLAN) + public EsbResp executeJob( + String username, + String appCode, + @AuditRequestBody EsbExecuteJobRequest request) { log.info("Execute task, request={}", JsonUtils.toJson(request)); ValidateResult checkResult = checkExecuteTaskRequest(request); @@ -98,9 +100,9 @@ public EsbResp executeJob(EsbExecuteJobRequest request) { taskVariableDTO.setName(globalVar.getName()); if ((globalVar.getIpList() != null || globalVar.getDynamicGroupIdList() != null || globalVar.getTargetServer() != null) && StringUtils.isEmpty(globalVar.getValue())) { - ServersDTO serversDTO = convertToServersDTO(globalVar.getTargetServer(), globalVar.getIpList(), - globalVar.getDynamicGroupIdList()); - taskVariableDTO.setTargetServers(serversDTO); + ExecuteTargetDTO executeTargetDTO = convertToServersDTO( + globalVar.getTargetServer(), globalVar.getIpList(), globalVar.getDynamicGroupIdList()); + taskVariableDTO.setExecuteTarget(executeTargetDTO); } else { taskVariableDTO.setValue(globalVar.getValue()); } @@ -112,11 +114,11 @@ public EsbResp executeJob(EsbExecuteJobRequest request) { .builder() .appId(request.getAppId()) .planId(request.getTaskId()) - .operator(request.getUserName()) + .operator(username) .executeVariableValues(executeVariableValues) .startupMode(TaskStartupModeEnum.API) .callbackUrl(request.getCallbackUrl()) - .appCode(request.getAppCode()) + .appCode(appCode) .build()); EsbJobExecuteDTO result = new EsbJobExecuteDTO(); @@ -143,43 +145,43 @@ private ValidateResult checkExecuteTaskRequest(EsbExecuteJobRequest request) { return ValidateResult.pass(); } - private ServersDTO convertToServersDTO(EsbServerDTO servers, List ipList, - List dynamicGroupIdList) { + private ExecuteTargetDTO convertToServersDTO(EsbServerDTO servers, List ipList, + List dynamicGroupIdList) { if (servers == null && ipList == null && dynamicGroupIdList == null) { return null; } - ServersDTO serversDTO = new ServersDTO(); + ExecuteTargetDTO executeTargetDTO = new ExecuteTargetDTO(); if (servers != null) { if (servers.getIps() != null) { List staticIpList = new ArrayList<>(); servers.getIps().forEach(ip -> staticIpList.add(new HostDTO(ip.getBkCloudId(), ip.getIp()))); - serversDTO.setStaticIpList(staticIpList); + executeTargetDTO.setStaticIpList(staticIpList); } if (servers.getDynamicGroupIds() != null) { List dynamicServerGroups = new ArrayList<>(); servers.getDynamicGroupIds().forEach( groupId -> dynamicServerGroups.add(new DynamicServerGroupDTO(groupId))); - serversDTO.setDynamicServerGroups(dynamicServerGroups); + executeTargetDTO.setDynamicServerGroups(dynamicServerGroups); } if (servers.getTopoNodes() != null) { List topoNodes = new ArrayList<>(); servers.getTopoNodes().forEach( topoNode -> topoNodes.add(new DynamicServerTopoNodeDTO(topoNode.getId(), topoNode.getNodeType()))); - serversDTO.setTopoNodes(topoNodes); + executeTargetDTO.setTopoNodes(topoNodes); } } else { if (ipList != null) { List staticIpList = new ArrayList<>(); ipList.forEach(ip -> staticIpList.add(new HostDTO(ip.getBkCloudId(), ip.getIp()))); - serversDTO.setStaticIpList(staticIpList); + executeTargetDTO.setStaticIpList(staticIpList); } if (dynamicGroupIdList != null) { List dynamicServerGroups = new ArrayList<>(); dynamicGroupIdList.forEach(groupId -> dynamicServerGroups.add(new DynamicServerGroupDTO(groupId))); - serversDTO.setDynamicServerGroups(dynamicServerGroups); + executeTargetDTO.setDynamicServerGroups(dynamicServerGroups); } } - return serversDTO; + return executeTargetDTO; } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbFastExecuteSQLResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbFastExecuteSQLResourceImpl.java index 679adc3c88..f3ff644d06 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbFastExecuteSQLResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbFastExecuteSQLResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,9 @@ package com.tencent.bk.job.execute.api.esb.v2.impl; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.audit.annotations.AuditRequestBody; +import com.tencent.bk.job.common.constant.AccountCategoryEnum; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.constant.JobConstants; import com.tencent.bk.job.common.esb.metrics.EsbApiTimed; @@ -34,7 +37,6 @@ import com.tencent.bk.job.common.i18n.service.MessageI18nService; import com.tencent.bk.job.common.metrics.CommonMetricNames; import com.tencent.bk.job.common.model.ValidateResult; -import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.common.util.ArrayUtil; import com.tencent.bk.job.common.util.Base64Util; import com.tencent.bk.job.common.util.date.DateUtils; @@ -52,11 +54,8 @@ import com.tencent.bk.job.execute.model.esb.v2.EsbJobExecuteDTO; import com.tencent.bk.job.execute.model.esb.v2.request.EsbFastExecuteSQLRequest; import com.tencent.bk.job.execute.service.AccountService; -import com.tencent.bk.job.execute.service.ScriptService; import com.tencent.bk.job.execute.service.TaskExecuteService; -import com.tencent.bk.job.manage.common.consts.account.AccountCategoryEnum; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; -import com.tencent.bk.job.manage.model.inner.ServiceScriptDTO; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -74,21 +73,13 @@ public class EsbFastExecuteSQLResourceImpl extends JobExecuteCommonProcessor imp private final MessageI18nService i18nService; - private final ScriptService scriptService; - - private final AppScopeMappingService appScopeMappingService; - @Autowired public EsbFastExecuteSQLResourceImpl(TaskExecuteService taskExecuteService, AccountService accountService, - MessageI18nService i18nService, - ScriptService scriptService, - AppScopeMappingService appScopeMappingService) { + MessageI18nService i18nService) { this.taskExecuteService = taskExecuteService; this.accountService = accountService; this.i18nService = i18nService; - this.scriptService = scriptService; - this.appScopeMappingService = appScopeMappingService; } @Override @@ -98,8 +89,11 @@ public EsbFastExecuteSQLResourceImpl(TaskExecuteService taskExecuteService, ExecuteMetricsConstants.TAG_KEY_START_MODE, ExecuteMetricsConstants.TAG_VALUE_START_MODE_API, ExecuteMetricsConstants.TAG_KEY_TASK_TYPE, ExecuteMetricsConstants.TAG_VALUE_TASK_TYPE_FAST_SQL }) - public EsbResp fastExecuteSQL(EsbFastExecuteSQLRequest request) { - request.fillAppResourceScope(appScopeMappingService); + @AuditEntry + public EsbResp fastExecuteSQL(String username, + String appCode, + @AuditRequestBody EsbFastExecuteSQLRequest request) { + ValidateResult validateResult = checkFastExecuteSQLRequest(request); if (!validateResult.isPass()) { @@ -109,16 +103,14 @@ public EsbResp fastExecuteSQL(EsbFastExecuteSQLRequest request request.trimIps(); - ServiceScriptDTO script = getAndCheckScript(request.getAppId(), request.getUserName(), request.getScriptId(), - scriptService); - TaskInstanceDTO taskInstance = buildFastSQLTaskInstance(request); - StepInstanceDTO stepInstance = buildFastSQLStepInstance(request, script); - long taskInstanceId = taskExecuteService.executeFastTask( + TaskInstanceDTO taskInstance = buildFastSQLTaskInstance(username, appCode, request); + StepInstanceDTO stepInstance = buildFastSQLStepInstance(username, request); + TaskInstanceDTO executeTaskInstance = taskExecuteService.executeFastTask( FastTaskDTO.builder().taskInstance(taskInstance).stepInstance(stepInstance).build() ); EsbJobExecuteDTO jobExecuteInfo = new EsbJobExecuteDTO(); - jobExecuteInfo.setTaskInstanceId(taskInstanceId); + jobExecuteInfo.setTaskInstanceId(executeTaskInstance.getId()); jobExecuteInfo.setTaskName(stepInstance.getName()); return EsbResp.buildSuccessResp(jobExecuteInfo); } @@ -129,9 +121,9 @@ private String generateDefaultFastTaskName() { } private ValidateResult checkFastExecuteSQLRequest(EsbFastExecuteSQLRequest request) { - if (request.getScriptId() != null) { - if (request.getScriptId() < 1) { - log.warn("Fast execute SQL, scriptId:{} is invalid", request.getScriptId()); + if (request.getScriptVersionId() != null) { + if (request.getScriptVersionId() < 1) { + log.warn("Fast execute SQL, scriptId:{} is invalid", request.getScriptVersionId()); return ValidateResult.fail(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME, "script_id"); } } else { @@ -156,7 +148,9 @@ private ValidateResult checkFastExecuteSQLRequest(EsbFastExecuteSQLRequest reque } - private TaskInstanceDTO buildFastSQLTaskInstance(EsbFastExecuteSQLRequest request) { + private TaskInstanceDTO buildFastSQLTaskInstance(String username, + String appCode, + EsbFastExecuteSQLRequest request) { TaskInstanceDTO taskInstance = new TaskInstanceDTO(); if (StringUtils.isNotBlank(request.getName())) { taskInstance.setName(request.getName()); @@ -165,8 +159,8 @@ private TaskInstanceDTO buildFastSQLTaskInstance(EsbFastExecuteSQLRequest reques } taskInstance.setCronTaskId(-1L); taskInstance.setAppId(request.getAppId()); - taskInstance.setOperator(request.getUserName()); - taskInstance.setTaskId(-1L); + taskInstance.setOperator(username); + taskInstance.setPlanId(-1L); taskInstance.setTaskTemplateId(-1L); taskInstance.setDebugTask(false); taskInstance.setStatus(RunStatusEnum.BLANK); @@ -174,13 +168,13 @@ private TaskInstanceDTO buildFastSQLTaskInstance(EsbFastExecuteSQLRequest reques taskInstance.setCurrentStepInstanceId(0L); taskInstance.setCreateTime(DateUtils.currentTimeMillis()); taskInstance.setType(TaskTypeEnum.SCRIPT.getValue()); - taskInstance.setAppCode(request.getAppCode()); + taskInstance.setAppCode(appCode); taskInstance.setCallbackUrl(request.getCallbackUrl()); return taskInstance; } - private StepInstanceDTO buildFastSQLStepInstance(EsbFastExecuteSQLRequest request, ServiceScriptDTO script) + private StepInstanceDTO buildFastSQLStepInstance(String username, EsbFastExecuteSQLRequest request) throws ServiceException { StepInstanceDTO stepInstance = new StepInstanceDTO(); stepInstance.setAppId(request.getAppId()); @@ -190,9 +184,9 @@ private StepInstanceDTO buildFastSQLStepInstance(EsbFastExecuteSQLRequest reques stepInstance.setName(generateDefaultFastTaskName()); } stepInstance.setStepId(-1L); - stepInstance.setScriptType(ScriptTypeEnum.SQL.getValue()); - if (script != null) { - stepInstance.setScriptContent(script.getContent()); + stepInstance.setScriptType(ScriptTypeEnum.SQL); + if (request.getScriptVersionId() != null && request.getScriptVersionId() > 0) { + stepInstance.setScriptVersionId(request.getScriptVersionId()); } else { // 对传入参数进行base64解码 stepInstance.setScriptContent(Base64Util.decodeContentToStr(request.getContent())); @@ -200,9 +194,9 @@ private StepInstanceDTO buildFastSQLStepInstance(EsbFastExecuteSQLRequest reques stepInstance.setTimeout(request.getTimeout() == null ? JobConstants.DEFAULT_JOB_TIMEOUT_SECONDS : request.getTimeout()); - stepInstance.setExecuteType(StepExecuteTypeEnum.EXECUTE_SQL.getValue()); + stepInstance.setExecuteType(StepExecuteTypeEnum.EXECUTE_SQL); stepInstance.setStatus(RunStatusEnum.BLANK); - stepInstance.setTargetServers(convertToStandardServers(request.getTargetServer(), request.getIpList(), + stepInstance.setTargetExecuteObjects(convertToStandardServers(request.getTargetServer(), request.getIpList(), request.getDynamicGroupIdList())); AccountDTO account = accountService.getAccountById(request.getDbAccountId()); @@ -218,7 +212,7 @@ private StepInstanceDTO buildFastSQLStepInstance(EsbFastExecuteSQLRequest reques } stepInstance.setAccountId(account.getDbSystemAccountId()); stepInstance.setDbAccountId(account.getId()); - stepInstance.setOperator(request.getUserName()); + stepInstance.setOperator(username); stepInstance.setCreateTime(DateUtils.currentTimeMillis()); return stepInstance; } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbFastExecuteScriptResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbFastExecuteScriptResourceImpl.java index a3355747fd..571a96cdcc 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbFastExecuteScriptResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbFastExecuteScriptResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,9 @@ package com.tencent.bk.job.execute.api.esb.v2.impl; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.audit.annotations.AuditRequestBody; +import com.tencent.bk.job.common.constant.AccountCategoryEnum; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.constant.JobConstants; import com.tencent.bk.job.common.esb.metrics.EsbApiTimed; @@ -34,7 +37,6 @@ import com.tencent.bk.job.common.i18n.service.MessageI18nService; import com.tencent.bk.job.common.metrics.CommonMetricNames; import com.tencent.bk.job.common.model.ValidateResult; -import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.common.util.ArrayUtil; import com.tencent.bk.job.common.util.Base64Util; import com.tencent.bk.job.common.util.date.DateUtils; @@ -52,11 +54,8 @@ import com.tencent.bk.job.execute.model.esb.v2.EsbJobExecuteDTO; import com.tencent.bk.job.execute.model.esb.v2.request.EsbFastExecuteScriptRequest; import com.tencent.bk.job.execute.service.AccountService; -import com.tencent.bk.job.execute.service.ScriptService; import com.tencent.bk.job.execute.service.TaskExecuteService; -import com.tencent.bk.job.manage.common.consts.account.AccountCategoryEnum; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; -import com.tencent.bk.job.manage.model.inner.ServiceScriptDTO; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -76,21 +75,13 @@ public class EsbFastExecuteScriptResourceImpl private final MessageI18nService i18nService; - private final ScriptService scriptService; - - private final AppScopeMappingService appScopeMappingService; - @Autowired public EsbFastExecuteScriptResourceImpl(TaskExecuteService taskExecuteService, AccountService accountService, - MessageI18nService i18nService, - ScriptService scriptService, - AppScopeMappingService appScopeMappingService) { + MessageI18nService i18nService) { this.taskExecuteService = taskExecuteService; this.accountService = accountService; this.i18nService = i18nService; - this.scriptService = scriptService; - this.appScopeMappingService = appScopeMappingService; } @Override @@ -100,8 +91,10 @@ public EsbFastExecuteScriptResourceImpl(TaskExecuteService taskExecuteService, ExecuteMetricsConstants.TAG_KEY_START_MODE, ExecuteMetricsConstants.TAG_VALUE_START_MODE_API, ExecuteMetricsConstants.TAG_KEY_TASK_TYPE, ExecuteMetricsConstants.TAG_VALUE_TASK_TYPE_FAST_SCRIPT }) - public EsbResp fastExecuteScript(EsbFastExecuteScriptRequest request) { - request.fillAppResourceScope(appScopeMappingService); + @AuditEntry + public EsbResp fastExecuteScript(String username, + String appCode, + @AuditRequestBody EsbFastExecuteScriptRequest request) { ValidateResult checkResult = checkFastExecuteScriptRequest(request); if (!checkResult.isPass()) { @@ -111,30 +104,29 @@ public EsbResp fastExecuteScript(EsbFastExecuteScriptRequest r request.trimIps(); - ServiceScriptDTO script = getAndCheckScript(request.getAppId(), request.getUserName(), request.getScriptId(), - scriptService); - TaskInstanceDTO taskInstance = buildFastScriptTaskInstance(request); - StepInstanceDTO stepInstance = buildFastScriptStepInstance(request, script); - long taskInstanceId = taskExecuteService.executeFastTask( + TaskInstanceDTO taskInstance = buildFastScriptTaskInstance(username, appCode, request); + StepInstanceDTO stepInstance = buildFastScriptStepInstance(username, request); + TaskInstanceDTO executeTaskInstance = taskExecuteService.executeFastTask( FastTaskDTO.builder().taskInstance(taskInstance).stepInstance(stepInstance).build() ); EsbJobExecuteDTO jobExecuteInfo = new EsbJobExecuteDTO(); - jobExecuteInfo.setTaskInstanceId(taskInstanceId); + jobExecuteInfo.setTaskInstanceId(executeTaskInstance.getId()); jobExecuteInfo.setTaskName(stepInstance.getName()); jobExecuteInfo.setStepInstanceId(stepInstance.getId()); return EsbResp.buildSuccessResp(jobExecuteInfo); } + private String generateDefaultFastTaskName() { return i18nService.getI18n("task.type.name.fast_execute_script") + "_" + DateUtils.formatLocalDateTime(LocalDateTime.now(), "yyyyMMddHHmmssSSS"); } private ValidateResult checkFastExecuteScriptRequest(EsbFastExecuteScriptRequest request) { - if (request.getScriptId() != null) { - if (request.getScriptId() < 1) { - log.warn("Fast execute script, scriptId:{} is invalid", request.getScriptId()); + if (request.getScriptVersionId() != null) { + if (request.getScriptVersionId() < 1) { + log.warn("Fast execute script, scriptId:{} is invalid", request.getScriptVersionId()); return ValidateResult.fail(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME, "script_id"); } } else { @@ -163,31 +155,33 @@ private ValidateResult checkFastExecuteScriptRequest(EsbFastExecuteScriptRequest } - private TaskInstanceDTO buildFastScriptTaskInstance(EsbFastExecuteScriptRequest request) { + private TaskInstanceDTO buildFastScriptTaskInstance(String username, + String appCode, + EsbFastExecuteScriptRequest request) { TaskInstanceDTO taskInstance = new TaskInstanceDTO(); if (StringUtils.isNotBlank(request.getName())) { taskInstance.setName(request.getName()); } else { taskInstance.setName(generateDefaultFastTaskName()); } - taskInstance.setTaskId(-1L); + taskInstance.setPlanId(-1L); taskInstance.setCronTaskId(-1L); taskInstance.setTaskTemplateId(-1L); taskInstance.setDebugTask(false); taskInstance.setAppId(request.getAppId()); taskInstance.setStartupMode(TaskStartupModeEnum.API.getValue()); taskInstance.setStatus(RunStatusEnum.BLANK); - taskInstance.setOperator(request.getUserName()); + taskInstance.setOperator(username); taskInstance.setCreateTime(DateUtils.currentTimeMillis()); taskInstance.setType(TaskTypeEnum.SCRIPT.getValue()); taskInstance.setCurrentStepInstanceId(0L); taskInstance.setCallbackUrl(request.getCallbackUrl()); - taskInstance.setAppCode(request.getAppCode()); + taskInstance.setAppCode(appCode); return taskInstance; } - private StepInstanceDTO buildFastScriptStepInstance(EsbFastExecuteScriptRequest request, ServiceScriptDTO script) + private StepInstanceDTO buildFastScriptStepInstance(String username, EsbFastExecuteScriptRequest request) throws ServiceException { StepInstanceDTO stepInstance = new StepInstanceDTO(); stepInstance.setAppId(request.getAppId()); @@ -197,13 +191,12 @@ private StepInstanceDTO buildFastScriptStepInstance(EsbFastExecuteScriptRequest stepInstance.setName(generateDefaultFastTaskName()); } stepInstance.setStepId(-1L); - if (script != null) { - stepInstance.setScriptContent(script.getContent()); - stepInstance.setScriptType(script.getType()); + if (request.getScriptVersionId() != null && request.getScriptVersionId() > 0) { + stepInstance.setScriptVersionId(request.getScriptVersionId()); } else { // 对传入参数进行base64解码 stepInstance.setScriptContent(Base64Util.decodeContentToStr(request.getContent())); - stepInstance.setScriptType(request.getScriptType()); + stepInstance.setScriptType(ScriptTypeEnum.valOf(request.getScriptType())); } if (StringUtils.isNotEmpty(request.getScriptParam())) { String scriptParam = Base64Util.decodeContentToStr(request.getScriptParam()); @@ -216,9 +209,9 @@ private StepInstanceDTO buildFastScriptStepInstance(EsbFastExecuteScriptRequest stepInstance.setTimeout(request.getTimeout() == null ? JobConstants.DEFAULT_JOB_TIMEOUT_SECONDS : request.getTimeout()); - stepInstance.setExecuteType(StepExecuteTypeEnum.EXECUTE_SCRIPT.getValue()); + stepInstance.setExecuteType(StepExecuteTypeEnum.EXECUTE_SCRIPT); stepInstance.setStatus(RunStatusEnum.BLANK); - stepInstance.setTargetServers(convertToStandardServers(request.getTargetServer(), request.getIpList(), + stepInstance.setTargetExecuteObjects(convertToStandardServers(request.getTargetServer(), request.getIpList(), request.getDynamicGroupIdList())); AccountDTO account = accountService.getSystemAccountByAlias(request.getAccount(), request.getAppId()); if (account == null) { @@ -230,7 +223,7 @@ private StepInstanceDTO buildFastScriptStepInstance(EsbFastExecuteScriptRequest throw new NotFoundException(ErrorCode.ACCOUNT_NOT_EXIST, ArrayUtil.toArray(request.getAccount())); } stepInstance.setAccountId(account.getId()); - stepInstance.setOperator(request.getUserName()); + stepInstance.setOperator(username); stepInstance.setCreateTime(DateUtils.currentTimeMillis()); // ESB接口默认为手工录入的本地脚本 stepInstance.setScriptSource(1); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbFastPushFileResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbFastPushFileResourceImpl.java index f3a82c8d2d..19963246f4 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbFastPushFileResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbFastPushFileResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,9 @@ package com.tencent.bk.job.execute.api.esb.v2.impl; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.audit.annotations.AuditRequestBody; +import com.tencent.bk.job.common.constant.AccountCategoryEnum; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.constant.JobConstants; import com.tencent.bk.job.common.constant.NotExistPathHandlerEnum; @@ -34,10 +37,12 @@ import com.tencent.bk.job.common.exception.NotFoundException; import com.tencent.bk.job.common.exception.ServiceException; import com.tencent.bk.job.common.i18n.service.MessageI18nService; +import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.metrics.CommonMetricNames; import com.tencent.bk.job.common.model.ValidateResult; -import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.common.util.ArrayUtil; +import com.tencent.bk.job.common.util.DataSizeConverter; +import com.tencent.bk.job.common.util.FilePathValidateUtil; import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.common.web.metrics.CustomTimed; import com.tencent.bk.job.execute.api.esb.v2.EsbFastPushFileResource; @@ -56,8 +61,7 @@ import com.tencent.bk.job.execute.model.esb.v2.request.EsbFastPushFileRequest; import com.tencent.bk.job.execute.service.AccountService; import com.tencent.bk.job.execute.service.TaskExecuteService; -import com.tencent.bk.job.manage.common.consts.account.AccountCategoryEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskFileTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskFileTypeEnum; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -67,8 +71,6 @@ import java.time.LocalDateTime; import java.util.ArrayList; import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; @RestController @Slf4j @@ -79,18 +81,13 @@ public class EsbFastPushFileResourceImpl extends JobExecuteCommonProcessor imple private final AccountService accountService; - private final AppScopeMappingService appScopeMappingService; - - @Autowired public EsbFastPushFileResourceImpl(TaskExecuteService taskExecuteService, MessageI18nService i18nService, - AccountService accountService, - AppScopeMappingService appScopeMappingService) { + AccountService accountService) { this.taskExecuteService = taskExecuteService; this.i18nService = i18nService; this.accountService = accountService; - this.appScopeMappingService = appScopeMappingService; } @Override @@ -100,9 +97,10 @@ public EsbFastPushFileResourceImpl(TaskExecuteService taskExecuteService, ExecuteMetricsConstants.TAG_KEY_START_MODE, ExecuteMetricsConstants.TAG_VALUE_START_MODE_API, ExecuteMetricsConstants.TAG_KEY_TASK_TYPE, ExecuteMetricsConstants.TAG_VALUE_TASK_TYPE_FAST_FILE }) - public EsbResp fastPushFile(EsbFastPushFileRequest request) { - request.fillAppResourceScope(appScopeMappingService); - + @AuditEntry(actionId = ActionId.QUICK_TRANSFER_FILE) + public EsbResp fastPushFile(String username, + String appCode, + @AuditRequestBody EsbFastPushFileRequest request) { ValidateResult checkResult = checkFastPushFileRequest(request); if (!checkResult.isPass()) { log.warn("Fast transfer file request is illegal!"); @@ -113,20 +111,20 @@ public EsbResp fastPushFile(EsbFastPushFileRequest request) { request.setName(generateDefaultFastTaskName()); } - TaskInstanceDTO taskInstance = buildFastFileTaskInstance(request); - StepInstanceDTO stepInstance = buildFastFileStepInstance(request); - long taskInstanceId = taskExecuteService.executeFastTask( + TaskInstanceDTO taskInstance = buildFastFileTaskInstance(username, appCode, request); + StepInstanceDTO stepInstance = buildFastFileStepInstance(username, request); + TaskInstanceDTO executeTaskInstance = taskExecuteService.executeFastTask( FastTaskDTO.builder().taskInstance(taskInstance).stepInstance(stepInstance).build() ); EsbJobExecuteDTO jobExecuteInfo = new EsbJobExecuteDTO(); - jobExecuteInfo.setTaskInstanceId(taskInstanceId); + jobExecuteInfo.setTaskInstanceId(executeTaskInstance.getId()); jobExecuteInfo.setTaskName(stepInstance.getName()); return EsbResp.buildSuccessResp(jobExecuteInfo); } private ValidateResult checkFastPushFileRequest(EsbFastPushFileRequest request) { - if (!validateFileSystemPath(request.getTargetPath())) { + if (!FilePathValidateUtil.validateFileSystemAbsolutePath(request.getTargetPath())) { log.warn("Fast transfer file, target path is invalid!path={}", request.getTargetPath()); return ValidateResult.fail(ErrorCode.MISSING_OR_ILLEGAL_PARAM_WITH_PARAM_NAME, "file_target_path"); } @@ -164,7 +162,7 @@ private ValidateResult validateFileSource(EsbFastPushFileRequest request) { return ValidateResult.fail(ErrorCode.MISSING_PARAM_WITH_PARAM_NAME, "file_source.files"); } for (String file : files) { - if (!validateFileSystemPath(file)) { + if (!FilePathValidateUtil.validateFileSystemAbsolutePath(file)) { log.warn("Invalid path:{}", file); return ValidateResult.fail(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME, "file_source.files"); } @@ -185,48 +183,22 @@ private ValidateResult validateFileSource(EsbFastPushFileRequest request) { return ValidateResult.pass(); } - private boolean validateFileSystemPath(String path) { - if (StringUtils.isBlank(path)) { - return false; - } - if (path.indexOf(' ') != -1) { - return false; - } - Pattern p1 = Pattern.compile("(//|\\\\)+"); - Matcher m1 = p1.matcher(path); - if (m1.matches()) { - return false; - } - - Pattern p2 = Pattern.compile("^[a-zA-Z]:(/|\\\\).*");//windows - Matcher m2 = p2.matcher(path); - - if (!m2.matches()) { //非windows - if (path.charAt(0) == '/') { - return !path.contains("\\\\"); - } else { - return false; - } - } - return true; - } - - private TaskInstanceDTO buildFastFileTaskInstance(EsbFastPushFileRequest request) { + private TaskInstanceDTO buildFastFileTaskInstance(String username, String appCode, EsbFastPushFileRequest request) { TaskInstanceDTO taskInstance = new TaskInstanceDTO(); taskInstance.setType(TaskTypeEnum.FILE.getValue()); taskInstance.setName(request.getName()); - taskInstance.setTaskId(-1L); + taskInstance.setPlanId(-1L); taskInstance.setCronTaskId(-1L); taskInstance.setTaskTemplateId(-1L); taskInstance.setAppId(request.getAppId()); taskInstance.setStatus(RunStatusEnum.BLANK); taskInstance.setStartupMode(TaskStartupModeEnum.API.getValue()); - taskInstance.setOperator(request.getUserName()); + taskInstance.setOperator(username); taskInstance.setCreateTime(DateUtils.currentTimeMillis()); taskInstance.setCurrentStepInstanceId(0L); taskInstance.setDebugTask(false); taskInstance.setCallbackUrl(request.getCallbackUrl()); - taskInstance.setAppCode(request.getAppCode()); + taskInstance.setAppCode(appCode); return taskInstance; } @@ -235,29 +207,29 @@ private String generateDefaultFastTaskName() { + DateUtils.formatLocalDateTime(LocalDateTime.now(), "yyyyMMddHHmmssSSS"); } - private StepInstanceDTO buildFastFileStepInstance(EsbFastPushFileRequest request) { + private StepInstanceDTO buildFastFileStepInstance(String username, EsbFastPushFileRequest request) { StepInstanceDTO stepInstance = new StepInstanceDTO(); stepInstance.setName(request.getName()); AccountDTO account = checkAndGetOsAccount(request.getAppId(), request.getAccount()); stepInstance.setAccountId(account.getId()); stepInstance.setAccount(account.getAccount()); stepInstance.setStepId(-1L); - stepInstance.setExecuteType(StepExecuteTypeEnum.SEND_FILE.getValue()); + stepInstance.setExecuteType(StepExecuteTypeEnum.SEND_FILE); stepInstance.setFileTargetPath(request.getTargetPath()); stepInstance.setFileSourceList(convertFileSource(request.getAppId(), request.getFileSources())); stepInstance.setAppId(request.getAppId()); - stepInstance.setTargetServers(convertToStandardServers(request.getTargetServer(), request.getIpList(), + stepInstance.setTargetExecuteObjects(convertToStandardServers(request.getTargetServer(), request.getIpList(), request.getDynamicGroupIdList())); - stepInstance.setOperator(request.getUserName()); + stepInstance.setOperator(username); stepInstance.setStatus(RunStatusEnum.BLANK); stepInstance.setCreateTime(DateUtils.currentTimeMillis()); stepInstance.setTimeout(request.getTimeout() == null ? JobConstants.DEFAULT_JOB_TIMEOUT_SECONDS : request.getTimeout()); if (request.getUploadSpeedLimit() != null && request.getUploadSpeedLimit() > 0) { - stepInstance.setFileUploadSpeedLimit(request.getUploadSpeedLimit() << 10); + stepInstance.setFileUploadSpeedLimit(DataSizeConverter.convertMBToKB(request.getUploadSpeedLimit())); } if (request.getDownloadSpeedLimit() != null && request.getDownloadSpeedLimit() > 0) { - stepInstance.setFileDownloadSpeedLimit(request.getDownloadSpeedLimit() << 10); + stepInstance.setFileDownloadSpeedLimit(DataSizeConverter.convertMBToKB(request.getDownloadSpeedLimit())); } // 新增参数兼容历史接口调用：默认直接创建 stepInstance.setNotExistPathHandler(NotExistPathHandlerEnum.CREATE_DIR.getValue()); @@ -294,9 +266,7 @@ private List convertFileSource(long appId, List fileSourceDTO.setFileType(TaskFileTypeEnum.SERVER.getType()); List files = new ArrayList<>(); if (fileSource.getFiles() != null) { - fileSource.getFiles().forEach(file -> { - files.add(new FileDetailDTO(file)); - }); + fileSource.getFiles().forEach(file -> files.add(new FileDetailDTO(file))); } fileSourceDTO.setFiles(files); fileSourceDTO.setServers(convertToStandardServers(fileSource.getTargetServer(), fileSource.getIpList(), diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbGetJobInstanceGlobalVarValueResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbGetJobInstanceGlobalVarValueResourceImpl.java index fa0d00d646..2a28b5f388 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbGetJobInstanceGlobalVarValueResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbGetJobInstanceGlobalVarValueResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -30,7 +30,6 @@ import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.metrics.CommonMetricNames; import com.tencent.bk.job.common.model.ValidateResult; -import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.execute.api.esb.v2.EsbGetJobInstanceGlobalVarValueResource; import com.tencent.bk.job.execute.api.esb.v3.EsbGetJobInstanceGlobalVarValueV3Resource; import com.tencent.bk.job.execute.model.esb.v2.EsbTaskInstanceGlobalVarValueDTO; @@ -49,27 +48,22 @@ @RestController @Slf4j -public class EsbGetJobInstanceGlobalVarValueResourceImpl - extends JobQueryCommonProcessor implements EsbGetJobInstanceGlobalVarValueResource { +public class EsbGetJobInstanceGlobalVarValueResourceImpl implements EsbGetJobInstanceGlobalVarValueResource { private final EsbGetJobInstanceGlobalVarValueV3Resource proxyGetJobInstanceGlobalVarService; - private final AppScopeMappingService appScopeMappingService; @Autowired public EsbGetJobInstanceGlobalVarValueResourceImpl( - EsbGetJobInstanceGlobalVarValueV3Resource proxyGetJobInstanceGlobalVarService, - AppScopeMappingService appScopeMappingService) { - this.appScopeMappingService = appScopeMappingService; + EsbGetJobInstanceGlobalVarValueV3Resource proxyGetJobInstanceGlobalVarService) { this.proxyGetJobInstanceGlobalVarService = proxyGetJobInstanceGlobalVarService; } @Override @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v2_get_job_instance_global_var_value"}) public EsbResp getJobInstanceGlobalVarValue( + String username, + String appCode, EsbGetJobInstanceGlobalVarValueRequest request) { - - request.fillAppResourceScope(appScopeMappingService); - ValidateResult checkResult = checkRequest(request); if (!checkResult.isPass()) { log.warn("Get job instance global var value, request is illegal!"); @@ -79,7 +73,7 @@ public EsbResp getJobInstanceGlobalVarValue( EsbGetJobInstanceGlobalVarValueV3Request newRequest = convertToEsbGetJobInstanceGlobalVarValueV3Request(request); EsbResp esbResp = - proxyGetJobInstanceGlobalVarService.getJobInstanceGlobalVarValueUsingPost(newRequest); + proxyGetJobInstanceGlobalVarService.getJobInstanceGlobalVarValueUsingPost(username, appCode, newRequest); return EsbResp.convertData(esbResp, this::convertToEsbJobInstanceGlobalVarValueDTO); } @@ -87,8 +81,6 @@ public EsbResp getJobInstanceGlobalVarValue( private EsbGetJobInstanceGlobalVarValueV3Request convertToEsbGetJobInstanceGlobalVarValueV3Request (EsbGetJobInstanceGlobalVarValueRequest request) { EsbGetJobInstanceGlobalVarValueV3Request newRequest = new EsbGetJobInstanceGlobalVarValueV3Request(); - newRequest.setAppCode(request.getAppCode()); - newRequest.setUserName(request.getUserName()); newRequest.setBizId(request.getBizId()); newRequest.setScopeType(request.getScopeType()); newRequest.setScopeId(request.getScopeId()); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbGetJobInstanceLogResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbGetJobInstanceLogResourceImpl.java index 58ba78f1bb..8d1b3e8bab 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbGetJobInstanceLogResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbGetJobInstanceLogResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,29 +25,31 @@ package com.tencent.bk.job.execute.api.esb.v2.impl; import com.google.common.collect.Lists; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.audit.annotations.AuditRequestBody; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.esb.metrics.EsbApiTimed; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.common.exception.InvalidParamException; +import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.metrics.CommonMetricNames; import com.tencent.bk.job.common.model.ValidateResult; import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.common.util.Utils; import com.tencent.bk.job.execute.api.esb.v2.EsbGetJobInstanceLogResource; -import com.tencent.bk.job.execute.common.constants.FileDistModeEnum; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; -import com.tencent.bk.job.execute.model.AgentTaskDetailDTO; -import com.tencent.bk.job.execute.model.AgentTaskResultGroupDTO; -import com.tencent.bk.job.execute.model.FileIpLogContent; -import com.tencent.bk.job.execute.model.ScriptHostLogContent; +import com.tencent.bk.job.execute.model.ExecuteObjectTask; +import com.tencent.bk.job.execute.model.FileExecuteObjectLogContent; +import com.tencent.bk.job.execute.model.ResultGroupDTO; +import com.tencent.bk.job.execute.model.ScriptExecuteObjectLogContent; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; -import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.model.esb.v2.EsbStepInstanceResultAndLog; import com.tencent.bk.job.execute.model.esb.v2.request.EsbGetJobInstanceLogRequest; -import com.tencent.bk.job.execute.service.FileAgentTaskService; -import com.tencent.bk.job.execute.service.GseTaskService; +import com.tencent.bk.job.execute.service.FileExecuteObjectTaskService; import com.tencent.bk.job.execute.service.LogService; -import com.tencent.bk.job.execute.service.ScriptAgentTaskService; +import com.tencent.bk.job.execute.service.ScriptExecuteObjectTaskService; +import com.tencent.bk.job.execute.service.StepInstanceService; +import com.tencent.bk.job.execute.service.TaskInstanceAccessProcessor; import com.tencent.bk.job.execute.service.TaskInstanceService; import lombok.extern.slf4j.Slf4j; import org.springframework.web.bind.annotation.RestController; @@ -57,33 +59,39 @@ @RestController @Slf4j -public class EsbGetJobInstanceLogResourceImpl extends JobQueryCommonProcessor implements EsbGetJobInstanceLogResource { +public class EsbGetJobInstanceLogResourceImpl implements EsbGetJobInstanceLogResource { private final TaskInstanceService taskInstanceService; - private final AppScopeMappingService appScopeMappingService; - private final ScriptAgentTaskService scriptAgentTaskService; - private final FileAgentTaskService fileAgentTaskService; - private final GseTaskService gseTaskService; + private final ScriptExecuteObjectTaskService scriptExecuteObjectTaskService; + private final FileExecuteObjectTaskService fileExecuteObjectTaskService; private final LogService logService; + private final TaskInstanceAccessProcessor taskInstanceAccessProcessor; + private final AppScopeMappingService appScopeMappingService; + private final StepInstanceService stepInstanceService; public EsbGetJobInstanceLogResourceImpl(TaskInstanceService taskInstanceService, + ScriptExecuteObjectTaskService scriptExecuteObjectTaskService, + FileExecuteObjectTaskService fileExecuteObjectTaskService, + LogService logService, + TaskInstanceAccessProcessor taskInstanceAccessProcessor, AppScopeMappingService appScopeMappingService, - ScriptAgentTaskService scriptAgentTaskService, - FileAgentTaskService fileAgentTaskService, - GseTaskService gseTaskService, - LogService logService) { + StepInstanceService stepInstanceService) { this.taskInstanceService = taskInstanceService; - this.appScopeMappingService = appScopeMappingService; - this.scriptAgentTaskService = scriptAgentTaskService; - this.fileAgentTaskService = fileAgentTaskService; - this.gseTaskService = gseTaskService; + this.scriptExecuteObjectTaskService = scriptExecuteObjectTaskService; + this.fileExecuteObjectTaskService = fileExecuteObjectTaskService; this.logService = logService; + this.taskInstanceAccessProcessor = taskInstanceAccessProcessor; + this.appScopeMappingService = appScopeMappingService; + this.stepInstanceService = stepInstanceService; } @Override @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v2_get_job_instance_log"}) - public EsbResp> getJobInstanceLogUsingPost(EsbGetJobInstanceLogRequest request) { - request.fillAppResourceScope(appScopeMappingService); + @AuditEntry(actionId = ActionId.VIEW_HISTORY) + public EsbResp> getJobInstanceLogUsingPost( + String username, + String appCode, + @AuditRequestBody EsbGetJobInstanceLogRequest request) { ValidateResult checkResult = checkRequest(request); if (!checkResult.isPass()) { @@ -91,12 +99,11 @@ public EsbResp> getJobInstanceLogUsingPost(Esb throw new InvalidParamException(checkResult); } - long taskInstanceId = request.getTaskInstanceId(); - TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(taskInstanceId); - authViewTaskInstance(request.getUserName(), request.getAppResourceScope(), taskInstance); + taskInstanceAccessProcessor.processBeforeAccess(username, + request.getAppResourceScope().getAppId(), request.getTaskInstanceId()); List stepInstanceList = - taskInstanceService.listStepInstanceByTaskInstanceId(taskInstanceId); + stepInstanceService.listBaseStepInstanceByTaskInstanceId(request.getTaskInstanceId()); List stepInstResultAndLogList = Lists.newArrayList(); for (StepInstanceBaseDTO stepInstance : stepInstanceList) { EsbStepInstanceResultAndLog stepInstResultAndLog = new EsbStepInstanceResultAndLog(); @@ -111,39 +118,39 @@ public EsbResp> getJobInstanceLogUsingPost(Esb } private List buildStepInstResult(StepInstanceBaseDTO stepInstance) { - List resultGroups = Collections.emptyList(); + List resultGroups = Collections.emptyList(); if (stepInstance.isScriptStep()) { - resultGroups = scriptAgentTaskService.listAndGroupAgentTasks(stepInstance, + resultGroups = scriptExecuteObjectTaskService.listAndGroupTasks(stepInstance, stepInstance.getExecuteCount(), null); } else if (stepInstance.isFileStep()) { - resultGroups = fileAgentTaskService.listAndGroupAgentTasks(stepInstance, + resultGroups = fileExecuteObjectTaskService.listAndGroupTasks(stepInstance, stepInstance.getExecuteCount(), null); } List stepInstResultList = Lists.newArrayListWithCapacity(resultGroups.size()); - for (AgentTaskResultGroupDTO resultGroup : resultGroups) { + for (ResultGroupDTO resultGroup : resultGroups) { EsbStepInstanceResultAndLog.StepInstResultDTO stepInstResult = new EsbStepInstanceResultAndLog.StepInstResultDTO(); stepInstResult.setIpStatus(resultGroup.getStatus()); stepInstResult.setTag(resultGroup.getTag()); - List agentTasks = resultGroup.getAgentTasks(); - addLogContent(stepInstance, agentTasks); + List executeObjectTasks = resultGroup.getExecuteObjectTasks(); + addLogContent(stepInstance, executeObjectTasks); List esbGseAgentTaskList = - Lists.newArrayListWithCapacity(agentTasks.size()); - for (AgentTaskDetailDTO agentTask : agentTasks) { + Lists.newArrayListWithCapacity(executeObjectTasks.size()); + for (ExecuteObjectTask executeObjectTask : executeObjectTasks) { EsbStepInstanceResultAndLog.EsbGseAgentTaskDTO esbGseAgentTaskDTO = new EsbStepInstanceResultAndLog.EsbGseAgentTaskDTO(); - esbGseAgentTaskDTO.setLogContent(Utils.htmlEncode(agentTask.getScriptLogContent())); - esbGseAgentTaskDTO.setExecuteCount(agentTask.getExecuteCount()); - esbGseAgentTaskDTO.setEndTime(agentTask.getEndTime()); - esbGseAgentTaskDTO.setStartTime(agentTask.getStartTime()); - esbGseAgentTaskDTO.setErrCode(agentTask.getErrorCode()); - esbGseAgentTaskDTO.setExitCode(agentTask.getExitCode()); - esbGseAgentTaskDTO.setTotalTime(agentTask.getTotalTime()); - esbGseAgentTaskDTO.setCloudAreaId(agentTask.getBkCloudId()); - esbGseAgentTaskDTO.setIp(agentTask.getIp()); + esbGseAgentTaskDTO.setLogContent(Utils.htmlEncode(executeObjectTask.getScriptLogContent())); + esbGseAgentTaskDTO.setExecuteCount(executeObjectTask.getExecuteCount()); + esbGseAgentTaskDTO.setEndTime(executeObjectTask.getEndTime()); + esbGseAgentTaskDTO.setStartTime(executeObjectTask.getStartTime()); + esbGseAgentTaskDTO.setErrCode(executeObjectTask.getErrorCode()); + esbGseAgentTaskDTO.setExitCode(executeObjectTask.getExitCode()); + esbGseAgentTaskDTO.setTotalTime(executeObjectTask.getTotalTime()); + esbGseAgentTaskDTO.setCloudAreaId(executeObjectTask.getExecuteObject().getHost().getBkCloudId()); + esbGseAgentTaskDTO.setIp(executeObjectTask.getExecuteObject().getHost().getIp()); esbGseAgentTaskList.add(esbGseAgentTaskDTO); } stepInstResult.setIpLogs(esbGseAgentTaskList); @@ -161,38 +168,40 @@ private ValidateResult checkRequest(EsbGetJobInstanceLogRequest request) { return ValidateResult.pass(); } - private void addLogContent(StepInstanceBaseDTO stepInstance, List agentTasks) { - long stepInstanceId = stepInstance.getId(); + private void addLogContent(StepInstanceBaseDTO stepInstance, List executeObjectTasks) { int executeCount = stepInstance.getExecuteCount(); - for (AgentTaskDetailDTO agentTask : agentTasks) { + for (ExecuteObjectTask executeObjectTask : executeObjectTasks) { if (stepInstance.isScriptStep()) { - ScriptHostLogContent scriptHostLogContent = logService.getScriptHostLogContent(stepInstanceId, - executeCount, - null, agentTask.getHost()); - agentTask.setScriptLogContent(scriptHostLogContent == null ? "" : scriptHostLogContent.getContent()); + ScriptExecuteObjectLogContent scriptExecuteObjectLogContent = + logService.getScriptExecuteObjectLogContent(stepInstance, executeCount, + null, executeObjectTask); + executeObjectTask.setScriptLogContent( + scriptExecuteObjectLogContent == null ? "" : scriptExecuteObjectLogContent.getContent()); } else if (stepInstance.isFileStep()) { - FileIpLogContent fileIpLogContent = logService.getFileIpLogContent(stepInstanceId, executeCount, - null, agentTask.getHost(), FileDistModeEnum.DOWNLOAD.getValue()); - agentTask.setScriptLogContent(fileIpLogContent == null ? "" : fileIpLogContent.getContent()); + FileExecuteObjectLogContent fileExecuteObjectLogContent = logService.getFileExecuteObjectLogContent( + stepInstance, executeCount, null, executeObjectTask); + executeObjectTask.setScriptLogContent( + fileExecuteObjectLogContent == null ? "" : fileExecuteObjectLogContent.getContent()); } } } @Override - public EsbResp> getJobInstanceLog(String appCode, - String username, + @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v2_get_job_instance_log"}) + @AuditEntry(actionId = ActionId.VIEW_HISTORY) + public EsbResp> getJobInstanceLog(String username, + String appCode, Long appId, String scopeType, String scopeId, Long taskInstanceId) { EsbGetJobInstanceLogRequest req = new EsbGetJobInstanceLogRequest(); - req.setAppCode(appCode); - req.setUserName(username); req.setBizId(appId); req.setScopeType(scopeType); req.setScopeId(scopeId); req.setTaskInstanceId(taskInstanceId); - return getJobInstanceLogUsingPost(req); + req.fillAppResourceScope(appScopeMappingService); + return getJobInstanceLogUsingPost(username, appCode, req); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbGetJobInstanceStatusResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbGetJobInstanceStatusResourceImpl.java index 5499a61518..a864b5e776 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbGetJobInstanceStatusResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbGetJobInstanceStatusResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,25 +25,29 @@ package com.tencent.bk.job.execute.api.esb.v2.impl; import com.google.common.collect.Lists; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.audit.annotations.AuditRequestBody; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.esb.metrics.EsbApiTimed; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.common.esb.util.EsbDTOAppScopeMappingHelper; import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.exception.NotFoundException; +import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.metrics.CommonMetricNames; import com.tencent.bk.job.common.model.ValidateResult; import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.execute.api.esb.v2.EsbGetJobInstanceStatusResource; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; -import com.tencent.bk.job.execute.model.AgentTaskDetailDTO; +import com.tencent.bk.job.execute.model.ExecuteObjectTask; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.model.esb.v2.EsbIpStatusDTO; import com.tencent.bk.job.execute.model.esb.v2.EsbJobInstanceStatusDTO; import com.tencent.bk.job.execute.model.esb.v2.request.EsbGetJobInstanceStatusRequest; -import com.tencent.bk.job.execute.service.FileAgentTaskService; -import com.tencent.bk.job.execute.service.ScriptAgentTaskService; +import com.tencent.bk.job.execute.service.FileExecuteObjectTaskService; +import com.tencent.bk.job.execute.service.ScriptExecuteObjectTaskService; +import com.tencent.bk.job.execute.service.StepInstanceService; import com.tencent.bk.job.execute.service.TaskInstanceService; import com.tencent.bk.job.logsvr.consts.FileTaskModeEnum; import lombok.extern.slf4j.Slf4j; @@ -58,30 +62,33 @@ @RestController @Slf4j -public class EsbGetJobInstanceStatusResourceImpl - extends JobQueryCommonProcessor - implements EsbGetJobInstanceStatusResource { +public class EsbGetJobInstanceStatusResourceImpl implements EsbGetJobInstanceStatusResource { private final TaskInstanceService taskInstanceService; + private final ScriptExecuteObjectTaskService scriptExecuteObjectTaskService; + private final FileExecuteObjectTaskService fileExecuteObjectTaskService; private final AppScopeMappingService appScopeMappingService; - private final ScriptAgentTaskService scriptAgentTaskService; - private final FileAgentTaskService fileAgentTaskService; + private final StepInstanceService stepInstanceService; public EsbGetJobInstanceStatusResourceImpl(TaskInstanceService taskInstanceService, + ScriptExecuteObjectTaskService scriptExecuteObjectTaskService, + FileExecuteObjectTaskService fileExecuteObjectTaskService, AppScopeMappingService appScopeMappingService, - ScriptAgentTaskService scriptAgentTaskService, - FileAgentTaskService fileAgentTaskService) { + StepInstanceService stepInstanceService) { this.taskInstanceService = taskInstanceService; + this.scriptExecuteObjectTaskService = scriptExecuteObjectTaskService; + this.fileExecuteObjectTaskService = fileExecuteObjectTaskService; this.appScopeMappingService = appScopeMappingService; - this.scriptAgentTaskService = scriptAgentTaskService; - this.fileAgentTaskService = fileAgentTaskService; + this.stepInstanceService = stepInstanceService; } @Override @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v2_get_job_instance_status"}) - public EsbResp getJobInstanceStatusUsingPost(EsbGetJobInstanceStatusRequest request) { - request.fillAppResourceScope(appScopeMappingService); - + @AuditEntry(actionId = ActionId.VIEW_HISTORY) + public EsbResp getJobInstanceStatusUsingPost( + String username, + String appCode, + @AuditRequestBody EsbGetJobInstanceStatusRequest request) { ValidateResult checkResult = checkRequest(request); if (!checkResult.isPass()) { log.warn("Get job instance status request is illegal!"); @@ -89,12 +96,11 @@ public EsbResp getJobInstanceStatusUsingPost(EsbGetJobI } long taskInstanceId = request.getTaskInstanceId(); + TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(username, + request.getAppResourceScope().getAppId(), request.getTaskInstanceId()); - TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(request.getTaskInstanceId()); - authViewTaskInstance(request.getUserName(), request.getAppResourceScope(), taskInstance); - - - List stepInstances = taskInstanceService.listStepInstanceByTaskInstanceId(taskInstanceId); + List stepInstances = + stepInstanceService.listBaseStepInstanceByTaskInstanceId(taskInstanceId); if (stepInstances == null || stepInstances.isEmpty()) { log.warn("Get job instance status by taskInstanceId:{}, stepInstanceList is empty!", taskInstanceId); throw new NotFoundException(ErrorCode.STEP_INSTANCE_NOT_EXIST); @@ -116,31 +122,31 @@ private ValidateResult checkRequest(EsbGetJobInstanceStatusRequest request) { private Map> getStepIpResult(List stepInstanceList) { Map> stepIpResult = new HashMap<>(); for (StepInstanceBaseDTO stepInstance : stepInstanceList) { - List agentTasks = null; + List executeObjectTasks = null; if (stepInstance.isScriptStep()) { - agentTasks = scriptAgentTaskService.listAgentTaskDetail(stepInstance, + executeObjectTasks = scriptExecuteObjectTaskService.listTasks(stepInstance, stepInstance.getExecuteCount(), null); } else if (stepInstance.isFileStep()) { - agentTasks = fileAgentTaskService.listAgentTaskDetail(stepInstance, + executeObjectTasks = fileExecuteObjectTaskService.listTasks(stepInstance, stepInstance.getExecuteCount(), null); - if (CollectionUtils.isNotEmpty(agentTasks)) { + if (CollectionUtils.isNotEmpty(executeObjectTasks)) { // 如果是文件分发任务，只返回目标Agent结果 - agentTasks = agentTasks.stream() - .filter(agentTask -> agentTask.getFileTaskMode() == FileTaskModeEnum.DOWNLOAD) + executeObjectTasks = executeObjectTasks.stream() + .filter(executeObjectTask -> executeObjectTask.getFileTaskMode() == FileTaskModeEnum.DOWNLOAD) .collect(Collectors.toList()); } } List ipResultList = Lists.newArrayList(); - if (CollectionUtils.isEmpty(agentTasks)) { + if (CollectionUtils.isEmpty(executeObjectTasks)) { stepIpResult.put(stepInstance.getId(), ipResultList); continue; } - for (AgentTaskDetailDTO agentTask : agentTasks) { + for (ExecuteObjectTask executeObjectTask : executeObjectTasks) { EsbIpStatusDTO ipStatus = new EsbIpStatusDTO(); - ipStatus.setIp(agentTask.getIp()); - ipStatus.setCloudAreaId(agentTask.getBkCloudId()); + ipStatus.setIp(executeObjectTask.getExecuteObject().getHost().getIp()); + ipStatus.setCloudAreaId(executeObjectTask.getExecuteObject().getHost().getBkCloudId()); - ipStatus.setStatus(agentTask.getStatus().getValue()); + ipStatus.setStatus(executeObjectTask.getStatus().getValue()); ipResultList.add(ipStatus); } stepIpResult.put(stepInstance.getId(), ipResultList); @@ -166,7 +172,7 @@ private EsbJobInstanceStatusDTO buildEsbJobInstanceStatusDTO(TaskInstanceDTO tas jobInstance.setEndTime(taskInstance.getEndTime()); jobInstance.setStartWay(taskInstance.getStartupMode()); jobInstance.setStatus(taskInstance.getStatus().getValue()); - jobInstance.setTaskId(taskInstance.getTaskId()); + jobInstance.setTaskId(taskInstance.getPlanId()); jobInstance.setTotalTime(taskInstance.getTotalTime()); jobInstanceStatus.setJobInstance(jobInstance); @@ -181,7 +187,7 @@ private EsbJobInstanceStatusDTO buildEsbJobInstanceStatusDTO(TaskInstanceDTO tas stepInst.setCreateTime(stepInstance.getCreateTime()); stepInst.setEndTime(stepInstance.getEndTime()); stepInst.setStartTime(stepInstance.getStartTime()); - stepInst.setType(stepInstance.getExecuteType()); + stepInst.setType(stepInstance.getExecuteType().getValue()); stepInst.setOperator(stepInstance.getOperator()); stepInst.setExecuteCount(stepInstance.getExecuteCount()); stepInst.setStatus(stepInstance.getStatus().getValue()); @@ -200,19 +206,20 @@ private EsbJobInstanceStatusDTO buildEsbJobInstanceStatusDTO(TaskInstanceDTO tas } @Override - public EsbResp getJobInstanceStatus(String appCode, - String username, + @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v2_get_job_instance_status"}) + @AuditEntry(actionId = ActionId.VIEW_HISTORY) + public EsbResp getJobInstanceStatus(String username, + String appCode, Long bizId, String scopeType, String scopeId, Long taskInstanceId) { EsbGetJobInstanceStatusRequest req = new EsbGetJobInstanceStatusRequest(); - req.setAppCode(appCode); - req.setUserName(username); req.setBizId(bizId); req.setScopeType(scopeType); req.setScopeId(scopeId); req.setTaskInstanceId(taskInstanceId); - return getJobInstanceStatusUsingPost(req); + req.fillAppResourceScope(appScopeMappingService); + return getJobInstanceStatusUsingPost(username, appCode, req); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbGetStepInstanceStatusResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbGetStepInstanceStatusResourceImpl.java index cf1f3e6c13..f6e33c59e9 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbGetStepInstanceStatusResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbGetStepInstanceStatusResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,8 @@ package com.tencent.bk.job.execute.api.esb.v2.impl; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.audit.annotations.AuditRequestBody; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.esb.metrics.EsbApiTimed; import com.tencent.bk.job.common.esb.model.EsbResp; @@ -32,55 +34,56 @@ import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.exception.NotFoundException; import com.tencent.bk.job.common.i18n.service.MessageI18nService; +import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.metrics.CommonMetricNames; import com.tencent.bk.job.common.model.ValidateResult; -import com.tencent.bk.job.common.service.AppScopeMappingService; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.execute.api.esb.v2.EsbGetStepInstanceStatusResource; -import com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum; -import com.tencent.bk.job.execute.model.AgentTaskDetailDTO; -import com.tencent.bk.job.execute.model.AgentTaskResultGroupDTO; +import com.tencent.bk.job.execute.engine.consts.ExecuteObjectTaskStatusEnum; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; +import com.tencent.bk.job.execute.model.ExecuteObjectTask; +import com.tencent.bk.job.execute.model.ResultGroupDTO; import com.tencent.bk.job.execute.model.StepExecutionDetailDTO; import com.tencent.bk.job.execute.model.StepExecutionResultQuery; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; -import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.model.esb.v2.EsbStepInstanceStatusDTO; import com.tencent.bk.job.execute.model.esb.v2.request.EsbGetStepInstanceStatusRequest; -import com.tencent.bk.job.execute.service.TaskInstanceService; +import com.tencent.bk.job.execute.service.StepInstanceService; import com.tencent.bk.job.execute.service.TaskResultService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.springframework.web.bind.annotation.RestController; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; @RestController @Slf4j -public class EsbGetStepInstanceStatusResourceImpl - extends JobQueryCommonProcessor - implements EsbGetStepInstanceStatusResource { - private final TaskInstanceService taskInstanceService; +public class EsbGetStepInstanceStatusResourceImpl implements EsbGetStepInstanceStatusResource { + private final StepInstanceService stepInstanceService; private final TaskResultService taskResultService; private final MessageI18nService i18nService; - private final AppScopeMappingService appScopeMappingService; public EsbGetStepInstanceStatusResourceImpl(MessageI18nService i18nService, - TaskInstanceService taskInstanceService, - AppScopeMappingService appScopeMappingService, + StepInstanceService stepInstanceService, TaskResultService taskResultService) { this.i18nService = i18nService; - this.taskInstanceService = taskInstanceService; - this.appScopeMappingService = appScopeMappingService; + this.stepInstanceService = stepInstanceService; this.taskResultService = taskResultService; } @Override @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v2_get_step_instance_status"}) - public EsbResp getJobStepInstanceStatus(EsbGetStepInstanceStatusRequest request) { - request.fillAppResourceScope(appScopeMappingService); - + @AuditEntry(actionId = ActionId.VIEW_HISTORY) + public EsbResp getJobStepInstanceStatus( + String username, + String appCode, + @AuditRequestBody EsbGetStepInstanceStatusRequest request) { ValidateResult checkResult = checkRequest(request); if (!checkResult.isPass()) { log.warn("Get step instance status request is illegal!"); @@ -90,16 +93,17 @@ public EsbResp getJobStepInstanceStatus(EsbGetStepInst EsbStepInstanceStatusDTO resultData = new EsbStepInstanceStatusDTO(); StepExecutionResultQuery query = StepExecutionResultQuery.builder() - .stepInstanceId(request.getStepInstanceId()).build(); - StepExecutionDetailDTO stepExecutionDetail = taskResultService.getStepExecutionResult(request.getUserName(), + .taskInstanceId(request.getTaskInstanceId()) + .stepInstanceId(request.getStepInstanceId()) + .build(); + StepExecutionDetailDTO stepExecutionDetail = taskResultService.getStepExecutionResult(username, request.getAppId(), query); - TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(request.getTaskInstanceId()); - authViewTaskInstance(request.getUserName(), request.getAppResourceScope(), taskInstance); resultData.setIsFinished(stepExecutionDetail.isFinished()); resultData.setAyalyseResult(convertToStandardAnalyseResult(stepExecutionDetail)); - StepInstanceBaseDTO stepInstance = taskInstanceService.getBaseStepInstance(request.getStepInstanceId()); + StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstance(request.getTaskInstanceId(), + request.getStepInstanceId()); if (stepInstance == null) { log.warn("Get step instance status by taskInstanceId:{}, stepInstanceId:{}, stepInstance is null!", request.getTaskInstanceId(), request.getStepInstanceId()); @@ -116,7 +120,13 @@ private EsbStepInstanceStatusDTO.StepInstance convertStepInstance(StepInstanceBa stepInst.setId(stepInstance.getId()); stepInst.setEndTime(stepInstance.getEndTime()); stepInst.setStartTime(stepInstance.getStartTime()); - stepInst.setIpList(stepInstance.getIpList()); + stepInst.setIpList( + convertToIpListStr( + stepInstance.getTargetExecuteObjects() + .getExecuteObjectsCompatibly() + .stream() + .map(ExecuteObject::getHost) + .collect(Collectors.toList()))); stepInst.setName(stepInstance.getName()); stepInst.setOperator(stepInstance.getOperator()); stepInst.setExecuteCount(stepInstance.getExecuteCount()); @@ -124,11 +134,16 @@ private EsbStepInstanceStatusDTO.StepInstance convertStepInstance(StepInstanceBa stepInst.setStepId(stepInstance.getStepId()); stepInst.setTaskInstanceId(stepInstance.getTaskInstanceId()); stepInst.setTotalTime(stepInstance.getTotalTime()); - stepInst.setType(stepInstance.getExecuteType()); + stepInst.setType(stepInstance.getExecuteType().getValue()); return stepInst; } + private String convertToIpListStr(Collection ips) { + return StringUtils.join(ips.stream().map(ipDTO -> + ipDTO.getBkCloudId() + ":" + ipDTO.getIp()).collect(Collectors.toList()), ","); + } + private ValidateResult checkRequest(EsbGetStepInstanceStatusRequest request) { if (request.getTaskInstanceId() == null || request.getTaskInstanceId() < 1) { @@ -144,19 +159,21 @@ private ValidateResult checkRequest(EsbGetStepInstanceStatusRequest request) { private List> convertToStandardAnalyseResult(StepExecutionDetailDTO stepExecutionDetail) { List> standardStepAnalyseResultList = new ArrayList<>(); - List resultGroups = stepExecutionDetail.getResultGroups(); + List resultGroups = stepExecutionDetail.getResultGroups(); if (resultGroups == null || resultGroups.isEmpty()) { return standardStepAnalyseResultList; } - for (AgentTaskResultGroupDTO resultGroup : resultGroups) { + for (ResultGroupDTO resultGroup : resultGroups) { Map standardStepAnalyseResult = new HashMap<>(); - List agentTasks = resultGroup.getAgentTasks(); - standardStepAnalyseResult.put("count", CollectionUtils.isEmpty(agentTasks) ? 0 : agentTasks.size()); - if (CollectionUtils.isNotEmpty(agentTasks)) { + List executeObjectTasks = resultGroup.getExecuteObjectTasks(); + standardStepAnalyseResult.put("count", + CollectionUtils.isEmpty(executeObjectTasks) ? 0 : executeObjectTasks.size()); + if (CollectionUtils.isNotEmpty(executeObjectTasks)) { List ips = new ArrayList<>(); - for (AgentTaskDetailDTO agentTask : agentTasks) { - ips.add(new EsbIpDTO(agentTask.getBkCloudId(), agentTask.getIp())); + for (ExecuteObjectTask executeObjectTask : executeObjectTasks) { + HostDTO host = executeObjectTask.getExecuteObject().getHost(); + ips.add(new EsbIpDTO(host.getHostId(), host.getBkCloudId(), host.getIp())); } standardStepAnalyseResult.put("ip_list", ips); @@ -164,7 +181,7 @@ private List> convertToStandardAnalyseResult(StepExecutionDe standardStepAnalyseResult.put("result_type", resultGroup.getStatus()); standardStepAnalyseResult.put("result_type_text", - i18nService.getI18n(AgentTaskStatusEnum.valueOf(resultGroup.getStatus()).getI18nKey())); + i18nService.getI18n(ExecuteObjectTaskStatusEnum.valOf(resultGroup.getStatus()).getI18nKey())); standardStepAnalyseResultList.add(standardStepAnalyseResult); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbOperateJobInstanceResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbOperateJobInstanceResourceImpl.java index 77ee405023..a6707ce8ea 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbOperateJobInstanceResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbOperateJobInstanceResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -29,7 +29,6 @@ import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.metrics.CommonMetricNames; -import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.execute.api.esb.v2.EsbOperateJobInstanceResource; import com.tencent.bk.job.execute.constants.TaskOperationEnum; @@ -43,24 +42,22 @@ @Slf4j public class EsbOperateJobInstanceResourceImpl implements EsbOperateJobInstanceResource { private final TaskExecuteService taskExecuteService; - private final AppScopeMappingService appScopeMappingService; - public EsbOperateJobInstanceResourceImpl(TaskExecuteService taskExecuteService, - AppScopeMappingService appScopeMappingService) { + public EsbOperateJobInstanceResourceImpl(TaskExecuteService taskExecuteService) { this.taskExecuteService = taskExecuteService; - this.appScopeMappingService = appScopeMappingService; } @Override @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v2_operate_job_instance"}) - public EsbResp operateJobInstance(EsbOperateJobInstanceRequest request) { + public EsbResp operateJobInstance(String username, + String appCode, + EsbOperateJobInstanceRequest request) { log.info("Operate task instance, request={}", JsonUtils.toJson(request)); - request.fillAppResourceScope(appScopeMappingService); if (!checkRequest(request)) { throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } TaskOperationEnum taskOperation = TaskOperationEnum.getTaskOperation(request.getOperationCode()); - taskExecuteService.doTaskOperation(request.getAppId(), request.getUserName(), + taskExecuteService.doTaskOperation(request.getAppId(), username, request.getTaskInstanceId(), taskOperation); EsbJobExecuteDTO result = new EsbJobExecuteDTO(); result.setTaskInstanceId(request.getTaskInstanceId()); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbOperateStepInstanceResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbOperateStepInstanceResourceImpl.java index 46fce21258..770027ce4d 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbOperateStepInstanceResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbOperateStepInstanceResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -29,7 +29,6 @@ import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.metrics.CommonMetricNames; -import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.execute.api.esb.v2.EsbOperateStepInstanceResource; import com.tencent.bk.job.execute.constants.StepOperationEnum; @@ -44,27 +43,26 @@ @Slf4j public class EsbOperateStepInstanceResourceImpl implements EsbOperateStepInstanceResource { private final TaskExecuteService taskExecuteService; - private final AppScopeMappingService appScopeMappingService; - public EsbOperateStepInstanceResourceImpl(TaskExecuteService taskExecuteService, - AppScopeMappingService appScopeMappingService) { + public EsbOperateStepInstanceResourceImpl(TaskExecuteService taskExecuteService) { this.taskExecuteService = taskExecuteService; - this.appScopeMappingService = appScopeMappingService; } @Override @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v2_operate_step_instance"}) - public EsbResp operateStepInstance(EsbOperateStepInstanceRequest request) { + public EsbResp operateStepInstance(String username, + String appCode, + EsbOperateStepInstanceRequest request) { log.info("Operate step instance, request={}", JsonUtils.toJson(request)); - request.fillAppResourceScope(appScopeMappingService); if (!checkRequest(request)) { throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } StepOperationEnum operationEnum = StepOperationEnum.getStepOperation(request.getOperationCode()); StepOperationDTO stepOperation = new StepOperationDTO(); + stepOperation.setTaskInstanceId(request.getTaskInstanceId()); stepOperation.setStepInstanceId(request.getStepInstanceId()); stepOperation.setOperation(operationEnum); - taskExecuteService.doStepOperation(request.getAppId(), request.getUserName(), stepOperation); + taskExecuteService.doStepOperation(request.getAppId(), username, stepOperation); EsbJobExecuteDTO result = new EsbJobExecuteDTO(); result.setTaskInstanceId(request.getTaskInstanceId()); result.setStepInstanceId(request.getStepInstanceId()); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbPushConfigFileResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbPushConfigFileResourceImpl.java index 7ba656406b..c550085f5d 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbPushConfigFileResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/EsbPushConfigFileResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,9 @@ package com.tencent.bk.job.execute.api.esb.v2.impl; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.audit.annotations.AuditRequestBody; +import com.tencent.bk.job.common.constant.AccountCategoryEnum; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.constant.JobConstants; import com.tencent.bk.job.common.esb.metrics.EsbApiTimed; @@ -31,9 +34,9 @@ import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.exception.NotFoundException; import com.tencent.bk.job.common.exception.ServiceException; +import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.metrics.CommonMetricNames; import com.tencent.bk.job.common.model.ValidateResult; -import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.common.util.ArrayUtil; import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.common.web.metrics.CustomTimed; @@ -43,9 +46,10 @@ import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; import com.tencent.bk.job.execute.common.constants.TaskStartupModeEnum; import com.tencent.bk.job.execute.common.constants.TaskTypeEnum; -import com.tencent.bk.job.execute.config.StorageSystemConfig; +import com.tencent.bk.job.execute.config.FileDistributeConfig; import com.tencent.bk.job.execute.metrics.ExecuteMetricsConstants; import com.tencent.bk.job.execute.model.AccountDTO; +import com.tencent.bk.job.execute.model.ExecuteTargetDTO; import com.tencent.bk.job.execute.model.FastTaskDTO; import com.tencent.bk.job.execute.model.FileDetailDTO; import com.tencent.bk.job.execute.model.FileSourceDTO; @@ -54,16 +58,16 @@ import com.tencent.bk.job.execute.model.esb.v2.EsbJobExecuteDTO; import com.tencent.bk.job.execute.model.esb.v2.request.EsbPushConfigFileRequest; import com.tencent.bk.job.execute.service.AccountService; -import com.tencent.bk.job.execute.service.AgentService; +import com.tencent.bk.job.execute.service.LocalFileDistributeSourceHostProvisioner; import com.tencent.bk.job.execute.service.TaskExecuteService; -import com.tencent.bk.job.manage.common.consts.account.AccountCategoryEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskFileTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskFileTypeEnum; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RestController; import java.util.ArrayList; +import java.util.Collections; import java.util.List; @RestController @@ -72,21 +76,18 @@ public class EsbPushConfigFileResourceImpl extends JobExecuteCommonProcessor imp private final TaskExecuteService taskExecuteService; private final AccountService accountService; - private final StorageSystemConfig storageSystemConfig; - private final AgentService agentService; - private final AppScopeMappingService appScopeMappingService; + private final FileDistributeConfig fileDistributeConfig; + private final LocalFileDistributeSourceHostProvisioner localFileDistributeSourceHostProvisioner; @Autowired public EsbPushConfigFileResourceImpl(TaskExecuteService taskExecuteService, AccountService accountService, - StorageSystemConfig storageSystemConfig, - AgentService agentService, - AppScopeMappingService appScopeMappingService) { + FileDistributeConfig fileDistributeConfig, + LocalFileDistributeSourceHostProvisioner localFileDistributeSourceHostProvisioner) { this.taskExecuteService = taskExecuteService; this.accountService = accountService; - this.storageSystemConfig = storageSystemConfig; - this.agentService = agentService; - this.appScopeMappingService = appScopeMappingService; + this.fileDistributeConfig = fileDistributeConfig; + this.localFileDistributeSourceHostProvisioner = localFileDistributeSourceHostProvisioner; } @Override @@ -96,9 +97,10 @@ public EsbPushConfigFileResourceImpl(TaskExecuteService taskExecuteService, ExecuteMetricsConstants.TAG_KEY_START_MODE, ExecuteMetricsConstants.TAG_VALUE_START_MODE_API, ExecuteMetricsConstants.TAG_KEY_TASK_TYPE, ExecuteMetricsConstants.TAG_VALUE_TASK_TYPE_FAST_CONFIG_FILE }) - public EsbResp pushConfigFile(EsbPushConfigFileRequest request) { - request.fillAppResourceScope(appScopeMappingService); - + @AuditEntry(actionId = ActionId.QUICK_TRANSFER_FILE) + public EsbResp pushConfigFile(String username, + String appCode, + @AuditRequestBody EsbPushConfigFileRequest request) { ValidateResult checkResult = checkPushConfigFileRequest(request); if (!checkResult.isPass()) { log.warn("Fast transfer file request is illegal!"); @@ -107,19 +109,21 @@ public EsbResp pushConfigFile(EsbPushConfigFileRequest request request.trimIps(); - TaskInstanceDTO taskInstance = buildFastFileTaskInstance(request); - StepInstanceDTO stepInstance = buildFastFileStepInstance(request, request.getFileList()); - long taskInstanceId = taskExecuteService.executeFastTask( + TaskInstanceDTO taskInstance = buildFastFileTaskInstance(username, appCode, request); + StepInstanceDTO stepInstance = buildFastFileStepInstance(username, request, request.getFileList()); + TaskInstanceDTO executeTaskInstance = taskExecuteService.executeFastTask( FastTaskDTO.builder().taskInstance(taskInstance).stepInstance(stepInstance).build() ); EsbJobExecuteDTO jobExecuteInfo = new EsbJobExecuteDTO(); - jobExecuteInfo.setTaskInstanceId(taskInstanceId); + jobExecuteInfo.setTaskInstanceId(executeTaskInstance.getId()); jobExecuteInfo.setTaskName(stepInstance.getName()); return EsbResp.buildSuccessResp(jobExecuteInfo); } - private TaskInstanceDTO buildFastFileTaskInstance(EsbPushConfigFileRequest request) { + private TaskInstanceDTO buildFastFileTaskInstance(String username, + String appCode, + EsbPushConfigFileRequest request) { TaskInstanceDTO taskInstance = new TaskInstanceDTO(); taskInstance.setType(TaskTypeEnum.FILE.getValue()); if (StringUtils.isNotEmpty(request.getName())) { @@ -127,21 +131,22 @@ private TaskInstanceDTO buildFastFileTaskInstance(EsbPushConfigFileRequest reque } else { taskInstance.setName("API_PUSH_CONFIG_FILE_" + DateUtils.currentTimeMillis()); } - taskInstance.setTaskId(-1L); + taskInstance.setPlanId(-1L); taskInstance.setCronTaskId(-1L); taskInstance.setTaskTemplateId(-1L); taskInstance.setAppId(request.getAppId()); taskInstance.setStatus(RunStatusEnum.BLANK); taskInstance.setStartupMode(TaskStartupModeEnum.API.getValue()); - taskInstance.setOperator(request.getUserName()); + taskInstance.setOperator(username); taskInstance.setCreateTime(DateUtils.currentTimeMillis()); taskInstance.setCurrentStepInstanceId(0L); taskInstance.setDebugTask(false); - taskInstance.setAppCode(request.getAppCode()); + taskInstance.setAppCode(appCode); return taskInstance; } - private StepInstanceDTO buildFastFileStepInstance(EsbPushConfigFileRequest request, + private StepInstanceDTO buildFastFileStepInstance(String username, + EsbPushConfigFileRequest request, List configFileList) { StepInstanceDTO stepInstance = new StepInstanceDTO(); if (StringUtils.isNotEmpty(request.getName())) { @@ -153,12 +158,12 @@ private StepInstanceDTO buildFastFileStepInstance(EsbPushConfigFileRequest reque stepInstance.setAccountId(account.getId()); stepInstance.setAccount(account.getAccount()); stepInstance.setStepId(-1L); - stepInstance.setExecuteType(StepExecuteTypeEnum.SEND_FILE.getValue()); + stepInstance.setExecuteType(StepExecuteTypeEnum.SEND_FILE); stepInstance.setFileTargetPath(request.getTargetPath()); - stepInstance.setFileSourceList(convertConfigFileSource(request.getUserName(), configFileList)); + stepInstance.setFileSourceList(convertConfigFileSource(username, configFileList)); stepInstance.setAppId(request.getAppId()); - stepInstance.setTargetServers(convertToStandardServers(null, request.getIpList(), null)); - stepInstance.setOperator(request.getUserName()); + stepInstance.setTargetExecuteObjects(convertToStandardServers(null, request.getIpList(), null)); + stepInstance.setOperator(username); stepInstance.setStatus(RunStatusEnum.BLANK); stepInstance.setCreateTime(DateUtils.currentTimeMillis()); stepInstance.setTimeout(JobConstants.DEFAULT_JOB_TIMEOUT_SECONDS); @@ -190,9 +195,9 @@ private List convertConfigFileSource(String userName, fileSourceDTO.setAccount("root"); fileSourceDTO.setLocalUpload(false); fileSourceDTO.setFileType(TaskFileTypeEnum.BASE64_FILE.getType()); - // 保存配置文件至机器 + // 保存配置文件至主机 String configFileLocalPath = ConfigFileUtil.saveConfigFileToLocal( - storageSystemConfig.getJobStorageRootPath(), + fileDistributeConfig.getJobDistributeRootPath(), userName, configFile.getFileName(), configFile.getContent() @@ -200,8 +205,12 @@ private List convertConfigFileSource(String userName, List files = new ArrayList<>(); files.add(new FileDetailDTO(configFileLocalPath)); fileSourceDTO.setFiles(files); - // 设置配置文件所在机器IP信息 - fileSourceDTO.setServers(agentService.getLocalServersDTO()); + // 设置配置文件所在主机信息 + ExecuteTargetDTO fileSourceExecuteObjects = new ExecuteTargetDTO(); + fileSourceExecuteObjects.setStaticIpList(Collections.singletonList( + localFileDistributeSourceHostProvisioner.getLocalFileDistributeSourceHost() + )); + fileSourceDTO.setServers(fileSourceExecuteObjects); fileSourceDTOS.add(fileSourceDTO); }); return fileSourceDTOS; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/JobExecuteCommonProcessor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/JobExecuteCommonProcessor.java index 5a7a3b9bde..745c67229c 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/JobExecuteCommonProcessor.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/JobExecuteCommonProcessor.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,17 +24,12 @@ package com.tencent.bk.job.execute.api.esb.v2.impl; -import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.esb.model.job.EsbIpDTO; import com.tencent.bk.job.common.esb.model.job.EsbServerDTO; -import com.tencent.bk.job.common.exception.NotFoundException; -import com.tencent.bk.job.common.exception.ServiceException; import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.execute.model.DynamicServerGroupDTO; import com.tencent.bk.job.execute.model.DynamicServerTopoNodeDTO; -import com.tencent.bk.job.execute.model.ServersDTO; -import com.tencent.bk.job.execute.service.ScriptService; -import com.tencent.bk.job.manage.model.inner.ServiceScriptDTO; +import com.tencent.bk.job.execute.model.ExecuteTargetDTO; import lombok.extern.slf4j.Slf4j; import java.util.ArrayList; @@ -45,18 +40,6 @@ */ @Slf4j public class JobExecuteCommonProcessor { - protected final ServiceScriptDTO getAndCheckScript(Long appId, String operator, Long scriptId, - ScriptService scriptService) throws ServiceException { - ServiceScriptDTO script = null; - if (scriptId != null && scriptId > 0) { - script = scriptService.getScriptByScriptVersionId(operator, appId, scriptId); - if (script == null) { - log.warn("Script:{} is not in app:{}", scriptId, appId); - throw new NotFoundException(ErrorCode.SCRIPT_NOT_EXIST); - } - } - return script; - } /** * 转换目标服务器 @@ -66,44 +49,44 @@ protected final ServiceScriptDTO getAndCheckScript(Long appId, String operator, * @param dynamicGroupIdList * @return */ - protected ServersDTO convertToStandardServers(EsbServerDTO requestTargetServers, List ipList, - List dynamicGroupIdList) { + protected ExecuteTargetDTO convertToStandardServers(EsbServerDTO requestTargetServers, List ipList, + List dynamicGroupIdList) { // 优先使用servers参数 if (requestTargetServers != null) { - ServersDTO serversDTO = new ServersDTO(); + ExecuteTargetDTO executeTargetDTO = new ExecuteTargetDTO(); if (requestTargetServers.getTopoNodes() != null) { List topoNodes = new ArrayList<>(); requestTargetServers.getTopoNodes().forEach( topoNode -> topoNodes.add(new DynamicServerTopoNodeDTO(topoNode.getId(), topoNode.getNodeType()))); - serversDTO.setTopoNodes(topoNodes); + executeTargetDTO.setTopoNodes(topoNodes); } if (requestTargetServers.getDynamicGroupIds() != null) { List dynamicServerGroups = new ArrayList<>(); requestTargetServers.getDynamicGroupIds().forEach( groupId -> dynamicServerGroups.add(new DynamicServerGroupDTO(groupId))); - serversDTO.setDynamicServerGroups(dynamicServerGroups); + executeTargetDTO.setDynamicServerGroups(dynamicServerGroups); } if (requestTargetServers.getIps() != null) { List staticIpList = new ArrayList<>(); requestTargetServers.getIps().forEach(ip -> staticIpList.add(new HostDTO(ip.getBkCloudId(), ip.getIp()))); - serversDTO.setStaticIpList(staticIpList); + executeTargetDTO.setStaticIpList(staticIpList); } - return serversDTO; + return executeTargetDTO; } else { // 兼容历史版本API - ServersDTO serversDTO = new ServersDTO(); + ExecuteTargetDTO executeTargetDTO = new ExecuteTargetDTO(); if (ipList != null) { List staticIpList = new ArrayList<>(); ipList.forEach(ip -> staticIpList.add(new HostDTO(ip.getBkCloudId(), ip.getIp()))); - serversDTO.setStaticIpList(staticIpList); + executeTargetDTO.setStaticIpList(staticIpList); } if (dynamicGroupIdList != null) { List dynamicServerGroups = new ArrayList<>(); dynamicGroupIdList.forEach(groupId -> dynamicServerGroups.add(new DynamicServerGroupDTO(groupId))); - serversDTO.setDynamicServerGroups(dynamicServerGroups); + executeTargetDTO.setDynamicServerGroups(dynamicServerGroups); } - return serversDTO; + return executeTargetDTO; } } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/JobQueryCommonProcessor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/JobQueryCommonProcessor.java deleted file mode 100644 index a1c5f205df..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v2/impl/JobQueryCommonProcessor.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.api.esb.v2.impl; - -import com.tencent.bk.job.common.constant.ErrorCode; -import com.tencent.bk.job.common.exception.NotFoundException; -import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; -import com.tencent.bk.job.common.iam.model.AuthResult; -import com.tencent.bk.job.common.model.dto.AppResourceScope; -import com.tencent.bk.job.execute.auth.ExecuteAuthService; -import com.tencent.bk.job.execute.model.TaskInstanceDTO; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; - -/** - * 作业查询基础类 - */ -@Slf4j -public class JobQueryCommonProcessor { - @Autowired - protected ExecuteAuthService executeAuthService; - - /** - * 查看作业实例鉴权 - * - * @param username 用户名 - * @param appResourceScope 资源范围 - * @param taskInstance 作业实例 - */ - protected void authViewTaskInstance(String username, AppResourceScope appResourceScope, - TaskInstanceDTO taskInstance) - throws PermissionDeniedException { - if (taskInstance == null) { - log.info("TaskInstance is null"); - throw new NotFoundException(ErrorCode.TASK_INSTANCE_NOT_EXIST); - } - if (!appResourceScope.getAppId().equals(taskInstance.getAppId())) { - log.info("TaskInstance {}|{} is not in resource scope : {}", taskInstance.getAppId(), - taskInstance.getId(), appResourceScope); - throw new NotFoundException(ErrorCode.TASK_INSTANCE_NOT_EXIST); - } - - AuthResult authResult = executeAuthService.authViewTaskInstance( - username, appResourceScope, taskInstance); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } - } -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBatchGetJobInstanceIpLogV3ResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBatchGetJobInstanceIpLogV3ResourceImpl.java index d71e6f6ef4..5ee262c03e 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBatchGetJobInstanceIpLogV3ResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBatchGetJobInstanceIpLogV3ResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,65 +24,67 @@ package com.tencent.bk.job.execute.api.esb.v3; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.audit.annotations.AuditRequestBody; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.esb.metrics.EsbApiTimed; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.common.esb.model.job.EsbIpDTO; import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.exception.NotFoundException; +import com.tencent.bk.job.common.gse.constants.FileDistModeEnum; +import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.metrics.CommonMetricNames; import com.tencent.bk.job.common.model.ValidateResult; import com.tencent.bk.job.common.model.dto.HostDTO; -import com.tencent.bk.job.common.service.AppScopeMappingService; -import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.common.util.ip.IpUtils; -import com.tencent.bk.job.execute.api.esb.v2.impl.JobQueryCommonProcessor; -import com.tencent.bk.job.execute.common.constants.FileDistModeEnum; -import com.tencent.bk.job.execute.model.ScriptHostLogContent; +import com.tencent.bk.job.execute.model.AtomicFileTaskLog; +import com.tencent.bk.job.execute.model.ExecuteObjectCompositeKey; +import com.tencent.bk.job.execute.model.FileExecuteObjectLogContent; +import com.tencent.bk.job.execute.model.ScriptExecuteObjectLogContent; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; -import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.model.esb.v3.EsbFileIpLogV3DTO; import com.tencent.bk.job.execute.model.esb.v3.EsbFileLogV3DTO; import com.tencent.bk.job.execute.model.esb.v3.EsbIpLogsV3DTO; -import com.tencent.bk.job.execute.model.esb.v3.EsbScriptIpLogV3DTO; +import com.tencent.bk.job.execute.model.esb.v3.EsbScriptHostLogV3DTO; import com.tencent.bk.job.execute.model.esb.v3.request.EsbBatchGetJobInstanceIpLogV3Request; import com.tencent.bk.job.execute.service.LogService; -import com.tencent.bk.job.execute.service.TaskInstanceService; +import com.tencent.bk.job.execute.service.StepInstanceService; +import com.tencent.bk.job.execute.service.TaskInstanceAccessProcessor; +import com.tencent.bk.job.execute.util.ExecuteObjectCompositeKeyUtils; import com.tencent.bk.job.logsvr.consts.LogTypeEnum; -import com.tencent.bk.job.logsvr.model.service.ServiceFileTaskLogDTO; -import com.tencent.bk.job.logsvr.model.service.ServiceHostLogsDTO; +import com.tencent.bk.job.logsvr.util.LogFieldUtil; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.web.bind.annotation.RestController; -import java.time.ZoneId; -import java.time.temporal.ChronoUnit; import java.util.List; import java.util.stream.Collectors; @RestController @Slf4j -public class EsbBatchGetJobInstanceIpLogV3ResourceImpl - extends JobQueryCommonProcessor - implements EsbBatchGetJobInstanceIpLogV3Resource { +public class EsbBatchGetJobInstanceIpLogV3ResourceImpl implements EsbBatchGetJobInstanceIpLogV3Resource { - private final TaskInstanceService taskInstanceService; + private final StepInstanceService stepInstanceService; private final LogService logService; - private final AppScopeMappingService appScopeMappingService; + private final TaskInstanceAccessProcessor taskInstanceAccessProcessor; public EsbBatchGetJobInstanceIpLogV3ResourceImpl(LogService logService, - TaskInstanceService taskInstanceService, - AppScopeMappingService appScopeMappingService) { - this.appScopeMappingService = appScopeMappingService; + StepInstanceService stepInstanceService, + TaskInstanceAccessProcessor taskInstanceAccessProcessor) { this.logService = logService; - this.taskInstanceService = taskInstanceService; + this.stepInstanceService = stepInstanceService; + this.taskInstanceAccessProcessor = taskInstanceAccessProcessor; } @Override @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_batch_get_job_instance_ip_log"}) - public EsbResp batchGetJobInstanceIpLogs(EsbBatchGetJobInstanceIpLogV3Request request) { - request.fillAppResourceScope(appScopeMappingService); + @AuditEntry(actionId = ActionId.VIEW_HISTORY) + public EsbResp batchGetJobInstanceIpLogs( + String username, + String appCode, + @AuditRequestBody EsbBatchGetJobInstanceIpLogV3Request request) { ValidateResult checkResult = checkRequest(request); if (!checkResult.isPass()) { log.warn("Batch get job instance ip log request is illegal!"); @@ -90,14 +92,11 @@ public EsbResp batchGetJobInstanceIpLogs(EsbBatchGetJobInstanceI } long taskInstanceId = request.getTaskInstanceId(); - TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(taskInstanceId); - if (taskInstance == null) { - throw new NotFoundException(ErrorCode.TASK_INSTANCE_NOT_EXIST); - } - - authViewTaskInstance(request.getUserName(), request.getAppResourceScope(), taskInstance); + taskInstanceAccessProcessor.processBeforeAccess(username, + request.getAppResourceScope().getAppId(), taskInstanceId); - StepInstanceBaseDTO stepInstance = taskInstanceService.getBaseStepInstance(request.getStepInstanceId()); + StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstance( + request.getTaskInstanceId(), request.getStepInstanceId()); if (stepInstance == null) { throw new NotFoundException(ErrorCode.TASK_INSTANCE_NOT_EXIST); } @@ -106,112 +105,89 @@ public EsbResp batchGetJobInstanceIpLogs(EsbBatchGetJobInstanceI ipLogs.setTaskInstanceId(taskInstanceId); ipLogs.setStepInstanceId(request.getStepInstanceId()); + List hostKeys = + ExecuteObjectCompositeKeyUtils.fromEsbHostParams(request.getHostIdList(), request.getIpList()); + if (stepInstance.isScriptStep()) { - buildScriptLogs(ipLogs, stepInstance, request.getIpList()); + buildScriptLogs(ipLogs, stepInstance, hostKeys); } else if (stepInstance.isFileStep()) { - buildFileLogs(ipLogs, stepInstance, request.getIpList()); + buildFileLogs(ipLogs, stepInstance, hostKeys); } return EsbResp.buildSuccessResp(ipLogs); } private ValidateResult checkRequest(EsbBatchGetJobInstanceIpLogV3Request request) { - if (request.getAppId() == null || request.getAppId() < 1) { - log.warn("App is empty or illegal, appId={}", request.getAppId()); - return ValidateResult.fail(ErrorCode.MISSING_OR_ILLEGAL_PARAM_WITH_PARAM_NAME, "bk_biz_id"); - } - if (request.getTaskInstanceId() == null || request.getTaskInstanceId() < 1) { - log.warn("TaskInstanceId is empty or illegal, taskInstanceId={}", request.getTaskInstanceId()); - return ValidateResult.fail(ErrorCode.MISSING_OR_ILLEGAL_PARAM_WITH_PARAM_NAME, "job_instance_id"); - } - if (request.getStepInstanceId() == null || request.getStepInstanceId() < 1) { - log.warn("StepInstanceId is empty or illegal, stepInstanceId={}", request.getStepInstanceId()); + if (CollectionUtils.isEmpty(request.getHostIdList()) && CollectionUtils.isEmpty(request.getIpList())) { return ValidateResult.fail(ErrorCode.MISSING_OR_ILLEGAL_PARAM_WITH_PARAM_NAME, - "step_instance_id"); - } - - ValidateResult ipCheckResult = checkIps(request.getIpList()); - if (!ipCheckResult.isPass()) { - return ipCheckResult; + "host_id_list/ip_list"); } - int ipSize = request.getIpList().size(); - if (ipSize > 500) { - log.warn("IpList size is gt 500, stepInstanceId={}, size: {}", request.getStepInstanceId(), ipSize); + int queryHostSize = CollectionUtils.isNotEmpty(request.getHostIdList()) ? + request.getHostIdList().size() : request.getIpList().size(); + if (queryHostSize > 500) { + log.warn("Host size is gt 500, stepInstanceId={}, size: {}", request.getStepInstanceId(), queryHostSize); return ValidateResult.fail(ErrorCode.MISSING_OR_ILLEGAL_PARAM_WITH_PARAM_NAME, - "ip_list"); + "host_id_list/ip_list"); } - return ValidateResult.pass(); - } - - private ValidateResult checkIps(List cloudIpList) { - if (CollectionUtils.isEmpty(cloudIpList)) { - log.warn("IpList is empty "); - return ValidateResult.fail(ErrorCode.MISSING_OR_ILLEGAL_PARAM_WITH_PARAM_NAME, "ip_list"); - } - for (EsbIpDTO cloudIp : cloudIpList) { - if (cloudIp.getBkCloudId() == null || cloudIp.getBkCloudId() < 0) { - log.warn("CloudAreaId is empty or illegal, cloudAreaId={}", cloudIp.getBkCloudId()); - return ValidateResult.fail(ErrorCode.MISSING_OR_ILLEGAL_PARAM_WITH_PARAM_NAME, - "bk_cloud_id"); - } - if (StringUtils.isBlank(cloudIp.getIp())) { - log.warn("Ip is empty"); - return ValidateResult.fail(ErrorCode.MISSING_PARAM_WITH_PARAM_NAME, "ip"); - } - if (!IpUtils.checkIp(cloudIp.getIp())) { - log.warn("Ip is illegal, ip={}", cloudIp.getIp()); - return ValidateResult.fail(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME, "ip"); - } - } return ValidateResult.pass(); } - private void buildScriptLogs(EsbIpLogsV3DTO ipLogs, StepInstanceBaseDTO stepInstance, - List ipList) { + + private void buildScriptLogs(EsbIpLogsV3DTO ipLogs, + StepInstanceBaseDTO stepInstance, + List hostKeys) { ipLogs.setLogType(LogTypeEnum.SCRIPT.getValue()); - String jobCreateDate = DateUtils.formatUnixTimestamp(stepInstance.getCreateTime(), ChronoUnit.MILLIS, - "yyyy_MM_dd", ZoneId.of("UTC")); - List ipLogContentList = logService.batchGetScriptHostLogContent(jobCreateDate, - stepInstance.getId(), stepInstance.getExecuteCount(), null, - ipList.stream().map(cloudIp -> new HostDTO(cloudIp.getBkCloudId(), cloudIp.getIp())) - .collect(Collectors.toList())); + String jobCreateDate = LogFieldUtil.buildJobCreateDate(stepInstance.getCreateTime()); + List hostLogContentList = logService.batchGetScriptExecuteObjectLogContent( + jobCreateDate, stepInstance, stepInstance.getExecuteCount(), null, hostKeys); - if (CollectionUtils.isEmpty(ipLogContentList)) { + if (CollectionUtils.isEmpty(hostLogContentList)) { return; } - List scriptTaskLogs = ipLogContentList.stream().map(ipLogContent -> { - EsbScriptIpLogV3DTO scriptIpLog = new EsbScriptIpLogV3DTO(); - HostDTO cloudIp = IpUtils.transform(ipLogContent.getIp()); - scriptIpLog.setCloudAreaId(cloudIp.getBkCloudId()); - scriptIpLog.setIp(cloudIp.getIp()); - scriptIpLog.setLogContent(ipLogContent.getContent()); - return scriptIpLog; + List scriptTaskLogs = hostLogContentList.stream().map(hostLogContent -> { + EsbScriptHostLogV3DTO scriptHostLog = new EsbScriptHostLogV3DTO(); + HostDTO host = hostLogContent.getExecuteObject().getHost(); + scriptHostLog.setHostId(host.getHostId()); + if (StringUtils.isNotEmpty(host.toCloudIp())) { + Long bkCloudId = IpUtils.extractBkCloudId(host.toCloudIp()); + String ip = IpUtils.extractIp(host.toCloudIp()); + scriptHostLog.setCloudAreaId(bkCloudId); + scriptHostLog.setIp(ip); + } + if (StringUtils.isNotEmpty(host.toCloudIpv6())) { + Long bkCloudId = IpUtils.extractBkCloudId(host.toCloudIpv6()); + String ipv6 = IpUtils.extractIp(host.toCloudIpv6()); + scriptHostLog.setCloudAreaId(bkCloudId); + scriptHostLog.setIpv6(ipv6); + } + scriptHostLog.setLogContent(hostLogContent.getContent()); + return scriptHostLog; }).collect(Collectors.toList()); ipLogs.setScriptTaskLogs(scriptTaskLogs); } - private void buildFileLogs(EsbIpLogsV3DTO esbIpLogs, StepInstanceBaseDTO stepInstance, - List ipList) { + private void buildFileLogs(EsbIpLogsV3DTO esbIpLogs, + StepInstanceBaseDTO stepInstance, + List hostKeys) { esbIpLogs.setLogType(LogTypeEnum.FILE.getValue()); - ServiceHostLogsDTO ipLogs = logService.batchGetFileIpLogContent( - stepInstance.getId(), stepInstance.getExecuteCount(), null, - ipList.stream().map(cloudIp -> new HostDTO(cloudIp.getBkCloudId(), cloudIp.getIp())) - .collect(Collectors.toList())); + List ipLogs = logService.batchGetFileExecuteObjectLogContent( + stepInstance.getTaskInstanceId(), stepInstance.getId(), stepInstance.getExecuteCount(), + null, null, hostKeys); - if (ipLogs == null || CollectionUtils.isEmpty(ipLogs.getIpLogs())) { + if (CollectionUtils.isEmpty(ipLogs)) { return; } - List fileTaskLogs = ipLogs.getIpLogs().stream().map(ipLog -> { - List ipFileLogs = ipLog.getFileTaskLogs(); + List fileTaskLogs = ipLogs.stream().map(ipLog -> { + List ipFileLogs = ipLog.getFileTaskLogs(); EsbFileIpLogV3DTO esbFileIpLog = new EsbFileIpLogV3DTO(); if (CollectionUtils.isNotEmpty(ipFileLogs)) { - HostDTO cloudIp = HostDTO.fromCloudIp(ipLog.getIp()); - esbFileIpLog.setCloudAreaId(cloudIp.getBkCloudId()); - esbFileIpLog.setIp(cloudIp.getIp()); + esbFileIpLog.setHostId(ipLog.getExecuteObject().getHost().getHostId()); + esbFileIpLog.setCloudAreaId(ipLog.getExecuteObject().getHost().getBkCloudId()); + esbFileIpLog.setIp(ipLog.getExecuteObject().getHost().getIp()); List esbFileLogs = ipFileLogs.stream() .map(this::toEsbFileLogV3DTO).collect(Collectors.toList()); esbFileIpLog.setFileLogs(esbFileLogs); @@ -222,19 +198,19 @@ private void buildFileLogs(EsbIpLogsV3DTO esbIpLogs, StepInstanceBaseDTO stepIns esbIpLogs.setFileTaskLogs(fileTaskLogs); } - private EsbFileLogV3DTO toEsbFileLogV3DTO(ServiceFileTaskLogDTO fileTaskLog) { + private EsbFileLogV3DTO toEsbFileLogV3DTO(AtomicFileTaskLog fileTaskLog) { EsbFileLogV3DTO fileLog = new EsbFileLogV3DTO(); fileLog.setMode(fileTaskLog.getMode()); - if (StringUtils.isNotBlank(fileTaskLog.getDisplaySrcIp())) { - EsbIpDTO srcIp = EsbIpDTO.fromCloudIp(fileTaskLog.getDisplaySrcIp()); + if (fileTaskLog.getSrcExecuteObject() != null) { + EsbIpDTO srcIp = EsbIpDTO.fromHost(fileTaskLog.getSrcExecuteObject().getHost()); if (srcIp != null) { fileLog.setSrcIp(srcIp); } } fileLog.setSrcPath(fileTaskLog.getDisplaySrcFile()); if (FileDistModeEnum.DOWNLOAD.getValue().equals(fileTaskLog.getMode())) { - if (StringUtils.isNotBlank(fileTaskLog.getDestIp())) { - EsbIpDTO destIp = EsbIpDTO.fromCloudIp(fileTaskLog.getDestIp()); + if (fileTaskLog.getDestExecuteObject() != null) { + EsbIpDTO destIp = EsbIpDTO.fromHost(fileTaskLog.getDestExecuteObject().getHost()); if (destIp != null) { fileLog.setDestIp(destIp); } @@ -243,6 +219,9 @@ private EsbFileLogV3DTO toEsbFileLogV3DTO(ServiceFileTaskLogDTO fileTaskLog) { } fileLog.setLogContent(fileTaskLog.getContent()); + fileLog.setSize(fileTaskLog.getSize()); + fileLog.setSpeed(fileTaskLog.getSpeed()); + fileLog.setProcess(fileTaskLog.getProcess()); fileLog.setStatus(fileTaskLog.getStatus()); return fileLog; } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBkCIPluginBatchGetJobInstanceExecuteObjectLogResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBkCIPluginBatchGetJobInstanceExecuteObjectLogResourceImpl.java new file mode 100644 index 0000000000..e184100238 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBkCIPluginBatchGetJobInstanceExecuteObjectLogResourceImpl.java @@ -0,0 +1,191 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.api.esb.v3; + +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.audit.annotations.AuditRequestBody; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.esb.metrics.EsbApiTimed; +import com.tencent.bk.job.common.esb.model.EsbResp; +import com.tencent.bk.job.common.exception.NotFoundException; +import com.tencent.bk.job.common.iam.constant.ActionId; +import com.tencent.bk.job.common.metrics.CommonMetricNames; +import com.tencent.bk.job.execute.model.AtomicFileTaskLog; +import com.tencent.bk.job.execute.model.ExecuteObjectCompositeKey; +import com.tencent.bk.job.execute.model.FileExecuteObjectLogContent; +import com.tencent.bk.job.execute.model.ScriptExecuteObjectLogContent; +import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; +import com.tencent.bk.job.execute.model.esb.v3.bkci.plugin.EsbBkCIPluginBatchGetJobInstanceExecuteObjectLogRequest; +import com.tencent.bk.job.execute.model.esb.v3.bkci.plugin.EsbExecuteObjectLogsDTO; +import com.tencent.bk.job.execute.model.esb.v3.bkci.plugin.EsbFileAtomicTaskLogDTO; +import com.tencent.bk.job.execute.model.esb.v3.bkci.plugin.EsbFileExecuteObjectLogDTO; +import com.tencent.bk.job.execute.model.esb.v3.bkci.plugin.EsbScriptExecuteObjectLogDTO; +import com.tencent.bk.job.execute.service.LogService; +import com.tencent.bk.job.execute.service.StepInstanceService; +import com.tencent.bk.job.execute.service.TaskInstanceAccessProcessor; +import com.tencent.bk.job.execute.util.ExecuteObjectCompositeKeyUtils; +import com.tencent.bk.job.logsvr.consts.LogTypeEnum; +import com.tencent.bk.job.logsvr.util.LogFieldUtil; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; +import java.util.stream.Collectors; + +@RestController +@Slf4j +public class EsbBkCIPluginBatchGetJobInstanceExecuteObjectLogResourceImpl + implements EsbBkCIPluginBatchGetJobInstanceExecuteObjectLogV3Resource { + + private final StepInstanceService stepInstanceService; + private final LogService logService; + private final TaskInstanceAccessProcessor taskInstanceAccessProcessor; + + public EsbBkCIPluginBatchGetJobInstanceExecuteObjectLogResourceImpl( + LogService logService, + StepInstanceService stepInstanceService, + TaskInstanceAccessProcessor taskInstanceAccessProcessor) { + + this.logService = logService; + this.stepInstanceService = stepInstanceService; + this.taskInstanceAccessProcessor = taskInstanceAccessProcessor; + } + + @Override + @EsbApiTimed(value = CommonMetricNames.ESB_API, + extraTags = {"api_name", "v3_bkci_plugin_batch_get_job_instance_execute_object_log"}) + @AuditEntry(actionId = ActionId.VIEW_HISTORY) + public EsbResp batchGetJobInstanceExecuteObjectLogs( + String username, + String appCode, + @AuditRequestBody + @Validated + EsbBkCIPluginBatchGetJobInstanceExecuteObjectLogRequest request) { + + long taskInstanceId = request.getTaskInstanceId(); + taskInstanceAccessProcessor.processBeforeAccess(username, + request.getAppResourceScope().getAppId(), taskInstanceId); + + StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstance( + request.getTaskInstanceId(), request.getStepInstanceId()); + if (stepInstance == null) { + throw new NotFoundException(ErrorCode.TASK_INSTANCE_NOT_EXIST); + } + + EsbExecuteObjectLogsDTO executeObjectLogs = new EsbExecuteObjectLogsDTO(); + executeObjectLogs.setTaskInstanceId(taskInstanceId); + executeObjectLogs.setStepInstanceId(request.getStepInstanceId()); + + + List executeObjectCompositeKeys = + ExecuteObjectCompositeKeyUtils.fromOpenApiExecuteObjectDTOList(request.getExecuteObjects()); + + if (stepInstance.isScriptStep()) { + buildScriptLogs(executeObjectLogs, stepInstance, executeObjectCompositeKeys); + } else if (stepInstance.isFileStep()) { + buildFileLogs(executeObjectLogs, stepInstance, executeObjectCompositeKeys); + } + return EsbResp.buildSuccessResp(executeObjectLogs); + } + + private void buildScriptLogs(EsbExecuteObjectLogsDTO executeObjectLogs, + StepInstanceBaseDTO stepInstance, + List executeObjectCompositeKeys) { + executeObjectLogs.setLogType(LogTypeEnum.SCRIPT.getValue()); + + String jobCreateDate = LogFieldUtil.buildJobCreateDate(stepInstance.getCreateTime()); + List executeObjectLogContentList = + logService.batchGetScriptExecuteObjectLogContent(jobCreateDate, stepInstance, + stepInstance.getExecuteCount(), null, executeObjectCompositeKeys); + + if (CollectionUtils.isEmpty(executeObjectLogContentList)) { + return; + } + + List scriptTaskLogs = + executeObjectLogContentList.stream().map(content -> { + EsbScriptExecuteObjectLogDTO executeObjectLog = new EsbScriptExecuteObjectLogDTO(); + executeObjectLog.setExecuteObject(content.getExecuteObject().toOpenApiExecuteObjectDTO()); + executeObjectLog.setLogContent(content.getContent()); + return executeObjectLog; + }).collect(Collectors.toList()); + executeObjectLogs.setScriptTaskLogs(scriptTaskLogs); + } + + private void buildFileLogs(EsbExecuteObjectLogsDTO esbExecuteObjectLogs, + StepInstanceBaseDTO stepInstance, + List executeObjectCompositeKeys) { + esbExecuteObjectLogs.setLogType(LogTypeEnum.FILE.getValue()); + + List executeObjectLogs = logService.batchGetFileExecuteObjectLogContent( + stepInstance.getTaskInstanceId(), stepInstance.getId(), stepInstance.getExecuteCount(), + null, null, executeObjectCompositeKeys); + + if (CollectionUtils.isEmpty(executeObjectLogs)) { + return; + } + + List esbFileExecuteObjectLogs = + executeObjectLogs.stream() + .map(executeObjectLog -> { + List atomicFileTaskLogs = executeObjectLog.getFileTaskLogs(); + EsbFileExecuteObjectLogDTO esbFileExecuteObjectLogDTO = new EsbFileExecuteObjectLogDTO(); + esbFileExecuteObjectLogDTO.setExecuteObject( + executeObjectLog.getExecuteObject().toOpenApiExecuteObjectDTO()); + if (CollectionUtils.isNotEmpty(atomicFileTaskLogs)) { + List esbFileAtomicTaskLogDTOS = atomicFileTaskLogs.stream() + .map(this::toEsbFileAtomicTaskLogDTO).collect(Collectors.toList()); + esbFileExecuteObjectLogDTO.setFileAtomicTaskLogs(esbFileAtomicTaskLogDTOS); + } + return esbFileExecuteObjectLogDTO; + }).collect(Collectors.toList()); + + esbExecuteObjectLogs.setFileTaskLogs(esbFileExecuteObjectLogs); + } + + private EsbFileAtomicTaskLogDTO toEsbFileAtomicTaskLogDTO(AtomicFileTaskLog fileTaskLog) { + EsbFileAtomicTaskLogDTO esbFileAtomicTaskLogDTO = new EsbFileAtomicTaskLogDTO(); + esbFileAtomicTaskLogDTO.setMode(fileTaskLog.getMode()); + if (fileTaskLog.getSrcExecuteObject() != null) { + esbFileAtomicTaskLogDTO.setSrcExecuteObject(fileTaskLog.getSrcExecuteObject().toOpenApiExecuteObjectDTO()); + } + if (fileTaskLog.getDestExecuteObject() != null) { + esbFileAtomicTaskLogDTO.setDestExecuteObject( + fileTaskLog.getDestExecuteObject().toOpenApiExecuteObjectDTO()); + } + + esbFileAtomicTaskLogDTO.setSrcPath(fileTaskLog.getDisplaySrcFile()); + esbFileAtomicTaskLogDTO.setDestPath(fileTaskLog.getDestFile()); + esbFileAtomicTaskLogDTO.setLogContent(fileTaskLog.getContent()); + esbFileAtomicTaskLogDTO.setSize(fileTaskLog.getSize()); + esbFileAtomicTaskLogDTO.setSpeed(fileTaskLog.getSpeed()); + esbFileAtomicTaskLogDTO.setProcess(fileTaskLog.getProcess()); + esbFileAtomicTaskLogDTO.setStatus(fileTaskLog.getStatus()); + return esbFileAtomicTaskLogDTO; + } + +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBkCIPluginFastExecuteScriptV3ResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBkCIPluginFastExecuteScriptV3ResourceImpl.java new file mode 100644 index 0000000000..659c9404f1 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBkCIPluginFastExecuteScriptV3ResourceImpl.java @@ -0,0 +1,181 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.api.esb.v3; + +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.audit.annotations.AuditRequestBody; +import com.tencent.bk.job.common.constant.JobConstants; +import com.tencent.bk.job.common.esb.metrics.EsbApiTimed; +import com.tencent.bk.job.common.esb.model.EsbResp; +import com.tencent.bk.job.common.i18n.service.MessageI18nService; +import com.tencent.bk.job.common.metrics.CommonMetricNames; +import com.tencent.bk.job.common.util.Base64Util; +import com.tencent.bk.job.common.util.date.DateUtils; +import com.tencent.bk.job.common.web.metrics.CustomTimed; +import com.tencent.bk.job.execute.common.constants.RunStatusEnum; +import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; +import com.tencent.bk.job.execute.common.constants.TaskStartupModeEnum; +import com.tencent.bk.job.execute.common.constants.TaskTypeEnum; +import com.tencent.bk.job.execute.metrics.ExecuteMetricsConstants; +import com.tencent.bk.job.execute.model.ExecuteTargetDTO; +import com.tencent.bk.job.execute.model.FastTaskDTO; +import com.tencent.bk.job.execute.model.StepInstanceDTO; +import com.tencent.bk.job.execute.model.StepRollingConfigDTO; +import com.tencent.bk.job.execute.model.TaskInstanceDTO; +import com.tencent.bk.job.execute.model.esb.v3.EsbJobExecuteV3DTO; +import com.tencent.bk.job.execute.model.esb.v3.bkci.plugin.EsbBkCIPluginFastExecuteScriptRequest; +import com.tencent.bk.job.execute.service.TaskExecuteService; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RestController; + +import java.time.LocalDateTime; + +@RestController +@Slf4j +public class EsbBkCIPluginFastExecuteScriptV3ResourceImpl extends JobExecuteCommonV3Processor + implements EsbBkCIPluginFastExecuteScriptV3Resource { + + private final TaskExecuteService taskExecuteService; + private final MessageI18nService i18nService; + + @Autowired + public EsbBkCIPluginFastExecuteScriptV3ResourceImpl(TaskExecuteService taskExecuteService, + MessageI18nService i18nService) { + this.taskExecuteService = taskExecuteService; + this.i18nService = i18nService; + } + + @Override + @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_bkci_plugin_fast_execute_script"}) + @CustomTimed(metricName = ExecuteMetricsConstants.NAME_JOB_TASK_START, + extraTags = { + ExecuteMetricsConstants.TAG_KEY_START_MODE, ExecuteMetricsConstants.TAG_VALUE_START_MODE_API, + ExecuteMetricsConstants.TAG_KEY_TASK_TYPE, ExecuteMetricsConstants.TAG_VALUE_TASK_TYPE_FAST_SCRIPT + }) + @AuditEntry + public EsbResp fastExecuteScript( + String username, + String appCode, + @AuditRequestBody EsbBkCIPluginFastExecuteScriptRequest request) { + + TaskInstanceDTO taskInstance = buildFastScriptTaskInstance(username, appCode, request); + StepInstanceDTO stepInstance = buildFastScriptStepInstance(username, request); + StepRollingConfigDTO rollingConfig = null; + if (request.getRollingConfig() != null) { + rollingConfig = StepRollingConfigDTO.fromEsbRollingConfig(request.getRollingConfig()); + } + TaskInstanceDTO executeTaskInstance = taskExecuteService.executeFastTask( + FastTaskDTO.builder() + .taskInstance(taskInstance) + .stepInstance(stepInstance) + .rollingConfig(rollingConfig) + .build() + ); + + EsbJobExecuteV3DTO jobExecuteInfo = new EsbJobExecuteV3DTO(); + jobExecuteInfo.setTaskInstanceId(executeTaskInstance.getId()); + jobExecuteInfo.setTaskName(stepInstance.getName()); + jobExecuteInfo.setStepInstanceId(stepInstance.getId()); + return EsbResp.buildSuccessResp(jobExecuteInfo); + } + + private String generateDefaultFastTaskName() { + return i18nService.getI18n("task.type.name.fast_execute_script") + "_" + + DateUtils.formatLocalDateTime(LocalDateTime.now(), "yyyyMMddHHmmssSSS"); + } + + private TaskInstanceDTO buildFastScriptTaskInstance(String username, + String appCode, + EsbBkCIPluginFastExecuteScriptRequest request) { + TaskInstanceDTO taskInstance = new TaskInstanceDTO(); + if (StringUtils.isNotBlank(request.getName())) { + taskInstance.setName(request.getName()); + } else { + taskInstance.setName(generateDefaultFastTaskName()); + } + taskInstance.setPlanId(-1L); + taskInstance.setCronTaskId(-1L); + taskInstance.setTaskTemplateId(-1L); + taskInstance.setDebugTask(false); + taskInstance.setAppId(request.getAppId()); + taskInstance.setStartupMode(TaskStartupModeEnum.API.getValue()); + taskInstance.setStatus(RunStatusEnum.BLANK); + taskInstance.setOperator(username); + taskInstance.setCreateTime(DateUtils.currentTimeMillis()); + taskInstance.setType(TaskTypeEnum.SCRIPT.getValue()); + taskInstance.setCurrentStepInstanceId(0L); + if (request.getCallback() != null) { + taskInstance.setCallbackUrl(request.getCallback().getUrl()); + } + taskInstance.setAppCode(appCode); + return taskInstance; + } + + private StepInstanceDTO buildFastScriptStepInstance(String username, + EsbBkCIPluginFastExecuteScriptRequest request) { + StepInstanceDTO stepInstance = new StepInstanceDTO(); + stepInstance.setAppId(request.getAppId()); + if (StringUtils.isNotBlank(request.getName())) { + stepInstance.setName(request.getName()); + } else { + stepInstance.setName(generateDefaultFastTaskName()); + } + stepInstance.setStepId(-1L); + + if (request.getScriptVersionId() != null && request.getScriptVersionId() > 0) { + stepInstance.setScriptVersionId(request.getScriptVersionId()); + } else if (StringUtils.isNotBlank(request.getScriptId())) { + stepInstance.setScriptId(request.getScriptId()); + } else if (StringUtils.isNotBlank(request.getContent())) { + stepInstance.setScriptContent(Base64Util.decodeContentToStr(request.getContent())); + stepInstance.setScriptType(ScriptTypeEnum.valOf(request.getScriptLanguage())); + } + + if (StringUtils.isNotEmpty(request.getScriptParam())) { + String scriptParam = Base64Util.decodeContentToStr(request.getScriptParam()); + // 需要把换行转换成空格，否则脚本执行报错 + if (StringUtils.isNotBlank(scriptParam)) { + stepInstance.setScriptParam(scriptParam.replace("\n", " ")); + } + } + stepInstance.setWindowsInterpreter(request.getTrimmedWindowsInterpreter()); + stepInstance.setSecureParam(request.isParamSensitive()); + stepInstance.setTimeout( + request.getTimeout() == null ? JobConstants.DEFAULT_JOB_TIMEOUT_SECONDS : request.getTimeout()); + + stepInstance.setExecuteType(StepExecuteTypeEnum.EXECUTE_SCRIPT); + stepInstance.setStatus(RunStatusEnum.BLANK); + stepInstance.setTargetExecuteObjects(ExecuteTargetDTO.buildFrom(request.getExecuteTarget())); + stepInstance.setAccountId(request.getAccountId()); + stepInstance.setAccountAlias(request.getAccountAlias()); + stepInstance.setOperator(username); + stepInstance.setCreateTime(DateUtils.currentTimeMillis()); + + return stepInstance; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBkCIPluginFastTransferFileV3ResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBkCIPluginFastTransferFileV3ResourceImpl.java new file mode 100644 index 0000000000..19283c8e8e --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBkCIPluginFastTransferFileV3ResourceImpl.java @@ -0,0 +1,295 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.api.esb.v3; + +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.constant.JobConstants; +import com.tencent.bk.job.common.constant.NotExistPathHandlerEnum; +import com.tencent.bk.job.common.esb.model.EsbResp; +import com.tencent.bk.job.common.esb.model.job.v3.EsbFileSourceV3DTO; +import com.tencent.bk.job.common.exception.InternalException; +import com.tencent.bk.job.common.exception.InvalidParamException; +import com.tencent.bk.job.common.exception.NotFoundException; +import com.tencent.bk.job.common.exception.ServiceException; +import com.tencent.bk.job.common.i18n.service.MessageI18nService; +import com.tencent.bk.job.common.model.InternalResponse; +import com.tencent.bk.job.common.model.ValidateResult; +import com.tencent.bk.job.common.util.DataSizeConverter; +import com.tencent.bk.job.common.util.date.DateUtils; +import com.tencent.bk.job.execute.common.constants.FileTransferModeEnum; +import com.tencent.bk.job.execute.common.constants.RunStatusEnum; +import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; +import com.tencent.bk.job.execute.common.constants.TaskStartupModeEnum; +import com.tencent.bk.job.execute.common.constants.TaskTypeEnum; +import com.tencent.bk.job.execute.model.ExecuteTargetDTO; +import com.tencent.bk.job.execute.model.FastTaskDTO; +import com.tencent.bk.job.execute.model.FileDetailDTO; +import com.tencent.bk.job.execute.model.FileSourceDTO; +import com.tencent.bk.job.execute.model.StepInstanceDTO; +import com.tencent.bk.job.execute.model.StepRollingConfigDTO; +import com.tencent.bk.job.execute.model.TaskInstanceDTO; +import com.tencent.bk.job.execute.model.esb.v3.EsbJobExecuteV3DTO; +import com.tencent.bk.job.execute.model.esb.v3.bkci.plugin.EsbBkCIPluginFastTransferFileV3Request; +import com.tencent.bk.job.execute.service.TaskExecuteService; +import com.tencent.bk.job.file_gateway.api.inner.ServiceFileSourceResource; +import com.tencent.bk.job.manage.api.common.constants.task.TaskFileTypeEnum; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.slf4j.helpers.MessageFormatter; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RestController; + +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.List; + +@RestController +@Slf4j +public class EsbBkCIPluginFastTransferFileV3ResourceImpl + extends EsbFileTransferProcessor + implements EsbBkCIPluginFastTransferFileV3Resource { + + private final MessageI18nService i18nService; + private final ServiceFileSourceResource fileSourceResource; + private final TaskExecuteService taskExecuteService; + + + @Autowired + public EsbBkCIPluginFastTransferFileV3ResourceImpl(MessageI18nService i18nService, + ServiceFileSourceResource fileSourceResource, + TaskExecuteService taskExecuteService) { + this.i18nService = i18nService; + this.fileSourceResource = fileSourceResource; + this.taskExecuteService = taskExecuteService; + } + + @Override + public EsbResp bkciPluginFastTransferFile( + String username, + String appCode, + EsbBkCIPluginFastTransferFileV3Request request + ) { + if (StringUtils.isEmpty(request.getName())) { + request.setName(generateDefaultFastTaskName()); + } + + ValidateResult validateResult = checkFastTransferFileRequest(request); + if (!validateResult.isPass()) { + log.warn("[EsbBkCIPluginFastTransferFileV3] request is invalid"); + throw new InvalidParamException(validateResult); + } + + TaskInstanceDTO taskInstanceDTO = buildFastFileTaskInstance(username, appCode, request); + StepInstanceDTO stepInstanceDTO = buildFastFileStepInstance(username, request); + StepRollingConfigDTO rollingConfig = null; + if (request.getRollingConfig() != null) { + rollingConfig = StepRollingConfigDTO.fromEsbRollingConfig(request.getRollingConfig()); + } + TaskInstanceDTO executeTaskInstanceDTO = taskExecuteService.executeFastTask( + FastTaskDTO.builder() + .taskInstance(taskInstanceDTO) + .stepInstance(stepInstanceDTO) + .rollingConfig(rollingConfig) + .build() + ); + + EsbJobExecuteV3DTO jobExecuteInfo = buildEsbJobExecuteV3Result( + executeTaskInstanceDTO, + stepInstanceDTO + ); + + return EsbResp.buildSuccessResp(jobExecuteInfo); + } + + private ValidateResult checkFastTransferFileRequest(EsbBkCIPluginFastTransferFileV3Request request) { + ValidateResult validateResult; + + validateResult = validateFilePath(request.trimTargetPath().getTargetPath()); + if (!validateResult.isPass()) { + return validateResult; + } + validateResult = validateAccount(request.getAccountId(), request.getAccountAlias()); + if (!validateResult.isPass()) { + return validateResult; + } + validateResult = validateExecuteObjects(request.getExecuteObject()); + if (!validateResult.isPass()) { + return validateResult; + } + // 校验文件源 + validateResult = validateBkCIPluginFileSources(request.getFileSources()); + if (!validateResult.isPass()) { + return validateResult; + } + + return ValidateResult.pass(); + } + + private ValidateResult validateBkCIPluginFileSources(List fileSources) { + // 插件不支持本地文件分发 + for (EsbFileSourceV3DTO fileSource : fileSources) { + if (fileSource.getFileType() != null && fileSource.getFileType() == TaskFileTypeEnum.LOCAL.getType()) { + String rejectReason = "bk-ci plugin does not support local file source"; + log.warn(rejectReason); + return ValidateResult.fail(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME_AND_REASON, + new Object[]{"file_source_list.file_type", rejectReason}); + } + } + return validateFileSources(fileSources); + } + + private TaskInstanceDTO buildFastFileTaskInstance(String username, + String appCode, + EsbBkCIPluginFastTransferFileV3Request request) { + TaskInstanceDTO taskInstance = new TaskInstanceDTO(); + taskInstance.setName(request.getName()); + taskInstance.setType(TaskTypeEnum.FILE.getValue()); + taskInstance.setPlanId(-1L); + taskInstance.setCronTaskId(-1L); + taskInstance.setTaskTemplateId(-1L); + taskInstance.setAppId(request.getAppId()); + taskInstance.setStatus(RunStatusEnum.BLANK); + taskInstance.setStartupMode(TaskStartupModeEnum.API.getValue()); + taskInstance.setOperator(username); + taskInstance.setCreateTime(DateUtils.currentTimeMillis()); + taskInstance.setCurrentStepInstanceId(0L); + taskInstance.setDebugTask(false); + taskInstance.setCallbackUrl(request.getCallbackUrl()); + taskInstance.setAppCode(appCode); + return taskInstance; + } + + private StepInstanceDTO buildFastFileStepInstance(String username, + EsbBkCIPluginFastTransferFileV3Request request) { + StepInstanceDTO stepInstance = new StepInstanceDTO(); + stepInstance.setName(request.getName()); + stepInstance.setAccountId(request.getAccountId()); + stepInstance.setAccountAlias(request.getAccountAlias()); + stepInstance.setStepId(-1L); + stepInstance.setExecuteType(StepExecuteTypeEnum.SEND_FILE); + stepInstance.setFileTargetPath(request.getTargetPath()); + stepInstance.setFileTargetName(request.getTargetName()); + stepInstance.setFileSourceList(convertFileSource(request.getAppId(), request.getFileSources())); + stepInstance.setAppId(request.getAppId()); + stepInstance.setTargetExecuteObjects(ExecuteTargetDTO.buildFrom(request.getExecuteObject())); + stepInstance.setOperator(username); + stepInstance.setStatus(RunStatusEnum.BLANK); + stepInstance.setCreateTime(DateUtils.currentTimeMillis()); + stepInstance.setTimeout(request.getTimeout() == null + ? JobConstants.DEFAULT_JOB_TIMEOUT_SECONDS : request.getTimeout()); + if (request.getUploadSpeedLimit() != null && request.getUploadSpeedLimit() > 0) { + stepInstance.setFileUploadSpeedLimit(DataSizeConverter.convertMBToKB(request.getUploadSpeedLimit())); + } + if (request.getDownloadSpeedLimit() != null && request.getDownloadSpeedLimit() > 0) { + stepInstance.setFileDownloadSpeedLimit(DataSizeConverter.convertMBToKB(request.getDownloadSpeedLimit())); + } + FileTransferModeEnum transferMode = FileTransferModeEnum.getFileTransferModeEnum(request.getTransferMode()); + if (transferMode == null) { + transferMode = FileTransferModeEnum.FORCE; + } + if (transferMode == FileTransferModeEnum.STRICT) { + stepInstance.setNotExistPathHandler(NotExistPathHandlerEnum.STEP_FAIL.getValue()); + } else { + stepInstance.setNotExistPathHandler(NotExistPathHandlerEnum.CREATE_DIR.getValue()); + } + return stepInstance; + } + + private List convertFileSource(Long appId, + List fileSources) throws ServiceException { + if (fileSources == null) { + return null; + } + List fileSourceDTOS = new ArrayList<>(); + fileSources.forEach(fileSource -> { + Integer fileType = fileSource.getFileType(); + if (fileType == null) { + // 默认服务器文件分发 + fileType = TaskFileTypeEnum.SERVER.getType(); + } + FileSourceDTO fileSourceDTO = new FileSourceDTO(); + if (TaskFileTypeEnum.SERVER.getType() == fileType) { + fileSourceDTO.setAccountId(fileSource.getAccount().getId()); + fileSourceDTO.setAccountAlias(fileSource.getAccount().getAlias()); + } + // 插件调用不支持本地文件分发，所以直接置为false + fileSourceDTO.setLocalUpload(false); + fileSourceDTO.setFileType(fileType); + List files = new ArrayList<>(); + if (fileSource.getTrimmedFiles() != null) { + for (String file : fileSource.getTrimmedFiles()) { + FileDetailDTO fileDetailDTO = new FileDetailDTO(file); + files.add(fileDetailDTO); + } + } + Integer fileSourceId = fileSource.getFileSourceId(); + String fileSourceCode = fileSource.getFileSourceCode(); + if (fileSourceId != null) { + fileSourceDTO.setFileSourceId(fileSource.getFileSourceId()); + } else if (StringUtils.isNotBlank(fileSourceCode)) { + try { + InternalResponse resp = fileSourceResource.getFileSourceIdByCode(appId, fileSourceCode); + if (resp != null && resp.isSuccess()) { + if (resp.getData() != null) { + fileSourceDTO.setFileSourceId(resp.getData()); + } else { + log.warn("fileSourceCode={},resp={}", fileSourceCode, resp); + throw new NotFoundException(ErrorCode.FAIL_TO_FIND_FILE_SOURCE_BY_CODE, + new String[]{fileSourceCode}); + } + } else { + log.warn("fileSourceCode={},resp={}", fileSourceCode, resp); + throw new NotFoundException(ErrorCode.FILE_SOURCE_SERVICE_INVALID); + } + } catch (Exception e) { + String msg = MessageFormatter.format( + "Fail to parse fileSourceCode to id:{}", + fileSourceCode + ).getMessage(); + log.error(msg, e); + throw new InternalException(ErrorCode.INTERNAL_ERROR); + } + } + fileSourceDTO.setFiles(files); + fileSourceDTO.setServers(buildFileSourceServer(fileSource.getServer())); + fileSourceDTOS.add(fileSourceDTO); + }); + return fileSourceDTOS; + } + + private EsbJobExecuteV3DTO buildEsbJobExecuteV3Result(TaskInstanceDTO executeTaskInstance, + StepInstanceDTO stepInstance) { + EsbJobExecuteV3DTO jobExecuteInfo = new EsbJobExecuteV3DTO(); + jobExecuteInfo.setTaskInstanceId(executeTaskInstance.getId()); + jobExecuteInfo.setStepInstanceId(stepInstance.getId()); + jobExecuteInfo.setTaskName(stepInstance.getName()); + return jobExecuteInfo; + } + + private String generateDefaultFastTaskName() { + return i18nService.getI18n("task.type.name.fast_push_file") + "_" + + DateUtils.formatLocalDateTime(LocalDateTime.now(), "yyyyMMddHHmmssSSS"); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBkCIPluginGetJobInstanceStatusV3ResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBkCIPluginGetJobInstanceStatusV3ResourceImpl.java new file mode 100644 index 0000000000..4e8cf046de --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbBkCIPluginGetJobInstanceStatusV3ResourceImpl.java @@ -0,0 +1,177 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.api.esb.v3; + +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.audit.annotations.AuditRequestBody; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.esb.metrics.EsbApiTimed; +import com.tencent.bk.job.common.esb.model.EsbResp; +import com.tencent.bk.job.common.esb.util.EsbDTOAppScopeMappingHelper; +import com.tencent.bk.job.common.exception.NotFoundException; +import com.tencent.bk.job.common.iam.constant.ActionId; +import com.tencent.bk.job.common.metrics.CommonMetricNames; +import com.tencent.bk.job.execute.common.constants.RunStatusEnum; +import com.tencent.bk.job.execute.model.ExecuteObjectTask; +import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; +import com.tencent.bk.job.execute.model.TaskInstanceDTO; +import com.tencent.bk.job.execute.model.esb.v3.bkci.plugin.EsbBkCIPluginGetJobInstanceStatusRequest; +import com.tencent.bk.job.execute.model.esb.v3.bkci.plugin.EsbJobInstanceStatusDTO; +import com.tencent.bk.job.execute.service.FileExecuteObjectTaskService; +import com.tencent.bk.job.execute.service.ScriptExecuteObjectTaskService; +import com.tencent.bk.job.execute.service.StepInstanceService; +import com.tencent.bk.job.execute.service.TaskInstanceService; +import com.tencent.bk.job.logsvr.consts.FileTaskModeEnum; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.RestController; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +@RestController +@Slf4j +public class EsbBkCIPluginGetJobInstanceStatusV3ResourceImpl implements EsbBkCIPluginGetJobInstanceStatusV3Resource { + + + private final TaskInstanceService taskInstanceService; + private final ScriptExecuteObjectTaskService scriptExecuteObjectTaskService; + private final FileExecuteObjectTaskService fileExecuteObjectTaskService; + private final StepInstanceService stepInstanceService; + + public EsbBkCIPluginGetJobInstanceStatusV3ResourceImpl( + TaskInstanceService taskInstanceService, + ScriptExecuteObjectTaskService scriptExecuteObjectTaskService, + FileExecuteObjectTaskService fileExecuteObjectTaskService, + StepInstanceService stepInstanceService) { + + this.taskInstanceService = taskInstanceService; + this.scriptExecuteObjectTaskService = scriptExecuteObjectTaskService; + this.fileExecuteObjectTaskService = fileExecuteObjectTaskService; + this.stepInstanceService = stepInstanceService; + } + + @Override + @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_bkci_plugin_get_job_instance_status"}) + @AuditEntry(actionId = ActionId.VIEW_HISTORY) + public EsbResp getJobInstanceStatus( + String username, + String appCode, + @AuditRequestBody + @Validated + EsbBkCIPluginGetJobInstanceStatusRequest request) { + + long taskInstanceId = request.getTaskInstanceId(); + + TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance( + username, request.getAppResourceScope().getAppId(), request.getTaskInstanceId()); + + List stepInstances = + stepInstanceService.listBaseStepInstanceByTaskInstanceId(taskInstanceId); + if (stepInstances == null || stepInstances.isEmpty()) { + log.warn("Get job instance status by taskInstanceId:{}, stepInstanceList is empty!", taskInstanceId); + throw new NotFoundException(ErrorCode.STEP_INSTANCE_NOT_EXIST); + } + + EsbJobInstanceStatusDTO jobInstanceStatus = buildEsbJobInstanceStatusDTO(taskInstance, stepInstances, + request.isIncludeExecuteObjectTaskResult()); + + return EsbResp.buildSuccessResp(jobInstanceStatus); + } + + private EsbJobInstanceStatusDTO buildEsbJobInstanceStatusDTO(TaskInstanceDTO taskInstance, + List stepInstances, + boolean isIncludeExecuteObjectTaskResult) { + EsbJobInstanceStatusDTO jobInstanceStatus = new EsbJobInstanceStatusDTO(); + jobInstanceStatus.setFinished(RunStatusEnum.isFinishedStatus(taskInstance.getStatus())); + + EsbJobInstanceStatusDTO.JobInstance jobInstance = new EsbJobInstanceStatusDTO.JobInstance(); + EsbDTOAppScopeMappingHelper.fillEsbAppScopeDTOByAppId(taskInstance.getAppId(), jobInstance); + jobInstance.setId(taskInstance.getId()); + jobInstance.setName(taskInstance.getName()); + jobInstance.setCreateTime(taskInstance.getCreateTime()); + jobInstance.setStartTime(taskInstance.getStartTime()); + jobInstance.setEndTime(taskInstance.getEndTime()); + jobInstance.setStatus(taskInstance.getStatus().getValue()); + jobInstance.setTotalTime(taskInstance.getTotalTime()); + jobInstanceStatus.setJobInstance(jobInstance); + + List stepResults = new ArrayList<>(stepInstances.size()); + for (StepInstanceBaseDTO stepInstance : stepInstances) { + EsbJobInstanceStatusDTO.StepResult stepResult = new EsbJobInstanceStatusDTO.StepResult(); + stepResult.setId(stepInstance.getId()); + stepResult.setName(stepInstance.getName()); + stepResult.setCreateTime(stepInstance.getCreateTime()); + stepResult.setEndTime(stepInstance.getEndTime()); + stepResult.setStartTime(stepInstance.getStartTime()); + stepResult.setType(stepInstance.getExecuteType().getValue()); + stepResult.setExecuteCount(stepInstance.getExecuteCount()); + stepResult.setStatus(stepInstance.getStatus().getValue()); + stepResult.setTotalTime(stepInstance.getTotalTime()); + + if (isIncludeExecuteObjectTaskResult) { + List executeObjectResults = new ArrayList<>(); + List executeObjectTaskList = null; + if (stepInstance.isScriptStep()) { + executeObjectTaskList = scriptExecuteObjectTaskService.listTasks(stepInstance, + stepInstance.getExecuteCount(), null); + } else if (stepInstance.isFileStep()) { + executeObjectTaskList = fileExecuteObjectTaskService.listTasks(stepInstance, + stepInstance.getExecuteCount(), null); + if (CollectionUtils.isNotEmpty(executeObjectTaskList)) { + // 如果是文件分发任务，只返回目标Agent结果 + executeObjectTaskList = executeObjectTaskList.stream() + .filter(executeObjectTask -> + executeObjectTask.getFileTaskMode() == FileTaskModeEnum.DOWNLOAD) + .collect(Collectors.toList()); + } + } + if (CollectionUtils.isNotEmpty(executeObjectTaskList)) { + for (ExecuteObjectTask executeObjectTask : executeObjectTaskList) { + EsbJobInstanceStatusDTO.ExecuteObjectResult executeObjectResult = + new EsbJobInstanceStatusDTO.ExecuteObjectResult(); + executeObjectResult.setExecuteObject( + executeObjectTask.getExecuteObject().toOpenApiExecuteObjectDTO()); + executeObjectResult.setExitCode(executeObjectTask.getExitCode()); + executeObjectResult.setErrorCode(executeObjectTask.getErrorCode()); + executeObjectResult.setStartTime(executeObjectTask.getStartTime()); + executeObjectResult.setEndTime(executeObjectTask.getEndTime()); + executeObjectResult.setTotalTime(executeObjectTask.getTotalTime()); + executeObjectResult.setTag(executeObjectTask.getTag()); + executeObjectResult.setStatus(executeObjectTask.getStatus().getValue()); + executeObjectResults.add(executeObjectResult); + } + } + stepResult.setExecuteObjectResults(executeObjectResults); + } + stepResults.add(stepResult); + } + jobInstanceStatus.setStepResults(stepResults); + + return jobInstanceStatus; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbCommonExecuteTaskProcessor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbCommonExecuteTaskProcessor.java new file mode 100644 index 0000000000..3b7fe548a2 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbCommonExecuteTaskProcessor.java @@ -0,0 +1,52 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.api.esb.v3; + +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.model.ValidateResult; +import com.tencent.bk.job.common.model.openapi.v4.OpenApiExecuteTargetDTO; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +@Slf4j +public class EsbCommonExecuteTaskProcessor { + + protected ValidateResult validateAccount(Long accountId, String accountAlias) { + if ((accountId == null || accountId <= 0L) && StringUtils.isBlank(accountAlias)) { + log.warn("Fast transfer file, account is empty!"); + return ValidateResult.fail(ErrorCode.MISSING_PARAM_WITH_PARAM_NAME, "account_id|account_alias"); + } + return ValidateResult.pass(); + } + + protected ValidateResult validateExecuteObjects(OpenApiExecuteTargetDTO executeTarget) { + if (executeTarget == null || !executeTarget.isValidExecuteTarget()) { + log.warn("Fast transfer file, targetServer is illegal!"); + return ValidateResult.fail(ErrorCode.MISSING_PARAM_WITH_PARAM_NAME, "target_server"); + } + return ValidateResult.pass(); + } + +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbExecuteJobPlanV3ResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbExecuteJobPlanV3ResourceImpl.java index 57365ff922..d57dce1e9c 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbExecuteJobPlanV3ResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbExecuteJobPlanV3ResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,28 +24,29 @@ package com.tencent.bk.job.execute.api.esb.v3; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.audit.annotations.AuditRequestBody; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.esb.metrics.EsbApiTimed; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.common.esb.model.job.v3.EsbGlobalVarV3DTO; import com.tencent.bk.job.common.esb.model.job.v3.EsbServerV3DTO; import com.tencent.bk.job.common.exception.InvalidParamException; +import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.metrics.CommonMetricNames; import com.tencent.bk.job.common.model.ValidateResult; -import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.common.web.metrics.CustomTimed; import com.tencent.bk.job.execute.common.constants.TaskStartupModeEnum; import com.tencent.bk.job.execute.engine.model.TaskVariableDTO; import com.tencent.bk.job.execute.metrics.ExecuteMetricsConstants; -import com.tencent.bk.job.execute.model.ServersDTO; +import com.tencent.bk.job.execute.model.ExecuteTargetDTO; import com.tencent.bk.job.execute.model.TaskExecuteParam; import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.model.esb.v3.EsbJobExecuteV3DTO; import com.tencent.bk.job.execute.model.esb.v3.request.EsbExecuteJobV3Request; import com.tencent.bk.job.execute.service.TaskExecuteService; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RestController; @@ -60,13 +61,10 @@ public class EsbExecuteJobPlanV3ResourceImpl implements EsbExecuteJobPlanV3Resource { private final TaskExecuteService taskExecuteService; - private final AppScopeMappingService appScopeMappingService; @Autowired - public EsbExecuteJobPlanV3ResourceImpl(TaskExecuteService taskExecuteService, - AppScopeMappingService appScopeMappingService) { + public EsbExecuteJobPlanV3ResourceImpl(TaskExecuteService taskExecuteService) { this.taskExecuteService = taskExecuteService; - this.appScopeMappingService = appScopeMappingService; } @Override @@ -76,8 +74,10 @@ public EsbExecuteJobPlanV3ResourceImpl(TaskExecuteService taskExecuteService, ExecuteMetricsConstants.TAG_KEY_START_MODE, ExecuteMetricsConstants.TAG_VALUE_START_MODE_API, ExecuteMetricsConstants.TAG_KEY_TASK_TYPE, ExecuteMetricsConstants.TAG_VALUE_TASK_TYPE_EXECUTE_PLAN }) - public EsbResp executeJobPlan(EsbExecuteJobV3Request request) { - request.fillAppResourceScope(appScopeMappingService); + @AuditEntry(actionId = ActionId.LAUNCH_JOB_PLAN) + public EsbResp executeJobPlan(String username, + String appCode, + @AuditRequestBody EsbExecuteJobV3Request request) { ValidateResult checkResult = checkExecuteTaskRequest(request); log.info("Execute task, request={}", JsonUtils.toJson(request)); if (!checkResult.isPass()) { @@ -94,12 +94,9 @@ public EsbResp executeJobPlan(EsbExecuteJobV3Request request taskVariableDTO.setId(globalVar.getId()); taskVariableDTO.setName(globalVar.getName()); EsbServerV3DTO server = globalVar.getServer(); - if (StringUtils.isEmpty(globalVar.getValue()) && server != null && - (CollectionUtils.isNotEmpty(server.getIps()) - || CollectionUtils.isNotEmpty(server.getTopoNodes()) - || CollectionUtils.isNotEmpty(server.getDynamicGroups()))) { - ServersDTO serversDTO = convertToServersDTO(globalVar.getServer()); - taskVariableDTO.setTargetServers(serversDTO); + if (StringUtils.isEmpty(globalVar.getValue()) && server != null && server.checkHostParamsNonEmpty()) { + ExecuteTargetDTO executeTargetDTO = convertToServersDTO(globalVar.getServer()); + taskVariableDTO.setExecuteTarget(executeTargetDTO); } else { taskVariableDTO.setValue(globalVar.getValue()); } @@ -111,11 +108,12 @@ public EsbResp executeJobPlan(EsbExecuteJobV3Request request .builder() .appId(request.getAppId()) .planId(request.getTaskId()) - .operator(request.getUserName()) + .operator(username) .executeVariableValues(executeVariableValues) .startupMode(TaskStartupModeEnum.API) .callbackUrl(request.getCallbackUrl()) - .appCode(request.getAppCode()) + .appCode(appCode) + .startTask(request.getStartTask()) .build()); EsbJobExecuteV3DTO result = new EsbJobExecuteV3DTO(); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastExecuteSQLV3ResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastExecuteSQLV3ResourceImpl.java index 1bae1a7ecc..183eae5a98 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastExecuteSQLV3ResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastExecuteSQLV3ResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,8 @@ package com.tencent.bk.job.execute.api.esb.v3; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.audit.annotations.AuditRequestBody; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.constant.JobConstants; import com.tencent.bk.job.common.esb.metrics.EsbApiTimed; @@ -32,7 +34,6 @@ import com.tencent.bk.job.common.i18n.service.MessageI18nService; import com.tencent.bk.job.common.metrics.CommonMetricNames; import com.tencent.bk.job.common.model.ValidateResult; -import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.common.util.Base64Util; import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.common.web.metrics.CustomTimed; @@ -47,7 +48,7 @@ import com.tencent.bk.job.execute.model.esb.v3.EsbJobExecuteV3DTO; import com.tencent.bk.job.execute.model.esb.v3.request.EsbFastExecuteSQLV3Request; import com.tencent.bk.job.execute.service.TaskExecuteService; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -63,15 +64,12 @@ public class EsbFastExecuteSQLV3ResourceImpl private final TaskExecuteService taskExecuteService; private final MessageI18nService i18nService; - private final AppScopeMappingService appScopeMappingService; @Autowired public EsbFastExecuteSQLV3ResourceImpl(TaskExecuteService taskExecuteService, - MessageI18nService i18nService, - AppScopeMappingService appScopeMappingService) { + MessageI18nService i18nService) { this.taskExecuteService = taskExecuteService; this.i18nService = i18nService; - this.appScopeMappingService = appScopeMappingService; } @Override @@ -81,9 +79,10 @@ public EsbFastExecuteSQLV3ResourceImpl(TaskExecuteService taskExecuteService, ExecuteMetricsConstants.TAG_KEY_START_MODE, ExecuteMetricsConstants.TAG_VALUE_START_MODE_API, ExecuteMetricsConstants.TAG_KEY_TASK_TYPE, ExecuteMetricsConstants.TAG_VALUE_TASK_TYPE_FAST_SQL }) - public EsbResp fastExecuteSQL(EsbFastExecuteSQLV3Request request) { - request.fillAppResourceScope(appScopeMappingService); - + @AuditEntry + public EsbResp fastExecuteSQL(String username, + String appCode, + @AuditRequestBody EsbFastExecuteSQLV3Request request) { ValidateResult validateResult = checkFastExecuteSQLRequest(request); if (!validateResult.isPass()) { log.warn("Fast execute sql request is illegal!"); @@ -92,14 +91,18 @@ public EsbResp fastExecuteSQL(EsbFastExecuteSQLV3Request req request.trimIps(); - TaskInstanceDTO taskInstance = buildFastSQLTaskInstance(request); - StepInstanceDTO stepInstance = buildFastSQLStepInstance(request); - long taskInstanceId = taskExecuteService.executeFastTask( - FastTaskDTO.builder().taskInstance(taskInstance).stepInstance(stepInstance).build() + TaskInstanceDTO taskInstance = buildFastSQLTaskInstance(username, appCode, request); + StepInstanceDTO stepInstance = buildFastSQLStepInstance(username, request); + TaskInstanceDTO executeTaskInstance = taskExecuteService.executeFastTask( + FastTaskDTO.builder() + .taskInstance(taskInstance) + .stepInstance(stepInstance) + .startTask(request.getStartTask()) + .build() ); EsbJobExecuteV3DTO jobExecuteInfo = new EsbJobExecuteV3DTO(); - jobExecuteInfo.setTaskInstanceId(taskInstanceId); + jobExecuteInfo.setTaskInstanceId(executeTaskInstance.getId()); jobExecuteInfo.setTaskName(stepInstance.getName()); return EsbResp.buildSuccessResp(jobExecuteInfo); } @@ -138,7 +141,9 @@ private ValidateResult checkFastExecuteSQLRequest(EsbFastExecuteSQLV3Request req return ValidateResult.pass(); } - private TaskInstanceDTO buildFastSQLTaskInstance(EsbFastExecuteSQLV3Request request) { + private TaskInstanceDTO buildFastSQLTaskInstance(String username, + String appCode, + EsbFastExecuteSQLV3Request request) { TaskInstanceDTO taskInstance = new TaskInstanceDTO(); if (StringUtils.isNotBlank(request.getName())) { taskInstance.setName(request.getName()); @@ -147,8 +152,8 @@ private TaskInstanceDTO buildFastSQLTaskInstance(EsbFastExecuteSQLV3Request requ } taskInstance.setCronTaskId(-1L); taskInstance.setAppId(request.getAppId()); - taskInstance.setOperator(request.getUserName()); - taskInstance.setTaskId(-1L); + taskInstance.setOperator(username); + taskInstance.setPlanId(-1L); taskInstance.setTaskTemplateId(-1L); taskInstance.setDebugTask(false); taskInstance.setStatus(RunStatusEnum.BLANK); @@ -156,13 +161,13 @@ private TaskInstanceDTO buildFastSQLTaskInstance(EsbFastExecuteSQLV3Request requ taskInstance.setCurrentStepInstanceId(0L); taskInstance.setCreateTime(DateUtils.currentTimeMillis()); taskInstance.setType(TaskTypeEnum.SCRIPT.getValue()); - taskInstance.setAppCode(request.getAppCode()); + taskInstance.setAppCode(appCode); taskInstance.setCallbackUrl(request.getCallbackUrl()); return taskInstance; } - private StepInstanceDTO buildFastSQLStepInstance(EsbFastExecuteSQLV3Request request) { + private StepInstanceDTO buildFastSQLStepInstance(String username, EsbFastExecuteSQLV3Request request) { StepInstanceDTO stepInstance = new StepInstanceDTO(); stepInstance.setAppId(request.getAppId()); if (StringUtils.isNotBlank(request.getName())) { @@ -171,7 +176,7 @@ private StepInstanceDTO buildFastSQLStepInstance(EsbFastExecuteSQLV3Request requ stepInstance.setName(generateDefaultFastTaskName()); } stepInstance.setStepId(-1L); - stepInstance.setScriptType(ScriptTypeEnum.SQL.getValue()); + stepInstance.setScriptType(ScriptTypeEnum.SQL); if (request.getScriptVersionId() != null && request.getScriptVersionId() > 0) { stepInstance.setScriptVersionId(request.getScriptVersionId()); } else if (StringUtils.isNotBlank(request.getScriptId())) { @@ -182,12 +187,12 @@ private StepInstanceDTO buildFastSQLStepInstance(EsbFastExecuteSQLV3Request requ stepInstance.setTimeout(request.getTimeout() == null ? JobConstants.DEFAULT_JOB_TIMEOUT_SECONDS : request.getTimeout()); - stepInstance.setExecuteType(StepExecuteTypeEnum.EXECUTE_SQL.getValue()); + stepInstance.setExecuteType(StepExecuteTypeEnum.EXECUTE_SQL); stepInstance.setStatus(RunStatusEnum.BLANK); - stepInstance.setTargetServers(convertToServersDTO(request.getTargetServer())); + stepInstance.setTargetExecuteObjects(convertToServersDTO(request.getTargetServer())); stepInstance.setDbAccountId(request.getDbAccountId()); - stepInstance.setOperator(request.getUserName()); + stepInstance.setOperator(username); stepInstance.setCreateTime(DateUtils.currentTimeMillis()); return stepInstance; } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastExecuteScriptV3ResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastExecuteScriptV3ResourceImpl.java index 81ed235175..a8b260ddf3 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastExecuteScriptV3ResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastExecuteScriptV3ResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,8 @@ package com.tencent.bk.job.execute.api.esb.v3; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.audit.annotations.AuditRequestBody; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.constant.JobConstants; import com.tencent.bk.job.common.esb.metrics.EsbApiTimed; @@ -33,7 +35,6 @@ import com.tencent.bk.job.common.i18n.service.MessageI18nService; import com.tencent.bk.job.common.metrics.CommonMetricNames; import com.tencent.bk.job.common.model.ValidateResult; -import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.common.util.Base64Util; import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.common.web.metrics.CustomTimed; @@ -50,7 +51,7 @@ import com.tencent.bk.job.execute.model.esb.v3.EsbJobExecuteV3DTO; import com.tencent.bk.job.execute.model.esb.v3.request.EsbFastExecuteScriptV3Request; import com.tencent.bk.job.execute.service.TaskExecuteService; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -65,17 +66,14 @@ public class EsbFastExecuteScriptV3ResourceImpl extends JobExecuteCommonV3Proces private final TaskExecuteService taskExecuteService; private final TaskEvictPolicyExecutor taskEvictPolicyExecutor; private final MessageI18nService i18nService; - private final AppScopeMappingService appScopeMappingService; @Autowired public EsbFastExecuteScriptV3ResourceImpl(TaskExecuteService taskExecuteService, TaskEvictPolicyExecutor taskEvictPolicyExecutor, - MessageI18nService i18nService, - AppScopeMappingService appScopeMappingService) { + MessageI18nService i18nService) { this.taskExecuteService = taskExecuteService; this.taskEvictPolicyExecutor = taskEvictPolicyExecutor; this.i18nService = i18nService; - this.appScopeMappingService = appScopeMappingService; } @Override @@ -85,9 +83,11 @@ public EsbFastExecuteScriptV3ResourceImpl(TaskExecuteService taskExecuteService, ExecuteMetricsConstants.TAG_KEY_START_MODE, ExecuteMetricsConstants.TAG_VALUE_START_MODE_API, ExecuteMetricsConstants.TAG_KEY_TASK_TYPE, ExecuteMetricsConstants.TAG_VALUE_TASK_TYPE_FAST_SCRIPT }) - public EsbResp fastExecuteScript(EsbFastExecuteScriptV3Request request) + @AuditEntry + public EsbResp fastExecuteScript(String username, + String appCode, + @AuditRequestBody EsbFastExecuteScriptV3Request request) throws ServiceException { - request.fillAppResourceScope(appScopeMappingService); ValidateResult checkResult = checkFastExecuteScriptRequest(request); if (!checkResult.isPass()) { log.warn("Fast execute script request is illegal!"); @@ -96,25 +96,23 @@ public EsbResp fastExecuteScript(EsbFastExecuteScriptV3Reque request.trimIps(); - TaskInstanceDTO taskInstance = buildFastScriptTaskInstance(request); - if (taskEvictPolicyExecutor.shouldEvictTask(taskInstance)) { - return EsbResp.buildCommonFailResp(ErrorCode.TASK_ABANDONED); - } - StepInstanceDTO stepInstance = buildFastScriptStepInstance(request); + TaskInstanceDTO taskInstance = buildFastScriptTaskInstance(username, appCode, request); + StepInstanceDTO stepInstance = buildFastScriptStepInstance(username, request); StepRollingConfigDTO rollingConfig = null; if (request.getRollingConfig() != null) { rollingConfig = StepRollingConfigDTO.fromEsbRollingConfig(request.getRollingConfig()); } - long taskInstanceId = taskExecuteService.executeFastTask( + TaskInstanceDTO executeTaskInstance = taskExecuteService.executeFastTask( FastTaskDTO.builder() .taskInstance(taskInstance) .stepInstance(stepInstance) .rollingConfig(rollingConfig) + .startTask(request.getStartTask()) .build() ); EsbJobExecuteV3DTO jobExecuteInfo = new EsbJobExecuteV3DTO(); - jobExecuteInfo.setTaskInstanceId(taskInstanceId); + jobExecuteInfo.setTaskInstanceId(executeTaskInstance.getId()); jobExecuteInfo.setTaskName(stepInstance.getName()); jobExecuteInfo.setStepInstanceId(stepInstance.getId()); return EsbResp.buildSuccessResp(jobExecuteInfo); @@ -161,30 +159,32 @@ private ValidateResult checkFastExecuteScriptRequest(EsbFastExecuteScriptV3Reque return ValidateResult.pass(); } - private TaskInstanceDTO buildFastScriptTaskInstance(EsbFastExecuteScriptV3Request request) { + private TaskInstanceDTO buildFastScriptTaskInstance(String username, + String appCode, + EsbFastExecuteScriptV3Request request) { TaskInstanceDTO taskInstance = new TaskInstanceDTO(); if (StringUtils.isNotBlank(request.getName())) { taskInstance.setName(request.getName()); } else { taskInstance.setName(generateDefaultFastTaskName()); } - taskInstance.setTaskId(-1L); + taskInstance.setPlanId(-1L); taskInstance.setCronTaskId(-1L); taskInstance.setTaskTemplateId(-1L); taskInstance.setDebugTask(false); taskInstance.setAppId(request.getAppId()); taskInstance.setStartupMode(TaskStartupModeEnum.API.getValue()); taskInstance.setStatus(RunStatusEnum.BLANK); - taskInstance.setOperator(request.getUserName()); + taskInstance.setOperator(username); taskInstance.setCreateTime(DateUtils.currentTimeMillis()); taskInstance.setType(TaskTypeEnum.SCRIPT.getValue()); taskInstance.setCurrentStepInstanceId(0L); taskInstance.setCallbackUrl(request.getCallbackUrl()); - taskInstance.setAppCode(request.getAppCode()); + taskInstance.setAppCode(appCode); return taskInstance; } - private StepInstanceDTO buildFastScriptStepInstance(EsbFastExecuteScriptV3Request request) { + private StepInstanceDTO buildFastScriptStepInstance(String username, EsbFastExecuteScriptV3Request request) { StepInstanceDTO stepInstance = new StepInstanceDTO(); stepInstance.setAppId(request.getAppId()); if (StringUtils.isNotBlank(request.getName())) { @@ -200,7 +200,7 @@ private StepInstanceDTO buildFastScriptStepInstance(EsbFastExecuteScriptV3Reques stepInstance.setScriptId(request.getScriptId()); } else if (StringUtils.isNotBlank(request.getContent())) { stepInstance.setScriptContent(Base64Util.decodeContentToStr(request.getContent())); - stepInstance.setScriptType(request.getScriptLanguage()); + stepInstance.setScriptType(ScriptTypeEnum.valOf(request.getScriptLanguage())); } if (StringUtils.isNotEmpty(request.getScriptParam())) { @@ -211,15 +211,16 @@ private StepInstanceDTO buildFastScriptStepInstance(EsbFastExecuteScriptV3Reques } } stepInstance.setSecureParam(request.getIsParamSensitive() != null && request.getIsParamSensitive() == 1); + stepInstance.setWindowsInterpreter(request.getTrimmedWindowsInterpreter()); stepInstance.setTimeout( request.getTimeout() == null ? JobConstants.DEFAULT_JOB_TIMEOUT_SECONDS : request.getTimeout()); - stepInstance.setExecuteType(StepExecuteTypeEnum.EXECUTE_SCRIPT.getValue()); + stepInstance.setExecuteType(StepExecuteTypeEnum.EXECUTE_SCRIPT); stepInstance.setStatus(RunStatusEnum.BLANK); - stepInstance.setTargetServers(convertToServersDTO(request.getTargetServer())); + stepInstance.setTargetExecuteObjects(convertToServersDTO(request.getTargetServer())); stepInstance.setAccountId(request.getAccountId()); stepInstance.setAccountAlias(request.getAccountAlias()); - stepInstance.setOperator(request.getUserName()); + stepInstance.setOperator(username); stepInstance.setCreateTime(DateUtils.currentTimeMillis()); return stepInstance; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastTransferFileV3ResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastTransferFileV3ResourceImpl.java index d472a7b898..951ebb147f 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastTransferFileV3ResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFastTransferFileV3ResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,8 @@ package com.tencent.bk.job.execute.api.esb.v3; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.audit.annotations.AuditRequestBody; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.constant.JobConstants; import com.tencent.bk.job.common.constant.NotExistPathHandlerEnum; @@ -36,13 +38,14 @@ import com.tencent.bk.job.common.exception.NotFoundException; import com.tencent.bk.job.common.exception.ServiceException; import com.tencent.bk.job.common.i18n.service.MessageI18nService; +import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.metrics.CommonMetricNames; import com.tencent.bk.job.common.model.InternalResponse; import com.tencent.bk.job.common.model.ValidateResult; -import com.tencent.bk.job.common.service.AppScopeMappingService; +import com.tencent.bk.job.common.util.DataSizeConverter; +import com.tencent.bk.job.common.util.FilePathValidateUtil; import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.common.web.metrics.CustomTimed; -import com.tencent.bk.job.execute.client.FileSourceResourceClient; import com.tencent.bk.job.execute.common.constants.FileTransferModeEnum; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; @@ -59,17 +62,17 @@ import com.tencent.bk.job.execute.model.esb.v3.request.EsbFastTransferFileV3Request; import com.tencent.bk.job.execute.service.ArtifactoryLocalFileService; import com.tencent.bk.job.execute.service.TaskExecuteService; -import com.tencent.bk.job.manage.common.consts.task.TaskFileTypeEnum; +import com.tencent.bk.job.file_gateway.api.inner.ServiceFileSourceResource; +import com.tencent.bk.job.manage.api.common.constants.task.TaskFileTypeEnum; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.slf4j.helpers.MessageFormatter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RestController; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; @RestController @Slf4j @@ -78,26 +81,21 @@ public class EsbFastTransferFileV3ResourceImpl implements EsbFastTransferFileV3Resource { private final TaskExecuteService taskExecuteService; - private final FileSourceResourceClient fileSourceService; + private final ServiceFileSourceResource fileSourceResource; private final MessageI18nService i18nService; private final ArtifactoryLocalFileService artifactoryLocalFileService; - private final AppScopeMappingService appScopeMappingService; - - @Autowired public EsbFastTransferFileV3ResourceImpl(TaskExecuteService taskExecuteService, - FileSourceResourceClient fileSourceService, + ServiceFileSourceResource fileSourceResource, MessageI18nService i18nService, - ArtifactoryLocalFileService artifactoryLocalFileService, - AppScopeMappingService appScopeMappingService) { + ArtifactoryLocalFileService artifactoryLocalFileService) { this.taskExecuteService = taskExecuteService; - this.fileSourceService = fileSourceService; + this.fileSourceResource = fileSourceResource; this.i18nService = i18nService; this.artifactoryLocalFileService = artifactoryLocalFileService; - this.appScopeMappingService = appScopeMappingService; } @Override @@ -107,9 +105,10 @@ public EsbFastTransferFileV3ResourceImpl(TaskExecuteService taskExecuteService, ExecuteMetricsConstants.TAG_KEY_START_MODE, ExecuteMetricsConstants.TAG_VALUE_START_MODE_API, ExecuteMetricsConstants.TAG_KEY_TASK_TYPE, ExecuteMetricsConstants.TAG_VALUE_TASK_TYPE_FAST_FILE }) - public EsbResp fastTransferFile(EsbFastTransferFileV3Request request) { - request.fillAppResourceScope(appScopeMappingService); - + @AuditEntry(actionId = ActionId.QUICK_TRANSFER_FILE) + public EsbResp fastTransferFile(String username, + String appCode, + @AuditRequestBody EsbFastTransferFileV3Request request) { ValidateResult checkResult = checkFastTransferFileRequest(request); if (!checkResult.isPass()) { log.warn("Fast transfer file request is illegal!"); @@ -120,22 +119,23 @@ public EsbResp fastTransferFile(EsbFastTransferFileV3Request request.setName(generateDefaultFastTaskName()); } - TaskInstanceDTO taskInstance = buildFastFileTaskInstance(request); - StepInstanceDTO stepInstance = buildFastFileStepInstance(request); + TaskInstanceDTO taskInstance = buildFastFileTaskInstance(username, appCode, request); + StepInstanceDTO stepInstance = buildFastFileStepInstance(username, request); StepRollingConfigDTO rollingConfig = null; if (request.getRollingConfig() != null) { rollingConfig = StepRollingConfigDTO.fromEsbRollingConfig(request.getRollingConfig()); } - long taskInstanceId = taskExecuteService.executeFastTask( + TaskInstanceDTO executeTaskInstance = taskExecuteService.executeFastTask( FastTaskDTO.builder() .taskInstance(taskInstance) .stepInstance(stepInstance) .rollingConfig(rollingConfig) + .startTask(request.getStartTask()) .build() ); EsbJobExecuteV3DTO jobExecuteInfo = new EsbJobExecuteV3DTO(); - jobExecuteInfo.setTaskInstanceId(taskInstanceId); + jobExecuteInfo.setTaskInstanceId(executeTaskInstance.getId()); jobExecuteInfo.setStepInstanceId(stepInstance.getId()); jobExecuteInfo.setTaskName(stepInstance.getName()); return EsbResp.buildSuccessResp(jobExecuteInfo); @@ -147,7 +147,7 @@ private ValidateResult checkFileSource(EsbFileSourceV3DTO fileSource) { if (fileType != null && !TaskFileTypeEnum.isValid(fileType)) { return ValidateResult.fail(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME, "file_source.file_type"); } - List files = fileSource.getFiles(); + List files = fileSource.getTrimmedFiles(); if (files == null || files.isEmpty()) { log.warn("File source contains empty file list"); return ValidateResult.fail(ErrorCode.MISSING_PARAM_WITH_PARAM_NAME, "file_source.file_list"); @@ -155,7 +155,7 @@ private ValidateResult checkFileSource(EsbFileSourceV3DTO fileSource) { for (String file : files) { if ((fileType == null || TaskFileTypeEnum.SERVER.getType() == fileType) - && !validateFileSystemPath(file)) { + && !FilePathValidateUtil.validateFileSystemAbsolutePath(file)) { log.warn("Invalid path:{}", file); return ValidateResult.fail(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME, "file_source.file_list"); } @@ -189,8 +189,8 @@ private ValidateResult checkFileSource(EsbFileSourceV3DTO fileSource) { } private ValidateResult checkFastTransferFileRequest(EsbFastTransferFileV3Request request) { - if (!validateFileSystemPath(request.getTargetPath())) { - log.warn("Fast transfer file, target path is invalid!path={}", request.getTargetPath()); + if (!FilePathValidateUtil.validateFileSystemAbsolutePath(request.getTrimmedTargetPath())) { + log.warn("Fast transfer file, target path is invalid!path={}", request.getTrimmedTargetPath()); return ValidateResult.fail(ErrorCode.MISSING_OR_ILLEGAL_PARAM_WITH_PARAM_NAME, "file_target_path"); } if ((request.getAccountId() == null || request.getAccountId() <= 0L) @@ -216,48 +216,24 @@ private ValidateResult checkFastTransferFileRequest(EsbFastTransferFileV3Request return ValidateResult.pass(); } - private boolean validateFileSystemPath(String path) { - if (StringUtils.isBlank(path)) { - return false; - } - if (path.indexOf(' ') != -1) { - return false; - } - Pattern p1 = Pattern.compile("(//|\\\\)+"); - Matcher m1 = p1.matcher(path); - if (m1.matches()) { - return false; - } - - Pattern p2 = Pattern.compile("^[a-zA-Z]:(/|\\\\).*");//windows - Matcher m2 = p2.matcher(path); - - if (!m2.matches()) { //非windows - if (path.charAt(0) == '/') { - return !path.contains("\\\\"); - } else { - return false; - } - } - return true; - } - - private TaskInstanceDTO buildFastFileTaskInstance(EsbFastTransferFileV3Request request) { + private TaskInstanceDTO buildFastFileTaskInstance(String username, + String appCode, + EsbFastTransferFileV3Request request) { TaskInstanceDTO taskInstance = new TaskInstanceDTO(); taskInstance.setType(TaskTypeEnum.FILE.getValue()); taskInstance.setName(request.getName()); - taskInstance.setTaskId(-1L); + taskInstance.setPlanId(-1L); taskInstance.setCronTaskId(-1L); taskInstance.setTaskTemplateId(-1L); taskInstance.setAppId(request.getAppId()); taskInstance.setStatus(RunStatusEnum.BLANK); taskInstance.setStartupMode(TaskStartupModeEnum.API.getValue()); - taskInstance.setOperator(request.getUserName()); + taskInstance.setOperator(username); taskInstance.setCreateTime(DateUtils.currentTimeMillis()); taskInstance.setCurrentStepInstanceId(0L); taskInstance.setDebugTask(false); taskInstance.setCallbackUrl(request.getCallbackUrl()); - taskInstance.setAppCode(request.getAppCode()); + taskInstance.setAppCode(appCode); return taskInstance; } @@ -266,28 +242,29 @@ private String generateDefaultFastTaskName() { + DateUtils.formatLocalDateTime(LocalDateTime.now(), "yyyyMMddHHmmssSSS"); } - private StepInstanceDTO buildFastFileStepInstance(EsbFastTransferFileV3Request request) { + private StepInstanceDTO buildFastFileStepInstance(String username, + EsbFastTransferFileV3Request request) { StepInstanceDTO stepInstance = new StepInstanceDTO(); stepInstance.setName(request.getName()); stepInstance.setAccountId(request.getAccountId()); stepInstance.setAccountAlias(request.getAccountAlias()); stepInstance.setStepId(-1L); - stepInstance.setExecuteType(StepExecuteTypeEnum.SEND_FILE.getValue()); - stepInstance.setFileTargetPath(request.getTargetPath()); + stepInstance.setExecuteType(StepExecuteTypeEnum.SEND_FILE); + stepInstance.setFileTargetPath(request.getTrimmedTargetPath()); stepInstance.setFileTargetName(request.getTargetName()); - stepInstance.setFileSourceList(convertFileSource(request.getFileSources())); + stepInstance.setFileSourceList(convertFileSource(request.getAppId(), request.getFileSources())); stepInstance.setAppId(request.getAppId()); - stepInstance.setTargetServers(convertToServersDTO(request.getTargetServer())); - stepInstance.setOperator(request.getUserName()); + stepInstance.setTargetExecuteObjects(convertToServersDTO(request.getTargetServer())); + stepInstance.setOperator(username); stepInstance.setStatus(RunStatusEnum.BLANK); stepInstance.setCreateTime(DateUtils.currentTimeMillis()); stepInstance.setTimeout(request.getTimeout() == null ? JobConstants.DEFAULT_JOB_TIMEOUT_SECONDS : request.getTimeout()); if (request.getUploadSpeedLimit() != null && request.getUploadSpeedLimit() > 0) { - stepInstance.setFileUploadSpeedLimit(request.getUploadSpeedLimit() << 10); + stepInstance.setFileUploadSpeedLimit(DataSizeConverter.convertMBToKB(request.getUploadSpeedLimit())); } if (request.getDownloadSpeedLimit() != null && request.getDownloadSpeedLimit() > 0) { - stepInstance.setFileDownloadSpeedLimit(request.getDownloadSpeedLimit() << 10); + stepInstance.setFileDownloadSpeedLimit(DataSizeConverter.convertMBToKB(request.getDownloadSpeedLimit())); } FileTransferModeEnum transferMode = FileTransferModeEnum.getFileTransferModeEnum(request.getTransferMode()); if (transferMode == null) { @@ -301,7 +278,7 @@ private StepInstanceDTO buildFastFileStepInstance(EsbFastTransferFileV3Request r return stepInstance; } - private List convertFileSource(List fileSources) throws ServiceException { + private List convertFileSource(Long appId, List fileSources) throws ServiceException { if (fileSources == null) { return null; } @@ -324,8 +301,8 @@ private List convertFileSource(List fileSourc } fileSourceDTO.setFileType(fileType); List files = new ArrayList<>(); - if (fileSource.getFiles() != null) { - for (String file : fileSource.getFiles()) { + if (fileSource.getTrimmedFiles() != null) { + for (String file : fileSource.getTrimmedFiles()) { FileDetailDTO fileDetailDTO; if (fileType == TaskFileTypeEnum.LOCAL.getType()) { // 从制品库获取本地文件信息 @@ -342,7 +319,7 @@ private List convertFileSource(List fileSourc fileSourceDTO.setFileSourceId(fileSource.getFileSourceId()); } else if (StringUtils.isNotBlank(fileSourceCode)) { try { - InternalResponse resp = fileSourceService.getFileSourceIdByCode(fileSourceCode); + InternalResponse resp = fileSourceResource.getFileSourceIdByCode(appId, fileSourceCode); if (resp != null && resp.isSuccess()) { if (resp.getData() != null) { fileSourceDTO.setFileSourceId(resp.getData()); @@ -356,7 +333,11 @@ private List convertFileSource(List fileSourc throw new NotFoundException(ErrorCode.FILE_SOURCE_SERVICE_INVALID); } } catch (Exception e) { - log.error("Fail to parse fileSourceCode to id:{}", fileSourceCode, e); + String msg = MessageFormatter.format( + "Fail to parse fileSourceCode to id:{}", + fileSourceCode + ).getMessage(); + log.error(msg, e); throw new InternalException(ErrorCode.INTERNAL_ERROR); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFileTransferProcessor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFileTransferProcessor.java new file mode 100644 index 0000000000..1dae072a77 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbFileTransferProcessor.java @@ -0,0 +1,165 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.api.esb.v3; + +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.esb.model.job.v3.EsbAccountV3BasicDTO; +import com.tencent.bk.job.common.esb.model.job.v3.EsbFileSourceV3DTO; +import com.tencent.bk.job.common.esb.model.job.v3.EsbServerV3DTO; +import com.tencent.bk.job.common.model.ValidateResult; +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.common.util.FilePathValidateUtil; +import com.tencent.bk.job.execute.model.DynamicServerGroupDTO; +import com.tencent.bk.job.execute.model.DynamicServerTopoNodeDTO; +import com.tencent.bk.job.execute.model.ExecuteTargetDTO; +import com.tencent.bk.job.manage.api.common.constants.task.TaskFileTypeEnum; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; + +import java.util.ArrayList; +import java.util.List; +import java.util.stream.Collectors; + +@Slf4j +public class EsbFileTransferProcessor extends EsbCommonExecuteTaskProcessor { + + protected ValidateResult validateFilePath(String filePath) { + if (!FilePathValidateUtil.validateFileSystemAbsolutePath(filePath)) { + log.warn("Fast transfer file, target path is invalid!path={}", filePath); + return ValidateResult.fail(ErrorCode.MISSING_OR_ILLEGAL_PARAM_WITH_PARAM_NAME, "file_target_path"); + } + return ValidateResult.pass(); + } + + protected ValidateResult validateFileSources(List fileSources) { + if (fileSources == null || fileSources.isEmpty()) { + log.warn("Fast transfer file, file source list is null or empty!"); + return ValidateResult.fail(ErrorCode.MISSING_PARAM_WITH_PARAM_NAME, "file_source_list"); + } + for (EsbFileSourceV3DTO fileSource : fileSources) { + ValidateResult result = validateSingleFileSource(fileSource); + if (result != null) { + return result; + } + } + return ValidateResult.pass(); + } + + protected ValidateResult validateSingleFileSource(EsbFileSourceV3DTO fileSource) { + Integer fileType = fileSource.getFileType(); + // fileType是后加的字段，为null则默认为服务器文件不校验 + if (fileType != null && !TaskFileTypeEnum.isValid(fileType)) { + return ValidateResult.fail(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME, "file_source.file_type"); + } + List files = fileSource.getTrimmedFiles(); + if (files == null || files.isEmpty()) { + log.warn("File source contains empty file list"); + return ValidateResult.fail(ErrorCode.MISSING_PARAM_WITH_PARAM_NAME, "file_source.file_list"); + } + for (String file : files) { + if ((fileType == null + || TaskFileTypeEnum.SERVER.getType() == fileType) + && !FilePathValidateUtil.validateFileSystemAbsolutePath(file)) { + log.warn("Invalid path:{}", file); + return ValidateResult.fail(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME, "file_source.file_list"); + } + } + if (null == fileType || TaskFileTypeEnum.SERVER.getType() == fileType) { + //对文件源类型为服务器文件的文件源校验账号和服务器信息 + EsbAccountV3BasicDTO account = fileSource.getAccount(); + if (account == null) { + log.warn("File source account is null!"); + return ValidateResult.fail(ErrorCode.MISSING_PARAM_WITH_PARAM_NAME, "file_source.account"); + } + if ((account.getId() == null || account.getId() < 1L) && StringUtils.isBlank(account.getAlias())) { + log.warn("File source account is empty!"); + return ValidateResult.fail(ErrorCode.MISSING_PARAM_WITH_PARAM_NAME, + "file_source.account.account_id|file_source.account.account_alias"); + } + if (!validFileSourceServer(fileSource.getServer()).isPass()) { + log.warn("File source server is empty!"); + return ValidateResult.fail(ErrorCode.MISSING_PARAM_WITH_PARAM_NAME, "file_source.server"); + } + } else if (TaskFileTypeEnum.FILE_SOURCE.getType() == fileType) { + //对文件源类型为第三方文件源的文件源校验Id与Code + Integer fileSourceId = fileSource.getFileSourceId(); + String fileSourceCode = fileSource.getFileSourceCode(); + if ((fileSourceId == null || fileSourceId <= 0) && StringUtils.isBlank(fileSourceCode)) { + return ValidateResult.fail(ErrorCode.MISSING_PARAM_WITH_PARAM_NAME, + "file_source.file_source_id/file_source.file_source_code"); + } + } + return null; + } + + protected ValidateResult validFileSourceServer(EsbServerV3DTO server) { + if (server == null) { + return ValidateResult.fail(ErrorCode.MISSING_PARAM_WITH_PARAM_NAME, "target_server"); + } + + if (!server.checkHostParamsNonEmpty()) { + return ValidateResult.fail(ErrorCode.SERVER_EMPTY); + } + return ValidateResult.pass(); + } + + protected ExecuteTargetDTO buildFileSourceServer(EsbServerV3DTO server) { + if (server == null) { + return null; + } + ExecuteTargetDTO executeTargetDTO = new ExecuteTargetDTO(); + + // 拓扑节点 + if (CollectionUtils.isNotEmpty(server.getTopoNodes())) { + List topoNodes = new ArrayList<>(); + server.getTopoNodes().forEach(topoNode -> topoNodes.add(new DynamicServerTopoNodeDTO(topoNode.getId(), + topoNode.getNodeType()))); + executeTargetDTO.setTopoNodes(topoNodes); + } + + // 动态分组 + if (CollectionUtils.isNotEmpty(server.getDynamicGroups())) { + List dynamicServerGroups = new ArrayList<>(); + server.getDynamicGroups().forEach( + group -> dynamicServerGroups.add(new DynamicServerGroupDTO(group.getId()))); + executeTargetDTO.setDynamicServerGroups(dynamicServerGroups); + } + + if (CollectionUtils.isNotEmpty(server.getHostIds())) { + executeTargetDTO.setStaticIpList( + server.getHostIds().stream().map(HostDTO::fromHostId).collect(Collectors.toList())); + } else if (CollectionUtils.isNotEmpty(server.getIps())) { + executeTargetDTO.setStaticIpList( + server.getIps().stream() + .map(host -> new HostDTO(host.getBkCloudId(), host.getIp())) + .collect(Collectors.toList())); + } + + return executeTargetDTO; + + } + +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceGlobalVarValueV3ResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceGlobalVarValueV3ResourceImpl.java index 9966910505..886797bdb4 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceGlobalVarValueV3ResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceGlobalVarValueV3ResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,23 +24,24 @@ package com.tencent.bk.job.execute.api.esb.v3; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.audit.annotations.AuditRequestBody; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.constant.TaskVariableTypeEnum; import com.tencent.bk.job.common.esb.metrics.EsbApiTimed; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.common.exception.InvalidParamException; +import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.metrics.CommonMetricNames; import com.tencent.bk.job.common.model.ValidateResult; import com.tencent.bk.job.common.service.AppScopeMappingService; -import com.tencent.bk.job.execute.api.esb.v2.impl.JobQueryCommonProcessor; import com.tencent.bk.job.execute.model.StepInstanceVariableValuesDTO; -import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.model.esb.v3.EsbJobInstanceGlobalVarValueV3DTO; import com.tencent.bk.job.execute.model.esb.v3.EsbJobInstanceGlobalVarValueV3DTO.EsbStepInstanceGlobalVarValuesV3DTO; import com.tencent.bk.job.execute.model.esb.v3.EsbJobInstanceGlobalVarValueV3DTO.GlobalVarValueV3DTO; import com.tencent.bk.job.execute.model.esb.v3.request.EsbGetJobInstanceGlobalVarValueV3Request; import com.tencent.bk.job.execute.service.StepInstanceVariableValueService; -import com.tencent.bk.job.execute.service.TaskInstanceService; +import com.tencent.bk.job.execute.service.TaskInstanceAccessProcessor; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.springframework.web.bind.annotation.RestController; @@ -51,28 +52,29 @@ @RestController @Slf4j public class EsbGetJobInstanceGlobalVarValueV3ResourceImpl - extends JobQueryCommonProcessor implements EsbGetJobInstanceGlobalVarValueV3Resource { - private final TaskInstanceService taskInstanceService; private final StepInstanceVariableValueService stepInstanceVariableValueService; + private final TaskInstanceAccessProcessor taskInstanceAccessProcessor; private final AppScopeMappingService appScopeMappingService; - public EsbGetJobInstanceGlobalVarValueV3ResourceImpl(TaskInstanceService taskInstanceService, - StepInstanceVariableValueService stepInstanceVariableValueService, - AppScopeMappingService appScopeMappingService) { - this.taskInstanceService = taskInstanceService; + + public EsbGetJobInstanceGlobalVarValueV3ResourceImpl( + StepInstanceVariableValueService stepInstanceVariableValueService, + TaskInstanceAccessProcessor taskInstanceAccessProcessor, + AppScopeMappingService appScopeMappingService) { this.stepInstanceVariableValueService = stepInstanceVariableValueService; + this.taskInstanceAccessProcessor = taskInstanceAccessProcessor; this.appScopeMappingService = appScopeMappingService; } @Override @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_get_job_instance_var_value"}) + @AuditEntry(actionId = ActionId.VIEW_HISTORY) public EsbResp getJobInstanceGlobalVarValueUsingPost( - EsbGetJobInstanceGlobalVarValueV3Request request) { - - request.fillAppResourceScope(appScopeMappingService); - + String username, + String appCode, + @AuditRequestBody EsbGetJobInstanceGlobalVarValueV3Request request) { ValidateResult checkResult = checkRequest(request); if (!checkResult.isPass()) { log.warn("Get job instance global var value, request is illegal!"); @@ -80,8 +82,8 @@ public EsbResp getJobInstanceGlobalVarValueUs } long taskInstanceId = request.getTaskInstanceId(); - TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(taskInstanceId); - authViewTaskInstance(request.getUserName(), request.getAppResourceScope(), taskInstance); + taskInstanceAccessProcessor.processBeforeAccess(username, + request.getAppResourceScope().getAppId(), taskInstanceId); EsbJobInstanceGlobalVarValueV3DTO result = new EsbJobInstanceGlobalVarValueV3DTO(); result.setTaskInstanceId(taskInstanceId); @@ -129,12 +131,11 @@ public EsbResp getJobInstanceGlobalVarValue(S String scopeId, Long taskInstanceId) { EsbGetJobInstanceGlobalVarValueV3Request request = new EsbGetJobInstanceGlobalVarValueV3Request(); - request.setUserName(username); - request.setAppCode(appCode); request.setBizId(bizId); request.setScopeType(scopeType); request.setScopeId(scopeId); request.setTaskInstanceId(taskInstanceId); - return getJobInstanceGlobalVarValueUsingPost(request); + request.fillAppResourceScope(appScopeMappingService); + return getJobInstanceGlobalVarValueUsingPost(username, appCode, request); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceIpLogV3ResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceIpLogV3ResourceImpl.java index be5e9973b7..760287f12d 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceIpLogV3ResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceIpLogV3ResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,30 +24,33 @@ package com.tencent.bk.job.execute.api.esb.v3; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.audit.annotations.AuditRequestBody; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.esb.metrics.EsbApiTimed; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.common.esb.model.job.EsbIpDTO; import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.exception.NotFoundException; +import com.tencent.bk.job.common.gse.constants.FileDistModeEnum; +import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.metrics.CommonMetricNames; import com.tencent.bk.job.common.model.ValidateResult; -import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.service.AppScopeMappingService; -import com.tencent.bk.job.common.util.ip.IpUtils; -import com.tencent.bk.job.execute.api.esb.v2.impl.JobQueryCommonProcessor; -import com.tencent.bk.job.execute.common.constants.FileDistModeEnum; -import com.tencent.bk.job.execute.model.FileIpLogContent; -import com.tencent.bk.job.execute.model.ScriptHostLogContent; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; +import com.tencent.bk.job.execute.model.AtomicFileTaskLog; +import com.tencent.bk.job.execute.model.ExecuteObjectCompositeKey; +import com.tencent.bk.job.execute.model.FileExecuteObjectLogContent; +import com.tencent.bk.job.execute.model.ScriptExecuteObjectLogContent; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; -import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.model.esb.v3.EsbFileLogV3DTO; import com.tencent.bk.job.execute.model.esb.v3.EsbIpLogV3DTO; import com.tencent.bk.job.execute.model.esb.v3.request.EsbGetJobInstanceIpLogV3Request; import com.tencent.bk.job.execute.service.LogService; -import com.tencent.bk.job.execute.service.TaskInstanceService; +import com.tencent.bk.job.execute.service.StepInstanceService; +import com.tencent.bk.job.execute.service.TaskInstanceAccessProcessor; +import com.tencent.bk.job.execute.util.ExecuteObjectCompositeKeyUtils; import com.tencent.bk.job.logsvr.consts.LogTypeEnum; -import com.tencent.bk.job.logsvr.model.service.ServiceFileTaskLogDTO; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -59,26 +62,30 @@ @RestController @Slf4j -public class EsbGetJobInstanceIpLogV3ResourceImpl extends JobQueryCommonProcessor - implements EsbGetJobInstanceIpLogV3Resource { +public class EsbGetJobInstanceIpLogV3ResourceImpl implements EsbGetJobInstanceIpLogV3Resource { - private final TaskInstanceService taskInstanceService; + private final StepInstanceService stepInstanceService; private final LogService logService; + private final TaskInstanceAccessProcessor taskInstanceAccessProcessor; private final AppScopeMappingService appScopeMappingService; public EsbGetJobInstanceIpLogV3ResourceImpl(LogService logService, - TaskInstanceService taskInstanceService, + StepInstanceService stepInstanceService, + TaskInstanceAccessProcessor taskInstanceAccessProcessor, AppScopeMappingService appScopeMappingService) { this.logService = logService; - this.taskInstanceService = taskInstanceService; + this.stepInstanceService = stepInstanceService; + this.taskInstanceAccessProcessor = taskInstanceAccessProcessor; this.appScopeMappingService = appScopeMappingService; } @Override @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_get_job_instance_ip_log"}) - public EsbResp getJobInstanceIpLogUsingPost(EsbGetJobInstanceIpLogV3Request request) { - request.fillAppResourceScope(appScopeMappingService); - + @AuditEntry(actionId = ActionId.VIEW_HISTORY) + public EsbResp getJobInstanceIpLogUsingPost( + String username, + String appCode, + @AuditRequestBody EsbGetJobInstanceIpLogV3Request request) { ValidateResult checkResult = checkRequest(request); if (!checkResult.isPass()) { log.warn("Get job instance ip log request is illegal!"); @@ -86,14 +93,11 @@ public EsbResp getJobInstanceIpLogUsingPost(EsbGetJobInstanceIpLo } long taskInstanceId = request.getTaskInstanceId(); - TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(taskInstanceId); - if (taskInstance == null) { - throw new NotFoundException(ErrorCode.TASK_INSTANCE_NOT_EXIST); - } - - authViewTaskInstance(request.getUserName(), request.getAppResourceScope(), taskInstance); + taskInstanceAccessProcessor.processBeforeAccess(username, + request.getAppResourceScope().getAppId(), taskInstanceId); - StepInstanceBaseDTO stepInstance = taskInstanceService.getBaseStepInstance(request.getStepInstanceId()); + StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstance( + request.getTaskInstanceId(), request.getStepInstanceId()); if (stepInstance == null) { throw new NotFoundException(ErrorCode.TASK_INSTANCE_NOT_EXIST); } @@ -101,12 +105,14 @@ public EsbResp getJobInstanceIpLogUsingPost(EsbGetJobInstanceIpLo EsbIpLogV3DTO ipLog = new EsbIpLogV3DTO(); ipLog.setCloudAreaId(request.getCloudAreaId()); ipLog.setIp(request.getIp()); + ipLog.setHostId(request.getHostId()); + ExecuteObjectCompositeKey hostKey = + ExecuteObjectCompositeKeyUtils.fromHostParam( + request.getHostId(), request.getCloudAreaId(), request.getIp()); if (stepInstance.isScriptStep()) { - buildScriptLog(ipLog, request.getStepInstanceId(), stepInstance.getExecuteCount(), request.getCloudAreaId(), - request.getIp()); + buildScriptLog(ipLog, stepInstance, stepInstance.getExecuteCount(), hostKey); } else if (stepInstance.isFileStep()) { - buildFileLog(ipLog, request.getStepInstanceId(), stepInstance.getExecuteCount(), request.getCloudAreaId(), - request.getIp()); + buildFileLog(ipLog, stepInstance, stepInstance.getExecuteCount(), hostKey); } return EsbResp.buildSuccessResp(ipLog); } @@ -120,80 +126,110 @@ private ValidateResult checkRequest(EsbGetJobInstanceIpLogV3Request request) { log.warn("StepInstanceId is empty or illegal, stepInstanceId={}", request.getStepInstanceId()); return ValidateResult.fail(ErrorCode.MISSING_OR_ILLEGAL_PARAM_WITH_PARAM_NAME, "step_instance_id"); } - if (request.getCloudAreaId() == null || request.getCloudAreaId() < 0) { - log.warn("CloudAreaId is empty or illegal, cloudAreaId={}", request.getCloudAreaId()); - return ValidateResult.fail(ErrorCode.MISSING_OR_ILLEGAL_PARAM_WITH_PARAM_NAME, "bk_cloud_id"); - } - if (StringUtils.isBlank(request.getIp())) { - log.warn("Ip is empty"); - return ValidateResult.fail(ErrorCode.MISSING_PARAM_WITH_PARAM_NAME, "ip"); - } - if (!IpUtils.checkIp(request.getIp())) { - log.warn("Ip is illegal, ip={}", request.getIp()); - return ValidateResult.fail(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME, "ip"); - } return ValidateResult.pass(); } - private void buildScriptLog(EsbIpLogV3DTO ipLog, Long stepInstanceId, Integer executeCount, - Long cloudAreaId, String ip) { + private void buildScriptLog(EsbIpLogV3DTO ipLog, + StepInstanceBaseDTO stepInstance, + Integer executeCount, + ExecuteObjectCompositeKey hostKey) { ipLog.setLogType(LogTypeEnum.SCRIPT.getValue()); - ScriptHostLogContent logContent = logService.getScriptHostLogContent(stepInstanceId, - executeCount, null, new HostDTO(cloudAreaId, ip)); + ScriptExecuteObjectLogContent logContent = logService.getScriptExecuteObjectLogContent(stepInstance, + executeCount, null, hostKey); if (logContent != null && StringUtils.isNotBlank(logContent.getContent())) { ipLog.setScriptLogContent(logContent.getContent()); + if (ipLog.getHostId() == null && logContent.getExecuteObject() != null) { + ipLog.setHostId(logContent.getExecuteObject().getHost().getHostId()); + } } } - private void buildFileLog(EsbIpLogV3DTO ipLog, Long stepInstanceId, Integer executeCount, - Long cloudAreaId, String ip) { + private void buildFileLog(EsbIpLogV3DTO ipLog, + StepInstanceBaseDTO stepInstance, + Integer executeCount, + ExecuteObjectCompositeKey hostKey) { ipLog.setLogType(LogTypeEnum.FILE.getValue()); - FileIpLogContent downloadIpLog = logService.getFileIpLogContent(stepInstanceId, executeCount, null, - new HostDTO(cloudAreaId, ip), FileDistModeEnum.DOWNLOAD.getValue()); - List uploadTaskLogs = logService.batchGetFileSourceIpLogContent( - stepInstanceId, executeCount, null); + FileExecuteObjectLogContent downloadIpLog = logService.getFileExecuteObjectLogContent(stepInstance, + executeCount, null, hostKey, FileDistModeEnum.DOWNLOAD.getValue()); + List uploadExecuteObjectLogs = + logService.batchGetFileSourceExecuteObjectLogContent( + stepInstance.getTaskInstanceId(), stepInstance.getId(), executeCount, null); List fileLogs = new ArrayList<>(); fileLogs.addAll(buildDownloadFileLogs(downloadIpLog)); - fileLogs.addAll(buildUploadFileLogs(uploadTaskLogs)); + fileLogs.addAll(buildUploadFileLogs(uploadExecuteObjectLogs)); + // 根据 bk_cloud_id + ip 条件查询，需要补全bk_host_id + if (ipLog.getHostId() == null && StringUtils.isNotEmpty(ipLog.getIp()) && ipLog.getCloudAreaId() != null) { + for (EsbFileLogV3DTO fileLog : fileLogs) { + if (fileLog.getSrcIp() != null) { + if (ipLog.getIp().equals(fileLog.getSrcIp().getIp()) && + ipLog.getCloudAreaId().equals(fileLog.getSrcIp().getBkCloudId())) { + ipLog.setHostId(fileLog.getSrcIp().getHostId()); + break; + } + } + if (fileLog.getDestIp() != null) { + if (ipLog.getIp().equals(fileLog.getDestIp().getIp()) && + ipLog.getCloudAreaId().equals(fileLog.getDestIp().getBkCloudId())) { + ipLog.setHostId(fileLog.getDestIp().getHostId()); + break; + } + } + } + } ipLog.setFileLogs(fileLogs); } - private List buildDownloadFileLogs(FileIpLogContent downloadIpLog) { + private List buildDownloadFileLogs(FileExecuteObjectLogContent executeObjectLogContent) { List downloadFileLogs = new ArrayList<>(); - if (downloadIpLog != null && CollectionUtils.isNotEmpty(downloadIpLog.getFileTaskLogs())) { - downloadFileLogs = downloadIpLog.getFileTaskLogs().stream().map(this::toEsbFileLogV3DTO) + if (executeObjectLogContent != null && CollectionUtils.isNotEmpty(executeObjectLogContent.getFileTaskLogs())) { + downloadFileLogs = executeObjectLogContent.getFileTaskLogs().stream().map(this::toEsbFileLogV3DTO) .collect(Collectors.toList()); } return downloadFileLogs; } - private List buildUploadFileLogs(List uploadTaskLogs) { + private List buildUploadFileLogs(List uploadExecuteObjectTaskLogs) { List uploadFileLogs = new ArrayList<>(); - if (CollectionUtils.isNotEmpty(uploadTaskLogs)) { - uploadFileLogs = uploadTaskLogs.stream().map(this::toEsbFileLogV3DTO).collect(Collectors.toList()); + if (CollectionUtils.isNotEmpty(uploadExecuteObjectTaskLogs)) { + uploadFileLogs = uploadExecuteObjectTaskLogs.stream() + .flatMap(executeObjectLogContent -> executeObjectLogContent.getFileTaskLogs().stream()) + .map(this::toEsbFileLogV3DTO) + .collect(Collectors.toList()); } return uploadFileLogs; } - private EsbFileLogV3DTO toEsbFileLogV3DTO(ServiceFileTaskLogDTO fileTaskLog) { + private EsbFileLogV3DTO toEsbFileLogV3DTO(AtomicFileTaskLog fileTaskLog) { EsbFileLogV3DTO fileLog = new EsbFileLogV3DTO(); fileLog.setMode(fileTaskLog.getMode()); - if (StringUtils.isNotBlank(fileTaskLog.getDisplaySrcIp())) { - EsbIpDTO srcIp = EsbIpDTO.fromCloudIp(fileTaskLog.getDisplaySrcIp()); - if (srcIp != null) { - fileLog.setSrcIp(srcIp); - } + + // source + EsbIpDTO srcHost = new EsbIpDTO(); + ExecuteObject srcExecuteObject = fileTaskLog.getSrcExecuteObject(); + srcHost.setHostId(srcExecuteObject.getHost().getHostId()); + EsbIpDTO srcIp = EsbIpDTO.fromCloudIp(srcExecuteObject.getHost().getIp()); + if (srcIp != null) { + srcHost.setBkCloudId(srcIp.getBkCloudId()); + srcHost.setIp(srcIp.getIp()); } + fileLog.setSrcIp(srcHost); fileLog.setSrcPath(fileTaskLog.getDisplaySrcFile()); + + // dest if (FileDistModeEnum.DOWNLOAD.getValue().equals(fileTaskLog.getMode())) { - if (StringUtils.isNotBlank(fileTaskLog.getDestIp())) { - EsbIpDTO destIp = EsbIpDTO.fromCloudIp(fileTaskLog.getDestIp()); + EsbIpDTO destHost = new EsbIpDTO(); + ExecuteObject destExecuteObject = fileTaskLog.getDestExecuteObject(); + destHost.setHostId(destExecuteObject.getHost().getHostId()); + if (StringUtils.isNotBlank(destExecuteObject.getHost().getIp())) { + EsbIpDTO destIp = EsbIpDTO.fromCloudIp(destExecuteObject.getHost().getIp()); if (destIp != null) { - fileLog.setDestIp(destIp); + destHost.setBkCloudId(destIp.getBkCloudId()); + destHost.setIp(destIp.getIp()); } } + fileLog.setDestIp(destHost); fileLog.setDestPath(fileTaskLog.getDestFile()); } @@ -210,18 +246,19 @@ public EsbResp getJobInstanceIpLog(String username, String scopeId, Long taskInstanceId, Long stepInstanceId, + Long hostId, Long cloudAreaId, String ip) { EsbGetJobInstanceIpLogV3Request request = new EsbGetJobInstanceIpLogV3Request(); - request.setUserName(username); - request.setAppCode(appCode); request.setBizId(bizId); request.setScopeType(scopeType); request.setScopeId(scopeId); request.setTaskInstanceId(taskInstanceId); request.setStepInstanceId(stepInstanceId); + request.setHostId(hostId); request.setCloudAreaId(cloudAreaId); request.setIp(ip); - return getJobInstanceIpLogUsingPost(request); + request.fillAppResourceScope(appScopeMappingService); + return getJobInstanceIpLogUsingPost(username, appCode, request); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceListV3ResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceListV3ResourceImpl.java index 6ecdb3c18c..88720f46ea 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceListV3ResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceListV3ResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -60,7 +60,7 @@ public class EsbGetJobInstanceListV3ResourceImpl implements EsbGetJobInstanceLis private final AppScopeMappingService appScopeMappingService; public EsbGetJobInstanceListV3ResourceImpl(TaskResultService taskResultService, - AppScopeMappingService appScopeMappingService) { + AppScopeMappingService appScopeMappingService) { this.taskResultService = taskResultService; this.appScopeMappingService = appScopeMappingService; } @@ -68,10 +68,9 @@ public EsbGetJobInstanceListV3ResourceImpl(TaskResultService taskResultService, @Override @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_get_job_instance_list"}) public EsbResp> getJobInstanceListUsingPost( + String username, + String appCode, EsbGetJobInstanceListV3Request request) { - - request.fillAppResourceScope(appScopeMappingService); - ValidateResult checkResult = checkRequest(request); if (!checkResult.isPass()) { log.warn("Get job instance ip log request is illegal!"); @@ -124,7 +123,7 @@ private EsbPageDataV3 convertToEsbTaskInstancePageData(Pag taskInstance.setEndTime(taskInstanceDTO.getEndTime()); taskInstance.setStartTime(taskInstanceDTO.getStartTime()); taskInstance.setStartupMode(taskInstanceDTO.getStartupMode()); - taskInstance.setTaskId(taskInstanceDTO.getTaskId()); + taskInstance.setTaskId(taskInstanceDTO.getPlanId()); taskInstance.setOperator(taskInstanceDTO.getOperator()); taskInstance.setStatus(taskInstanceDTO.getStatus().getValue()); taskInstance.setTemplateId(taskInstanceDTO.getTaskTemplateId()); @@ -147,8 +146,13 @@ private ValidateResult checkRequest(EsbGetJobInstanceListV3Request request) { return ValidateResult.fail(ErrorCode.MISSING_OR_ILLEGAL_PARAM_WITH_PARAM_NAME, "create_time_end"); } long period = request.getCreateTimeEnd() - request.getCreateTimeStart(); - if (period > Consts.MAX_SEARCH_TASK_HISTORY_RANGE_MILLS) { - log.warn("Search time range greater than 30 days!"); + if (period <= 0) { + log.warn("CreateStartTime is greater or equal to createTimeEnd, getCreateTimeStart={}, createTimeEnd={}", + request.getCreateTimeStart(), request.getCreateTimeEnd()); + return ValidateResult.fail(ErrorCode.MISSING_OR_ILLEGAL_PARAM_WITH_PARAM_NAME, + "create_time_start|create_time_end"); + } else if (period > Consts.MAX_SEARCH_TASK_HISTORY_RANGE_MILLS) { + log.warn("Search time range greater than {} days!", Consts.MAX_SEARCH_TASK_HISTORY_RANGE_MILLS); return ValidateResult.fail(ErrorCode.ILLEGAL_PARAM, "create_time_start|create_time_end"); } if (request.getTaskType() != null && TaskTypeEnum.valueOf(request.getTaskType()) == null) { @@ -173,11 +177,10 @@ public EsbResp> getJobInstanceList(String us Integer taskType, Integer taskStatus, String ip, + Long cronId, Integer start, Integer length) { EsbGetJobInstanceListV3Request request = new EsbGetJobInstanceListV3Request(); - request.setUserName(username); - request.setAppCode(appCode); request.setBizId(bizId); request.setScopeType(scopeType); request.setScopeId(scopeId); @@ -190,8 +193,10 @@ public EsbResp> getJobInstanceList(String us request.setTaskType(taskType); request.setTaskStatus(taskStatus); request.setIp(ip); + request.setCronId(cronId); request.setStart(start); request.setLength(length); - return getJobInstanceListUsingPost(request); + request.fillAppResourceScope(appScopeMappingService); + return getJobInstanceListUsingPost(username, appCode, request); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceStatusV3ResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceStatusV3ResourceImpl.java index 27da93865e..e1c808a6ef 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceStatusV3ResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetJobInstanceStatusV3ResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,24 +24,28 @@ package com.tencent.bk.job.execute.api.esb.v3; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.audit.annotations.AuditRequestBody; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.esb.metrics.EsbApiTimed; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.common.esb.util.EsbDTOAppScopeMappingHelper; import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.exception.NotFoundException; +import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.metrics.CommonMetricNames; import com.tencent.bk.job.common.model.ValidateResult; import com.tencent.bk.job.common.service.AppScopeMappingService; -import com.tencent.bk.job.execute.api.esb.v2.impl.JobQueryCommonProcessor; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; -import com.tencent.bk.job.execute.model.AgentTaskDetailDTO; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; +import com.tencent.bk.job.execute.model.ExecuteObjectTask; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.model.esb.v3.EsbJobInstanceStatusV3DTO; import com.tencent.bk.job.execute.model.esb.v3.request.EsbGetJobInstanceStatusV3Request; -import com.tencent.bk.job.execute.service.FileAgentTaskService; -import com.tencent.bk.job.execute.service.ScriptAgentTaskService; +import com.tencent.bk.job.execute.service.FileExecuteObjectTaskService; +import com.tencent.bk.job.execute.service.ScriptExecuteObjectTaskService; +import com.tencent.bk.job.execute.service.StepInstanceService; import com.tencent.bk.job.execute.service.TaskInstanceService; import com.tencent.bk.job.logsvr.consts.FileTaskModeEnum; import lombok.extern.slf4j.Slf4j; @@ -54,30 +58,33 @@ @RestController @Slf4j -public class EsbGetJobInstanceStatusV3ResourceImpl - extends JobQueryCommonProcessor - implements EsbGetJobInstanceStatusV3Resource { +public class EsbGetJobInstanceStatusV3ResourceImpl implements EsbGetJobInstanceStatusV3Resource { private final TaskInstanceService taskInstanceService; + private final ScriptExecuteObjectTaskService scriptExecuteObjectTaskService; + private final FileExecuteObjectTaskService fileExecuteObjectTaskService; private final AppScopeMappingService appScopeMappingService; - private final ScriptAgentTaskService scriptAgentTaskService; - private final FileAgentTaskService fileAgentTaskService; + private final StepInstanceService stepInstanceService; public EsbGetJobInstanceStatusV3ResourceImpl(TaskInstanceService taskInstanceService, + ScriptExecuteObjectTaskService scriptExecuteObjectTaskService, + FileExecuteObjectTaskService fileExecuteObjectTaskService, AppScopeMappingService appScopeMappingService, - ScriptAgentTaskService scriptAgentTaskService, - FileAgentTaskService fileAgentTaskService) { + StepInstanceService stepInstanceService) { this.taskInstanceService = taskInstanceService; + this.scriptExecuteObjectTaskService = scriptExecuteObjectTaskService; + this.fileExecuteObjectTaskService = fileExecuteObjectTaskService; this.appScopeMappingService = appScopeMappingService; - this.scriptAgentTaskService = scriptAgentTaskService; - this.fileAgentTaskService = fileAgentTaskService; + this.stepInstanceService = stepInstanceService; } @Override @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_get_job_instance_status"}) - public EsbResp getJobInstanceStatusUsingPost(EsbGetJobInstanceStatusV3Request request) { - request.fillAppResourceScope(appScopeMappingService); - + @AuditEntry(actionId = ActionId.VIEW_HISTORY) + public EsbResp getJobInstanceStatusUsingPost( + String username, + String appCode, + @AuditRequestBody EsbGetJobInstanceStatusV3Request request) { ValidateResult checkResult = checkRequest(request); if (!checkResult.isPass()) { log.warn("Get job instance status request is illegal!"); @@ -86,12 +93,11 @@ public EsbResp getJobInstanceStatusUsingPost(EsbGetJo long taskInstanceId = request.getTaskInstanceId(); - TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(request.getTaskInstanceId()); - - authViewTaskInstance(request.getUserName(), request.getAppResourceScope(), taskInstance); + TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance( + username, request.getAppResourceScope().getAppId(), request.getTaskInstanceId()); - - List stepInstances = taskInstanceService.listStepInstanceByTaskInstanceId(taskInstanceId); + List stepInstances = + stepInstanceService.listBaseStepInstanceByTaskInstanceId(taskInstanceId); if (stepInstances == null || stepInstances.isEmpty()) { log.warn("Get job instance status by taskInstanceId:{}, stepInstanceList is empty!", taskInstanceId); throw new NotFoundException(ErrorCode.STEP_INSTANCE_NOT_EXIST); @@ -137,39 +143,42 @@ private EsbJobInstanceStatusV3DTO buildEsbJobInstanceStatusDTO(TaskInstanceDTO t stepInst.setCreateTime(stepInstance.getCreateTime()); stepInst.setEndTime(stepInstance.getEndTime()); stepInst.setStartTime(stepInstance.getStartTime()); - stepInst.setType(stepInstance.getExecuteType()); + stepInst.setType(stepInstance.getExecuteType().getValue()); stepInst.setExecuteCount(stepInstance.getExecuteCount()); stepInst.setStatus(stepInstance.getStatus().getValue()); stepInst.setTotalTime(stepInstance.getTotalTime()); if (isReturnIpResult) { List stepIpResults = new ArrayList<>(); - List agentTaskList = null; + List executeObjectTaskList = null; if (stepInstance.isScriptStep()) { - agentTaskList = scriptAgentTaskService.listAgentTaskDetail(stepInstance, + executeObjectTaskList = scriptExecuteObjectTaskService.listTasks(stepInstance, stepInstance.getExecuteCount(), null); } else if (stepInstance.isFileStep()) { - agentTaskList = fileAgentTaskService.listAgentTaskDetail(stepInstance, + executeObjectTaskList = fileExecuteObjectTaskService.listTasks(stepInstance, stepInstance.getExecuteCount(), null); - if (CollectionUtils.isNotEmpty(agentTaskList)) { + if (CollectionUtils.isNotEmpty(executeObjectTaskList)) { // 如果是文件分发任务，只返回目标Agent结果 - agentTaskList = agentTaskList.stream() - .filter(agentTask -> agentTask.getFileTaskMode() == FileTaskModeEnum.DOWNLOAD) + executeObjectTaskList = executeObjectTaskList.stream() + .filter(executeObjectTask -> + executeObjectTask.getFileTaskMode() == FileTaskModeEnum.DOWNLOAD) .collect(Collectors.toList()); } } - if (CollectionUtils.isNotEmpty(agentTaskList)) { - for (AgentTaskDetailDTO agentTask : agentTaskList) { + if (CollectionUtils.isNotEmpty(executeObjectTaskList)) { + for (ExecuteObjectTask executeObjectTask : executeObjectTaskList) { + ExecuteObject executeObject = executeObjectTask.getExecuteObject(); EsbJobInstanceStatusV3DTO.IpResult stepIpResult = new EsbJobInstanceStatusV3DTO.IpResult(); - stepIpResult.setCloudAreaId(agentTask.getBkCloudId()); - stepIpResult.setIp(agentTask.getIp()); - stepIpResult.setExitCode(agentTask.getExitCode()); - stepIpResult.setErrorCode(agentTask.getErrorCode()); - stepIpResult.setStartTime(agentTask.getStartTime()); - stepIpResult.setEndTime(agentTask.getEndTime()); - stepIpResult.setTotalTime(agentTask.getTotalTime()); - stepIpResult.setTag(agentTask.getTag()); - stepIpResult.setStatus(agentTask.getStatus().getValue()); + stepIpResult.setHostId(executeObject.getHost().getHostId()); + stepIpResult.setCloudAreaId(executeObject.getHost().getBkCloudId()); + stepIpResult.setIp(executeObject.getHost().getIp()); + stepIpResult.setExitCode(executeObjectTask.getExitCode()); + stepIpResult.setErrorCode(executeObjectTask.getErrorCode()); + stepIpResult.setStartTime(executeObjectTask.getStartTime()); + stepIpResult.setEndTime(executeObjectTask.getEndTime()); + stepIpResult.setTotalTime(executeObjectTask.getTotalTime()); + stepIpResult.setTag(executeObjectTask.getTag()); + stepIpResult.setStatus(executeObjectTask.getStatus().getValue()); stepIpResults.add(stepIpResult); } } @@ -183,6 +192,7 @@ private EsbJobInstanceStatusV3DTO buildEsbJobInstanceStatusDTO(TaskInstanceDTO t } @Override + @AuditEntry(actionId = ActionId.VIEW_HISTORY) public EsbResp getJobInstanceStatus(String username, String appCode, Long bizId, @@ -191,13 +201,12 @@ public EsbResp getJobInstanceStatus(String username, Long taskInstanceId, boolean returnIpResult) { EsbGetJobInstanceStatusV3Request request = new EsbGetJobInstanceStatusV3Request(); - request.setUserName(username); - request.setAppCode(appCode); request.setBizId(bizId); request.setScopeType(scopeType); request.setScopeId(scopeId); request.setTaskInstanceId(taskInstanceId); request.setReturnIpResult(returnIpResult); - return getJobInstanceStatusUsingPost(request); + request.fillAppResourceScope(appScopeMappingService); + return getJobInstanceStatusUsingPost(username, appCode, request); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetStepInstanceDetailV3ResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetStepInstanceDetailV3ResourceImpl.java new file mode 100644 index 0000000000..e4f11e3bdf --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetStepInstanceDetailV3ResourceImpl.java @@ -0,0 +1,215 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.api.esb.v3; + +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.job.common.esb.model.EsbResp; +import com.tencent.bk.job.common.esb.model.job.v3.EsbAccountV3BasicDTO; +import com.tencent.bk.job.common.esb.model.job.v3.EsbFileDestinationV3DTO; +import com.tencent.bk.job.common.esb.model.job.v3.EsbFileSourceV3DTO; +import com.tencent.bk.job.common.esb.model.job.v3.resp.EsbApprovalStepV3DTO; +import com.tencent.bk.job.common.esb.model.job.v3.resp.EsbFileStepV3DTO; +import com.tencent.bk.job.common.esb.model.job.v3.resp.EsbScriptStepV3DTO; +import com.tencent.bk.job.common.esb.model.job.v3.resp.EsbStepV3DTO; +import com.tencent.bk.job.common.iam.constant.ActionId; +import com.tencent.bk.job.common.service.AppScopeMappingService; +import com.tencent.bk.job.execute.model.FileDetailDTO; +import com.tencent.bk.job.execute.model.FileSourceDTO; +import com.tencent.bk.job.execute.model.StepInstanceDTO; +import com.tencent.bk.job.execute.service.StepInstanceService; +import com.tencent.bk.job.execute.service.TaskInstanceAccessProcessor; +import com.tencent.bk.job.execute.service.TaskInstanceService; +import com.tencent.bk.job.execute.util.FileTransferModeUtil; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.util.CollectionUtils; +import org.springframework.web.bind.annotation.RestController; + +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +@RestController +@Slf4j +public class EsbGetStepInstanceDetailV3ResourceImpl implements EsbGetStepInstanceDetailV3Resource { + + private final TaskInstanceService taskInstanceService; + private final AppScopeMappingService appScopeMappingService; + private final TaskInstanceAccessProcessor taskInstanceAccessProcessor; + private final StepInstanceService stepInstanceService; + + public EsbGetStepInstanceDetailV3ResourceImpl(TaskInstanceService taskInstanceService, + AppScopeMappingService appScopeMappingService, + TaskInstanceAccessProcessor taskInstanceAccessProcessor, + StepInstanceService stepInstanceService) { + this.taskInstanceService = taskInstanceService; + this.appScopeMappingService = appScopeMappingService; + this.taskInstanceAccessProcessor = taskInstanceAccessProcessor; + this.stepInstanceService = stepInstanceService; + } + + @Override + @AuditEntry(actionId = ActionId.VIEW_HISTORY) + public EsbResp getStepInstanceDetail(String username, + String appCode, + String scopeType, + String scopeId, + Long taskInstanceId, + Long stepInstanceId) { + long appId = appScopeMappingService.getAppIdByScope(scopeType, scopeId); + + StepInstanceDTO stepInstance = stepInstanceService.getStepInstanceDetail(taskInstanceId, stepInstanceId); + + taskInstanceAccessProcessor.processBeforeAccess(username, appId, stepInstance.getTaskInstanceId()); + + EsbStepV3DTO esbStepV3DTO = convertToEsbStepV3DTO(stepInstance); + return EsbResp.buildSuccessResp(esbStepV3DTO); + } + + private EsbStepV3DTO convertToEsbStepV3DTO(StepInstanceDTO stepInstance) { + if (stepInstance == null) { + return null; + } + EsbStepV3DTO esbStepV3DTO = new EsbStepV3DTO(); + esbStepV3DTO.setId(stepInstance.getId()); + esbStepV3DTO.setName(stepInstance.getName()); + esbStepV3DTO.setType(stepInstance.getStepType().getValue()); + switch (stepInstance.getStepType()) { + case SCRIPT: + EsbScriptStepV3DTO scriptStepInfo = new EsbScriptStepV3DTO(); + scriptStepInfo.setType(stepInstance.getScriptSource()); + scriptStepInfo.setScriptId(stepInstance.getScriptId()); + scriptStepInfo.setScriptVersionId(stepInstance.getScriptVersionId()); + scriptStepInfo.setContent(stepInstance.getScriptContent()); + scriptStepInfo.setLanguage(stepInstance.getScriptType().getValue()); + if (stepInstance.isSecureParam()) { + scriptStepInfo.setSecureParam(1); + scriptStepInfo.setScriptParam("******"); + } else { + scriptStepInfo.setSecureParam(0); + if (StringUtils.isNotEmpty(stepInstance.getResolvedScriptParam())) { + scriptStepInfo.setScriptParam(stepInstance.getResolvedScriptParam()); + } else { + scriptStepInfo.setScriptParam(stepInstance.getScriptParam()); + } + } + scriptStepInfo.setScriptTimeout(getTimeout(stepInstance)); + scriptStepInfo.setAccount(getAccount(stepInstance)); + scriptStepInfo.setServer(stepInstance.getTargetExecuteObjects().toEsbServerV3DTO()); + scriptStepInfo.setIgnoreError(stepInstance.isIgnoreError() ? 1 : 0); + esbStepV3DTO.setScriptInfo(scriptStepInfo); + break; + case FILE: + EsbFileStepV3DTO fileStepInfo = new EsbFileStepV3DTO(); + fileStepInfo.setFileSourceList(convertToFileSourceList(stepInstance.getFileSourceList())); + fileStepInfo.setFileDestination(getFileDestination(stepInstance)); + fileStepInfo.setTimeout(getTimeout(stepInstance)); + Long uploadSpeedLimit = getFileUploadSpeedLimit(stepInstance); + if (uploadSpeedLimit != null) { + fileStepInfo.setSourceSpeedLimit(uploadSpeedLimit >> 10); + } + Long downloadSpeedLimit = getFileDownloadSpeedLimit(stepInstance); + if (downloadSpeedLimit != null) { + fileStepInfo.setDestinationSpeedLimit(downloadSpeedLimit >> 10); + } + Integer transferMode = FileTransferModeUtil.getTransferMode( + stepInstance.getFileDuplicateHandle(), + stepInstance.getNotExistPathHandler() + ).getValue(); + fileStepInfo.setTransferMode(transferMode); + fileStepInfo.setIgnoreError(stepInstance.isIgnoreError() ? 1 : 0); + esbStepV3DTO.setFileInfo(fileStepInfo); + break; + case APPROVAL: + EsbApprovalStepV3DTO esbApprovalStepV3DTO = new EsbApprovalStepV3DTO(); + esbApprovalStepV3DTO.setApprovalMessage(stepInstance.getConfirmMessage()); + esbStepV3DTO.setApprovalInfo(esbApprovalStepV3DTO); + break; + } + return esbStepV3DTO; + } + + private Long getTimeout(StepInstanceDTO stepInstance) { + Integer timeout = stepInstance.getTimeout(); + return timeout == null ? null : timeout.longValue(); + } + + private EsbAccountV3BasicDTO getAccount(StepInstanceDTO stepInstance) { + EsbAccountV3BasicDTO account = new EsbAccountV3BasicDTO(); + account.setId(stepInstance.getAccountId()); + account.setName(stepInstance.getAccount()); + return account; + } + + private Long getFileUploadSpeedLimit(StepInstanceDTO stepInstance) { + Integer fileUploadSpeedLimit = stepInstance.getFileUploadSpeedLimit(); + return fileUploadSpeedLimit == null ? null : fileUploadSpeedLimit.longValue(); + } + + private Long getFileDownloadSpeedLimit(StepInstanceDTO stepInstance) { + Integer fileDownloadSpeedLimit = stepInstance.getFileDownloadSpeedLimit(); + return fileDownloadSpeedLimit == null ? null : fileDownloadSpeedLimit.longValue(); + } + + private List convertToFileSourceList(List fileSourceList) { + if (fileSourceList == null) { + return null; + } + if (CollectionUtils.isEmpty(fileSourceList)) { + return Collections.emptyList(); + } + return fileSourceList.stream().map(this::convertToEsbFileSourceV3DTO).collect(Collectors.toList()); + } + + private EsbFileSourceV3DTO convertToEsbFileSourceV3DTO(FileSourceDTO fileSourceDTO) { + if (fileSourceDTO == null) { + return null; + } + EsbFileSourceV3DTO esbFileSourceV3DTO = new EsbFileSourceV3DTO(); + esbFileSourceV3DTO.setFiles(fileSourceDTO.getFiles() + .stream().map(FileDetailDTO::getFilePath) + .collect(Collectors.toList()) + ); + EsbAccountV3BasicDTO account = new EsbAccountV3BasicDTO(); + account.setId(fileSourceDTO.getAccountId()); + account.setName(fileSourceDTO.getAccount()); + esbFileSourceV3DTO.setAccount(account); + esbFileSourceV3DTO.setServer(fileSourceDTO.getServers().toEsbServerV3DTO()); + esbFileSourceV3DTO.setFileType(fileSourceDTO.getFileType()); + esbFileSourceV3DTO.setFileSourceId(fileSourceDTO.getFileSourceId()); + return esbFileSourceV3DTO; + } + + private EsbFileDestinationV3DTO getFileDestination(StepInstanceDTO stepInstance) { + if (stepInstance == null) { + return null; + } + EsbFileDestinationV3DTO esbFileDestinationV3DTO = new EsbFileDestinationV3DTO(); + esbFileDestinationV3DTO.setPath(stepInstance.getFileTargetPath()); + esbFileDestinationV3DTO.setAccount(getAccount(stepInstance)); + esbFileDestinationV3DTO.setServer(stepInstance.getTargetExecuteObjects().toEsbServerV3DTO()); + return esbFileDestinationV3DTO; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetStepInstanceStatusV3ResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetStepInstanceStatusV3ResourceImpl.java new file mode 100644 index 0000000000..6bff8a92ca --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbGetStepInstanceStatusV3ResourceImpl.java @@ -0,0 +1,164 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.api.esb.v3; + +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.esb.metrics.EsbApiTimed; +import com.tencent.bk.job.common.esb.model.EsbResp; +import com.tencent.bk.job.common.exception.NotFoundException; +import com.tencent.bk.job.common.i18n.service.MessageI18nService; +import com.tencent.bk.job.common.iam.constant.ActionId; +import com.tencent.bk.job.common.metrics.CommonMetricNames; +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.common.service.AppScopeMappingService; +import com.tencent.bk.job.execute.engine.consts.ExecuteObjectTaskStatusEnum; +import com.tencent.bk.job.execute.model.ExecuteObjectTask; +import com.tencent.bk.job.execute.model.ResultGroupDTO; +import com.tencent.bk.job.execute.model.StepExecutionDetailDTO; +import com.tencent.bk.job.execute.model.StepExecutionResultQuery; +import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; +import com.tencent.bk.job.execute.model.esb.v3.EsbStepInstanceStatusV3DTO; +import com.tencent.bk.job.execute.service.TaskResultService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.web.bind.annotation.RestController; + +import java.util.ArrayList; +import java.util.List; + +@RestController +@Slf4j +public class EsbGetStepInstanceStatusV3ResourceImpl implements EsbGetStepInstanceStatusV3Resource { + + private final AppScopeMappingService appScopeMappingService; + private final TaskResultService taskResultService; + private final MessageI18nService messageI18nService; + + public EsbGetStepInstanceStatusV3ResourceImpl(AppScopeMappingService appScopeMappingService, + TaskResultService taskResultService, + MessageI18nService messageI18nService) { + this.appScopeMappingService = appScopeMappingService; + this.taskResultService = taskResultService; + this.messageI18nService = messageI18nService; + } + + private EsbStepInstanceStatusV3DTO buildEsbStepInstanceStatusV3DTO(StepExecutionDetailDTO executionResult) { + StepInstanceBaseDTO stepInstance = executionResult.getStepInstance(); + EsbStepInstanceStatusV3DTO stepInst = new EsbStepInstanceStatusV3DTO(); + stepInst.setId(stepInstance.getId()); + stepInst.setExecuteCount(stepInstance.getExecuteCount()); + stepInst.setName(stepInstance.getName()); + stepInst.setType(stepInstance.getExecuteType().getValue()); + stepInst.setStatus(stepInstance.getStatus().getValue()); + stepInst.setCreateTime(stepInstance.getCreateTime()); + stepInst.setStartTime(executionResult.getStartTime()); + stepInst.setEndTime(executionResult.getEndTime()); + stepInst.setTotalTime(executionResult.getTotalTime()); + + List stepResultGroupList = new ArrayList<>(); + + List resultGroups = executionResult.getResultGroups(); + for (ResultGroupDTO resultGroup : resultGroups) { + List executeObjectTasks = resultGroup.getExecuteObjectTasks(); + + EsbStepInstanceStatusV3DTO.StepResultGroup stepResultGroup = + new EsbStepInstanceStatusV3DTO.StepResultGroup(); + stepResultGroup.setResultType(resultGroup.getStatus()); + ExecuteObjectTaskStatusEnum taskStatusEnum = ExecuteObjectTaskStatusEnum.valOf(resultGroup.getStatus()); + if (taskStatusEnum != null) { + stepResultGroup.setResultTypeDesc(messageI18nService.getI18n(taskStatusEnum.getI18nKey())); + } + stepResultGroup.setTag(resultGroup.getTag()); + stepResultGroup.setHostSize(resultGroup.getTotal()); + List hostResults = new ArrayList<>(); + + if (executeObjectTasks != null) { + for (ExecuteObjectTask executeObjectTask : executeObjectTasks) { + EsbStepInstanceStatusV3DTO.HostResult stepHostResult = new EsbStepInstanceStatusV3DTO.HostResult(); + HostDTO host = executeObjectTask.getExecuteObject().getHost(); + stepHostResult.setHostId(host.getHostId()); + stepHostResult.setIp(host.getIp()); + stepHostResult.setIpv6(host.getIpv6()); + stepHostResult.setCloudAreaId(host.getBkCloudId()); + stepHostResult.setAgentId(host.getAgentId()); + stepHostResult.setCloudAreaName(host.getBkCloudName()); + stepHostResult.setStatus(executeObjectTask.getStatus().getValue()); + stepHostResult.setStatusDesc(messageI18nService.getI18n(executeObjectTask.getStatus().getI18nKey())); + stepHostResult.setTag(executeObjectTask.getTag()); + stepHostResult.setExitCode(executeObjectTask.getExitCode()); + stepHostResult.setStartTime(executeObjectTask.getStartTime()); + stepHostResult.setEndTime(executeObjectTask.getEndTime()); + stepHostResult.setTotalTime(executeObjectTask.getTotalTime()); + hostResults.add(stepHostResult); + } + stepResultGroup.setHostResultList(hostResults); + } + stepResultGroupList.add(stepResultGroup); + } + stepInst.setStepResultGroupList(stepResultGroupList); + return stepInst; + } + + @Override + @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_get_step_instance_status"}) + @AuditEntry(actionId = ActionId.VIEW_HISTORY) + public EsbResp getStepInstanceStatus(String username, + String appCode, + String scopeType, + String scopeId, + Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch, + Integer maxHostNumPerGroup, + String keyword, + String searchIp, + Integer status, + String tag) { + long appId = appScopeMappingService.getAppIdByScope(scopeType, scopeId); + + StepExecutionResultQuery query = StepExecutionResultQuery.builder() + .taskInstanceId(taskInstanceId) + .stepInstanceId(stepInstanceId) + .executeCount(executeCount) + .batch(batch == null ? null : (batch == 0 ? null : batch)) + .filterByLatestBatch(batch == null) + .status(status) + .tag(tag) + .logKeyword(keyword) + .searchIp(searchIp) + .maxTasksForResultGroup(maxHostNumPerGroup) + .fetchAllGroupData(status == null) + .build(); + + StepExecutionDetailDTO executionResult = taskResultService.getStepExecutionResult(username, appId, query); + if (executionResult == null) { + throw new NotFoundException(ErrorCode.STEP_INSTANCE_NOT_EXIST); + } + EsbStepInstanceStatusV3DTO jobInstanceStatus = buildEsbStepInstanceStatusV3DTO(executionResult); + + return EsbResp.buildSuccessResp(jobInstanceStatus); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbOperateJobInstanceV3ResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbOperateJobInstanceV3ResourceImpl.java index af787db961..d1d79b6bda 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbOperateJobInstanceV3ResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbOperateJobInstanceV3ResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,12 +24,12 @@ package com.tencent.bk.job.execute.api.esb.v3; +import com.tencent.bk.audit.annotations.AuditRequestBody; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.esb.metrics.EsbApiTimed; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.metrics.CommonMetricNames; -import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.execute.constants.TaskOperationEnum; import com.tencent.bk.job.execute.model.esb.v3.EsbJobExecuteV3DTO; @@ -42,24 +42,22 @@ @Slf4j public class EsbOperateJobInstanceV3ResourceImpl implements EsbOperateJobInstanceV3Resource { private final TaskExecuteService taskExecuteService; - private final AppScopeMappingService appScopeMappingService; - public EsbOperateJobInstanceV3ResourceImpl(TaskExecuteService taskExecuteService, - AppScopeMappingService appScopeMappingService) { + public EsbOperateJobInstanceV3ResourceImpl(TaskExecuteService taskExecuteService) { this.taskExecuteService = taskExecuteService; - this.appScopeMappingService = appScopeMappingService; } @Override @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_operate_job_instance"}) - public EsbResp operateJobInstance(EsbOperateJobInstanceV3Request request) { + public EsbResp operateJobInstance(String username, + String appCode, + @AuditRequestBody EsbOperateJobInstanceV3Request request) { log.info("Operate task instance, request={}", JsonUtils.toJson(request)); - request.fillAppResourceScope(appScopeMappingService); if (!checkRequest(request)) { throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } TaskOperationEnum taskOperation = TaskOperationEnum.getTaskOperation(request.getOperationCode()); - taskExecuteService.doTaskOperation(request.getAppId(), request.getUserName(), + taskExecuteService.doTaskOperation(request.getAppId(), username, request.getTaskInstanceId(), taskOperation); EsbJobExecuteV3DTO result = new EsbJobExecuteV3DTO(); result.setTaskInstanceId(request.getTaskInstanceId()); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbOperateStepInstanceV3ResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbOperateStepInstanceV3ResourceImpl.java index 525c9f9181..b3b0af2dae 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbOperateStepInstanceV3ResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbOperateStepInstanceV3ResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,12 +24,12 @@ package com.tencent.bk.job.execute.api.esb.v3; +import com.tencent.bk.audit.annotations.AuditRequestBody; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.esb.metrics.EsbApiTimed; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.metrics.CommonMetricNames; -import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.execute.constants.StepOperationEnum; import com.tencent.bk.job.execute.model.StepOperationDTO; @@ -43,27 +43,26 @@ @Slf4j public class EsbOperateStepInstanceV3ResourceImpl implements EsbOperateStepInstanceV3Resource { private final TaskExecuteService taskExecuteService; - private final AppScopeMappingService appScopeMappingService; - public EsbOperateStepInstanceV3ResourceImpl(TaskExecuteService taskExecuteService, - AppScopeMappingService appScopeMappingService) { + public EsbOperateStepInstanceV3ResourceImpl(TaskExecuteService taskExecuteService) { this.taskExecuteService = taskExecuteService; - this.appScopeMappingService = appScopeMappingService; } @Override @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_operate_step_instance"}) - public EsbResp operateStepInstance(EsbOperateStepInstanceV3Request request) { + public EsbResp operateStepInstance(String username, + String appCode, + @AuditRequestBody EsbOperateStepInstanceV3Request request) { log.info("Operate step instance, request={}", JsonUtils.toJson(request)); - request.fillAppResourceScope(appScopeMappingService); if (!checkRequest(request)) { throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } StepOperationEnum operationEnum = StepOperationEnum.getStepOperation(request.getOperationCode()); StepOperationDTO stepOperation = new StepOperationDTO(); + stepOperation.setTaskInstanceId(request.getTaskInstanceId()); stepOperation.setStepInstanceId(request.getStepInstanceId()); stepOperation.setOperation(operationEnum); - taskExecuteService.doStepOperation(request.getAppId(), request.getUserName(), stepOperation); + taskExecuteService.doStepOperation(request.getAppId(), username, stepOperation); EsbJobExecuteV3DTO result = new EsbJobExecuteV3DTO(); result.setTaskInstanceId(request.getTaskInstanceId()); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbPushConfigFileResourceV3Impl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbPushConfigFileResourceV3Impl.java index eccf28a855..7bc3c9de0d 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbPushConfigFileResourceV3Impl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbPushConfigFileResourceV3Impl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,15 +24,17 @@ package com.tencent.bk.job.execute.api.esb.v3; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.audit.annotations.AuditRequestBody; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.constant.JobConstants; import com.tencent.bk.job.common.esb.metrics.EsbApiTimed; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.exception.ServiceException; +import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.metrics.CommonMetricNames; import com.tencent.bk.job.common.model.ValidateResult; -import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.common.web.metrics.CustomTimed; import com.tencent.bk.job.execute.api.esb.common.ConfigFileUtil; @@ -40,8 +42,9 @@ import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; import com.tencent.bk.job.execute.common.constants.TaskStartupModeEnum; import com.tencent.bk.job.execute.common.constants.TaskTypeEnum; -import com.tencent.bk.job.execute.config.StorageSystemConfig; +import com.tencent.bk.job.execute.config.FileDistributeConfig; import com.tencent.bk.job.execute.metrics.ExecuteMetricsConstants; +import com.tencent.bk.job.execute.model.ExecuteTargetDTO; import com.tencent.bk.job.execute.model.FastTaskDTO; import com.tencent.bk.job.execute.model.FileDetailDTO; import com.tencent.bk.job.execute.model.FileSourceDTO; @@ -49,15 +52,16 @@ import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.model.esb.v3.EsbJobExecuteV3DTO; import com.tencent.bk.job.execute.model.esb.v3.request.EsbPushConfigFileV3Request; -import com.tencent.bk.job.execute.service.AgentService; +import com.tencent.bk.job.execute.service.LocalFileDistributeSourceHostProvisioner; import com.tencent.bk.job.execute.service.TaskExecuteService; -import com.tencent.bk.job.manage.common.consts.task.TaskFileTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskFileTypeEnum; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RestController; import java.util.ArrayList; +import java.util.Collections; import java.util.List; @RestController @@ -66,19 +70,16 @@ public class EsbPushConfigFileResourceV3Impl extends JobExecuteCommonV3Processor implements EsbPushConfigFileV3Resource { private final TaskExecuteService taskExecuteService; - private final StorageSystemConfig storageSystemConfig; - private final AgentService agentService; - private final AppScopeMappingService appScopeMappingService; + private final FileDistributeConfig fileDistributeConfig; + private final LocalFileDistributeSourceHostProvisioner localFileDistributeSourceHostProvisioner; @Autowired public EsbPushConfigFileResourceV3Impl(TaskExecuteService taskExecuteService, - StorageSystemConfig storageSystemConfig, - AgentService agentService, - AppScopeMappingService appScopeMappingService) { + FileDistributeConfig fileDistributeConfig, + LocalFileDistributeSourceHostProvisioner localFileDistributeSourceHostProvisioner) { this.taskExecuteService = taskExecuteService; - this.storageSystemConfig = storageSystemConfig; - this.agentService = agentService; - this.appScopeMappingService = appScopeMappingService; + this.fileDistributeConfig = fileDistributeConfig; + this.localFileDistributeSourceHostProvisioner = localFileDistributeSourceHostProvisioner; } @Override @@ -88,8 +89,10 @@ public EsbPushConfigFileResourceV3Impl(TaskExecuteService taskExecuteService, ExecuteMetricsConstants.TAG_KEY_START_MODE, ExecuteMetricsConstants.TAG_VALUE_START_MODE_API, ExecuteMetricsConstants.TAG_KEY_TASK_TYPE, ExecuteMetricsConstants.TAG_VALUE_TASK_TYPE_FAST_CONFIG_FILE }) - public EsbResp pushConfigFile(EsbPushConfigFileV3Request request) { - request.fillAppResourceScope(appScopeMappingService); + @AuditEntry(actionId = ActionId.QUICK_TRANSFER_FILE) + public EsbResp pushConfigFile(String username, + String appCode, + @AuditRequestBody EsbPushConfigFileV3Request request) { ValidateResult checkResult = checkPushConfigFileRequest(request); if (!checkResult.isPass()) { log.warn("Fast transfer file request is illegal!"); @@ -98,18 +101,20 @@ public EsbResp pushConfigFile(EsbPushConfigFileV3Request req request.trimIps(); - TaskInstanceDTO taskInstance = buildFastFileTaskInstance(request); - StepInstanceDTO stepInstance = buildFastFileStepInstance(request, request.getFileList()); - long taskInstanceId = taskExecuteService.executeFastTask( + TaskInstanceDTO taskInstance = buildFastFileTaskInstance(username, appCode, request); + StepInstanceDTO stepInstance = buildFastFileStepInstance(username, request, request.getFileList()); + TaskInstanceDTO executeTaskInstance = taskExecuteService.executeFastTask( FastTaskDTO.builder().taskInstance(taskInstance).stepInstance(stepInstance).build()); EsbJobExecuteV3DTO jobExecuteInfo = new EsbJobExecuteV3DTO(); - jobExecuteInfo.setTaskInstanceId(taskInstanceId); + jobExecuteInfo.setTaskInstanceId(executeTaskInstance.getId()); jobExecuteInfo.setTaskName(stepInstance.getName()); return EsbResp.buildSuccessResp(jobExecuteInfo); } - private TaskInstanceDTO buildFastFileTaskInstance(EsbPushConfigFileV3Request request) { + private TaskInstanceDTO buildFastFileTaskInstance(String username, + String appCode, + EsbPushConfigFileV3Request request) { TaskInstanceDTO taskInstance = new TaskInstanceDTO(); taskInstance.setType(TaskTypeEnum.FILE.getValue()); if (StringUtils.isNotEmpty(request.getName())) { @@ -117,22 +122,23 @@ private TaskInstanceDTO buildFastFileTaskInstance(EsbPushConfigFileV3Request req } else { taskInstance.setName("API_PUSH_CONFIG_FILE_" + DateUtils.currentTimeMillis()); } - taskInstance.setTaskId(-1L); + taskInstance.setPlanId(-1L); taskInstance.setCronTaskId(-1L); taskInstance.setTaskTemplateId(-1L); taskInstance.setAppId(request.getAppId()); taskInstance.setStatus(RunStatusEnum.BLANK); taskInstance.setStartupMode(TaskStartupModeEnum.API.getValue()); - taskInstance.setOperator(request.getUserName()); + taskInstance.setOperator(username); taskInstance.setCreateTime(DateUtils.currentTimeMillis()); taskInstance.setCurrentStepInstanceId(0L); taskInstance.setDebugTask(false); - taskInstance.setAppCode(request.getAppCode()); + taskInstance.setAppCode(appCode); taskInstance.setCallbackUrl(request.getCallbackUrl()); return taskInstance; } private StepInstanceDTO buildFastFileStepInstance( + String username, EsbPushConfigFileV3Request request, List configFileList ) { @@ -145,12 +151,12 @@ private StepInstanceDTO buildFastFileStepInstance( stepInstance.setAccountId(request.getAccountId()); stepInstance.setAccountAlias(request.getAccountAlias()); stepInstance.setStepId(-1L); - stepInstance.setExecuteType(StepExecuteTypeEnum.SEND_FILE.getValue()); + stepInstance.setExecuteType(StepExecuteTypeEnum.SEND_FILE); stepInstance.setFileTargetPath(request.getTargetPath()); - stepInstance.setFileSourceList(convertConfigFileSource(request.getUserName(), configFileList)); + stepInstance.setFileSourceList(convertConfigFileSource(username, configFileList)); stepInstance.setAppId(request.getAppId()); - stepInstance.setTargetServers(convertToServersDTO(request.getTargetServer())); - stepInstance.setOperator(request.getUserName()); + stepInstance.setTargetExecuteObjects(convertToServersDTO(request.getTargetServer())); + stepInstance.setOperator(username); stepInstance.setStatus(RunStatusEnum.BLANK); stepInstance.setCreateTime(DateUtils.currentTimeMillis()); stepInstance.setTimeout(JobConstants.DEFAULT_JOB_TIMEOUT_SECONDS); @@ -171,17 +177,21 @@ private List convertConfigFileSource( fileSourceDTO.setLocalUpload(false); fileSourceDTO.setFileType(TaskFileTypeEnum.BASE64_FILE.getType()); List files = new ArrayList<>(); - // 保存配置文件至机器 + // 保存配置文件至主机 String configFileLocalPath = ConfigFileUtil.saveConfigFileToLocal( - storageSystemConfig.getJobStorageRootPath(), + fileDistributeConfig.getJobDistributeRootPath(), userName, configFile.getFileName(), configFile.getContent() ); files.add(new FileDetailDTO(configFileLocalPath)); fileSourceDTO.setFiles(files); - // 设置配置文件所在机器IP信息 - fileSourceDTO.setServers(agentService.getLocalServersDTO()); + // 设置配置文件所在主机信息 + ExecuteTargetDTO fileSourceExecuteObjects = new ExecuteTargetDTO(); + fileSourceExecuteObjects.setStaticIpList(Collections.singletonList( + localFileDistributeSourceHostProvisioner.getLocalFileDistributeSourceHost() + )); + fileSourceDTO.setServers(fileSourceExecuteObjects); fileSourceDTOS.add(fileSourceDTO); }); return fileSourceDTOS; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbQueryToolsV3ResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbQueryToolsV3ResourceImpl.java new file mode 100644 index 0000000000..c26899df5b --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/EsbQueryToolsV3ResourceImpl.java @@ -0,0 +1,231 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.api.esb.v3; + +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.esb.model.EsbResp; +import com.tencent.bk.job.common.esb.util.EsbDTOAppScopeMappingHelper; +import com.tencent.bk.job.common.iam.constant.ActionId; +import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; +import com.tencent.bk.job.common.iam.model.AuthResult; +import com.tencent.bk.job.common.iam.service.AppAuthService; +import com.tencent.bk.job.common.model.dto.AppResourceScope; +import com.tencent.bk.job.common.model.dto.ResourceScope; +import com.tencent.bk.job.common.service.AppScopeMappingService; +import com.tencent.bk.job.execute.model.GseTaskSimpleDTO; +import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; +import com.tencent.bk.job.execute.model.StepInstanceRollingTaskDTO; +import com.tencent.bk.job.execute.model.esb.v3.EsbTaskLinkV3DTO; +import com.tencent.bk.job.execute.service.GseTaskService; +import com.tencent.bk.job.execute.service.StepInstanceRollingTaskService; +import com.tencent.bk.job.execute.service.StepInstanceService; +import org.apache.commons.collections4.CollectionUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.web.bind.annotation.RestController; + +import java.util.ArrayList; +import java.util.List; + +@RestController +public class EsbQueryToolsV3ResourceImpl implements EsbQueryToolsV3Resource { + private final StepInstanceService stepInstanceService; + private final GseTaskService gseTaskService; + private final AppAuthService appAuthService; + private final AppScopeMappingService appScopeMappingService; + private final StepInstanceRollingTaskService stepInstanceRollingTaskService; + + @Value("${job.web.url:}") + private String jobWebUrl; + + /** + * 快速执行链接地址 + */ + private static final String FAST_LINK = "%s/api_execute/%s?stepInstanceId=%s&retryCount=%s&batch=%s"; + + /** + * 作业执行链接地址 + */ + private static final String TASK_LINK = "%s/api_execute_step/%s/%s?retryCount=%s&batch=%s"; + + @Autowired + public EsbQueryToolsV3ResourceImpl(StepInstanceService stepInstanceService, + GseTaskService gseTaskService, + AppAuthService appAuthService, + AppScopeMappingService appScopeMappingService, + StepInstanceRollingTaskService stepInstanceRollingTaskService) { + this.stepInstanceService = stepInstanceService; + this.gseTaskService = gseTaskService; + this.appAuthService = appAuthService; + this.appScopeMappingService = appScopeMappingService; + this.stepInstanceRollingTaskService = stepInstanceRollingTaskService; + } + + @Override + public EsbResp> queryGSETaskByStep(String username, String appCode, Long stepInstanceId) { + StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstanceById(stepInstanceId); + if (stepInstance == null) { + String reason = String.format("step not found by id: %s", stepInstanceId); + return EsbResp.buildCommonFailResp( + ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME_AND_REASON, + new String[]{String.valueOf(stepInstanceId), reason}, + null + ); + } + + Long appId = stepInstance.getAppId(); + ResourceScope resourceScope = appScopeMappingService.getAppResourceScope(appId); + AppResourceScope appResourceScope = new AppResourceScope(appId, resourceScope); + // 鉴权 + auth(username, appResourceScope); + + List gseTaskResult = new ArrayList<>(); + List stepInstanceRollingTaskDTOS = + stepInstanceRollingTaskService.listRollingTasksByStep(stepInstance.getTaskInstanceId(), stepInstanceId); + + if (CollectionUtils.isNotEmpty(stepInstanceRollingTaskDTOS)) { + // 步骤为滚动执行 + for (StepInstanceRollingTaskDTO stepInstanceRollingTaskDTO : stepInstanceRollingTaskDTOS) { + List gseTaskSimpleDTOS = gseTaskService.listGseTaskSimpleInfo( + stepInstanceId, + stepInstanceRollingTaskDTO.getExecuteCount(), + stepInstanceRollingTaskDTO.getBatch() + ); + gseTaskResult.addAll(gseTaskSimpleDTOS); + } + } else { + // 步骤不是滚动执行 + gseTaskResult.addAll(gseTaskService.listGseTaskSimpleInfo( + stepInstanceId, + null, + null + )); + } + + List taskLinks = new ArrayList<>(); + for (GseTaskSimpleDTO gseTaskSimpleDTO : gseTaskResult) { + taskLinks.add(buildEsbTaskLinkV3DTO( + appId, + stepInstance, + gseTaskSimpleDTO.getExecuteCount(), + gseTaskSimpleDTO.getBatch(), + gseTaskSimpleDTO.getGseTaskId() + )); + } + return EsbResp.buildSuccessResp(taskLinks); + } + + @Override + public EsbResp queryJobInstanceByGseTask(String username, String appCode, String gseTaskId) { + GseTaskSimpleDTO gseTask = gseTaskService.getGseTaskSimpleInfo(gseTaskId); + if (gseTask == null) { + String reason = String.format("gseTask not found by id: %s", gseTaskId); + return EsbResp.buildCommonFailResp( + ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME_AND_REASON, + new String[]{gseTaskId, reason}, + null + ); + } + + StepInstanceBaseDTO stepInstanceBaseDTO = stepInstanceService.getBaseStepInstance( + gseTask.getTaskInstanceId(), + gseTask.getStepInstanceId() + ); + Long appId = stepInstanceBaseDTO.getAppId(); + ResourceScope resourceScope = appScopeMappingService.getScopeByAppId(appId); + AppResourceScope appResourceScope = new AppResourceScope(appId, resourceScope); + auth(username, appResourceScope); + + return EsbResp.buildSuccessResp(buildEsbTaskLinkV3DTO( + appId, + stepInstanceBaseDTO, + gseTask.getExecuteCount(), + gseTask.getBatch(), + gseTaskId + )); + } + + private void auth(String username, AppResourceScope appResourceScope) { + AuthResult authResult = appAuthService.auth( + username, + ActionId.ACCESS_BUSINESS, + appResourceScope + ); + if (!authResult.isPass()) { + throw new PermissionDeniedException(authResult); + } + + authResult = appAuthService.auth( + username, + ActionId.VIEW_HISTORY, + appResourceScope + ); + if (!authResult.isPass()) { + throw new PermissionDeniedException(authResult); + } + } + + private EsbTaskLinkV3DTO buildEsbTaskLinkV3DTO( + Long appId, + StepInstanceBaseDTO stepInstanceBase, + int retryCount, + int batch, + String gseTaskId + ) { + + EsbTaskLinkV3DTO esbTaskLinkV3DTO = new EsbTaskLinkV3DTO(); + esbTaskLinkV3DTO.setJobInstanceId(stepInstanceBase.getTaskInstanceId()); + esbTaskLinkV3DTO.setStepInstanceId(stepInstanceBase.getId()); + EsbDTOAppScopeMappingHelper.fillEsbAppScopeDTOByAppId(appId, esbTaskLinkV3DTO); + esbTaskLinkV3DTO.setRetryCount(retryCount); + esbTaskLinkV3DTO.setBatch(batch); + esbTaskLinkV3DTO.setGseTaskId(gseTaskId); + esbTaskLinkV3DTO.setLink(buildLink(esbTaskLinkV3DTO, stepInstanceBase)); + return esbTaskLinkV3DTO; + } + + private List buildLink(EsbTaskLinkV3DTO esbTaskLinkV3DTO, StepInstanceBaseDTO stepInstanceBase) { + List links = new ArrayList<>(); + String linkTemplate = FAST_LINK; + // 是作业执行 + if (stepInstanceBase.getStepId() != -1L) { + linkTemplate = TASK_LINK; + } + + String[] urls = jobWebUrl.split(","); + for (String url : urls) { + String link = String.format( + linkTemplate, + url, + esbTaskLinkV3DTO.getJobInstanceId(), + esbTaskLinkV3DTO.getStepInstanceId(), + esbTaskLinkV3DTO.getRetryCount(), + esbTaskLinkV3DTO.getBatch() + ); + links.add(link); + } + return links; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/JobExecuteCommonV3Processor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/JobExecuteCommonV3Processor.java index c5aba13938..cb8c21fed2 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/JobExecuteCommonV3Processor.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/esb/v3/JobExecuteCommonV3Processor.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -30,12 +30,13 @@ import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.execute.model.DynamicServerGroupDTO; import com.tencent.bk.job.execute.model.DynamicServerTopoNodeDTO; -import com.tencent.bk.job.execute.model.ServersDTO; +import com.tencent.bk.job.execute.model.ExecuteTargetDTO; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; /** * 作业执行公用 @@ -48,35 +49,45 @@ protected ValidateResult checkServer(EsbServerV3DTO server) { return ValidateResult.fail(ErrorCode.MISSING_PARAM_WITH_PARAM_NAME, "target_server"); } - if (CollectionUtils.isEmpty(server.getIps()) && CollectionUtils.isEmpty(server.getTopoNodes()) - && CollectionUtils.isEmpty(server.getDynamicGroups())) { + if (!server.checkHostParamsNonEmpty()) { return ValidateResult.fail(ErrorCode.SERVER_EMPTY); } return ValidateResult.pass(); } - protected ServersDTO convertToServersDTO(EsbServerV3DTO server) { + protected ExecuteTargetDTO convertToServersDTO(EsbServerV3DTO server) { if (server == null) { return null; } - ServersDTO serversDTO = new ServersDTO(); + ExecuteTargetDTO executeTargetDTO = new ExecuteTargetDTO(); + + // 拓扑节点 if (CollectionUtils.isNotEmpty(server.getTopoNodes())) { List topoNodes = new ArrayList<>(); server.getTopoNodes().forEach(topoNode -> topoNodes.add(new DynamicServerTopoNodeDTO(topoNode.getId(), topoNode.getNodeType()))); - serversDTO.setTopoNodes(topoNodes); + executeTargetDTO.setTopoNodes(topoNodes); } + + // 动态分组 if (CollectionUtils.isNotEmpty(server.getDynamicGroups())) { List dynamicServerGroups = new ArrayList<>(); server.getDynamicGroups().forEach( group -> dynamicServerGroups.add(new DynamicServerGroupDTO(group.getId()))); - serversDTO.setDynamicServerGroups(dynamicServerGroups); + executeTargetDTO.setDynamicServerGroups(dynamicServerGroups); } - if (CollectionUtils.isNotEmpty(server.getIps())) { - List staticIpList = new ArrayList<>(); - server.getIps().forEach(ip -> staticIpList.add(new HostDTO(ip.getBkCloudId(), ip.getIp()))); - serversDTO.setStaticIpList(staticIpList); + + // 主机列表，优先使用hostId + if (CollectionUtils.isNotEmpty(server.getHostIds())) { + executeTargetDTO.setStaticIpList( + server.getHostIds().stream().map(HostDTO::fromHostId).collect(Collectors.toList())); + } else if (CollectionUtils.isNotEmpty(server.getIps())) { + executeTargetDTO.setStaticIpList( + server.getIps().stream() + .map(host -> new HostDTO(host.getBkCloudId(), host.getIp())) + .collect(Collectors.toList())); } - return serversDTO; + + return executeTargetDTO; } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceExecuteTaskResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceExecuteTaskResourceImpl.java index 3dc71c002b..832a82f2df 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceExecuteTaskResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceExecuteTaskResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -38,7 +38,7 @@ import com.tencent.bk.job.execute.metrics.ExecuteMetricsConstants; import com.tencent.bk.job.execute.model.DynamicServerGroupDTO; import com.tencent.bk.job.execute.model.DynamicServerTopoNodeDTO; -import com.tencent.bk.job.execute.model.ServersDTO; +import com.tencent.bk.job.execute.model.ExecuteTargetDTO; import com.tencent.bk.job.execute.model.TaskExecuteParam; import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.model.inner.ServiceTargetServers; @@ -54,6 +54,8 @@ import java.util.ArrayList; import java.util.List; +import static com.tencent.bk.job.common.constant.TaskVariableTypeEnum.CIPHER; + @RestController @Slf4j public class ServiceExecuteTaskResourceImpl implements ServiceExecuteTaskResource { @@ -86,31 +88,13 @@ public InternalResponse executeTask(ServiceTaskExecute ServiceTaskExecuteResult result = new ServiceTaskExecuteResult(); result.setTaskInstanceId(taskInstanceDTO.getId()); result.setName(taskInstanceDTO.getName()); + result.setCreateTime(taskInstanceDTO.getCreateTime()); return InternalResponse.buildSuccessResp(result); } private TaskExecuteParam buildExecuteParam(ServiceTaskExecuteRequest request) { List executeVariableValues = new ArrayList<>(); - if (request.getTaskVariables() != null) { - for (ServiceTaskVariable serviceTaskVariable : request.getTaskVariables()) { - TaskVariableDTO taskVariableDTO = new TaskVariableDTO(); - taskVariableDTO.setId(serviceTaskVariable.getId()); - if (serviceTaskVariable.getType() == TaskVariableTypeEnum.STRING.getType() - || serviceTaskVariable.getType() == TaskVariableTypeEnum.CIPHER.getType() - || serviceTaskVariable.getType() == TaskVariableTypeEnum.INDEX_ARRAY.getType() - || serviceTaskVariable.getType() == TaskVariableTypeEnum.ASSOCIATIVE_ARRAY.getType()) { - taskVariableDTO.setValue(serviceTaskVariable.getStringValue()); - } else if (serviceTaskVariable.getType() == TaskVariableTypeEnum.HOST_LIST.getType()) { - ServiceTargetServers serviceServers = serviceTaskVariable.getServerValue(); - ServersDTO serversDTO = convertToServersDTO(serviceServers); - taskVariableDTO.setTargetServers(serversDTO); - } else if (serviceTaskVariable.getType() == TaskVariableTypeEnum.NAMESPACE.getType()) { - taskVariableDTO.setValue(serviceTaskVariable.getNamespaceValue()); - } - executeVariableValues.add(taskVariableDTO); - } - } - return TaskExecuteParam + TaskExecuteParam taskExecuteParam = TaskExecuteParam .builder() .appId(request.getAppId()) .planId(request.getPlanId()) @@ -121,27 +105,56 @@ private TaskExecuteParam buildExecuteParam(ServiceTaskExecuteRequest request) { .taskName(request.getTaskName()) .skipAuth(request.getCronTaskId() != null && request.isSkipAuth()) .build(); + if (request.getTaskVariables() == null) { + return taskExecuteParam; + } + // 解析构造全局变量 + for (ServiceTaskVariable serviceTaskVariable : request.getTaskVariables()) { + TaskVariableDTO taskVariableDTO = new TaskVariableDTO(); + taskVariableDTO.setId(serviceTaskVariable.getId()); + if (serviceTaskVariable.getType() == TaskVariableTypeEnum.STRING.getType() + || serviceTaskVariable.getType() == TaskVariableTypeEnum.INDEX_ARRAY.getType() + || serviceTaskVariable.getType() == TaskVariableTypeEnum.ASSOCIATIVE_ARRAY.getType()) { + taskVariableDTO.setValue(serviceTaskVariable.getStringValue()); + } else if (serviceTaskVariable.getType() == CIPHER.getType()) { + // 如果密码类型的变量传入为空或者“******”，那么密码使用系统中保存的 + if (serviceTaskVariable.getStringValue() == null || "******".equals(serviceTaskVariable.getStringValue())) { + continue; + } else { + taskVariableDTO.setValue(serviceTaskVariable.getStringValue()); + } + } else if (serviceTaskVariable.getType() == TaskVariableTypeEnum.HOST_LIST.getType()) { + ServiceTargetServers serviceServers = serviceTaskVariable.getServerValue(); + ExecuteTargetDTO executeTargetDTO = convertToServersDTO(serviceServers); + taskVariableDTO.setExecuteTarget(executeTargetDTO); + } else if (serviceTaskVariable.getType() == TaskVariableTypeEnum.NAMESPACE.getType()) { + taskVariableDTO.setValue(serviceTaskVariable.getNamespaceValue()); + } + executeVariableValues.add(taskVariableDTO); + } + taskExecuteParam.setExecuteVariableValues(executeVariableValues); + return taskExecuteParam; } - private ServersDTO convertToServersDTO(ServiceTargetServers servers) { + private ExecuteTargetDTO convertToServersDTO(ServiceTargetServers servers) { if (servers == null) { return null; } - ServersDTO serversDTO = new ServersDTO(); - serversDTO.setStaticIpList(servers.getIps()); + ExecuteTargetDTO executeTargetDTO = new ExecuteTargetDTO(); + executeTargetDTO.setStaticIpList(servers.getIps()); if (servers.getDynamicGroupIds() != null) { List dynamicServerGroups = new ArrayList<>(); servers.getDynamicGroupIds() .forEach(groupId -> dynamicServerGroups.add(new DynamicServerGroupDTO(groupId))); - serversDTO.setDynamicServerGroups(dynamicServerGroups); + executeTargetDTO.setDynamicServerGroups(dynamicServerGroups); } if (servers.getTopoNodes() != null) { List topoNodes = new ArrayList<>(); servers.getTopoNodes().forEach(topoNode -> topoNodes.add(new DynamicServerTopoNodeDTO(topoNode.getId(), topoNode.getNodeType()))); - serversDTO.setTopoNodes(topoNodes); + executeTargetDTO.setTopoNodes(topoNodes); } - return serversDTO; + return executeTargetDTO; } private boolean checkExecuteTaskRequest(ServiceTaskExecuteRequest request) { diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceMetricsResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceMetricsResourceImpl.java index 5f9b36fe9c..8b5d0cd9cb 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceMetricsResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceMetricsResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,16 +24,11 @@ package com.tencent.bk.job.execute.api.inner; +import com.tencent.bk.job.analysis.api.dto.StatisticsDTO; import com.tencent.bk.job.common.model.InternalResponse; -import com.tencent.bk.job.common.statistics.model.dto.StatisticsDTO; -import com.tencent.bk.job.execute.common.constants.RunStatusEnum; -import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; -import com.tencent.bk.job.execute.common.constants.TaskStartupModeEnum; -import com.tencent.bk.job.execute.common.constants.TaskTypeEnum; import com.tencent.bk.job.execute.model.inner.request.ServiceTriggerStatisticsRequest; import com.tencent.bk.job.execute.service.TaskInstanceService; import com.tencent.bk.job.execute.statistics.StatisticsService; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; import io.micrometer.core.annotation.Timed; import lombok.extern.slf4j.Slf4j; import org.springframework.web.bind.annotation.RestController; @@ -44,7 +39,7 @@ import static com.tencent.bk.job.execute.monitor.ExecuteMetricTags.IGNORE_TAG; @Slf4j -@RestController +@RestController("executeMetricsResource") public class ServiceMetricsResourceImpl implements ServiceMetricsResource { private final TaskInstanceService taskInstanceService; private final StatisticsService statisticsService; @@ -67,46 +62,13 @@ public InternalResponse hasExecuteHistory(Long appId, Long cronTaskId, toTime)); } - @Override - @Timed(extraTags = {IGNORE_TAG, BOOLEAN_TRUE_TAG_VALUE}) - public InternalResponse countFastPushFile(Long appId, Integer transferMode, Boolean localUpload, - RunStatusEnum runStatus, Long fromTime, Long toTime) { - return InternalResponse.buildSuccessResp(taskInstanceService.countFastPushFile(appId, transferMode, - localUpload, runStatus, fromTime, toTime)); - } - - @Override - @Timed(extraTags = {IGNORE_TAG, BOOLEAN_TRUE_TAG_VALUE}) - public InternalResponse countStepInstances(Long appId, List stepIdList, - StepExecuteTypeEnum stepExecuteType, Integer scriptType, - RunStatusEnum runStatus, Long fromTime, Long toTime) { - return InternalResponse.buildSuccessResp(taskInstanceService.countStepInstances(appId, stepIdList, - stepExecuteType, ScriptTypeEnum.valueOf(scriptType), runStatus, fromTime, toTime)); - } - - @Override - @Timed(extraTags = {IGNORE_TAG, BOOLEAN_TRUE_TAG_VALUE}) - public InternalResponse countTaskInstances(Long appId, Long minTotalTime, Long maxTotalTime, - TaskStartupModeEnum taskStartupMode, TaskTypeEnum taskType, - List runStatusList, Long fromTime, Long toTime) { - return InternalResponse.buildSuccessResp(taskInstanceService.countTaskInstances(appId, minTotalTime, - maxTotalTime, taskStartupMode, taskType, runStatusList, fromTime, toTime)); - } - @Override public InternalResponse getStatistics(Long appId, String resource, String dimension, - String dimensionValue, String dateStr) { + String dimensionValue, String dateStr) { return InternalResponse.buildSuccessResp(statisticsService.getStatistics(appId, resource, dimension, dimensionValue, dateStr)); } - @Override - public InternalResponse> listStatistics(Long appId, String resource, String dimension, - String dimensionValue, String dateStr) { - return InternalResponse.buildSuccessResp(statisticsService.listStatistics(appId, resource, dimension, - dimensionValue, dateStr)); - } - @Override public InternalResponse triggerStatistics(ServiceTriggerStatisticsRequest request) { return InternalResponse.buildSuccessResp(statisticsService.triggerStatistics(request.getDateList())); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceResourceQuotaResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceResourceQuotaResourceImpl.java new file mode 100644 index 0000000000..0c5aeaf15c --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceResourceQuotaResourceImpl.java @@ -0,0 +1,65 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.api.inner; + +import com.tencent.bk.job.common.model.InternalResponse; +import com.tencent.bk.job.common.resource.quota.ResourceQuotaLimit; +import com.tencent.bk.job.common.service.quota.ResourceQuotaStore; +import com.tencent.bk.job.execute.engine.quota.limit.RunningJobResourceQuotaManager; +import com.tencent.bk.job.execute.model.inner.RunningJobQuotaUsage; +import lombok.extern.slf4j.Slf4j; +import org.springframework.web.bind.annotation.RestController; + +import java.util.Map; + +@Slf4j +@RestController +public class ServiceResourceQuotaResourceImpl implements ServiceResourceQuotaResource { + + private final ResourceQuotaStore resourceQuotaStore; + + private final RunningJobResourceQuotaManager runningJobResourceQuotaManager; + + public ServiceResourceQuotaResourceImpl(ResourceQuotaStore resourceQuotaStore, + RunningJobResourceQuotaManager runningJobResourceQuotaManager) { + this.resourceQuotaStore = resourceQuotaStore; + this.runningJobResourceQuotaManager = runningJobResourceQuotaManager; + } + + @Override + public InternalResponse> getResourceQuotaConfig() { + return InternalResponse.buildSuccessResp(resourceQuotaStore.getAll()); + } + + @Override + public InternalResponse getRunningJobQuotaUsage() { + + RunningJobQuotaUsage runningJobQuotaUsage = new RunningJobQuotaUsage(); + runningJobQuotaUsage.setTotal(runningJobResourceQuotaManager.getRunningJobTotal()); + runningJobQuotaUsage.setAppCount(runningJobResourceQuotaManager.getAppRunningJobCount()); + runningJobQuotaUsage.setResourceScopeCount(runningJobResourceQuotaManager.getResourceScopeRunningJobCount()); + return InternalResponse.buildSuccessResp(runningJobQuotaUsage); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceStepInstanceResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceStepInstanceResourceImpl.java new file mode 100644 index 0000000000..dfb6f60645 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceStepInstanceResourceImpl.java @@ -0,0 +1,71 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.api.inner; + +import com.tencent.bk.job.common.exception.NotFoundException; +import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; +import com.tencent.bk.job.common.iam.model.AuthResult; +import com.tencent.bk.job.common.model.InternalResponse; +import com.tencent.bk.job.common.model.iam.AuthResultDTO; +import com.tencent.bk.job.execute.model.StepInstanceDTO; +import com.tencent.bk.job.execute.model.inner.ServiceStepInstanceDTO; +import com.tencent.bk.job.execute.service.StepInstanceService; +import com.tencent.bk.job.execute.service.TaskInstanceAccessProcessor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@Slf4j +public class ServiceStepInstanceResourceImpl implements ServiceStepInstanceResource { + + private final TaskInstanceAccessProcessor taskInstanceAccessProcessor; + private final StepInstanceService stepInstanceService; + + @Autowired + public ServiceStepInstanceResourceImpl(TaskInstanceAccessProcessor taskInstanceAccessProcessor, + StepInstanceService stepInstanceService) { + this.taskInstanceAccessProcessor = taskInstanceAccessProcessor; + this.stepInstanceService = stepInstanceService; + } + + @Override + public InternalResponse getStepInstance(String username, + Long appId, + Long taskInstanceId, + Long stepInstanceId) { + try { + StepInstanceDTO stepInstance = stepInstanceService.getStepInstanceDetail(appId, + taskInstanceId, stepInstanceId); + taskInstanceAccessProcessor.processBeforeAccess(username, appId, stepInstance.getTaskInstanceId()); + return InternalResponse.buildSuccessResp(stepInstance.toServiceStepInstanceDTO()); + } catch (NotFoundException e) { + return InternalResponse.buildCommonFailResp(e); + } catch (PermissionDeniedException e) { + AuthResultDTO authResult = AuthResult.toAuthResultDTO(e.getAuthResult()); + return InternalResponse.buildAuthFailResp(authResult); + } + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceTaskEvictPolicyResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceTaskEvictPolicyResourceImpl.java index ff46856269..1609933e00 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceTaskEvictPolicyResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceTaskEvictPolicyResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceTaskExecuteResultResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceTaskExecuteResultResourceImpl.java index d232f30ed5..53eaa0b94c 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceTaskExecuteResultResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/inner/ServiceTaskExecuteResultResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/op/SwitchableTopoPathResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/op/SwitchableTopoPathResourceImpl.java new file mode 100644 index 0000000000..2d958723c4 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/op/SwitchableTopoPathResourceImpl.java @@ -0,0 +1,58 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.api.op; + +import com.tencent.bk.job.common.model.Response; +import com.tencent.bk.job.execute.auth.impl.SwitchableTopoPathService; +import com.tencent.bk.job.execute.model.op.SwitchStatusReq; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RestController; + +import java.util.HashSet; +import java.util.List; +import java.util.Map; + +@RestController +@Slf4j +public class SwitchableTopoPathResourceImpl implements SwitchableTopoPathResource { + + private final SwitchableTopoPathService switchableTopoPathService; + + @Autowired + public SwitchableTopoPathResourceImpl(SwitchableTopoPathService switchableTopoPathService) { + this.switchableTopoPathService = switchableTopoPathService; + } + + @Override + public Response>> getTopoPathByHostIds(List hostIdList) { + return Response.buildSuccessResp(switchableTopoPathService.getTopoPathByHostIds(new HashSet<>(hostIdList))); + } + + @Override + public Response switchStatus(SwitchStatusReq req) { + return Response.buildSuccessResp(switchableTopoPathService.switchStatus(req.isStatus())); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebDangerousRecordResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebDangerousRecordResourceImpl.java index a3d936f8a1..66c99c533e 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebDangerousRecordResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebDangerousRecordResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,10 @@ package com.tencent.bk.job.execute.api.web.impl; +import com.tencent.bk.audit.annotations.ActionAuditRecord; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.job.common.audit.constants.EventContentConstants; +import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; import com.tencent.bk.job.common.model.Response; @@ -55,6 +59,11 @@ public WebDangerousRecordResourceImpl(DangerousRecordService dangerousRecordServ } @Override + @AuditEntry(actionId = ActionId.HIGH_RISK_DETECT_RECORD) + @ActionAuditRecord( + actionId = ActionId.HIGH_RISK_DETECT_RECORD, + content = EventContentConstants.VIEW_HIGH_RISK_DETECT_RECORD + ) public Response> pageListDangerousRecords(String username, Long id, String scopeType, diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebExecuteTaskResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebExecuteTaskResourceImpl.java index b6ae6a26d0..c74ea1fa6d 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebExecuteTaskResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebExecuteTaskResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,12 +24,19 @@ package com.tencent.bk.job.execute.api.web.impl; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.audit.annotations.AuditRequestBody; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.CompatibleType; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.constant.TaskVariableTypeEnum; import com.tencent.bk.job.common.exception.InvalidParamException; +import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.model.Response; import com.tencent.bk.job.common.model.dto.AppResourceScope; -import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.common.model.vo.TaskTargetVO; +import com.tencent.bk.job.common.util.DataSizeConverter; +import com.tencent.bk.job.common.util.FilePathValidateUtil; import com.tencent.bk.job.common.util.check.IlegalCharChecker; import com.tencent.bk.job.common.util.check.MaxLengthChecker; import com.tencent.bk.job.common.util.check.NotEmptyChecker; @@ -46,12 +53,11 @@ import com.tencent.bk.job.execute.constants.StepOperationEnum; import com.tencent.bk.job.execute.engine.model.TaskVariableDTO; import com.tencent.bk.job.execute.metrics.ExecuteMetricsConstants; -import com.tencent.bk.job.execute.model.DynamicServerGroupDTO; -import com.tencent.bk.job.execute.model.DynamicServerTopoNodeDTO; +import com.tencent.bk.job.execute.model.ExecuteTargetDTO; import com.tencent.bk.job.execute.model.FastTaskDTO; import com.tencent.bk.job.execute.model.FileDetailDTO; import com.tencent.bk.job.execute.model.FileSourceDTO; -import com.tencent.bk.job.execute.model.ServersDTO; +import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.model.StepOperationDTO; import com.tencent.bk.job.execute.model.StepRollingConfigDTO; @@ -64,16 +70,14 @@ import com.tencent.bk.job.execute.model.web.request.WebTaskExecuteRequest; import com.tencent.bk.job.execute.model.web.vo.ExecuteFileDestinationInfoVO; import com.tencent.bk.job.execute.model.web.vo.ExecuteFileSourceInfoVO; -import com.tencent.bk.job.execute.model.web.vo.ExecuteHostVO; -import com.tencent.bk.job.execute.model.web.vo.ExecuteServersVO; -import com.tencent.bk.job.execute.model.web.vo.ExecuteTargetVO; import com.tencent.bk.job.execute.model.web.vo.ExecuteVariableVO; import com.tencent.bk.job.execute.model.web.vo.StepExecuteVO; import com.tencent.bk.job.execute.model.web.vo.StepOperationVO; import com.tencent.bk.job.execute.model.web.vo.TaskExecuteVO; +import com.tencent.bk.job.execute.service.StepInstanceService; import com.tencent.bk.job.execute.service.TaskExecuteService; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskFileTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskFileTypeEnum; import lombok.extern.slf4j.Slf4j; import org.apache.commons.codec.binary.Base64; import org.apache.commons.collections4.CollectionUtils; @@ -96,10 +100,13 @@ @Slf4j public class WebExecuteTaskResourceImpl implements WebExecuteTaskResource { private final TaskExecuteService taskExecuteService; + private final StepInstanceService stepInstanceService; @Autowired - public WebExecuteTaskResourceImpl(TaskExecuteService taskExecuteService) { + public WebExecuteTaskResourceImpl(TaskExecuteService taskExecuteService, + StepInstanceService stepInstanceService) { this.taskExecuteService = taskExecuteService; + this.stepInstanceService = stepInstanceService; } @Override @@ -108,11 +115,12 @@ public WebExecuteTaskResourceImpl(TaskExecuteService taskExecuteService) { ExecuteMetricsConstants.TAG_KEY_START_MODE, ExecuteMetricsConstants.TAG_VALUE_START_MODE_WEB, ExecuteMetricsConstants.TAG_KEY_TASK_TYPE, ExecuteMetricsConstants.TAG_VALUE_TASK_TYPE_EXECUTE_PLAN }) + @AuditEntry public Response executeTask(String username, AppResourceScope appResourceScope, String scopeType, String scopeId, - WebTaskExecuteRequest request) { + @AuditRequestBody WebTaskExecuteRequest request) { log.info("Execute task, request={}", request); if (!checkExecuteTaskRequest(request)) { @@ -123,7 +131,7 @@ public Response executeTask(String username, TaskInstanceDTO taskInstanceDTO = taskExecuteService.executeJobPlan( TaskExecuteParam.builder().appId(appResourceScope.getAppId()).planId(request.getTaskId()) .operator(username).executeVariableValues(executeVariableValues) - .startupMode(TaskStartupModeEnum.NORMAL).build()); + .startupMode(TaskStartupModeEnum.WEB).build()); TaskExecuteVO result = new TaskExecuteVO(); result.setTaskInstanceId(taskInstanceDTO.getId()); @@ -165,7 +173,7 @@ public Response redoTask(String username, } List executeVariableValues = buildExecuteVariables(request.getTaskVariables()); - TaskInstanceDTO taskInstanceDTO = taskExecuteService.createTaskInstanceForRedo(appResourceScope.getAppId(), + TaskInstanceDTO taskInstanceDTO = taskExecuteService.redoJob(appResourceScope.getAppId(), request.getTaskInstanceId(), username, executeVariableValues); TaskExecuteVO result = new TaskExecuteVO(); @@ -191,9 +199,9 @@ private List buildExecuteVariables(List webT taskVariableDTO.setValue(webTaskVariable.getValue()); } } else if (webTaskVariable.getType() == HOST_LIST.getType()) { - ExecuteTargetVO webServers = webTaskVariable.getTargetValue(); - ServersDTO serversDTO = convertToServersDTO(webServers); - taskVariableDTO.setTargetServers(serversDTO); + TaskTargetVO taskTarget = webTaskVariable.getTargetValue(); + ExecuteTargetDTO executeTargetDTO = ExecuteTargetDTO.fromTaskTargetVO(taskTarget); + taskVariableDTO.setExecuteTarget(executeTargetDTO); } else if (webTaskVariable.getType() == NAMESPACE.getType()) { taskVariableDTO.setValue(webTaskVariable.getValue()); } @@ -224,12 +232,15 @@ private boolean checkRedoTaskRequest(RedoTaskRequest request) { ExecuteMetricsConstants.TAG_KEY_START_MODE, ExecuteMetricsConstants.TAG_VALUE_START_MODE_WEB, ExecuteMetricsConstants.TAG_KEY_TASK_TYPE, ExecuteMetricsConstants.TAG_VALUE_TASK_TYPE_FAST_SCRIPT }) + @AuditEntry public Response fastExecuteScript(String username, AppResourceScope appResourceScope, String scopeType, String scopeId, - WebFastExecuteScriptRequest request) { - log.debug("Fast execute script, scope={}, operator={}, request={}", appResourceScope, username, request); + @AuditRequestBody WebFastExecuteScriptRequest request) { + if (log.isDebugEnabled()) { + log.debug("Fast execute script, scope={}, operator={}, request={}", appResourceScope, username, request); + } if (!checkFastExecuteScriptRequest(request)) { log.warn("Fast execute script request is illegal!"); @@ -269,20 +280,14 @@ private boolean checkFastExecuteScriptRequest(WebFastExecuteScriptRequest reques log.warn("Fast execute script, script type is invalid! scriptType={}", request.getScriptLanguage()); return false; } - ExecuteTargetVO targetServers = request.getTargetServers(); - if (targetServers == null || targetServers.getHostNodeInfo() == null) { - log.warn("Fast execute script, target server is null!"); - return false; - } - if (CollectionUtils.isEmpty(targetServers.getHostNodeInfo().getIpList()) && - CollectionUtils.isEmpty(targetServers.getHostNodeInfo().getTopoNodeList()) - && CollectionUtils.isEmpty(targetServers.getHostNodeInfo().getDynamicGroupList())) { - log.warn("Fast execute script, target server is null!"); - return false; - } - if (!checkIpValid(targetServers.getHostNodeInfo().getIpList())) { + + TaskTargetVO taskTarget = request.getTaskTarget(); + if (taskTarget == null) { + log.warn("Fast execute script, target is null!"); return false; } + taskTarget.validate(); + if (request.getAccount() == null || request.getAccount() < 1) { log.warn("Fast execute script, accountId is invalid! accountId={}", request.getAccount()); return false; @@ -290,32 +295,15 @@ private boolean checkFastExecuteScriptRequest(WebFastExecuteScriptRequest reques return true; } - private boolean checkIpValid(List hosts) { - if (CollectionUtils.isEmpty(hosts)) { - return true; - } - for (ExecuteHostVO host : hosts) { - if (host.getCloudAreaInfo() == null || host.getCloudAreaInfo().getId() == null) { - log.warn("Check host:{}, cloudAreaId is empty!", host); - return false; - } - if (StringUtils.isEmpty(host.getIp())) { - log.warn("Check host:{}, ip is empty!", host); - return false; - } - } - return true; - } - private TaskInstanceDTO buildFastScriptTaskInstance(String username, Long appId, WebFastExecuteScriptRequest request) { TaskInstanceDTO taskInstance = new TaskInstanceDTO(); taskInstance.setName(request.getName()); - taskInstance.setTaskId(-1L); + taskInstance.setPlanId(-1L); taskInstance.setCronTaskId(-1L); taskInstance.setTaskTemplateId(-1L); taskInstance.setAppId(appId); - taskInstance.setStartupMode(TaskStartupModeEnum.NORMAL.getValue()); + taskInstance.setStartupMode(TaskStartupModeEnum.WEB.getValue()); taskInstance.setStatus(RunStatusEnum.BLANK); taskInstance.setOperator(username); taskInstance.setCreateTime(DateUtils.currentTimeMillis()); @@ -335,23 +323,24 @@ private StepInstanceDTO buildFastScriptStepInstance(String userName, Long appId, stepInstance.setName(request.getName()); stepInstance.setStepId(-1L); stepInstance.setAppId(appId); - stepInstance.setTargetServers(convertToServersDTO(request.getTargetServers())); + stepInstance.setTargetExecuteObjects(ExecuteTargetDTO.fromTaskTargetVO(request.getTaskTarget())); if (request.getScriptLanguage().equals(ScriptTypeEnum.SQL.getValue())) { stepInstance.setDbAccountId(request.getAccount()); - stepInstance.setExecuteType(StepExecuteTypeEnum.EXECUTE_SQL.getValue()); + stepInstance.setExecuteType(StepExecuteTypeEnum.EXECUTE_SQL); } else { stepInstance.setAccountId(request.getAccount()); - stepInstance.setExecuteType(StepExecuteTypeEnum.EXECUTE_SCRIPT.getValue()); + stepInstance.setExecuteType(StepExecuteTypeEnum.EXECUTE_SCRIPT); } stepInstance.setOperator(userName); stepInstance.setStatus(RunStatusEnum.BLANK); stepInstance.setCreateTime(DateUtils.currentTimeMillis()); stepInstance.setScriptSource(request.getScriptSource()); - stepInstance.setScriptType(request.getScriptLanguage()); + stepInstance.setScriptType(ScriptTypeEnum.valOf(request.getScriptLanguage())); stepInstance.setScriptContent(request.getContent()); stepInstance.setScriptId(request.getScriptId()); stepInstance.setScriptVersionId(request.getScriptVersionId()); stepInstance.setTimeout(request.getTimeout()); + stepInstance.setWindowsInterpreter(request.getTrimmedWindowsInterpreter()); stepInstance.setScriptParam(request.getScriptParam()); stepInstance.setSecureParam(request.getSecureParam() != null && request.getSecureParam() == 1); return stepInstance; @@ -363,11 +352,12 @@ private StepInstanceDTO buildFastScriptStepInstance(String userName, Long appId, ExecuteMetricsConstants.TAG_KEY_START_MODE, ExecuteMetricsConstants.TAG_VALUE_START_MODE_WEB, ExecuteMetricsConstants.TAG_KEY_TASK_TYPE, ExecuteMetricsConstants.TAG_VALUE_TASK_TYPE_FAST_FILE }) + @AuditEntry(actionId = ActionId.QUICK_TRANSFER_FILE) public Response fastPushFile(String username, AppResourceScope appResourceScope, String scopeType, String scopeId, - WebFastPushFileRequest request) { + @AuditRequestBody WebFastPushFileRequest request) { log.debug("Fast send file, scope={}, operator={}, request={}", appResourceScope, username, request); if (!checkFastPushFileRequest(request)) { log.warn("Fast send file request is illegal!"); @@ -390,14 +380,14 @@ private Response createAndStartFastTask(boolean isRedoTask, StepRollingConfigDTO rollingConfig) { FastTaskDTO fastTask = FastTaskDTO.builder().taskInstance(taskInstance).stepInstance(stepInstance) .rollingConfig(rollingConfig).build(); - long taskInstanceId; + TaskInstanceDTO executeTaskInstance; if (!isRedoTask) { - taskInstanceId = taskExecuteService.executeFastTask(fastTask); + executeTaskInstance = taskExecuteService.executeFastTask(fastTask); } else { - taskInstanceId = taskExecuteService.redoFastTask(fastTask); + executeTaskInstance = taskExecuteService.redoFastTask(fastTask); } StepExecuteVO stepExecuteVO = new StepExecuteVO(); - stepExecuteVO.setTaskInstanceId(taskInstanceId); + stepExecuteVO.setTaskInstanceId(executeTaskInstance.getId()); stepExecuteVO.setStepInstanceId(stepInstance.getId()); stepExecuteVO.setStepName(stepInstance.getName()); return Response.buildSuccessResp(stepExecuteVO); @@ -421,20 +411,12 @@ private boolean checkFastPushFileRequest(WebFastPushFileRequest request) { log.warn("Fast send file, fileDestination is null!"); return false; } - ExecuteTargetVO targetServers = fileDestination.getServer(); - if (targetServers == null || targetServers.getHostNodeInfo() == null) { - log.warn("Fast send file, target server is null!"); - return false; - } - if (CollectionUtils.isEmpty(targetServers.getHostNodeInfo().getIpList()) && - CollectionUtils.isEmpty(targetServers.getHostNodeInfo().getTopoNodeList()) - && CollectionUtils.isEmpty(targetServers.getHostNodeInfo().getDynamicGroupList())) { - log.warn("Fast send file, target server is null!"); - return false; - } - if (!checkIpValid(targetServers.getHostNodeInfo().getIpList())) { + TaskTargetVO targetServers = fileDestination.getServer(); + if (targetServers == null) { return false; } + targetServers.validate(); + if (fileDestination.getAccountId() == null || fileDestination.getAccountId() < 1) { log.warn("Fast send file, accountId is invalid! accountId={}", fileDestination.getAccountId()); return false; @@ -444,7 +426,7 @@ private boolean checkFastPushFileRequest(WebFastPushFileRequest request) { return false; } for (ExecuteFileSourceInfoVO fileSource : request.getFileSourceList()) { - if (CollectionUtils.isEmpty(fileSource.getFileLocation())) { + if (CollectionUtils.isEmpty(fileSource.getTrimmedFileLocation())) { log.warn("Fast send file ,files are empty"); return false; } @@ -453,10 +435,16 @@ private boolean checkFastPushFileRequest(WebFastPushFileRequest request) { log.warn("Fast send file, account is empty!"); return false; } + for (String file : fileSource.getTrimmedFileLocation()) { + if (!FilePathValidateUtil.validateFileSystemAbsolutePath(file)) { + log.warn("Fast send file, fileLocation is null or illegal!"); + return false; + } + } } } - if (StringUtils.isBlank(fileDestination.getPath())) { - log.warn("Fast send file, targetPath is empty"); + if (!FilePathValidateUtil.validateFileSystemAbsolutePath(fileDestination.getTrimmedPath())) { + log.warn("Fast send file, fileDestinationPath is null or illegal!"); return false; } @@ -468,12 +456,12 @@ private TaskInstanceDTO buildFastFileTaskInstance(String username, Long appId, W TaskInstanceDTO taskInstance = new TaskInstanceDTO(); taskInstance.setType(TaskTypeEnum.FILE.getValue()); taskInstance.setName(request.getName()); - taskInstance.setTaskId(-1L); + taskInstance.setPlanId(-1L); taskInstance.setCronTaskId(-1L); taskInstance.setTaskTemplateId(-1L); taskInstance.setAppId(appId); taskInstance.setStatus(RunStatusEnum.BLANK); - taskInstance.setStartupMode(TaskStartupModeEnum.NORMAL.getValue()); + taskInstance.setStartupMode(TaskStartupModeEnum.WEB.getValue()); taskInstance.setOperator(username); taskInstance.setCreateTime(DateUtils.currentTimeMillis()); taskInstance.setCurrentStepInstanceId(0L); @@ -487,10 +475,10 @@ private StepInstanceDTO buildFastFileStepInstance(String userName, Long appId, W stepInstance.setName(request.getName()); ExecuteFileDestinationInfoVO fileDestination = request.getFileDestination(); stepInstance.setAccountId(fileDestination.getAccountId()); - stepInstance.setTargetServers(convertToServersDTO(fileDestination.getServer())); - stepInstance.setFileTargetPath(fileDestination.getPath()); + stepInstance.setTargetExecuteObjects(ExecuteTargetDTO.fromTaskTargetVO(fileDestination.getServer())); + stepInstance.setFileTargetPath(fileDestination.getTrimmedPath()); stepInstance.setStepId(-1L); - stepInstance.setExecuteType(StepExecuteTypeEnum.SEND_FILE.getValue()); + stepInstance.setExecuteType(StepExecuteTypeEnum.SEND_FILE); stepInstance.setFileSourceList(convertFileSource(request.getFileSourceList())); stepInstance.setAppId(appId); stepInstance.setOperator(userName); @@ -498,11 +486,11 @@ private StepInstanceDTO buildFastFileStepInstance(String userName, Long appId, W stepInstance.setCreateTime(DateUtils.currentTimeMillis()); if (request.getDownloadSpeedLimit() != null && request.getDownloadSpeedLimit() > 0) { // MB->KB - stepInstance.setFileDownloadSpeedLimit(request.getDownloadSpeedLimit() << 10); + stepInstance.setFileDownloadSpeedLimit(DataSizeConverter.convertMBToKB(request.getDownloadSpeedLimit())); } if (request.getUploadSpeedLimit() != null && request.getUploadSpeedLimit() > 0) { // MB->KB - stepInstance.setFileUploadSpeedLimit(request.getUploadSpeedLimit() << 10); + stepInstance.setFileUploadSpeedLimit(DataSizeConverter.convertMBToKB(request.getUploadSpeedLimit())); } stepInstance.setTimeout(request.getTimeout()); stepInstance.setFileDuplicateHandle(request.getDuplicateHandler()); @@ -510,32 +498,6 @@ private StepInstanceDTO buildFastFileStepInstance(String userName, Long appId, W return stepInstance; } - private ServersDTO convertToServersDTO(ExecuteTargetVO target) { - if (target == null || target.getHostNodeInfo() == null) { - return null; - } - ExecuteServersVO hostNode = target.getHostNodeInfo(); - ServersDTO serversDTO = new ServersDTO(); - if (CollectionUtils.isNotEmpty(hostNode.getIpList())) { - List staticIpList = new ArrayList<>(); - hostNode.getIpList().forEach(host -> staticIpList.add(new HostDTO(host.getCloudAreaInfo().getId(), - host.getIp()))); - serversDTO.setStaticIpList(staticIpList); - } - if (CollectionUtils.isNotEmpty(hostNode.getDynamicGroupList())) { - List dynamicServerGroups = new ArrayList<>(); - hostNode.getDynamicGroupList().forEach( - groupId -> dynamicServerGroups.add(new DynamicServerGroupDTO(groupId))); - serversDTO.setDynamicServerGroups(dynamicServerGroups); - } - if (CollectionUtils.isNotEmpty(hostNode.getTopoNodeList())) { - List topoNodes = new ArrayList<>(); - hostNode.getTopoNodeList().forEach( - topoNode -> topoNodes.add(new DynamicServerTopoNodeDTO(topoNode.getId(), topoNode.getType()))); - serversDTO.setTopoNodes(topoNodes); - } - return serversDTO; - } private List convertFileSource(List fileSources) { if (fileSources == null) { @@ -550,8 +512,8 @@ private List convertFileSource(List file fileSourceDTO.setFileType(fileType.getType()); fileSourceDTO.setFileSourceId(fileSource.getFileSourceId()); List files = new ArrayList<>(); - if (fileSource.getFileLocation() != null) { - for (String file : fileSource.getFileLocation()) { + if (fileSource.getTrimmedFileLocation() != null) { + for (String file : fileSource.getTrimmedFileLocation()) { if (TaskFileTypeEnum.LOCAL == fileType) { files.add(new FileDetailDTO(true, file, fileSource.getFileHash(), Long.valueOf(fileSource.getFileSize()))); @@ -564,7 +526,7 @@ private List convertFileSource(List file fileSourceDTO.setFiles(files); if (fileType == TaskFileTypeEnum.SERVER) { // 服务器文件分发才需要解析主机参数 - fileSourceDTO.setServers(convertToServersDTO(fileSource.getHost())); + fileSourceDTO.setServers(ExecuteTargetDTO.fromTaskTargetVO(fileSource.getHost())); } fileSourceDTOS.add(fileSourceDTO); }); @@ -572,17 +534,34 @@ private List convertFileSource(List file } @Override + @CompatibleImplementation(name = "dao_add_task_instance_id", deprecatedVersion = "3.11.x", + type = CompatibleType.DEPLOY, explain = "发布完成后可以删除") public Response doStepOperation(String username, AppResourceScope appResourceScope, String scopeType, String scopeId, Long stepInstanceId, WebStepOperation operation) { + // 兼容代码，部署完成后删除 + StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstanceById(stepInstanceId); + return doStepOperationV2(username, appResourceScope, scopeType, scopeId, + stepInstance.getTaskInstanceId(), stepInstanceId, operation); + } + + @Override + public Response doStepOperationV2(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + Long taskInstanceId, + Long stepInstanceId, + WebStepOperation operation) { StepOperationEnum stepOperationEnum = StepOperationEnum.getStepOperation(operation.getOperationCode()); if (stepOperationEnum == null) { throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } StepOperationDTO stepOperation = new StepOperationDTO(); + stepOperation.setTaskInstanceId(taskInstanceId); stepOperation.setStepInstanceId(stepInstanceId); stepOperation.setOperation(stepOperationEnum); stepOperation.setConfirmReason(operation.getConfirmReason()); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebPermissionResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebPermissionResourceImpl.java index 66f4192b10..8568ef5017 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebPermissionResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebPermissionResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -40,7 +40,7 @@ import org.apache.commons.lang3.StringUtils; import org.springframework.web.bind.annotation.RestController; -@RestController +@RestController("jobExecuteWebPermissionResourceImpl") @Slf4j public class WebPermissionResourceImpl implements WebPermissionResource { private final WebAuthService webAuthService; @@ -85,15 +85,11 @@ public Response checkOperationPermission(String username, case "task_instance": long taskInstanceId = Long.parseLong(resourceId); TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(taskInstanceId); - if (taskInstance == null) { - return Response.buildSuccessResp( - AuthResultVO.fail()); - } switch (action) { case "view": case "redo": - AuthResult authResult = executeAuthService.authViewTaskInstance(username, appResourceScope, - taskInstanceId); + AuthResult authResult = executeAuthService.checkViewTaskInstancePermission(username, + appResourceScope, taskInstance); return Response.buildSuccessResp(webAuthService.toAuthResultVO(isReturnApplyUrl, authResult)); } break; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebSearchToolsResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebSearchToolsResourceImpl.java new file mode 100644 index 0000000000..32b4c742ff --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebSearchToolsResourceImpl.java @@ -0,0 +1,204 @@ +package com.tencent.bk.job.execute.api.web.impl; + +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.iam.constant.ActionId; +import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; +import com.tencent.bk.job.common.iam.model.AuthResult; +import com.tencent.bk.job.common.iam.service.AppAuthService; +import com.tencent.bk.job.common.model.Response; +import com.tencent.bk.job.common.model.dto.AppResourceScope; +import com.tencent.bk.job.common.model.dto.ResourceScope; +import com.tencent.bk.job.common.service.AppScopeMappingService; +import com.tencent.bk.job.execute.api.web.WebSearchToolsResource; +import com.tencent.bk.job.execute.model.GseTaskSimpleDTO; +import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; +import com.tencent.bk.job.execute.model.StepInstanceRollingTaskDTO; +import com.tencent.bk.job.execute.model.web.vo.TaskLinkVO; +import com.tencent.bk.job.execute.service.GseTaskService; +import com.tencent.bk.job.execute.service.StepInstanceRollingTaskService; +import com.tencent.bk.job.execute.service.StepInstanceService; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.web.bind.annotation.RestController; + +import java.util.ArrayList; +import java.util.List; + +@RestController +@Slf4j +public class WebSearchToolsResourceImpl implements WebSearchToolsResource { + private final StepInstanceService stepInstanceService; + private final GseTaskService gseTaskService; + private final AppAuthService appAuthService; + private final AppScopeMappingService appScopeMappingService; + private final StepInstanceRollingTaskService stepInstanceRollingTaskService; + + /** + * 作业平台web访问地址，可配置多个，用","分隔 + */ + @Value("${job.web.url:}") + private String jobWebUrl; + + /** + * 快速执行链接地址 + */ + private static final String FAST_LINK = "%s/api_execute/%s?stepInstanceId=%s&retryCount=%s&batch=%s"; + + /** + * 作业执行链接地址 + */ + private static final String TASK_LINK = "%s/api_execute_step/%s/%s?retryCount=%s&batch=%s"; + + @Autowired + public WebSearchToolsResourceImpl(StepInstanceService stepInstanceService, + GseTaskService gseTaskService, + AppAuthService appAuthService, + AppScopeMappingService appScopeMappingService, + StepInstanceRollingTaskService stepInstanceRollingTaskService) { + this.stepInstanceService = stepInstanceService; + this.gseTaskService = gseTaskService; + this.appAuthService = appAuthService; + this.appScopeMappingService = appScopeMappingService; + this.stepInstanceRollingTaskService = stepInstanceRollingTaskService; + } + + @Override + public Response getTaskLink(String username, + String gseTaskId) { + GseTaskSimpleDTO gseTaskSimpleInfo = gseTaskService.getGseTaskSimpleInfo(gseTaskId); + if (gseTaskSimpleInfo == null) { + String errorMsg = "not found gseTask by " + gseTaskId; + log.warn(errorMsg); + return Response.buildCommonFailResp(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME_AND_REASON, + new String[]{String.valueOf(gseTaskId), errorMsg}); + } + StepInstanceBaseDTO stepInstanceBase = stepInstanceService.getBaseStepInstance( + gseTaskSimpleInfo.getTaskInstanceId(), gseTaskSimpleInfo.getStepInstanceId()); + Long appId = stepInstanceBase.getAppId(); + ResourceScope resourceScope = appScopeMappingService.getScopeByAppId(appId); + AppResourceScope appResourceScope = new AppResourceScope(appId, resourceScope); + // 鉴权 + auth(username, appResourceScope); + TaskLinkVO taskLinkVO = convertToTaskLinVO(resourceScope, + stepInstanceBase, + gseTaskSimpleInfo.getExecuteCount(), + gseTaskSimpleInfo.getBatch(), + gseTaskId); + return Response.buildSuccessResp(taskLinkVO); + } + + @Override + public Response> getTaskLinkByStepId(String username, + Long stepInstanceId) { + StepInstanceBaseDTO stepInstanceBase = stepInstanceService.getBaseStepInstanceById(stepInstanceId); + if (stepInstanceBase == null) { + String errorMsg = "not found StepInstance by " + stepInstanceId; + log.warn(errorMsg); + return Response.buildCommonFailResp(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME_AND_REASON, + new String[]{String.valueOf(stepInstanceId), errorMsg}); + } + Long appId = stepInstanceBase.getAppId(); + ResourceScope resourceScope = appScopeMappingService.getScopeByAppId(appId); + AppResourceScope appResourceScope = new AppResourceScope(appId, resourceScope); + // 鉴权 + auth(username, appResourceScope); + + List stepInstanceRollingTaskDTOS = + stepInstanceRollingTaskService.listRollingTasksByStep(stepInstanceBase.getTaskInstanceId(), stepInstanceId); + List gseTaskDTOList = new ArrayList<>(); + // 是否滚动执行 + if (CollectionUtils.isNotEmpty(stepInstanceRollingTaskDTOS)) { + for (StepInstanceRollingTaskDTO stepInstanceRollingTaskDTO : stepInstanceRollingTaskDTOS) { + List gseTaskSimpleDTOList = gseTaskService.listGseTaskSimpleInfo(stepInstanceId, + stepInstanceRollingTaskDTO.getExecuteCount(), + stepInstanceRollingTaskDTO.getBatch()); + if (CollectionUtils.isNotEmpty(gseTaskSimpleDTOList)) { + gseTaskDTOList.addAll(gseTaskSimpleDTOList); + } + } + } else { + List gseTaskSimpleDTOList = gseTaskService.listGseTaskSimpleInfo(stepInstanceId, + null, + null); + if (CollectionUtils.isNotEmpty(gseTaskSimpleDTOList)) { + gseTaskDTOList.addAll(gseTaskSimpleDTOList); + } + } + + List taskLinkVOList = new ArrayList<>(); + for (GseTaskSimpleDTO gseTaskSimpleDTO : gseTaskDTOList) { + taskLinkVOList.add(convertToTaskLinVO(resourceScope, + stepInstanceBase, + gseTaskSimpleDTO.getExecuteCount(), + gseTaskSimpleDTO.getBatch(), + gseTaskSimpleDTO.getGseTaskId())); + } + return Response.buildSuccessResp(taskLinkVOList); + } + + private AuthResult auth(String username, + AppResourceScope appResourceScope) { + AuthResult authResult = appAuthService.auth(username, + ActionId.ACCESS_BUSINESS, + appResourceScope); + if (authResult.isPass()) { + authResult = appAuthService.auth(username, + ActionId.VIEW_HISTORY, + appResourceScope); + } + if (!authResult.isPass()) { + throw new PermissionDeniedException(authResult); + } + return authResult; + } + + private TaskLinkVO convertToTaskLinVO(ResourceScope resourceScope, + StepInstanceBaseDTO stepInstanceBase, + int retryCount, + int batch, + String gseTaskId) { + TaskLinkVO taskLinkVO = new TaskLinkVO(); + taskLinkVO.setScopeType(resourceScope.getType().getValue()); + taskLinkVO.setScopeId(resourceScope.getId()); + taskLinkVO.setAppId(stepInstanceBase.getAppId()); + taskLinkVO.setJobInstanceId(stepInstanceBase.getTaskInstanceId()); + taskLinkVO.setStepInstanceId(stepInstanceBase.getId()); + taskLinkVO.setRetryCount(retryCount); + taskLinkVO.setBatch(batch); + taskLinkVO.setGseTaskId(gseTaskId); + taskLinkVO.setLink(buildLink(taskLinkVO, stepInstanceBase)); + return taskLinkVO; + } + + /** + * 拼接链接地址 + */ + private List buildLink(TaskLinkVO taskLinkVO, StepInstanceBaseDTO stepInstanceBase) { + List links = new ArrayList(); + String linkTemplate = FAST_LINK; + if (stepInstanceBase.getStepId() != -1L) { + linkTemplate = TASK_LINK; + } + if (jobWebUrl.indexOf(",") != -1) { + String[] jobWebUrls = jobWebUrl.split(","); + for (String webUrl : jobWebUrls) { + links.add(String.format(linkTemplate, + webUrl, + taskLinkVO.getJobInstanceId(), + taskLinkVO.getStepInstanceId(), + taskLinkVO.getRetryCount(), + taskLinkVO.getBatch())); + } + } else { + links.add(String.format(linkTemplate, + jobWebUrl, + taskLinkVO.getJobInstanceId(), + taskLinkVO.getStepInstanceId(), + taskLinkVO.getRetryCount(), + taskLinkVO.getBatch())); + } + return links; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskExecutionResultResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskExecutionResultResourceImpl.java index 8c0960046c..10461e4484 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskExecutionResultResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskExecutionResultResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,20 +27,22 @@ import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.job.common.constant.Bool; import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.constant.ExecuteObjectTypeEnum; import com.tencent.bk.job.common.constant.Order; import com.tencent.bk.job.common.constant.TaskVariableTypeEnum; +import com.tencent.bk.job.common.exception.FailedPreconditionException; import com.tencent.bk.job.common.exception.InvalidParamException; -import com.tencent.bk.job.common.exception.NotFoundException; +import com.tencent.bk.job.common.gse.constants.FileDistModeEnum; +import com.tencent.bk.job.common.gse.util.AgentUtils; import com.tencent.bk.job.common.i18n.service.MessageI18nService; -import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; -import com.tencent.bk.job.common.iam.model.AuthResult; -import com.tencent.bk.job.common.iam.service.WebAuthService; +import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.InternalResponse; import com.tencent.bk.job.common.model.PageData; import com.tencent.bk.job.common.model.Response; -import com.tencent.bk.job.common.model.ValidateResult; import com.tencent.bk.job.common.model.dto.AppResourceScope; import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.util.CustomCollectionUtils; @@ -49,20 +51,20 @@ import com.tencent.bk.job.common.util.ip.IpUtils; import com.tencent.bk.job.execute.api.web.WebTaskExecutionResultResource; import com.tencent.bk.job.execute.auth.ExecuteAuthService; -import com.tencent.bk.job.execute.client.ServiceNotificationResourceClient; -import com.tencent.bk.job.execute.common.constants.FileDistModeEnum; import com.tencent.bk.job.execute.common.constants.FileDistStatusEnum; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; -import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; import com.tencent.bk.job.execute.common.constants.TaskStartupModeEnum; import com.tencent.bk.job.execute.common.constants.TaskTotalTimeTypeEnum; import com.tencent.bk.job.execute.common.constants.TaskTypeEnum; -import com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum; +import com.tencent.bk.job.execute.engine.consts.ExecuteObjectTaskStatusEnum; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; import com.tencent.bk.job.execute.engine.model.TaskVariableDTO; -import com.tencent.bk.job.execute.model.AgentTaskDetailDTO; -import com.tencent.bk.job.execute.model.AgentTaskResultGroupDTO; -import com.tencent.bk.job.execute.model.FileIpLogContent; -import com.tencent.bk.job.execute.model.ScriptHostLogContent; +import com.tencent.bk.job.execute.model.AtomicFileTaskLog; +import com.tencent.bk.job.execute.model.ExecuteObjectCompositeKey; +import com.tencent.bk.job.execute.model.ExecuteObjectTask; +import com.tencent.bk.job.execute.model.FileExecuteObjectLogContent; +import com.tencent.bk.job.execute.model.ResultGroupDTO; +import com.tencent.bk.job.execute.model.ScriptExecuteObjectLogContent; import com.tencent.bk.job.execute.model.StepExecutionDTO; import com.tencent.bk.job.execute.model.StepExecutionDetailDTO; import com.tencent.bk.job.execute.model.StepExecutionRecordDTO; @@ -78,12 +80,18 @@ import com.tencent.bk.job.execute.model.VariableValueDTO; import com.tencent.bk.job.execute.model.converter.TaskInstanceConverter; import com.tencent.bk.job.execute.model.web.vo.AgentTaskExecutionVO; +import com.tencent.bk.job.execute.model.web.vo.ExecuteObjectFileLogVO; +import com.tencent.bk.job.execute.model.web.vo.ExecuteObjectScriptLogVO; +import com.tencent.bk.job.execute.model.web.vo.ExecuteObjectTaskVO; +import com.tencent.bk.job.execute.model.web.vo.ExecuteObjectVO; import com.tencent.bk.job.execute.model.web.vo.ExecuteVariableVO; +import com.tencent.bk.job.execute.model.web.vo.ExecutionResultGroupV2VO; import com.tencent.bk.job.execute.model.web.vo.ExecutionResultGroupVO; +import com.tencent.bk.job.execute.model.web.vo.FileDistributionDetailV2VO; import com.tencent.bk.job.execute.model.web.vo.FileDistributionDetailVO; -import com.tencent.bk.job.execute.model.web.vo.IpFileLogContentVO; import com.tencent.bk.job.execute.model.web.vo.IpScriptLogContentVO; import com.tencent.bk.job.execute.model.web.vo.RollingStepBatchTaskVO; +import com.tencent.bk.job.execute.model.web.vo.StepExecutionDetailV2VO; import com.tencent.bk.job.execute.model.web.vo.StepExecutionDetailVO; import com.tencent.bk.job.execute.model.web.vo.StepExecutionRecordVO; import com.tencent.bk.job.execute.model.web.vo.StepExecutionVO; @@ -91,13 +99,16 @@ import com.tencent.bk.job.execute.model.web.vo.TaskExecutionVO; import com.tencent.bk.job.execute.model.web.vo.TaskInstanceVO; import com.tencent.bk.job.execute.service.LogService; +import com.tencent.bk.job.execute.service.StepInstanceService; import com.tencent.bk.job.execute.service.StepInstanceVariableValueService; +import com.tencent.bk.job.execute.service.TaskInstanceAccessProcessor; import com.tencent.bk.job.execute.service.TaskInstanceService; import com.tencent.bk.job.execute.service.TaskInstanceVariableService; import com.tencent.bk.job.execute.service.TaskResultService; -import com.tencent.bk.job.logsvr.model.service.ServiceFileTaskLogDTO; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskStepTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskFileTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskStepTypeEnum; +import com.tencent.bk.job.manage.api.inner.ServiceNotificationResource; import com.tencent.bk.job.manage.model.inner.ServiceAppRoleDTO; import com.tencent.bk.job.manage.model.inner.ServiceNotifyChannelDTO; import lombok.extern.slf4j.Slf4j; @@ -107,9 +118,6 @@ import org.springframework.web.bind.annotation.RestController; import java.nio.charset.StandardCharsets; -import java.time.LocalDate; -import java.time.LocalDateTime; -import java.time.LocalTime; import java.time.ZoneId; import java.time.temporal.ChronoUnit; import java.util.ArrayList; @@ -122,6 +130,7 @@ import static com.tencent.bk.job.execute.constants.Consts.MAX_SEARCH_TASK_HISTORY_RANGE_MILLS; +@SuppressWarnings("Duplicates") @RestController @Slf4j public class WebTaskExecutionResultResourceImpl implements WebTaskExecutionResultResource { @@ -131,51 +140,52 @@ public class WebTaskExecutionResultResourceImpl implements WebTaskExecutionResul private final StepInstanceVariableValueService stepInstanceVariableValueService; private final TaskInstanceService taskInstanceService; private final TaskInstanceVariableService taskInstanceVariableService; - private final ServiceNotificationResourceClient notifyResource; + private final ServiceNotificationResource notifyResource; private final ExecuteAuthService executeAuthService; - private final WebAuthService webAuthService; + private final TaskInstanceAccessProcessor taskInstanceAccessProcessor; + private final StepInstanceService stepInstanceService; private final LoadingCache> roleCache = CacheBuilder.newBuilder() .maximumSize(10).expireAfterWrite(10, TimeUnit.MINUTES). - build(new CacheLoader>() { - @Override - public Map load(String lang) { - InternalResponse> resp = notifyResource.getNotifyRoles(lang); - log.info("Get notify roles, resp={}", resp); - if (!resp.isSuccess() || resp.getData() == null) { - return new HashMap<>(); - } else { - List appRoles = resp.getData(); - Map codeNameMap = new HashMap<>(); - if (appRoles != null) { - appRoles.forEach(role -> codeNameMap.put(role.getCode(), role.getName())); - } - return codeNameMap; + build(new CacheLoader>() { + @Override + public Map load(String lang) { + InternalResponse> resp = notifyResource.getNotifyRoles(lang); + log.info("Get notify roles, resp={}", resp); + if (!resp.isSuccess() || resp.getData() == null) { + return new HashMap<>(); + } else { + List appRoles = resp.getData(); + Map codeNameMap = new HashMap<>(); + if (appRoles != null) { + appRoles.forEach(role -> codeNameMap.put(role.getCode(), role.getName())); } + return codeNameMap; } } - ); + } + ); private final LoadingCache> channelCache = CacheBuilder.newBuilder() .maximumSize(10).expireAfterWrite(10, TimeUnit.MINUTES). - build(new CacheLoader>() { - @Override - public Map load(String lang) { - InternalResponse> resp = notifyResource.getNotifyChannels(lang); - log.info("Get notify channels, resp={}", resp); - if (!resp.isSuccess() || resp.getData() == null) { - return new HashMap<>(); - } else { - List channels = resp.getData(); - Map typeNameMap = new HashMap<>(); - if (channels != null) { - channels.forEach(channel -> typeNameMap.put(channel.getType(), channel.getName())); - } - return typeNameMap; + build(new CacheLoader>() { + @Override + public Map load(String lang) { + InternalResponse> resp = notifyResource.getNotifyChannels(lang); + log.info("Get notify channels, resp={}", resp); + if (!resp.isSuccess() || resp.getData() == null) { + return new HashMap<>(); + } else { + List channels = resp.getData(); + Map typeNameMap = new HashMap<>(); + if (channels != null) { + channels.forEach(channel -> typeNameMap.put(channel.getType(), channel.getName())); } + return typeNameMap; } } - ); + } + ); @Autowired public WebTaskExecutionResultResourceImpl(TaskResultService taskResultService, @@ -184,9 +194,10 @@ public WebTaskExecutionResultResourceImpl(TaskResultService taskResultService, StepInstanceVariableValueService stepInstanceVariableValueService, TaskInstanceService taskInstanceService, TaskInstanceVariableService taskInstanceVariableService, - ServiceNotificationResourceClient notifyResource, + ServiceNotificationResource notifyResource, ExecuteAuthService executeAuthService, - WebAuthService webAuthService) { + TaskInstanceAccessProcessor taskInstanceAccessProcessor, + StepInstanceService stepInstanceService) { this.taskResultService = taskResultService; this.i18nService = i18nService; this.logService = logService; @@ -195,7 +206,8 @@ public WebTaskExecutionResultResourceImpl(TaskResultService taskResultService, this.taskInstanceVariableService = taskInstanceVariableService; this.notifyResource = notifyResource; this.executeAuthService = executeAuthService; - this.webAuthService = webAuthService; + this.taskInstanceAccessProcessor = taskInstanceAccessProcessor; + this.stepInstanceService = stepInstanceService; } @Override @@ -214,19 +226,55 @@ public Response> getTaskHistoryList(String username, TaskTotalTimeTypeEnum totalTimeType, Integer start, Integer pageSize, + Boolean countPageTotal, Long cronTaskId, String startupModes, String ip) { + TaskInstanceQuery taskQuery = buildListTaskInstanceQuery(appResourceScope, taskName, taskInstanceId, + status, operator, taskType, startTime, endTime, timeRange, totalTimeType, cronTaskId, startupModes, ip); + BaseSearchCondition baseSearchCondition = BaseSearchCondition.pageCondition(start, pageSize, + countPageTotal == null ? true : countPageTotal); + + PageData pageData = taskResultService.listPageTaskInstance(taskQuery, baseSearchCondition); + if (pageData == null) { + return Response.buildSuccessResp(PageData.emptyPageData(start, pageSize)); + } + + PageData pageDataVO = new PageData<>(); + pageDataVO.setTotal(pageData.getTotal()); + pageDataVO.setStart(pageData.getStart()); + pageDataVO.setPageSize(pageData.getPageSize()); + + List taskInstanceVOS = new ArrayList<>(); + if (pageData.getData() != null) { + pageData.getData().forEach(taskInstanceDTO -> taskInstanceVOS.add(TaskInstanceConverter + .convertToTaskInstanceVO(taskInstanceDTO))); + } + pageDataVO.setData(taskInstanceVOS); + batchSetPermissionsForTaskInstance(username, appResourceScope, taskInstanceVOS); + return Response.buildSuccessResp(pageDataVO); + } + + private TaskInstanceQuery buildListTaskInstanceQuery(AppResourceScope appResourceScope, + String taskName, + Long taskInstanceId, + Integer status, + String operator, + Integer taskType, + String startTime, + String endTime, + Integer timeRange, + TaskTotalTimeTypeEnum totalTimeType, + Long cronTaskId, + String startupModes, + String ip) { TaskInstanceQuery taskQuery = new TaskInstanceQuery(); taskQuery.setTaskInstanceId(taskInstanceId); taskQuery.setAppId(appResourceScope.getAppId()); taskQuery.setTaskName(taskName); taskQuery.setCronTaskId(cronTaskId); - ValidateResult validateResult = validateAndSetQueryTimeRange(taskQuery, startTime, endTime, timeRange); - if (!validateResult.isPass()) { - return Response.buildValidateFailResp(validateResult); - } + validateAndSetQueryTimeRange(taskQuery, startTime, endTime, timeRange); setTotalTimeCondition(taskQuery, totalTimeType); taskQuery.setOperator(operator); @@ -247,47 +295,27 @@ public Response> getTaskHistoryList(String username, } } taskQuery.setIp(ip); - BaseSearchCondition baseSearchCondition = new BaseSearchCondition(); - baseSearchCondition.setStart(start); - baseSearchCondition.setLength(pageSize); - - PageData pageData = taskResultService.listPageTaskInstance(taskQuery, baseSearchCondition); - if (pageData == null) { - return Response.buildSuccessResp(PageData.emptyPageData(start, pageSize)); - } - - PageData pageDataVO = new PageData<>(); - pageDataVO.setTotal(pageData.getTotal()); - pageDataVO.setStart(pageData.getStart()); - pageDataVO.setPageSize(pageData.getPageSize()); - - List taskInstanceVOS = new ArrayList<>(); - if (pageData.getData() != null) { - pageData.getData().forEach(taskInstanceDTO -> taskInstanceVOS.add(TaskInstanceConverter - .convertToTaskInstanceVO(taskInstanceDTO))); - } - pageDataVO.setData(taskInstanceVOS); - batchSetPermissionsForTaskInstance(username, appResourceScope, taskInstanceVOS); - return Response.buildSuccessResp(pageDataVO); + return taskQuery; } - private ValidateResult validateAndSetQueryTimeRange(TaskInstanceQuery taskInstanceQuery, String startTime, - String endTime, Integer timeRange) { + private void validateAndSetQueryTimeRange(TaskInstanceQuery taskInstanceQuery, + String startTime, + String endTime, + Integer timeRange) { Long start = null; Long end = null; if (timeRange != null) { if (timeRange < 1) { log.warn("Param timeRange should greater than 0"); - return ValidateResult.fail(ErrorCode.ILLEGAL_PARAM); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } if (timeRange > 30) { log.warn("Param timeRange should less then 30"); - return ValidateResult.fail(ErrorCode.TASK_INSTANCE_QUERY_TIME_SPAN_MORE_THAN_30_DAYS); + throw new FailedPreconditionException(ErrorCode.TASK_INSTANCE_QUERY_TIME_SPAN_MORE_THAN_30_DAYS); } - // 当天结束时间 - long todayMaxMills = LocalDateTime.of(LocalDate.now(), LocalTime.MAX).getSecond() * 1000L; - start = todayMaxMills - 30 * 24 * 3600 * 1000L; + // 当天结束时间 - 往前的天数 + start = DateUtils.getUTCCurrentDayEndTimestamp() - timeRange * 24 * 3600 * 1000L; } else { if (StringUtils.isNotBlank(startTime)) { start = DateUtils.convertUnixTimestampFromDateTimeStr(startTime, "yyyy-MM-dd HH:mm:ss", @@ -300,20 +328,19 @@ private ValidateResult validateAndSetQueryTimeRange(TaskInstanceQuery taskInstan if (start == null) { log.info("StartTime should not be empty!"); - return ValidateResult.fail(ErrorCode.ILLEGAL_PARAM); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } if (end == null) { end = System.currentTimeMillis(); } if (end - start > MAX_SEARCH_TASK_HISTORY_RANGE_MILLS) { log.info("Query task instance history time span must be less than 30 days"); - return ValidateResult.fail(ErrorCode.TASK_INSTANCE_QUERY_TIME_SPAN_MORE_THAN_30_DAYS); + throw new FailedPreconditionException(ErrorCode.TASK_INSTANCE_QUERY_TIME_SPAN_MORE_THAN_30_DAYS); } } taskInstanceQuery.setStartTime(start); taskInstanceQuery.setEndTime(end); - return ValidateResult.pass(); } private void setTotalTimeCondition(TaskInstanceQuery taskQuery, TaskTotalTimeTypeEnum totalTimeType) { @@ -364,6 +391,7 @@ private void batchSetPermissionsForTaskInstance(String username, AppResourceScop @Override + @AuditEntry(actionId = ActionId.VIEW_HISTORY) public Response getTaskExecutionResult(String username, AppResourceScope appResourceScope, String scopeType, @@ -400,6 +428,7 @@ private TaskExecuteResultVO convertToTaskExecuteResultVO(TaskExecuteResultDTO ta StepExecutionVO stepExecutionVO = new StepExecutionVO(); stepExecutionVO.setName(stepExecutionDTO.getName()); stepExecutionVO.setRetryCount(stepExecutionDTO.getExecuteCount()); + stepExecutionVO.setExecuteCount(stepExecutionDTO.getExecuteCount()); stepExecutionVO.setStepInstanceId(stepExecutionDTO.getStepInstanceId()); stepExecutionVO.setStartTime(stepExecutionDTO.getStartTime()); stepExecutionVO.setEndTime(stepExecutionDTO.getEndTime()); @@ -429,7 +458,7 @@ private void fillConfirmStepDetail(StepExecutionVO stepExecutionVO, StepExecutio stepExecutionVO.setUserList(stepExecutionDTO.getConfirmUsers()); if (stepExecutionDTO.getConfirmRoles() != null && !stepExecutionDTO.getConfirmRoles().isEmpty()) { List roleNames = new ArrayList<>(); - Map roleCodeAndName = null; + Map roleCodeAndName; try { roleCodeAndName = roleCache.get(JobContextUtil.getUserLang()); } catch (Exception e) { @@ -471,6 +500,7 @@ private void fillConfirmStepDetail(StepExecutionVO stepExecutionVO, StepExecutio } @Override + @AuditEntry(actionId = ActionId.VIEW_HISTORY) public Response getStepExecutionResult(String username, AppResourceScope appResourceScope, String scopeType, @@ -494,7 +524,7 @@ public Response getStepExecutionResult(String username, .tag(tag) .logKeyword(keyword) .searchIp(searchIp) - .maxAgentTasksForResultGroup(maxIpsPerResultGroup) + .maxTasksForResultGroup(maxIpsPerResultGroup) .orderField(orderField) .order(Order.valueOf(order)) .build(); @@ -504,31 +534,6 @@ public Response getStepExecutionResult(String username, return Response.buildSuccessResp(convertToStepInstanceExecutionDetailVO(executionResult)); } - @Override - public Response getFastTaskStepExecutionResult(String username, - AppResourceScope appResourceScope, - String scopeType, - String scopeId, - Long taskInstanceId, - Integer batch, - Integer resultType, - String tag, - Integer maxIpsPerResultGroup, - String orderField, - Integer order) { - StepExecutionResultQuery query = StepExecutionResultQuery.builder() - .batch(batch) - .status(resultType) - .tag(tag) - .maxAgentTasksForResultGroup(maxIpsPerResultGroup) - .orderField(orderField) - .order(Order.valueOf(order)) - .build(); - StepExecutionDetailDTO executionResult = taskResultService.getFastTaskStepExecutionResult(username, - appResourceScope.getAppId(), taskInstanceId, query); - return Response.buildSuccessResp(convertToStepInstanceExecutionDetailVO(executionResult)); - } - private StepExecutionDetailVO convertToStepInstanceExecutionDetailVO(StepExecutionDetailDTO executionDetail) { StepExecutionDetailVO stepExecutionDetailVO = new StepExecutionDetailVO(); stepExecutionDetailVO.setFinished(executionDetail.isFinished()); @@ -546,32 +551,35 @@ private StepExecutionDetailVO convertToStepInstanceExecutionDetailVO(StepExecuti stepExecutionDetailVO.setRunMode(executionDetail.getRunMode().getValue()); List resultGroupVOS = new ArrayList<>(); - for (AgentTaskResultGroupDTO resultGroup : executionDetail.getResultGroups()) { + for (ResultGroupDTO resultGroup : executionDetail.getResultGroups()) { ExecutionResultGroupVO executionResultGroupVO = new ExecutionResultGroupVO(); executionResultGroupVO.setResultType(resultGroup.getStatus()); executionResultGroupVO.setResultTypeDesc( - i18nService.getI18n(AgentTaskStatusEnum.valueOf(resultGroup.getStatus()).getI18nKey())); + i18nService.getI18n(ExecuteObjectTaskStatusEnum.valOf(resultGroup.getStatus()).getI18nKey())); executionResultGroupVO.setTag(resultGroup.getTag()); - executionResultGroupVO.setAgentTaskSize(resultGroup.getTotalAgentTasks()); + executionResultGroupVO.setAgentTaskSize(resultGroup.getTotal()); List agentTaskExecutionVOS = new ArrayList<>(); - if (resultGroup.getAgentTasks() != null) { - for (AgentTaskDetailDTO agentTask : resultGroup.getAgentTasks()) { + if (resultGroup.getExecuteObjectTasks() != null) { + for (ExecuteObjectTask executeObjectTask : resultGroup.getExecuteObjectTasks()) { + HostDTO host = executeObjectTask.getExecuteObject().getHost(); AgentTaskExecutionVO agentTaskVO = new AgentTaskExecutionVO(); - agentTaskVO.setIp(agentTask.getCloudIp()); - agentTaskVO.setDisplayIp(agentTask.getDisplayIp()); - agentTaskVO.setEndTime(agentTask.getEndTime()); - agentTaskVO.setStartTime(agentTask.getStartTime()); - agentTaskVO.setStatus(agentTask.getStatus().getValue()); - agentTaskVO.setStatusDesc(i18nService.getI18n(agentTask.getStatus().getI18nKey())); - agentTaskVO.setErrorCode(agentTask.getErrorCode()); - agentTaskVO.setExitCode(agentTask.getExitCode()); - agentTaskVO.setTag(agentTask.getTag()); - agentTaskVO.setTotalTime(agentTask.getTotalTime()); - agentTaskVO.setCloudAreaId(agentTask.getBkCloudId()); - agentTaskVO.setCloudAreaName(agentTask.getBkCloudName()); - agentTaskVO.setRetryCount(agentTask.getExecuteCount()); - agentTaskVO.setBatch(agentTask.getBatch()); + agentTaskVO.setHostId(host.getHostId()); + agentTaskVO.setAgentId(AgentUtils.displayAsRealAgentId(host.getAgentId())); + agentTaskVO.setIpv4(host.getIp()); + agentTaskVO.setIpv6(host.getIpv6()); + agentTaskVO.setEndTime(executeObjectTask.getEndTime()); + agentTaskVO.setStartTime(executeObjectTask.getStartTime()); + agentTaskVO.setStatus(executeObjectTask.getStatus().getValue()); + agentTaskVO.setStatusDesc(i18nService.getI18n(executeObjectTask.getStatus().getI18nKey())); + agentTaskVO.setErrorCode(executeObjectTask.getErrorCode()); + agentTaskVO.setExitCode(executeObjectTask.getExitCode()); + agentTaskVO.setTag(executeObjectTask.getTag()); + agentTaskVO.setTotalTime(executeObjectTask.getTotalTime()); + agentTaskVO.setCloudAreaId(host.getBkCloudId()); + agentTaskVO.setCloudAreaName(host.getBkCloudName()); + agentTaskVO.setRetryCount(executeObjectTask.getExecuteCount()); + agentTaskVO.setBatch(executeObjectTask.getBatch()); agentTaskExecutionVOS.add(agentTaskVO); } } @@ -589,6 +597,65 @@ private StepExecutionDetailVO convertToStepInstanceExecutionDetailVO(StepExecuti return stepExecutionDetailVO; } + private StepExecutionDetailV2VO convertToStepInstanceExecutionDetailV2VO(StepExecutionDetailDTO executionDetail) { + StepExecutionDetailV2VO stepExecutionDetailVO = new StepExecutionDetailV2VO(); + stepExecutionDetailVO.setExecuteObjectType( + executionDetail.getStepInstance().determineStepExecuteObjectType()); + stepExecutionDetailVO.setFinished(executionDetail.isFinished()); + stepExecutionDetailVO.setName(executionDetail.getName()); + stepExecutionDetailVO.setStepInstanceId(executionDetail.getStepInstanceId()); + stepExecutionDetailVO.setExecuteCount(executionDetail.getExecuteCount()); + stepExecutionDetailVO.setStatus(executionDetail.getStatus().getValue()); + stepExecutionDetailVO.setStatusDesc( + i18nService.getI18n(executionDetail.getStatus().getI18nKey())); + stepExecutionDetailVO.setStartTime(executionDetail.getStartTime()); + stepExecutionDetailVO.setEndTime(executionDetail.getEndTime()); + stepExecutionDetailVO.setTotalTime(executionDetail.getTotalTime()); + stepExecutionDetailVO.setIsLastStep(executionDetail.isLastStep()); + stepExecutionDetailVO.setType(executionDetail.getStepType().getValue()); + stepExecutionDetailVO.setRunMode(executionDetail.getRunMode().getValue()); + + List resultGroupVOS = new ArrayList<>(); + for (ResultGroupDTO resultGroup : executionDetail.getResultGroups()) { + ExecutionResultGroupV2VO executionResultGroupVO = new ExecutionResultGroupV2VO(); + executionResultGroupVO.setResultType(resultGroup.getStatus()); + executionResultGroupVO.setResultTypeDesc( + i18nService.getI18n(ExecuteObjectTaskStatusEnum.valOf(resultGroup.getStatus()).getI18nKey())); + executionResultGroupVO.setTag(resultGroup.getTag()); + executionResultGroupVO.setTaskSize(resultGroup.getTotal()); + + List executeObjectTaskVOs = new ArrayList<>(); + if (resultGroup.getExecuteObjectTasks() != null) { + for (ExecuteObjectTask executeObjectTask : resultGroup.getExecuteObjectTasks()) { + ExecuteObjectTaskVO executeObjectTaskVO = new ExecuteObjectTaskVO(); + executeObjectTaskVO.setEndTime(executeObjectTask.getEndTime()); + executeObjectTaskVO.setStartTime(executeObjectTask.getStartTime()); + executeObjectTaskVO.setStatus(executeObjectTask.getStatus().getValue()); + executeObjectTaskVO.setStatusDesc(i18nService.getI18n(executeObjectTask.getStatus().getI18nKey())); + executeObjectTaskVO.setErrorCode(executeObjectTask.getErrorCode()); + executeObjectTaskVO.setExitCode(executeObjectTask.getExitCode()); + executeObjectTaskVO.setTag(executeObjectTask.getTag()); + executeObjectTaskVO.setTotalTime(executeObjectTask.getTotalTime()); + executeObjectTaskVO.setExecuteCount(executeObjectTask.getExecuteCount()); + executeObjectTaskVO.setBatch(executeObjectTask.getBatch()); + executeObjectTaskVO.setExecuteObject(executeObjectTask.getExecuteObject().toExecuteObjectVO()); + executeObjectTaskVOs.add(executeObjectTaskVO); + } + } + executionResultGroupVO.setTasks(executeObjectTaskVOs); + + resultGroupVOS.add(executionResultGroupVO); + } + stepExecutionDetailVO.setResultGroups(resultGroupVOS); + + if (CollectionUtils.isNotEmpty(executionDetail.getRollingTasks())) { + stepExecutionDetailVO.setRollingTasks(toRollingStepBatchTaskVOs(executionDetail.getLatestBatch(), + executionDetail.getRollingTasks())); + } + + return stepExecutionDetailVO; + } + private List toRollingStepBatchTaskVOs(Integer latestBatch, List stepInstanceRollingTasks) { return stepInstanceRollingTasks.stream().map(stepInstanceRollingTask -> { @@ -601,143 +668,163 @@ private List toRollingStepBatchTaskVOs(Integer latestBat } @Override - public Response getScriptLogContentByIp(String username, - AppResourceScope appResourceScope, - String scopeType, - String scopeId, - Long stepInstanceId, - Integer executeCount, - String ip, - Integer batch) { - - if (stepInstanceId == null || executeCount == null || ip == null) { - log.warn("Get ip log content, param is illegal!"); - throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); - } - if (!IpUtils.checkCloudIp(ip)) { - log.warn("Get ip log content, param ip is illegal! ip={}", ip); - throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); - } - - authViewStepInstance(username, appResourceScope, stepInstanceId); - - ScriptHostLogContent scriptHostLogContent = logService.getScriptHostLogContent(stepInstanceId, executeCount, - batch, HostDTO.fromCloudIp(ip)); + @AuditEntry(actionId = ActionId.VIEW_HISTORY) + public Response getScriptLogContentByHost(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + Long stepInstanceId, + Integer executeCount, + Long hostId, + Integer batch) { + StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstanceById(stepInstanceId); + auditAndAuthViewStepInstance(username, appResourceScope, stepInstance); + + ScriptExecuteObjectLogContent scriptExecuteObjectLogContent = + logService.getScriptExecuteObjectLogContent(stepInstance, executeCount, + batch, ExecuteObjectCompositeKey.ofHostId(hostId)); IpScriptLogContentVO ipScriptLogContentVO = new IpScriptLogContentVO(); - ipScriptLogContentVO.setDisplayIp(ip); - if (scriptHostLogContent != null) { - ipScriptLogContentVO.setLogContent(scriptHostLogContent.getContent()); - ipScriptLogContentVO.setFinished(scriptHostLogContent.isFinished()); + if (scriptExecuteObjectLogContent != null) { + ipScriptLogContentVO.setLogContent(scriptExecuteObjectLogContent.getContent()); + ipScriptLogContentVO.setFinished(scriptExecuteObjectLogContent.isFinished()); } return Response.buildSuccessResp(ipScriptLogContentVO); } - private AuthResult authViewStepInstance(String username, AppResourceScope appResourceScope, - StepInstanceBaseDTO stepInstance) { - String operator = stepInstance.getOperator(); - if (username.equals(operator)) { - return AuthResult.pass(); - } - AuthResult authResult = executeAuthService.authViewTaskInstance( - username, appResourceScope, stepInstance.getTaskInstanceId()); - if (!authResult.isPass()) { - authResult.setApplyUrl(webAuthService.getApplyUrl(authResult.getRequiredActionResources())); - } - return authResult; - } - - @Override - public Response> getStepVariableByIp(String username, - AppResourceScope appResourceScope, - String scopeType, - String scopeId, - Long stepInstanceId, - String ip) { - StepInstanceDTO stepInstance = taskInstanceService.getStepInstanceDetail(stepInstanceId); - if (stepInstance == null) { - return Response.buildSuccessResp(Collections.emptyList()); - } - if (!stepInstance.getExecuteType().equals(StepExecuteTypeEnum.EXECUTE_SCRIPT.getValue()) - || !stepInstance.getScriptType().equals(ScriptTypeEnum.SHELL.getValue())) { + @AuditEntry(actionId = ActionId.VIEW_HISTORY) + public Response> getStepVariableByHost(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + Long stepInstanceId, + Long hostId, + String ip) { + StepInstanceDTO stepInstance = stepInstanceService.getStepInstanceDetail( + null, stepInstanceId); + if (!stepInstance.isScriptStep() || stepInstance.getScriptType() != ScriptTypeEnum.SHELL) { return Response.buildSuccessResp(Collections.emptyList()); } - AuthResult authResult = authViewStepInstance(username, appResourceScope, stepInstance); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } + taskInstanceAccessProcessor.processBeforeAccess(username, + appResourceScope.getAppId(), stepInstance.getTaskInstanceId()); + + List taskVariableVOS = getStepVariableByExecuteObject(stepInstance, + ExecuteObjectTypeEnum.HOST, hostId); + return Response.buildSuccessResp(taskVariableVOS); + } + private List getStepVariableByExecuteObject(StepInstanceBaseDTO stepInstance, + ExecuteObjectTypeEnum executeObjectType, + Long executeObjectResourceId) { List taskVars = taskInstanceVariableService.getByTaskInstanceId(stepInstance.getTaskInstanceId()); if (taskVars == null || taskVars.isEmpty()) { - return Response.buildSuccessResp(Collections.emptyList()); + return Collections.emptyList(); } List taskVariableVOS = new ArrayList<>(); - List changeableVarNames = new ArrayList<>(); - List namespaceVarNames = new ArrayList<>(); - Map taskVariablesMap = new HashMap<>(); + List changeableVars = new ArrayList<>(); for (TaskVariableDTO taskVar : taskVars) { - taskVariablesMap.put(taskVar.getName(), taskVar); + // 主机变量无需返回 if (taskVar.getType() == TaskVariableTypeEnum.HOST_LIST.getType()) { continue; } if (!taskVar.isChangeable()) { taskVariableVOS.add(convertToTaskVariableVO(taskVar)); } else { - changeableVarNames.add(taskVar.getName()); - if (taskVar.getType() == TaskVariableTypeEnum.NAMESPACE.getType()) { - namespaceVarNames.add(taskVar.getName()); - } + changeableVars.add(taskVar); } } - if (!changeableVarNames.isEmpty()) { - StepInstanceVariableValuesDTO inputStepInstanceValues = stepInstanceVariableValueService - .computeInputStepInstanceVariableValues(stepInstance.getTaskInstanceId(), stepInstanceId, taskVars); - if (inputStepInstanceValues == null) { - taskVars.stream().filter(var -> !var.getType().equals(TaskVariableTypeEnum.HOST_LIST.getType()) && - var.isChangeable()).forEach(var -> taskVariableVOS.add(convertToTaskVariableVO(var))); - return Response.buildSuccessResp(taskVariableVOS); - } + if (CollectionUtils.isNotEmpty(changeableVars)) { + appendStepChangeableVars(taskVariableVOS, stepInstance, changeableVars, + executeObjectType, executeObjectResourceId); + } + + return taskVariableVOS; + } + + private void appendStepChangeableVars(List taskVariableVOS, + StepInstanceBaseDTO stepInstance, + List changeableVars, + ExecuteObjectTypeEnum executeObjectType, + Long executeObjectResourceId) { + StepInstanceVariableValuesDTO inputStepInstanceValues = stepInstanceVariableValueService + .computeInputStepInstanceVariableValues(stepInstance, changeableVars); + if (inputStepInstanceValues == null) { + changeableVars.stream().filter(var -> !var.getType().equals(TaskVariableTypeEnum.HOST_LIST.getType()) && + var.isChangeable()).forEach(var -> taskVariableVOS.add(convertToTaskVariableVO(var))); + return; + } + + Map taskVariablesMap = new HashMap<>(); + for (TaskVariableDTO taskVar : changeableVars) { + taskVariablesMap.put(taskVar.getName(), taskVar); + } + + // 命名空间变量 + if (executeObjectType == ExecuteObjectTypeEnum.HOST) { + // 当前只有主机支持命名空间变量 + appendNamespaceVars(taskVariableVOS, changeableVars, inputStepInstanceValues, + taskVariablesMap, executeObjectResourceId); + } + + // 全局变量(除命名空间变量) + appendGlobalChangeableVars(taskVariableVOS, inputStepInstanceValues, taskVariablesMap); + } - namespaceVarNames.forEach(paramName -> { + private void appendNamespaceVars(List taskVariableVOS, + List changeableVars, + StepInstanceVariableValuesDTO inputStepInstanceValues, + Map taskVariablesMap, + Long hostId) { + Map hostVariables = new HashMap<>(); + if (inputStepInstanceValues.getNamespaceParamsMap() != null + && !inputStepInstanceValues.getNamespaceParamsMap().isEmpty()) { + // Map + inputStepInstanceValues.getNamespaceParamsMap() + .forEach((host, hostVars) -> { + if (host.getHostId() != null && host.getHostId().equals(hostId)) { + hostVars.forEach(hostVariables::put); + } + }); + } + changeableVars + .stream() + .filter(var -> var.getType().equals(TaskVariableTypeEnum.NAMESPACE.getType())) + .forEach(var -> { ExecuteVariableVO vo = new ExecuteVariableVO(); - vo.setName(paramName); - String paramValue = (inputStepInstanceValues.getNamespaceParamsMap() != null - && inputStepInstanceValues.getNamespaceParamsMap().get(ip) != null - && inputStepInstanceValues.getNamespaceParamsMap().get(ip).get(paramName) != null) - ? inputStepInstanceValues.getNamespaceParamsMap().get(ip).get(paramName).getValue() - : taskVariablesMap.get(paramName).getValue(); - vo.setValue(paramValue); - vo.setChangeable(1); + String varName = var.getName(); + vo.setName(varName); + String varValue = hostVariables.get(varName) != null + ? hostVariables.get(varName).getValue() : taskVariablesMap.get(varName).getValue(); + vo.setValue(varValue); + vo.setChangeable(Bool.TRUE.intValue()); vo.setType(TaskVariableTypeEnum.NAMESPACE.getType()); taskVariableVOS.add(vo); }); + } - List globalVars = inputStepInstanceValues.getGlobalParams(); - if (globalVars != null) { - for (VariableValueDTO varValue : globalVars) { - if (varValue.getType().equals(TaskVariableTypeEnum.HOST_LIST.getType())) { - // 过滤掉主机变量 - continue; - } - TaskVariableDTO taskVariable = taskVariablesMap.get(varValue.getName()); - if (taskVariable == null || !taskVariable.isChangeable()) { - // 过滤掉常量 - continue; - } - - ExecuteVariableVO vo = new ExecuteVariableVO(); - vo.setName(varValue.getName()); - vo.setValue(varValue.getValue()); - vo.setChangeable(1); - vo.setType(taskVariable.getType()); - taskVariableVOS.add(vo); + private void appendGlobalChangeableVars(List taskVariableVOS, + StepInstanceVariableValuesDTO inputStepInstanceValues, + Map taskVariablesMap) { + List globalVars = inputStepInstanceValues.getGlobalParams(); + if (CollectionUtils.isNotEmpty(globalVars)) { + for (VariableValueDTO varValue : globalVars) { + TaskVariableDTO taskVariable = taskVariablesMap.get(varValue.getName()); + if (taskVariable == null || !taskVariable.isChangeable()) { + // 过滤掉常量 + continue; } + + ExecuteVariableVO vo = new ExecuteVariableVO(); + vo.setName(varValue.getName()); + vo.setValue(varValue.getValue()); + vo.setChangeable(Bool.TRUE.intValue()); + vo.setType(taskVariable.getType()); + taskVariableVOS.add(vo); } } - return Response.buildSuccessResp(taskVariableVOS); } private ExecuteVariableVO convertToTaskVariableVO(TaskVariableDTO taskVariable) { @@ -756,70 +843,6 @@ private ExecuteVariableVO convertToTaskVariableVO(TaskVariableDTO taskVariable) return vo; } - @Override - @SuppressWarnings({"unchecked", "rawtypes"}) - public Response getFileLogContentByIp(String username, - AppResourceScope appResourceScope, - String scopeType, - String scopeId, - Long stepInstanceId, - Integer executeCount, - String ip, - String mode, - Integer batch) { - - if (stepInstanceId == null || executeCount == null || ip == null) { - log.warn("Get ip log content, param is illegal!"); - throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); - } - if (!IpUtils.checkCloudIp(ip)) { - log.warn("Get ip log content, param ip is illegal! ip={}", ip); - throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); - } - - authViewStepInstance(username, appResourceScope, stepInstanceId); - - IpFileLogContentVO result = new IpFileLogContentVO(); - List fileDistDetailVOS = new ArrayList<>(); - result.setFileDistributionDetails(fileDistDetailVOS); - - if ("download".equals(mode)) { - FileIpLogContent downloadLog = logService.getFileIpLogContent(stepInstanceId, executeCount, batch, - HostDTO.fromCloudIp(ip), FileDistModeEnum.DOWNLOAD.getValue()); - // downloadLog为null说明步骤还未下发至GSE就被终止 - if (downloadLog != null && CollectionUtils.isNotEmpty(downloadLog.getFileTaskLogs())) { - downloadLog.getFileTaskLogs().forEach(fileLog -> { - if (fileLog.getMode().equals(FileDistModeEnum.UPLOAD.getValue())) { - return; - } - fileDistDetailVOS.add(convertToFileDistributionDetailVO(fileLog)); - }); - result.setFinished(downloadLog.isFinished()); - } - Collections.sort(fileDistDetailVOS); - } else { - List fileTaskLogs = logService.batchGetFileSourceIpLogContent(stepInstanceId, - executeCount, batch); - if (CollectionUtils.isNotEmpty(fileTaskLogs)) { - fileTaskLogs.forEach(fileTaskLog -> { - if (fileTaskLog.getMode().equals(FileDistModeEnum.DOWNLOAD.getValue())) { - return; - } - fileDistDetailVOS.add(convertToFileDistributionDetailVO(fileTaskLog)); - }); - Collections.sort(fileDistDetailVOS); - result.setFinished(fileTaskLogs.stream().noneMatch(fileLog -> - (fileLog.getStatus().equals(FileDistStatusEnum.DOWNLOADING.getValue()) - || fileLog.getStatus().equals(FileDistStatusEnum.UPLOADING.getValue()) - || fileLog.getStatus().equals(FileDistStatusEnum.WAITING.getValue())) - || fileLog.getStatus().equals(FileDistStatusEnum.PULLING.getValue()))); - } - } - boolean includingLogContent = !removeFileLogContentIfResultIsLarge(fileDistDetailVOS); - result.setIncludingLogContent(includingLogContent); - return Response.buildSuccessResp(result); - } - private boolean removeFileLogContentIfResultIsLarge(List fileDistDetailVOS) { // 超过128K boolean removeFileLogContent = calculateFileLogContentLength(fileDistDetailVOS) > 131072L; @@ -840,63 +863,93 @@ private long calculateFileLogContentLength(List fileDi return length; } - private FileDistributionDetailVO convertToFileDistributionDetailVO(ServiceFileTaskLogDTO fileLog) { + private boolean removeFileLogContentIfLarge(List fileDistDetailVOS) { + // 超过128K + boolean removeFileLogContent = sumFileLogContentLength(fileDistDetailVOS) > 131072L; + if (removeFileLogContent) { + fileDistDetailVOS.forEach(fileDistributionDetailVO -> { + fileDistributionDetailVO.setLogContent(null); + }); + } + return removeFileLogContent; + } + + private long sumFileLogContentLength(List fileDistDetailVOS) { + long length = 0; + for (FileDistributionDetailV2VO fileDistributionDetailVO : fileDistDetailVOS) { + length += (StringUtils.isEmpty(fileDistributionDetailVO.getLogContent()) ? + 0 : fileDistributionDetailVO.getLogContent().getBytes(StandardCharsets.UTF_8).length); + } + return length; + } + + private FileDistributionDetailVO convertToFileDistributionDetailVO(AtomicFileTaskLog fileLog) { FileDistributionDetailVO fileDistDetailVO = new FileDistributionDetailVO(); fileDistDetailVO.setTaskId(fileLog.getTaskId()); fileDistDetailVO.setMode(fileLog.getMode()); + if (FileDistModeEnum.UPLOAD.getValue().equals(fileLog.getMode())) { - fileDistDetailVO.setSrcIp(fileLog.getDisplaySrcIp()); fileDistDetailVO.setFileName(fileLog.getDisplaySrcFile()); } else { - fileDistDetailVO.setSrcIp(fileLog.getDisplaySrcIp()); - fileDistDetailVO.setDestIp(fileLog.getDestIp()); + fileDistDetailVO.setDestIp(IpUtils.extractIp(fileLog.getDestExecuteObject().getHost().getIp())); + fileDistDetailVO.setDestIpv6(IpUtils.extractIp(fileLog.getDestExecuteObject().getHost().getIpv6())); fileDistDetailVO.setFileName(fileLog.getDestFile()); } + boolean hideSrcIp = fileLog.getSrcFileType() != null + && TaskFileTypeEnum.valueOf(fileLog.getSrcFileType()) != TaskFileTypeEnum.SERVER; + fileDistDetailVO.setSrcIp(hideSrcIp ? "--" : + IpUtils.extractIp(fileLog.getSrcExecuteObject().getHost().getIp())); + fileDistDetailVO.setSrcIpv6(hideSrcIp ? "--" : + IpUtils.extractIp(fileLog.getSrcExecuteObject().getHost().getIpv6())); fileDistDetailVO.setFileSize(fileLog.getSize()); fileDistDetailVO.setProgress(fileLog.getProcess()); fileDistDetailVO.setSpeed(fileLog.getSpeed()); FileDistStatusEnum fileDistStatus = FileDistStatusEnum.getFileDistStatus(fileLog.getStatus()); - fileDistDetailVO.setStatusDesc(fileDistStatus != null ? fileDistStatus.getName() : ""); + fileDistDetailVO.setStatusDesc(fileDistStatus.getName()); fileDistDetailVO.setStatus(fileLog.getStatus()); fileDistDetailVO.setLogContent(fileLog.getContent()); return fileDistDetailVO; } - @Override - public Response> getFileLogContentByFileTaskIds(String username, - AppResourceScope appResourceScope, - String scopeType, - String scopeId, - Long stepInstanceId, - Integer executeCount, - Integer batch, - List taskIds) { - authViewStepInstance(username, appResourceScope, stepInstanceId); - - List fileTaskLogs = logService.getFileLogContentByTaskIds(stepInstanceId, executeCount, - batch, taskIds); - if (CollectionUtils.isEmpty(fileTaskLogs)) { - return Response.buildSuccessResp(null); + private FileDistributionDetailV2VO convertToFileDistributionDetailV2VO(AtomicFileTaskLog fileLog) { + FileDistributionDetailV2VO fileDistDetailVO = new FileDistributionDetailV2VO(); + fileDistDetailVO.setTaskId(fileLog.getTaskId()); + fileDistDetailVO.setMode(fileLog.getMode()); + if (fileLog.getSrcExecuteObject() != null) { + fileDistDetailVO.setSrcExecuteObject(fileLog.getSrcExecuteObject().toExecuteObjectVO()); + } else { + log.warn("Missing src execute object. fileLog: {}", fileLog); } - List fileDistDetailVOS = new ArrayList<>(); - fileTaskLogs.forEach(fileLog -> { - fileDistDetailVOS.add(convertToFileDistributionDetailVO(fileLog)); - }); - return Response.buildSuccessResp(fileDistDetailVOS); + if (FileDistModeEnum.UPLOAD.getValue().equals(fileLog.getMode())) { + fileDistDetailVO.setFileName(fileLog.getDisplaySrcFile()); + } else { + if (fileLog.getDestExecuteObject() != null) { + fileDistDetailVO.setDestExecuteObject(fileLog.getDestExecuteObject().toExecuteObjectVO()); + } else { + log.warn("Missing dest execute object. fileLog: {}", fileLog); + } + fileDistDetailVO.setFileName(fileLog.getDestFile()); + } + + fileDistDetailVO.setFileSize(fileLog.getSize()); + fileDistDetailVO.setProgress(fileLog.getProcess()); + fileDistDetailVO.setSpeed(fileLog.getSpeed()); + FileDistStatusEnum fileDistStatus = FileDistStatusEnum.getFileDistStatus(fileLog.getStatus()); + fileDistDetailVO.setStatusDesc(fileDistStatus.getName()); + fileDistDetailVO.setStatus(fileLog.getStatus()); + fileDistDetailVO.setLogContent(fileLog.getContent()); + return fileDistDetailVO; } - private void authViewStepInstance(String username, AppResourceScope appResourceScope, Long stepInstanceId) { - StepInstanceBaseDTO stepInstance = taskInstanceService.getBaseStepInstance(stepInstanceId); - if (stepInstance == null) { - throw new NotFoundException(ErrorCode.STEP_INSTANCE_NOT_EXIST); - } - AuthResult authResult = authViewStepInstance(username, appResourceScope, stepInstance); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } + private void auditAndAuthViewStepInstance(String username, + AppResourceScope appResourceScope, + StepInstanceBaseDTO stepInstance) { + taskInstanceAccessProcessor.processBeforeAccess(username, + appResourceScope.getAppId(), stepInstance.getTaskInstanceId()); } @Override + @AuditEntry(actionId = ActionId.VIEW_HISTORY) public Response> getHostsByResultType(String username, AppResourceScope appResourceScope, String scopeType, @@ -907,28 +960,291 @@ public Response> getHostsByResultType(String username, Integer resultType, String tag, String keyword) { - List hosts = taskResultService.getHostsByResultType(username, appResourceScope.getAppId(), - stepInstanceId, executeCount, batch, resultType, tag, keyword); - return Response.buildSuccessResp(hosts); + List executeObjects = taskResultService.getExecuteObjectsByResultType( + username, + appResourceScope.getAppId(), + null, + stepInstanceId, + executeCount, + batch, + resultType, + tag, + keyword + ); + if (CollectionUtils.isEmpty(executeObjects)) { + return Response.buildSuccessResp(Collections.emptyList()); + } + return Response.buildSuccessResp( + executeObjects.stream() + .map(ExecuteObject::getHost) + .collect(Collectors.toList()) + ); } @Override + @AuditEntry(actionId = ActionId.VIEW_HISTORY) public Response> listStepExecutionHistory(String username, AppResourceScope appResourceScope, String scopeType, String scopeId, Long stepInstanceId, Integer batch) { + // 兼容代码，部署完成后删除 + StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstanceById(stepInstanceId); + return listStepExecutionHistoryV2(username, appResourceScope, scopeType, scopeId, + stepInstance.getTaskInstanceId(), stepInstanceId, batch); + } + @Override + public Response> listStepExecutionHistoryV2(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + Long taskInstanceId, + Long stepInstanceId, + Integer batch) { List stepExecutionRecords = taskResultService.listStepExecutionHistory(username, - appResourceScope.getAppId(), stepInstanceId, batch); + appResourceScope.getAppId(), taskInstanceId, stepInstanceId, batch); return Response.buildSuccessResp(stepExecutionRecords.stream().map(stepExecutionRecord -> { StepExecutionRecordVO vo = new StepExecutionRecordVO(); + vo.setTaskInstanceId(taskInstanceId); vo.setStepInstanceId(stepInstanceId); vo.setRetryCount(stepExecutionRecord.getRetryCount()); + vo.setExecuteCount(stepExecutionRecord.getRetryCount()); vo.setCreateTime(stepExecutionRecord.getCreateTime()); return vo; }).collect(Collectors.toList())); } + + @Override + @AuditEntry(actionId = ActionId.VIEW_HISTORY) + public Response getStepExecutionResult(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch, + Integer resultType, + String tag, + Integer maxTasksPerResultGroup, + String keyword, + String searchIp, + String orderField, + Integer order) { + StepExecutionResultQuery query = StepExecutionResultQuery.builder() + .taskInstanceId(taskInstanceId) + .stepInstanceId(stepInstanceId) + .executeCount(executeCount) + .batch(batch == null ? null : (batch == 0 ? null : batch)) + .filterByLatestBatch(batch == null) + .status(resultType) + .tag(tag) + .logKeyword(keyword) + .searchIp(searchIp) + .maxTasksForResultGroup(maxTasksPerResultGroup) + .orderField(orderField) + .order(Order.valueOf(order)) + .build(); + + StepExecutionDetailDTO executionResult = taskResultService.getStepExecutionResult(username, + appResourceScope.getAppId(), query); + return Response.buildSuccessResp(convertToStepInstanceExecutionDetailV2VO(executionResult)); + } + + @Override + @AuditEntry(actionId = ActionId.VIEW_HISTORY) + public Response getScriptLogContentByExecuteObject(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + Long taskInstanceId, + Long stepInstanceId, + Integer executeObjectType, + Long executeObjectResourceId, + Integer executeCount, + Integer batch) { + StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstance( + appResourceScope.getAppId(), taskInstanceId, stepInstanceId); + auditAndAuthViewStepInstance(username, appResourceScope, stepInstance); + + int actualExecuteCount = computeActualExecuteCount(stepInstance, executeCount); + ScriptExecuteObjectLogContent scriptExecuteObjectLogContent = + logService.getScriptExecuteObjectLogContent(stepInstance, actualExecuteCount, batch, + ExecuteObjectCompositeKey.ofExecuteObjectResource( + ExecuteObjectTypeEnum.valOf(executeObjectType), executeObjectResourceId)); + ExecuteObjectScriptLogVO executeObjectScriptLogVO = new ExecuteObjectScriptLogVO(); + if (scriptExecuteObjectLogContent != null) { + executeObjectScriptLogVO.setLogContent(scriptExecuteObjectLogContent.getContent()); + executeObjectScriptLogVO.setFinished(scriptExecuteObjectLogContent.isFinished()); + executeObjectScriptLogVO.setExecuteObject( + scriptExecuteObjectLogContent.getExecuteObject().toExecuteObjectVO()); + } + return Response.buildSuccessResp(executeObjectScriptLogVO); + } + + private int computeActualExecuteCount(StepInstanceBaseDTO stepInstance, Integer queryExecuteCount) { + return queryExecuteCount == null ? stepInstance.getExecuteCount() : queryExecuteCount; + } + + @Override + @AuditEntry(actionId = ActionId.VIEW_HISTORY) + public Response getFileLogContentByExecuteObject(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + Long taskInstanceId, + Long stepInstanceId, + Integer executeObjectType, + Long executeObjectResourceId, + Integer executeCount, + Integer batch, + Integer mode) { + StepInstanceDTO stepInstance = + stepInstanceService.getStepInstanceDetail(taskInstanceId, stepInstanceId); + auditAndAuthViewStepInstance(username, appResourceScope, stepInstance); + + ExecuteObjectFileLogVO result = new ExecuteObjectFileLogVO(); + List fileDistDetailVOS = new ArrayList<>(); + + FileDistModeEnum fileDistMode = FileDistModeEnum.getFileDistMode(mode); + int actualExecuteCount = computeActualExecuteCount(stepInstance, executeCount); + switch (fileDistMode) { + case DOWNLOAD: + FileExecuteObjectLogContent downloadLog = + logService.getFileExecuteObjectLogContent( + stepInstance, + actualExecuteCount, + batch, + ExecuteObjectCompositeKey.ofExecuteObjectResource( + ExecuteObjectTypeEnum.valOf(executeObjectType), executeObjectResourceId), + FileDistModeEnum.DOWNLOAD.getValue()); + if (downloadLog == null || CollectionUtils.isEmpty(downloadLog.getFileTaskLogs())) { + log.info("Can not find file execute object download log, stepInstanceId: {}, executeCount: {}, " + + "batch: {}, executeObjectId: {}", + stepInstanceId, actualExecuteCount, batch, executeObjectType + ":" + executeObjectResourceId); + break; + } + fileDistDetailVOS = + downloadLog.getFileTaskLogs().stream() + .map(this::convertToFileDistributionDetailV2VO) + .collect(Collectors.toList()); + result.setFinished(downloadLog.isFinished()); + break; + case UPLOAD: + List executeObjectLogContents = + logService.batchGetFileSourceExecuteObjectLogContent(taskInstanceId, stepInstanceId, + actualExecuteCount, batch); + if (CollectionUtils.isEmpty(executeObjectLogContents)) { + log.info("Can not find file execute object upload log, stepInstanceId: {}, executeCount: {}, " + + "batch: {}, executeObjectId: {}", + stepInstanceId, actualExecuteCount, batch, executeObjectType + ":" + executeObjectResourceId); + break; + } + fileDistDetailVOS = + executeObjectLogContents.stream() + .flatMap(executeObjectLogContent -> executeObjectLogContent.getFileTaskLogs().stream()) + .map(this::convertToFileDistributionDetailV2VO) + .collect(Collectors.toList()); + result.setFinished( + executeObjectLogContents.stream() + .flatMap(executeObjectLogContent -> executeObjectLogContent.getFileTaskLogs().stream()) + .allMatch(fileLog -> FileDistStatusEnum.isFinishedStatus(fileLog.getStatus()))); + break; + } + Collections.sort(fileDistDetailVOS); + result.setFileDistributionDetails(fileDistDetailVOS); + + boolean includingLogContent = !removeFileLogContentIfLarge(fileDistDetailVOS); + result.setIncludingLogContent(includingLogContent); + + return Response.buildSuccessResp(result); + } + + @Override + @AuditEntry(actionId = ActionId.VIEW_HISTORY) + public Response> getFileLogContentByFileTaskIds(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch, + List taskIds) { + StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstance( + taskInstanceId, stepInstanceId); + auditAndAuthViewStepInstance(username, appResourceScope, stepInstance); + + int actualExecuteCount = computeActualExecuteCount(stepInstance, executeCount); + List fileTaskLogs = logService.getAtomicFileTaskLogByTaskIds(taskInstanceId, stepInstanceId, + actualExecuteCount, batch, taskIds); + if (CollectionUtils.isEmpty(fileTaskLogs)) { + return Response.buildSuccessResp(null); + } + List fileDistDetailVOS = new ArrayList<>(); + fileTaskLogs.forEach(fileLog -> fileDistDetailVOS.add(convertToFileDistributionDetailV2VO(fileLog))); + return Response.buildSuccessResp(fileDistDetailVOS); + } + + @Override + @AuditEntry(actionId = ActionId.VIEW_HISTORY) + public Response> getStepVariableByExecuteObject(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + Long taskInstanceId, + Long stepInstanceId, + Integer executeObjectType, + Long executeObjectResourceId) { + StepInstanceDTO stepInstance = stepInstanceService.getStepInstanceDetail( + taskInstanceId, stepInstanceId); + if (!stepInstance.isScriptStep() || stepInstance.getScriptType() != ScriptTypeEnum.SHELL) { + return Response.buildSuccessResp(Collections.emptyList()); + } + + taskInstanceAccessProcessor.processBeforeAccess(username, + appResourceScope.getAppId(), stepInstance.getTaskInstanceId()); + + List taskVariableVOS = getStepVariableByExecuteObject(stepInstance, + ExecuteObjectTypeEnum.valOf(executeObjectType), executeObjectResourceId); + return Response.buildSuccessResp(taskVariableVOS); + } + + @Override + @AuditEntry(actionId = ActionId.VIEW_HISTORY) + public Response> getExecuteObjectsByResultType(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch, + Integer resultType, + String tag, + String keyword) { + List executeObjects = + taskResultService.getExecuteObjectsByResultType( + username, + appResourceScope.getAppId(), + taskInstanceId, + stepInstanceId, + executeCount, + batch, + resultType, + tag, + keyword + ); + if (CollectionUtils.isEmpty(executeObjects)) { + return Response.buildSuccessResp(Collections.emptyList()); + } + return Response.buildSuccessResp( + executeObjects.stream() + .map(ExecuteObject::toExecuteObjectVO) + .collect(Collectors.toList()) + ); + } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskInstanceResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskInstanceResourceImpl.java index ce448fe37f..d6c5494228 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskInstanceResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskInstanceResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,57 +24,56 @@ package com.tencent.bk.job.execute.api.web.impl; -import com.tencent.bk.job.common.constant.DuplicateHandlerEnum; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.CompatibleType; import com.tencent.bk.job.common.constant.ErrorCode; -import com.tencent.bk.job.common.constant.NotExistPathHandlerEnum; import com.tencent.bk.job.common.constant.TaskVariableTypeEnum; import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.exception.NotFoundException; import com.tencent.bk.job.common.i18n.service.MessageI18nService; +import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; import com.tencent.bk.job.common.iam.model.AuthResult; import com.tencent.bk.job.common.iam.service.BusinessAuthService; import com.tencent.bk.job.common.model.Response; import com.tencent.bk.job.common.model.dto.AppResourceScope; -import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.common.model.vo.TaskTargetVO; import com.tencent.bk.job.common.util.Base64Util; import com.tencent.bk.job.execute.api.web.WebTaskInstanceResource; -import com.tencent.bk.job.execute.auth.ExecuteAuthService; import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; import com.tencent.bk.job.execute.common.constants.TaskStartupModeEnum; import com.tencent.bk.job.execute.constants.UserOperationEnum; import com.tencent.bk.job.execute.engine.model.TaskVariableDTO; +import com.tencent.bk.job.execute.model.ExecuteTargetDTO; import com.tencent.bk.job.execute.model.FileSourceDTO; import com.tencent.bk.job.execute.model.OperationLogDTO; import com.tencent.bk.job.execute.model.RollingConfigDTO; -import com.tencent.bk.job.execute.model.ServersDTO; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.model.converter.TaskInstanceConverter; import com.tencent.bk.job.execute.model.db.RollingConfigDetailDO; import com.tencent.bk.job.execute.model.web.vo.ExecuteApprovalStepVO; -import com.tencent.bk.job.execute.model.web.vo.ExecuteCloudAreaInfoVO; import com.tencent.bk.job.execute.model.web.vo.ExecuteFileDestinationInfoVO; import com.tencent.bk.job.execute.model.web.vo.ExecuteFileSourceInfoVO; import com.tencent.bk.job.execute.model.web.vo.ExecuteFileStepVO; -import com.tencent.bk.job.execute.model.web.vo.ExecuteHostVO; import com.tencent.bk.job.execute.model.web.vo.ExecuteScriptStepVO; -import com.tencent.bk.job.execute.model.web.vo.ExecuteServersVO; import com.tencent.bk.job.execute.model.web.vo.ExecuteStepVO; -import com.tencent.bk.job.execute.model.web.vo.ExecuteTargetVO; import com.tencent.bk.job.execute.model.web.vo.ExecuteVariableVO; import com.tencent.bk.job.execute.model.web.vo.RollingConfigVO; import com.tencent.bk.job.execute.model.web.vo.TaskInstanceDetailVO; import com.tencent.bk.job.execute.model.web.vo.TaskInstanceVO; import com.tencent.bk.job.execute.model.web.vo.TaskOperationLogVO; -import com.tencent.bk.job.execute.service.HostService; import com.tencent.bk.job.execute.service.RollingConfigService; +import com.tencent.bk.job.execute.service.StepInstanceService; +import com.tencent.bk.job.execute.service.TaskInstanceAccessProcessor; import com.tencent.bk.job.execute.service.TaskInstanceService; import com.tencent.bk.job.execute.service.TaskInstanceVariableService; import com.tencent.bk.job.execute.service.TaskOperationLogService; -import com.tencent.bk.job.manage.common.consts.task.TaskFileTypeEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskStepTypeEnum; +import com.tencent.bk.job.execute.util.FileTransferModeUtil; +import com.tencent.bk.job.manage.api.common.constants.task.TaskFileTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskStepTypeEnum; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -90,51 +89,59 @@ public class WebTaskInstanceResourceImpl implements WebTaskInstanceResource { private final TaskInstanceService taskInstanceService; private final TaskInstanceVariableService taskInstanceVariableService; - private final HostService hostService; private final TaskOperationLogService taskOperationLogService; private final MessageI18nService i18nService; - private final ExecuteAuthService executeAuthService; private final BusinessAuthService businessAuthService; private final RollingConfigService rollingConfigService; + private final TaskInstanceAccessProcessor taskInstanceAccessProcessor; + private final StepInstanceService stepInstanceService; @Autowired public WebTaskInstanceResourceImpl(TaskInstanceService taskInstanceService, TaskInstanceVariableService taskInstanceVariableService, - HostService hostService, TaskOperationLogService taskOperationLogService, MessageI18nService i18nService, - ExecuteAuthService executeAuthService, BusinessAuthService businessAuthService, - RollingConfigService rollingConfigService) { + RollingConfigService rollingConfigService, + TaskInstanceAccessProcessor taskInstanceAccessProcessor, + StepInstanceService stepInstanceService) { this.taskInstanceService = taskInstanceService; this.taskInstanceVariableService = taskInstanceVariableService; - this.hostService = hostService; this.taskOperationLogService = taskOperationLogService; this.i18nService = i18nService; - this.executeAuthService = executeAuthService; this.businessAuthService = businessAuthService; this.rollingConfigService = rollingConfigService; + this.taskInstanceAccessProcessor = taskInstanceAccessProcessor; + this.stepInstanceService = stepInstanceService; } @Override + @AuditEntry(actionId = ActionId.VIEW_HISTORY) + @CompatibleImplementation(name = "dao_add_task_instance_id", deprecatedVersion = "3.11.x", + type = CompatibleType.DEPLOY, explain = "发布完成后可以删除") public Response getStepInstanceDetail(String username, AppResourceScope appResourceScope, String scopeType, String scopeId, Long stepInstanceId) { + // 兼容代码，部署完成后删除 + StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstanceById(stepInstanceId); + return getStepInstanceDetailV2(username, appResourceScope, scopeType, scopeId, + stepInstance.getTaskInstanceId(), stepInstanceId); + } - StepInstanceDTO stepInstance = taskInstanceService.getStepInstanceDetail(stepInstanceId); - if (stepInstance == null) { - throw new NotFoundException(ErrorCode.STEP_INSTANCE_NOT_EXIST); - } - if (!stepInstance.getAppId().equals(appResourceScope.getAppId())) { - log.warn("StepInstance:{} is not in app:{}", stepInstanceId, appResourceScope.getAppId()); - throw new NotFoundException(ErrorCode.STEP_INSTANCE_NOT_EXIST); - } - AuthResult authResult = authViewStepInstance(username, appResourceScope, stepInstance); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } + @Override + @AuditEntry(actionId = ActionId.VIEW_HISTORY) + public Response getStepInstanceDetailV2(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + Long taskInstanceId, + Long stepInstanceId) { + StepInstanceDTO stepInstance = stepInstanceService.getStepInstanceDetail(taskInstanceId, stepInstanceId); + + taskInstanceAccessProcessor.processBeforeAccess(username, + appResourceScope.getAppId(), stepInstance.getTaskInstanceId()); ExecuteStepVO stepVO = convertToStepVO(stepInstance); fillRollingConfigForRollingStep(stepVO, stepInstance); @@ -146,7 +153,8 @@ private void fillRollingConfigForRollingStep(ExecuteStepVO stepVO, StepInstanceB if (stepInstance.isRollingStep()) { RollingConfigVO rollingConfigVO = new RollingConfigVO(); RollingConfigDTO rollingConfigDTO = - rollingConfigService.getRollingConfig(stepInstance.getRollingConfigId()); + rollingConfigService.getRollingConfig(stepInstance.getTaskInstanceId(), + stepInstance.getRollingConfigId()); RollingConfigDetailDO rollingConfig = rollingConfigDTO.getConfigDetail(); rollingConfigVO.setMode(rollingConfig.getMode()); if (rollingConfigDTO.isBatchRollingStep(stepInstance.getId())) { @@ -157,21 +165,6 @@ private void fillRollingConfigForRollingStep(ExecuteStepVO stepVO, StepInstanceB } } - private AuthResult authViewTaskInstance(String username, AppResourceScope appResourceScope, - TaskInstanceDTO taskInstance) { - return executeAuthService.authViewTaskInstance(username, appResourceScope, taskInstance); - } - - private AuthResult authViewStepInstance(String username, AppResourceScope appResourceScope, - StepInstanceDTO stepInstance) { - String operator = stepInstance.getOperator(); - if (username.equals(operator)) { - return AuthResult.pass(); - } - return executeAuthService.authViewTaskInstance( - username, appResourceScope, stepInstance.getTaskInstanceId()); - } - private void convertFileSources(ExecuteFileStepVO fileStepVO, StepInstanceDTO stepInstance) { List fileSources = new ArrayList<>(); for (FileSourceDTO fileSource : stepInstance.getFileSourceList()) { @@ -214,7 +207,7 @@ private void convertFileSources(ExecuteFileStepVO fileStepVO, StepInstanceDTO st }); fileSourceVO.setFileLocation(files); } - fileSourceVO.setHost(convertToServers(fileSource.getServers())); + fileSourceVO.setHost(fileSource.getServers().convertToTaskTargetVO()); fileSources.add(fileSourceVO); } fileStepVO.setFileSourceList(fileSources); @@ -223,7 +216,7 @@ private void convertFileSources(ExecuteFileStepVO fileStepVO, StepInstanceDTO st private ExecuteStepVO convertToStepVO(StepInstanceDTO stepInstance) { ExecuteStepVO stepVO = new ExecuteStepVO(); stepVO.setName(stepInstance.getName()); - StepExecuteTypeEnum stepType = StepExecuteTypeEnum.valueOf(stepInstance.getExecuteType()); + StepExecuteTypeEnum stepType = stepInstance.getExecuteType(); if (stepType == StepExecuteTypeEnum.EXECUTE_SCRIPT || stepType == StepExecuteTypeEnum.EXECUTE_SQL) { stepVO.setType(TaskStepTypeEnum.SCRIPT.getValue()); ExecuteScriptStepVO scriptStepVO = new ExecuteScriptStepVO(); @@ -246,12 +239,13 @@ private ExecuteStepVO convertToStepVO(StepInstanceDTO stepInstance) { } scriptStepVO.setSecureParam(0); } + scriptStepVO.setWindowsInterpreter(stepInstance.getWindowsInterpreter()); scriptStepVO.setTimeout(stepInstance.getTimeout()); - scriptStepVO.setScriptLanguage(stepInstance.getScriptType()); + scriptStepVO.setScriptLanguage(stepInstance.getScriptType().getValue()); scriptStepVO.setScriptSource(stepInstance.getScriptSource()); scriptStepVO.setScriptId(stepInstance.getScriptId()); scriptStepVO.setScriptVersionId(stepInstance.getScriptVersionId()); - scriptStepVO.setExecuteTarget(convertToServers(stepInstance.getTargetServers())); + scriptStepVO.setExecuteTarget(stepInstance.getTargetExecuteObjects().convertToTaskTargetVO()); scriptStepVO.setIgnoreError(stepInstance.isIgnoreError() ? 1 : 0); stepVO.setScriptStepInfo(scriptStepVO); } else if (stepType == StepExecuteTypeEnum.SEND_FILE) { @@ -266,13 +260,15 @@ private ExecuteStepVO convertToStepVO(StepInstanceDTO stepInstance) { } else { fileDestinationInfoVO.setPath(stepInstance.getFileTargetPath()); } - fileDestinationInfoVO.setServer(convertToServers(stepInstance.getTargetServers())); + fileDestinationInfoVO.setServer(stepInstance.getTargetExecuteObjects().convertToTaskTargetVO()); fileStepVO.setFileDestination(fileDestinationInfoVO); fileStepVO.setIgnoreError(stepInstance.isIgnoreError() ? 1 : 0); - fileStepVO.setTransferMode( - ExecuteFileStepVO.getTransferMode(DuplicateHandlerEnum.valueOf(stepInstance.getFileDuplicateHandle()), - NotExistPathHandlerEnum.valueOf(stepInstance.getNotExistPathHandler()))); + Integer transferMode = FileTransferModeUtil.getTransferMode( + stepInstance.getFileDuplicateHandle(), + stepInstance.getNotExistPathHandler() + ).getValue(); + fileStepVO.setTransferMode(transferMode); if (stepInstance.getFileDownloadSpeedLimit() != null) { fileStepVO.setTargetSpeedLimit(stepInstance.getFileDownloadSpeedLimit() >> 10); } @@ -294,58 +290,27 @@ private ExecuteStepVO convertToStepVO(StepInstanceDTO stepInstance) { return stepVO; } - private ExecuteTargetVO convertToServers(ServersDTO serversDTO) { - if (serversDTO == null) { - return null; - } - ExecuteTargetVO targetServer = new ExecuteTargetVO(); - targetServer.setVariable(serversDTO.getVariable()); - ExecuteServersVO taskHostNodeVO = new ExecuteServersVO(); - if (serversDTO.getIpList() != null) { - List hosts = new ArrayList<>(); - for (HostDTO ip : serversDTO.getIpList()) { - ExecuteHostVO host = new ExecuteHostVO(); - ExecuteCloudAreaInfoVO cloudAreaInfoVO = new ExecuteCloudAreaInfoVO(ip.getBkCloudId(), ip.getIp()); - host.setIp(ip.getIp()); - host.setAlive(ip.getAlive()); - host.setCloudAreaInfo(cloudAreaInfoVO); - hosts.add(host); - } - taskHostNodeVO.setIpList(hosts); - targetServer.setHostNodeInfo(taskHostNodeVO); - } - return targetServer; - } - @Override + @AuditEntry(actionId = ActionId.VIEW_HISTORY) public Response> getTaskInstanceVariables(String username, AppResourceScope appResourceScope, String scopeType, String scopeId, Long taskInstanceId) { - TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(taskInstanceId); - if (taskInstance == null || !taskInstance.getAppId().equals(appResourceScope.getAppId())) { - log.warn("TaskInstance:{} is not in app:{}", taskInstanceId, appResourceScope.getAppId()); - throw new NotFoundException(ErrorCode.TASK_INSTANCE_NOT_EXIST); - } - - AuthResult authResult = authViewTaskInstance(username, appResourceScope, taskInstance); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } + taskInstanceAccessProcessor.processBeforeAccess(username, appResourceScope.getAppId(), taskInstanceId); List taskVariables = taskInstanceVariableService.getByTaskInstanceId(taskInstanceId); List variableVOS = new ArrayList<>(); if (taskVariables != null) { taskVariables.forEach(variable -> { - variableVOS.add(convertToVariableVO(appResourceScope.getAppId(), variable)); + variableVOS.add(convertToVariableVO(variable)); }); } return Response.buildSuccessResp(variableVOS); } - private ExecuteVariableVO convertToVariableVO(long appId, TaskVariableDTO variable) { + private ExecuteVariableVO convertToVariableVO(TaskVariableDTO variable) { ExecuteVariableVO vo = new ExecuteVariableVO(); vo.setId(variable.getId()); vo.setName(variable.getName()); @@ -353,24 +318,11 @@ private ExecuteVariableVO convertToVariableVO(long appId, TaskVariableDTO variab vo.setChangeable(variable.isChangeable() ? 1 : 0); vo.setRequired(variable.isRequired() ? 1 : 0); if (variable.getType() == TaskVariableTypeEnum.HOST_LIST.getType()) { - ServersDTO servers = variable.getTargetServers(); - ExecuteTargetVO taskTargetVO = new ExecuteTargetVO(); - if (servers.getIpList() != null) { - ExecuteServersVO taskHostNodeVO = new ExecuteServersVO(); - List hosts = new ArrayList<>(servers.getIpList().size()); - for (HostDTO ip : servers.getIpList()) { - ExecuteHostVO host = new ExecuteHostVO(); - host.setIp(ip.getIp()); - host.setAlive(ip.getAlive()); - ExecuteCloudAreaInfoVO cloudAreaInfoVO = new ExecuteCloudAreaInfoVO(ip.getBkCloudId(), - hostService.getCloudAreaName(ip.getBkCloudId())); - host.setCloudAreaInfo(cloudAreaInfoVO); - hosts.add(host); - } - taskHostNodeVO.setIpList(hosts); - taskTargetVO.setHostNodeInfo(taskHostNodeVO); + ExecuteTargetDTO executeTarget = variable.getExecuteTarget(); + if (executeTarget != null && executeTarget.getExecuteObjectsCompatibly() != null) { + TaskTargetVO taskTargetVO = executeTarget.convertToTaskTargetVO(); + vo.setTargetValue(taskTargetVO); } - vo.setTargetValue(taskTargetVO); } else if (variable.getType().equals(TaskVariableTypeEnum.CIPHER.getType())) { vo.setValue("******"); } else { @@ -380,22 +332,14 @@ private ExecuteVariableVO convertToVariableVO(long appId, TaskVariableDTO variab } @Override + @AuditEntry(actionId = ActionId.VIEW_HISTORY) public Response> getTaskInstanceOperationLog(String username, AppResourceScope appResourceScope, String scopeType, String scopeId, Long taskInstanceId) { - TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(taskInstanceId); - if (taskInstance == null || !taskInstance.getAppId().equals(appResourceScope.getAppId())) { - log.warn("TaskInstance:{} is not in app:{}", taskInstanceId, appResourceScope.getAppId()); - throw new NotFoundException(ErrorCode.TASK_INSTANCE_NOT_EXIST); - } - - AuthResult authResult = authViewTaskInstance(username, appResourceScope, taskInstance); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } + taskInstanceAccessProcessor.processBeforeAccess(username, appResourceScope.getAppId(), taskInstanceId); List operationLogs = taskOperationLogService.listOperationLog(taskInstanceId); if (operationLogs == null || operationLogs.isEmpty()) { @@ -424,7 +368,7 @@ public Response> getTaskInstanceOperationLog(String use private String buildDetail(UserOperationEnum operationCode, OperationLogDTO.OperationDetail detail) { switch (operationCode) { case START: - if (detail.getStartupMode().equals(TaskStartupModeEnum.NORMAL.getValue())) { + if (detail.getStartupMode().equals(TaskStartupModeEnum.WEB.getValue())) { return i18nService.getI18n("user.operation.detail.start.web"); } else if (detail.getStartupMode().equals(TaskStartupModeEnum.CRON.getValue())) { return i18nService.getI18n("user.operation.detail.start.cron"); @@ -456,22 +400,16 @@ private String buildDetail(UserOperationEnum operationCode, OperationLogDTO.Oper } @Override + @AuditEntry(actionId = ActionId.VIEW_HISTORY) public Response getTaskInstanceDetail(String username, AppResourceScope appResourceScope, String scopeType, String scopeId, Long taskInstanceId) { - TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstanceDetail(taskInstanceId); - if (taskInstance == null || !taskInstance.getAppId().equals(appResourceScope.getAppId())) { - log.warn("TaskInstance:{} is not in app:{}", taskInstanceId, appResourceScope.getAppId()); - throw new NotFoundException(ErrorCode.TASK_INSTANCE_NOT_EXIST); - } + TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstanceDetail(username, + appResourceScope.getAppId(), taskInstanceId); - AuthResult authResult = authViewTaskInstance(username, appResourceScope, taskInstance); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } return Response.buildSuccessResp(convertToTaskInstanceDetailVO(taskInstance)); } @@ -492,7 +430,7 @@ private TaskInstanceDetailVO convertToTaskInstanceDetailVO(TaskInstanceDTO taskI if (taskInstanceDTO.getVariables() != null && !taskInstanceDTO.getVariables().isEmpty()) { List taskVariableVOS = new ArrayList<>(); taskInstanceDTO.getVariables().forEach(variable -> { - taskVariableVOS.add(convertToVariableVO(taskInstanceDTO.getAppId(), variable)); + taskVariableVOS.add(convertToVariableVO(variable)); }); taskInstanceDetailVO.setVariables(taskVariableVOS); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskLogResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskLogResourceImpl.java index d94b0132f2..d5d522c2cb 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskLogResourceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebTaskLogResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,32 +24,36 @@ package com.tencent.bk.job.execute.api.web.impl; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.job.common.artifactory.config.ArtifactoryConfig; import com.tencent.bk.job.common.artifactory.model.dto.NodeDTO; import com.tencent.bk.job.common.artifactory.sdk.ArtifactoryClient; import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.constant.ExecuteObjectTypeEnum; import com.tencent.bk.job.common.constant.JobConstants; import com.tencent.bk.job.common.exception.InternalException; import com.tencent.bk.job.common.exception.NotFoundException; +import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.model.Response; import com.tencent.bk.job.common.model.dto.AppResourceScope; import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.execute.api.web.WebTaskLogResource; -import com.tencent.bk.job.execute.config.ArtifactoryConfig; import com.tencent.bk.job.execute.config.LogExportConfig; -import com.tencent.bk.job.execute.config.StorageSystemConfig; +import com.tencent.bk.job.execute.config.StorageConfig; import com.tencent.bk.job.execute.engine.consts.FileDirTypeConf; import com.tencent.bk.job.execute.engine.util.NFSUtils; import com.tencent.bk.job.execute.model.LogExportJobInfoDTO; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.web.vo.LogExportJobInfoVO; import com.tencent.bk.job.execute.service.LogExportService; -import com.tencent.bk.job.execute.service.TaskInstanceService; +import com.tencent.bk.job.execute.service.StepInstanceService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; import org.apache.http.client.methods.HttpRequestBase; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.http.HttpHeaders; import org.springframework.http.MediaType; import org.springframework.http.ResponseEntity; @@ -67,22 +71,22 @@ @Slf4j public class WebTaskLogResourceImpl implements WebTaskLogResource { private final String logFileDir; - private final TaskInstanceService taskInstanceService; + private final StepInstanceService stepInstanceService; private final LogExportService logExportService; private final ArtifactoryClient artifactoryClient; private final ArtifactoryConfig artifactoryConfig; private final LogExportConfig logExportConfig; @Autowired - public WebTaskLogResourceImpl(TaskInstanceService taskInstanceService, - StorageSystemConfig storageSystemConfig, + public WebTaskLogResourceImpl(StepInstanceService stepInstanceService, + StorageConfig storageConfig, LogExportService logExportService, - ArtifactoryClient artifactoryClient, + @Qualifier("jobArtifactoryClient") ArtifactoryClient artifactoryClient, ArtifactoryConfig artifactoryConfig, LogExportConfig logExportConfig) { - this.taskInstanceService = taskInstanceService; + this.stepInstanceService = stepInstanceService; this.logExportService = logExportService; - this.logFileDir = NFSUtils.getFileDir(storageSystemConfig.getJobStorageRootPath(), + this.logFileDir = NFSUtils.getFileDir(storageConfig.getJobStorageRootPath(), FileDirTypeConf.JOB_INSTANCE_PATH); this.artifactoryClient = artifactoryClient; this.artifactoryConfig = artifactoryConfig; @@ -113,13 +117,15 @@ private boolean existsLogZipFileInArtifactory(String zipFileName) { } @Override + @AuditEntry(actionId = ActionId.VIEW_HISTORY) public Response requestDownloadLogFile(String username, AppResourceScope appResourceScope, String scopeType, String scopeId, + Long taskInstanceId, Long stepInstanceId, - Long hostId, - String cloudIp, + Integer executeObjectType, + Long executeObjectResourceId, Boolean repackage) { Long appId = appResourceScope.getAppId(); @@ -127,14 +133,25 @@ public Response requestDownloadLogFile(String username, repackage = false; } - StepInstanceBaseDTO stepInstance = taskInstanceService.getBaseStepInstance(stepInstanceId); + StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstance(taskInstanceId, stepInstanceId); if (!stepInstance.getAppId().equals(appResourceScope.getAppId())) { throw new NotFoundException(ErrorCode.STEP_INSTANCE_NOT_EXIST); } + boolean getByExecuteObject = executeObjectResourceId != null; + ExecuteObjectTypeEnum executeObjectTypeEnum = null; + if (getByExecuteObject) { + executeObjectTypeEnum = ExecuteObjectTypeEnum.valOf(executeObjectType); + } + if (!repackage) { - log.debug("Do not need repackage, check exist job"); - LogExportJobInfoDTO exportInfo = logExportService.getExportInfo(appId, stepInstanceId, hostId, cloudIp); + if (log.isDebugEnabled()) { + log.debug("Do not need repackage, check exist job " + + "|stepInstanceId={}|executeObjectType={}|executeObjectResourceId={}", + stepInstanceId, executeObjectType, executeObjectResourceId); + } + LogExportJobInfoDTO exportInfo = logExportService.getExportInfo(appId, stepInstanceId, + executeObjectTypeEnum, executeObjectResourceId); if (exportInfo != null) { log.debug("Find exist job info|{}", exportInfo); switch (exportInfo.getStatus()) { @@ -157,6 +174,7 @@ public Response requestDownloadLogFile(String username, if (!repackage) { return Response.buildSuccessResp(LogExportJobInfoDTO.toVO(exportInfo)); } + log.warn("Not exist zip file, needs to be repackaged! |{}", exportInfo); break; default: throw new InternalException(ErrorCode.EXPORT_STEP_EXECUTION_LOG_FAIL); @@ -166,18 +184,36 @@ public Response requestDownloadLogFile(String username, int executeCount = stepInstance.getExecuteCount(); - String logFileName = getLogFileName(stepInstanceId, hostId, cloudIp, executeCount); + String logFileName = getLogFileName(stepInstanceId, executeCount, + executeObjectTypeEnum, executeObjectResourceId); if (StringUtils.isBlank(logFileName)) { + log.warn("Log File Name is blank! request fail! " + + "|stepInstanceId={}|executeObjectType={}|executeObjectResourceId={}|executeCount={}", + stepInstanceId, executeObjectType, executeObjectResourceId, executeCount); throw new InternalException(ErrorCode.EXPORT_STEP_EXECUTION_LOG_FAIL); } - LogExportJobInfoDTO exportInfo = logExportService.packageLogFile(username, appId, stepInstanceId, hostId, cloudIp, - executeCount, logFileDir, logFileName, repackage); + LogExportJobInfoDTO exportInfo = logExportService.packageLogFile( + username, + appId, + taskInstanceId, + stepInstanceId, + executeObjectTypeEnum, + executeObjectResourceId, + executeCount, + logFileDir, + logFileName, + repackage + ); return Response.buildSuccessResp(LogExportJobInfoDTO.toVO(exportInfo)); } - private String getLogFileName(Long stepInstanceId, Long hostId, String cloudIp, int executeCount) { - String fileName = makeExportLogFileName(stepInstanceId, executeCount, hostId, cloudIp); + private String getLogFileName(Long stepInstanceId, + int executeCount, + ExecuteObjectTypeEnum executeObjectType, + Long executeObjectResourceId) { + String fileName = makeExportLogFileName(stepInstanceId, executeCount, + executeObjectType, executeObjectResourceId); String logFileName = fileName + ".log"; File dir = new File(logFileDir); @@ -238,17 +274,19 @@ private Pair getFileSizeAndStreamFromArtifactory( } @Override + @AuditEntry(actionId = ActionId.VIEW_HISTORY) public ResponseEntity downloadLogFile(HttpServletResponse response, String username, AppResourceScope appResourceScope, String scopeType, String scopeId, + Long taskInstanceId, Long stepInstanceId, - Long hostId, - String cloudIp) { + Integer executeObjectType, + Long executeObjectResourceId) { Long appId = appResourceScope.getAppId(); - StepInstanceBaseDTO stepInstance = taskInstanceService.getBaseStepInstance(stepInstanceId); + StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstance(taskInstanceId, stepInstanceId); if (!stepInstance.getAppId().equals(appId)) { log.info("StepInstance: {} is not in app: {}", stepInstance.getId(), appResourceScope.getAppId()); return ResponseEntity.notFound().build(); @@ -258,16 +296,31 @@ public ResponseEntity downloadLogFile(HttpServletResponse LogExportJobInfoDTO exportInfo; - boolean isGetByHost = hostId != null || StringUtils.isNotBlank(cloudIp); - if (isGetByHost) { - String logFileName = getLogFileName(stepInstanceId, hostId, cloudIp, executeCount); + boolean isGetByExecuteObject = executeObjectResourceId != null; + if (isGetByExecuteObject) { + ExecuteObjectTypeEnum executeObjectTypeEnum = ExecuteObjectTypeEnum.valOf(executeObjectType); + String logFileName = getLogFileName(stepInstanceId, executeCount, + executeObjectTypeEnum, executeObjectResourceId); if (StringUtils.isBlank(logFileName)) { + log.warn("Log File Name is blank! download fail! " + + "|stepInstanceId={}|executeObjectType={}|executeObjectResourceId={}|executeCount={}", + stepInstanceId, executeObjectType, executeObjectResourceId, executeCount); return ResponseEntity.notFound().build(); } - exportInfo = logExportService.packageLogFile(username, appId, stepInstanceId, hostId, cloudIp, executeCount, - logFileDir, logFileName, false); + exportInfo = logExportService.packageLogFile( + username, + appId, + taskInstanceId, + stepInstanceId, + executeObjectTypeEnum, + executeObjectResourceId, + executeCount, + logFileDir, + logFileName, + false + ); } else { - exportInfo = logExportService.getExportInfo(appId, stepInstanceId, hostId, cloudIp); + exportInfo = logExportService.getExportInfo(appId, stepInstanceId, null, null); } if (exportInfo != null) { @@ -310,17 +363,20 @@ public ResponseEntity downloadLogFile(HttpServletResponse default: } } + log.warn("Not exist job info.|appId={}|stepInstanceId={}|executeObjectType={}|executeObjectResourceId={}", + appId, stepInstanceId, executeObjectType, executeObjectResourceId); return ResponseEntity.notFound().build(); } - private String makeExportLogFileName(Long stepInstanceId, Integer executeCount, Long hostId, String cloudIp) { + private String makeExportLogFileName(Long stepInstanceId, + int executeCount, + ExecuteObjectTypeEnum executeObjectType, + Long executeObjectResourceId) { StringBuilder fileName = new StringBuilder(); fileName.append("bk_job_export_log_"); fileName.append("step_").append(stepInstanceId).append("_").append(executeCount).append("_"); - if (hostId != null) { - fileName.append(hostId).append("_"); - } else if (StringUtils.isNotBlank(cloudIp)) { - fileName.append(cloudIp).append("_"); + if (executeObjectResourceId != null) { + fileName.append(executeObjectType.getValue()).append("_").append(executeObjectResourceId).append("_"); } fileName.append(DateUtils.formatLocalDateTime(LocalDateTime.now(), "yyyyMMddHHmmssSSS")); return fileName.toString(); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebVersionResourceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebVersionResourceImpl.java deleted file mode 100644 index f10a898314..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/api/web/impl/WebVersionResourceImpl.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.api.web.impl; - -import com.tencent.bk.job.common.web.controller.WebVersionResource; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.actuate.autoconfigure.endpoint.condition.ConditionalOnAvailableEndpoint; -import org.springframework.boot.actuate.info.InfoEndpoint; -import org.springframework.web.bind.annotation.RestController; - -import java.util.Map; - -/** - * @since 17/7/2020 13:47 - */ -@Slf4j -@RestController -@ConditionalOnAvailableEndpoint(endpoint = InfoEndpoint.class) -public class WebVersionResourceImpl implements WebVersionResource { - - private final InfoEndpoint infoEndpoint; - - @Autowired - public WebVersionResourceImpl(InfoEndpoint infoEndpoint) { - this.infoEndpoint = infoEndpoint; - } - - @Override - public Map getVersion() { - return infoEndpoint.info(); - } -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/audit/ExecuteJobAuditEventBuilder.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/audit/ExecuteJobAuditEventBuilder.java new file mode 100644 index 0000000000..5ff2828df2 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/audit/ExecuteJobAuditEventBuilder.java @@ -0,0 +1,63 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.audit; + +import com.tencent.bk.audit.DefaultAuditEventBuilder; +import com.tencent.bk.audit.context.ActionAuditContext; +import com.tencent.bk.audit.model.AuditEvent; +import com.tencent.bk.audit.utils.AuditInstanceUtils; +import com.tencent.bk.job.common.iam.constant.ResourceTypeId; + +import java.util.Collections; +import java.util.List; + +/** + * 作业执行事件生成 + */ +public class ExecuteJobAuditEventBuilder extends DefaultAuditEventBuilder { + private final ActionAuditContext actionAuditContext; + + public ExecuteJobAuditEventBuilder() { + this.actionAuditContext = ActionAuditContext.current(); + } + + @Override + public List build() { + AuditEvent auditEvent = buildBasicAuditEvent(); + + // 事件实例 + auditEvent.setResourceTypeId(ResourceTypeId.HOST); + auditEvent.setInstanceId(AuditInstanceUtils.extractInstanceIds(actionAuditContext.getInstanceIdList(), + instance -> instance)); + auditEvent.setInstanceName(AuditInstanceUtils.extractInstanceIds(actionAuditContext.getInstanceNameList(), + instance -> instance)); + + // 事件描述 + auditEvent.setContent(resolveAttributes(actionAuditContext.getContent(), actionAuditContext.getAttributes())); + auditEvent.setExtendData(actionAuditContext.getExtendData()); + + return Collections.singletonList(auditEvent); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/auth/ExecuteAuthService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/auth/ExecuteAuthService.java index f6e9f7440a..8c24ba3ec9 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/auth/ExecuteAuthService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/auth/ExecuteAuthService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,9 +24,10 @@ package com.tencent.bk.job.execute.auth; +import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; import com.tencent.bk.job.common.iam.model.AuthResult; import com.tencent.bk.job.common.model.dto.AppResourceScope; -import com.tencent.bk.job.execute.model.ServersDTO; +import com.tencent.bk.job.execute.model.ExecuteTargetDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; import java.util.Collection; @@ -40,20 +41,22 @@ public interface ExecuteAuthService { * * @param username 用户名 * @param appResourceScope 业务范围 - * @param servers 服务器 + * @param executeTarget 执行目标 * @return 鉴权结果 */ - AuthResult authFastPushFile(String username, AppResourceScope appResourceScope, ServersDTO servers); + AuthResult authFastPushFile(String username, AppResourceScope appResourceScope, ExecuteTargetDTO executeTarget); /** * 快速执行脚本鉴权 * * @param username 用户名 * @param appResourceScope 业务范围 - * @param servers 服务器 + * @param executeTarget 执行目标 * @return 鉴权结果 */ - AuthResult authFastExecuteScript(String username, AppResourceScope appResourceScope, ServersDTO servers); + AuthResult authFastExecuteScript(String username, + AppResourceScope appResourceScope, + ExecuteTargetDTO executeTarget); /** * 执行业务脚本鉴权 @@ -62,11 +65,14 @@ public interface ExecuteAuthService { * @param appResourceScope 业务范围 * @param scriptId 脚本ID * @param scriptName 脚本名称，如果传入为空，则会调用ResourceNameQueryService查询 - * @param servers 服务器 + * @param executeTarget 执行目标 * @return 鉴权结果 */ - AuthResult authExecuteAppScript(String username, AppResourceScope appResourceScope, - String scriptId, String scriptName, ServersDTO servers); + AuthResult authExecuteAppScript(String username, + AppResourceScope appResourceScope, + String scriptId, + String scriptName, + ExecuteTargetDTO executeTarget); /** * 执行公共脚本鉴权 @@ -75,11 +81,14 @@ AuthResult authExecuteAppScript(String username, AppResourceScope appResourceSco * @param appResourceScope 业务范围 * @param scriptId 脚本ID * @param scriptName 脚本名称，如果传入为空，则会调用ResourceNameQueryService查询 - * @param servers 服务器 + * @param executeTarget 执行目标 * @return 鉴权结果 */ - AuthResult authExecutePublicScript(String username, AppResourceScope appResourceScope, - String scriptId, String scriptName, ServersDTO servers); + AuthResult authExecutePublicScript(String username, + AppResourceScope appResourceScope, + String scriptId, + String scriptName, + ExecuteTargetDTO executeTarget); /** * 执行执行方案鉴权 @@ -88,11 +97,15 @@ AuthResult authExecutePublicScript(String username, AppResourceScope appResource * @param appResourceScope 业务范围 * @param planId 执行方案ID * @param planName 执行方案名称，如果传入为空，则会调用ResourceNameQueryService查询 - * @param servers 服务器 + * @param executeTarget 执行目标 * @return 鉴权结果 */ - AuthResult authExecutePlan(String username, AppResourceScope appResourceScope, Long templateId, - Long planId, String planName, ServersDTO servers); + AuthResult authExecutePlan(String username, + AppResourceScope appResourceScope, + Long templateId, + Long planId, + String planName, + ExecuteTargetDTO executeTarget); /** * 作业模板调试鉴权 @@ -100,31 +113,35 @@ AuthResult authExecutePlan(String username, AppResourceScope appResourceScope, L * @param username 用户名 * @param appResourceScope 业务范围 * @param templateId 作业模板ID - * @param servers 服务器 + * @param executeTarget 执行目标 * @return 鉴权结果 */ - AuthResult authDebugTemplate(String username, AppResourceScope appResourceScope, - Long templateId, ServersDTO servers); + AuthResult authDebugTemplate(String username, + AppResourceScope appResourceScope, + Long templateId, + ExecuteTargetDTO executeTarget); /** * 作业执行实例查看权限鉴权 * * @param username 用户名 * @param appResourceScope 业务范围 - * @param taskInstanceId 作业实例ID - * @return 鉴权结果 + * @param taskInstance 作业实例 + * @throws PermissionDeniedException 用户无权限 */ - AuthResult authViewTaskInstance(String username, AppResourceScope appResourceScope, long taskInstanceId); + void authViewTaskInstance(String username, AppResourceScope appResourceScope, TaskInstanceDTO taskInstance) + throws PermissionDeniedException; /** - * 作业执行实例查看权限鉴权 + * 检查是否拥有作业执行实例查看权限 * * @param username 用户名 * @param appResourceScope 业务范围 * @param taskInstance 作业实例 - * @return 鉴权结果 */ - AuthResult authViewTaskInstance(String username, AppResourceScope appResourceScope, TaskInstanceDTO taskInstance); + AuthResult checkViewTaskInstancePermission(String username, + AppResourceScope appResourceScope, + TaskInstanceDTO taskInstance); /** * 用户是否具有查看所有作业实例权限 diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/auth/impl/CMDBTopoPathService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/auth/impl/CMDBTopoPathService.java new file mode 100644 index 0000000000..6bf961bf24 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/auth/impl/CMDBTopoPathService.java @@ -0,0 +1,107 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.auth.impl; + +import com.google.common.collect.Lists; +import com.tencent.bk.job.common.cc.model.result.HostBizRelationDTO; +import com.tencent.bk.job.common.cc.sdk.BizCmdbClient; +import com.tencent.bk.job.common.iam.constant.ResourceTypeId; +import com.tencent.bk.job.common.util.ConcurrencyUtil; +import com.tencent.bk.sdk.iam.service.TopoPathService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.stream.Collectors; + +/** + * 主机拓扑路径服务，根据主机ID从CMDB接口获取主机拓扑路径 + */ +@Slf4j +@Service("cmdbTopoPathService") +public class CMDBTopoPathService implements TopoPathService { + + private final BizCmdbClient bizCmdbClient; + private final ExecutorService executorService; + + @Autowired + public CMDBTopoPathService(BizCmdbClient bizCmdbClient, + @Qualifier("getHostTopoPathExecutor") + ExecutorService executorService) { + this.bizCmdbClient = bizCmdbClient; + this.executorService = executorService; + } + + @Override + public Map> getTopoPathByHostIds(Set hostIds) { + List hostIdList = hostIds.stream().map(Long::parseLong).collect(Collectors.toList()); + // CMDB接口限制每次最多查询500台主机 + int batchSize = 500; + List> hostIdsSubList = Lists.partition(hostIdList, batchSize); + // 多线程并发拉取 + List hostBizRelationDTOList = ConcurrencyUtil.getResultWithThreads( + hostIdsSubList, + executorService, + bizCmdbClient::findHostBizRelations + ); + return buildTopoPathMap(hostBizRelationDTOList); + } + + private Map> buildTopoPathMap(List hostBizRelationDTOList) { + if (CollectionUtils.isEmpty(hostBizRelationDTOList)) { + return Collections.emptyMap(); + } + Map> topoPathMap = new HashMap<>(); + hostBizRelationDTOList.forEach(hostBizRelationDTO -> { + Long hostId = hostBizRelationDTO.getHostId(); + Long bizId = hostBizRelationDTO.getBizId(); + Long setId = hostBizRelationDTO.getSetId(); + Long moduleId = hostBizRelationDTO.getModuleId(); + String topoPath = buildTopoPath(bizId, setId, moduleId); + String hostIdKey = hostId.toString(); + if (!topoPathMap.containsKey(hostIdKey)) { + topoPathMap.put(hostIdKey, new ArrayList<>()); + } + topoPathMap.get(hostIdKey).add(topoPath); + }); + return topoPathMap; + } + + private String buildTopoPath(Long bizId, Long setId, Long moduleId) { + return "/" + ResourceTypeId.BIZ + "," + bizId + + "/" + ResourceTypeId.SET + "," + setId + + "/" + ResourceTypeId.MODULE + "," + moduleId + + "/"; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/auth/impl/CachedTopoPathServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/auth/impl/CachedTopoPathServiceImpl.java new file mode 100644 index 0000000000..4798e4640b --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/auth/impl/CachedTopoPathServiceImpl.java @@ -0,0 +1,171 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.auth.impl; + +import com.tencent.bk.sdk.iam.service.TopoPathService; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections4.MapUtils; +import org.slf4j.helpers.MessageFormatter; +import org.springframework.util.StopWatch; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * 优先使用缓存数据的主机拓扑路径查询服务 + */ +@Slf4j +public class CachedTopoPathServiceImpl implements TopoPathService { + + private final TopoPathService delegate; + private final HostTopoPathCache hostTopoPathCache; + + public CachedTopoPathServiceImpl(TopoPathService delegate, HostTopoPathCache hostTopoPathCache) { + this.delegate = delegate; + this.hostTopoPathCache = hostTopoPathCache; + } + + @Override + public Map> getTopoPathByHostIds(Set hostIds) { + StopWatch watch = new StopWatch(); + try { + Map> resultMap = getTopoPathByHostIds(hostIds, watch); + if (watch.getTotalTimeMillis() >= 3000) { + log.warn("getTopoPathByHostIds slow:{}", watch.prettyPrint()); + } else if (watch.getTotalTimeMillis() >= 1000) { + log.info("getTopoPathByHostIds cost:{}", watch.prettyPrint()); + } + return resultMap; + } catch (Exception e) { + log.warn("getTopoPathByHostIds error, use delegate directly", e); + return delegate.getTopoPathByHostIds(hostIds); + } + } + + private Map> getTopoPathByHostIds(Set hostIds, StopWatch watch) { + // 1.优先从缓存中获取拓扑路径 + List hostIdList = hostIds.stream().map(Long::parseLong).collect(Collectors.toList()); + List hostTopoPathEntryList; + try { + watch.start("getHostTopoPathFromCache"); + hostTopoPathEntryList = hostTopoPathCache.batchGetHostTopoPathByHostIds(hostIdList); + if (hostTopoPathEntryList != null) { + hostTopoPathEntryList = hostTopoPathEntryList.stream() + .filter(hostTopoPathEntry -> + hostTopoPathEntry != null + && CollectionUtils.isNotEmpty(hostTopoPathEntry.getTopoPathList())) + .collect(Collectors.toList()); + } + } catch (Exception e) { + String message = MessageFormatter.format( + "Fail to get hostTopoPath from cache, hostIds={}", + hostIds + ).getMessage(); + log.warn(message, e); + return delegate.getTopoPathByHostIds(hostIds); + } finally { + watch.stop(); + } + if (CollectionUtils.isEmpty(hostTopoPathEntryList)) { + if (log.isDebugEnabled()) { + log.debug("Get empty hostTopoPath from cache, hostIds={}", hostIds); + } + watch.start("getAllTopoPathByHostIds"); + Map> resultMap = delegate.getTopoPathByHostIds(hostIds); + watch.stop(); + watch.start("updateHostTopoPathCache"); + updateHostTopoPathCache(resultMap); + watch.stop(); + return resultMap; + } + if (log.isDebugEnabled()) { + log.debug( + "Get {} hostTopoPath from cache, hostTopoPathEntryList={}", + hostTopoPathEntryList.size(), + hostTopoPathEntryList + ); + } + watch.start("buildHostTopoPathMap"); + Map> hostTopoPathMap = new HashMap<>(hostIds.size()); + Set cachedHostIds = new HashSet<>(); + for (HostTopoPathEntry hostTopoPathEntry : hostTopoPathEntryList) { + if (hostTopoPathEntry == null) { + continue; + } + String hostIdStr = hostTopoPathEntry.getHostId().toString(); + cachedHostIds.add(hostIdStr); + hostTopoPathMap.put(hostIdStr, hostTopoPathEntry.getTopoPathList()); + } + watch.stop(); + // 2.计算出缓存中不存在的拓扑路径，从CMDB获取 + Collection notCachedHostIds = CollectionUtils.subtract(hostIds, cachedHostIds); + if (CollectionUtils.isEmpty(notCachedHostIds)) { + return hostTopoPathMap; + } + watch.start("getNotInCacheTopoPath"); + Map> notCachedHostTopoPathMap = + delegate.getTopoPathByHostIds(new HashSet<>(notCachedHostIds)); + watch.stop(); + + if (MapUtils.isNotEmpty(notCachedHostTopoPathMap)) { + // 3.汇总拓扑路径信息 + hostTopoPathMap.putAll(notCachedHostTopoPathMap); + // 4.将未缓存的拓扑路径信息更新到缓存 + watch.start("updateHostTopoPathCache"); + updateHostTopoPathCache(notCachedHostTopoPathMap); + watch.stop(); + } + return hostTopoPathMap; + } + + /** + * 更新主机拓扑路径信息到缓存 + * + * @param hostTopoPathMap 主机拓扑路径信息Map + */ + private void updateHostTopoPathCache(Map> hostTopoPathMap) { + List topoPathEntryList = new ArrayList<>(); + hostTopoPathMap.forEach((hostIdStr, topoPathList) -> + topoPathEntryList.add(new HostTopoPathEntry(Long.parseLong(hostIdStr), topoPathList)) + ); + hostTopoPathCache.batchAddOrUpdateHostTopoPaths(topoPathEntryList); + if (log.isDebugEnabled()) { + log.debug( + "updateHostTopoPathCache, size={}, topoPathEntryList={}", + topoPathEntryList.size(), + topoPathEntryList + ); + } else if (log.isInfoEnabled()) { + log.info("updateHostTopoPathCache, size={}", topoPathEntryList.size()); + } + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/auth/impl/CompositeTopoPathService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/auth/impl/CompositeTopoPathService.java new file mode 100644 index 0000000000..7aa35ad063 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/auth/impl/CompositeTopoPathService.java @@ -0,0 +1,74 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.auth.impl; + +import com.tencent.bk.sdk.iam.service.TopoPathService; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.slf4j.helpers.MessageFormatter; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * 支持从多种来源查询主机拓扑路径的服务，先添加的TopoPathService实例优先级高，若查询失败则遍历列表中其他服务进行查询，直至成功为止 + */ +@NoArgsConstructor +@Slf4j +public class CompositeTopoPathService implements TopoPathService { + + private final List topoPathServiceList = new ArrayList<>(); + + public void addTopoPathService(TopoPathService topoPathService) { + topoPathServiceList.add(topoPathService); + } + + @Override + public Map> getTopoPathByHostIds(Set hostIds) { + for (TopoPathService topoPathService : topoPathServiceList) { + try { + if (log.isDebugEnabled()) { + log.debug( + "Use {} to get topoPath, hostIds={}", + topoPathService.getClass().getName(), + hostIds + ); + } + return topoPathService.getTopoPathByHostIds(hostIds); + } catch (Exception e) { + String message = MessageFormatter.format( + "Fail to get topoPath by hostIds using {}, hostIds={}", + topoPathService.getClass().getName(), + hostIds + ).getMessage(); + log.warn(message, e); + } + } + return Collections.emptyMap(); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/auth/impl/ExecuteAuthServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/auth/impl/ExecuteAuthServiceImpl.java index 90cc2e3097..16095b2510 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/auth/impl/ExecuteAuthServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/auth/impl/ExecuteAuthServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,14 +24,15 @@ package com.tencent.bk.job.execute.auth.impl; +import com.tencent.bk.job.common.cc.model.InstanceTopologyDTO; import com.tencent.bk.job.common.constant.CcNodeTypeEnum; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.constant.FeatureToggleModeEnum; -import com.tencent.bk.job.common.exception.NotFoundException; import com.tencent.bk.job.common.exception.NotImplementedException; import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.iam.constant.ResourceTypeEnum; import com.tencent.bk.job.common.iam.constant.ResourceTypeId; +import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; import com.tencent.bk.job.common.iam.model.AuthResult; import com.tencent.bk.job.common.iam.model.PermissionResource; import com.tencent.bk.job.common.iam.service.AppAuthService; @@ -41,9 +42,10 @@ import com.tencent.bk.job.common.model.dto.AppResourceScope; import com.tencent.bk.job.execute.auth.ExecuteAuthService; import com.tencent.bk.job.execute.config.JobExecuteConfig; -import com.tencent.bk.job.execute.model.ServersDTO; +import com.tencent.bk.job.execute.model.DynamicServerTopoNodeDTO; +import com.tencent.bk.job.execute.model.ExecuteTargetDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; -import com.tencent.bk.job.execute.service.TaskInstanceService; +import com.tencent.bk.job.execute.service.TopoService; import com.tencent.bk.sdk.iam.constants.SystemId; import com.tencent.bk.sdk.iam.dto.InstanceDTO; import com.tencent.bk.sdk.iam.dto.PathInfoDTO; @@ -53,13 +55,16 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -70,31 +75,36 @@ public class ExecuteAuthServiceImpl implements ExecuteAuthService { private final ResourceNameQueryService resourceNameQueryService; private final AuthService authService; private final AppAuthService appAuthService; - private final TaskInstanceService taskInstanceService; private final JobExecuteConfig jobExecuteConfig; + private final TopoService topoService; @Autowired public ExecuteAuthServiceImpl(AuthHelper authHelper, + @Qualifier("jobExecuteResourceNameQueryService") ResourceNameQueryService resourceNameQueryService, AuthService authService, AppAuthService appAuthService, - TaskInstanceService taskInstanceService, - JobExecuteConfig jobExecuteConfig) { + JobExecuteConfig jobExecuteConfig, + TopoService topoService) { this.authHelper = authHelper; this.resourceNameQueryService = resourceNameQueryService; this.authService = authService; this.appAuthService = appAuthService; - this.taskInstanceService = taskInstanceService; this.jobExecuteConfig = jobExecuteConfig; + this.topoService = topoService; this.authService.setResourceNameQueryService(resourceNameQueryService); this.appAuthService.setResourceNameQueryService(resourceNameQueryService); } - public AuthResult authFastExecuteScript(String username, AppResourceScope appResourceScope, ServersDTO servers) { - List hostInstanceList = buildHostInstances(appResourceScope, servers); + public AuthResult authFastExecuteScript(String username, + AppResourceScope appResourceScope, + ExecuteTargetDTO executeTarget) { - log.debug("Auth fast execute script, username:{}, appResourceScope:{}, hostInstances:{}", username, - appResourceScope, hostInstanceList); + List hostInstanceList = buildHostInstances(appResourceScope, executeTarget); + if (log.isDebugEnabled()) { + log.debug("Auth fast execute script, username:{}, appResourceScope:{}, hostInstances:{}", username, + appResourceScope, hostInstanceList); + } boolean isAllowed = authHelper.isAllowed( username, ActionId.QUICK_EXECUTE_SCRIPT, null, hostInstanceList); @@ -105,17 +115,22 @@ public AuthResult authFastExecuteScript(String username, AppResourceScope appRes AuthResult authResult = AuthResult.fail(); List hostResources = convertHostsToPermissionResourceList( - appResourceScope, servers); + appResourceScope, executeTarget); authResult.addRequiredPermissions(ActionId.QUICK_EXECUTE_SCRIPT, hostResources); - log.debug("Auth execute script, authResult:{}", authResult); + if (log.isDebugEnabled()) { + log.debug("Auth execute script, authResult:{}", authResult); + } return authResult; } - public AuthResult authFastPushFile(String username, AppResourceScope appResourceScope, ServersDTO servers) { - List hostInstanceList = buildHostInstances(appResourceScope, servers); - - log.debug("Auth Fast transfer file, username:{}, appResourceScope:{}, hostInstances:{}", username, - appResourceScope, hostInstanceList); + public AuthResult authFastPushFile(String username, + AppResourceScope appResourceScope, + ExecuteTargetDTO executeTarget) { + List hostInstanceList = buildHostInstances(appResourceScope, executeTarget); + if (log.isDebugEnabled()) { + log.debug("Auth Fast transfer file, username:{}, appResourceScope:{}, hostInstances:{}", username, + appResourceScope, hostInstanceList); + } boolean isAllowed = authHelper.isAllowed( username, ActionId.QUICK_TRANSFER_FILE, null, hostInstanceList); @@ -125,22 +140,25 @@ public AuthResult authFastPushFile(String username, AppResourceScope appResource AuthResult authResult = AuthResult.fail(); - List hostResources = convertHostsToPermissionResourceList(appResourceScope, servers); + List hostResources = convertHostsToPermissionResourceList(appResourceScope, executeTarget); authResult.addRequiredPermissions(ActionId.QUICK_TRANSFER_FILE, hostResources); - log.debug("Auth execute script, authResult:{}", authResult); + if (log.isDebugEnabled()) { + log.debug("Auth execute script, authResult:{}", authResult); + } return authResult; } public AuthResult authExecuteAppScript(String username, AppResourceScope appResourceScope, - String scriptId, String scriptName, ServersDTO servers) { - List hostInstanceList = buildHostInstances(appResourceScope, servers); + String scriptId, String scriptName, ExecuteTargetDTO executeTarget) { + List hostInstanceList = buildHostInstances(appResourceScope, executeTarget); InstanceDTO scriptInstance = buildExecutableInstance(appResourceScope, ResourceTypeEnum.SCRIPT, scriptId, null); - log.debug("Auth execute script, username:{}, appResourceScope:{}, scriptId:{}, scriptInstance:{}, " + - "hostInstances:{}", - username, - appResourceScope, scriptId, scriptInstance, hostInstanceList); + if (log.isDebugEnabled()) { + log.debug("Auth execute script, username:{}, appResourceScope:{}, scriptId:{}, scriptInstance:{}, " + + "hostInstances:{}", + username, appResourceScope, scriptId, scriptInstance, hostInstanceList); + } boolean isAllowed = authHelper.isAllowed(username, ActionId.EXECUTE_SCRIPT, scriptInstance, hostInstanceList); if (isAllowed) { @@ -161,13 +179,16 @@ public AuthResult authExecuteAppScript(String username, AppResourceScope appReso authResult.addRequiredPermission(ActionId.EXECUTE_SCRIPT, scriptResource); List hostResources = convertHostsToPermissionResourceList( - appResourceScope, servers); + appResourceScope, executeTarget); authResult.addRequiredPermissions(ActionId.EXECUTE_SCRIPT, hostResources); - log.debug("Auth execute script, authResult:{}", authResult); + if (log.isDebugEnabled()) { + log.debug("Auth execute script, authResult:{}", authResult); + } return authResult; } - private InstanceDTO buildExecutableInstance(AppResourceScope appResourceScope, ResourceTypeEnum resourceType, + private InstanceDTO buildExecutableInstance(AppResourceScope appResourceScope, + ResourceTypeEnum resourceType, String resourceId, PathInfoDTO pathInfo) { InstanceDTO executeInstance = new InstanceDTO(); @@ -182,16 +203,20 @@ private InstanceDTO buildExecutableInstance(AppResourceScope appResourceScope, R return executeInstance; } - public AuthResult authExecutePublicScript(String username, AppResourceScope appResourceScope, - String scriptId, String scriptName, ServersDTO servers) { - List hostInstanceList = buildHostInstances(appResourceScope, servers); + public AuthResult authExecutePublicScript(String username, + AppResourceScope appResourceScope, + String scriptId, + String scriptName, + ExecuteTargetDTO executeTarget) { + List hostInstanceList = buildHostInstances(appResourceScope, executeTarget); InstanceDTO scriptInstance = buildExecutableInstance( appResourceScope, ResourceTypeEnum.PUBLIC_SCRIPT, scriptId, null); - log.debug("Auth execute public script, username:{}, appResourceScope:{}, scriptId:{}, scriptInstance:{}, " + - "hostInstances:{}", username, - appResourceScope, scriptId, scriptInstance, hostInstanceList); + if (log.isDebugEnabled()) { + log.debug("Auth execute public script, username:{}, appResourceScope:{}, scriptId:{}, scriptInstance:{}, " + + "hostInstances:{}", username, appResourceScope, scriptId, scriptInstance, hostInstanceList); + } boolean isAllowed = authHelper.isAllowed(username, ActionId.EXECUTE_PUBLIC_SCRIPT, scriptInstance, hostInstanceList); @@ -214,23 +239,33 @@ public AuthResult authExecutePublicScript(String username, AppResourceScope appR authResult.addRequiredPermission(ActionId.EXECUTE_PUBLIC_SCRIPT, scriptResource); List hostResources = convertHostsToPermissionResourceList( - appResourceScope, servers); + appResourceScope, executeTarget); authResult.addRequiredPermissions(ActionId.EXECUTE_PUBLIC_SCRIPT, hostResources); - log.debug("Auth execute script, authResult:{}", authResult); + if (log.isDebugEnabled()) { + log.debug("Auth execute script, authResult:{}", authResult); + } return authResult; } - public AuthResult authExecutePlan(String username, AppResourceScope appResourceScope, Long templateId, - Long planId, String planName, ServersDTO servers) { - List hostInstanceList = buildHostInstances(appResourceScope, servers); + public AuthResult authExecutePlan(String username, + AppResourceScope appResourceScope, + Long templateId, + Long planId, + String planName, + ExecuteTargetDTO executeTarget) { + List hostInstanceList = buildHostInstances(appResourceScope, executeTarget); InstanceDTO planInstance = buildExecutableInstance( appResourceScope, - ResourceTypeEnum.PLAN, planId.toString(), + ResourceTypeEnum.PLAN, + planId.toString(), buildAppScopeResourcePath(appResourceScope, ResourceTypeEnum.TEMPLATE, templateId.toString())); - log.debug("Auth execute plan, username:{}, appResourceScope:{}, planId:{}, planInstance:{}, hostInstances:{}", - username, appResourceScope, planId, planInstance, hostInstanceList); + if (log.isDebugEnabled()) { + log.debug("Auth execute plan, username:{}, appResourceScope:{}, planId:{}, planInstance:{}," + + " hostInstances:{}", + username, appResourceScope, planId, planInstance, hostInstanceList); + } boolean isAllowed = authHelper.isAllowed(username, ActionId.LAUNCH_JOB_PLAN, planInstance, hostInstanceList); if (isAllowed) { @@ -252,22 +287,26 @@ public AuthResult authExecutePlan(String username, AppResourceScope appResourceS authResult.addRequiredPermission(ActionId.LAUNCH_JOB_PLAN, planResource); List hostResources = convertHostsToPermissionResourceList( - appResourceScope, servers); + appResourceScope, executeTarget); authResult.addRequiredPermissions(ActionId.LAUNCH_JOB_PLAN, hostResources); - log.debug("Auth execute plan, authResult:{}", authResult); + if (log.isDebugEnabled()) { + log.debug("Auth execute plan, authResult:{}", authResult); + } return authResult; } @Override public AuthResult authDebugTemplate(String username, AppResourceScope appResourceScope, Long templateId, - ServersDTO servers) { - List hostInstanceList = buildHostInstances(appResourceScope, servers); + ExecuteTargetDTO executeTarget) { + List hostInstanceList = buildHostInstances(appResourceScope, executeTarget); InstanceDTO jobTemplateInstance = buildExecutableInstance(appResourceScope, ResourceTypeEnum.TEMPLATE, templateId.toString(), null); - log.debug("Auth execute job template, username:{}, appResourceScope:{}, planId:{}, templateInstance:{}, " + - "hostInstances:{}", username, appResourceScope, templateId, jobTemplateInstance, hostInstanceList); + if (log.isDebugEnabled()) { + log.debug("Auth execute job template, username:{}, appResourceScope:{}, planId:{}, templateInstance:{}, " + + "hostInstances:{}", username, appResourceScope, templateId, jobTemplateInstance, hostInstanceList); + } boolean isAllowed = authHelper.isAllowed(username, ActionId.DEBUG_JOB_TEMPLATE, jobTemplateInstance, hostInstanceList); @@ -286,26 +325,83 @@ public AuthResult authDebugTemplate(String username, AppResourceScope appResourc authResult.addRequiredPermission(ActionId.DEBUG_JOB_TEMPLATE, jobTemplateResource); List hostResources = convertHostsToPermissionResourceList( - appResourceScope, servers); + appResourceScope, executeTarget); authResult.addRequiredPermissions(ActionId.DEBUG_JOB_TEMPLATE, hostResources); - log.debug("Auth execute job template, authResult:{}", authResult); + if (log.isDebugEnabled()) { + log.debug("Auth execute job template, authResult:{}", authResult); + } return authResult; } - private List buildAppTopoNodeHostInstances(AppResourceScope appResourceScope) { - List topoNodeInstanceList = new ArrayList<>(1); - InstanceDTO topoNodeInstance = new InstanceDTO(); - topoNodeInstance.setType(ResourceTypeEnum.HOST.getId()); - topoNodeInstance.setSystem(SystemId.CMDB); - topoNodeInstance.setPath(buildAppScopePath(appResourceScope)); - topoNodeInstanceList.add(topoNodeInstance); + private List buildAppTopoNodeHostInstances(AppResourceScope appResourceScope, + List topoNodes) { + long bizId = Long.parseLong(appResourceScope.getId()); + List topoNodeTopologyList = topoService.batchGetTopoNodeHierarchy(bizId, topoNodes); + Map topoNodeTopologyMap = new HashMap<>(); + for (InstanceTopologyDTO instanceTopologyDTO : topoNodeTopologyList) { + topoNodeTopologyMap.put(instanceTopologyDTO.getUniqueKey(), instanceTopologyDTO); + } + List topoNodeInstanceList = new ArrayList<>(topoNodes.size()); + for (DynamicServerTopoNodeDTO topoNode : topoNodes) { + String topoNodeKey = topoNode.getUniqueKey(); + InstanceTopologyDTO nodeTopology = topoNodeTopologyMap.get(topoNodeKey); + if (nodeTopology == null) { + log.info("Cannot find topoPath for node {}, ignore", topoNodeKey); + continue; + } + InstanceDTO topoNodeInstance = new InstanceDTO(); + topoNodeInstance.setName(String.valueOf(topoNode.getTopoNodeId())); + topoNodeInstance.setType(ResourceTypeEnum.HOST.getId()); + topoNodeInstance.setSystem(SystemId.CMDB); + topoNodeInstance.setPath(buildIamPathForTopoNode(nodeTopology)); + topoNodeInstanceList.add(topoNodeInstance); + } return topoNodeInstanceList; } + private PathInfoDTO buildIamPathForTopoNode(InstanceTopologyDTO nodeTopology) { + List parents = nodeTopology.getParents(); + if (parents == null) { + parents = Collections.emptyList(); + } + List pathNodeList = new ArrayList<>(parents); + pathNodeList.add(nodeTopology); + // 权限路径不支持自定义节点，过滤掉 + pathNodeList = pathNodeList.stream().filter(parent -> + CcNodeTypeEnum.BIZ.getType().equals(parent.getObjectId()) + || CcNodeTypeEnum.SET.getType().equals(parent.getObjectId()) + || CcNodeTypeEnum.MODULE.getType().equals(parent.getObjectId()) + ).collect(Collectors.toList()); + if (CollectionUtils.isEmpty(pathNodeList)) { + return null; + } + return buildPathInfoDTO(pathNodeList); + } + + private PathInfoDTO buildPathInfoDTO(List pathNodeList) { + PathInfoDTO rootPathInfo = null; + PathInfoDTO endPathInfo = null; + for (InstanceTopologyDTO pathNode : pathNodeList) { + if (rootPathInfo == null) { + rootPathInfo = new PathInfoDTO(); + rootPathInfo.setType(pathNode.getObjectId()); + rootPathInfo.setId(pathNode.getInstanceId().toString()); + endPathInfo = rootPathInfo; + } else { + PathInfoDTO childPathInfo = new PathInfoDTO(); + childPathInfo.setType(pathNode.getObjectId()); + childPathInfo.setId(pathNode.getInstanceId().toString()); + endPathInfo.setChild(childPathInfo); + endPathInfo = childPathInfo; + } + } + return rootPathInfo; + } + private List buildBizStaticHostInstances(AppResourceScope appResourceScope, - ServersDTO servers) { + ExecuteTargetDTO executeTarget) { List hostInstanceList = new ArrayList<>(); - servers.getStaticIpList().forEach(host -> { + executeTarget.getStaticIpList().forEach(host -> { InstanceDTO hostInstance = new InstanceDTO(); String hostIdStr = host.getHostId().toString(); hostInstance.setId(hostIdStr); @@ -320,13 +416,13 @@ private List buildBizStaticHostInstances(AppResourceScope appResour } private List buildHostInstances(AppResourceScope appResourceScope, - ServersDTO servers) { + ExecuteTargetDTO executeObjects) { List hostInstanceList = new ArrayList<>(); // 静态IP - if (!CollectionUtils.isEmpty(servers.getStaticIpList())) { + if (!CollectionUtils.isEmpty(executeObjects.getStaticIpList())) { switch (appResourceScope.getType()) { case BIZ: - hostInstanceList.addAll(buildBizStaticHostInstances(appResourceScope, servers)); + hostInstanceList.addAll(buildBizStaticHostInstances(appResourceScope, executeObjects)); break; case BIZ_SET: InstanceDTO hostInstance = new InstanceDTO(); @@ -337,18 +433,18 @@ private List buildHostInstances(AppResourceScope appResourceScope, break; default: throw new NotImplementedException( - "Unsupport appScopeType:" + appResourceScope.getType().getValue(), + "Unsupported appScopeType:" + appResourceScope.getType().getValue(), ErrorCode.NOT_SUPPORT_FEATURE); } } // 动态topo节点 - if (!CollectionUtils.isEmpty(servers.getTopoNodes())) { - // CMDB未提供权限中心使用的topo视图，暂时使用“业务”这个topo节点进行鉴权，不细化到集群、模块 - hostInstanceList.addAll(buildAppTopoNodeHostInstances(appResourceScope)); + List topoNodes = executeObjects.getTopoNodes(); + if (!CollectionUtils.isEmpty(topoNodes)) { + hostInstanceList.addAll(buildAppTopoNodeHostInstances(appResourceScope, topoNodes)); } // 动态分组 - if (!CollectionUtils.isEmpty(servers.getDynamicServerGroups())) { - servers.getDynamicServerGroups().forEach(serverGroup -> { + if (!CollectionUtils.isEmpty(executeObjects.getDynamicServerGroups())) { + executeObjects.getDynamicServerGroups().forEach(serverGroup -> { InstanceDTO serverGroupInstance = new InstanceDTO(); serverGroupInstance.setType(ResourceTypeEnum.HOST.getId()); serverGroupInstance.setSystem(SystemId.CMDB); @@ -360,12 +456,21 @@ private List buildHostInstances(AppResourceScope appResourceScope, hostInstanceList.add(serverGroupInstance); }); } + // 静态容器 + if (!CollectionUtils.isEmpty(executeObjects.getStaticContainerList())) { + // 静态容器按照业务鉴权 + InstanceDTO hostInstance = new InstanceDTO(); + hostInstance.setType(ResourceTypeEnum.HOST.getId()); + hostInstance.setSystem(SystemId.CMDB); + hostInstance.setPath(buildAppScopePath(appResourceScope)); + hostInstanceList.add(hostInstance); + } return hostInstanceList; } - private List convertBizStaticIpToPermissionResourceList(ServersDTO servers) { + private List convertBizStaticIpToPermissionResourceList(ExecuteTargetDTO executeTarget) { List hostResources = new ArrayList<>(); - servers.getStaticIpList().forEach(host -> { + executeTarget.getStaticIpList().forEach(host -> { PermissionResource resource = new PermissionResource(); resource.setResourceId(String.valueOf(host.getHostId())); resource.setResourceType(ResourceTypeEnum.HOST); @@ -403,24 +508,48 @@ private List convertBizSetStaticIpToPermissionResourceList(A return hostResources; } - private List convertTopoNodesToPermissionResourceList(AppResourceScope appResourceScope) { - List hostResources = new ArrayList<>(); - PermissionResource resource = new PermissionResource(); - resource.setResourceId(appResourceScope.getId()); - resource.setResourceType(ResourceTypeEnum.HOST); - resource.setSubResourceType("topo"); - resource.setResourceName(getResourceName(appResourceScope)); - resource.setSystemId(SystemId.CMDB); - resource.setType(CcNodeTypeEnum.BIZ.getType()); - resource.setParentHierarchicalResources(null); - hostResources.add(resource); - return hostResources; + private List convertTopoNodesToPermissionResourceList(AppResourceScope appResourceScope, + List topoNodes) { + List hostInstanceList = buildAppTopoNodeHostInstances(appResourceScope, topoNodes); + List finalPermissionResourceList = new ArrayList<>(); + for (InstanceDTO instanceDTO : hostInstanceList) { + List permissionResourceList = convert(instanceDTO.getPath()); + if (CollectionUtils.isEmpty(permissionResourceList)) { + continue; + } + int lastNodeIndex = permissionResourceList.size() - 1; + PermissionResource lastNode = permissionResourceList.get(lastNodeIndex); + permissionResourceList.remove(lastNodeIndex); + lastNode.setParentHierarchicalResources(permissionResourceList); + finalPermissionResourceList.add(lastNode); + } + return finalPermissionResourceList; + } + + private List convert(PathInfoDTO pathInfoDTO) { + List permissionResourceList = new ArrayList<>(); + if (pathInfoDTO == null) { + return permissionResourceList; + } + PathInfoDTO currentNode = pathInfoDTO; + while (currentNode != null) { + PermissionResource resource = new PermissionResource(); + resource.setResourceId(currentNode.getId()); + resource.setResourceType(ResourceTypeEnum.HOST); + resource.setSubResourceType("topo"); + resource.setResourceName(currentNode.getType() + "_" + currentNode.getId()); + resource.setSystemId(SystemId.CMDB); + resource.setType(currentNode.getType()); + permissionResourceList.add(resource); + currentNode = currentNode.getChild(); + } + return permissionResourceList; } private List convertDynamicGroupsToPermissionResourceList(AppResourceScope appResourceScope, - ServersDTO servers) { + ExecuteTargetDTO executeTarget) { List hostResources = new ArrayList<>(); - servers.getDynamicServerGroups().forEach(serverGroup -> { + executeTarget.getDynamicServerGroups().forEach(serverGroup -> { PermissionResource resource = new PermissionResource(); String groupId = serverGroup.getGroupId(); resource.setResourceId(groupId); @@ -435,14 +564,29 @@ private List convertDynamicGroupsToPermissionResourceList(Ap return hostResources; } + private List convertContainersToPermissionResourceList(AppResourceScope appResourceScope) { + List hostResources = new ArrayList<>(); + PermissionResource resource = new PermissionResource(); + resource.setResourceId(appResourceScope.getId()); + resource.setResourceType(ResourceTypeEnum.HOST); + resource.setSubResourceType("container"); + resource.setResourceName(getResourceName(appResourceScope)); + resource.setSystemId(SystemId.CMDB); + resource.setType(CcNodeTypeEnum.BIZ.getType()); + resource.setParentHierarchicalResources(null); + hostResources.add(resource); + return hostResources; + } + private List convertHostsToPermissionResourceList(AppResourceScope appResourceScope, - ServersDTO servers) { + ExecuteTargetDTO executeTarget) { List hostResources = new ArrayList<>(); - if (!CollectionUtils.isEmpty(servers.getStaticIpList())) { + // 静态IP + if (!CollectionUtils.isEmpty(executeTarget.getStaticIpList())) { switch (appResourceScope.getType()) { case BIZ: - hostResources.addAll(convertBizStaticIpToPermissionResourceList(servers)); + hostResources.addAll(convertBizStaticIpToPermissionResourceList(executeTarget)); break; case BIZ_SET: hostResources.addAll( @@ -454,11 +598,18 @@ private List convertHostsToPermissionResourceList(AppResourc ErrorCode.NOT_SUPPORT_FEATURE); } } - if (!CollectionUtils.isEmpty(servers.getTopoNodes())) { - hostResources.addAll(convertTopoNodesToPermissionResourceList(appResourceScope)); + // 动态topo节点 + List topoNodes = executeTarget.getTopoNodes(); + if (CollectionUtils.isNotEmpty(topoNodes)) { + hostResources.addAll(convertTopoNodesToPermissionResourceList(appResourceScope, topoNodes)); + } + // 动态分组 + if (!CollectionUtils.isEmpty(executeTarget.getDynamicServerGroups())) { + hostResources.addAll(convertDynamicGroupsToPermissionResourceList(appResourceScope, executeTarget)); } - if (!CollectionUtils.isEmpty(servers.getDynamicServerGroups())) { - hostResources.addAll(convertDynamicGroupsToPermissionResourceList(appResourceScope, servers)); + // 静态容器 + if (!CollectionUtils.isEmpty(executeTarget.getStaticContainerList())) { + hostResources.addAll(convertContainersToPermissionResourceList(appResourceScope)); } return hostResources; } @@ -481,21 +632,20 @@ private PermissionResource buildAppResource(AppResourceScope appResourceScope) { } @Override - public AuthResult authViewTaskInstance(String username, AppResourceScope appResourceScope, long taskInstanceId) { - TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(taskInstanceId); - if (taskInstance == null) { - throw new NotFoundException(ErrorCode.TASK_INSTANCE_NOT_EXIST); - } + public void authViewTaskInstance(String username, + AppResourceScope appResourceScope, + TaskInstanceDTO taskInstance) throws PermissionDeniedException { if (username.equals(taskInstance.getOperator())) { - return AuthResult.pass(); + return; } - return appAuthService.auth(username, ActionId.VIEW_HISTORY, appResourceScope); + AuthResult authResult = appAuthService.auth(username, ActionId.VIEW_HISTORY, appResourceScope); + authResult.denyIfNoPermission(); } @Override - public AuthResult authViewTaskInstance(String username, - AppResourceScope appResourceScope, - TaskInstanceDTO taskInstance) { + public AuthResult checkViewTaskInstancePermission(String username, + AppResourceScope appResourceScope, + TaskInstanceDTO taskInstance) { if (username.equals(taskInstance.getOperator())) { return AuthResult.pass(); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/auth/impl/HostTopoPathCache.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/auth/impl/HostTopoPathCache.java new file mode 100644 index 0000000000..5c7ca359fd --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/auth/impl/HostTopoPathCache.java @@ -0,0 +1,169 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.auth.impl; + +import com.tencent.bk.job.common.redis.BaseRedisCache; +import com.tencent.bk.job.common.util.TimeUtil; +import com.tencent.bk.job.execute.config.IamHostTopoPathProperties; +import io.micrometer.core.instrument.MeterRegistry; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.jetbrains.annotations.NotNull; +import org.springframework.dao.DataAccessException; +import org.springframework.data.redis.core.RedisOperations; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.data.redis.core.SessionCallback; + +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +/** + * 主机拓扑路径缓存 + */ +@Slf4j +public class HostTopoPathCache extends BaseRedisCache { + + private final RedisTemplate redisTemplate; + private final IamHostTopoPathProperties iamHostTopoPathProperties; + + public HostTopoPathCache(RedisTemplate redisTemplate, + IamHostTopoPathProperties iamHostTopoPathProperties, + MeterRegistry meterRegistry) { + super(meterRegistry, "HostTopoPathCache"); + this.redisTemplate = redisTemplate; + this.iamHostTopoPathProperties = iamHostTopoPathProperties; + } + + /** + * 批量根据hostId查询缓存的主机拓扑路径实例 + * + * @param hostIds 主机ID列表 + * @return 缓存的主机拓扑路径列表, 按照传入的hostIds排序。如果该主机不存在，List中对应索引的值为null + */ + public List batchGetHostTopoPathByHostIds(List hostIds) { + List keys = hostIds.stream().map(this::buildHostIdKey).collect(Collectors.toList()); + return getHostTopoPathByKeys(keys); + } + + private List getHostTopoPathByKeys(List keys) { + long hitCount = 0; + long missCount = 0; + try { + List results = redisTemplate.opsForValue().multiGet(keys); + // 通过 Object 间接强制转换 List + List foundHosts = results == null ? + Collections.emptyList() : (List) (Object) results; + + // multiGet 获取到的 list 中的元素可能为 null （如果key 不存在) + hitCount = foundHosts.stream().filter(Objects::nonNull).count(); + missCount = keys.size() - hitCount; + + return foundHosts; + } catch (Exception e) { + log.warn("Batch get host in cache exception", e); + hitCount = 0; + missCount = keys.size(); + return Collections.emptyList(); + } finally { + if (hitCount > 0) { + addHits(hitCount); + } + if (missCount > 0) { + addMisses(missCount); + } + } + } + + /** + * 删除缓存中的主机拓扑路径实例 + * + * @param hostId 主机ID + */ + public void deleteHostTopoPath(Long hostId) { + String hostIdKey = buildHostIdKey(hostId); + redisTemplate.delete(hostIdKey); + } + + /** + * 更新缓存中的主机拓扑路径实例 + * + * @param hostTopoPathEntry 主机拓扑路径实例 + */ + public void addOrUpdateHostTopoPath(HostTopoPathEntry hostTopoPathEntry) { + String timeFormat = "yyyy-MM-dd HH:mm:ss.SSS"; + hostTopoPathEntry.setCacheTime(TimeUtil.formatTime(System.currentTimeMillis(), timeFormat)); + log.debug( + "Update hostTopoPath cache, hostId: {}, hostTopoPath: {}", + hostTopoPathEntry.getHostId(), + hostTopoPathEntry + ); + String hostIdKey = buildHostIdKey(hostTopoPathEntry.getHostId()); + redisTemplate.opsForValue().set( + hostIdKey, + hostTopoPathEntry, + iamHostTopoPathProperties.getCache().getExpireSeconds(), + TimeUnit.SECONDS + ); + } + + /** + * 批量更新缓存中的主机拓扑路径实例 + * + * @param hostTopoPathList 主机拓扑路径缓存实例列表 + */ + public void batchAddOrUpdateHostTopoPaths(List hostTopoPathList) { + if (CollectionUtils.isEmpty(hostTopoPathList)) { + return; + } + if (hostTopoPathList.size() == 1) { + addOrUpdateHostTopoPath(hostTopoPathList.get(0)); + return; + } + + long start = System.currentTimeMillis(); + redisTemplate.executePipelined(new SessionCallback() { + @Override + public Object execute(@NotNull RedisOperations operations) throws DataAccessException { + hostTopoPathList.forEach(host -> addOrUpdateHostTopoPath(host)); + return null; + } + }); + long cost = System.currentTimeMillis() - start; + if (cost > 1000) { + log.info( + "BatchAddOrUpdateHostTopoPaths slow, hostTopoPathListSize: {}, cost: {}", + hostTopoPathList.size(), + cost + ); + } + } + + private String buildHostIdKey(long hostId) { + return "job:execute:hostTopoPath:hostId:" + hostId; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/auth/impl/HostTopoPathEntry.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/auth/impl/HostTopoPathEntry.java new file mode 100644 index 0000000000..e2875c6723 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/auth/impl/HostTopoPathEntry.java @@ -0,0 +1,64 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.auth.impl; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +import java.util.List; + +@NoArgsConstructor +@Getter +@Setter +public class HostTopoPathEntry { + + /** + * 主机ID + */ + private Long hostId; + /** + * 拓扑路径列表 + */ + private List topoPathList; + /** + * 缓存时间 + */ + private String cacheTime; + + public HostTopoPathEntry(Long hostId, List topoPathList) { + this.hostId = hostId; + this.topoPathList = topoPathList; + } + + @Override + public String toString() { + return "HostTopoPathEntry(" + + "hostId=" + hostId + + ", topoPathList=" + topoPathList + + ", cacheTime=" + cacheTime + + ')'; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/auth/impl/LocalDBTopoPathService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/auth/impl/LocalDBTopoPathService.java new file mode 100644 index 0000000000..619911148a --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/auth/impl/LocalDBTopoPathService.java @@ -0,0 +1,98 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.auth.impl; + +import com.google.common.collect.Lists; +import com.tencent.bk.job.common.util.ConcurrencyUtil; +import com.tencent.bk.job.manage.api.inner.ServiceHostResource; +import com.tencent.bk.job.manage.model.inner.request.ServiceBatchGetHostToposReq; +import com.tencent.bk.job.manage.model.inner.resp.ServiceHostTopoDTO; +import com.tencent.bk.sdk.iam.service.TopoPathService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ExecutorService; +import java.util.stream.Collectors; + +/** + * 主机拓扑路径服务，根据主机ID从本地DB缓存数据获取主机拓扑路径 + */ +@Slf4j +@Service("localDBTopoPathService") +public class LocalDBTopoPathService implements TopoPathService { + + private final ServiceHostResource hostResource; + private final ExecutorService executorService; + + @Autowired + public LocalDBTopoPathService(@Lazy ServiceHostResource serviceHostResource, + @Qualifier("getHostTopoPathExecutor") + ExecutorService executorService) { + this.hostResource = serviceHostResource; + this.executorService = executorService; + } + + @Override + public Map> getTopoPathByHostIds(Set hostIds) { + if (CollectionUtils.isEmpty(hostIds)) { + return Collections.emptyMap(); + } + List hostIdList = hostIds.stream().map(Long::parseLong).collect(Collectors.toList()); + // 分批从job-manage查询，每次最多查询5000台主机 + int batchSize = 5000; + List> hostIdsSubList = Lists.partition(hostIdList, batchSize); + // 多线程并发拉取 + List hostTopoList = ConcurrencyUtil.getResultWithThreads( + hostIdsSubList, + executorService, + (subList) -> hostResource.batchGetHostTopos(new ServiceBatchGetHostToposReq(subList)).getData() + ); + Map> hostTopoPathMap = new HashMap<>(); + hostTopoList.forEach(hostTopoDTO -> { + String hostIdStr = hostTopoDTO.getHostId().toString(); + if (!hostTopoPathMap.containsKey(hostIdStr)) { + hostTopoPathMap.put(hostIdStr, new ArrayList<>()); + } + hostTopoPathMap.get(hostIdStr).add(buildTopoPath(hostTopoDTO)); + }); + return hostTopoPathMap; + } + + private String buildTopoPath(ServiceHostTopoDTO hostTopo) { + return "/biz," + hostTopo.getBizId() + + "/set," + hostTopo.getSetId() + + "/module," + hostTopo.getModuleId() + "/"; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/auth/impl/SwitchableTopoPathService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/auth/impl/SwitchableTopoPathService.java new file mode 100644 index 0000000000..03eb934fad --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/auth/impl/SwitchableTopoPathService.java @@ -0,0 +1,66 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.auth.impl; + +import com.tencent.bk.job.execute.config.IamHostTopoPathProperties; +import com.tencent.bk.sdk.iam.service.TopoPathService; +import lombok.extern.slf4j.Slf4j; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * 可以动态开关的主机拓扑路径查询服务 + */ +@Slf4j +public class SwitchableTopoPathService implements TopoPathService { + + /** + * 是否开启 + */ + private volatile boolean status; + private final TopoPathService delegate; + + public SwitchableTopoPathService(TopoPathService delegate, IamHostTopoPathProperties iamHostTopoPathProperties) { + this.delegate = delegate; + this.status = iamHostTopoPathProperties.getEnabled(); + } + + @Override + public Map> getTopoPathByHostIds(Set hostIds) { + if (status) { + return delegate.getTopoPathByHostIds(hostIds); + } + return Collections.emptyMap(); + } + + public boolean switchStatus(boolean status) { + log.info("Switch status from {} to {}", this.status, status); + this.status = status; + return status; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/AccountResourceClient.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/AccountResourceClient.java deleted file mode 100644 index e47467952c..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/AccountResourceClient.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.client; - -import com.tencent.bk.job.manage.api.inner.ServiceAccountResource; -import org.springframework.cloud.openfeign.FeignClient; - -/** - * 系统账号服务远程调用客户端 - */ -@FeignClient(value = "job-manage", contextId = "account") -public interface AccountResourceClient extends ServiceAccountResource { -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/ApplicationResourceClient.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/ApplicationResourceClient.java deleted file mode 100644 index 4730274dc5..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/ApplicationResourceClient.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.client; - -import com.tencent.bk.job.manage.api.inner.ServiceApplicationResource; -import org.springframework.cloud.openfeign.FeignClient; - -/** - * 业务服务远程调用客户端 - */ -@FeignClient(value = "job-manage", contextId = "app") -public interface ApplicationResourceClient extends ServiceApplicationResource { -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/FileSourceResourceClient.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/FileSourceResourceClient.java deleted file mode 100644 index 6fe3025433..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/FileSourceResourceClient.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.client; - -import com.tencent.bk.job.file_gateway.api.inner.ServiceFileSourceResource; -import org.springframework.cloud.openfeign.FeignClient; - -/** - * 文件源服务远程调用客户端 - */ -@FeignClient(value = "job-file-gateway", contextId = "fileSource") -public interface FileSourceResourceClient extends ServiceFileSourceResource { -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/FileSourceTaskResourceClient.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/FileSourceTaskResourceClient.java deleted file mode 100644 index 8813afe388..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/FileSourceTaskResourceClient.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.client; - -import com.tencent.bk.job.file_gateway.api.inner.ServiceFileSourceTaskResource; -import org.springframework.cloud.openfeign.FeignClient; - -/** - * 文件源任务服务远程调用客户端 - */ -@FeignClient(value = "job-file-gateway", contextId = "fileSourceTask") -public interface FileSourceTaskResourceClient extends ServiceFileSourceTaskResource { -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/GlobalSettingsClient.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/GlobalSettingsClient.java deleted file mode 100644 index 290cab8f55..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/GlobalSettingsClient.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.client; - -import com.tencent.bk.job.manage.api.inner.ServiceGlobalSettingsResource; -import org.springframework.cloud.openfeign.FeignClient; - -/** - * 作业管理-全局配置 - */ -@FeignClient(value = "job-manage", contextId = "globalSettings") -public interface GlobalSettingsClient extends ServiceGlobalSettingsResource { -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/LogServiceResourceClient.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/LogServiceResourceClient.java deleted file mode 100644 index e9b833a996..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/LogServiceResourceClient.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.client; - -import com.tencent.bk.job.logsvr.api.ServiceLogResource; -import org.springframework.cloud.openfeign.FeignClient; - -/** - * 日志服务远程调用客户端 - */ -@FeignClient(value = "job-logsvr", contextId = "log") -public interface LogServiceResourceClient extends ServiceLogResource { -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/ScriptCheckResourceClient.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/ScriptCheckResourceClient.java deleted file mode 100644 index 1d4ae4ecc0..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/ScriptCheckResourceClient.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.client; - -import com.tencent.bk.job.manage.api.inner.ServiceCheckScriptResource; -import org.springframework.cloud.openfeign.FeignClient; - -/** - * 系统账号服务远程调用客户端 - */ -@FeignClient(value = "job-manage", contextId = "scriptCheck") -public interface ScriptCheckResourceClient extends ServiceCheckScriptResource { -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/ScriptResourceClient.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/ScriptResourceClient.java deleted file mode 100644 index b55ad6cf12..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/ScriptResourceClient.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.client; - -import com.tencent.bk.job.manage.api.inner.ServiceScriptResource; -import org.springframework.cloud.openfeign.FeignClient; - -/** - * 作业管理-脚本服务远程调用客户端 - */ -@FeignClient(value = "job-manage", contextId = "script") -public interface ScriptResourceClient extends ServiceScriptResource { -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/ServiceHostResourceClient.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/ServiceHostResourceClient.java deleted file mode 100644 index 69f534cfaa..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/ServiceHostResourceClient.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.client; - -import com.tencent.bk.job.manage.api.inner.ServiceHostResource; -import org.springframework.cloud.openfeign.FeignClient; - -/** - * 业务主机服务远程调用客户端 - */ -@FeignClient(value = "job-manage", contextId = "host") -public interface ServiceHostResourceClient extends ServiceHostResource { -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/ServiceNotificationResourceClient.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/ServiceNotificationResourceClient.java deleted file mode 100644 index 27d590fd73..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/ServiceNotificationResourceClient.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.client; - -import com.tencent.bk.job.manage.api.inner.ServiceNotificationResource; -import org.springframework.cloud.openfeign.FeignClient; - -/** - * 消息通知服务远程调用客户端 - */ -@FeignClient(value = "job-manage", contextId = "notification") -public interface ServiceNotificationResourceClient extends ServiceNotificationResource { -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/ServiceUserResourceClient.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/ServiceUserResourceClient.java deleted file mode 100644 index 4c794be466..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/ServiceUserResourceClient.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.client; - -import com.tencent.bk.job.manage.api.inner.ServiceUserResource; -import org.springframework.cloud.openfeign.FeignClient; - -/** - * 作业管理-用户服务调用客户端 - */ -@FeignClient(value = "job-manage", contextId = "user-service") -public interface ServiceUserResourceClient extends ServiceUserResource { -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/SyncResourceClient.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/SyncResourceClient.java deleted file mode 100644 index 098ddf928c..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/SyncResourceClient.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.client; - -import com.tencent.bk.job.manage.api.inner.ServiceSyncResource; -import org.springframework.cloud.openfeign.FeignClient; - -/** - * 从job-manage同步业务、主机服务 - */ -@FeignClient(value = "job-manage", contextId = "sync") -public interface SyncResourceClient extends ServiceSyncResource { -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/TaskPlanResourceClient.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/TaskPlanResourceClient.java deleted file mode 100644 index 427057a6f6..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/TaskPlanResourceClient.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.client; - -import com.tencent.bk.job.manage.api.inner.ServiceTaskPlanResource; -import org.springframework.cloud.openfeign.FeignClient; - -/** - * 作业管理-执行方案服务远程调用客户端 - */ -@FeignClient(value = "job-manage", contextId = "plan") -public interface TaskPlanResourceClient extends ServiceTaskPlanResource { - -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/TaskTemplateResourceClient.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/TaskTemplateResourceClient.java deleted file mode 100644 index 5925f8921b..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/TaskTemplateResourceClient.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.client; - -import com.tencent.bk.job.manage.api.inner.ServiceTaskTemplateResource; -import org.springframework.cloud.openfeign.FeignClient; - -/** - * 作业管理-作业模板服务远程调用客户端 - */ -@FeignClient(value = "job-manage", contextId = "template") -public interface TaskTemplateResourceClient extends ServiceTaskTemplateResource { - -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/WhiteIpResourceClient.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/WhiteIpResourceClient.java deleted file mode 100644 index 93da5452fb..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/client/WhiteIpResourceClient.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.client; - -import com.tencent.bk.job.manage.api.inner.ServiceWhiteIPResource; -import org.springframework.cloud.openfeign.FeignClient; - -/** - * Ip白名单服务远程调用客户端 - */ -@FeignClient(value = "job-manage", contextId = "white-ip") -public interface WhiteIpResourceClient extends ServiceWhiteIPResource { -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/cache/WhiteHostCache.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/cache/WhiteHostCache.java new file mode 100644 index 0000000000..ef470834ed --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/cache/WhiteHostCache.java @@ -0,0 +1,132 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.common.cache; + +import com.tencent.bk.job.common.model.InternalResponse; +import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.manage.api.inner.ServiceWhiteIPResource; +import com.tencent.bk.job.manage.model.inner.ServiceWhiteIPInfo; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReadWriteLock; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +/** + * 主机白名单本地缓存 + */ +@Component +@Slf4j +public class WhiteHostCache { + private final ServiceWhiteIPResource whiteIpResource; + + private volatile boolean isWhiteIpConfigLoaded = false; + /** + * 主机白名单缓存， key: hostId, value: 白名单配置 + */ + private final Map whiteHostConfig = new HashMap<>(); + + private static final ReadWriteLock rwLock = new ReentrantReadWriteLock(); + private static final Lock rLock = rwLock.readLock(); + private static final Lock wLock = rwLock.writeLock(); + + @Autowired + public WhiteHostCache(ServiceWhiteIPResource whiteIpResource) { + this.whiteIpResource = whiteIpResource; + } + + @Scheduled(cron = "0 * * * * ?") + public void syncWhiteIpConfig() { + log.info("Sync white host config!"); + isWhiteIpConfigLoaded = true; + long start = System.currentTimeMillis(); + InternalResponse> resp = whiteIpResource.listWhiteIPInfos(); + if (resp == null || !resp.isSuccess()) { + log.warn("Get all white host config return fail resp! resp: {}", JsonUtils.toJson(resp)); + return; + } + log.info("Sync white host config, resp: {}", JsonUtils.toJson(resp)); + + refreshCache(resp.getData()); + + long cost = System.currentTimeMillis() - start; + if (cost > 1000L) { + log.warn("Sync white host config is slow, cost: {}", cost); + } + log.info("Sync white host config success!"); + } + + private void refreshCache(List whiteIpInfos) { + try { + wLock.lock(); + whiteHostConfig.clear(); + whiteIpInfos.forEach(whiteIpInfo -> { + if (whiteIpInfo.getHostId() != null) { + whiteHostConfig.put(whiteIpInfo.getHostId(), whiteIpInfo); + } + }); + log.info("Refresh white host cache success. whiteHostConfig: {}", JsonUtils.toJson(whiteHostConfig)); + } finally { + wLock.unlock(); + } + } + + public List getHostAllowedAction(long appId, long hostId) { + try { + if (!isWhiteIpConfigLoaded) { + syncWhiteIpConfig(); + } + try { + rLock.lock(); + ServiceWhiteIPInfo whiteIPInfo = whiteHostConfig.get(hostId); + if (whiteIPInfo == null) { + return null; + } + if (whiteIPInfo.isForAllApp()) { + return whiteIPInfo.getAllAppActionScopeList(); + } else { + if (whiteIPInfo.getAppIdActionScopeMap() != null + && !whiteIPInfo.getAppIdActionScopeMap().isEmpty()) { + return whiteIPInfo.getAppIdActionScopeMap().get(appId); + } else { + return null; + } + } + } finally { + rLock.unlock(); + } + } catch (Exception e) { + log.warn("GetHostAllowedAction fail!", e); + return null; + } + } + +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/context/JobExecuteContext.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/context/JobExecuteContext.java new file mode 100644 index 0000000000..1978c2b9dc --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/context/JobExecuteContext.java @@ -0,0 +1,54 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.common.context; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.tencent.bk.job.common.model.dto.AppResourceScope; +import lombok.Data; + +/** + * 作业执行上下文信息，用于在全局共享、异步传播(跨线程、跨 mq等场景) + */ +@Data +@JsonInclude(JsonInclude.Include.NON_EMPTY) +public class JobExecuteContext { + + public static final String KEY = "JobExecuteContext"; + + /** + * 业务信息 + */ + private AppResourceScope resourceScope; + + /** + * 用户账号 + */ + private String username; + + /** + * 作业实例 + */ + private JobInstanceContext jobInstanceContext; +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/context/JobExecuteContextThreadLocalRepo.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/context/JobExecuteContextThreadLocalRepo.java new file mode 100644 index 0000000000..4f69542c01 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/context/JobExecuteContextThreadLocalRepo.java @@ -0,0 +1,54 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.common.context; + +import lombok.extern.slf4j.Slf4j; + +/** + * 作业执行上下文托管（ThreadLocal方式) + */ +@Slf4j +public class JobExecuteContextThreadLocalRepo { + + public static final ThreadLocal HOLDER = new ThreadLocal<>(); + + public static void set(JobExecuteContext context) { + if (log.isDebugEnabled()) { + log.info("SetJobExecuteContextThreadLocalRepo, context: {}", context); + } + HOLDER.set(context); + } + + public static void unset() { + if (log.isDebugEnabled()) { + log.debug("RemoveFromJobExecuteContextThreadLocalRepo"); + } + HOLDER.remove(); + } + + public static JobExecuteContext get() { + return HOLDER.get(); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/context/JobInstanceContext.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/context/JobInstanceContext.java new file mode 100644 index 0000000000..0a5be95abb --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/context/JobInstanceContext.java @@ -0,0 +1,52 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.common.context; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.tencent.bk.job.common.constant.ArchiveFlag; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * 可传播的Job作业实例上下文，用于在异步线程、跨进程(http/mq 通信等)场景下传递、共享作业实例数据 + */ +@Data +@JsonInclude(JsonInclude.Include.NON_EMPTY) +@NoArgsConstructor +public class JobInstanceContext { + + private Long taskInstanceId; + + /** + * 数据归档状态 + * + * @see ArchiveFlag + */ + private int archiveFlag = ArchiveFlag.HOT.getValue(); + + public JobInstanceContext(Long taskInstanceId) { + this.taskInstanceId = taskInstanceId; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/converter/StepTypeExecuteTypeConverter.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/converter/StepTypeExecuteTypeConverter.java index e0fc33fb24..abdaf5121b 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/converter/StepTypeExecuteTypeConverter.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/converter/StepTypeExecuteTypeConverter.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,21 +25,9 @@ package com.tencent.bk.job.execute.common.converter; import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskStepTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskStepTypeEnum; public class StepTypeExecuteTypeConverter { - public static Integer convertToStepType(Integer executeType) { - if (executeType.equals(StepExecuteTypeEnum.EXECUTE_SCRIPT.getValue()) - || executeType.equals(StepExecuteTypeEnum.EXECUTE_SQL.getValue())) { - return TaskStepTypeEnum.SCRIPT.getValue(); - } else if (executeType.equals(StepExecuteTypeEnum.SEND_FILE.getValue())) { - return TaskStepTypeEnum.FILE.getValue(); - } else if (executeType.equals(StepExecuteTypeEnum.MANUAL_CONFIRM.getValue())) { - return TaskStepTypeEnum.APPROVAL.getValue(); - } else { - throw new IllegalArgumentException("executeType"); - } - } public static TaskStepTypeEnum convertToStepType(StepExecuteTypeEnum executeType) { switch (executeType) { diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/exception/MessageHandleException.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/exception/MessageHandleException.java index b20f69ddc1..b813f44a4e 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/exception/MessageHandleException.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/exception/MessageHandleException.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/exception/MessageHandlerUnavailableException.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/exception/MessageHandlerUnavailableException.java index 124dc22eb9..bed36f5c14 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/exception/MessageHandlerUnavailableException.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/exception/MessageHandlerUnavailableException.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/exception/ObtainHostServiceException.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/exception/ObtainHostServiceException.java index 61990217e5..82063f1c95 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/exception/ObtainHostServiceException.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/exception/ObtainHostServiceException.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/exception/ReadTimeoutException.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/exception/ReadTimeoutException.java deleted file mode 100644 index 00514eda58..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/exception/ReadTimeoutException.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.common.exception; - -/** - * 消息处理异常 - */ -public class ReadTimeoutException extends RuntimeException { - - public ReadTimeoutException() { - super("Read timeout"); - } - - public ReadTimeoutException(String errorMessage) { - super(errorMessage); - } -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/exception/RollingExprParseException.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/exception/RollingExprParseException.java index 77892a5686..762f5d58eb 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/exception/RollingExprParseException.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/exception/RollingExprParseException.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/exception/RunningJobQuotaLimitExceedException.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/exception/RunningJobQuotaLimitExceedException.java new file mode 100644 index 0000000000..fc7304d309 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/exception/RunningJobQuotaLimitExceedException.java @@ -0,0 +1,37 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.common.exception; + +import com.tencent.bk.job.common.exception.ResourceExhaustedException; + +/** + * 正在执行作业数量超过配额异常 + */ +public class RunningJobQuotaLimitExceedException extends ResourceExhaustedException { + + public RunningJobQuotaLimitExceedException(Integer errorCode) { + super(errorCode); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/ha/DestroyOrder.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/ha/DestroyOrder.java index feac725f36..8545d1e52c 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/ha/DestroyOrder.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/ha/DestroyOrder.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/ha/GracefulShutdownGlobalHandler.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/ha/GracefulShutdownGlobalHandler.java index e4978f1fa3..f499dc29ca 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/ha/GracefulShutdownGlobalHandler.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/ha/GracefulShutdownGlobalHandler.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/ha/mq/JobListenerContainerCustomizer.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/ha/mq/JobListenerContainerCustomizer.java index 173bc1e2a6..ddcdaee866 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/ha/mq/JobListenerContainerCustomizer.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/ha/mq/JobListenerContainerCustomizer.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextHttpInterceptor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextHttpInterceptor.java new file mode 100644 index 0000000000..1bced70d91 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextHttpInterceptor.java @@ -0,0 +1,84 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.common.interceptor; + +import com.tencent.bk.job.common.annotation.JobInterceptor; +import com.tencent.bk.job.common.constant.InterceptorOrder; +import com.tencent.bk.job.common.context.JobContext; +import com.tencent.bk.job.common.util.JobContextUtil; +import com.tencent.bk.job.execute.common.context.JobExecuteContext; +import com.tencent.bk.job.execute.common.context.JobExecuteContextThreadLocalRepo; +import lombok.extern.slf4j.Slf4j; +import org.springframework.lang.NonNull; +import org.springframework.stereotype.Component; +import org.springframework.web.servlet.AsyncHandlerInterceptor; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/** + * 在请求处理之前，初始话作业执行上下文(JobExecuteContext）；在请求结束之前，删除 JobExecuteContext + */ +@Slf4j +@Component +@JobInterceptor(order = InterceptorOrder.Init.LOWEST, pathPatterns = "/**") +public class JobExecuteContextHttpInterceptor implements AsyncHandlerInterceptor { + + @Override + public boolean preHandle(@NonNull HttpServletRequest request, + @NonNull HttpServletResponse response, + @NonNull Object handler) { + if (!shouldFilter(request)) { + return true; + } + + JobContext jobContext = JobContextUtil.getContext(); + if (jobContext != null) { + JobExecuteContext jobExecuteContext = new JobExecuteContext(); + jobExecuteContext.setResourceScope(jobContext.getAppResourceScope()); + jobExecuteContext.setUsername(jobContext.getUsername()); + JobExecuteContextThreadLocalRepo.set(jobExecuteContext); + if (log.isDebugEnabled()) { + log.debug("JobExecuteContextInterceptor -> Set JobExecuteContext : {}", jobExecuteContext); + } + } + + return true; + } + + private boolean shouldFilter(HttpServletRequest request) { + String uri = request.getRequestURI(); + // 只拦截web/service/esb的API请求 + return uri.startsWith("/web/") || uri.startsWith("/service/") || uri.startsWith("/esb/"); + } + + @Override + public void afterCompletion(@NonNull HttpServletRequest request, + @NonNull HttpServletResponse response, + @NonNull Object handler, + Exception ex) { + JobExecuteContextThreadLocalRepo.unset(); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextPropagateMqInterceptor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextPropagateMqInterceptor.java new file mode 100644 index 0000000000..4f4081e97c --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteContextPropagateMqInterceptor.java @@ -0,0 +1,61 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.common.interceptor; + +import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.execute.common.context.JobExecuteContext; +import com.tencent.bk.job.execute.common.context.JobExecuteContextThreadLocalRepo; +import lombok.extern.slf4j.Slf4j; +import org.springframework.integration.config.GlobalChannelInterceptor; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageChannel; +import org.springframework.messaging.support.ChannelInterceptor; +import org.springframework.messaging.support.MessageBuilder; +import org.springframework.stereotype.Component; + +/** + * Mq 消息全局拦截 - JobExecuteContext 上下文传递 + */ +@Component +@Slf4j +@GlobalChannelInterceptor +public class JobExecuteContextPropagateMqInterceptor implements ChannelInterceptor { + + @Override + public Message preSend(Message message, MessageChannel channel) { + JobExecuteContext context = JobExecuteContextThreadLocalRepo.get(); + if (context == null) { + return ChannelInterceptor.super.preSend(message, channel); + } + if (log.isDebugEnabled()) { + log.debug("SetJobExecuteContextMessageHeader, context: {}", context); + } + Message newMessage = + MessageBuilder.fromMessage(message) + .setHeader(JobExecuteContext.KEY, JsonUtils.toJson(context)) + .build(); + return ChannelInterceptor.super.preSend(newMessage, channel); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteUriPermissionInterceptor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteUriPermissionInterceptor.java new file mode 100644 index 0000000000..f4d3bf7628 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/JobExecuteUriPermissionInterceptor.java @@ -0,0 +1,74 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.common.interceptor; + +import com.tencent.bk.job.common.annotation.JobInterceptor; +import com.tencent.bk.job.common.constant.InterceptorOrder; +import com.tencent.bk.job.common.iam.constant.ActionId; +import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; +import com.tencent.bk.job.common.iam.model.AuthResult; +import com.tencent.bk.job.common.iam.service.AuthService; +import com.tencent.bk.job.common.util.JobContextUtil; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.web.servlet.handler.HandlerInterceptorAdapter; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/** + * Uri权限控制拦截 + */ +@Slf4j +@Component +@JobInterceptor(pathPatterns = {"/web/dangerous-record/**"}, + order = InterceptorOrder.AUTH.AUTH_COMMON) +public class JobExecuteUriPermissionInterceptor extends HandlerInterceptorAdapter { + private final AuthService authService; + + @Autowired + public JobExecuteUriPermissionInterceptor(AuthService authService) { + this.authService = authService; + } + + @Override + public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) { + String username = JobContextUtil.getUsername(); + AuthResult authResult = authService.auth(username, ActionId.HIGH_RISK_DETECT_RECORD); + if (!authResult.isPass()) { + throw new PermissionDeniedException(authResult); + } + return true; + } + + @Override + public void afterCompletion(HttpServletRequest request, + HttpServletResponse response, + Object handler, + Exception ex) { + + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/UriPermissionInterceptor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/UriPermissionInterceptor.java deleted file mode 100644 index 2d2d3c0f85..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/interceptor/UriPermissionInterceptor.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.common.interceptor; - -import com.tencent.bk.job.common.RequestIdLogger; -import com.tencent.bk.job.common.iam.constant.ActionId; -import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; -import com.tencent.bk.job.common.iam.model.AuthResult; -import com.tencent.bk.job.common.iam.service.AuthService; -import com.tencent.bk.job.common.util.JobContextUtil; -import com.tencent.bk.job.common.util.SimpleRequestIdLogger; -import lombok.extern.slf4j.Slf4j; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import org.springframework.util.AntPathMatcher; -import org.springframework.util.PathMatcher; -import org.springframework.web.servlet.handler.HandlerInterceptorAdapter; - -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import java.util.Collections; -import java.util.List; - -/** - * Uri权限控制拦截 - */ -@Slf4j -@Component -public class UriPermissionInterceptor extends HandlerInterceptorAdapter { - private static final RequestIdLogger logger = - new SimpleRequestIdLogger(LoggerFactory.getLogger(UriPermissionInterceptor.class)); - private final String URI_PATTERN_DANGEROUS_RECORD = "/web/dangerous-record/**"; - private final AuthService authService; - private final PathMatcher pathMatcher; - - @Autowired - public UriPermissionInterceptor(AuthService authService) { - this.authService = authService; - this.pathMatcher = new AntPathMatcher(); - } - - public String[] getControlUriPatterns() { - Object[] rawArr = getControlUriPatternsList().toArray(); - String[] arr = new String[rawArr.length]; - for (int i = 0; i < rawArr.length; i++) { - arr[i] = (String) rawArr[i]; - } - return arr; - } - - private List getControlUriPatternsList() { - return Collections.singletonList( - // 高危语句拦截记录 - URI_PATTERN_DANGEROUS_RECORD - ); - } - - @Override - public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) - throws Exception { - String username = JobContextUtil.getUsername(); - String uri = request.getRequestURI(); - logger.infoWithRequestId("PermissionControlInterceptor.preHandle:username=" + username + ", uri=" + uri + ", " + - "controlUriPatterns=" + getControlUriPatternsList()); - if (pathMatcher.match(URI_PATTERN_DANGEROUS_RECORD, uri)) { - AuthResult authResult = authService.auth(username, ActionId.HIGH_RISK_DETECT_RECORD); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } - } - return true; - } - - @Override - public void afterCompletion(HttpServletRequest request, HttpServletResponse response, Object handler, Exception ex) - throws Exception { - - } -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/util/JooqDataTypeUtil.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/util/JooqDataTypeUtil.java deleted file mode 100644 index 763a24345e..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/util/JooqDataTypeUtil.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.common.util; - -import org.jooq.types.UByte; - -public class JooqDataTypeUtil { - - public static UByte toUByte(Integer value) { - if (value == null) { - return null; - } - return UByte.valueOf(value); - } - - public static Byte toByte(Integer value) { - if (value == null) { - return null; - } - return Byte.valueOf(String.valueOf(value)); - } - - public static Short toShort(Integer value) { - if (value == null) { - return null; - } - return Short.valueOf(String.valueOf(value)); - } - - public static Integer toInteger(Byte value) { - if (value == null) { - return null; - } - return value.intValue(); - } - -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/util/TaskCostCalculator.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/util/TaskCostCalculator.java index 0618e5cd89..32dbfa07ac 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/util/TaskCostCalculator.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/util/TaskCostCalculator.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/util/VariableValueResolver.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/util/VariableValueResolver.java index b1ef9660b2..fc8459832b 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/util/VariableValueResolver.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/common/util/VariableValueResolver.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,7 @@ package com.tencent.bk.job.execute.common.util; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import java.util.Map; @@ -33,7 +34,15 @@ /** * 变量值解析 */ +@Slf4j public class VariableValueResolver { + /** + * 解析字符串中的变量值 + * + * @param param 字符串参数 + * @param variableMap 变量KV + * @return 解析后的字符串 + */ public static String resolve(String param, Map variableMap) { if (StringUtils.isBlank(param) || variableMap == null || variableMap.isEmpty()) { return param; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/ArtifactoryConfig.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/ArtifactoryConfig.java deleted file mode 100644 index ca634e0ddf..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/ArtifactoryConfig.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.config; - -import lombok.Data; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Configuration; - -@Data -@Configuration -public class ArtifactoryConfig { - - @Value("${artifactory.base-url:}") - private String artifactoryBaseUrl; - - @Value("${artifactory.admin.username:admin}") - private String artifactoryAdminUsername; - - @Value("${artifactory.admin.password:blueking}") - private String artifactoryAdminPassword; - - @Value("${artifactory.job.username:bkjob}") - private String artifactoryJobUsername; - - @Value("${artifactory.job.password:bkjob}") - private String artifactoryJobPassword; - - @Value("${artifactory.job.project:bkjob}") - private String artifactoryJobProject; - -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/ConditionalOnExternalAgentDisabled.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/ConditionalOnExternalAgentDisabled.java new file mode 100644 index 0000000000..9a1203cffc --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/ConditionalOnExternalAgentDisabled.java @@ -0,0 +1,46 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.config; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target(ElementType.METHOD) +@Retention(RetentionPolicy.RUNTIME) +@Documented +@Inherited +@ConditionalOnProperty( + value = "external-gse-agent.enabled", + havingValue = "false", + matchIfMissing = true +) +public @interface ConditionalOnExternalAgentDisabled { +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/ConditionalOnExternalAgentEnabled.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/ConditionalOnExternalAgentEnabled.java new file mode 100644 index 0000000000..b6c8cfc036 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/ConditionalOnExternalAgentEnabled.java @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.config; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target(ElementType.METHOD) +@Retention(RetentionPolicy.RUNTIME) +@Documented +@Inherited +@ConditionalOnProperty( + value = "external-gse-agent.enabled", + havingValue = "true" +) +public @interface ConditionalOnExternalAgentEnabled { +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/ConditionalOnIamHostTopoPathCacheEnabled.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/ConditionalOnIamHostTopoPathCacheEnabled.java new file mode 100644 index 0000000000..80658c30e2 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/ConditionalOnIamHostTopoPathCacheEnabled.java @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.config; + +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; + +import java.lang.annotation.Documented; +import java.lang.annotation.ElementType; +import java.lang.annotation.Inherited; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +@Documented +@Inherited +@ConditionalOnProperty( + value = "job.execute.iam.host-topo-path.cache.enabled", + havingValue = "true", + matchIfMissing = true +) +public @interface ConditionalOnIamHostTopoPathCacheEnabled { + +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/DSLContextConfiguration.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/DSLContextConfiguration.java new file mode 100644 index 0000000000..0136d5e99f --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/DSLContextConfiguration.java @@ -0,0 +1,357 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.config; + +import com.tencent.bk.job.common.mysql.MySQLProperties; +import com.tencent.bk.job.common.mysql.dynamic.ds.HorizontalShardingDSLContextProvider; +import com.tencent.bk.job.common.mysql.dynamic.ds.MigrateDynamicDSLContextProvider; +import com.tencent.bk.job.common.mysql.dynamic.ds.StandaloneDSLContextProvider; +import com.tencent.bk.job.common.mysql.dynamic.ds.VerticalShardingDSLContextProvider; +import com.tencent.bk.job.common.mysql.util.JooqConfigurationUtil; +import com.tencent.bk.job.common.util.toggle.prop.PropToggleStore; +import com.tencent.bk.job.execute.dao.common.DSLContextProviderFactory; +import com.tencent.bk.job.execute.dao.common.JobExecuteVerticalShardingDSLContextProvider; +import com.tencent.bk.job.execute.dao.common.PropBasedDynamicDataSource; +import com.tencent.bk.job.execute.dao.common.ReadWriteLockDbMigrateAspect; +import lombok.extern.slf4j.Slf4j; +import org.jooq.ConnectionProvider; +import org.jooq.DSLContext; +import org.jooq.impl.DataSourceConnectionProvider; +import org.jooq.impl.DefaultDSLContext; +import org.jooq.impl.DefaultExecuteListenerProvider; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.jdbc.DataSourceBuilder; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.DependsOn; +import org.springframework.context.annotation.Primary; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.DataSourceTransactionManager; +import org.springframework.jdbc.datasource.TransactionAwareDataSourceProxy; + +import javax.sql.DataSource; + +@Slf4j +@Configuration(value = "jobExecuteDSLContextConfiguration") +public class DSLContextConfiguration { + + @ConditionalOnProperty(value = "mysql.standalone.enabled", havingValue = "true", matchIfMissing = false) + protected static class StandaloneDslContextConfiguration { + @Qualifier("job-execute-data-source") + @Bean(name = "job-execute-data-source") + @ConfigurationProperties(prefix = "spring.datasource.job-execute") + public DataSource dataSource() { + return DataSourceBuilder.create().build(); + } + + @Qualifier("job-execute-transaction-manager") + @Bean(name = "job-execute-transaction-manager") + @DependsOn("job-execute-data-source") + public DataSourceTransactionManager transactionManager( + @Qualifier("job-execute-data-source") DataSource dataSource) { + return new DataSourceTransactionManager(dataSource); + } + + @Qualifier("job-execute-jdbc-template") + @Bean(name = "job-execute-jdbc-template") + public JdbcTemplate jdbcTemplate(@Qualifier("job-execute-data-source") DataSource dataSource) { + return new JdbcTemplate(dataSource); + } + + @Qualifier("job-execute-dsl-context") + @Bean(name = "job-execute-dsl-context") + public DSLContext dslContext(@Qualifier("job-execute-jooq-conf") org.jooq.Configuration configuration) { + log.info("Init DSLContext job-execute-standalone"); + return new DefaultDSLContext(configuration); + } + + @Qualifier("job-execute-jooq-conf") + @Bean(name = "job-execute-jooq-conf") + public org.jooq.Configuration jooqConf( + @Qualifier("job-execute-conn-provider") ConnectionProvider connectionProvider, + DefaultExecuteListenerProvider jooqExecuteListenerProvider + ) { + return JooqConfigurationUtil.getConfiguration(connectionProvider, jooqExecuteListenerProvider); + } + + @Qualifier("job-execute-conn-provider") + @Bean(name = "job-execute-conn-provider") + public ConnectionProvider connectionProvider( + @Qualifier("job-execute-transaction-aware-data-source") DataSource dataSource) { + return new DataSourceConnectionProvider(dataSource); + } + + @Qualifier("job-execute-transaction-aware-data-source") + @Bean(name = "job-execute-transaction-aware-data-source") + public TransactionAwareDataSourceProxy + transactionAwareDataSourceProxy(@Qualifier("job-execute-data-source") DataSource dataSource) { + return new TransactionAwareDataSourceProxy(dataSource); + } + + @Qualifier("job-execute-standalone-dsl-context-provider") + @Bean(name = "job-execute-standalone-dsl-context-provider") + @Primary + public StandaloneDSLContextProvider standaloneDSLContextProvider( + @Qualifier("job-execute-dsl-context") DSLContext dslContext + ) { + log.info("Init StandaloneDSLContextProvider"); + return new StandaloneDSLContextProvider(dslContext); + } + } + + @ConditionalOnProperty(value = "mysql.verticalSharding.enabled", + havingValue = "true", matchIfMissing = false) + protected static class VerticalDslContextConfiguration { + // 配置垂直分片数据源-a + @Qualifier("job-execute-data-source-a") + @Bean(name = "job-execute-data-source-a") + @ConfigurationProperties(prefix = "spring.datasource.job-execute-vertical-a") + public DataSource dataSourceA() { + return DataSourceBuilder.create().build(); + } + + @Qualifier("job-execute-transaction-manager-a") + @Bean(name = "job-execute-transaction-manager-a") + @DependsOn("job-execute-data-source-a") + public DataSourceTransactionManager transactionManagerA( + @Qualifier("job-execute-data-source-a") DataSource dataSource) { + return new DataSourceTransactionManager(dataSource); + } + + @Qualifier("job-execute-jdbc-template-a") + @Bean(name = "job-execute-jdbc-template-a") + public JdbcTemplate jdbcTemplateA(@Qualifier("job-execute-data-source-a") DataSource dataSource) { + return new JdbcTemplate(dataSource); + } + + @Qualifier("job-execute-dsl-context-a") + @Bean(name = "job-execute-dsl-context-a") + public DSLContext dslContextA(@Qualifier("job-execute-jooq-conf-a") org.jooq.Configuration configuration) { + log.info("Init DSLContext job-execute-vertical-a"); + return new DefaultDSLContext(configuration); + } + + @Qualifier("job-execute-jooq-conf-a") + @Bean(name = "job-execute-jooq-conf-a") + public org.jooq.Configuration jooqConfA( + @Qualifier("job-execute-conn-provider-a") ConnectionProvider connectionProvider, + DefaultExecuteListenerProvider jooqExecuteListenerProvider + ) { + return JooqConfigurationUtil.getConfiguration(connectionProvider, jooqExecuteListenerProvider); + } + + @Qualifier("job-execute-conn-provider-a") + @Bean(name = "job-execute-conn-provider-a") + public ConnectionProvider connectionProviderA( + @Qualifier("job-execute-transaction-aware-data-source-a") DataSource dataSource) { + return new DataSourceConnectionProvider(dataSource); + } + + @Qualifier("job-execute-transaction-aware-data-source-a") + @Bean(name = "job-execute-transaction-aware-data-source-a") + public TransactionAwareDataSourceProxy + transactionAwareDataSourceProxyA(@Qualifier("job-execute-data-source-a") DataSource dataSource) { + return new TransactionAwareDataSourceProxy(dataSource); + } + + // 配置垂直分片数据源-b + @Qualifier("job-execute-data-source-b") + @Bean(name = "job-execute-data-source-b") + @ConfigurationProperties(prefix = "spring.datasource.job-execute-vertical-b") + public DataSource dataSourceB() { + return DataSourceBuilder.create().build(); + } + + @Qualifier("job-execute-transaction-manager-b") + @Bean(name = "job-execute-transaction-manager-b") + @DependsOn("job-execute-data-source-b") + public DataSourceTransactionManager transactionManagerB( + @Qualifier("job-execute-data-source-b") DataSource dataSource) { + return new DataSourceTransactionManager(dataSource); + } + + @Qualifier("job-execute-jdbc-template-b") + @Bean(name = "job-execute-jdbc-template-b") + public JdbcTemplate jdbcTemplateB(@Qualifier("job-execute-data-source-b") DataSource dataSource) { + return new JdbcTemplate(dataSource); + } + + @Qualifier("job-execute-dsl-context-b") + @Bean(name = "job-execute-dsl-context-b") + public DSLContext dslContextB(@Qualifier("job-execute-jooq-conf-b") org.jooq.Configuration configuration) { + log.info("Init DSLContext job-execute-vertical-b"); + return new DefaultDSLContext(configuration); + } + + @Qualifier("job-execute-jooq-conf-b") + @Bean(name = "job-execute-jooq-conf-b") + public org.jooq.Configuration jooqConfB( + @Qualifier("job-execute-conn-provider-b") ConnectionProvider connectionProvider, + DefaultExecuteListenerProvider jooqExecuteListenerProvider + ) { + return JooqConfigurationUtil.getConfiguration(connectionProvider, jooqExecuteListenerProvider); + } + + @Qualifier("job-execute-conn-provider-b") + @Bean(name = "job-execute-conn-provider-b") + public ConnectionProvider connectionProviderB( + @Qualifier("job-execute-transaction-aware-data-source-b") DataSource dataSource) { + return new DataSourceConnectionProvider(dataSource); + } + + @Qualifier("job-execute-transaction-aware-data-source-b") + @Bean(name = "job-execute-transaction-aware-data-source-b") + public TransactionAwareDataSourceProxy + transactionAwareDataSourceProxyB(@Qualifier("job-execute-data-source-b") DataSource dataSource) { + return new TransactionAwareDataSourceProxy(dataSource); + } + + // 配置垂直分片数据源-c + @Qualifier("job-execute-data-source-c") + @Bean(name = "job-execute-data-source-c") + @ConfigurationProperties(prefix = "spring.datasource.job-execute-vertical-c") + public DataSource dataSourceC() { + return DataSourceBuilder.create().build(); + } + + @Qualifier("job-execute-transaction-manager-c") + @Bean(name = "job-execute-transaction-manager-c") + @DependsOn("job-execute-data-source-c") + public DataSourceTransactionManager transactionManagerC( + @Qualifier("job-execute-data-source-c") DataSource dataSource) { + return new DataSourceTransactionManager(dataSource); + } + + @Qualifier("job-execute-jdbc-template-c") + @Bean(name = "job-execute-jdbc-template-c") + public JdbcTemplate jdbcTemplateC(@Qualifier("job-execute-data-source-c") DataSource dataSource) { + return new JdbcTemplate(dataSource); + } + + @Qualifier("job-execute-dsl-context-c") + @Bean(name = "job-execute-dsl-context-c") + public DSLContext dslContextC(@Qualifier("job-execute-jooq-conf-c") org.jooq.Configuration configuration) { + log.info("Init DSLContext job-execute-vertical-c"); + return new DefaultDSLContext(configuration); + } + + @Qualifier("job-execute-jooq-conf-c") + @Bean(name = "job-execute-jooq-conf-c") + public org.jooq.Configuration jooqConfC( + @Qualifier("job-execute-conn-provider-c") ConnectionProvider connectionProvider, + DefaultExecuteListenerProvider jooqExecuteListenerProvider + ) { + return JooqConfigurationUtil.getConfiguration(connectionProvider, jooqExecuteListenerProvider); + } + + @Qualifier("job-execute-conn-provider-c") + @Bean(name = "job-execute-conn-provider-c") + public ConnectionProvider connectionProviderC( + @Qualifier("job-execute-transaction-aware-data-source-c") DataSource dataSource) { + return new DataSourceConnectionProvider(dataSource); + } + + @Qualifier("job-execute-transaction-aware-data-source-c") + @Bean(name = "job-execute-transaction-aware-data-source-c") + public TransactionAwareDataSourceProxy + transactionAwareDataSourceProxyC(@Qualifier("job-execute-data-source-c") DataSource dataSource) { + return new TransactionAwareDataSourceProxy(dataSource); + } + + @Qualifier("job-execute-vertical-sharding-dsl-context-provider") + @Bean(name = "job-execute-vertical-sharding-dsl-context-provider") + @Primary + public VerticalShardingDSLContextProvider verticalShardingDSLContextProvider( + @Qualifier("job-execute-dsl-context-a") DSLContext dslContextA, + @Qualifier("job-execute-dsl-context-b") DSLContext dslContextB, + @Qualifier("job-execute-dsl-context-c") DSLContext dslContextC + ) { + log.info("Init JobExecuteVerticalShardingDSLContextProvider"); + return new JobExecuteVerticalShardingDSLContextProvider( + dslContextA, + dslContextB, + dslContextC + ); + } + } + + + /** + * Db 迁移配置 + */ + @ConditionalOnProperty(value = "mysql.migration.enabled", havingValue = "true", matchIfMissing = false) + protected static class DbMigratingConfiguration { + @Bean("job-execute-migrate-dynamic-dsl-context-provider") + public MigrateDynamicDSLContextProvider dynamicDSLContextProvider() { + return new MigrateDynamicDSLContextProvider(); + } + + @Bean("jobExecutePropBasedDynamicDataSource") + public PropBasedDynamicDataSource propBasedDynamicDataSource( + ObjectProvider standaloneDSLContextProvider, + ObjectProvider verticalShardingDSLContextProvider, + @Qualifier("jsonRedisTemplate") RedisTemplate redisTemplate, + PropToggleStore propToggleStore + ) { + return new PropBasedDynamicDataSource( + standaloneDSLContextProvider.getIfAvailable(), + verticalShardingDSLContextProvider.getIfAvailable(), + redisTemplate, + propToggleStore + ); + } + + + @Bean("readWriteLockDbMigrateAspect") + public ReadWriteLockDbMigrateAspect dbMigrateAspect( + @Qualifier("job-execute-migrate-dynamic-dsl-context-provider") + MigrateDynamicDSLContextProvider migrateDSLContextDynamicProvider, + PropBasedDynamicDataSource propBasedDynamicDataSource) { + log.info("Init ReadWriteLockDbMigrateAspect"); + return new ReadWriteLockDbMigrateAspect(migrateDSLContextDynamicProvider, propBasedDynamicDataSource); + } + } + + @Bean("jobExecuteDslContextProviderFactory") + public DSLContextProviderFactory dslContextProviderFactory( + ObjectProvider standaloneDSLContextProviderObjectProvider, + ObjectProvider verticalShardingDSLContextProviderObjectProvider, + ObjectProvider horizontalShardingDSLContextProviderObjectProvider, + ObjectProvider migrateDynamicDSLContextProviderObjectProvider, + MySQLProperties mySQLProperties + ) { + return new DSLContextProviderFactory( + standaloneDSLContextProviderObjectProvider.getIfAvailable(), + verticalShardingDSLContextProviderObjectProvider.getIfAvailable(), + horizontalShardingDSLContextProviderObjectProvider.getIfAvailable(), + migrateDynamicDSLContextProviderObjectProvider.getIfAvailable(), + mySQLProperties + ); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/DistributeSourceHostConfiguration.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/DistributeSourceHostConfiguration.java new file mode 100644 index 0000000000..b1c9b353bc --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/DistributeSourceHostConfiguration.java @@ -0,0 +1,116 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.config; + +import com.tencent.bk.job.common.gse.service.AgentStateClient; +import com.tencent.bk.job.execute.engine.prepare.third.FileWorkerHostService; +import com.tencent.bk.job.execute.service.AgentService; +import com.tencent.bk.job.execute.service.ExternalAgentService; +import com.tencent.bk.job.execute.service.HostService; +import com.tencent.bk.job.execute.service.LocalFileDistributeSourceHostProvisioner; +import com.tencent.bk.job.execute.service.ThirdFileDistributeSourceHostProvisioner; +import com.tencent.bk.job.execute.service.impl.ExternalAgentServiceImpl; +import com.tencent.bk.job.execute.service.impl.FileWorkerHostProvisioner; +import com.tencent.bk.job.execute.service.impl.LocalFileExternalAgentHostProvisioner; +import com.tencent.bk.job.execute.service.impl.LocalAgentHostProvisioner; +import com.tencent.bk.job.execute.service.impl.ThirdFileExternalAgentHostProvisioner; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import static com.tencent.bk.job.execute.config.GseConfig.EXECUTE_BEAN_AGENT_STATE_CLIENT; + +/** + * 通过开关控制使用job机器分发文件 还是使用 配置的集群外机器分发文件 + */ +@Configuration +@Slf4j +public class DistributeSourceHostConfiguration { + + /** + * 采用集群外机器作为分发源 + */ + @ConditionalOnExternalAgentEnabled + @Bean + ExternalAgentService externalAgentService(NFSExternalAgentHostConfig nfsExternalAgentHostConfig, + HostService hostService, + @Qualifier(EXECUTE_BEAN_AGENT_STATE_CLIENT) + AgentStateClient agentStateClient) { + return new ExternalAgentServiceImpl( + nfsExternalAgentHostConfig, + hostService, + agentStateClient + ); + } + + /** + * 文件源文件分发，采用集群外机器作为分发源 + * + */ + @ConditionalOnExternalAgentEnabled + @Bean + ThirdFileDistributeSourceHostProvisioner thirdFileExternalAgentHostProvisioner( + ExternalAgentService externalAgentService + ) { + log.info("init ThirdFileExternalAgentHostProvisioner"); + return new ThirdFileExternalAgentHostProvisioner(externalAgentService); + } + + /** + * 文件源文件分发，采用 job-file-worker 所在node进行分发 + * + */ + @ConditionalOnExternalAgentDisabled + @Bean + ThirdFileDistributeSourceHostProvisioner fileWorkerHostProvisioner(FileWorkerHostService fileWorkerHostService) { + log.info("init FileWorkerHostProvisioner"); + return new FileWorkerHostProvisioner(fileWorkerHostService); + } + + /** + * 本地文件分发，采用集群外的机器作为分发源 + * + */ + @ConditionalOnExternalAgentEnabled + @Bean + LocalFileDistributeSourceHostProvisioner localFileExternalAgentHostProvisioner( + ExternalAgentService externalAgentService + ) { + log.info("init LocalFileExternalAgentHostProvisioner"); + return new LocalFileExternalAgentHostProvisioner(externalAgentService); + } + + /** + * 本地文件分发，采用job-execute所在node进行分发 + * + */ + @ConditionalOnExternalAgentDisabled + @Bean + LocalFileDistributeSourceHostProvisioner localAgentHostProvisioner(AgentService agentService) { + log.info("init LocalAgentHostProvisioner"); + return new LocalAgentHostProvisioner(agentService); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/ExecutorConfiguration.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/ExecutorConfiguration.java index eeae58ee69..89d870cb7c 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/ExecutorConfiguration.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/ExecutorConfiguration.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,47 +25,79 @@ package com.tencent.bk.job.execute.config; import com.google.common.util.concurrent.ThreadFactoryBuilder; +import com.tencent.bk.job.common.WatchableThreadPoolExecutor; +import com.tencent.bk.job.execute.util.ContextExecutorService; +import io.micrometer.core.instrument.MeterRegistry; import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import java.util.concurrent.ExecutorService; import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.SynchronousQueue; import java.util.concurrent.ThreadFactory; -import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; @Slf4j -@Configuration +@Configuration(value = "jobExecuteExecutorConfig") public class ExecutorConfiguration { @Bean("logExportExecutor") - public ThreadPoolExecutor logExportExecutor() { + public ExecutorService logExportExecutor(MeterRegistry meterRegistry) { ThreadFactory threadFactory = new ThreadFactoryBuilder().setNameFormat("log-export-thread-%d").build(); - return new ThreadPoolExecutor( + return ContextExecutorService.wrap(new WatchableThreadPoolExecutor( + meterRegistry, + "logExportExecutor", 10, 100, 60, TimeUnit.SECONDS, new LinkedBlockingQueue<>(), threadFactory - ); + )); } @Bean("getHostsByTopoExecutor") - public ThreadPoolExecutor getHostsByTopoExecutor() { - return new ThreadPoolExecutor( + public ExecutorService getHostsByTopoExecutor(MeterRegistry meterRegistry) { + return ContextExecutorService.wrap(new WatchableThreadPoolExecutor( + meterRegistry, + "getHostsByTopoExecutor", 50, 100, 60, TimeUnit.SECONDS, new LinkedBlockingQueue<>() - ); + )); } - @Bean("localFilePrepareExecutor") - public ThreadPoolExecutor localFilePrepareExecutor(LocalFileConfigForExecute localFileConfigForExecute) { + @Bean("getHostTopoPathExecutor") + public ExecutorService getHostTopoPathExecutor(MeterRegistry meterRegistry) { + return ContextExecutorService.wrap(new WatchableThreadPoolExecutor( + meterRegistry, + "getHostTopoPathExecutor", + 5, + 50, + 60, + TimeUnit.SECONDS, + new LinkedBlockingQueue<>(5), + (r, executor) -> { + //使用请求的线程直接拉取数据 + log.error( + "getHostTopoPath runnable rejected," + + " use current thread({}), plz add more threads", + Thread.currentThread().getName()); + r.run(); + } + )); + } + + @Bean("localFileDownloadExecutor") + public ExecutorService localFileDownloadExecutor(LocalFileConfigForExecute localFileConfigForExecute, + MeterRegistry meterRegistry) { int concurrency = localFileConfigForExecute.getDownloadConcurrency(); - return new ThreadPoolExecutor( + return ContextExecutorService.wrap(new WatchableThreadPoolExecutor( + meterRegistry, + "localFileDownloadExecutor", concurrency, concurrency, 180L, @@ -79,17 +111,40 @@ public ThreadPoolExecutor localFilePrepareExecutor(LocalFileConfigForExecute loc Thread.currentThread().getName()); r.run(); } - ); + )); + } + + @Bean("localFileWatchExecutor") + public ExecutorService localFileWatchExecutor(MeterRegistry meterRegistry) { + return ContextExecutorService.wrap(new WatchableThreadPoolExecutor( + meterRegistry, + "localFileWatchExecutor", + 0, + 50, + 60L, + TimeUnit.SECONDS, + new SynchronousQueue<>(), + (r, executor) -> { + //使用请求的线程直接拉取数据 + log.error( + "watch localupload file from artifactory runnable rejected," + + " use current thread({}), plz add more job-execute instances", + Thread.currentThread().getName()); + r.run(); + } + )); } @Bean("shutdownExecutor") - public ThreadPoolExecutor shutdownExecutor() { - return new ThreadPoolExecutor( + public ExecutorService shutdownExecutor(MeterRegistry meterRegistry) { + return ContextExecutorService.wrap(new WatchableThreadPoolExecutor( + meterRegistry, + "shutdownExecutor", 10, 20, 120, TimeUnit.SECONDS, new LinkedBlockingQueue<>() - ); + )); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/FileDistributeConfig.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/FileDistributeConfig.java new file mode 100644 index 0000000000..e4c27cfeec --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/FileDistributeConfig.java @@ -0,0 +1,42 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.config; + +import lombok.Getter; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Configuration; + +/** + * 文件分发配置 + */ +@Getter +@Configuration("jobExecuteFileDistributeConfig") +public class FileDistributeConfig { + /** + * 要分发的文件存储根目录，二进制环境下与临时文件存储根目录相同 + */ + @Value("${job.execute.file.distribute.root-path:/data/bkee/job/data}") + private String jobDistributeRootPath; +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/GseClientAutoConfig.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/GseClientAutoConfig.java deleted file mode 100644 index dcda01c6f2..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/GseClientAutoConfig.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.config; - -import com.tencent.bk.job.common.gse.service.QueryAgentStatusClient; -import com.tencent.bk.job.common.gse.service.QueryAgentStatusClientImpl; -import io.micrometer.core.instrument.MeterRegistry; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Configuration -public class GseClientAutoConfig { - - @Bean - @Autowired - public QueryAgentStatusClient gseCacheClient(GseConfig gseConfigForExecute, - MeterRegistry meterRegistry) { - com.tencent.bk.job.common.gse.config.GseConfig gseConfig = new com.tencent.bk.job.common.gse.config.GseConfig(); - gseConfig.setEnableSsl(gseConfigForExecute.isGseSSLEnable()); - String[] cacheApiServerHosts = gseConfigForExecute.getGseCacheApiServerHost().split(","); - gseConfig.setGseCacheApiServerHost(cacheApiServerHosts); - gseConfig.setGseCacheApiServerPort(gseConfigForExecute.getGseCacheApiServerPort()); - gseConfig.setKeyStore(gseConfigForExecute.getGseSSLKeystore()); - gseConfig.setKeyStorePass(gseConfigForExecute.getGseSSLKeystorePassword()); - gseConfig.setTrustStore(gseConfigForExecute.getGseSSLTruststore()); - gseConfig.setTrustStorePass(gseConfigForExecute.getGseSSLTruststorePassword()); - gseConfig.setTrustStoreType(gseConfigForExecute.getGseSSLTruststoreStoreType()); - gseConfig.setTrustManagerType(gseConfigForExecute.getGseSSLTruststoreManagerType()); - gseConfig.setQueryBatchSize(gseConfigForExecute.getGseQueryBatchSize()); - gseConfig.setQueryThreadsNum(gseConfigForExecute.getGseQueryThreadsNum()); - return new QueryAgentStatusClientImpl(gseConfig, meterRegistry); - } -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/GseConfig.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/GseConfig.java index be2adec2b9..f25d1c29b9 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/GseConfig.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/GseConfig.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,56 +24,79 @@ package com.tencent.bk.job.execute.config; -import lombok.Getter; -import lombok.Setter; -import lombok.ToString; -import org.springframework.beans.factory.annotation.Value; +import com.tencent.bk.job.common.gse.config.AgentStateQueryConfig; +import com.tencent.bk.job.common.gse.constants.DefaultBeanNames; +import com.tencent.bk.job.common.gse.service.AgentStateClient; +import com.tencent.bk.job.common.gse.service.AutoChoosingAgentStateClientImpl; +import com.tencent.bk.job.common.gse.service.BizHostInfoQueryService; +import com.tencent.bk.job.common.gse.service.GseV1AgentStateClientImpl; +import com.tencent.bk.job.common.gse.service.GseV2AgentStateClientImpl; +import com.tencent.bk.job.common.gse.service.UseV2ByFeatureAgentStateClientImpl; +import com.tencent.bk.job.common.gse.v1.GseV1ApiClient; +import com.tencent.bk.job.common.gse.v2.GseV2ApiClient; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; -/** - * GSE配置 - */ -@Configuration("gseConfig") -@Getter -@Setter -@ToString(exclude = {"gseSSLKeystorePassword", "gseSSLTruststorePassword"}) -public class GseConfig { - @Value("${gse.taskserver.host:}") - private String taskServerHost; - - @Value("${gse.taskserver.port:48673}") - private int taskServerPort; - - @Value("${gse.cache.apiserver.host:}") - private String gseCacheApiServerHost; - - @Value("${gse.cache.apiserver.port:59313}") - private int gseCacheApiServerPort; +import java.util.concurrent.ThreadPoolExecutor; - @Value("${gse.ssl.enabled:true}") - private boolean gseSSLEnable; - - @Value("${gse.ssl.keystore.path:}") - private String gseSSLKeystore; - - @Value("${gse.ssl.keystore.password:}") - private String gseSSLKeystorePassword; - - @Value("${gse.ssl.truststore.path:}") - private String gseSSLTruststore; +@Configuration(value = "jobExecuteGseConfig") +public class GseConfig { - @Value("${gse.ssl.truststore.password:}") - private String gseSSLTruststorePassword; + public static final String EXECUTE_BEAN_PREFIX = "jobExecute"; + public static final String EXECUTE_BEAN_GSE_V1_AGENT_STATE_CLIENT = EXECUTE_BEAN_PREFIX + "GseV1AgentStateClient"; + public static final String EXECUTE_BEAN_GSE_V2_AGENT_STATE_CLIENT = EXECUTE_BEAN_PREFIX + "GseV2AgentStateClient"; + public static final String EXECUTE_BEAN_AGENT_STATE_CLIENT = EXECUTE_BEAN_PREFIX + "AgentStateClient"; + public static final String EXECUTE_BEAN_USE_V2_BY_FEATURE_AGENT_STATE_CLIENT = + EXECUTE_BEAN_PREFIX + DefaultBeanNames.USE_V2_BY_FEATURE_AGENT_STATE_CLIENT; - @Value("${gse.ssl.truststore.manager-type:SunX509}") - private String gseSSLTruststoreManagerType; + @Bean(EXECUTE_BEAN_GSE_V1_AGENT_STATE_CLIENT) + public GseV1AgentStateClientImpl gseV1AgentStateClient(AgentStateQueryConfig agentStateQueryConfig, + ObjectProvider gseV1ApiClient, + @Qualifier(DefaultBeanNames.AGENT_STATUS_QUERY_THREAD_POOL_EXECUTOR) + ThreadPoolExecutor threadPoolExecutor) { + return new GseV1AgentStateClientImpl( + agentStateQueryConfig, + gseV1ApiClient.getIfAvailable(), + threadPoolExecutor + ); + } - @Value("${gse.ssl.truststore.store-type:JKS}") - private String gseSSLTruststoreStoreType; + @Bean(EXECUTE_BEAN_GSE_V2_AGENT_STATE_CLIENT) + public GseV2AgentStateClientImpl gseV2AgentStateClient(AgentStateQueryConfig agentStateQueryConfig, + ObjectProvider gseV2ApiClient, + @Qualifier(DefaultBeanNames.AGENT_STATUS_QUERY_THREAD_POOL_EXECUTOR) + ThreadPoolExecutor threadPoolExecutor) { + return new GseV2AgentStateClientImpl( + agentStateQueryConfig, + gseV2ApiClient.getIfAvailable(), + threadPoolExecutor + ); + } - @Value("${gse.query.threads.num:5}") - private int gseQueryThreadsNum; + @Bean(EXECUTE_BEAN_USE_V2_BY_FEATURE_AGENT_STATE_CLIENT) + public AgentStateClient useV2ByFeatureAgentStateClient(@Qualifier(EXECUTE_BEAN_GSE_V1_AGENT_STATE_CLIENT) + GseV1AgentStateClientImpl gseV1AgentStateClient, + @Qualifier(EXECUTE_BEAN_GSE_V2_AGENT_STATE_CLIENT) + GseV2AgentStateClientImpl gseV2AgentStateClient, + @Qualifier("jobExecuteBizHostInfoQueryService") + BizHostInfoQueryService bizHostInfoQueryService) { + return new UseV2ByFeatureAgentStateClientImpl( + gseV1AgentStateClient, + gseV2AgentStateClient, + bizHostInfoQueryService + ); + } - @Value("${gse.query.batchSize:5000}") - private int gseQueryBatchSize; + @Primary + @Bean(EXECUTE_BEAN_AGENT_STATE_CLIENT) + public AgentStateClient AutoChoosingAgentStateClientImpl( + @Qualifier(DefaultBeanNames.PREFER_V2_AGENT_STATE_CLIENT) AgentStateClient preferV2AgentStateClient, + @Qualifier(EXECUTE_BEAN_USE_V2_BY_FEATURE_AGENT_STATE_CLIENT) + AgentStateClient useV2ByFeatureAgentStateClient + ) { + return new AutoChoosingAgentStateClientImpl(preferV2AgentStateClient, useV2ByFeatureAgentStateClient); + } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/GseServerAutoConfig.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/GseServerAutoConfig.java deleted file mode 100644 index 3fb19dbc78..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/GseServerAutoConfig.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.config; - -import com.tencent.bk.job.execute.engine.gse.GseServer; -import com.tencent.bk.job.execute.engine.gse.GseZkServer; -import com.tencent.bk.job.execute.engine.gse.IntervalIncrementForeverRetry; -import org.apache.curator.framework.CuratorFramework; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -/** - * @since 22/12/2020 21:41 - */ -@Configuration -public class GseServerAutoConfig { - - private static final String ZOOKEEPER_SERVER_TYPE = "zookeeper"; - - /** - * GSE ZooKeeper 连接信息 - */ - @Value("${gse.server.zookeeper.connect.string:}") - private String gseServerZkConnectString; - - /** - * 储存 GSE 接入服务器信息的 ZooKeeper 节点 - */ - @Value("${gse.server.zookeeper.path:/gse/config/server/task/accessv3}") - private String gseServerZkPath; - - /** - * ZooKeeper 会话超时时间 - */ - @Value("${gse.server.zookeeper.timeout.session:60000}") - private Integer gseServerZkSessionTimeOutMills; - - /** - * ZooKeeper 连接超时时间 - */ - @Value("${gse.server.zookeeper.timeout.connect:60000}") - private Integer gseServerZkConnectTimeOutMills; - - /** - * 不存在 gseServer，回落到 domain/ip list 方式 - */ - @Bean("gseServer") - @ConditionalOnMissingBean(name = "gseServer") - public GseServer gseServer() { - return new GseServer(); - } - - /** - * 启用 ZooKeeper 方式，初始化 GseZkServer - * - * @param curatorFramework ZooKeeper 连接组件 - */ - @Bean("gseServer") - @ConditionalOnProperty(name = "gse.server.discovery.type", havingValue = ZOOKEEPER_SERVER_TYPE) - public GseServer gseServer(CuratorFramework curatorFramework) { - return new GseZkServer(curatorFramework, gseServerZkPath); - } - - /** - * 启用 ZooKeeper 方式，初始化 ZooKeeper 连接组件 - */ - @Bean(initMethod = "start", destroyMethod = "stop") - @ConditionalOnProperty(name = "gse.server.discovery.type", havingValue = ZOOKEEPER_SERVER_TYPE) - public CuratorFrameworkFactoryBean curatorFrameworkFactoryBean() { - return new CuratorFrameworkFactoryBean(gseServerZkConnectString, gseServerZkSessionTimeOutMills, - gseServerZkConnectTimeOutMills, new IntervalIncrementForeverRetry(60000)); - } -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/IamConfiguration.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/IamConfiguration.java new file mode 100644 index 0000000000..a2c144ce54 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/IamConfiguration.java @@ -0,0 +1,93 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.config; + +import com.tencent.bk.job.execute.auth.impl.CachedTopoPathServiceImpl; +import com.tencent.bk.job.execute.auth.impl.CompositeTopoPathService; +import com.tencent.bk.job.execute.auth.impl.HostTopoPathCache; +import com.tencent.bk.job.execute.auth.impl.SwitchableTopoPathService; +import com.tencent.bk.sdk.iam.service.TopoPathService; +import io.micrometer.core.instrument.MeterRegistry; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; +import org.springframework.data.redis.core.RedisTemplate; + +@Slf4j +@Configuration(value = "jobExecuteIamConfiguration") +@EnableConfigurationProperties(IamHostTopoPathProperties.class) +public class IamConfiguration { + + @Bean + public TopoPathService topoPathService(@Qualifier("cmdbTopoPathService") + TopoPathService cmdbTopoPathService, + @Qualifier("localDBTopoPathService") + TopoPathService localDBTopoPathService) { + CompositeTopoPathService compositeTopoPathService = new CompositeTopoPathService(); + compositeTopoPathService.addTopoPathService(cmdbTopoPathService); + compositeTopoPathService.addTopoPathService(localDBTopoPathService); + return compositeTopoPathService; + } + + @ConditionalOnIamHostTopoPathCacheEnabled + @Configuration + public static class HostTopoPathCacheConfiguration { + + @Bean + public HostTopoPathCache hostTopoPathCache(@Qualifier("jsonRedisTemplate") + RedisTemplate redisTemplate, + IamHostTopoPathProperties iamHostTopoPathProperties, + MeterRegistry meterRegistry) { + return new HostTopoPathCache(redisTemplate, iamHostTopoPathProperties, meterRegistry); + } + + @Bean + public TopoPathService cachedTopoPathService(@Qualifier("topoPathService") + TopoPathService topoPathService, + HostTopoPathCache hostTopoPathCache) { + log.info("cachedTopoPathService init"); + return new CachedTopoPathServiceImpl(topoPathService, hostTopoPathCache); + } + } + + @Bean + @Primary + public SwitchableTopoPathService switchableTopoPathService(@Qualifier("topoPathService") + TopoPathService topoPathService, + @Autowired(required = false) + @Qualifier("cachedTopoPathService") + TopoPathService cachedTopoPathService, + IamHostTopoPathProperties iamHostTopoPathProperties) { + if (cachedTopoPathService != null) { + return new SwitchableTopoPathService(cachedTopoPathService, iamHostTopoPathProperties); + } + return new SwitchableTopoPathService(topoPathService, iamHostTopoPathProperties); + } + +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/IamHostTopoPathProperties.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/IamHostTopoPathProperties.java new file mode 100644 index 0000000000..5f7789e94c --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/IamHostTopoPathProperties.java @@ -0,0 +1,59 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.config; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.springframework.boot.context.properties.ConfigurationProperties; + +@Data +@ConfigurationProperties(prefix = "job.execute.iam.host-topo-path") +@NoArgsConstructor +public class IamHostTopoPathProperties { + + /** + * 是否开启主机拓扑路径鉴权服务，默认开启 + */ + private Boolean enabled = true; + /** + * 主机拓扑路径缓存相关配置 + */ + private CacheConfig cache; + + @Data + @AllArgsConstructor + @NoArgsConstructor + public static class CacheConfig { + /** + * 是否开启缓存 + */ + private Boolean enabled = true; + /** + * 缓存过期时间默认为10s + */ + private Integer expireSeconds = 10; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/IdGenConfiguration.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/IdGenConfiguration.java new file mode 100644 index 0000000000..33b5d6e75e --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/IdGenConfiguration.java @@ -0,0 +1,106 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.config; + +import com.tencent.bk.job.common.mysql.dynamic.id.IdGenType; +import com.tencent.bk.job.common.util.toggle.prop.PropToggleStore; +import com.tencent.bk.job.execute.dao.common.AutoIncrementIdGen; +import com.tencent.bk.job.execute.dao.common.PropBasedDynamicIdGen; +import com.tencent.bk.job.execute.dao.common.SegmentIdGen; +import com.tencent.devops.leaf.service.SegmentService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.condition.AllNestedConditions; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Conditional; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.redis.core.RedisTemplate; + +@Configuration(value = "jobExecuteIdGenConfiguration") +@Slf4j +public class IdGenConfiguration { + + @Conditional(AutoIncrementIdGenCondition.class) + @Bean + public AutoIncrementIdGen autoIncrementIdGen() { + log.info("Init AutoIncrementIdGen"); + return new AutoIncrementIdGen(); + } + + static class AutoIncrementIdGenCondition extends AllNestedConditions { + public AutoIncrementIdGenCondition() { + super(ConfigurationPhase.REGISTER_BEAN); + } + + @ConditionalOnProperty(value = "idGen.type", havingValue = IdGenType.Constants.AUTO_INCREMENT) + static class IdGenTypeCondition { + + } + + @ConditionalOnProperty(value = "idGen.migration.enabled", havingValue = "false", matchIfMissing = true) + static class MigrationDisable { + + } + } + + @Conditional(SegmentIdGenCondition.class) + @Bean + public SegmentIdGen segmentIdGen(SegmentService segmentService) { + log.info("Init SegmentIdGen"); + return new SegmentIdGen(segmentService); + } + + static class SegmentIdGenCondition extends AllNestedConditions { + public SegmentIdGenCondition() { + super(ConfigurationPhase.REGISTER_BEAN); + } + + @ConditionalOnProperty(value = "idGen.type", havingValue = IdGenType.Constants.LEAF_SEGMENT) + static class IdGenTypeCondition { + + } + + @ConditionalOnProperty(value = "idGen.migration.enabled", havingValue = "false", matchIfMissing = true) + static class MigrationDisable { + + } + } + + @ConditionalOnProperty(value = "idGen.migration.enabled", havingValue = "true") + @Bean + public PropBasedDynamicIdGen propBasedDynamicIdGen( + SegmentService segmentService, + @Qualifier("jsonRedisTemplate") RedisTemplate redisTemplate, + PropToggleStore propToggleStore + ) { + log.info("Init PropBasedDynamicIdGen"); + AutoIncrementIdGen autoIncrementIdGen = new AutoIncrementIdGen(); + SegmentIdGen segmentIdGen = new SegmentIdGen(segmentService); + return new PropBasedDynamicIdGen(autoIncrementIdGen, segmentIdGen, redisTemplate, propToggleStore); + } + + +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/JobExecuteAutoConfiguration.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/JobExecuteAutoConfiguration.java deleted file mode 100644 index f1be73e260..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/JobExecuteAutoConfiguration.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.config; - -import com.tencent.bk.job.common.artifactory.sdk.ArtifactoryClient; -import com.tencent.bk.job.common.esb.metrics.EsbApiTimedAspect; -import com.tencent.bk.job.common.service.AppScopeMappingService; -import com.tencent.bk.job.execute.client.ApplicationResourceClient; -import com.tencent.bk.job.manage.AppScopeMappingServiceImpl; -import io.micrometer.core.instrument.MeterRegistry; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Configuration -public class JobExecuteAutoConfiguration { - @Bean - public EsbApiTimedAspect esbApiTimedAspect(@Autowired MeterRegistry meterRegistry) { - return new EsbApiTimedAspect(meterRegistry); - } - - @Bean - public ArtifactoryClient artifactoryClient(@Autowired ArtifactoryConfig artifactoryConfig, - @Autowired MeterRegistry meterRegistry) { - return new ArtifactoryClient( - artifactoryConfig.getArtifactoryBaseUrl(), - artifactoryConfig.getArtifactoryJobUsername(), - artifactoryConfig.getArtifactoryJobPassword(), - meterRegistry - ); - } - - static class CmdbConfigSetter { - CmdbConfigSetter() { - } - } - - @Bean - AppScopeMappingService appScopeMappingService(ApplicationResourceClient applicationResource) { - return new AppScopeMappingServiceImpl(applicationResource); - } -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/JobExecuteConfig.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/JobExecuteConfig.java index 77a5dfe2c8..94823a7bac 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/JobExecuteConfig.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/JobExecuteConfig.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -32,9 +32,6 @@ @Data public class JobExecuteConfig { - @Value("${swagger.url:swagger.job.com}") - private String swaggerUrl; - /** * 功能开关 - 启用账号鉴权 */ @@ -50,12 +47,6 @@ public class JobExecuteConfig { @Value("${job.execute.result.handle.tasks.limit: 2000}") private int resultHandleTasksLimit; - /** - * 作业平台web访问地址 - */ - @Value("${job.web.url:}") - private String jobWebUrl; - /** * Symmetric encryption password */ @@ -70,4 +61,11 @@ public class JobExecuteConfig { @Value("${gse.script.rootPath:/tmp/bkjob}") private String gseScriptFileRootPath; + + /** + * GSE 脚本任务执行结果查询 API 单次返回的执行输出内容长度 + * 默认值：512M + */ + @Value("${job.execute.scriptTask.query.contentSizeLimit:512MB}") + private String scriptTaskQueryContentSizeLimit; } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/JobFunctionConfiguration.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/JobFunctionConfiguration.java new file mode 100644 index 0000000000..158a0456a0 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/JobFunctionConfiguration.java @@ -0,0 +1,93 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.config; + +import com.tencent.bk.job.execute.engine.listener.CallbackListener; +import com.tencent.bk.job.execute.engine.listener.GseTaskListener; +import com.tencent.bk.job.execute.engine.listener.JobListener; +import com.tencent.bk.job.execute.engine.listener.NotifyMsgListener; +import com.tencent.bk.job.execute.engine.listener.ResultHandleResumeListener; +import com.tencent.bk.job.execute.engine.listener.StepListener; +import com.tencent.bk.job.execute.engine.listener.event.GseTaskEvent; +import com.tencent.bk.job.execute.engine.listener.event.JobEvent; +import com.tencent.bk.job.execute.engine.listener.event.ResultHandleTaskResumeEvent; +import com.tencent.bk.job.execute.engine.listener.event.StepEvent; +import com.tencent.bk.job.execute.engine.model.JobCallbackDTO; +import com.tencent.bk.job.execute.model.TaskNotifyDTO; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.messaging.Message; + +import java.util.function.Consumer; + +/** + * spring cloud function 定义 + *

+ * 注意：方法名与配置文件中的spring.cloud.function.definition对应，修改需要注意！！！ + */ +@Configuration +@Slf4j +public class JobFunctionConfiguration { + @Bean + public Consumer> handleJobEvent(@Autowired JobListener jobListener) { + log.info("Init handleJobEvent consumer"); + + return jobListener::onEvent; + } + + @Bean + public Consumer> handleStepEvent(@Autowired StepListener stepListener) { + log.info("Init handleStepEvent consumer"); + return stepListener::onEvent; + } + + @Bean + public Consumer> handleGseTaskEvent(@Autowired GseTaskListener gseTaskListener) { + log.info("Init handleGseTaskEvent consumer"); + return gseTaskListener::onEvent; + } + + @Bean + public Consumer> handleResultHandleResumeEvent( + @Autowired ResultHandleResumeListener resultHandleResumeListener) { + log.info("Init handleResultHandleResumeEvent consumer"); + return resultHandleResumeListener::onEvent; + } + + @Bean + public Consumer> handleNotifyMsg(@Autowired NotifyMsgListener notifyMsgListener) { + log.info("Init handleNotifyMsg consumer"); + return notifyMsgListener::onEvent; + } + + @Bean + public Consumer> handleCallbackMsg(@Autowired CallbackListener callbackListener) { + log.info("Init handleCallbackMsg consumer"); + return callbackListener::onEvent; + } + +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/LocalFileConfigForExecute.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/LocalFileConfigForExecute.java index e379cdc982..e0de322b28 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/LocalFileConfigForExecute.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/LocalFileConfigForExecute.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -40,4 +40,7 @@ public class LocalFileConfigForExecute { @Value("${local-file.artifactory.download.concurrency:10}") private Integer downloadConcurrency; + + @Value("${local-file.expire-days:7}") + private Integer expireDays; } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/LogExportConfig.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/LogExportConfig.java index 299444ce48..44fb7bfc7e 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/LogExportConfig.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/LogExportConfig.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -32,13 +32,13 @@ @Configuration public class LogExportConfig { - @Value("${log-export.storage-backend:local}") + @Value("${job.execute.log-export.storage-backend:local}") private String storageBackend; - @Value("${log-export.artifactory.repo:filedata}") + @Value("${job.execute.log-export.artifactory.repo:filedata}") private String logExportRepo; - @Value("${log-export.artifactory.file-expire-days:7}") + @Value("${job.execute.log-export.artifactory.file-expire-days:7}") private Integer artifactoryFileExpireDays; } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/MetricsAutoConfig.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/MetricsAutoConfig.java deleted file mode 100644 index a68d2335e9..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/MetricsAutoConfig.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.config; - -import com.tencent.bk.job.common.gse.constants.GseConstants; -import com.tencent.bk.job.common.metrics.CommonMetricNames; -import com.tencent.bk.job.execute.monitor.ExecuteMetricNames; -import com.tencent.bk.job.execute.monitor.ExecuteMetricTags; -import io.micrometer.core.instrument.Meter; -import io.micrometer.core.instrument.config.MeterFilter; -import io.micrometer.core.instrument.config.MeterFilterReply; -import io.micrometer.core.instrument.distribution.DistributionStatisticConfig; -import org.apache.commons.lang3.StringUtils; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Configuration -public class MetricsAutoConfig { - @Bean - public MeterFilter distributionMeterFilter() { - return new MeterFilter() { - @Override - public DistributionStatisticConfig configure(Meter.Id id, DistributionStatisticConfig config) { - String metricName = id.getName(); - if (metricName.startsWith("http.server.request") || metricName.startsWith(CommonMetricNames.ESB_API)) { - return DistributionStatisticConfig.builder().percentilesHistogram(true) - // [10ms,3s] - .minimumExpectedValue(10_000_000.0).maximumExpectedValue(3_000_000_000.0) - .build().merge(config); - } else if (metricName.startsWith(GseConstants.GSE_API_METRICS_NAME_PREFIX)) { - return DistributionStatisticConfig.builder().percentilesHistogram(true) - // [10ms,1s] - .minimumExpectedValue(10_000_000.0).maximumExpectedValue(1_000_000_000.0) - .build().merge(config); - } else if (metricName.startsWith(ExecuteMetricNames.RESULT_HANDLE_TASK_SCHEDULE_PREFIX)) { - return DistributionStatisticConfig.builder().percentilesHistogram(true) - // [10ms,5s] - .minimumExpectedValue(10_000_000.0).maximumExpectedValue(5_000_000_000.0) - .build().merge(config); - } else { - return config; - } - } - }; - } - - @Bean - public MeterFilter denyIgnoreHttpRequestMeterFilter() { - return new MeterFilter() { - @Override - public MeterFilterReply accept(Meter.Id id) { - String metricName = id.getName(); - if (metricName.startsWith("http.server.request")) { - String ignore = id.getTag(ExecuteMetricTags.IGNORE_TAG); - if (StringUtils.isNotEmpty(ignore) && ExecuteMetricTags.BOOLEAN_TRUE_TAG_VALUE.equals(ignore)) { - return MeterFilterReply.DENY; - } else { - return MeterFilterReply.NEUTRAL; - } - } else { - return MeterFilterReply.NEUTRAL; - } - } - }; - } -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/MetricsConfiguration.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/MetricsConfiguration.java new file mode 100644 index 0000000000..ccf90e6ea9 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/MetricsConfiguration.java @@ -0,0 +1,85 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.config; + +import com.tencent.bk.job.common.metrics.CommonMetricNames; +import com.tencent.bk.job.execute.monitor.ExecuteMetricNames; +import com.tencent.bk.job.execute.monitor.ExecuteMetricTags; +import io.micrometer.core.instrument.Meter; +import io.micrometer.core.instrument.config.MeterFilter; +import io.micrometer.core.instrument.config.MeterFilterReply; +import io.micrometer.core.instrument.distribution.DistributionStatisticConfig; +import org.apache.commons.lang3.StringUtils; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * 监控指标配置 + */ +@Configuration +public class MetricsConfiguration { + @Bean + public MeterFilter distributionMeterFilter() { + return new MeterFilter() { + @Override + public DistributionStatisticConfig configure(Meter.Id id, DistributionStatisticConfig config) { + String metricName = id.getName(); + if (metricName.startsWith("http.server.request") || metricName.startsWith(CommonMetricNames.ESB_API)) { + return DistributionStatisticConfig.builder().percentilesHistogram(true) + // [10ms,3s] + .minimumExpectedValue(10_000_000.0).maximumExpectedValue(3_000_000_000.0) + .build().merge(config); + } else if (metricName.startsWith(ExecuteMetricNames.RESULT_HANDLE_TASK_SCHEDULE_PREFIX)) { + return DistributionStatisticConfig.builder().percentilesHistogram(true) + // [10ms,5s] + .minimumExpectedValue(10_000_000.0).maximumExpectedValue(5_000_000_000.0) + .build().merge(config); + } else { + return config; + } + } + }; + } + + @Bean + public MeterFilter denyIgnoreHttpRequestMeterFilter() { + return new MeterFilter() { + @Override + public MeterFilterReply accept(Meter.Id id) { + String metricName = id.getName(); + if (metricName.startsWith("http.server.request")) { + String ignore = id.getTag(ExecuteMetricTags.IGNORE_TAG); + if (StringUtils.isNotEmpty(ignore) && ExecuteMetricTags.BOOLEAN_TRUE_TAG_VALUE.equals(ignore)) { + return MeterFilterReply.DENY; + } else { + return MeterFilterReply.NEUTRAL; + } + } else { + return MeterFilterReply.NEUTRAL; + } + } + }; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/NFSExternalAgentHostConfig.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/NFSExternalAgentHostConfig.java new file mode 100644 index 0000000000..3c371bf94a --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/NFSExternalAgentHostConfig.java @@ -0,0 +1,49 @@ +/* + * + * * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * * + * * Copyright (C) 2021 Tencent. All rights reserved. + * * + * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * * + * * License for BK-JOB蓝鲸智云作业平台: + * * -------------------------------------------------------------------- + * * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * * + * * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * * the Software. + * * + * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * * IN THE SOFTWARE. + * + */ + +package com.tencent.bk.job.execute.config; + +import com.tencent.bk.job.execute.model.ExternalHostDTO; +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +import java.util.List; + +/** + * 集群外部共享了文件分发路径的NFS客户端配置 + */ +@Configuration +@ConfigurationProperties(prefix = "external-gse-agent") +@Getter +@Setter +public class NFSExternalAgentHostConfig { + + private Boolean enabled; + + private List hosts; +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/ScheduleAutoConfig.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/ScheduleAutoConfig.java index b4ec7e115e..121a05772b 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/ScheduleAutoConfig.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/ScheduleAutoConfig.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/StatisticConfig.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/StatisticConfig.java index fadbcae3ba..c0b7cebc58 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/StatisticConfig.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/StatisticConfig.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,7 +28,7 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Configuration; -@Configuration +@Configuration("jobExecuteStatisticConfig") @Data public class StatisticConfig { diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/StorageConfig.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/StorageConfig.java new file mode 100644 index 0000000000..e208fb6fd5 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/StorageConfig.java @@ -0,0 +1,42 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.config; + +import lombok.Getter; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Configuration; + +/** + * 存储与分发配置 + */ +@Getter +@Configuration("jobExecuteStorageConfig") +public class StorageConfig { + /** + * 临时文件存储根目录 + */ + @Value("${job.storage.root-path:/data/bkee/job/data}") + private String jobStorageRootPath; +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/StorageSystemConfig.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/StorageSystemConfig.java deleted file mode 100644 index 045b32d468..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/config/StorageSystemConfig.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.config; - -import lombok.Getter; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Configuration; - -/** - * 存储配置 - */ -@Getter -@Configuration -public class StorageSystemConfig { - @Value("${job.storage.root-path:/data/bkee/job/data}") - private String jobStorageRootPath; -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/constants/Consts.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/constants/Consts.java index 4defab4699..a0f45cf710 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/constants/Consts.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/constants/Consts.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/constants/LogExportStatusEnum.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/constants/LogExportStatusEnum.java index 880dbecea3..4dbda4cd2c 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/constants/LogExportStatusEnum.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/constants/LogExportStatusEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/constants/RedisKeys.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/constants/RedisKeys.java new file mode 100644 index 0000000000..167cfa42bd --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/constants/RedisKeys.java @@ -0,0 +1,36 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.constants; + +/** + * redis key 定义 + */ +public interface RedisKeys { + /** + * 正在执行中的作业-ZSET + */ + String RUNNING_JOB_ZSET_KEY = "job:execute:running:job"; + +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/constants/ScriptSourceEnum.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/constants/ScriptSourceEnum.java index 9698a32503..5c71d2af17 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/constants/ScriptSourceEnum.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/constants/ScriptSourceEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/constants/StepOperationEnum.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/constants/StepOperationEnum.java index aa1c4db510..ab630d30a4 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/constants/StepOperationEnum.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/constants/StepOperationEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/constants/TaskOperationEnum.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/constants/TaskOperationEnum.java index 7f9ba88d32..aef9d8f732 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/constants/TaskOperationEnum.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/constants/TaskOperationEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,7 +28,8 @@ * 作业执行操作 */ public enum TaskOperationEnum { - TERMINATE_JOB(1, "终止任务"); + TERMINATE_JOB(1, "终止任务"), + START_JOB(2, "启动任务"); private final Integer value; private final String name; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/constants/TopoNodeTypeEnum.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/constants/TopoNodeTypeEnum.java deleted file mode 100644 index 13d999e77c..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/constants/TopoNodeTypeEnum.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.constants; - -/** - * 作业类型 - */ -public enum TopoNodeTypeEnum { - /** - * 业务 - */ - APP("biz"), - /** - * 集群 - */ - SET("set"), - /** - * 模块 - */ - MODULE("module"); - - - private final String nodeType; - - TopoNodeTypeEnum(String nodeType) { - this.nodeType = nodeType; - } - - public static TopoNodeTypeEnum getTopoNodeType(String nodeType) { - for (TopoNodeTypeEnum nodeTypeEnum : values()) { - if (nodeTypeEnum.getValue().equals(nodeType)) { - return nodeTypeEnum; - } - } - return null; - } - - public String getValue() { - return nodeType; - } -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/constants/UserOperationEnum.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/constants/UserOperationEnum.java index 3fa6e4079e..83a38689a2 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/constants/UserOperationEnum.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/constants/UserOperationEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -37,7 +37,8 @@ public enum UserOperationEnum { CONFIRM_CONTINUE(7, "人工确认-继续"), CONFIRM_TERMINATE(8, "人工确认-终止"), NEXT_STEP(9, "进入下一步"), - CONFIRM_RESTART(10, "重新发起确认"); + CONFIRM_RESTART(10, "重新发起确认"), + CREATE_JOB(11, "创建作业"); private final Integer value; private final String name; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/constants/VariableValueTypeEnum.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/constants/VariableValueTypeEnum.java index d1bfada677..1bf0025195 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/constants/VariableValueTypeEnum.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/constants/VariableValueTypeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/DangerousRecordDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/DangerousRecordDAO.java index 2c3a05da45..2fe58bc358 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/DangerousRecordDAO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/DangerousRecordDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileAgentTaskDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileAgentTaskDAO.java index c788a5fe46..3edc623b7b 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileAgentTaskDAO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileAgentTaskDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,9 +24,11 @@ package com.tencent.bk.job.execute.dao; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.CompatibleType; import com.tencent.bk.job.common.constant.Order; -import com.tencent.bk.job.execute.model.AgentTaskDTO; -import com.tencent.bk.job.execute.model.AgentTaskResultGroupBaseDTO; +import com.tencent.bk.job.execute.model.ExecuteObjectTask; +import com.tencent.bk.job.execute.model.ResultGroupBaseDTO; import com.tencent.bk.job.logsvr.consts.FileTaskModeEnum; import java.util.Collection; @@ -35,57 +37,67 @@ /** * FileAgentTaskDAO */ +@Deprecated +@CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA) public interface FileAgentTaskDAO { /** * 批量新增Agent任务 * * @param agentTasks Agent任务列表 */ - void batchSaveAgentTasks(Collection agentTasks); + void batchSaveAgentTasks(Collection agentTasks); /** * 批量更新Agent任务 * * @param agentTasks Agent任务 */ - void batchUpdateAgentTasks(Collection agentTasks); + void batchUpdateAgentTasks(Collection agentTasks); /** * 获取步骤成功执行的Agent任务数量 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 * @return 步骤成功执行的Agent任务数量 */ - int getSuccessAgentTaskCount(long stepInstanceId, int executeCount); + int getSuccessAgentTaskCount(Long taskInstanceId, long stepInstanceId, int executeCount); /** * 查询执行结果分组 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 * @param batch 滚动执行批次；如果传入null或者0，忽略该参数 * @return 执行结果分组 */ - List listResultGroups(long stepInstanceId, int executeCount, Integer batch); + List listResultGroups(Long taskInstanceId, + long stepInstanceId, + int executeCount, + Integer batch); /** * 根据执行结果查询Agent任务 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 * @param batch 滚动执行批次；如果传入null或者0，忽略该参数 * @param status 任务状态 * @return Agent任务 */ - List listAgentTaskByResultGroup(Long stepInstanceId, - Integer executeCount, - Integer batch, - Integer status); + List listAgentTaskByResultGroup(Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch, + Integer status); /** * 根据执行结果查询Agent任务(排序、限制返回数量) * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 * @param batch 滚动执行批次；如果传入null或者0，忽略该参数 @@ -95,39 +107,44 @@ List listAgentTaskByResultGroup(Long stepInstanceId, * @param order 排序方式 * @return Agent任务 */ - List listAgentTaskByResultGroup(Long stepInstanceId, - Integer executeCount, - Integer batch, - Integer status, - Integer limit, - String orderField, - Order order); + List listAgentTaskByResultGroup(Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch, + Integer status, + Integer limit, + String orderField, + Order order); /** * 获取agent任务 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 * @param batch 滚动执行批次；传入null或者0将忽略该参数 * @param fileTaskMode 文件分发任务模式;传入null表示忽略该过滤条件 * @return agent任务 */ - List listAgentTasks(Long stepInstanceId, - Integer executeCount, - Integer batch, - FileTaskModeEnum fileTaskMode); + List listAgentTasks(Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch, + FileTaskModeEnum fileTaskMode); /** * 根据GSE任务ID获取agent任务 * - * @param gseTaskId GSE任务ID + * @param taskInstanceId 作业实例 ID + * @param gseTaskId GSE任务ID * @return agent任务 */ - List listAgentTasksByGseTaskId(Long gseTaskId); + List listAgentTasksByGseTaskId(Long taskInstanceId, Long gseTaskId); /** * 根据hostId查询Agent任务 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 * @param batch 滚动执行批次；传入null或者0将忽略该参数 @@ -135,26 +152,26 @@ List listAgentTasks(Long stepInstanceId, * @param hostId 主机ID * @return Agent任务 */ - AgentTaskDTO getAgentTaskByHostId(Long stepInstanceId, Integer executeCount, Integer batch, - FileTaskModeEnum mode, long hostId); + ExecuteObjectTask getAgentTaskByHostId(Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch, + FileTaskModeEnum mode, + long hostId); - /** - * 判断步骤实例的Agent Task 记录是否存在 - * - * @param stepInstanceId 步骤实例ID - */ - boolean isStepInstanceRecordExist(long stepInstanceId); /** * 批量更新AgentTask的字段 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 条件 - 步骤实例ID * @param executeCount 条件 - 重试次数 * @param batch 条件 - 滚动执行批次；传入null将忽略该条件 * @param actualExecuteCount 值 - Agent任务实际执行的步骤重试次数；如果传入null，则不更新 * @param gseTaskId 值 - Agent任务对应的GSE_TASK_ID；如果传入null，则不更新 */ - void updateAgentTaskFields(long stepInstanceId, + void updateAgentTaskFields(Long taskInstanceId, + long stepInstanceId, int executeCount, Integer batch, Integer actualExecuteCount, diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileExecuteObjectTaskDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileExecuteObjectTaskDAO.java new file mode 100644 index 0000000000..411ab9f8c4 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileExecuteObjectTaskDAO.java @@ -0,0 +1,181 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.dao; + +import com.tencent.bk.job.common.constant.Order; +import com.tencent.bk.job.execute.model.ExecuteObjectTask; +import com.tencent.bk.job.execute.model.ResultGroupBaseDTO; +import com.tencent.bk.job.logsvr.consts.FileTaskModeEnum; + +import java.util.Collection; +import java.util.List; + +/** + * FileExecuteObjectTaskDAO + */ +public interface FileExecuteObjectTaskDAO { + /** + * 批量新增任务 + * + * @param tasks 任务列表 + */ + void batchSaveTasks(Collection tasks); + + /** + * 批量更新任务 + * + * @param tasks 任务 + */ + void batchUpdateTasks(Collection tasks); + + /** + * 获取步骤成功执行的任务数量 + * + * @param taskInstanceId 作业实例 ID + * @param stepInstanceId 步骤实例ID + * @param executeCount 执行次数 + * @return 步骤成功执行的任务数量 + */ + int getSuccessTaskCount(Long taskInstanceId, long stepInstanceId, int executeCount); + + /** + * 查询执行结果分组 + * + * @param taskInstanceId 作业实例 ID + * @param stepInstanceId 步骤实例ID + * @param executeCount 执行次数 + * @param batch 滚动执行批次；如果传入null或者0，忽略该参数 + * @return 执行结果分组 + */ + List listResultGroups(Long taskInstanceId, + long stepInstanceId, + int executeCount, + Integer batch); + + /** + * 根据执行结果查询任务 + * + * @param stepInstanceId 步骤实例ID + * @param executeCount 执行次数 + * @param batch 滚动执行批次；如果传入null或者0，忽略该参数 + * @param status 任务状态 + * @return 任务 + */ + List listTaskByResultGroup(Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch, + Integer status); + + /** + * 根据执行结果查询任务(排序、限制返回数量) + * + * @param taskInstanceId 作业实例 ID + * @param stepInstanceId 步骤实例ID + * @param executeCount 执行次数 + * @param batch 滚动执行批次；如果传入null或者0，忽略该参数 + * @param status 任务状态 + * @param limit 最大返回数量 + * @param orderField 排序字段 + * @param order 排序方式 + * @return 任务 + */ + List listTaskByResultGroup(Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch, + Integer status, + Integer limit, + String orderField, + Order order); + + /** + * 获取agent任务 + * + * @param taskInstanceId 作业实例 ID + * @param stepInstanceId 步骤实例ID + * @param executeCount 执行次数 + * @param batch 滚动执行批次；传入null或者0将忽略该参数 + * @param fileTaskMode 文件分发任务模式;传入null表示忽略该过滤条件 + * @return agent任务 + */ + List listTasks(Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch, + FileTaskModeEnum fileTaskMode); + + /** + * 根据GSE任务ID获取agent任务 + * + * @param gseTaskId GSE任务ID + * @return agent任务 + */ + List listTasksByGseTaskId(Long taskInstanceId, Long gseTaskId); + + /** + * 根据执行对象ID查询任务 + * + * @param taskInstanceId 作业实例 ID + * @param stepInstanceId 步骤实例ID + * @param executeCount 执行次数 + * @param batch 滚动执行批次；传入null或者0将忽略该参数 + * @param mode 文件分发任务模式 + * @param executeObjectId 执行对象ID + * @return 任务 + */ + ExecuteObjectTask getTaskByExecuteObjectId(Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch, + FileTaskModeEnum mode, + String executeObjectId); + + + /** + * 判断步骤实例的Agent Task 记录是否存在 + * + * @param stepInstanceId 步骤实例ID + */ + boolean isStepInstanceRecordExist(Long taskInstanceId, long stepInstanceId); + + /** + * 批量更新Task的字段 + * + * @param taskInstanceId 作业实例 ID + * @param stepInstanceId 条件 - 步骤实例ID + * @param executeCount 条件 - 重试次数 + * @param batch 条件 - 滚动执行批次；传入null将忽略该条件 + * @param actualExecuteCount 值 - 任务实际执行的步骤重试次数；如果传入null，则不更新 + * @param gseTaskId 值 - 任务对应的GSE_TASK_ID；如果传入null，则不更新 + */ + void updateTaskFields(Long taskInstanceId, + long stepInstanceId, + int executeCount, + Integer batch, + Integer actualExecuteCount, + Long gseTaskId); + +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileSourceTaskLogDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileSourceTaskLogDAO.java index 102b7912ac..ffd15c2278 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileSourceTaskLogDAO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/FileSourceTaskLogDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,16 +27,19 @@ import com.tencent.bk.job.execute.model.FileSourceTaskLogDTO; public interface FileSourceTaskLogDAO { - FileSourceTaskLogDTO getStepLastExecuteLog(long stepInstanceId); - void saveFileSourceTaskLog(FileSourceTaskLogDTO fileSourceTaskLog); + int insertFileSourceTaskLog(FileSourceTaskLogDTO fileSourceTaskLog); - FileSourceTaskLogDTO getFileSourceTaskLog(long stepInstanceId, int executeCount); + int updateFileSourceTaskLogByStepInstance(FileSourceTaskLogDTO fileSourceTaskLog); - FileSourceTaskLogDTO getFileSourceTaskLogByBatchTaskId(String fileSourceBatchTaskId); + FileSourceTaskLogDTO getFileSourceTaskLog(Long taskInstanceId, long stepInstanceId, int executeCount); - int updateTimeConsumingByBatchTaskId(String fileSourceBatchTaskId, Long startTime, Long endTime, Long totalTime); + FileSourceTaskLogDTO getFileSourceTaskLogByBatchTaskId(Long taskInstanceId, String fileSourceBatchTaskId); - void deleteFileSourceTaskLog(long stepInstanceId, int executeCount); + int updateTimeConsumingByBatchTaskId(Long taskInstanceId, + String fileSourceBatchTaskId, + Long startTime, + Long endTime, + Long totalTime); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/GseTaskDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/GseTaskDAO.java index 572c693b55..dc0af5639e 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/GseTaskDAO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/GseTaskDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,6 +25,9 @@ package com.tencent.bk.job.execute.dao; import com.tencent.bk.job.execute.model.GseTaskDTO; +import com.tencent.bk.job.execute.model.GseTaskSimpleDTO; + +import java.util.List; /** * GseTaskDAO @@ -50,19 +53,41 @@ public interface GseTaskDAO { /** * 获取 GSE 任务 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 步骤执行次数 * @param batch 滚动执行批次 * @return GSE 任务 */ - GseTaskDTO getGseTask(long stepInstanceId, int executeCount, Integer batch); + GseTaskDTO getGseTask(Long taskInstanceId, long stepInstanceId, int executeCount, Integer batch); /** * 获取 GSE 任务 * - * @param gseTaskId GSE任务ID + * @param taskInstanceId 作业实例 ID + * @param gseTaskId GSE任务ID * @return GSE 任务 */ - GseTaskDTO getGseTask(long gseTaskId); + GseTaskDTO getGseTask(Long taskInstanceId, long gseTaskId); + + /** + * 获取 GSE 任务 + * + * @param gseTaskId GSE任务ID + * @return stepInstanceId + */ + GseTaskSimpleDTO getGseTaskSimpleInfo(String gseTaskId); + + /** + * 获取 GSE 任务列表 + * + * @param stepInstanceId 步骤实例ID + * @param executeCount 执行次数 + * @param batch 批次 + * @return GSE 任务列表 + */ + List ListGseTaskSimpleInfo(Long stepInstanceId, + Integer executeCount, + Integer batch); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/GseTaskIpLogDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/GseTaskIpLogDAO.java deleted file mode 100644 index 620e7f15c7..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/GseTaskIpLogDAO.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.dao; - -import com.tencent.bk.job.common.annotation.CompatibleImplementation; -import com.tencent.bk.job.common.constant.Order; -import com.tencent.bk.job.execute.model.AgentTaskDTO; -import com.tencent.bk.job.execute.model.AgentTaskResultGroupBaseDTO; - -import java.util.List; - -/** - * GseTaskIpLogDAO - */ -@CompatibleImplementation(name = "rolling_execute", explain = "兼容老版本数据，过1-2个大版本之后删除", version = "3.7.x") -@Deprecated -public interface GseTaskIpLogDAO { - - /** - * 获取步骤成功执行的Agent任务数量 - * - * @param stepInstanceId 步骤实例ID - * @param executeCount 执行次数 - * @return 步骤成功执行的Agent任务数量 - */ - int getSuccessAgentTaskCount(long stepInstanceId, int executeCount); - - /** - * 查询执行结果分组 - * - * @param stepInstanceId 步骤实例ID - * @param executeCount 执行次数 - * @return 执行结果分组 - */ - List listResultGroups(long stepInstanceId, int executeCount); - - /** - * 根据执行结果查询Agent任务 - * - * @param stepInstanceId 步骤实例ID - * @param executeCount 执行次数 - * @param status 任务状态 - * @param tag 用户自定义分组标签 - * @return Agent任务 - */ - List listAgentTaskByResultGroup(Long stepInstanceId, - Integer executeCount, - Integer status, - String tag); - - /** - * 根据执行结果查询Agent任务(排序、限制返回数量) - * - * @param stepInstanceId 步骤实例ID - * @param executeCount 执行次数 - * @param status 任务状态 - * @param tag 用户自定义分组标签 - * @param limit 最大返回数量 - * @param orderField 排序字段 - * @param order 排序方式 - * @return Agent任务 - */ - List listAgentTaskByResultGroup(Long stepInstanceId, - Integer executeCount, - Integer status, - String tag, - Integer limit, - String orderField, - Order order); - - /** - * 获取Agent任务 - * - * @param stepInstanceId 步骤实例ID - * @param executeCount 执行次数 - * @return Agent任务信息 - */ - List listAgentTasks(Long stepInstanceId, - Integer executeCount); - - /** - * 根据hostId查询Agent任务 - * - * @param stepInstanceId 步骤实例ID - * @param executeCount 执行次数 - * @param ip 主机ip - * @return Agent任务 - */ - AgentTaskDTO getAgentTaskByIp(Long stepInstanceId, Integer executeCount, String ip); - - /** - * 获取Agent任务实际执行成功的executeCount值(重试场景) - * - * @param stepInstanceId 步骤实例ID - * @param cloudIp 主机ip - * @return Agent任务实际执行成功的executeCount值 - */ - int getActualSuccessExecuteCount(long stepInstanceId, String cloudIp); -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/OperationLogDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/OperationLogDAO.java index 3cc6f5cccc..6e4cffd02b 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/OperationLogDAO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/OperationLogDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/RollingConfigDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/RollingConfigDAO.java index cfdcf2e8cf..b82ceb4555 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/RollingConfigDAO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/RollingConfigDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -41,8 +41,17 @@ public interface RollingConfigDAO { /** * 通过滚动配置ID获取滚动配置 * + * @param taskInstanceId 任务id * @param rollingConfigId 滚动配置ID * @return 滚动配置 */ - RollingConfigDTO queryRollingConfigById(Long rollingConfigId); + RollingConfigDTO queryRollingConfigById(Long taskInstanceId, Long rollingConfigId); + + /** + * 任务是否启用了滚动执行 + * + * @param taskInstanceId 任务id + * @return boolean true启用，false未启用 + */ + boolean existsRollingConfig(long taskInstanceId); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/ScriptAgentTaskDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/ScriptAgentTaskDAO.java index cd723f3ff4..2d809c73e7 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/ScriptAgentTaskDAO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/ScriptAgentTaskDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,9 +24,11 @@ package com.tencent.bk.job.execute.dao; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.CompatibleType; import com.tencent.bk.job.common.constant.Order; -import com.tencent.bk.job.execute.model.AgentTaskDTO; -import com.tencent.bk.job.execute.model.AgentTaskResultGroupBaseDTO; +import com.tencent.bk.job.execute.model.ExecuteObjectTask; +import com.tencent.bk.job.execute.model.ResultGroupBaseDTO; import java.util.Collection; import java.util.List; @@ -34,43 +36,51 @@ /** * ScriptAgentTaskDAO */ +@Deprecated +@CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA) public interface ScriptAgentTaskDAO { /** * 批量保存Agent任务 * * @param agentTasks Agent任务 */ - void batchSaveAgentTasks(Collection agentTasks); + void batchSaveAgentTasks(Collection agentTasks); /** * 批量更新Agent任务 * * @param agentTasks Agent任务 */ - void batchUpdateAgentTasks(Collection agentTasks); + void batchUpdateAgentTasks(Collection agentTasks); /** * 获取步骤成功执行的Agent任务数量 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 * @return 步骤成功执行的Agent任务数量 */ - int getSuccessAgentTaskCount(long stepInstanceId, int executeCount); + int getSuccessAgentTaskCount(Long taskInstanceId, long stepInstanceId, int executeCount); /** * 查询执行结果分组 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 * @param batch 滚动执行批次；如果传入null或者0，忽略该参数 * @return 执行结果分组 */ - List listResultGroups(long stepInstanceId, int executeCount, Integer batch); + List listResultGroups(Long taskInstanceId, + long stepInstanceId, + int executeCount, + Integer batch); /** * 根据执行结果查询Agent任务 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 * @param batch 滚动执行批次；如果传入null或者0，忽略该参数 @@ -78,15 +88,17 @@ public interface ScriptAgentTaskDAO { * @param tag 用户自定义分组标签 * @return Agent任务 */ - List listAgentTaskByResultGroup(Long stepInstanceId, - Integer executeCount, - Integer batch, - Integer status, - String tag); + List listAgentTaskByResultGroup(Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch, + Integer status, + String tag); /** * 根据执行结果查询Agent任务(排序、限制返回数量) * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 * @param batch 滚动执行批次；如果传入null或者0，忽略该参数 @@ -97,63 +109,67 @@ List listAgentTaskByResultGroup(Long stepInstanceId, * @param order 排序方式 * @return Agent任务 */ - List listAgentTaskByResultGroup(Long stepInstanceId, - Integer executeCount, - Integer batch, - Integer status, - String tag, - Integer limit, - String orderField, - Order order); + List listAgentTaskByResultGroup(Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch, + Integer status, + String tag, + Integer limit, + String orderField, + Order order); /** * 获取Agent任务 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 * @param batch 滚动执行批次；传入null或者0将忽略该参数 * @return Agent任务信息 */ - List listAgentTasks(Long stepInstanceId, - Integer executeCount, - Integer batch); + List listAgentTasks(Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch); /** * 根据GSE任务ID获取Agent任务 * - * @param gseTaskId GSE任务ID + * @param taskInstanceId 作业实例 ID + * @param gseTaskId GSE任务ID * @return Agent任务 */ - List listAgentTasksByGseTaskId(Long gseTaskId); + List listAgentTasksByGseTaskId(Long taskInstanceId, Long gseTaskId); /** * 根据hostId查询Agent任务 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 * @param batch 滚动执行批次；传入null或者0将忽略该参数 * @param hostId 主机ID * @return Agent任务 */ - AgentTaskDTO getAgentTaskByHostId(Long stepInstanceId, Integer executeCount, Integer batch, long hostId); + ExecuteObjectTask getAgentTaskByHostId(Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch, + long hostId); /** - * 判断步骤实例的Agent Task 记录是否存在 - * - * @param stepInstanceId 步骤实例ID - */ - boolean isStepInstanceRecordExist(long stepInstanceId); - - /** - * 批量更新AgentTask的字段 + * 部分更新AgentTask的字段 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 条件 - 步骤实例ID * @param executeCount 条件 - 重试次数 * @param batch 条件 - 滚动执行批次；传入null将忽略该条件 * @param actualExecuteCount 值 - Agent任务实际执行的步骤重试次数；如果传入null，则不更新 * @param gseTaskId 值 - Agent任务对应的GSE_TASK_ID；如果传入null，则不更新 */ - void updateAgentTaskFields(long stepInstanceId, + void updateAgentTaskFields(Long taskInstanceId, + long stepInstanceId, int executeCount, Integer batch, Integer actualExecuteCount, diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/ScriptExecuteObjectTaskDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/ScriptExecuteObjectTaskDAO.java new file mode 100644 index 0000000000..cacb5bb8ae --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/ScriptExecuteObjectTaskDAO.java @@ -0,0 +1,196 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.dao; + +import com.tencent.bk.job.common.constant.Order; +import com.tencent.bk.job.execute.model.ExecuteObjectTask; +import com.tencent.bk.job.execute.model.ResultGroupBaseDTO; + +import java.util.Collection; +import java.util.List; + +/** + * ScriptExecuteObjectTaskDAO + */ +public interface ScriptExecuteObjectTaskDAO { + /** + * 批量保存任务 + * + * @param tasks 任务 + */ + void batchSaveTasks(Collection tasks); + + /** + * 批量更新任务 + * + * @param tasks 任务 + */ + void batchUpdateTasks(Collection tasks); + + /** + * 获取步骤成功执行的任务数量 + * + * @param stepInstanceId 步骤实例ID + * @param executeCount 执行次数 + * @return 步骤成功执行的任务数量 + */ + int getSuccessTaskCount(Long taskInstanceId, long stepInstanceId, int executeCount); + + /** + * 查询执行结果分组 + * + * @param taskInstanceId 作业实例 ID + * @param stepInstanceId 步骤实例ID + * @param executeCount 执行次数 + * @param batch 滚动执行批次；如果传入null或者0，忽略该参数 + * @return 执行结果分组 + */ + List listResultGroups(Long taskInstanceId, + long stepInstanceId, + int executeCount, + Integer batch); + + /** + * 根据执行结果查询任务 + * + * @param taskInstanceId 作业实例 ID + * @param stepInstanceId 步骤实例ID + * @param executeCount 执行次数 + * @param batch 滚动执行批次；如果传入null或者0，忽略该参数 + * @param status 任务状态 + * @param tag 用户自定义分组标签 + * @return 任务 + */ + List listTasksByResultGroup(Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch, + Integer status, + String tag); + + /** + * 根据执行结果查询任务(排序、限制返回数量) + * + * @param taskInstanceId 作业实例 ID + * @param stepInstanceId 步骤实例ID + * @param executeCount 执行次数 + * @param batch 滚动执行批次；如果传入null或者0，忽略该参数 + * @param status 任务状态 + * @param tag 用户自定义分组标签 + * @param limit 最大返回数量 + * @param orderField 排序字段 + * @param order 排序方式 + * @return 任务 + */ + List listTasksByResultGroup(Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch, + Integer status, + String tag, + Integer limit, + String orderField, + Order order); + + /** + * 获取任务 + * + * @param taskInstanceId 作业实例 ID + * @param stepInstanceId 步骤实例ID + * @param executeCount 执行次数 + * @param batch 滚动执行批次；传入null或者0将忽略该参数 + * @return 任务信息 + */ + List listTasks(Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch); + + /** + * 根据GSE任务ID获取任务 + * + * @param taskInstanceId 作业实例 ID + * @param gseTaskId GSE任务ID + * @return 任务 + */ + List listTasksByGseTaskId(Long taskInstanceId, Long gseTaskId); + + /** + * 根据执行对象ID查询任务 + * + * @param taskInstanceId 作业实例 ID + * @param stepInstanceId 步骤实例ID + * @param executeCount 执行次数 + * @param batch 滚动执行批次；传入null或者0将忽略该参数 + * @param executeObjectId 执行对象ID + * @return 任务 + */ + ExecuteObjectTask getTaskByExecuteObjectId(Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch, + String executeObjectId); + + /** + * 根据执行对象ID批量查询任务 + * + * @param taskInstanceId 作业实例 ID + * @param stepInstanceId 步骤实例ID + * @param executeCount 执行次数 + * @param batch 滚动执行批次；传入null或者0将忽略该参数 + * @param executeObjectIds 执行对象ID集合 + * @return 任务 + */ + List getTaskByExecuteObjectIds(Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch, + Collection executeObjectIds); + + /** + * 判断步骤实例的执行对象任务记录是否存在 + * + * @param taskInstanceId 作业实例 ID + * @param stepInstanceId 步骤实例ID + */ + boolean isStepInstanceRecordExist(Long taskInstanceId, long stepInstanceId); + + /** + * 部分更新任务的字段 + * + * @param taskInstanceId 作业实例 ID + * @param stepInstanceId 条件 - 步骤实例ID + * @param executeCount 条件 - 重试次数 + * @param batch 条件 - 滚动执行批次；传入null将忽略该条件 + * @param actualExecuteCount 值 - 任务实际执行的步骤重试次数；如果传入null，则不更新 + * @param gseTaskId 值 - 任务对应的GSE_TASK_ID；如果传入null，则不更新 + */ + void updateTaskFields(Long taskInstanceId, + long stepInstanceId, + int executeCount, + Integer batch, + Integer actualExecuteCount, + Long gseTaskId); +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StatisticsDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StatisticsDAO.java index 62c3995afe..50f36c1567 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StatisticsDAO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StatisticsDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,56 +24,31 @@ package com.tencent.bk.job.execute.dao; -import com.tencent.bk.job.common.statistics.model.dto.StatisticsDTO; -import org.jooq.DSLContext; +import com.tencent.bk.job.analysis.api.dto.StatisticsDTO; +import com.tencent.bk.job.execute.statistics.StatisticsKey; import java.util.List; public interface StatisticsDAO { - Long insertStatistics(DSLContext dslContext, StatisticsDTO statisticsDTO); - - Long upsertStatistics(DSLContext dslContext, StatisticsDTO statisticsDTO); - - int updateStatisticsById(DSLContext dslContext, StatisticsDTO statisticsDTO); - - int deleteStatisticsById(DSLContext dslContext, Long id); int deleteStatisticsByDate(String date); int deleteOneDayStatistics(Long appId, String date); - StatisticsDTO getStatisticsById(Long id); StatisticsDTO getStatistics(Long appId, String resource, String dimension, String dimensionValue, String date); - List getStatisticsListByAppId(Long appId, String resource, String dimension, String dimensionValue - , String sinceDate); - - List getStatisticsList(Long appId, String resource, String dimension, String date); - - List getStatisticsList(List inAppIdList, String resource, String dimension, String date); List getStatisticsList(List inAppIdList, List notInAppIdList, String resource, String dimension, String dimensionValue, String date); - Long getTotalValueOfStatisticsList(List inAppIdList, List notInAppIdList, String resource, - String dimension, String dimensionValue, String date); - - List getStatisticsListBetweenDate(List inAppIdList, List notInAppIdList, - String resource, String dimension, String startDate, - String endDate); - - List getStatisticsListBetweenDate(List inAppIdList, List notInAppIdList, - String resource, String dimension, String dimensionValue, - String startDate, String endDate); - - List getStatisticsListBetweenDate(Long appId, String resource, String dimension, - String dimensionValue, String startDate, String endDate); - - List listAllStatistics(); - - Integer countStatisticsByDate(String date); - - Integer countStatistics(List inAppIdList, List notInAppIdList, String resource, String dimension, - String dimensionValue, String date); + /** + * 增加统计指标的值 + * + * @param date 日期 + * @param statisticsKey 统计 key + * @param incrementValue 增长值 + * @return affectRows 更新的行数 + */ + int increaseStatisticValue(String date, StatisticsKey statisticsKey, Integer incrementValue); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceDAO.java index 88a7a2970b..65b20c2a49 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceDAO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,20 +24,20 @@ package com.tencent.bk.job.execute.dao; -import com.tencent.bk.job.common.constant.DuplicateHandlerEnum; -import com.tencent.bk.job.common.constant.NotExistPathHandlerEnum; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; -import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; import com.tencent.bk.job.execute.model.ConfirmStepInstanceDTO; import com.tencent.bk.job.execute.model.FileSourceDTO; import com.tencent.bk.job.execute.model.FileStepInstanceDTO; import com.tencent.bk.job.execute.model.ScriptStepInstanceDTO; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; import java.util.List; +import java.util.Map; +/** + * 任务步骤实例 DAO + */ public interface StepInstanceDAO { Long addStepInstanceBase(StepInstanceBaseDTO stepInstance); @@ -47,6 +47,13 @@ public interface StepInstanceDAO { void addConfirmStepInstance(StepInstanceDTO stepInstance); + StepInstanceBaseDTO getStepInstanceBase(Long taskInstanceId, long stepInstanceId); + + /** + * 获取步骤信息。注意，在分库分表的部署架构下，由于缺少 taskInstanceId （分片键）作为查询条件，该 SQL 执行性能较低；慎用！！ + * + * @param stepInstanceId 步骤实例 ID + */ StepInstanceBaseDTO getStepInstanceBase(long stepInstanceId); /** @@ -55,7 +62,7 @@ public interface StepInstanceDAO { * @param taskInstanceId 作业实例ID * @return 作业第一个步骤实例 */ - StepInstanceBaseDTO getFirstStepInstanceBase(long taskInstanceId); + StepInstanceBaseDTO getFirstStepInstanceBase(Long taskInstanceId); /** * 获取下一个步骤实例 @@ -64,25 +71,25 @@ public interface StepInstanceDAO { * @param currentStepOrder 当前步骤的顺序 * @return 步骤实例；如果当前为最后一个步骤实例，那么返回null */ - StepInstanceBaseDTO getNextStepInstance(long taskInstanceId, int currentStepOrder); + StepInstanceBaseDTO getNextStepInstance(Long taskInstanceId, int currentStepOrder); - ScriptStepInstanceDTO getScriptStepInstance(long stepInstanceId); + ScriptStepInstanceDTO getScriptStepInstance(Long taskInstanceId, long stepInstanceId); - FileStepInstanceDTO getFileStepInstance(long stepInstanceId); + FileStepInstanceDTO getFileStepInstance(Long taskInstanceId, long stepInstanceId); - ConfirmStepInstanceDTO getConfirmStepInstance(long stepInstanceId); + ConfirmStepInstanceDTO getConfirmStepInstance(Long taskInstanceId, long stepInstanceId); - List listStepInstanceBaseByTaskInstanceId(long taskInstanceId); + List listStepInstanceBaseByTaskInstanceId(Long taskInstanceId); - void resetStepStatus(long stepInstanceId); + void resetStepStatus(Long taskInstanceId, long stepInstanceId); - void resetStepExecuteInfoForRetry(long stepInstanceId); + void resetStepExecuteInfoForRetry(Long taskInstanceId, long stepInstanceId); - void addStepExecuteCount(long stepInstanceId); + void addStepExecuteCount(Long taskInstanceId, long stepInstanceId); - void updateStepStatus(long stepInstanceId, int status); + void updateStepStatus(Long taskInstanceId, long stepInstanceId, int status); - void updateStepStartTime(long stepInstanceId, Long startTime); + void updateStepStartTime(Long taskInstanceId, long stepInstanceId, Long startTime); /** * 更新步骤启动时间 - 仅当启动时间为空的的场景 @@ -90,125 +97,142 @@ public interface StepInstanceDAO { * @param stepInstanceId 步骤实例ID * @param startTime 启动时间 */ - void updateStepStartTimeIfNull(long stepInstanceId, Long startTime); + void updateStepStartTimeIfNull(Long taskInstanceId, long stepInstanceId, Long startTime); - void updateStepEndTime(long stepInstanceId, Long endTime); + void updateStepEndTime(Long taskInstanceId, long stepInstanceId, Long endTime); - void addStepInstanceExecuteCount(long stepInstanceId); + void addStepInstanceExecuteCount(Long taskInstanceId, long stepInstanceId); - void updateStepTotalTime(long stepInstanceId, long totalTime); + void updateStepTotalTime(Long taskInstanceId, long stepInstanceId, long totalTime); /** * 更新步骤的执行信息 * + * @param taskInstanceId 任务实例 ID * @param stepInstanceId 步骤实例ID * @param status 步骤执行状态 * @param startTime 开始时间 * @param endTime 结束时间 * @param totalTime 总耗时 */ - void updateStepExecutionInfo(long stepInstanceId, RunStatusEnum status, Long startTime, Long endTime, + void updateStepExecutionInfo(Long taskInstanceId, + long stepInstanceId, + RunStatusEnum status, + Long startTime, + Long endTime, Long totalTime); /** * 更新解析之后的脚本参数 * + * @param taskInstanceId 任务实例 ID * @param stepInstanceId 步骤实例ID + * @param isSecureParam 是否为敏感参数 * @param resolvedScriptParam 解析之后的脚本参数 */ - void updateResolvedScriptParam(long stepInstanceId, String resolvedScriptParam); + void updateResolvedScriptParam(Long taskInstanceId, + long stepInstanceId, + boolean isSecureParam, + String resolvedScriptParam); /** * 更新变量解析之后的源文件 * + * @param taskInstanceId 任务实例 ID * @param stepInstanceId 步骤实例ID - * @param resolvedFileSources + * @param resolvedFileSources 解析后的源文件信息 */ - void updateResolvedSourceFile(long stepInstanceId, List resolvedFileSources); + void updateResolvedSourceFile(Long taskInstanceId, + long stepInstanceId, + List resolvedFileSources); /** * 更新变量解析之后的目标路径 * + * @param taskInstanceId 任务实例 ID * @param stepInstanceId 步骤实例ID * @param resolvedTargetPath 解析之后的目标路径 */ - void updateResolvedTargetPath(long stepInstanceId, String resolvedTargetPath); + void updateResolvedTargetPath(Long taskInstanceId, long stepInstanceId, String resolvedTargetPath); /** * 更新确认理由 * + * @param taskInstanceId 任务实例 ID * @param stepInstanceId 步骤实例ID * @param confirmReason 确认理由 */ - void updateConfirmReason(long stepInstanceId, String confirmReason); + void updateConfirmReason(Long taskInstanceId, long stepInstanceId, String confirmReason); /** * 更新步骤实例操作者 * + * @param taskInstanceId 任务实例 ID * @param stepInstanceId 步骤实例ID * @param operator 操作者 */ - void updateStepOperator(long stepInstanceId, String operator); - - Integer count(Long appId, List stepIdList, StepExecuteTypeEnum stepExecuteType, ScriptTypeEnum scriptType, - RunStatusEnum runStatus, Long fromTime, Long toTime); - - Integer countFastPushFile(Long appId, DuplicateHandlerEnum fileDupliateHandle, Boolean fileDupliateHandleNull, - NotExistPathHandlerEnum notExistPathHandler, Boolean notExistPathHandlerNull, - RunStatusEnum runStatus, Long fromTime, Long toTime); - - List> listFastPushFileSource(Long appId, DuplicateHandlerEnum fileDupliateHandle, - Boolean fileDupliateHandleNull, - NotExistPathHandlerEnum notExistPathHandler, - Boolean notExistPathHandlerNull, RunStatusEnum runStatus, - Long fromTime, Long toTime); + void updateStepOperator(Long taskInstanceId, long stepInstanceId, String operator); /** * 获取前一个可执行步骤实例 * - * @param taskInstanceId 任务实例ID - * @param stepInstanceId 当前步骤实例ID + * @param taskInstanceId 任务实例ID + * @param currentStepOrder 当前步骤的顺序 * @return 可执行步骤实例 */ - StepInstanceBaseDTO getPreExecutableStepInstance(long taskInstanceId, long stepInstanceId); + StepInstanceBaseDTO getPreExecutableStepInstance(Long taskInstanceId, int currentStepOrder); + + /** + * 获取步骤实例 ID 和步骤顺序的映射关系 + * + * @param taskInstanceId 任务实例ID + * @return 步骤实例 ID 和步骤顺序的映射关系 + */ + Map listStepInstanceIdAndStepOrderMapping(Long taskInstanceId); /** * 根据taskInstanceId获取一个stepInstanceId，用于快速脚本/文件任务 * - * @param taskInstanceId - * @return + * @param taskInstanceId 作业实例ID + * @return 步骤实例ID */ - Long getStepInstanceId(long taskInstanceId); + Long getStepInstanceId(Long taskInstanceId); /** * 根据stepInstanceId获取脚本类型 * - * @param stepInstanceId - * @return + * @param taskInstanceId 任务实例ID + * @param stepInstanceId 步骤实例ID + * @return 脚本类型 */ - Byte getScriptTypeByStepInstanceId(long stepInstanceId); + Byte getScriptTypeByStepInstanceId(Long taskInstanceId, long stepInstanceId); /** * 更新步骤实例的当前滚动执行批次 * + * @param taskInstanceId 任务实例ID * @param stepInstanceId 步骤实例ID * @param batch 滚动执行批次 */ - void updateStepCurrentBatch(long stepInstanceId, int batch); + void updateStepCurrentBatch(Long taskInstanceId, long stepInstanceId, int batch); /** * 更新步骤实例的当前滚动执行批次 * + * @param taskInstanceId 任务实例ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 */ - void updateStepCurrentExecuteCount(long stepInstanceId, int executeCount); + void updateStepCurrentExecuteCount(Long taskInstanceId, long stepInstanceId, int executeCount); /** * 更新步骤实例的滚动配置ID * + * @param taskInstanceId 任务实例ID * @param stepInstanceId 步骤实例ID * @param rollingConfigId 滚动配置ID */ - void updateStepRollingConfigId(long stepInstanceId, long rollingConfigId); + void updateStepRollingConfigId(Long taskInstanceId, long stepInstanceId, long rollingConfigId); + + List getTaskStepInstanceIdList(Long taskInstanceId); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceRollingTaskDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceRollingTaskDAO.java index d7a0f9f3e6..4859ab9f23 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceRollingTaskDAO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceRollingTaskDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -37,24 +37,28 @@ public interface StepInstanceRollingTaskDAO { /** * 查询步骤滚动任务 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 步骤执行次数 * @param batch 滚动批次 * @return 步骤滚动任务 */ - StepInstanceRollingTaskDTO queryRollingTask(long stepInstanceId, + StepInstanceRollingTaskDTO queryRollingTask(Long taskInstanceId, + long stepInstanceId, int executeCount, int batch); /** * 查询步骤滚动任务 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数;如果不为null，会根据executeCount过滤 * @param batch 滚动执行批次;如果不为null，会根据batch过滤 * @return 步骤滚动任务 */ - List listRollingTasks(long stepInstanceId, + List listRollingTasks(Long taskInstanceId, + long stepInstanceId, Integer executeCount, Integer batch); @@ -69,6 +73,7 @@ List listRollingTasks(long stepInstanceId, /** * 更新滚动任务 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 步骤执行次数 * @param batch 滚动执行批次 @@ -77,7 +82,8 @@ List listRollingTasks(long stepInstanceId, * @param endTime 任务结束时间；如果不更新传入null * @param totalTime 任务执行总时间；如果不更新传入null */ - void updateRollingTask(long stepInstanceId, + void updateRollingTask(Long taskInstanceId, + long stepInstanceId, int executeCount, int batch, RunStatusEnum status, diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceVariableDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceVariableDAO.java index a72a3d1132..d7578a5cf6 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceVariableDAO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/StepInstanceVariableDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -43,12 +43,15 @@ public interface StepInstanceVariableDAO { /** * 获取步骤变量值 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 * @param variableValueType 参数值类型 * @return 步骤变量值 */ - StepInstanceVariableValuesDTO getStepVariableValues(long stepInstanceId, int executeCount, + StepInstanceVariableValuesDTO getStepVariableValues(Long taskInstanceId, + long stepInstanceId, + int executeCount, VariableValueTypeEnum variableValueType); /** @@ -58,13 +61,4 @@ StepInstanceVariableValuesDTO getStepVariableValues(long stepInstanceId, int exe * @return 输出变量值 */ List listStepOutputVariableValuesByTaskInstanceId(long taskInstanceId); - - /** - * 获取前面步骤的所有输出变量值 - * - * @param taskInstanceId 作业实例ID - * @param stepInstanceId 步骤实例ID - * @return 每个步骤的变量值。返回值按照执行先后排序 - */ - List listSortedPreStepOutputVariableValues(long taskInstanceId, long stepInstanceId); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/TaskInstanceDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/TaskInstanceDAO.java index 3fcd69b4ee..a53d8d2d01 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/TaskInstanceDAO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/TaskInstanceDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,8 +28,6 @@ import com.tencent.bk.job.common.model.PageData; import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; -import com.tencent.bk.job.execute.common.constants.TaskStartupModeEnum; -import com.tencent.bk.job.execute.common.constants.TaskTypeEnum; import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.model.TaskInstanceQuery; @@ -44,35 +42,22 @@ public interface TaskInstanceDAO { TaskInstanceDTO getTaskInstance(long taskInstanceId); - List getTaskInstanceByTaskId(long taskId); - void updateTaskStatus(long taskInstanceId, int status); - void updateTaskStartTime(long taskInstanceId, Long startTime); - - void updateTaskEndTime(long taskInstanceId, Long endTime); - - List getTaskStepInstanceIdList(long taskInstanceId); - void updateTaskCurrentStepId(Long taskInstanceId, Long stepInstanceId); void resetTaskStatus(Long taskInstanceId); - void cleanTaskEndTime(Long taskInstanceId); - - void updateTaskTotalTime(Long taskInstanceId, Long totalTime); - /** * 分页查询作业执行实例 * - * @param taskQuery - * @param baseSearchCondition - * @return + * @param taskQuery 查询 + * @param baseSearchCondition 基础查询条件 + * @return 分页结果 */ PageData listPageTaskInstance(TaskInstanceQuery taskQuery, BaseSearchCondition baseSearchCondition); - void addCallbackUrl(long taskInstanceId, String callBackUrl); /** * 获取定时作业执行情况 @@ -82,10 +67,13 @@ PageData listPageTaskInstance(TaskInstanceQuery taskQuery, * @param latestTimeInSeconds 时间范围 * @param status 任务状态,如果为NULL,那么返回所有状态 * @param limit 返回记录个数；如果未NULL,那么不限制返回数量 - * @return + * @return 作业实例列表 */ - List listLatestCronTaskInstance(long appId, Long cronTaskId, - Long latestTimeInSeconds, RunStatusEnum status, Integer limit); + List listLatestCronTaskInstance(long appId, + Long cronTaskId, + Long latestTimeInSeconds, + RunStatusEnum status, + Integer limit); /** * 更新作业的执行信息 @@ -97,8 +85,12 @@ List listLatestCronTaskInstance(long appId, Long cronTaskId, * @param endTime 结束时间 * @param totalTime 总耗时 */ - void updateTaskExecutionInfo(long taskInstanceId, RunStatusEnum status, Long currentStepId, - Long startTime, Long endTime, Long totalTime); + void updateTaskExecutionInfo(long taskInstanceId, + RunStatusEnum status, + Long currentStepId, + Long startTime, + Long endTime, + Long totalTime); /** * 重置作业执行状态 @@ -107,22 +99,6 @@ void updateTaskExecutionInfo(long taskInstanceId, RunStatusEnum status, Long cur */ void resetTaskExecuteInfoForRetry(long taskInstanceId); - /** - * 根据条件统计执行过的任务 - * - * @param appId 业务Id - * @param minTotalTime 最小执行时间 - * @param maxTotalTime 最大执行时间 - * @param taskStartupMode 启动模式 - * @param taskType 任务类型 - * @param runStatusList 任务状态 - * @param fromTime 统计时间起点 - * @param toTime 统计时间终点 - * @return - */ - Integer countTaskInstances(Long appId, Long minTotalTime, Long maxTotalTime, TaskStartupModeEnum taskStartupMode, - TaskTypeEnum taskType, List runStatusList, Long fromTime, Long toTime); - /** * 查询大于某个时间的定时任务执行记录对应的业务Id * ！！！调用时需要注意查询效率，可能慢查询 @@ -130,7 +106,7 @@ Integer countTaskInstances(Long appId, Long minTotalTime, Long maxTotalTime, Tas * @param inAppIdList 指定appId的范围 * @param cronTaskId 定时任务Id * @param minCreateTime 最小创建时间 - * @return + * @return 业务 ID 列表 */ List listTaskInstanceAppId(List inAppIdList, Long cronTaskId, Long minCreateTime); @@ -141,7 +117,7 @@ Integer countTaskInstances(Long appId, Long minTotalTime, Long maxTotalTime, Tas * @param cronTaskId 定时任务Id * @param fromTime 起始时间 * @param toTime 终止时间 - * @return + * @return 是否存在执行记录 */ boolean hasExecuteHistory(Long appId, Long cronTaskId, Long fromTime, Long toTime); @@ -150,9 +126,10 @@ Integer countTaskInstances(Long appId, Long minTotalTime, Long maxTotalTime, Tas /** * 保存作业实例与主机的关系，便于根据ip/ipv6检索作业实例 * + * @param appId 业务 ID * @param taskInstanceId 作业实例ID * @param hosts 主机列表 */ - void saveTaskInstanceHosts(long taskInstanceId, Collection hosts); + void saveTaskInstanceHosts(long appId, long taskInstanceId, Collection hosts); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/TaskInstanceVariableDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/TaskInstanceVariableDAO.java index 4cfe2cfe1e..61e6fef40c 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/TaskInstanceVariableDAO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/TaskInstanceVariableDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/AbstractPropBasedDynamicComponent.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/AbstractPropBasedDynamicComponent.java new file mode 100644 index 0000000000..f37ad0075c --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/AbstractPropBasedDynamicComponent.java @@ -0,0 +1,321 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.dao.common; + +import com.tencent.bk.job.common.mysql.dynamic.ds.MigrationStatus; +import com.tencent.bk.job.common.redis.util.LockUtils; +import com.tencent.bk.job.common.util.ThreadUtils; +import com.tencent.bk.job.common.util.ip.IpUtils; +import com.tencent.bk.job.common.util.toggle.prop.PropChangeEventListener; +import com.tencent.bk.job.common.util.toggle.prop.PropToggle; +import com.tencent.bk.job.common.util.toggle.prop.PropToggleStore; +import org.apache.commons.collections4.CollectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.data.redis.core.RedisTemplate; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static com.tencent.bk.job.common.mysql.dynamic.ds.MigrationStatus.FAIL; +import static com.tencent.bk.job.common.mysql.dynamic.ds.MigrationStatus.MIGRATED; +import static com.tencent.bk.job.common.mysql.dynamic.ds.MigrationStatus.MIGRATING; +import static com.tencent.bk.job.common.mysql.dynamic.ds.MigrationStatus.NOT_START; +import static com.tencent.bk.job.common.mysql.dynamic.ds.MigrationStatus.PREPARING; + +/** + * 基于属性动态控制的组件 + */ + +public abstract class AbstractPropBasedDynamicComponent implements PropChangeEventListener { + private final Logger log = LoggerFactory.getLogger(this.getClass()); + /** + * 候选的组件。key: 属性名称；value: 组件实例 + */ + private Map candidateComponents; + + private volatile MigrationStatus status = NOT_START; + + private final Object lock = new Object(); + + private final RedisTemplate redisTemplate; + + private volatile C current; + + private final PropToggleStore propToggleStore; + + /** + * 动态属性名称 - 迁移目标 + */ + private final String migrateTargetPropName; + /** + * 动态属性名称 - 参与迁移的微服务实例数量 + */ + private static final String PROP_NAME_SERVICE_INSTANCE_COUNT = "job_execute_service_instance_count"; + + /** + * 迁移使用的分布式锁 Redis KEY + */ + private final String migrationLockRedisKey; + /** + * 整体迁移状态 Redis KEY + */ + private final String globalMigrationStatusRedisKey; + /** + * 服务实例迁移状态 Redis KEY + */ + private final String serviceInstanceMigrationStatusRedisKey; + /** + * 服务节点 IP + */ + private final String serviceNodeIp; + + private static final String REDIS_KEY_PREFIX = "job:comp:mig:"; + + public AbstractPropBasedDynamicComponent( + RedisTemplate redisTemplate, + PropToggleStore propToggleStore, + String migrateComponentName) { + this.redisTemplate = redisTemplate; + this.propToggleStore = propToggleStore; + + this.serviceNodeIp = IpUtils.getFirstMachineIP(); + this.migrateTargetPropName = migrateComponentName; + this.migrationLockRedisKey = REDIS_KEY_PREFIX + migrateComponentName; + this.globalMigrationStatusRedisKey = REDIS_KEY_PREFIX + "global:status:" + migrateComponentName; + this.serviceInstanceMigrationStatusRedisKey = REDIS_KEY_PREFIX + ":svc:status:" + migrateComponentName; + + // 注册监听属性变化 + this.propToggleStore.addPropChangeEventListener(migrateTargetPropName, this); + } + + protected void initCandidateComponents(Map candidateComponents) { + this.candidateComponents = candidateComponents; + initCurrentComponent(); + } + + private void initCurrentComponent() { + // 从配置文件读取组件名称，并初始化 + String propValue = propToggleStore.getPropToggle(migrateTargetPropName).getDefaultValue(); + log.info("Init default component by prop: {}:{}", migrateTargetPropName, propValue); + this.current = getComponentByProp(this.candidateComponents, propValue); + if (this.current == null) { + log.error("No match component for {}", propValue); + throw new IllegalStateException("Unsupported component define by prop: " + propValue); + } + log.info("Use {} as default component", current.getClass()); + } + + /** + * 获取当前组件 + * 如果组件处于迁移状态中，会阻塞当前线程直到切换完成 + */ + public C getCurrent(boolean blockWhenMigration) { + if (status == MigrationStatus.MIGRATING && blockWhenMigration) { + // 如果组件正在迁移中，并且 blockWhenMigration = true, 需要等待迁移完成；当前线程阻塞 + try { + synchronized (lock) { + log.info("Component is migrating, block until migrated"); + lock.wait(); + log.info("Continue after component migrated"); + } + } catch (InterruptedException e) { + log.error("Get current component error", e); + } + } + return current; + } + + + @Override + public void handlePropChangeEvent(String propName, PropToggle currentValue) { + log.info("Handle prop change event, propName: {}, value: {}", propName, currentValue.getDefaultValue()); + C targetComponent = getComponentByProp(this.candidateComponents, currentValue.getDefaultValue()); + if (targetComponent == null) { + log.warn("No match component for prop : {}", currentValue.getDefaultValue()); + return; + } + if (targetComponent == current) { + // 迁移目标不变，无需处理 + log.info("Component is not changed, skip migration!"); + return; + } + + migrateComponent(targetComponent); + } + + private void migrateComponent(C targetComponent) { + boolean success = false; + try { + long startTime = System.currentTimeMillis(); + log.info("Migrate component from {} to {} start...", current.getClass(), targetComponent.getClass()); + PropToggle migrateServiceInstanceCountProp = + propToggleStore.getPropToggle(PROP_NAME_SERVICE_INSTANCE_COUNT); + if (migrateServiceInstanceCountProp == null) { + log.error("Prop [" + PROP_NAME_SERVICE_INSTANCE_COUNT + "] is required. Skip migration"); + return; + } + int migrateServiceInstanceCount = Integer.parseInt(migrateServiceInstanceCountProp.getDefaultValue()); + if (status == NOT_START) { + synchronized (this) { + if (status == NOT_START) { + status = PREPARING; + initServiceInstanceMigrationStatus(); + // 为了在同一时间(接近）在多个微服务实例切换到新的组件，需要判断处于 PREPARING 状态的服务实例是否符合预期数量 + updateServiceInstanceMigrationStatus(PREPARING); + while (true) { + if (System.currentTimeMillis() - startTime > 60000L) { + // 超过一分钟，放弃本次迁移 + log.info("Prepare migration cost 1min, terminate migration"); + updateServiceInstanceMigrationStatus(MigrationStatus.FAIL); + return; + } + long prepareServiceInstanceCount = + redisTemplate.opsForHash().size(serviceInstanceMigrationStatusRedisKey); + if (prepareServiceInstanceCount >= migrateServiceInstanceCount) { + log.info("All service node are ready, actualReadyNodeCount: {}, expected: {}", + prepareServiceInstanceCount, migrateServiceInstanceCount); + // 所有服务实例都确认收到组件迁移事件，准备启动迁移 + updateGlobalMigrationStatus(MIGRATING); + updateServiceInstanceMigrationStatus(MIGRATING); + // 等待 5s，等待当前组件正在执行的操作完成（比如 db 读写请求) + log.info("Wait 5s before migration"); + ThreadUtils.sleep(5000L); + break; + } else { + log.info("Wait all service instance ready, actual: {}, expected: {}", + prepareServiceInstanceCount, migrateServiceInstanceCount); + ThreadUtils.sleep(100L); + } + } + this.current = targetComponent; + updateServiceInstanceMigrationStatus(MIGRATED); + success = true; + } else { + log.warn("Unexpected migration status {}, skip migration", status); + } + } + } else { + log.warn("Unexpected migration status {}, skip migration", status); + } + } catch (Throwable e) { + log.error("Migrate component error", e); + } finally { + status = NOT_START; + clearAfterMigrated(); + synchronized (lock) { + lock.notifyAll(); + } + log.info("Migrate component done, isSuccess : {}", success); + } + } + + private void clearAfterMigrated() { + try { + List allServiceInstanceMigStatus = + redisTemplate.opsForHash().values(serviceInstanceMigrationStatusRedisKey); + List serviceInstancesMigStatusList = + castList(allServiceInstanceMigStatus, k -> MigrationStatus.valOf((Integer) k)); + if (serviceInstancesMigStatusList.stream().allMatch(status -> status == MIGRATED || status == FAIL)) { + // 所有服务实例都完成了组件迁移，开始清理 + log.info("All migration done. Delete all migration temporary data"); + redisTemplate.delete(serviceInstanceMigrationStatusRedisKey); + redisTemplate.delete(globalMigrationStatusRedisKey); + } else { + log.info("Some service instance node not yet complete migration"); + } + } catch (Throwable e) { + log.error("Clear migration caught exception", e); + } + } + + private void updateServiceInstanceMigrationStatus(MigrationStatus migrationStatus) { + log.info("Update service instance migration status {}", migrationStatus); + redisTemplate.opsForHash().put(serviceInstanceMigrationStatusRedisKey, + serviceNodeIp, migrationStatus.getStatus()); + this.status = migrationStatus; + } + + private void updateGlobalMigrationStatus(MigrationStatus migrationStatus) { + log.info("Update global migration status {}", migrationStatus); + redisTemplate.opsForValue().set(globalMigrationStatusRedisKey, + migrationStatus.getStatus()); + } + + private List castList(List source, Function mapping) { + if (CollectionUtils.isEmpty(source)) { + return Collections.emptyList(); + } + List list = new ArrayList<>(source.size()); + list.addAll(source.stream().map(mapping).collect(Collectors.toList())); + return list; + } + + private void initServiceInstanceMigrationStatus() { + MigrationStatus migrationStatus = getGlobalMigrationStatus(); + if (migrationStatus == null) { + boolean locked = false; + try { + locked = lockForUpdate(); + if (locked) { + // 二次确认，保证获取锁期间数据没有发生改变 + migrationStatus = getGlobalMigrationStatus(); + if (migrationStatus == null) { + // 初始化迁移状态 + log.info("Init migration status"); + redisTemplate.opsForValue().set(globalMigrationStatusRedisKey, PREPARING.getStatus()); + redisTemplate.delete(serviceInstanceMigrationStatusRedisKey); + } + } + } finally { + if (locked) { + unlock(); + } + } + } + } + + private MigrationStatus getGlobalMigrationStatus() { + Object migrationStatus = redisTemplate.opsForValue().get(globalMigrationStatusRedisKey); + if (migrationStatus == null) { + return null; + } + return MigrationStatus.valOf((Integer) migrationStatus); + } + + private boolean lockForUpdate() { + return LockUtils.lock(migrationLockRedisKey, serviceNodeIp, 60000L, 60); + } + + private boolean unlock() { + return LockUtils.releaseDistributedLock(migrationLockRedisKey, serviceNodeIp); + } + + protected abstract C getComponentByProp(Map candidateComponents, String propValue); +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/AutoIncrementIdGen.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/AutoIncrementIdGen.java new file mode 100644 index 0000000000..bff677716e --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/AutoIncrementIdGen.java @@ -0,0 +1,82 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.dao.common; + +public class AutoIncrementIdGen implements IdGen { + @Override + public Long genTaskInstanceId() { + return null; + } + + @Override + public Long genStepInstanceId() { + return null; + } + + @Override + public Long genGseTaskId() { + return null; + } + + @Override + public Long genOperationLogId() { + return null; + } + + @Override + public Long genFileSourceTaskLogId() { + return null; + } + + @Override + public Long genGseFileExecuteObjTaskId() { + return null; + } + + @Override + public Long genGseScriptExecuteObjTaskId() { + return null; + } + + @Override + public Long genRollingConfigId() { + return null; + } + + @Override + public Long genStepInstanceRollingTaskId() { + return null; + } + + @Override + public Long genStepInstanceVariableId() { + return null; + } + + @Override + public Long genTaskInstanceVariableId() { + return null; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/DSLContextProviderFactory.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/DSLContextProviderFactory.java new file mode 100644 index 0000000000..4880cae908 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/DSLContextProviderFactory.java @@ -0,0 +1,77 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.dao.common; + +import com.tencent.bk.job.common.mysql.MySQLProperties; +import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; +import com.tencent.bk.job.common.mysql.dynamic.ds.DataSourceMode; +import com.tencent.bk.job.common.mysql.dynamic.ds.HorizontalShardingDSLContextProvider; +import com.tencent.bk.job.common.mysql.dynamic.ds.MigrateDynamicDSLContextProvider; +import com.tencent.bk.job.common.mysql.dynamic.ds.StandaloneDSLContextProvider; +import com.tencent.bk.job.common.mysql.dynamic.ds.VerticalShardingDSLContextProvider; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class DSLContextProviderFactory { + + private DSLContextProvider dslContextProvider; + + public DSLContextProviderFactory(StandaloneDSLContextProvider standaloneDSLContextProvider, + VerticalShardingDSLContextProvider verticalShardingDSLContextProvider, + HorizontalShardingDSLContextProvider horizontalShardingDSLContextProvider, + MigrateDynamicDSLContextProvider migrateDynamicDSLContextProvider, + MySQLProperties mySQLProperties) { + if (mySQLProperties.getMigration() != null && mySQLProperties.getMigration().isEnabled()) { + checkDSLContextProvider(migrateDynamicDSLContextProvider); + this.dslContextProvider = migrateDynamicDSLContextProvider; + log.info("Use MigrateDynamicDSLContextProvider"); + } else { + DataSourceMode dataSourceMode = DataSourceMode.valOf(mySQLProperties.getDataSourceMode()); + if (dataSourceMode == DataSourceMode.STANDALONE) { + checkDSLContextProvider(standaloneDSLContextProvider); + this.dslContextProvider = standaloneDSLContextProvider; + log.info("Use StandaloneDSLContextProvider"); + } else if (dataSourceMode == DataSourceMode.VERTICAL_SHARDING) { + checkDSLContextProvider(verticalShardingDSLContextProvider); + this.dslContextProvider = verticalShardingDSLContextProvider; + log.info("Use VerticalShardingDSLContextProvider"); + } else if (dataSourceMode == DataSourceMode.HORIZONTAL_SHARDING) { + checkDSLContextProvider(horizontalShardingDSLContextProvider); + this.dslContextProvider = horizontalShardingDSLContextProvider; + log.info("Use HorizontalShardingDSLContextProvider"); + } + } + } + + private void checkDSLContextProvider(DSLContextProvider dslContextProvider) { + if (dslContextProvider == null) { + throw new IllegalStateException("DSLContextProvider is empty"); + } + } + + public DSLContextProvider get() { + return this.dslContextProvider; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/IdGen.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/IdGen.java new file mode 100644 index 0000000000..4fd3c56256 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/IdGen.java @@ -0,0 +1,52 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.dao.common; + +/** + * ID 生成器 + */ +public interface IdGen { + Long genTaskInstanceId(); + + Long genStepInstanceId(); + + Long genGseTaskId(); + + Long genOperationLogId(); + + Long genFileSourceTaskLogId(); + + Long genGseFileExecuteObjTaskId(); + + Long genGseScriptExecuteObjTaskId(); + + Long genRollingConfigId(); + + Long genStepInstanceRollingTaskId(); + + Long genStepInstanceVariableId(); + + Long genTaskInstanceVariableId(); +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/JobExecuteVerticalShardingDSLContextProvider.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/JobExecuteVerticalShardingDSLContextProvider.java new file mode 100644 index 0000000000..3bcb54e4d7 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/JobExecuteVerticalShardingDSLContextProvider.java @@ -0,0 +1,75 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.dao.common; + +import com.tencent.bk.job.common.mysql.dynamic.ds.VerticalShardingDSLContextProvider; +import lombok.extern.slf4j.Slf4j; +import org.jooq.DSLContext; + +import java.util.HashMap; +import java.util.Map; + +/** + * 垂直分库 MySQL Jooq DSLContext 提供者 + */ +@Slf4j +public class JobExecuteVerticalShardingDSLContextProvider extends VerticalShardingDSLContextProvider { + + /** + * 垂直分库，固定分成 3 个分库，每个分库与表的关系也是固定的。 + * + * @param dslContextA 分库 a + * @param dslContextB 分库 b + * @param dslContextC 分库 c + */ + public JobExecuteVerticalShardingDSLContextProvider(DSLContext dslContextA, + DSLContext dslContextB, + DSLContext dslContextC) { + super(); + Map tableNameAndDslContextMap = new HashMap<>(); + tableNameAndDslContextMap.put("task_instance", dslContextA); + tableNameAndDslContextMap.put("task_instance_host", dslContextA); + tableNameAndDslContextMap.put("gse_script_agent_task", dslContextB); + tableNameAndDslContextMap.put("gse_script_execute_obj_task", dslContextB); + tableNameAndDslContextMap.put("dangerous_record", dslContextC); + tableNameAndDslContextMap.put("file_source_task_log", dslContextC); + tableNameAndDslContextMap.put("gse_file_agent_task", dslContextC); + tableNameAndDslContextMap.put("gse_file_execute_obj_task", dslContextC); + tableNameAndDslContextMap.put("gse_task", dslContextC); + tableNameAndDslContextMap.put("gse_task_ip_log", dslContextC); + tableNameAndDslContextMap.put("gse_task_log", dslContextC); + tableNameAndDslContextMap.put("operation_log", dslContextC); + tableNameAndDslContextMap.put("rolling_config", dslContextC); + tableNameAndDslContextMap.put("statistics", dslContextC); + tableNameAndDslContextMap.put("step_instance", dslContextC); + tableNameAndDslContextMap.put("step_instance_confirm", dslContextC); + tableNameAndDslContextMap.put("step_instance_file", dslContextC); + tableNameAndDslContextMap.put("step_instance_rolling_task", dslContextC); + tableNameAndDslContextMap.put("step_instance_script", dslContextC); + tableNameAndDslContextMap.put("step_instance_variable", dslContextC); + tableNameAndDslContextMap.put("task_instance_variable", dslContextC); + super.initTableNameAndDslContextMap(tableNameAndDslContextMap); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/PropBasedDynamicDataSource.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/PropBasedDynamicDataSource.java new file mode 100644 index 0000000000..c85afa8d20 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/PropBasedDynamicDataSource.java @@ -0,0 +1,87 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.dao.common; + +import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; +import com.tencent.bk.job.common.mysql.dynamic.ds.DataSourceMode; +import com.tencent.bk.job.common.mysql.dynamic.ds.DbOperationEnum; +import com.tencent.bk.job.common.mysql.dynamic.ds.MySQLOperation; +import com.tencent.bk.job.common.mysql.dynamic.ds.StandaloneDSLContextProvider; +import com.tencent.bk.job.common.mysql.dynamic.ds.VerticalShardingDSLContextProvider; +import com.tencent.bk.job.common.util.toggle.prop.PropToggleStore; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.redis.core.RedisTemplate; + +import java.util.HashMap; +import java.util.Map; + +/** + * 基于属性动态控制的数据源，可以根据属性值切换到不同的数据源 + */ +@Slf4j +public class PropBasedDynamicDataSource extends AbstractPropBasedDynamicComponent { + + public PropBasedDynamicDataSource( + StandaloneDSLContextProvider standaloneDSLContextProvider, + VerticalShardingDSLContextProvider verticalShardingDSLContextProvider, + RedisTemplate redisTemplate, + PropToggleStore propToggleStore) { + super(redisTemplate, propToggleStore, "execute_mysql"); + + Map candidateDSLContextProviders = new HashMap<>(); + if (standaloneDSLContextProvider != null) { + candidateDSLContextProviders.put(DataSourceMode.Constants.STANDALONE, standaloneDSLContextProvider); + } + if (verticalShardingDSLContextProvider != null) { + candidateDSLContextProviders.put(DataSourceMode.Constants.VERTICAL_SHARDING, + verticalShardingDSLContextProvider); + } + super.initCandidateComponents(candidateDSLContextProviders); + } + + /** + * 获取当前 DSLContextProvider。 + * 如果处于db 数据源切换中，会阻塞当前线程直到切换完成 + * + * @return DSLContextProvider + */ + public DSLContextProvider getCurrent(MySQLOperation op) { + if (op.op() == DbOperationEnum.WRITE) { + return getCurrent(true); + } else { + return getCurrent(false); + } + } + + @Override + protected DSLContextProvider getComponentByProp(Map candidateComponents, + String propValue) { + if (!DataSourceMode.checkValid(propValue)) { + log.error("Invalid target datasource mode : {}, skip migration", propValue); + return null; + } + return candidateComponents.get(propValue); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/PropBasedDynamicIdGen.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/PropBasedDynamicIdGen.java new file mode 100644 index 0000000000..20c29b2f4d --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/PropBasedDynamicIdGen.java @@ -0,0 +1,124 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.dao.common; + +import com.tencent.bk.job.common.mysql.dynamic.id.IdGenType; +import com.tencent.bk.job.common.util.toggle.prop.PropToggleStore; +import lombok.extern.slf4j.Slf4j; +import org.springframework.data.redis.core.RedisTemplate; + +import java.util.HashMap; +import java.util.Map; + +/** + * 基于属性动态控制的ID生成器 + */ +@Slf4j +public class PropBasedDynamicIdGen extends AbstractPropBasedDynamicComponent implements IdGen { + + public PropBasedDynamicIdGen( + AutoIncrementIdGen autoIncrementIdGenerator, + SegmentIdGen segmentIdGenerator, + RedisTemplate redisTemplate, + PropToggleStore propToggleStore) { + super(redisTemplate, propToggleStore, "execute_id_gen"); + Map candidateIdGens = new HashMap<>(); + if (autoIncrementIdGenerator != null) { + candidateIdGens.put(IdGenType.Constants.AUTO_INCREMENT, autoIncrementIdGenerator); + } + if (segmentIdGenerator != null) { + candidateIdGens.put(IdGenType.Constants.LEAF_SEGMENT, segmentIdGenerator); + } + super.initCandidateComponents(candidateIdGens); + } + + private IdGen currentIdGen() { + return getCurrent(true); + } + + @Override + protected IdGen getComponentByProp(Map candidateComponents, String propValue) { + if (!IdGenType.checkValid(propValue)) { + log.error("Invalid target IdGenType : {}, skip migration", propValue); + return null; + } + return candidateComponents.get(propValue); + } + + @Override + public Long genTaskInstanceId() { + return currentIdGen().genTaskInstanceId(); + } + + @Override + public Long genStepInstanceId() { + return currentIdGen().genStepInstanceId(); + } + + @Override + public Long genGseTaskId() { + return currentIdGen().genGseTaskId(); + } + + @Override + public Long genOperationLogId() { + return currentIdGen().genOperationLogId(); + } + + @Override + public Long genFileSourceTaskLogId() { + return currentIdGen().genFileSourceTaskLogId(); + } + + @Override + public Long genGseFileExecuteObjTaskId() { + return currentIdGen().genGseFileExecuteObjTaskId(); + } + + @Override + public Long genGseScriptExecuteObjTaskId() { + return currentIdGen().genGseScriptExecuteObjTaskId(); + } + + @Override + public Long genRollingConfigId() { + return currentIdGen().genRollingConfigId(); + } + + @Override + public Long genStepInstanceRollingTaskId() { + return currentIdGen().genStepInstanceRollingTaskId(); + } + + @Override + public Long genStepInstanceVariableId() { + return currentIdGen().genStepInstanceVariableId(); + } + + @Override + public Long genTaskInstanceVariableId() { + return currentIdGen().genTaskInstanceVariableId(); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/ReadWriteLockDbMigrateAspect.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/ReadWriteLockDbMigrateAspect.java new file mode 100644 index 0000000000..afc2d110cb --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/ReadWriteLockDbMigrateAspect.java @@ -0,0 +1,71 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.dao.common; + +import com.tencent.bk.job.common.mysql.dynamic.ds.MigrateDynamicDSLContextProvider; +import com.tencent.bk.job.common.mysql.dynamic.ds.MySQLOperation; +import lombok.extern.slf4j.Slf4j; +import org.aspectj.lang.ProceedingJoinPoint; +import org.aspectj.lang.annotation.Around; +import org.aspectj.lang.annotation.Aspect; +import org.aspectj.lang.annotation.Pointcut; +import org.aspectj.lang.reflect.MethodSignature; +import org.springframework.core.Ordered; +import org.springframework.core.annotation.Order; + +import java.lang.reflect.Method; + +@Aspect +@Slf4j +@Order(Ordered.HIGHEST_PRECEDENCE + 20) +public class ReadWriteLockDbMigrateAspect { + + private final MigrateDynamicDSLContextProvider migrateDynamicDSLContextProvider; + + private final PropBasedDynamicDataSource propBasedDynamicDataSource; + + + public ReadWriteLockDbMigrateAspect(MigrateDynamicDSLContextProvider migrateDynamicDSLContextProvider, + PropBasedDynamicDataSource propBasedDynamicDataSource) { + this.migrateDynamicDSLContextProvider = migrateDynamicDSLContextProvider; + this.propBasedDynamicDataSource = propBasedDynamicDataSource; + } + + @Pointcut("@annotation(com.tencent.bk.job.common.mysql.dynamic.ds.MySQLOperation)") + public void mysqlOperation() { + } + + @Around("mysqlOperation()") + public Object around(ProceedingJoinPoint pjp) throws Throwable { + try { + Method method = ((MethodSignature) pjp.getSignature()).getMethod(); + MySQLOperation operation = method.getAnnotation(MySQLOperation.class); + migrateDynamicDSLContextProvider.setProvider(propBasedDynamicDataSource.getCurrent(operation)); + return pjp.proceed(); + } finally { + migrateDynamicDSLContextProvider.setProvider(null); + } + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/SegmentIdGen.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/SegmentIdGen.java new file mode 100644 index 0000000000..3c5bf202b7 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/common/SegmentIdGen.java @@ -0,0 +1,85 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.dao.common; + +import com.tencent.bk.job.common.sharding.mysql.SegmentIdKeys; +import com.tencent.devops.leaf.service.SegmentService; + +public class SegmentIdGen implements IdGen { + + private final SegmentService segmentService; + + public SegmentIdGen(SegmentService segmentServices) { + this.segmentService = segmentServices; + } + + private Long gen(String key) { + return segmentService.getId(key).getId(); + } + + public Long genTaskInstanceId() { + return gen(SegmentIdKeys.KEY_JOB_EXECUTE_TASK_INSTANCE); + } + + public Long genStepInstanceId() { + return gen(SegmentIdKeys.KEY_JOB_EXECUTE_STEP_INSTANCE); + } + + public Long genGseTaskId() { + return gen(SegmentIdKeys.KEY_JOB_EXECUTE_GSE_TASK); + } + + public Long genOperationLogId() { + return gen(SegmentIdKeys.KEY_JOB_EXECUTE_OPERATION_LOG); + } + + public Long genFileSourceTaskLogId() { + return gen(SegmentIdKeys.KEY_JOB_EXECUTE_FILE_SOURCE_TASK_LOG); + } + + public Long genGseFileExecuteObjTaskId() { + return gen(SegmentIdKeys.KEY_JOB_EXECUTE_GSE_FILE_EXECUTE_OBJ_TASK); + } + + public Long genGseScriptExecuteObjTaskId() { + return gen(SegmentIdKeys.KEY_JOB_EXECUTE_GSE_SCRIPT_EXECUTE_OBJ_TASK); + } + + public Long genRollingConfigId() { + return gen(SegmentIdKeys.KEY_JOB_EXECUTE_ROLLING_CONFIG); + } + + public Long genStepInstanceRollingTaskId() { + return gen(SegmentIdKeys.KEY_JOB_EXECUTE_STEP_INSTANCE_ROLLING_TASK); + } + + public Long genStepInstanceVariableId() { + return gen(SegmentIdKeys.KEY_JOB_EXECUTE_STEP_INSTANCE_VARIABLE); + } + + public Long genTaskInstanceVariableId() { + return gen(SegmentIdKeys.KEY_JOB_EXECUTE_TASK_INSTANCE_VARIABLE); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/BaseDAO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/BaseDAO.java new file mode 100644 index 0000000000..3d532190ec --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/BaseDAO.java @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.dao.impl; + +import com.tencent.bk.job.common.mysql.dynamic.ds.DSLContextProvider; +import com.tencent.bk.job.execute.dao.common.DSLContextProviderFactory; +import org.jooq.DSLContext; + +/** + * DAO 层基础类 + */ +public class BaseDAO { + private final DSLContextProvider dslContextProvider; + + private String tableName; + + public BaseDAO(DSLContextProviderFactory dslContextProviderFactory, String tableName) { + this.dslContextProvider = dslContextProviderFactory.get(); + this.tableName = tableName; + } + + protected DSLContext dsl() { + return this.dslContextProvider.get(tableName); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/DangerousRecordDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/DangerousRecordDAOImpl.java index 197276e50e..31de7f1098 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/DangerousRecordDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/DangerousRecordDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,19 +27,20 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; +import com.tencent.bk.job.common.mysql.dynamic.ds.DbOperationEnum; +import com.tencent.bk.job.common.mysql.dynamic.ds.MySQLOperation; +import com.tencent.bk.job.common.mysql.jooq.JooqDataTypeUtil; import com.tencent.bk.job.common.util.json.JsonUtils; -import com.tencent.bk.job.execute.common.util.JooqDataTypeUtil; import com.tencent.bk.job.execute.dao.DangerousRecordDAO; +import com.tencent.bk.job.execute.dao.common.DSLContextProviderFactory; import com.tencent.bk.job.execute.model.DangerousRecordDTO; import com.tencent.bk.job.execute.model.ScriptCheckResultDTO; +import com.tencent.bk.job.execute.model.tables.DangerousRecord; import org.apache.commons.lang3.StringUtils; import org.jooq.Condition; -import org.jooq.DSLContext; import org.jooq.Record; import org.jooq.Result; -import org.jooq.generated.tables.DangerousRecord; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; import java.util.ArrayList; @@ -47,25 +48,37 @@ import java.util.Map; @Repository -public class DangerousRecordDAOImpl implements DangerousRecordDAO { +public class DangerousRecordDAOImpl extends BaseDAO implements DangerousRecordDAO { private static final DangerousRecord T = DangerousRecord.DANGEROUS_RECORD; - private final DSLContext ctx; @Autowired - public DangerousRecordDAOImpl(@Qualifier("job-execute-dsl-context") DSLContext ctx) { - this.ctx = ctx; + public DangerousRecordDAOImpl(DSLContextProviderFactory dslContextProviderFactory) { + super(dslContextProviderFactory, T.getName()); } @Override + @MySQLOperation(table = "dangerous_record", op = DbOperationEnum.READ) public PageData listPageDangerousRecord(DangerousRecordDTO query, BaseSearchCondition baseSearchCondition) { int count = getPageDangerousRecordCount(query, baseSearchCondition); int start = baseSearchCondition.getStartOrDefault(0); int length = baseSearchCondition.getLengthOrDefault(10); - Result result = ctx.select(T.ID, T.RULE_ID, T.RULE_EXPRESSION, T.APP_ID, T.APP_NAME, T.OPERATOR, - T.SCRIPT_LANGUAGE, - T.SCRIPT_CONTENT, T.CREATE_TIME, T.STARTUP_MODE, T.CLIENT, T.ACTION, T.CHECK_RESULT, T.EXT_DATA) + Result result = dsl().select( + T.ID, + T.RULE_ID, + T.RULE_EXPRESSION, + T.APP_ID, + T.APP_NAME, + T.OPERATOR, + T.SCRIPT_LANGUAGE, + T.SCRIPT_CONTENT, + T.CREATE_TIME, + T.STARTUP_MODE, + T.CLIENT, + T.ACTION, + T.CHECK_RESULT, + T.EXT_DATA) .from(T) .where(buildSearchCondition(query, baseSearchCondition)) .orderBy(T.ID.desc()) @@ -77,7 +90,7 @@ public PageData listPageDangerousRecord(DangerousRecordDTO q @SuppressWarnings("all") private int getPageDangerousRecordCount(DangerousRecordDTO query, BaseSearchCondition baseSearchCondition) { List conditions = buildSearchCondition(query, baseSearchCondition); - return ctx.selectCount().from(T).where(conditions).fetchOne(0, Integer.class); + return dsl().selectCount().from(T).where(conditions).fetchOne(0, Integer.class); } private List buildSearchCondition(DangerousRecordDTO query, BaseSearchCondition baseSearchCondition) { @@ -145,7 +158,7 @@ private DangerousRecordDTO extractInfo(Record record) { dangerousRecord.setScriptContent(record.get(T.SCRIPT_CONTENT)); dangerousRecord.setCheckResult(JsonUtils.fromJson(record.get(T.CHECK_RESULT), new TypeReference() { - })); + })); String extData = record.get(T.EXT_DATA); if (StringUtils.isNotEmpty(extData)) { dangerousRecord.setExtData(JsonUtils.fromJson(extData, new TypeReference>() { @@ -155,9 +168,11 @@ private DangerousRecordDTO extractInfo(Record record) { } @Override + @MySQLOperation(table = "dangerous_record", op = DbOperationEnum.WRITE) public boolean saveDangerousRecord(DangerousRecordDTO record) { - int count = ctx.insertInto(T, T.RULE_ID, T.RULE_EXPRESSION, T.APP_ID, T.APP_NAME, T.OPERATOR, T.SCRIPT_LANGUAGE, - T.SCRIPT_CONTENT, T.CREATE_TIME, T.STARTUP_MODE, T.CLIENT, T.ACTION, T.CHECK_RESULT, T.EXT_DATA) + int count = dsl().insertInto(T, T.RULE_ID, T.RULE_EXPRESSION, T.APP_ID, T.APP_NAME, + T.OPERATOR, T.SCRIPT_LANGUAGE, + T.SCRIPT_CONTENT, T.CREATE_TIME, T.STARTUP_MODE, T.CLIENT, T.ACTION, T.CHECK_RESULT, T.EXT_DATA) .values(record.getRuleId(), record.getRuleExpression(), record.getAppId(), diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImpl.java index 80c3038a14..42001dc4f0 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileAgentTaskDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,16 +24,22 @@ package com.tencent.bk.job.execute.dao.impl; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.CompatibleType; import com.tencent.bk.job.common.constant.Order; +import com.tencent.bk.job.common.mysql.dynamic.ds.DbOperationEnum; +import com.tencent.bk.job.common.mysql.dynamic.ds.MySQLOperation; import com.tencent.bk.job.execute.dao.FileAgentTaskDAO; -import com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum; -import com.tencent.bk.job.execute.model.AgentTaskDTO; -import com.tencent.bk.job.execute.model.AgentTaskResultGroupBaseDTO; +import com.tencent.bk.job.execute.dao.common.DSLContextProviderFactory; +import com.tencent.bk.job.execute.engine.consts.ExecuteObjectTaskStatusEnum; +import com.tencent.bk.job.execute.model.ExecuteObjectTask; +import com.tencent.bk.job.execute.model.ResultGroupBaseDTO; +import com.tencent.bk.job.execute.model.tables.GseFileAgentTask; +import com.tencent.bk.job.execute.model.tables.records.GseFileAgentTaskRecord; import com.tencent.bk.job.logsvr.consts.FileTaskModeEnum; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.jooq.Condition; -import org.jooq.DSLContext; import org.jooq.OrderField; import org.jooq.Record; import org.jooq.Result; @@ -44,10 +50,7 @@ import org.jooq.UpdateConditionStep; import org.jooq.UpdateSetMoreStep; import org.jooq.UpdateSetStep; -import org.jooq.generated.tables.GseFileAgentTask; -import org.jooq.generated.tables.records.GseFileAgentTaskRecord; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; import java.util.ArrayList; @@ -59,10 +62,13 @@ import static org.jooq.impl.DSL.count; @Repository -public class FileAgentTaskDAOImpl implements FileAgentTaskDAO { +@Deprecated +@CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA) +public class FileAgentTaskDAOImpl extends BaseDAO implements FileAgentTaskDAO { private static final GseFileAgentTask T_GSE_FILE_AGENT_TASK = GseFileAgentTask.GSE_FILE_AGENT_TASK; private static final TableField[] ALL_FIELDS = { + T_GSE_FILE_AGENT_TASK.TASK_INSTANCE_ID, T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID, T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT, T_GSE_FILE_AGENT_TASK.ACTUAL_EXECUTE_COUNT, @@ -78,89 +84,107 @@ public class FileAgentTaskDAOImpl implements FileAgentTaskDAO { T_GSE_FILE_AGENT_TASK.ERROR_CODE }; - private final DSLContext CTX; - @Autowired - public FileAgentTaskDAOImpl(@Qualifier("job-execute-dsl-context") DSLContext CTX) { - this.CTX = CTX; + public FileAgentTaskDAOImpl(DSLContextProviderFactory dslContextProviderFactory) { + super(dslContextProviderFactory, T_GSE_FILE_AGENT_TASK.getName()); } @Override - public void batchSaveAgentTasks(Collection agentTasks) { - String sql = "insert into gse_file_agent_task (step_instance_id, execute_count, actual_execute_count, batch," - + "mode, host_id, agent_id ,gse_task_id,status, start_time, end_time, total_time, error_code)" - + " values (?,?,?,?,?,?,?,?,?,?,?,?,?)"; - Object[][] params = new Object[agentTasks.size()][13]; + @MySQLOperation(table = "gse_file_agent_task", op = DbOperationEnum.WRITE) + public void batchSaveAgentTasks(Collection agentTasks) { + String sql = "insert into gse_file_agent_task (task_instance_id, step_instance_id, execute_count, " + + "actual_execute_count, batch, mode, host_id, agent_id ,gse_task_id,status, start_time, " + + "end_time, total_time, error_code) " + + "values (?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; + Object[][] params = new Object[agentTasks.size()][14]; int batchCount = 0; - for (AgentTaskDTO agentTask : agentTasks) { - Object[] param = new Object[13]; - param[0] = agentTask.getStepInstanceId(); - param[1] = agentTask.getExecuteCount(); - param[2] = agentTask.getActualExecuteCount(); - param[3] = agentTask.getBatch(); - param[4] = agentTask.getFileTaskMode().getValue(); - param[5] = agentTask.getHostId(); - param[6] = agentTask.getAgentId(); - param[7] = agentTask.getGseTaskId(); - param[8] = agentTask.getStatus().getValue(); - param[9] = agentTask.getStartTime(); - param[10] = agentTask.getEndTime(); - param[11] = agentTask.getTotalTime(); - param[12] = agentTask.getErrorCode(); + for (ExecuteObjectTask agentTask : agentTasks) { + Object[] param = new Object[14]; + param[0] = agentTask.getTaskInstanceId(); + param[1] = agentTask.getStepInstanceId(); + param[2] = agentTask.getExecuteCount(); + param[3] = agentTask.getActualExecuteCount(); + param[4] = agentTask.getBatch(); + param[5] = agentTask.getFileTaskMode().getValue(); + param[6] = agentTask.getHostId(); + param[7] = agentTask.getAgentId() == null ? "" : agentTask.getAgentId(); + param[8] = agentTask.getGseTaskId(); + param[9] = agentTask.getStatus().getValue(); + param[10] = agentTask.getStartTime(); + param[11] = agentTask.getEndTime(); + param[12] = agentTask.getTotalTime(); + param[13] = agentTask.getErrorCode(); params[batchCount++] = param; } - CTX.batch(sql, params).execute(); + dsl().batch(sql, params).execute(); } @Override - public void batchUpdateAgentTasks(Collection agentTasks) { + @MySQLOperation(table = "gse_file_agent_task", op = DbOperationEnum.WRITE) + public void batchUpdateAgentTasks(Collection agentTasks) { if (CollectionUtils.isEmpty(agentTasks)) { return; } String sql = "update gse_file_agent_task set gse_task_id = ?, status = ?, start_time = ?, end_time = ?" + ", total_time = ?, error_code = ?" - + " where step_instance_id = ? and execute_count = ? and batch = ? and mode = ? and host_id = ?"; - Object[][] params = new Object[agentTasks.size()][11]; + + " where task_instance_id = ? and step_instance_id = ? and execute_count = ?" + + " and batch = ? and mode = ? and host_id = ?"; + Object[][] params = new Object[agentTasks.size()][12]; int batchCount = 0; - for (AgentTaskDTO agentTask : agentTasks) { - Object[] param = new Object[11]; + for (ExecuteObjectTask agentTask : agentTasks) { + Object[] param = new Object[12]; param[0] = agentTask.getGseTaskId(); param[1] = agentTask.getStatus().getValue(); param[2] = agentTask.getStartTime(); param[3] = agentTask.getEndTime(); param[4] = agentTask.getTotalTime(); param[5] = agentTask.getErrorCode(); - param[6] = agentTask.getStepInstanceId(); - param[7] = agentTask.getExecuteCount(); - param[8] = agentTask.getBatch(); - param[9] = agentTask.getFileTaskMode().getValue(); - param[10] = agentTask.getHostId(); + param[6] = agentTask.getTaskInstanceId(); + param[7] = agentTask.getStepInstanceId(); + param[8] = agentTask.getExecuteCount(); + param[9] = agentTask.getBatch(); + param[10] = agentTask.getFileTaskMode().getValue(); + param[11] = agentTask.getHostId(); params[batchCount++] = param; } - CTX.batch(sql, params).execute(); + dsl().batch(sql, params).execute(); } @Override - public int getSuccessAgentTaskCount(long stepInstanceId, int executeCount) { - Integer count = CTX.selectCount() + @MySQLOperation(table = "gse_file_agent_task", op = DbOperationEnum.READ) + public int getSuccessAgentTaskCount(Long taskInstanceId, long stepInstanceId, int executeCount) { + Integer count = dsl().selectCount() .from(T_GSE_FILE_AGENT_TASK) - .where(T_GSE_FILE_AGENT_TASK.STATUS.in(AgentTaskStatusEnum.LAST_SUCCESS.getValue(), - AgentTaskStatusEnum.SUCCESS.getValue())) + .where(T_GSE_FILE_AGENT_TASK.STATUS.in(ExecuteObjectTaskStatusEnum.LAST_SUCCESS.getValue(), + ExecuteObjectTaskStatusEnum.SUCCESS.getValue())) .and(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq((short) executeCount)) .and(T_GSE_FILE_AGENT_TASK.MODE.eq(FileTaskModeEnum.DOWNLOAD.getValue().byteValue())) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)) .fetchOne(0, Integer.class); return count == null ? 0 : count; } + private Condition buildTaskInstanceIdQueryCondition(Long taskInstanceId) { + return TaskInstanceIdDynamicCondition.build( + taskInstanceId, + T_GSE_FILE_AGENT_TASK.TASK_INSTANCE_ID::eq + ); + } + @Override - public List listResultGroups(long stepInstanceId, int executeCount, Integer batch) { + @MySQLOperation(table = "gse_file_agent_task", op = DbOperationEnum.READ) + public List listResultGroups(Long taskInstanceId, + long stepInstanceId, + int executeCount, + Integer batch) { SelectConditionStep selectConditionStep = - CTX.select(T_GSE_FILE_AGENT_TASK.STATUS, count().as("ip_count")) + dsl().select(T_GSE_FILE_AGENT_TASK.STATUS, count().as("ip_count")) .from(T_GSE_FILE_AGENT_TASK) .where(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq((short) executeCount)) - .and(T_GSE_FILE_AGENT_TASK.MODE.eq(FileTaskModeEnum.DOWNLOAD.getValue().byteValue())); + .and(T_GSE_FILE_AGENT_TASK.MODE.eq(FileTaskModeEnum.DOWNLOAD.getValue().byteValue())) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)); if (batch != null && batch > 0) { selectConditionStep.and(T_GSE_FILE_AGENT_TASK.BATCH.eq(batch.shortValue())); } @@ -169,35 +193,38 @@ public List listResultGroups(long stepInstanceId, i .orderBy(T_GSE_FILE_AGENT_TASK.STATUS.asc()) .fetch(); - List resultGroups = new ArrayList<>(); + List resultGroups = new ArrayList<>(); result.forEach(record -> { - AgentTaskResultGroupBaseDTO resultGroup = new AgentTaskResultGroupBaseDTO(); + ResultGroupBaseDTO resultGroup = new ResultGroupBaseDTO(); resultGroup.setStatus(record.get(T_GSE_FILE_AGENT_TASK.STATUS)); resultGroup.setTag(""); Object ipCount = record.get("ip_count"); - resultGroup.setTotalAgentTasks(ipCount == null ? 0 : (int) ipCount); + resultGroup.setTotal(ipCount == null ? 0 : (int) ipCount); resultGroups.add(resultGroup); }); return resultGroups; } @Override - public List listAgentTaskByResultGroup(Long stepInstanceId, - Integer executeCount, - Integer batch, - Integer status) { - SelectConditionStep selectConditionStep = CTX.select(ALL_FIELDS) + @MySQLOperation(table = "gse_file_agent_task", op = DbOperationEnum.READ) + public List listAgentTaskByResultGroup(Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch, + Integer status) { + SelectConditionStep selectConditionStep = dsl().select(ALL_FIELDS) .from(T_GSE_FILE_AGENT_TASK) .where(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())) .and(T_GSE_FILE_AGENT_TASK.STATUS.eq(status)) - .and(T_GSE_FILE_AGENT_TASK.MODE.eq(FileTaskModeEnum.DOWNLOAD.getValue().byteValue())); + .and(T_GSE_FILE_AGENT_TASK.MODE.eq(FileTaskModeEnum.DOWNLOAD.getValue().byteValue())) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)); if (batch != null && batch > 0) { selectConditionStep.and(T_GSE_FILE_AGENT_TASK.BATCH.eq(batch.shortValue())); } Result result = selectConditionStep.fetch(); - List agentTasks = new ArrayList<>(); + List agentTasks = new ArrayList<>(); if (result.size() > 0) { result.forEach(record -> agentTasks.add(extract(record))); } @@ -205,20 +232,23 @@ public List listAgentTaskByResultGroup(Long stepInstanceId, } @Override - public List listAgentTaskByResultGroup(Long stepInstanceId, - Integer executeCount, - Integer batch, - Integer status, - Integer limit, - String orderField, - Order order) { + @MySQLOperation(table = "gse_file_agent_task", op = DbOperationEnum.READ) + public List listAgentTaskByResultGroup(Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch, + Integer status, + Integer limit, + String orderField, + Order order) { List conditions = new ArrayList<>(); conditions.add(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)); conditions.add(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())); conditions.add(T_GSE_FILE_AGENT_TASK.STATUS.eq(status)); conditions.add(T_GSE_FILE_AGENT_TASK.MODE.eq(FileTaskModeEnum.DOWNLOAD.getValue().byteValue())); + conditions.add(buildTaskInstanceIdQueryCondition(taskInstanceId)); - SelectConditionStep select = CTX.select(ALL_FIELDS) + SelectConditionStep select = dsl().select(ALL_FIELDS) .from(T_GSE_FILE_AGENT_TASK) .where(conditions); @@ -241,7 +271,7 @@ public List listAgentTaskByResultGroup(Long stepInstanceId, } } - List agentTasks = new ArrayList<>(); + List agentTasks = new ArrayList<>(); Result result; if (selectLimitPercentStep != null) { result = selectLimitPercentStep.fetch(); @@ -272,14 +302,17 @@ private OrderField buildOrderField(String field, Order order) { } @Override - public List listAgentTasks(Long stepInstanceId, - Integer executeCount, - Integer batch, - FileTaskModeEnum fileTaskMode) { - SelectConditionStep selectConditionStep = CTX.select(ALL_FIELDS) + @MySQLOperation(table = "gse_file_agent_task", op = DbOperationEnum.READ) + public List listAgentTasks(Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch, + FileTaskModeEnum fileTaskMode) { + SelectConditionStep selectConditionStep = dsl().select(ALL_FIELDS) .from(T_GSE_FILE_AGENT_TASK) .where(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())); + .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)); if (batch != null && batch > 0) { selectConditionStep.and(T_GSE_FILE_AGENT_TASK.BATCH.eq(batch.shortValue())); } @@ -287,7 +320,7 @@ public List listAgentTasks(Long stepInstanceId, selectConditionStep.and(T_GSE_FILE_AGENT_TASK.MODE.eq(fileTaskMode.getValue().byteValue())); } Result result = selectConditionStep.fetch(); - List agentTaskList = new ArrayList<>(); + List agentTaskList = new ArrayList<>(); if (result.size() != 0) { result.map(record -> { agentTaskList.add(extract(record)); @@ -297,11 +330,12 @@ public List listAgentTasks(Long stepInstanceId, return agentTaskList; } - private AgentTaskDTO extract(Record record) { + private ExecuteObjectTask extract(Record record) { if (record == null) { return null; } - AgentTaskDTO agentTask = new AgentTaskDTO(); + ExecuteObjectTask agentTask = new ExecuteObjectTask(); + agentTask.setTaskInstanceId(record.get(T_GSE_FILE_AGENT_TASK.TASK_INSTANCE_ID)); agentTask.setStepInstanceId(record.get(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID)); agentTask.setExecuteCount(record.get(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT)); Short actualExecuteCount = record.get(T_GSE_FILE_AGENT_TASK.ACTUAL_EXECUTE_COUNT); @@ -311,7 +345,7 @@ private AgentTaskDTO extract(Record record) { agentTask.setHostId(record.get(T_GSE_FILE_AGENT_TASK.HOST_ID)); agentTask.setAgentId(record.get(T_GSE_FILE_AGENT_TASK.AGENT_ID)); agentTask.setGseTaskId(record.get(T_GSE_FILE_AGENT_TASK.GSE_TASK_ID)); - agentTask.setStatus(AgentTaskStatusEnum.valueOf(record.get(T_GSE_FILE_AGENT_TASK.STATUS))); + agentTask.setStatus(ExecuteObjectTaskStatusEnum.valOf(record.get(T_GSE_FILE_AGENT_TASK.STATUS))); agentTask.setStartTime(record.get(T_GSE_FILE_AGENT_TASK.START_TIME)); agentTask.setEndTime(record.get(T_GSE_FILE_AGENT_TASK.END_TIME)); agentTask.setTotalTime(record.get(T_GSE_FILE_AGENT_TASK.TOTAL_TIME)); @@ -320,16 +354,18 @@ private AgentTaskDTO extract(Record record) { } @Override - public List listAgentTasksByGseTaskId(Long gseTaskId) { + @MySQLOperation(table = "gse_file_agent_task", op = DbOperationEnum.READ) + public List listAgentTasksByGseTaskId(Long taskInstanceId, Long gseTaskId) { if (gseTaskId == null || gseTaskId <= 0) { return Collections.emptyList(); } - List agentTaskList = new ArrayList<>(); + List agentTaskList = new ArrayList<>(); - Result result = CTX.select(ALL_FIELDS) + Result result = dsl().select(ALL_FIELDS) .from(T_GSE_FILE_AGENT_TASK) .where(T_GSE_FILE_AGENT_TASK.GSE_TASK_ID.eq(gseTaskId)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)) .fetch(); if (result.size() > 0) { result.forEach(record -> agentTaskList.add(extract(record))); @@ -338,31 +374,39 @@ public List listAgentTasksByGseTaskId(Long gseTaskId) { } @Override - public AgentTaskDTO getAgentTaskByHostId(Long stepInstanceId, Integer executeCount, Integer batch, - FileTaskModeEnum mode, long hostId) { - Record record = CTX.select(ALL_FIELDS) - .from(T_GSE_FILE_AGENT_TASK) - .where(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())) - .and(T_GSE_FILE_AGENT_TASK.BATCH.eq(batch == null ? 0 : batch.shortValue())) - .and(T_GSE_FILE_AGENT_TASK.MODE.eq(mode.getValue().byteValue())) - .and(T_GSE_FILE_AGENT_TASK.HOST_ID.eq(hostId)) - .fetchOne(); + @MySQLOperation(table = "gse_file_agent_task", op = DbOperationEnum.READ) + public ExecuteObjectTask getAgentTaskByHostId(Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch, + FileTaskModeEnum mode, + long hostId) { + SelectConditionStep selectConditionStep = + dsl().select(ALL_FIELDS) + .from(T_GSE_FILE_AGENT_TASK) + .where(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())) + .and(T_GSE_FILE_AGENT_TASK.MODE.eq(mode.getValue().byteValue())) + .and(T_GSE_FILE_AGENT_TASK.HOST_ID.eq(hostId)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)); + if (batch != null && batch > 0) { + // 滚动执行批次，传入null或者0将忽略该参数 + selectConditionStep.and(T_GSE_FILE_AGENT_TASK.BATCH.eq(batch.shortValue())); + } + selectConditionStep.limit(1); + Record record = selectConditionStep.fetchOne(); return extract(record); } @Override - public boolean isStepInstanceRecordExist(long stepInstanceId) { - return CTX.fetchExists(T_GSE_FILE_AGENT_TASK, T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)); - } - - @Override - public void updateAgentTaskFields(long stepInstanceId, + @MySQLOperation(table = "gse_file_agent_task", op = DbOperationEnum.WRITE) + public void updateAgentTaskFields(Long taskInstanceId, + long stepInstanceId, int executeCount, Integer batch, Integer actualExecuteCount, Long gseTaskId) { - UpdateSetStep updateSetStep = CTX.update(T_GSE_FILE_AGENT_TASK); + UpdateSetStep updateSetStep = dsl().update(T_GSE_FILE_AGENT_TASK); boolean needUpdate = false; if (actualExecuteCount != null) { updateSetStep = updateSetStep.set(T_GSE_FILE_AGENT_TASK.ACTUAL_EXECUTE_COUNT, @@ -384,7 +428,8 @@ public void updateAgentTaskFields(long stepInstanceId, UpdateConditionStep updateConditionStep = updateSetMoreStep .where(T_GSE_FILE_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq((short) executeCount)); + .and(T_GSE_FILE_AGENT_TASK.EXECUTE_COUNT.eq((short) executeCount)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)); if (batch != null) { updateConditionStep.and(T_GSE_FILE_AGENT_TASK.BATCH.eq(batch.shortValue())); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileExecuteObjectTaskDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileExecuteObjectTaskDAOImpl.java new file mode 100644 index 0000000000..e024d324f0 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileExecuteObjectTaskDAOImpl.java @@ -0,0 +1,447 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.dao.impl; + +import com.tencent.bk.job.common.constant.ExecuteObjectTypeEnum; +import com.tencent.bk.job.common.constant.Order; +import com.tencent.bk.job.common.mysql.dynamic.ds.DbOperationEnum; +import com.tencent.bk.job.common.mysql.dynamic.ds.MySQLOperation; +import com.tencent.bk.job.execute.dao.FileExecuteObjectTaskDAO; +import com.tencent.bk.job.execute.dao.common.DSLContextProviderFactory; +import com.tencent.bk.job.execute.engine.consts.ExecuteObjectTaskStatusEnum; +import com.tencent.bk.job.execute.model.ExecuteObjectTask; +import com.tencent.bk.job.execute.model.ResultGroupBaseDTO; +import com.tencent.bk.job.execute.model.tables.GseFileExecuteObjTask; +import com.tencent.bk.job.execute.model.tables.records.GseFileExecuteObjTaskRecord; +import com.tencent.bk.job.logsvr.consts.FileTaskModeEnum; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.jooq.Condition; +import org.jooq.OrderField; +import org.jooq.Record; +import org.jooq.Result; +import org.jooq.SelectConditionStep; +import org.jooq.SelectLimitPercentStep; +import org.jooq.SelectSeekStep1; +import org.jooq.TableField; +import org.jooq.UpdateConditionStep; +import org.jooq.UpdateSetMoreStep; +import org.jooq.UpdateSetStep; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Repository; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +import static com.tencent.bk.job.common.constant.Order.DESCENDING; +import static org.jooq.impl.DSL.count; + +@Repository +public class FileExecuteObjectTaskDAOImpl extends BaseDAO implements FileExecuteObjectTaskDAO { + + private static final GseFileExecuteObjTask T = GseFileExecuteObjTask.GSE_FILE_EXECUTE_OBJ_TASK; + private static final TableField[] ALL_FIELDS = { + T.TASK_INSTANCE_ID, + T.STEP_INSTANCE_ID, + T.EXECUTE_COUNT, + T.ACTUAL_EXECUTE_COUNT, + T.BATCH, + T.MODE, + T.EXECUTE_OBJ_TYPE, + T.EXECUTE_OBJ_ID, + T.GSE_TASK_ID, + T.STATUS, + T.START_TIME, + T.END_TIME, + T.TOTAL_TIME, + T.ERROR_CODE + }; + + public static final String BATCH_INSERT_SQL = + "insert into gse_file_execute_obj_task (id,task_instance_id,step_instance_id,execute_count," + + "actual_execute_count, " + + "batch,mode,execute_obj_type,execute_obj_id,gse_task_id,status,start_time,end_time,total_time,error_code)" + + " values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; + public static final String BATCH_UPDATE_SQL = "update gse_file_execute_obj_task set gse_task_id = ?, status = ?," + + "start_time = ?, end_time = ? , total_time = ?, error_code = ?" + + " where step_instance_id = ? and execute_count = ? and batch = ? and mode = ? and execute_obj_id = ?"; + + @Autowired + public FileExecuteObjectTaskDAOImpl(DSLContextProviderFactory dslContextProviderFactory) { + super(dslContextProviderFactory, T.getName()); + } + + @Override + @MySQLOperation(table = "gse_file_execute_obj_task", op = DbOperationEnum.WRITE) + public void batchSaveTasks(Collection tasks) { + Object[][] params = new Object[tasks.size()][15]; + int batchCount = 0; + for (ExecuteObjectTask task : tasks) { + Object[] param = new Object[15]; + param[0] = task.getId(); + param[1] = task.getTaskInstanceId(); + param[2] = task.getStepInstanceId(); + param[3] = task.getExecuteCount(); + param[4] = task.getActualExecuteCount(); + param[5] = task.getBatch(); + param[6] = task.getFileTaskMode().getValue(); + param[7] = task.getExecuteObjectType().getValue(); + param[8] = task.getExecuteObjectId(); + param[9] = task.getGseTaskId(); + param[10] = task.getStatus().getValue(); + param[11] = task.getStartTime(); + param[12] = task.getEndTime(); + param[13] = task.getTotalTime(); + param[14] = task.getErrorCode(); + params[batchCount++] = param; + } + dsl().batch(BATCH_INSERT_SQL, params).execute(); + } + + @Override + @MySQLOperation(table = "gse_file_execute_obj_task", op = DbOperationEnum.WRITE) + public void batchUpdateTasks(Collection tasks) { + if (CollectionUtils.isEmpty(tasks)) { + return; + } + Object[][] params = new Object[tasks.size()][11]; + int batchCount = 0; + for (ExecuteObjectTask task : tasks) { + Object[] param = new Object[11]; + param[0] = task.getGseTaskId(); + param[1] = task.getStatus().getValue(); + param[2] = task.getStartTime(); + param[3] = task.getEndTime(); + param[4] = task.getTotalTime(); + param[5] = task.getErrorCode(); + param[6] = task.getStepInstanceId(); + param[7] = task.getExecuteCount(); + param[8] = task.getBatch(); + param[9] = task.getFileTaskMode().getValue(); + param[10] = task.getExecuteObjectId(); + params[batchCount++] = param; + } + dsl().batch(BATCH_UPDATE_SQL, params).execute(); + } + + @Override + @MySQLOperation(table = "gse_file_execute_obj_task", op = DbOperationEnum.READ) + public int getSuccessTaskCount(Long taskInstanceId, long stepInstanceId, int executeCount) { + Integer count = dsl().selectCount() + .from(T) + .where(T.STATUS.in(ExecuteObjectTaskStatusEnum.LAST_SUCCESS.getValue(), + ExecuteObjectTaskStatusEnum.SUCCESS.getValue())) + .and(T.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(T.EXECUTE_COUNT.eq((short) executeCount)) + .and(T.MODE.eq(FileTaskModeEnum.DOWNLOAD.getValue().byteValue())) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)) + .fetchOne(0, Integer.class); + return count == null ? 0 : count; + } + + private Condition buildTaskInstanceIdQueryCondition(Long taskInstanceId) { + return TaskInstanceIdDynamicCondition.build( + taskInstanceId, + T.TASK_INSTANCE_ID::eq + ); + } + + @Override + @MySQLOperation(table = "gse_file_execute_obj_task", op = DbOperationEnum.READ) + public List listResultGroups(Long taskInstanceId, + long stepInstanceId, + int executeCount, + Integer batch) { + SelectConditionStep selectConditionStep = + dsl().select(T.STATUS, count().as("ip_count")) + .from(T) + .where(T.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)) + .and(T.EXECUTE_COUNT.eq((short) executeCount)) + .and(T.MODE.eq(FileTaskModeEnum.DOWNLOAD.getValue().byteValue())); + if (batch != null && batch > 0) { + selectConditionStep.and(T.BATCH.eq(batch.shortValue())); + } + + Result result = selectConditionStep.groupBy(T.STATUS) + .orderBy(T.STATUS.asc()) + .fetch(); + + List resultGroups = new ArrayList<>(); + result.forEach(record -> { + ResultGroupBaseDTO resultGroup = new ResultGroupBaseDTO(); + resultGroup.setStatus(record.get(T.STATUS)); + resultGroup.setTag(""); + Object ipCount = record.get("ip_count"); + resultGroup.setTotal(ipCount == null ? 0 : (int) ipCount); + resultGroups.add(resultGroup); + }); + return resultGroups; + } + + @Override + @MySQLOperation(table = "gse_file_execute_obj_task", op = DbOperationEnum.READ) + public List listTaskByResultGroup(Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch, + Integer status) { + SelectConditionStep selectConditionStep = dsl().select(ALL_FIELDS) + .from(T) + .where(T.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)) + .and(T.EXECUTE_COUNT.eq(executeCount.shortValue())) + .and(T.STATUS.eq(status)) + .and(T.MODE.eq(FileTaskModeEnum.DOWNLOAD.getValue().byteValue())); + if (batch != null && batch > 0) { + selectConditionStep.and(T.BATCH.eq(batch.shortValue())); + } + Result result = selectConditionStep.fetch(); + + List tasks = new ArrayList<>(); + if (result.size() > 0) { + result.forEach(record -> tasks.add(extract(record))); + } + return tasks; + } + + @Override + @MySQLOperation(table = "gse_file_execute_obj_task", op = DbOperationEnum.READ) + public List listTaskByResultGroup(Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch, + Integer status, + Integer limit, + String orderField, + Order order) { + List conditions = new ArrayList<>(); + conditions.add(buildTaskInstanceIdQueryCondition(taskInstanceId)); + conditions.add(T.STEP_INSTANCE_ID.eq(stepInstanceId)); + conditions.add(T.EXECUTE_COUNT.eq(executeCount.shortValue())); + conditions.add(T.STATUS.eq(status)); + conditions.add(T.MODE.eq(FileTaskModeEnum.DOWNLOAD.getValue().byteValue())); + + SelectConditionStep select = dsl().select(ALL_FIELDS) + .from(T) + .where(conditions); + + if (batch != null && batch > 0) { + select.and(T.BATCH.eq(batch.shortValue())); + } + + SelectSeekStep1 selectSeekStep = null; + OrderField orderFieldEntity = buildOrderField(orderField, order); + if (orderFieldEntity != null) { + selectSeekStep = select.orderBy(orderFieldEntity); + } + + SelectLimitPercentStep selectLimitPercentStep = null; + if (limit != null && limit > 0) { + if (selectSeekStep != null) { + selectLimitPercentStep = selectSeekStep.limit(limit); + } else { + selectLimitPercentStep = select.limit(limit); + } + } + + List tasks = new ArrayList<>(); + Result result; + if (selectLimitPercentStep != null) { + result = selectLimitPercentStep.fetch(); + } else if (selectSeekStep != null) { + result = selectSeekStep.fetch(); + } else { + result = select.fetch(); + } + + if (result.size() > 0) { + result.into(record -> tasks.add(extract(record))); + } + return tasks; + } + + private OrderField buildOrderField(String field, Order order) { + OrderField orderField = null; + if (StringUtils.isNotBlank(field)) { + if (field.equals(T.TOTAL_TIME.getName())) { + if (order == DESCENDING) { + orderField = T.TOTAL_TIME.desc(); + } else { + orderField = T.TOTAL_TIME.asc(); + } + } + } + return orderField; + } + + @Override + @MySQLOperation(table = "gse_file_execute_obj_task", op = + DbOperationEnum.READ) + public List listTasks(Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch, + FileTaskModeEnum fileTaskMode) { + SelectConditionStep selectConditionStep = dsl().select(ALL_FIELDS) + .from(T) + .where(T.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)) + .and(T.EXECUTE_COUNT.eq(executeCount.shortValue())); + if (batch != null && batch > 0) { + selectConditionStep.and(T.BATCH.eq(batch.shortValue())); + } + if (fileTaskMode != null) { + selectConditionStep.and(T.MODE.eq(fileTaskMode.getValue().byteValue())); + } + Result result = selectConditionStep.fetch(); + List executeObjectList = new ArrayList<>(); + if (result.size() != 0) { + result.map(record -> { + executeObjectList.add(extract(record)); + return null; + }); + } + return executeObjectList; + } + + private ExecuteObjectTask extract(Record record) { + if (record == null) { + return null; + } + ExecuteObjectTask executeObject = new ExecuteObjectTask(); + executeObject.setTaskInstanceId(record.get(T.TASK_INSTANCE_ID)); + executeObject.setStepInstanceId(record.get(T.STEP_INSTANCE_ID)); + executeObject.setExecuteCount(record.get(T.EXECUTE_COUNT)); + Short actualExecuteCount = record.get(T.ACTUAL_EXECUTE_COUNT); + executeObject.setActualExecuteCount(actualExecuteCount != null ? actualExecuteCount.intValue() : null); + executeObject.setBatch(record.get(T.BATCH)); + executeObject.setFileTaskMode(FileTaskModeEnum.getFileTaskMode(record.get(T.MODE).intValue())); + executeObject.setExecuteObjectType(ExecuteObjectTypeEnum.valOf(record.get(T.EXECUTE_OBJ_TYPE).intValue())); + executeObject.setExecuteObjectId(record.get(T.EXECUTE_OBJ_ID)); + executeObject.setGseTaskId(record.get(T.GSE_TASK_ID)); + executeObject.setStatus(ExecuteObjectTaskStatusEnum.valOf(record.get(T.STATUS))); + executeObject.setStartTime(record.get(T.START_TIME)); + executeObject.setEndTime(record.get(T.END_TIME)); + executeObject.setTotalTime(record.get(T.TOTAL_TIME)); + executeObject.setErrorCode(record.get(T.ERROR_CODE)); + return executeObject; + } + + @Override + @MySQLOperation(table = "gse_file_execute_obj_task", op = DbOperationEnum.READ) + public List listTasksByGseTaskId(Long taskInstanceId, Long gseTaskId) { + if (gseTaskId == null || gseTaskId <= 0) { + return Collections.emptyList(); + } + + List executeObjectList = new ArrayList<>(); + + Result result = dsl().select(ALL_FIELDS) + .from(T) + .where(T.GSE_TASK_ID.eq(gseTaskId)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)) + .fetch(); + if (result.size() > 0) { + result.forEach(record -> executeObjectList.add(extract(record))); + } + return executeObjectList; + } + + @Override + @MySQLOperation(table = "gse_file_execute_obj_task", op = DbOperationEnum.READ) + public ExecuteObjectTask getTaskByExecuteObjectId(Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch, + FileTaskModeEnum mode, + String executeObjectId) { + SelectConditionStep selectConditionStep = + dsl().select(ALL_FIELDS) + .from(T) + .where(T.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)) + .and(T.EXECUTE_COUNT.eq(executeCount.shortValue())) + .and(T.MODE.eq(mode.getValue().byteValue())) + .and(T.EXECUTE_OBJ_ID.eq(executeObjectId)); + if (batch != null && batch > 0) { + // 滚动执行批次，传入null或者0将忽略该参数 + selectConditionStep.and(T.BATCH.eq(batch.shortValue())); + } + selectConditionStep.limit(1); + Record record = selectConditionStep.fetchOne(); + return extract(record); + } + + @Override + @MySQLOperation(table = "gse_file_execute_obj_task", op = DbOperationEnum.READ) + public boolean isStepInstanceRecordExist(Long taskInstanceId, long stepInstanceId) { + return dsl().fetchExists( + T, + T.STEP_INSTANCE_ID.eq(stepInstanceId), + buildTaskInstanceIdQueryCondition(taskInstanceId) + ); + } + + @Override + @MySQLOperation(table = "gse_file_execute_obj_task", op = DbOperationEnum.WRITE) + public void updateTaskFields(Long taskInstanceId, + long stepInstanceId, + int executeCount, + Integer batch, + Integer actualExecuteCount, + Long gseTaskId) { + UpdateSetStep updateSetStep = dsl().update(T); + boolean needUpdate = false; + if (actualExecuteCount != null) { + updateSetStep = updateSetStep.set(T.ACTUAL_EXECUTE_COUNT, + actualExecuteCount.shortValue()); + needUpdate = true; + } + if (gseTaskId != null) { + updateSetStep = updateSetStep.set(T.GSE_TASK_ID, gseTaskId); + needUpdate = true; + } + + if (!needUpdate) { + return; + } + + UpdateSetMoreStep updateSetMoreStep = + (UpdateSetMoreStep) updateSetStep; + + UpdateConditionStep updateConditionStep = + updateSetMoreStep + .where(T.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)) + .and(T.EXECUTE_COUNT.eq((short) executeCount)); + if (batch != null) { + updateConditionStep.and(T.BATCH.eq(batch.shortValue())); + } + updateConditionStep.execute(); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileSourceTaskLogDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileSourceTaskLogDAOImpl.java index 42daf5870d..c84891639a 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileSourceTaskLogDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/FileSourceTaskLogDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,40 +24,46 @@ package com.tencent.bk.job.execute.dao.impl; -import com.tencent.bk.job.execute.common.util.JooqDataTypeUtil; +import com.tencent.bk.job.common.mysql.dynamic.ds.DbOperationEnum; +import com.tencent.bk.job.common.mysql.dynamic.ds.MySQLOperation; +import com.tencent.bk.job.common.mysql.jooq.JooqDataTypeUtil; import com.tencent.bk.job.execute.dao.FileSourceTaskLogDAO; +import com.tencent.bk.job.execute.dao.common.DSLContextProviderFactory; import com.tencent.bk.job.execute.model.FileSourceTaskLogDTO; -import org.jooq.DSLContext; +import com.tencent.bk.job.execute.model.tables.FileSourceTaskLog; +import com.tencent.bk.job.execute.model.tables.records.FileSourceTaskLogRecord; +import org.jooq.Condition; import org.jooq.Record; +import org.jooq.TableField; import org.jooq.UpdateSetFirstStep; import org.jooq.UpdateSetMoreStep; -import org.jooq.generated.tables.FileSourceTaskLog; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; +import java.util.ArrayList; +import java.util.List; + @Repository -public class FileSourceTaskLogDAOImpl implements FileSourceTaskLogDAO { - FileSourceTaskLog defaultTable = FileSourceTaskLog.FILE_SOURCE_TASK_LOG; - private DSLContext defaultContext; +public class FileSourceTaskLogDAOImpl extends BaseDAO implements FileSourceTaskLogDAO { - @Autowired - public FileSourceTaskLogDAOImpl(@Qualifier("job-execute-dsl-context") DSLContext defaultContext) { - this.defaultContext = defaultContext; - } + private static final FileSourceTaskLog defaultTable = FileSourceTaskLog.FILE_SOURCE_TASK_LOG; - @Override - public FileSourceTaskLogDTO getStepLastExecuteLog(long stepInstanceId) { - FileSourceTaskLog t = FileSourceTaskLog.FILE_SOURCE_TASK_LOG; - Record record = defaultContext.select(t.STEP_INSTANCE_ID, t.EXECUTE_COUNT, t.START_TIME, t.END_TIME, - t.TOTAL_TIME, - t.STATUS, t.FILE_SOURCE_BATCH_TASK_ID) - .from(t) - .where(t.STEP_INSTANCE_ID.eq(stepInstanceId)) - .orderBy(t.EXECUTE_COUNT.desc()) - .limit(1) - .fetchOne(); - return extractInfo(record); + private static final TableField[] ALL_FIELDS = { + defaultTable.ID, + defaultTable.TASK_INSTANCE_ID, + defaultTable.STEP_INSTANCE_ID, + defaultTable.EXECUTE_COUNT, + defaultTable.START_TIME, + defaultTable.END_TIME, + defaultTable.TOTAL_TIME, + defaultTable.STATUS, + defaultTable.FILE_SOURCE_BATCH_TASK_ID + }; + + + @Autowired + public FileSourceTaskLogDAOImpl(DSLContextProviderFactory dslContextProviderFactory) { + super(dslContextProviderFactory, defaultTable.getName()); } private FileSourceTaskLogDTO extractInfo(Record record) { @@ -67,6 +73,8 @@ private FileSourceTaskLogDTO extractInfo(Record record) { FileSourceTaskLogDTO fileSourceTaskLogDTO = new FileSourceTaskLogDTO(); FileSourceTaskLog t = FileSourceTaskLog.FILE_SOURCE_TASK_LOG; + fileSourceTaskLogDTO.setId(record.get(t.ID)); + fileSourceTaskLogDTO.setTaskInstanceId(record.get(t.TASK_INSTANCE_ID)); fileSourceTaskLogDTO.setStepInstanceId(record.get(t.STEP_INSTANCE_ID)); fileSourceTaskLogDTO.setExecuteCount(record.get(t.EXECUTE_COUNT)); fileSourceTaskLogDTO.setStartTime(record.get(t.START_TIME)); @@ -78,52 +86,92 @@ private FileSourceTaskLogDTO extractInfo(Record record) { } @Override - public void saveFileSourceTaskLog(FileSourceTaskLogDTO fileSourceTaskLog) { + @MySQLOperation(table = "file_source_task_log", op = DbOperationEnum.WRITE) + public int insertFileSourceTaskLog(FileSourceTaskLogDTO fileSourceTaskLog) { FileSourceTaskLog t = FileSourceTaskLog.FILE_SOURCE_TASK_LOG; - defaultContext.insertInto(t, t.STEP_INSTANCE_ID, t.EXECUTE_COUNT, t.START_TIME, t.END_TIME, t.TOTAL_TIME, - t.STATUS, t.FILE_SOURCE_BATCH_TASK_ID) - .values(fileSourceTaskLog.getStepInstanceId(), - fileSourceTaskLog.getExecuteCount(), - fileSourceTaskLog.getStartTime(), - fileSourceTaskLog.getEndTime(), - fileSourceTaskLog.getTotalTime(), - JooqDataTypeUtil.toByte(fileSourceTaskLog.getStatus()), - fileSourceTaskLog.getFileSourceBatchTaskId()) - .onDuplicateKeyUpdate() - .set(t.START_TIME, fileSourceTaskLog.getStartTime()) - .set(t.END_TIME, fileSourceTaskLog.getEndTime()) - .set(t.TOTAL_TIME, fileSourceTaskLog.getTotalTime()) - .set(t.STATUS, JooqDataTypeUtil.toByte(fileSourceTaskLog.getStatus())).set(t.FILE_SOURCE_BATCH_TASK_ID, fileSourceTaskLog.getFileSourceBatchTaskId()) + return dsl().insertInto( + t, + t.ID, + t.TASK_INSTANCE_ID, + t.STEP_INSTANCE_ID, + t.EXECUTE_COUNT, + t.START_TIME, + t.END_TIME, + t.TOTAL_TIME, + t.STATUS, + t.FILE_SOURCE_BATCH_TASK_ID + ).values( + fileSourceTaskLog.getId(), + fileSourceTaskLog.getTaskInstanceId(), + fileSourceTaskLog.getStepInstanceId(), + fileSourceTaskLog.getExecuteCount(), + fileSourceTaskLog.getStartTime(), + fileSourceTaskLog.getEndTime(), + fileSourceTaskLog.getTotalTime(), + JooqDataTypeUtil.toByte(fileSourceTaskLog.getStatus()), + fileSourceTaskLog.getFileSourceBatchTaskId() + ).execute(); + } + + @Override + @MySQLOperation(table = "file_source_task_log", op = DbOperationEnum.WRITE) + public int updateFileSourceTaskLogByStepInstance(FileSourceTaskLogDTO fileSourceTaskLog) { + List conditionList = new ArrayList<>(); + conditionList.add(defaultTable.TASK_INSTANCE_ID.eq(fileSourceTaskLog.getTaskInstanceId())); + conditionList.add(defaultTable.STEP_INSTANCE_ID.eq(fileSourceTaskLog.getStepInstanceId())); + conditionList.add(defaultTable.EXECUTE_COUNT.eq(fileSourceTaskLog.getExecuteCount())); + return dsl().update(defaultTable) + .set(defaultTable.START_TIME, fileSourceTaskLog.getStartTime()) + .set(defaultTable.END_TIME, fileSourceTaskLog.getEndTime()) + .set(defaultTable.TOTAL_TIME, fileSourceTaskLog.getTotalTime()) + .set(defaultTable.STATUS, JooqDataTypeUtil.toByte(fileSourceTaskLog.getStatus())) + .set(defaultTable.FILE_SOURCE_BATCH_TASK_ID, fileSourceTaskLog.getFileSourceBatchTaskId()) + .where(conditionList) + .limit(1) .execute(); } @Override - public FileSourceTaskLogDTO getFileSourceTaskLog(long stepInstanceId, int executeCount) { + @MySQLOperation(table = "file_source_task_log", op = DbOperationEnum.READ) + public FileSourceTaskLogDTO getFileSourceTaskLog(Long taskInstanceId, long stepInstanceId, int executeCount) { FileSourceTaskLog t = FileSourceTaskLog.FILE_SOURCE_TASK_LOG; - Record record = defaultContext.select(t.STEP_INSTANCE_ID, t.EXECUTE_COUNT, t.START_TIME, t.END_TIME, - t.TOTAL_TIME, - t.STATUS, t.FILE_SOURCE_BATCH_TASK_ID).from(t) + Record record = dsl().select( + ALL_FIELDS + ).from(t) .where(t.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)) .and(t.EXECUTE_COUNT.eq(executeCount)) .fetchOne(); return extractInfo(record); } + private Condition buildTaskInstanceIdQueryCondition(Long taskInstanceId) { + return TaskInstanceIdDynamicCondition.build( + taskInstanceId, + FileSourceTaskLog.FILE_SOURCE_TASK_LOG.TASK_INSTANCE_ID::eq + ); + } + @Override - public FileSourceTaskLogDTO getFileSourceTaskLogByBatchTaskId(String fileSourceBatchTaskId) { - Record record = defaultContext.select(defaultTable.STEP_INSTANCE_ID, defaultTable.EXECUTE_COUNT, - defaultTable.START_TIME, defaultTable.END_TIME, defaultTable.TOTAL_TIME, - defaultTable.STATUS, defaultTable.FILE_SOURCE_BATCH_TASK_ID).from(defaultTable) + @MySQLOperation(table = "file_source_task_log", op = DbOperationEnum.READ) + public FileSourceTaskLogDTO getFileSourceTaskLogByBatchTaskId(Long taskInstanceId, String fileSourceBatchTaskId) { + Record record = dsl().select(ALL_FIELDS) + .from(defaultTable) .where(defaultTable.FILE_SOURCE_BATCH_TASK_ID.eq(fileSourceBatchTaskId)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)) .fetchOne(); return extractInfo(record); } @Override - public int updateTimeConsumingByBatchTaskId(String fileSourceBatchTaskId, Long startTime, Long endTime, + @MySQLOperation(table = "file_source_task_log", op = DbOperationEnum.WRITE) + public int updateTimeConsumingByBatchTaskId(Long taskInstanceId, + String fileSourceBatchTaskId, + Long startTime, + Long endTime, Long totalTime) { - UpdateSetFirstStep firstStep = defaultContext.update(defaultTable); - UpdateSetMoreStep moreStep = null; + UpdateSetFirstStep firstStep = dsl().update(defaultTable); + UpdateSetMoreStep moreStep = null; if (startTime != null) { moreStep = firstStep.set(defaultTable.START_TIME, startTime); } @@ -142,19 +190,13 @@ public int updateTimeConsumingByBatchTaskId(String fileSourceBatchTaskId, Long s } } if (moreStep != null) { - return moreStep.where(defaultTable.FILE_SOURCE_BATCH_TASK_ID.eq(fileSourceBatchTaskId)) + return moreStep.where( + defaultTable.FILE_SOURCE_BATCH_TASK_ID.eq(fileSourceBatchTaskId)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)) .execute(); } else { return 0; } } - @Override - public void deleteFileSourceTaskLog(long stepInstanceId, int executeCount) { - FileSourceTaskLog t = FileSourceTaskLog.FILE_SOURCE_TASK_LOG; - defaultContext.deleteFrom(t).where(t.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(t.EXECUTE_COUNT.eq(executeCount)) - .execute(); - } - } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/GseTaskDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/GseTaskDAOImpl.java index ac88f92822..f3187adc19 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/GseTaskDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/GseTaskDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,28 +24,53 @@ package com.tencent.bk.job.execute.dao.impl; -import com.tencent.bk.job.execute.common.util.JooqDataTypeUtil; +import com.tencent.bk.job.common.mysql.dynamic.ds.DbOperationEnum; +import com.tencent.bk.job.common.mysql.dynamic.ds.MySQLOperation; +import com.tencent.bk.job.common.mysql.jooq.JooqDataTypeUtil; import com.tencent.bk.job.execute.dao.GseTaskDAO; +import com.tencent.bk.job.execute.dao.common.DSLContextProviderFactory; import com.tencent.bk.job.execute.model.GseTaskDTO; -import org.jooq.DSLContext; +import com.tencent.bk.job.execute.model.GseTaskSimpleDTO; +import com.tencent.bk.job.execute.model.tables.GseTask; +import org.jooq.Condition; import org.jooq.Record; +import org.jooq.Result; +import org.jooq.SelectConditionStep; import org.jooq.TableField; -import org.jooq.generated.tables.GseTask; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; +import java.util.ArrayList; +import java.util.List; + @Repository -public class GseTaskDAOImpl implements GseTaskDAO { - private final DSLContext dslContext; +public class GseTaskDAOImpl extends BaseDAO implements GseTaskDAO { private static final GseTask TABLE = GseTask.GSE_TASK; - private static final TableField[] ALL_FIELDS = {TABLE.ID, TABLE.STEP_INSTANCE_ID, TABLE.EXECUTE_COUNT, - TABLE.BATCH, TABLE.START_TIME, TABLE.END_TIME, TABLE.TOTAL_TIME, TABLE.STATUS, TABLE.GSE_TASK_ID}; + private static final TableField[] ALL_FIELDS = { + TABLE.ID, + TABLE.STEP_INSTANCE_ID, + TABLE.EXECUTE_COUNT, + TABLE.BATCH, + TABLE.START_TIME, + TABLE.END_TIME, + TABLE.TOTAL_TIME, + TABLE.STATUS, + TABLE.GSE_TASK_ID, + TABLE.TASK_INSTANCE_ID + }; + + private static final TableField[] SIMPLE_FIELDS = { + TABLE.STEP_INSTANCE_ID, + TABLE.EXECUTE_COUNT, + TABLE.BATCH, + TABLE.GSE_TASK_ID, + TABLE.TASK_INSTANCE_ID + }; @Autowired - public GseTaskDAOImpl(@Qualifier("job-execute-dsl-context") DSLContext dslContext) { - this.dslContext = dslContext; + public GseTaskDAOImpl(DSLContextProviderFactory dslContextProviderFactory) { + super(dslContextProviderFactory, TABLE.getName()); } private GseTaskDTO extractInfo(Record record) { @@ -55,6 +80,7 @@ private GseTaskDTO extractInfo(Record record) { GseTaskDTO gseTaskDTO = new GseTaskDTO(); gseTaskDTO.setId(record.get(TABLE.ID)); + gseTaskDTO.setTaskInstanceId(record.get(TABLE.TASK_INSTANCE_ID)); gseTaskDTO.setStepInstanceId(record.get(TABLE.STEP_INSTANCE_ID)); gseTaskDTO.setExecuteCount(record.get(TABLE.EXECUTE_COUNT).intValue()); gseTaskDTO.setBatch(record.get(TABLE.BATCH)); @@ -66,19 +92,36 @@ private GseTaskDTO extractInfo(Record record) { return gseTaskDTO; } + private GseTaskSimpleDTO extractSimpleInfo(Record record) { + if (record == null) { + return null; + } + GseTaskSimpleDTO gseTaskSimpleDTO = new GseTaskSimpleDTO(); + gseTaskSimpleDTO.setTaskInstanceId(record.get(TABLE.TASK_INSTANCE_ID)); + gseTaskSimpleDTO.setStepInstanceId(record.get(TABLE.STEP_INSTANCE_ID)); + gseTaskSimpleDTO.setExecuteCount(record.get(TABLE.EXECUTE_COUNT).intValue()); + gseTaskSimpleDTO.setBatch(record.get(TABLE.BATCH).intValue()); + gseTaskSimpleDTO.setGseTaskId(record.get(TABLE.GSE_TASK_ID)); + return gseTaskSimpleDTO; + } + @Override + @MySQLOperation(table = "gse_task", op = DbOperationEnum.WRITE) public long saveGseTask(GseTaskDTO gseTask) { - Record record = dslContext.insertInto( - TABLE, - TABLE.STEP_INSTANCE_ID, - TABLE.EXECUTE_COUNT, - TABLE.BATCH, - TABLE.START_TIME, - TABLE.END_TIME, - TABLE.TOTAL_TIME, - TABLE.STATUS, - TABLE.GSE_TASK_ID) + Record record = dsl().insertInto( + TABLE, + TABLE.ID, + TABLE.STEP_INSTANCE_ID, + TABLE.EXECUTE_COUNT, + TABLE.BATCH, + TABLE.START_TIME, + TABLE.END_TIME, + TABLE.TOTAL_TIME, + TABLE.STATUS, + TABLE.GSE_TASK_ID, + TABLE.TASK_INSTANCE_ID) .values( + gseTask.getId(), gseTask.getStepInstanceId(), gseTask.getExecuteCount().shortValue(), (short) gseTask.getBatch(), @@ -86,41 +129,91 @@ public long saveGseTask(GseTaskDTO gseTask) { gseTask.getEndTime(), gseTask.getTotalTime(), JooqDataTypeUtil.toByte(gseTask.getStatus()), - gseTask.getGseTaskId()) + gseTask.getGseTaskId(), + gseTask.getTaskInstanceId()) .returning(TABLE.ID) .fetchOne(); + return gseTask.getId() != null ? gseTask.getId() : record.getValue(TABLE.ID); - return record == null ? 0 : record.get(TABLE.ID); } @Override + @MySQLOperation(table = "gse_task", op = DbOperationEnum.WRITE) public boolean updateGseTask(GseTaskDTO gseTask) { - int affectRows = dslContext.update(TABLE) + int affectRows = dsl().update(TABLE) .set(TABLE.START_TIME, gseTask.getStartTime()) .set(TABLE.END_TIME, gseTask.getEndTime()) .set(TABLE.TOTAL_TIME, gseTask.getTotalTime()) .set(TABLE.STATUS, gseTask.getStatus().byteValue()) .set(TABLE.GSE_TASK_ID, gseTask.getGseTaskId()) .where(TABLE.ID.eq(gseTask.getId())) + .and(TaskInstanceIdDynamicCondition.build(gseTask.getTaskInstanceId(), TABLE.TASK_INSTANCE_ID::eq)) .execute(); return affectRows > 0; } @Override - public GseTaskDTO getGseTask(long stepInstanceId, int executeCount, Integer batch) { - Record record = dslContext.select(ALL_FIELDS).from(TABLE) - .where(TABLE.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(TABLE.EXECUTE_COUNT.eq((short) executeCount)) - .and(TABLE.BATCH.eq(batch == null ? 0 : batch.shortValue())) - .fetchOne(); + @MySQLOperation(table = "gse_task", op = DbOperationEnum.READ) + public GseTaskDTO getGseTask(Long taskInstanceId, long stepInstanceId, int executeCount, Integer batch) { + SelectConditionStep selectConditionStep = + dsl().select(ALL_FIELDS).from(TABLE) + .where(TABLE.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(TABLE.EXECUTE_COUNT.eq((short) executeCount)) + .and(TaskInstanceIdDynamicCondition.build(taskInstanceId, TABLE.TASK_INSTANCE_ID::eq)); + if (batch != null && batch > 0) { + // 滚动执行批次，传入null或者0将忽略该参数 + selectConditionStep.and(TABLE.BATCH.eq(batch.shortValue())); + } + selectConditionStep.limit(1); + Record record = selectConditionStep.fetchOne(); return extractInfo(record); } @Override - public GseTaskDTO getGseTask(long gseTaskId) { - Record record = dslContext.select(ALL_FIELDS).from(TABLE) + @MySQLOperation(table = "gse_task", op = DbOperationEnum.READ) + public GseTaskDTO getGseTask(Long taskInstanceId, long gseTaskId) { + Record record = dsl().select(ALL_FIELDS).from(TABLE) .where(TABLE.ID.eq(gseTaskId)) + .and(TaskInstanceIdDynamicCondition.build(taskInstanceId, TABLE.TASK_INSTANCE_ID::eq)) .fetchOne(); return extractInfo(record); } + + @Override + @MySQLOperation(table = "gse_task", op = DbOperationEnum.READ) + public GseTaskSimpleDTO getGseTaskSimpleInfo(String gseTaskId) { + Result records = dsl().select(SIMPLE_FIELDS).from(TABLE) + .where(TABLE.GSE_TASK_ID.eq(gseTaskId)) + .limit(1) + .fetch(); + if (records.isEmpty()) { + return null; + } + return extractSimpleInfo(records.get(0)); + } + + @Override + @MySQLOperation(table = "gse_task", op = DbOperationEnum.READ) + public List ListGseTaskSimpleInfo(Long stepInstanceId, Integer executeCount, Integer batch) { + List conditions = new ArrayList<>(); + if (stepInstanceId != null) { + conditions.add(TABLE.STEP_INSTANCE_ID.eq(stepInstanceId)); + } + if (executeCount != null) { + conditions.add(TABLE.EXECUTE_COUNT.eq(executeCount.shortValue())); + } + if (batch != null) { + conditions.add(TABLE.BATCH.eq(batch.shortValue())); + } + Result records = dsl().select(SIMPLE_FIELDS).from(TABLE) + .where(conditions) + .fetch(); + List results = new ArrayList<>(); + if (records.isNotEmpty()) { + records.into(record -> { + results.add(extractSimpleInfo(record)); + }); + } + return results; + } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/GseTaskIpLogDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/GseTaskIpLogDAOImpl.java deleted file mode 100644 index 347350ad62..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/GseTaskIpLogDAOImpl.java +++ /dev/null @@ -1,279 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.dao.impl; - -import com.tencent.bk.job.common.annotation.CompatibleImplementation; -import com.tencent.bk.job.common.constant.Bool; -import com.tencent.bk.job.common.constant.Order; -import com.tencent.bk.job.execute.dao.GseTaskIpLogDAO; -import com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum; -import com.tencent.bk.job.execute.model.AgentTaskDTO; -import com.tencent.bk.job.execute.model.AgentTaskResultGroupBaseDTO; -import com.tencent.bk.job.logsvr.consts.FileTaskModeEnum; -import org.apache.commons.lang3.StringUtils; -import org.jooq.Condition; -import org.jooq.DSLContext; -import org.jooq.OrderField; -import org.jooq.Record; -import org.jooq.Result; -import org.jooq.SelectConditionStep; -import org.jooq.SelectLimitPercentStep; -import org.jooq.SelectSeekStep1; -import org.jooq.generated.tables.GseTaskIpLog; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.stereotype.Repository; - -import java.util.ArrayList; -import java.util.List; - -import static com.tencent.bk.job.common.constant.Order.DESCENDING; -import static org.jooq.impl.DSL.count; - -@CompatibleImplementation(name = "rolling_execute", explain = "兼容老版本数据，过1-2个大版本之后删除", version = "3.7.x") -@Repository -public class GseTaskIpLogDAOImpl implements GseTaskIpLogDAO { - - private final DSLContext CTX; - - @Autowired - public GseTaskIpLogDAOImpl(@Qualifier("job-execute-dsl-context") DSLContext CTX) { - this.CTX = CTX; - } - - @Override - public int getSuccessAgentTaskCount(long stepInstanceId, int executeCount) { - GseTaskIpLog t = GseTaskIpLog.GSE_TASK_IP_LOG; - - return CTX.selectCount().from(t) - .where(t.STATUS.in(AgentTaskStatusEnum.LAST_SUCCESS.getValue(), - AgentTaskStatusEnum.SUCCESS.getValue())) - .and(t.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(t.EXECUTE_COUNT.eq(executeCount)) - .and(t.IS_TARGET.eq(Bool.TRUE.getValue())) - .fetchOne(0, Integer.class); - } - - @Override - public List listResultGroups(long stepInstanceId, int executeCount) { - GseTaskIpLog t = GseTaskIpLog.GSE_TASK_IP_LOG; - Result result = CTX.select(t.STATUS, t.TAG, count().as("ip_count")).from(t) - .where(t.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(t.EXECUTE_COUNT.eq(executeCount)) - .and(t.IS_TARGET.eq(Bool.TRUE.getValue())) - .groupBy(t.STATUS, t.TAG) - .orderBy(t.STATUS.asc()) - .fetch(); - - List resultGroups = new ArrayList<>(); - result.into(record -> { - AgentTaskResultGroupBaseDTO resultGroup = new AgentTaskResultGroupBaseDTO(); - resultGroup.setStatus(record.get(t.STATUS)); - resultGroup.setTag(record.get(t.TAG)); - resultGroup.setTotalAgentTasks((int) record.get("ip_count")); - resultGroups.add(resultGroup); - }); - return resultGroups; - } - - @Override - public List listAgentTaskByResultGroup(Long stepInstanceId, - Integer executeCount, - Integer status, - String tag) { - GseTaskIpLog t = GseTaskIpLog.GSE_TASK_IP_LOG; - Result result = CTX.select(t.STEP_INSTANCE_ID, t.EXECUTE_COUNT, t.IP, t.STATUS, t.START_TIME, t.END_TIME, - t.TOTAL_TIME, t.ERROR_CODE, t.EXIT_CODE, t.TAG, t.LOG_OFFSET, t.DISPLAY_IP, t.IS_TARGET, t.IS_SOURCE) - .from(t) - .where(t.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(t.EXECUTE_COUNT.eq(executeCount)) - .and(t.STATUS.eq(status)) - .and(t.TAG.eq(tag == null ? "" : tag)) - .and(t.IS_TARGET.eq(Bool.TRUE.getValue())) - .fetch(); - - List agentTasks = new ArrayList<>(); - if (result.size() > 0) { - result.into(record -> agentTasks.add(extract(record))); - } - return agentTasks; - } - - @Override - public List listAgentTaskByResultGroup(Long stepInstanceId, - Integer executeCount, - Integer status, - String tag, - Integer limit, - String orderField, - Order order) { - GseTaskIpLog t = GseTaskIpLog.GSE_TASK_IP_LOG; - - List conditions = new ArrayList<>(); - conditions.add(t.STEP_INSTANCE_ID.eq(stepInstanceId)); - conditions.add(t.EXECUTE_COUNT.eq(executeCount)); - conditions.add(t.STATUS.eq(status)); - conditions.add(t.TAG.eq(tag == null ? "" : tag)); - conditions.add(t.IS_TARGET.eq(Bool.TRUE.getValue())); - - SelectConditionStep select = CTX.select(t.STEP_INSTANCE_ID, t.EXECUTE_COUNT, t.IP, t.STATUS, t.START_TIME, - t.END_TIME, - t.TOTAL_TIME, t.ERROR_CODE, t.EXIT_CODE, t.TAG, t.LOG_OFFSET, t.DISPLAY_IP, t.IS_TARGET, t.IS_SOURCE) - .from(t) - .where(conditions); - - SelectSeekStep1 selectSeekStep = null; - OrderField orderFieldEntity = buildOrderField(orderField, order); - if (orderFieldEntity != null) { - selectSeekStep = select.orderBy(orderFieldEntity); - } - - SelectLimitPercentStep selectLimitPercentStep = null; - if (limit != null && limit > 0) { - if (selectSeekStep != null) { - selectLimitPercentStep = selectSeekStep.limit(limit); - } else { - selectLimitPercentStep = select.limit(limit); - } - } - - List agentTasks = new ArrayList<>(); - Result result; - if (selectLimitPercentStep != null) { - result = selectLimitPercentStep.fetch(); - } else if (selectSeekStep != null) { - result = selectSeekStep.fetch(); - } else { - result = select.fetch(); - } - - if (result.size() > 0) { - result.into(record -> { - agentTasks.add(extract(record)); - }); - } - return agentTasks; - } - - private OrderField buildOrderField(String field, Order order) { - OrderField orderField = null; - if (StringUtils.isNotBlank(field)) { - if (field.equals(GseTaskIpLog.GSE_TASK_IP_LOG.TOTAL_TIME.getName())) { - if (order == DESCENDING) { - orderField = GseTaskIpLog.GSE_TASK_IP_LOG.TOTAL_TIME.desc(); - } else { - orderField = GseTaskIpLog.GSE_TASK_IP_LOG.TOTAL_TIME.asc(); - } - } else if (field.equals(GseTaskIpLog.GSE_TASK_IP_LOG.EXIT_CODE.getName())) { - if (order == DESCENDING) { - orderField = GseTaskIpLog.GSE_TASK_IP_LOG.EXIT_CODE.desc(); - } else { - orderField = GseTaskIpLog.GSE_TASK_IP_LOG.EXIT_CODE.asc(); - } - } else if (field.equals(GseTaskIpLog.GSE_TASK_IP_LOG.IP.getName())) { - if (order == DESCENDING) { - orderField = GseTaskIpLog.GSE_TASK_IP_LOG.IP.desc(); - } else { - orderField = GseTaskIpLog.GSE_TASK_IP_LOG.IP.asc(); - } - } - } - return orderField; - } - - - @Override - public List listAgentTasks(Long stepInstanceId, Integer executeCount) { - GseTaskIpLog t = GseTaskIpLog.GSE_TASK_IP_LOG; - Result result = CTX.select(t.STEP_INSTANCE_ID, t.EXECUTE_COUNT, t.IP, t.STATUS, - t.START_TIME, t.END_TIME, - t.TOTAL_TIME, t.ERROR_CODE, t.EXIT_CODE, t.TAG, t.LOG_OFFSET, t.DISPLAY_IP, t.IS_TARGET, t.IS_SOURCE) - .from(t) - .where(t.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(t.EXECUTE_COUNT.eq(executeCount)) - .fetch(); - List agentTaskList = new ArrayList<>(); - if (result.size() != 0) { - result.map(record -> { - agentTaskList.add(extract(record)); - return null; - }); - } - return agentTaskList; - } - - private AgentTaskDTO extract(Record record) { - if (record == null) { - return null; - } - GseTaskIpLog t = GseTaskIpLog.GSE_TASK_IP_LOG; - AgentTaskDTO agentTask = new AgentTaskDTO(); - agentTask.setStepInstanceId(record.get(t.STEP_INSTANCE_ID)); - agentTask.setExecuteCount(record.get(t.EXECUTE_COUNT)); - agentTask.setCloudIp(record.get(t.IP)); - agentTask.setDisplayIp(record.get(t.DISPLAY_IP)); - agentTask.setAgentId(record.get(t.IP)); - agentTask.setStatus(AgentTaskStatusEnum.valueOf(record.get(t.STATUS))); - agentTask.setStartTime(record.get(t.START_TIME)); - agentTask.setEndTime(record.get(t.END_TIME)); - agentTask.setTotalTime(record.get(t.TOTAL_TIME)); - agentTask.setErrorCode(record.get(t.ERROR_CODE)); - agentTask.setExitCode(record.get(t.EXIT_CODE, Integer.class)); - agentTask.setTag(record.get(t.TAG)); - agentTask.setScriptLogOffset(record.get(t.LOG_OFFSET)); - boolean isUploadMode = record.get(t.IS_SOURCE) == Bool.TRUE.getValue(); - agentTask.setFileTaskMode(isUploadMode ? FileTaskModeEnum.UPLOAD : FileTaskModeEnum.DOWNLOAD); - return agentTask; - } - - @Override - public AgentTaskDTO getAgentTaskByIp(Long stepInstanceId, Integer executeCount, String ip) { - GseTaskIpLog t = GseTaskIpLog.GSE_TASK_IP_LOG; - Record record = CTX.select(t.STEP_INSTANCE_ID, t.EXECUTE_COUNT, t.IP, t.STATUS, t.START_TIME, t.END_TIME, - t.TOTAL_TIME, t.ERROR_CODE, t.EXIT_CODE, t.TAG, t.LOG_OFFSET, t.DISPLAY_IP, t.IS_TARGET, t.IS_SOURCE) - .from(t) - .where(t.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(t.EXECUTE_COUNT.eq(executeCount)) - .and(t.IP.eq(ip)) - .fetchOne(); - return extract(record); - } - - @Override - public int getActualSuccessExecuteCount(long stepInstanceId, String cloudIp) { - GseTaskIpLog t = GseTaskIpLog.GSE_TASK_IP_LOG; - Record record = CTX.select(t.EXECUTE_COUNT).from(t).where(t.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(t.IP.eq(cloudIp)) - .and(t.STATUS.eq(AgentTaskStatusEnum.SUCCESS.getValue())) - .orderBy(t.EXECUTE_COUNT.desc()) - .limit(1) - .fetchOne(); - if (record != null && record.size() > 0) { - return record.getValue(t.EXECUTE_COUNT); - } else { - return 0; - } - } -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/OperationLogDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/OperationLogDAOImpl.java index d4237635bd..d1640b0db5 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/OperationLogDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/OperationLogDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,50 +24,68 @@ package com.tencent.bk.job.execute.dao.impl; +import com.tencent.bk.job.common.mysql.dynamic.ds.DbOperationEnum; +import com.tencent.bk.job.common.mysql.dynamic.ds.MySQLOperation; +import com.tencent.bk.job.common.mysql.jooq.JooqDataTypeUtil; import com.tencent.bk.job.common.util.json.JsonUtils; -import com.tencent.bk.job.execute.common.util.JooqDataTypeUtil; import com.tencent.bk.job.execute.constants.UserOperationEnum; import com.tencent.bk.job.execute.dao.OperationLogDAO; +import com.tencent.bk.job.execute.dao.common.DSLContextProviderFactory; import com.tencent.bk.job.execute.model.OperationLogDTO; +import com.tencent.bk.job.execute.model.tables.OperationLog; import org.apache.commons.lang3.StringUtils; -import org.jooq.DSLContext; import org.jooq.Record; import org.jooq.Result; -import org.jooq.generated.tables.OperationLog; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; import java.util.ArrayList; import java.util.List; @Repository -public class OperationLogDAOImpl implements OperationLogDAO { +public class OperationLogDAOImpl extends BaseDAO implements OperationLogDAO { private static final OperationLog TABLE = OperationLog.OPERATION_LOG; - private DSLContext ctx; @Autowired - public OperationLogDAOImpl(@Qualifier("job-execute-dsl-context") DSLContext ctx) { - this.ctx = ctx; + public OperationLogDAOImpl(DSLContextProviderFactory dslContextProviderFactory) { + super(dslContextProviderFactory, TABLE.getName()); } @Override + @MySQLOperation(table = "operation_log", op = DbOperationEnum.WRITE) public long saveOperationLog(OperationLogDTO operationLog) { - Record record = ctx.insertInto(TABLE, TABLE.TASK_INSTANCE_ID, TABLE.OP_CODE, TABLE.OPERATOR, - TABLE.CREATE_TIME, TABLE.DETAIL) - .values(operationLog.getTaskInstanceId(), + Record record = dsl().insertInto( + TABLE, + TABLE.ID, + TABLE.TASK_INSTANCE_ID, + TABLE.OP_CODE, + TABLE.OPERATOR, + TABLE.CREATE_TIME, + TABLE.DETAIL) + .values( + operationLog.getId(), + operationLog.getTaskInstanceId(), JooqDataTypeUtil.toByte(operationLog.getOperationEnum().getValue()), operationLog.getOperator(), operationLog.getCreateTime(), JsonUtils.toJson(operationLog.getDetail())) - .returning(TABLE.ID).fetchOne(); - return record.getValue(TABLE.ID); + .returning(TABLE.ID) + .fetchOne(); + + return operationLog.getId() != null ? operationLog.getId() : record.getValue(TABLE.ID); } @Override + @MySQLOperation(table = "operation_log", op = DbOperationEnum.READ) public List listOperationLog(long taskInstanceId) { - Result result = ctx.select(TABLE.ID, TABLE.TASK_INSTANCE_ID, TABLE.OP_CODE, TABLE.OPERATOR, TABLE.CREATE_TIME - , TABLE.DETAIL) + Result result = dsl() + .select( + TABLE.ID, + TABLE.TASK_INSTANCE_ID, + TABLE.OP_CODE, + TABLE.OPERATOR, + TABLE.CREATE_TIME, + TABLE.DETAIL) .from(TABLE) .where(TABLE.TASK_INSTANCE_ID.eq(taskInstanceId)) .orderBy(TABLE.CREATE_TIME.desc()) diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/RollingConfigDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/RollingConfigDAOImpl.java index e3db47f84a..a242eae04d 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/RollingConfigDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/RollingConfigDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,58 +24,78 @@ package com.tencent.bk.job.execute.dao.impl; +import com.tencent.bk.job.common.mysql.dynamic.ds.DbOperationEnum; +import com.tencent.bk.job.common.mysql.dynamic.ds.MySQLOperation; import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.execute.dao.RollingConfigDAO; +import com.tencent.bk.job.execute.dao.common.DSLContextProviderFactory; import com.tencent.bk.job.execute.model.RollingConfigDTO; import com.tencent.bk.job.execute.model.db.RollingConfigDetailDO; -import org.jooq.DSLContext; +import com.tencent.bk.job.execute.model.tables.RollingConfig; +import org.apache.commons.collections4.CollectionUtils; import org.jooq.Record; -import org.jooq.generated.tables.RollingConfig; +import org.jooq.Record1; +import org.jooq.Result; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; @Repository -public class RollingConfigDAOImpl implements RollingConfigDAO { +public class RollingConfigDAOImpl extends BaseDAO implements RollingConfigDAO { private static final RollingConfig TABLE = RollingConfig.ROLLING_CONFIG; - private final DSLContext CTX; @Autowired - public RollingConfigDAOImpl(@Qualifier("job-execute-dsl-context") DSLContext ctx) { - this.CTX = ctx; + public RollingConfigDAOImpl(DSLContextProviderFactory dslContextProviderFactory) { + super(dslContextProviderFactory, TABLE.getName()); } @Override + @MySQLOperation(table = "rolling_config", op = DbOperationEnum.WRITE) public long saveRollingConfig(RollingConfigDTO rollingConfig) { - Record record = CTX.insertInto( - TABLE, - TABLE.TASK_INSTANCE_ID, - TABLE.CONFIG_NAME, - TABLE.CONFIG) + Record record = dsl().insertInto( + TABLE, + TABLE.ID, + TABLE.TASK_INSTANCE_ID, + TABLE.CONFIG_NAME, + TABLE.CONFIG) .values( + rollingConfig.getId(), rollingConfig.getTaskInstanceId(), rollingConfig.getConfigName(), JsonUtils.toJson(rollingConfig.getConfigDetail())) .returning(TABLE.ID) .fetchOne(); - assert record != null; - return record.get(TABLE.ID); + + return rollingConfig.getId() != null ? rollingConfig.getId() : record.getValue(TABLE.ID); + } @Override - public RollingConfigDTO queryRollingConfigById(Long rollingConfigId) { - Record record = CTX.select( - TABLE.ID, - TABLE.TASK_INSTANCE_ID, - TABLE.CONFIG_NAME, - TABLE.CONFIG) + @MySQLOperation(table = "rolling_config", op = DbOperationEnum.READ) + public RollingConfigDTO queryRollingConfigById(Long taskInstanceId, Long rollingConfigId) { + Record record = dsl().select( + TABLE.ID, + TABLE.TASK_INSTANCE_ID, + TABLE.CONFIG_NAME, + TABLE.CONFIG) .from(TABLE) - .where(TABLE.ID.eq(rollingConfigId)) + .where(TaskInstanceIdDynamicCondition.build(taskInstanceId, TABLE.TASK_INSTANCE_ID::eq)) + .and(TABLE.ID.eq(rollingConfigId)) .fetchOne(); return extract(record); } + @Override + @MySQLOperation(table = "rolling_config", op = DbOperationEnum.READ) + public boolean existsRollingConfig(long taskInstanceId) { + Result> records = dsl().selectOne() + .from(TABLE) + .where(TABLE.TASK_INSTANCE_ID.eq(taskInstanceId)) + .limit(1) + .fetch(); + return CollectionUtils.isNotEmpty(records); + } + private RollingConfigDTO extract(Record record) { if (record == null) { return null; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImpl.java index 43674c6265..699367101f 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptAgentTaskDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,16 +24,22 @@ package com.tencent.bk.job.execute.dao.impl; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.CompatibleType; import com.tencent.bk.job.common.constant.JobConstants; import com.tencent.bk.job.common.constant.Order; +import com.tencent.bk.job.common.mysql.dynamic.ds.DbOperationEnum; +import com.tencent.bk.job.common.mysql.dynamic.ds.MySQLOperation; import com.tencent.bk.job.execute.dao.ScriptAgentTaskDAO; -import com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum; -import com.tencent.bk.job.execute.model.AgentTaskDTO; -import com.tencent.bk.job.execute.model.AgentTaskResultGroupBaseDTO; +import com.tencent.bk.job.execute.dao.common.DSLContextProviderFactory; +import com.tencent.bk.job.execute.engine.consts.ExecuteObjectTaskStatusEnum; +import com.tencent.bk.job.execute.model.ExecuteObjectTask; +import com.tencent.bk.job.execute.model.ResultGroupBaseDTO; +import com.tencent.bk.job.execute.model.tables.GseScriptAgentTask; +import com.tencent.bk.job.execute.model.tables.records.GseScriptAgentTaskRecord; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.jooq.Condition; -import org.jooq.DSLContext; import org.jooq.OrderField; import org.jooq.Record; import org.jooq.Result; @@ -44,10 +50,7 @@ import org.jooq.UpdateConditionStep; import org.jooq.UpdateSetMoreStep; import org.jooq.UpdateSetStep; -import org.jooq.generated.tables.GseScriptAgentTask; -import org.jooq.generated.tables.records.GseScriptAgentTaskRecord; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; import java.util.ArrayList; @@ -59,10 +62,13 @@ import static org.jooq.impl.DSL.count; @Repository -public class ScriptAgentTaskDAOImpl implements ScriptAgentTaskDAO { +@Deprecated +@CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA) +public class ScriptAgentTaskDAOImpl extends BaseDAO implements ScriptAgentTaskDAO { private static final GseScriptAgentTask T_GSE_SCRIPT_AGENT_TASK = GseScriptAgentTask.GSE_SCRIPT_AGENT_TASK; private static final TableField[] ALL_FIELDS = { + T_GSE_SCRIPT_AGENT_TASK.TASK_INSTANCE_ID, T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID, T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT, T_GSE_SCRIPT_AGENT_TASK.ACTUAL_EXECUTE_COUNT, @@ -80,55 +86,57 @@ public class ScriptAgentTaskDAOImpl implements ScriptAgentTaskDAO { T_GSE_SCRIPT_AGENT_TASK.LOG_OFFSET }; - private final DSLContext CTX; - @Autowired - public ScriptAgentTaskDAOImpl(@Qualifier("job-execute-dsl-context") DSLContext CTX) { - this.CTX = CTX; + public ScriptAgentTaskDAOImpl(DSLContextProviderFactory dslContextProviderFactory) { + super(dslContextProviderFactory, T_GSE_SCRIPT_AGENT_TASK.getName()); } @Override - public void batchSaveAgentTasks(Collection agentTasks) { - String sql = "insert into gse_script_agent_task (step_instance_id, execute_count, actual_execute_count, batch," - + " host_id, agent_id, gse_task_id, status, start_time, end_time, total_time, error_code, exit_code, tag," - + " log_offset)" - + " values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; - Object[][] params = new Object[agentTasks.size()][15]; + @MySQLOperation(table = "gse_script_agent_task", op = DbOperationEnum.WRITE) + public void batchSaveAgentTasks(Collection agentTasks) { + String sql = "insert into gse_script_agent_task (task_instance_id, step_instance_id, execute_count, " + + "actual_execute_count, batch, host_id, agent_id, gse_task_id, status, start_time, end_time, " + + "total_time, error_code, exit_code, tag, log_offset) " + + "values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; + Object[][] params = new Object[agentTasks.size()][16]; int batchCount = 0; - for (AgentTaskDTO agentTask : agentTasks) { - Object[] param = new Object[15]; - param[0] = agentTask.getStepInstanceId(); - param[1] = agentTask.getExecuteCount(); - param[2] = agentTask.getActualExecuteCount(); - param[3] = agentTask.getBatch(); - param[4] = agentTask.getHostId(); - param[5] = agentTask.getAgentId(); - param[6] = agentTask.getGseTaskId(); - param[7] = agentTask.getStatus().getValue(); - param[8] = agentTask.getStartTime(); - param[9] = agentTask.getEndTime(); - param[10] = agentTask.getTotalTime(); - param[11] = agentTask.getErrorCode(); - param[12] = agentTask.getExitCode(); - param[13] = StringUtils.truncate(agentTask.getTag(), JobConstants.RESULT_GROUP_TAG_MAX_LENGTH); - param[14] = agentTask.getScriptLogOffset(); + for (ExecuteObjectTask agentTask : agentTasks) { + Object[] param = new Object[16]; + param[0] = agentTask.getTaskInstanceId(); + param[1] = agentTask.getStepInstanceId(); + param[2] = agentTask.getExecuteCount(); + param[3] = agentTask.getActualExecuteCount(); + param[4] = agentTask.getBatch(); + param[5] = agentTask.getHostId(); + param[6] = agentTask.getAgentId() == null ? "" : agentTask.getAgentId(); + param[7] = agentTask.getGseTaskId(); + param[8] = agentTask.getStatus().getValue(); + param[9] = agentTask.getStartTime(); + param[10] = agentTask.getEndTime(); + param[11] = agentTask.getTotalTime(); + param[12] = agentTask.getErrorCode(); + param[13] = agentTask.getExitCode(); + param[14] = StringUtils.truncate(agentTask.getTag(), JobConstants.RESULT_GROUP_TAG_MAX_LENGTH); + param[15] = agentTask.getScriptLogOffset(); params[batchCount++] = param; } - CTX.batch(sql, params).execute(); + dsl().batch(sql, params).execute(); } @Override - public void batchUpdateAgentTasks(Collection agentTasks) { + @MySQLOperation(table = "gse_script_agent_task", op = DbOperationEnum.WRITE) + public void batchUpdateAgentTasks(Collection agentTasks) { if (CollectionUtils.isEmpty(agentTasks)) { return; } String sql = "update gse_script_agent_task set gse_task_id = ?, status = ?, start_time = ?, end_time = ?" + ", total_time = ?, error_code = ?, exit_code = ?, tag = ?, log_offset = ?" - + " where step_instance_id = ? and execute_count = ? and batch = ? and host_id = ?"; - Object[][] params = new Object[agentTasks.size()][13]; + + " where task_instance_id = ? and step_instance_id = ? and execute_count = ?" + + " and batch = ? and host_id = ?"; + Object[][] params = new Object[agentTasks.size()][14]; int batchCount = 0; - for (AgentTaskDTO agentTask : agentTasks) { - Object[] param = new Object[13]; + for (ExecuteObjectTask agentTask : agentTasks) { + Object[] param = new Object[14]; param[0] = agentTask.getGseTaskId(); param[1] = agentTask.getStatus().getValue(); param[2] = agentTask.getStartTime(); @@ -138,34 +146,49 @@ public void batchUpdateAgentTasks(Collection agentTasks) { param[6] = agentTask.getExitCode(); param[7] = StringUtils.truncate(agentTask.getTag(), JobConstants.RESULT_GROUP_TAG_MAX_LENGTH); param[8] = agentTask.getScriptLogOffset(); - param[9] = agentTask.getStepInstanceId(); - param[10] = agentTask.getExecuteCount(); - param[11] = agentTask.getBatch(); - param[12] = agentTask.getHostId(); + param[9] = agentTask.getTaskInstanceId(); + param[10] = agentTask.getStepInstanceId(); + param[11] = agentTask.getExecuteCount(); + param[12] = agentTask.getBatch(); + param[13] = agentTask.getHostId(); params[batchCount++] = param; } - CTX.batch(sql, params).execute(); + dsl().batch(sql, params).execute(); } @Override - public int getSuccessAgentTaskCount(long stepInstanceId, int executeCount) { - Integer count = CTX.selectCount() + @MySQLOperation(table = "gse_script_agent_task", op = DbOperationEnum.READ) + public int getSuccessAgentTaskCount(Long taskInstanceId, long stepInstanceId, int executeCount) { + Integer count = dsl().selectCount() .from(T_GSE_SCRIPT_AGENT_TASK) - .where(T_GSE_SCRIPT_AGENT_TASK.STATUS.in(AgentTaskStatusEnum.LAST_SUCCESS.getValue(), - AgentTaskStatusEnum.SUCCESS.getValue())) + .where(T_GSE_SCRIPT_AGENT_TASK.STATUS.in(ExecuteObjectTaskStatusEnum.LAST_SUCCESS.getValue(), + ExecuteObjectTaskStatusEnum.SUCCESS.getValue())) .and(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq((short) executeCount)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)) .fetchOne(0, Integer.class); return count == null ? 0 : count; } + private Condition buildTaskInstanceIdQueryCondition(Long taskInstanceId) { + return TaskInstanceIdDynamicCondition.build( + taskInstanceId, + T_GSE_SCRIPT_AGENT_TASK.TASK_INSTANCE_ID::eq + ); + } + @Override - public List listResultGroups(long stepInstanceId, int executeCount, Integer batch) { + @MySQLOperation(table = "gse_script_agent_task", op = DbOperationEnum.READ) + public List listResultGroups(Long taskInstanceId, + long stepInstanceId, + int executeCount, + Integer batch) { SelectConditionStep selectConditionStep = - CTX.select(T_GSE_SCRIPT_AGENT_TASK.STATUS, T_GSE_SCRIPT_AGENT_TASK.TAG, count().as("ip_count")) + dsl().select(T_GSE_SCRIPT_AGENT_TASK.STATUS, T_GSE_SCRIPT_AGENT_TASK.TAG, count().as("ip_count")) .from(T_GSE_SCRIPT_AGENT_TASK) .where(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq((short) executeCount)); + .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq((short) executeCount)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)); if (batch != null && batch > 0) { selectConditionStep.and(T_GSE_SCRIPT_AGENT_TASK.BATCH.eq(batch.shortValue())); } @@ -174,58 +197,64 @@ public List listResultGroups(long stepInstanceId, i .orderBy(T_GSE_SCRIPT_AGENT_TASK.STATUS.asc()) .fetch(); - List resultGroups = new ArrayList<>(); + List resultGroups = new ArrayList<>(); result.forEach(record -> { - AgentTaskResultGroupBaseDTO resultGroup = new AgentTaskResultGroupBaseDTO(); + ResultGroupBaseDTO resultGroup = new ResultGroupBaseDTO(); resultGroup.setStatus(record.get(T_GSE_SCRIPT_AGENT_TASK.STATUS)); resultGroup.setTag(record.get(T_GSE_SCRIPT_AGENT_TASK.TAG)); Object ipCount = record.get("ip_count"); - resultGroup.setTotalAgentTasks(ipCount == null ? 0 : (int) ipCount); + resultGroup.setTotal(ipCount == null ? 0 : (int) ipCount); resultGroups.add(resultGroup); }); return resultGroups; } @Override - public List listAgentTaskByResultGroup(Long stepInstanceId, - Integer executeCount, - Integer batch, - Integer status, - String tag) { - SelectConditionStep selectConditionStep = CTX.select(ALL_FIELDS) + @MySQLOperation(table = "gse_script_agent_task", op = DbOperationEnum.READ) + public List listAgentTaskByResultGroup(Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch, + Integer status, + String tag) { + SelectConditionStep selectConditionStep = dsl().select(ALL_FIELDS) .from(T_GSE_SCRIPT_AGENT_TASK) .where(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())) .and(T_GSE_SCRIPT_AGENT_TASK.STATUS.eq(status)) - .and(T_GSE_SCRIPT_AGENT_TASK.TAG.eq(tag == null ? "" : tag)); + .and(T_GSE_SCRIPT_AGENT_TASK.TAG.eq(tag == null ? "" : tag)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)); if (batch != null && batch > 0) { selectConditionStep.and(T_GSE_SCRIPT_AGENT_TASK.BATCH.eq(batch.shortValue())); } Result result = selectConditionStep.fetch(); - List agentTasks = new ArrayList<>(); - if (result.size() > 0) { + List agentTasks = new ArrayList<>(); + if (!result.isEmpty()) { result.forEach(record -> agentTasks.add(extract(record))); } return agentTasks; } @Override - public List listAgentTaskByResultGroup(Long stepInstanceId, - Integer executeCount, - Integer batch, - Integer status, - String tag, - Integer limit, - String orderField, - Order order) { + @MySQLOperation(table = "gse_script_agent_task", op = DbOperationEnum.READ) + public List listAgentTaskByResultGroup(Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch, + Integer status, + String tag, + Integer limit, + String orderField, + Order order) { List conditions = new ArrayList<>(); conditions.add(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)); conditions.add(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())); conditions.add(T_GSE_SCRIPT_AGENT_TASK.STATUS.eq(status)); conditions.add(T_GSE_SCRIPT_AGENT_TASK.TAG.eq(tag == null ? "" : tag)); + conditions.add(buildTaskInstanceIdQueryCondition(taskInstanceId)); - SelectConditionStep select = CTX.select(ALL_FIELDS) + SelectConditionStep select = dsl().select(ALL_FIELDS) .from(T_GSE_SCRIPT_AGENT_TASK) .where(conditions); @@ -248,7 +277,7 @@ public List listAgentTaskByResultGroup(Long stepInstanceId, } } - List agentTasks = new ArrayList<>(); + List agentTasks = new ArrayList<>(); Result result; if (selectLimitPercentStep != null) { result = selectLimitPercentStep.fetch(); @@ -258,7 +287,7 @@ public List listAgentTaskByResultGroup(Long stepInstanceId, result = select.fetch(); } - if (result.size() > 0) { + if (!result.isEmpty()) { result.into(record -> agentTasks.add(extract(record))); } return agentTasks; @@ -285,19 +314,22 @@ private OrderField buildOrderField(String field, Order order) { } @Override - public List listAgentTasks(Long stepInstanceId, - Integer executeCount, - Integer batch) { - SelectConditionStep selectConditionStep = CTX.select(ALL_FIELDS) + @MySQLOperation(table = "gse_script_agent_task", op = DbOperationEnum.READ) + public List listAgentTasks(Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch) { + SelectConditionStep selectConditionStep = dsl().select(ALL_FIELDS) .from(T_GSE_SCRIPT_AGENT_TASK) .where(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())); + .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)); if (batch != null && batch > 0) { selectConditionStep.and(T_GSE_SCRIPT_AGENT_TASK.BATCH.eq(batch.shortValue())); } Result result = selectConditionStep.fetch(); - List agentTaskList = new ArrayList<>(); - if (result.size() != 0) { + List agentTaskList = new ArrayList<>(); + if (!result.isEmpty()) { result.map(record -> { agentTaskList.add(extract(record)); return null; @@ -306,11 +338,12 @@ public List listAgentTasks(Long stepInstanceId, return agentTaskList; } - private AgentTaskDTO extract(Record record) { + private ExecuteObjectTask extract(Record record) { if (record == null) { return null; } - AgentTaskDTO agentTask = new AgentTaskDTO(); + ExecuteObjectTask agentTask = new ExecuteObjectTask(); + agentTask.setTaskInstanceId(record.get(T_GSE_SCRIPT_AGENT_TASK.TASK_INSTANCE_ID)); agentTask.setStepInstanceId(record.get(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID)); agentTask.setExecuteCount(record.get(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT)); Short actualExecuteCount = record.get(T_GSE_SCRIPT_AGENT_TASK.ACTUAL_EXECUTE_COUNT); @@ -319,7 +352,7 @@ private AgentTaskDTO extract(Record record) { agentTask.setHostId(record.get(T_GSE_SCRIPT_AGENT_TASK.HOST_ID)); agentTask.setAgentId(record.get(T_GSE_SCRIPT_AGENT_TASK.AGENT_ID)); agentTask.setGseTaskId(record.get(T_GSE_SCRIPT_AGENT_TASK.GSE_TASK_ID)); - agentTask.setStatus(AgentTaskStatusEnum.valueOf(record.get(T_GSE_SCRIPT_AGENT_TASK.STATUS))); + agentTask.setStatus(ExecuteObjectTaskStatusEnum.valOf(record.get(T_GSE_SCRIPT_AGENT_TASK.STATUS))); agentTask.setStartTime(record.get(T_GSE_SCRIPT_AGENT_TASK.START_TIME)); agentTask.setEndTime(record.get(T_GSE_SCRIPT_AGENT_TASK.END_TIME)); agentTask.setTotalTime(record.get(T_GSE_SCRIPT_AGENT_TASK.TOTAL_TIME)); @@ -331,48 +364,58 @@ private AgentTaskDTO extract(Record record) { } @Override - public List listAgentTasksByGseTaskId(Long gseTaskId) { + @MySQLOperation(table = "gse_script_agent_task", op = DbOperationEnum.READ) + public List listAgentTasksByGseTaskId(Long taskInstanceId, Long gseTaskId) { if (gseTaskId == null || gseTaskId <= 0) { return Collections.emptyList(); } - List agentTaskList = new ArrayList<>(); + List agentTaskList = new ArrayList<>(); - Result result = CTX.select(ALL_FIELDS) + Result result = dsl().select(ALL_FIELDS) .from(T_GSE_SCRIPT_AGENT_TASK) .where(T_GSE_SCRIPT_AGENT_TASK.GSE_TASK_ID.eq(gseTaskId)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)) .fetch(); - if (result.size() > 0) { + if (!result.isEmpty()) { result.forEach(record -> agentTaskList.add(extract(record))); } return agentTaskList; } @Override - public AgentTaskDTO getAgentTaskByHostId(Long stepInstanceId, Integer executeCount, Integer batch, long hostId) { - Record record = CTX.select(ALL_FIELDS) - .from(T_GSE_SCRIPT_AGENT_TASK) - .where(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())) - .and(T_GSE_SCRIPT_AGENT_TASK.BATCH.eq(batch == null ? 0 : batch.shortValue())) - .and(T_GSE_SCRIPT_AGENT_TASK.HOST_ID.eq(hostId)) - .fetchOne(); + @MySQLOperation(table = "gse_script_agent_task", op = DbOperationEnum.READ) + public ExecuteObjectTask getAgentTaskByHostId(Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch, + long hostId) { + SelectConditionStep selectConditionStep = + dsl().select(ALL_FIELDS) + .from(T_GSE_SCRIPT_AGENT_TASK) + .where(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq(executeCount.shortValue())) + .and(T_GSE_SCRIPT_AGENT_TASK.HOST_ID.eq(hostId)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)); + if (batch != null && batch > 0) { + // 滚动执行批次，传入null或者0将忽略该参数 + selectConditionStep.and(T_GSE_SCRIPT_AGENT_TASK.BATCH.eq(batch.shortValue())); + } + selectConditionStep.limit(1); + Record record = selectConditionStep.fetchOne(); return extract(record); } - @Override - public boolean isStepInstanceRecordExist(long stepInstanceId) { - return CTX.fetchExists(T_GSE_SCRIPT_AGENT_TASK, T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)); - } - @Override - public void updateAgentTaskFields(long stepInstanceId, + @MySQLOperation(table = "gse_script_agent_task", op = DbOperationEnum.WRITE) + public void updateAgentTaskFields(Long taskInstanceId, + long stepInstanceId, int executeCount, Integer batch, Integer actualExecuteCount, Long gseTaskId) { - UpdateSetStep updateSetStep = CTX.update(T_GSE_SCRIPT_AGENT_TASK); + UpdateSetStep updateSetStep = dsl().update(T_GSE_SCRIPT_AGENT_TASK); boolean needUpdate = false; if (actualExecuteCount != null) { updateSetStep = updateSetStep.set(T_GSE_SCRIPT_AGENT_TASK.ACTUAL_EXECUTE_COUNT, @@ -394,7 +437,8 @@ public void updateAgentTaskFields(long stepInstanceId, UpdateConditionStep updateConditionStep = updateSetMoreStep .where(T_GSE_SCRIPT_AGENT_TASK.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq((short) executeCount)); + .and(T_GSE_SCRIPT_AGENT_TASK.EXECUTE_COUNT.eq((short) executeCount)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)); if (batch != null) { updateConditionStep.and(T_GSE_SCRIPT_AGENT_TASK.BATCH.eq(batch.shortValue())); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptExecuteObjectTaskDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptExecuteObjectTaskDAOImpl.java new file mode 100644 index 0000000000..b3f4507182 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/ScriptExecuteObjectTaskDAOImpl.java @@ -0,0 +1,477 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.dao.impl; + +import com.tencent.bk.job.common.constant.ExecuteObjectTypeEnum; +import com.tencent.bk.job.common.constant.JobConstants; +import com.tencent.bk.job.common.constant.Order; +import com.tencent.bk.job.common.mysql.dynamic.ds.DbOperationEnum; +import com.tencent.bk.job.common.mysql.dynamic.ds.MySQLOperation; +import com.tencent.bk.job.execute.dao.ScriptExecuteObjectTaskDAO; +import com.tencent.bk.job.execute.dao.common.DSLContextProviderFactory; +import com.tencent.bk.job.execute.engine.consts.ExecuteObjectTaskStatusEnum; +import com.tencent.bk.job.execute.model.ExecuteObjectTask; +import com.tencent.bk.job.execute.model.ResultGroupBaseDTO; +import com.tencent.bk.job.execute.model.tables.GseScriptExecuteObjTask; +import com.tencent.bk.job.execute.model.tables.records.GseScriptExecuteObjTaskRecord; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.jooq.Condition; +import org.jooq.OrderField; +import org.jooq.Record; +import org.jooq.Result; +import org.jooq.SelectConditionStep; +import org.jooq.SelectLimitPercentStep; +import org.jooq.SelectSeekStep1; +import org.jooq.TableField; +import org.jooq.UpdateConditionStep; +import org.jooq.UpdateSetMoreStep; +import org.jooq.UpdateSetStep; +import org.springframework.stereotype.Repository; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; + +import static com.tencent.bk.job.common.constant.Order.DESCENDING; +import static org.jooq.impl.DSL.count; + +@Repository +public class ScriptExecuteObjectTaskDAOImpl extends BaseDAO implements ScriptExecuteObjectTaskDAO { + + private static final GseScriptExecuteObjTask T = GseScriptExecuteObjTask.GSE_SCRIPT_EXECUTE_OBJ_TASK; + + private static final TableField[] ALL_FIELDS = { + T.TASK_INSTANCE_ID, + T.STEP_INSTANCE_ID, + T.EXECUTE_COUNT, + T.ACTUAL_EXECUTE_COUNT, + T.BATCH, + T.EXECUTE_OBJ_TYPE, + T.EXECUTE_OBJ_ID, + T.GSE_TASK_ID, + T.STATUS, + T.START_TIME, + T.END_TIME, + T.TOTAL_TIME, + T.ERROR_CODE, + T.EXIT_CODE, + T.TAG, + T.LOG_OFFSET + }; + + private static final String BATCH_INSERT_SQL = + "insert into gse_script_execute_obj_task (id,task_instance_id,step_instance_id,execute_count," + + "actual_execute_count,batch,execute_obj_type,execute_obj_id,gse_task_id,status,start_time,end_time," + + "total_time,error_code,exit_code,tag,log_offset) values (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"; + + private static final String BATCH_UPDATE_SQL = + "update gse_script_execute_obj_task set gse_task_id = ?, status = ?, start_time = ?, end_time = ?" + + ", total_time = ?, error_code = ?, exit_code = ?, tag = ?, log_offset = ?" + + " where task_instance_id = ? and step_instance_id = ? and execute_count = ? and batch = ?" + + " and execute_obj_id = ?"; + + public ScriptExecuteObjectTaskDAOImpl(DSLContextProviderFactory dslContextProviderFactory) { + super(dslContextProviderFactory, T.getName()); + } + + @Override + @MySQLOperation(table = "gse_script_execute_obj_task", op = DbOperationEnum.WRITE) + public void batchSaveTasks(Collection tasks) { + Object[][] params = new Object[tasks.size()][17]; + int batchCount = 0; + for (ExecuteObjectTask task : tasks) { + Object[] param = new Object[17]; + param[0] = task.getId(); + param[1] = task.getTaskInstanceId(); + param[2] = task.getStepInstanceId(); + param[3] = task.getExecuteCount(); + param[4] = task.getActualExecuteCount(); + param[5] = task.getBatch(); + param[6] = task.getExecuteObjectType().getValue(); + param[7] = task.getExecuteObjectId(); + param[8] = task.getGseTaskId(); + param[9] = task.getStatus().getValue(); + param[10] = task.getStartTime(); + param[11] = task.getEndTime(); + param[12] = task.getTotalTime(); + param[13] = task.getErrorCode(); + param[14] = task.getExitCode(); + param[15] = StringUtils.truncate(task.getTag(), JobConstants.RESULT_GROUP_TAG_MAX_LENGTH); + param[16] = task.getScriptLogOffset(); + params[batchCount++] = param; + } + dsl().batch(BATCH_INSERT_SQL, params).execute(); + } + + @Override + @MySQLOperation(table = "gse_script_execute_obj_task", op = DbOperationEnum.WRITE) + public void batchUpdateTasks(Collection tasks) { + if (CollectionUtils.isEmpty(tasks)) { + return; + } + Object[][] params = new Object[tasks.size()][14]; + int batchCount = 0; + for (ExecuteObjectTask task : tasks) { + Object[] param = new Object[14]; + param[0] = task.getGseTaskId(); + param[1] = task.getStatus().getValue(); + param[2] = task.getStartTime(); + param[3] = task.getEndTime(); + param[4] = task.getTotalTime(); + param[5] = task.getErrorCode(); + param[6] = task.getExitCode(); + param[7] = StringUtils.truncate(task.getTag(), JobConstants.RESULT_GROUP_TAG_MAX_LENGTH); + param[8] = task.getScriptLogOffset(); + param[9] = task.getTaskInstanceId(); + param[10] = task.getStepInstanceId(); + param[11] = task.getExecuteCount(); + param[12] = task.getBatch(); + param[13] = task.getExecuteObjectId(); + params[batchCount++] = param; + } + dsl().batch(BATCH_UPDATE_SQL, params).execute(); + } + + @Override + @MySQLOperation(table = "gse_script_execute_obj_task", op = DbOperationEnum.READ) + public int getSuccessTaskCount(Long taskInstanceId, long stepInstanceId, int executeCount) { + Integer count = dsl().selectCount() + .from(T) + .where(T.STATUS.in(ExecuteObjectTaskStatusEnum.LAST_SUCCESS.getValue(), + ExecuteObjectTaskStatusEnum.SUCCESS.getValue())) + .and(T.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(T.EXECUTE_COUNT.eq((short) executeCount)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)) + .fetchOne(0, Integer.class); + return count == null ? 0 : count; + } + + private Condition buildTaskInstanceIdQueryCondition(Long taskInstanceId) { + return TaskInstanceIdDynamicCondition.build( + taskInstanceId, + T.TASK_INSTANCE_ID::eq + ); + } + + @Override + @MySQLOperation(table = "gse_script_execute_obj_task", op = DbOperationEnum.READ) + public List listResultGroups(Long taskInstanceId, + long stepInstanceId, + int executeCount, + Integer batch) { + SelectConditionStep selectConditionStep = + dsl().select(T.STATUS, T.TAG, count().as("task_count")) + .from(T) + .where(T.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(T.EXECUTE_COUNT.eq((short) executeCount)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)); + if (batch != null && batch > 0) { + selectConditionStep.and(T.BATCH.eq(batch.shortValue())); + } + + Result result = selectConditionStep.groupBy(T.STATUS, T.TAG) + .orderBy(T.STATUS.asc()) + .fetch(); + + List resultGroups = new ArrayList<>(); + result.forEach(record -> { + ResultGroupBaseDTO resultGroup = new ResultGroupBaseDTO(); + resultGroup.setStatus(record.get(T.STATUS).intValue()); + resultGroup.setTag(record.get(T.TAG)); + Object taskCount = record.get("task_count"); + resultGroup.setTotal(taskCount == null ? 0 : (int) taskCount); + resultGroups.add(resultGroup); + }); + return resultGroups; + } + + @Override + @MySQLOperation(table = "gse_script_execute_obj_task", op = DbOperationEnum.READ) + public List listTasksByResultGroup(Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch, + Integer status, + String tag) { + SelectConditionStep selectConditionStep = dsl().select(ALL_FIELDS) + .from(T) + .where(T.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(T.EXECUTE_COUNT.eq(executeCount.shortValue())) + .and(T.STATUS.eq(status)) + .and(T.TAG.eq(tag == null ? "" : tag)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)); + if (batch != null && batch > 0) { + selectConditionStep.and(T.BATCH.eq(batch.shortValue())); + } + Result result = selectConditionStep.fetch(); + + List executeObjectTasks = new ArrayList<>(); + if (result.size() > 0) { + result.forEach(record -> executeObjectTasks.add(extract(record))); + } + return executeObjectTasks; + } + + private ExecuteObjectTask extract(Record record) { + if (record == null) { + return null; + } + ExecuteObjectTask executeObjectTask = new ExecuteObjectTask(); + executeObjectTask.setTaskInstanceId(record.get(T.TASK_INSTANCE_ID)); + executeObjectTask.setStepInstanceId(record.get(T.STEP_INSTANCE_ID)); + executeObjectTask.setExecuteCount(record.get(T.EXECUTE_COUNT)); + Short actualExecuteCount = record.get(T.ACTUAL_EXECUTE_COUNT); + executeObjectTask.setActualExecuteCount(actualExecuteCount != null ? actualExecuteCount.intValue() : null); + executeObjectTask.setBatch(record.get(T.BATCH)); + executeObjectTask.setExecuteObjectId(record.get(T.EXECUTE_OBJ_ID)); + executeObjectTask.setExecuteObjectType(ExecuteObjectTypeEnum.valOf(record.get(T.EXECUTE_OBJ_TYPE).intValue())); + executeObjectTask.setGseTaskId(record.get(T.GSE_TASK_ID)); + executeObjectTask.setStatus(ExecuteObjectTaskStatusEnum.valOf(record.get(T.STATUS))); + executeObjectTask.setStartTime(record.get(T.START_TIME)); + executeObjectTask.setEndTime(record.get(T.END_TIME)); + executeObjectTask.setTotalTime(record.get(T.TOTAL_TIME)); + executeObjectTask.setErrorCode(record.get(T.ERROR_CODE)); + executeObjectTask.setExitCode(record.get(T.EXIT_CODE, Integer.class)); + executeObjectTask.setTag(record.get(T.TAG)); + executeObjectTask.setScriptLogOffset(record.get(T.LOG_OFFSET)); + return executeObjectTask; + } + + @Override + @MySQLOperation(table = "gse_script_execute_obj_task", op = DbOperationEnum.READ) + public List listTasksByResultGroup(Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch, + Integer status, + String tag, + Integer limit, + String orderField, + Order order) { + List conditions = new ArrayList<>(); + conditions.add(T.STEP_INSTANCE_ID.eq(stepInstanceId)); + conditions.add(T.EXECUTE_COUNT.eq(executeCount.shortValue())); + conditions.add(T.STATUS.eq(status)); + conditions.add(T.TAG.eq(tag == null ? "" : tag)); + conditions.add(buildTaskInstanceIdQueryCondition(taskInstanceId)); + + SelectConditionStep select = dsl().select(ALL_FIELDS) + .from(T) + .where(conditions); + + if (batch != null && batch > 0) { + select.and(T.BATCH.eq(batch.shortValue())); + } + + SelectSeekStep1 selectSeekStep = null; + OrderField orderFieldEntity = buildOrderField(orderField, order); + if (orderFieldEntity != null) { + selectSeekStep = select.orderBy(orderFieldEntity); + } + + SelectLimitPercentStep selectLimitPercentStep = null; + if (limit != null && limit > 0) { + if (selectSeekStep != null) { + selectLimitPercentStep = selectSeekStep.limit(limit); + } else { + selectLimitPercentStep = select.limit(limit); + } + } + + List executeObjectTasks = new ArrayList<>(); + Result result; + if (selectLimitPercentStep != null) { + result = selectLimitPercentStep.fetch(); + } else if (selectSeekStep != null) { + result = selectSeekStep.fetch(); + } else { + result = select.fetch(); + } + + if (result.size() > 0) { + result.into(record -> executeObjectTasks.add(extract(record))); + } + return executeObjectTasks; + } + + private OrderField buildOrderField(String field, Order order) { + OrderField orderField = null; + if (StringUtils.isNotBlank(field)) { + if (field.equals(T.TOTAL_TIME.getName())) { + if (order == DESCENDING) { + orderField = T.TOTAL_TIME.desc(); + } else { + orderField = T.TOTAL_TIME.asc(); + } + } else if (field.equals(T.EXIT_CODE.getName())) { + if (order == DESCENDING) { + orderField = T.EXIT_CODE.desc(); + } else { + orderField = T.EXIT_CODE.asc(); + } + } + } + return orderField; + } + + @Override + @MySQLOperation(table = "gse_script_execute_obj_task", op = DbOperationEnum.READ) + public List listTasks(Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch) { + SelectConditionStep selectConditionStep = dsl().select(ALL_FIELDS) + .from(T) + .where(T.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(T.EXECUTE_COUNT.eq(executeCount.shortValue())) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)); + if (batch != null && batch > 0) { + selectConditionStep.and(T.BATCH.eq(batch.shortValue())); + } + Result result = selectConditionStep.fetch(); + List executeObjectList = new ArrayList<>(); + if (result.size() != 0) { + result.map(record -> { + executeObjectList.add(extract(record)); + return null; + }); + } + return executeObjectList; + } + + @Override + @MySQLOperation(table = "gse_script_execute_obj_task", op = DbOperationEnum.READ) + public List listTasksByGseTaskId(Long taskInstanceId, Long gseTaskId) { + if (gseTaskId == null || gseTaskId <= 0) { + return Collections.emptyList(); + } + + List executeObjectList = new ArrayList<>(); + + Result result = dsl().select(ALL_FIELDS) + .from(T) + .where(T.GSE_TASK_ID.eq(gseTaskId)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)) + .fetch(); + if (result.size() > 0) { + result.forEach(record -> executeObjectList.add(extract(record))); + } + return executeObjectList; + } + + @Override + @MySQLOperation(table = "gse_script_execute_obj_task", op = DbOperationEnum.READ) + public ExecuteObjectTask getTaskByExecuteObjectId(Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch, + String executeObjectId) { + SelectConditionStep selectConditionStep = + dsl().select(ALL_FIELDS) + .from(T) + .where(buildTaskInstanceIdQueryCondition(taskInstanceId)) + .and(T.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(T.EXECUTE_COUNT.eq(executeCount.shortValue())) + .and(T.EXECUTE_OBJ_ID.eq(executeObjectId)); + if (batch != null && batch > 0) { + // 滚动执行批次，传入null或者0将忽略该参数 + selectConditionStep.and(T.BATCH.eq(batch.shortValue())); + } + selectConditionStep.limit(1); + Record record = selectConditionStep.fetchOne(); + return extract(record); + } + + @Override + @MySQLOperation(table = "gse_script_execute_obj_task", op = DbOperationEnum.READ) + public List getTaskByExecuteObjectIds(Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch, + Collection executeObjectIds) { + SelectConditionStep selectConditionStep = + dsl().select(ALL_FIELDS) + .from(T) + .where(buildTaskInstanceIdQueryCondition(taskInstanceId)) + .and(T.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(T.EXECUTE_COUNT.eq(executeCount.shortValue())) + .and(T.EXECUTE_OBJ_ID.in(executeObjectIds)); + if (batch != null && batch > 0) { + // 滚动执行批次，传入null或者0将忽略该参数 + selectConditionStep.and(T.BATCH.eq(batch.shortValue())); + } + Result records = selectConditionStep.fetch(); + return records.map(this::extract); + } + + @Override + @MySQLOperation(table = "gse_script_execute_obj_task", op = DbOperationEnum.READ) + public boolean isStepInstanceRecordExist(Long taskInstanceId, long stepInstanceId) { + return dsl().fetchExists( + T, + T.STEP_INSTANCE_ID.eq(stepInstanceId), + buildTaskInstanceIdQueryCondition(taskInstanceId)); + } + + @Override + @MySQLOperation(table = "gse_script_execute_obj_task", op = DbOperationEnum.WRITE) + public void updateTaskFields(Long taskInstanceId, + long stepInstanceId, + int executeCount, + Integer batch, + Integer actualExecuteCount, + Long gseTaskId) { + UpdateSetStep updateSetStep = dsl().update(T); + boolean needUpdate = false; + if (actualExecuteCount != null) { + updateSetStep = updateSetStep.set(T.ACTUAL_EXECUTE_COUNT, + actualExecuteCount.shortValue()); + needUpdate = true; + } + if (gseTaskId != null) { + updateSetStep = updateSetStep.set(T.GSE_TASK_ID, gseTaskId); + needUpdate = true; + } + + if (!needUpdate) { + return; + } + + UpdateSetMoreStep updateSetMoreStep = + (UpdateSetMoreStep) updateSetStep; + + UpdateConditionStep updateConditionStep = + updateSetMoreStep + .where(T.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)) + .and(T.EXECUTE_COUNT.eq((short) executeCount)); + if (batch != null) { + updateConditionStep.and(T.BATCH.eq(batch.shortValue())); + } + updateConditionStep.execute(); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StatisticsDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StatisticsDAOImpl.java index 4c466a8a16..fa0d36069d 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StatisticsDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StatisticsDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,161 +24,53 @@ package com.tencent.bk.job.execute.dao.impl; -import com.tencent.bk.job.common.statistics.model.dto.StatisticsDTO; -import com.tencent.bk.job.common.util.Wrapper; +import com.tencent.bk.job.analysis.api.dto.StatisticsDTO; +import com.tencent.bk.job.common.mysql.dynamic.ds.DbOperationEnum; +import com.tencent.bk.job.common.mysql.dynamic.ds.MySQLOperation; import com.tencent.bk.job.execute.dao.StatisticsDAO; +import com.tencent.bk.job.execute.dao.common.DSLContextProviderFactory; +import com.tencent.bk.job.execute.model.tables.Statistics; +import com.tencent.bk.job.execute.model.tables.records.StatisticsRecord; +import com.tencent.bk.job.execute.statistics.StatisticsKey; import lombok.extern.slf4j.Slf4j; import lombok.val; import lombok.var; import org.apache.commons.lang3.StringUtils; import org.jooq.Condition; -import org.jooq.Configuration; import org.jooq.DSLContext; -import org.jooq.Record1; +import org.jooq.Record2; import org.jooq.Result; -import org.jooq.TransactionalRunnable; import org.jooq.conf.ParamType; -import org.jooq.generated.tables.Statistics; -import org.jooq.generated.tables.records.StatisticsRecord; +import org.jooq.exception.DataAccessException; import org.jooq.impl.DSL; import org.jooq.types.ULong; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Repository; -import java.math.BigDecimal; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; -/** - * @Description - * @Date 2020/3/6 - * @Version 1.0 - */ -@Repository +@Repository("jobExecuteStatisticsDAOImpl") @Slf4j -public class StatisticsDAOImpl implements StatisticsDAO { +public class StatisticsDAOImpl extends BaseDAO implements StatisticsDAO { private static final Statistics defaultTable = Statistics.STATISTICS; - private final DSLContext defaultDSLContext; @Autowired - public StatisticsDAOImpl(DSLContext dslContext) { - this.defaultDSLContext = dslContext; - } - - @Override - public Long insertStatistics(DSLContext dslContext, StatisticsDTO statisticsDTO) { - if (statisticsDTO == null) { - return -1L; - } - if (statisticsDTO.getCreateTime() == null) { - statisticsDTO.setCreateTime(System.currentTimeMillis()); - } - val query = dslContext.insertInto(defaultTable, - defaultTable.ID, - defaultTable.APP_ID, - defaultTable.RESOURCE, - defaultTable.DIMENSION, - defaultTable.DIMENSION_VALUE, - defaultTable.DATE, - defaultTable.VALUE, - defaultTable.CREATE_TIME, - defaultTable.LAST_MODIFY_TIME - ).values( - null, - statisticsDTO.getAppId(), - statisticsDTO.getResource(), - statisticsDTO.getDimension(), - statisticsDTO.getDimensionValue(), - statisticsDTO.getDate(), - statisticsDTO.getValue(), - ULong.valueOf(statisticsDTO.getCreateTime()), - ULong.valueOf(System.currentTimeMillis()) - ).returning(defaultTable.ID); - val sql = query.getSQL(ParamType.INLINED); - try { - return query.fetchOne().getId(); - } catch (Exception e) { - log.error(sql); - throw e; - } - } - - private Collection genConditions(Long appId, String resource, String dimension, String dimensionValue, - String date) { - List conditions = new ArrayList<>(); - conditions.add(defaultTable.APP_ID.eq(appId)); - conditions.add(defaultTable.RESOURCE.eq(resource)); - conditions.add(defaultTable.DIMENSION.eq(dimension)); - conditions.add(defaultTable.DIMENSION_VALUE.eq(dimensionValue)); - conditions.add(defaultTable.DATE.eq(date)); - return conditions; - } - - public Boolean exist(DSLContext dslContext, Long appId, String resource, String dimension, String dimensionValue, - String date) { - Collection conditions = genConditions(appId, resource, dimension, dimensionValue, date); - return dslContext.fetchExists(defaultTable, conditions); - } - - @Override - public Long upsertStatistics(DSLContext dslContext, StatisticsDTO statisticsDTO) { - Wrapper idWrapper = new Wrapper<>(-1L); - dslContext.transaction(new TransactionalRunnable() { - @Override - public void run(Configuration configuration) throws Throwable { - DSLContext context = DSL.using(configuration); - StatisticsDTO oldStatisticsDTO = getStatistics(context, statisticsDTO.getAppId(), - statisticsDTO.getResource(), statisticsDTO.getDimension(), statisticsDTO.getDimensionValue(), - statisticsDTO.getDate()); - if (oldStatisticsDTO == null) { - idWrapper.setValue(insertStatistics(context, statisticsDTO)); - } else { - oldStatisticsDTO.setValue(statisticsDTO.getValue()); - updateStatisticsById(context, oldStatisticsDTO); - idWrapper.setValue(oldStatisticsDTO.getId()); - } - } - }); - return idWrapper.getValue(); - } - - @Override - public int updateStatisticsById(DSLContext dslContext, StatisticsDTO statisticsDTO) { - val query = dslContext.update(defaultTable) - .set(defaultTable.APP_ID, statisticsDTO.getAppId()) - .set(defaultTable.RESOURCE, statisticsDTO.getResource()) - .set(defaultTable.DIMENSION, statisticsDTO.getDimension()) - .set(defaultTable.DIMENSION_VALUE, statisticsDTO.getDimensionValue()) - .set(defaultTable.DATE, statisticsDTO.getDate()) - .set(defaultTable.VALUE, statisticsDTO.getValue()) - .set(defaultTable.CREATE_TIME, ULong.valueOf(statisticsDTO.getCreateTime())) - .set(defaultTable.LAST_MODIFY_TIME, ULong.valueOf(System.currentTimeMillis())) - .where(defaultTable.ID.eq(statisticsDTO.getId())); - val sql = query.getSQL(ParamType.INLINED); - try { - return query.execute(); - } catch (Exception e) { - log.error(sql); - throw e; - } - } - - @Override - public int deleteStatisticsById(DSLContext dslContext, Long id) { - return dslContext.deleteFrom(defaultTable).where( - defaultTable.ID.eq(id) - ).execute(); + public StatisticsDAOImpl(DSLContextProviderFactory dslContextProviderFactory) { + super(dslContextProviderFactory, defaultTable.getName()); } @Override + @MySQLOperation(table = "statistics", op = DbOperationEnum.WRITE) public int deleteStatisticsByDate(String date) { int totalAffectedRows = 0; int affectedRows; do { - affectedRows = defaultDSLContext.deleteFrom(defaultTable).where( + affectedRows = dsl().deleteFrom(defaultTable).where( defaultTable.DATE.lessThan(date) ).limit(10000).execute(); totalAffectedRows += affectedRows; @@ -187,6 +79,7 @@ public int deleteStatisticsByDate(String date) { } @Override + @MySQLOperation(table = "statistics", op = DbOperationEnum.WRITE) public int deleteOneDayStatistics(Long appId, String date) { int totalAffectedRows = 0; int affectedRows; @@ -198,7 +91,7 @@ public int deleteOneDayStatistics(Long appId, String date) { conditions.add(defaultTable.DATE.eq(date)); } do { - affectedRows = defaultDSLContext.deleteFrom(defaultTable) + affectedRows = dsl().deleteFrom(defaultTable) .where(conditions) .limit(10000).execute(); totalAffectedRows += affectedRows; @@ -206,22 +99,12 @@ public int deleteOneDayStatistics(Long appId, String date) { return totalAffectedRows; } - @Override - public StatisticsDTO getStatisticsById(Long id) { - val record = defaultDSLContext.selectFrom(defaultTable).where( - defaultTable.ID.eq(id) - ).fetchOne(); - if (record == null) { - return null; - } else { - return convert(record); - } - } @Override + @MySQLOperation(table = "statistics", op = DbOperationEnum.READ) public StatisticsDTO getStatistics(Long appId, String resource, String dimension, String dimensionValue, String date) { - val record = defaultDSLContext.selectFrom(defaultTable) + val record = dsl().selectFrom(defaultTable) .where(defaultTable.APP_ID.eq(appId)) .and(defaultTable.RESOURCE.eq(resource)) .and(defaultTable.DIMENSION.eq(dimension)) @@ -236,85 +119,13 @@ val record = defaultDSLContext.selectFrom(defaultTable) } @Override - public List getStatisticsListByAppId(Long appId, String resource, String dimension, - String dimensionValue, String sinceDate) { - List conditions = new ArrayList<>(); - if (appId != null) { - conditions.add(defaultTable.APP_ID.eq(appId)); - } - if (StringUtils.isNotBlank(resource)) { - conditions.add(defaultTable.RESOURCE.eq(resource)); - } - if (StringUtils.isNotBlank(dimension)) { - conditions.add(defaultTable.DIMENSION.eq(dimension)); - } - if (StringUtils.isNotBlank(dimensionValue)) { - conditions.add(defaultTable.DIMENSION_VALUE.eq(dimensionValue)); - } - if (StringUtils.isNotBlank(sinceDate)) { - conditions.add(defaultTable.DATE.lessOrEqual(sinceDate)); - } - return listStatisticsWithConditions(defaultDSLContext, conditions); - } - - @Override - public List getStatisticsList(Long appId, String resource, String dimension, String date) { - List conditions = new ArrayList<>(); - if (appId != null) { - conditions.add(defaultTable.APP_ID.eq(appId)); - } - if (StringUtils.isNotBlank(resource)) { - conditions.add(defaultTable.RESOURCE.eq(resource)); - } - if (StringUtils.isNotBlank(dimension)) { - conditions.add(defaultTable.DIMENSION.eq(dimension)); - } - if (StringUtils.isNotBlank(date)) { - conditions.add(defaultTable.DATE.eq(date)); - } - return listStatisticsWithConditions(defaultDSLContext, conditions); - } - - @Override - public List getStatisticsList(List inAppIdList, String resource, String dimension, - String date) { - List conditions = new ArrayList<>(); - if (inAppIdList != null) { - conditions.add(defaultTable.APP_ID.in(inAppIdList)); - } - if (StringUtils.isNotBlank(resource)) { - conditions.add(defaultTable.RESOURCE.eq(resource)); - } - if (StringUtils.isNotBlank(dimension)) { - conditions.add(defaultTable.DIMENSION.eq(dimension)); - } - if (StringUtils.isNotBlank(date)) { - conditions.add(defaultTable.DATE.eq(date)); - } - return listStatisticsWithConditions(defaultDSLContext, conditions); - } - - @Override + @MySQLOperation(table = "statistics", op = DbOperationEnum.READ) public List getStatisticsList(List inAppIdList, List notInAppIdList, String resource, String dimension, String dimensionValue, String date) { - return listStatisticsWithConditions(defaultDSLContext, genConditions(inAppIdList, notInAppIdList, resource, + return listStatisticsWithConditions(dsl(), genConditions(inAppIdList, notInAppIdList, resource, dimension, dimensionValue, date)); } - @Override - public Long getTotalValueOfStatisticsList(List inAppIdList, List notInAppIdList, String resource, - String dimension, String dimensionValue, String date) { - return getTotalValueOfStatisticsWithConditions(defaultDSLContext, genConditions(inAppIdList, notInAppIdList, - resource, dimension, dimensionValue, date)); - } - - @Override - public Integer countStatistics(List inAppIdList, List notInAppIdList, String resource, - String dimension, String dimensionValue, String date) { - return countStatisticsByConditions(genConditions(inAppIdList, notInAppIdList, resource, dimension, - dimensionValue, date)); - } - public Collection genConditions(List inAppIdList, List notInAppIdList, String resource, String dimension, String dimensionValue, String date) { List conditions = new ArrayList<>(); @@ -339,128 +150,6 @@ public Collection genConditions(List inAppIdList, List no return conditions; } - @Override - public List getStatisticsListBetweenDate(List inAppIdList, List notInAppIdList, - String resource, String dimension, String startDate, - String endDate) { - List conditions = new ArrayList<>(); - if (inAppIdList != null) { - conditions.add(defaultTable.APP_ID.in(inAppIdList)); - } - if (notInAppIdList != null) { - conditions.add(defaultTable.APP_ID.notIn(notInAppIdList)); - } - if (StringUtils.isNotBlank(resource)) { - conditions.add(defaultTable.RESOURCE.eq(resource)); - } - if (StringUtils.isNotBlank(dimension)) { - conditions.add(defaultTable.DIMENSION.eq(dimension)); - } - if (StringUtils.isNotBlank(startDate)) { - conditions.add(defaultTable.DATE.greaterOrEqual(startDate)); - } - if (StringUtils.isNotBlank(endDate)) { - conditions.add(defaultTable.DATE.lessOrEqual(endDate)); - } - return listStatisticsWithConditions(defaultDSLContext, conditions); - } - - @Override - public List getStatisticsListBetweenDate(List inAppIdList, List notInAppIdList, - String resource, String dimension, String dimensionValue, - String startDate, String endDate) { - List conditions = new ArrayList<>(); - if (inAppIdList != null) { - conditions.add(defaultTable.APP_ID.in(inAppIdList)); - } - if (notInAppIdList != null) { - conditions.add(defaultTable.APP_ID.notIn(notInAppIdList)); - } - if (StringUtils.isNotBlank(resource)) { - conditions.add(defaultTable.RESOURCE.eq(resource)); - } - if (StringUtils.isNotBlank(dimension)) { - conditions.add(defaultTable.DIMENSION.eq(dimension)); - } - if (StringUtils.isNotBlank(dimensionValue)) { - conditions.add(defaultTable.DIMENSION_VALUE.eq(dimensionValue)); - } - if (StringUtils.isNotBlank(startDate)) { - conditions.add(defaultTable.DATE.greaterOrEqual(startDate)); - } - if (StringUtils.isNotBlank(endDate)) { - conditions.add(defaultTable.DATE.lessOrEqual(endDate)); - } - return listStatisticsWithConditions(defaultDSLContext, conditions); - } - - @Override - public List getStatisticsListBetweenDate(Long appId, String resource, String dimension, - String dimensionValue, String startDate, String endDate) { - List conditions = new ArrayList<>(); - if (appId != null) { - conditions.add(defaultTable.APP_ID.eq(appId)); - } - if (StringUtils.isNotBlank(resource)) { - conditions.add(defaultTable.RESOURCE.eq(resource)); - } - if (StringUtils.isNotBlank(dimension)) { - conditions.add(defaultTable.DIMENSION.eq(dimension)); - } - if (StringUtils.isNotBlank(dimensionValue)) { - conditions.add(defaultTable.DIMENSION_VALUE.eq(dimensionValue)); - } - if (StringUtils.isNotBlank(startDate)) { - conditions.add(defaultTable.DATE.greaterOrEqual(startDate)); - } - if (StringUtils.isNotBlank(endDate)) { - conditions.add(defaultTable.DATE.lessOrEqual(endDate)); - } - return listStatisticsWithConditions(defaultDSLContext, conditions); - } - - public StatisticsDTO getStatistics(DSLContext dslContext, Long appId, String resource, String dimension, - String dimensionValue, String date) { - Collection conditions = genConditions(appId, resource, dimension, dimensionValue, date); - val query = dslContext.selectFrom(defaultTable).where(conditions); - val sql = query.getSQL(ParamType.INLINED); - try { - val record = query.fetchOne(); - if (record == null) { - return null; - } else { - return convert(record); - } - } catch (Exception e) { - log.error(sql); - throw e; - } - } - - @Override - public List listAllStatistics() { - return listStatisticsWithConditions(defaultDSLContext, Collections.emptyList()); - } - - @Override - public Integer countStatisticsByDate(String date) { - Collection conditions = new ArrayList<>(); - if (StringUtils.isNotBlank(date)) { - conditions.add(defaultTable.DATE.eq(date)); - } - var query = defaultDSLContext.selectCount().from(defaultTable).where( - conditions - ); - return query.fetchOne().get(0, Integer.class); - } - - public Integer countStatisticsByConditions(Collection conditions) { - var query = defaultDSLContext.selectCount().from(defaultTable).where( - conditions - ); - return query.fetchOne().get(0, Integer.class); - } - private List listStatisticsWithConditions(DSLContext dslContext, Collection conditions) { var query = dslContext.selectFrom(defaultTable).where( conditions @@ -480,25 +169,6 @@ private List listStatisticsWithConditions(DSLContext dslContext, } } - private Long getTotalValueOfStatisticsWithConditions(DSLContext dslContext, Collection conditions) { - var query = dslContext.select(DSL.sum(defaultTable.VALUE.cast(Long.class))).from(defaultTable).where( - conditions - ); - Result> records; - val sql = query.getSQL(ParamType.INLINED); - try { - records = query.fetch(); - } catch (Exception e) { - log.error(sql); - throw e; - } - if (records == null || records.isEmpty()) { - return 0L; - } else { - return records.get(0).value1().longValue(); - } - } - private StatisticsDTO convert(StatisticsRecord record) { return new StatisticsDTO( record.getId(), @@ -513,4 +183,84 @@ private StatisticsDTO convert(StatisticsRecord record) { ); } + @MySQLOperation(table = "statistics", op = DbOperationEnum.WRITE) + public int increaseStatisticValue(String date, StatisticsKey statisticsKey, Integer incrementValue) { + Long appId = statisticsKey.getAppId(); + String resource = statisticsKey.getResource(); + String dimension = statisticsKey.getDimension(); + String dimensionValue = statisticsKey.getDimensionValue(); + AtomicInteger affectedRows = new AtomicInteger(0); + dsl().transaction(configuration -> { + DSLContext context = DSL.using(configuration); + List conditions = new ArrayList<>(); + conditions.add(defaultTable.APP_ID.eq(appId)); + conditions.add(defaultTable.RESOURCE.eq(resource)); + conditions.add(defaultTable.DIMENSION.eq(dimension)); + conditions.add(defaultTable.DIMENSION_VALUE.eq(dimensionValue)); + conditions.add(defaultTable.DATE.eq(date)); + try { + Long oldValue = 0L; + Long id; + val selectQuery = context.select(defaultTable.ID, defaultTable.VALUE) + .from(defaultTable) + .where(conditions) + .forUpdate(); + log.debug("selectQuery=" + selectQuery.getSQL(ParamType.INLINED)); + Result> records = selectQuery.fetch(); + if (records.isEmpty()) { + log.debug("records is empty"); + // 记录不存在，先插入 + val query = context.insertInto(defaultTable, + defaultTable.ID, + defaultTable.APP_ID, + defaultTable.RESOURCE, + defaultTable.DIMENSION, + defaultTable.DIMENSION_VALUE, + defaultTable.DATE, + defaultTable.VALUE, + defaultTable.CREATE_TIME, + defaultTable.LAST_MODIFY_TIME + ).values( + null, + appId, + resource, + dimension, + dimensionValue, + date, + "0", + ULong.valueOf(System.currentTimeMillis()), + ULong.valueOf(System.currentTimeMillis()) + ).returning(defaultTable.ID); + id = query.fetchOne().getId(); + } else { + if (records.size() > 1) { + log.warn("more than 1 records, statisticsKey:{}", statisticsKey); + } + id = records.get(0).get(defaultTable.ID); + oldValue = Long.parseLong(records.get(0).get(defaultTable.VALUE)); + } + // 更新 + log.debug("Update record {} from {} to {}", id, oldValue, (oldValue + incrementValue)); + affectedRows.set(context.update(defaultTable) + .set(defaultTable.VALUE, "" + (oldValue + incrementValue)) + .where(defaultTable.ID.eq(id)) + .and(defaultTable.VALUE.eq("" + oldValue)) + .execute()); + if (affectedRows.get() == 0) { + log.debug("Record {} updated by other thread just now, retry", statisticsKey); + } + } catch (DataAccessException dataAccessException) { + if (dataAccessException.getMessage().contains("Duplicate entry")) { + //多个实例并发插入导致的主键冲突，忽略，改为更新 + } else { + log.warn("dataAccessException when update/create", dataAccessException); + } + } catch (Throwable t) { + log.info("May fail to lock", t); + } + log.debug("affectedRows={}", affectedRows.get()); + }); + return affectedRows.get(); + } + } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceDAOImpl.java index 25128a2302..174020bdb0 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,55 +25,60 @@ package com.tencent.bk.job.execute.dao.impl; import com.fasterxml.jackson.core.type.TypeReference; -import com.tencent.bk.job.common.constant.DuplicateHandlerEnum; -import com.tencent.bk.job.common.constant.NotExistPathHandlerEnum; +import com.tencent.bk.job.common.constant.AccountCategoryEnum; +import com.tencent.bk.job.common.crypto.scenario.DbPasswordCryptoService; +import com.tencent.bk.job.common.crypto.scenario.SensitiveParamCryptoService; +import com.tencent.bk.job.common.mysql.dynamic.ds.DbOperationEnum; +import com.tencent.bk.job.common.mysql.dynamic.ds.MySQLOperation; +import com.tencent.bk.job.common.mysql.jooq.JooqDataTypeUtil; import com.tencent.bk.job.common.util.Utils; import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; -import com.tencent.bk.job.execute.common.util.JooqDataTypeUtil; import com.tencent.bk.job.execute.dao.StepInstanceDAO; +import com.tencent.bk.job.execute.dao.common.DSLContextProviderFactory; import com.tencent.bk.job.execute.model.ConfirmStepInstanceDTO; +import com.tencent.bk.job.execute.model.ExecuteTargetDTO; import com.tencent.bk.job.execute.model.FileSourceDTO; import com.tencent.bk.job.execute.model.FileStepInstanceDTO; import com.tencent.bk.job.execute.model.ScriptStepInstanceDTO; -import com.tencent.bk.job.execute.model.ServersDTO; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; +import com.tencent.bk.job.execute.model.tables.StepInstance; +import com.tencent.bk.job.execute.model.tables.StepInstanceConfirm; +import com.tencent.bk.job.execute.model.tables.StepInstanceFile; +import com.tencent.bk.job.execute.model.tables.StepInstanceScript; +import com.tencent.bk.job.execute.model.tables.records.StepInstanceRecord; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; import lombok.extern.slf4j.Slf4j; -import lombok.val; import org.apache.commons.lang3.StringUtils; import org.jooq.Condition; -import org.jooq.DSLContext; import org.jooq.Record; import org.jooq.Record1; +import org.jooq.Record2; import org.jooq.Result; +import org.jooq.Table; import org.jooq.TableField; import org.jooq.UpdateSetMoreStep; -import org.jooq.conf.ParamType; -import org.jooq.generated.tables.StepInstance; -import org.jooq.generated.tables.StepInstanceConfirm; -import org.jooq.generated.tables.StepInstanceFile; -import org.jooq.generated.tables.StepInstanceScript; -import org.jooq.generated.tables.records.StepInstanceRecord; import org.jooq.types.UByte; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; import java.util.ArrayList; -import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; @Slf4j @Repository -public class StepInstanceDAOImpl implements StepInstanceDAO { +public class StepInstanceDAOImpl extends BaseDAO implements StepInstanceDAO { private static final StepInstance T_STEP_INSTANCE = StepInstance.STEP_INSTANCE; - private static final StepInstanceScript TABLE_STEP_INSTANCE_SCRIPT = StepInstanceScript.STEP_INSTANCE_SCRIPT; - private static final StepInstanceFile TABLE_STEP_INSTANCE_FILE = StepInstanceFile.STEP_INSTANCE_FILE; + private static final StepInstanceScript T_STEP_INSTANCE_SCRIPT = StepInstanceScript.STEP_INSTANCE_SCRIPT; + private static final StepInstanceFile T_STEP_INSTANCE_FILE = StepInstanceFile.STEP_INSTANCE_FILE; + private static final StepInstanceConfirm T_STEP_INSTANCE_CONFIRM = StepInstanceConfirm.STEP_INSTANCE_CONFIRM; + private static final TableField[] T_STEP_INSTANCE_ALL_FIELDS = { T_STEP_INSTANCE.ID, T_STEP_INSTANCE.STEP_ID, @@ -95,43 +100,116 @@ public class StepInstanceDAOImpl implements StepInstanceDAO { T_STEP_INSTANCE.BATCH, T_STEP_INSTANCE.ROLLING_CONFIG_ID }; + private static final TableField[] T_STEP_INSTANCE_SCRIPT_ALL_FIELDS = { + T_STEP_INSTANCE_SCRIPT.STEP_INSTANCE_ID, + T_STEP_INSTANCE_SCRIPT.TASK_INSTANCE_ID, + T_STEP_INSTANCE_SCRIPT.SCRIPT_CONTENT, + T_STEP_INSTANCE_SCRIPT.SCRIPT_TYPE, + T_STEP_INSTANCE_SCRIPT.SCRIPT_PARAM, + T_STEP_INSTANCE_SCRIPT.RESOLVED_SCRIPT_PARAM, + T_STEP_INSTANCE_SCRIPT.EXECUTION_TIMEOUT, + T_STEP_INSTANCE_SCRIPT.SYSTEM_ACCOUNT_ID, + T_STEP_INSTANCE_SCRIPT.SYSTEM_ACCOUNT, + T_STEP_INSTANCE_SCRIPT.DB_ACCOUNT_ID, + T_STEP_INSTANCE_SCRIPT.DB_ACCOUNT, + T_STEP_INSTANCE_SCRIPT.DB_TYPE, + T_STEP_INSTANCE_SCRIPT.DB_PASSWORD, + T_STEP_INSTANCE_SCRIPT.DB_PORT, + T_STEP_INSTANCE_SCRIPT.SCRIPT_SOURCE, + T_STEP_INSTANCE_SCRIPT.SCRIPT_ID, + T_STEP_INSTANCE_SCRIPT.SCRIPT_VERSION_ID, + T_STEP_INSTANCE_SCRIPT.IS_SECURE_PARAM, + T_STEP_INSTANCE_SCRIPT.WINDOWS_INTERPRETER + }; + private static final TableField[] T_STEP_INSTANCE_FILE_ALL_FIELDS = { + T_STEP_INSTANCE_FILE.TASK_INSTANCE_ID, + T_STEP_INSTANCE_FILE.STEP_INSTANCE_ID, + T_STEP_INSTANCE_FILE.FILE_SOURCE, + T_STEP_INSTANCE_FILE.FILE_TARGET_PATH, + T_STEP_INSTANCE_FILE.FILE_TARGET_NAME, + T_STEP_INSTANCE_FILE.RESOLVED_FILE_TARGET_PATH, + T_STEP_INSTANCE_FILE.FILE_UPLOAD_SPEED_LIMIT, + T_STEP_INSTANCE_FILE.FILE_DOWNLOAD_SPEED_LIMIT, + T_STEP_INSTANCE_FILE.FILE_DUPLICATE_HANDLE, + T_STEP_INSTANCE_FILE.NOT_EXIST_PATH_HANDLER, + T_STEP_INSTANCE_FILE.EXECUTION_TIMEOUT, + T_STEP_INSTANCE_FILE.SYSTEM_ACCOUNT_ID, + T_STEP_INSTANCE_FILE.SYSTEM_ACCOUNT + }; + private static final TableField[] T_STEP_INSTANCE_CONFIRM_ALL_FIELDS = { + T_STEP_INSTANCE_CONFIRM.TASK_INSTANCE_ID, + T_STEP_INSTANCE_CONFIRM.STEP_INSTANCE_ID, + T_STEP_INSTANCE_CONFIRM.CONFIRM_MESSAGE, + T_STEP_INSTANCE_CONFIRM.CONFIRM_REASON, + T_STEP_INSTANCE_CONFIRM.CONFIRM_USERS, + T_STEP_INSTANCE_CONFIRM.CONFIRM_ROLES, + T_STEP_INSTANCE_CONFIRM.NOTIFY_CHANNELS + }; - private final DSLContext CTX; + private final SensitiveParamCryptoService sensitiveParamCryptoService; + private final DbPasswordCryptoService dbPasswordCryptoService; @Autowired - public StepInstanceDAOImpl(@Qualifier("job-execute-dsl-context") DSLContext CTX) { - this.CTX = CTX; + public StepInstanceDAOImpl(DSLContextProviderFactory dslContextProviderFactory, + SensitiveParamCryptoService sensitiveParamCryptoService, + DbPasswordCryptoService dbPasswordCryptoService) { + super(dslContextProviderFactory, T_STEP_INSTANCE.getName()); + this.sensitiveParamCryptoService = sensitiveParamCryptoService; + this.dbPasswordCryptoService = dbPasswordCryptoService; } @Override + @MySQLOperation(table = "step_instance", op = DbOperationEnum.WRITE) public Long addStepInstanceBase(StepInstanceBaseDTO stepInstance) { StepInstance t = StepInstance.STEP_INSTANCE; - Record record = CTX.insertInto(t, t.STEP_ID, t.TASK_INSTANCE_ID, t.APP_ID, t.NAME, t.TYPE, - t.OPERATOR, t.STATUS, t.EXECUTE_COUNT, t.START_TIME, t.END_TIME, t.TOTAL_TIME, - t.TARGET_SERVERS, t.CREATE_TIME, t.IGNORE_ERROR, t.STEP_NUM, - t.STEP_ORDER, t.BATCH) - .values(stepInstance.getStepId(), + Record record = dsl().insertInto( + t, + t.ID, + t.STEP_ID, + t.TASK_INSTANCE_ID, + t.APP_ID, + t.NAME, + t.TYPE, + t.OPERATOR, + t.STATUS, + t.EXECUTE_COUNT, + t.START_TIME, + t.END_TIME, + t.TOTAL_TIME, + t.TARGET_SERVERS, + t.CREATE_TIME, + t.IGNORE_ERROR, + t.STEP_NUM, + t.STEP_ORDER, + t.BATCH + ).values( + stepInstance.getId(), + stepInstance.getStepId(), stepInstance.getTaskInstanceId(), stepInstance.getAppId(), stepInstance.getName(), - JooqDataTypeUtil.toByte(stepInstance.getExecuteType()), + stepInstance.getExecuteType().getValue().byteValue(), stepInstance.getOperator(), stepInstance.getStatus().getValue().byteValue(), stepInstance.getExecuteCount(), stepInstance.getStartTime(), stepInstance.getEndTime(), stepInstance.getTotalTime(), - stepInstance.getTargetServers() == null ? null : JsonUtils.toJson(stepInstance.getTargetServers()), + stepInstance.getTargetExecuteObjects() == null ? null : + JsonUtils.toJson(stepInstance.getTargetExecuteObjects()), stepInstance.getCreateTime(), stepInstance.isIgnoreError() ? Byte.valueOf("1") : Byte.valueOf("0"), stepInstance.getStepNum(), stepInstance.getStepOrder(), - (short) stepInstance.getBatch()) - .returning(t.ID).fetchOne(); - return record.getValue(t.ID); + (short) stepInstance.getBatch() + ).returning(t.ID) + .fetchOne(); + return stepInstance.getId() != null ? stepInstance.getId() : record.getValue(t.ID); + } @Override + @MySQLOperation(table = "step_instance_script", op = DbOperationEnum.WRITE) public void addScriptStepInstance(StepInstanceDTO stepInstance) { Integer scriptSource = stepInstance.getScriptSource(); byte scriptSourceByteValue = 1; @@ -139,56 +217,99 @@ public void addScriptStepInstance(StepInstanceDTO stepInstance) { scriptSourceByteValue = scriptSource.byteValue(); } StepInstanceScript t = StepInstanceScript.STEP_INSTANCE_SCRIPT; - CTX.insertInto(t, t.STEP_INSTANCE_ID, t.SCRIPT_CONTENT, t.SCRIPT_TYPE, t.SCRIPT_PARAM, t.RESOLVED_SCRIPT_PARAM, - t.EXECUTION_TIMEOUT, t.SYSTEM_ACCOUNT_ID, t.SYSTEM_ACCOUNT, t.DB_ACCOUNT_ID, - t.DB_TYPE, t.DB_ACCOUNT, t.DB_PASSWORD, t.DB_PORT, t.SCRIPT_SOURCE, t.SCRIPT_ID, t.SCRIPT_VERSION_ID, - t.IS_SECURE_PARAM) - .values(stepInstance.getId(), - stepInstance.getScriptContent(), - JooqDataTypeUtil.toByte(stepInstance.getScriptType()), - stepInstance.getScriptParam(), - stepInstance.getResolvedScriptParam(), - stepInstance.getTimeout(), - stepInstance.getAccountId(), - stepInstance.getAccount(), - stepInstance.getDbAccountId(), - JooqDataTypeUtil.toByte(stepInstance.getDbType()), - stepInstance.getDbAccount(), - stepInstance.getDbPass(), - stepInstance.getDbPort(), - scriptSourceByteValue, - stepInstance.getScriptId(), - stepInstance.getScriptVersionId(), - stepInstance.isSecureParam() ? JooqDataTypeUtil.toByte(1) : - JooqDataTypeUtil.toByte(0) - ).execute(); - } - - @Override + dsl().insertInto(t, + t.STEP_INSTANCE_ID, + t.TASK_INSTANCE_ID, + t.SCRIPT_CONTENT, + t.SCRIPT_TYPE, + t.SCRIPT_PARAM, + t.RESOLVED_SCRIPT_PARAM, + t.EXECUTION_TIMEOUT, + t.SYSTEM_ACCOUNT_ID, + t.SYSTEM_ACCOUNT, + t.DB_ACCOUNT_ID, + t.DB_TYPE, + t.DB_ACCOUNT, + t.DB_PASSWORD, + t.DB_PORT, + t.SCRIPT_SOURCE, + t.SCRIPT_ID, + t.SCRIPT_VERSION_ID, + t.IS_SECURE_PARAM, + t.WINDOWS_INTERPRETER + ).values( + stepInstance.getId(), + stepInstance.getTaskInstanceId(), + stepInstance.getScriptContent(), + stepInstance.getScriptType().getValue().byteValue(), + sensitiveParamCryptoService.encryptParamIfNeeded( + stepInstance.isSecureParam(), stepInstance.getScriptParam()), + sensitiveParamCryptoService.encryptParamIfNeeded( + stepInstance.isSecureParam(), stepInstance.getResolvedScriptParam()), + stepInstance.getTimeout(), + stepInstance.getAccountId(), + stepInstance.getAccount(), + stepInstance.getDbAccountId(), + JooqDataTypeUtil.toByte(stepInstance.getDbType()), + stepInstance.getDbAccount(), + dbPasswordCryptoService.encryptDbPasswordIfNeeded(AccountCategoryEnum.DB, stepInstance.getDbPass()), + stepInstance.getDbPort(), + scriptSourceByteValue, + stepInstance.getScriptId(), + stepInstance.getScriptVersionId(), + stepInstance.isSecureParam() ? JooqDataTypeUtil.toByte(1) : + JooqDataTypeUtil.toByte(0), + stepInstance.getWindowsInterpreter() + ).execute(); + } + + @Override + @MySQLOperation(table = "step_instance_file", op = DbOperationEnum.WRITE) public void addFileStepInstance(StepInstanceDTO stepInstance) { StepInstanceFile t = StepInstanceFile.STEP_INSTANCE_FILE; - CTX.insertInto(t, t.STEP_INSTANCE_ID, t.FILE_SOURCE, t.FILE_TARGET_PATH, t.FILE_TARGET_NAME, - t.FILE_UPLOAD_SPEED_LIMIT, t.FILE_DOWNLOAD_SPEED_LIMIT, t.FILE_DUPLICATE_HANDLE, t.NOT_EXIST_PATH_HANDLER, - t.EXECUTION_TIMEOUT, t.SYSTEM_ACCOUNT_ID, t.SYSTEM_ACCOUNT) - .values(stepInstance.getId(), - JsonUtils.toJson(stepInstance.getFileSourceList()), - stepInstance.getFileTargetPath(), - stepInstance.getFileTargetName(), - stepInstance.getFileUploadSpeedLimit(), - stepInstance.getFileDownloadSpeedLimit(), - JooqDataTypeUtil.toByte(stepInstance.getFileDuplicateHandle()), - JooqDataTypeUtil.toUByte(stepInstance.getNotExistPathHandler()), - stepInstance.getTimeout(), - stepInstance.getAccountId(), - stepInstance.getAccount() - ).execute(); - } - - @Override + dsl().insertInto(t, + t.STEP_INSTANCE_ID, + t.TASK_INSTANCE_ID, + t.FILE_SOURCE, + t.FILE_TARGET_PATH, + t.FILE_TARGET_NAME, + t.FILE_UPLOAD_SPEED_LIMIT, + t.FILE_DOWNLOAD_SPEED_LIMIT, + t.FILE_DUPLICATE_HANDLE, + t.NOT_EXIST_PATH_HANDLER, + t.EXECUTION_TIMEOUT, + t.SYSTEM_ACCOUNT_ID, + t.SYSTEM_ACCOUNT + ).values( + stepInstance.getId(), + stepInstance.getTaskInstanceId(), + JsonUtils.toJson(stepInstance.getFileSourceList()), + stepInstance.getFileTargetPath(), + stepInstance.getFileTargetName(), + stepInstance.getFileUploadSpeedLimit(), + stepInstance.getFileDownloadSpeedLimit(), + JooqDataTypeUtil.toByte(stepInstance.getFileDuplicateHandle()), + JooqDataTypeUtil.toUByte(stepInstance.getNotExistPathHandler()), + stepInstance.getTimeout(), + stepInstance.getAccountId(), + stepInstance.getAccount() + ).execute(); + } + + @Override + @MySQLOperation(table = "step_instance_confirm", op = DbOperationEnum.WRITE) public void addConfirmStepInstance(StepInstanceDTO stepInstance) { StepInstanceConfirm t = StepInstanceConfirm.STEP_INSTANCE_CONFIRM; - CTX.insertInto(t, t.STEP_INSTANCE_ID, t.CONFIRM_MESSAGE, t.CONFIRM_USERS, t.CONFIRM_ROLES, t.NOTIFY_CHANNELS) - .values(stepInstance.getId(), + dsl().insertInto(t, + t.STEP_INSTANCE_ID, + t.TASK_INSTANCE_ID, + t.CONFIRM_MESSAGE, + t.CONFIRM_USERS, + t.CONFIRM_ROLES, + t.NOTIFY_CHANNELS) + .values( + stepInstance.getId(), + stepInstance.getTaskInstanceId(), stepInstance.getConfirmMessage(), stepInstance.getConfirmUsers() == null ? null : Utils.concatStringWithSeperator(stepInstance.getConfirmUsers(), ","), @@ -200,37 +321,86 @@ public void addConfirmStepInstance(StepInstanceDTO stepInstance) { } @Override - public ScriptStepInstanceDTO getScriptStepInstance(long stepInstanceId) { - StepInstanceScript t = StepInstanceScript.STEP_INSTANCE_SCRIPT; - Record record = CTX.select(t.STEP_INSTANCE_ID, t.SCRIPT_CONTENT, t.SCRIPT_TYPE, t.SCRIPT_PARAM, - t.RESOLVED_SCRIPT_PARAM, t.EXECUTION_TIMEOUT, t.SYSTEM_ACCOUNT_ID, t.SYSTEM_ACCOUNT, - t.DB_ACCOUNT_ID, t.DB_ACCOUNT, t.DB_TYPE, t.DB_PASSWORD, t.DB_PORT, t.SCRIPT_SOURCE, t.SCRIPT_ID, - t.SCRIPT_VERSION_ID, t.IS_SECURE_PARAM - ).from(t) - .where(t.STEP_INSTANCE_ID.eq(stepInstanceId)).fetchOne(); + @MySQLOperation(table = "step_instance_script", op = DbOperationEnum.READ) + public ScriptStepInstanceDTO getScriptStepInstance(Long taskInstanceId, long stepInstanceId) { + Record record = dsl().select(T_STEP_INSTANCE_SCRIPT_ALL_FIELDS) + .from(T_STEP_INSTANCE_SCRIPT) + .where(buildTaskInstanceIdQueryCondition(T_STEP_INSTANCE_SCRIPT, taskInstanceId)) + .and(T_STEP_INSTANCE_SCRIPT.STEP_INSTANCE_ID.eq(stepInstanceId)) + .fetchOne(); return extractScriptInfo(record); } + private Condition buildTaskInstanceIdQueryCondition(Table table, + Long taskInstanceId) { + if (table instanceof StepInstance) { + return TaskInstanceIdDynamicCondition.build( + taskInstanceId, + T_STEP_INSTANCE.TASK_INSTANCE_ID::eq + ); + } else if (table instanceof StepInstanceScript) { + return TaskInstanceIdDynamicCondition.build( + taskInstanceId, + T_STEP_INSTANCE_SCRIPT.TASK_INSTANCE_ID::eq + ); + } else if (table instanceof StepInstanceFile) { + return TaskInstanceIdDynamicCondition.build( + taskInstanceId, + T_STEP_INSTANCE_FILE.TASK_INSTANCE_ID::eq + ); + } else if (table instanceof StepInstanceConfirm) { + return TaskInstanceIdDynamicCondition.build( + taskInstanceId, + T_STEP_INSTANCE_CONFIRM.TASK_INSTANCE_ID::eq + ); + } else { + throw new IllegalArgumentException("Invalid table for building task_instance_id query condition"); + } + } + + private ScriptStepInstanceDTO extractScriptInfo(Record record) { if (record == null) { return null; } StepInstanceScript t = StepInstanceScript.STEP_INSTANCE_SCRIPT; ScriptStepInstanceDTO stepInstance = new ScriptStepInstanceDTO(); + stepInstance.setTaskInstanceId(record.get(t.TASK_INSTANCE_ID)); stepInstance.setStepInstanceId(record.get(t.STEP_INSTANCE_ID)); stepInstance.setScriptContent(record.get(t.SCRIPT_CONTENT)); - stepInstance.setScriptType(JooqDataTypeUtil.toInteger(record.get(t.SCRIPT_TYPE))); - stepInstance.setScriptParam(record.get(t.SCRIPT_PARAM)); - stepInstance.setResolvedScriptParam(record.get(t.RESOLVED_SCRIPT_PARAM)); + stepInstance.setScriptType(ScriptTypeEnum.valOf(record.get(t.SCRIPT_TYPE).intValue())); + stepInstance.setSecureParam(record.get(t.IS_SECURE_PARAM).intValue() == 1); + String encryptedScriptParam = record.get(t.SCRIPT_PARAM); + + // 敏感参数解密 + String scriptParam = sensitiveParamCryptoService.decryptParamIfNeeded( + stepInstance.isSecureParam(), + encryptedScriptParam + ); + stepInstance.setScriptParam(scriptParam); + String encryptedResolvedScriptParam = record.get(t.RESOLVED_SCRIPT_PARAM); + String resolvedScriptParam = sensitiveParamCryptoService.decryptParamIfNeeded( + stepInstance.isSecureParam(), + encryptedResolvedScriptParam + ); + stepInstance.setResolvedScriptParam(resolvedScriptParam); + stepInstance.setTimeout(record.get(t.EXECUTION_TIMEOUT)); stepInstance.setAccountId(record.get(t.SYSTEM_ACCOUNT_ID)); stepInstance.setAccount(record.get(t.SYSTEM_ACCOUNT)); stepInstance.setDbAccountId(record.get(t.DB_ACCOUNT_ID)); stepInstance.setDbType(JooqDataTypeUtil.toInteger(record.get(t.DB_TYPE))); stepInstance.setDbAccount(record.get(t.DB_ACCOUNT)); - stepInstance.setDbPass(record.get(t.DB_PASSWORD)); + + // 账号密码解密 + String encryptedDbPassword = record.get(t.DB_PASSWORD); + String dbPassword = dbPasswordCryptoService.decryptDbPasswordIfNeeded( + AccountCategoryEnum.DB, + encryptedDbPassword + ); + + stepInstance.setDbPass(dbPassword); stepInstance.setDbPort(record.get(t.DB_PORT)); - stepInstance.setSecureParam(record.get(t.IS_SECURE_PARAM).intValue() == 1); Byte scriptSource = record.get(t.SCRIPT_SOURCE); if (scriptSource == null) { stepInstance.setScriptSource(1); @@ -239,18 +409,18 @@ private ScriptStepInstanceDTO extractScriptInfo(Record record) { } stepInstance.setScriptId(record.get(t.SCRIPT_ID)); stepInstance.setScriptVersionId(record.get(t.SCRIPT_VERSION_ID)); + stepInstance.setWindowsInterpreter(record.get(t.WINDOWS_INTERPRETER)); return stepInstance; } @Override - public FileStepInstanceDTO getFileStepInstance(long stepInstanceId) { - StepInstanceFile t = StepInstanceFile.STEP_INSTANCE_FILE; - Record record = CTX.select(t.STEP_INSTANCE_ID, t.FILE_SOURCE, t.RESOLVED_FILE_SOURCE, t.FILE_TARGET_PATH, - t.FILE_TARGET_NAME, t.RESOLVED_FILE_TARGET_PATH, t.FILE_UPLOAD_SPEED_LIMIT, t.FILE_DOWNLOAD_SPEED_LIMIT, - t.FILE_DUPLICATE_HANDLE, - t.NOT_EXIST_PATH_HANDLER, t.EXECUTION_TIMEOUT, t.SYSTEM_ACCOUNT_ID, t.SYSTEM_ACCOUNT) - .from(t) - .where(t.STEP_INSTANCE_ID.eq(stepInstanceId)).fetchOne(); + @MySQLOperation(table = "step_instance_file", op = DbOperationEnum.READ) + public FileStepInstanceDTO getFileStepInstance(Long taskInstanceId, long stepInstanceId) { + Record record = dsl().select(T_STEP_INSTANCE_FILE_ALL_FIELDS) + .from(T_STEP_INSTANCE_FILE) + .where(buildTaskInstanceIdQueryCondition(T_STEP_INSTANCE_FILE, taskInstanceId)) + .and(T_STEP_INSTANCE_FILE.STEP_INSTANCE_ID.eq(stepInstanceId)) + .fetchOne(); return extractFileInfo(record); } @@ -260,18 +430,13 @@ private FileStepInstanceDTO extractFileInfo(Record record) { } StepInstanceFile t = StepInstanceFile.STEP_INSTANCE_FILE; FileStepInstanceDTO stepInstance = new FileStepInstanceDTO(); + stepInstance.setTaskInstanceId(record.get(t.TASK_INSTANCE_ID)); stepInstance.setStepInstanceId(record.get(t.STEP_INSTANCE_ID)); stepInstance.setTimeout(record.get(t.EXECUTION_TIMEOUT)); List fileSourceList = JsonUtils.fromJson(record.get(t.FILE_SOURCE), new TypeReference>() { }); stepInstance.setFileSourceList(fileSourceList); - if (StringUtils.isNotEmpty(record.get(t.RESOLVED_FILE_SOURCE))) { - List resolvedFileSourceList = JsonUtils.fromJson(record.get(t.RESOLVED_FILE_SOURCE), - new TypeReference>() { - }); - stepInstance.setResolvedFileSourceList(resolvedFileSourceList); - } stepInstance.setFileTargetPath(record.get(t.FILE_TARGET_PATH)); stepInstance.setFileTargetName(record.get(t.FILE_TARGET_NAME)); stepInstance.setResolvedFileTargetPath(record.get(t.RESOLVED_FILE_TARGET_PATH)); @@ -291,11 +456,13 @@ private FileStepInstanceDTO extractFileInfo(Record record) { } @Override - public ConfirmStepInstanceDTO getConfirmStepInstance(long stepInstanceId) { - StepInstanceConfirm t = StepInstanceConfirm.STEP_INSTANCE_CONFIRM; - Record record = CTX.select(t.STEP_INSTANCE_ID, t.CONFIRM_MESSAGE, t.CONFIRM_REASON, t.CONFIRM_USERS, - t.CONFIRM_ROLES, t.NOTIFY_CHANNELS).from(t) - .where(t.STEP_INSTANCE_ID.eq(stepInstanceId)).fetchOne(); + @MySQLOperation(table = "step_instance_confirm", op = DbOperationEnum.READ) + public ConfirmStepInstanceDTO getConfirmStepInstance(Long taskInstanceId, long stepInstanceId) { + Record record = dsl().select(T_STEP_INSTANCE_CONFIRM_ALL_FIELDS) + .from(T_STEP_INSTANCE_CONFIRM) + .where(buildTaskInstanceIdQueryCondition(T_STEP_INSTANCE_CONFIRM, taskInstanceId)) + .and(T_STEP_INSTANCE_CONFIRM.STEP_INSTANCE_ID.eq(stepInstanceId)) + .fetchOne(); return extractConfirmInfo(record); } @@ -305,6 +472,7 @@ private ConfirmStepInstanceDTO extractConfirmInfo(Record record) { } StepInstanceConfirm t = StepInstanceConfirm.STEP_INSTANCE_CONFIRM; ConfirmStepInstanceDTO stepInstanceDTO = new ConfirmStepInstanceDTO(); + stepInstanceDTO.setTaskInstanceId(record.get(t.TASK_INSTANCE_ID)); stepInstanceDTO.setStepInstanceId(record.get(t.STEP_INSTANCE_ID)); stepInstanceDTO.setConfirmMessage(record.get(t.CONFIRM_MESSAGE)); stepInstanceDTO.setConfirmReason(record.get(t.CONFIRM_REASON)); @@ -318,11 +486,13 @@ private ConfirmStepInstanceDTO extractConfirmInfo(Record record) { } @Override - public StepInstanceBaseDTO getStepInstanceBase(long stepInstanceId) { - Record record = CTX + @MySQLOperation(table = "step_instance", op = DbOperationEnum.READ) + public StepInstanceBaseDTO getStepInstanceBase(Long taskInstanceId, long stepInstanceId) { + Record record = dsl() .select(T_STEP_INSTANCE_ALL_FIELDS) .from(T_STEP_INSTANCE) - .where(T_STEP_INSTANCE.ID.eq(stepInstanceId)) + .where(buildTaskInstanceIdQueryCondition(T_STEP_INSTANCE, taskInstanceId)) + .and(T_STEP_INSTANCE.ID.eq(stepInstanceId)) .fetchOne(); return extractBaseInfo(record); } @@ -338,7 +508,7 @@ private StepInstanceBaseDTO extractBaseInfo(Record record) { stepInstance.setStepId(record.get(t.STEP_ID)); stepInstance.setTaskInstanceId(record.get(t.TASK_INSTANCE_ID)); stepInstance.setName(record.get(t.NAME)); - stepInstance.setExecuteType(JooqDataTypeUtil.toInteger(record.get(t.TYPE))); + stepInstance.setExecuteType(StepExecuteTypeEnum.valOf(JooqDataTypeUtil.toInteger(record.get(t.TYPE)))); stepInstance.setOperator(record.get(t.OPERATOR)); stepInstance.setStatus(RunStatusEnum.valueOf(record.get(t.STATUS))); stepInstance.setExecuteCount(record.get(t.EXECUTE_COUNT)); @@ -346,9 +516,8 @@ private StepInstanceBaseDTO extractBaseInfo(Record record) { stepInstance.setEndTime(record.get(t.END_TIME)); stepInstance.setTotalTime(record.get(t.TOTAL_TIME)); if (StringUtils.isNotBlank(record.get(t.TARGET_SERVERS))) { - ServersDTO targetServers = JsonUtils.fromJson(record.get(t.TARGET_SERVERS), ServersDTO.class); - stepInstance.setTargetServers(targetServers); - stepInstance.setIpList(targetServers.buildIpListStr()); + ExecuteTargetDTO targetServers = JsonUtils.fromJson(record.get(t.TARGET_SERVERS), ExecuteTargetDTO.class); + stepInstance.setTargetExecuteObjects(targetServers); } stepInstance.setCreateTime(record.get(t.CREATE_TIME)); stepInstance.setIgnoreError(JooqDataTypeUtil.toInteger(record.get(t.IGNORE_ERROR)) != null @@ -361,20 +530,33 @@ private StepInstanceBaseDTO extractBaseInfo(Record record) { } @Override - public StepInstanceBaseDTO getFirstStepInstanceBase(long taskInstanceId) { - Record record = CTX + @MySQLOperation(table = "step_instance", op = DbOperationEnum.READ) + public StepInstanceBaseDTO getStepInstanceBase(long stepInstanceId) { + Record record = dsl() + .select(T_STEP_INSTANCE_ALL_FIELDS) + .from(T_STEP_INSTANCE) + .where(T_STEP_INSTANCE.ID.eq(stepInstanceId)) + .fetchOne(); + return extractBaseInfo(record); + } + + @Override + @MySQLOperation(table = "step_instance", op = DbOperationEnum.READ) + public StepInstanceBaseDTO getFirstStepInstanceBase(Long taskInstanceId) { + Record record = dsl() .select(T_STEP_INSTANCE_ALL_FIELDS) .from(T_STEP_INSTANCE) .where(T_STEP_INSTANCE.TASK_INSTANCE_ID.eq(taskInstanceId)) - .orderBy(T_STEP_INSTANCE.ID.asc()) + .orderBy(T_STEP_INSTANCE.STEP_ORDER.asc()) .limit(1) .fetchOne(); return extractBaseInfo(record); } @Override - public StepInstanceBaseDTO getNextStepInstance(long taskInstanceId, int currentStepOrder) { - Record record = CTX + @MySQLOperation(table = "step_instance", op = DbOperationEnum.READ) + public StepInstanceBaseDTO getNextStepInstance(Long taskInstanceId, int currentStepOrder) { + Record record = dsl() .select(T_STEP_INSTANCE_ALL_FIELDS) .from(T_STEP_INSTANCE) .where(T_STEP_INSTANCE.TASK_INSTANCE_ID.eq(taskInstanceId)) @@ -386,12 +568,13 @@ public StepInstanceBaseDTO getNextStepInstance(long taskInstanceId, int currentS } @Override - public List listStepInstanceBaseByTaskInstanceId(long taskInstanceId) { - Result result = CTX + @MySQLOperation(table = "step_instance", op = DbOperationEnum.READ) + public List listStepInstanceBaseByTaskInstanceId(Long taskInstanceId) { + Result result = dsl() .select(T_STEP_INSTANCE_ALL_FIELDS) .from(T_STEP_INSTANCE) .where(T_STEP_INSTANCE.TASK_INSTANCE_ID.eq(taskInstanceId)) - .orderBy(T_STEP_INSTANCE.ID.asc()) + .orderBy(T_STEP_INSTANCE.STEP_ORDER.asc()) .fetch(); List stepInstanceList = new ArrayList<>(); result.into(record -> stepInstanceList.add(extractBaseInfo(record))); @@ -399,76 +582,116 @@ public List listStepInstanceBaseByTaskInstanceId(long taskI } @Override - public void resetStepStatus(long stepInstanceId) { + @MySQLOperation(table = "step_instance", op = DbOperationEnum.WRITE) + public void resetStepStatus(Long taskInstanceId, long stepInstanceId) { StepInstance t = StepInstance.STEP_INSTANCE; - CTX.update(t).setNull(t.START_TIME).setNull(t.END_TIME).setNull(t.TOTAL_TIME) - .where(t.ID.eq(stepInstanceId)).execute(); + dsl().update(t) + .setNull(t.START_TIME) + .setNull(t.END_TIME) + .setNull(t.TOTAL_TIME) + .where(buildTaskInstanceIdQueryCondition(T_STEP_INSTANCE, taskInstanceId)) + .and(t.ID.eq(stepInstanceId)) + .execute(); } @Override - public void resetStepExecuteInfoForRetry(long stepInstanceId) { + @MySQLOperation(table = "step_instance", op = DbOperationEnum.WRITE) + public void resetStepExecuteInfoForRetry(Long taskInstanceId, long stepInstanceId) { StepInstance t = StepInstance.STEP_INSTANCE; - CTX.update(t) + dsl().update(t) .set(t.STATUS, RunStatusEnum.RUNNING.getValue().byteValue()) .setNull(t.END_TIME) .setNull(t.TOTAL_TIME) - .where(t.ID.eq(stepInstanceId)).execute(); + .where(buildTaskInstanceIdQueryCondition(T_STEP_INSTANCE, taskInstanceId)) + .and(t.ID.eq(stepInstanceId)) + .execute(); } @Override - public void addStepExecuteCount(long stepInstanceId) { + @MySQLOperation(table = "step_instance", op = DbOperationEnum.WRITE) + public void addStepExecuteCount(Long taskInstanceId, long stepInstanceId) { StepInstance t = StepInstance.STEP_INSTANCE; - CTX.update(t).set(t.EXECUTE_COUNT, t.EXECUTE_COUNT.plus(1)) - .where(t.ID.eq(stepInstanceId)).execute(); + dsl().update(t) + .set(t.EXECUTE_COUNT, t.EXECUTE_COUNT.plus(1)) + .where(buildTaskInstanceIdQueryCondition(T_STEP_INSTANCE, taskInstanceId)) + .and(t.ID.eq(stepInstanceId)) + .execute(); } @Override - public void updateStepStatus(long stepInstanceId, int status) { + @MySQLOperation(table = "step_instance", op = DbOperationEnum.WRITE) + public void updateStepStatus(Long taskInstanceId, long stepInstanceId, int status) { StepInstance t = StepInstance.STEP_INSTANCE; - CTX.update(t).set(t.STATUS, JooqDataTypeUtil.toByte(status)) - .where(t.ID.eq(stepInstanceId)).execute(); + dsl().update(t) + .set(t.STATUS, JooqDataTypeUtil.toByte(status)) + .where(buildTaskInstanceIdQueryCondition(T_STEP_INSTANCE, taskInstanceId)) + .and(t.ID.eq(stepInstanceId)) + .execute(); } @Override - public void updateStepStartTime(long stepInstanceId, Long startTime) { + @MySQLOperation(table = "step_instance", op = DbOperationEnum.WRITE) + public void updateStepStartTime(Long taskInstanceId, long stepInstanceId, Long startTime) { StepInstance t = StepInstance.STEP_INSTANCE; - CTX.update(t).set(t.START_TIME, startTime) - .where(t.ID.eq(stepInstanceId)).execute(); + dsl().update(t) + .set(t.START_TIME, startTime) + .where(buildTaskInstanceIdQueryCondition(T_STEP_INSTANCE, taskInstanceId)) + .and(t.ID.eq(stepInstanceId)) + .execute(); } @Override - public void updateStepStartTimeIfNull(long stepInstanceId, Long startTime) { + @MySQLOperation(table = "step_instance", op = DbOperationEnum.WRITE) + public void updateStepStartTimeIfNull(Long taskInstanceId, long stepInstanceId, Long startTime) { StepInstance t = StepInstance.STEP_INSTANCE; - CTX.update(t).set(t.START_TIME, startTime) - .where(t.ID.eq(stepInstanceId)) + dsl().update(t) + .set(t.START_TIME, startTime) + .where(buildTaskInstanceIdQueryCondition(T_STEP_INSTANCE, taskInstanceId)) + .and(t.ID.eq(stepInstanceId)) .and(t.START_TIME.isNull()) .execute(); } @Override - public void updateStepEndTime(long stepInstanceId, Long endTime) { + @MySQLOperation(table = "step_instance", op = DbOperationEnum.WRITE) + public void updateStepEndTime(Long taskInstanceId, long stepInstanceId, Long endTime) { StepInstance t = StepInstance.STEP_INSTANCE; - CTX.update(t).set(t.END_TIME, endTime) - .where(t.ID.eq(stepInstanceId)) + dsl().update(t) + .set(t.END_TIME, endTime) + .where(buildTaskInstanceIdQueryCondition(T_STEP_INSTANCE, taskInstanceId)) + .and(t.ID.eq(stepInstanceId)) .execute(); } @Override - public void addStepInstanceExecuteCount(long stepInstanceId) { + @MySQLOperation(table = "step_instance", op = DbOperationEnum.WRITE) + public void addStepInstanceExecuteCount(Long taskInstanceId, long stepInstanceId) { StepInstance t = StepInstance.STEP_INSTANCE; - CTX.update(t).set(t.EXECUTE_COUNT, t.EXECUTE_COUNT.plus(1)) - .where(t.ID.eq(stepInstanceId)).execute(); + dsl().update(t) + .set(t.EXECUTE_COUNT, t.EXECUTE_COUNT.plus(1)) + .where(buildTaskInstanceIdQueryCondition(T_STEP_INSTANCE, taskInstanceId)) + .and(t.ID.eq(stepInstanceId)) + .execute(); } @Override - public void updateStepTotalTime(long stepInstanceId, long totalTime) { + @MySQLOperation(table = "step_instance", op = DbOperationEnum.WRITE) + public void updateStepTotalTime(Long taskInstanceId, long stepInstanceId, long totalTime) { StepInstance t = StepInstance.STEP_INSTANCE; - CTX.update(t).set(t.TOTAL_TIME, totalTime) - .where(t.ID.eq(stepInstanceId)).execute(); + dsl().update(t) + .set(t.TOTAL_TIME, totalTime) + .where(buildTaskInstanceIdQueryCondition(T_STEP_INSTANCE, taskInstanceId)) + .and(t.ID.eq(stepInstanceId)) + .execute(); } @Override - public void updateStepExecutionInfo(long stepInstanceId, RunStatusEnum status, Long startTime, Long endTime, + @MySQLOperation(table = "step_instance", op = DbOperationEnum.WRITE) + public void updateStepExecutionInfo(Long taskInstanceId, + long stepInstanceId, + RunStatusEnum status, + Long startTime, + Long endTime, Long totalTime) { StepInstance t = StepInstance.STEP_INSTANCE; UpdateSetMoreStep updateSetMoreStep = buildBasicUpdateSetMoreStep(status, @@ -476,33 +699,39 @@ public void updateStepExecutionInfo(long stepInstanceId, RunStatusEnum status, L if (updateSetMoreStep == null) { return; } - updateSetMoreStep.where(t.ID.eq(stepInstanceId)).execute(); + updateSetMoreStep + .where(buildTaskInstanceIdQueryCondition(T_STEP_INSTANCE, taskInstanceId)) + .and(t.ID.eq(stepInstanceId)) + .execute(); } - private UpdateSetMoreStep buildBasicUpdateSetMoreStep(RunStatusEnum status, Long startTime, - Long endTime, Long totalTime) { + private UpdateSetMoreStep buildBasicUpdateSetMoreStep(RunStatusEnum status, + Long startTime, + Long endTime, + Long totalTime) { StepInstance t = StepInstance.STEP_INSTANCE; UpdateSetMoreStep updateSetMoreStep = null; if (status != null) { - updateSetMoreStep = CTX.update(t).set(t.STATUS, JooqDataTypeUtil.toByte(status.getValue())); + updateSetMoreStep = dsl().update(t).set(t.STATUS, + JooqDataTypeUtil.toByte(status.getValue())); } if (startTime != null) { if (updateSetMoreStep == null) { - updateSetMoreStep = CTX.update(t).set(t.START_TIME, startTime); + updateSetMoreStep = dsl().update(t).set(t.START_TIME, startTime); } else { updateSetMoreStep.set(t.START_TIME, startTime); } } if (endTime != null) { if (updateSetMoreStep == null) { - updateSetMoreStep = CTX.update(t).set(t.END_TIME, endTime); + updateSetMoreStep = dsl().update(t).set(t.END_TIME, endTime); } else { updateSetMoreStep.set(t.END_TIME, endTime); } } if (totalTime != null) { if (updateSetMoreStep == null) { - updateSetMoreStep = CTX.update(t).set(t.TOTAL_TIME, totalTime); + updateSetMoreStep = dsl().update(t).set(t.TOTAL_TIME, totalTime); } else { updateSetMoreStep.set(t.TOTAL_TIME, totalTime); } @@ -511,326 +740,171 @@ private UpdateSetMoreStep buildBasicUpdateSetMoreStep(RunSta } @Override - public void updateResolvedScriptParam(long stepInstanceId, String resolvedScriptParam) { + @MySQLOperation(table = "step_instance_script", op = DbOperationEnum.WRITE) + public void updateResolvedScriptParam(Long taskInstanceId, + long stepInstanceId, + boolean isSecureParam, + String resolvedScriptParam) { StepInstanceScript t = StepInstanceScript.STEP_INSTANCE_SCRIPT; - CTX.update(t).set(t.RESOLVED_SCRIPT_PARAM, resolvedScriptParam) - .where(t.STEP_INSTANCE_ID.eq(stepInstanceId)) + dsl().update(t) + .set(t.RESOLVED_SCRIPT_PARAM, sensitiveParamCryptoService.encryptParamIfNeeded( + isSecureParam, resolvedScriptParam + )) + .where(buildTaskInstanceIdQueryCondition(T_STEP_INSTANCE_SCRIPT, taskInstanceId)) + .and(t.STEP_INSTANCE_ID.eq(stepInstanceId)) .execute(); } @Override - public void updateResolvedSourceFile(long stepInstanceId, List resolvedFileSources) { + @MySQLOperation(table = "step_instance_file", op = DbOperationEnum.WRITE) + public void updateResolvedSourceFile(Long taskInstanceId, + long stepInstanceId, + List resolvedFileSources) { StepInstanceFile t = StepInstanceFile.STEP_INSTANCE_FILE; - CTX.update(t).set(t.FILE_SOURCE, JsonUtils.toJson(resolvedFileSources)) - .where(t.STEP_INSTANCE_ID.eq(stepInstanceId)) + dsl().update(t) + .set(t.FILE_SOURCE, JsonUtils.toJson(resolvedFileSources)) + .where(buildTaskInstanceIdQueryCondition(T_STEP_INSTANCE_FILE, taskInstanceId)) + .and(t.STEP_INSTANCE_ID.eq(stepInstanceId)) .execute(); } @Override - public void updateResolvedTargetPath(long stepInstanceId, String resolvedTargetPath) { + @MySQLOperation(table = "step_instance_file", op = DbOperationEnum.WRITE) + public void updateResolvedTargetPath(Long taskInstanceId, long stepInstanceId, String resolvedTargetPath) { StepInstanceFile t = StepInstanceFile.STEP_INSTANCE_FILE; - CTX.update(t).set(t.RESOLVED_FILE_TARGET_PATH, resolvedTargetPath) - .where(t.STEP_INSTANCE_ID.eq(stepInstanceId)) + dsl().update(t) + .set(t.RESOLVED_FILE_TARGET_PATH, resolvedTargetPath) + .where(buildTaskInstanceIdQueryCondition(T_STEP_INSTANCE_FILE, taskInstanceId)) + .and(t.STEP_INSTANCE_ID.eq(stepInstanceId)) .execute(); } @Override - public void updateConfirmReason(long stepInstanceId, String confirmReason) { + @MySQLOperation(table = "step_instance_confirm", op = DbOperationEnum.WRITE) + public void updateConfirmReason(Long taskInstanceId, long stepInstanceId, String confirmReason) { StepInstanceConfirm t = StepInstanceConfirm.STEP_INSTANCE_CONFIRM; - CTX.update(t).set(t.CONFIRM_REASON, confirmReason) - .where(t.STEP_INSTANCE_ID.eq(stepInstanceId)) + dsl().update(t).set(t.CONFIRM_REASON, confirmReason) + .where(buildTaskInstanceIdQueryCondition(T_STEP_INSTANCE_CONFIRM, taskInstanceId)) + .and(t.STEP_INSTANCE_ID.eq(stepInstanceId)) .execute(); } @Override - public void updateStepOperator(long stepInstanceId, String operator) { + @MySQLOperation(table = "step_instance", op = DbOperationEnum.WRITE) + public void updateStepOperator(Long taskInstanceId, long stepInstanceId, String operator) { StepInstance t = StepInstance.STEP_INSTANCE; - CTX.update(t).set(t.OPERATOR, operator) - .where(t.ID.eq(stepInstanceId)) + dsl().update(t).set(t.OPERATOR, operator) + .where(buildTaskInstanceIdQueryCondition(T_STEP_INSTANCE, taskInstanceId)) + .and(t.ID.eq(stepInstanceId)) .execute(); } @Override - public StepInstanceBaseDTO getPreExecutableStepInstance(long taskInstanceId, long stepInstanceId) { - Record record = CTX + @MySQLOperation(table = "step_instance", op = DbOperationEnum.READ) + public StepInstanceBaseDTO getPreExecutableStepInstance(Long taskInstanceId, int currentStepOrder) { + Record record = dsl() .select(T_STEP_INSTANCE_ALL_FIELDS) .from(T_STEP_INSTANCE) .where(T_STEP_INSTANCE.TASK_INSTANCE_ID.eq(taskInstanceId)) - .and(T_STEP_INSTANCE.ID.lt(stepInstanceId)) + .and(T_STEP_INSTANCE.STEP_ORDER.lt(currentStepOrder)) .and(T_STEP_INSTANCE.TYPE.notIn(StepExecuteTypeEnum.MANUAL_CONFIRM.getValue().byteValue())) - .orderBy(T_STEP_INSTANCE.ID.desc()) + .orderBy(T_STEP_INSTANCE.STEP_ORDER.desc()) .limit(1) .fetchOne(); return extractBaseInfo(record); } @Override - public Long getStepInstanceId(long taskInstanceId) { - Result> records = CTX.select(T_STEP_INSTANCE.ID) + public Map listStepInstanceIdAndStepOrderMapping(Long taskInstanceId) { + Result> result = dsl() + .select(T_STEP_INSTANCE.ID, T_STEP_INSTANCE.STEP_ORDER) .from(T_STEP_INSTANCE) .where(T_STEP_INSTANCE.TASK_INSTANCE_ID.eq(taskInstanceId)) - .limit(1) .fetch(); - if (records.isEmpty()) { - return null; - } else { - return records.get(0).get(T_STEP_INSTANCE.ID); + if (result.isEmpty()) { + return Collections.emptyMap(); } + Map map = new HashMap<>(); + result.forEach(record -> map.put(record.get(T_STEP_INSTANCE.ID), record.get(T_STEP_INSTANCE.STEP_ORDER))); + return map; } @Override - public Byte getScriptTypeByStepInstanceId(long stepInstanceId) { - Result> records = CTX.select(TABLE_STEP_INSTANCE_SCRIPT.SCRIPT_TYPE) - .from(TABLE_STEP_INSTANCE_SCRIPT) - .where(TABLE_STEP_INSTANCE_SCRIPT.STEP_INSTANCE_ID.eq(stepInstanceId)) + @MySQLOperation(table = "step_instance", op = DbOperationEnum.READ) + public Long getStepInstanceId(Long taskInstanceId) { + Result> records = dsl().select(T_STEP_INSTANCE.ID) + .from(T_STEP_INSTANCE) + .where(T_STEP_INSTANCE.TASK_INSTANCE_ID.eq(taskInstanceId)) .limit(1) .fetch(); if (records.isEmpty()) { return null; } else { - return records.get(0).get(TABLE_STEP_INSTANCE_SCRIPT.SCRIPT_TYPE); - } - } - - public Integer countStepInstanceByConditions(Collection conditions) { - return CTX.selectCount().from(T_STEP_INSTANCE) - .where(conditions).fetchOne().value1(); - } - - public Integer countStepInstanceScriptByConditions(Collection conditions) { - return CTX.selectCount().from(TABLE_STEP_INSTANCE_SCRIPT) - .where(conditions).fetchOne().value1(); - } - - public Integer countStepInstanceFileByConditions(Collection conditions) { - return CTX.selectCount().from(TABLE_STEP_INSTANCE_FILE) - .where(conditions).fetchOne().value1(); - } - - @Override - public Integer count(Long appId, List stepIdList, StepExecuteTypeEnum stepExecuteType, - ScriptTypeEnum scriptType, RunStatusEnum runStatus, Long fromTime, Long toTime) { - List stepInstanceConditions = genStepInstanceConditions(appId, stepIdList, stepExecuteType, - runStatus, fromTime, toTime); - if (StepExecuteTypeEnum.EXECUTE_SCRIPT == stepExecuteType && scriptType != null) { - int totalCount = 0; - int offset = 0; - int limit = 10000; - List stepInstanceIds; - do { - // 1.查stepInstanceId - stepInstanceIds = listStepInstanceIds(stepInstanceConditions, offset, limit); - if (stepInstanceIds == null || stepInstanceIds.isEmpty()) { - break; - } - // 2.分批统计脚本步骤实例 - List stepInstanceScriptConditions = new ArrayList<>(); - stepInstanceScriptConditions.add(TABLE_STEP_INSTANCE_SCRIPT.STEP_INSTANCE_ID.in(stepInstanceIds)); - stepInstanceScriptConditions.add(TABLE_STEP_INSTANCE_SCRIPT.SCRIPT_TYPE.eq(scriptType.getValue().byteValue())); - totalCount += countStepInstanceScriptByConditions(stepInstanceScriptConditions); - offset += limit; - } while (stepInstanceIds.size() == limit); - return totalCount; - } else { - return countStepInstanceByConditions(stepInstanceConditions); + return records.get(0).get(T_STEP_INSTANCE.ID); } } - private List listStepInstanceIds(Collection conditions, int offset, int limit) { - Result> records = CTX.select(T_STEP_INSTANCE.ID) - .from(T_STEP_INSTANCE) - .where(conditions) - .limit(offset, limit) + @Override + @MySQLOperation(table = "step_instance_script", op = DbOperationEnum.READ) + public Byte getScriptTypeByStepInstanceId(Long taskInstanceId, long stepInstanceId) { + Result> records = dsl().select(T_STEP_INSTANCE_SCRIPT.SCRIPT_TYPE) + .from(T_STEP_INSTANCE_SCRIPT) + .where(buildTaskInstanceIdQueryCondition(T_STEP_INSTANCE_SCRIPT, taskInstanceId)) + .and(T_STEP_INSTANCE_SCRIPT.STEP_INSTANCE_ID.eq(stepInstanceId)) + .limit(1) .fetch(); if (records.isEmpty()) { - return Collections.emptyList(); - } else { - return records.map(it -> it.get(T_STEP_INSTANCE.ID)); - } - } - - private List genStepInstanceConditions(Long appId, List stepIdList, - StepExecuteTypeEnum stepExecuteType, RunStatusEnum runStatus, - Long fromTime, Long toTime) { - List conditions = new ArrayList<>(); - if (appId != null) { - conditions.add(T_STEP_INSTANCE.APP_ID.eq(appId)); - } - if (stepIdList != null && !stepIdList.isEmpty()) { - conditions.add(T_STEP_INSTANCE.STEP_ID.in(stepIdList)); - } - if (stepExecuteType != null) { - conditions.add(T_STEP_INSTANCE.TYPE.eq(stepExecuteType.getValue().byteValue())); - } - if (runStatus != null) { - conditions.add(T_STEP_INSTANCE.STATUS.eq(runStatus.getValue().byteValue())); - } - if (fromTime != null) { - conditions.add(T_STEP_INSTANCE.CREATE_TIME.greaterOrEqual(fromTime)); - } - if (toTime != null) { - conditions.add(T_STEP_INSTANCE.CREATE_TIME.lessThan(toTime)); - } - return conditions; - } - - private List genStepInstanceFileConditions(Collection stepInstanceIds, - DuplicateHandlerEnum fileDupliateHandle, - Boolean fileDupliateHandleNull, - NotExistPathHandlerEnum notExistPathHandler, - Boolean notExistPathHandlerNull) { - List conditions = new ArrayList<>(); - if (stepInstanceIds != null) { - conditions.add(TABLE_STEP_INSTANCE_FILE.STEP_INSTANCE_ID.in(stepInstanceIds)); - } - if (fileDupliateHandle != null) { - conditions.add(TABLE_STEP_INSTANCE_FILE.FILE_DUPLICATE_HANDLE.eq((byte) fileDupliateHandle.getId())); - } else { - if (fileDupliateHandleNull != null) { - if (fileDupliateHandleNull) { - conditions.add(TABLE_STEP_INSTANCE_FILE.FILE_DUPLICATE_HANDLE.isNull()); - } else { - conditions.add(TABLE_STEP_INSTANCE_FILE.FILE_DUPLICATE_HANDLE.isNotNull()); - } - } else { - // fileDupliateHandle与fileDupliateHandleNull同时为null表示FILE_DUPLICATE_HANDLE字段不作任何筛选 - } - } - if (notExistPathHandler != null) { - conditions.add(TABLE_STEP_INSTANCE_FILE.NOT_EXIST_PATH_HANDLER.eq(UByte.valueOf(notExistPathHandler.getValue()))); - } else { - if (notExistPathHandlerNull != null) { - if (notExistPathHandlerNull) { - conditions.add(TABLE_STEP_INSTANCE_FILE.NOT_EXIST_PATH_HANDLER.isNull()); - } else { - conditions.add(TABLE_STEP_INSTANCE_FILE.NOT_EXIST_PATH_HANDLER.isNotNull()); - } - } else { - // notExistPathHandler与notExistPathHandlerNull同时为null表示NOT_EXIST_PATH_HANDLER字段不作任何筛选 - } - } - return conditions; - } - - private List genConditions(Long appId, DuplicateHandlerEnum fileDupliateHandle, - Boolean fileDupliateHandleNull, NotExistPathHandlerEnum notExistPathHandler - , Boolean notExistPathHandlerNull, RunStatusEnum runStatus, Long fromTime, Long toTime) { - List conditions = new ArrayList<>(); - if (appId != null) { - conditions.add(T_STEP_INSTANCE.APP_ID.eq(appId)); - } - if (fileDupliateHandle != null) { - conditions.add(TABLE_STEP_INSTANCE_FILE.FILE_DUPLICATE_HANDLE.eq((byte) fileDupliateHandle.getId())); - } else { - if (fileDupliateHandleNull != null) { - if (fileDupliateHandleNull) { - conditions.add(TABLE_STEP_INSTANCE_FILE.FILE_DUPLICATE_HANDLE.isNull()); - } else { - conditions.add(TABLE_STEP_INSTANCE_FILE.FILE_DUPLICATE_HANDLE.isNotNull()); - } - } else { - // fileDupliateHandle与fileDupliateHandleNull同时为null表示FILE_DUPLICATE_HANDLE字段不作任何筛选 - } - } - if (notExistPathHandler != null) { - conditions.add(TABLE_STEP_INSTANCE_FILE.NOT_EXIST_PATH_HANDLER.eq(UByte.valueOf(notExistPathHandler.getValue()))); + return null; } else { - if (notExistPathHandlerNull != null) { - if (notExistPathHandlerNull) { - conditions.add(TABLE_STEP_INSTANCE_FILE.NOT_EXIST_PATH_HANDLER.isNull()); - } else { - conditions.add(TABLE_STEP_INSTANCE_FILE.NOT_EXIST_PATH_HANDLER.isNotNull()); - } - } else { - // notExistPathHandler与notExistPathHandlerNull同时为null表示NOT_EXIST_PATH_HANDLER字段不作任何筛选 - } - } - if (runStatus != null) { - conditions.add(T_STEP_INSTANCE.STATUS.eq(runStatus.getValue().byteValue())); - } - if (fromTime != null) { - conditions.add(T_STEP_INSTANCE.CREATE_TIME.greaterOrEqual(fromTime)); - } - if (toTime != null) { - conditions.add(T_STEP_INSTANCE.CREATE_TIME.lessThan(toTime)); - } - return conditions; - } - - @Override - public List> listFastPushFileSource(Long appId, DuplicateHandlerEnum fileDupliateHandle, - Boolean fileDupliateHandleNull, - NotExistPathHandlerEnum notExistPathHandler, - Boolean notExistPathHandlerNull, RunStatusEnum runStatus, - Long fromTime, Long toTime) { - List conditions = genConditions(appId, fileDupliateHandle, fileDupliateHandleNull, - notExistPathHandler, notExistPathHandlerNull, runStatus, fromTime, toTime); - val query = - CTX.select(TABLE_STEP_INSTANCE_FILE.FILE_SOURCE).from(T_STEP_INSTANCE).join(TABLE_STEP_INSTANCE_FILE) - .on(T_STEP_INSTANCE.ID.eq(TABLE_STEP_INSTANCE_FILE.STEP_INSTANCE_ID)) - .where(conditions); - String sql = query.getSQL(ParamType.INLINED); - try { - Result> records = query.fetch(); - List> resultList = new ArrayList<>(); - records.forEach(record -> { - resultList.add(convertStringToFileSourceDTO((String) record.get(0))); - }); - return resultList; - } catch (Exception e) { - log.error("Fail to query:{}", sql, e); - throw new RuntimeException("Fail to listFastPushFileSource", e); + return records.get(0).get(T_STEP_INSTANCE_SCRIPT.SCRIPT_TYPE); } } @Override - public Integer countFastPushFile(Long appId, DuplicateHandlerEnum fileDupliateHandle, - Boolean fileDupliateHandleNull, NotExistPathHandlerEnum notExistPathHandler, - Boolean notExistPathHandlerNull, RunStatusEnum runStatus, Long fromTime, - Long toTime) { - // 1.查stepInstanceId - int totalCount = 0; - int offset = 0; - int limit = 10000; - List stepInstanceIds; - do { - stepInstanceIds = listStepInstanceIds(genStepInstanceConditions(appId, null, null, null, fromTime, - toTime), offset, limit); - if (stepInstanceIds == null || stepInstanceIds.isEmpty()) { - break; - } - // 2.分批统计文件步骤实例 - List stepInstanceFileConditions = genStepInstanceFileConditions(stepInstanceIds, - fileDupliateHandle, fileDupliateHandleNull, notExistPathHandler, notExistPathHandlerNull); - totalCount += countStepInstanceFileByConditions(stepInstanceFileConditions); - offset += limit; - } while (stepInstanceIds.size() == limit); - return totalCount; - } - - private List convertStringToFileSourceDTO(String str) { - return JsonUtils.fromJson(str, new TypeReference>() { - }); + @MySQLOperation(table = "step_instance", op = DbOperationEnum.WRITE) + public void updateStepCurrentBatch(Long taskInstanceId, long stepInstanceId, int batch) { + dsl().update(T_STEP_INSTANCE) + .set(T_STEP_INSTANCE.BATCH, JooqDataTypeUtil.toShort(batch)) + .where(buildTaskInstanceIdQueryCondition(T_STEP_INSTANCE, taskInstanceId)) + .and(T_STEP_INSTANCE.ID.eq(stepInstanceId)) + .execute(); } @Override - public void updateStepCurrentBatch(long stepInstanceId, int batch) { - CTX.update(T_STEP_INSTANCE).set(T_STEP_INSTANCE.BATCH, JooqDataTypeUtil.toShort(batch)) - .where(T_STEP_INSTANCE.ID.eq(stepInstanceId)) + @MySQLOperation(table = "step_instance", op = DbOperationEnum.WRITE) + public void updateStepCurrentExecuteCount(Long taskInstanceId, long stepInstanceId, int executeCount) { + dsl().update(T_STEP_INSTANCE) + .set(T_STEP_INSTANCE.EXECUTE_COUNT, executeCount) + .where(buildTaskInstanceIdQueryCondition(T_STEP_INSTANCE, taskInstanceId)) + .and(T_STEP_INSTANCE.ID.eq(stepInstanceId)) .execute(); } @Override - public void updateStepCurrentExecuteCount(long stepInstanceId, int executeCount) { - CTX.update(T_STEP_INSTANCE).set(T_STEP_INSTANCE.EXECUTE_COUNT, executeCount) - .where(T_STEP_INSTANCE.ID.eq(stepInstanceId)) + @MySQLOperation(table = "step_instance", op = DbOperationEnum.WRITE) + public void updateStepRollingConfigId(Long taskInstanceId, long stepInstanceId, long rollingConfigId) { + dsl().update(T_STEP_INSTANCE) + .set(T_STEP_INSTANCE.ROLLING_CONFIG_ID, rollingConfigId) + .where(buildTaskInstanceIdQueryCondition(T_STEP_INSTANCE, taskInstanceId)) + .and(T_STEP_INSTANCE.ID.eq(stepInstanceId)) .execute(); } @Override - public void updateStepRollingConfigId(long stepInstanceId, long rollingConfigId) { - CTX.update(T_STEP_INSTANCE).set(T_STEP_INSTANCE.ROLLING_CONFIG_ID, rollingConfigId) - .where(T_STEP_INSTANCE.ID.eq(stepInstanceId)) - .execute(); + @MySQLOperation(table = "step_instance", op = DbOperationEnum.READ) + public List getTaskStepInstanceIdList(Long taskInstanceId) { + Result result = + dsl().select(StepInstance.STEP_INSTANCE.ID).from(StepInstance.STEP_INSTANCE) + .where(StepInstance.STEP_INSTANCE.TASK_INSTANCE_ID.eq(taskInstanceId)) + .orderBy(StepInstance.STEP_INSTANCE.ID.asc()) + .fetch(); + List stepInstanceIdList = new ArrayList<>(); + result.into(record -> { + Long stepInstanceId = record.getValue(StepInstance.STEP_INSTANCE.ID); + if (stepInstanceId != null) { + stepInstanceIdList.add(stepInstanceId); + } + }); + return stepInstanceIdList; } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceRollingTaskDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceRollingTaskDAOImpl.java index 2d7c1d7339..405ffa11b2 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceRollingTaskDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceRollingTaskDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,21 +24,23 @@ package com.tencent.bk.job.execute.dao.impl; +import com.tencent.bk.job.common.mysql.dynamic.ds.DbOperationEnum; +import com.tencent.bk.job.common.mysql.dynamic.ds.MySQLOperation; +import com.tencent.bk.job.common.mysql.jooq.JooqDataTypeUtil; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; -import com.tencent.bk.job.execute.common.util.JooqDataTypeUtil; import com.tencent.bk.job.execute.dao.StepInstanceRollingTaskDAO; +import com.tencent.bk.job.execute.dao.common.DSLContextProviderFactory; import com.tencent.bk.job.execute.model.StepInstanceRollingTaskDTO; +import com.tencent.bk.job.execute.model.tables.StepInstanceRollingTask; +import com.tencent.bk.job.execute.model.tables.records.StepInstanceRollingTaskRecord; import lombok.extern.slf4j.Slf4j; -import org.jooq.DSLContext; +import org.jooq.Condition; import org.jooq.Record; import org.jooq.Result; import org.jooq.SelectConditionStep; import org.jooq.TableField; import org.jooq.UpdateSetMoreStep; -import org.jooq.generated.tables.StepInstanceRollingTask; -import org.jooq.generated.tables.records.StepInstanceRollingTaskRecord; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; import java.util.ArrayList; @@ -46,11 +48,12 @@ @Repository @Slf4j -public class StepInstanceRollingTaskDAOImpl implements StepInstanceRollingTaskDAO { +public class StepInstanceRollingTaskDAOImpl extends BaseDAO implements StepInstanceRollingTaskDAO { private static final StepInstanceRollingTask TABLE = StepInstanceRollingTask.STEP_INSTANCE_ROLLING_TASK; private static final TableField[] ALL_FIELDS = { TABLE.ID, + TABLE.TASK_INSTANCE_ID, TABLE.STEP_INSTANCE_ID, TABLE.EXECUTE_COUNT, TABLE.BATCH, @@ -59,30 +62,42 @@ public class StepInstanceRollingTaskDAOImpl implements StepInstanceRollingTaskDA TABLE.END_TIME, TABLE.TOTAL_TIME }; - private final DSLContext CTX; @Autowired - public StepInstanceRollingTaskDAOImpl(@Qualifier("job-execute-dsl-context") DSLContext CTX) { - this.CTX = CTX; + public StepInstanceRollingTaskDAOImpl(DSLContextProviderFactory dslContextProviderFactory) { + super(dslContextProviderFactory, TABLE.getName()); } @Override - public StepInstanceRollingTaskDTO queryRollingTask(long stepInstanceId, int executeCount, int batch) { - Record record = CTX.select(ALL_FIELDS) + @MySQLOperation(table = "step_instance_rolling_task", op = DbOperationEnum.READ) + public StepInstanceRollingTaskDTO queryRollingTask(Long taskInstanceId, + long stepInstanceId, + int executeCount, + int batch) { + Record record = dsl().select(ALL_FIELDS) .from(TABLE) .where(TABLE.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)) .and(TABLE.EXECUTE_COUNT.eq(JooqDataTypeUtil.toShort(executeCount))) .and(TABLE.BATCH.eq(JooqDataTypeUtil.toShort(batch))) .fetchOne(); return extract(record); } + private Condition buildTaskInstanceIdQueryCondition(Long taskInstanceId) { + return TaskInstanceIdDynamicCondition.build( + taskInstanceId, + TABLE.TASK_INSTANCE_ID::eq + ); + } + private StepInstanceRollingTaskDTO extract(Record record) { if (record == null) { return null; } StepInstanceRollingTaskDTO stepInstanceRollingTask = new StepInstanceRollingTaskDTO(); stepInstanceRollingTask.setId(record.get(TABLE.ID)); + stepInstanceRollingTask.setTaskInstanceId(record.get(TABLE.TASK_INSTANCE_ID)); stepInstanceRollingTask.setStepInstanceId(record.get(TABLE.STEP_INSTANCE_ID)); stepInstanceRollingTask.setExecuteCount(record.get(TABLE.EXECUTE_COUNT).intValue()); stepInstanceRollingTask.setBatch(record.get(TABLE.BATCH).intValue()); @@ -94,12 +109,15 @@ private StepInstanceRollingTaskDTO extract(Record record) { } @Override - public List listRollingTasks(long stepInstanceId, + @MySQLOperation(table = "step_instance_rolling_task", op = DbOperationEnum.READ) + public List listRollingTasks(Long taskInstanceId, + long stepInstanceId, Integer executeCount, Integer batch) { - SelectConditionStep selectConditionStep = CTX.select(ALL_FIELDS) + SelectConditionStep selectConditionStep = dsl().select(ALL_FIELDS) .from(TABLE) - .where(TABLE.STEP_INSTANCE_ID.eq(stepInstanceId)); + .where(TABLE.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)); if (executeCount != null) { selectConditionStep.and(TABLE.EXECUTE_COUNT.eq(executeCount.shortValue())); } @@ -110,24 +128,29 @@ public List listRollingTasks(long stepInstanceId, Result result = selectConditionStep.orderBy(TABLE.BATCH.asc()).fetch(); List stepInstanceRollingTasks = new ArrayList<>(); - if (result.size() > 0) { + if (!result.isEmpty()) { stepInstanceRollingTasks = result.map(this::extract); } return stepInstanceRollingTasks; } @Override + @MySQLOperation(table = "step_instance_rolling_task", op = DbOperationEnum.WRITE) public long saveRollingTask(StepInstanceRollingTaskDTO rollingTask) { - Record record = CTX.insertInto( - TABLE, - TABLE.STEP_INSTANCE_ID, - TABLE.EXECUTE_COUNT, - TABLE.BATCH, - TABLE.STATUS, - TABLE.START_TIME, - TABLE.END_TIME, - TABLE.TOTAL_TIME) + Record record = dsl().insertInto( + TABLE, + TABLE.ID, + TABLE.TASK_INSTANCE_ID, + TABLE.STEP_INSTANCE_ID, + TABLE.EXECUTE_COUNT, + TABLE.BATCH, + TABLE.STATUS, + TABLE.START_TIME, + TABLE.END_TIME, + TABLE.TOTAL_TIME) .values( + rollingTask.getId(), + rollingTask.getTaskInstanceId(), rollingTask.getStepInstanceId(), JooqDataTypeUtil.toShort(rollingTask.getExecuteCount()), JooqDataTypeUtil.toShort(rollingTask.getBatch()), @@ -137,12 +160,14 @@ public long saveRollingTask(StepInstanceRollingTaskDTO rollingTask) { rollingTask.getTotalTime()) .returning(TABLE.ID) .fetchOne(); - assert record != null; - return record.get(TABLE.ID); + return rollingTask.getId() != null ? rollingTask.getId() : record.getValue(TABLE.ID); + } @Override - public void updateRollingTask(long stepInstanceId, + @MySQLOperation(table = "step_instance_rolling_task", op = DbOperationEnum.WRITE) + public void updateRollingTask(Long taskInstanceId, + long stepInstanceId, int executeCount, int batch, RunStatusEnum status, @@ -151,25 +176,25 @@ public void updateRollingTask(long stepInstanceId, Long totalTime) { UpdateSetMoreStep updateSetMoreStep = null; if (status != null) { - updateSetMoreStep = CTX.update(TABLE).set(TABLE.STATUS, JooqDataTypeUtil.toByte(status.getValue())); + updateSetMoreStep = dsl().update(TABLE).set(TABLE.STATUS, JooqDataTypeUtil.toByte(status.getValue())); } if (startTime != null) { if (updateSetMoreStep == null) { - updateSetMoreStep = CTX.update(TABLE).set(TABLE.START_TIME, startTime); + updateSetMoreStep = dsl().update(TABLE).set(TABLE.START_TIME, startTime); } else { updateSetMoreStep.set(TABLE.START_TIME, startTime); } } if (endTime != null) { if (updateSetMoreStep == null) { - updateSetMoreStep = CTX.update(TABLE).set(TABLE.END_TIME, endTime); + updateSetMoreStep = dsl().update(TABLE).set(TABLE.END_TIME, endTime); } else { updateSetMoreStep.set(TABLE.END_TIME, endTime); } } if (totalTime != null) { if (updateSetMoreStep == null) { - updateSetMoreStep = CTX.update(TABLE).set(TABLE.TOTAL_TIME, totalTime); + updateSetMoreStep = dsl().update(TABLE).set(TABLE.TOTAL_TIME, totalTime); } else { updateSetMoreStep.set(TABLE.TOTAL_TIME, totalTime); } @@ -181,6 +206,7 @@ public void updateRollingTask(long stepInstanceId, updateSetMoreStep.where(TABLE.STEP_INSTANCE_ID.eq(stepInstanceId)) .and(TABLE.EXECUTE_COUNT.eq(JooqDataTypeUtil.toByte(executeCount).shortValue())) .and(TABLE.BATCH.eq(JooqDataTypeUtil.toShort(batch))) + .and(buildTaskInstanceIdQueryCondition(taskInstanceId)) .execute(); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceVariableDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceVariableDAOImpl.java index 8bb05d97d8..d91e11e3b8 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceVariableDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/StepInstanceVariableDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,56 +24,80 @@ package com.tencent.bk.job.execute.dao.impl; +import com.tencent.bk.job.common.mysql.dynamic.ds.DbOperationEnum; +import com.tencent.bk.job.common.mysql.dynamic.ds.MySQLOperation; +import com.tencent.bk.job.common.mysql.jooq.JooqDataTypeUtil; import com.tencent.bk.job.common.util.json.JsonUtils; -import com.tencent.bk.job.execute.common.util.JooqDataTypeUtil; import com.tencent.bk.job.execute.constants.VariableValueTypeEnum; import com.tencent.bk.job.execute.dao.StepInstanceVariableDAO; +import com.tencent.bk.job.execute.dao.common.DSLContextProviderFactory; import com.tencent.bk.job.execute.model.StepInstanceVariableValuesDTO; +import com.tencent.bk.job.execute.model.tables.StepInstanceVariable; import org.apache.commons.lang3.StringUtils; -import org.jooq.DSLContext; import org.jooq.Record; import org.jooq.Result; import org.jooq.TableField; -import org.jooq.generated.tables.StepInstanceVariable; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; import java.util.ArrayList; import java.util.List; @Repository -public class StepInstanceVariableDAOImpl implements StepInstanceVariableDAO { +public class StepInstanceVariableDAOImpl extends BaseDAO implements StepInstanceVariableDAO { private static final StepInstanceVariable TABLE = StepInstanceVariable.STEP_INSTANCE_VARIABLE; - private DSLContext ctx; - private TableField[] FIELDS = {TABLE.TASK_INSTANCE_ID, TABLE.STEP_INSTANCE_ID, - TABLE.EXECUTE_COUNT, TABLE.TYPE, TABLE.PARAM_VALUES}; + + private final TableField[] FIELDS = { + TABLE.ID, + TABLE.TASK_INSTANCE_ID, + TABLE.STEP_INSTANCE_ID, + TABLE.EXECUTE_COUNT, + TABLE.TYPE, + TABLE.PARAM_VALUES + }; @Autowired - public StepInstanceVariableDAOImpl(@Qualifier("job-execute-dsl-context") DSLContext ctx) { - this.ctx = ctx; + public StepInstanceVariableDAOImpl(DSLContextProviderFactory dslContextProviderFactory) { + super(dslContextProviderFactory, TABLE.getName()); } @Override + @MySQLOperation(table = "step_instance_variable", op = DbOperationEnum.WRITE) public void saveVariableValues(StepInstanceVariableValuesDTO variableValues) { - ctx.insertInto(TABLE, FIELDS) - .values(variableValues.getTaskInstanceId(), variableValues.getStepInstanceId(), - variableValues.getExecuteCount(), JooqDataTypeUtil.toByte(variableValues.getType()), - JsonUtils.toJson(variableValues)) - .execute(); + dsl().insertInto( + TABLE, + TABLE.ID, + TABLE.TASK_INSTANCE_ID, + TABLE.STEP_INSTANCE_ID, + TABLE.EXECUTE_COUNT, + TABLE.TYPE, + TABLE.PARAM_VALUES) + .values( + variableValues.getId(), + variableValues.getTaskInstanceId(), + variableValues.getStepInstanceId(), + variableValues.getExecuteCount(), + JooqDataTypeUtil.toByte(variableValues.getType()), + JsonUtils.toJson(variableValues)) + .execute(); } @Override - public StepInstanceVariableValuesDTO getStepVariableValues(long stepInstanceId, int executeCount, + @MySQLOperation(table = "step_instance_variable", op = DbOperationEnum.READ) + public StepInstanceVariableValuesDTO getStepVariableValues(Long taskInstanceId, + long stepInstanceId, + int executeCount, VariableValueTypeEnum variableValueType) { - Record record = ctx.select(FIELDS) - .from(TABLE) - .where(TABLE.STEP_INSTANCE_ID.eq(stepInstanceId)) - .and(TABLE.EXECUTE_COUNT.eq(executeCount)) - .and(TABLE.TYPE.eq(JooqDataTypeUtil.toByte(variableValueType.getValue()))) - .limit(1) - .fetchOne(); + Record record = dsl().select(FIELDS) + .from(TABLE) + .where(TaskInstanceIdDynamicCondition.build(taskInstanceId, + TABLE.TASK_INSTANCE_ID::eq)) + .and(TABLE.STEP_INSTANCE_ID.eq(stepInstanceId)) + .and(TABLE.EXECUTE_COUNT.eq(executeCount)) + .and(TABLE.TYPE.eq(JooqDataTypeUtil.toByte(variableValueType.getValue()))) + .limit(1) + .fetchOne(); return extract(record); } @@ -83,6 +107,7 @@ private StepInstanceVariableValuesDTO extract(Record record) { } StepInstanceVariableValuesDTO stepInstanceVariableValues = new StepInstanceVariableValuesDTO(); + stepInstanceVariableValues.setId(record.get(TABLE.ID)); stepInstanceVariableValues.setTaskInstanceId(record.get(TABLE.TASK_INSTANCE_ID)); stepInstanceVariableValues.setStepInstanceId(record.get(TABLE.STEP_INSTANCE_ID)); stepInstanceVariableValues.setExecuteCount(record.get(TABLE.EXECUTE_COUNT)); @@ -98,28 +123,11 @@ private StepInstanceVariableValuesDTO extract(Record record) { } @Override + @MySQLOperation(table = "step_instance_variable", op = DbOperationEnum.READ) public List listStepOutputVariableValuesByTaskInstanceId(long taskInstanceId) { - Result result = ctx.select(FIELDS) - .from(TABLE) - .where(TABLE.TASK_INSTANCE_ID.eq(taskInstanceId)) - .and(TABLE.TYPE.eq(JooqDataTypeUtil.toByte(VariableValueTypeEnum.OUTPUT.getValue()))) - .orderBy(TABLE.STEP_INSTANCE_ID.asc(), TABLE.EXECUTE_COUNT.asc()) - .fetch(); - - List varList = new ArrayList<>(); - if (!result.isEmpty()) { - result.into(record -> varList.add(extract(record))); - } - return varList; - } - - @Override - public List listSortedPreStepOutputVariableValues(long taskInstanceId, - long stepInstanceId) { - Result result = ctx.select(FIELDS) + Result result = dsl().select(FIELDS) .from(TABLE) .where(TABLE.TASK_INSTANCE_ID.eq(taskInstanceId)) - .and(TABLE.STEP_INSTANCE_ID.lt(stepInstanceId)) .and(TABLE.TYPE.eq(JooqDataTypeUtil.toByte(VariableValueTypeEnum.OUTPUT.getValue()))) .orderBy(TABLE.STEP_INSTANCE_ID.asc(), TABLE.EXECUTE_COUNT.asc()) .fetch(); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceDAOImpl.java index 9c6c140ed6..43d3671a4a 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,32 +27,32 @@ import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.common.mysql.dynamic.ds.DbOperationEnum; +import com.tencent.bk.job.common.mysql.dynamic.ds.MySQLOperation; +import com.tencent.bk.job.common.mysql.jooq.JooqDataTypeUtil; +import com.tencent.bk.job.common.util.BatchUtil; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; -import com.tencent.bk.job.execute.common.constants.TaskStartupModeEnum; -import com.tencent.bk.job.execute.common.constants.TaskTypeEnum; -import com.tencent.bk.job.execute.common.util.JooqDataTypeUtil; import com.tencent.bk.job.execute.dao.TaskInstanceDAO; +import com.tencent.bk.job.execute.dao.common.DSLContextProviderFactory; import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.model.TaskInstanceQuery; +import com.tencent.bk.job.execute.model.tables.TaskInstance; +import com.tencent.bk.job.execute.model.tables.TaskInstanceHost; +import com.tencent.bk.job.execute.model.tables.records.TaskInstanceRecord; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.jooq.BatchBindStep; import org.jooq.Condition; -import org.jooq.DSLContext; import org.jooq.Record; import org.jooq.Record1; import org.jooq.Result; import org.jooq.SelectSeekStep1; import org.jooq.SortField; +import org.jooq.TableField; import org.jooq.UpdateSetMoreStep; import org.jooq.conf.ParamType; -import org.jooq.generated.tables.StepInstance; -import org.jooq.generated.tables.TaskInstance; -import org.jooq.generated.tables.TaskInstanceHost; -import org.jooq.generated.tables.records.TaskInstanceRecord; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; import java.time.Instant; @@ -65,26 +65,62 @@ */ @Slf4j @Repository -public class TaskInstanceDAOImpl implements TaskInstanceDAO { +public class TaskInstanceDAOImpl extends BaseDAO implements TaskInstanceDAO { private static final TaskInstance TASK_INSTANCE = TaskInstance.TASK_INSTANCE; private static final TaskInstanceHost TASK_INSTANCE_HOST = TaskInstanceHost.TASK_INSTANCE_HOST; - private final DSLContext ctx; + + private static final TableField[] ALL_FIELDS = { + TASK_INSTANCE.ID, + TASK_INSTANCE.TASK_ID, + TASK_INSTANCE.CRON_TASK_ID, + TASK_INSTANCE.TASK_TEMPLATE_ID, + TASK_INSTANCE.IS_DEBUG_TASK, + TASK_INSTANCE.APP_ID, + TASK_INSTANCE.NAME, + TASK_INSTANCE.OPERATOR, + TASK_INSTANCE.STARTUP_MODE, + TASK_INSTANCE.CURRENT_STEP_ID, + TASK_INSTANCE.STATUS, + TASK_INSTANCE.START_TIME, + TASK_INSTANCE.END_TIME, + TASK_INSTANCE.TOTAL_TIME, + TASK_INSTANCE.CREATE_TIME, + TASK_INSTANCE.CALLBACK_URL, + TASK_INSTANCE.TYPE, + TASK_INSTANCE.APP_CODE + }; @Autowired - public TaskInstanceDAOImpl(@Qualifier("job-execute-dsl-context") DSLContext ctx) { - this.ctx = ctx; + public TaskInstanceDAOImpl(DSLContextProviderFactory dslContextProviderFactory) { + super(dslContextProviderFactory, TASK_INSTANCE.getName()); } @Override + @MySQLOperation(table = "task_instance", op = DbOperationEnum.WRITE) public Long addTaskInstance(TaskInstanceDTO taskInstance) { - Record record = ctx.insertInto(TASK_INSTANCE, TASK_INSTANCE.TASK_ID, TASK_INSTANCE.CRON_TASK_ID, - TASK_INSTANCE.TASK_TEMPLATE_ID, - TASK_INSTANCE.IS_DEBUG_TASK, TASK_INSTANCE.APP_ID, TASK_INSTANCE.NAME, TASK_INSTANCE.OPERATOR, - TASK_INSTANCE.STARTUP_MODE, TASK_INSTANCE.CURRENT_STEP_ID, - TASK_INSTANCE.STATUS, TASK_INSTANCE.START_TIME, - TASK_INSTANCE.END_TIME, TASK_INSTANCE.TOTAL_TIME, TASK_INSTANCE.CREATE_TIME, TASK_INSTANCE.CALLBACK_URL, - TASK_INSTANCE.TYPE, TASK_INSTANCE.APP_CODE) - .values(taskInstance.getTaskId(), + Record record = dsl().insertInto( + TASK_INSTANCE, + TASK_INSTANCE.ID, + TASK_INSTANCE.TASK_ID, + TASK_INSTANCE.CRON_TASK_ID, + TASK_INSTANCE.TASK_TEMPLATE_ID, + TASK_INSTANCE.IS_DEBUG_TASK, + TASK_INSTANCE.APP_ID, + TASK_INSTANCE.NAME, + TASK_INSTANCE.OPERATOR, + TASK_INSTANCE.STARTUP_MODE, + TASK_INSTANCE.CURRENT_STEP_ID, + TASK_INSTANCE.STATUS, + TASK_INSTANCE.START_TIME, + TASK_INSTANCE.END_TIME, + TASK_INSTANCE.TOTAL_TIME, + TASK_INSTANCE.CREATE_TIME, + TASK_INSTANCE.CALLBACK_URL, + TASK_INSTANCE.TYPE, + TASK_INSTANCE.APP_CODE) + .values( + taskInstance.getId(), + taskInstance.getPlanId(), taskInstance.getCronTaskId(), taskInstance.getTaskTemplateId(), taskInstance.isDebugTask() ? (byte) 1 : (byte) 0, @@ -101,21 +137,19 @@ public Long addTaskInstance(TaskInstanceDTO taskInstance) { taskInstance.getCallbackUrl(), JooqDataTypeUtil.toByte(taskInstance.getType()), taskInstance.getAppCode()) - .returning(TASK_INSTANCE.ID).fetchOne(); - return record.getValue(TASK_INSTANCE.ID); + .returning(TASK_INSTANCE.ID) + .fetchOne(); + + return taskInstance.getId() != null ? taskInstance.getId() : record.getValue(TASK_INSTANCE.ID); } @Override + @MySQLOperation(table = "task_instance", op = DbOperationEnum.READ) public TaskInstanceDTO getTaskInstance(long taskInstanceId) { - Record record = ctx.select(TASK_INSTANCE.ID, TASK_INSTANCE.TASK_ID, TASK_INSTANCE.CRON_TASK_ID, - TASK_INSTANCE.TASK_TEMPLATE_ID, - TASK_INSTANCE.IS_DEBUG_TASK, TASK_INSTANCE.APP_ID, TASK_INSTANCE.NAME, TASK_INSTANCE.OPERATOR, - TASK_INSTANCE.STARTUP_MODE, TASK_INSTANCE.CURRENT_STEP_ID, - TASK_INSTANCE.STATUS, - TASK_INSTANCE.START_TIME, TASK_INSTANCE.END_TIME, TASK_INSTANCE.TOTAL_TIME, TASK_INSTANCE.CREATE_TIME, - TASK_INSTANCE.CALLBACK_URL, TASK_INSTANCE.TYPE, - TASK_INSTANCE.APP_CODE).from(TASK_INSTANCE) - .where(TASK_INSTANCE.ID.eq(taskInstanceId)).fetchOne(); + Record record = dsl().select(ALL_FIELDS) + .from(TASK_INSTANCE) + .where(TASK_INSTANCE.ID.eq(taskInstanceId)) + .fetchOne(); return extractInfo(record); } @@ -125,7 +159,7 @@ private TaskInstanceDTO extractInfo(Record record) { } TaskInstanceDTO taskInstance = new TaskInstanceDTO(); taskInstance.setId(record.get(TaskInstance.TASK_INSTANCE.ID)); - taskInstance.setTaskId(record.get(TaskInstance.TASK_INSTANCE.TASK_ID)); + taskInstance.setPlanId(record.get(TaskInstance.TASK_INSTANCE.TASK_ID)); taskInstance.setCronTaskId(record.get(TaskInstance.TASK_INSTANCE.CRON_TASK_ID)); taskInstance.setTaskTemplateId(record.get(TaskInstance.TASK_INSTANCE.TASK_TEMPLATE_ID)); taskInstance.setDebugTask(record.get(TaskInstance.TASK_INSTANCE.IS_DEBUG_TASK) == 1); @@ -146,73 +180,29 @@ private TaskInstanceDTO extractInfo(Record record) { } @Override - public List getTaskInstanceByTaskId(long taskId) { - Result result = ctx.select(TASK_INSTANCE.ID, TASK_INSTANCE.TASK_ID, TASK_INSTANCE.CRON_TASK_ID, - TASK_INSTANCE.TASK_TEMPLATE_ID, - TASK_INSTANCE.IS_DEBUG_TASK, TASK_INSTANCE.APP_ID, TASK_INSTANCE.NAME, TASK_INSTANCE.OPERATOR, - TASK_INSTANCE.STARTUP_MODE, TASK_INSTANCE.CURRENT_STEP_ID, - TASK_INSTANCE.STATUS, - TASK_INSTANCE.START_TIME, TASK_INSTANCE.END_TIME, TASK_INSTANCE.TOTAL_TIME, TASK_INSTANCE.CREATE_TIME, - TASK_INSTANCE.CALLBACK_URL, TASK_INSTANCE.TYPE, - TASK_INSTANCE.APP_CODE).from(TASK_INSTANCE) - .where(TASK_INSTANCE.TASK_ID.eq(taskId)).fetch(); - List taskInstances = new ArrayList<>(); - result.into(record -> { - TaskInstanceDTO taskInstance = extractInfo(record); - if (taskInstance != null) { - taskInstances.add(taskInstance); - } - }); - return taskInstances; - } - - @Override + @MySQLOperation(table = "task_instance", op = DbOperationEnum.WRITE) public void updateTaskStatus(long taskInstanceId, int status) { - ctx.update(TASK_INSTANCE).set(TASK_INSTANCE.STATUS, Byte.valueOf(String.valueOf(status))) - .where(TASK_INSTANCE.ID.eq(taskInstanceId)) - .execute(); - } - - @Override - public void updateTaskStartTime(long taskInstanceId, Long startTime) { - ctx.update(TASK_INSTANCE).set(TASK_INSTANCE.START_TIME, startTime) - .where(TASK_INSTANCE.ID.eq(taskInstanceId)) - .execute(); - } - - @Override - public void updateTaskEndTime(long taskInstanceId, Long endTime) { - ctx.update(TASK_INSTANCE).set(TASK_INSTANCE.END_TIME, endTime) + dsl().update(TASK_INSTANCE).set(TASK_INSTANCE.STATUS, + Byte.valueOf(String.valueOf(status))) .where(TASK_INSTANCE.ID.eq(taskInstanceId)) .execute(); } - @Override - public List getTaskStepInstanceIdList(long taskInstanceId) { - Result result = ctx.select(StepInstance.STEP_INSTANCE.ID).from(StepInstance.STEP_INSTANCE) - .where(StepInstance.STEP_INSTANCE.TASK_INSTANCE_ID.eq(taskInstanceId)) - .orderBy(StepInstance.STEP_INSTANCE.ID.asc()) - .fetch(); - List stepInstanceIdList = new ArrayList<>(); - result.into(record -> { - Long stepInstanceId = record.getValue(StepInstance.STEP_INSTANCE.ID); - if (stepInstanceId != null) { - stepInstanceIdList.add(stepInstanceId); - } - }); - return stepInstanceIdList; - } @Override + @MySQLOperation(table = "task_instance", op = DbOperationEnum.WRITE) public void updateTaskCurrentStepId(Long taskInstanceId, Long stepInstanceId) { - ctx.update(TASK_INSTANCE).set(TASK_INSTANCE.CURRENT_STEP_ID, stepInstanceId) + dsl().update(TASK_INSTANCE).set(TASK_INSTANCE.CURRENT_STEP_ID, stepInstanceId) .where(TASK_INSTANCE.ID.eq(taskInstanceId)) .execute(); } @Override + @MySQLOperation(table = "task_instance", op = DbOperationEnum.WRITE) public void resetTaskStatus(Long taskInstanceId) { - ctx.update(TASK_INSTANCE).setNull(TASK_INSTANCE.START_TIME).setNull(TASK_INSTANCE.END_TIME).setNull(TASK_INSTANCE.TOTAL_TIME) + dsl().update(TASK_INSTANCE) + .setNull(TASK_INSTANCE.START_TIME).setNull(TASK_INSTANCE.END_TIME) + .setNull(TASK_INSTANCE.TOTAL_TIME) .setNull(TASK_INSTANCE.CURRENT_STEP_ID) .set(TASK_INSTANCE.STATUS, JooqDataTypeUtil.toByte(RunStatusEnum.BLANK.getValue())) .where(TASK_INSTANCE.ID.eq(taskInstanceId)) @@ -220,20 +210,7 @@ public void resetTaskStatus(Long taskInstanceId) { } @Override - public void cleanTaskEndTime(Long taskInstanceId) { - ctx.update(TASK_INSTANCE).setNull(TASK_INSTANCE.END_TIME).setNull(TASK_INSTANCE.TOTAL_TIME) - .where(TASK_INSTANCE.ID.eq(taskInstanceId)) - .execute(); - } - - @Override - public void updateTaskTotalTime(Long taskInstanceId, Long totalTime) { - ctx.update(TASK_INSTANCE).set(TASK_INSTANCE.TOTAL_TIME, totalTime) - .where(TASK_INSTANCE.ID.eq(taskInstanceId)) - .execute(); - } - - @Override + @MySQLOperation(table = "task_instance", op = DbOperationEnum.READ) public PageData listPageTaskInstance(TaskInstanceQuery taskQuery, BaseSearchCondition baseSearchCondition) { if (StringUtils.isNotEmpty(taskQuery.getIp()) || StringUtils.isNotEmpty(taskQuery.getIpv6())) { @@ -248,26 +225,20 @@ private PageData listPageTaskInstanceByBasicInfo(TaskInstanceQu int start = baseSearchCondition.getStartOrDefault(0); int length = baseSearchCondition.getLengthOrDefault(10); - int count = getPageTaskInstanceCount(taskQuery); - if (count == 0) { - return PageData.emptyPageData(start, length); - } - Collection> orderFields = new ArrayList<>(); orderFields.add(TASK_INSTANCE.CREATE_TIME.desc()); - Result result = ctx.select(TASK_INSTANCE.ID, TASK_INSTANCE.TASK_ID, TASK_INSTANCE.CRON_TASK_ID, - TASK_INSTANCE.TASK_TEMPLATE_ID, - TASK_INSTANCE.IS_DEBUG_TASK, TASK_INSTANCE.APP_ID, TASK_INSTANCE.NAME, TASK_INSTANCE.OPERATOR, - TASK_INSTANCE.STARTUP_MODE, TASK_INSTANCE.CURRENT_STEP_ID, - TASK_INSTANCE.STATUS, - TASK_INSTANCE.START_TIME, TASK_INSTANCE.END_TIME, TASK_INSTANCE.TOTAL_TIME, TASK_INSTANCE.CREATE_TIME, - TASK_INSTANCE.CALLBACK_URL, TASK_INSTANCE.TYPE, - TASK_INSTANCE.APP_CODE) + Result result = dsl().select(ALL_FIELDS) .from(TaskInstanceDAOImpl.TASK_INSTANCE) .where(buildSearchCondition(taskQuery)) .orderBy(orderFields) .limit(start, length) .fetch(); + + int count = 0; + if (baseSearchCondition.isCountPageTotal()) { + count = getPageTaskInstanceCount(taskQuery); + } + return buildTaskInstancePageData(start, length, count, result); } @@ -281,23 +252,19 @@ private PageData listPageTaskInstanceByIp(TaskInstanceQuery tas } int start = baseSearchCondition.getStartOrDefault(0); int length = baseSearchCondition.getLengthOrDefault(10); - int count = ctx.selectCount().from(TaskInstance.TASK_INSTANCE) - .leftJoin(TASK_INSTANCE_HOST).on(TaskInstance.TASK_INSTANCE.ID.eq(TASK_INSTANCE_HOST.TASK_INSTANCE_ID)) - .where(conditions) - .fetchOne(0, Integer.class); - if (count == 0) { - return PageData.emptyPageData(start, length); + Integer count = 0; + if (baseSearchCondition.isCountPageTotal()) { + count = dsl().selectCount().from(TaskInstance.TASK_INSTANCE) + .leftJoin(TASK_INSTANCE_HOST).on(TaskInstance.TASK_INSTANCE.ID.eq(TASK_INSTANCE_HOST.TASK_INSTANCE_ID)) + .where(conditions) + .fetchOne(0, Integer.class); + if (count == null || count == 0) { + return PageData.emptyPageData(start, length); + } } Collection> orderFields = new ArrayList<>(); orderFields.add(TASK_INSTANCE.ID.desc()); - Result result = ctx.select(TASK_INSTANCE.ID, TASK_INSTANCE.TASK_ID, TASK_INSTANCE.CRON_TASK_ID, - TASK_INSTANCE.TASK_TEMPLATE_ID, - TASK_INSTANCE.IS_DEBUG_TASK, TASK_INSTANCE.APP_ID, TASK_INSTANCE.NAME, TASK_INSTANCE.OPERATOR, - TASK_INSTANCE.STARTUP_MODE, TASK_INSTANCE.CURRENT_STEP_ID, - TASK_INSTANCE.STATUS, - TASK_INSTANCE.START_TIME, TASK_INSTANCE.END_TIME, TASK_INSTANCE.TOTAL_TIME, TASK_INSTANCE.CREATE_TIME, - TASK_INSTANCE.CALLBACK_URL, TASK_INSTANCE.TYPE, - TASK_INSTANCE.APP_CODE) + Result result = dsl().select(ALL_FIELDS) .from(TaskInstanceDAOImpl.TASK_INSTANCE) .leftJoin(TASK_INSTANCE_HOST).on(TaskInstance.TASK_INSTANCE.ID.eq(TASK_INSTANCE_HOST.TASK_INSTANCE_ID)) .where(conditions) @@ -305,13 +272,17 @@ private PageData listPageTaskInstanceByIp(TaskInstanceQuery tas .orderBy(orderFields) .limit(start, length) .fetch(); + return buildTaskInstancePageData(start, length, count, result); } - private PageData buildTaskInstancePageData(int start, int length, int count, Result result) { + private PageData buildTaskInstancePageData(int start, + int length, + int count, + Result result) { List taskInstances = new ArrayList<>(); - if (result != null && result.size() > 0) { - result.into(record -> taskInstances.add(extractInfo(record))); + if (result != null && !result.isEmpty()) { + result.forEach(record -> taskInstances.add(extractInfo(record))); } PageData pageData = new PageData<>(); pageData.setData(taskInstances); @@ -324,7 +295,8 @@ private PageData buildTaskInstancePageData(int start, int lengt @SuppressWarnings("all") private int getPageTaskInstanceCount(TaskInstanceQuery taskQuery) { List conditions = buildSearchCondition(taskQuery); - return ctx.selectCount().from(TASK_INSTANCE).where(conditions).fetchOne(0, Integer.class); + return dsl().selectCount().from(TASK_INSTANCE).where(conditions).fetchOne(0, + Integer.class); } private List buildSearchCondition(TaskInstanceQuery taskQuery) { @@ -344,8 +316,9 @@ private List buildSearchCondition(TaskInstanceQuery taskQuery) { conditions.add(TASK_INSTANCE.STATUS.eq(JooqDataTypeUtil.toByte(taskQuery.getStatus().getValue()))); } if (CollectionUtils.isNotEmpty(taskQuery.getStartupModes())) { - if (taskQuery.getStartupModes().size() == 0) { - conditions.add(TASK_INSTANCE.STARTUP_MODE.eq(JooqDataTypeUtil.toByte(taskQuery.getStartupModes().get(0).getValue()))); + if (taskQuery.getStartupModes().size() == 1) { + conditions.add(TASK_INSTANCE.STARTUP_MODE.eq( + JooqDataTypeUtil.toByte(taskQuery.getStartupModes().get(0).getValue()))); } else { conditions.add(TASK_INSTANCE.STARTUP_MODE.in(taskQuery.getStartupModeValues())); } @@ -372,45 +345,38 @@ private List buildSearchCondition(TaskInstanceQuery taskQuery) { } @Override - public void addCallbackUrl(long taskInstanceId, String callBackUrl) { - ctx.update(TASK_INSTANCE).set(TASK_INSTANCE.CALLBACK_URL, callBackUrl) - .where(TASK_INSTANCE.ID.eq(taskInstanceId)) - .execute(); - } - - @Override - public List listLatestCronTaskInstance(long appId, Long cronTaskId, Long latestTimeInSeconds, - RunStatusEnum status, Integer limit) { - TaskInstance TABLE = TaskInstance.TASK_INSTANCE; + @MySQLOperation(table = "task_instance", op = DbOperationEnum.READ) + public List listLatestCronTaskInstance(long appId, + Long cronTaskId, + Long latestTimeInSeconds, + RunStatusEnum status, + Integer limit) { List conditions = new ArrayList<>(); - conditions.add(TABLE.APP_ID.eq(appId)); - conditions.add(TABLE.CRON_TASK_ID.eq(cronTaskId)); + conditions.add(TASK_INSTANCE.APP_ID.eq(appId)); + conditions.add(TASK_INSTANCE.CRON_TASK_ID.eq(cronTaskId)); if (latestTimeInSeconds != null) { long fromTimeInMillSecond = Instant.now().minusMillis(1000 * latestTimeInSeconds).toEpochMilli(); - conditions.add(TABLE.CREATE_TIME.ge(fromTimeInMillSecond)); + conditions.add(TASK_INSTANCE.CREATE_TIME.ge(fromTimeInMillSecond)); } if (status != null) { - conditions.add(TABLE.STATUS.eq(status.getValue().byteValue())); + conditions.add(TASK_INSTANCE.STATUS.eq(status.getValue().byteValue())); } - SelectSeekStep1 select = ctx.select(TABLE.ID, TABLE.TASK_ID, TABLE.CRON_TASK_ID, TABLE.TASK_TEMPLATE_ID, - TABLE.IS_DEBUG_TASK, TABLE.APP_ID, TABLE.NAME, TABLE.OPERATOR, TABLE.STARTUP_MODE, TABLE.CURRENT_STEP_ID, - TABLE.STATUS, TABLE.START_TIME, TABLE.END_TIME, TABLE.TOTAL_TIME, TABLE.CREATE_TIME, - TABLE.CALLBACK_URL, TABLE.TYPE, TABLE.APP_CODE) - .from(TABLE) + SelectSeekStep1 select = dsl().select(ALL_FIELDS) + .from(TASK_INSTANCE) .where(conditions) - .orderBy(TABLE.CREATE_TIME.desc()); + .orderBy(TASK_INSTANCE.CREATE_TIME.desc()); if (log.isDebugEnabled()) { - log.debug("SQL=", select.getSQL(ParamType.INLINED)); + log.debug("SQL={}", select.getSQL(ParamType.INLINED)); } - Result result; + Result result; if (limit != null && limit > 0) { result = select.limit(0, limit.intValue()).fetch(); } else { result = select.fetch(); } List taskInstances = new ArrayList<>(); - result.into(record -> { + result.forEach(record -> { TaskInstanceDTO taskInstance = extractInfo(record); if (taskInstance != null) { taskInstances.add(taskInstance); @@ -420,37 +386,47 @@ public List listLatestCronTaskInstance(long appId, Long cronTas } @Override - public void updateTaskExecutionInfo(long taskInstanceId, RunStatusEnum status, Long currentStepId, - Long startTime, Long endTime, Long totalTime) { + @MySQLOperation(table = "task_instance", op = DbOperationEnum.WRITE) + public void updateTaskExecutionInfo(long taskInstanceId, + RunStatusEnum status, + Long currentStepId, + Long startTime, + Long endTime, + Long totalTime) { UpdateSetMoreStep updateSetMoreStep = null; if (status != null) { - updateSetMoreStep = ctx.update(TASK_INSTANCE).set(TASK_INSTANCE.STATUS, + updateSetMoreStep = dsl().update(TASK_INSTANCE).set(TASK_INSTANCE.STATUS, JooqDataTypeUtil.toByte(status.getValue())); } if (currentStepId != null) { if (updateSetMoreStep == null) { - updateSetMoreStep = ctx.update(TASK_INSTANCE).set(TASK_INSTANCE.CURRENT_STEP_ID, currentStepId); + updateSetMoreStep = + dsl().update(TASK_INSTANCE).set(TASK_INSTANCE.CURRENT_STEP_ID, + currentStepId); } else { updateSetMoreStep.set(TASK_INSTANCE.CURRENT_STEP_ID, currentStepId); } } if (startTime != null) { if (updateSetMoreStep == null) { - updateSetMoreStep = ctx.update(TASK_INSTANCE).set(TASK_INSTANCE.START_TIME, startTime); + updateSetMoreStep = dsl().update(TASK_INSTANCE).set(TASK_INSTANCE.START_TIME, + startTime); } else { updateSetMoreStep.set(TASK_INSTANCE.START_TIME, startTime); } } if (endTime != null) { if (updateSetMoreStep == null) { - updateSetMoreStep = ctx.update(TASK_INSTANCE).set(TASK_INSTANCE.END_TIME, endTime); + updateSetMoreStep = dsl().update(TASK_INSTANCE).set(TASK_INSTANCE.END_TIME, + endTime); } else { updateSetMoreStep.set(TASK_INSTANCE.END_TIME, endTime); } } if (totalTime != null) { if (updateSetMoreStep == null) { - updateSetMoreStep = ctx.update(TASK_INSTANCE).set(TASK_INSTANCE.TOTAL_TIME, totalTime); + updateSetMoreStep = dsl().update(TASK_INSTANCE).set(TASK_INSTANCE.TOTAL_TIME, + totalTime); } else { updateSetMoreStep.set(TASK_INSTANCE.TOTAL_TIME, totalTime); } @@ -462,8 +438,9 @@ public void updateTaskExecutionInfo(long taskInstanceId, RunStatusEnum status, L } @Override + @MySQLOperation(table = "task_instance", op = DbOperationEnum.WRITE) public void resetTaskExecuteInfoForRetry(long taskInstanceId) { - ctx.update(TASK_INSTANCE) + dsl().update(TASK_INSTANCE) .setNull(TASK_INSTANCE.END_TIME) .setNull(TASK_INSTANCE.TOTAL_TIME) .set(TASK_INSTANCE.STATUS, RunStatusEnum.RUNNING.getValue().byteValue()) @@ -471,44 +448,8 @@ public void resetTaskExecuteInfoForRetry(long taskInstanceId) { .execute(); } - public Integer countTaskInstanceByConditions(Collection conditions) { - return ctx.selectCount().from(TASK_INSTANCE) - .where(conditions).fetchOne().value1(); - } - - @Override - public Integer countTaskInstances(Long appId, Long minTotalTime, Long maxTotalTime, - TaskStartupModeEnum taskStartupMode, TaskTypeEnum taskType, - List runStatusList, Long fromTime, Long toTime) { - List conditions = new ArrayList<>(); - if (appId != null) { - conditions.add(TASK_INSTANCE.APP_ID.eq(appId)); - } - if (taskStartupMode != null) { - conditions.add(TASK_INSTANCE.STARTUP_MODE.eq((byte) (taskStartupMode.getValue()))); - } - if (taskType != null) { - conditions.add(TASK_INSTANCE.TYPE.eq(taskType.getValue().byteValue())); - } - if (runStatusList != null) { - conditions.add(TASK_INSTANCE.STATUS.in(runStatusList)); - } - if (minTotalTime != null) { - conditions.add(TASK_INSTANCE.TOTAL_TIME.greaterOrEqual(minTotalTime * 1000)); - } - if (maxTotalTime != null) { - conditions.add(TASK_INSTANCE.TOTAL_TIME.lessOrEqual(maxTotalTime * 1000)); - } - if (fromTime != null) { - conditions.add(TASK_INSTANCE.CREATE_TIME.greaterOrEqual(fromTime)); - } - if (toTime != null) { - conditions.add(TASK_INSTANCE.CREATE_TIME.lessThan(toTime)); - } - return countTaskInstanceByConditions(conditions); - } - @Override + @MySQLOperation(table = "task_instance", op = DbOperationEnum.READ) public List listTaskInstanceAppId(List inAppIdList, Long cronTaskId, Long minCreateTime) { List conditions = new ArrayList<>(); if (inAppIdList != null) { @@ -520,11 +461,12 @@ public List listTaskInstanceAppId(List inAppIdList, Long cronTaskId, if (minCreateTime != null) { conditions.add(TASK_INSTANCE.CREATE_TIME.greaterOrEqual(minCreateTime)); } - Result result = ctx.selectDistinct(TASK_INSTANCE.APP_ID).from(TASK_INSTANCE) - .where(conditions) - .fetch(); + Result result = + dsl().selectDistinct(TASK_INSTANCE.APP_ID).from(TASK_INSTANCE) + .where(conditions) + .fetch(); List appIdList = new ArrayList<>(); - result.into(record -> { + result.forEach(record -> { Long appId = record.getValue(TASK_INSTANCE.APP_ID); if (appId != null) { appIdList.add(appId); @@ -534,6 +476,7 @@ public List listTaskInstanceAppId(List inAppIdList, Long cronTaskId, } @Override + @MySQLOperation(table = "task_instance", op = DbOperationEnum.READ) public boolean hasExecuteHistory(Long appId, Long cronTaskId, Long fromTime, Long toTime) { List conditions = new ArrayList<>(); if (appId != null) { @@ -548,14 +491,15 @@ public boolean hasExecuteHistory(Long appId, Long cronTaskId, Long fromTime, Lon if (toTime != null) { conditions.add(TASK_INSTANCE.CREATE_TIME.lessOrEqual(toTime)); } - Result> result = ctx.select(TASK_INSTANCE.APP_ID).from(TASK_INSTANCE) + Result> result = dsl().select(TASK_INSTANCE.APP_ID).from(TASK_INSTANCE) .where(conditions) .limit(1) .fetch(); - return result.size() > 0; + return !result.isEmpty(); } @Override + @MySQLOperation(table = "task_instance", op = DbOperationEnum.READ) public List listTaskInstanceId(Long appId, Long fromTime, Long toTime, int offset, int limit) { List conditions = new ArrayList<>(); if (appId != null) { @@ -567,7 +511,7 @@ public List listTaskInstanceId(Long appId, Long fromTime, Long toTime, int if (toTime != null) { conditions.add(TASK_INSTANCE.CREATE_TIME.lessThan(toTime)); } - Result> result = ctx.select(TASK_INSTANCE.ID).from(TASK_INSTANCE) + Result> result = dsl().select(TASK_INSTANCE.ID).from(TASK_INSTANCE) .where(conditions) .limit(offset, limit) .fetch(); @@ -579,22 +523,33 @@ public List listTaskInstanceId(Long appId, Long fromTime, Long toTime, int } @Override - public void saveTaskInstanceHosts(long taskInstanceId, + @MySQLOperation(table = "task_instance_host", op = DbOperationEnum.WRITE) + public void saveTaskInstanceHosts(long appId, + long taskInstanceId, Collection hosts) { - BatchBindStep batchInsert = ctx.batch( - ctx.insertInto(TASK_INSTANCE_HOST, TASK_INSTANCE_HOST.TASK_INSTANCE_ID, - TASK_INSTANCE_HOST.HOST_ID, TASK_INSTANCE_HOST.IP, TASK_INSTANCE_HOST.IPV6) - .values((Long) null, null, null, null) - ); - - for (HostDTO host : hosts) { - batchInsert = batchInsert.bind( - taskInstanceId, - host.getHostId(), - host.getIp(), - host.getIpv6() + BatchUtil.executeBatch(hosts, 2000, batchHosts -> { + BatchBindStep batchInsert = dsl().batch( + dsl().insertInto( + TASK_INSTANCE_HOST, + TASK_INSTANCE_HOST.TASK_INSTANCE_ID, + TASK_INSTANCE_HOST.HOST_ID, + TASK_INSTANCE_HOST.IP, + TASK_INSTANCE_HOST.IPV6, + TASK_INSTANCE_HOST.APP_ID + ) + .values((Long) null, null, null, null, null) ); - } - batchInsert.execute(); + + for (HostDTO host : batchHosts) { + batchInsert = batchInsert.bind( + taskInstanceId, + host.getHostId(), + host.getIp(), + host.getIpv6(), + appId + ); + } + batchInsert.execute(); + }); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceIdDynamicCondition.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceIdDynamicCondition.java new file mode 100644 index 0000000000..01aa1821c2 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceIdDynamicCondition.java @@ -0,0 +1,95 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.dao.impl; + +import com.tencent.bk.job.common.model.dto.ResourceScope; +import com.tencent.bk.job.common.util.toggle.ToggleEvaluateContext; +import com.tencent.bk.job.common.util.toggle.ToggleStrategyContextParams; +import com.tencent.bk.job.common.util.toggle.feature.FeatureIdConstants; +import com.tencent.bk.job.common.util.toggle.feature.FeatureToggle; +import com.tencent.bk.job.execute.common.context.JobExecuteContext; +import com.tencent.bk.job.execute.common.context.JobExecuteContextThreadLocalRepo; +import lombok.extern.slf4j.Slf4j; +import org.jooq.Condition; +import org.jooq.impl.DSL; + +import java.util.function.Function; + +/** + * task_instance_id DAO 层查询条件动态构造。 + * 背景：由于分库分表改造，原有的一些表(比如step_instance_script)中需要加入 task_instance_id 字段作为分片键。从数据的唯一性来说， + * 查询的时候传入 task_instance_id 条件是多余的，涉及的这些表的唯一业务主键都是(step_instance_id+execute_count+其它字段)。 + *

+ * 考虑到 db 变更过程中表中 task_instance_id 可能为 0 的情况（历史数据）， + * 所以，需要通过业务 ID 、task_instance_id 的值动态去构造查询条件，兼容历史数据的查询 + */ +@Slf4j +public class TaskInstanceIdDynamicCondition { + + public static Condition build(Long taskInstanceId, + Function taskInstanceIdConditionBuilder) { + ToggleEvaluateContext toggleEvaluateContext; + JobExecuteContext jobExecuteContext = JobExecuteContextThreadLocalRepo.get(); + if (jobExecuteContext == null) { + // JobExecuteContext 正常应该不会为 null 。为了不影响请求正常处理，忽略错误,直接返回 TRUE Condition + // (不会影响 DAO 查询，task_instance_id 仅作为分片功能实用，实际业务数据关系并不强依赖 task_instance_id) + toggleEvaluateContext = ToggleEvaluateContext.EMPTY; + } else { + ResourceScope resourceScope = jobExecuteContext.getResourceScope(); + if (resourceScope != null) { + toggleEvaluateContext = ToggleEvaluateContext.builder() + .addContextParam(ToggleStrategyContextParams.CTX_PARAM_RESOURCE_SCOPE, resourceScope); + } else { + toggleEvaluateContext = ToggleEvaluateContext.EMPTY; + } + } + + if (FeatureToggle.checkFeature(FeatureIdConstants.DAO_ADD_TASK_INSTANCE_ID, toggleEvaluateContext)) { + if (taskInstanceId == null || taskInstanceId <= 0L) { + log.info("TaskInstanceIdDynamicCondition : InvalidTaskInstanceId : {}", taskInstanceId); + // 输出堆栈信息，便于发现问题；后续功能稳定之后需要删除 + safePrintStackTrace(); + // 为了不影响兼容性，忽略错误 + return DSL.trueCondition(); + } else { + return taskInstanceIdConditionBuilder.apply(taskInstanceId); + } + } else { + return DSL.trueCondition(); + } + } + + private static void safePrintStackTrace() { + try { + StringBuilder message = new StringBuilder(); + for (StackTraceElement stackTraceElement : Thread.currentThread().getStackTrace()) { + message.append(System.lineSeparator()).append(stackTraceElement.toString()); + } + log.info("InvalidTaskInstanceIdConditionStackTrace: {}", message); + } catch (Throwable e) { + // ignore + } + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceVariableDAOImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceVariableDAOImpl.java index a9651f0ef7..bd3140eb12 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceVariableDAOImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/dao/impl/TaskInstanceVariableDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,17 +24,21 @@ package com.tencent.bk.job.execute.dao.impl; -import com.tencent.bk.job.execute.common.util.JooqDataTypeUtil; +import com.tencent.bk.job.common.constant.TaskVariableTypeEnum; +import com.tencent.bk.job.common.crypto.scenario.CipherVariableCryptoService; +import com.tencent.bk.job.common.mysql.dynamic.ds.DbOperationEnum; +import com.tencent.bk.job.common.mysql.dynamic.ds.MySQLOperation; +import com.tencent.bk.job.common.mysql.jooq.JooqDataTypeUtil; import com.tencent.bk.job.execute.dao.TaskInstanceVariableDAO; +import com.tencent.bk.job.execute.dao.common.DSLContextProviderFactory; import com.tencent.bk.job.execute.engine.model.TaskVariableDTO; -import org.jooq.DSLContext; -import org.jooq.InsertValuesStep5; +import com.tencent.bk.job.execute.model.tables.TaskInstanceVariable; +import com.tencent.bk.job.execute.model.tables.records.TaskInstanceVariableRecord; +import org.jooq.InsertValuesStep6; import org.jooq.Record; +import org.jooq.Record6; import org.jooq.Result; -import org.jooq.generated.tables.TaskInstanceVariable; -import org.jooq.generated.tables.records.TaskInstanceVariableRecord; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; import java.util.ArrayList; @@ -44,24 +48,33 @@ * 作业实例全局变量DAO */ @Repository -public class TaskInstanceVariableDAOImpl implements TaskInstanceVariableDAO { +public class TaskInstanceVariableDAOImpl extends BaseDAO implements TaskInstanceVariableDAO { private static final TaskInstanceVariable TABLE = TaskInstanceVariable.TASK_INSTANCE_VARIABLE; - private DSLContext ctx; + + private final CipherVariableCryptoService cipherVariableCryptoService; @Autowired - public TaskInstanceVariableDAOImpl(@Qualifier("job-execute-dsl-context") DSLContext ctx) { - this.ctx = ctx; + public TaskInstanceVariableDAOImpl(DSLContextProviderFactory dslContextProviderFactory, + CipherVariableCryptoService cipherVariableCryptoService) { + super(dslContextProviderFactory, TABLE.getName()); + this.cipherVariableCryptoService = cipherVariableCryptoService; } @Override + @MySQLOperation(table = "task_instance_variable", op = DbOperationEnum.READ) public List getByTaskInstanceId(long taskInstanceId) { - Result result = ctx.select(TABLE.ID, TABLE.TASK_INSTANCE_ID, TABLE.NAME, TABLE.TYPE, TABLE.IS_CHANGEABLE, - TABLE.VALUE) - .from(TABLE) + Result> result = dsl().select( + TABLE.ID, + TABLE.TASK_INSTANCE_ID, + TABLE.NAME, + TABLE.TYPE, + TABLE.IS_CHANGEABLE, + TABLE.VALUE + ).from(TABLE) .where(TABLE.TASK_INSTANCE_ID.eq(taskInstanceId)) .fetch(); List taskVariables = new ArrayList<>(); - if (result.size() > 0) { + if (!result.isEmpty()) { result.into(record -> taskVariables.add(extract((record)))); } return taskVariables; @@ -73,30 +86,51 @@ private TaskVariableDTO extract(Record record) { taskVariable.setTaskInstanceId(record.get(TABLE.TASK_INSTANCE_ID)); taskVariable.setName(record.get(TABLE.NAME)); taskVariable.setType(JooqDataTypeUtil.toInteger(record.get(TABLE.TYPE))); - taskVariable.setValue(record.get(TABLE.VALUE)); + TaskVariableTypeEnum taskVarType = TaskVariableTypeEnum.valOf(taskVariable.getType()); + String encryptedValue = record.get(TABLE.VALUE); + String value = cipherVariableCryptoService.decryptTaskVariableIfNeeded(taskVarType, encryptedValue); + taskVariable.setValue(value); taskVariable.setChangeable(JooqDataTypeUtil.toInteger(record.get(TABLE.IS_CHANGEABLE)).equals(1)); return taskVariable; } @Override + @MySQLOperation(table = "task_instance_variable", op = DbOperationEnum.WRITE) public void batchDeleteByTaskInstanceIds(List taskInstanceIdList) { } @Override + @MySQLOperation(table = "task_instance_variable", op = DbOperationEnum.WRITE) public void deleteByTaskInstanceId(long taskInstanceId) { } @Override + @MySQLOperation(table = "task_instance_variable", op = DbOperationEnum.WRITE) public void saveTaskInstanceVariables(List taskVarList) { - InsertValuesStep5 insertStep = - ctx.insertInto(TABLE).columns(TABLE.TASK_INSTANCE_ID, TABLE.NAME, TABLE.TYPE, TABLE.VALUE, - TABLE.IS_CHANGEABLE); + InsertValuesStep6 insertStep = + dsl().insertInto(TABLE) + .columns( + TABLE.ID, + TABLE.TASK_INSTANCE_ID, + TABLE.NAME, + TABLE.TYPE, + TABLE.VALUE, + TABLE.IS_CHANGEABLE + ); - taskVarList.forEach(taskVar -> insertStep.values(taskVar.getTaskInstanceId(), - taskVar.getName(), JooqDataTypeUtil.toByte(taskVar.getType()), - taskVar.getValue(), JooqDataTypeUtil.toByte(taskVar.isChangeable() ? 1 : 0))); + taskVarList.forEach(taskVar -> { + TaskVariableTypeEnum taskVarType = TaskVariableTypeEnum.valOf(taskVar.getType()); + insertStep.values( + taskVar.getId(), + taskVar.getTaskInstanceId(), + taskVar.getName(), + JooqDataTypeUtil.toByte(taskVar.getType()), + cipherVariableCryptoService.encryptTaskVariableIfNeeded(taskVarType, taskVar.getValue()), + JooqDataTypeUtil.toByte(taskVar.isChangeable() ? 1 : 0) + ); + }); insertStep.execute(); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/EngineDependentServiceHolder.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/EngineDependentServiceHolder.java new file mode 100644 index 0000000000..3b615de791 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/EngineDependentServiceHolder.java @@ -0,0 +1,110 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted; free of charge; to any person obtaining a copy of this software and associated + * documentation files (the "Software"); to deal in the Software without restriction; including without limitation + * the rights to use; copy; modify; merge; publish; distribute; sublicense; and/or sell copies of the Software; and + * to permit persons to whom the Software is furnished to do so; subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS"; WITHOUT WARRANTY OF ANY KIND; EXPRESS OR IMPLIED; INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY; FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM; DAMAGES OR OTHER LIABILITY; WHETHER IN AN ACTION OF + * CONTRACT; TORT OR OTHERWISE; ARISING FROM; OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine; + +import com.tencent.bk.job.common.gse.GseClient; +import com.tencent.bk.job.execute.engine.evict.TaskEvictPolicyExecutor; +import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; +import com.tencent.bk.job.execute.engine.quota.limit.RunningJobKeepaliveManager; +import com.tencent.bk.job.execute.engine.result.ResultHandleManager; +import com.tencent.bk.job.execute.engine.result.ha.ResultHandleTaskKeepaliveManager; +import com.tencent.bk.job.execute.engine.variable.JobBuildInVariableResolver; +import com.tencent.bk.job.execute.monitor.metrics.ExecuteMonitor; +import com.tencent.bk.job.execute.monitor.metrics.GseTasksExceptionCounter; +import com.tencent.bk.job.execute.service.AccountService; +import com.tencent.bk.job.execute.service.AgentService; +import com.tencent.bk.job.execute.service.GseTaskService; +import com.tencent.bk.job.execute.service.LogService; +import com.tencent.bk.job.execute.service.StepInstanceService; +import com.tencent.bk.job.execute.service.StepInstanceVariableValueService; +import com.tencent.bk.job.execute.service.TaskInstanceService; +import com.tencent.bk.job.execute.service.TaskInstanceVariableService; +import lombok.Getter; +import org.springframework.cloud.sleuth.Tracer; +import org.springframework.stereotype.Component; + +/** + * 执行引擎依赖的服务的托管类。用于简化一些较为复杂的类构造函数的参数列表 + */ +@Component +@Getter +public class EngineDependentServiceHolder { + private final ResultHandleManager resultHandleManager; + private final TaskInstanceService taskInstanceService; + private final GseTaskService gseTaskService; + private final AccountService accountService; + private final TaskInstanceVariableService taskInstanceVariableService; + private final StepInstanceVariableValueService stepInstanceVariableValueService; + private final AgentService agentService; + private final LogService logService; + private final TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher; + private final ResultHandleTaskKeepaliveManager resultHandleTaskKeepaliveManager; + private final ExecuteMonitor executeMonitor; + private final TaskEvictPolicyExecutor taskEvictPolicyExecutor; + private final GseTasksExceptionCounter gseTasksExceptionCounter; + private final StepInstanceService stepInstanceService; + private final Tracer tracer; + private final GseClient gseClient; + private final RunningJobKeepaliveManager runningJobKeepaliveManager; + private final JobBuildInVariableResolver jobBuildInVariableResolver; + + public EngineDependentServiceHolder(ResultHandleManager resultHandleManager, + TaskInstanceService taskInstanceService, + GseTaskService gseTaskService, + AccountService accountService, + TaskInstanceVariableService taskInstanceVariableService, + StepInstanceVariableValueService stepInstanceVariableValueService, + AgentService agentService, + LogService logService, + TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher, + ResultHandleTaskKeepaliveManager resultHandleTaskKeepaliveManager, + ExecuteMonitor executeMonitor, + TaskEvictPolicyExecutor taskEvictPolicyExecutor, + GseTasksExceptionCounter gseTasksExceptionCounter, + StepInstanceService stepInstanceService, + Tracer tracer, + GseClient gseClient, + RunningJobKeepaliveManager runningJobKeepaliveManager, + JobBuildInVariableResolver jobBuildInVariableResolver) { + this.resultHandleManager = resultHandleManager; + this.taskInstanceService = taskInstanceService; + this.gseTaskService = gseTaskService; + this.accountService = accountService; + this.taskInstanceVariableService = taskInstanceVariableService; + this.stepInstanceVariableValueService = stepInstanceVariableValueService; + this.agentService = agentService; + this.logService = logService; + this.taskExecuteMQEventDispatcher = taskExecuteMQEventDispatcher; + this.resultHandleTaskKeepaliveManager = resultHandleTaskKeepaliveManager; + this.executeMonitor = executeMonitor; + this.taskEvictPolicyExecutor = taskEvictPolicyExecutor; + this.gseTasksExceptionCounter = gseTasksExceptionCounter; + this.stepInstanceService = stepInstanceService; + this.tracer = tracer; + this.gseClient = gseClient; + this.runningJobKeepaliveManager = runningJobKeepaliveManager; + this.jobBuildInVariableResolver = jobBuildInVariableResolver; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/AgentTaskStatusEnum.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/AgentTaskStatusEnum.java deleted file mode 100644 index d0e112cb21..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/AgentTaskStatusEnum.java +++ /dev/null @@ -1,204 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.engine.consts; - -/** - * Agent 任务状态 - */ -public enum AgentTaskStatusEnum { - /** - * 未知错误 - */ - UNKNOWN_ERROR(0), - /** - * Agent异常 - */ - AGENT_ERROR(1), - /** - * 无效主机 - */ - HOST_NOT_EXIST(2), - /** - * 上次已成功 - */ - LAST_SUCCESS(3), - /** - * 等待执行 - */ - WAITING(5), - /** - * 正在执行 - */ - RUNNING(7), - /** - * 执行成功 - */ - SUCCESS(9), - /** - * 执行失败 - */ - FAILED(11), - /** - * 任务下发失败 - */ - SUBMIT_FAILED(12), - /** - * 任务超时 - */ - TASK_TIMEOUT(13), - /** - * 任务日志错误 - */ - LOG_ERROR(15), - /** - * GSE脚本日志超时 - */ - GSE_SCRIPT_TIMEOUT(16), - /** - * GSE文件日志超时 - */ - GSE_FILE_TIMEOUT(17), - /** - * 脚本执行失败 - */ - SCRIPT_FAILED(101), - /** - * 脚本执行超时 - */ - SCRIPT_TIMEOUT(102), - /** - * 脚本执行被终止 - */ - SCRIPT_TERMINATE(103), - /** - * 脚本返回码非零 - */ - SCRIPT_NOT_ZERO_EXIT_CODE(104), - /** - * 文件传输失败 - */ - COPYFILE_FAILED(202), - /** - * 源文件不存在 - */ - COPYFILE_SOURCE_FILE_NOT_EXIST(203), - /** - * 文件任务系统错误-未分类的 - */ - FILE_ERROR_UNCLASSIFIED(301), - /** - * 文件任务超时 - */ - GSE_TIMEOUT(303), - /** - * Agent异常 - */ - GSE_AGENT_ERROR(310), - /** - * 用户名不存在 - */ - GSE_USER_ERROR(311), - /** - * 用户密码错误 - */ - GSE_USER_PWD_ERROR(312), - /** - * 文件获取失败 - */ - GSE_FILE_ERROR(320), - /** - * 文件超出限制 - */ - GSE_FILE_SIZE_EXCEED(321), - /** - * 文件传输错误 - */ - GSE_FILE_TASK_ERROR(329), - /** - * 任务执行出错 - */ - GSE_TASK_ERROR(399), - /** - * GSE 任务强制终止成功 - */ - GSE_TASK_TERMINATE_SUCCESS(403), - /** - * GSE 任务强制终止失败 - */ - GSE_TASK_TERMINATE_FAILED(404), - /** - * 未知状态 - */ - UNKNOWN(500); - - - private final int status; - - AgentTaskStatusEnum(int status) { - this.status = status; - } - - public static AgentTaskStatusEnum valueOf(Integer status) { - if (status == null) { - return null; - } - for (AgentTaskStatusEnum agentTaskStatus : values()) { - if (agentTaskStatus.status == status) { - return agentTaskStatus; - } - } - return UNKNOWN_ERROR; - } - - public final int getValue() { - return status; - } - - public String getI18nKey() { - return "agent.task.status." + this.name().toLowerCase(); - } - - public static boolean isSuccess(AgentTaskStatusEnum status) { - if (status == null) { - return false; - } - return status == SUCCESS || status == LAST_SUCCESS; - } - - public static boolean isSuccess(Integer status) { - if (status == null) { - return false; - } - return status.equals(SUCCESS.getValue()) || status.equals(LAST_SUCCESS.getValue()); - } - - /** - * 是否完成状态 - */ - public boolean isFinished() { - return this != AgentTaskStatusEnum.RUNNING && this != AgentTaskStatusEnum.WAITING; - } - -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/Consts.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/Consts.java index f00227b021..ea6f888342 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/Consts.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/Consts.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,28 +28,28 @@ import java.util.HashMap; import java.util.Map; -import static com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum.GSE_AGENT_ERROR; -import static com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum.GSE_FILE_ERROR; -import static com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum.GSE_FILE_SIZE_EXCEED; -import static com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum.GSE_FILE_TASK_ERROR; -import static com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum.GSE_TASK_ERROR; -import static com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum.GSE_TASK_TERMINATE_SUCCESS; -import static com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum.GSE_TIMEOUT; -import static com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum.GSE_USER_ERROR; -import static com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum.GSE_USER_PWD_ERROR; +import static com.tencent.bk.job.execute.engine.consts.ExecuteObjectTaskStatusEnum.GSE_AGENT_ERROR; +import static com.tencent.bk.job.execute.engine.consts.ExecuteObjectTaskStatusEnum.GSE_FILE_ERROR; +import static com.tencent.bk.job.execute.engine.consts.ExecuteObjectTaskStatusEnum.GSE_FILE_SIZE_EXCEED; +import static com.tencent.bk.job.execute.engine.consts.ExecuteObjectTaskStatusEnum.GSE_FILE_TASK_ERROR; +import static com.tencent.bk.job.execute.engine.consts.ExecuteObjectTaskStatusEnum.GSE_TASK_ERROR; +import static com.tencent.bk.job.execute.engine.consts.ExecuteObjectTaskStatusEnum.GSE_TASK_TERMINATE_SUCCESS; +import static com.tencent.bk.job.execute.engine.consts.ExecuteObjectTaskStatusEnum.GSE_TIMEOUT; +import static com.tencent.bk.job.execute.engine.consts.ExecuteObjectTaskStatusEnum.GSE_USER_ERROR; +import static com.tencent.bk.job.execute.engine.consts.ExecuteObjectTaskStatusEnum.GSE_USER_PWD_ERROR; public class Consts { /** - * GSE 错误码与 AgentTask 执行结果的映射关系 + * GSE 错误码与执行对象任务结果的映射关系 */ - public static final Map GSE_ERROR_CODE_2_STATUS_MAP; + public static final Map GSE_ERROR_CODE_2_STATUS_MAP; /** * 直连云区域ID */ public static final int DEFAULT_CLOUD_ID = 0; static { - Map gseErrorCode2StatusMap = new HashMap<>(); + Map gseErrorCode2StatusMap = new HashMap<>(); // 文件任务超时 gseErrorCode2StatusMap.put(120, GSE_TIMEOUT); // agent异常 diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/EventSourceTypeEnum.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/EventSourceTypeEnum.java index ec344d17e7..25cfefe747 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/EventSourceTypeEnum.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/EventSourceTypeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/ExecuteObjectTaskStatusEnum.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/ExecuteObjectTaskStatusEnum.java new file mode 100644 index 0000000000..76d63e3fc3 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/ExecuteObjectTaskStatusEnum.java @@ -0,0 +1,208 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.consts; + +/** + * 单个执行对象的任务状态，体现在任务详情页左侧状态栏 + */ +public enum ExecuteObjectTaskStatusEnum { + /** + * 未知错误 + */ + UNKNOWN_ERROR(0), + /** + * Agent异常 + */ + AGENT_ERROR(1), + /** + * 无效执行对象 + */ + INVALID_EXECUTE_OBJECT(2), + /** + * 上次已成功 + */ + LAST_SUCCESS(3), + /** + * 等待执行 + */ + WAITING(5), + /** + * 正在执行 + */ + RUNNING(7), + /** + * 执行成功 + */ + SUCCESS(9), + /** + * 执行失败 + */ + FAILED(11), + /** + * 任务下发失败 + */ + SUBMIT_FAILED(12), + /** + * 任务超时 + */ + TASK_TIMEOUT(13), + /** + * 任务日志错误 + */ + LOG_ERROR(15), + /** + * GSE脚本日志超时 + */ + GSE_SCRIPT_TIMEOUT(16), + /** + * GSE文件日志超时 + */ + GSE_FILE_TIMEOUT(17), + /** + * Agent未安装 + */ + AGENT_NOT_INSTALLED(18), + /** + * 脚本执行失败 + */ + SCRIPT_FAILED(101), + /** + * 脚本执行超时 + */ + SCRIPT_TIMEOUT(102), + /** + * 脚本执行被终止 + */ + SCRIPT_TERMINATE(103), + /** + * 脚本返回码非零 + */ + SCRIPT_NOT_ZERO_EXIT_CODE(104), + /** + * 文件传输失败 + */ + COPYFILE_FAILED(202), + /** + * 源文件不存在 + */ + COPYFILE_SOURCE_FILE_NOT_EXIST(203), + /** + * 文件任务系统错误-未分类的 + */ + FILE_ERROR_UNCLASSIFIED(301), + /** + * 文件任务超时 + */ + GSE_TIMEOUT(303), + /** + * Agent异常 + */ + GSE_AGENT_ERROR(310), + /** + * 用户名不存在 + */ + GSE_USER_ERROR(311), + /** + * 用户密码错误 + */ + GSE_USER_PWD_ERROR(312), + /** + * 文件获取失败 + */ + GSE_FILE_ERROR(320), + /** + * 文件超出限制 + */ + GSE_FILE_SIZE_EXCEED(321), + /** + * 文件传输错误 + */ + GSE_FILE_TASK_ERROR(329), + /** + * 任务执行出错 + */ + GSE_TASK_ERROR(399), + /** + * GSE 任务强制终止成功 + */ + GSE_TASK_TERMINATE_SUCCESS(403), + /** + * GSE 任务强制终止失败 + */ + GSE_TASK_TERMINATE_FAILED(404), + /** + * 未知状态 + */ + UNKNOWN(500); + + + private final int status; + + ExecuteObjectTaskStatusEnum(int status) { + this.status = status; + } + + public static ExecuteObjectTaskStatusEnum valOf(Integer status) { + if (status == null) { + return null; + } + for (ExecuteObjectTaskStatusEnum executeObjectTaskStatus : values()) { + if (executeObjectTaskStatus.status == status) { + return executeObjectTaskStatus; + } + } + return UNKNOWN_ERROR; + } + + public final int getValue() { + return status; + } + + public String getI18nKey() { + return "agent.task.status." + this.name().toLowerCase(); + } + + public static boolean isSuccess(ExecuteObjectTaskStatusEnum status) { + if (status == null) { + return false; + } + return status == SUCCESS || status == LAST_SUCCESS; + } + + public static boolean isSuccess(Integer status) { + if (status == null) { + return false; + } + return status.equals(SUCCESS.getValue()) || status.equals(LAST_SUCCESS.getValue()); + } + + /** + * 是否完成状态 + */ + public boolean isFinished() { + return this != ExecuteObjectTaskStatusEnum.RUNNING && this != ExecuteObjectTaskStatusEnum.WAITING; + } + +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/FileDirTypeConf.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/FileDirTypeConf.java index 79812eab06..276500fafa 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/FileDirTypeConf.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/FileDirTypeConf.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/GseTaskActionEnum.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/GseTaskActionEnum.java index 46373f7a2a..bb27bee866 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/GseTaskActionEnum.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/GseTaskActionEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/JobActionEnum.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/JobActionEnum.java index 73c1f2537f..e91a45d34f 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/JobActionEnum.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/JobActionEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/JobBuildInVariables.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/JobBuildInVariables.java index cbf6be0889..bd67cfe427 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/JobBuildInVariables.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/JobBuildInVariables.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/StepActionEnum.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/StepActionEnum.java index c58fdc9f59..347ad121e7 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/StepActionEnum.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/consts/StepActionEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -83,7 +83,11 @@ public enum StepActionEnum { /** * 刷新步骤状态 */ - REFRESH(14); + REFRESH(14), + /** + * 准备文件 + */ + PREPARE_FILE(15); private final int value; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/evict/AppCodeTaskEvictPolicy.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/evict/AppCodeTaskEvictPolicy.java index ae46e17b97..f8d62a7b3d 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/evict/AppCodeTaskEvictPolicy.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/evict/AppCodeTaskEvictPolicy.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/evict/AppIdTaskEvictPolicy.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/evict/AppIdTaskEvictPolicy.java index b9a18ece77..157aec44bd 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/evict/AppIdTaskEvictPolicy.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/evict/AppIdTaskEvictPolicy.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/evict/ComposedTaskEvictPolicy.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/evict/ComposedTaskEvictPolicy.java index 5a25566fa8..40f69bff54 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/evict/ComposedTaskEvictPolicy.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/evict/ComposedTaskEvictPolicy.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/evict/ITaskEvictPolicy.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/evict/ITaskEvictPolicy.java index 51c7555ced..1599b1ef7d 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/evict/ITaskEvictPolicy.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/evict/ITaskEvictPolicy.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/evict/PolicyFactory.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/evict/PolicyFactory.java index 8848470659..de52b8887e 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/evict/PolicyFactory.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/evict/PolicyFactory.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/evict/TaskEvictPolicyExecutor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/evict/TaskEvictPolicyExecutor.java index dab9865c0c..e64943a991 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/evict/TaskEvictPolicyExecutor.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/evict/TaskEvictPolicyExecutor.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,30 +24,23 @@ package com.tencent.bk.job.execute.engine.evict; -import com.tencent.bk.job.execute.common.constants.RunStatusEnum; -import com.tencent.bk.job.execute.common.util.TaskCostCalculator; -import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; -import com.tencent.bk.job.execute.service.TaskInstanceService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; /** - * 任务驱逐策略执行器，用于判定某个任务在当前驱逐策略下是否应当被驱逐，提供任务被驱逐后更新任务相关状态的方法 + * 任务驱逐策略执行器，用于判定某个任务在当前驱逐策略下是否应当被驱逐 */ @Slf4j @Component public class TaskEvictPolicyExecutor { private final TaskEvictPolicyManager taskEvictPolicyManager; - private final TaskInstanceService taskInstanceService; @Autowired - public TaskEvictPolicyExecutor(TaskEvictPolicyManager taskEvictPolicyManager, - TaskInstanceService taskInstanceService) { + public TaskEvictPolicyExecutor(TaskEvictPolicyManager taskEvictPolicyManager) { this.taskEvictPolicyManager = taskEvictPolicyManager; - this.taskInstanceService = taskInstanceService; } /** @@ -63,58 +56,4 @@ public boolean shouldEvictTask(TaskInstanceDTO taskInstance) { } return policy.needToEvict(taskInstance); } - - /** - * 更新被驱逐的任务相关状态为被丢弃状态 - * - * @param taskInstance 任务实例 - * @param stepInstance 步骤实例 - */ - public void updateEvictedTaskStatus(TaskInstanceDTO taskInstance, StepInstanceBaseDTO stepInstance) { - long endTime = System.currentTimeMillis(); - Long taskInstanceId = stepInstance.getTaskInstanceId(); - // 将进行中的被驱逐任务的步骤实例状态更新为“被丢弃”状态 - if (!RunStatusEnum.isFinishedStatus(stepInstance.getStatus())) { - long totalTime = TaskCostCalculator.calculate( - stepInstance.getStartTime(), - endTime, - stepInstance.getTotalTime() - ); - taskInstanceService.updateStepExecutionInfo( - stepInstance.getId(), - RunStatusEnum.ABANDONED, - null, - endTime, - totalTime - ); - } else { - log.info( - "stepInstance {} already enter a final state:{}", - stepInstance.getId(), - stepInstance.getStatus() - ); - } - // 将进行中的被驱逐任务外层状态更新为“被丢弃”状态 - if (!RunStatusEnum.isFinishedStatus(taskInstance.getStatus())) { - long totalTime = TaskCostCalculator.calculate( - taskInstance.getStartTime(), - endTime, - taskInstance.getTotalTime() - ); - taskInstanceService.updateTaskExecutionInfo( - taskInstanceId, - RunStatusEnum.ABANDONED, - null, - null, - endTime, - totalTime - ); - } else { - log.info( - "taskInstance {} already enter a final state:{}", - taskInstanceId, - taskInstance.getStatus() - ); - } - } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/evict/TaskEvictPolicyManager.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/evict/TaskEvictPolicyManager.java index c7bf0f1292..1419812217 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/evict/TaskEvictPolicyManager.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/evict/TaskEvictPolicyManager.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/evict/TaskInstanceIdEvictPolicy.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/evict/TaskInstanceIdEvictPolicy.java index 69df453d68..5d1b3132d9 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/evict/TaskInstanceIdEvictPolicy.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/evict/TaskInstanceIdEvictPolicy.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/AbstractGseTaskCommand.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/AbstractGseTaskCommand.java index 0a1a832e41..a999bbfa5f 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/AbstractGseTaskCommand.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/AbstractGseTaskCommand.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,32 +24,33 @@ package com.tencent.bk.job.execute.engine.executor; +import com.tencent.bk.job.common.gse.GseClient; +import com.tencent.bk.job.execute.engine.EngineDependentServiceHolder; import com.tencent.bk.job.execute.model.AccountDTO; import com.tencent.bk.job.execute.model.GseTaskDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.service.AccountService; import com.tencent.bk.job.execute.service.AgentService; -import com.tencent.bk.job.execute.service.AgentTaskService; +import com.tencent.bk.job.execute.service.ExecuteObjectTaskService; import com.tencent.bk.job.execute.service.GseTaskService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.cloud.sleuth.Tracer; -import java.util.HashMap; -import java.util.Map; - /** * GSE 任务命令基础实现 */ @Slf4j public abstract class AbstractGseTaskCommand implements GseTaskCommand { + protected final EngineDependentServiceHolder engineDependentServiceHolder; protected final AgentService agentService; protected final AccountService accountService; protected final GseTaskService gseTaskService; - protected final AgentTaskService agentTaskService; + protected final ExecuteObjectTaskService executeObjectTaskService; protected final Tracer tracer; + protected final GseClient gseClient; /** * 步骤实例 @@ -80,58 +81,40 @@ public abstract class AbstractGseTaskCommand implements GseTaskCommand { */ protected int batch; /** - * GSE 任务唯一名称 + * GSE 任务信息 + */ + protected String gseTaskInfo; + /** + * 是否是GSE V2 Task */ - protected String gseTaskUniqueName; + protected boolean gseV2Task; - public AbstractGseTaskCommand(AgentService agentService, - AccountService accountService, - GseTaskService gseTaskService, - AgentTaskService agentTaskService, - Tracer tracer, + public AbstractGseTaskCommand(EngineDependentServiceHolder engineDependentServiceHolder, + ExecuteObjectTaskService executeObjectTaskService, TaskInstanceDTO taskInstance, StepInstanceDTO stepInstance, GseTaskDTO gseTask) { - this.agentService = agentService; - this.accountService = accountService; - this.gseTaskService = gseTaskService; - this.agentTaskService = agentTaskService; - this.tracer = tracer; + this.engineDependentServiceHolder = engineDependentServiceHolder; + this.agentService = engineDependentServiceHolder.getAgentService(); + this.accountService = engineDependentServiceHolder.getAccountService(); + this.gseTaskService = engineDependentServiceHolder.getGseTaskService(); + this.tracer = engineDependentServiceHolder.getTracer(); + this.gseClient = engineDependentServiceHolder.getGseClient(); + this.executeObjectTaskService = executeObjectTaskService; this.taskInstance = taskInstance; this.stepInstance = stepInstance; + this.gseV2Task = stepInstance.isTargetGseV2Agent(); this.gseTask = gseTask; this.taskInstanceId = taskInstance.getId(); this.stepInstanceId = gseTask.getStepInstanceId(); this.executeCount = gseTask.getExecuteCount(); this.batch = gseTask.getBatch(); - this.gseTaskUniqueName = gseTask.getTaskUniqueName(); + this.gseTaskInfo = buildGseTaskInfo(stepInstance.getTaskInstanceId(), gseTask); } - /** - * 生成GSE trace 信息 - */ - protected Map buildGSETraceInfo() { - // 捕获所有异常，避免影响任务下发主流程 - Map traceInfoMap = new HashMap<>(); - try { - traceInfoMap.put("CALLER_NAME", "JOB"); - traceInfoMap.put("JOB_ID", stepInstance.getTaskInstanceId().toString()); - traceInfoMap.put("STEP_ID", stepInstance.getId().toString()); - traceInfoMap.put("EXECUTE_COUNT", String.valueOf(stepInstance.getExecuteCount())); - traceInfoMap.put("JOB_BIZ_ID", taskInstance.getAppId().toString()); - if (tracer != null) { - traceInfoMap.put("REQUEST_ID", tracer.currentSpan().context().traceId()); - } - if (StringUtils.isNotEmpty(taskInstance.getAppCode())) { - traceInfoMap.put("APP_CODE", taskInstance.getAppCode()); - } - traceInfoMap.put("CALLER_IP", agentService.getLocalAgentHost().getIp()); - traceInfoMap.put("TASK_ACCOUNT", stepInstance.getOperator()); - } catch (Throwable e) { - log.error("Build trace info map for gse failed"); - } - return traceInfoMap; + private String buildGseTaskInfo(Long jobInstanceId, GseTaskDTO gseTask) { + return "Job:" + jobInstanceId + "-" + gseTask.getTaskUniqueName(); } /** diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/AbstractGseTaskStartCommand.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/AbstractGseTaskStartCommand.java index e082cdcc1f..4a8cedbcc4 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/AbstractGseTaskStartCommand.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/AbstractGseTaskStartCommand.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,29 +25,29 @@ package com.tencent.bk.job.execute.engine.executor; import com.tencent.bk.job.common.constant.TaskVariableTypeEnum; -import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.common.gse.v2.model.ExecuteObjectGseKey; +import com.tencent.bk.job.common.gse.v2.model.GseTaskResponse; +import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; import com.tencent.bk.job.execute.config.JobExecuteConfig; +import com.tencent.bk.job.execute.engine.EngineDependentServiceHolder; import com.tencent.bk.job.execute.engine.evict.TaskEvictPolicyExecutor; import com.tencent.bk.job.execute.engine.listener.event.EventSource; import com.tencent.bk.job.execute.engine.listener.event.StepEvent; import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; -import com.tencent.bk.job.execute.engine.model.GseTaskResponse; import com.tencent.bk.job.execute.engine.model.TaskVariableDTO; import com.tencent.bk.job.execute.engine.model.TaskVariablesAnalyzeResult; +import com.tencent.bk.job.execute.engine.quota.limit.RunningJobKeepaliveManager; import com.tencent.bk.job.execute.engine.result.ResultHandleManager; import com.tencent.bk.job.execute.engine.result.ha.ResultHandleTaskKeepaliveManager; -import com.tencent.bk.job.execute.model.AgentTaskDTO; +import com.tencent.bk.job.execute.model.ExecuteObjectTask; import com.tencent.bk.job.execute.model.GseTaskDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.model.StepInstanceVariableValuesDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.monitor.metrics.ExecuteMonitor; import com.tencent.bk.job.execute.monitor.metrics.GseTasksExceptionCounter; -import com.tencent.bk.job.execute.service.AccountService; -import com.tencent.bk.job.execute.service.AgentService; -import com.tencent.bk.job.execute.service.AgentTaskService; -import com.tencent.bk.job.execute.service.GseTaskService; +import com.tencent.bk.job.execute.service.ExecuteObjectTaskService; import com.tencent.bk.job.execute.service.LogService; import com.tencent.bk.job.execute.service.StepInstanceService; import com.tencent.bk.job.execute.service.StepInstanceVariableValueService; @@ -56,12 +56,12 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; -import org.springframework.cloud.sleuth.Tracer; import org.springframework.util.StopWatch; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; @Slf4j public abstract class AbstractGseTaskStartCommand extends AbstractGseTaskCommand { @@ -78,18 +78,15 @@ public abstract class AbstractGseTaskStartCommand extends AbstractGseTaskCommand protected final TaskEvictPolicyExecutor taskEvictPolicyExecutor; protected final JobExecuteConfig jobExecuteConfig; protected final StepInstanceService stepInstanceService; - + protected final RunningJobKeepaliveManager runningJobKeepaliveManager; /** * 任务下发请求ID,防止重复下发任务 */ protected String requestId; - - /** - * GSE 目标 Agent 任务, Map + * GSE任务与JOB执行对象任务的映射关系 */ - protected Map targetAgentTaskMap = new HashMap<>(); - + protected Map targetExecuteObjectTaskMap = new HashMap<>(); /** * 全局参数分析结果 */ @@ -102,57 +99,41 @@ public abstract class AbstractGseTaskStartCommand extends AbstractGseTaskCommand * 全局变量参数定义与初始值 */ protected Map globalVariables = new HashMap<>(); - /** - * Agent ID 与 host 映射关系 + * 目标执行对象任务列表(全量，包含非法的任务) */ - protected Map agentIdHostMap; + protected List targetExecuteObjectTasks; - AbstractGseTaskStartCommand(ResultHandleManager resultHandleManager, - TaskInstanceService taskInstanceService, - GseTaskService gseTaskService, - AgentTaskService agentTaskService, - AccountService accountService, - TaskInstanceVariableService taskInstanceVariableService, - StepInstanceVariableValueService stepInstanceVariableValueService, - AgentService agentService, - LogService logService, - TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher, - ResultHandleTaskKeepaliveManager resultHandleTaskKeepaliveManager, - ExecuteMonitor executeMonitor, + AbstractGseTaskStartCommand(EngineDependentServiceHolder engineDependentServiceHolder, + ExecuteObjectTaskService executeObjectTaskService, JobExecuteConfig jobExecuteConfig, - TaskEvictPolicyExecutor taskEvictPolicyExecutor, - GseTasksExceptionCounter gseTasksExceptionCounter, - Tracer tracer, String requestId, TaskInstanceDTO taskInstance, StepInstanceDTO stepInstance, - GseTaskDTO gseTask, - StepInstanceService stepInstanceService) { - super(agentService, - accountService, - gseTaskService, - agentTaskService, - tracer, + GseTaskDTO gseTask) { + super( + engineDependentServiceHolder, + executeObjectTaskService, taskInstance, stepInstance, - gseTask); - this.resultHandleManager = resultHandleManager; - this.taskInstanceService = taskInstanceService; - this.taskInstanceVariableService = taskInstanceVariableService; - this.stepInstanceVariableValueService = stepInstanceVariableValueService; - this.logService = logService; - this.taskExecuteMQEventDispatcher = taskExecuteMQEventDispatcher; - this.resultHandleTaskKeepaliveManager = resultHandleTaskKeepaliveManager; - this.executeMonitor = executeMonitor; + gseTask + ); + + this.taskInstanceVariableService = engineDependentServiceHolder.getTaskInstanceVariableService(); + this.stepInstanceVariableValueService = engineDependentServiceHolder.getStepInstanceVariableValueService(); + this.logService = engineDependentServiceHolder.getLogService(); + this.taskExecuteMQEventDispatcher = engineDependentServiceHolder.getTaskExecuteMQEventDispatcher(); + this.resultHandleTaskKeepaliveManager = engineDependentServiceHolder.getResultHandleTaskKeepaliveManager(); + this.executeMonitor = engineDependentServiceHolder.getExecuteMonitor(); + this.taskEvictPolicyExecutor = engineDependentServiceHolder.getTaskEvictPolicyExecutor(); + this.gseTasksExceptionCounter = engineDependentServiceHolder.getGseTasksExceptionCounter(); + this.runningJobKeepaliveManager = engineDependentServiceHolder.getRunningJobKeepaliveManager(); + this.stepInstanceService = engineDependentServiceHolder.getStepInstanceService(); + this.resultHandleManager = engineDependentServiceHolder.getResultHandleManager(); + this.taskInstanceService = engineDependentServiceHolder.getTaskInstanceService(); this.jobExecuteConfig = jobExecuteConfig; - this.taskEvictPolicyExecutor = taskEvictPolicyExecutor; - this.gseTasksExceptionCounter = gseTasksExceptionCounter; this.requestId = requestId; - this.stepInstanceService = stepInstanceService; - - this.agentIdHostMap = stepInstanceService.computeStepHosts(stepInstance, HostDTO::toCloudIp); } @@ -161,13 +142,26 @@ public abstract class AbstractGseTaskStartCommand extends AbstractGseTaskCommand */ @Override public void execute() { - StopWatch watch = new StopWatch("startGseTask-" + this.gseTaskUniqueName); + StopWatch watch = new StopWatch("startGseTask-" + this.gseTaskInfo); + // 初始化任务执行上下文 watch.start("initExecutionContext"); initExecutionContext(); watch.stop(); - startGseTaskIfNotAvailable(watch); + // 检查任务 + watch.start("checkGseTaskExecutable"); + if (!checkGseTaskExecutable()) { + finishGseTask(RunStatusEnum.FAIL); + return; + } + watch.stop(); + + // 下发任务给GSE + boolean startSuccess = startGseTaskIfNotAvailable(watch); + if (!startSuccess) { + return; + } // 添加执行结果处理后台任务 watch.start("addResultHandleTask"); @@ -181,64 +175,67 @@ public void execute() { } } - private void startGseTaskIfNotAvailable(StopWatch watch) { + /** + * 下发 GSE 任务。 + * + * @param watch watch + * @return 如果任务下发成功/任务已下发，返回true;下发失败返回false + */ + private boolean startGseTaskIfNotAvailable(StopWatch watch) { boolean isGseTaskStarted = StringUtils.isNotEmpty(gseTask.getGseTaskId()); if (!isGseTaskStarted) { watch.start("sendGseTask"); gseTask.setStartTime(System.currentTimeMillis()); - log.info("[{}] Sending task to gse server", this.gseTaskUniqueName); + log.info("[{}] Sending task to gse server", this.gseTaskInfo); GseTaskResponse gseTaskResponse = startGseTask(); watch.stop(); watch.start("handleGseResponse"); if (GseTaskResponse.ERROR_CODE_SUCCESS != gseTaskResponse.getErrorCode()) { + log.error("[{}] Start gse task fail, response: {}", this.gseTaskInfo, gseTaskResponse); handleStartGseTaskError(gseTaskResponse); gseTasksExceptionCounter.increment(); - taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.refreshStep(stepInstanceId, - EventSource.buildGseTaskEventSource(stepInstanceId, executeCount, batch, gseTask.getId()))); + taskExecuteMQEventDispatcher.dispatchStepEvent( + StepEvent.refreshStep( + taskInstanceId, + stepInstanceId, + EventSource.buildGseTaskEventSource( + taskInstanceId, + stepInstanceId, + executeCount, + batch, + gseTask.getId())) + ); watch.stop(); - return; + return false; } else { - updateGseTaskExecutionInfo(gseTaskResponse.getGseTaskId(), RunStatusEnum.RUNNING, - System.currentTimeMillis(), null, null); + log.info("[{}] Start gse task successfully, gseTaskId: {}", this.gseTaskInfo, + gseTaskResponse.getGseTaskId()); + updateGseTaskExecutionInfo(gseTaskResponse.getGseTaskId(), RunStatusEnum.RUNNING, null); } watch.stop(); + return true; } else { // GSE 任务已经下发过，不做处理 - log.info("[{}] Gse Task had already started!", this.gseTaskUniqueName); + log.info("[{}] Gse Task had already started!", this.gseTaskInfo); + return true; } } - /** - * 初始化执行上下文，在GSE任务下发前调用 - */ - private void initExecutionContext() { - initTargetAgentTasks(); - initVariables(); - preExecute(); + private void initExecuteObjectTasks() { + targetExecuteObjectTasks = + executeObjectTaskService.listTasksByGseTaskId(stepInstance, gseTask.getId()) + .stream() + .filter(ExecuteObjectTask::isTarget) + .collect(Collectors.toList()); + + targetExecuteObjectTasks.stream() + .filter(executeObjectTask -> executeObjectTask.getExecuteObject().isExecutable()) + .forEach(executeObjectTask -> + this.targetExecuteObjectTaskMap.put( + executeObjectTask.getExecuteObject().toExecuteObjectGseKey(), executeObjectTask)); } - private void initTargetAgentTasks() { - List agentTasks = agentTaskService.listAgentTasksByGseTaskId(gseTask.getId()); - fillIpForAgentTasks(agentTasks); - agentTasks.stream() - .filter(AgentTaskDTO::isTarget) - .forEach(agentTask -> this.targetAgentTaskMap.put(agentTask.getAgentId(), agentTask)); - } - - protected void fillIpForAgentTasks(List agentTasks) { - Map hosts = stepInstanceService.computeStepHosts(stepInstance, HostDTO::getHostId); - agentTasks.forEach(agentTask -> { - if (agentTask.getHostId() != null) { - HostDTO host = hosts.get(agentTask.getHostId()); - if (host != null) { - agentTask.setCloudIp(host.toCloudIp()); - } - } - }); - } - - private void initVariables() { if (taskInstance.isPlanInstance()) { List taskVariables = @@ -251,7 +248,7 @@ private void initVariables() { return; } stepInputVariables = stepInstanceVariableValueService.computeInputStepInstanceVariableValues( - taskInstance.getId(), stepInstance.getId(), taskVariables); + stepInstance, taskVariables); log.info("Compute step input variable, stepInputVariables:{}", stepInputVariables); } else { taskVariablesAnalyzeResult = new TaskVariablesAnalyzeResult(null); @@ -264,35 +261,29 @@ private void initVariables() { * * @param gseTaskId GSE返回的任务ID;如果不需要更新，传入null * @param status 任务状态;如果不需要更新，传入null - * @param startTime 任务开始时间;如果不需要更新，传入null * @param endTime 任务结束时间;如果不需要更新，传入null - * @param totalTime 任务耗时;如果不需要更新，传入null */ protected void updateGseTaskExecutionInfo(String gseTaskId, RunStatusEnum status, - Long startTime, - Long endTime, - Long totalTime) { + Long endTime) { if (StringUtils.isNotEmpty(gseTaskId)) { gseTask.setGseTaskId(gseTaskId); } if (status != null) { gseTask.setStatus(status.getValue()); } - if (startTime != null) { + if (gseTask.getStartTime() == null) { gseTask.setStartTime(System.currentTimeMillis()); } if (endTime != null) { gseTask.setEndTime(endTime); - } - if (totalTime != null) { - gseTask.setTotalTime(totalTime); + gseTask.setTotalTime(endTime - gseTask.getStartTime()); } gseTaskService.updateGseTask(gseTask); } /** - * 获取字符类型全局变量的变量名和值 + * 获取字符类型（TaskVariableTypeEnum.STRING|TaskVariableTypeEnum.CIPHER)全局变量的变量名和值 * * @param stepInputVariables 步骤入参 * @return 字符类型全局变量的变量名和值 @@ -303,17 +294,43 @@ protected Map buildStringGlobalVarKV(StepInstanceVariableValuesD return globalVarValueMap; } stepInputVariables.getGlobalParams().forEach(globalParam -> { - if (TaskVariableTypeEnum.valOf(globalParam.getType()) == TaskVariableTypeEnum.STRING) { + TaskVariableTypeEnum variableType = TaskVariableTypeEnum.valOf(globalParam.getType()); + if (variableType == TaskVariableTypeEnum.STRING || variableType == TaskVariableTypeEnum.CIPHER) { globalVarValueMap.put(globalParam.getName(), globalParam.getValue()); } }); return globalVarValueMap; } + private void finishGseTask(RunStatusEnum gseTaskStatus) { + updateGseTaskExecutionInfo(null, gseTaskStatus, DateUtils.currentTimeMillis()); + taskExecuteMQEventDispatcher.dispatchStepEvent( + StepEvent.refreshStep( + taskInstanceId, + stepInstanceId, + EventSource.buildGseTaskEventSource( + taskInstanceId, + stepInstanceId, + executeCount, + batch, + gseTask.getId() + ))); + } + + /** + * 初始化执行上下文，在GSE任务下发前调用 + */ + protected void initExecutionContext() { + initExecuteObjectTasks(); + initVariables(); + } + /** - * GSE任务下发前调用 + * 检查任务 + * + * @return 是否可以执行 */ - protected abstract void preExecute(); + protected abstract boolean checkGseTaskExecutable(); /** * 下发GSE任务 diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/FileGseTaskStartCommand.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/FileGseTaskStartCommand.java index 2d65d9f701..c30aa0b413 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/FileGseTaskStartCommand.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/FileGseTaskStartCommand.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,54 +24,44 @@ package com.tencent.bk.job.execute.engine.executor; -import com.tencent.bk.gse.taskapi.api_agent; -import com.tencent.bk.gse.taskapi.api_copy_fileinfoV2; +import com.tencent.bk.job.common.constant.AccountCategoryEnum; import com.tencent.bk.job.common.constant.NotExistPathHandlerEnum; -import com.tencent.bk.job.common.model.dto.HostDTO; -import com.tencent.bk.job.execute.common.constants.FileDistModeEnum; +import com.tencent.bk.job.common.gse.v2.model.Agent; +import com.tencent.bk.job.common.gse.v2.model.ExecuteObjectGseKey; +import com.tencent.bk.job.common.gse.v2.model.FileTransferTask; +import com.tencent.bk.job.common.gse.v2.model.GseTaskResponse; +import com.tencent.bk.job.common.gse.v2.model.SourceFile; +import com.tencent.bk.job.common.gse.v2.model.TargetFile; +import com.tencent.bk.job.common.gse.v2.model.TransferFileRequest; +import com.tencent.bk.job.common.util.CollectionUtil; +import com.tencent.bk.job.common.util.DataSizeConverter; +import com.tencent.bk.job.common.util.FilePathUtils; +import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.execute.common.constants.FileDistStatusEnum; +import com.tencent.bk.job.execute.common.constants.RunStatusEnum; import com.tencent.bk.job.execute.common.util.VariableValueResolver; import com.tencent.bk.job.execute.config.JobExecuteConfig; -import com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum; -import com.tencent.bk.job.execute.engine.consts.FileDirTypeConf; -import com.tencent.bk.job.execute.engine.evict.TaskEvictPolicyExecutor; -import com.tencent.bk.job.execute.engine.gse.GseRequestUtils; -import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; +import com.tencent.bk.job.execute.engine.EngineDependentServiceHolder; +import com.tencent.bk.job.execute.engine.consts.ExecuteObjectTaskStatusEnum; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; import com.tencent.bk.job.execute.engine.model.FileDest; -import com.tencent.bk.job.execute.engine.model.GseTaskResponse; import com.tencent.bk.job.execute.engine.model.JobFile; import com.tencent.bk.job.execute.engine.result.FileResultHandleTask; -import com.tencent.bk.job.execute.engine.result.ResultHandleManager; -import com.tencent.bk.job.execute.engine.result.ha.ResultHandleTaskKeepaliveManager; -import com.tencent.bk.job.execute.engine.util.FilePathUtils; import com.tencent.bk.job.execute.engine.util.JobSrcFileUtils; import com.tencent.bk.job.execute.engine.util.MacroUtil; -import com.tencent.bk.job.execute.engine.util.NFSUtils; import com.tencent.bk.job.execute.model.AccountDTO; -import com.tencent.bk.job.execute.model.AgentTaskDTO; +import com.tencent.bk.job.execute.model.ExecuteObjectCompositeKey; +import com.tencent.bk.job.execute.model.ExecuteObjectTask; import com.tencent.bk.job.execute.model.FileDetailDTO; import com.tencent.bk.job.execute.model.FileSourceDTO; import com.tencent.bk.job.execute.model.GseTaskDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; -import com.tencent.bk.job.execute.monitor.metrics.ExecuteMonitor; -import com.tencent.bk.job.execute.monitor.metrics.GseTasksExceptionCounter; -import com.tencent.bk.job.execute.service.AccountService; -import com.tencent.bk.job.execute.service.AgentService; -import com.tencent.bk.job.execute.service.FileAgentTaskService; -import com.tencent.bk.job.execute.service.GseTaskService; -import com.tencent.bk.job.execute.service.LogService; -import com.tencent.bk.job.execute.service.StepInstanceService; -import com.tencent.bk.job.execute.service.StepInstanceVariableValueService; -import com.tencent.bk.job.execute.service.TaskInstanceService; -import com.tencent.bk.job.execute.service.TaskInstanceVariableService; +import com.tencent.bk.job.execute.service.FileExecuteObjectTaskService; import com.tencent.bk.job.logsvr.consts.FileTaskModeEnum; -import com.tencent.bk.job.logsvr.model.service.ServiceFileTaskLogDTO; -import com.tencent.bk.job.logsvr.model.service.ServiceHostLogDTO; -import com.tencent.bk.job.manage.common.consts.account.AccountCategoryEnum; +import com.tencent.bk.job.logsvr.model.service.ServiceExecuteObjectLogDTO; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; -import org.springframework.cloud.sleuth.Tracer; import java.util.ArrayList; import java.util.HashMap; @@ -79,118 +69,122 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; @Slf4j public class FileGseTaskStartCommand extends AbstractGseTaskStartCommand { + private final FileExecuteObjectTaskService fileExecuteObjectTaskService; /** - * 本地文件服务器 Agent + * 待分发文件，文件传输的源文件 */ - private HostDTO localAgentHost; + private Set srcFiles; /** - * 待分发文件，文件传输的源文件 + * 所有的源文件，包含非法的 */ - private Set sendFiles; + private Set allSrcFiles; /** * 本地文件的存储根目录 */ private final String fileStorageRootPath; + /** - * 源文件路径与目标路径映射关系 - * 格式： Map<源IP:源文件路径，目标路径> + * 文件源-GSE任务与JOB执行对象任务的映射关系 */ - private final Map sourceDestPathMap = new HashMap<>(); + protected Map sourceExecuteObjectTaskMap = new HashMap<>(); /** - * 源文件原始文件路径与展示文件路径的映射 + * 源文件与目标文件路径映射关系 */ - private Map sourceFileDisplayMap = new HashMap<>(); + private final Map srcDestFileMap = new HashMap<>(); /** - * 本地文件存储目录 + * 源文件与目标文件路径映射关系, 包含非法主机 */ - private final String localUploadDir; + private Map allSrcDestFileMap; /** - * GSE 源 Agent 任务, Map + * 源执行对象任务列表(全量，包含非法的任务) */ - protected Map sourceAgentTaskMap = new HashMap<>(); - - private final FileAgentTaskService fileAgentTaskService; - - - public FileGseTaskStartCommand(ResultHandleManager resultHandleManager, - TaskInstanceService taskInstanceService, - GseTaskService gseTaskService, - FileAgentTaskService fileAgentTaskService, - AccountService accountService, - TaskInstanceVariableService taskInstanceVariableService, - StepInstanceVariableValueService stepInstanceVariableValueService, - AgentService agentService, - LogService logService, - TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher, - ResultHandleTaskKeepaliveManager resultHandleTaskKeepaliveManager, - ExecuteMonitor executeMonitor, + protected List sourceExecuteObjectTasks; + + + public FileGseTaskStartCommand(EngineDependentServiceHolder engineDependentServiceHolder, + FileExecuteObjectTaskService fileExecuteObjectTaskService, JobExecuteConfig jobExecuteConfig, - TaskEvictPolicyExecutor taskEvictPolicyExecutor, - GseTasksExceptionCounter gseTasksExceptionCounter, - StepInstanceService stepInstanceService, - Tracer tracer, String requestId, TaskInstanceDTO taskInstance, StepInstanceDTO stepInstance, GseTaskDTO gseTask, String fileStorageRootPath) { - super(resultHandleManager, - taskInstanceService, - gseTaskService, - fileAgentTaskService, - accountService, - taskInstanceVariableService, - stepInstanceVariableValueService, - agentService, - logService, - taskExecuteMQEventDispatcher, - resultHandleTaskKeepaliveManager, - executeMonitor, + super(engineDependentServiceHolder, + fileExecuteObjectTaskService, jobExecuteConfig, - taskEvictPolicyExecutor, - gseTasksExceptionCounter, - tracer, requestId, taskInstance, stepInstance, - gseTask, - stepInstanceService); - - this.fileAgentTaskService = fileAgentTaskService; + gseTask); + this.fileExecuteObjectTaskService = fileExecuteObjectTaskService; this.fileStorageRootPath = fileStorageRootPath; - this.localUploadDir = NFSUtils.getFileDir(fileStorageRootPath, FileDirTypeConf.UPLOAD_FILE_DIR); } @Override - protected void preExecute() { - // 设置本地文件服务器的Agent Ip - this.localAgentHost = agentService.getLocalAgentHost(); + protected void initExecutionContext() { + super.initExecutionContext(); + // 解析文件源 + resolveFileSource(); // 解析文件传输的源文件, 得到List - parseSendFileList(); + parseSrcFiles(); + // 解析目标路径中的变量 resolvedTargetPathWithVariable(); - initFileSourceGseAgentTasks(); + // 解析源<->目标文件映射 + parseSrcDestFileMap(); + // 初始化执行对象任务 + initFileSourceExecuteObjectTasks(); + // 保存文件子任务的初始状态 + saveInitialFileTaskLogs(); + } + + private void parseSrcDestFileMap() { + // 路径中变量解析与路径标准化预处理 + String targetDir = FilePathUtils.standardizedDirPath(stepInstance.getResolvedFileTargetPath()); + // 构造源路径与目标路径映射 + allSrcDestFileMap = JobSrcFileUtils.buildSourceDestPathMapping(allSrcFiles, targetDir, + stepInstance.getFileTargetName()); + allSrcDestFileMap.forEach((sreFile, destFile) -> { + if (sreFile.getExecuteObject().isExecutable()) { + srcDestFileMap.put(sreFile, destFile); + } + }); + } + + /** + * 解析文件源 + */ + private void resolveFileSource() { + List fileSourceList = stepInstance.getFileSourceList(); + if (CollectionUtils.isNotEmpty(fileSourceList)) { + // 解析源文件路径中的全局变量 + resolveVariableForSourceFilePath(fileSourceList, buildStringGlobalVarKV(stepInputVariables)); + + stepInstanceService.updateResolvedSourceFile(stepInstance.getTaskInstanceId(), + stepInstance.getId(), stepInstance.getFileSourceList()); + } } /** * 解析源文件 */ - private void parseSendFileList() { - resolveVariableForSourceFilePath(stepInstance.getFileSourceList(), - buildStringGlobalVarKV(stepInputVariables)); - sendFiles = JobSrcFileUtils.parseSendFileList(stepInstance, localAgentHost, fileStorageRootPath); - setAccountInfoForSourceFiles(sendFiles); - // 初始化显示名称映射Map - sourceFileDisplayMap = JobSrcFileUtils.buildSourceFileDisplayMapping(sendFiles, localUploadDir); + private void parseSrcFiles() { + allSrcFiles = JobSrcFileUtils.parseSrcFiles(stepInstance, fileStorageRootPath); + srcFiles = allSrcFiles.stream() + .filter(file -> file.getExecuteObject().isExecutable()) + .collect(Collectors.toSet()); + // 设置源文件所在主机账号信息 + setAccountInfoForSourceFiles(srcFiles); } private void setAccountInfoForSourceFiles(Set sendFiles) { Map accounts = new HashMap<>(); sendFiles.forEach(sendFile -> { - String accountKey = sendFile.getAccountId() == null ? ("id_" + sendFile.getAccountId()) + String accountKey = sendFile.getAccountId() == null ? ("id_null") : ("alias_" + sendFile.getAccountAlias()); AccountDTO account = accounts.computeIfAbsent(accountKey, k -> accountService.getAccount(sendFile.getAccountId(), AccountCategoryEnum.SYSTEM, @@ -209,7 +203,6 @@ private void resolveVariableForSourceFilePath(List fileSources, if (stepInputGlobalVariableValueMap == null || stepInputGlobalVariableValueMap.isEmpty()) { return; } - boolean isContainsVar = false; for (FileSourceDTO fileSource : fileSources) { if (CollectionUtils.isNotEmpty(fileSource.getFiles())) { for (FileDetailDTO file : fileSource.getFiles()) { @@ -217,14 +210,10 @@ private void resolveVariableForSourceFilePath(List fileSources, stepInputGlobalVariableValueMap); if (!resolvedFilePath.equals(file.getFilePath())) { file.setResolvedFilePath(resolvedFilePath); - isContainsVar = true; } } } } - if (isContainsVar) { - taskInstanceService.updateResolvedSourceFile(stepInstance.getId(), stepInstance.getFileSourceList()); - } } /** @@ -236,40 +225,55 @@ private void resolvedTargetPathWithVariable() { resolvedTargetPath = MacroUtil.resolveDateWithStrfTime(resolvedTargetPath); stepInstance.setResolvedFileTargetPath(resolvedTargetPath); if (!resolvedTargetPath.equals(stepInstance.getFileTargetPath())) { - taskInstanceService.updateResolvedTargetPath(stepInstance.getId(), resolvedTargetPath); + stepInstanceService.updateResolvedTargetPath(stepInstance.getTaskInstanceId(), + stepInstance.getId(), resolvedTargetPath); } } /** - * 初始化源文件服务器上传任务状态 + * 初始化源文件上传任务状态 */ - private void initFileSourceGseAgentTasks() { - Set sourceHosts = new HashSet<>(); - if (sendFiles != null) { - for (JobFile sendFile : sendFiles) { - if (sendFile.getHost() != null) { - sourceHosts.add(sendFile.getHost()); + private void initFileSourceExecuteObjectTasks() { + Set sourceExecuteObjects = new HashSet<>(); + if (allSrcFiles != null) { + for (JobFile sendFile : allSrcFiles) { + if (sendFile.getExecuteObject() != null) { + sourceExecuteObjects.add(sendFile.getExecuteObject()); } } } - List fileSourceGseAgentTasks = new ArrayList<>(); - for (HostDTO sourceHost : sourceHosts) { - AgentTaskDTO agentTask = new AgentTaskDTO(stepInstanceId, executeCount, batch, sourceHost.getHostId(), - sourceHost.getAgentId()); - agentTask.setActualExecuteCount(executeCount); - agentTask.setCloudIp(sourceHost.toCloudIp()); - agentTask.setDisplayIp(sourceHost.getDisplayIp()); - agentTask.setFileTaskMode(FileTaskModeEnum.UPLOAD); - agentTask.setStatus(AgentTaskStatusEnum.WAITING); - agentTask.setGseTaskId(gseTask.getId()); - sourceAgentTaskMap.put(sourceHost.getAgentId(), agentTask); - fileSourceGseAgentTasks.add(agentTask); + sourceExecuteObjectTasks = new ArrayList<>(); + for (ExecuteObject sourceExecuteObject : sourceExecuteObjects) { + ExecuteObjectTask executeObjectTask = new ExecuteObjectTask( + taskInstanceId, + stepInstanceId, + executeCount, + batch, + sourceExecuteObject + ); + executeObjectTask.setActualExecuteCount(executeCount); + executeObjectTask.setFileTaskMode(FileTaskModeEnum.UPLOAD); + executeObjectTask.setGseTaskId(gseTask.getId()); + + if (sourceExecuteObject.isAgentIdEmpty()) { + executeObjectTask.setStatus(ExecuteObjectTaskStatusEnum.AGENT_NOT_INSTALLED); + } else if (sourceExecuteObject.isInvalid()) { + executeObjectTask.setStatus(ExecuteObjectTaskStatusEnum.INVALID_EXECUTE_OBJECT); + } else { + executeObjectTask.setStatus(ExecuteObjectTaskStatusEnum.WAITING); + sourceExecuteObjectTaskMap.put(sourceExecuteObject.toExecuteObjectGseKey(), executeObjectTask); + } + + sourceExecuteObjectTasks.add(executeObjectTask); } - fileAgentTaskService.batchSaveAgentTasks(fileSourceGseAgentTasks); + fileExecuteObjectTaskService.batchSaveTasks(sourceExecuteObjectTasks); } @Override protected GseTaskResponse startGseTask() { + TransferFileRequest request = new TransferFileRequest(); + request.setGseV2Task(gseV2Task); + // 账号信息查询与填充 AccountDTO accountInfo = accountService.getAccount(stepInstance.getAccountId(), AccountCategoryEnum.SYSTEM, stepInstance.getAccountAlias(), stepInstance.getAppId()); @@ -277,106 +281,70 @@ protected GseTaskResponse startGseTask() { log.error("Start gse task fail, account is null!"); return GseTaskResponse.fail(GseTaskResponse.ERROR_CODE_FAIL, "account is empty"); } - List dst = GseRequestUtils.buildAgentList(targetAgentTaskMap.keySet(), accountInfo.getAccount(), + + List targetAgents = gseClient.fillAgentAuthInfo( + targetExecuteObjectTaskMap.values().stream() + .map(executeObjectTask -> executeObjectTask.getExecuteObject().toGseAgent()) + .collect(Collectors.toList()), + accountInfo.getAccount(), accountInfo.getPassword()); + // 构造GSE文件分发请求 + for (JobFile file : srcFiles) { + Agent srcAgent = buildAgent(file.getExecuteObject(), file.getAccount(), file.getPassword()); + SourceFile sourceFile = new SourceFile(file.getFileName(), file.getDir(), srcAgent); - List copyFileInfoList = new ArrayList<>(); - // 路径中变量解析与路径标准化预处理 - String targetDir = FilePathUtils.standardizedDirPath(stepInstance.getResolvedFileTargetPath()); - // 构造源路径与目标路径映射Map - Map srcAndDestMap = JobSrcFileUtils.buildSourceDestPathMapping(sendFiles, targetDir, - stepInstance.getFileTargetName()); - // 构造源路径与目标路径映射Map供后续状态判定使用 - initSourceDestPathMap(srcAndDestMap); + FileDest fileDest = srcDestFileMap.get(file); + TargetFile targetFile = new TargetFile(fileDest.getDestName(), fileDest.getDestDirPath(), targetAgents); - // 构造GSE文件分发请求 - for (JobFile file : sendFiles) { - FileDest fileDest = srcAndDestMap.get(file.getUniqueKey()); - api_agent src = GseRequestUtils.buildAgent(file.getHost().toCloudIp(), file.getAccount(), - file.getPassword()); - api_copy_fileinfoV2 copyFileInfo = GseRequestUtils.buildCopyFileInfo(src, file.getDir(), file.getFileName(), - dst, fileDest.getDestDirPath(), fileDest.getDestName(), - stepInstance.getFileDownloadSpeedLimit(), stepInstance.getFileUploadSpeedLimit(), - stepInstance.getTimeout()); - - // 设置文件路径不存在的处理方式 - if (stepInstance.getNotExistPathHandler() == NotExistPathHandlerEnum.CREATE_DIR.getValue()) { - // 直接创建 - copyFileInfo.setMkdirflag(1); - } else if (stepInstance.getNotExistPathHandler() == NotExistPathHandlerEnum.STEP_FAIL.getValue()) { - // 直接失败 - copyFileInfo.setMkdirflag(0); - } else { - log.warn("NotExistPathHandler not supported：{}, supported handlers are:{}", - stepInstance.getNotExistPathHandler(), NotExistPathHandlerEnum.getDescStr()); - } - copyFileInfoList.add(copyFileInfo); + FileTransferTask fileTask = new FileTransferTask(sourceFile, targetFile); + request.addFileTask(fileTask); } - if (CollectionUtils.isNotEmpty(copyFileInfoList)) { - copyFileInfoList.get(0).setM_caller(buildGSETraceInfo()); + + // 设置文件路径不存在的处理方式 + if (stepInstance.getNotExistPathHandler() == NotExistPathHandlerEnum.CREATE_DIR.getValue()) { + // 直接创建 + request.setAutoMkdir(true); + } else if (stepInstance.getNotExistPathHandler() == NotExistPathHandlerEnum.STEP_FAIL.getValue()) { + // 直接失败 + request.setAutoMkdir(false); + } + + setSpeedLimit(request, stepInstance); + request.setTimeout(stepInstance.getTimeout()); + + return gseClient.asyncTransferFile(request); + } + + private Agent buildAgent(ExecuteObject executeObject, String user, String password) { + Agent agent = executeObject.toGseAgent(); + agent.setUser(user); + agent.setPwd(password); + return agent; + } + + private void setSpeedLimit(TransferFileRequest request, StepInstanceDTO stepInstance) { + if (stepInstance.getFileDownloadSpeedLimit() != null && stepInstance.getFileDownloadSpeedLimit() > 0) { + // KB -> MB + request.setDownloadSpeed(DataSizeConverter.convertKBToMB(stepInstance.getFileDownloadSpeedLimit())); + } else { + request.setDownloadSpeed(0); + } + if (stepInstance.getFileUploadSpeedLimit() != null && stepInstance.getFileUploadSpeedLimit() > 0) { + // KB -> MB + request.setUploadSpeed(DataSizeConverter.convertKBToMB(stepInstance.getFileUploadSpeedLimit())); + } else { + request.setUploadSpeed(0); } - saveInitialFileTaskLogs(sourceDestPathMap); - return GseRequestUtils.sendCopyFileTaskRequest(stepInstanceId, copyFileInfoList); } /** * 保存文件执行日志初始状态 */ - private void saveInitialFileTaskLogs(Map sourceDestPathMap) { + private void saveInitialFileTaskLogs() { try { - log.debug("[{}] SourceDestPathMap: {}", stepInstanceId, sourceDestPathMap); - Map logs = new HashMap<>(); - // 每个要分发的源文件一条上传日志 - for (JobFile file : sendFiles) { - Long sourceHostId = file.getHost().getHostId(); - ServiceHostLogDTO hostTaskLog = initServiceLogDTOIfAbsent(logs, stepInstanceId, executeCount, - sourceHostId, file.getHost().toCloudIp()); - hostTaskLog.addFileTaskLog( - new ServiceFileTaskLogDTO( - FileDistModeEnum.UPLOAD.getValue(), - null, - null, - null, - sourceHostId, - file.getHost().toCloudIp(), - file.getHost().toCloudIp(), - file.getStandardFilePath(), - file.getDisplayFilePath(), - "--", - FileDistStatusEnum.WAITING.getValue(), - FileDistStatusEnum.WAITING.getName(), - "--", - "--", - null) - ); - } - // 每个目标IP从每个要分发的源文件下载的一条下载日志 - for (AgentTaskDTO targetAgentTask : targetAgentTaskMap.values()) { - HostDTO targetHost = agentIdHostMap.get(targetAgentTask.getAgentId()); - ServiceHostLogDTO ipTaskLog = initServiceLogDTOIfAbsent(logs, stepInstanceId, executeCount, - targetHost.getHostId(), targetHost.toCloudIp()); - for (JobFile file : sendFiles) { - Long sourceHostId = file.getHost().getHostId(); - ipTaskLog.addFileTaskLog( - new ServiceFileTaskLogDTO( - FileDistModeEnum.DOWNLOAD.getValue(), - targetAgentTask.getHostId(), - targetHost.toCloudIp(), - getDestPath(file), - sourceHostId, - file.getHost().toCloudIp(), - file.getHost().toCloudIp(), - file.getStandardFilePath(), - file.getDisplayFilePath(), - "--", - FileDistStatusEnum.WAITING.getValue(), - FileDistStatusEnum.WAITING.getName(), - "--", - "--", - null) - ); - } - } + Map logs = new HashMap<>(); + addInitialFileUploadTaskLogs(logs); + addInitialFileDownloadTaskLogs(logs); // 调用logService写入MongoDB writeLogs(logs); } catch (Throwable e) { @@ -384,62 +352,142 @@ private void saveInitialFileTaskLogs(Map sourceDestPathMap) { } } - private ServiceHostLogDTO initServiceLogDTOIfAbsent(Map logs, long stepInstanceId, - int executeCount, Long hostId, String cloudIp) { - ServiceHostLogDTO hostTaskLog = logs.get(hostId); - if (hostTaskLog == null) { - hostTaskLog = new ServiceHostLogDTO(); - hostTaskLog.setStepInstanceId(stepInstanceId); - hostTaskLog.setExecuteCount(executeCount); - hostTaskLog.setHostId(hostId); - hostTaskLog.setIp(cloudIp); - logs.put(hostId, hostTaskLog); + private void addInitialFileUploadTaskLogs(Map logs) { + // 每个要分发的源文件一条上传日志 + for (JobFile file : allSrcFiles) { + boolean isSourceValid = !file.getExecuteObject().isInvalid(); + boolean isSourceAgentInstalled = !file.getExecuteObject().isAgentIdEmpty(); + FileDistStatusEnum status = isSourceValid && isSourceAgentInstalled ? + FileDistStatusEnum.WAITING : FileDistStatusEnum.FAILED; + logService.addFileTaskLog( + stepInstance, + logs, + file.getExecuteObject(), + logService.buildUploadServiceFileTaskLogDTO( + stepInstance, + file, + status, + "--", + "--", + "--", + buildInitialFileTaskUploadLogContent(isSourceValid, isSourceAgentInstalled) + ) + ); + } + } + + private String buildInitialFileTaskUploadLogContent(boolean isSourceValid, + boolean isSourceAgentInstalled) { + if (!isSourceValid) { + return "Source execute object is invalid, please check whether the host belongs to the current business"; + } else if (!isSourceAgentInstalled) { + return "Agent is not installed"; + } else { + // 源、目标正常，无需写入错误日志 + return null; } - return hostTaskLog; } - private void writeLogs(Map executionLogs) { - log.debug("Write file task initial logs, executionLogs: {}", executionLogs); - logService.writeFileLogs(taskInstance.getCreateTime(), new ArrayList<>(executionLogs.values())); + private void addInitialFileDownloadTaskLogs(Map logs) { + // 每个目标IP从每个要分发的源文件下载的一条下载日志 + targetExecuteObjectTasks + .forEach(targetExecuteObjectTask -> { + boolean isTargetValid = !targetExecuteObjectTask.getExecuteObject().isInvalid(); + boolean isTargetAgentInstalled = !targetExecuteObjectTask.getExecuteObject().isAgentIdEmpty(); + for (JobFile file : allSrcFiles) { + boolean isSourceValid = !file.getExecuteObject().isInvalid(); + boolean isSourceAgentInstalled = !file.getExecuteObject().isAgentIdEmpty(); + FileDistStatusEnum status = isTargetValid && isTargetAgentInstalled && isSourceValid + && isSourceAgentInstalled ? FileDistStatusEnum.WAITING : FileDistStatusEnum.FAILED; + logService.addFileTaskLog( + stepInstance, + logs, + targetExecuteObjectTask.getExecuteObject(), + logService.buildDownloadServiceFileTaskLogDTO( + stepInstance, + file, + targetExecuteObjectTask.getExecuteObject(), + getDestPath(file), + status, + "--", + "--", + "--", + buildInitialFileTaskDownloadLogContent( + isTargetValid, + isTargetAgentInstalled, + isSourceValid, + isSourceAgentInstalled + ) + ) + ); + } + }); + } + + private String buildInitialFileTaskDownloadLogContent(boolean isTargetValid, + boolean isTargetAgentInstalled, + boolean isSourceValid, + boolean isSourceAgentInstalled) { + if (!isTargetValid) { + return "Target execute object is invalid, please check whether the host belongs to the current business"; + } else if (!isTargetAgentInstalled) { + return "Agent is not installed"; + } else if (!isSourceValid) { + return "Source execute object is invalid, please check whether the host belongs to the current business"; + } else if (!isSourceAgentInstalled) { + return "Source agent is not installed"; + } else { + // 源、目标正常，无需错误日志 + return null; + } } - private void initSourceDestPathMap(Map srcAndDestMap) { - srcAndDestMap.forEach((fileKey, dest) -> this.sourceDestPathMap.put(fileKey, dest.getDestPath())); + private void writeLogs(Map executionLogs) { + if (log.isDebugEnabled()) { + log.debug("Write file task initial logs, executionLogs: {}", executionLogs); + } + logService.writeFileLogs(taskInstance, new ArrayList<>(executionLogs.values())); } + private String getDestPath(JobFile sourceFile) { - return sourceDestPathMap.get(sourceFile.getUniqueKey()); + return allSrcDestFileMap.get(sourceFile).getDestPath(); } @Override protected void handleStartGseTaskError(GseTaskResponse gseTaskResponse) { - + updateGseTaskExecutionInfo(null, RunStatusEnum.FAIL, DateUtils.currentTimeMillis()); } @Override protected void addResultHandleTask() { FileResultHandleTask fileResultHandleTask = - new FileResultHandleTask(taskInstanceService, - gseTaskService, - logService, - taskInstanceVariableService, - stepInstanceVariableValueService, - taskExecuteMQEventDispatcher, - resultHandleTaskKeepaliveManager, - taskEvictPolicyExecutor, - fileAgentTaskService, - stepInstanceService, + new FileResultHandleTask( + engineDependentServiceHolder, + fileExecuteObjectTaskService, taskInstance, stepInstance, taskVariablesAnalyzeResult, - targetAgentTaskMap, - sourceAgentTaskMap, + targetExecuteObjectTaskMap, + sourceExecuteObjectTaskMap, gseTask, - sendFiles, - fileStorageRootPath, - sourceDestPathMap, - sourceFileDisplayMap, - requestId); + srcDestFileMap, + requestId, + CollectionUtil.mergeToArrayList(targetExecuteObjectTasks, sourceExecuteObjectTasks)); resultHandleManager.handleDeliveredTask(fileResultHandleTask); } + + + @Override + protected boolean checkGseTaskExecutable() { + if (this.targetExecuteObjectTaskMap.isEmpty()) { + log.warn("File gse task target agent is empty, can not execute! Set gse task status fail"); + return false; + } + if (this.sourceExecuteObjectTaskMap.isEmpty()) { + log.warn("File gse task source agent is empty, can not execute! Set gse task status fail"); + return false; + } + return true; + } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/FileGseTaskStopCommand.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/FileGseTaskStopCommand.java index 572e625b82..ffc519d9e3 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/FileGseTaskStopCommand.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/FileGseTaskStopCommand.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,44 +24,32 @@ package com.tencent.bk.job.execute.engine.executor; -import com.tencent.bk.gse.taskapi.api_agent; -import com.tencent.bk.gse.taskapi.api_stop_task_request; +import com.tencent.bk.job.common.gse.v2.model.Agent; +import com.tencent.bk.job.common.gse.v2.model.GseTaskResponse; +import com.tencent.bk.job.common.gse.v2.model.TerminateGseTaskRequest; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; -import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; -import com.tencent.bk.job.execute.engine.gse.GseRequestUtils; -import com.tencent.bk.job.execute.engine.model.GseTaskResponse; -import com.tencent.bk.job.execute.model.AccountDTO; -import com.tencent.bk.job.execute.model.AgentTaskDTO; +import com.tencent.bk.job.execute.engine.EngineDependentServiceHolder; +import com.tencent.bk.job.execute.model.ExecuteObjectTask; import com.tencent.bk.job.execute.model.GseTaskDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; -import com.tencent.bk.job.execute.service.AccountService; -import com.tencent.bk.job.execute.service.AgentService; -import com.tencent.bk.job.execute.service.AgentTaskService; -import com.tencent.bk.job.execute.service.GseTaskService; +import com.tencent.bk.job.execute.service.ExecuteObjectTaskService; import lombok.extern.slf4j.Slf4j; -import org.springframework.cloud.sleuth.Tracer; +import org.apache.commons.lang3.StringUtils; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; @Slf4j public class FileGseTaskStopCommand extends AbstractGseTaskCommand { - public FileGseTaskStopCommand(AgentService agentService, - AccountService accountService, - GseTaskService gseTaskService, - AgentTaskService agentTaskService, - Tracer tracer, + public FileGseTaskStopCommand(EngineDependentServiceHolder engineDependentServiceHolder, + ExecuteObjectTaskService executeObjectTaskService, TaskInstanceDTO taskInstance, StepInstanceDTO stepInstance, GseTaskDTO gseTask) { - super(agentService, - accountService, - gseTaskService, - agentTaskService, - tracer, + super(engineDependentServiceHolder, + executeObjectTaskService, taskInstance, stepInstance, gseTask); @@ -69,29 +57,22 @@ public FileGseTaskStopCommand(AgentService agentService, @Override public void execute() { - log.info("Stop gse task, gseTask:" + gseTaskUniqueName); - List agentTasks = agentTaskService.listAgentTasksByGseTaskId(gseTask.getId()); - AccountDTO targetAccount = getAccountBean(stepInstance.getAccountId(), stepInstance.getAccount(), - stepInstance.getAppId()); - Set agentIds = agentTasks.stream() - .map(AgentTaskDTO::getAgentId) - .collect(Collectors.toSet()); - //目标机器的agent - List agentList = GseRequestUtils.buildAgentList(agentIds, targetAccount.getAccount(), - targetAccount.getPassword()); + log.info("Stop gse file task, gseTask:" + gseTaskInfo); + List executeObjectTasks = + executeObjectTaskService.listTasksByGseTaskId(stepInstance, gseTask.getId()); + List agents = executeObjectTasks.stream() + .map(executeObjectTask -> executeObjectTask.getExecuteObject().toGseAgent()) + .filter(agent -> agent != null && StringUtils.isNotEmpty(agent.getAgentId())) + .distinct() + .collect(Collectors.toList()); - api_stop_task_request stopTaskRequest = new api_stop_task_request(); - stopTaskRequest.setStop_task_id(gseTask.getGseTaskId()); - stopTaskRequest.setAgents(agentList); - stopTaskRequest.setType(StepExecuteTypeEnum.SEND_FILE.getValue()); - stopTaskRequest.setM_caller(buildGSETraceInfo()); - GseTaskResponse gseTaskResponse = GseRequestUtils.sendForceStopTaskRequest(stepInstance.getId(), - stopTaskRequest); + TerminateGseTaskRequest request = new TerminateGseTaskRequest(gseTask.getGseTaskId(), agents, gseV2Task); + GseTaskResponse gseTaskResponse = gseClient.terminateGseFileTask(request); if (GseTaskResponse.ERROR_CODE_SUCCESS != gseTaskResponse.getErrorCode()) { - log.error("Terminate gse task failed! gseTask: {}", gseTaskUniqueName); + log.error("Terminate gse task failed! gseTask: {}", gseTaskInfo); } else { - log.info("Terminate gse task response success! gseTask: {}", gseTaskUniqueName); + log.info("Terminate gse task response success! gseTask: {}", gseTaskInfo); gseTask.setStatus(RunStatusEnum.STOPPING.getValue()); gseTaskService.updateGseTask(gseTask); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/GseTaskCommand.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/GseTaskCommand.java index d3bc6444a9..4f2b33ad24 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/GseTaskCommand.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/GseTaskCommand.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/GseTaskManager.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/GseTaskManager.java index 9014922a60..a99e2f2d1b 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/GseTaskManager.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/GseTaskManager.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -31,44 +31,32 @@ import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; import com.tencent.bk.job.execute.common.exception.MessageHandlerUnavailableException; import com.tencent.bk.job.execute.common.ha.DestroyOrder; +import com.tencent.bk.job.execute.config.FileDistributeConfig; import com.tencent.bk.job.execute.config.JobExecuteConfig; -import com.tencent.bk.job.execute.config.StorageSystemConfig; +import com.tencent.bk.job.execute.engine.EngineDependentServiceHolder; import com.tencent.bk.job.execute.engine.evict.TaskEvictPolicyExecutor; import com.tencent.bk.job.execute.engine.listener.event.EventSource; import com.tencent.bk.job.execute.engine.listener.event.StepEvent; import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; -import com.tencent.bk.job.execute.engine.result.ResultHandleManager; -import com.tencent.bk.job.execute.engine.result.ha.ResultHandleTaskKeepaliveManager; import com.tencent.bk.job.execute.engine.util.RunningTaskCounter; -import com.tencent.bk.job.execute.engine.variable.JobBuildInVariableResolver; import com.tencent.bk.job.execute.model.GseTaskDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.monitor.ExecuteMetricNames; import com.tencent.bk.job.execute.monitor.metrics.ExecuteMonitor; -import com.tencent.bk.job.execute.monitor.metrics.GseTasksExceptionCounter; -import com.tencent.bk.job.execute.service.AccountService; -import com.tencent.bk.job.execute.service.AgentService; -import com.tencent.bk.job.execute.service.FileAgentTaskService; +import com.tencent.bk.job.execute.service.FileExecuteObjectTaskService; import com.tencent.bk.job.execute.service.GseTaskService; -import com.tencent.bk.job.execute.service.LogService; -import com.tencent.bk.job.execute.service.ScriptAgentTaskService; +import com.tencent.bk.job.execute.service.ScriptExecuteObjectTaskService; import com.tencent.bk.job.execute.service.StepInstanceService; -import com.tencent.bk.job.execute.service.StepInstanceVariableValueService; import com.tencent.bk.job.execute.service.TaskInstanceService; -import com.tencent.bk.job.execute.service.TaskInstanceVariableService; -import com.tencent.bk.job.manage.common.consts.task.TaskStepTypeEnum; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.cloud.sleuth.Tracer; import org.springframework.context.SmartLifecycle; import org.springframework.stereotype.Component; import org.springframework.util.StopWatch; -import java.util.Map; import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -78,33 +66,20 @@ @Component @Slf4j public class GseTaskManager implements SmartLifecycle { - private final ResultHandleManager resultHandleManager; + private final EngineDependentServiceHolder engineDependentServiceHolder; private final TaskInstanceService taskInstanceService; private final GseTaskService gseTaskService; - private final ScriptAgentTaskService scriptAgentTaskService; - private final FileAgentTaskService fileAgentTaskService; + private final ScriptExecuteObjectTaskService scriptExecuteObjectTaskService; + private final FileExecuteObjectTaskService fileExecuteObjectTaskService; private final TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher; - private final AccountService accountService; - private final LogService logService; - private final TaskInstanceVariableService taskInstanceVariableService; - private final StepInstanceVariableValueService stepInstanceVariableValueService; - private final AgentService agentService; - private final ResultHandleTaskKeepaliveManager resultHandleTaskKeepaliveManager; - private final JobBuildInVariableResolver jobBuildInVariableResolver; - private final Tracer tracer; private final ExecuteMonitor executeMonitor; - private final StorageSystemConfig storageSystemConfig; + private final FileDistributeConfig fileDistributeConfig; private final JobExecuteConfig jobExecuteConfig; private final TaskEvictPolicyExecutor taskEvictPolicyExecutor; - private final GseTasksExceptionCounter gseTasksExceptionCounter; private final StepInstanceService stepInstanceService; private final Object lifecycleMonitor = new Object(); private final RunningTaskCounter counter = new RunningTaskCounter<>("GseTask-Counter"); - /** - * 正在执行中的任务 - */ - private final Map startingGseTasks = new ConcurrentHashMap<>(); private volatile boolean running = false; private volatile boolean active = false; @@ -133,42 +108,24 @@ public class GseTaskManager implements SmartLifecycle { * GseTaskManager Constructor */ @Autowired - public GseTaskManager(ResultHandleManager resultHandleManager, + public GseTaskManager(EngineDependentServiceHolder engineDependentServiceHolder, TaskInstanceService taskInstanceService, GseTaskService gseTaskService, TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher, - AccountService accountService, - LogService logService, - TaskInstanceVariableService taskInstanceVariableService, - StepInstanceVariableValueService stepInstanceVariableValueService, - JobBuildInVariableResolver jobBuildInVariableResolver, - StorageSystemConfig storageSystemConfig, - AgentService agentService, - ResultHandleTaskKeepaliveManager resultHandleTaskKeepaliveManager, - GseTasksExceptionCounter gseTasksExceptionCounter, - Tracer tracer, + FileDistributeConfig fileDistributeConfig, ExecuteMonitor executeMonitor, JobExecuteConfig jobExecuteConfig, TaskEvictPolicyExecutor taskEvictPolicyExecutor, - ScriptAgentTaskService scriptAgentTaskService, - FileAgentTaskService fileAgentTaskService, + ScriptExecuteObjectTaskService scriptExecuteObjectTaskService, + FileExecuteObjectTaskService fileExecuteObjectTaskService, StepInstanceService stepInstanceService) { - this.resultHandleManager = resultHandleManager; + this.engineDependentServiceHolder = engineDependentServiceHolder; this.taskInstanceService = taskInstanceService; this.gseTaskService = gseTaskService; - this.scriptAgentTaskService = scriptAgentTaskService; - this.fileAgentTaskService = fileAgentTaskService; + this.scriptExecuteObjectTaskService = scriptExecuteObjectTaskService; + this.fileExecuteObjectTaskService = fileExecuteObjectTaskService; this.taskExecuteMQEventDispatcher = taskExecuteMQEventDispatcher; - this.accountService = accountService; - this.logService = logService; - this.taskInstanceVariableService = taskInstanceVariableService; - this.stepInstanceVariableValueService = stepInstanceVariableValueService; - this.jobBuildInVariableResolver = jobBuildInVariableResolver; - this.storageSystemConfig = storageSystemConfig; - this.agentService = agentService; - this.resultHandleTaskKeepaliveManager = resultHandleTaskKeepaliveManager; - this.gseTasksExceptionCounter = gseTasksExceptionCounter; - this.tracer = tracer; + this.fileDistributeConfig = fileDistributeConfig; this.executeMonitor = executeMonitor; this.jobExecuteConfig = jobExecuteConfig; this.taskEvictPolicyExecutor = taskEvictPolicyExecutor; @@ -206,13 +163,14 @@ public void startTask(GseTaskDTO gseTask, String requestId) { watch.stop(); watch.start("loadTaskAndCheck"); - StepInstanceDTO stepInstance = taskInstanceService.getStepInstanceDetail(stepInstanceId); + StepInstanceDTO stepInstance = stepInstanceService.getStepInstanceDetail( + gseTask.getTaskInstanceId(), stepInstanceId); TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(stepInstance.getTaskInstanceId()); // 如果任务应当被驱逐，直接置为被丢弃状态 if (taskEvictPolicyExecutor.shouldEvictTask(taskInstance)) { log.warn("Evict job, taskInstanceId: {}, gseTask: {}", taskInstance.getId(), taskName); - taskEvictPolicyExecutor.updateEvictedTaskStatus(taskInstance, stepInstance); + finishGseTask(gseTask, RunStatusEnum.ABANDONED); watch.stop(); return; } @@ -221,11 +179,7 @@ public void startTask(GseTaskDTO gseTask, String requestId) { if (taskInstance.getStatus() == RunStatusEnum.STOPPING) { log.info("Task instance status is stopping, stop executing the step! taskInstanceId:{}, " + "stepInstanceId:{}", taskInstance.getId(), stepInstance.getId()); - gseTask.setStatus(RunStatusEnum.STOP_SUCCESS.getValue()); - gseTaskService.updateGseTask(gseTask); - taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.refreshStep(stepInstanceId, - EventSource.buildGseTaskEventSource(stepInstanceId, stepInstance.getExecuteCount(), - stepInstance.getBatch(), gseTask.getId()))); + finishGseTask(gseTask, RunStatusEnum.STOP_SUCCESS); watch.stop(); return; } @@ -237,7 +191,7 @@ public void startTask(GseTaskDTO gseTask, String requestId) { watch.start("executeTask"); counter.add(taskName); - executeTask(startCommand, gseTask); + executeTask(startCommand); watch.stop(); success = true; } finally { @@ -251,11 +205,35 @@ public void startTask(GseTaskDTO gseTask, String requestId) { log.warn("GseTaskManager-> start gse task is slow, statistics:{}", watch.prettyPrint()); } executeMonitor.getMeterRegistry().timer(ExecuteMetricNames.EXECUTE_TASK_PREFIX, - "task_type", getTaskTypeDesc(startCommand), "status", success ? "ok" : "error") + "task_type", getTaskTypeDesc(startCommand), "status", success ? "ok" : "error") .record(watch.getTotalTimeNanos(), TimeUnit.NANOSECONDS); } } + private void finishGseTask(GseTaskDTO gseTask, RunStatusEnum status) { + gseTask.setStatus(status.getValue()); + long endTime = System.currentTimeMillis(); + gseTask.setEndTime(endTime); + if (gseTask.getStartTime() == null) { + gseTask.setStartTime(endTime); + } + gseTask.setTotalTime(endTime - gseTask.getStartTime()); + gseTaskService.updateGseTask(gseTask); + taskExecuteMQEventDispatcher.dispatchStepEvent( + StepEvent.refreshStep( + gseTask.getTaskInstanceId(), + gseTask.getStepInstanceId(), + EventSource.buildGseTaskEventSource( + gseTask.getTaskInstanceId(), + gseTask.getStepInstanceId(), + gseTask.getExecuteCount(), + gseTask.getBatch(), + gseTask.getId() + ) + ) + ); + } + private String buildGseTaskLockKey(GseTaskDTO gseTask) { return "job:running:gse:task:" + gseTask.getId(); } @@ -309,83 +287,39 @@ private AbstractGseTaskStartCommand initGseTaskStartCommand(String requestId, TaskInstanceDTO taskInstance, GseTaskDTO gseTask) { AbstractGseTaskStartCommand gseTaskStartCommand = null; - int executeType = stepInstance.getExecuteType(); - if (executeType == StepExecuteTypeEnum.EXECUTE_SCRIPT.getValue()) { + StepExecuteTypeEnum executeType = stepInstance.getExecuteType(); + if (executeType == StepExecuteTypeEnum.EXECUTE_SCRIPT) { gseTaskStartCommand = new ScriptGseTaskStartCommand( - resultHandleManager, - taskInstanceService, - stepInstanceService, - gseTaskService, - scriptAgentTaskService, - accountService, - taskInstanceVariableService, - stepInstanceVariableValueService, - agentService, - logService, - taskExecuteMQEventDispatcher, - resultHandleTaskKeepaliveManager, - executeMonitor, + engineDependentServiceHolder, + scriptExecuteObjectTaskService, jobExecuteConfig, - taskEvictPolicyExecutor, - gseTasksExceptionCounter, - jobBuildInVariableResolver, - tracer, requestId, taskInstance, stepInstance, gseTask ); scriptTaskCounter.incrementAndGet(); - } else if (executeType == StepExecuteTypeEnum.EXECUTE_SQL.getValue()) { + } else if (executeType == StepExecuteTypeEnum.EXECUTE_SQL) { gseTaskStartCommand = new SQLScriptGseTaskStartCommand( - resultHandleManager, - taskInstanceService, - stepInstanceService, - gseTaskService, - scriptAgentTaskService, - accountService, - taskInstanceVariableService, - stepInstanceVariableValueService, - agentService, - logService, - taskExecuteMQEventDispatcher, - resultHandleTaskKeepaliveManager, - executeMonitor, + engineDependentServiceHolder, + scriptExecuteObjectTaskService, jobExecuteConfig, - taskEvictPolicyExecutor, - gseTasksExceptionCounter, - jobBuildInVariableResolver, - tracer, requestId, taskInstance, stepInstance, gseTask ); scriptTaskCounter.incrementAndGet(); - } else if (executeType == TaskStepTypeEnum.FILE.getValue()) { + } else if (executeType == StepExecuteTypeEnum.SEND_FILE) { gseTaskStartCommand = new FileGseTaskStartCommand( - resultHandleManager, - taskInstanceService, - gseTaskService, - fileAgentTaskService, - accountService, - taskInstanceVariableService, - stepInstanceVariableValueService, - agentService, - logService, - taskExecuteMQEventDispatcher, - resultHandleTaskKeepaliveManager, - executeMonitor, + engineDependentServiceHolder, + fileExecuteObjectTaskService, jobExecuteConfig, - taskEvictPolicyExecutor, - gseTasksExceptionCounter, - stepInstanceService, - tracer, requestId, taskInstance, stepInstance, gseTask, - storageSystemConfig.getJobStorageRootPath() + fileDistributeConfig.getJobDistributeRootPath() ); fileTaskCounter.incrementAndGet(); } @@ -398,14 +332,11 @@ private AbstractGseTaskStartCommand initGseTaskStartCommand(String requestId, return gseTaskStartCommand; } - private void executeTask(AbstractGseTaskStartCommand startCommand, - GseTaskDTO gseTask) { + private void executeTask(AbstractGseTaskStartCommand startCommand) { try { - startingGseTasks.put(gseTask.getTaskUniqueName(), startCommand); incrementRunningTasksCount(startCommand); startCommand.execute(); } finally { - startingGseTasks.remove(gseTask.getTaskUniqueName()); decrementRunningTasksCount(startCommand); } } @@ -425,7 +356,8 @@ public void stopTask(GseTaskDTO gseTask) { GseTaskCommand stopCommand; try { watch.start("loadTaskAndCheck"); - StepInstanceDTO stepInstance = taskInstanceService.getStepInstanceDetail(stepInstanceId); + StepInstanceDTO stepInstance = stepInstanceService.getStepInstanceDetail( + gseTask.getTaskInstanceId(), stepInstanceId); TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(stepInstance.getTaskInstanceId()); watch.stop(); @@ -454,11 +386,8 @@ private GseTaskCommand initGseTaskStopCommand(StepInstanceDTO stepInstance, GseTaskCommand gseTaskStopCommand = null; if (stepInstance.isScriptStep()) { gseTaskStopCommand = new ScriptGseTaskStopCommand( - agentService, - accountService, - gseTaskService, - scriptAgentTaskService, - tracer, + engineDependentServiceHolder, + scriptExecuteObjectTaskService, taskInstance, stepInstance, gseTask @@ -466,11 +395,8 @@ private GseTaskCommand initGseTaskStopCommand(StepInstanceDTO stepInstance, scriptTaskCounter.incrementAndGet(); } else if (stepInstance.isFileStep()) { gseTaskStopCommand = new FileGseTaskStopCommand( - agentService, - accountService, - gseTaskService, - fileAgentTaskService, - tracer, + engineDependentServiceHolder, + fileExecuteObjectTaskService, taskInstance, stepInstance, gseTask @@ -534,33 +460,6 @@ public int getPhase() { return DestroyOrder.GSE_TASK_HANDLER; } - /** - * 返回正在执行的任务数量 - * - * @return 任务数量 - */ - public int getRunningTaskCount() { - return this.runningTasks.get(); - } - - /** - * 返回正在执行的文件任务数量 - * - * @return 任务数量 - */ - public int getRunningFileTaskCount() { - return this.runningFileTasks.get(); - } - - /** - * 返回正在执行的脚本任务数量 - * - * @return 任务数量 - */ - public int getRunningScriptTaskCount() { - return this.runningScriptTasks.get(); - } - /** * 返回累计处理的文件任务数量 * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/SQLScriptGseTaskStartCommand.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/SQLScriptGseTaskStartCommand.java index 53a107d1e2..cd102a04ed 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/SQLScriptGseTaskStartCommand.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/SQLScriptGseTaskStartCommand.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,42 +25,21 @@ package com.tencent.bk.job.execute.engine.executor; import com.google.common.collect.Maps; -import com.tencent.bk.gse.taskapi.api_agent; -import com.tencent.bk.gse.taskapi.api_script_request; -import com.tencent.bk.job.common.constant.ErrorCode; -import com.tencent.bk.job.common.exception.InternalException; -import com.tencent.bk.job.common.util.Base64Util; -import com.tencent.bk.job.common.util.crypto.AESUtils; +import com.tencent.bk.job.common.gse.util.ScriptRequestBuilder; +import com.tencent.bk.job.common.gse.v2.model.Agent; +import com.tencent.bk.job.common.gse.v2.model.ExecuteScriptRequest; import com.tencent.bk.job.execute.config.JobExecuteConfig; -import com.tencent.bk.job.execute.engine.evict.TaskEvictPolicyExecutor; -import com.tencent.bk.job.execute.engine.gse.GseRequestUtils; -import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; -import com.tencent.bk.job.execute.engine.model.RunSQLScriptFile; -import com.tencent.bk.job.execute.engine.result.ResultHandleManager; -import com.tencent.bk.job.execute.engine.result.ha.ResultHandleTaskKeepaliveManager; +import com.tencent.bk.job.execute.engine.EngineDependentServiceHolder; import com.tencent.bk.job.execute.engine.util.TimeoutUtils; -import com.tencent.bk.job.execute.engine.variable.JobBuildInVariableResolver; -import com.tencent.bk.job.execute.model.AccountDTO; import com.tencent.bk.job.execute.model.GseTaskDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; -import com.tencent.bk.job.execute.monitor.metrics.ExecuteMonitor; -import com.tencent.bk.job.execute.monitor.metrics.GseTasksExceptionCounter; -import com.tencent.bk.job.execute.service.AccountService; -import com.tencent.bk.job.execute.service.AgentService; -import com.tencent.bk.job.execute.service.GseTaskService; -import com.tencent.bk.job.execute.service.LogService; -import com.tencent.bk.job.execute.service.ScriptAgentTaskService; -import com.tencent.bk.job.execute.service.StepInstanceService; -import com.tencent.bk.job.execute.service.StepInstanceVariableValueService; -import com.tencent.bk.job.execute.service.TaskInstanceService; -import com.tencent.bk.job.execute.service.TaskInstanceVariableService; -import com.tencent.bk.job.manage.common.consts.account.AccountTypeEnum; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; +import com.tencent.bk.job.execute.service.ScriptExecuteObjectTaskService; +import com.tencent.bk.job.manage.api.common.constants.account.AccountTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; -import org.springframework.cloud.sleuth.Tracer; import java.io.InputStream; import java.io.StringWriter; @@ -117,46 +96,17 @@ public class SQLScriptGseTaskStartCommand extends ScriptGseTaskStartCommand { } } - public SQLScriptGseTaskStartCommand(ResultHandleManager resultHandleManager, - TaskInstanceService taskInstanceService, - StepInstanceService stepInstanceService, - GseTaskService gseTaskService, - ScriptAgentTaskService scriptAgentTaskService, - AccountService accountService, - TaskInstanceVariableService taskInstanceVariableService, - StepInstanceVariableValueService stepInstanceVariableValueService, - AgentService agentService, - LogService logService, - TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher, - ResultHandleTaskKeepaliveManager resultHandleTaskKeepaliveManager, - ExecuteMonitor executeMonitor, + public SQLScriptGseTaskStartCommand(EngineDependentServiceHolder engineDependentServiceHolder, + ScriptExecuteObjectTaskService scriptExecuteObjectTaskService, JobExecuteConfig jobExecuteConfig, - TaskEvictPolicyExecutor taskEvictPolicyExecutor, - GseTasksExceptionCounter gseTasksExceptionCounter, - JobBuildInVariableResolver jobBuildInVariableResolver, - Tracer tracer, String requestId, TaskInstanceDTO taskInstance, StepInstanceDTO stepInstance, GseTaskDTO gseTask) { - super(resultHandleManager, - taskInstanceService, - stepInstanceService, - gseTaskService, - scriptAgentTaskService, - accountService, - taskInstanceVariableService, - stepInstanceVariableValueService, - agentService, - logService, - taskExecuteMQEventDispatcher, - resultHandleTaskKeepaliveManager, - executeMonitor, + super( + engineDependentServiceHolder, + scriptExecuteObjectTaskService, jobExecuteConfig, - taskEvictPolicyExecutor, - gseTasksExceptionCounter, - jobBuildInVariableResolver, - tracer, requestId, taskInstance, stepInstance, @@ -164,51 +114,43 @@ public SQLScriptGseTaskStartCommand(ResultHandleManager resultHandleManager, } @Override - protected api_script_request buildScriptRequest() { + protected ExecuteScriptRequest buildScriptRequest() { String sqlScriptContent = stepInstance.getScriptContent(); String sqlScriptFileName = buildScriptFileName(stepInstance); String publicScriptContent = sqlMap.get(stepInstance.getDbType()); int timeout = TimeoutUtils.adjustTaskTimeout(stepInstance.getTimeout()); - String publicScriptName = this.scriptFileNamePrefix - + ScriptTypeEnum.getExtByValue(ScriptTypeEnum.SHELL.getValue()); + String publicScriptName = this.scriptFileNamePrefix + ScriptTypeEnum.SHELL.getExt(); - RunSQLScriptFile param = new RunSQLScriptFile(); - param.setTimeout(timeout); - param.setDownloadPath(scriptFilePath); - param.setPublicScriptContent(publicScriptContent); - param.setPublicScriptName(publicScriptName); - param.setSqlScriptContent(sqlScriptContent); - param.setSqlScriptFileName(sqlScriptFileName); - StringBuilder sqlParam = new StringBuilder(255); - sqlParam.append(stepInstance.getDbPort()); + ScriptRequestBuilder builder = new ScriptRequestBuilder(); + builder.addScriptFile(scriptFilePath, publicScriptName, publicScriptContent); + builder.addScriptFile(scriptFilePath, sqlScriptFileName, sqlScriptContent); + + List agentList = buildTargetAgents(); + + builder.addScriptTask(agentList, scriptFilePath, publicScriptName, buildRunSqlShellParams(sqlScriptFileName), + timeout); + + ExecuteScriptRequest request = builder.build(); + request.setGseV2Task(gseV2Task); + return request; + } + + private String buildRunSqlShellParams(String sqlScriptFileName) { + StringBuilder sb = new StringBuilder(255); + sb.append(stepInstance.getDbPort()); if (StringUtils.isNotBlank(stepInstance.getDbAccount()) && !StringUtils.equals("null", stepInstance.getDbAccount())) { - sqlParam.append(" ").append(stepInstance.getDbAccount()); + sb.append(" ").append(stepInstance.getDbAccount()); } else { - sqlParam.append(" EMPTY"); + sb.append(" EMPTY"); } if (StringUtils.isNotBlank(stepInstance.getDbPass()) && !StringUtils.equals("null", stepInstance.getDbPass())) { - String dbPassword; - try { - dbPassword = AESUtils.decryptToPlainText(Base64Util.decodeContentToByte(stepInstance.getDbPass()), - jobExecuteConfig.getEncryptPassword()); - } catch (Exception e) { - log.error("Decrypt db password failed!", e); - throw new InternalException(ErrorCode.INTERNAL_ERROR); - } - sqlParam.append(" ").append(dbPassword); + sb.append(" ").append(stepInstance.getDbPass()); } else { - sqlParam.append(" EMPTY"); + sb.append(" EMPTY"); } - param.setParamForDBInfo(sqlParam.toString()); - - AccountDTO accountInfo = getAccountBean(stepInstance.getAccountId(), stepInstance.getAccount(), - stepInstance.getAppId()); - - List agentList = GseRequestUtils.buildAgentList(targetAgentTaskMap.keySet(), - accountInfo.getAccount(), accountInfo.getPassword()); - - return GseRequestUtils.buildScriptRequestWithSQL(agentList, param); + return scriptFilePath + "/" + sqlScriptFileName + " " + sb.toString(); } + } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/ScriptGseTaskStartCommand.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/ScriptGseTaskStartCommand.java index d1a50499ed..d067ff9ebd 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/ScriptGseTaskStartCommand.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/ScriptGseTaskStartCommand.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,26 +24,23 @@ package com.tencent.bk.job.execute.engine.executor; -import com.tencent.bk.gse.taskapi.api_agent; -import com.tencent.bk.gse.taskapi.api_script_request; import com.tencent.bk.job.common.constant.TaskVariableTypeEnum; +import com.tencent.bk.job.common.gse.util.ScriptRequestBuilder; +import com.tencent.bk.job.common.gse.v2.model.Agent; +import com.tencent.bk.job.common.gse.v2.model.ExecuteScriptRequest; +import com.tencent.bk.job.common.gse.v2.model.GseTaskResponse; import com.tencent.bk.job.common.service.VariableResolver; import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; import com.tencent.bk.job.execute.common.util.TaskCostCalculator; import com.tencent.bk.job.execute.common.util.VariableValueResolver; import com.tencent.bk.job.execute.config.JobExecuteConfig; -import com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum; -import com.tencent.bk.job.execute.engine.evict.TaskEvictPolicyExecutor; -import com.tencent.bk.job.execute.engine.gse.GseRequestUtils; -import com.tencent.bk.job.execute.engine.gse.ScriptRequestBuilder; -import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; -import com.tencent.bk.job.execute.engine.model.GseTaskResponse; +import com.tencent.bk.job.execute.engine.EngineDependentServiceHolder; +import com.tencent.bk.job.execute.engine.consts.ExecuteObjectTaskStatusEnum; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; import com.tencent.bk.job.execute.engine.model.TaskVariableDTO; import com.tencent.bk.job.execute.engine.model.TaskVariablesAnalyzeResult; -import com.tencent.bk.job.execute.engine.result.ResultHandleManager; import com.tencent.bk.job.execute.engine.result.ScriptResultHandleTask; -import com.tencent.bk.job.execute.engine.result.ha.ResultHandleTaskKeepaliveManager; import com.tencent.bk.job.execute.engine.util.MacroUtil; import com.tencent.bk.job.execute.engine.util.TimeoutUtils; import com.tencent.bk.job.execute.engine.variable.JobBuildInVariableResolver; @@ -51,40 +48,30 @@ import com.tencent.bk.job.execute.engine.variable.VariableResolveResult; import com.tencent.bk.job.execute.engine.variable.VariableResolveUtils; import com.tencent.bk.job.execute.model.AccountDTO; -import com.tencent.bk.job.execute.model.AgentTaskDTO; +import com.tencent.bk.job.execute.model.ExecuteObjectTask; import com.tencent.bk.job.execute.model.GseTaskDTO; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.model.VariableValueDTO; -import com.tencent.bk.job.execute.monitor.metrics.ExecuteMonitor; -import com.tencent.bk.job.execute.monitor.metrics.GseTasksExceptionCounter; -import com.tencent.bk.job.execute.service.AccountService; -import com.tencent.bk.job.execute.service.AgentService; -import com.tencent.bk.job.execute.service.GseTaskService; -import com.tencent.bk.job.execute.service.LogService; -import com.tencent.bk.job.execute.service.ScriptAgentTaskService; -import com.tencent.bk.job.execute.service.StepInstanceService; -import com.tencent.bk.job.execute.service.StepInstanceVariableValueService; -import com.tencent.bk.job.execute.service.TaskInstanceService; -import com.tencent.bk.job.execute.service.TaskInstanceVariableService; -import com.tencent.bk.job.logsvr.model.service.ServiceScriptLogDTO; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; +import com.tencent.bk.job.execute.service.ScriptExecuteObjectTaskService; +import com.tencent.bk.job.logsvr.model.service.ServiceExecuteObjectScriptLogDTO; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; -import org.springframework.cloud.sleuth.Tracer; import java.io.File; import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; @Slf4j public class ScriptGseTaskStartCommand extends AbstractGseTaskStartCommand { - private final ScriptAgentTaskService scriptAgentTaskService; + private final ScriptExecuteObjectTaskService scriptExecuteObjectTaskService; private final JobBuildInVariableResolver jobBuildInVariableResolver; @@ -99,51 +86,45 @@ public class ScriptGseTaskStartCommand extends AbstractGseTaskStartCommand { private final String GSE_SCRIPT_FILE_NAME_PREFIX = "bk_gse_script_"; - public ScriptGseTaskStartCommand(ResultHandleManager resultHandleManager, - TaskInstanceService taskInstanceService, - StepInstanceService stepInstanceService, - GseTaskService gseTaskService, - ScriptAgentTaskService scriptAgentTaskService, - AccountService accountService, - TaskInstanceVariableService taskInstanceVariableService, - StepInstanceVariableValueService stepInstanceVariableValueService, - AgentService agentService, - LogService logService, - TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher, - ResultHandleTaskKeepaliveManager resultHandleTaskKeepaliveManager, - ExecuteMonitor executeMonitor, + /** + * 脚本默认的解释器声明 + */ + private static final String DEFAULT_SHEBANG = "#!/bin/bash"; + + /** + * 从用户脚本提取的解释器声明 + */ + private String extractedShebang; + + public ScriptGseTaskStartCommand(EngineDependentServiceHolder engineDependentServiceHolder, + ScriptExecuteObjectTaskService scriptExecuteObjectTaskService, JobExecuteConfig jobExecuteConfig, - TaskEvictPolicyExecutor taskEvictPolicyExecutor, - GseTasksExceptionCounter gseTasksExceptionCounter, - JobBuildInVariableResolver jobBuildInVariableResolver, - Tracer tracer, String requestId, TaskInstanceDTO taskInstance, StepInstanceDTO stepInstance, GseTaskDTO gseTask) { - super(resultHandleManager, - taskInstanceService, - gseTaskService, - scriptAgentTaskService, - accountService, - taskInstanceVariableService, - stepInstanceVariableValueService, - agentService, - logService, - taskExecuteMQEventDispatcher, - resultHandleTaskKeepaliveManager, - executeMonitor, + super(engineDependentServiceHolder, + scriptExecuteObjectTaskService, jobExecuteConfig, - taskEvictPolicyExecutor, - gseTasksExceptionCounter, - tracer, requestId, taskInstance, stepInstance, - gseTask, stepInstanceService); - this.scriptAgentTaskService = scriptAgentTaskService; - this.jobBuildInVariableResolver = jobBuildInVariableResolver; + gseTask); + this.scriptExecuteObjectTaskService = scriptExecuteObjectTaskService; + this.jobBuildInVariableResolver = engineDependentServiceHolder.getJobBuildInVariableResolver(); this.scriptFileNamePrefix = buildScriptFileNamePrefix(stepInstance); + this.extractedShebang = extractShebang(stepInstance.getScriptContent()); + } + + /** + * 提取用户脚本中的解释器申明 + */ + private String extractShebang(String scriptContent) { + if (StringUtils.isEmpty(scriptContent)) { + return DEFAULT_SHEBANG; + } + String firstLine = scriptContent.split("\\r?\\n", 2)[0].trim(); + return firstLine.startsWith("#!") ? firstLine : DEFAULT_SHEBANG; } private String buildScriptFileNamePrefix(StepInstanceDTO stepInstance) { @@ -153,7 +134,8 @@ private String buildScriptFileNamePrefix(StepInstanceDTO stepInstance) { } @Override - protected void preExecute() { + protected void initExecutionContext() { + super.initExecutionContext(); scriptFilePath = buildScriptFilePath(jobExecuteConfig.getGseScriptFileRootPath(), stepInstance.getAccount()); } @@ -166,73 +148,38 @@ private String buildScriptFilePath(String gseScriptFileRootPath, String account) } } - @Override protected GseTaskResponse startGseTask() { - return GseRequestUtils.sendScriptTaskRequest(stepInstanceId, buildScriptRequest()); + return gseClient.asyncExecuteScript(buildScriptRequest()); } - protected api_script_request buildScriptRequest() { - api_script_request request; + protected ExecuteScriptRequest buildScriptRequest() { + ExecuteScriptRequest request; // shell 脚本需要支持全局变量传参，需要特殊的处理逻辑 - if (stepInstance.getScriptType().equals(ScriptTypeEnum.SHELL.getValue())) { - request = buildShellScriptRequest(); - } else { - request = buildNonShellScriptRequest(); - } - request.setM_caller(buildGSETraceInfo()); - - return request; - } - - private api_script_request buildNonShellScriptRequest() { - String scriptContent = stepInstance.getScriptContent(); - String scriptFileName = buildScriptFileName(stepInstance); - - String scriptParam = MacroUtil.resolveDateWithStrfTime(stepInstance.getScriptParam()); - String resolvedScriptParam = resolveScriptParamVariables(scriptParam); - int timeout = TimeoutUtils.adjustTaskTimeout(stepInstance.getTimeout()); - - AccountDTO accountInfo = getAccountBean(stepInstance.getAccountId(), stepInstance.getAccount(), - stepInstance.getAppId()); - List agentList = GseRequestUtils.buildAgentList(targetAgentTaskMap.keySet(), - accountInfo.getAccount(), accountInfo.getPassword()); - api_script_request request = GseRequestUtils.buildScriptRequest(agentList, scriptContent, scriptFileName, - scriptFilePath, resolvedScriptParam, timeout); - request.setM_caller(buildGSETraceInfo()); - - return request; - } - - - protected String buildScriptFileName(StepInstanceDTO stepInstance) { - return this.scriptFileNamePrefix + ScriptTypeEnum.getExtByValue(stepInstance.getScriptType()); - } - - /** - * 构建shell脚本下发的请求 - */ - private api_script_request buildShellScriptRequest() { - api_script_request request; - if (taskInstance.isPlanInstance()) { + if (stepInstance.getScriptType() == ScriptTypeEnum.SHELL + && taskInstance.isPlanInstance()) { // 执行方案脚本执行步骤，需要处理变量 request = buildShellScriptRequestForPlan(); } else { - // 快速执行脚本 request = buildRequestWithoutAnyParam(stepInstance); } + request.setGseV2Task(gseV2Task); return request; } - private api_script_request buildShellScriptRequestForPlan() { - api_script_request request; + protected String buildScriptFileName(StepInstanceDTO stepInstance) { + return this.scriptFileNamePrefix + stepInstance.getScriptType().getExt(); + } + + private ExecuteScriptRequest buildShellScriptRequestForPlan() { + ExecuteScriptRequest request; boolean containsAnyImportedVariable = false; List importVariables = null; if (shouldParseBuildInVariables(stepInstance)) { importVariables = VariableResolver.resolveJobImportVariables(stepInstance.getScriptContent()); if (CollectionUtils.isNotEmpty(importVariables)) { - log.info("Parse imported variables, stepInstanceId:{}, variables: {}", stepInstance.getId(), + log.info("Parse imported attributes, stepInstanceId:{}, attributes: {}", stepInstance.getId(), importVariables); containsAnyImportedVariable = true; } @@ -275,14 +222,19 @@ private String resolveScriptParamVariables(String scriptParam) { private void updateResolvedScriptParamIfNecessary(String originParam, String resolvedScriptParam) { // 只有存在变量解析之后才需要update if (!resolvedScriptParam.equals(originParam)) { - taskInstanceService.updateResolvedScriptParam(stepInstance.getId(), resolvedScriptParam); + stepInstanceService.updateResolvedScriptParam( + stepInstance.getTaskInstanceId(), + stepInstance.getId(), + stepInstance.isSecureParam(), + resolvedScriptParam + ); } } /** * 创建下发请求-不带任何全局参数 */ - private api_script_request buildRequestWithoutAnyParam(StepInstanceDTO stepInstance) { + private ExecuteScriptRequest buildRequestWithoutAnyParam(StepInstanceDTO stepInstance) { String scriptContent = stepInstance.getScriptContent(); String scriptFileName = buildScriptFileName(stepInstance); @@ -293,21 +245,29 @@ private api_script_request buildRequestWithoutAnyParam(StepInstanceDTO stepInsta ScriptRequestBuilder builder = new ScriptRequestBuilder(); builder.addScriptFile(scriptFilePath, scriptFileName, scriptContent); - AccountDTO accountInfo = getAccountBean(stepInstance.getAccountId(), stepInstance.getAccount(), - stepInstance.getAppId()); - List agentList = GseRequestUtils.buildAgentList(targetAgentTaskMap.keySet(), - accountInfo.getAccount(), accountInfo.getPassword()); + List agents = buildTargetAgents(); - builder.addScriptTask(agentList, scriptFilePath, scriptFileName, resolvedScriptParam, timeout); + builder.addScriptTask(agents, scriptFilePath, scriptFileName, resolvedScriptParam, timeout); + builder.setWindowsInterpreter(stepInstance.getWindowsInterpreter()); return builder.build(); } + protected List buildTargetAgents() { + AccountDTO account = getAccountBean(stepInstance.getAccountId(), stepInstance.getAccount(), + stepInstance.getAppId()); + return gseClient.fillAgentAuthInfo( + targetExecuteObjectTaskMap.values().stream().map(executeObjectTask -> + executeObjectTask.getExecuteObject().toGseAgent()).collect(Collectors.toList()), + account.getAccount(), + account.getPassword()); + } + /** * 创建下发请求-仅包含常量 */ - private api_script_request buildRequestWithConstParamOnly(StepInstanceDTO stepInstance, - List taskVars, - List importVariables) { + private ExecuteScriptRequest buildRequestWithConstParamOnly(StepInstanceDTO stepInstance, + List taskVars, + List importVariables) { ScriptRequestBuilder builder = new ScriptRequestBuilder(); //用户原始脚本 String userScriptContent = stepInstance.getScriptContent(); @@ -328,17 +288,16 @@ private api_script_request buildRequestWithConstParamOnly(StepInstanceDTO stepIn String resolvedScriptParam = resolveScriptParamVariables(scriptParam); int timeout = TimeoutUtils.adjustTaskTimeout(stepInstance.getTimeout()); - AccountDTO accountInfo = getAccountBean(stepInstance.getAccountId(), stepInstance.getAccount(), - stepInstance.getAppId()); - List agentList = GseRequestUtils.buildAgentList(targetAgentTaskMap.keySet(), - accountInfo.getAccount(), accountInfo.getPassword()); - builder.addScriptTask(agentList, scriptFilePath, wrapperScriptFileName, resolvedScriptParam, timeout); + List agents = buildTargetAgents(); + + builder.addScriptTask(agents, scriptFilePath, wrapperScriptFileName, resolvedScriptParam, timeout); + builder.setWindowsInterpreter(stepInstance.getWindowsInterpreter()); return builder.build(); } private String buildConstVarDeclareScript(List taskVars, List importVariables) { StringBuffer sb = new StringBuffer(1024); - sb.append("#!/bin/bash\n"); + sb.append(extractedShebang).append("\n"); sb.append("set -e\n"); for (TaskVariableDTO taskVar : taskVars) { buildDeclareScript(taskVar, sb); @@ -391,9 +350,10 @@ private void appendImportVariablesDeclareScript(StringBuffer sb, List importVariables) { + private ExecuteScriptRequest buildRequestWithChangeableParam(StepInstanceDTO stepInstance, + TaskVariablesAnalyzeResult taskVariablesAnalyzeResult, + List importVariables) { String scriptParam = MacroUtil.resolveDateWithStrfTime(stepInstance.getScriptParam()); String resolvedScriptParam = resolveScriptParamVariables(scriptParam); int timeout = TimeoutUtils.adjustTaskTimeout(stepInstance.getTimeout()); - AccountDTO accountInfo = getAccountBean(stepInstance.getAccountId(), stepInstance.getAccount(), - stepInstance.getAppId()); - List agentList = GseRequestUtils.buildAgentList(targetAgentTaskMap.keySet(), - accountInfo.getAccount(), accountInfo.getPassword()); + List agents = buildTargetAgents(); ScriptRequestBuilder builder = new ScriptRequestBuilder(); @@ -483,8 +441,9 @@ private api_script_request buildRequestWithChangeableParam(StepInstanceDTO stepI builder.addScriptFile(scriptFilePath, wrapperScriptFileName, wrapperScriptContent); builder.addScriptFile(scriptFilePath, getJobParamScriptFileName, getJobParamScriptContent); - builder.addScriptTask(agentList, scriptFilePath, wrapperScriptFileName, resolvedScriptParam, timeout); - builder.addScriptTask(agentList, scriptFilePath, getJobParamScriptFileName, null, timeout); + builder.addScriptTask(agents, scriptFilePath, wrapperScriptFileName, resolvedScriptParam, timeout); + builder.addScriptTask(agents, scriptFilePath, getJobParamScriptFileName, null, timeout); + builder.setWindowsInterpreter(stepInstance.getWindowsInterpreter()); return builder.build(); } @@ -499,7 +458,7 @@ private String buildChangeableVarDeclareScript(TaskVariablesAnalyzeResult taskVa List importVariables) { List globalVars = taskVariablesAnalyzeResult.getTaskVars(); StringBuffer sb = new StringBuffer(1024); - sb.append("#!/bin/bash\n"); + sb.append(extractedShebang).append("\n"); sb.append("set -e\n"); //从作业参数中初始化输入参数 @@ -566,7 +525,7 @@ private String buildWrapperScriptWhenExistChangeableVar(TaskVariablesAnalyzeResu String allParamsOutputFileName, String scriptParam) { StringBuilder sb = new StringBuilder(1024); - sb.append("#!/bin/bash\n"); + sb.append(extractedShebang).append("\n"); sb.append("BASE_PATH=\"\"\n"); sb.append("OS_TYPE=`uname -s`\n"); sb.append("if [ \"`echo ${OS_TYPE}|grep -i 'CYGWIN'`\" ];then\n"); @@ -634,9 +593,10 @@ private String buildOutputVarsOnExitFunction(TaskVariablesAnalyzeResult taskVari return sb.toString(); } + @SuppressWarnings("StringBufferReplaceableByString") private String buildGetJobParamsScript(String varOutputFileName) { StringBuilder sb = new StringBuilder(1024); - sb.append("#!/bin/bash\n"); + sb.append(extractedShebang).append("\n"); sb.append("BASE_PATH=\"\"\n"); sb.append("OS_TYPE=`uname -s`\n"); sb.append("if [ \"`echo ${OS_TYPE}|grep -i 'CYGWIN'`\" ];then\n"); @@ -666,22 +626,16 @@ private String getNamespaceParamsOutputFilePath() { protected final void addResultHandleTask() { ScriptResultHandleTask scriptResultHandleTask = new ScriptResultHandleTask( - taskInstanceService, - gseTaskService, - logService, - taskInstanceVariableService, - stepInstanceVariableValueService, - taskExecuteMQEventDispatcher, - resultHandleTaskKeepaliveManager, - taskEvictPolicyExecutor, - scriptAgentTaskService, - stepInstanceService, + engineDependentServiceHolder, + scriptExecuteObjectTaskService, + jobExecuteConfig, taskInstance, stepInstance, taskVariablesAnalyzeResult, - targetAgentTaskMap, + targetExecuteObjectTaskMap, gseTask, - requestId); + requestId, + targetExecuteObjectTasks); resultHandleManager.handleDeliveredTask(scriptResultHandleTask); } @@ -689,26 +643,40 @@ protected final void addResultHandleTask() { @Override protected final void handleStartGseTaskError(GseTaskResponse gseTaskResponse) { long now = DateUtils.currentTimeMillis(); - updateGseTaskExecutionInfo(null, RunStatusEnum.FAIL, null, now, now - gseTask.getStartTime()); + updateGseTaskExecutionInfo(null, RunStatusEnum.FAIL, now); String errorMsg = "GSE Job failed:" + gseTaskResponse.getErrorMessage(); int errorMsgLength = errorMsg.length(); - List scriptLogs = new ArrayList<>(targetAgentTaskMap.size()); - for (AgentTaskDTO agentTask : targetAgentTaskMap.values()) { + List scriptLogs = new ArrayList<>(targetExecuteObjectTaskMap.size()); + for (ExecuteObjectTask executeObjectTask : targetExecuteObjectTaskMap.values()) { + ExecuteObject executeObject = executeObjectTask.getExecuteObject(); // 日志输出 - ServiceScriptLogDTO scriptLog = logService.buildSystemScriptLog(agentTask.getHost(), errorMsg, - agentTask.getScriptLogOffset() + errorMsgLength, now); + ServiceExecuteObjectScriptLogDTO scriptLog = logService.buildSystemScriptLog( + stepInstance, + executeObject, + errorMsg, + executeObjectTask.getScriptLogOffset() + errorMsgLength, + now); scriptLogs.add(scriptLog); - // AgentTask 结果更新 - agentTask.setGseTaskId(gseTask.getId()); - agentTask.setStartTime(gseTask.getStartTime()); - agentTask.setEndTime(now); - agentTask.setTotalTime(TaskCostCalculator.calculate(gseTask.getStartTime(), now, null)); - agentTask.setStatus(AgentTaskStatusEnum.SUBMIT_FAILED); + // ExecuteObjectTask 结果更新 + executeObjectTask.setGseTaskId(gseTask.getId()); + executeObjectTask.setStartTime(gseTask.getStartTime()); + executeObjectTask.setEndTime(now); + executeObjectTask.setTotalTime(TaskCostCalculator.calculate(gseTask.getStartTime(), now, null)); + executeObjectTask.setStatus(ExecuteObjectTaskStatusEnum.SUBMIT_FAILED); + } + logService.batchWriteScriptLog(taskInstance, stepInstanceId, executeCount, batch, scriptLogs); + scriptExecuteObjectTaskService.batchUpdateTasks(targetExecuteObjectTaskMap.values()); + } + + @Override + protected boolean checkGseTaskExecutable() { + if (this.targetExecuteObjectTaskMap.isEmpty()) { + log.warn("Script gse task target agent is empty, can not execute! Set gse task status fail"); + return false; } - logService.batchWriteScriptLog(taskInstance.getCreateTime(), stepInstanceId, executeCount, batch, scriptLogs); - scriptAgentTaskService.batchUpdateAgentTasks(targetAgentTaskMap.values()); + return true; } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/ScriptGseTaskStopCommand.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/ScriptGseTaskStopCommand.java index 497fa07f98..e8c4f7c2c8 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/ScriptGseTaskStopCommand.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/executor/ScriptGseTaskStopCommand.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,44 +24,32 @@ package com.tencent.bk.job.execute.engine.executor; -import com.tencent.bk.gse.taskapi.api_agent; -import com.tencent.bk.gse.taskapi.api_stop_task_request; +import com.tencent.bk.job.common.gse.v2.model.Agent; +import com.tencent.bk.job.common.gse.v2.model.GseTaskResponse; +import com.tencent.bk.job.common.gse.v2.model.TerminateGseTaskRequest; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; -import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; -import com.tencent.bk.job.execute.engine.gse.GseRequestUtils; -import com.tencent.bk.job.execute.engine.model.GseTaskResponse; -import com.tencent.bk.job.execute.model.AccountDTO; -import com.tencent.bk.job.execute.model.AgentTaskDTO; +import com.tencent.bk.job.execute.engine.EngineDependentServiceHolder; +import com.tencent.bk.job.execute.model.ExecuteObjectTask; import com.tencent.bk.job.execute.model.GseTaskDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; -import com.tencent.bk.job.execute.service.AccountService; -import com.tencent.bk.job.execute.service.AgentService; -import com.tencent.bk.job.execute.service.AgentTaskService; -import com.tencent.bk.job.execute.service.GseTaskService; +import com.tencent.bk.job.execute.service.ExecuteObjectTaskService; import lombok.extern.slf4j.Slf4j; -import org.springframework.cloud.sleuth.Tracer; +import org.apache.commons.lang3.StringUtils; import java.util.List; -import java.util.Set; import java.util.stream.Collectors; @Slf4j public class ScriptGseTaskStopCommand extends AbstractGseTaskCommand { - public ScriptGseTaskStopCommand(AgentService agentService, - AccountService accountService, - GseTaskService gseTaskService, - AgentTaskService agentTaskService, - Tracer tracer, + public ScriptGseTaskStopCommand(EngineDependentServiceHolder engineDependentServiceHolder, + ExecuteObjectTaskService executeObjectTaskService, TaskInstanceDTO taskInstance, StepInstanceDTO stepInstance, GseTaskDTO gseTask) { - super(agentService, - accountService, - gseTaskService, - agentTaskService, - tracer, + super(engineDependentServiceHolder, + executeObjectTaskService, taskInstance, stepInstance, gseTask); @@ -69,26 +57,21 @@ public ScriptGseTaskStopCommand(AgentService agentService, @Override public void execute() { - log.info("Stop gse task, gseTask:" + gseTaskUniqueName); - List agentTasks = agentTaskService.listAgentTasksByGseTaskId(gseTask.getId()); - Set terminateAgentIds = agentTasks.stream() - .map(AgentTaskDTO::getAgentId) - .collect(Collectors.toSet()); + log.info("Stop gse script task, gseTask:" + gseTaskInfo); + List executeObjectTasks = + executeObjectTaskService.listTasksByGseTaskId(stepInstance, gseTask.getId()); + List terminateAgents = executeObjectTasks.stream() + .map(executeObjectTask -> executeObjectTask.getExecuteObject().toGseAgent()) + .filter(agent -> agent != null && StringUtils.isNotEmpty(agent.getAgentId())) + .distinct() + .collect(Collectors.toList()); - AccountDTO accountInfo = getAccountBean(stepInstance.getAccountId(), stepInstance.getAccount(), - stepInstance.getAppId()); - List agentList = GseRequestUtils.buildAgentList(terminateAgentIds, accountInfo.getAccount(), - accountInfo.getPassword()); - api_stop_task_request stopTaskRequest = new api_stop_task_request(); - stopTaskRequest.setStop_task_id(gseTask.getGseTaskId()); - stopTaskRequest.setAgents(agentList); - stopTaskRequest.setType(StepExecuteTypeEnum.EXECUTE_SCRIPT.getValue()); - stopTaskRequest.setM_caller(buildGSETraceInfo()); - GseTaskResponse gseTaskResponse = GseRequestUtils.sendForceStopTaskRequest(stepInstance.getId(), - stopTaskRequest); + TerminateGseTaskRequest request = new TerminateGseTaskRequest(gseTask.getGseTaskId(), + terminateAgents, gseV2Task); + GseTaskResponse gseTaskResponse = gseClient.terminateGseScriptTask(request); if (GseTaskResponse.ERROR_CODE_SUCCESS != gseTaskResponse.getErrorCode()) { - log.error("Terminate gse task failed! gseTask: {}", gseTaskUniqueName); + log.error("Terminate gse task failed! gseTask: {}", gseTaskInfo); } else { log.info("Terminate gse task response success!"); gseTask.setStatus(RunStatusEnum.STOPPING.getValue()); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/gse/BKTSSLTransportFactory.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/gse/BKTSSLTransportFactory.java deleted file mode 100644 index 624a98edd5..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/gse/BKTSSLTransportFactory.java +++ /dev/null @@ -1,289 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.engine.gse; - -import org.apache.thrift.transport.TServerSocket; -import org.apache.thrift.transport.TSocket; -import org.apache.thrift.transport.TTransportException; - -import javax.net.ssl.*; -import java.io.FileInputStream; -import java.io.IOException; -import java.net.InetAddress; -import java.security.KeyStore; -import java.security.SecureRandom; -import java.security.cert.X509Certificate; -import java.util.Arrays; - -/** - * 针对Thrift的SSL做免TrustManager验证，主要是因为centos 7.x以上系统，可能存在认证异常 - * Caused by: sun.security.validator.ValidatorException: KeyUsage does not allow key encipherment - * at sun.security.validator.EndEntityChecker.checkTLSServer(EndEntityChecker.java:264) - * at sun.security.validator.EndEntityChecker.check(EndEntityChecker.java:141) - * at sun.security.validator.Validator.validate(Validator.java:264) - * at sun.security.ssl.X509TrustManagerImpl.validate(X509TrustManagerImpl.java:324) - * at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:229) - * at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:124) - * at sun.security.ssl.ClientHandshaker.serverCertificate(ClientHandshaker.java:1496) - * - * @version 1.0 - * @time 2017/6/19. - */ -public class BKTSSLTransportFactory { - private final static X509TrustManager trustManager = new X509TrustManager() { - public void checkClientTrusted(X509Certificate[] xcs, String string) { - } - - public void checkServerTrusted(X509Certificate[] xcs, String string) { - } - - public X509Certificate[] getAcceptedIssuers() { - return null; - } - }; - - public BKTSSLTransportFactory() { - } - - public static TServerSocket getServerSocket(int port) throws TTransportException { - return getServerSocket(port, 0); - } - - public static TServerSocket getServerSocket(int port, int clientTimeout) throws TTransportException { - return getServerSocket(port, clientTimeout, false, (InetAddress) null); - } - - public static TServerSocket getServerSocket(int port, int clientTimeout, boolean clientAuth, - InetAddress ifAddress) throws TTransportException { - SSLServerSocketFactory factory = (SSLServerSocketFactory) SSLServerSocketFactory.getDefault(); - return createServer(factory, port, clientTimeout, clientAuth, ifAddress, null); - } - - public static TServerSocket getServerSocket(int port, int clientTimeout, InetAddress ifAddress, - TSSLTransportParameters params) throws TTransportException { - if (params != null && (params.isKeyStoreSet || params.isTrustStoreSet)) { - SSLContext ctx = createSSLContext(params); - return createServer(ctx.getServerSocketFactory(), port, clientTimeout, params.clientAuth, ifAddress, - params); - } else { - throw new TTransportException("Either one of the KeyStore or TrustStore must be set for " + - "SSLTransportParameters"); - } - } - - private static TServerSocket createServer(SSLServerSocketFactory factory, int port, int timeout, - boolean clientAuth, InetAddress ifAddress, - TSSLTransportParameters params) throws TTransportException { - try { - SSLServerSocket serverSocket = (SSLServerSocket) factory.createServerSocket(port, 100, ifAddress); - serverSocket.setSoTimeout(timeout); - serverSocket.setNeedClientAuth(clientAuth); - if (params != null && params.cipherSuites != null) { - serverSocket.setEnabledCipherSuites(params.cipherSuites); - } - - return new TServerSocket((new TServerSocket.ServerSocketTransportArgs()) - .serverSocket(serverSocket).clientTimeout(timeout)); - } catch (Exception var7) { - throw new TTransportException("Could not bind to port " + port, var7); - } - } - - public static TSocket getClientSocket(String host, int port, int timeout) throws TTransportException { - SSLSocketFactory factory = (SSLSocketFactory) SSLSocketFactory.getDefault(); - return createClient(factory, host, port, timeout); - } - - public static TSocket getClientSocket(String host, int port) throws TTransportException { - return getClientSocket(host, port, 0); - } - - public static TSocket getClientSocket(String host, int port, int timeout, - TSSLTransportParameters params) throws TTransportException { - if (params != null && (params.isKeyStoreSet || params.isTrustStoreSet)) { - SSLContext ctx = createSSLContext(params); - return createClient(ctx.getSocketFactory(), host, port, timeout); - } else { - throw new TTransportException("Either one of the KeyStore or TrustStore must be set for " + - "SSLTransportParameters"); - } - } - - private static SSLContext createSSLContext(TSSLTransportParameters params) throws TTransportException { - FileInputStream fin = null; - FileInputStream fis = null; - - SSLContext ctx; - try { - ctx = SSLContext.getInstance(params.protocol); - TrustManagerFactory tmf = null; - KeyManagerFactory kmf = null; - KeyStore ks; - if (params.isTrustStoreSet) { - tmf = TrustManagerFactory.getInstance(params.trustManagerType); - ks = KeyStore.getInstance(params.trustStoreType); - fin = new FileInputStream(params.trustStore); - ks.load(fin, params.trustPass != null ? params.trustPass.toCharArray() : null); - tmf.init(ks); - } - - if (params.isKeyStoreSet) { - kmf = KeyManagerFactory.getInstance(params.keyManagerType); - ks = KeyStore.getInstance(params.keyStoreType); - fis = new FileInputStream(params.keyStore); - ks.load(fis, params.keyPass.toCharArray()); - kmf.init(ks, params.keyPass.toCharArray()); - } - - if (params.isKeyStoreSet && params.isTrustStoreSet) { - ctx.init(kmf.getKeyManagers(), new TrustManager[]{trustManager}, (SecureRandom) null); - } else if (params.isKeyStoreSet) { - ctx.init(kmf.getKeyManagers(), (TrustManager[]) null, (SecureRandom) null); - } else { - ctx.init((KeyManager[]) null, new TrustManager[]{trustManager}, (SecureRandom) null); - } - } catch (Exception var17) { - throw new TTransportException("Error creating the transport", var17); - } finally { - if (fin != null) { - try { - fin.close(); - } catch (IOException var16) { - var16.printStackTrace(); - } - } - - if (fis != null) { - try { - fis.close(); - } catch (IOException var15) { - var15.printStackTrace(); - } - } - - } - - return ctx; - } - - private static TSocket createClient(SSLSocketFactory factory, String host, int port, - int timeout) throws TTransportException { - try { - SSLSocket socket = (SSLSocket) factory.createSocket(host, port); - socket.setSoTimeout(timeout); - return new TSocket(socket); - } catch (Exception var5) { - throw new TTransportException("Could not connect to " + host + " on port " + port, var5); - } - } - - public static class TSSLTransportParameters { - protected String protocol; - protected String keyStore; - protected String keyPass; - protected String keyManagerType; - protected String keyStoreType; - protected String trustStore; - protected String trustPass; - protected String trustManagerType; - protected String trustStoreType; - protected String[] cipherSuites; - protected boolean clientAuth; - protected boolean isKeyStoreSet; - protected boolean isTrustStoreSet; - - public TSSLTransportParameters() { - this.protocol = "TLS"; - this.keyManagerType = KeyManagerFactory.getDefaultAlgorithm(); - this.keyStoreType = "JKS"; - this.trustManagerType = TrustManagerFactory.getDefaultAlgorithm(); - this.trustStoreType = "JKS"; - this.clientAuth = false; - this.isKeyStoreSet = false; - this.isTrustStoreSet = false; - } - - public TSSLTransportParameters(String protocol, String[] cipherSuites) { - this(protocol, cipherSuites, false); - } - - public TSSLTransportParameters(String protocol, String[] cipherSuites, boolean clientAuth) { - this.protocol = "TLS"; - this.keyManagerType = KeyManagerFactory.getDefaultAlgorithm(); - this.keyStoreType = "JKS"; - this.trustManagerType = TrustManagerFactory.getDefaultAlgorithm(); - this.trustStoreType = "JKS"; - this.clientAuth = false; - this.isKeyStoreSet = false; - this.isTrustStoreSet = false; - if (protocol != null) { - this.protocol = protocol; - } - - this.cipherSuites = (String[]) Arrays.copyOf(cipherSuites, cipherSuites.length); - this.clientAuth = clientAuth; - } - - public void setKeyStore(String keyStore, String keyPass, String keyManagerType, String keyStoreType) { - this.keyStore = keyStore; - this.keyPass = keyPass; - if (keyManagerType != null) { - this.keyManagerType = keyManagerType; - } - - if (keyStoreType != null) { - this.keyStoreType = keyStoreType; - } - - this.isKeyStoreSet = true; - } - - public void setKeyStore(String keyStore, String keyPass) { - this.setKeyStore(keyStore, keyPass, (String) null, (String) null); - } - - public void setTrustStore(String trustStore, String trustPass, String trustManagerType, String trustStoreType) { - this.trustStore = trustStore; - this.trustPass = trustPass; - if (trustManagerType != null) { - this.trustManagerType = trustManagerType; - } - - if (trustStoreType != null) { - this.trustStoreType = trustStoreType; - } - - this.isTrustStoreSet = true; - } - - public void setTrustStore(String trustStore, String trustPass) { - this.setTrustStore(trustStore, trustPass, (String) null, (String) null); - } - - public void requireClientAuth(boolean clientAuth) { - this.clientAuth = clientAuth; - } - } -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/gse/GseClient.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/gse/GseClient.java deleted file mode 100644 index ab9e990124..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/gse/GseClient.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.engine.gse; - -import com.tencent.bk.gse.taskapi.doSomeCmdV3; -import com.tencent.bk.job.common.util.ApplicationContextRegister; -import com.tencent.bk.job.execute.config.GseConfig; -import lombok.extern.slf4j.Slf4j; -import org.apache.thrift.TConfiguration; -import org.apache.thrift.TException; -import org.apache.thrift.protocol.TBinaryProtocol; -import org.apache.thrift.transport.TSocket; -import org.apache.thrift.transport.TTransport; -import org.springframework.beans.BeansException; - -import java.io.Closeable; - -@Slf4j -public class GseClient implements Closeable { - - private static final boolean ENABLE_SSL; - private static final String KEY_STORE; - private static final String TRUST_STORE; - private static final String KEY_STORE_PASS; - private static final String TRUST_STORE_PASS; - private static final String TRUST_MANAGER_TYPE; - private static final String TRUST_STORE_TYPE; - - static { - GseConfig gseConfig = ApplicationContextRegister.getBean("gseConfig", GseConfig.class); - ENABLE_SSL = gseConfig.isGseSSLEnable(); - KEY_STORE = gseConfig.getGseSSLKeystore(); - KEY_STORE_PASS = gseConfig.getGseSSLKeystorePassword(); - TRUST_STORE = gseConfig.getGseSSLTruststore(); - TRUST_STORE_PASS = gseConfig.getGseSSLTruststorePassword(); - TRUST_MANAGER_TYPE = gseConfig.getGseSSLTruststoreManagerType(); - TRUST_STORE_TYPE = gseConfig.getGseSSLTruststoreStoreType(); - } - - /** - * gse服务接口 - */ - private final doSomeCmdV3.Client gseAgentClient; - private final TTransport transport; - - private GseClient(String ip, int port) throws TException { - if (ENABLE_SSL) { - BKTSSLTransportFactory.TSSLTransportParameters params = - new BKTSSLTransportFactory.TSSLTransportParameters(); - params.setTrustStore(TRUST_STORE, TRUST_STORE_PASS, TRUST_MANAGER_TYPE, TRUST_STORE_TYPE); - params.setKeyStore(KEY_STORE, KEY_STORE_PASS); - transport = BKTSSLTransportFactory.getClientSocket(ip, port, 60000, params); - } else { - this.transport = new TSocket(new TConfiguration(), ip, port, 60000); - } - TBinaryProtocol tProtocol = new TBinaryProtocol(transport); - this.gseAgentClient = new doSomeCmdV3.Client(tProtocol); - if (!transport.isOpen()) - transport.open(); - } - - public static GseClient getClient() { - try { - GseServer gseServer = ApplicationContextRegister.getBean("gseServer"); - return gseServer.getClient(); - } catch (BeansException ignored) { - log.error("load gseSever failed!", ignored); - } - return null; - } - - /** - * 构建gse访问客户端, 并连接服务端 - * - * @param ip GSE IP - * @param port GSE Port - * @return GseClient - */ - static GseClient getClient(String ip, int port) throws TException { - return new GseClient(ip, port); - } - - public doSomeCmdV3.Client getGseAgentClient() { - return gseAgentClient; - } - - /** - * 关闭连接 - */ - public void tearDown() { - try { - transport.close(); - } catch (Throwable ignored) { - } - } - - @Override - public void close() { - tearDown(); - } -} - diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/gse/GseRequestUtils.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/gse/GseRequestUtils.java deleted file mode 100644 index bca1973d92..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/gse/GseRequestUtils.java +++ /dev/null @@ -1,605 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.engine.gse; - -import com.tencent.bk.gse.taskapi.api_agent; -import com.tencent.bk.gse.taskapi.api_auth; -import com.tencent.bk.gse.taskapi.api_auto_task; -import com.tencent.bk.gse.taskapi.api_base_file_info; -import com.tencent.bk.gse.taskapi.api_comm_rsp; -import com.tencent.bk.gse.taskapi.api_copy_fileinfoV2; -import com.tencent.bk.gse.taskapi.api_host; -import com.tencent.bk.gse.taskapi.api_map_rsp; -import com.tencent.bk.gse.taskapi.api_process_base_info; -import com.tencent.bk.gse.taskapi.api_process_extra_info; -import com.tencent.bk.gse.taskapi.api_process_info; -import com.tencent.bk.gse.taskapi.api_process_req; -import com.tencent.bk.gse.taskapi.api_query_agent_info_v2; -import com.tencent.bk.gse.taskapi.api_query_atom_task_info; -import com.tencent.bk.gse.taskapi.api_query_task_info_v2; -import com.tencent.bk.gse.taskapi.api_script_file; -import com.tencent.bk.gse.taskapi.api_script_request; -import com.tencent.bk.gse.taskapi.api_stop_task_request; -import com.tencent.bk.gse.taskapi.api_task_detail_result; -import com.tencent.bk.gse.taskapi.api_task_request; -import com.tencent.bk.job.common.gse.constants.GseConstants; -import com.tencent.bk.job.common.util.ApplicationContextRegister; -import com.tencent.bk.job.common.util.ThreadUtils; -import com.tencent.bk.job.execute.common.exception.ReadTimeoutException; -import com.tencent.bk.job.execute.engine.model.GseTaskResponse; -import com.tencent.bk.job.execute.engine.model.LogPullProgress; -import com.tencent.bk.job.execute.engine.model.RunSQLScriptFile; -import com.tencent.bk.job.execute.gse.model.ProcessOperateTypeEnum; -import io.micrometer.core.instrument.MeterRegistry; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.apache.thrift.TException; -import org.apache.thrift.transport.TTransportException; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.stream.Collectors; - -/** - * GSE请求构建工具类 - */ -@Slf4j -public class GseRequestUtils { - private static final MeterRegistry meterRegistry; - - static { - meterRegistry = ApplicationContextRegister.getBean(MeterRegistry.class); - } - - /** - * 构建 GSE 拷贝文件信息 - * - * @param src 源agent - * @param srcPath 源文件路径 - * @param srcName 源文件名称 - * @param dests 目标agent - * @param destPath 目标路径 - * @param destName 目标目录 - * @param downloadSpeedLimit 下载限速，KB - * @param uploadSpeedLimit 上传限速，KB - * @param timeout 超时时间，单位s - * @return 拷贝文件信息 - */ - public static api_copy_fileinfoV2 buildCopyFileInfo(api_agent src, String srcPath, String srcName, - List dests, String destPath, - String destName, Integer downloadSpeedLimit, - Integer uploadSpeedLimit, Integer timeout) { - api_copy_fileinfoV2 copyFileInfo = new api_copy_fileinfoV2(); - - api_base_file_info baseFileInfo = new api_base_file_info(); - baseFileInfo.setName(srcName); - baseFileInfo.setPath(srcPath); - baseFileInfo.setDest_path(destPath); - baseFileInfo.setDest_name(destName); - baseFileInfo.setBackup_name(StringUtils.EMPTY); - baseFileInfo.setBackup_path(StringUtils.EMPTY); - baseFileInfo.setOwner(StringUtils.EMPTY); - baseFileInfo.setMd5(StringUtils.EMPTY); - - copyFileInfo.setFile(baseFileInfo); - copyFileInfo.setSrc_agent(src); - copyFileInfo.setDest_agents(dests); - if (uploadSpeedLimit != null) { - copyFileInfo.setUpload_speed(uploadSpeedLimit); - } - if (downloadSpeedLimit != null) { - copyFileInfo.setDownload_speed(downloadSpeedLimit); - } - if (timeout != null) { - copyFileInfo.setTimeout(timeout); - } - return copyFileInfo; - } - - - public static List buildAgentList(Collection agentIds, String userName, String password) { - List agentList = new ArrayList<>(); - for (String agentId : agentIds) { - api_agent agent = buildAgent(agentId, userName, password); - agentList.add(agent); - } - return agentList; - } - - public static api_agent buildAgent(String agentId, String userName, String password) { - // 在gse api v1, agentId=云区域:IP - int source = 0; - String ip; - String[] ipArray = agentId.split(":"); - if (ipArray.length > 1) { - source = Integer.parseInt(ipArray[0]); - ip = ipArray[1]; - } else { - ip = ipArray[0]; - } - - api_auth auth = new api_auth(); - auth.setUser(userName); - if (password != null) { - auth.setPassword(password); - } else { - auth.setPassword(""); - } - - api_agent agent = new api_agent(); - agent.setIp(ip); - agent.setAuth(auth); - if (source > 1) { - agent.setGse_composite_id(source); - } - - return agent; - } - - /** - * 创建GSE执行脚本请求(以文件做参数） - */ - public static api_script_request buildScriptRequestWithSQL(List agentList, - RunSQLScriptFile runSQLScriptFile) { - api_script_request scriptReq = new api_script_request(); - - api_script_file scriptFile = new api_script_file(); - scriptFile.setDownload_path(runSQLScriptFile.getDownloadPath()); - scriptFile.setMd5(""); - scriptFile.setName(runSQLScriptFile.getPublicScriptName()); - scriptFile.setContent(runSQLScriptFile.getPublicScriptContent()); - api_script_file sqlScriptFile = new api_script_file(); - sqlScriptFile.setDownload_path(runSQLScriptFile.getDownloadPath()); - sqlScriptFile.setMd5(""); - sqlScriptFile.setName(runSQLScriptFile.getSqlScriptFileName()); - sqlScriptFile.setContent(runSQLScriptFile.getSqlScriptContent()); - List scriptFileList = new ArrayList<>(); - scriptFileList.add(scriptFile); - scriptFileList.add(sqlScriptFile); - scriptReq.setScripts(scriptFileList); - - api_task_request req = new api_task_request(); - req.setAtomic_task_num(1); - req.setCrond(""); - - List taskList = new ArrayList<>(); - api_auto_task autoTask = new api_auto_task(); - autoTask.setAtomic_task_id((byte) 0); - - String exeCmd = runSQLScriptFile.getDownloadPath() + "/" + runSQLScriptFile.getPublicScriptName() - + " " + runSQLScriptFile.getDownloadPath() + "/" + runSQLScriptFile.getSqlScriptFileName() - + " " + runSQLScriptFile.getParamForDBInfo(); - autoTask.setCmd(exeCmd); - autoTask.setTimeout(runSQLScriptFile.getTimeout()); - taskList.add(autoTask); - - req.setAtomic_tasks(taskList); - req.setAgent_list(agentList); - req.setRel_list(new ArrayList<>()); - req.setVersion("1.0"); - scriptReq.setTasks(req); - - return scriptReq; - } - - /** - * 创建GSE执行脚本请求 - * - * @param agentList Agent列表 - * @param scriptContent 脚本内容 - * @param scriptFileName 脚本名称 - * @param downloadPath 脚本存放路径 - * @param scriptParam 脚本参数 - * @param timeout 脚本任务超时时间，单位秒 - * @return 执行脚本任务请求 - */ - public static api_script_request buildScriptRequest(List agentList, String scriptContent, - String scriptFileName, String downloadPath, String scriptParam, - int timeout) { - api_script_request scriptReq = new api_script_request(); - - // 脚本文件 - api_script_file scriptFile = new api_script_file(); - scriptFile.setDownload_path(downloadPath); - scriptFile.setMd5(""); - scriptFile.setName(scriptFileName); - scriptFile.setContent(scriptContent); - List scriptFileList = new ArrayList<>(); - scriptFileList.add(scriptFile); - scriptReq.setScripts(scriptFileList); - - api_task_request req = new api_task_request(); - req.setAtomic_task_num(1); - req.setCrond(""); - - List taskList = new ArrayList<>(); - api_auto_task autoTask = new api_auto_task(); - autoTask.setAtomic_task_id((byte) 0); - - String exeCmd = downloadPath + "/" + scriptFileName; - if (StringUtils.isNotBlank(scriptParam)) { - exeCmd += " " + scriptParam; - } - - autoTask.setCmd(exeCmd); - autoTask.setTimeout(timeout); - taskList.add(autoTask); - - req.setAtomic_tasks(taskList); - req.setAgent_list(agentList); - req.setRel_list(new ArrayList<>()); - req.setVersion("1.0"); - scriptReq.setTasks(req); - - return scriptReq; - } - - /** - * 下发脚本任务 - * - * @param id 任务ID - * @param scriptRequest 请求内容 - * @return GSE Server响应 - */ - public static GseTaskResponse sendScriptTaskRequest(long id, api_script_request scriptRequest) { - return sendCmd("" + id, new GseTaskResponseCaller() { - @Override - public GseTaskResponse callback(GseClient gseClient) throws TException { - GseTaskResponse gseResponse = new GseTaskResponse(); - if (log.isDebugEnabled()) { - log.debug("[{}]: runScriptTaskRequest={}", id, scriptRequest); - } else if (log.isInfoEnabled()) { - log.info("[{}]: runScriptTaskRequest={}", id, - GseRequestPrinter.simplifyScriptRequest(scriptRequest)); - } - api_comm_rsp commRsp = gseClient.getGseAgentClient().run_bash(scriptRequest); - log.info("[{}]: runScriptTaskResponse={}", id, commRsp.toString()); - - gseResponse.setErrorCode(commRsp.getBk_error_code()); - gseResponse.setErrorMessage(commRsp.getBk_error_msg()); - gseResponse.setGseTaskId(commRsp.getTask_id()); - return gseResponse; - } - - @Override - public String getApiName() { - return "run_bash"; - } - }); - } - - - public static api_process_req buildProcessRequest(List agentList, api_process_base_info processBaseInfo, - api_process_extra_info processExtraInfo, int reqType) { - api_process_req processReq = new api_process_req(); - - api_process_info processInfo = new api_process_info(); - processInfo.setBase_info(processBaseInfo); - processInfo.setExtra_info(processExtraInfo); - processReq.setAgents(agentList); - processReq.setProc_info(processInfo); - processReq.setOperate_type(reqType); - - return processReq; - } - - /** - * 下发文件分发任务 - * - * @param id 任务ID - * @param copyFileInfoList 请求内容 - * @return GSE Server响应 - */ - public static GseTaskResponse sendCopyFileTaskRequest(long id, List copyFileInfoList) { - return sendCmd("" + id, new GseTaskResponseCaller() { - @Override - public GseTaskResponse callback(GseClient gseClient) throws TException { - GseTaskResponse gseResponse = new GseTaskResponse(); - log.info("[{}]: fileCopyTaskRequest={}", id, copyFileInfoList); - api_comm_rsp commRsp = gseClient.getGseAgentClient().copy_file_v2(copyFileInfoList); - log.info("[{}]: fileCopyTaskResponse={}", id, commRsp); - gseResponse.setErrorCode(commRsp.getBk_error_code()); - gseResponse.setErrorMessage(commRsp.getBk_error_msg()); - gseResponse.setGseTaskId(commRsp.getTask_id()); - return gseResponse; - } - - @Override - public String getApiName() { - return "copy_file_v2"; - } - }); - } - - /** - * 强制终止任务 - * - * @param id 任务ID - * @param stopTaskRequest 终止任务请求 - * @return GSE SERVER 响应 - */ - public static GseTaskResponse sendForceStopTaskRequest(long id, api_stop_task_request stopTaskRequest) { - return sendCmd("" + id, new GseTaskResponseCaller() { - @Override - public GseTaskResponse callback(GseClient gseClient) throws TException { - GseTaskResponse gseResponse = new GseTaskResponse(); - log.info("[{}]: sendForceStopTaskRequest={}", id, stopTaskRequest); - api_comm_rsp commRsp = gseClient.getGseAgentClient().stop_task(stopTaskRequest); - log.info("[{}]: sendForceStopTaskResponse={}", id, commRsp); - gseResponse.setErrorCode(commRsp.getBk_error_code()); - gseResponse.setErrorMessage(commRsp.getBk_error_msg()); - gseResponse.setGseTaskId(commRsp.getTask_id()); - return gseResponse; - } - - @Override - public String getApiName() { - return "stop_task"; - } - }); - } - - private static List buildQueryAtomTaskInfoList(int offset, int mid) { - api_query_atom_task_info queryAtomTaskInfo = new api_query_atom_task_info(); - queryAtomTaskInfo.setId((byte) mid); - queryAtomTaskInfo.setOffset(offset); - - List queryAtomTaskList = new ArrayList<>(); - queryAtomTaskList.add(queryAtomTaskInfo); - return queryAtomTaskList; - } - - /** - * 拉取脚本执行任务日志 V2 - * - * @param gseTaskId gse任务ID - * @param cloudIps 目标服务器列表 - * @param runBushOffsetMap 日志偏移量 - * @return 响应 - */ - public static api_query_task_info_v2 buildScriptLogRequestV2(String gseTaskId, Collection cloudIps, - Map runBushOffsetMap) { - List queryAgentList = new ArrayList<>(); - for (String cloudIp : cloudIps) { - api_query_agent_info_v2 queryAgentInfo = new api_query_agent_info_v2(); - - api_host apiHost = convertToApiHost(cloudIp); - queryAgentInfo.setHost(apiHost); - - LogPullProgress process = runBushOffsetMap.get(cloudIp); - if (null == process) { - queryAgentInfo.setAtomic_tasks(buildQueryAtomTaskInfoList(0, 0)); - } else { - queryAgentInfo.setAtomic_tasks(buildQueryAtomTaskInfoList(process.getByteOffset(), process.getMid())); - } - - queryAgentList.add(queryAgentInfo); - } - - // 请求消息 - api_query_task_info_v2 queryTaskInfo = new api_query_task_info_v2(); - queryTaskInfo.setAgents(queryAgentList); - queryTaskInfo.setTask_id(gseTaskId); - return queryTaskInfo; - } - - private static api_host convertToApiHost(String cloudIp) { - String[] ipArray = cloudIp.split(":"); - - int cloudAreaId = 0; - String ip; - if (ipArray.length > 1) { - cloudAreaId = Integer.parseInt(ipArray[0]); - ip = ipArray[1]; - } else { - ip = ipArray[0]; - } - - api_host apiHost = new api_host(); - apiHost.setIp(ip); - if (cloudAreaId > 1) { - apiHost.setGse_composite_id(cloudAreaId); - } else { - apiHost.setGse_composite_id(0); - } - return apiHost; - } - - /** - * 获取文件任务执行结果 - * - * @param id job任务ID - * @param gseTaskId gse任务ID - * @return 结果 - */ - public static api_map_rsp pullCopyFileTaskLog(long id, String gseTaskId) { - return sendCmd("" + id, new GseApiCallback() { - @Override - public api_map_rsp callback(GseClient gseClient) throws TException { - log.info("[{}]: pullCopyFileTaskLog|gseTaskId:{}", id, gseTaskId); - api_map_rsp copyFileTaskLog = gseClient.getGseAgentClient().get_copy_file_result(gseTaskId); - log.info("[{}]: pullCopyFileTaskLog|response:{}", id, copyFileTaskLog); - return copyFileTaskLog; - } - - @Override - public String getApiName() { - return "get_copy_file_result"; - } - }); - } - - /** - * 拉取文件任务执行结果-根据IP - */ - public static api_map_rsp pullCopyFileTaskLog(long id, String gseTaskId, Collection cloudIps) { - return sendCmd("" + id, new GseApiCallback() { - @Override - public api_map_rsp callback(GseClient gseClient) throws TException { - List hosts = cloudIps.stream() - .map(GseRequestUtils::convertToApiHost).collect(Collectors.toList()); - log.info("[{}]: pullCopyFileTaskLogByIp|gseTaskId:{}|hosts:{}", id, gseTaskId, hosts); - api_map_rsp copyFileTaskLog = gseClient.getGseAgentClient() - .get_copy_file_result_by_ip_v2(gseTaskId, hosts); - log.info("[{}]: pullCopyFileTaskLogByIp|response:{}", id, copyFileTaskLog); - return copyFileTaskLog; - } - - @Override - public String getApiName() { - return "get_copy_file_result_by_ip_v2"; - } - }); - } - - /** - * 脚本任务概览信息 - */ - public static api_task_detail_result getScriptTaskDetailRst(long id, api_query_task_info_v2 taskQuery) { - return sendCmd("" + id, new GseApiCallback() { - @Override - public api_task_detail_result callback(GseClient gseClient) throws TException { - log.info("[{}]: getScriptTaskDetailRequest={}", id, taskQuery); - api_task_detail_result taskDetailRst = gseClient.getGseAgentClient().get_task_detail_result(taskQuery); - log.info("[{}]: getScriptTaskDetailResponse={}", id, - GseRequestPrinter.printScriptTaskResult(taskDetailRst)); - return taskDetailRst; - } - - @Override - public String getApiName() { - return "get_task_detail_result"; - } - }); - } - - public static GseTaskResponse sendProcessRequest(String id, List processRequestList, int reqType) { - return sendCmd(id, new GseTaskResponseCaller() { - @Override - public GseTaskResponse callback(GseClient gseClient) throws TException { - GseTaskResponse gseResponse = new GseTaskResponse(); - log.info("[{}]: sendProcessRequest: {}", id, processRequestList.toString()); - api_comm_rsp commRsp; - if (reqType == ProcessOperateTypeEnum.REGISTER_PROC.getValue()) { - commRsp = gseClient.getGseAgentClient().register_hosting_process(processRequestList); - } else if (reqType == ProcessOperateTypeEnum.UNREGISTER_PROC.getValue()) { - commRsp = gseClient.getGseAgentClient().unregister_hosting_process(processRequestList); - } else { - commRsp = gseClient.getGseAgentClient().process_operate(processRequestList); - log.info("[{}]: sendProcessOperResponse: {}", id, commRsp.toString()); - } - gseResponse.setErrorCode(commRsp.getBk_error_code()); - gseResponse.setErrorMessage(commRsp.getBk_error_msg()); - gseResponse.setGseTaskId(commRsp.getTask_id()); - return gseResponse; - } - - @Override - public String getApiName() { - return "process_operate"; - } - }); - } - - public static api_map_rsp getGetProcRst(String id, String gseTaskId) { - return sendCmd(id, new GseApiCallback() { - @Override - public api_map_rsp callback(GseClient gseClient) throws TException { - log.info("[{}]: getGetProcRst: gseTaskId={}", id, gseTaskId); - api_map_rsp getGetProcRst = gseClient.getGseAgentClient().get_proc_result(gseTaskId); - log.info("[{}]: getGetProcRst: {}", id, getGetProcRst); - return getGetProcRst; - } - - @Override - public String getApiName() { - return "get_proc_result"; - } - }); - } - - private static T sendCmd(String id, GseApiCallback caller) { - int retry = 1; - boolean connect = true; - do { - String status = "ok"; - long start = System.nanoTime(); - try (GseClient gseClient = GseClient.getClient()) { - connect = true; - if (gseClient == null) { - connect = false; - // 如果拿不到连接 ，则等待3s重试 - ThreadUtils.sleep(3000L); - continue; - } - return caller.callback(gseClient); - } catch (TTransportException e) { - // 由于无法捕获到底层的java.net.SocketTimeoutException，所以只能对errorMessage进行判断。读超时无需重试 - status = "error"; - if ("java.net.SocketTimeoutException: Read timed out".equalsIgnoreCase(e.getMessage())) { - log.error("[" + id + "]: Invoke gse api fail", e); - throw new ReadTimeoutException(e.getMessage()); - } - } catch (Exception e) { - log.error("[" + id + "]: Invoke gse api fail", e); - status = "error"; - } finally { - long end = System.nanoTime(); - meterRegistry.timer(GseConstants.GSE_API_METRICS_NAME_PREFIX, "api_name", caller.getApiName(), - "status", status).record(end - start, TimeUnit.NANOSECONDS); - } - } while (retry-- > 0); //重试1次 - return caller.fail(connect); - } - - public interface GseApiCallback { - - T callback(GseClient gseClient) throws TException; - - String getApiName(); - - default T fail(boolean connect) { - return null; - } - } - - public interface GseTaskResponseCaller extends GseApiCallback { - - @Override - default GseTaskResponse fail(boolean connect) { - GseTaskResponse gseResponse = new GseTaskResponse(); - gseResponse.setErrorCode(-1); - if (connect) { - gseResponse.setErrorMessage("Send GSE Job failed!"); - } else { - gseResponse.setErrorMessage("Connect GSE Server failed!"); - } - return gseResponse; - } - } -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/gse/ScriptRequestBuilder.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/gse/ScriptRequestBuilder.java deleted file mode 100644 index 354a9a94c5..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/gse/ScriptRequestBuilder.java +++ /dev/null @@ -1,122 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.engine.gse; - -import com.tencent.bk.gse.taskapi.api_agent; -import com.tencent.bk.gse.taskapi.api_auto_task; -import com.tencent.bk.gse.taskapi.api_script_file; -import com.tencent.bk.gse.taskapi.api_script_request; -import com.tencent.bk.gse.taskapi.api_task_relation; -import com.tencent.bk.gse.taskapi.api_task_request; -import org.apache.commons.lang3.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.util.ArrayList; -import java.util.List; - -/** - * GSE脚本下发请求参数构造器 - */ -public class ScriptRequestBuilder { - private static final Logger LOG = LoggerFactory.getLogger(ScriptRequestBuilder.class); - private api_script_request scriptRqeuest; - - public ScriptRequestBuilder() { - scriptRqeuest = new api_script_request(); - } - - /** - * 添加脚本文件 - * - * @param downloadPath - * @param scriptFileName - * @param scriptContent - * @return - */ - public ScriptRequestBuilder addScriptFile(String downloadPath, String scriptFileName, String scriptContent) { - api_script_file scriptFile = new api_script_file(); - scriptFile.setDownload_path(downloadPath); - scriptFile.setMd5(""); - scriptFile.setName(scriptFileName); - scriptFile.setContent(scriptContent); - scriptRqeuest.addToScripts(scriptFile); - return this; - } - - /** - * 新增脚本任务 - * - * @param agentList - * @param downloadPath - * @param scriptFileName - * @param scriptParam - * @param timeout - * @return - */ - public ScriptRequestBuilder addScriptTask(List agentList, String downloadPath, String scriptFileName, - String scriptParam, int timeout) { - api_task_request taskRequest = scriptRqeuest.getTasks(); - if (taskRequest == null) { - taskRequest = new api_task_request(); - scriptRqeuest.setTasks(taskRequest); - } - - taskRequest.setAtomic_task_num(taskRequest.getAtomic_task_num() + 1); - taskRequest.setCrond(""); - - api_auto_task autoTask = new api_auto_task(); - int atomicTaskId = taskRequest.getAtomic_task_num() - 1; - if (atomicTaskId > 0) { - //为了保证任务串行执行，需要设置relation参数 - List dependencyTaskIdList = new ArrayList<>(); - int dependencyTaskId = 0; - while (dependencyTaskId < atomicTaskId) { - dependencyTaskIdList.add((byte) dependencyTaskId); - dependencyTaskId++; - } - api_task_relation apiTaskRelation = new api_task_relation((byte) atomicTaskId, dependencyTaskIdList); - taskRequest.addToRel_list(apiTaskRelation); - } else { - taskRequest.setRel_list(new ArrayList<>(0)); - } - - autoTask.setAtomic_task_id((byte) (taskRequest.getAtomic_task_num() - 1)); - String exeCmd = downloadPath + "/" + scriptFileName; - if (StringUtils.isNotBlank(scriptParam)) { - exeCmd += " " + scriptParam; - } - autoTask.setCmd(exeCmd); - autoTask.setTimeout(timeout); - taskRequest.addToAtomic_tasks(autoTask); - taskRequest.setAgent_list(agentList); - taskRequest.setVersion("1.0"); - return this; - } - - public api_script_request build() { - return scriptRqeuest; - } -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/gse/model/GSEFileTaskResult.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/gse/model/GSEFileTaskResult.java deleted file mode 100644 index ed33b1c8a9..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/gse/model/GSEFileTaskResult.java +++ /dev/null @@ -1,327 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.engine.gse.model; - -import com.fasterxml.jackson.annotation.JsonProperty; -import com.tencent.bk.job.execute.common.constants.FileDistModeEnum; -import com.tencent.bk.job.execute.engine.util.FilePathUtils; -import lombok.Getter; -import lombok.Setter; -import lombok.ToString; -import org.apache.commons.lang3.StringUtils; - -import java.util.StringJoiner; - -/** - * GSE 文件任务结果 - */ -@Getter -@Setter -@ToString -public class GSEFileTaskResult { - - /** - * 0:upload， 1:download - * - * @see FileDistModeEnum - */ - private Integer mode; - - /** - * 完成进度，取值0-100 - */ - private Integer process; - - /** - * 文件总字节数 - */ - private Long size; - - /** - * 传输的速度，单位bps - */ - private Integer speed; - - /** - * 任务状态信息，比如checksum/upload/download - */ - private String type; - - /** - * 任务开始时间 - */ - @JsonProperty("start_time") - private Long startTime; - - /** - * 任务开始时间-兼容字段 - */ - @JsonProperty("starttime") - private String startTimeStr; - - /** - * 任务结束时间-兼容字段 - */ - @JsonProperty("endtime") - private String endTimeStr; - - /** - * 任务结束时间 - */ - @JsonProperty("end_time") - private Long endTime; - - /** - * 文件源IP - */ - @JsonProperty("source") - private String sourceIp; - - /** - * 文件源云区域 - */ - @JsonProperty("source_cloudid") - private Long sourceCloudId; - - /** - * 文件源主机AgentId - */ - @JsonProperty("source_agent_id") - private String sourceAgentId; - - /** - * 分发目标IP - */ - @JsonProperty("dest") - private String destIp; - - /** - * 分发目标云区域 - */ - @JsonProperty("dest_cloudid") - private Long destCloudId; - - /** - * 文件目标主机AgentId - */ - @JsonProperty("dest_agent_id") - private String destAgentId; - - /** - * 源文件目录 - */ - @JsonProperty("source_file_dir") - private String srcDirPath; - - /** - * 源文件名 - */ - @JsonProperty("source_file_name") - private String srcFileName; - - /** - * 目标文件目录 - */ - @JsonProperty("dest_file_dir") - private String destDirPath; - - /** - * 目标文件名 - */ - @JsonProperty("dest_file_name") - private String destFileName; - - /** - * GSE 文件任务状态 - */ - @JsonProperty("status_code") - private Integer status; - - /** - * GSE 文件任务状态描述 - */ - @JsonProperty("status_info") - private String statusDesc; - - /** - * GSE 协议版本(0 - 未知版本；1 - 初始版本 ; 2 - 解除valuekey依赖版本) - */ - @JsonProperty("protover") - private Integer protocolVersion; - - // ------------------ 非协议字段 ---------------------- - /** - * 用来表示文件任务ID - */ - private String taskId; - - /** - * 标准化之后的源文件路径 - */ - private String standardSourceFilePath; - - /** - * 标准化之后的目标文件路径 - */ - private String standardDestFilePath; - - /** - * 任务类型，1-文件分发，2-目录分发，3-正则分发，4-通配符分发 - */ - private TaskType taskType; - - /** - * 文件源主机agentId - * 按照与GSE的约定，在没有bk_agent_id的场景下，使用{云区域:IP}作为agentId - */ - public String getSourceAgentId() { - if (sourceAgentId == null) { - sourceAgentId = sourceCloudId + ":" + sourceIp; - } - return sourceAgentId; - } - - /** - * 目标主机agentId - * 按照与GSE的约定，在没有bk_agent_id的场景下，使用{云区域:IP}作为agentId - */ - public String getDestAgentId() { - if (destAgentId == null) { - destAgentId = destCloudId + ":" + destIp; - } - return destAgentId; - } - - public boolean isDownloadMode() { - return FileDistModeEnum.DOWNLOAD.getValue().equals(this.mode); - } - - public String getStandardSourceFilePath() { - if (standardSourceFilePath != null) { - return standardSourceFilePath; - } else { - standardSourceFilePath = FilePathUtils.appendFileName(srcDirPath, srcFileName); - } - return standardSourceFilePath; - } - - public String getStandardDestFilePath() { - if (standardDestFilePath != null) { - return standardDestFilePath; - } else { - if (protocolVersion != null && protocolVersion == 2) { - if (StringUtils.isEmpty(srcFileName)) { - //目录分发 - String srcDirName = FilePathUtils.parseDirName(srcDirPath); - standardDestFilePath = FilePathUtils.appendDirName(destDirPath, srcDirName); - } else { - // 文件分发、正则分发、通配符分发 - standardDestFilePath = FilePathUtils.appendFileName(destDirPath, srcFileName); - } - } else { - standardDestFilePath = FilePathUtils.appendFileName(destDirPath, destFileName); - } - } - return standardDestFilePath; - } - - public String getTaskId() { - if (taskId == null) { - this.taskId = buildTaskId(mode, getSourceAgentId(), getStandardSourceFilePath(), getDestAgentId(), - getStandardDestFilePath()); - } - return taskId; - } - - public static String buildTaskId(Integer mode, String sourceAgentId, String sourceFilePath, String destAgentId, - String destFilePath) { - String taskId; - if (FileDistModeEnum.getFileDistMode(mode) == FileDistModeEnum.DOWNLOAD) { - taskId = concat(mode.toString(), sourceAgentId, FilePathUtils.standardizedGSEFilePath(sourceFilePath), - destAgentId, destFilePath); - } else { - taskId = concat(mode.toString(), sourceAgentId, FilePathUtils.standardizedGSEFilePath(sourceFilePath)); - } - return taskId; - } - - private static String concat(String... strArgs) { - StringJoiner sj = new StringJoiner(":"); - for (String strArg : strArgs) { - if (StringUtils.isEmpty(strArg)) { - sj.add(""); - } else { - sj.add(strArg); - } - } - return sj.toString(); - } - - public Long getStartTime() { - if (startTime != null) { - return startTime; - } - if (StringUtils.isNotEmpty(startTimeStr)) { - return Long.valueOf(startTimeStr); - } - return null; - } - - public Long getEndTime() { - if (endTime != null) { - return endTime; - } - if (StringUtils.isNotEmpty(endTimeStr)) { - return Long.valueOf(endTimeStr); - } - return null; - } - - private enum TaskType { - FILE(1), DIR(2), REGEX(3), WILDCARD(4); - - private final int value; - - TaskType(int taskType) { - this.value = taskType; - } - - public static TaskType valueOf(Integer taskType) { - if (taskType == null) { - return null; - } - for (TaskType inst : values()) { - if (inst.value == taskType) { - return inst; - } - } - return null; - } - - public final int getValue() { - return value; - } - } -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/AbstractStepEventHandler.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/AbstractStepEventHandler.java new file mode 100644 index 0000000000..d790ce2b2e --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/AbstractStepEventHandler.java @@ -0,0 +1,101 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.listener; + +import com.tencent.bk.job.execute.common.constants.RunStatusEnum; +import com.tencent.bk.job.execute.common.util.TaskCostCalculator; +import com.tencent.bk.job.execute.engine.listener.event.EventSource; +import com.tencent.bk.job.execute.engine.listener.event.JobEvent; +import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; +import com.tencent.bk.job.execute.model.StepInstanceDTO; +import com.tencent.bk.job.execute.service.StepInstanceService; +import com.tencent.bk.job.execute.service.TaskInstanceService; +import lombok.extern.slf4j.Slf4j; + +/** + * 步骤事件处理-抽象实现 + */ +@Slf4j +public abstract class AbstractStepEventHandler implements StepEventHandler { + + protected final TaskInstanceService taskInstanceService; + protected final StepInstanceService stepInstanceService; + protected final TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher; + + + public AbstractStepEventHandler(TaskInstanceService taskInstanceService, + StepInstanceService stepInstanceService, + TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher) { + this.taskInstanceService = taskInstanceService; + this.stepInstanceService = stepInstanceService; + this.taskExecuteMQEventDispatcher = taskExecuteMQEventDispatcher; + } + + /** + * 当步骤处理异常，安全地结束作业步骤执行的调度 + */ + protected void safelyFinishStepWhenCaughtException(StepInstanceDTO stepInstance) { + try { + finishStepWithAbnormalState(stepInstance); + taskExecuteMQEventDispatcher.dispatchJobEvent( + JobEvent.refreshJob(stepInstance.getTaskInstanceId(), + EventSource.buildStepEventSource(stepInstance.getTaskInstanceId(), stepInstance.getId()))); + } catch (Throwable e) { + log.error("Finish step instance safely caught exception", e); + } + } + + protected void finishStepWithAbnormalState(StepInstanceDTO stepInstance) { + finishStep(stepInstance, RunStatusEnum.ABNORMAL_STATE); + } + + protected void finishStep(StepInstanceDTO stepInstance, RunStatusEnum status) { + long endTime = System.currentTimeMillis(); + if (!RunStatusEnum.isFinishedStatus(stepInstance.getStatus())) { + long totalTime = TaskCostCalculator.calculate(stepInstance.getStartTime(), endTime, + stepInstance.getTotalTime()); + stepInstanceService.updateStepExecutionInfo( + stepInstance.getTaskInstanceId(), + stepInstance.getId(), + status, + null, + endTime, + totalTime + ); + } else { + log.info( + "StepInstance {} already enter a final state:{}", + stepInstance.getId(), + stepInstance.getStatus() + ); + } + } + + protected void onAbandonState(StepInstanceDTO stepInstance) { + finishStep(stepInstance, RunStatusEnum.ABANDONED); + } + + +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/BaseJobMqListener.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/BaseJobMqListener.java new file mode 100644 index 0000000000..d1df8712d9 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/BaseJobMqListener.java @@ -0,0 +1,63 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.listener; + +import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.execute.common.context.JobExecuteContext; +import com.tencent.bk.job.execute.common.context.JobExecuteContextThreadLocalRepo; +import com.tencent.bk.job.execute.engine.listener.event.JobMessage; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageHeaders; + +@Slf4j +public abstract class BaseJobMqListener { + + + public final void onEvent(Message message) { + beforeHandleMessage(message); + try { + handleEvent(message); + } finally { + afterHandle(message); + } + } + + private void beforeHandleMessage(Message message) { + MessageHeaders headers = message.getHeaders(); + String jobExecuteContextJson = (String) headers.get(JobExecuteContext.KEY); + if (StringUtils.isNotEmpty(jobExecuteContextJson)) { + JobExecuteContextThreadLocalRepo.set(JsonUtils.fromJson(jobExecuteContextJson, + JobExecuteContext.class)); + } + } + + private void afterHandle(Message message) { + JobExecuteContextThreadLocalRepo.unset(); + } + + protected abstract void handleEvent(Message message); +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/CallbackListener.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/CallbackListener.java index 46eb4f64ea..03a325e58e 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/CallbackListener.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/CallbackListener.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,13 +24,22 @@ package com.tencent.bk.job.execute.engine.listener; +import com.tencent.bk.job.common.metrics.CommonMetricTags; import com.tencent.bk.job.common.util.http.HttpConPoolUtil; +import com.tencent.bk.job.common.util.http.HttpResponse; import com.tencent.bk.job.common.util.json.JsonUtils; -import com.tencent.bk.job.execute.engine.message.CallbackProcessor; +import com.tencent.bk.job.execute.engine.listener.event.JobMessage; import com.tencent.bk.job.execute.engine.model.JobCallbackDTO; +import com.tencent.bk.job.execute.metrics.ExecuteMetricsConstants; +import com.tencent.bk.job.execute.model.TaskInstanceDTO; +import com.tencent.bk.job.execute.service.TaskInstanceService; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tag; +import io.micrometer.core.instrument.Tags; import lombok.extern.slf4j.Slf4j; -import org.springframework.cloud.stream.annotation.EnableBinding; -import org.springframework.cloud.stream.annotation.StreamListener; +import org.apache.http.HttpStatus; +import org.slf4j.helpers.MessageFormatter; +import org.springframework.messaging.Message; import org.springframework.stereotype.Component; import java.net.MalformedURLException; @@ -40,44 +49,115 @@ * 任务执行结束回调处理 */ @Component -@EnableBinding({CallbackProcessor.class}) @Slf4j -public class CallbackListener { +public class CallbackListener extends BaseJobMqListener { + private final MeterRegistry meterRegistry; + private final TaskInstanceService taskInstanceService; + + public CallbackListener(MeterRegistry meterRegistry, + TaskInstanceService taskInstanceService) { + this.meterRegistry = meterRegistry; + this.taskInstanceService = taskInstanceService; + } /** * 处理回调请求 */ - @StreamListener(CallbackProcessor.INPUT) - public void handleMessage(JobCallbackDTO callbackDTO) { + @Override + public void handleEvent(Message message) { + JobCallbackDTO callbackDTO = (JobCallbackDTO) message.getPayload(); long taskInstanceId = callbackDTO.getId(); + String callbackUrl = callbackDTO.getCallbackUrl(); + callbackDTO.setCallbackUrl(null); + String bodyStr = JsonUtils.toJson(callbackDTO); + boolean postJsonSuccess = false; try { - log.info("Handle callback, taskInstanceId: {}, msg: {}", taskInstanceId, callbackDTO); - String callbackUrl = callbackDTO.getCallbackUrl(); - try { - new URL(callbackUrl); - } catch (MalformedURLException var5) { - log.warn("Callback fail, bad url: {}", callbackUrl); - return; - } - callbackDTO.setCallbackUrl(null); - try { - // TODO 需要优化，返回application/json - try { - String rst = HttpConPoolUtil.post(callbackUrl, JsonUtils.toJson(callbackDTO)); - log.info("Callback success, taskInstanceId: {}, result: {}", taskInstanceId, rst); - } catch (Throwable e) { //出错重试一次 - String errorMsg = "Callback fail, taskInstanceId: " + taskInstanceId; - log.warn(errorMsg, e); - String rst = HttpConPoolUtil.post(callbackUrl, JsonUtils.toJson(callbackDTO)); - log.info("Retry callback success, taskInstanceId: {}, result: {}", taskInstanceId, rst); + log.info("Handle callback, taskInstanceId: {}, callbackDTO: {}", taskInstanceId, callbackDTO); + validateUrl(callbackUrl); + + // 先用JSON数据格式请求一次 + HttpResponse response = HttpConPoolUtil.postJson(callbackUrl, bodyStr); + + if (response.getStatusCode() == HttpStatus.SC_OK) { + postJsonSuccess = true; + recordCallbackMetrics( + String.valueOf(response.getStatusCode()), + ExecuteMetricsConstants.TAG_VALUE_RESULT_POST_JSON_SUCCESS, + callbackDTO + ); + } else { + // 回调状态码不是200，用表单数据格式重试一次 + log.warn( + "Callback using json body failed, use form data to retry. TaskInstanceId={}, statusCode={}", + taskInstanceId, + response.getStatusCode() + ); + response = HttpConPoolUtil.postFormData(callbackUrl, bodyStr); + if (response.getStatusCode() == HttpStatus.SC_OK) { + recordCallbackMetrics( + String.valueOf(response.getStatusCode()), + ExecuteMetricsConstants.TAG_VALUE_RESULT_POST_FORM_DATA_SUCCESS, + callbackDTO + ); + } else { + // 回调状态码不是200，记录为失败 + log.warn("Callback using form data failed, statusCode={}", response.getStatusCode()); + recordCallbackMetrics( + String.valueOf(response.getStatusCode()), + ExecuteMetricsConstants.TAG_VALUE_RESULT_FAILED, + callbackDTO + ); } - } catch (Throwable e) { - String errorMsg = "Callback fail, taskInstanceId: " + taskInstanceId; - log.warn(errorMsg, e); } - } catch (Throwable e) { - String errorMsg = "Callback fail, taskInstanceId: " + taskInstanceId; + log.info( + "Callback result: taskInstanceId={}, postJsonSuccess={}, httpStatusCode={}, result={}", + taskInstanceId, + postJsonSuccess, + response.getStatusCode(), + response.getEntity() + ); + } catch (MalformedURLException e) { + log.warn("Invalid callbackUrl: " + callbackUrl, e); + recordCallbackMetrics( + CommonMetricTags.VALUE_HTTP_STATUS_UNKNOWN, + ExecuteMetricsConstants.TAG_VALUE_RESULT_FAILED, + callbackDTO + ); + } catch (Exception e) { + String errorMsg = MessageFormatter.format( + "Callback request failed, callbackUrl={}, bodyStr={}", + callbackUrl, + bodyStr + ).getMessage(); log.warn(errorMsg, e); + recordCallbackMetrics( + CommonMetricTags.VALUE_HTTP_STATUS_UNKNOWN, + ExecuteMetricsConstants.TAG_VALUE_RESULT_FAILED, + callbackDTO + ); } } + + /** + * 校验URL是否合法 + */ + private void validateUrl(String callbackUrl) throws MalformedURLException { + new URL(callbackUrl); + } + + /** + * 记录回调请求的监控指标 + */ + private void recordCallbackMetrics(String httpStatusCode, + String result, + JobCallbackDTO callbackDTO) { + TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(callbackDTO.getId()); + Iterable tags = Tags.of( + Tag.of(CommonMetricTags.KEY_APP_ID, String.valueOf(taskInstance.getAppId())), + Tag.of(CommonMetricTags.KEY_APP_CODE, String.valueOf(taskInstance.getAppCode())), + Tag.of(CommonMetricTags.KEY_HTTP_STATUS, String.valueOf(httpStatusCode)), + Tag.of(ExecuteMetricsConstants.TAG_KEY_RESULT, String.valueOf(result)) + ); + meterRegistry.counter(ExecuteMetricsConstants.NAME_JOB_TASK_CALLBACK, tags).increment(); + } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/ConfirmStepEventHandler.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/ConfirmStepEventHandler.java index 21909481b6..4dee840e60 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/ConfirmStepEventHandler.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/ConfirmStepEventHandler.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -36,6 +36,7 @@ import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.service.NotifyService; +import com.tencent.bk.job.execute.service.StepInstanceService; import com.tencent.bk.job.execute.service.TaskInstanceService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -47,18 +48,16 @@ */ @Component @Slf4j -public class ConfirmStepEventHandler implements StepEventHandler { +public class ConfirmStepEventHandler extends AbstractStepEventHandler { - private final TaskInstanceService taskInstanceService; - private final TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher; private final NotifyService notifyService; @Autowired public ConfirmStepEventHandler(TaskInstanceService taskInstanceService, TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher, - NotifyService notifyService) { - this.taskInstanceService = taskInstanceService; - this.taskExecuteMQEventDispatcher = taskExecuteMQEventDispatcher; + NotifyService notifyService, + StepInstanceService stepInstanceService) { + super(taskInstanceService, stepInstanceService, taskExecuteMQEventDispatcher); this.notifyService = notifyService; } @@ -68,10 +67,6 @@ public void handleEvent(StepEvent stepEvent, long stepInstanceId = stepEvent.getStepInstanceId(); try { StepActionEnum action = StepActionEnum.valueOf(stepEvent.getAction()); - if (action == null) { - log.error("Invalid step action: {}, ignore step event!", stepEvent.getAction()); - return; - } switch (action) { case START: @@ -92,6 +87,7 @@ public void handleEvent(StepEvent stepEvent, } catch (Throwable e) { String errorMsg = "Handling step event error,stepInstanceId:" + stepInstanceId; log.error(errorMsg, e); + safelyFinishStepWhenCaughtException(stepInstance); } } @@ -108,8 +104,8 @@ private void confirmStepTerminate(StepInstanceDTO stepInstance) { null, endTime, taskTotalTime); long stepTotalTime = TaskCostCalculator.calculate(stepInstance.getStartTime(), endTime, stepInstance.getTotalTime()); - taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.CONFIRM_TERMINATED, null, - endTime, stepTotalTime); + stepInstanceService.updateStepExecutionInfo(stepInstance.getTaskInstanceId(), stepInstanceId, + RunStatusEnum.CONFIRM_TERMINATED, null, endTime, stepTotalTime); } else { log.warn("Unsupported step instance status for confirm step terminate action, stepInstanceId:{}, " + "status:{}", stepInstanceId, stepInstance.getStatus()); @@ -147,7 +143,7 @@ private void executeConfirmStep(StepInstanceBaseDTO stepInstance) { stepOperator = taskInstance.getOperator(); stepInstance.setOperator(stepOperator); } - taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.WAITING_USER, + stepInstanceService.updateStepExecutionInfo(taskInstanceId, stepInstanceId, RunStatusEnum.WAITING_USER, System.currentTimeMillis(), null, null); taskInstanceService.updateTaskStatus(taskInstanceId, RunStatusEnum.WAITING_USER.getValue()); notifyService.asyncSendMQConfirmNotification(taskInstance, stepInstance); @@ -169,10 +165,10 @@ private void confirmStepContinue(StepInstanceBaseDTO stepInstance) { long totalTime = TaskCostCalculator.calculate(stepInstance.getStartTime(), endTime, stepInstance.getTotalTime()); // 人工确认通过，该步骤状态标识为成功；终止成功的步骤保持状态不变 - taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.SUCCESS, null, endTime, - totalTime); + stepInstanceService.updateStepExecutionInfo(taskInstanceId, stepInstanceId, RunStatusEnum.SUCCESS, + null, endTime, totalTime); taskExecuteMQEventDispatcher.dispatchJobEvent( - JobEvent.refreshJob(taskInstanceId, EventSource.buildStepEventSource(stepInstanceId))); + JobEvent.refreshJob(taskInstanceId, EventSource.buildStepEventSource(taskInstanceId, stepInstanceId))); } else { log.warn("Unsupported step instance status for confirm-step-continue step action, stepInstanceId:{}, " + "status:{}", stepInstanceId, stepInstance.getStatus()); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseStepEventHandler.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseStepEventHandler.java index 239c7e295f..4b9f428d81 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseStepEventHandler.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseStepEventHandler.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,35 +27,34 @@ import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.constant.RollingModeEnum; import com.tencent.bk.job.common.exception.NotImplementedException; -import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; import com.tencent.bk.job.execute.common.util.TaskCostCalculator; -import com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum; +import com.tencent.bk.job.execute.engine.consts.ExecuteObjectTaskStatusEnum; import com.tencent.bk.job.execute.engine.consts.StepActionEnum; import com.tencent.bk.job.execute.engine.listener.event.EventSource; import com.tencent.bk.job.execute.engine.listener.event.GseTaskEvent; import com.tencent.bk.job.execute.engine.listener.event.JobEvent; import com.tencent.bk.job.execute.engine.listener.event.StepEvent; import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; import com.tencent.bk.job.execute.engine.prepare.FilePrepareService; -import com.tencent.bk.job.execute.model.AgentTaskDTO; +import com.tencent.bk.job.execute.model.ExecuteObjectTask; import com.tencent.bk.job.execute.model.GseTaskDTO; import com.tencent.bk.job.execute.model.RollingConfigDTO; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.model.StepInstanceRollingTaskDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; -import com.tencent.bk.job.execute.model.db.RollingHostsBatchDO; -import com.tencent.bk.job.execute.service.FileAgentTaskService; +import com.tencent.bk.job.execute.model.db.RollingExecuteObjectsBatchDO; +import com.tencent.bk.job.execute.service.FileExecuteObjectTaskService; import com.tencent.bk.job.execute.service.GseTaskService; import com.tencent.bk.job.execute.service.RollingConfigService; -import com.tencent.bk.job.execute.service.ScriptAgentTaskService; +import com.tencent.bk.job.execute.service.ScriptExecuteObjectTaskService; import com.tencent.bk.job.execute.service.StepInstanceRollingTaskService; import com.tencent.bk.job.execute.service.StepInstanceService; import com.tencent.bk.job.execute.service.TaskInstanceService; import com.tencent.bk.job.logsvr.consts.FileTaskModeEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskStepTypeEnum; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -71,17 +70,14 @@ */ @Component @Slf4j -public class GseStepEventHandler implements StepEventHandler { +public class GseStepEventHandler extends AbstractStepEventHandler { - private final TaskInstanceService taskInstanceService; - private final StepInstanceService stepInstanceService; - private final TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher; private final FilePrepareService filePrepareService; private final GseTaskService gseTaskService; private final RollingConfigService rollingConfigService; private final StepInstanceRollingTaskService stepInstanceRollingTaskService; - private final ScriptAgentTaskService scriptAgentTaskService; - private final FileAgentTaskService fileAgentTaskService; + private final ScriptExecuteObjectTaskService scriptExecuteObjectTaskService; + private final FileExecuteObjectTaskService fileExecuteObjectTaskService; @Autowired public GseStepEventHandler(TaskInstanceService taskInstanceService, @@ -91,17 +87,15 @@ public GseStepEventHandler(TaskInstanceService taskInstanceService, GseTaskService gseTaskService, RollingConfigService rollingConfigService, StepInstanceRollingTaskService stepInstanceRollingTaskService, - ScriptAgentTaskService scriptAgentTaskService, - FileAgentTaskService fileAgentTaskService) { - this.taskInstanceService = taskInstanceService; - this.stepInstanceService = stepInstanceService; - this.taskExecuteMQEventDispatcher = taskExecuteMQEventDispatcher; + ScriptExecuteObjectTaskService scriptExecuteObjectTaskService, + FileExecuteObjectTaskService fileExecuteObjectTaskService) { + super(taskInstanceService, stepInstanceService, taskExecuteMQEventDispatcher); this.filePrepareService = filePrepareService; this.gseTaskService = gseTaskService; this.rollingConfigService = rollingConfigService; this.stepInstanceRollingTaskService = stepInstanceRollingTaskService; - this.scriptAgentTaskService = scriptAgentTaskService; - this.fileAgentTaskService = fileAgentTaskService; + this.scriptExecuteObjectTaskService = scriptExecuteObjectTaskService; + this.fileExecuteObjectTaskService = fileExecuteObjectTaskService; } @Override @@ -139,6 +133,9 @@ public void handleEvent(StepEvent stepEvent, case CONTINUE_FILE_PUSH: continueGseFileStep(stepInstance); break; + case PREPARE_FILE: + prepareFile(stepInstance); + break; case CLEAR: clearStep(stepInstance); break; @@ -148,10 +145,12 @@ public void handleEvent(StepEvent stepEvent, } catch (Throwable e) { String errorMsg = "Handling step event error,stepInstanceId:" + stepInstanceId; log.error(errorMsg, e); + safelyFinishStepWhenCaughtException(stepInstance); } } private void startStep(StepEvent stepEvent, StepInstanceDTO stepInstance) { + long taskInstanceId = stepInstance.getTaskInstanceId(); long stepInstanceId = stepInstance.getId(); boolean isRollingStep = stepInstance.isRollingStep(); if (isRollingStep) { @@ -174,22 +173,31 @@ private void startStep(StepEvent stepEvent, StepInstanceDTO stepInstance) { RollingConfigDTO rollingConfig = null; if (isRollingStep) { - rollingConfig = rollingConfigService.getRollingConfig(stepInstance.getRollingConfigId()); + rollingConfig = rollingConfigService.getRollingConfig(stepInstance.getTaskInstanceId(), + stepInstance.getRollingConfigId()); log.info("Rolling config: {}", rollingConfig); // 更新滚动进度 - stepInstanceService.updateStepCurrentBatch(stepInstanceId, stepInstance.getBatch()); + stepInstanceService.updateStepCurrentBatch(taskInstanceId, stepInstanceId, stepInstance.getBatch()); // 初始化步骤滚动任务 saveInitialStepInstanceRollingTask(stepInstance); } Long gseTaskId = saveInitialGseTask(stepInstance); - saveGseAgentTasksForStartStep(gseTaskId, stepInstance, rollingConfig); + saveExecuteObjectTasksForStartStep(gseTaskId, stepInstance, rollingConfig); - taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.RUNNING, + stepInstanceService.updateStepExecutionInfo(taskInstanceId, stepInstanceId, RunStatusEnum.RUNNING, stepInstance.getStartTime() == null ? DateUtils.currentTimeMillis() : null, null, null); if (isRollingStep) { - stepInstanceRollingTaskService.updateRollingTask(stepInstanceId, stepInstance.getExecuteCount(), - stepInstance.getBatch(), RunStatusEnum.RUNNING, System.currentTimeMillis(), null, null); + stepInstanceRollingTaskService.updateRollingTask( + taskInstanceId, + stepInstanceId, + stepInstance.getExecuteCount(), + stepInstance.getBatch(), + RunStatusEnum.RUNNING, + System.currentTimeMillis(), + null, + null + ); } startGseTask(stepInstance, gseTaskId); @@ -207,19 +215,39 @@ private void startStep(StepEvent stepEvent, StepInstanceDTO stepInstance) { */ private void startGseTask(StepInstanceDTO stepInstance, Long gseTaskId) { if (stepInstance.isScriptStep()) { - taskExecuteMQEventDispatcher.dispatchGseTaskEvent(GseTaskEvent.startGseTask( - stepInstance.getId(), stepInstance.getExecuteCount(), stepInstance.getBatch(), gseTaskId, null)); + taskExecuteMQEventDispatcher.dispatchGseTaskEvent( + GseTaskEvent.startGseTask( + stepInstance.getTaskInstanceId(), + stepInstance.getId(), + stepInstance.getExecuteCount(), + stepInstance.getBatch(), + gseTaskId, + null)); } else if (stepInstance.isFileStep()) { if (filePrepareService.needToPrepareSourceFilesForGseTask(stepInstance)) { filePrepareService.prepareFileForGseTask(stepInstance); } else { - taskExecuteMQEventDispatcher.dispatchGseTaskEvent(GseTaskEvent.startGseTask( - stepInstance.getId(), stepInstance.getExecuteCount(), stepInstance.getBatch(), gseTaskId, - null)); + taskExecuteMQEventDispatcher.dispatchGseTaskEvent( + GseTaskEvent.startGseTask( + stepInstance.getTaskInstanceId(), + stepInstance.getId(), + stepInstance.getExecuteCount(), + stepInstance.getBatch(), + gseTaskId, + null)); } } } + /** + * 再次准备要分发的文件 + * + * @param stepInstance 步骤实例 + */ + private void prepareFile(StepInstanceDTO stepInstance) { + filePrepareService.prepareFileForGseTask(stepInstance); + } + /** * 初始化的GSE任务 * @@ -227,71 +255,79 @@ private void startGseTask(StepInstanceDTO stepInstance, Long gseTaskId) { * @return GSE 任务ID */ private Long saveInitialGseTask(StepInstanceDTO stepInstance) { - GseTaskDTO gseTask = new GseTaskDTO(stepInstance.getId(), stepInstance.getExecuteCount(), - stepInstance.getBatch()); - gseTask.setStatus(RunStatusEnum.WAITING_USER.getValue()); + GseTaskDTO gseTask = new GseTaskDTO(stepInstance.getTaskInstanceId(), stepInstance.getId(), + stepInstance.getExecuteCount(), stepInstance.getBatch()); + gseTask.setStatus(RunStatusEnum.BLANK.getValue()); return gseTaskService.saveGseTask(gseTask); } /** - * 启动步骤的时候保存 GSE Agent 任务 + * 启动步骤的时候保存执行对象任务 * * @param gseTaskId GSE任务ID * @param stepInstance 步骤实例 * @param rollingConfig 滚动配置 */ - private void saveGseAgentTasksForStartStep(Long gseTaskId, - StepInstanceDTO stepInstance, - RollingConfigDTO rollingConfig) { + private void saveExecuteObjectTasksForStartStep(Long gseTaskId, + StepInstanceDTO stepInstance, + RollingConfigDTO rollingConfig) { long stepInstanceId = stepInstance.getId(); int executeCount = stepInstance.getExecuteCount(); int batch = stepInstance.getBatch(); if (stepInstance.isRollingStep()) { // 滚动步骤 - saveGseAgentTasksForStartRollingStep(gseTaskId, stepInstance, rollingConfig); + saveGseExecuteObjectTasksForStartRollingStep(gseTaskId, stepInstance, rollingConfig); } else { - // 普通步骤，启动的时候需要初始化所有AgentTask - List agentTasks = new ArrayList<>( - buildGseAgentTasks(stepInstanceId, executeCount, executeCount, batch, - gseTaskId, stepInstance.getTargetServers().getIpList(), AgentTaskStatusEnum.WAITING)); - saveAgentTasks(stepInstance, agentTasks); + // 普通步骤，启动的时候需要初始化所有ExecuteObjectTask + List executeObjectTasks = new ArrayList<>( + buildInitialExecuteObjectTasks( + stepInstance.getTaskInstanceId(), + stepInstanceId, + executeCount, + executeCount, + batch, + gseTaskId, + stepInstance.getTargetExecuteObjects().getExecuteObjectsCompatibly()) + ); + saveExecuteObjectTasks(stepInstance, executeObjectTasks); } } /** - * 启动滚动执行步骤的时候保存 GSE Agent 任务 + * 启动滚动执行步骤的时候保存执行对象任务 * * @param gseTaskId GSE任务ID * @param stepInstance 步骤实例 * @param rollingConfig 滚动配置 */ - private void saveGseAgentTasksForStartRollingStep(Long gseTaskId, - StepInstanceDTO stepInstance, - RollingConfigDTO rollingConfig) { + private void saveGseExecuteObjectTasksForStartRollingStep(Long gseTaskId, + StepInstanceDTO stepInstance, + RollingConfigDTO rollingConfig) { long stepInstanceId = stepInstance.getId(); int executeCount = stepInstance.getExecuteCount(); int batch = stepInstance.getBatch(); if (stepInstance.isFirstRollingBatch()) { - // 如果是第一批次的执行，需要提前初始化所有批次的agent任务（作业详情查询主机任务列表需要) - List agentTasks = new ArrayList<>(); + // 如果是第一批次的执行，需要提前初始化所有批次的执行对象任务（作业详情查询主机任务列表需要) + List executeObjectTasks = new ArrayList<>(); if (rollingConfig.isBatchRollingStep(stepInstanceId)) { - List serverBatchList = - rollingConfig.getConfigDetail().getHostsBatchList(); - serverBatchList.forEach(serverBatch -> { - agentTasks.addAll( - buildGseAgentTasks( + List executeObjectsBatchList = + rollingConfig.getConfigDetail().getExecuteObjectsBatchListCompatibly(); + executeObjectsBatchList.forEach(executeObjectsBatch -> { + executeObjectTasks.addAll( + buildInitialExecuteObjectTasks( + stepInstance.getTaskInstanceId(), stepInstanceId, executeCount, - serverBatch.getBatch() == 1 ? executeCount : null, - serverBatch.getBatch(), - serverBatch.getBatch() == 1 ? gseTaskId : 0, - serverBatch.getHosts(), - AgentTaskStatusEnum.WAITING) + executeObjectsBatch.getBatch() == 1 ? executeCount : null, + executeObjectsBatch.getBatch(), + executeObjectsBatch.getBatch() == 1 ? gseTaskId : 0, + executeObjectsBatch.getExecuteObjectsCompatibly() + ) ); }); - saveAgentTasks(stepInstance, agentTasks); + saveExecuteObjectTasks(stepInstance, executeObjectTasks); } else { // 暂时不支持，滚动执行二期需求 log.warn("All rolling step is not supported!"); @@ -299,50 +335,55 @@ private void saveGseAgentTasksForStartRollingStep(Long gseTaskId, ErrorCode.NOT_SUPPORT_FEATURE); } } else { - // 滚动执行步骤除了第一批次，后续的批次仅更新 AgentTask 的 actualExecuteCount、gse_task_id + // 滚动执行步骤除了第一批次，后续的批次仅更新 ExecuteObjectTask 的 actualExecuteCount、gseTaskId if (stepInstance.isScriptStep()) { - scriptAgentTaskService.updateAgentTaskFields(stepInstanceId, executeCount, batch, executeCount, + scriptExecuteObjectTaskService.updateTaskFields(stepInstance, executeCount, batch, executeCount, gseTaskId); } else if (stepInstance.isFileStep()) { - fileAgentTaskService.updateAgentTaskFields(stepInstanceId, executeCount, batch, executeCount, + fileExecuteObjectTaskService.updateTaskFields(stepInstance, executeCount, batch, executeCount, gseTaskId); } } } - private List buildGseAgentTasks(long stepInstanceId, - int executeCount, - Integer actualExecuteCount, - int batch, - Long gseTaskId, - List hosts, - AgentTaskStatusEnum status) { - return hosts.stream() - .map(host -> buildGseAgentTask(stepInstanceId, executeCount, actualExecuteCount, - batch, gseTaskId, host, status)) + private List buildInitialExecuteObjectTasks(long taskInstanceId, + long stepInstanceId, + int executeCount, + Integer actualExecuteCount, + int batch, + Long gseTaskId, + List executeObjects) { + return executeObjects.stream() + .map(executeObject -> { + ExecuteObjectTask executeObjectTask = new ExecuteObjectTask( + taskInstanceId, + stepInstanceId, + executeCount, + batch, + executeObject + ); + executeObjectTask.setStepInstanceId(stepInstanceId); + executeObjectTask.setExecuteCount(executeCount); + executeObjectTask.setActualExecuteCount(actualExecuteCount); + executeObjectTask.setBatch(batch); + executeObjectTask.setGseTaskId(gseTaskId); + executeObjectTask.setStatus(executeObject.isExecutable() ? + ExecuteObjectTaskStatusEnum.WAITING : + executeObject.isAgentIdEmpty() ? + ExecuteObjectTaskStatusEnum.AGENT_NOT_INSTALLED : + ExecuteObjectTaskStatusEnum.INVALID_EXECUTE_OBJECT); + if (!executeObject.isExecutable()) { + executeObjectTask.setStartTime(System.currentTimeMillis()); + executeObjectTask.setEndTime(System.currentTimeMillis()); + executeObjectTask.setTotalTime(0L); + } + executeObjectTask.setFileTaskMode(FileTaskModeEnum.DOWNLOAD); + executeObjectTask.setExecuteObject(executeObject); + return executeObjectTask; + }) .collect(Collectors.toList()); } - protected AgentTaskDTO buildGseAgentTask(long stepInstanceId, - int executeCount, - Integer actualExecuteCount, - int batch, - Long gseTaskId, - HostDTO host, - AgentTaskStatusEnum status) { - AgentTaskDTO agentTask = new AgentTaskDTO(); - agentTask.setStepInstanceId(stepInstanceId); - agentTask.setExecuteCount(executeCount); - agentTask.setActualExecuteCount(actualExecuteCount); - agentTask.setBatch(batch); - agentTask.setGseTaskId(gseTaskId); - agentTask.setStatus(status); - agentTask.setFileTaskMode(FileTaskModeEnum.DOWNLOAD); - agentTask.setHostId(host.getHostId()); - agentTask.setAgentId(host.toCloudIp()); - return agentTask; - } - /** * 保存初始化的步骤实例上的滚动任务 * @@ -350,6 +391,7 @@ protected AgentTaskDTO buildGseAgentTask(long stepInstanceId, */ private void saveInitialStepInstanceRollingTask(StepInstanceDTO stepInstance) { StepInstanceRollingTaskDTO stepInstanceRollingTask = new StepInstanceRollingTaskDTO(); + stepInstanceRollingTask.setTaskInstanceId(stepInstance.getTaskInstanceId()); stepInstanceRollingTask.setStepInstanceId(stepInstance.getId()); stepInstanceRollingTask.setBatch(stepInstance.getBatch()); stepInstanceRollingTask.setExecuteCount(stepInstance.getExecuteCount()); @@ -371,10 +413,15 @@ private void nextStep(StepInstanceDTO stepInstance) { long totalTime = TaskCostCalculator.calculate(stepInstance.getStartTime(), endTime, stepInstance.getTotalTime()); // 终止成功，进入下一步，该步骤设置为“跳过” - taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.SKIPPED, null, endTime, + stepInstanceService.updateStepExecutionInfo( + taskInstanceId, + stepInstanceId, + RunStatusEnum.SKIPPED, + null, + endTime, totalTime); taskExecuteMQEventDispatcher.dispatchJobEvent( - JobEvent.refreshJob(taskInstanceId, EventSource.buildStepEventSource(stepInstanceId))); + JobEvent.refreshJob(taskInstanceId, EventSource.buildStepEventSource(taskInstanceId, stepInstanceId))); } else { log.warn("Unsupported step instance status for next step action, stepInstanceId:{}, status:{}", stepInstanceId, stepInstance.getStatus()); @@ -390,6 +437,7 @@ private void ignoreError(StepInstanceDTO stepInstance) { return; } + long taskInstanceId = stepInstance.getTaskInstanceId(); long stepInstanceId = stepInstance.getId(); boolean isRollingStep = stepInstance.isRollingStep(); if (isRollingStep) { @@ -399,21 +447,22 @@ private void ignoreError(StepInstanceDTO stepInstance) { log.info("Retry-fail for step, stepInstanceId={}", stepInstanceId); } - taskInstanceService.updateStepStatus(stepInstance.getId(), RunStatusEnum.IGNORE_ERROR.getValue()); + stepInstanceService.updateStepStatus(taskInstanceId, stepInstance.getId(), + RunStatusEnum.IGNORE_ERROR.getValue()); taskInstanceService.resetTaskExecuteInfoForRetry(stepInstance.getTaskInstanceId()); if (isRollingStep) { StepInstanceRollingTaskDTO stepInstanceRollingTask = - stepInstanceRollingTaskService.queryRollingTask(stepInstanceId, stepInstance.getExecuteCount(), - stepInstance.getBatch()); + stepInstanceRollingTaskService.queryRollingTask(taskInstanceId, stepInstanceId, + stepInstance.getExecuteCount(), stepInstance.getBatch()); if (stepInstanceRollingTask != null) { - finishRollingTask(stepInstanceId, stepInstance.getExecuteCount(), stepInstance.getBatch(), - RunStatusEnum.IGNORE_ERROR); + finishRollingTask(taskInstanceId, stepInstanceId, stepInstance.getExecuteCount(), + stepInstance.getBatch(), RunStatusEnum.IGNORE_ERROR); } } taskExecuteMQEventDispatcher.dispatchJobEvent( JobEvent.refreshJob(stepInstance.getTaskInstanceId(), - EventSource.buildStepEventSource(stepInstance.getId()))); + EventSource.buildStepEventSource(taskInstanceId, stepInstance.getId()))); } @@ -433,13 +482,13 @@ private void skipStep(StepInstanceDTO stepInstance) { // 只有当步骤状态为'终止中'时可以跳过步骤 if (RunStatusEnum.STOPPING == stepStatus) { long now = DateUtils.currentTimeMillis(); - taskInstanceService.updateStepStartTimeIfNull(stepInstanceId, now); - taskInstanceService.updateStepStatus(stepInstanceId, RunStatusEnum.SKIPPED.getValue()); - taskInstanceService.updateStepEndTime(stepInstanceId, now); + stepInstanceService.updateStepStartTimeIfNull(taskInstanceId, stepInstanceId, now); + stepInstanceService.updateStepStatus(taskInstanceId, stepInstanceId, RunStatusEnum.SKIPPED.getValue()); + stepInstanceService.updateStepEndTime(taskInstanceId, stepInstanceId, now); taskInstanceService.updateTaskStatus(taskInstanceId, RunStatusEnum.RUNNING.getValue()); taskExecuteMQEventDispatcher.dispatchJobEvent( - JobEvent.refreshJob(taskInstanceId, EventSource.buildStepEventSource(stepInstanceId))); + JobEvent.refreshJob(taskInstanceId, EventSource.buildStepEventSource(taskInstanceId, stepInstanceId))); } else { log.warn("Unsupported step instance run status for skipping step, stepInstanceId={}, status={}", stepInstanceId, stepStatus); @@ -447,6 +496,7 @@ private void skipStep(StepInstanceDTO stepInstance) { } private void stopStep(StepInstanceDTO stepInstance) { + long taskInstanceId = stepInstance.getTaskInstanceId(); long stepInstanceId = stepInstance.getId(); log.info("Force stop step, stepInstanceId={}", stepInstanceId); @@ -455,14 +505,19 @@ private void stopStep(StepInstanceDTO stepInstance) { log.info("Step status is WAITING_USER, set step status stop_success directly!"); // 等待用户的步骤可以直接结束 long endTime = DateUtils.currentTimeMillis(); - taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.STOP_SUCCESS, - null, endTime, TaskCostCalculator.calculate(stepInstance.getStartTime(), endTime, - stepInstance.getTotalTime())); + stepInstanceService.updateStepExecutionInfo( + taskInstanceId, + stepInstanceId, + RunStatusEnum.STOP_SUCCESS, + null, + endTime, + TaskCostCalculator.calculate(stepInstance.getStartTime(), endTime, stepInstance.getTotalTime())); + taskExecuteMQEventDispatcher.dispatchJobEvent(JobEvent.refreshJob(stepInstance.getTaskInstanceId(), - EventSource.buildStepEventSource(stepInstanceId))); + EventSource.buildStepEventSource(taskInstanceId, stepInstanceId))); } else if (stepStatus == RunStatusEnum.RUNNING) { // 正在运行中的任务无法立即结束，需要等待任务调度引擎检测到停止状态;这里只需要处理设置步骤状态即可 - taskInstanceService.updateStepStatus(stepInstanceId, RunStatusEnum.STOPPING.getValue()); + stepInstanceService.updateStepStatus(taskInstanceId, stepInstanceId, RunStatusEnum.STOPPING.getValue()); } } @@ -475,9 +530,11 @@ private void continueGseFileStep(StepInstanceDTO stepInstance) { log.info("Continue file push step, stepInstanceId={}", stepInstance.getId()); GseTaskDTO gseTask = - gseTaskService.getGseTask(stepInstance.getId(), stepInstance.getExecuteCount(), stepInstance.getBatch()); + gseTaskService.getGseTask(stepInstance.getTaskInstanceId(), stepInstance.getId(), + stepInstance.getExecuteCount(), stepInstance.getBatch()); taskExecuteMQEventDispatcher.dispatchGseTaskEvent(GseTaskEvent.startGseTask( - gseTask.getStepInstanceId(), gseTask.getExecuteCount(), gseTask.getBatch(), gseTask.getId(), null)); + stepInstance.getTaskInstanceId(), gseTask.getStepInstanceId(), gseTask.getExecuteCount(), + gseTask.getBatch(), gseTask.getId(), null)); } /** @@ -504,7 +561,7 @@ private void retryStepFail(StepInstanceDTO stepInstance) { } Long gseTaskId = saveInitialGseTask(stepInstance); - saveAgentTasksForRetryFail(stepInstance, stepInstance.getExecuteCount(), stepInstance.getBatch(), + saveExecuteObjectTasksForRetryFail(stepInstance, stepInstance.getExecuteCount(), stepInstance.getBatch(), gseTaskId); startGseTask(stepInstance, gseTaskId); @@ -519,61 +576,70 @@ private boolean isStepSupportRetry(RunStatusEnum stepStatus) { || RunStatusEnum.STOP_SUCCESS == stepStatus; } - private void saveAgentTasksForRetryFail(StepInstanceBaseDTO stepInstance, int executeCount, Integer batch, - Long gseTaskId) { - List retryAgentTasks = listTargetAgentTasks(stepInstance, executeCount - 1); + private void saveExecuteObjectTasksForRetryFail(StepInstanceBaseDTO stepInstance, + int executeCount, + Integer batch, + Long gseTaskId) { + List retryExecuteObjectTasks = listTargetExecuteObjectTasks(stepInstance, executeCount - 1); - for (AgentTaskDTO retryAgentTask : retryAgentTasks) { - retryAgentTask.setExecuteCount(executeCount); - if (batch != null && retryAgentTask.getBatch() != batch) { + for (ExecuteObjectTask retryExecuteObjectTask : retryExecuteObjectTasks) { + retryExecuteObjectTask.setExecuteCount(executeCount); + if (batch != null && retryExecuteObjectTask.getBatch() != batch) { continue; } // 只有失败的目标主机才需要参与重试 - if (!AgentTaskStatusEnum.isSuccess(retryAgentTask.getStatus())) { - retryAgentTask.setActualExecuteCount(executeCount); - retryAgentTask.resetTaskInitialStatus(); - retryAgentTask.setGseTaskId(gseTaskId); + if (!ExecuteObjectTaskStatusEnum.isSuccess(retryExecuteObjectTask.getStatus())) { + if (retryExecuteObjectTask.getExecuteObject().isExecutable()) { + retryExecuteObjectTask.setActualExecuteCount(executeCount); + retryExecuteObjectTask.resetTaskInitialStatus(); + } + retryExecuteObjectTask.setGseTaskId(gseTaskId); } } - saveAgentTasks(stepInstance, retryAgentTasks); + saveExecuteObjectTasks(stepInstance, retryExecuteObjectTasks); } - private void saveAgentTasksForRetryAll(StepInstanceBaseDTO stepInstance, int executeCount, Integer batch, - Long gseTaskId) { - List retryAgentTasks = listTargetAgentTasks(stepInstance, executeCount - 1); + private void saveExecuteObjectTasksForRetryAll(StepInstanceBaseDTO stepInstance, + int executeCount, + Integer batch, + Long gseTaskId) { + List retryExecuteObjectTasks = listTargetExecuteObjectTasks(stepInstance, executeCount - 1); - for (AgentTaskDTO retryAgentTask : retryAgentTasks) { - retryAgentTask.setExecuteCount(executeCount); - if (batch != null && retryAgentTask.getBatch() != batch) { + for (ExecuteObjectTask retryExecuteObjectTask : retryExecuteObjectTasks) { + retryExecuteObjectTask.setExecuteCount(executeCount); + if (batch != null && retryExecuteObjectTask.getBatch() != batch) { continue; } - retryAgentTask.setActualExecuteCount(executeCount); - retryAgentTask.resetTaskInitialStatus(); - retryAgentTask.setGseTaskId(gseTaskId); + if (retryExecuteObjectTask.getExecuteObject().isExecutable()) { + // 重置运行数据 + retryExecuteObjectTask.setActualExecuteCount(executeCount); + retryExecuteObjectTask.resetTaskInitialStatus(); + } + retryExecuteObjectTask.setGseTaskId(gseTaskId); } - saveAgentTasks(stepInstance, retryAgentTasks); + saveExecuteObjectTasks(stepInstance, retryExecuteObjectTasks); } - private List listTargetAgentTasks(StepInstanceBaseDTO stepInstance, int executeCount) { - List agentTasks = Collections.emptyList(); + private List listTargetExecuteObjectTasks(StepInstanceBaseDTO stepInstance, int executeCount) { + List executeObjectTasks = Collections.emptyList(); if (stepInstance.isScriptStep()) { - agentTasks = scriptAgentTaskService.listAgentTasks(stepInstance.getId(), executeCount, null); + executeObjectTasks = scriptExecuteObjectTaskService.listTasks(stepInstance, executeCount, null); } else if (stepInstance.isFileStep()) { - agentTasks = fileAgentTaskService.listAgentTasks(stepInstance.getId(), executeCount, null, + executeObjectTasks = fileExecuteObjectTaskService.listTasks(stepInstance, executeCount, null, FileTaskModeEnum.DOWNLOAD); } - return agentTasks; + return executeObjectTasks; } - private void saveAgentTasks(StepInstanceBaseDTO stepInstance, List agentTasks) { - if (CollectionUtils.isNotEmpty(agentTasks)) { + private void saveExecuteObjectTasks(StepInstanceBaseDTO stepInstance, List executeObjectTasks) { + if (CollectionUtils.isNotEmpty(executeObjectTasks)) { if (stepInstance.isScriptStep()) { - scriptAgentTaskService.batchSaveAgentTasks(agentTasks); + scriptExecuteObjectTaskService.batchSaveTasks(executeObjectTasks); } else if (stepInstance.isFileStep()) { - fileAgentTaskService.batchSaveAgentTasks(agentTasks); + fileExecuteObjectTaskService.batchSaveTasks(executeObjectTasks); } } } @@ -603,7 +669,7 @@ private void retryStepAll(StepInstanceDTO stepInstance) { } Long gseTaskId = saveInitialGseTask(stepInstance); - saveAgentTasksForRetryAll(stepInstance, stepInstance.getExecuteCount(), stepInstance.getBatch(), + saveExecuteObjectTasksForRetryAll(stepInstance, stepInstance.getExecuteCount(), stepInstance.getBatch(), gseTaskId); startGseTask(stepInstance, gseTaskId); @@ -617,12 +683,10 @@ private void retryStepAll(StepInstanceDTO stepInstance) { * 清理执行完的步骤 */ private void clearStep(StepInstanceDTO stepInstance) { - log.info("Clear step, stepInstanceId={}", stepInstance.getId()); - - int executeType = stepInstance.getExecuteType(); // 当前仅有文件分发类步骤需要清理中间文件 - if (TaskStepTypeEnum.FILE.getValue() == executeType) { - filePrepareService.clearPreparedTmpFile(stepInstance.getId()); + if (stepInstance.isFileStep()) { + log.info("Clear file step, stepInstanceId={}", stepInstance.getId()); + filePrepareService.clearPreparedTmpFile(stepInstance.getTaskInstanceId(), stepInstance.getId()); } } @@ -635,7 +699,7 @@ private void resetExecutionInfoForRetry(StepInstanceDTO stepInstance) { long stepInstanceId = stepInstance.getId(); long taskInstanceId = stepInstance.getTaskInstanceId(); - taskInstanceService.resetStepExecuteInfoForRetry(stepInstanceId); + stepInstanceService.resetStepExecuteInfoForRetry(taskInstanceId, stepInstanceId); taskInstanceService.resetTaskExecuteInfoForRetry(taskInstanceId); } @@ -644,7 +708,7 @@ private void refreshStep(StepEvent stepEvent, StepInstanceDTO stepInstance) { long stepInstanceId = stepInstance.getId(); EventSource eventSource = stepEvent.getSource(); - GseTaskDTO gseTask = gseTaskService.getGseTask(eventSource.getGseTaskId()); + GseTaskDTO gseTask = gseTaskService.getGseTask(eventSource.getJobInstanceId(), eventSource.getGseTaskId()); RunStatusEnum gseTaskStatus = RunStatusEnum.valueOf(gseTask.getStatus()); log.info("Refresh step according to gse task status, stepInstanceId: {}, gseTaskStatus: {}", @@ -661,22 +725,27 @@ private void refreshStep(StepEvent stepEvent, StepInstanceDTO stepInstance) { onStopSuccess(stepInstance); break; case ABNORMAL_STATE: - onAbnormalState(stepInstance); + finishStepWithAbnormalState(stepInstance); + break; + case ABANDONED: + onAbandonState(stepInstance); break; default: log.error("Refresh step fail because of unexpected gse task status. stepInstanceId: {}, " + "gseTaskStatus: {}", stepInstanceId, gseTaskStatus.getValue()); - setAbnormalStatusForStep(stepInstance); + finishStep(stepInstance, RunStatusEnum.ABNORMAL_STATE); break; } // 更新作业状态 taskExecuteMQEventDispatcher.dispatchJobEvent( - JobEvent.refreshJob(stepInstance.getTaskInstanceId(), EventSource.buildStepEventSource(stepInstanceId))); + JobEvent.refreshJob(stepInstance.getTaskInstanceId(), + EventSource.buildStepEventSource(stepInstance.getTaskInstanceId(), stepInstanceId))); } private void onSuccess(StepInstanceDTO stepInstance) { + long taskInstanceId = stepInstance.getTaskInstanceId(); long stepInstanceId = stepInstance.getId(); long endTime = System.currentTimeMillis(); long startTime = stepInstance.getStartTime(); @@ -684,22 +753,23 @@ private void onSuccess(StepInstanceDTO stepInstance) { if (stepInstance.isRollingStep()) { RollingConfigDTO rollingConfig = - rollingConfigService.getRollingConfig(stepInstance.getRollingConfigId()); - finishRollingTask(stepInstanceId, stepInstance.getExecuteCount(), stepInstance.getBatch(), + rollingConfigService.getRollingConfig(stepInstance.getTaskInstanceId(), + stepInstance.getRollingConfigId()); + finishRollingTask(taskInstanceId, stepInstanceId, stepInstance.getExecuteCount(), stepInstance.getBatch(), RunStatusEnum.SUCCESS); int totalBatch = rollingConfig.getConfigDetail().getTotalBatch(); boolean isLastBatch = totalBatch == stepInstance.getBatch(); if (isLastBatch) { - taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.SUCCESS, + stepInstanceService.updateStepExecutionInfo(taskInstanceId, stepInstanceId, RunStatusEnum.SUCCESS, startTime, endTime, totalTime); // 步骤执行成功后清理产生的临时文件 clearStep(stepInstance); } else { - taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.ROLLING_WAITING, - startTime, endTime, totalTime); + stepInstanceService.updateStepExecutionInfo(taskInstanceId, stepInstanceId, + RunStatusEnum.ROLLING_WAITING, startTime, endTime, totalTime); } } else { - taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.SUCCESS, + stepInstanceService.updateStepExecutionInfo(taskInstanceId, stepInstanceId, RunStatusEnum.SUCCESS, startTime, endTime, totalTime); // 步骤执行成功后清理产生的临时文件 clearStep(stepInstance); @@ -707,50 +777,53 @@ private void onSuccess(StepInstanceDTO stepInstance) { } private void onFail(StepInstanceDTO stepInstance) { + long taskInstanceId = stepInstance.getTaskInstanceId(); long stepInstanceId = stepInstance.getId(); long endTime = System.currentTimeMillis(); long startTime = stepInstance.getStartTime(); long totalTime = endTime - startTime; if (stepInstance.isIgnoreError()) { log.info("Ignore error for step: {}", stepInstanceId); - taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.IGNORE_ERROR, + stepInstanceService.updateStepExecutionInfo(taskInstanceId, stepInstanceId, RunStatusEnum.IGNORE_ERROR, startTime, endTime, totalTime); if (stepInstance.isRollingStep()) { - finishRollingTask(stepInstanceId, stepInstance.getExecuteCount(), stepInstance.getBatch(), - RunStatusEnum.IGNORE_ERROR); + finishRollingTask(taskInstanceId, stepInstanceId, stepInstance.getExecuteCount(), + stepInstance.getBatch(), RunStatusEnum.IGNORE_ERROR); } return; } if (stepInstance.isRollingStep()) { RollingConfigDTO rollingConfig = - rollingConfigService.getRollingConfig(stepInstance.getRollingConfigId()); + rollingConfigService.getRollingConfig(stepInstance.getTaskInstanceId(), + stepInstance.getRollingConfigId()); RollingModeEnum rollingMode = RollingModeEnum.valOf(rollingConfig.getConfigDetail().getMode()); switch (rollingMode) { case IGNORE_ERROR: log.info("Ignore error for rolling step, rollingMode: {}", rollingMode); - finishRollingTask(stepInstanceId, stepInstance.getExecuteCount(), stepInstance.getBatch(), - RunStatusEnum.IGNORE_ERROR); - taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.IGNORE_ERROR, - startTime, endTime, totalTime); + finishRollingTask(taskInstanceId, stepInstanceId, stepInstance.getExecuteCount(), + stepInstance.getBatch(), RunStatusEnum.IGNORE_ERROR); + stepInstanceService.updateStepExecutionInfo(taskInstanceId, stepInstanceId, + RunStatusEnum.IGNORE_ERROR, startTime, endTime, totalTime); break; case PAUSE_IF_FAIL: case MANUAL: - finishRollingTask(stepInstanceId, stepInstance.getExecuteCount(), stepInstance.getBatch(), - RunStatusEnum.FAIL); - taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.FAIL, + finishRollingTask(taskInstanceId, stepInstanceId, stepInstance.getExecuteCount(), + stepInstance.getBatch(), RunStatusEnum.FAIL); + stepInstanceService.updateStepExecutionInfo(taskInstanceId, stepInstanceId, RunStatusEnum.FAIL, startTime, endTime, totalTime); break; default: log.error("Invalid rolling mode: {}", rollingMode); } } else { - taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.FAIL, + stepInstanceService.updateStepExecutionInfo(taskInstanceId, stepInstanceId, RunStatusEnum.FAIL, startTime, endTime, totalTime); } } private void onStopSuccess(StepInstanceDTO stepInstance) { + long taskInstanceId = stepInstance.getTaskInstanceId(); long stepInstanceId = stepInstance.getId(); long endTime = System.currentTimeMillis(); long startTime = stepInstance.getStartTime(); @@ -758,11 +831,11 @@ private void onStopSuccess(StepInstanceDTO stepInstance) { RunStatusEnum stepStatus = stepInstance.getStatus(); if (stepStatus == RunStatusEnum.STOPPING || stepStatus == RunStatusEnum.RUNNING) { - taskInstanceService.updateStepExecutionInfo(stepInstanceId, RunStatusEnum.STOP_SUCCESS, + stepInstanceService.updateStepExecutionInfo(taskInstanceId, stepInstanceId, RunStatusEnum.STOP_SUCCESS, startTime, endTime, totalTime); if (stepInstance.isRollingStep()) { - finishRollingTask(stepInstanceId, stepInstance.getExecuteCount(), stepInstance.getBatch(), - RunStatusEnum.STOP_SUCCESS); + finishRollingTask(taskInstanceId, stepInstanceId, stepInstance.getExecuteCount(), + stepInstance.getBatch(), RunStatusEnum.STOP_SUCCESS); } } else { log.error("Refresh step fail, stepInstanceId: {}, stepStatus: {}, gseTaskStatus: {}", @@ -770,38 +843,29 @@ private void onStopSuccess(StepInstanceDTO stepInstance) { } } - private void onAbnormalState(StepInstanceDTO stepInstance) { - setAbnormalStatusForStep(stepInstance); + protected void finishStepWithAbnormalState(StepInstanceDTO stepInstance) { + super.finishStepWithAbnormalState(stepInstance); if (stepInstance.isRollingStep()) { - finishRollingTask(stepInstance.getId(), stepInstance.getExecuteCount(), stepInstance.getBatch(), - RunStatusEnum.ABNORMAL_STATE); + finishRollingTask(stepInstance.getTaskInstanceId(), stepInstance.getId(), stepInstance.getExecuteCount(), + stepInstance.getBatch(), RunStatusEnum.ABNORMAL_STATE); } } - private void setAbnormalStatusForStep(StepInstanceDTO stepInstance) { - long endTime = System.currentTimeMillis(); - if (!RunStatusEnum.isFinishedStatus(stepInstance.getStatus())) { - long totalTime = TaskCostCalculator.calculate(stepInstance.getStartTime(), endTime, - stepInstance.getTotalTime()); - taskInstanceService.updateStepExecutionInfo( - stepInstance.getId(), - RunStatusEnum.ABNORMAL_STATE, - null, - endTime, - totalTime - ); - } else { - log.info( - "StepInstance {} already enter a final state:{}", - stepInstance.getId(), - stepInstance.getStatus() - ); + protected void onAbandonState(StepInstanceDTO stepInstance) { + finishStep(stepInstance, RunStatusEnum.ABANDONED); + if (stepInstance.isRollingStep()) { + finishRollingTask(stepInstance.getTaskInstanceId(), stepInstance.getId(), stepInstance.getExecuteCount(), + stepInstance.getBatch(), RunStatusEnum.ABANDONED); } } - private void finishRollingTask(long stepInstanceId, int executeCount, int batch, RunStatusEnum status) { + private void finishRollingTask(Long taskInstanceId, + long stepInstanceId, + int executeCount, + int batch, + RunStatusEnum status) { StepInstanceRollingTaskDTO rollingTask = - stepInstanceRollingTaskService.queryRollingTask(stepInstanceId, executeCount, batch); + stepInstanceRollingTaskService.queryRollingTask(taskInstanceId, stepInstanceId, executeCount, batch); if (rollingTask == null) { log.error("Rolling task is not exist, skip update! stepInstanceId: {}, executeCount: {}, batch: {}", stepInstanceId, executeCount, batch); @@ -810,7 +874,7 @@ private void finishRollingTask(long stepInstanceId, int executeCount, int batch, long now = System.currentTimeMillis(); long startTime = rollingTask.getStartTime() != null ? rollingTask.getStartTime() : now; - stepInstanceRollingTaskService.updateRollingTask(stepInstanceId, executeCount, + stepInstanceRollingTaskService.updateRollingTask(taskInstanceId, stepInstanceId, executeCount, batch, status, startTime, now, now - startTime); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseTaskListener.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseTaskListener.java index 150c77210c..34703bb551 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseTaskListener.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/GseTaskListener.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,53 +24,59 @@ package com.tencent.bk.job.execute.engine.listener; +import com.tencent.bk.job.execute.common.constants.RunStatusEnum; import com.tencent.bk.job.execute.common.exception.MessageHandleException; import com.tencent.bk.job.execute.common.exception.MessageHandlerUnavailableException; import com.tencent.bk.job.execute.engine.consts.GseTaskActionEnum; import com.tencent.bk.job.execute.engine.executor.GseTaskManager; +import com.tencent.bk.job.execute.engine.listener.event.EventSource; import com.tencent.bk.job.execute.engine.listener.event.GseTaskEvent; -import com.tencent.bk.job.execute.engine.message.GseTaskProcessor; +import com.tencent.bk.job.execute.engine.listener.event.JobMessage; +import com.tencent.bk.job.execute.engine.listener.event.StepEvent; +import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; import com.tencent.bk.job.execute.model.GseTaskDTO; import com.tencent.bk.job.execute.monitor.metrics.GseTasksExceptionCounter; import com.tencent.bk.job.execute.service.GseTaskService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.cloud.stream.annotation.EnableBinding; -import org.springframework.cloud.stream.annotation.StreamListener; -import org.springframework.messaging.handler.annotation.Payload; +import org.springframework.messaging.Message; import org.springframework.stereotype.Component; /** * 执行引擎事件处理-GSE任务 */ @Component -@EnableBinding({GseTaskProcessor.class}) @Slf4j -public class GseTaskListener { +public class GseTaskListener extends BaseJobMqListener { private final GseTaskManager gseTaskManager; private final GseTaskService gseTaskService; private final GseTasksExceptionCounter gseTasksExceptionCounter; + private final TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher; @Autowired public GseTaskListener(GseTaskManager gseTaskManager, GseTaskService gseTaskService, - GseTasksExceptionCounter gseTasksExceptionCounter) { + GseTasksExceptionCounter gseTasksExceptionCounter, + TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher) { this.gseTaskManager = gseTaskManager; this.gseTaskService = gseTaskService; this.gseTasksExceptionCounter = gseTasksExceptionCounter; + this.taskExecuteMQEventDispatcher = taskExecuteMQEventDispatcher; } /** * 处理GSE任务相关的事件 * - * @param gseTaskEvent GSE任务事件 + * @param message 消息 */ - @StreamListener(GseTaskProcessor.INPUT) - public void handleEvent(@Payload GseTaskEvent gseTaskEvent) { - log.info("Handel gse task event: {}", gseTaskEvent); - GseTaskDTO gseTask = gseTaskService.getGseTask(gseTaskEvent.getGseTaskId()); - String requestId = gseTaskEvent.getRequestId(); + @Override + public void handleEvent(Message message) { + GseTaskEvent gseTaskEvent = (GseTaskEvent) message.getPayload(); + GseTaskDTO gseTask = null; try { + log.info("Handle gse task event: {}, duration: {}ms", gseTaskEvent, gseTaskEvent.duration()); + gseTask = gseTaskService.getGseTask(gseTaskEvent.getJobInstanceId(), gseTaskEvent.getGseTaskId()); + String requestId = gseTaskEvent.getRequestId(); int action = gseTaskEvent.getAction(); if (GseTaskActionEnum.START.getValue() == action) { gseTaskManager.startTask(gseTask, requestId); @@ -82,15 +88,43 @@ public void handleEvent(@Payload GseTaskEvent gseTaskEvent) { } catch (Throwable e) { String errorMsg = "Handling gse task event error, event:" + gseTaskEvent; log.error(errorMsg, e); - handleException(e); + handleException(gseTask, e); } } - private void handleException(Throwable e) throws MessageHandleException { + private void handleException(GseTaskDTO gseTask, Throwable e) throws MessageHandleException { // 服务关闭，消息被拒绝，重新入队列 if (e instanceof MessageHandlerUnavailableException) { throw (MessageHandlerUnavailableException) e; } + + updateGseTaskResult(gseTask); + gseTasksExceptionCounter.increment(); + + taskExecuteMQEventDispatcher.dispatchStepEvent( + StepEvent.refreshStep( + gseTask.getTaskInstanceId(), + gseTask.getStepInstanceId(), + EventSource.buildGseTaskEventSource( + gseTask.getTaskInstanceId(), + gseTask.getStepInstanceId(), + gseTask.getExecuteCount(), + gseTask.getBatch(), + gseTask.getId() + ) + ) + ); + } + + private void updateGseTaskResult(GseTaskDTO gseTask) { + gseTask.setStatus(RunStatusEnum.ABNORMAL_STATE.getValue()); + if (gseTask.getStartTime() == null) { + gseTask.setStartTime(System.currentTimeMillis()); + } + long endTime = System.currentTimeMillis(); + gseTask.setEndTime(endTime); + gseTask.setTotalTime(endTime - gseTask.getStartTime()); + gseTaskService.updateGseTask(gseTask); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/JobListener.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/JobListener.java index ef20a51c86..55c5a800d8 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/JobListener.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/JobListener.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -31,10 +31,11 @@ import com.tencent.bk.job.execute.common.util.TaskCostCalculator; import com.tencent.bk.job.execute.engine.consts.JobActionEnum; import com.tencent.bk.job.execute.engine.listener.event.JobEvent; +import com.tencent.bk.job.execute.engine.listener.event.JobMessage; import com.tencent.bk.job.execute.engine.listener.event.StepEvent; import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; -import com.tencent.bk.job.execute.engine.message.TaskProcessor; import com.tencent.bk.job.execute.engine.model.JobCallbackDTO; +import com.tencent.bk.job.execute.engine.quota.limit.RunningJobResourceQuotaManager; import com.tencent.bk.job.execute.model.RollingConfigDTO; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; @@ -44,11 +45,11 @@ import com.tencent.bk.job.execute.service.StepInstanceService; import com.tencent.bk.job.execute.service.TaskInstanceService; import com.tencent.bk.job.execute.statistics.StatisticsService; +import com.tencent.bk.job.manage.GlobalAppScopeMappingService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.cloud.stream.annotation.EnableBinding; -import org.springframework.cloud.stream.annotation.StreamListener; +import org.springframework.messaging.Message; import org.springframework.stereotype.Component; import java.util.Collection; @@ -57,10 +58,9 @@ /** * 执行引擎事件处理-作业 */ -@Component -@EnableBinding({TaskProcessor.class}) +@Component("jobEventListener") @Slf4j -public class JobListener { +public class JobListener extends BaseJobMqListener { private final TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher; private final StatisticsService statisticsService; @@ -69,34 +69,40 @@ public class JobListener { private final RollingConfigService rollingConfigService; private final NotifyService notifyService; + private final RunningJobResourceQuotaManager runningJobResourceQuotaManager; + @Autowired public JobListener(TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher, StatisticsService statisticsService, TaskInstanceService taskInstanceService, StepInstanceService stepInstanceService, RollingConfigService rollingConfigService, - NotifyService notifyService) { + NotifyService notifyService, + RunningJobResourceQuotaManager runningJobResourceQuotaManager) { this.taskExecuteMQEventDispatcher = taskExecuteMQEventDispatcher; this.statisticsService = statisticsService; this.taskInstanceService = taskInstanceService; this.stepInstanceService = stepInstanceService; this.rollingConfigService = rollingConfigService; this.notifyService = notifyService; + this.runningJobResourceQuotaManager = runningJobResourceQuotaManager; } /** * 处理作业执行相关的事件 * - * @param jobEvent 作业执行相关的事件 + * @param message 消息 */ - @StreamListener(TaskProcessor.INPUT) - public void handleEvent(JobEvent jobEvent) { - log.info("Handle job event, event: {}", jobEvent); + @Override + public void handleEvent(Message message) { + JobEvent jobEvent = (JobEvent) message.getPayload(); + log.info("Handle job event, event: {}, duration: {}ms", jobEvent, jobEvent.duration()); long jobInstanceId = jobEvent.getJobInstanceId(); JobActionEnum action = JobActionEnum.valueOf(jobEvent.getAction()); + TaskInstanceDTO taskInstance; try { - TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(jobInstanceId); + taskInstance = taskInstanceService.getTaskInstance(jobInstanceId); switch (action) { case START: startJob(taskInstance); @@ -104,9 +110,6 @@ public void handleEvent(JobEvent jobEvent) { case STOP: stopJob(taskInstance); break; - case RESTART: - restartJob(taskInstance); - break; case REFRESH: refreshJob(taskInstance); break; @@ -128,7 +131,7 @@ private void startJob(TaskInstanceDTO taskInstance) { long jobInstanceId = taskInstance.getId(); // 首先验证作业的状态，只有状态为“未执行”的作业可以启动 if (RunStatusEnum.BLANK == taskInstance.getStatus()) { - StepInstanceBaseDTO stepInstance = taskInstanceService.getFirstStepInstance(jobInstanceId); + StepInstanceBaseDTO stepInstance = stepInstanceService.getFirstStepInstance(jobInstanceId); taskInstanceService.updateTaskExecutionInfo(jobInstanceId, RunStatusEnum.RUNNING, stepInstance.getId(), DateUtils.currentTimeMillis(), null, null); startStep(stepInstance); @@ -153,42 +156,14 @@ private void stopJob(TaskInstanceDTO taskInstance) { if (RunStatusEnum.RUNNING == taskStatus || RunStatusEnum.WAITING_USER == taskStatus) { taskInstanceService.updateTaskStatus(jobInstanceId, RunStatusEnum.STOPPING.getValue()); long currentStepInstanceId = taskInstance.getCurrentStepInstanceId(); - taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.stopStep(currentStepInstanceId)); + taskExecuteMQEventDispatcher.dispatchStepEvent( + StepEvent.stopStep(jobInstanceId, currentStepInstanceId)); } else { log.warn("Unsupported job instance run status for stop task, jobInstanceId={}, status={}", jobInstanceId, taskInstance.getStatus()); } } - /** - * 重头执行作业 - * - * @param taskInstance 作业实例 - */ - private void restartJob(TaskInstanceDTO taskInstance) { - long jobInstanceId = taskInstance.getId(); - RunStatusEnum taskStatus = taskInstance.getStatus(); - // 验证作业状态，只有“执行失败”和“等待用户”的作业可以重头执行 - if (RunStatusEnum.WAITING_USER == taskStatus || RunStatusEnum.FAIL == taskStatus) { - - // 重置作业状态 - taskInstanceService.resetTaskStatus(jobInstanceId); - taskInstanceService.addStepInstanceExecuteCount(jobInstanceId); - - // 重置作业下步骤的状态、开始时间和结束时间等。 - List stepInstanceIdList = taskInstanceService.getTaskStepIdList(jobInstanceId); - for (long stepInstanceId : stepInstanceIdList) { - taskInstanceService.updateStepStatus(stepInstanceId, RunStatusEnum.BLANK.getValue()); - taskInstanceService.resetStepStatus(stepInstanceId); - } - - taskExecuteMQEventDispatcher.dispatchJobEvent(JobEvent.startJob(jobInstanceId)); - } else { - log.warn("Unsupported job instance run status for restart task, jobInstanceId={}, status={}", - jobInstanceId, taskInstance.getStatus()); - } - } - /** * 作业状态流转 * @@ -199,7 +174,8 @@ private void refreshJob(TaskInstanceDTO taskInstance) { RunStatusEnum taskStatus = taskInstance.getStatus(); long currentStepInstanceId = taskInstance.getCurrentStepInstanceId(); - StepInstanceBaseDTO currentStepInstance = taskInstanceService.getBaseStepInstance(currentStepInstanceId); + StepInstanceBaseDTO currentStepInstance = stepInstanceService.getBaseStepInstance( + jobInstanceId, currentStepInstanceId); RunStatusEnum stepStatus = currentStepInstance.getStatus(); // 验证作业状态，只有'正在执行'、'强制终止中'的作业可以刷新状态进入下一步或者结束 @@ -218,7 +194,9 @@ private void refreshJob(TaskInstanceDTO taskInstance) { || RunStatusEnum.IGNORE_ERROR == stepStatus || RunStatusEnum.ROLLING_WAITING == stepStatus) { nextStep(taskInstance, currentStepInstance); - } else if (RunStatusEnum.FAIL == stepStatus || RunStatusEnum.ABNORMAL_STATE == stepStatus) { + } else if (RunStatusEnum.FAIL == stepStatus + || RunStatusEnum.ABNORMAL_STATE == stepStatus + || RunStatusEnum.ABANDONED == stepStatus) { // 步骤失败，任务结束 finishJob(taskInstance, currentStepInstance, stepStatus); } else { @@ -234,7 +212,8 @@ private void refreshJob(TaskInstanceDTO taskInstance) { private void nextStep(TaskInstanceDTO taskInstance, StepInstanceBaseDTO currentStepInstance) { if (currentStepInstance.isRollingStep()) { RollingConfigDTO taskInstanceRollingConfig = - rollingConfigService.getRollingConfig(currentStepInstance.getRollingConfigId()); + rollingConfigService.getRollingConfig(currentStepInstance.getTaskInstanceId(), + currentStepInstance.getRollingConfigId()); RollingConfigDetailDO rollingConfig = taskInstanceRollingConfig.getConfigDetail(); StepInstanceBaseDTO nextStepInstance = getNextStepInstance(taskInstance, currentStepInstance, rollingConfig); @@ -245,7 +224,7 @@ private void nextStep(TaskInstanceDTO taskInstance, StepInstanceBaseDTO currentS && rollingConfig.isFirstRollingStep(nextStepInstance.getId())) { log.info("Manual mode for rolling step[{}], pause and wait for user confirmation", nextStepInstance.getId()); - taskInstanceService.updateStepStatus(nextStepInstance.getId(), + stepInstanceService.updateStepStatus(taskInstance.getId(), nextStepInstance.getId(), RunStatusEnum.WAITING_USER.getValue()); taskInstanceService.updateTaskStatus(taskInstance.getId(), RunStatusEnum.WAITING_USER.getValue()); } else { @@ -269,27 +248,36 @@ private void finishJob(TaskInstanceDTO taskInstance, StepInstanceBaseDTO stepInstance, RunStatusEnum jobStatus) { long jobInstanceId = taskInstance.getId(); - long stepInstanceId = stepInstance.getId(); - Long endTime = DateUtils.currentTimeMillis(); - long totalTime = TaskCostCalculator.calculate(taskInstance.getStartTime(), endTime, - taskInstance.getTotalTime()); - taskInstance.setEndTime(endTime); - taskInstance.setTotalTime(totalTime); - taskInstance.setStatus(jobStatus); - taskInstanceService.updateTaskExecutionInfo(jobInstanceId, jobStatus, null, null, endTime, totalTime); + try { + Long endTime = DateUtils.currentTimeMillis(); + long totalTime = TaskCostCalculator.calculate(taskInstance.getStartTime(), endTime, + taskInstance.getTotalTime()); + taskInstance.setEndTime(endTime); + taskInstance.setTotalTime(totalTime); + taskInstance.setStatus(jobStatus); + taskInstanceService.updateTaskExecutionInfo(jobInstanceId, jobStatus, null, null, endTime, totalTime); - // 作业执行结果消息通知 - if (RunStatusEnum.SUCCESS == jobStatus || RunStatusEnum.IGNORE_ERROR == jobStatus) { - notifyService.asyncSendMQSuccessTaskNotification(taskInstance, stepInstance); - } else { - notifyService.asyncSendMQFailTaskNotification(taskInstance, stepInstance); - } + // 作业执行结果消息通知 + if (RunStatusEnum.SUCCESS == jobStatus || RunStatusEnum.IGNORE_ERROR == jobStatus) { + notifyService.asyncSendMQSuccessTaskNotification(taskInstance, stepInstance); + } else { + notifyService.asyncSendMQFailTaskNotification(taskInstance, stepInstance); + } + + // 触发作业结束统计分析 + statisticsService.updateEndJobStatistics(taskInstance); - // 触发作业结束统计分析 - statisticsService.updateEndJobStatistics(taskInstance); + // 作业执行完成回调 + callback(taskInstance); + } finally { + // 从资源配额中删除该作业实例 + runningJobResourceQuotaManager.removeJob( + taskInstance.getAppCode(), + GlobalAppScopeMappingService.get().getScopeByAppId(taskInstance.getAppId()), + jobInstanceId + ); + } - // 作业执行完成回调 - callback(taskInstance, jobInstanceId, jobStatus.getValue(), stepInstanceId, stepInstance.getStatus()); } /** @@ -323,7 +311,8 @@ private StepInstanceBaseDTO getNextStepInstance(TaskInstanceDTO taskInstance, currentStepInstance.getId()); } if (nextRollingStepInstanceId != null) { - nextStepInstance = taskInstanceService.getBaseStepInstance(nextRollingStepInstanceId); + nextStepInstance = stepInstanceService.getBaseStepInstance(taskInstance.getId(), + nextRollingStepInstanceId); } } } else { @@ -345,29 +334,36 @@ private Long getNextStepInstanceId(List stepInstanceIdList, long currentSt private void startStep(StepInstanceBaseDTO stepInstance) { taskInstanceService.updateTaskCurrentStepId(stepInstance.getTaskInstanceId(), stepInstance.getId()); if (stepInstance.isRollingStep()) { - taskInstanceService.updateStepStatus(stepInstance.getId(), RunStatusEnum.ROLLING_WAITING.getValue()); - taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.startStep(stepInstance.getId(), - stepInstance.getBatch() + 1)); + stepInstanceService.updateStepStatus(stepInstance.getTaskInstanceId(), + stepInstance.getId(), RunStatusEnum.ROLLING_WAITING.getValue()); + taskExecuteMQEventDispatcher.dispatchStepEvent( + StepEvent.startStep(stepInstance.getTaskInstanceId(), stepInstance.getId(), + stepInstance.getBatch() + 1)); } else { - taskInstanceService.updateStepStatus(stepInstance.getId(), RunStatusEnum.BLANK.getValue()); - taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.startStep(stepInstance.getId(), null)); + stepInstanceService.updateStepStatus(stepInstance.getTaskInstanceId(), + stepInstance.getId(), RunStatusEnum.BLANK.getValue()); + taskExecuteMQEventDispatcher.dispatchStepEvent( + StepEvent.startStep(stepInstance.getTaskInstanceId(), stepInstance.getId(), null)); } } - private void callback(TaskInstanceDTO taskInstance, long jobInstanceId, int taskStatus, long currentStepId, - RunStatusEnum stepStatus) { + private void callback(TaskInstanceDTO taskInstance) { if (StringUtils.isNotBlank(taskInstance.getCallbackUrl())) { JobCallbackDTO callback = new JobCallbackDTO(); - callback.setId(jobInstanceId); - callback.setStatus(taskStatus); + callback.setId(taskInstance.getId()); + callback.setStatus(taskInstance.getStatus().getValue()); callback.setCallbackUrl(taskInstance.getCallbackUrl()); - Collection stepInstanceList = Lists.newArrayList(); - JobCallbackDTO.StepInstanceStatus stepInstance = new JobCallbackDTO.StepInstanceStatus(); - stepInstance.setId(currentStepId); - stepInstance.setStatus(stepStatus.getValue()); - stepInstanceList.add(stepInstance); - callback.setStepInstances(stepInstanceList); - taskExecuteMQEventDispatcher.sendCallback(callback); + Collection stepInstanceStatusList = Lists.newArrayList(); + List stepInstanceList = + stepInstanceService.listBaseStepInstanceByTaskInstanceId(taskInstance.getId()); + stepInstanceList.forEach(stepInstance -> { + JobCallbackDTO.StepInstanceStatus stepInstanceStatus = new JobCallbackDTO.StepInstanceStatus(); + stepInstanceStatus.setId(stepInstance.getId()); + stepInstanceStatus.setStatus(stepInstance.getStatus().getValue()); + stepInstanceStatusList.add(stepInstanceStatus); + }); + callback.setStepInstances(stepInstanceStatusList); + taskExecuteMQEventDispatcher.dispatchCallbackMsg(callback); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/NotifyMsgListener.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/NotifyMsgListener.java index 8ac740df95..0ea64c6515 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/NotifyMsgListener.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/NotifyMsgListener.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,23 +25,21 @@ package com.tencent.bk.job.execute.engine.listener; import com.tencent.bk.job.common.util.json.JsonUtils; -import com.tencent.bk.job.execute.engine.message.NotifyMsgProcessor; +import com.tencent.bk.job.execute.engine.listener.event.JobMessage; import com.tencent.bk.job.execute.model.TaskNotifyDTO; import com.tencent.bk.job.execute.service.NotifyService; -import com.tencent.bk.job.manage.common.consts.notify.ExecuteStatusEnum; +import com.tencent.bk.job.manage.api.common.constants.notify.ExecuteStatusEnum; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.cloud.stream.annotation.EnableBinding; -import org.springframework.cloud.stream.annotation.StreamListener; +import org.springframework.messaging.Message; import org.springframework.stereotype.Component; /** * 任务执行结果通知处理 */ @Component -@EnableBinding({NotifyMsgProcessor.class}) @Slf4j -public class NotifyMsgListener { +public class NotifyMsgListener extends BaseJobMqListener { private final NotifyService notifyService; @Autowired @@ -49,8 +47,9 @@ public NotifyMsgListener(NotifyService notifyService) { this.notifyService = notifyService; } - @StreamListener(NotifyMsgProcessor.INPUT) - public void handleMessage(TaskNotifyDTO taskNotifyDTO) { + @Override + public void handleEvent(Message message) { + TaskNotifyDTO taskNotifyDTO = (TaskNotifyDTO) message.getPayload(); log.info("Begin to send msg:{}", JsonUtils.toJson(taskNotifyDTO)); ExecuteStatusEnum executeStatus = ExecuteStatusEnum.get(taskNotifyDTO.getResourceExecuteStatus()); if (executeStatus == null) { diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/ResultHandleResumeListener.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/ResultHandleResumeListener.java index d6d06d6bc1..95b828c286 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/ResultHandleResumeListener.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/ResultHandleResumeListener.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,13 +24,14 @@ package com.tencent.bk.job.execute.engine.listener; +import com.tencent.bk.job.common.gse.v2.model.ExecuteObjectGseKey; +import com.tencent.bk.job.common.util.FilePathUtils; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; -import com.tencent.bk.job.execute.config.StorageSystemConfig; -import com.tencent.bk.job.execute.engine.consts.FileDirTypeConf; -import com.tencent.bk.job.execute.engine.evict.TaskEvictPolicyExecutor; +import com.tencent.bk.job.execute.config.FileDistributeConfig; +import com.tencent.bk.job.execute.config.JobExecuteConfig; +import com.tencent.bk.job.execute.engine.EngineDependentServiceHolder; +import com.tencent.bk.job.execute.engine.listener.event.JobMessage; import com.tencent.bk.job.execute.engine.listener.event.ResultHandleTaskResumeEvent; -import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; -import com.tencent.bk.job.execute.engine.message.TaskResultHandleResumeProcessor; import com.tencent.bk.job.execute.engine.model.FileDest; import com.tencent.bk.job.execute.engine.model.JobFile; import com.tencent.bk.job.execute.engine.model.TaskVariableDTO; @@ -38,28 +39,21 @@ import com.tencent.bk.job.execute.engine.result.FileResultHandleTask; import com.tencent.bk.job.execute.engine.result.ResultHandleManager; import com.tencent.bk.job.execute.engine.result.ScriptResultHandleTask; -import com.tencent.bk.job.execute.engine.result.ha.ResultHandleTaskKeepaliveManager; -import com.tencent.bk.job.execute.engine.util.FilePathUtils; import com.tencent.bk.job.execute.engine.util.JobSrcFileUtils; -import com.tencent.bk.job.execute.engine.util.NFSUtils; -import com.tencent.bk.job.execute.model.AgentTaskDTO; +import com.tencent.bk.job.execute.model.ExecuteObjectTask; import com.tencent.bk.job.execute.model.GseTaskDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; -import com.tencent.bk.job.execute.service.AgentService; -import com.tencent.bk.job.execute.service.FileAgentTaskService; +import com.tencent.bk.job.execute.service.FileExecuteObjectTaskService; import com.tencent.bk.job.execute.service.GseTaskService; -import com.tencent.bk.job.execute.service.LogService; -import com.tencent.bk.job.execute.service.ScriptAgentTaskService; +import com.tencent.bk.job.execute.service.ScriptExecuteObjectTaskService; import com.tencent.bk.job.execute.service.StepInstanceService; -import com.tencent.bk.job.execute.service.StepInstanceVariableValueService; import com.tencent.bk.job.execute.service.TaskInstanceService; import com.tencent.bk.job.execute.service.TaskInstanceVariableService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.cloud.stream.annotation.EnableBinding; -import org.springframework.cloud.stream.annotation.StreamListener; +import org.springframework.messaging.Message; import org.springframework.stereotype.Component; import java.util.HashMap; @@ -72,9 +66,10 @@ * 执行引擎事件处理-任务恢复 */ @Component -@EnableBinding({TaskResultHandleResumeProcessor.class}) @Slf4j -public class ResultHandleResumeListener { +public class ResultHandleResumeListener extends BaseJobMqListener { + + private final EngineDependentServiceHolder engineDependentServiceHolder; private final TaskInstanceService taskInstanceService; private final ResultHandleManager resultHandleManager; @@ -83,71 +78,54 @@ public class ResultHandleResumeListener { private final GseTaskService gseTaskService; - private final StorageSystemConfig storageSystemConfig; - - private final AgentService agentService; - - private final LogService logService; - - private final StepInstanceVariableValueService stepInstanceVariableValueService; - - private final TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher; - - private final ResultHandleTaskKeepaliveManager resultHandleTaskKeepaliveManager; + private final FileDistributeConfig fileDistributeConfig; - private final TaskEvictPolicyExecutor taskEvictPolicyExecutor; + private final ScriptExecuteObjectTaskService scriptExecuteObjectTaskService; - private final ScriptAgentTaskService scriptAgentTaskService; - - private final FileAgentTaskService fileAgentTaskService; + private final FileExecuteObjectTaskService fileExecuteObjectTaskService; private final StepInstanceService stepInstanceService; + private final JobExecuteConfig jobExecuteConfig; + @Autowired - public ResultHandleResumeListener(TaskInstanceService taskInstanceService, + public ResultHandleResumeListener(EngineDependentServiceHolder engineDependentServiceHolder, + TaskInstanceService taskInstanceService, ResultHandleManager resultHandleManager, TaskInstanceVariableService taskInstanceVariableService, GseTaskService gseTaskService, - StorageSystemConfig storageSystemConfig, - AgentService agentService, - LogService logService, - StepInstanceVariableValueService stepInstanceVariableValueService, - TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher, - ResultHandleTaskKeepaliveManager resultHandleTaskKeepaliveManager, - TaskEvictPolicyExecutor taskEvictPolicyExecutor, - ScriptAgentTaskService scriptAgentTaskService, - FileAgentTaskService fileAgentTaskService, - StepInstanceService stepInstanceService) { + FileDistributeConfig fileDistributeConfig, + ScriptExecuteObjectTaskService scriptExecuteObjectTaskService, + FileExecuteObjectTaskService fileExecuteObjectTaskService, + StepInstanceService stepInstanceService, + JobExecuteConfig jobExecuteConfig) { + this.engineDependentServiceHolder = engineDependentServiceHolder; this.taskInstanceService = taskInstanceService; this.resultHandleManager = resultHandleManager; this.taskInstanceVariableService = taskInstanceVariableService; this.gseTaskService = gseTaskService; - this.storageSystemConfig = storageSystemConfig; - this.agentService = agentService; - this.logService = logService; - this.stepInstanceVariableValueService = stepInstanceVariableValueService; - this.taskExecuteMQEventDispatcher = taskExecuteMQEventDispatcher; - this.resultHandleTaskKeepaliveManager = resultHandleTaskKeepaliveManager; - this.taskEvictPolicyExecutor = taskEvictPolicyExecutor; - this.scriptAgentTaskService = scriptAgentTaskService; - this.fileAgentTaskService = fileAgentTaskService; + this.fileDistributeConfig = fileDistributeConfig; + this.scriptExecuteObjectTaskService = scriptExecuteObjectTaskService; + this.fileExecuteObjectTaskService = fileExecuteObjectTaskService; this.stepInstanceService = stepInstanceService; + this.jobExecuteConfig = jobExecuteConfig; } /** * 恢复被中断的作业结果处理任务 */ - @StreamListener(TaskResultHandleResumeProcessor.INPUT) - public void handleEvent(ResultHandleTaskResumeEvent event) { - log.info("Receive gse task result handle task resume event: {}", event); - GseTaskDTO gseTask = gseTaskService.getGseTask(event.getGseTaskId()); + public void handleEvent(Message message) { + ResultHandleTaskResumeEvent event = (ResultHandleTaskResumeEvent) message.getPayload(); + log.info("Receive gse task result handle task resume event: {}, duration: {}ms", event, event.duration()); + GseTaskDTO gseTask = gseTaskService.getGseTask(event.getJobInstanceId(), event.getGseTaskId()); long stepInstanceId = gseTask.getStepInstanceId(); String requestId = StringUtils.isNotEmpty(event.getRequestId()) ? event.getRequestId() : UUID.randomUUID().toString(); try { - StepInstanceDTO stepInstance = taskInstanceService.getStepInstanceDetail(stepInstanceId); + StepInstanceDTO stepInstance = stepInstanceService.getStepInstanceDetail( + event.getJobInstanceId(), stepInstanceId); TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(stepInstance.getTaskInstanceId()); if (!checkIsTaskResumeable(stepInstance, gseTask)) { @@ -178,27 +156,25 @@ private void resumeScriptTask(TaskInstanceDTO taskInstance, TaskVariablesAnalyzeResult taskVariablesAnalyzeResult, GseTaskDTO gseTask, String requestId) { - Map agentTaskMap = new HashMap<>(); - List agentTasks = scriptAgentTaskService.listAgentTasksByGseTaskId(gseTask.getId()); - agentTasks.forEach(agentTask -> agentTaskMap.put(agentTask.getAgentId(), agentTask)); + Map executeObjectTaskMap = new HashMap<>(); + List executeObjectTasks + = scriptExecuteObjectTaskService.listTasksByGseTaskId(stepInstance, gseTask.getId()); + executeObjectTasks.stream() + .filter(executeObjectTask -> !executeObjectTask.getExecuteObject().isAgentIdEmpty()) + .forEach(executeObjectTask -> executeObjectTaskMap.put( + executeObjectTask.getExecuteObject().toExecuteObjectGseKey(), executeObjectTask)); ScriptResultHandleTask scriptResultHandleTask = new ScriptResultHandleTask( - taskInstanceService, - gseTaskService, - logService, - taskInstanceVariableService, - stepInstanceVariableValueService, - taskExecuteMQEventDispatcher, - resultHandleTaskKeepaliveManager, - taskEvictPolicyExecutor, - scriptAgentTaskService, - stepInstanceService, + engineDependentServiceHolder, + scriptExecuteObjectTaskService, + jobExecuteConfig, taskInstance, stepInstance, taskVariablesAnalyzeResult, - agentTaskMap, + executeObjectTaskMap, gseTask, - requestId); + requestId, + executeObjectTasks); resultHandleManager.handleDeliveredTask(scriptResultHandleTask); } @@ -207,59 +183,43 @@ private void resumeFileTask(TaskInstanceDTO taskInstance, TaskVariablesAnalyzeResult taskVariablesAnalyzeResult, GseTaskDTO gseTask, String requestId) { - Set sendFiles = JobSrcFileUtils.parseSendFileList(stepInstance, - agentService.getLocalAgentHost(), - storageSystemConfig.getJobStorageRootPath()); + Set sendFiles = JobSrcFileUtils.parseSrcFiles(stepInstance, + fileDistributeConfig.getJobDistributeRootPath()); String targetDir = FilePathUtils.standardizedDirPath(stepInstance.getResolvedFileTargetPath()); - Map srcAndDestMap = JobSrcFileUtils.buildSourceDestPathMapping( + Map srcAndDestMap = JobSrcFileUtils.buildSourceDestPathMapping( sendFiles, targetDir, stepInstance.getFileTargetName()); - Map sourceDestPathMap = buildSourceDestPathMap(srcAndDestMap); - // 初始化显示名称映射Map - Map sourceFileDisplayMap = JobSrcFileUtils.buildSourceFileDisplayMapping(sendFiles, - NFSUtils.getFileDir(storageSystemConfig.getJobStorageRootPath(), FileDirTypeConf.UPLOAD_FILE_DIR)); - Map sourceAgentTaskMap = new HashMap<>(); - Map targetAgentTaskMap = new HashMap<>(); - List agentTasks = fileAgentTaskService.listAgentTasksByGseTaskId(gseTask.getId()); - agentTasks.forEach(agentTask -> { - if (agentTask.isTarget()) { - targetAgentTaskMap.put(agentTask.getAgentId(), agentTask); - } else { - sourceAgentTaskMap.put(agentTask.getAgentId(), agentTask); - } - }); + Map sourceAgentTaskMap = new HashMap<>(); + Map targetAgentTaskMap = new HashMap<>(); + List executeObjectTasks + = fileExecuteObjectTaskService.listTasksByGseTaskId(stepInstance, gseTask.getId()); + executeObjectTasks.stream() + .filter(executeObjectTask -> !executeObjectTask.getExecuteObject().isAgentIdEmpty()) + .forEach(executeObjectTask -> { + if (executeObjectTask.isTarget()) { + targetAgentTaskMap.put( + executeObjectTask.getExecuteObject().toExecuteObjectGseKey(), executeObjectTask); + } else { + sourceAgentTaskMap.put( + executeObjectTask.getExecuteObject().toExecuteObjectGseKey(), executeObjectTask); + } + }); FileResultHandleTask fileResultHandleTask = new FileResultHandleTask( - taskInstanceService, - gseTaskService, - logService, - taskInstanceVariableService, - stepInstanceVariableValueService, - taskExecuteMQEventDispatcher, - resultHandleTaskKeepaliveManager, - taskEvictPolicyExecutor, - fileAgentTaskService, - stepInstanceService, + engineDependentServiceHolder, + fileExecuteObjectTaskService, taskInstance, stepInstance, taskVariablesAnalyzeResult, targetAgentTaskMap, sourceAgentTaskMap, gseTask, - sendFiles, - storageSystemConfig.getJobStorageRootPath(), - sourceDestPathMap, - sourceFileDisplayMap, - requestId); + srcAndDestMap, + requestId, + executeObjectTasks); resultHandleManager.handleDeliveredTask(fileResultHandleTask); } - private Map buildSourceDestPathMap(Map srcAndDestMap) { - Map sourceDestPathMap = new HashMap<>(); - srcAndDestMap.forEach((fileKey, dest) -> sourceDestPathMap.put(fileKey, dest.getDestPath())); - return sourceDestPathMap; - } - private boolean checkIsTaskResumeable(StepInstanceDTO stepInstance, GseTaskDTO gseTask) { RunStatusEnum stepStatus = stepInstance.getStatus(); RunStatusEnum gseTaskStatus = RunStatusEnum.valueOf(gseTask.getStatus()); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/StepEventHandler.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/StepEventHandler.java index 7cc313ba41..b592a32f30 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/StepEventHandler.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/StepEventHandler.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/StepListener.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/StepListener.java index 98bd40084b..f99ea9f1bc 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/StepListener.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/StepListener.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,32 +25,30 @@ package com.tencent.bk.job.execute.engine.listener; import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; +import com.tencent.bk.job.execute.engine.listener.event.JobMessage; import com.tencent.bk.job.execute.engine.listener.event.StepEvent; -import com.tencent.bk.job.execute.engine.message.StepProcessor; import com.tencent.bk.job.execute.model.StepInstanceDTO; -import com.tencent.bk.job.execute.service.TaskInstanceService; +import com.tencent.bk.job.execute.service.StepInstanceService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.cloud.stream.annotation.EnableBinding; -import org.springframework.cloud.stream.annotation.StreamListener; +import org.springframework.messaging.Message; import org.springframework.stereotype.Component; /** * 执行引擎事件处理-步骤 */ @Component -@EnableBinding({StepProcessor.class}) @Slf4j -public class StepListener { - private final TaskInstanceService taskInstanceService; +public class StepListener extends BaseJobMqListener { + private final StepInstanceService stepInstanceService; private final GseStepEventHandler gseStepEventHandler; private final ConfirmStepEventHandler confirmStepEventHandler; @Autowired - public StepListener(TaskInstanceService taskInstanceService, + public StepListener(StepInstanceService stepInstanceService, GseStepEventHandler gseStepEventHandler, ConfirmStepEventHandler confirmStepEventHandler) { - this.taskInstanceService = taskInstanceService; + this.stepInstanceService = stepInstanceService; this.gseStepEventHandler = gseStepEventHandler; this.confirmStepEventHandler = confirmStepEventHandler; } @@ -58,14 +56,16 @@ public StepListener(TaskInstanceService taskInstanceService, /** * 处理步骤执行相关的事件 * - * @param stepEvent 步骤执行相关的事件 + * @param message 消息 */ - @StreamListener(StepProcessor.INPUT) - public void handleEvent(StepEvent stepEvent) { - log.info("Handle step event: {}", stepEvent); + @Override + public void handleEvent(Message message) { + StepEvent stepEvent = (StepEvent) message.getPayload(); + log.info("Handle step event: {}, duration: {}ms", stepEvent, stepEvent.duration()); long stepInstanceId = stepEvent.getStepInstanceId(); try { - StepInstanceDTO stepInstance = taskInstanceService.getStepInstanceDetail(stepInstanceId); + StepInstanceDTO stepInstance = stepInstanceService.getStepInstanceDetail( + stepEvent.getJobInstanceId(), stepInstanceId); dispatchEvent(stepEvent, stepInstance); } catch (Throwable e) { String errorMsg = "Handling step event error,stepInstanceId:" + stepInstanceId; @@ -75,7 +75,7 @@ public void handleEvent(StepEvent stepEvent) { } private void dispatchEvent(StepEvent stepEvent, StepInstanceDTO stepInstance) { - StepExecuteTypeEnum stepType = StepExecuteTypeEnum.valueOf(stepInstance.getExecuteType()); + StepExecuteTypeEnum stepType = stepInstance.getExecuteType(); switch (stepType) { case EXECUTE_SCRIPT: diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/Event.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/Event.java index 89ed821450..a1f3b14167 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/Event.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/Event.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,6 +25,7 @@ package com.tencent.bk.job.execute.engine.listener.event; import com.fasterxml.jackson.annotation.JsonInclude; +import com.tencent.bk.job.common.util.date.DateUtils; import lombok.Data; import java.time.LocalDateTime; @@ -34,7 +35,7 @@ */ @Data @JsonInclude(JsonInclude.Include.NON_NULL) -public class Event { +public class Event implements JobMessage { /** * 事件源 */ @@ -43,4 +44,12 @@ public class Event { * 事件发生时间 */ protected LocalDateTime time; + + public long duration() { + if (time != null) { + return DateUtils.calculateMillsBetweenDateTime(time, LocalDateTime.now()); + } else { + return 0; + } + } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/EventSource.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/EventSource.java index 475c3fb06e..2c6497b387 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/EventSource.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/EventSource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -67,11 +67,13 @@ public class EventSource { /** * 构造事件源 - 步骤 * + * @param jobInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @return 执行引擎事件源 */ - public static EventSource buildStepEventSource(long stepInstanceId) { + public static EventSource buildStepEventSource(Long jobInstanceId, long stepInstanceId) { EventSource eventSource = new EventSource(); + eventSource.setJobInstanceId(jobInstanceId); eventSource.setStepInstanceId(stepInstanceId); eventSource.setSourceType(EventSourceTypeEnum.STEP); return eventSource; @@ -80,17 +82,20 @@ public static EventSource buildStepEventSource(long stepInstanceId) { /** * 构造事件源 - GSE 任务 * + * @param jobInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 步骤执行次数 * @param batch 滚动执行批次 * @param gseTaskId GSE 任务ID * @return 执行引擎事件源 */ - public static EventSource buildGseTaskEventSource(Long stepInstanceId, + public static EventSource buildGseTaskEventSource(Long jobInstanceId, + Long stepInstanceId, Integer executeCount, Integer batch, Long gseTaskId) { EventSource eventSource = new EventSource(); + eventSource.setJobInstanceId(jobInstanceId); eventSource.setStepInstanceId(stepInstanceId); eventSource.setExecuteCount(executeCount); eventSource.setBatch(batch); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/GseTaskEvent.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/GseTaskEvent.java index dbd4494e00..c44b16a5d0 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/GseTaskEvent.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/GseTaskEvent.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -49,6 +49,10 @@ public class GseTaskEvent extends Event { * @see com.tencent.bk.job.execute.engine.consts.GseTaskActionEnum */ private int action; + /** + * 步骤实例ID + */ + private Long jobInstanceId; /** * 步骤实例ID */ @@ -73,6 +77,7 @@ public class GseTaskEvent extends Event { /** * 构造启动 GSE 任务 事件 * + * @param jobInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 * @param batch 滚动批次 @@ -80,12 +85,13 @@ public class GseTaskEvent extends Event { * @param requestId 请求ID,防止重复下发任务 * @return 事件 */ - public static GseTaskEvent startGseTask(Long stepInstanceId, + public static GseTaskEvent startGseTask(Long jobInstanceId, + Long stepInstanceId, Integer executeCount, Integer batch, Long gseTaskId, String requestId) { - GseTaskEvent gseTaskEvent = buildGseTaskEvent(stepInstanceId, executeCount, batch, gseTaskId); + GseTaskEvent gseTaskEvent = buildGseTaskEvent(jobInstanceId, stepInstanceId, executeCount, batch, gseTaskId); if (StringUtils.isNotEmpty(requestId)) { gseTaskEvent.setRequestId(requestId); } else { @@ -98,27 +104,31 @@ public static GseTaskEvent startGseTask(Long stepInstanceId, /** * 构造停止 GSE 任务 事件 * + * @param jobInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 * @param batch 滚动批次 * @param gseTaskId GSE任务ID * @return 事件 */ - public static GseTaskEvent stopGseTask(Long stepInstanceId, + public static GseTaskEvent stopGseTask(Long jobInstanceId, + Long stepInstanceId, Integer executeCount, Integer batch, Long gseTaskId) { - GseTaskEvent gseTaskEvent = buildGseTaskEvent(stepInstanceId, executeCount, batch, gseTaskId); + GseTaskEvent gseTaskEvent = buildGseTaskEvent(jobInstanceId, stepInstanceId, executeCount, batch, gseTaskId); gseTaskEvent.setAction(GseTaskActionEnum.STOP.getValue()); return gseTaskEvent; } - private static GseTaskEvent buildGseTaskEvent(Long stepInstanceId, + private static GseTaskEvent buildGseTaskEvent(Long jobInstanceId, + Long stepInstanceId, Integer executeCount, Integer batch, Long gseTaskId) { GseTaskEvent gseTaskEvent = new GseTaskEvent(); gseTaskEvent.setGseTaskId(gseTaskId); + gseTaskEvent.setJobInstanceId(jobInstanceId); gseTaskEvent.setStepInstanceId(stepInstanceId); gseTaskEvent.setExecuteCount(executeCount); gseTaskEvent.setBatch(batch); @@ -133,6 +143,10 @@ public String toString() { .add("action=" + action) .add("actionDesc=" + GseTaskActionEnum.valueOf(action)) .add("gseTaskId=" + gseTaskId) + .add("jobInstanceId=" + jobInstanceId) + .add("stepInstanceId=" + stepInstanceId) + .add("executeCount=" + executeCount) + .add("batch=" + batch) .add("requestId='" + requestId + "'") .add("source=" + source) .add("time=" + time) diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/JobEvent.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/JobEvent.java index 00f822e46a..5bda9ff06d 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/JobEvent.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/JobEvent.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/JobMessage.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/JobMessage.java new file mode 100644 index 0000000000..6ac0c2cbe7 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/JobMessage.java @@ -0,0 +1,32 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.listener.event; + +/** + * MQ Job消息接口 + */ +public interface JobMessage { + +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/ResultHandleTaskResumeEvent.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/ResultHandleTaskResumeEvent.java index 8ec4909866..658fa80c04 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/ResultHandleTaskResumeEvent.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/ResultHandleTaskResumeEvent.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -41,6 +41,10 @@ @NoArgsConstructor @JsonInclude(JsonInclude.Include.NON_NULL) public class ResultHandleTaskResumeEvent extends Event { + /** + * 作业实例ID + */ + private Long jobInstanceId; /** * 步骤实例ID */ @@ -72,12 +76,14 @@ public class ResultHandleTaskResumeEvent extends Event { * @param requestId 请求ID,防止重复下发任务 * @return 事件 */ - public static ResultHandleTaskResumeEvent resume(Long stepInstanceId, + public static ResultHandleTaskResumeEvent resume(Long jobInstanceId, + Long stepInstanceId, Integer executeCount, Integer batch, Long gseTaskId, String requestId) { ResultHandleTaskResumeEvent event = new ResultHandleTaskResumeEvent(); + event.setJobInstanceId(jobInstanceId); event.setStepInstanceId(stepInstanceId); event.setExecuteCount(executeCount); event.setBatch(batch); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/StepEvent.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/StepEvent.java index 6b326b176e..8440ceb44a 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/StepEvent.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/StepEvent.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -47,6 +47,10 @@ public class StepEvent extends Event { * @see com.tencent.bk.job.execute.engine.consts.StepActionEnum */ private int action; + /** + * 作业实例ID + */ + private Long jobInstanceId; /** * 步骤实例ID */ @@ -59,11 +63,13 @@ public class StepEvent extends Event { /** * 构造`忽略错误`事件 * + * @param jobInstanceId 作业实例ID * @param stepInstanceId 步骤实例ID * @return 事件 */ - public static StepEvent ignoreError(long stepInstanceId) { + public static StepEvent ignoreError(Long jobInstanceId, long stepInstanceId) { StepEvent stepEvent = new StepEvent(); + stepEvent.setJobInstanceId(jobInstanceId); stepEvent.setStepInstanceId(stepInstanceId); stepEvent.setAction(StepActionEnum.IGNORE_ERROR.getValue()); stepEvent.setTime(LocalDateTime.now()); @@ -73,11 +79,13 @@ public static StepEvent ignoreError(long stepInstanceId) { /** * 构造`下一步`事件 * + * @param jobInstanceId 作业实例ID * @param stepInstanceId 步骤实例ID * @return 事件 */ - public static StepEvent nextStep(long stepInstanceId) { + public static StepEvent nextStep(Long jobInstanceId, long stepInstanceId) { StepEvent stepEvent = new StepEvent(); + stepEvent.setJobInstanceId(jobInstanceId); stepEvent.setStepInstanceId(stepInstanceId); stepEvent.setAction(StepActionEnum.NEXT_STEP.getValue()); stepEvent.setTime(LocalDateTime.now()); @@ -87,11 +95,13 @@ public static StepEvent nextStep(long stepInstanceId) { /** * 构造`人工确认-重新发起确认`事件 * + * @param jobInstanceId 作业实例ID * @param stepInstanceId 步骤实例ID * @return 事件 */ - public static StepEvent confirmStepContinue(long stepInstanceId) { + public static StepEvent confirmStepContinue(Long jobInstanceId, long stepInstanceId) { StepEvent stepEvent = new StepEvent(); + stepEvent.setJobInstanceId(jobInstanceId); stepEvent.setStepInstanceId(stepInstanceId); stepEvent.setAction(StepActionEnum.CONFIRM_CONTINUE.getValue()); stepEvent.setTime(LocalDateTime.now()); @@ -101,11 +111,13 @@ public static StepEvent confirmStepContinue(long stepInstanceId) { /** * 构造`人工确认-终止流程`事件 * + * @param jobInstanceId 作业实例ID * @param stepInstanceId 步骤实例ID * @return 事件 */ - public static StepEvent confirmStepTerminate(long stepInstanceId) { + public static StepEvent confirmStepTerminate(Long jobInstanceId, long stepInstanceId) { StepEvent stepEvent = new StepEvent(); + stepEvent.setJobInstanceId(jobInstanceId); stepEvent.setStepInstanceId(stepInstanceId); stepEvent.setAction(StepActionEnum.CONFIRM_TERMINATE.getValue()); stepEvent.setTime(LocalDateTime.now()); @@ -115,11 +127,13 @@ public static StepEvent confirmStepTerminate(long stepInstanceId) { /** * 构造`人工确认-重新发起确认`事件 * + * @param jobInstanceId 作业实例ID * @param stepInstanceId 步骤实例ID * @return 事件 */ - public static StepEvent confirmStepRestart(long stepInstanceId) { + public static StepEvent confirmStepRestart(Long jobInstanceId, long stepInstanceId) { StepEvent stepEvent = new StepEvent(); + stepEvent.setJobInstanceId(jobInstanceId); stepEvent.setStepInstanceId(stepInstanceId); stepEvent.setAction(StepActionEnum.CONFIRM_RESTART.getValue()); stepEvent.setTime(LocalDateTime.now()); @@ -129,12 +143,14 @@ public static StepEvent confirmStepRestart(long stepInstanceId) { /** * 构造`启动步骤`事件 * + * @param jobInstanceId 作业实例ID * @param stepInstanceId 步骤实例ID * @param batch 滚动执行批次 * @return 事件 */ - public static StepEvent startStep(long stepInstanceId, Integer batch) { + public static StepEvent startStep(Long jobInstanceId, long stepInstanceId, Integer batch) { StepEvent stepEvent = new StepEvent(); + stepEvent.setJobInstanceId(jobInstanceId); stepEvent.setStepInstanceId(stepInstanceId); stepEvent.setBatch(batch); stepEvent.setAction(StepActionEnum.START.getValue()); @@ -145,11 +161,13 @@ public static StepEvent startStep(long stepInstanceId, Integer batch) { /** * 构造`跳过步骤`事件 * + * @param jobInstanceId 作业实例ID * @param stepInstanceId 步骤实例ID * @return 事件 */ - public static StepEvent skipStep(long stepInstanceId) { + public static StepEvent skipStep(Long jobInstanceId, long stepInstanceId) { StepEvent stepEvent = new StepEvent(); + stepEvent.setJobInstanceId(jobInstanceId); stepEvent.setStepInstanceId(stepInstanceId); stepEvent.setAction(StepActionEnum.SKIP.getValue()); stepEvent.setTime(LocalDateTime.now()); @@ -159,11 +177,13 @@ public static StepEvent skipStep(long stepInstanceId) { /** * 构造`停止步骤`事件 * + * @param jobInstanceId 作业实例ID * @param stepInstanceId 步骤实例ID * @return 事件 */ - public static StepEvent stopStep(long stepInstanceId) { + public static StepEvent stopStep(Long jobInstanceId, long stepInstanceId) { StepEvent stepEvent = new StepEvent(); + stepEvent.setJobInstanceId(jobInstanceId); stepEvent.setStepInstanceId(stepInstanceId); stepEvent.setAction(StepActionEnum.STOP.getValue()); stepEvent.setTime(LocalDateTime.now()); @@ -173,11 +193,13 @@ public static StepEvent stopStep(long stepInstanceId) { /** * 构造`失败IP重试`事件 * + * @param jobInstanceId 作业实例ID * @param stepInstanceId 步骤实例ID * @return 事件 */ - public static StepEvent retryStepFail(long stepInstanceId) { + public static StepEvent retryStepFail(Long jobInstanceId, long stepInstanceId) { StepEvent stepEvent = new StepEvent(); + stepEvent.setJobInstanceId(jobInstanceId); stepEvent.setStepInstanceId(stepInstanceId); stepEvent.setAction(StepActionEnum.RETRY_FAIL.getValue()); stepEvent.setTime(LocalDateTime.now()); @@ -187,11 +209,13 @@ public static StepEvent retryStepFail(long stepInstanceId) { /** * 构造`全部重试`事件 * + * @param jobInstanceId 作业实例ID * @param stepInstanceId 步骤实例ID * @return 事件 */ - public static StepEvent retryStepAll(long stepInstanceId) { + public static StepEvent retryStepAll(Long jobInstanceId, long stepInstanceId) { StepEvent stepEvent = new StepEvent(); + stepEvent.setJobInstanceId(jobInstanceId); stepEvent.setStepInstanceId(stepInstanceId); stepEvent.setAction(StepActionEnum.RETRY_ALL.getValue()); stepEvent.setTime(LocalDateTime.now()); @@ -199,27 +223,47 @@ public static StepEvent retryStepAll(long stepInstanceId) { } /** - * 构造`第三方文件源文件拉取完成后继续GSE分发`事件 + * 构造`文件准备完成后继续GSE分发`事件 * + * @param jobInstanceId 作业实例ID * @param stepInstanceId 步骤实例ID * @return 事件 */ - public static StepEvent continueGseFileStep(long stepInstanceId) { + public static StepEvent continueGseFileStep(Long jobInstanceId, long stepInstanceId) { StepEvent stepEvent = new StepEvent(); + stepEvent.setJobInstanceId(jobInstanceId); stepEvent.setStepInstanceId(stepInstanceId); stepEvent.setAction(StepActionEnum.CONTINUE_FILE_PUSH.getValue()); stepEvent.setTime(LocalDateTime.now()); return stepEvent; } + /** + * 构造`准备文件`事件 + * + * @param jobInstanceId 作业实例ID + * @param stepInstanceId 步骤实例ID + * @return 事件 + */ + public static StepEvent prepareFile(Long jobInstanceId, long stepInstanceId) { + StepEvent stepEvent = new StepEvent(); + stepEvent.setJobInstanceId(jobInstanceId); + stepEvent.setStepInstanceId(stepInstanceId); + stepEvent.setAction(StepActionEnum.PREPARE_FILE.getValue()); + stepEvent.setTime(LocalDateTime.now()); + return stepEvent; + } + /** * 构造`刷新步骤状态`事件 * + * @param jobInstanceId 作业实例ID * @param stepInstanceId 步骤实例ID * @return 事件 */ - public static StepEvent refreshStep(long stepInstanceId, EventSource eventSource) { + public static StepEvent refreshStep(Long jobInstanceId, long stepInstanceId, EventSource eventSource) { StepEvent stepEvent = new StepEvent(); + stepEvent.setJobInstanceId(jobInstanceId); stepEvent.setStepInstanceId(stepInstanceId); stepEvent.setAction(StepActionEnum.REFRESH.getValue()); stepEvent.setSource(eventSource); @@ -232,6 +276,7 @@ public String toString() { return new StringJoiner(", ", StepEvent.class.getSimpleName() + "[", "]") .add("action=" + action) .add("actionDesc=" + StepActionEnum.valueOf(action)) + .add("jobInstanceId=" + jobInstanceId) .add("stepInstanceId=" + stepInstanceId) .add("batch=" + batch) .add("source=" + source) diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/TaskExecuteMQEventDispatcher.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/TaskExecuteMQEventDispatcher.java index dbf64ed009..d53c8dfc90 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/TaskExecuteMQEventDispatcher.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/listener/event/TaskExecuteMQEventDispatcher.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,67 +25,28 @@ package com.tencent.bk.job.execute.engine.listener.event; import com.tencent.bk.job.common.util.json.JsonUtils; -import com.tencent.bk.job.execute.engine.message.CallbackProcessor; -import com.tencent.bk.job.execute.engine.message.GseTaskProcessor; -import com.tencent.bk.job.execute.engine.message.NotifyMsgProcessor; -import com.tencent.bk.job.execute.engine.message.StepProcessor; -import com.tencent.bk.job.execute.engine.message.TaskProcessor; -import com.tencent.bk.job.execute.engine.message.TaskResultHandleResumeProcessor; import com.tencent.bk.job.execute.engine.model.JobCallbackDTO; import com.tencent.bk.job.execute.model.TaskNotifyDTO; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.messaging.MessageChannel; -import org.springframework.messaging.support.MessageBuilder; +import org.springframework.cloud.stream.function.StreamBridge; import org.springframework.stereotype.Component; /** - * 作业执行MQ事件分发 + * 作业执行MQ事件分发. + *

+ * 基于Spring cloud stream + spring cloud function 实现。文档链接: + * https://docs.spring.io/spring-cloud-stream/docs/3.1.0/reference/html/spring-cloud-stream.html#spring_cloud_function */ @Component @Slf4j public class TaskExecuteMQEventDispatcher { - /** - * 消息通道-作业 - */ - private final MessageChannel taskOutput; - /** - * 消息通道-步骤 - */ - private final MessageChannel stepOutput; - /** - * 消息通道-GSE任务 - */ - private final MessageChannel gseTaskOutput; - /** - * 消息通道-消息通知 - */ - private final MessageChannel notifyMsgOutput; - /** - * 消息通道-作业执行完成回调 - */ - private final MessageChannel callbackOutput; - - /** - * 消息通道-作业执行完成回调 - */ - private final MessageChannel resultHandleTaskResumeOutput; + private final StreamBridge streamBridge; @Autowired - public TaskExecuteMQEventDispatcher(@Qualifier(TaskProcessor.OUTPUT) MessageChannel taskOutput, - @Qualifier(StepProcessor.OUTPUT) MessageChannel stepOutput, - @Qualifier(GseTaskProcessor.OUTPUT) MessageChannel gseTaskOutput, - @Qualifier(NotifyMsgProcessor.OUTPUT) MessageChannel notifyMsgOutput, - @Qualifier(CallbackProcessor.OUTPUT) MessageChannel callbackOutput, - @Qualifier(TaskResultHandleResumeProcessor.OUTPUT) MessageChannel resultHandleTaskResumeOutput) { - this.taskOutput = taskOutput; - this.stepOutput = stepOutput; - this.gseTaskOutput = gseTaskOutput; - this.notifyMsgOutput = notifyMsgOutput; - this.callbackOutput = callbackOutput; - this.resultHandleTaskResumeOutput = resultHandleTaskResumeOutput; + public TaskExecuteMQEventDispatcher(StreamBridge streamBridge) { + this.streamBridge = streamBridge; } /** @@ -95,7 +56,8 @@ public TaskExecuteMQEventDispatcher(@Qualifier(TaskProcessor.OUTPUT) MessageChan */ public void dispatchJobEvent(JobEvent jobEvent) { log.info("Begin to dispatch job event, event: {}", jobEvent); - taskOutput.send(MessageBuilder.withPayload(jobEvent).build()); + String taskOutput = "task-out-0"; + streamBridge.send(taskOutput, jobEvent); log.info("Dispatch job event successfully, event: {}", jobEvent); } @@ -106,7 +68,8 @@ public void dispatchJobEvent(JobEvent jobEvent) { */ public void dispatchStepEvent(StepEvent stepEvent) { log.info("Begin to dispatch step event, event: {}", stepEvent); - stepOutput.send(MessageBuilder.withPayload(stepEvent).build()); + String stepOutput = "step-out-0"; + streamBridge.send(stepOutput, stepEvent); log.info("Dispatch step event successfully, event: {}", stepEvent); } @@ -117,7 +80,8 @@ public void dispatchStepEvent(StepEvent stepEvent) { */ public void dispatchGseTaskEvent(GseTaskEvent gseTaskEvent) { log.info("Begin to dispatch gse task event, event: {}", gseTaskEvent); - gseTaskOutput.send(MessageBuilder.withPayload(gseTaskEvent).build()); + String gseTaskOutput = "gseTask-out-0"; + streamBridge.send(gseTaskOutput, gseTaskEvent); log.info("Dispatch gse task event successfully, event: {}", gseTaskEvent); } @@ -128,7 +92,8 @@ public void dispatchGseTaskEvent(GseTaskEvent gseTaskEvent) { */ public void dispatchResultHandleTaskResumeEvent(ResultHandleTaskResumeEvent event) { log.info("Begin to dispatch gse task result handle resume event, event: {}", event); - resultHandleTaskResumeOutput.send(MessageBuilder.withPayload(event).build()); + String resultHandleTaskResumeOutput = "resultHandleTaskResume-out-0"; + streamBridge.send(resultHandleTaskResumeOutput, event); log.info("Dispatch gse task result handle resume event successfully, event: {}", event); } @@ -137,9 +102,11 @@ public void dispatchResultHandleTaskResumeEvent(ResultHandleTaskResumeEvent even * * @param notification 消息内容 */ - public void asyncSendNotifyMsg(TaskNotifyDTO notification) { - log.info("Async send notification event:{}", JsonUtils.toJson(notification)); - notifyMsgOutput.send(MessageBuilder.withPayload(JsonUtils.toJson(notification)).build()); + public void dispatchNotifyMsg(TaskNotifyDTO notification) { + log.info("Begin to dispatch notification event:{}", JsonUtils.toJson(notification)); + String notifyMsgOutput = "notifyMsg-out-0"; + streamBridge.send(notifyMsgOutput, notification); + log.info("Dispatch notification event successfully"); } /** @@ -147,8 +114,10 @@ public void asyncSendNotifyMsg(TaskNotifyDTO notification) { * * @param jobCallback 回调内容 */ - public void sendCallback(JobCallbackDTO jobCallback) { - log.info("Async invoke callback url, callback:{}", JsonUtils.toJson(jobCallback)); - callbackOutput.send(MessageBuilder.withPayload(JsonUtils.toJson(jobCallback)).build()); + public void dispatchCallbackMsg(JobCallbackDTO jobCallback) { + log.info("Begin to dispatch job callback event, callback:{}", JsonUtils.toJson(jobCallback)); + String callbackOutput = "callback-out-0"; + streamBridge.send(callbackOutput, jobCallback); + log.info("Dispatch job callback event successfully"); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/message/CallbackProcessor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/message/CallbackProcessor.java deleted file mode 100644 index d98ff0361a..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/message/CallbackProcessor.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.engine.message; - -import org.springframework.cloud.stream.annotation.Input; -import org.springframework.cloud.stream.annotation.Output; -import org.springframework.messaging.MessageChannel; -import org.springframework.messaging.SubscribableChannel; - -public interface CallbackProcessor { - String INPUT = "callbackInput"; - String OUTPUT = "callbackOutput"; - - @Input(CallbackProcessor.INPUT) - SubscribableChannel input(); - - @Output(CallbackProcessor.OUTPUT) - MessageChannel output(); -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/message/GseTaskProcessor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/message/GseTaskProcessor.java deleted file mode 100644 index be902fdba7..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/message/GseTaskProcessor.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.engine.message; - -import org.springframework.cloud.stream.annotation.Input; -import org.springframework.cloud.stream.annotation.Output; -import org.springframework.messaging.MessageChannel; -import org.springframework.messaging.SubscribableChannel; - -public interface GseTaskProcessor { - String INPUT = "gseTaskInput"; - String OUTPUT = "gseTaskOutput"; - - @Input(GseTaskProcessor.INPUT) - SubscribableChannel input(); - - @Output(GseTaskProcessor.OUTPUT) - MessageChannel output(); -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/message/GseTaskResumeProcessor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/message/GseTaskResumeProcessor.java deleted file mode 100644 index 315ef3ac7d..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/message/GseTaskResumeProcessor.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.engine.message; - -import org.springframework.cloud.stream.annotation.Input; -import org.springframework.cloud.stream.annotation.Output; -import org.springframework.messaging.MessageChannel; -import org.springframework.messaging.SubscribableChannel; - -public interface GseTaskResumeProcessor { - String INPUT = "gseTaskResumeInput"; - String OUTPUT = "gseTaskResumeOutput"; - - @Input(GseTaskResumeProcessor.INPUT) - SubscribableChannel input(); - - @Output(GseTaskResumeProcessor.OUTPUT) - MessageChannel output(); -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/message/NotifyMsgProcessor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/message/NotifyMsgProcessor.java deleted file mode 100644 index 3685c3c4c5..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/message/NotifyMsgProcessor.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.engine.message; - -import org.springframework.cloud.stream.annotation.Input; -import org.springframework.cloud.stream.annotation.Output; -import org.springframework.messaging.MessageChannel; -import org.springframework.messaging.SubscribableChannel; - -public interface NotifyMsgProcessor { - String INPUT = "notifyMsgInput"; - String OUTPUT = "notifyMsgOutput"; - - @Input(NotifyMsgProcessor.INPUT) - SubscribableChannel input(); - - @Output(NotifyMsgProcessor.OUTPUT) - MessageChannel output(); -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/message/StepProcessor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/message/StepProcessor.java deleted file mode 100644 index 74af117425..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/message/StepProcessor.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.engine.message; - -import org.springframework.cloud.stream.annotation.Input; -import org.springframework.cloud.stream.annotation.Output; -import org.springframework.messaging.MessageChannel; -import org.springframework.messaging.SubscribableChannel; - -public interface StepProcessor { - String INPUT = "stepInput"; - String OUTPUT = "stepOutput"; - - @Input(StepProcessor.INPUT) - SubscribableChannel input(); - - @Output(StepProcessor.OUTPUT) - MessageChannel output(); -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/message/TaskProcessor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/message/TaskProcessor.java deleted file mode 100644 index 67571bd220..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/message/TaskProcessor.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.engine.message; - -import org.springframework.cloud.stream.annotation.Input; -import org.springframework.cloud.stream.annotation.Output; -import org.springframework.messaging.MessageChannel; -import org.springframework.messaging.SubscribableChannel; - -public interface TaskProcessor { - String INPUT = "taskInput"; - String OUTPUT = "taskOutput"; - - @Input(TaskProcessor.INPUT) - SubscribableChannel input(); - - @Output(TaskProcessor.OUTPUT) - MessageChannel output(); -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/message/TaskResultHandleResumeProcessor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/message/TaskResultHandleResumeProcessor.java deleted file mode 100644 index a2cf14f348..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/message/TaskResultHandleResumeProcessor.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.engine.message; - -import org.springframework.cloud.stream.annotation.Input; -import org.springframework.cloud.stream.annotation.Output; -import org.springframework.messaging.MessageChannel; -import org.springframework.messaging.SubscribableChannel; - -public interface TaskResultHandleResumeProcessor { - String INPUT = "taskResultHandleResumeInput"; - String OUTPUT = "taskResultHandleResumeOutput"; - - @Input(TaskResultHandleResumeProcessor.INPUT) - SubscribableChannel input(); - - @Output(TaskResultHandleResumeProcessor.OUTPUT) - MessageChannel output(); -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/ExecuteObject.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/ExecuteObject.java new file mode 100644 index 0000000000..ef1ed02a37 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/ExecuteObject.java @@ -0,0 +1,297 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.model; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.annotation.PersistenceObject; +import com.tencent.bk.job.common.constant.CompatibleType; +import com.tencent.bk.job.common.constant.ExecuteObjectTypeEnum; +import com.tencent.bk.job.common.gse.util.K8sUtils; +import com.tencent.bk.job.common.gse.v2.model.Agent; +import com.tencent.bk.job.common.gse.v2.model.ExecuteObjectGseKey; +import com.tencent.bk.job.common.model.dto.Container; +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.common.model.openapi.v4.OpenApiExecuteObjectDTO; +import com.tencent.bk.job.execute.model.ExecuteObjectCompositeKey; +import com.tencent.bk.job.execute.model.inner.ServiceExecuteObject; +import com.tencent.bk.job.execute.model.web.vo.ExecuteObjectVO; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +import java.util.Objects; + +/** + * 作业执行对象通用模型 + */ +@Setter +@Getter +@NoArgsConstructor +@JsonInclude(JsonInclude.Include.NON_NULL) +@ToString +@PersistenceObject +@Slf4j +public class ExecuteObject implements Cloneable { + /** + * 执行对象 ID + */ + private String id; + + /** + * 执行对象类型 + * + * @see ExecuteObjectTypeEnum + */ + private ExecuteObjectTypeEnum type; + + /** + * 执行对象资源 ID（主机 ID/容器 ID) + */ + private Long resourceId; + + /** + * 容器 + */ + private Container container; + + /** + * 主机 + */ + private HostDTO host; + + /** + * 执行对象对应的 。不会被持久化 + */ + @JsonIgnore + private ExecuteObjectGseKey executeObjectGseKey; + + /** + * 是否非法执行对象 + */ + private boolean invalid; + + public ExecuteObject(Container container) { + this.container = container; + this.type = ExecuteObjectTypeEnum.CONTAINER; + this.resourceId = container.getId(); + this.id = buildExecuteObjectId(ExecuteObjectTypeEnum.CONTAINER, container.getId()); + } + + public ExecuteObject(HostDTO host) { + this.host = host; + this.type = ExecuteObjectTypeEnum.HOST; + this.resourceId = host.getHostId(); + this.id = buildExecuteObjectId(ExecuteObjectTypeEnum.HOST, host.getHostId()); + } + + private String buildExecuteObjectId(ExecuteObjectTypeEnum executeObjectType, Long executeObjectResoruceId) { + return executeObjectType.getValue() + ":" + executeObjectResoruceId; + } + + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA, + explain = "数据失效后该构造方法可以删除") + public static ExecuteObject buildCompatibleExecuteObject(HostDTO host) { + ExecuteObject executeObject = new ExecuteObject(); + executeObject.setType(ExecuteObjectTypeEnum.HOST); + executeObject.setResourceId(host.getHostId()); + executeObject.setHost(host); + return executeObject; + } + + @JsonCreator(mode = JsonCreator.Mode.DELEGATING) + public static ExecuteObjectTypeEnum fromExecuteObjectTypeValue(int type) { + return ExecuteObjectTypeEnum.valOf(type); + } + + @Override + public ExecuteObject clone() { + ExecuteObject clone = new ExecuteObject(); + clone.setId(id); + clone.setType(type); + clone.setResourceId(resourceId); + if (host != null) { + clone.setHost(host.clone()); + } + if (container != null) { + clone.setContainer(container.clone()); + } + clone.setInvalid(invalid); + return clone; + } + + @JsonIgnore + public boolean isHostExecuteObject() { + return type == ExecuteObjectTypeEnum.HOST; + } + + @JsonIgnore + public boolean isContainerExecuteObject() { + return type == ExecuteObjectTypeEnum.CONTAINER; + } + + public ExecuteObjectGseKey toExecuteObjectGseKey() { + if (executeObjectGseKey != null) { + return executeObjectGseKey; + } + if (isHostExecuteObject()) { + executeObjectGseKey = ExecuteObjectGseKey.ofHost(host.getAgentId()); + } else { + executeObjectGseKey = ExecuteObjectGseKey.ofContainer(container.getNodeAgentId(), + container.getContainerId()); + } + return executeObjectGseKey; + } + + @JsonIgnore + public boolean isAgentIdEmpty() { + if (isHostExecuteObject()) { + return StringUtils.isEmpty(getHost().getAgentId()); + } else if (isContainerExecuteObject()) { + return StringUtils.isEmpty(getContainer().getNodeAgentId()); + } else { + throw new IllegalArgumentException("Invalid execute object type: " + type); + } + } + + /** + * 判断是否可作为执行目标 + */ + @JsonIgnore + public boolean isExecutable() { + return !isInvalid() && !isAgentIdEmpty(); + } + + public Agent toGseAgent() { + Agent agent = new Agent(); + if (isHostExecuteObject()) { + agent.setAgentId(host.getAgentId()); + } else if (isContainerExecuteObject()) { + agent.setAgentId(container.getNodeAgentId()); + agent.setContainerId(K8sUtils.removeContainerIdType(container.getContainerId())); + } + return agent; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ExecuteObject that = (ExecuteObject) o; + if (id != null) { + // 优先使用 ID 计算 + return id.equals(that.getId()); + } else if (type != null && resourceId != null) { + return type == that.getType() && resourceId.equals(that.getResourceId()); + } else { + return false; + } + } + + @Override + public int hashCode() { + return Objects.hash(type.getValue(), resourceId); + } + + /** + * 获取可读性的执行对象名称 + */ + @JsonIgnore + public String getExecuteObjectName() { + if (isHostExecuteObject()) { + return host.getPrimaryIpWithBkNetId(); + } else if (isContainerExecuteObject()) { + return container.getReadabilityName(); + } else { + return "Unknown"; + } + } + + public ExecuteObjectCompositeKey toResourceIdCompositeKey() { + return ExecuteObjectCompositeKey.ofExecuteObjectResource(type, resourceId); + } + + public ExecuteObjectCompositeKey toExecuteObjectCompositeKey( + ExecuteObjectCompositeKey.CompositeKeyType compositeKeyType) { + switch (compositeKeyType) { + case EXECUTE_OBJECT_ID: + return ExecuteObjectCompositeKey.ofExecuteObjectId(id); + case RESOURCE_ID: + return ExecuteObjectCompositeKey.ofExecuteObjectResource(type, resourceId); + case HOST_CLOUD_IP: + return ExecuteObjectCompositeKey.ofHostIp(host.toCloudIp()); + } + return null; + } + + public ExecuteObjectVO toExecuteObjectVO() { + ExecuteObjectVO vo = new ExecuteObjectVO(); + vo.setType(type); + vo.setExecuteObjectResourceId(resourceId); + if (host != null) { + vo.setHost(host.toHostInfoVO()); + } + if (container != null) { + vo.setContainer(container.toContainerVO()); + } + return vo; + } + + public OpenApiExecuteObjectDTO toOpenApiExecuteObjectDTO() { + OpenApiExecuteObjectDTO executeObject = new OpenApiExecuteObjectDTO(); + executeObject.setType(type.getValue()); + executeObject.setResourceId(String.valueOf(resourceId)); + if (container != null) { + executeObject.setContainer(container.toOpenApiContainerDTO()); + } + if (host != null) { + executeObject.setHost(host.toOpenApiHostDTO()); + } + return executeObject; + } + + /** + * 判断是否支持执行对象特性 + */ + @JsonIgnore + public boolean isSupportExecuteObjectFeature() { + return id != null; + } + + public ServiceExecuteObject toServiceExecuteObject() { + ServiceExecuteObject serviceExecuteObject = new ServiceExecuteObject(); + serviceExecuteObject.setId(id); + serviceExecuteObject.setType(type.getValue()); + serviceExecuteObject.setResourceId(resourceId); + serviceExecuteObject.setHost(host); + serviceExecuteObject.setContainer(container); + return serviceExecuteObject; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/FileDest.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/FileDest.java index 7305e60c3c..92f2683541 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/FileDest.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/FileDest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/FileGseTaskResult.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/FileGseTaskResult.java new file mode 100644 index 0000000000..44c06210de --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/FileGseTaskResult.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.model; + + +import com.tencent.bk.job.common.gse.v2.model.FileTaskResult; +import org.springframework.util.CollectionUtils; + +/** + * 文件任务查询结果 + */ +public class FileGseTaskResult extends GseTaskResult { + + public FileGseTaskResult(FileTaskResult result) { + super(result); + } + + @Override + public boolean isEmptyResult() { + return getResult() == null || CollectionUtils.isEmpty(getResult().getAtomicFileTaskResults()); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/FileTaskLog.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/FileTaskLog.java deleted file mode 100644 index 3f7b309881..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/FileTaskLog.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.engine.model; - - -import com.tencent.bk.gse.taskapi.api_map_rsp; -import com.tencent.bk.job.execute.engine.consts.GSECode; - -/** - * @version 1.0 - * @time 2017/6/5. - */ -public class FileTaskLog extends GseLog { - - public FileTaskLog(api_map_rsp gseLog) { - super(gseLog); - } - - @Override - public boolean isError() { - return GSECode.ErrorCode.getErrorCode(getGseLog().getBk_error_code()) == GSECode.ErrorCode.ERROR; - } - - @Override - public String getErrorMsg() { - return getGseLog().getBk_error_msg(); - } - - @Override - public boolean isNullResp() { - return getGseLog().getResult() == null || getGseLog().getResult().isEmpty(); - } -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/GseLog.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/GseLog.java deleted file mode 100644 index e0d3baa4f5..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/GseLog.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.engine.model; - -/** - * GSE 任务执行结果 - * - * @param - */ -public abstract class GseLog { - - /** - * GSE 任务执行结果 - */ - private final T gseLog; - - GseLog(T gseLog) { - this.gseLog = gseLog; - } - - public T getGseLog() { - return gseLog; - } - - /** - * 是否协议结构中有空结果 - */ - public boolean isNullResp() { - return false; - } - - /** - * 是否出错 - * - * @return 是否出错 - */ - public abstract boolean isError(); - - /** - * 错误信息 - * - * @return 错误信息 - */ - public abstract String getErrorMsg(); -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/GseLogBatchPullResult.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/GseLogBatchPullResult.java index 9739626172..354547b995 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/GseLogBatchPullResult.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/GseLogBatchPullResult.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -33,14 +33,6 @@ @Data @NoArgsConstructor public class GseLogBatchPullResult { - /** - * 是否成功拉取到日志 - */ - private boolean success; - /** - * 错误信息 - */ - private String errorMsg; /** * 是否最后一批 */ @@ -48,12 +40,10 @@ public class GseLogBatchPullResult { /** * GSE 任务执行日志 */ - private GseLog gseLog; + private GseTaskResult gseTaskResult; - public GseLogBatchPullResult(boolean success, boolean isLastBatch, GseLog gseLog, String errorMsg) { - this.success = success; + public GseLogBatchPullResult(boolean isLastBatch, GseTaskResult gseTaskResult) { this.lastBatch = isLastBatch; - this.gseLog = gseLog; - this.errorMsg = errorMsg; + this.gseTaskResult = gseTaskResult; } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/GseTaskExecuteResult.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/GseTaskExecuteResult.java index 17445880d1..80f6f95852 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/GseTaskExecuteResult.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/GseTaskExecuteResult.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -40,11 +40,11 @@ public class GseTaskExecuteResult { public static final int RESULT_CODE_SUCCESS = 1; public static final int RESULT_CODE_FAILED = 2; public static final int RESULT_CODE_STOP_SUCCESS = 3; - public static final int RESULT_CODE_STOP_FAILED = 4; +// public static final int RESULT_CODE_STOP_FAILED = 4; /** * 任务被打断 */ - public static final int RESULT_CODE_INTERRUPTED = 5; +// public static final int RESULT_CODE_INTERRUPTED = 5; public static final int RESULT_CODE_RUNNING = 6; public static final int RESULT_CODE_SKIPPED = 7; /** @@ -59,8 +59,8 @@ public class GseTaskExecuteResult { public static final GseTaskExecuteResult SUCCESS = new GseTaskExecuteResult(RESULT_CODE_SUCCESS); public static final GseTaskExecuteResult FAILED = new GseTaskExecuteResult(RESULT_CODE_FAILED); public static final GseTaskExecuteResult STOP_SUCCESS = new GseTaskExecuteResult(RESULT_CODE_STOP_SUCCESS); - public static final GseTaskExecuteResult STOP_FAILED = new GseTaskExecuteResult(RESULT_CODE_STOP_FAILED); - public static final GseTaskExecuteResult INTERRUPTED = new GseTaskExecuteResult(RESULT_CODE_INTERRUPTED); + // public static final GseTaskExecuteResult STOP_FAILED = new GseTaskExecuteResult(RESULT_CODE_STOP_FAILED); +// public static final GseTaskExecuteResult INTERRUPTED = new GseTaskExecuteResult(RESULT_CODE_INTERRUPTED); public static final GseTaskExecuteResult RUNNING = new GseTaskExecuteResult(RESULT_CODE_RUNNING); public static final GseTaskExecuteResult SKIPPED = new GseTaskExecuteResult(RESULT_CODE_SKIPPED); public static final GseTaskExecuteResult DISCARDED = new GseTaskExecuteResult(RESULT_CODE_DISCARDED); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/GseTaskResult.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/GseTaskResult.java index ca10901e1f..a5156ff407 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/GseTaskResult.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/GseTaskResult.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,31 +24,30 @@ package com.tencent.bk.job.execute.engine.model; -import lombok.Data; - -import java.util.List; -import java.util.Map; - /** - * GSE任务下单响应信息 + * GSE 任务执行结果 + * + * @param */ -@Data -public class GseTaskResult { +public abstract class GseTaskResult { /** - * 错误码 + * GSE 任务执行结果 */ - private int errorCode; + private final T result; - /** - * 错误信息 - */ - private String errorMessage; + GseTaskResult(T result) { + this.result = result; + } + + public T getResult() { + return result; + } /** - * 数据 + * 任务执行结果是否为空 */ - private List> data; - - + public boolean isEmptyResult() { + return false; + } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/IpExecuteStatus.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/IpExecuteStatus.java index f68b94ba5e..8bf7285f20 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/IpExecuteStatus.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/IpExecuteStatus.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/JobCallbackDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/JobCallbackDTO.java index b29e87aa2b..b4cbd14654 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/JobCallbackDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/JobCallbackDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,6 +26,7 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.execute.engine.listener.event.JobMessage; import lombok.Getter; import lombok.Setter; import lombok.ToString; @@ -36,7 +37,7 @@ @Getter @ToString @JsonInclude(JsonInclude.Include.NON_NULL) -public class JobCallbackDTO { +public class JobCallbackDTO implements JobMessage { @JsonProperty("job_instance_id") private long id; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/JobFile.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/JobFile.java index db1e06fba4..c447a78758 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/JobFile.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/JobFile.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,9 +24,8 @@ package com.tencent.bk.job.execute.engine.model; -import com.tencent.bk.job.common.model.dto.HostDTO; -import com.tencent.bk.job.execute.engine.util.FilePathUtils; -import com.tencent.bk.job.manage.common.consts.task.TaskFileTypeEnum; +import com.tencent.bk.job.common.util.FilePathUtils; +import com.tencent.bk.job.manage.api.common.constants.task.TaskFileTypeEnum; import lombok.Data; import lombok.ToString; import org.apache.commons.lang3.StringUtils; @@ -42,9 +41,9 @@ public class JobFile { */ private TaskFileTypeEnum fileType; /** - * 源文件主机 + * 源文件执行对象 */ - private HostDTO host; + private ExecuteObject executeObject; /** * 文件路径(用户输入) */ @@ -94,7 +93,7 @@ public class JobFile { /** * @param fileType 文件类型 - * @param host 源文件主机 + * @param executeObject 源文件执行对象 * @param filePath 文件路径 * @param dir 目录名称 * @param fileName 文件名 @@ -103,7 +102,7 @@ public class JobFile { * @param displayFilePath 要展示的文件路径 */ public JobFile(TaskFileTypeEnum fileType, - HostDTO host, + ExecuteObject executeObject, String filePath, String dir, String fileName, @@ -111,7 +110,7 @@ public JobFile(TaskFileTypeEnum fileType, String password, String displayFilePath) { this.fileType = fileType; - this.host = host; + this.executeObject = executeObject; this.filePath = filePath; this.dir = dir; this.fileName = fileName; @@ -122,7 +121,7 @@ public JobFile(TaskFileTypeEnum fileType, /** * @param fileType 文件类型 - * @param host 源文件主机 + * @param executeObject 源文件执行对象 * @param filePath 文件路径 * @param displayFilePath 要展示的文件路径 * @param dir 目录名称 @@ -132,7 +131,7 @@ public JobFile(TaskFileTypeEnum fileType, * @param accountAlias 账号别名 */ public JobFile(TaskFileTypeEnum fileType, - HostDTO host, + ExecuteObject executeObject, String filePath, String displayFilePath, String dir, @@ -141,7 +140,7 @@ public JobFile(TaskFileTypeEnum fileType, Long accountId, String accountAlias) { this.fileType = fileType; - this.host = host; + this.executeObject = executeObject; this.filePath = filePath; this.displayFilePath = displayFilePath; this.dir = dir; @@ -178,14 +177,14 @@ public int hashCode() { * @return 文件KEY */ public String getUniqueKey() { - if (!StringUtils.isEmpty(this.uniqueKey)) { + if (StringUtils.isNotEmpty(this.uniqueKey)) { return this.uniqueKey; } StringBuilder sb = new StringBuilder(); sb.append(fileType.name()).append(":"); if (fileType == TaskFileTypeEnum.SERVER) { - // 远程文件分发，需要源主机信息才能唯一确定一个源文件 - sb.append(host.getUniqueKey()).append(":"); + // 远程文件分发，需要源执行对象信息才能唯一确定一个源文件 + sb.append(executeObject.toResourceIdCompositeKey()).append(":"); } sb.append(getStandardFilePath()); this.uniqueKey = sb.toString(); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/LogPullProgress.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/LogPullProgress.java index 6551bd2e27..e73b97eb6b 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/LogPullProgress.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/LogPullProgress.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,7 @@ package com.tencent.bk.job.execute.engine.model; +import com.tencent.bk.job.common.gse.v2.model.ExecuteObjectGseKey; import lombok.Data; /** @@ -32,13 +33,13 @@ @Data public class LogPullProgress { /** - * bk_agent_id + * 执行对象GSE KEY */ - private String agentId; + private ExecuteObjectGseKey executeObjectGseKey; /** * 任务编号 */ - private int mid; + private int atomicTaskId; /** * 日志偏移 - 字节 */ diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/RunSQLScriptFile.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/RunSQLScriptFile.java index c9aaa3958e..faf3b871b3 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/RunSQLScriptFile.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/RunSQLScriptFile.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/ScriptGseTaskResult.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/ScriptGseTaskResult.java new file mode 100644 index 0000000000..acc9a330d4 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/ScriptGseTaskResult.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.model; + + +import com.tencent.bk.job.common.gse.v2.model.ScriptTaskResult; +import org.springframework.util.CollectionUtils; + +/** + * 脚本任务结果 + */ +public class ScriptGseTaskResult extends GseTaskResult { + + public ScriptGseTaskResult(ScriptTaskResult result) { + super(result); + } + + @Override + public boolean isEmptyResult() { + return getResult() == null || CollectionUtils.isEmpty(getResult().getResult()); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/ScriptTaskLog.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/ScriptTaskLog.java deleted file mode 100644 index 6056d59993..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/ScriptTaskLog.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.engine.model; - - -import com.tencent.bk.gse.taskapi.api_task_detail_result; -import com.tencent.bk.job.execute.engine.consts.GSECode; - -/** - * 脚本任务结果 - * - * @version 1.0 - * @time 2017/6/5. - */ -public class ScriptTaskLog extends GseLog { - - public ScriptTaskLog(api_task_detail_result gseLog) { - super(gseLog); - } - - @Override - public boolean isError() { - return GSECode.ErrorCode.getErrorCode(getGseLog().getBk_error_code()) == GSECode.ErrorCode.ERROR; - } - - @Override - public boolean isNullResp() { - return getGseLog() == null || getGseLog().getResult() == null || getGseLog().getResult().isEmpty(); - } - - @Override - public String getErrorMsg() { - return getGseLog().getBk_error_msg(); - } -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/TaskResultHandleResumeMessage.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/TaskResultHandleResumeMessage.java index 0701d1c9ea..691beae362 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/TaskResultHandleResumeMessage.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/TaskResultHandleResumeMessage.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/TaskVariableDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/TaskVariableDTO.java index 6b2527c7fc..7014082629 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/TaskVariableDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/TaskVariableDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,7 +25,7 @@ package com.tencent.bk.job.execute.engine.model; import com.tencent.bk.job.common.constant.TaskVariableTypeEnum; -import com.tencent.bk.job.execute.model.ServersDTO; +import com.tencent.bk.job.execute.model.ExecuteTargetDTO; import lombok.Getter; import lombok.Setter; @@ -48,7 +48,7 @@ public class TaskVariableDTO { * * @see TaskVariableTypeEnum */ - private Integer type = 1; + private Integer type; /** * 名称 @@ -60,7 +60,10 @@ public class TaskVariableDTO { */ private String value; - private ServersDTO targetServers; + /** + * 执行目标；当 type = TaskVariableTypeEnum.HOST_LIST 的时候，该值不为null + */ + private ExecuteTargetDTO executeTarget; /** @@ -84,7 +87,7 @@ public String toString() { .add("id=" + id) .add("type=" + type) .add("name='" + name + "'") - .add("targetServers=" + targetServers) + .add("executeTarget=" + executeTarget) .add("changeable=" + changeable) .add("required=" + required) .add("taskInstanceId=" + taskInstanceId); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/TaskVariablesAnalyzeResult.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/TaskVariablesAnalyzeResult.java index 9b0d92fa8e..5f50b79528 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/TaskVariablesAnalyzeResult.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/model/TaskVariablesAnalyzeResult.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/DefaultFilePrepareTaskResultHandler.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/DefaultFilePrepareTaskResultHandler.java new file mode 100644 index 0000000000..0146fe600e --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/DefaultFilePrepareTaskResultHandler.java @@ -0,0 +1,132 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.prepare; + +import com.tencent.bk.job.execute.common.constants.RunStatusEnum; +import com.tencent.bk.job.execute.engine.listener.event.EventSource; +import com.tencent.bk.job.execute.engine.listener.event.JobEvent; +import com.tencent.bk.job.execute.engine.listener.event.StepEvent; +import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; +import com.tencent.bk.job.execute.model.StepInstanceDTO; +import com.tencent.bk.job.execute.service.StepInstanceService; +import lombok.extern.slf4j.Slf4j; +import org.slf4j.helpers.FormattingTuple; +import org.slf4j.helpers.MessageFormatter; + +import java.util.List; + +@Slf4j +public class DefaultFilePrepareTaskResultHandler implements FilePrepareTaskResultHandler { + + private final StepInstanceService stepInstanceService; + private final TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher; + + public DefaultFilePrepareTaskResultHandler(StepInstanceService stepInstanceService, + TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher) { + this.stepInstanceService = stepInstanceService; + this.taskExecuteMQEventDispatcher = taskExecuteMQEventDispatcher; + } + + @Override + public void onFinished(StepInstanceDTO stepInstance, List resultList) { + log.info("[{}]: prepareTask finished", stepInstance.getUniqueKey()); + handleFinalTaskResult(resultList, stepInstance); + } + + @Override + public void onTimeout(StepInstanceDTO stepInstance) { + log.info("[{}]: prepareTask timeout", stepInstance.getUniqueKey()); + onFailed(stepInstance, null); + } + + @Override + public void onException(StepInstanceDTO stepInstance, Throwable t) { + FormattingTuple msg = MessageFormatter.format( + "[{}]: prepareTask exception", + stepInstance.getUniqueKey() + ); + log.error(msg.getMessage(), t); + onFailed(stepInstance, null); + } + + private void handleFinalTaskResult(List resultList, StepInstanceDTO stepInstance) { + FilePrepareTaskResult finalResult = combineTaskResult(resultList); + if (finalResult == null) return; + if (finalResult.status == FilePrepareTaskResult.STATUS_SUCCESS) { + onSuccess(stepInstance, finalResult); + } else if (finalResult.status == FilePrepareTaskResult.STATUS_STOPPED) { + onStopped(stepInstance, finalResult); + } else if (finalResult.status == FilePrepareTaskResult.STATUS_FAILED) { + onFailed(stepInstance, finalResult); + } else { + log.warn("Unknown status:{}", finalResult.status); + } + } + + private FilePrepareTaskResult combineTaskResult(List resultList) { + int size = resultList.size(); + if (size == 0) return null; + if (size == 1) return resultList.get(0); + for (FilePrepareTaskResult filePrepareTaskResult : resultList) { + if (filePrepareTaskResult.status == FilePrepareTaskResult.STATUS_FAILED) { + return filePrepareTaskResult; + } + } + for (FilePrepareTaskResult filePrepareTaskResult : resultList) { + if (filePrepareTaskResult.status == FilePrepareTaskResult.STATUS_STOPPED) { + return filePrepareTaskResult; + } + } + return resultList.get(0); + } + + private void onSuccess(StepInstanceDTO stepInstance, FilePrepareTaskResult finalResult) { + if (!finalResult.getTaskContext().isForRetry()) { + // 直接进行下一步 + taskExecuteMQEventDispatcher.dispatchStepEvent( + StepEvent.continueGseFileStep(stepInstance.getTaskInstanceId(), stepInstance.getId())); + } + } + + private void onStopped(StepInstanceDTO stepInstance, FilePrepareTaskResult finalResult) { + // 步骤状态变更 + stepInstanceService.updateStepStatus(stepInstance.getTaskInstanceId(), + stepInstance.getId(), RunStatusEnum.STOP_SUCCESS.getValue()); + // 任务状态变更 + taskExecuteMQEventDispatcher.dispatchJobEvent( + JobEvent.refreshJob(stepInstance.getTaskInstanceId(), + EventSource.buildStepEventSource(stepInstance.getTaskInstanceId(), stepInstance.getId()))); + } + + private void onFailed(StepInstanceDTO stepInstance, FilePrepareTaskResult finalResult) { + // 文件源文件下载失败 + stepInstanceService.updateStepStatus(stepInstance.getTaskInstanceId(), + stepInstance.getId(), RunStatusEnum.FAIL.getValue()); + taskExecuteMQEventDispatcher.dispatchJobEvent( + JobEvent.refreshJob(stepInstance.getTaskInstanceId(), + EventSource.buildStepEventSource(stepInstance.getTaskInstanceId(), stepInstance.getId()))); + } + +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareControlTask.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareControlTask.java index 1fa619ab98..7631358ea9 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareControlTask.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareControlTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,17 +25,23 @@ package com.tencent.bk.job.execute.engine.prepare; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; +import com.tencent.bk.job.execute.engine.listener.event.StepEvent; +import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; import com.tencent.bk.job.execute.engine.result.ContinuousScheduledTask; import com.tencent.bk.job.execute.engine.result.ScheduleStrategy; import com.tencent.bk.job.execute.engine.result.StopTaskCounter; +import com.tencent.bk.job.execute.engine.result.TaskContext; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; +import com.tencent.bk.job.execute.service.StepInstanceService; import com.tencent.bk.job.execute.service.TaskInstanceService; +import lombok.extern.slf4j.Slf4j; import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; +@Slf4j public class FilePrepareControlTask implements ContinuousScheduledTask { /** @@ -46,40 +52,38 @@ public class FilePrepareControlTask implements ContinuousScheduledTask { * 任务是否已停止 */ private volatile boolean isStopped = false; + private volatile boolean isRunning = false; volatile AtomicBoolean isDoneWrapper = new AtomicBoolean(false); private final FilePrepareService filePrepareService; private final TaskInstanceService taskInstanceService; + private final TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher; private final StepInstanceDTO stepInstance; private final CountDownLatch latch; private final List resultList; private final FilePrepareTaskResultHandler filePrepareTaskResultHandler; + private final StepInstanceService stepInstanceService; private final long startTimeMills; - // 默认30分钟 - private long timeoutMills = 30 * 60 * 1000L; + private final TaskContext taskContext; public FilePrepareControlTask( FilePrepareService filePrepareService, TaskInstanceService taskInstanceService, + TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher, StepInstanceDTO stepInstance, CountDownLatch latch, List resultList, - FilePrepareTaskResultHandler filePrepareTaskResultHandler - ) { - this.startTimeMills = System.currentTimeMillis(); + FilePrepareTaskResultHandler filePrepareTaskResultHandler, + StepInstanceService stepInstanceService) { this.filePrepareService = filePrepareService; this.taskInstanceService = taskInstanceService; + this.taskExecuteMQEventDispatcher = taskExecuteMQEventDispatcher; this.stepInstance = stepInstance; this.latch = latch; this.resultList = resultList; this.filePrepareTaskResultHandler = filePrepareTaskResultHandler; - } - - public void setDoneStatus() { - isDoneWrapper.set(true); - } - - public void setTimeoutMills(long timeoutMills) { - this.timeoutMills = timeoutMills; + this.stepInstanceService = stepInstanceService; + this.startTimeMills = System.currentTimeMillis(); + this.taskContext = new TaskContext(stepInstance.getTaskInstanceId()); } @Override @@ -94,26 +98,16 @@ public ScheduleStrategy getScheduleStrategy() { } @Override - public void stop() { - synchronized (stopMonitor) { - if (!isStopped) { - StopTaskCounter.getInstance().decrement(getTaskId()); - this.isStopped = true; - } - } - } - - private boolean needToStop(StepInstanceDTO stepInstance) { - TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(stepInstance.getTaskInstanceId()); - // 刷新步骤状态 - stepInstance = taskInstanceService.getStepInstanceDetail(stepInstance.getId()); - // 如果任务处于“终止中”状态，触发任务终止 - if (taskInstance.getStatus() == RunStatusEnum.STOPPING) { - // 已经发送过停止命令的就不再重复发送了 - return !(RunStatusEnum.STOPPING == stepInstance.getStatus() - || RunStatusEnum.STOP_SUCCESS == stepInstance.getStatus()); + public void execute() { + try { + isRunning = true; + doExecute(); + } catch (Throwable t) { + filePrepareTaskResultHandler.onException(stepInstance, t); + setDoneStatus(); + } finally { + isRunning = false; } - return false; } private void doExecute() { @@ -128,19 +122,56 @@ private void doExecute() { setDoneStatus(); } // 超时判定 + // 默认30分钟 + long timeoutMills = 30 * 60 * 1000L; if (System.currentTimeMillis() - startTimeMills >= timeoutMills) { filePrepareTaskResultHandler.onTimeout(stepInstance); setDoneStatus(); } } + private boolean needToStop(StepInstanceDTO stepInstance) { + TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(stepInstance.getTaskInstanceId()); + // 刷新步骤状态 + stepInstance = stepInstanceService.getStepInstanceDetail( + stepInstance.getTaskInstanceId(), stepInstance.getId()); + // 如果任务处于“终止中”状态，触发任务终止 + if (taskInstance.getStatus() == RunStatusEnum.STOPPING) { + // 已经发送过停止命令的就不再重复发送了 + return !(RunStatusEnum.STOPPING == stepInstance.getStatus() + || RunStatusEnum.STOP_SUCCESS == stepInstance.getStatus()); + } + return false; + } + + public void setDoneStatus() { + isDoneWrapper.set(true); + } + @Override - public void execute() { - try { - doExecute(); - } catch (Throwable t) { - filePrepareTaskResultHandler.onException(stepInstance, t); - setDoneStatus(); + public void stop() { + synchronized (stopMonitor) { + if (!isStopped) { + gracefulStop(); + } else { + log.info("{} already stopped, ignore", getTaskId()); + } + } + } + + private void gracefulStop() { + if (!this.isRunning) { + log.info("gracefulStop begin:{}", getTaskId()); + // 1.停止正在进行的所有文件准备任务 + filePrepareService.stopPrepareFile(stepInstance); + // 2.MQ消息通知其他实例准备文件 + taskExecuteMQEventDispatcher.dispatchStepEvent( + StepEvent.prepareFile(stepInstance.getTaskInstanceId(), stepInstance.getId())); + this.isStopped = true; + StopTaskCounter.getInstance().decrement(getTaskId()); + log.info("gracefulStop end:{}", getTaskId()); + } else { + log.info("{} is running, wait for next schedule to gracefulStop", getTaskId()); } } @@ -148,4 +179,14 @@ public void execute() { public String getTaskId() { return "FilePrepareControlTask-" + stepInstance.getId() + "_" + stepInstance.getExecuteCount(); } + + @Override + public TaskContext getTaskContext() { + return taskContext; + } + + @Override + public String toString() { + return getTaskId(); + } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareService.java index 5b47e34e05..44e460f82d 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -41,19 +41,13 @@ public interface FilePrepareService { */ void stopPrepareFile(StepInstanceDTO stepInstance); - /** - * 重新为分发准备文件 - * - * @param stepInstanceId 步骤实例Id - */ - void retryPrepareFile(long stepInstanceId); - /** * 清理已准备完成的临时文件 * + * @param taskInstanceId 作业实例Id * @param stepInstanceId 步骤实例Id */ - void clearPreparedTmpFile(long stepInstanceId); + void clearPreparedTmpFile(long taskInstanceId, long stepInstanceId); /** * 是否需要准备源文件 diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareServiceImpl.java index d2d1ae2fc9..c646ae708e 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,23 +24,18 @@ package com.tencent.bk.job.execute.engine.prepare; -import com.tencent.bk.job.execute.common.constants.RunStatusEnum; -import com.tencent.bk.job.execute.engine.listener.event.EventSource; -import com.tencent.bk.job.execute.engine.listener.event.JobEvent; -import com.tencent.bk.job.execute.engine.listener.event.StepEvent; import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; +import com.tencent.bk.job.execute.engine.prepare.local.DefaultLocalFilePrepareTaskResultHandler; import com.tencent.bk.job.execute.engine.prepare.local.LocalFilePrepareService; -import com.tencent.bk.job.execute.engine.prepare.local.LocalFilePrepareTaskResultHandler; +import com.tencent.bk.job.execute.engine.prepare.third.DefaultThirdFilePrepareTaskResultHandler; import com.tencent.bk.job.execute.engine.prepare.third.ThirdFilePrepareService; -import com.tencent.bk.job.execute.engine.prepare.third.ThirdFilePrepareTaskResultHandler; import com.tencent.bk.job.execute.engine.result.ResultHandleManager; import com.tencent.bk.job.execute.model.FileSourceDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; +import com.tencent.bk.job.execute.service.StepInstanceService; import com.tencent.bk.job.execute.service.TaskInstanceService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; -import org.slf4j.helpers.FormattingTuple; -import org.slf4j.helpers.MessageFormatter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Component; @@ -62,29 +57,27 @@ public class FilePrepareServiceImpl implements FilePrepareService { private final TaskInstanceService taskInstanceService; private final TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher; private final ResultHandleManager resultHandleManager; + private final StepInstanceService stepInstanceService; @Autowired public FilePrepareServiceImpl(LocalFilePrepareService localFilePrepareService, ThirdFilePrepareService thirdFilePrepareService, TaskInstanceService taskInstanceService, TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher, - ResultHandleManager resultHandleManager) { + ResultHandleManager resultHandleManager, + StepInstanceService stepInstanceService) { this.localFilePrepareService = localFilePrepareService; this.thirdFilePrepareService = thirdFilePrepareService; this.taskInstanceService = taskInstanceService; this.taskExecuteMQEventDispatcher = taskExecuteMQEventDispatcher; this.resultHandleManager = resultHandleManager; + this.stepInstanceService = stepInstanceService; } @Override - public void retryPrepareFile(long stepInstanceId) { - // TODO - } - - @Override - public void clearPreparedTmpFile(long stepInstanceId) { - localFilePrepareService.clearPreparedTmpFile(stepInstanceId); - thirdFilePrepareService.clearPreparedTmpFile(stepInstanceId); + public void clearPreparedTmpFile(long taskInstanceId, long stepInstanceId) { + localFilePrepareService.clearPreparedTmpFile(taskInstanceId, stepInstanceId); + thirdFilePrepareService.clearPreparedTmpFile(taskInstanceId, stepInstanceId); } private void startPrepareLocalFileTask(StepInstanceDTO stepInstance, @@ -95,28 +88,8 @@ private void startPrepareLocalFileTask(StepInstanceDTO stepInstance, localFilePrepareService.prepareLocalFilesAsync( stepInstance, fileSourceList, - new LocalFilePrepareTaskResultHandler() { - @Override - public void onSuccess(JobTaskContext taskContext) { - log.info("[{}]:LocalFilePrepareTask success", stepInstance.getUniqueKey()); - resultList.add(new FilePrepareTaskResult(FilePrepareTaskResult.STATUS_SUCCESS, taskContext)); - latch.countDown(); - } - - @Override - public void onStopped(JobTaskContext taskContext) { - log.info("[{}]:LocalFilePrepareTask stopped", stepInstance.getUniqueKey()); - resultList.add(new FilePrepareTaskResult(FilePrepareTaskResult.STATUS_STOPPED, taskContext)); - latch.countDown(); - } - - @Override - public void onFailed(JobTaskContext taskContext) { - log.warn("[{}]:LocalFilePrepareTask failed", stepInstance.getUniqueKey()); - resultList.add(new FilePrepareTaskResult(FilePrepareTaskResult.STATUS_FAILED, taskContext)); - latch.countDown(); - } - }); + new DefaultLocalFilePrepareTaskResultHandler(stepInstance, resultList, latch) + ); } private void startPrepareThirdFileTask(StepInstanceDTO stepInstance, @@ -125,29 +98,7 @@ private void startPrepareThirdFileTask(StepInstanceDTO stepInstance, ) { thirdFilePrepareService.prepareThirdFileAsync( stepInstance, - new ThirdFilePrepareTaskResultHandler() { - - @Override - public void onSuccess(JobTaskContext taskContext) { - log.info("[{}]: ThirdFilePrepareTask success", stepInstance.getUniqueKey()); - resultList.add(new FilePrepareTaskResult(FilePrepareTaskResult.STATUS_SUCCESS, taskContext)); - latch.countDown(); - } - - @Override - public void onStopped(JobTaskContext taskContext) { - log.info("[{}]: ThirdFilePrepareTask stopped", stepInstance.getUniqueKey()); - resultList.add(new FilePrepareTaskResult(FilePrepareTaskResult.STATUS_STOPPED, taskContext)); - latch.countDown(); - } - - @Override - public void onFailed(JobTaskContext taskContext) { - log.warn("[{}]: ThirdFilePrepareTask failed", stepInstance.getUniqueKey()); - resultList.add(new FilePrepareTaskResult(FilePrepareTaskResult.STATUS_FAILED, taskContext)); - latch.countDown(); - } - } + new DefaultThirdFilePrepareTaskResultHandler(stepInstance, resultList, latch) ); } @@ -194,38 +145,20 @@ public void prepareFileForGseTask(StepInstanceDTO stepInstance) { startPrepareThirdFileTask(stepInstance, resultList, latch); } // 文件准备任务结果处理 - FilePrepareTaskResultHandler filePrepareTaskResultHandler = new FilePrepareTaskResultHandler() { - @Override - public void onFinished(StepInstanceDTO stepInstance, List resultList) { - log.info("[{}]: prepareTask finished", stepInstance.getUniqueKey()); - handleFinalTaskResult(resultList, stepInstance); - } - - @Override - public void onTimeout(StepInstanceDTO stepInstance) { - log.info("[{}]: prepareTask timeout", stepInstance.getUniqueKey()); - onFailed(stepInstance, null); - } - - @Override - public void onException(StepInstanceDTO stepInstance, Throwable t) { - FormattingTuple msg = MessageFormatter.format( - "[{}]: prepareTask exception", - stepInstance.getUniqueKey() - ); - log.error(msg.getMessage(), t); - onFailed(stepInstance, null); - } - }; + FilePrepareTaskResultHandler filePrepareTaskResultHandler = new DefaultFilePrepareTaskResultHandler( + stepInstanceService, + taskExecuteMQEventDispatcher + ); FilePrepareControlTask filePrepareControlTask = new FilePrepareControlTask( this, taskInstanceService, + taskExecuteMQEventDispatcher, stepInstance, latch, resultList, - filePrepareTaskResultHandler - ); + filePrepareTaskResultHandler, + stepInstanceService); resultHandleManager.handleDeliveredTask(filePrepareControlTask); } @@ -253,61 +186,6 @@ public void stopPrepareFile(StepInstanceDTO stepInstance) { thirdFilePrepareService.stopPrepareThirdFileAsync(stepInstance); } - private FilePrepareTaskResult combineTaskResult(List resultList) { - int size = resultList.size(); - if (size == 0) return null; - if (size == 1) return resultList.get(0); - for (FilePrepareTaskResult filePrepareTaskResult : resultList) { - if (filePrepareTaskResult.status == FilePrepareTaskResult.STATUS_FAILED) { - return filePrepareTaskResult; - } - } - for (FilePrepareTaskResult filePrepareTaskResult : resultList) { - if (filePrepareTaskResult.status == FilePrepareTaskResult.STATUS_STOPPED) { - return filePrepareTaskResult; - } - } - return resultList.get(0); - } - - private void handleFinalTaskResult(List resultList, StepInstanceDTO stepInstance) { - FilePrepareTaskResult finalResult = combineTaskResult(resultList); - if (finalResult == null) return; - if (finalResult.status == FilePrepareTaskResult.STATUS_SUCCESS) { - onSuccess(stepInstance, finalResult); - } else if (finalResult.status == FilePrepareTaskResult.STATUS_STOPPED) { - onStopped(stepInstance, finalResult); - } else if (finalResult.status == FilePrepareTaskResult.STATUS_FAILED) { - onFailed(stepInstance, finalResult); - } else { - log.warn("Unknown status:{}", finalResult.status); - } - } - - private void onSuccess(StepInstanceDTO stepInstance, FilePrepareTaskResult finalResult) { - if (!finalResult.getTaskContext().isForRetry()) { - // 直接进行下一步 - taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.continueGseFileStep(stepInstance.getId())); - } - } - - private void onStopped(StepInstanceDTO stepInstance, FilePrepareTaskResult finalResult) { - // 步骤状态变更 - taskInstanceService.updateStepStatus(stepInstance.getId(), RunStatusEnum.STOP_SUCCESS.getValue()); - // 任务状态变更 - taskExecuteMQEventDispatcher.dispatchJobEvent( - JobEvent.refreshJob(stepInstance.getTaskInstanceId(), - EventSource.buildStepEventSource(stepInstance.getId()))); - } - - private void onFailed(StepInstanceDTO stepInstance, FilePrepareTaskResult finalResult) { - // 文件源文件下载失败 - taskInstanceService.updateStepStatus(stepInstance.getId(), RunStatusEnum.FAIL.getValue()); - taskExecuteMQEventDispatcher.dispatchJobEvent( - JobEvent.refreshJob(stepInstance.getTaskInstanceId(), - EventSource.buildStepEventSource(stepInstance.getId()))); - } - @Override public boolean needToPrepareSourceFilesForGseTask(StepInstanceDTO stepInstance) { List fileSourceList = stepInstance.getFileSourceList(); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareTaskResult.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareTaskResult.java index 512afb004a..0e3ad2e00f 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareTaskResult.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareTaskResult.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -39,7 +39,7 @@ public JobTaskContext getTaskContext() { return taskContext; } - FilePrepareTaskResult(int status, JobTaskContext taskContext) { + public FilePrepareTaskResult(int status, JobTaskContext taskContext) { this.status = status; this.taskContext = taskContext; } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareTaskResultHandler.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareTaskResultHandler.java index 6e1552c6e2..c4572e8b7b 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareTaskResultHandler.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/FilePrepareTaskResultHandler.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/JobTaskContext.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/JobTaskContext.java index 2875b5d4b4..7648333980 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/JobTaskContext.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/JobTaskContext.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/JobTaskContextUtil.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/JobTaskContextUtil.java index 9b44970500..ef374c3242 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/JobTaskContextUtil.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/JobTaskContextUtil.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/local/ArtifactoryLocalFilePrepareTask.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/local/ArtifactoryLocalFilePrepareTask.java index 18ac67d5c7..759ae208a3 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/local/ArtifactoryLocalFilePrepareTask.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/local/ArtifactoryLocalFilePrepareTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,34 +24,19 @@ package com.tencent.bk.job.execute.engine.prepare.local; -import com.tencent.bk.job.common.artifactory.model.dto.NodeDTO; import com.tencent.bk.job.common.artifactory.sdk.ArtifactoryClient; -import com.tencent.bk.job.common.util.FileUtil; -import com.tencent.bk.job.common.util.ListUtil; -import com.tencent.bk.job.common.util.file.PathUtil; -import com.tencent.bk.job.execute.constants.Consts; import com.tencent.bk.job.execute.engine.prepare.JobTaskContext; import com.tencent.bk.job.execute.model.FileDetailDTO; import com.tencent.bk.job.execute.model.FileSourceDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; -import com.tencent.bk.job.manage.common.consts.task.TaskFileTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskFileTypeEnum; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.http.client.methods.HttpRequestBase; -import org.slf4j.helpers.FormattingTuple; -import org.slf4j.helpers.MessageFormatter; -import java.io.File; -import java.io.InputStream; import java.util.ArrayList; import java.util.List; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Future; import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicInteger; /** * 本地文件下载进度拉取任务调度 @@ -68,8 +53,9 @@ public class ArtifactoryLocalFilePrepareTask implements JobTaskContext { private final String artifactoryRepo; private final String jobStorageRootPath; private final List> futureList = new ArrayList<>(); - private final ThreadPoolExecutor threadPoolExecutor; - public static FinalResultHandler finalResultHandler = null; + private final ExecutorService localFileDownloadExecutor; + private final ExecutorService localFileWatchExecutor; + public static Future localFileWatchFuture = null; public ArtifactoryLocalFilePrepareTask( StepInstanceDTO stepInstance, @@ -80,7 +66,8 @@ public ArtifactoryLocalFilePrepareTask( String artifactoryProject, String artifactoryRepo, String jobStorageRootPath, - ThreadPoolExecutor threadPoolExecutor + ExecutorService localFileDownloadExecutor, + ExecutorService localFileWatchExecutor ) { this.stepInstance = stepInstance; this.isForRetry = isForRetry; @@ -90,7 +77,8 @@ public ArtifactoryLocalFilePrepareTask( this.artifactoryProject = artifactoryProject; this.artifactoryRepo = artifactoryRepo; this.jobStorageRootPath = jobStorageRootPath; - this.threadPoolExecutor = threadPoolExecutor; + this.localFileDownloadExecutor = localFileDownloadExecutor; + this.localFileWatchExecutor = localFileWatchExecutor; } @Override @@ -104,12 +92,13 @@ public void stop() { future.cancel(true); } } - if (finalResultHandler != null) { - finalResultHandler.interrupt(); + if (localFileWatchFuture != null) { + localFileWatchFuture.cancel(true); } } public void execute() { + int localFileCount = 0; for (FileSourceDTO fileSourceDTO : fileSourceList) { if (fileSourceDTO == null) { log.warn("[{}]:fileSourceDTO is null", stepInstance.getUniqueKey()); @@ -118,154 +107,28 @@ public void execute() { if (fileSourceDTO.isLocalUpload() || fileSourceDTO.getFileType() == TaskFileTypeEnum.LOCAL.getType()) { List files = fileSourceDTO.getFiles(); for (FileDetailDTO file : files) { - FileDownloadTask task = new FileDownloadTask(file); - Future future = threadPoolExecutor.submit(task); + LocalFileDownloadTask task = new LocalFileDownloadTask( + stepInstance, + artifactoryClient, + artifactoryProject, + artifactoryRepo, + jobStorageRootPath, + file + ); + Future future = localFileDownloadExecutor.submit(task); futureList.add(future); + localFileCount += 1; } } } - finalResultHandler = new FinalResultHandler(this, futureList, resultHandler); - finalResultHandler.start(); + LocalFileDownloadResultWatcher resultWatcher = new LocalFileDownloadResultWatcher( + this, + stepInstance, + futureList, + resultHandler + ); + localFileWatchFuture = localFileWatchExecutor.submit(resultWatcher); + log.info("[{}]: {} localFile downloadTask committed", stepInstance.getUniqueKey(), localFileCount); } - class FinalResultHandler extends Thread { - - ArtifactoryLocalFilePrepareTask artifactoryLocalFilePrepareTask; - List> futureList; - LocalFilePrepareTaskResultHandler resultHandler; - - FinalResultHandler( - ArtifactoryLocalFilePrepareTask artifactoryLocalFilePrepareTask, - List> futureList, - LocalFilePrepareTaskResultHandler resultHandler - ) { - this.artifactoryLocalFilePrepareTask = artifactoryLocalFilePrepareTask; - this.futureList = futureList; - this.resultHandler = resultHandler; - } - - @Override - public void run() { - List resultList = new ArrayList<>(); - for (Future future : futureList) { - try { - resultList.add(future.get(30, TimeUnit.MINUTES)); - } catch (InterruptedException e) { - log.info("[{}]:task stopped", stepInstance.getUniqueKey()); - resultHandler.onStopped(artifactoryLocalFilePrepareTask); - return; - } catch (ExecutionException e) { - log.info("[{}]:task download failed", stepInstance.getUniqueKey()); - resultHandler.onFailed(artifactoryLocalFilePrepareTask); - return; - } catch (TimeoutException e) { - log.info("[{}]:task download timeout", stepInstance.getUniqueKey()); - resultHandler.onFailed(artifactoryLocalFilePrepareTask); - return; - } - } - if (ListUtil.isAllTrue(resultList)) { - log.info("[{}]:all task success", stepInstance.getUniqueKey()); - resultHandler.onSuccess(artifactoryLocalFilePrepareTask); - } else { - int failCount = 0; - for (Boolean result : resultList) { - if (!result) { - failCount++; - } - } - log.warn( - "[{}]:{}/{} localFile prepare tasks failed", - stepInstance.getUniqueKey(), - failCount, - resultList.size() - ); - resultHandler.onFailed(artifactoryLocalFilePrepareTask); - } - } - } - - class FileDownloadTask implements Callable { - - private final FileDetailDTO file; - - FileDownloadTask(FileDetailDTO file) { - this.file = file; - } - - @Override - public Boolean call() { - try { - return doCall(); - } catch (Throwable t) { - FormattingTuple msg = MessageFormatter.format( - "[{}]:Unexpected error when prepare localFile {}", - stepInstance.getUniqueKey(), - file.getFilePath() - ); - log.error(msg.getMessage(), t); - return false; - } - } - - private Boolean doCall() { - String filePath = file.getFilePath(); - // 本地存储路径 - String localPath = PathUtil.joinFilePath(jobStorageRootPath, Consts.LOCAL_FILE_DIR_NAME); - localPath = PathUtil.joinFilePath(localPath, filePath); - File localFile = new File(localPath); - // 如果本地文件还未下载就已存在，说明是分发配置文件，直接完成准备阶段 - if (localFile.exists()) { - log.debug("[{}]:local file already exists", stepInstance.getUniqueKey()); - return true; - } - // 制品库的完整路径 - NodeDTO nodeDTO = artifactoryClient.queryNodeDetail(artifactoryProject, artifactoryRepo, filePath); - if (nodeDTO == null) { - log.warn( - "[{}]:File {} not exists in project {} repo {}", - stepInstance.getUniqueKey(), - filePath, - artifactoryProject, - artifactoryRepo - ); - return false; - } - Pair pair = artifactoryClient.getFileInputStream( - artifactoryProject, - artifactoryRepo, - filePath - ); - InputStream ins = pair.getLeft(); - Long fileSize = nodeDTO.getSize(); - // 保存到本地临时目录 - AtomicInteger speed = new AtomicInteger(0); - AtomicInteger process = new AtomicInteger(0); - try { - log.debug( - "[{}]:begin to download {} to {}", - stepInstance.getUniqueKey(), - filePath, - localPath - ); - FileUtil.writeInsToFile(ins, localPath, fileSize, speed, process); - return true; - } catch (InterruptedException e) { - log.warn( - "[{}]:Interrupted:Download {} to {}", - stepInstance.getUniqueKey(), - filePath, - localPath - ); - } catch (Exception e) { - log.error( - "[{}]:Fail to download {} to {}", - stepInstance.getUniqueKey(), - filePath, - localPath - ); - } - return false; - } - } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/local/DefaultLocalFilePrepareTaskResultHandler.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/local/DefaultLocalFilePrepareTaskResultHandler.java new file mode 100644 index 0000000000..5239346bc9 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/local/DefaultLocalFilePrepareTaskResultHandler.java @@ -0,0 +1,70 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.prepare.local; + +import com.tencent.bk.job.execute.engine.prepare.FilePrepareTaskResult; +import com.tencent.bk.job.execute.engine.prepare.JobTaskContext; +import com.tencent.bk.job.execute.model.StepInstanceDTO; +import lombok.extern.slf4j.Slf4j; + +import java.util.List; +import java.util.concurrent.CountDownLatch; + +@Slf4j +public class DefaultLocalFilePrepareTaskResultHandler implements LocalFilePrepareTaskResultHandler { + + private final StepInstanceDTO stepInstance; + private final List resultList; + private final CountDownLatch latch; + + public DefaultLocalFilePrepareTaskResultHandler(StepInstanceDTO stepInstance, + List resultList, + CountDownLatch latch) { + this.stepInstance = stepInstance; + this.resultList = resultList; + this.latch = latch; + } + + @Override + public void onSuccess(JobTaskContext taskContext) { + log.info("[{}]:LocalFilePrepareTask success", stepInstance.getUniqueKey()); + resultList.add(new FilePrepareTaskResult(FilePrepareTaskResult.STATUS_SUCCESS, taskContext)); + latch.countDown(); + } + + @Override + public void onStopped(JobTaskContext taskContext) { + log.info("[{}]:LocalFilePrepareTask stopped", stepInstance.getUniqueKey()); + resultList.add(new FilePrepareTaskResult(FilePrepareTaskResult.STATUS_STOPPED, taskContext)); + latch.countDown(); + } + + @Override + public void onFailed(JobTaskContext taskContext) { + log.warn("[{}]:LocalFilePrepareTask failed", stepInstance.getUniqueKey()); + resultList.add(new FilePrepareTaskResult(FilePrepareTaskResult.STATUS_FAILED, taskContext)); + latch.countDown(); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/local/LocalFileDownloadResultWatcher.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/local/LocalFileDownloadResultWatcher.java new file mode 100644 index 0000000000..2aee52cf54 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/local/LocalFileDownloadResultWatcher.java @@ -0,0 +1,98 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.prepare.local; + +import com.tencent.bk.job.common.util.ListUtil; +import com.tencent.bk.job.execute.model.StepInstanceDTO; +import lombok.extern.slf4j.Slf4j; + +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +@Slf4j +public class LocalFileDownloadResultWatcher implements Runnable { + + private final ArtifactoryLocalFilePrepareTask artifactoryLocalFilePrepareTask; + private final StepInstanceDTO stepInstance; + private final List> futureList; + private final LocalFilePrepareTaskResultHandler resultHandler; + + LocalFileDownloadResultWatcher( + ArtifactoryLocalFilePrepareTask artifactoryLocalFilePrepareTask, + StepInstanceDTO stepInstance, + List> futureList, + LocalFilePrepareTaskResultHandler resultHandler + ) { + this.artifactoryLocalFilePrepareTask = artifactoryLocalFilePrepareTask; + this.stepInstance = stepInstance; + this.futureList = futureList; + this.resultHandler = resultHandler; + } + + @Override + public void run() { + List resultList = new ArrayList<>(); + for (Future future : futureList) { + try { + resultList.add(future.get(30, TimeUnit.MINUTES)); + } catch (InterruptedException e) { + log.info("[{}]:task stopped", stepInstance.getUniqueKey()); + resultHandler.onStopped(artifactoryLocalFilePrepareTask); + return; + } catch (ExecutionException e) { + log.info("[{}]:task download failed", stepInstance.getUniqueKey()); + resultHandler.onFailed(artifactoryLocalFilePrepareTask); + return; + } catch (TimeoutException e) { + log.info("[{}]:task download timeout", stepInstance.getUniqueKey()); + resultHandler.onFailed(artifactoryLocalFilePrepareTask); + return; + } + } + if (ListUtil.isAllTrue(resultList)) { + log.info("[{}]:All {} localFile download task(s) success", stepInstance.getUniqueKey(), futureList.size()); + resultHandler.onSuccess(artifactoryLocalFilePrepareTask); + } else { + int failCount = 0; + for (Boolean result : resultList) { + if (!result) { + failCount++; + } + } + log.warn( + "[{}]:{}/{} localFile download tasks failed", + stepInstance.getUniqueKey(), + failCount, + resultList.size() + ); + resultHandler.onFailed(artifactoryLocalFilePrepareTask); + } + } +} + diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/local/LocalFileDownloadTask.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/local/LocalFileDownloadTask.java new file mode 100644 index 0000000000..d0dc7c93f9 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/local/LocalFileDownloadTask.java @@ -0,0 +1,208 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.prepare.local; + +import com.tencent.bk.job.common.artifactory.model.dto.NodeDTO; +import com.tencent.bk.job.common.artifactory.sdk.ArtifactoryClient; +import com.tencent.bk.job.common.util.TimeUtil; +import com.tencent.bk.job.common.util.file.FileUtil; +import com.tencent.bk.job.common.util.file.PathUtil; +import com.tencent.bk.job.execute.constants.Consts; +import com.tencent.bk.job.execute.model.FileDetailDTO; +import com.tencent.bk.job.execute.model.StepInstanceDTO; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.codec.digest.DigestUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.http.client.methods.HttpRequestBase; +import org.slf4j.helpers.FormattingTuple; +import org.slf4j.helpers.MessageFormatter; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.concurrent.Callable; +import java.util.concurrent.atomic.AtomicInteger; + +@Slf4j +public class LocalFileDownloadTask implements Callable { + + private final StepInstanceDTO stepInstance; + private final ArtifactoryClient artifactoryClient; + private final String artifactoryProject; + private final String artifactoryRepo; + private final String jobStorageRootPath; + private final FileDetailDTO file; + + public LocalFileDownloadTask(StepInstanceDTO stepInstance, + ArtifactoryClient artifactoryClient, + String artifactoryProject, + String artifactoryRepo, + String jobStorageRootPath, + FileDetailDTO file) { + this.stepInstance = stepInstance; + this.artifactoryClient = artifactoryClient; + this.artifactoryProject = artifactoryProject; + this.artifactoryRepo = artifactoryRepo; + this.jobStorageRootPath = jobStorageRootPath; + this.file = file; + } + + @Override + public Boolean call() { + try { + return doCall(); + } catch (Throwable t) { + FormattingTuple msg = MessageFormatter.format( + "[{}]:Unexpected error when prepare localFile {}", + stepInstance.getUniqueKey(), + file.getFilePath() + ); + log.error(msg.getMessage(), t); + return false; + } + } + + private Boolean doCall() { + String filePath = file.getFilePath(); + // 制品库的完整路径 + NodeDTO nodeDTO = artifactoryClient.queryNodeDetail(artifactoryProject, artifactoryRepo, filePath); + if (nodeDTO == null) { + log.warn( + "[{}]:File {} not exists in project {} repo {}", + stepInstance.getUniqueKey(), + filePath, + artifactoryProject, + artifactoryRepo + ); + return false; + } + // 本地存储路径 + String localPath = PathUtil.joinFilePath(jobStorageRootPath, Consts.LOCAL_FILE_DIR_NAME); + localPath = PathUtil.joinFilePath(localPath, filePath); + // 如果本地文件还未下载就已存在并且Md5正确，直接完成准备阶段 + if (currentLocalFileValid(localPath, nodeDTO)) { + // 将最后修改时间设置为当前时间，避免在分发过程中被清理 + File localFile = new File(localPath); + boolean lastModifyTimeSet = localFile.setLastModified(System.currentTimeMillis()); + if (!lastModifyTimeSet) { + log.warn("Fail to set lastModifyTime for {}", localPath); + } + return true; + } + Pair pair = artifactoryClient.getFileInputStream( + artifactoryProject, + artifactoryRepo, + filePath + ); + InputStream ins = pair.getLeft(); + Long fileSize = nodeDTO.getSize(); + // 保存到本地临时目录 + AtomicInteger speed = new AtomicInteger(0); + AtomicInteger process = new AtomicInteger(0); + try { + log.debug( + "[{}]:begin to download {} to {}", + stepInstance.getUniqueKey(), + filePath, + localPath + ); + FileUtil.writeInsToFile(ins, localPath, fileSize, speed, process); + log.info( + "[{}]:success: {} -> {}", + stepInstance.getUniqueKey(), + filePath, + localPath + ); + return true; + } catch (InterruptedException e) { + log.warn( + "[{}]:Interrupted:Download {} to {}", + stepInstance.getUniqueKey(), + filePath, + localPath + ); + } catch (Exception e) { + log.error( + "[{}]:Fail to download {} to {}", + stepInstance.getUniqueKey(), + filePath, + localPath + ); + } + return false; + } + + /** + * 判断当前的本地文件是否可以直接使用 + * + * @param localPath 本地文件绝对路径 + * @param nodeDTO 制品库中的节点信息 + * @return 当前的本地文件是否可以直接使用 + */ + private boolean currentLocalFileValid(String localPath, NodeDTO nodeDTO) { + File localFile = new File(localPath); + if (!localFile.exists()) { + return false; + } + String md5 = nodeDTO.getMd5(); + if (StringUtils.isBlank(md5)) { + log.warn("Md5 from node is blank, node={}", nodeDTO); + return false; + } + String lastModifyTimeFormat = "yyyy-MM-dd HH:mm:ss.SSS"; + try (InputStream fis = new FileInputStream(localFile)) { + String localFileMd5 = DigestUtils.md5Hex(fis); + if (!md5.equals(localFileMd5)) { + log.info( + "[{}]:local file {} exists with wrong md5:{}, expect md5:{}," + + " lastModifyTime:{}, download again", + stepInstance.getUniqueKey(), + localPath, + localFileMd5, + md5, + TimeUtil.formatTime(localFile.lastModified(), lastModifyTimeFormat) + ); + return false; + } + log.info( + "[{}]:local file {} already exists with same md5:{}, lastModifyTime:{}", + stepInstance.getUniqueKey(), + localPath, + localFileMd5, + TimeUtil.formatTime(localFile.lastModified(), lastModifyTimeFormat) + ); + return true; + } catch (IOException e) { + String msg = MessageFormatter.format( + "Fail to check md5 of localFile: {}, download again", + localPath + ).getMessage(); + log.warn(msg, e); + } + return false; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/local/LocalFilePrepareService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/local/LocalFilePrepareService.java index 557a16f31d..f6a69bb8fb 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/local/LocalFilePrepareService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/local/LocalFilePrepareService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,56 +24,62 @@ package com.tencent.bk.job.execute.engine.prepare.local; +import com.tencent.bk.job.common.artifactory.config.ArtifactoryConfig; import com.tencent.bk.job.common.artifactory.sdk.ArtifactoryClient; import com.tencent.bk.job.common.constant.JobConstants; -import com.tencent.bk.job.execute.config.ArtifactoryConfig; +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.execute.config.FileDistributeConfig; import com.tencent.bk.job.execute.config.LocalFileConfigForExecute; -import com.tencent.bk.job.execute.config.StorageSystemConfig; import com.tencent.bk.job.execute.engine.prepare.JobTaskContext; +import com.tencent.bk.job.execute.model.ExecuteTargetDTO; import com.tencent.bk.job.execute.model.FileSourceDTO; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; -import com.tencent.bk.job.execute.service.AgentService; -import com.tencent.bk.job.execute.service.TaskInstanceService; -import com.tencent.bk.job.manage.common.consts.task.TaskFileTypeEnum; +import com.tencent.bk.job.execute.service.LocalFileDistributeSourceHostProvisioner; +import com.tencent.bk.job.execute.service.StepInstanceService; +import com.tencent.bk.job.manage.api.common.constants.task.TaskFileTypeEnum; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.ExecutorService; @Slf4j @Service public class LocalFilePrepareService { - private final StorageSystemConfig storageSystemConfig; + private final FileDistributeConfig fileDistributeConfig; private final ArtifactoryConfig artifactoryConfig; private final LocalFileConfigForExecute localFileConfigForExecute; - private final AgentService agentService; - private final TaskInstanceService taskInstanceService; + private final LocalFileDistributeSourceHostProvisioner localFileDistributeSourceHostProvisioner; + private final StepInstanceService stepInstanceService; private final ArtifactoryClient artifactoryClient; private final Map taskMap = new ConcurrentHashMap<>(); - private final ThreadPoolExecutor localFilePrepareExecutor; + private final ExecutorService localFileDownloadExecutor; + private final ExecutorService localFileWatchExecutor; @Autowired - public LocalFilePrepareService(StorageSystemConfig storageSystemConfig, + public LocalFilePrepareService(FileDistributeConfig fileDistributeConfig, ArtifactoryConfig artifactoryConfig, LocalFileConfigForExecute localFileConfigForExecute, - AgentService agentService, - TaskInstanceService taskInstanceService, - ArtifactoryClient artifactoryClient, - @Qualifier("localFilePrepareExecutor") ThreadPoolExecutor localFilePrepareExecutor) { - this.storageSystemConfig = storageSystemConfig; + LocalFileDistributeSourceHostProvisioner localFileDistributeSourceHostProvisioner, + StepInstanceService stepInstanceService, + @Qualifier("jobArtifactoryClient") ArtifactoryClient artifactoryClient, + @Qualifier("localFileDownloadExecutor") ExecutorService localFileDownloadExecutor, + @Qualifier("localFileWatchExecutor") ExecutorService localFileWatchExecutor) { + this.fileDistributeConfig = fileDistributeConfig; this.artifactoryConfig = artifactoryConfig; this.localFileConfigForExecute = localFileConfigForExecute; - this.agentService = agentService; - this.taskInstanceService = taskInstanceService; + this.localFileDistributeSourceHostProvisioner = localFileDistributeSourceHostProvisioner; + this.stepInstanceService = stepInstanceService; this.artifactoryClient = artifactoryClient; - this.localFilePrepareExecutor = localFilePrepareExecutor; + this.localFileDownloadExecutor = localFileDownloadExecutor; + this.localFileWatchExecutor = localFileWatchExecutor; } public void stopPrepareLocalFilesAsync( @@ -107,24 +113,37 @@ public void prepareLocalFilesAsync( artifactoryClient, artifactoryConfig.getArtifactoryJobProject(), localFileConfigForExecute.getLocalUploadRepo(), - storageSystemConfig.getJobStorageRootPath(), - localFilePrepareExecutor + fileDistributeConfig.getJobDistributeRootPath(), + localFileDownloadExecutor, + localFileWatchExecutor ); taskMap.put(stepInstance.getUniqueKey(), task); task.execute(); } private void fillLocalFileSourceHost(List fileSourceList, StepInstanceBaseDTO stepInstance) { + boolean isGseV2Task = stepInstance.isTargetGseV2Agent(); fileSourceList.forEach(fileSourceDTO -> { if (fileSourceDTO.getFileType() == TaskFileTypeEnum.LOCAL.getType() || fileSourceDTO.isLocalUpload()) { - fileSourceDTO.setServers(agentService.getLocalServersDTO()); + HostDTO localHost = localFileDistributeSourceHostProvisioner.getLocalFileDistributeSourceHost().clone(); + if (!isGseV2Task) { + // 如果目标Agent是GSE V1, 那么源Agent也必须要GSE1.0 Agent，设置agentId={云区域:ip} + localHost.setAgentId(localHost.toCloudIp()); + } + ExecuteTargetDTO fileSourceExecuteObjects = new ExecuteTargetDTO(); + fileSourceExecuteObjects.setStaticIpList(Collections.singletonList(localHost)); + fileSourceExecuteObjects.buildMergedExecuteObjects(stepInstance.isSupportExecuteObjectFeature()); + fileSourceDTO.setServers(fileSourceExecuteObjects); + log.info("FillLocalFileSourceHost -> stepInstanceId: {}, isGseV2Task: {}, localFileSource: {}", + stepInstance.getId(), isGseV2Task, fileSourceDTO); } }); // 更新本地文件任务内容 - taskInstanceService.updateResolvedSourceFile(stepInstance.getId(), fileSourceList); + stepInstanceService.updateResolvedSourceFile(stepInstance.getTaskInstanceId(), + stepInstance.getId(), fileSourceList); } - public void clearPreparedTmpFile(long stepInstanceId) { + public void clearPreparedTmpFile(long taskInstanceId, long stepInstanceId) { // 本地文件暂不支持实时清理，依赖定时清理 } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/local/LocalFilePrepareTaskResultHandler.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/local/LocalFilePrepareTaskResultHandler.java index e7b5b23795..72671f06b3 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/local/LocalFilePrepareTaskResultHandler.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/local/LocalFilePrepareTaskResultHandler.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/local/NFSLocalFilePrepareTask.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/local/NFSLocalFilePrepareTask.java index b43a347e66..7bec49524d 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/local/NFSLocalFilePrepareTask.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/local/NFSLocalFilePrepareTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/DefaultThirdFilePrepareTaskResultHandler.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/DefaultThirdFilePrepareTaskResultHandler.java new file mode 100644 index 0000000000..e6d29f88c6 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/DefaultThirdFilePrepareTaskResultHandler.java @@ -0,0 +1,70 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.prepare.third; + +import com.tencent.bk.job.execute.engine.prepare.FilePrepareTaskResult; +import com.tencent.bk.job.execute.engine.prepare.JobTaskContext; +import com.tencent.bk.job.execute.model.StepInstanceDTO; +import lombok.extern.slf4j.Slf4j; + +import java.util.List; +import java.util.concurrent.CountDownLatch; + +@Slf4j +public class DefaultThirdFilePrepareTaskResultHandler implements ThirdFilePrepareTaskResultHandler { + + private final StepInstanceDTO stepInstance; + private final List resultList; + private final CountDownLatch latch; + + public DefaultThirdFilePrepareTaskResultHandler(StepInstanceDTO stepInstance, + List resultList, + CountDownLatch latch) { + this.stepInstance = stepInstance; + this.resultList = resultList; + this.latch = latch; + } + + @Override + public void onSuccess(JobTaskContext taskContext) { + log.info("[{}]: ThirdFilePrepareTask success", stepInstance.getUniqueKey()); + resultList.add(new FilePrepareTaskResult(FilePrepareTaskResult.STATUS_SUCCESS, taskContext)); + latch.countDown(); + } + + @Override + public void onStopped(JobTaskContext taskContext) { + log.info("[{}]: ThirdFilePrepareTask stopped", stepInstance.getUniqueKey()); + resultList.add(new FilePrepareTaskResult(FilePrepareTaskResult.STATUS_STOPPED, taskContext)); + latch.countDown(); + } + + @Override + public void onFailed(JobTaskContext taskContext) { + log.warn("[{}]: ThirdFilePrepareTask failed", stepInstance.getUniqueKey()); + resultList.add(new FilePrepareTaskResult(FilePrepareTaskResult.STATUS_FAILED, taskContext)); + latch.countDown(); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/FileWorkerHostService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/FileWorkerHostService.java new file mode 100644 index 0000000000..0873564a0c --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/FileWorkerHostService.java @@ -0,0 +1,100 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.prepare.third; + +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.common.util.ip.IpUtils; +import com.tencent.bk.job.execute.service.HostService; +import com.tencent.bk.job.manage.model.inner.ServiceHostDTO; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Triple; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Primary; +import org.springframework.stereotype.Component; + +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; + +/** + * 第三方源文件File-Worker的主机解析服务 + */ +@Slf4j +@Primary +@Component +public class FileWorkerHostService { + + private final HostService hostService; + + @Autowired + public FileWorkerHostService(HostService hostService) { + this.hostService = hostService; + } + + // 查询file-worker对应主机信息使用60s缓存，避免短时间内多次重复查询 + private final LoadingCache, HostDTO> fileWorkerHostCache = CacheBuilder.newBuilder() + .maximumSize(1).expireAfterWrite(60, TimeUnit.SECONDS). + build(new CacheLoader, HostDTO>() { + @SuppressWarnings("all") + @Override + public HostDTO load(Triple triple) { + return parseFileWorkerHost(triple.getLeft(), triple.getMiddle(), triple.getRight()); + } + } + ); + + public HostDTO parseFileWorkerHostWithCache(Long cloudAreaId, String ipProtocol, String ip) { + try { + return fileWorkerHostCache.get(buildCacheKey(cloudAreaId, ipProtocol, ip)); + } catch (ExecutionException e) { + log.error("Fail to parseFileWorkerHostWithCache", e); + return null; + } + } + + private Triple buildCacheKey(Long cloudAreaId, String ipProtocol, String ip) { + return Triple.of(cloudAreaId, ipProtocol, ip); + } + + private HostDTO parseFileWorkerHost(Long cloudAreaId, String ipProtocol, String ip) { + if (StringUtils.isBlank(ipProtocol)) { + ipProtocol = IpUtils.inferProtocolByIp(ip); + log.info("ipProtocol is null or blank, use {} inferred by ip {}", ipProtocol, ip); + } + HostDTO hostDTO; + if (IpUtils.PROTOCOL_IP_V6.equalsIgnoreCase(ipProtocol)) { + hostDTO = ServiceHostDTO.toHostDTO(hostService.getHostByCloudIpv6(cloudAreaId, ip)); + } else { + hostDTO = ServiceHostDTO.toHostDTO(hostService.getHost(new HostDTO(cloudAreaId, ip))); + } + if (log.isDebugEnabled()) { + log.debug("host get by ({},{},{}) is {}", ipProtocol, cloudAreaId, ip, hostDTO); + } + return hostDTO; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/ThirdFilePrepareService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/ThirdFilePrepareService.java index daf77a7fcd..575f05115f 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/ThirdFilePrepareService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/ThirdFilePrepareService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,21 +28,24 @@ import com.tencent.bk.job.common.model.InternalResponse; import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.util.file.PathUtil; -import com.tencent.bk.job.execute.client.FileSourceTaskResourceClient; -import com.tencent.bk.job.execute.dao.FileSourceTaskLogDAO; import com.tencent.bk.job.execute.engine.listener.event.GseTaskEvent; import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; import com.tencent.bk.job.execute.engine.prepare.JobTaskContext; import com.tencent.bk.job.execute.engine.result.ResultHandleManager; +import com.tencent.bk.job.execute.model.ExecuteTargetDTO; import com.tencent.bk.job.execute.model.FileDetailDTO; import com.tencent.bk.job.execute.model.FileSourceDTO; import com.tencent.bk.job.execute.model.FileSourceTaskLogDTO; -import com.tencent.bk.job.execute.model.ServersDTO; +import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; +import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.service.AccountService; -import com.tencent.bk.job.execute.service.HostService; +import com.tencent.bk.job.execute.service.FileSourceTaskLogService; import com.tencent.bk.job.execute.service.LogService; +import com.tencent.bk.job.execute.service.StepInstanceService; import com.tencent.bk.job.execute.service.TaskInstanceService; +import com.tencent.bk.job.execute.service.ThirdFileDistributeSourceHostProvisioner; +import com.tencent.bk.job.file_gateway.api.inner.ServiceFileSourceTaskResource; import com.tencent.bk.job.file_gateway.consts.TaskStatusEnum; import com.tencent.bk.job.file_gateway.model.req.inner.ClearTaskFilesReq; import com.tencent.bk.job.file_gateway.model.req.inner.FileSourceBatchDownloadTaskReq; @@ -52,6 +55,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; +import org.jooq.exception.DataAccessException; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Primary; import org.springframework.stereotype.Component; @@ -71,11 +75,13 @@ @Component public class ThirdFilePrepareService { private final ResultHandleManager resultHandleManager; - private final FileSourceTaskResourceClient fileSourceTaskResource; + private final ServiceFileSourceTaskResource fileSourceTaskResource; private final TaskInstanceService taskInstanceService; - private final FileSourceTaskLogDAO fileSourceTaskLogDAO; + private final StepInstanceService stepInstanceService; + private final FileSourceTaskLogService fileSourceTaskLogService; private final AccountService accountService; - private final HostService hostService; + private final FileWorkerHostService fileWorkerHostService; + private final ThirdFileDistributeSourceHostProvisioner thirdFileDistributeSourceHostProvisioner; private final LogService logService; private final TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher; // 记录第三方文件准备任务信息，用于在需要时查找并终止任务 @@ -83,21 +89,25 @@ public class ThirdFilePrepareService { @Autowired public ThirdFilePrepareService(ResultHandleManager resultHandleManager, - FileSourceTaskResourceClient fileSourceTaskResource, + ServiceFileSourceTaskResource fileSourceTaskResource, TaskInstanceService taskInstanceService, - FileSourceTaskLogDAO fileSourceTaskLogDAO, + StepInstanceService stepInstanceService, + FileSourceTaskLogService fileSourceTaskLogService, AccountService accountService, - HostService hostService, + FileWorkerHostService fileWorkerHostService, LogService logService, - TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher) { + TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher, + ThirdFileDistributeSourceHostProvisioner thirdFileDistributeSourceHostProvisioner) { this.resultHandleManager = resultHandleManager; this.fileSourceTaskResource = fileSourceTaskResource; this.taskInstanceService = taskInstanceService; - this.fileSourceTaskLogDAO = fileSourceTaskLogDAO; + this.stepInstanceService = stepInstanceService; + this.fileSourceTaskLogService = fileSourceTaskLogService; this.accountService = accountService; - this.hostService = hostService; + this.fileWorkerHostService = fileWorkerHostService; this.logService = logService; this.taskExecuteMQEventDispatcher = taskExecuteMQEventDispatcher; + this.thirdFileDistributeSourceHostProvisioner = thirdFileDistributeSourceHostProvisioner; } /** @@ -106,15 +116,28 @@ public ThirdFilePrepareService(ResultHandleManager resultHandleManager, * @param taskInfoDTO 文件源文件下载任务信息 * @param fileSourceDTO 分发源文件数据 */ - private void setTaskInfoIntoThirdFileSource(TaskInfoDTO taskInfoDTO, FileSourceDTO fileSourceDTO) { + private void setTaskInfoIntoThirdFileSource(StepInstanceBaseDTO stepInstance, + TaskInfoDTO taskInfoDTO, + FileSourceDTO fileSourceDTO) { String fileSourceTaskId = taskInfoDTO.getTaskId(); if (fileSourceDTO.getServers() == null) { - fileSourceDTO.setServers(new ServersDTO()); + fileSourceDTO.setServers(new ExecuteTargetDTO()); } List hostDTOList = new ArrayList<>(); - hostDTOList.add(new HostDTO(taskInfoDTO.getCloudId(), taskInfoDTO.getIp())); + HostDTO hostDTO = fileWorkerHostService.parseFileWorkerHostWithCache( + taskInfoDTO.getCloudId(), + taskInfoDTO.getIpProtocol(), + taskInfoDTO.getIp() + ); + log.info( + "[{}]: fileSourceTaskId={} start, sourceHost={}", + stepInstance.getUniqueKey(), + fileSourceTaskId, + hostDTO + ); + hostDTOList.add(hostDTO); fileSourceDTO.getServers().setStaticIpList(hostDTOList); - fileSourceDTO.getServers().setIpList(hostDTOList); + fileSourceDTO.getServers().buildMergedExecuteObjects(stepInstance.isSupportExecuteObjectFeature()); fileSourceDTO.setFileSourceTaskId(fileSourceTaskId); fileSourceDTO.getFiles().forEach(fileDetailDTO -> { // 含文件源名称的文件路径 @@ -130,7 +153,8 @@ private void setTaskInfoIntoThirdFileSource(TaskInfoDTO taskInfoDTO, FileSourceD * @param batchTaskInfoDTO 文件源批量任务信息 * @param thirdFileSourceList 第三方文件源列表 */ - private void setBatchTaskInfoIntoThirdFileSource(BatchTaskInfoDTO batchTaskInfoDTO, + private void setBatchTaskInfoIntoThirdFileSource(StepInstanceDTO stepInstance, + BatchTaskInfoDTO batchTaskInfoDTO, List thirdFileSourceList) { List taskInfoList = batchTaskInfoDTO.getTaskInfoList(); for (int i = 0; i < taskInfoList.size(); i++) { @@ -138,7 +162,7 @@ private void setBatchTaskInfoIntoThirdFileSource(BatchTaskInfoDTO batchTaskInfoD FileSourceDTO fileSourceDTO = thirdFileSourceList.get(i); Integer fileSourceId = fileSourceDTO.getFileSourceId(); if (fileSourceId != null && fileSourceId > 0) { - setTaskInfoIntoThirdFileSource(taskInfoDTO, fileSourceDTO); + setTaskInfoIntoThirdFileSource(stepInstance, taskInfoDTO, fileSourceDTO); } } } @@ -176,8 +200,9 @@ private Pair, List> parseThirdFileSou } } // 收集第三方文件源文件路径 - List filePaths = files.parallelStream() + List filePaths = files.stream() .map(FileDetailDTO::getThirdFilePath) + .distinct() .collect(Collectors.toList()); // 收集文件源文件任务 fileSourceTaskList.add(new FileSourceTaskContent(fileSourceId, filePaths)); @@ -216,13 +241,14 @@ public void prepareThirdFileAsync( batchTaskInfoDTO.getTaskInfoList() ); // 填充任务信息到分发源文件数据 - setBatchTaskInfoIntoThirdFileSource(batchTaskInfoDTO, thirdFileSourceList); + setBatchTaskInfoIntoThirdFileSource(stepInstance, batchTaskInfoDTO, thirdFileSourceList); log.debug("[{}]: fileSourceList={}", stepInstance.getUniqueKey(), fileSourceList); // 放进文件源下载任务进度表中 FileSourceTaskLogDTO fileSourceTaskLogDTO = buildInitFileSourceTaskLog(stepInstance, batchTaskInfoDTO); - fileSourceTaskLogDAO.saveFileSourceTaskLog(fileSourceTaskLogDTO); + insertOrUpdateFileSourceTaskLog(fileSourceTaskLogDTO); // 更新文件源任务状态 - taskInstanceService.updateResolvedSourceFile(stepInstance.getId(), fileSourceList); + stepInstanceService.updateResolvedSourceFile(stepInstance.getTaskInstanceId(), + stepInstance.getId(), fileSourceList); // 异步轮询文件下载任务 ThirdFilePrepareTask task = asyncWatchThirdFilePulling( stepInstance, @@ -234,6 +260,28 @@ public void prepareThirdFileAsync( taskMap.put(stepInstance.getUniqueKey(), task); } + private void insertOrUpdateFileSourceTaskLog(FileSourceTaskLogDTO fileSourceTaskLogDTO) { + boolean shouldRetry; + do { + try { + int insertedNum = fileSourceTaskLogService.addFileSourceTaskLog(fileSourceTaskLogDTO); + log.info("{} fileSourceTaskLog inserted", insertedNum); + return; + } catch (DataAccessException e) { + String message = e.getMessage(); + if (message != null && message.equalsIgnoreCase("Deadlock found")) { + log.info("Deadlock found when insert fileSourceTaskLog, retry", e); + shouldRetry = true; + } else { + log.info("Fail to insert fileSourceTaskLog, update instead", e); + shouldRetry = false; + } + } + } while (shouldRetry); + int updatedNum = fileSourceTaskLogService.updateFileSourceTaskLog(fileSourceTaskLogDTO); + log.info("{} fileSourceTaskLog updated", updatedNum); + } + /** * 立即继续步骤 * @@ -242,6 +290,7 @@ public void prepareThirdFileAsync( private void continueStepAtOnce(StepInstanceDTO stepInstance) { taskExecuteMQEventDispatcher.dispatchGseTaskEvent( GseTaskEvent.startGseTask( + stepInstance.getTaskInstanceId(), stepInstance.getId(), stepInstance.getExecuteCount(), stepInstance.getBatch(), @@ -261,6 +310,7 @@ private void continueStepAtOnce(StepInstanceDTO stepInstance) { private FileSourceTaskLogDTO buildInitFileSourceTaskLog(StepInstanceDTO stepInstance, BatchTaskInfoDTO batchTaskInfoDTO) { FileSourceTaskLogDTO fileSourceTaskLogDTO = new FileSourceTaskLogDTO(); + fileSourceTaskLogDTO.setTaskInstanceId(stepInstance.getTaskInstanceId()); fileSourceTaskLogDTO.setStepInstanceId(stepInstance.getId()); fileSourceTaskLogDTO.setExecuteCount(stepInstance.getExecuteCount()); fileSourceTaskLogDTO.setFileSourceBatchTaskId(batchTaskInfoDTO.getBatchTaskId()); @@ -276,8 +326,8 @@ public void stopPrepareThirdFileAsync(StepInstanceDTO stepInstance) { } } - public void clearPreparedTmpFile(long stepInstanceId) { - StepInstanceDTO stepInstance = taskInstanceService.getStepInstanceDetail(stepInstanceId); + public void clearPreparedTmpFile(long taskInstanceId, long stepInstanceId) { + StepInstanceDTO stepInstance = stepInstanceService.getStepInstanceDetail(taskInstanceId, stepInstanceId); // 找出所有第三方文件源的TaskId进行清理 List fileSourceList = stepInstance.getFileSourceList(); List fileSourceTaskIdList = findFileSourceTaskIds(stepInstanceId, fileSourceList); @@ -347,17 +397,24 @@ private ThirdFilePrepareTask asyncWatchThirdFilePulling( boolean isForRetry, ThirdFilePrepareTaskResultHandler resultHandler ) { - ThirdFilePrepareTask batchResultHandleTask = - new ThirdFilePrepareTask( - stepInstance, - fileSourceList, - batchTaskId, - isForRetry, - new RecordableThirdFilePrepareTaskResultHandler(stepInstance, resultHandler) - ); + TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(stepInstance.getTaskInstanceId()); + ThirdFilePrepareTask batchResultHandleTask = new ThirdFilePrepareTask( + taskInstance, + stepInstance, + fileSourceList, + batchTaskId, + isForRetry, + new RecordableThirdFilePrepareTaskResultHandler(stepInstance, resultHandler) + ); batchResultHandleTask.initDependentService( - fileSourceTaskResource, taskInstanceService, accountService, - hostService, logService, taskExecuteMQEventDispatcher, fileSourceTaskLogDAO + fileSourceTaskResource, + stepInstanceService, + accountService, + fileWorkerHostService, + logService, + taskExecuteMQEventDispatcher, + fileSourceTaskLogService, + thirdFileDistributeSourceHostProvisioner ); resultHandleManager.handleDeliveredTask(batchResultHandleTask); return batchResultHandleTask; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/ThirdFilePrepareTask.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/ThirdFilePrepareTask.java index 81925b7f00..3550f2387c 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/ThirdFilePrepareTask.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/ThirdFilePrepareTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,39 +24,44 @@ package com.tencent.bk.job.execute.engine.prepare.third; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.InternalException; import com.tencent.bk.job.common.model.InternalResponse; import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.util.json.JsonUtils; -import com.tencent.bk.job.execute.client.FileSourceTaskResourceClient; -import com.tencent.bk.job.execute.common.constants.FileDistModeEnum; +import com.tencent.bk.job.execute.common.constants.FileDistStatusEnum; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; -import com.tencent.bk.job.execute.dao.FileSourceTaskLogDAO; import com.tencent.bk.job.execute.engine.listener.event.EventSource; import com.tencent.bk.job.execute.engine.listener.event.JobEvent; import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; import com.tencent.bk.job.execute.engine.prepare.JobTaskContext; import com.tencent.bk.job.execute.engine.result.ContinuousScheduledTask; import com.tencent.bk.job.execute.engine.result.ScheduleStrategy; import com.tencent.bk.job.execute.engine.result.StopTaskCounter; +import com.tencent.bk.job.execute.engine.result.TaskContext; import com.tencent.bk.job.execute.model.AccountDTO; +import com.tencent.bk.job.execute.model.ExecuteTargetDTO; import com.tencent.bk.job.execute.model.FileDetailDTO; import com.tencent.bk.job.execute.model.FileSourceDTO; import com.tencent.bk.job.execute.model.FileSourceTaskLogDTO; -import com.tencent.bk.job.execute.model.ServersDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; +import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.service.AccountService; -import com.tencent.bk.job.execute.service.HostService; +import com.tencent.bk.job.execute.service.FileSourceTaskLogService; import com.tencent.bk.job.execute.service.LogService; -import com.tencent.bk.job.execute.service.TaskInstanceService; +import com.tencent.bk.job.execute.service.StepInstanceService; +import com.tencent.bk.job.execute.service.ThirdFileDistributeSourceHostProvisioner; +import com.tencent.bk.job.file_gateway.api.inner.ServiceFileSourceTaskResource; import com.tencent.bk.job.file_gateway.consts.TaskStatusEnum; import com.tencent.bk.job.file_gateway.model.req.inner.StopBatchTaskReq; import com.tencent.bk.job.file_gateway.model.resp.inner.BatchTaskStatusDTO; import com.tencent.bk.job.file_gateway.model.resp.inner.FileLogPieceDTO; import com.tencent.bk.job.file_gateway.model.resp.inner.FileSourceTaskStatusDTO; import com.tencent.bk.job.file_gateway.model.resp.inner.ThirdFileSourceTaskLogDTO; +import com.tencent.bk.job.logsvr.model.service.ServiceExecuteObjectLogDTO; import com.tencent.bk.job.logsvr.model.service.ServiceFileTaskLogDTO; -import com.tencent.bk.job.logsvr.model.service.ServiceHostLogDTO; -import com.tencent.bk.job.manage.model.inner.ServiceHostDTO; +import com.tencent.bk.job.manage.api.common.constants.task.TaskFileTypeEnum; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -67,7 +72,6 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.concurrent.atomic.AtomicBoolean; @@ -79,6 +83,7 @@ @Slf4j public class ThirdFilePrepareTask implements ContinuousScheduledTask, JobTaskContext { + private final TaskInstanceDTO taskInstance; private final StepInstanceDTO stepInstance; private final List fileSourceList; private final String batchTaskId; @@ -88,15 +93,15 @@ public class ThirdFilePrepareTask implements ContinuousScheduledTask, JobTaskCon */ private final Object stopMonitor = new Object(); volatile AtomicBoolean isDoneWrapper = new AtomicBoolean(false); - volatile AtomicBoolean isReadyForNextStepWrapper = new AtomicBoolean(false); - private FileSourceTaskResourceClient fileSourceTaskResource; - private TaskInstanceService taskInstanceService; + private ServiceFileSourceTaskResource fileSourceTaskResource; private AccountService accountService; - private HostService hostService; + private FileWorkerHostService fileWorkerHostService; private LogService logService; private TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher; - private FileSourceTaskLogDAO fileSourceTaskLogDAO; + private FileSourceTaskLogService fileSourceTaskLogService; private final ThirdFilePrepareTaskResultHandler resultHandler; + private StepInstanceService stepInstanceService; + private ThirdFileDistributeSourceHostProvisioner thirdFileDistributeSourceHostProvisioner; private int pullTimes = 0; private long logStart = 0L; @@ -105,36 +110,42 @@ public class ThirdFilePrepareTask implements ContinuousScheduledTask, JobTaskCon */ private volatile boolean isStopped = false; + private final TaskContext taskContext; + public ThirdFilePrepareTask( + TaskInstanceDTO taskInstance, StepInstanceDTO stepInstance, List fileSourceList, String batchTaskId, boolean isForRetry, - ThirdFilePrepareTaskResultHandler resultHandler - ) { + ThirdFilePrepareTaskResultHandler resultHandler) { + this.taskInstance = taskInstance; this.stepInstance = stepInstance; this.fileSourceList = fileSourceList; this.batchTaskId = batchTaskId; this.isForRetry = isForRetry; this.resultHandler = resultHandler; + this.taskContext = new TaskContext(stepInstance.getTaskInstanceId()); } public void initDependentService( - FileSourceTaskResourceClient fileSourceTaskResource, - TaskInstanceService taskInstanceService, + ServiceFileSourceTaskResource fileSourceTaskResource, + StepInstanceService stepInstanceService, AccountService accountService, - HostService hostService, + FileWorkerHostService fileWorkerHostService, LogService logService, TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher, - FileSourceTaskLogDAO fileSourceTaskLogDAO + FileSourceTaskLogService fileSourceTaskLogService, + ThirdFileDistributeSourceHostProvisioner thirdFileDistributeSourceHostProvisioner ) { this.fileSourceTaskResource = fileSourceTaskResource; - this.taskInstanceService = taskInstanceService; this.accountService = accountService; - this.hostService = hostService; + this.fileWorkerHostService = fileWorkerHostService; this.logService = logService; this.taskExecuteMQEventDispatcher = taskExecuteMQEventDispatcher; - this.fileSourceTaskLogDAO = fileSourceTaskLogDAO; + this.fileSourceTaskLogService = fileSourceTaskLogService; + this.stepInstanceService = stepInstanceService; + this.thirdFileDistributeSourceHostProvisioner = thirdFileDistributeSourceHostProvisioner; } @Override @@ -142,10 +153,6 @@ public boolean isFinished() { return this.isDoneWrapper.get(); } - public boolean isReadyForNext() { - return this.isReadyForNextStepWrapper.get(); - } - @Override public ScheduleStrategy getScheduleStrategy() { // 每秒拉取一次任务状态 @@ -210,7 +217,7 @@ public BatchTaskStatusDTO getFileSourceBatchTaskResults(StepInstanceDTO stepInst for (FileSourceTaskStatusDTO fileSourceTaskStatusDTO : fileSourceTaskStatusInfoList) { List logList = fileSourceTaskStatusDTO.getLogList(); if (logList != null && !logList.isEmpty()) { - writeLogs(stepInstance, logList); + writeLogs(stepInstance, fileSourceTaskStatusDTO, logList); if (logList.size() > maxLogSize) { maxLogSize = logList.size(); } @@ -222,6 +229,13 @@ public BatchTaskStatusDTO getFileSourceBatchTaskResults(StepInstanceDTO stepInst } // 任务结束了，且日志拉取完毕才算结束 isDone = batchTaskStatusDTO.isDone() && allLogDone; + log.info( + "[{}]: batchTaskDone={}, allLogDone={}, logStart={}", + stepInstance.getUniqueKey(), + batchTaskStatusDTO.isDone(), + allLogDone, + logStart + ); } catch (Exception e) { FormattingTuple msg = MessageFormatter.format( "[{}][{}]:Exception occurred when getFileSourceTaskStatus, tried {} times", @@ -244,32 +258,16 @@ public BatchTaskStatusDTO getFileSourceBatchTaskResults(StepInstanceDTO stepInst if (isDone) { isDoneWrapper.set(true); handleFileSourceTaskResult(stepInstance, batchTaskStatusDTO); - isReadyForNextStepWrapper.set(true); } return batchTaskStatusDTO; } - private void fillHostInfo(HostDTO hostDTO, ServiceHostDTO serviceHostDTO) { - if (hostDTO == null || serviceHostDTO == null) { - return; - } - hostDTO.setHostId(serviceHostDTO.getHostId()); - hostDTO.setBkCloudId(serviceHostDTO.getCloudAreaId()); - hostDTO.setIp(serviceHostDTO.getIp()); - hostDTO.setAgentId(serviceHostDTO.getCloudIp()); - } - private void handleFileSourceTaskResult( StepInstanceDTO stepInstance, BatchTaskStatusDTO batchTaskStatusDTO ) { // 更新文件源拉取任务耗时数据 - FileSourceTaskLogDTO fileSourceTaskLogDTO = fileSourceTaskLogDAO.getFileSourceTaskLogByBatchTaskId(batchTaskId); - if (fileSourceTaskLogDTO != null) { - Long endTime = System.currentTimeMillis(); - fileSourceTaskLogDAO.updateTimeConsumingByBatchTaskId(batchTaskId, null, endTime, - endTime - fileSourceTaskLogDTO.getStartTime()); - } + updateBatchTaskTimeStatistics(); List taskStatusList = batchTaskStatusDTO.getFileSourceTaskStatusInfoList(); if (taskStatusList.isEmpty()) { // 直接成功 @@ -286,7 +284,12 @@ private void handleFileSourceTaskResult( boolean allSuccess = statePair.getLeft(); boolean stopped = statePair.getRight(); if (allSuccess) { - onSuccess(taskStatusList); + log.info( + "[{}]: batchTaskId={}, fileSourceTaskStatus all success", + stepInstance.getUniqueKey(), + batchTaskId + ); + onSuccess(stepInstance, taskStatusList); } else if (stopped) { resultHandler.onStopped(this); } else { @@ -294,6 +297,22 @@ private void handleFileSourceTaskResult( } } + private void updateBatchTaskTimeStatistics() { + FileSourceTaskLogDTO fileSourceTaskLogDTO = fileSourceTaskLogService.getFileSourceTaskLogByBatchTaskId( + stepInstance.getTaskInstanceId(), batchTaskId); + if (fileSourceTaskLogDTO == null) { + return; + } + Long endTime = System.currentTimeMillis(); + fileSourceTaskLogService.updateTimeConsumingByBatchTaskId( + stepInstance.getTaskInstanceId(), + batchTaskId, + null, + endTime, + endTime - fileSourceTaskLogDTO.getStartTime() + ); + } + private Pair checkSuccessAndStopState(List fileSourceTaskStatusList) { boolean allSuccess = true; boolean stopped = false; @@ -322,103 +341,132 @@ private Pair checkSuccessAndStopState(List fileSourceTaskStatusList) { + private void onSuccess(StepInstanceDTO stepInstance, List fileSourceTaskStatusList) { Map map = new HashMap<>(); fileSourceTaskStatusList.forEach(taskStatus -> map.put(taskStatus.getTaskId(), taskStatus)); //添加服务器文件信息 + boolean isGseV2Task = stepInstance.isTargetGseV2Agent(); + int updatedNum = 0; for (FileSourceDTO fileSourceDTO : fileSourceList) { String fileSourceTaskId = fileSourceDTO.getFileSourceTaskId(); - if (StringUtils.isNotBlank(fileSourceTaskId)) { - FileSourceTaskStatusDTO fileSourceTaskStatusDTO = map.get(fileSourceTaskId); - fileSourceDTO.setAccount("root"); - AccountDTO accountDTO = accountService.getAccountByAccountName(stepInstance.getAppId(), "root"); - if (accountDTO == null) { - //业务无root账号，报错提示 - log.error( - "[{}]: No root account in appId={}, plz config one", - stepInstance.getUniqueKey(), - stepInstance.getAppId() - ); - taskInstanceService.updateStepStatus(stepInstance.getId(), RunStatusEnum.FAIL.getValue()); - taskExecuteMQEventDispatcher.dispatchJobEvent( - JobEvent.refreshJob(stepInstance.getTaskInstanceId(), - EventSource.buildStepEventSource(stepInstance.getId()))); - return; - } - fileSourceDTO.setAccountId(accountDTO.getId()); - fileSourceDTO.setLocalUpload(false); - ServersDTO servers = new ServersDTO(); - HostDTO hostDTO = new HostDTO( - fileSourceTaskStatusDTO.getCloudId(), - fileSourceTaskStatusDTO.getIp() - ); - ServiceHostDTO serviceHostDTO = hostService.getHost(hostDTO); - if (serviceHostDTO == null) { - log.warn( - "[{}]: Cannot find file-worker host info by {}, " + - "plz check whether file-worker gse agent is installed", - stepInstance.getUniqueKey(), - hostDTO - ); - } - fillHostInfo(hostDTO, serviceHostDTO); - List hostDTOList = Collections.singletonList(hostDTO); - servers.addStaticIps(hostDTOList); - if (servers.getIpList() == null) { - servers.setIpList(hostDTOList); - } else { - servers.getIpList().addAll(hostDTOList); - // 去重 - servers.setIpList(new ArrayList<>(new HashSet<>(servers.getIpList()))); - } - fileSourceDTO.setServers(servers); - Map filePathMap = fileSourceTaskStatusDTO.getFilePathMap(); - log.debug( - "[{}]: filePathMap={}", - stepInstance.getUniqueKey(), - filePathMap - ); - List files = fileSourceDTO.getFiles(); - // 设置downloadPath进行后续GSE分发 - for (FileDetailDTO file : files) { - String downloadPath = filePathMap.get(file.getThirdFilePath()); - file.setFilePath(downloadPath); - file.setResolvedFilePath(downloadPath); - } + if (StringUtils.isBlank(fileSourceTaskId)) { + continue; } + updateServerInfoForFileSource(map, fileSourceTaskId, fileSourceDTO, isGseV2Task); + updatedNum += 1; + } + if (updatedNum > 0) { + log.info("[{}]: {} serverInfo updated", stepInstance.getUniqueKey(), updatedNum); + } else { + log.warn("[{}]: no serverInfo updated", stepInstance.getUniqueKey()); } //更新StepInstance - taskInstanceService.updateResolvedSourceFile(stepInstance.getId(), fileSourceList); + stepInstanceService.updateResolvedSourceFile( + stepInstance.getTaskInstanceId(), stepInstance.getId(), fileSourceList); resultHandler.onSuccess(this); } - private void writeLogs(StepInstanceDTO stepInstance, List logDTOList) { - for (ThirdFileSourceTaskLogDTO logDTO : logDTOList) { - HostDTO host = buildHost(logDTO); - logService.writeFileLogWithTimestamp( - stepInstance.getCreateTime(), - stepInstance.getId(), - stepInstance.getExecuteCount(), - stepInstance.getBatch(), - host, - buildServiceHostLogDTO(host, logDTO), - System.currentTimeMillis() + private void updateServerInfoForFileSource(Map map, + String fileSourceTaskId, + FileSourceDTO fileSourceDTO, + boolean isGseV2Task) { + FileSourceTaskStatusDTO fileSourceTaskStatusDTO = map.get(fileSourceTaskId); + fileSourceDTO.setAccount("root"); + AccountDTO accountDTO = accountService.getAccountByAccountName(stepInstance.getAppId(), "root"); + if (accountDTO == null) { + //业务无root账号，报错提示 + log.error( + "[{}]: No root account in appId={}, plz config one", + stepInstance.getUniqueKey(), + stepInstance.getAppId() + ); + stepInstanceService.updateStepStatus(stepInstance.getTaskInstanceId(), stepInstance.getId(), + RunStatusEnum.FAIL.getValue()); + taskExecuteMQEventDispatcher.dispatchJobEvent( + JobEvent.refreshJob(stepInstance.getTaskInstanceId(), + EventSource.buildStepEventSource(stepInstance.getTaskInstanceId(), stepInstance.getId()))); + return; + } + fileSourceDTO.setAccountId(accountDTO.getId()); + fileSourceDTO.setLocalUpload(false); + + log.info( + "third file download to file-worker completed, taskId={}, file-worker=(cloudId:{},protocol:{}, ip{})", + stepInstance.getTaskInstanceId(), + fileSourceTaskStatusDTO.getCloudId(), + fileSourceTaskStatusDTO.getIpProtocol(), + fileSourceTaskStatusDTO.getIp() + ); + ExecuteTargetDTO executeTargetDTO = new ExecuteTargetDTO(); + HostDTO hostDTO = thirdFileDistributeSourceHostProvisioner.getThirdFileDistributeSourceHost( + fileSourceTaskStatusDTO.getCloudId(), + fileSourceTaskStatusDTO.getIpProtocol(), + fileSourceTaskStatusDTO.getIp() + ); + if (hostDTO == null) { + log.error( + "[{}]: Cannot find file-worker host info by IP{} (cloudAreaId={}, ip={}), " + + "plz check whether file-worker gse agent is installed", + stepInstance.getUniqueKey(), + fileSourceTaskStatusDTO.getIpProtocol(), + fileSourceTaskStatusDTO.getCloudId(), + fileSourceTaskStatusDTO.getIp() ); + throw new InternalException(ErrorCode.FILE_WORKER_NOT_FOUND); + } + + HostDTO sourceHost = hostDTO.clone(); + if (isGseV2Task) { + if (StringUtils.isBlank(sourceHost.getAgentId())) { + log.error("Using gseV2, source host agent id is empty! host: {}", sourceHost); + throw new InternalException(ErrorCode.CAN_NOT_FIND_AVAILABLE_FILE_WORKER); + } + } else { + sourceHost.setAgentId(sourceHost.toCloudIp()); + } + log.info( + "[{}]: fileSourceTaskId={} success, sourceHost={}", + stepInstance.getUniqueKey(), + fileSourceTaskId, + sourceHost + ); + List hostDTOList = Collections.singletonList(sourceHost); + executeTargetDTO.addStaticHosts(hostDTOList); + executeTargetDTO.buildMergedExecuteObjects(stepInstance.isSupportExecuteObjectFeature()); + fileSourceDTO.setServers(executeTargetDTO); + Map filePathMap = fileSourceTaskStatusDTO.getFilePathMap(); + log.debug( + "[{}]: filePathMap={}", + stepInstance.getUniqueKey(), + filePathMap + ); + List files = fileSourceDTO.getFiles(); + // 设置downloadPath进行后续GSE分发 + for (FileDetailDTO file : files) { + String downloadPath = filePathMap.get(file.getThirdFilePath()); + file.setFilePath(downloadPath); + file.setResolvedFilePath(downloadPath); } } - /** - * 根据第三方文件源任务日志中的file-worker所在IP获取对应的主机信息 - * - * @param thirdFileSourceTaskLog 第三方文件源任务日志 - * @return 主机对象 - */ - private HostDTO buildHost(ThirdFileSourceTaskLogDTO thirdFileSourceTaskLog) { - String cloudIp = thirdFileSourceTaskLog.getIp(); - HostDTO host = HostDTO.fromCloudIp(cloudIp); - ServiceHostDTO serviceHost = hostService.getHost(host); - fillHostInfo(host, serviceHost); - return host; + + private void writeLogs(StepInstanceDTO stepInstance, + FileSourceTaskStatusDTO fileSourceTaskStatusDTO, + List logDTOList) { + List serviceExecuteObjectLogDTOList = new ArrayList<>(); + for (ThirdFileSourceTaskLogDTO logDTO : logDTOList) { + HostDTO host = fileWorkerHostService.parseFileWorkerHostWithCache( + fileSourceTaskStatusDTO.getCloudId(), + fileSourceTaskStatusDTO.getIpProtocol(), + fileSourceTaskStatusDTO.getIp() + ); + serviceExecuteObjectLogDTOList.add(buildServiceHostLogDTO(host, logDTO)); + } + logService.writeFileLogsWithTimestamp( + taskInstance, + serviceExecuteObjectLogDTOList, + System.currentTimeMillis() + ); } /** @@ -428,48 +476,54 @@ private HostDTO buildHost(ThirdFileSourceTaskLogDTO thirdFileSourceTaskLog) { * @param thirdFileSourceTaskLog 第三方源文件下载任务日志 * @return 统一格式日志实体 */ - private ServiceHostLogDTO buildServiceHostLogDTO(HostDTO host, ThirdFileSourceTaskLogDTO thirdFileSourceTaskLog) { + private ServiceExecuteObjectLogDTO buildServiceHostLogDTO(HostDTO host, + ThirdFileSourceTaskLogDTO thirdFileSourceTaskLog) { if (thirdFileSourceTaskLog == null) { return null; } - ServiceHostLogDTO serviceHostLog = new ServiceHostLogDTO(); + ServiceExecuteObjectLogDTO serviceHostLog = new ServiceExecuteObjectLogDTO(); serviceHostLog.setStepInstanceId(stepInstance.getId()); serviceHostLog.setExecuteCount(stepInstance.getExecuteCount()); serviceHostLog.setBatch(stepInstance.getBatch()); - serviceHostLog.setHostId(host.getHostId()); - serviceHostLog.setIp(host.getIp()); - serviceHostLog.setFileTaskLogs(buildFileTaskLogs(host, thirdFileSourceTaskLog.getFileTaskLogs())); + ExecuteObject executeObject; + if (stepInstance.isSupportExecuteObjectFeature()) { + executeObject = new ExecuteObject(host); + serviceHostLog.setExecuteObjectId(executeObject.getId()); + } else { + serviceHostLog.setHostId(host.getHostId()); + serviceHostLog.setCloudIp(host.getIp()); + executeObject = ExecuteObject.buildCompatibleExecuteObject(host); + } + serviceHostLog.setFileTaskLogs(buildFileTaskLogs(executeObject, thirdFileSourceTaskLog.getFileTaskLogs())); return serviceHostLog; } /** * 将第三方文件源任务日志转为Job系统统一格式日志用于存储与展示 * - * @param host 日志关联的主机信息 + * @param executeObject 日志关联的执行对象信息 * @param fileLogPieces 文件下载日志列表 * @return Job日志列表 */ - private List buildFileTaskLogs(HostDTO host, List fileLogPieces) { + private List buildFileTaskLogs(ExecuteObject executeObject, + List fileLogPieces) { if (CollectionUtils.isEmpty(fileLogPieces)) { return Collections.emptyList(); } - return fileLogPieces.stream().map(fileLogPiece -> { - ServiceFileTaskLogDTO serviceFileTaskLog = new ServiceFileTaskLogDTO(); - // 第三方源文件下载后用于分发上传，仅有源文件相关属性 - serviceFileTaskLog.setMode(FileDistModeEnum.UPLOAD.getValue()); - serviceFileTaskLog.setSrcIp(fileLogPiece.getSrcIp()); - serviceFileTaskLog.setSrcHostId(host.getHostId()); - serviceFileTaskLog.setDisplaySrcIp(fileLogPiece.getDisplaySrcIp()); - serviceFileTaskLog.setSrcFile(fileLogPiece.getSrcFile()); - serviceFileTaskLog.setDisplaySrcFile(fileLogPiece.getDisplaySrcFile()); - serviceFileTaskLog.setSize(fileLogPiece.getSize()); - serviceFileTaskLog.setStatus(fileLogPiece.getStatus()); - serviceFileTaskLog.setStatusDesc(fileLogPiece.getStatusDesc()); - serviceFileTaskLog.setSpeed(fileLogPiece.getSpeed()); - serviceFileTaskLog.setProcess(fileLogPiece.getProcess()); - serviceFileTaskLog.setContent(fileLogPiece.getContent()); - return serviceFileTaskLog; - }).collect(Collectors.toList()); + return fileLogPieces.stream().map(fileLogPiece -> + logService.buildUploadServiceFileTaskLogDTO( + stepInstance, + TaskFileTypeEnum.FILE_SOURCE, + fileLogPiece.getSrcFile(), + fileLogPiece.getDisplaySrcFile(), + executeObject, + FileDistStatusEnum.getFileDistStatus(fileLogPiece.getStatus()), + fileLogPiece.getSize(), + fileLogPiece.getSpeed(), + fileLogPiece.getProcess(), + fileLogPiece.getContent() + ) + ).collect(Collectors.toList()); } @Override @@ -487,6 +541,11 @@ public String getTaskId() { return "file_source_batch_task:" + this.stepInstance.getId() + ":" + this.stepInstance.getExecuteCount(); } + @Override + public TaskContext getTaskContext() { + return taskContext; + } + @Override public boolean isForRetry() { return isForRetry; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/ThirdFilePrepareTaskResultHandler.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/ThirdFilePrepareTaskResultHandler.java index 50a61b6411..18df087577 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/ThirdFilePrepareTaskResultHandler.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/prepare/third/ThirdFilePrepareTaskResultHandler.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/quota/limit/NotAliveJobDetector.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/quota/limit/NotAliveJobDetector.java new file mode 100644 index 0000000000..c0cd5240da --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/quota/limit/NotAliveJobDetector.java @@ -0,0 +1,91 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.quota.limit; + +import com.tencent.bk.job.common.redis.util.LockUtils; +import com.tencent.bk.job.execute.model.TaskInstanceDTO; +import com.tencent.bk.job.execute.service.TaskInstanceService; +import com.tencent.bk.job.manage.GlobalAppScopeMappingService; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.util.Set; +import java.util.UUID; + +/** + * 未存活作业检查与处理 + */ +@Slf4j +@Component +@EnableScheduling +public class NotAliveJobDetector { + private final RunningJobResourceQuotaManager runningJobResourceQuotaManager; + + private final TaskInstanceService taskInstanceService; + + private final String requestId = UUID.randomUUID().toString(); + + public NotAliveJobDetector(RunningJobResourceQuotaManager runningJobResourceQuotaManager, + TaskInstanceService taskInstanceService) { + this.runningJobResourceQuotaManager = runningJobResourceQuotaManager; + this.taskInstanceService = taskInstanceService; + } + + /** + * 兜底方案。为了防止系统异常、程序 bug 等原因导致 redis 中的作业记录没有被清理，需要定时清理。每10min触发一次 + */ + @Scheduled(cron = "0 0/10 * * * ?") + public void detectNotAliveJob() { + try { + if (LockUtils.tryGetDistributedLock("job:execute:not:alive:job:detect:lock", requestId, 60000L)) { + log.info("Detect not alive job start ..."); + Set notAliveJobInstanceIds = runningJobResourceQuotaManager.getNotAliveJobInstanceIds(); + if (CollectionUtils.isEmpty(notAliveJobInstanceIds)) { + return; + } + log.info("Found not alive job, notAliveJobInstanceIds : {}", notAliveJobInstanceIds); + notAliveJobInstanceIds.forEach(notAliveJobInstanceId -> { + TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(notAliveJobInstanceId); + if (taskInstance != null) { + log.info("Remove not alive job : {}", notAliveJobInstanceId); + runningJobResourceQuotaManager.removeJob( + taskInstance.getAppCode(), + GlobalAppScopeMappingService.get().getScopeByAppId(taskInstance.getAppId()), + notAliveJobInstanceId + ); + } else { + log.error("Job instance record not found, notAliveJobInstanceId : {}", notAliveJobInstanceId); + } + }); + } + } catch (Throwable e) { + log.error("Detect not alive job caught exception", e); + } + } + +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/quota/limit/ResourceQuotaCheckResultEnum.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/quota/limit/ResourceQuotaCheckResultEnum.java new file mode 100644 index 0000000000..30d9230e8f --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/quota/limit/ResourceQuotaCheckResultEnum.java @@ -0,0 +1,52 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.quota.limit; + +public enum ResourceQuotaCheckResultEnum { + + NO_LIMIT("no_limit"), + RESOURCE_SCOPE_LIMIT("resource_scope_quota_limit"), + APP_LIMIT("app_quota_limit"), + SYSTEM_LIMIT("system_quota_limit"); + + private final String value; + + ResourceQuotaCheckResultEnum(String value) { + this.value = value; + } + + public static ResourceQuotaCheckResultEnum valOf(String value) { + for (ResourceQuotaCheckResultEnum resultEnum : values()) { + if (resultEnum.value.equals(value)) { + return resultEnum; + } + } + throw new IllegalArgumentException("No ResourceQuotaCheckResultEnum constant: " + value); + } + + public boolean isExceedLimit() { + return this != NO_LIMIT; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/quota/limit/RunningJobKeepaliveManager.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/quota/limit/RunningJobKeepaliveManager.java new file mode 100644 index 0000000000..44623915f4 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/quota/limit/RunningJobKeepaliveManager.java @@ -0,0 +1,173 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.quota.limit; + + +import com.tencent.bk.job.common.util.ThreadUtils; +import com.tencent.bk.job.execute.constants.RedisKeys; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.redis.connection.RedisZSetCommands; +import org.springframework.data.redis.core.RedisCallback; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.data.redis.serializer.RedisSerializer; +import org.springframework.scheduling.annotation.EnableScheduling; +import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.stereotype.Component; + +import java.util.Collection; +import java.util.HashSet; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +/** + * 正在执行中的作业存活状态维持 + */ +@Slf4j +@Component +@EnableScheduling +public class RunningJobKeepaliveManager { + private final Object lock = new Object(); + private final RedisTemplate redisTemplate; + private final Map runningJobKeepaliveTasks = new ConcurrentHashMap<>(); + + @Autowired + public RunningJobKeepaliveManager(StringRedisTemplate redisTemplate) { + this.redisTemplate = redisTemplate; + } + + public void addKeepaliveTask(long jobInstanceId) { + KeepaliveTask keepaliveTask = updateTaskKeepaliveInfo(jobInstanceId); + if (keepaliveTask != null) { + log.info("Add running job keepalive task: {}", keepaliveTask); + this.runningJobKeepaliveTasks.put(jobInstanceId, keepaliveTask); + } + } + + private KeepaliveTask updateTaskKeepaliveInfo(long jobInstanceId) { + int maxWaitingSeconds = 600; + while (maxWaitingSeconds > 0) { + try { + long timestamp = System.currentTimeMillis(); + updateJobLatestAliveTimestamp(jobInstanceId, timestamp); + return new KeepaliveTask(jobInstanceId, timestamp); + } catch (Throwable e) { + log.error("Update running job keepalive task error, jobInstanceId: " + jobInstanceId, e); + log.info("Wait for 5 seconds retry!"); + ThreadUtils.sleep(5000L); + maxWaitingSeconds -= 5; + } + } + return null; + } + + private void updateJobLatestAliveTimestamp(Long jobInstanceId, long currentTimestamp) { + RedisSerializer stringRedisSerializer = redisTemplate.getStringSerializer(); + byte[] key = stringRedisSerializer.serialize(RedisKeys.RUNNING_JOB_ZSET_KEY); + byte[] member = stringRedisSerializer.serialize(String.valueOf(jobInstanceId)); + redisTemplate.execute((RedisCallback) connection -> { + connection.zAdd( + Objects.requireNonNull(key), + currentTimestamp, + Objects.requireNonNull(member), + // 只有 member 存在的时候才会更新，避免写入一些异常的作业数据 + RedisZSetCommands.ZAddArgs.ifExists() + ); + return null; + }); + } + + public void stopKeepaliveTask(long jobInstanceId) { + log.info("Stop running job keepalive task : {}", jobInstanceId); + KeepaliveTask keepaliveTask = runningJobKeepaliveTasks.get(jobInstanceId); + if (keepaliveTask == null) { + log.warn("keepalive task task already stopped, jobInstanceId: {}!", jobInstanceId); + return; + } + synchronized (lock) { + this.runningJobKeepaliveTasks.remove(jobInstanceId); + } + } + + /** + * 定时刷新作业存活心跳。 1min周期 + */ + @Scheduled(cron = "0 * * * * ?") + public void refreshTaskKeepaliveInfo() { + log.info("Refresh running job keepalive task start..."); + if (runningJobKeepaliveTasks.isEmpty()) { + return; + } + Set refreshJobInstanceIds = new HashSet<>(); + long startInMills = System.currentTimeMillis(); + synchronized (lock) { + Collection keepaliveTasks = runningJobKeepaliveTasks.values(); + long currentTimestamp = System.currentTimeMillis(); + keepaliveTasks.stream() + // 设置刷新间隔最小为 30 秒，降低写 redis 的频率 + .filter(keepaliveTask -> currentTimestamp - keepaliveTask.getTimestamp() > 30000L) + .forEach(keepaliveTask -> { + long jobInstanceId = keepaliveTask.getJobInstanceId(); + try { + // 二次确认，防止在运行期间任务被移除 + if (runningJobKeepaliveTasks.get(jobInstanceId) != null) { + updateJobLatestAliveTimestamp(jobInstanceId, currentTimestamp); + keepaliveTask.setTimestamp(currentTimestamp); + refreshJobInstanceIds.add(jobInstanceId); + } + } catch (Throwable e) { + String errorMsg = + "Refresh running job keepalive task fail, jobInstanceId: " + jobInstanceId; + log.error(errorMsg, e); + } + }); + } + log.info("Refresh running job keepalive task done! taskSize: {},refreshJobInstanceIds: {}", + refreshJobInstanceIds.size(), refreshJobInstanceIds); + long cost = System.currentTimeMillis() - startInMills; + if (cost > 1000L) { + log.info("Refresh running job keepalive task is slow, taskSize: {}, cost: {}", + refreshJobInstanceIds.size(), cost); + } + } + + @Getter + @Setter + @ToString + @NoArgsConstructor + @AllArgsConstructor + private static class KeepaliveTask { + private long jobInstanceId; + private long timestamp; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/quota/limit/RunningJobResourceQuotaManager.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/quota/limit/RunningJobResourceQuotaManager.java new file mode 100644 index 0000000000..62943a788b --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/quota/limit/RunningJobResourceQuotaManager.java @@ -0,0 +1,269 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.quota.limit; + +import com.tencent.bk.job.common.exception.JobMicroServiceBootException; +import com.tencent.bk.job.common.metrics.CommonMetricTags; +import com.tencent.bk.job.common.metrics.CommonMetricValues; +import com.tencent.bk.job.common.model.dto.ResourceScope; +import com.tencent.bk.job.common.service.quota.RunningJobResourceQuotaStore; +import com.tencent.bk.job.execute.constants.RedisKeys; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tags; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.core.io.ClassPathResource; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.data.redis.core.script.RedisScript; +import org.springframework.stereotype.Component; + +import java.io.BufferedReader; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * 正在执行作业配额限制管理 + */ +@Component +@Slf4j +public class RunningJobResourceQuotaManager { + + private final RedisTemplate redisTemplate; + private final RunningJobResourceQuotaStore runningJobResourceQuotaStore; + + private final MeterRegistry meterRegistry; + + private static final String RESOURCE_SCOPE_RUNNING_JOB_COUNT_HASH_KEY = + "job:execute:running:job:count:resource_scope"; + private static final String APP_RUNNING_JOB_COUNT_HASH_KEY = "job:execute:running:job:count:app"; + + private static final List LUA_SCRIPT_KEYS = new ArrayList<>(); + + private static String CHECK_QUOTA_LUA_SCRIPT; + + private static String ADD_JOB_LUA_SCRIPT; + private static String REMOVE_JOB_LUA_SCRIPT; + + private static final String METRIC_RUNNING_JOB_RESOURCE_QUOTA_LIMIT_EXCEED_TOTAL = + "job_running_job_resource_quota_limit_exceed_total"; + + private static final long JOB_EXPIRE_TIME = 3600 * 1000L; + + static { + LUA_SCRIPT_KEYS.add(RedisKeys.RUNNING_JOB_ZSET_KEY); + LUA_SCRIPT_KEYS.add(RESOURCE_SCOPE_RUNNING_JOB_COUNT_HASH_KEY); + LUA_SCRIPT_KEYS.add(APP_RUNNING_JOB_COUNT_HASH_KEY); + + loadLuaScript(); + } + + private static void loadLuaScript() { + try { + log.info("Load running job resource quota lua script start"); + + CHECK_QUOTA_LUA_SCRIPT = readContentFromClasspathFile("lua/check_running_job_quota_limit.lua"); + ADD_JOB_LUA_SCRIPT = readContentFromClasspathFile("lua/add_running_job.lua"); + REMOVE_JOB_LUA_SCRIPT = readContentFromClasspathFile("lua/remove_running_job.lua"); + + log.info("Load running job resource quota lua script successfully!"); + } catch (Throwable e) { + throw new JobMicroServiceBootException("Load running job resource quota lua script error", e); + } + } + + private static String readContentFromClasspathFile(String path) throws Exception { + try (InputStream inputStream = + new ClassPathResource(path).getInputStream(); + BufferedReader reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8))) { + return reader.lines().collect(Collectors.joining("\n")); + } + } + + + public RunningJobResourceQuotaManager(StringRedisTemplate redisTemplate, + RunningJobResourceQuotaStore runningJobResourceQuotaStore, + MeterRegistry meterRegistry) { + this.redisTemplate = redisTemplate; + this.runningJobResourceQuotaStore = runningJobResourceQuotaStore; + this.meterRegistry = meterRegistry; + } + + public void addJob(String appCode, ResourceScope resourceScope, long jobInstanceId) { + long startTime = System.currentTimeMillis(); + RedisScript script = RedisScript.of(ADD_JOB_LUA_SCRIPT, Void.class); + + redisTemplate.execute( + script, + LUA_SCRIPT_KEYS, + String.valueOf(jobInstanceId), + resourceScope.toResourceScopeUniqueId(), + convertAppCode(appCode), + String.valueOf(System.currentTimeMillis()) + ); + + long cost = System.currentTimeMillis() - startTime; + if (log.isDebugEnabled()) { + log.debug("RunningJobResourceQuotaManager - Add job cost : {} ms", cost); + } + } + + private String convertAppCode(String appCode) { + return StringUtils.isNotBlank(appCode) ? appCode : "None"; + } + + public void removeJob(String appCode, ResourceScope resourceScope, long jobInstanceId) { + long startTime = System.currentTimeMillis(); + RedisScript script = RedisScript.of(REMOVE_JOB_LUA_SCRIPT, Void.class); + + redisTemplate.execute( + script, + LUA_SCRIPT_KEYS, + String.valueOf(jobInstanceId), + resourceScope.toResourceScopeUniqueId(), + convertAppCode(appCode) + ); + + long cost = System.currentTimeMillis() - startTime; + if (log.isDebugEnabled()) { + log.debug("RunningJobResourceQuotaManager - Remove job cost : {} ms", cost); + } + } + + /** + * 业务是否超过正在执行的任务数量配额 + * + * @param appCode 作业发起的蓝鲸应用 code + * @param resourceScope 资源管理空间 + */ + public ResourceQuotaCheckResultEnum checkResourceQuotaLimit(String appCode, + ResourceScope resourceScope) { + if (!runningJobResourceQuotaStore.isQuotaLimitEnabled()) { + return ResourceQuotaCheckResultEnum.NO_LIMIT; + } + long startTime = System.currentTimeMillis(); + RedisScript script = RedisScript.of(CHECK_QUOTA_LUA_SCRIPT, String.class); + + long systemLimit = runningJobResourceQuotaStore.getSystemQuotaLimit(); + long resourceScopeLimit = runningJobResourceQuotaStore.getQuotaLimitByResourceScope(resourceScope); + // 是否通过第三方应用 Open API 方式调用作业平台产生的作业 + boolean isJobFrom3rdApp = StringUtils.isNotEmpty(appCode); + long appLimit = isJobFrom3rdApp ? runningJobResourceQuotaStore.getQuotaLimitByAppCode(appCode) : + Long.MAX_VALUE; + + String checkResourceQuotaResult = redisTemplate.execute( + script, + LUA_SCRIPT_KEYS, + resourceScope.toResourceScopeUniqueId(), + convertAppCode(appCode), + String.valueOf(systemLimit), + String.valueOf(resourceScopeLimit), + String.valueOf(appLimit) + ); + + long cost = System.currentTimeMillis() - startTime; + if (log.isDebugEnabled()) { + log.debug("CheckRunningJobResourceQuotaLimit cost: {} ms", cost); + } + ResourceQuotaCheckResultEnum checkResult = ResourceQuotaCheckResultEnum.valOf(checkResourceQuotaResult); + if (checkResult.isExceedLimit()) { + recordExceedQuotaLimitRecord(appCode, resourceScope); + } + + return checkResult; + } + + private void recordExceedQuotaLimitRecord(String appCode, ResourceScope resourceScope) { + meterRegistry.counter( + METRIC_RUNNING_JOB_RESOURCE_QUOTA_LIMIT_EXCEED_TOTAL, + Tags.of(CommonMetricTags.KEY_RESOURCE_SCOPE, resourceScope.toResourceScopeUniqueId()) + .and(CommonMetricTags.KEY_APP_CODE, StringUtils.isNotBlank(appCode) ? + appCode : CommonMetricValues.NONE)) + .increment(); + } + + public long getRunningJobTotal() { + Long count = redisTemplate.opsForZSet().size(RedisKeys.RUNNING_JOB_ZSET_KEY); + return count != null ? count : 0L; + } + + public Map getAppRunningJobCount() { + Map countMap = + redisTemplate.opsForHash().entries(APP_RUNNING_JOB_COUNT_HASH_KEY); + if (countMap.isEmpty()) { + return null; + } + return filterEmptyCountAndConvert(countMap); + } + + public Map getResourceScopeRunningJobCount() { + Map countMap = + redisTemplate.opsForHash().entries(RESOURCE_SCOPE_RUNNING_JOB_COUNT_HASH_KEY); + if (countMap.isEmpty()) { + return null; + } + return filterEmptyCountAndConvert(countMap); + } + + private Map filterEmptyCountAndConvert(Map map) { + Map finalMap = new HashMap<>(); + map.forEach((k, v) -> { + long count = Long.parseLong(v); + if (count == 0) { + return; + } + finalMap.put(k, count); + }); + return finalMap; + } + + public Set getNotAliveJobInstanceIds() { + try { + // 1 小时过期 + long EXPIRE_AT = System.currentTimeMillis() - JOB_EXPIRE_TIME; + Set notAliveJobInstanceIds = redisTemplate.opsForZSet() + .rangeByScore( + RedisKeys.RUNNING_JOB_ZSET_KEY, + -1, + EXPIRE_AT + ); + if (CollectionUtils.isEmpty(notAliveJobInstanceIds)) { + return Collections.emptySet(); + } + return notAliveJobInstanceIds.stream().map(Long::parseLong).collect(Collectors.toSet()); + } catch (Throwable e) { + return Collections.emptySet(); + } + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/quota/limit/package-info.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/quota/limit/package-info.java new file mode 100644 index 0000000000..d538776acf --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/quota/limit/package-info.java @@ -0,0 +1,29 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.quota.limit; + +/* + * 作业配额限制，防止单业务占用所有执行引擎的调度资源 + */ diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/AbstractResultHandleTask.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/AbstractResultHandleTask.java index 6a0c357b5c..b885970a79 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/AbstractResultHandleTask.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/AbstractResultHandleTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,30 +25,33 @@ package com.tencent.bk.job.execute.engine.result; import com.tencent.bk.job.common.constant.JobConstants; -import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.common.gse.GseClient; +import com.tencent.bk.job.common.gse.v2.model.ExecuteObjectGseKey; import com.tencent.bk.job.common.redis.util.LockUtils; import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; -import com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum; +import com.tencent.bk.job.execute.engine.EngineDependentServiceHolder; +import com.tencent.bk.job.execute.engine.consts.ExecuteObjectTaskStatusEnum; import com.tencent.bk.job.execute.engine.evict.TaskEvictPolicyExecutor; import com.tencent.bk.job.execute.engine.listener.event.EventSource; import com.tencent.bk.job.execute.engine.listener.event.GseTaskEvent; import com.tencent.bk.job.execute.engine.listener.event.ResultHandleTaskResumeEvent; import com.tencent.bk.job.execute.engine.listener.event.StepEvent; import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; -import com.tencent.bk.job.execute.engine.model.GseLog; import com.tencent.bk.job.execute.engine.model.GseLogBatchPullResult; import com.tencent.bk.job.execute.engine.model.GseTaskExecuteResult; +import com.tencent.bk.job.execute.engine.model.GseTaskResult; import com.tencent.bk.job.execute.engine.model.TaskVariableDTO; import com.tencent.bk.job.execute.engine.model.TaskVariablesAnalyzeResult; +import com.tencent.bk.job.execute.engine.quota.limit.RunningJobKeepaliveManager; import com.tencent.bk.job.execute.engine.result.ha.ResultHandleTaskKeepaliveManager; -import com.tencent.bk.job.execute.model.AgentTaskDTO; +import com.tencent.bk.job.execute.model.ExecuteObjectTask; import com.tencent.bk.job.execute.model.GseTaskDTO; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; -import com.tencent.bk.job.execute.service.AgentTaskService; +import com.tencent.bk.job.execute.service.ExecuteObjectTaskService; import com.tencent.bk.job.execute.service.GseTaskService; import com.tencent.bk.job.execute.service.LogService; import com.tencent.bk.job.execute.service.StepInstanceService; @@ -78,9 +81,9 @@ @Slf4j public abstract class AbstractResultHandleTask implements ContinuousScheduledTask { /** - * GSE任务执行结果为空,Job最大容忍时间,5min.用于异常情况下的任务自动终止，防止长时间占用系统资源 + * GSE任务执行结果为空,Job最大容忍时间1min.用于异常情况下的任务自动终止，防止长时间占用系统资源 */ - private static final int GSE_TASK_EMPTY_RESULT_MAX_TOLERATION_MILLS = 300_000; + private static final int GSE_TASK_EMPTY_RESULT_MAX_TOLERATION_MILLS = 60_000; /** * GSE任务超时未结束,Job最大容忍时间。5min.用于异常情况下的任务自动终止，防止长时间占用系统资源 */ @@ -98,8 +101,9 @@ public abstract class AbstractResultHandleTask implements ContinuousScheduled protected TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher; protected ResultHandleTaskKeepaliveManager resultHandleTaskKeepaliveManager; protected TaskEvictPolicyExecutor taskEvictPolicyExecutor; - protected AgentTaskService agentTaskService; + protected ExecuteObjectTaskService executeObjectTaskService; protected StepInstanceService stepInstanceService; + protected GseClient gseClient; /** * 任务请求的requestId，用于防止重复下发任务 */ @@ -125,13 +129,17 @@ public abstract class AbstractResultHandleTask implements ContinuousScheduled */ protected long appId; /** - * GSE 任务执行结果 + * GSE 任务执行 */ protected GseTaskDTO gseTask; /** - * GSE 主机任务执行结果，Map + * 执行对象任务列表(全量) */ - protected Map targetAgentTasks; + protected List executeObjectTasks; + /** + * GSE任务与JOB执行对象任务的映射关系 + */ + protected Map targetExecuteObjectTasks; /** * 全局参数分析结果 */ @@ -143,29 +151,19 @@ public abstract class AbstractResultHandleTask implements ContinuousScheduled /** * 任务包含的所有目标服务器 */ - protected Set targetAgentIds = new HashSet<>(); + protected Set targetExecuteObjectGseKeys = new HashSet<>(); /** - * 未开始任务的目标服务器 + * 未结束的目标服务器 */ - protected Set notStartedTargetAgentIds = new HashSet<>(); - /** - * 正在执行任务的目标服务器 - */ - protected Set runningTargetAgentIds = new HashSet<>(); - - // ---------------- analysed task execution result for server -------------------- + protected Set notFinishedTargetExecuteObjectGseKeys = new HashSet<>(); /** * 已经分析结果完成的目标服务器 */ - protected Set analyseFinishedTargetAgentIds = new HashSet<>(); + protected Set analyseFinishedTargetExecuteObjectGseKeys = new HashSet<>(); /** * 执行成功的目标服务器 */ - protected Set successTargetAgentIds = new HashSet<>(); - /** - * Agent ID 与 host 映射关系 - */ - protected Map agentIdHostMap; + protected Set successTargetExecuteObjectGseKeys = new HashSet<>(); /** * 任务成功被终止 */ @@ -178,7 +176,7 @@ public abstract class AbstractResultHandleTask implements ContinuousScheduled * 任务是否已停止 */ protected volatile boolean isStopped = false; - // ---------------- analysed task execution result for server -------------------- + /** * 任务是否启用 */ @@ -188,7 +186,7 @@ public abstract class AbstractResultHandleTask implements ContinuousScheduled */ private final AtomicInteger pullLogTimes = new AtomicInteger(0); - // ---------------- task lifecycle properties -------------------- + /** * 最近一次成功拉取GSE执行结果的时间 */ @@ -201,51 +199,63 @@ public abstract class AbstractResultHandleTask implements ContinuousScheduled * GSE任务是否处于终止状态 */ private volatile boolean isGseTaskTerminating = false; - // ---------------- task lifecycle properties -------------------- - - - protected AbstractResultHandleTask(TaskInstanceService taskInstanceService, - GseTaskService gseTaskService, - LogService logService, - TaskInstanceVariableService taskInstanceVariableService, - StepInstanceVariableValueService stepInstanceVariableValueService, - TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher, - ResultHandleTaskKeepaliveManager resultHandleTaskKeepaliveManager, - TaskEvictPolicyExecutor taskEvictPolicyExecutor, - AgentTaskService agentTaskService, - StepInstanceService stepInstanceService, + /** + * 是否是GSE V2 TASK + */ + protected boolean gseV2Task; + /** + * 是否存在不可执行的目标执行对象 + */ + protected boolean existNoExecutableTargetExecuteObject; + /** + * GSE 任务信息，用于日志输出 + */ + protected String gseTaskInfo; + + protected final RunningJobKeepaliveManager runningJobKeepaliveManager; + + private final TaskContext taskContext; + + protected AbstractResultHandleTask(EngineDependentServiceHolder engineDependentServiceHolder, + ExecuteObjectTaskService executeObjectTaskService, TaskInstanceDTO taskInstance, StepInstanceDTO stepInstance, TaskVariablesAnalyzeResult taskVariablesAnalyzeResult, - Map targetAgentTasks, + Map targetExecuteObjectTasks, GseTaskDTO gseTask, - String requestId) { - this.taskInstanceService = taskInstanceService; - this.gseTaskService = gseTaskService; - this.logService = logService; - this.taskInstanceVariableService = taskInstanceVariableService; - this.stepInstanceVariableValueService = stepInstanceVariableValueService; - this.taskExecuteMQEventDispatcher = taskExecuteMQEventDispatcher; - this.resultHandleTaskKeepaliveManager = resultHandleTaskKeepaliveManager; - this.taskEvictPolicyExecutor = taskEvictPolicyExecutor; - this.agentTaskService = agentTaskService; - this.stepInstanceService = stepInstanceService; + String requestId, + List executeObjectTasks) { + this.taskInstanceService = engineDependentServiceHolder.getTaskInstanceService(); + this.gseTaskService = engineDependentServiceHolder.getGseTaskService(); + this.logService = engineDependentServiceHolder.getLogService(); + this.taskInstanceVariableService = engineDependentServiceHolder.getTaskInstanceVariableService(); + this.stepInstanceVariableValueService = engineDependentServiceHolder.getStepInstanceVariableValueService(); + this.taskExecuteMQEventDispatcher = engineDependentServiceHolder.getTaskExecuteMQEventDispatcher(); + this.resultHandleTaskKeepaliveManager = engineDependentServiceHolder.getResultHandleTaskKeepaliveManager(); + this.taskEvictPolicyExecutor = engineDependentServiceHolder.getTaskEvictPolicyExecutor(); + this.stepInstanceService = engineDependentServiceHolder.getStepInstanceService(); + this.gseClient = engineDependentServiceHolder.getGseClient(); + this.runningJobKeepaliveManager = engineDependentServiceHolder.getRunningJobKeepaliveManager(); + + this.executeObjectTaskService = executeObjectTaskService; + this.requestId = requestId; this.taskInstance = taskInstance; this.taskInstanceId = taskInstance.getId(); this.stepInstance = stepInstance; + this.gseV2Task = stepInstance.isTargetGseV2Agent(); this.appId = stepInstance.getAppId(); this.stepInstanceId = stepInstance.getId(); this.taskVariablesAnalyzeResult = taskVariablesAnalyzeResult; - this.targetAgentTasks = targetAgentTasks; + this.targetExecuteObjectTasks = targetExecuteObjectTasks; this.gseTask = gseTask; + this.executeObjectTasks = executeObjectTasks; + this.gseTaskInfo = buildGseTaskInfo(stepInstance.getTaskInstanceId(), gseTask); + this.taskContext = new TaskContext(taskInstanceId); - targetAgentTasks.values().forEach(agentTask -> { - this.targetAgentIds.add(agentTask.getAgentId()); - }); - this.notStartedTargetAgentIds.addAll(targetAgentIds); - - this.agentIdHostMap = stepInstanceService.computeStepHosts(stepInstance, HostDTO::toCloudIp); + targetExecuteObjectTasks.values().forEach(executeObjectTask -> + this.targetExecuteObjectGseKeys.add(executeObjectTask.getExecuteObject().toExecuteObjectGseKey())); + this.notFinishedTargetExecuteObjectGseKeys.addAll(targetExecuteObjectGseKeys); // 如果是执行方案，需要初始化全局变量 if (taskInstance.isPlanInstance()) { @@ -254,6 +264,15 @@ protected AbstractResultHandleTask(TaskInstanceService taskInstanceService, taskVariables.forEach(var -> initialVariables.put(var.getName(), var)); } } + + this.existNoExecutableTargetExecuteObject = + executeObjectTasks.stream(). + filter(ExecuteObjectTask::isTarget) + .anyMatch(executeObjectTask -> !executeObjectTask.getExecuteObject().isExecutable()); + } + + private String buildGseTaskInfo(Long jobInstanceId, GseTaskDTO gseTask) { + return "Job:" + jobInstanceId + "-" + gseTask.getTaskUniqueName(); } /** @@ -266,10 +285,10 @@ private boolean checkAndEvictTaskIfNeed(StopWatch watch) { watch.start("check-evict-task"); if (taskEvictPolicyExecutor.shouldEvictTask(taskInstance)) { log.info("taskInstance {} evicted", taskInstance.getId()); - // 更新任务与步骤状态 - taskEvictPolicyExecutor.updateEvictedTaskStatus(taskInstance, stepInstance); // 停止日志拉取调度 this.executeResult = GseTaskExecuteResult.DISCARDED; + finishGseTask(this.executeResult); + watch.stop(); return true; } @@ -278,40 +297,40 @@ private boolean checkAndEvictTaskIfNeed(StopWatch watch) { } /** - * 拉取GSE日志 + * 拉取GSE任务结果并分析 * * @param watch 外部传入的耗时统计watch对象 * @return 是否应当继续后续流程 */ - private boolean pullGSELogsAndCheckPullResult(StopWatch watch) { - log.info("[{}]: Start pull gse task result, times: {}", stepInstanceId, pullLogTimes.addAndGet(1)); + private boolean pullGSEResultAndAnalyse(StopWatch watch) { + log.info("[{}]: Start pull gse task result, times: {}", gseTaskInfo, + pullLogTimes.addAndGet(1)); GseLogBatchPullResult gseLogBatchPullResult; int batch = 0; do { batch++; - if (checkAndEvictTaskIfNeed(watch)) return false; + if (checkAndEvictTaskIfNeed(watch)) { + return false; + } watch.start("pull-task-result-batch-" + batch); // 分批拉取GSE任务执行结果 gseLogBatchPullResult = pullGseTaskResultInBatches(); - // 拉取结果校验 - if (!checkPullResult(gseLogBatchPullResult)) { + // 检查任务异常并处理 + GseTaskResult gseTaskResult = gseLogBatchPullResult.getGseTaskResult(); + if (determineTaskAbnormal(gseTaskResult)) { return false; } - // 检查任务异常并处理 - GseLog gseLog = gseLogBatchPullResult.getGseLog(); - if (determineTaskAbnormal(gseLog)) return false; - watch.stop(); try { watch.start("analyse-task-result-batch-" + batch); - this.executeResult = analyseGseTaskResult(gseLog); + this.executeResult = analyseGseTaskResult(gseTaskResult); watch.stop(); } catch (Throwable e) { - log.error("[" + stepInstanceId + "]: analyse gse task result error.", e); + log.error("[" + gseTaskInfo + "]: analyse gse task result error.", e); throw e; } } while (!gseLogBatchPullResult.isLastBatch()); @@ -340,7 +359,7 @@ public void execute() { watch.start("check-skip-or-stop"); if (shouldSkipStep()) { this.executeResult = GseTaskExecuteResult.SKIPPED; - log.info("[{}]: Skip task, set unfinished ip task status to unknown!", stepInstanceId); + log.info("[{}]: Skip task, set unfinished ip task status to unknown!", gseTaskInfo); saveStatusWhenSkip(); return; } @@ -348,17 +367,20 @@ public void execute() { watch.stop(); // 拉取执行结果日志 - if (!pullGSELogsAndCheckPullResult(watch)) { + if (!pullGSEResultAndAnalyse(watch)) { return; } - watch.start("handle-execute-result"); - handleExecuteResult(this.executeResult); - watch.stop(); + // 如果任务已结束 + if (this.executeResult != GseTaskExecuteResult.RUNNING) { + watch.start("finish-gse-task"); + finishGseTask(this.executeResult); + watch.stop(); + } } catch (Throwable e) { - log.error("[" + stepInstanceId + "]: result handle error.", e); + log.error("[" + gseTaskInfo + "]: result handle error.", e); this.executeResult = GseTaskExecuteResult.EXCEPTION; - handleExecuteResult(this.executeResult); + finishGseTask(this.executeResult); } finally { this.isRunning = false; LockUtils.releaseDistributedLock(lockKey, requestId); @@ -378,13 +400,13 @@ private String buildGseTaskLockKey(GseTaskDTO gseTask) { private boolean checkTaskActiveAndSetRunningStatus() { if (!isActive) { - log.info("Task is inactive, stepInstanceId: {}", stepInstanceId); + log.info("Task is inactive, task: {}", gseTaskInfo); return false; } this.isRunning = true; // 二次确认，防止isActive在设置this.isRunning=true期间发生变化 if (!isActive) { - log.info("Task is inactive, stepInstanceId: {}", stepInstanceId); + log.info("Task is inactive, task: {}", gseTaskInfo); return false; } return true; @@ -396,10 +418,10 @@ private void terminateGseTaskIfDetectTaskStatusIsStopping() { // 如果任务处于“终止中”状态，触发任务终止 if (this.taskInstance.getStatus() == RunStatusEnum.STOPPING) { log.info("Task instance status is stopping, stop executing the step! taskInstanceId:{}, " + - "stepInstanceId:{}", - taskInstance.getId(), stepInstance.getId()); + "task:{}", taskInstance.getId(), gseTaskInfo); taskExecuteMQEventDispatcher.dispatchGseTaskEvent(GseTaskEvent.stopGseTask( - gseTask.getStepInstanceId(), gseTask.getExecuteCount(), gseTask.getBatch(), gseTask.getId())); + taskInstanceId, gseTask.getStepInstanceId(), gseTask.getExecuteCount(), + gseTask.getBatch(), gseTask.getId())); this.isGseTaskTerminating = true; log.info("Send stop gse step control action successfully!"); } @@ -407,28 +429,29 @@ private void terminateGseTaskIfDetectTaskStatusIsStopping() { } private boolean shouldSkipStep() { - StepInstanceBaseDTO stepInstance = taskInstanceService.getBaseStepInstance(stepInstanceId); + StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstance(taskInstanceId, stepInstanceId); return stepInstance == null || RunStatusEnum.SKIPPED == stepInstance.getStatus(); } private void saveStatusWhenSkip() { - List notFinishedGseAgentTasks = - targetAgentTasks.values().stream().filter(not(AgentTaskDTO::isFinished)).collect(Collectors.toList()); - if (CollectionUtils.isNotEmpty(notFinishedGseAgentTasks)) { - notFinishedGseAgentTasks.forEach(agentTask -> { - agentTask.setStatus(AgentTaskStatusEnum.UNKNOWN); - agentTask.setEndTime(System.currentTimeMillis()); - + List notFinishedExecuteObjectTasks = + targetExecuteObjectTasks.values().stream() + .filter(not(ExecuteObjectTask::isFinished)) + .collect(Collectors.toList()); + if (CollectionUtils.isNotEmpty(notFinishedExecuteObjectTasks)) { + notFinishedExecuteObjectTasks.forEach(executeObjectTask -> { + executeObjectTask.setStatus(ExecuteObjectTaskStatusEnum.UNKNOWN); + executeObjectTask.setEndTime(System.currentTimeMillis()); }); } - agentTaskService.batchUpdateAgentTasks(notFinishedGseAgentTasks); + executeObjectTaskService.batchUpdateTasks(notFinishedExecuteObjectTasks); } /* * 检查任务是否异常，执行结果持续为空、超时未结束等 */ - private boolean determineTaskAbnormal(GseLog gseLog) { - return checkTaskTimeout() || checkEmptyGseResult(gseLog); + private boolean determineTaskAbnormal(GseTaskResult gseTaskResult) { + return checkTaskTimeout() || checkEmptyGseResult(gseTaskResult); } private boolean checkTaskTimeout() { @@ -439,12 +462,12 @@ private boolean checkTaskTimeout() { long timeout = stepInstance.getTimeout() == null ? JobConstants.DEFAULT_JOB_TIMEOUT_SECONDS : stepInstance.getTimeout(); if (runDuration - 1000 * timeout >= GSE_TASK_TIMEOUT_MAX_TOLERATION_MILLS) { - log.warn("[{}]: Task execution timeout! runDuration: {}ms, timeout: {}s", stepInstanceId, runDuration, - stepInstance.getTimeout()); + log.warn("[{}]: Task execution timeout! runDuration: {}ms, timeout: {}s", gseTaskInfo, + runDuration, stepInstance.getTimeout()); this.executeResult = GseTaskExecuteResult.FAILED; - saveFailInfoForUnfinishedAgentTask(AgentTaskStatusEnum.LOG_ERROR, + saveFailInfoForUnfinishedExecuteObjectTask(ExecuteObjectTaskStatusEnum.LOG_ERROR, "Task execution may be abnormal or timeout."); - handleExecuteResult(GseTaskExecuteResult.FAILED); + finishGseTask(GseTaskExecuteResult.FAILED); isTimeout = true; } return isTimeout; @@ -453,19 +476,21 @@ private boolean checkTaskTimeout() { /* * 检查从GSE 拉取的任务执行结果是否持续为空 */ - private boolean checkEmptyGseResult(GseLog gseLog) { + private boolean checkEmptyGseResult(GseTaskResult gseTaskResult) { if (latestPullGseLogSuccessTimeMillis == 0) { latestPullGseLogSuccessTimeMillis = System.currentTimeMillis(); } boolean isAbnormal = false; - if (null == gseLog || gseLog.isNullResp()) { + if (null == gseTaskResult || gseTaskResult.isEmptyResult()) { long currentTimeMillis = System.currentTimeMillis(); - // 执行结果持续为空 + // 执行结果持续为空,并且超出 Job 最大容忍时间，需要终止调度任务 if (currentTimeMillis - latestPullGseLogSuccessTimeMillis >= GSE_TASK_EMPTY_RESULT_MAX_TOLERATION_MILLS) { - log.warn("[{}]: Execution result log always empty!", stepInstanceId); + log.warn("[{}]: Execution result log always empty!", gseTaskInfo); this.executeResult = GseTaskExecuteResult.FAILED; - saveFailInfoForUnfinishedAgentTask(AgentTaskStatusEnum.LOG_ERROR, "Execution result log always empty."); - handleExecuteResult(GseTaskExecuteResult.FAILED); + saveFailInfoForUnfinishedExecuteObjectTask(ExecuteObjectTaskStatusEnum.LOG_ERROR, "Execution result " + + "log " + + "always empty."); + finishGseTask(GseTaskExecuteResult.FAILED); isAbnormal = true; } } else { @@ -474,115 +499,120 @@ private boolean checkEmptyGseResult(GseLog gseLog) { return isAbnormal; } - private boolean checkPullResult(GseLogBatchPullResult gseLogBatchPullResult) { - if (!gseLogBatchPullResult.isSuccess()) { - log.error("[{}] Pull gse task result error, errorMsg: {}", stepInstanceId, - gseLogBatchPullResult.getErrorMsg()); - this.executeResult = GseTaskExecuteResult.FAILED; - saveFailInfoForUnfinishedAgentTask(AgentTaskStatusEnum.LOG_ERROR, gseLogBatchPullResult.getErrorMsg()); - handleExecuteResult(GseTaskExecuteResult.FAILED); - return false; - } - return true; - } - - /** * 设置目标gent任务结束状态 * - * @param agentId agentId - * @param startTime 起始时间 - * @param endTime 终止时间 - * @param agentTask 日志 - */ - protected void dealTargetAgentFinish(String agentId, Long startTime, Long endTime, AgentTaskDTO agentTask) { - log.info("[{}]: Deal target agent finished| agentId={}| startTime:{}, endTime:{}, agentTask:{}", - stepInstanceId, agentId, startTime, endTime, JsonUtils.toJsonWithoutSkippedFields(agentTask)); - - notStartedTargetAgentIds.remove(agentId); - runningTargetAgentIds.remove(agentId); - analyseFinishedTargetAgentIds.add(agentId); + * @param executeObjectGseKey executeObjectGseKey + * @param startTime 起始时间 + * @param endTime 终止时间 + * @param executeObjectTask 执行对象任务 + */ + protected void dealTargetExecuteObjectFinish(ExecuteObjectGseKey executeObjectGseKey, + Long startTime, + Long endTime, + ExecuteObjectTask executeObjectTask) { + log.info("[{}]: Deal target agent finished| executeObjectGseKey={}| startTime:{}, endTime:{}, " + + "executeObjectTask:{}", + gseTaskInfo, executeObjectGseKey, startTime, endTime, + JsonUtils.toJsonWithoutSkippedFields(executeObjectTask)); + + notFinishedTargetExecuteObjectGseKeys.remove(executeObjectGseKey); + analyseFinishedTargetExecuteObjectGseKeys.add(executeObjectGseKey); if (endTime - startTime <= 0) { - agentTask.setTotalTime(100L); + executeObjectTask.setTotalTime(100L); } else { - agentTask.setTotalTime(endTime - startTime); + executeObjectTask.setTotalTime(endTime - startTime); } - agentTask.setStartTime(startTime); - agentTask.setEndTime(endTime); + executeObjectTask.setStartTime(startTime); + executeObjectTask.setEndTime(endTime); } /** - * 处理任务结果 + * 设置GSE TASK 完成状态并分发事件 * * @param result 任务执行结果 */ - private void handleExecuteResult(GseTaskExecuteResult result) { + private void finishGseTask(GseTaskExecuteResult result) { int gseTaskExecuteResult = result.getResultCode(); - // 如果任务正在执行中 - if (gseTaskExecuteResult == GseTaskExecuteResult.RESULT_CODE_RUNNING) { - return; - } // 处理GSE任务执行结果 - log.info("Handle execute result, stepInstanceId:{}, executeResult:{}", stepInstanceId, gseTaskExecuteResult); + log.info("Finish gse task, task:{}, executeResult:{}", gseTaskInfo, gseTaskExecuteResult); long startTime = this.gseTask.getStartTime(); long endTime = DateUtils.currentTimeMillis(); long gseTotalTime = endTime - startTime; - int targetAgentNum = this.targetAgentIds.size(); - int allSuccessAgentNum = this.successTargetAgentIds.size(); - boolean isSuccess = !stepInstance.hasInvalidHost() && allSuccessAgentNum == targetAgentNum; - - updateGseTaskExecutionInfo(result, isSuccess, endTime, gseTotalTime); - taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.refreshStep(stepInstanceId, - EventSource.buildGseTaskEventSource(stepInstanceId, stepInstance.getExecuteCount(), - stepInstance.getBatch(), gseTask.getId()))); + updateGseTaskExecutionInfo(result, endTime, gseTotalTime); + + taskExecuteMQEventDispatcher.dispatchStepEvent( + StepEvent.refreshStep( + taskInstanceId, + stepInstanceId, + EventSource.buildGseTaskEventSource( + taskInstanceId, + stepInstanceId, + stepInstance.getExecuteCount(), + stepInstance.getBatch(), + gseTask.getId()) + ) + ); } - private void updateGseTaskExecutionInfo(GseTaskExecuteResult result, boolean isSuccess, long endTime, - long totalTime) { - if (GseTaskExecuteResult.RESULT_CODE_STOP_SUCCESS == result.getResultCode()) { - gseTask.setStatus(RunStatusEnum.STOP_SUCCESS.getValue()); - } else { - gseTask.setStatus(isSuccess ? RunStatusEnum.SUCCESS.getValue() : RunStatusEnum.FAIL.getValue()); - } + private void updateGseTaskExecutionInfo(GseTaskExecuteResult result, long endTime, long totalTime) { + gseTask.setStatus(analyseGseTaskStatus(result).getValue()); gseTask.setEndTime(endTime); gseTask.setTotalTime(totalTime); gseTaskService.updateGseTask(gseTask); } + private RunStatusEnum analyseGseTaskStatus(GseTaskExecuteResult result) { + if (result.equals(GseTaskExecuteResult.RUNNING)) { + return RunStatusEnum.RUNNING; + } else if (result.equals(GseTaskExecuteResult.SUCCESS)) { + return RunStatusEnum.SUCCESS; + } else if (result.equals(GseTaskExecuteResult.FAILED)) { + return RunStatusEnum.FAIL; + } else if (result.equals(GseTaskExecuteResult.STOP_SUCCESS)) { + return RunStatusEnum.STOP_SUCCESS; + } else if (result.equals(GseTaskExecuteResult.DISCARDED)) { + return RunStatusEnum.ABANDONED; + } else if (result.equals(GseTaskExecuteResult.EXCEPTION)) { + return RunStatusEnum.ABNORMAL_STATE; + } else if (result.equals(GseTaskExecuteResult.SKIPPED)) { + return RunStatusEnum.SKIPPED; + } else { + return RunStatusEnum.FAIL; + } + } + /** - * 批量更新AgentTask并重置changed标志 + * 批量更新执行对象任务并重置changed标志 * - * @param agentTasks agent任务列表 - */ - protected void batchSaveChangedGseAgentTasks(Collection agentTasks) { - if (CollectionUtils.isNotEmpty(agentTasks)) { - List changedGseAgentTasks = - agentTasks.stream().filter(AgentTaskDTO::isChanged).collect(Collectors.toList()); - agentTaskService.batchUpdateAgentTasks(changedGseAgentTasks); - changedGseAgentTasks.forEach(agentTask -> agentTask.setChanged(false)); + * @param executeObjectTasks 执行对象任务列表 + */ + protected void batchSaveChangedExecuteObjectTasks(Collection executeObjectTasks) { + if (CollectionUtils.isNotEmpty(executeObjectTasks)) { + List changedTasks = + executeObjectTasks.stream().filter(ExecuteObjectTask::isChanged).collect(Collectors.toList()); + executeObjectTaskService.batchUpdateTasks(changedTasks); + changedTasks.forEach(executeObjectTask -> executeObjectTask.setChanged(false)); } } - protected void saveFailInfoForUnfinishedAgentTask(AgentTaskStatusEnum status, String errorMsg) { - log.info("[{}]: Deal unfinished agent result| noStartJobAgentIds : {}| runningJobAgentIds : {}", - stepInstanceId, notStartedTargetAgentIds, runningTargetAgentIds); - Set unfinishedAgentIds = new HashSet<>(); - unfinishedAgentIds.addAll(notStartedTargetAgentIds); - unfinishedAgentIds.addAll(runningTargetAgentIds); + protected void saveFailInfoForUnfinishedExecuteObjectTask(ExecuteObjectTaskStatusEnum status, String errorMsg) { + log.info("[{}]: Deal unfinished agent result| notFinishedTargetAgentIds : {}", + gseTaskInfo, notFinishedTargetExecuteObjectGseKeys); long startTime = (gseTask != null && gseTask.getStartTime() != null) ? gseTask.getStartTime() : System.currentTimeMillis(); - for (String agentId : unfinishedAgentIds) { - AgentTaskDTO agentTask = targetAgentTasks.get(agentId); - agentTask.setStartTime(startTime); - agentTask.setEndTime(System.currentTimeMillis()); - agentTask.setStatus(status); + for (ExecuteObjectGseKey executeObjectGseKey : notFinishedTargetExecuteObjectGseKeys) { + ExecuteObjectTask executeObjectTask = targetExecuteObjectTasks.get(executeObjectGseKey); + executeObjectTask.setStartTime(startTime); + executeObjectTask.setEndTime(System.currentTimeMillis()); + executeObjectTask.setStatus(status); } - batchSaveChangedGseAgentTasks(targetAgentTasks.values()); + batchSaveChangedExecuteObjectTasks(targetExecuteObjectTasks.values()); } @Override @@ -592,22 +622,24 @@ public void stop() { this.isActive = false; tryStopImmediately(); } else { - log.info("Task is stopped, stepInstanceId: {}", stepInstanceId); + log.info("Task is stopped, task: {}", gseTaskInfo); } } } private void tryStopImmediately() { if (!this.isRunning) { - log.info("ResultHandleTask-onStop start, stepInstanceId: {}", stepInstanceId); + log.info("ResultHandleTask-onStop start, task: {}", gseTaskInfo); resultHandleTaskKeepaliveManager.stopKeepaliveInfoTask(getTaskId()); + runningJobKeepaliveManager.stopKeepaliveTask(taskInstanceId); + taskExecuteMQEventDispatcher.dispatchResultHandleTaskResumeEvent( - ResultHandleTaskResumeEvent.resume(gseTask.getStepInstanceId(), + ResultHandleTaskResumeEvent.resume(stepInstance.getTaskInstanceId(), gseTask.getStepInstanceId(), gseTask.getExecuteCount(), gseTask.getBatch(), gseTask.getId(), requestId)); this.isStopped = true; StopTaskCounter.getInstance().decrement(getTaskId()); - log.info("ResultHandleTask-onStop end, stepInstanceId: {}", stepInstanceId); + log.info("ResultHandleTask-onStop end, task: {}", gseTaskInfo); } else { log.info("ResultHandleTask-onStop, task is running now, will stop when idle. stepInstanceId: {}", stepInstanceId); @@ -635,7 +667,51 @@ public String getTaskType() { } /** - * 获取G步骤执行结果 + * 是否所有目标上的执行对象任务都完成 + */ + protected boolean isAllTargetExecuteObjectTasksDone() { + return this.analyseFinishedTargetExecuteObjectGseKeys.size() == this.targetExecuteObjectGseKeys.size(); + } + + /** + * 是否所有目标上的执行对象任务都执行成功 + */ + protected boolean isAllTargetExecuteObjectTasksSuccess() { + return this.targetExecuteObjectGseKeys.size() == this.successTargetExecuteObjectGseKeys.size(); + } + + /** + * 所有执行对象任务结束的时候，分析整体GSE任务状态 + */ + protected GseTaskExecuteResult analyseFinishedExecuteResult() { + GseTaskExecuteResult rst; + if (isAllTargetExecuteObjectTasksSuccess()) { + // 如果源/目标包含非法主机，设置任务状态为失败 + if (existNoExecutableExecuteObject()) { + log.info("Gse task contains invalid execute object, set execute result fail"); + rst = GseTaskExecuteResult.FAILED; + } else { + rst = GseTaskExecuteResult.SUCCESS; + } + } else { + if (this.isTerminatedSuccess) { + rst = GseTaskExecuteResult.STOP_SUCCESS; + } else { + rst = GseTaskExecuteResult.FAILED; + } + } + return rst; + } + + /** + * 任务是否包含不可执行的执行对象 + */ + protected boolean existNoExecutableExecuteObject() { + return this.existNoExecutableTargetExecuteObject; + } + + /** + * 获取执行结果 * * @return 执行结果 */ @@ -644,17 +720,22 @@ protected final GseTaskExecuteResult getExecuteResult() { } /** - * 分批拉取GSE日志 + * 分批拉取GSE任务结果 * * @return 日志 */ abstract GseLogBatchPullResult pullGseTaskResultInBatches(); /** - * 解析GSE日志并获取结果 + * 解析GSE任务结果 * - * @param gseLog GSE日志 + * @param gseTaskResult GSE日志 * @return 任务执行结果 */ - abstract GseTaskExecuteResult analyseGseTaskResult(GseLog gseLog); + abstract GseTaskExecuteResult analyseGseTaskResult(GseTaskResult gseTaskResult); + + @Override + public TaskContext getTaskContext() { + return this.taskContext; + } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ContinuousScheduledTask.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ContinuousScheduledTask.java index 68abf9f7b0..a7dbedef3a 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ContinuousScheduledTask.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ContinuousScheduledTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/DelayedTask.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/DelayedTask.java index 5f09e0bbe4..6363f019d4 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/DelayedTask.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/DelayedTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -117,4 +117,9 @@ public String toString() { public String getTaskId() { return this.task.getTaskId(); } + + @Override + public TaskContext getTaskContext() { + return task.getTaskContext(); + } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/FileResultHandleTask.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/FileResultHandleTask.java index 067abe467a..c14f505576 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/FileResultHandleTask.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/FileResultHandleTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,54 +25,55 @@ package com.tencent.bk.job.execute.engine.result; import com.google.common.collect.Sets; -import com.tencent.bk.gse.taskapi.api_map_rsp; -import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.InternalException; +import com.tencent.bk.job.common.gse.constants.FileDistModeEnum; +import com.tencent.bk.job.common.gse.constants.GSECode; +import com.tencent.bk.job.common.gse.v2.model.AtomicFileTaskResult; +import com.tencent.bk.job.common.gse.v2.model.AtomicFileTaskResultContent; +import com.tencent.bk.job.common.gse.v2.model.ExecuteObjectGseKey; +import com.tencent.bk.job.common.gse.v2.model.FileTaskResult; +import com.tencent.bk.job.common.gse.v2.model.GetTransferFileResultRequest; import com.tencent.bk.job.common.util.ip.IpUtils; import com.tencent.bk.job.common.util.json.JsonUtils; -import com.tencent.bk.job.execute.common.constants.FileDistModeEnum; +import com.tencent.bk.job.common.util.toggle.ToggleEvaluateContext; +import com.tencent.bk.job.common.util.toggle.ToggleStrategyContextParams; +import com.tencent.bk.job.common.util.toggle.feature.FeatureIdConstants; +import com.tencent.bk.job.common.util.toggle.feature.FeatureToggle; import com.tencent.bk.job.execute.common.constants.FileDistStatusEnum; -import com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum; -import com.tencent.bk.job.execute.engine.consts.FileDirTypeConf; -import com.tencent.bk.job.execute.engine.consts.GSECode; -import com.tencent.bk.job.execute.engine.evict.TaskEvictPolicyExecutor; -import com.tencent.bk.job.execute.engine.gse.GseRequestUtils; -import com.tencent.bk.job.execute.engine.gse.model.CopyFileRsp; -import com.tencent.bk.job.execute.engine.gse.model.GSEFileTaskResult; -import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; -import com.tencent.bk.job.execute.engine.model.FileTaskLog; -import com.tencent.bk.job.execute.engine.model.GseLog; +import com.tencent.bk.job.execute.engine.EngineDependentServiceHolder; +import com.tencent.bk.job.execute.engine.consts.ExecuteObjectTaskStatusEnum; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; +import com.tencent.bk.job.execute.engine.model.FileDest; +import com.tencent.bk.job.execute.engine.model.FileGseTaskResult; import com.tencent.bk.job.execute.engine.model.GseLogBatchPullResult; import com.tencent.bk.job.execute.engine.model.GseTaskExecuteResult; +import com.tencent.bk.job.execute.engine.model.GseTaskResult; import com.tencent.bk.job.execute.engine.model.JobFile; import com.tencent.bk.job.execute.engine.model.TaskVariablesAnalyzeResult; -import com.tencent.bk.job.execute.engine.result.ha.ResultHandleTaskKeepaliveManager; -import com.tencent.bk.job.execute.engine.util.FilePathUtils; import com.tencent.bk.job.execute.engine.util.GseUtils; -import com.tencent.bk.job.execute.engine.util.NFSUtils; -import com.tencent.bk.job.execute.engine.util.WindowsHelper; -import com.tencent.bk.job.execute.model.AgentTaskDTO; +import com.tencent.bk.job.execute.model.ExecuteObjectCompositeKey; +import com.tencent.bk.job.execute.model.ExecuteObjectTask; import com.tencent.bk.job.execute.model.GseTaskDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; -import com.tencent.bk.job.execute.service.FileAgentTaskService; -import com.tencent.bk.job.execute.service.GseTaskService; -import com.tencent.bk.job.execute.service.LogService; -import com.tencent.bk.job.execute.service.StepInstanceService; -import com.tencent.bk.job.execute.service.StepInstanceVariableValueService; -import com.tencent.bk.job.execute.service.TaskInstanceService; -import com.tencent.bk.job.execute.service.TaskInstanceVariableService; -import com.tencent.bk.job.logsvr.model.service.ServiceFileTaskLogDTO; -import com.tencent.bk.job.logsvr.model.service.ServiceHostLogDTO; +import com.tencent.bk.job.execute.service.FileExecuteObjectTaskService; +import com.tencent.bk.job.logsvr.model.service.ServiceExecuteObjectLogDTO; +import com.tencent.bk.job.manage.GlobalAppScopeMappingService; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.Pair; import org.springframework.util.StopWatch; import java.text.DecimalFormat; import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.Set; @@ -80,82 +81,66 @@ * 文件任务执行结果处理 */ @Slf4j -public class FileResultHandleTask extends AbstractResultHandleTask { - private static final String FILE_TASK_MODE_DOWNLOAD = "download"; - private static final String FILE_TASK_MODE_UPLOAD = "upload"; +public class FileResultHandleTask extends AbstractResultHandleTask { /** - * GSE 源 Agent 任务, Map + * 文件源-GSE任务与JOB执行对象任务的映射关系 */ - private final Map sourceAgentTasks; - /** - * 待分发文件，文件传输的源文件 - */ - private final Set sendFiles; + private final Map sourceExecuteObjectTaskMap; /** * 任务包含的源服务器 */ - private final Set sourceAgentIds = new HashSet<>(); + private final Set sourceExecuteObjectGseKeys = new HashSet<>(); /** * 已经分析结果完成的源服务器 */ - protected Set analyseFinishedSourceAgentIds = new HashSet<>(); + protected Set analyseFinishedSourceExecuteObjectGseKeys = new HashSet<>(); /** - * 成功的文件下载任务，key: agentId, value: 成功的文件任务名称 + * 成功的文件下载任务，key: ExecuteObjectGseKey, value: 成功的文件任务名称 */ - private final Map> successDownloadFileMap = new HashMap<>(); + private final Map> successDownloadFileMap = new HashMap<>(); /** - * 已经结束的文件下载任务，key: agentId, value: 已完成文件任务名称 + * 已经结束的文件下载任务，key: ExecuteObjectGseKey, value: 已完成文件任务名称 */ - private final Map> finishedDownloadFileMap = new HashMap<>(); + private final Map> finishedDownloadFileMap = new HashMap<>(); /** - * 下载文件任务数，key: agentId, value: 主机对应的文件任务数 + * 下载文件任务数，key: ExecuteObjectGseKey, value: 主机对应的文件任务数 */ - private final Map fileDownloadTaskNumMap = new HashMap<>(); + private final Map fileDownloadTaskNumMap = new HashMap<>(); /** - * 成功的文件上传任务，key: agentId, value: 成功的文件任务名称 + * 成功的文件上传任务，key: ExecuteObjectGseKey, value: 成功的文件任务名称 */ - private final Map> successUploadFileMap = new HashMap<>(); + private final Map> successUploadFileMap = new HashMap<>(); /** - * 已经结束的文件上传任务，key: agentId, value: 已完成文件任务名称 + * 已经结束的文件上传任务，key: ExecuteObjectGseKey, value: 已完成文件任务名称 */ - private final Map> finishedUploadFileMap = new HashMap<>(); + private final Map> finishedUploadFileMap = new HashMap<>(); /** - * 上传文件任务数，key: agentId, value: 主机对应的文件任务数 + * 上传文件任务数，key: ExecuteObjectGseKey, value: 主机对应的文件任务数 */ - private final Map fileUploadTaskNumMap = new HashMap<>(); + private final Map fileUploadTaskNumMap = new HashMap<>(); /** * 文件任务进度表 */ private final Map processMap = new HashMap<>(); + + private final Map srcDestFileMap; + + // @tmp 兼容gse agent < 1.7.2 之前的版本， Map<目标路径, Map<源IP,源文件>> + private Map> compatibleDestSrcMap = null; + /** - * 本地文件上传目录 - */ - private final String localUploadDir; - /** - * 源文件ip与云区域IP的映射关系；为了规避GSE BUG，GSE Server的源IP在多文件的场景下会返回-1； - * 为了解决GSE Server日志不返回源IP的云区域ID的问题 - */ - private final Map intSourceIpMapping = new HashMap<>(); - /** - * 分发的源文件路径与分发之后的目标文件路径的映射关系 - */ - private final Map sourceDestPathMap; - /** - * 源文件真实路径与显示路径的映射关系，用于本地文件分发场景下的真实路径隐藏 + * 源文件. key: file-key; value: 文件 */ - private final Map sourceFileDisplayMap; + private final Map srcFilesMap = new HashMap<>(); + /** * 下载全部结束的时间 */ private long downloadFinishedTime = 0; /** - * 未开始任务的文件源服务器 - */ - private final Set notStartedFileSourceAgentIds = new HashSet<>(); - /** - * 正在执行任务的文件源服务器 + * 未结束的目标服务器 */ - private final Set runningFileSourceAgentIds = new HashSet<>(); + protected Set notFinishedSourceExecuteObjectGseKeys = new HashSet<>(); /** * 文件任务执行结果处理调度策略 */ @@ -164,152 +149,136 @@ public class FileResultHandleTask extends AbstractResultHandleTask * 任务基本信息，用于日志输出 */ private String taskInfo; + /** + * 是否存在不可执行的源执行对象 + */ + protected boolean existNoExecutableSourceExecuteObject; + - public FileResultHandleTask(TaskInstanceService taskInstanceService, - GseTaskService gseTaskService, - LogService logService, - TaskInstanceVariableService taskInstanceVariableService, - StepInstanceVariableValueService stepInstanceVariableValueService, - TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher, - ResultHandleTaskKeepaliveManager resultHandleTaskKeepaliveManager, - TaskEvictPolicyExecutor taskEvictPolicyExecutor, - FileAgentTaskService fileAgentTaskService, - StepInstanceService stepInstanceService, + public FileResultHandleTask(EngineDependentServiceHolder engineDependentServiceHolder, + FileExecuteObjectTaskService fileExecuteObjectTaskService, TaskInstanceDTO taskInstance, StepInstanceDTO stepInstance, TaskVariablesAnalyzeResult taskVariablesAnalyzeResult, - Map targetAgentTasks, - Map sourceAgentTasks, + Map targetExecuteObjectTaskMap, + Map sourceExecuteObjectTaskMap, GseTaskDTO gseTask, - Set sendFiles, - String storageRootPath, - Map sourceDestPathMap, - Map sourceFileDisplayMap, - String requestId) { - super(taskInstanceService, - gseTaskService, - logService, - taskInstanceVariableService, - stepInstanceVariableValueService, - taskExecuteMQEventDispatcher, - resultHandleTaskKeepaliveManager, - taskEvictPolicyExecutor, - fileAgentTaskService, - stepInstanceService, + Map srcDestFileMap, + String requestId, + List executeObjectTasks) { + super(engineDependentServiceHolder, + fileExecuteObjectTaskService, taskInstance, stepInstance, taskVariablesAnalyzeResult, - targetAgentTasks, + targetExecuteObjectTaskMap, gseTask, - requestId); - this.sourceAgentTasks = sourceAgentTasks; - this.sendFiles = sendFiles; - this.localUploadDir = NFSUtils.getFileDir(storageRootPath, FileDirTypeConf.UPLOAD_FILE_DIR); - if (sourceDestPathMap != null) { - this.sourceDestPathMap = sourceDestPathMap; - } else { - this.sourceDestPathMap = new HashMap<>(); - } - this.sourceFileDisplayMap = sourceFileDisplayMap; + requestId, + executeObjectTasks); + this.sourceExecuteObjectTaskMap = sourceExecuteObjectTaskMap; + this.srcDestFileMap = srcDestFileMap; + + initSrcFilesMap(srcDestFileMap.keySet()); initFileTaskNumMap(); - initSourceAgentIds(); - initFileSourceIntIpMapping(); + initSourceExecuteObjectGseKeys(); + + this.existNoExecutableSourceExecuteObject = + executeObjectTasks.stream().anyMatch( + executeObjectTask -> !executeObjectTask.isTarget() && + !executeObjectTask.getExecuteObject().isExecutable()); + + log.info("InitFileResultHandleTask|stepInstanceId: {}|sourceExecuteObjectGseKeys: {}" + + "|targetExecuteObjectGseKeys: {}|fileUploadTaskNumMap: {}|fileDownloadTaskNumMap: {}" + + "|existNoExecutableSourceExecuteObject: {}", + stepInstance.getId(), sourceExecuteObjectGseKeys, targetExecuteObjectGseKeys, fileUploadTaskNumMap, + fileDownloadTaskNumMap, existNoExecutableSourceExecuteObject); + } - log.info("InitFileResultHandleTask|stepInstanceId: {}|sendFiles: {}|fileSourceAgentIds: {}|targetAgentIds: {}|" - + "fileUploadTaskNumMap: {}|fileDownloadTaskNumMap: {}", - stepInstance.getId(), sendFiles, sourceAgentIds, targetAgentIds, fileUploadTaskNumMap, - fileDownloadTaskNumMap); + private void initSrcFilesMap(Collection srcFiles) { + srcFiles.forEach(srcFile -> + srcFilesMap.put( + buildSrcFileKey(srcFile.getExecuteObject().toExecuteObjectGseKey(), srcFile.getStandardFilePath()), + srcFile) + ); } + private String buildSrcFileKey(ExecuteObjectGseKey executeObjectGseKey, String standardSrcFilePath) { + return executeObjectGseKey.getKey() + ":" + standardSrcFilePath; + } /** * 初始化文件任务计数器 */ private void initFileTaskNumMap() { - for (String agentId : this.targetAgentIds) { - this.fileDownloadTaskNumMap.put(agentId, this.sendFiles.size()); + for (ExecuteObjectGseKey executeObjectGseKey : this.targetExecuteObjectGseKeys) { + this.fileDownloadTaskNumMap.put(executeObjectGseKey, this.srcDestFileMap.size()); } - for (JobFile sendFile : this.sendFiles) { - String agentId = sendFile.getHost().getAgentId(); - this.fileUploadTaskNumMap.put(agentId, this.fileUploadTaskNumMap.get(agentId) == null ? 1 : - (this.fileUploadTaskNumMap.get(agentId) + 1)); + for (JobFile sendFile : this.srcDestFileMap.keySet()) { + ExecuteObjectGseKey executeObjectGseKey = sendFile.getExecuteObject().toExecuteObjectGseKey(); + this.fileUploadTaskNumMap.put(executeObjectGseKey, + this.fileUploadTaskNumMap.get(executeObjectGseKey) == null ? + 1 : (this.fileUploadTaskNumMap.get(executeObjectGseKey) + 1)); } } - private void initSourceAgentIds() { - sourceAgentTasks.values().forEach(agentTask -> { - this.notStartedFileSourceAgentIds.add(agentTask.getAgentId()); - this.sourceAgentIds.add(agentTask.getAgentId()); + private void initSourceExecuteObjectGseKeys() { + sourceExecuteObjectTaskMap.values().forEach(executeObjectTask -> { + this.notFinishedSourceExecuteObjectGseKeys.add( + executeObjectTask.getExecuteObject().toExecuteObjectGseKey()); + this.sourceExecuteObjectGseKeys.add(executeObjectTask.getExecuteObject().toExecuteObjectGseKey()); }); } - private void initFileSourceIntIpMapping() { - // gse bug，使用int保存ip会溢出，所以必须根据当前的源ip反向推出key中源ip - if (this.intSourceIpMapping.isEmpty()) { - for (String cloudIp : this.sourceAgentIds) { - this.intSourceIpMapping.put( - String.valueOf((int) IpUtils.getStringIpToLong(cloudIp.split(":")[1])), - cloudIp); - } - } - } - @Override - GseLogBatchPullResult pullGseTaskResultInBatches() { - api_map_rsp gseLog; - if (CollectionUtils.isNotEmpty(this.analyseFinishedSourceAgentIds) - || CollectionUtils.isNotEmpty(this.analyseFinishedTargetAgentIds)) { - Set notFinishedAgentIds = new HashSet<>(); - notFinishedAgentIds.addAll(notStartedFileSourceAgentIds); - notFinishedAgentIds.addAll(runningFileSourceAgentIds); - notFinishedAgentIds.addAll(notStartedTargetAgentIds); - notFinishedAgentIds.addAll(runningTargetAgentIds); - gseLog = GseRequestUtils.pullCopyFileTaskLog(this.stepInstanceId, this.gseTask.getGseTaskId(), - notFinishedAgentIds); - } else { - gseLog = GseRequestUtils.pullCopyFileTaskLog(this.stepInstanceId, this.gseTask.getGseTaskId()); + GseLogBatchPullResult pullGseTaskResultInBatches() { + GetTransferFileResultRequest request = new GetTransferFileResultRequest(); + request.setGseV2Task(gseV2Task); + request.setTaskId(gseTask.getGseTaskId()); + + if (CollectionUtils.isNotEmpty(this.analyseFinishedSourceExecuteObjectGseKeys) + || CollectionUtils.isNotEmpty(this.analyseFinishedTargetExecuteObjectGseKeys)) { + List notFinishedExecuteObjectGseKeys = new ArrayList<>(); + notFinishedExecuteObjectGseKeys.addAll(notFinishedSourceExecuteObjectGseKeys); + notFinishedExecuteObjectGseKeys.addAll(notFinishedTargetExecuteObjectGseKeys); + request.batchAddAgentQuery(notFinishedExecuteObjectGseKeys); } - GseLogBatchPullResult pullResult; - if (gseLog != null) { - pullResult = new GseLogBatchPullResult<>( - true, true, new FileTaskLog(gseLog), null); + FileTaskResult result = gseClient.getTransferFileResult(request); + GseLogBatchPullResult pullResult; + if (result != null) { + pullResult = new GseLogBatchPullResult<>(true, new FileGseTaskResult(result)); } else { - pullResult = new GseLogBatchPullResult<>(true, true, null, null); + pullResult = new GseLogBatchPullResult<>(true, null); } return pullResult; } @Override - GseTaskExecuteResult analyseGseTaskResult(GseLog taskDetail) { - Set> ipResults = taskDetail.getGseLog().getResult().entrySet(); - // 执行日志, Map - Map executionLogs = new HashMap<>(); + GseTaskExecuteResult analyseGseTaskResult(GseTaskResult taskDetail) { + if (taskDetail == null || taskDetail.isEmptyResult()) { + log.info("Analyse gse task result, result is empty!"); + return analyseExecuteResult(); + } + // 执行日志Map,按照执行对象分组 + Map executionLogs = new HashMap<>(); StopWatch watch = new StopWatch("analyse-gse-file-task"); watch.start("analyse"); - for (Map.Entry ipResult : ipResults) { - CopyFileRsp copyFileRsp = parseCopyFileRspFromGSELog(ipResult); - if (copyFileRsp == null) { + for (AtomicFileTaskResult result : taskDetail.getResult().getAtomicFileTaskResults()) { + compatibleProtocolBeforeV2(result.getContent()); + if (!shouldAnalyse(result)) { continue; } - GSEFileTaskResult fileTaskResult = copyFileRsp.getGseFileTaskResult(); - if (!shouldAnalyse(fileTaskResult)) { - continue; - } - - boolean isDownloadResult = fileTaskResult.isDownloadMode(); - String agentId = isDownloadResult ? - fileTaskResult.getDestAgentId() : fileTaskResult.getSourceAgentId(); + log.info("[{}] Analyse file result: {}", gseTaskInfo, JsonUtils.toJson(result)); + JobAtomicFileTaskResult jobAtomicFileTaskResult = buildJobAtomicFileTaskResult(result); - if (isDownloadResult && this.targetAgentIds.contains(agentId)) { - this.runningTargetAgentIds.add(agentId); - } else if (!isDownloadResult && this.sourceAgentIds.contains(agentId)) { - this.runningFileSourceAgentIds.add(agentId); - } - analyseFileResult(agentId, copyFileRsp, executionLogs, isDownloadResult); + AtomicFileTaskResultContent content = result.getContent(); + boolean isDownloadResult = content.isDownloadMode(); + ExecuteObjectGseKey executeObjectGseKey = isDownloadResult ? + content.getDestExecuteObjectGseKey() : content.getSourceExecuteObjectGseKey(); + analyseFileResult(executeObjectGseKey, jobAtomicFileTaskResult, executionLogs, isDownloadResult); } watch.stop(); @@ -319,22 +288,21 @@ GseTaskExecuteResult analyseGseTaskResult(GseLog taskDetail) { watch.stop(); // 保存任务执行结果 - watch.start("saveAgentTasks"); - batchSaveChangedGseAgentTasks(targetAgentTasks.values()); - batchSaveChangedGseAgentTasks(sourceAgentTasks.values()); + watch.start("saveExecuteObjectTasks"); + batchSaveChangedExecuteObjectTasks(targetExecuteObjectTasks.values()); + batchSaveChangedExecuteObjectTasks(sourceExecuteObjectTaskMap.values()); watch.stop(); - log.info("Analyse gse task log [{}] -> runningTargetAgentIds={}, " + - "notStartedTargetAgentIds={}, runningFileSourceAgentIds={}, notStartedFileSourceAgentIds={}, " + - "analyseFinishedTargetAgentIds={}, analyseFinishedSourceAgentIds={}, finishedDownloadFileMap={}, " + - "successDownloadFileMap={}, finishedUploadFileMap={}, successUploadFileMap={}", - this.stepInstanceId, - this.runningTargetAgentIds, - this.notStartedTargetAgentIds, - this.runningFileSourceAgentIds, - this.notStartedFileSourceAgentIds, - this.analyseFinishedTargetAgentIds, - this.analyseFinishedSourceAgentIds, + log.info("[{}] Analyse gse task result -> " + + "notFinishedTargetExecuteObjectGseKeys={}, notFinishedSourceExecuteObjectGseKeys={}, " + + "analyseFinishedTargetExecuteObjectGseKeys={}, analyseFinishedSourceExecuteObjectGseKeys={}" + + ", finishedDownloadFileMap={}, successDownloadFileMap={}, finishedUploadFileMap={}," + + " successUploadFileMap={}", + this.gseTaskInfo, + this.notFinishedTargetExecuteObjectGseKeys, + this.notFinishedSourceExecuteObjectGseKeys, + this.analyseFinishedTargetExecuteObjectGseKeys, + this.analyseFinishedSourceExecuteObjectGseKeys, this.finishedDownloadFileMap, this.successDownloadFileMap, this.finishedUploadFileMap, @@ -347,70 +315,190 @@ GseTaskExecuteResult analyseGseTaskResult(GseLog taskDetail) { return analyseExecuteResult(); } - private void analyseFileResult(String agentId, CopyFileRsp copyFileRsp, - Map executionLogs, + private void compatibleProtocolBeforeV2(AtomicFileTaskResultContent content) { + if (content == null) { + return; + } + if (content.isApiProtocolBeforeV2() && content.isDownloadMode() && isSupportProtocolBeforeV2()) { + // 老版本协议(协议版本<2.0，gse agent 版本 < 1.7.2),下载任务结果存在问题（没有源主机云区域ID、没有源文件路径), 需要根据任务上下文推测并补充 + if (compatibleDestSrcMap == null) { + compatibleDestSrcMap = new HashMap<>(); + srcDestFileMap.forEach((jobFile, fileDest) -> { + // GSE BUG, 如果是目录分发，那么下载结果中只会返回目标目录的上一级 + // 例如：download:-1:-1:/tmp/test/:0:127.0.0.1， 表示从源分发一个目录到目标/tmp/test/目录下 + String destPath = jobFile.isDir() ? fileDest.getDestDirPath() : fileDest.getDestPath(); + compatibleDestSrcMap.compute(destPath, (dest, map) -> { + if (map == null) { + map = new HashMap<>(); + } + if (StringUtils.isNotBlank(jobFile.getExecuteObject().getHost().getIp())) { + map.put(jobFile.getExecuteObject().getHost().getIp(), jobFile); + } + return map; + }); + }); + log.info("[CompatibleProtocolBeforeV2] Init destSrcMap: {}", compatibleDestSrcMap); + } + String destPath = content.getStandardDestFilePath(); + Map srcIpAndSrcFile = compatibleDestSrcMap.get(destPath); + if (srcIpAndSrcFile == null) { + log.error("[CompatibleProtocolBeforeV2] Can not get srcFile by destPath: {}. ", destPath); + return; + } + String srcAgentId = content.getSourceAgentId(); + // 源主机没有包含云区域信息，需要处理 + String srcIp = IpUtils.extractIp(srcAgentId); + JobFile srcFile = srcIpAndSrcFile.get(srcIp); + if (srcFile == null) { + log.error("[CompatibleProtocolBeforeV2] Can not get srcFile by destPath: {} and srcIp: {}. ", + destPath, srcIp); + return; + } + content.setSourceAgentId(srcFile.getExecuteObject().getHost().toCloudIp()); + content.setSourceFileDir(srcFile.getDir()); + content.setSourceFileName(srcFile.getFileName()); + // 重置 + content.setStandardSourceFilePath(null); + content.setTaskId(null); + } + } + + private boolean isSupportProtocolBeforeV2() { + return FeatureToggle.checkFeature( + FeatureIdConstants.GSE_FILE_PROTOCOL_BEFORE_V2, + ToggleEvaluateContext.builder() + .addContextParam(ToggleStrategyContextParams.CTX_PARAM_RESOURCE_SCOPE, + GlobalAppScopeMappingService.get().getScopeByAppId(appId)) + ); + } + + private void analyseFileResult(ExecuteObjectGseKey executeObjectGseKey, + JobAtomicFileTaskResult result, + Map executionLogs, boolean isDownloadResult) { - AgentTaskDTO agentTask = getAgentTask(isDownloadResult, agentId); - if (agentTask.getStartTime() == null) { - agentTask.setStartTime(System.currentTimeMillis()); + ExecuteObjectTask executeObjectTask = result.getExecuteObjectTask(); + if (executeObjectTask.getStartTime() == null) { + executeObjectTask.setStartTime(System.currentTimeMillis()); } - agentTask.setErrorCode(copyFileRsp.getFinalErrorCode()); - GSECode.AtomicErrorCode errorCode = GSECode.AtomicErrorCode.getErrorCode(copyFileRsp.getFinalErrorCode()); - GSEFileTaskResult fileTaskResult = copyFileRsp.getGseFileTaskResult(); + executeObjectTask.setErrorCode(result.getResult().getErrorCode()); + GSECode.AtomicErrorCode errorCode = GSECode.AtomicErrorCode.getErrorCode(result.getResult().getErrorCode()); switch (errorCode) { case RUNNING: - parseExecutionLog(copyFileRsp, executionLogs); - agentTask.setStatus(AgentTaskStatusEnum.RUNNING); - if (isDownloadResult) { - this.notStartedTargetAgentIds.remove(agentId); - } + analyseRunningFileResult(result, executionLogs, executeObjectTask); break; case FINISHED: - parseExecutionLog(copyFileRsp, executionLogs); - if (fileTaskResult.getProcess() == 100) { - if (isDownloadResult) { - addFinishedFile(true, true, - fileTaskResult.getDestAgentId(), fileTaskResult.getTaskId()); - } else { - addFinishedFile(true, false, - fileTaskResult.getSourceAgentId(), fileTaskResult.getTaskId()); - } - // 分析日志，更新successAgentIds、notStartedAgentIds等状态集合 - analyseAgentTaskResult(errorCode.getValue(), agentId, fileTaskResult.getStartTime(), - fileTaskResult.getEndTime(), isDownloadResult); - } else { - agentTask.setStatus(AgentTaskStatusEnum.RUNNING); - this.notStartedTargetAgentIds.remove(agentId); - } + analyseSuccessFileResult(executeObjectGseKey, result, executionLogs, isDownloadResult); break; case TERMINATE: - parseExecutionLog(copyFileRsp, executionLogs); - if (isDownloadResult) { - addFinishedFile(false, true, - fileTaskResult.getDestAgentId(), fileTaskResult.getTaskId()); - } else { - addFinishedFile(false, false, - fileTaskResult.getSourceAgentId(), fileTaskResult.getTaskId()); - } - analyseAgentTaskResult(errorCode.getValue(), agentId, fileTaskResult.getStartTime(), - fileTaskResult.getEndTime(), isDownloadResult); - this.isTerminatedSuccess = true; + analyseTerminatedFileResult(executeObjectGseKey, result, executionLogs, isDownloadResult); break; default: - dealIpTaskFail(copyFileRsp, executionLogs, isDownloadResult); + analyseFailedFileResult(result, executionLogs, isDownloadResult); break; } } - private AgentTaskDTO getAgentTask(boolean isDownloadResult, String agentId) { + private void analyseRunningFileResult(JobAtomicFileTaskResult result, + Map executionLogs, + ExecuteObjectTask executeObjectTask) { + parseExecutionLog(result, executionLogs); + executeObjectTask.setStatus(ExecuteObjectTaskStatusEnum.RUNNING); + executeObjectTask.setStartTime(result.getResult().getContent().getStartTime()); + } + + private void analyseSuccessFileResult(ExecuteObjectGseKey executeObjectGseKey, + JobAtomicFileTaskResult result, + Map executionLogs, + boolean isDownloadResult) { + AtomicFileTaskResultContent content = result.getResult().getContent(); + parseExecutionLog(result, executionLogs); if (isDownloadResult) { - return targetAgentTasks.get(agentId); + addFinishedFile(true, true, + content.getDestExecuteObjectGseKey(), content.getTaskId()); } else { - return sourceAgentTasks.get(agentId); + addFinishedFile(true, false, + content.getSourceExecuteObjectGseKey(), content.getTaskId()); } + // 分析执行对象任务结果 + analyseExecuteObjectTaskResult(GSECode.AtomicErrorCode.FINISHED.getValue(), executeObjectGseKey, + content.getStartTime(), content.getEndTime(), isDownloadResult); } - /* + private void analyseTerminatedFileResult(ExecuteObjectGseKey executeObjectGseKey, + JobAtomicFileTaskResult result, + Map executionLogs, + boolean isDownloadResult) { + AtomicFileTaskResultContent content = result.getResult().getContent(); + parseExecutionLog(result, executionLogs); + if (isDownloadResult) { + addFinishedFile(false, true, + content.getDestExecuteObjectGseKey(), content.getTaskId()); + } else { + addFinishedFile(false, false, + content.getSourceExecuteObjectGseKey(), content.getTaskId()); + } + analyseExecuteObjectTaskResult(GSECode.AtomicErrorCode.TERMINATE.getValue(), executeObjectGseKey, + content.getStartTime(), content.getEndTime(), isDownloadResult); + this.isTerminatedSuccess = true; + } + + private JobAtomicFileTaskResult buildJobAtomicFileTaskResult(AtomicFileTaskResult atomicFileTaskResult) { + AtomicFileTaskResultContent content = atomicFileTaskResult.getContent(); + boolean isDownloadResult = content.isDownloadMode(); + ExecuteObjectGseKey executeObjectGseKey = isDownloadResult ? + content.getDestExecuteObjectGseKey() : content.getSourceExecuteObjectGseKey(); + JobFile srcFile = srcFilesMap.get(buildSrcFileKey(content.getSourceExecuteObjectGseKey(), + content.getStandardSourceFilePath())); + if (srcFile == null) { + log.error("Src file not found, sourceAgentId: {}, filePath:{}", content.getSourceAgentId(), + content.getStandardSourceFilePath()); + throw new InternalException("Parse src file fail", ErrorCode.INTERNAL_ERROR); + } + + ExecuteObjectTask executeObjectTask = getExecuteObjectTask(isDownloadResult, executeObjectGseKey); + + ExecuteObject sourceExecuteObject = findExecuteObject(sourceExecuteObjectTaskMap, + content.getSourceExecuteObjectGseKey()); + if (sourceExecuteObject == null) { + log.error("Src execute object not found, sourceExecuteObjectGseKey: {}", + content.getSourceExecuteObjectGseKey()); + throw new InternalException("Parse src execute object fail", ErrorCode.INTERNAL_ERROR); + } + + ExecuteObject targetExecuteObject = null; + if (isDownloadResult) { + targetExecuteObject = findExecuteObject(targetExecuteObjectTasks, + content.getDestExecuteObjectGseKey()); + if (targetExecuteObject == null) { + log.error("Target execute object not found, destExecuteObjectGseKey: {}", + content.getDestExecuteObjectGseKey()); + throw new InternalException("Parse target host fail", ErrorCode.INTERNAL_ERROR); + } + } + + return new JobAtomicFileTaskResult(atomicFileTaskResult, sourceExecuteObject, targetExecuteObject, + srcFile, executeObjectTask); + } + + private ExecuteObject findExecuteObject(Map executeObjectTaskMap, + ExecuteObjectGseKey executeObjectGseKey) { + ExecuteObjectTask executeObjectTask = executeObjectTaskMap.get(executeObjectGseKey); + if (executeObjectGseKey == null) { + return null; + } + return executeObjectTask.getExecuteObject(); + } + + private ExecuteObjectTask getExecuteObjectTask(boolean isDownloadResult, + ExecuteObjectGseKey executeObjectGseKey) { + if (isDownloadResult) { + return targetExecuteObjectTasks.get(executeObjectGseKey); + } else { + return sourceExecuteObjectTaskMap.get(executeObjectGseKey); + } + } + + /** * 分析执行结果 * * @return 任务执行结果 @@ -418,22 +506,12 @@ private AgentTaskDTO getAgentTask(boolean isDownloadResult, String agentId) { private GseTaskExecuteResult analyseExecuteResult() { GseTaskExecuteResult rst; // 目标下载全部完成 - if (this.notStartedTargetAgentIds.isEmpty() && this.runningTargetAgentIds.isEmpty()) { + if (isAllTargetExecuteObjectTasksDone()) { // 源上传全部完成 - if (this.notStartedFileSourceAgentIds.isEmpty() && this.runningFileSourceAgentIds.isEmpty()) { - int successTargetIpNum = this.successTargetAgentIds.size(); - int targetIPNum = this.targetAgentIds.size(); - boolean isSuccess = !stepInstance.hasInvalidHost() && successTargetIpNum == targetIPNum; - if (isSuccess) { - rst = GseTaskExecuteResult.SUCCESS; - } else { - if (this.isTerminatedSuccess) { - rst = GseTaskExecuteResult.STOP_SUCCESS; - } else { - rst = GseTaskExecuteResult.FAILED; - } - } - log.info("[{}] AnalyseExecuteResult-> Result: finished. All source and target ip have completed tasks", + if (isAllSourceExecuteObjectTasksDone()) { + rst = analyseFinishedExecuteResult(); + log.info("[{}] AnalyseExecuteResult-> Result: finished. All source and target execute object " + + "have completed tasks", this.stepInstanceId); } else { // 场景：下载任务已全部结束，但是GSE未更新上传任务的状态。如果超过15s没有结束上传任务，那么任务结束 @@ -441,10 +519,7 @@ private GseTaskExecuteResult analyseExecuteResult() { this.downloadFinishedTime = System.currentTimeMillis(); } if (System.currentTimeMillis() - this.downloadFinishedTime > 15_000L) { - int targetIPNum = this.targetAgentIds.size(); - int successTargetIpNum = this.successTargetAgentIds.size(); - boolean isSuccess = !stepInstance.hasInvalidHost() && successTargetIpNum == targetIPNum; - rst = isSuccess ? GseTaskExecuteResult.SUCCESS : GseTaskExecuteResult.FAILED; + rst = analyseFinishedExecuteResult(); log.info("[{}] AnalyseExecuteResult-> Result: finished. Download tasks are finished, " + "but upload tasks are not finished after 15 seconds. Ignore upload tasks", this.stepInstanceId); @@ -463,453 +538,363 @@ private GseTaskExecuteResult analyseExecuteResult() { return rst; } - private boolean shouldAnalyse(GSEFileTaskResult fileTaskResult) { - if (fileTaskResult == null) { - return false; - } - boolean isDownloadMode = fileTaskResult.isDownloadMode(); - String agentId = isDownloadMode ? - fileTaskResult.getDestAgentId() : fileTaskResult.getSourceAgentId(); - boolean shouldAnalyse = true; - if (isDownloadMode) { - if (this.analyseFinishedTargetAgentIds.contains(agentId) // 该ip已经日志分析结束，不要再分析 - // 该文件下载任务已结束 - || (this.finishedDownloadFileMap.get(agentId) != null - && this.finishedDownloadFileMap.get(agentId).contains(fileTaskResult.getTaskId())) - // 非目标IP - || !this.fileDownloadTaskNumMap.containsKey(agentId)) { - shouldAnalyse = false; - } - } else { - if ((this.finishedUploadFileMap.get(agentId) != null - && this.finishedUploadFileMap.get(agentId).contains(fileTaskResult.getTaskId())) - // 非源IP - || !this.fileUploadTaskNumMap.containsKey(agentId)) { - shouldAnalyse = false; - } - } - return shouldAnalyse; - } - - private CopyFileRsp parseCopyFileRspFromGSELog(Map.Entry ipResult) { - log.info("[{}]: ParseIpResult: {}", this.stepInstanceId, ipResult); - String taskInfo = ipResult.getValue(); - CopyFileRsp copyFileRsp; - try { - copyFileRsp = JsonUtils.fromJson(taskInfo, CopyFileRsp.class); - if (copyFileRsp == null) { - return null; - } - } catch (Throwable e) { - log.error("[" + this.stepInstanceId + "]: Convert to CopyFileRsp error", e); - return null; - } - - // 新版GSE, 把所有任务相关的信息全部放入fileTaskResult，并且新增了status字段 - boolean isStandardGSEProtocol = isStandardGSEProtocol(copyFileRsp.getGseFileTaskResult()); - if (!isStandardGSEProtocol) { - copyFileRsp = parseCopyFileRspFromResultKey(copyFileRsp, ipResult.getKey()); - if (copyFileRsp != null) { - log.debug("Parse from resultKey, copyFileRsp: {}", copyFileRsp); - } - } - return copyFileRsp; + @Override + protected boolean existNoExecutableExecuteObject() { + return this.existNoExecutableTargetExecuteObject || this.existNoExecutableSourceExecuteObject; } - private boolean isStandardGSEProtocol(GSEFileTaskResult fileTaskResult) { - return fileTaskResult != null && fileTaskResult.getProtocolVersion() != null - && fileTaskResult.getProtocolVersion() > 1; + private boolean isAllSourceExecuteObjectTasksDone() { + return this.notFinishedSourceExecuteObjectGseKeys.isEmpty() + && this.notFinishedTargetExecuteObjectGseKeys.isEmpty(); } - private CopyFileRsp parseCopyFileRspFromResultKey(CopyFileRsp copyFileRsp, String resultKey) { - boolean isGSEAgentLog = - resultKey.startsWith(FILE_TASK_MODE_DOWNLOAD) || resultKey.startsWith(FILE_TASK_MODE_UPLOAD); - // 过滤GSE Server 115状态的日志（GSE BUG, TO BE FIXED) - if (!isGSEAgentLog && copyFileRsp.getFinalErrorCode().equals(GSECode.AtomicErrorCode.RUNNING.getValue())) { - return null; + private boolean shouldAnalyse(AtomicFileTaskResult result) { + if (result == null) { + return false; } - // 从key中提取任务信息 - String[] taskProps = resultKey.split(":"); - GSEFileTaskResult fileTaskResult = copyFileRsp.getGseFileTaskResult(); - if (fileTaskResult == null) { - fileTaskResult = new GSEFileTaskResult(); - copyFileRsp.setGseFileTaskResult(fileTaskResult); - } - if (fileTaskResult.getMode() == null) { - fileTaskResult.setMode(parseFileTaskModeFromKey(isGSEAgentLog, taskProps).getValue()); - } - HostDTO cloudIp = parseCloudIpFromKey(taskProps); - if (FileDistModeEnum.DOWNLOAD.getValue().equals(fileTaskResult.getMode())) { - fileTaskResult.setDestIp(cloudIp.getIp()); - fileTaskResult.setDestCloudId(cloudIp.getBkCloudId()); - HostDTO fileSourceCloudIp = parseFileSourceIpFromKey(isGSEAgentLog, cloudIp, taskProps); - if (fileSourceCloudIp != null) { - fileTaskResult.setSourceCloudId(fileSourceCloudIp.getBkCloudId()); - fileTaskResult.setSourceIp(fileSourceCloudIp.getIp()); + AtomicFileTaskResultContent content = result.getContent(); + boolean isDownloadMode = content.isDownloadMode(); + ExecuteObjectGseKey executeObjectGseKey = isDownloadMode ? + content.getDestExecuteObjectGseKey() : content.getSourceExecuteObjectGseKey(); + if (isDownloadMode) { + if (this.analyseFinishedTargetExecuteObjectGseKeys.contains(executeObjectGseKey) // 该Agent已经分析结束，不需要再分析 + // 该文件下载任务已结束 + || (this.finishedDownloadFileMap.get(executeObjectGseKey) != null + && this.finishedDownloadFileMap.get(executeObjectGseKey).contains(content.getTaskId()))) { + return false; } - // GSE BUG, 只有目标文件信息，没有源文件信息 - String destFilePath = parseFilePathFromKey(taskProps); - Pair dirAndFileName = FilePathUtils.parseDirAndFileName(destFilePath); - fileTaskResult.setDestDirPath(dirAndFileName.getLeft()); - fileTaskResult.setDestFileName(dirAndFileName.getRight()); - } else { - fileTaskResult.setSourceIp(cloudIp.getIp()); - fileTaskResult.setSourceCloudId(cloudIp.getBkCloudId()); - String sourceFilePath = parseFilePathFromKey(taskProps); - Pair dirAndFileName = FilePathUtils.parseDirAndFileName(sourceFilePath); - fileTaskResult.setSrcDirPath(dirAndFileName.getLeft()); - fileTaskResult.setSrcFileName(dirAndFileName.getRight()); - } - return copyFileRsp; - } - - private String parseFilePathFromKey(String[] taskProps) { - String filePath = taskProps[taskProps.length - 3]; - if (taskProps.length > 4 && taskProps[taskProps.length - 4] != null) { - // 如果是正则的文件， /tmp/REGEX:abc.*.txt 这种有:，在key中会被分开，要拼回去 - // GSE 的Redis Key问题 可能引入空格变=号，导致key被当成key=value, 所以要判断 taskProps.length > 4 - // Windows路径包含: - if (taskProps[taskProps.length - 4].endsWith("REGEX") - || WindowsHelper.isWindowsDiskPartition(taskProps[taskProps.length - 4])) { - filePath = taskProps[taskProps.length - 4] + ":" + filePath; + // 不属于当前任务的目标Agent + if (!this.fileDownloadTaskNumMap.containsKey(executeObjectGseKey)) { + log.warn("[{}] Unexpected target executeObjectGseKey {}. result: {}", gseTaskInfo, executeObjectGseKey, + JsonUtils.toJson(result)); + return false; } - } - return filePath; - } - - - private HostDTO parseCloudIpFromKey(String[] taskProps) { - String ip = taskProps[taskProps.length - 1]; - long cloudAreaId = Long.parseLong(taskProps[taskProps.length - 2].trim()); - return new HostDTO(cloudAreaId, ip); - } - - private FileDistModeEnum parseFileTaskModeFromKey(boolean isGseAgentLog, String[] taskProps) { - FileDistModeEnum fileDistMode; - if (isGseAgentLog) { - fileDistMode = taskProps[0].equals(FILE_TASK_MODE_DOWNLOAD) ? FileDistModeEnum.DOWNLOAD : - FileDistModeEnum.UPLOAD; } else { - HostDTO cloudIp = parseCloudIpFromKey(taskProps); - // GSE Task Server 日志， 如果该key为目标服务器ip的日志，并且key的第一个字段如果为"-1"或者为源文件ip, 表示download日志 - String fileSourceCloudIp = this.intSourceIpMapping.get(taskProps[0]); - if (this.targetAgentIds.contains(cloudIp.toCloudIp()) - && (taskProps[0].equals("-1") - || (fileSourceCloudIp != null && this.sourceAgentIds.contains(fileSourceCloudIp)))) { - fileDistMode = FileDistModeEnum.DOWNLOAD; - } else { - fileDistMode = FileDistModeEnum.UPLOAD; + if (this.analyseFinishedSourceExecuteObjectGseKeys.contains(executeObjectGseKey) // 该Agent已经分析结束，不需要再分析 + || (this.finishedUploadFileMap.get(executeObjectGseKey) != null + && this.finishedUploadFileMap.get(executeObjectGseKey).contains(content.getTaskId()))) { + return false; + } + // 不属于当前任务的源Agent + if (!this.fileUploadTaskNumMap.containsKey(executeObjectGseKey)) { + log.warn("[{}] Unexpected source executeObjectGseKey {}. result: {}", gseTaskInfo, executeObjectGseKey, + JsonUtils.toJson(result)); + return false; } } - return fileDistMode; + return true; } - private HostDTO parseFileSourceIpFromKey(boolean isGseAgentLog, HostDTO cloudIp, String[] taskProps) { - HostDTO fileSourceIp = null; - if (isGseAgentLog) { - if (FILE_TASK_MODE_DOWNLOAD.equalsIgnoreCase(taskProps[0])) { - // GSE BUG, download日志无法获取到源IP的云区域 - String fileSourceCloudIp = guessFileSourceCloudIp(taskProps[2]); - if (StringUtils.isNotEmpty(fileSourceCloudIp)) { - fileSourceIp = HostDTO.fromCloudIp(fileSourceCloudIp); - } - } else { - fileSourceIp = cloudIp; - } + private void analyseFailedFileResult(JobAtomicFileTaskResult result, + Map executionLogs, + boolean isDownloadResult) { + if (isDownloadResult) { + dealDownloadTaskFail(result.getResult(), executionLogs); } else { - // GSE Task Server 日志， key的第一个字段为源IP; - // GSE BUG, GSE Task Server 日志， key的第一个字段可能为-1 - String fileSourceCloudIp = guessFileSourceCloudIp(taskProps[0]); - if (StringUtils.isNotEmpty(fileSourceCloudIp)) { - fileSourceIp = HostDTO.fromCloudIp(fileSourceCloudIp); - } else { - if (this.sourceAgentIds.contains(cloudIp.toCloudIp())) { - fileSourceIp = cloudIp; - } - } + dealUploadTaskFail(result, executionLogs); } - return fileSourceIp; } - private String guessFileSourceCloudIp(String intIp) { - // gse bug，使用int保存ip会溢出，所以必须根据当前的源ip反向推出key中源ip - return this.intSourceIpMapping.get(intIp); + private void dealDownloadTaskFail(AtomicFileTaskResult result, + Map executionLogs) { + AtomicFileTaskResultContent content = result.getContent(); + dealDownloadTaskFail(executionLogs, content.getSourceExecuteObjectGseKey(), content.getStandardSourceFilePath(), + content.getDestExecuteObjectGseKey(), content.getStandardDestFilePath(), result.getErrorCode(), + buildErrorLogContent(result), content.getStartTime(), content.getEndTime()); } - private void recordDownloadFileFail(CopyFileRsp copyFileRsp, - Map executionLogs) { - GSEFileTaskResult taskResult = copyFileRsp.getGseFileTaskResult(); - String destAgentId = taskResult.getDestAgentId(); - String sourceAgentId = taskResult.getSourceAgentId(); - AgentTaskDTO destAgentTask = targetAgentTasks.get(destAgentId); - AgentTaskDTO sourceAgentTask = sourceAgentTasks.get(sourceAgentId); - boolean isAddSuccess = addFinishedFile(false, true, destAgentId, - GSEFileTaskResult.buildTaskId(taskResult.getMode(), sourceAgentId, taskResult.getStandardSourceFilePath(), - destAgentId, taskResult.getStandardDestFilePath())); - if (isAddSuccess) { - addFileTaskLog(executionLogs, - new ServiceFileTaskLogDTO( - FileDistModeEnum.DOWNLOAD.getValue(), - destAgentTask.getHostId(), - agentIdHostMap.get(destAgentTask.getAgentId()).toCloudIp(), - taskResult.getStandardDestFilePath(), - sourceAgentTask.getHostId(), - agentIdHostMap.get(sourceAgentTask.getAgentId()).toCloudIp(), - agentIdHostMap.get(sourceAgentTask.getAgentId()).getDisplayIp(), - taskResult.getStandardSourceFilePath(), - taskResult.getStandardSourceFilePath() == null ? null : - sourceFileDisplayMap.get(taskResult.getStandardSourceFilePath()), null, - FileDistStatusEnum.FAILED.getValue(), - FileDistStatusEnum.FAILED.getName(), - null, - null, - copyFileRsp.getFinalErrorMsg()) - ); - } - } - - private long getStartTimeOrDefault(GSEFileTaskResult content) { - return (content != null && content.getStartTime() != null && content.getStartTime() > 0) ? - content.getStartTime() : System.currentTimeMillis(); + private void dealDownloadTaskFail(Map executionLogs, + ExecuteObjectGseKey sourceExecuteObjectGseKey, + String sourceFilePath, + ExecuteObjectGseKey destExecuteObjectGseKey, + String destFilePath, + Integer errorCode, + String errorMsg, + Long startTime, + Long endTime) { + JobFile srcFile = srcFilesMap.get(buildSrcFileKey(sourceExecuteObjectGseKey, sourceFilePath)); + ExecuteObject targetExecuteObject = findExecuteObject(targetExecuteObjectTasks, destExecuteObjectGseKey); + FileDest fileDest = srcDestFileMap.get(srcFile); + + // 记录目标IP单个文件下载任务的结束状态 + addFinishedFile(false, + true, + destExecuteObjectGseKey, + AtomicFileTaskResultContent.buildTaskId( + FileDistModeEnum.DOWNLOAD.getValue(), + sourceExecuteObjectGseKey, + sourceFilePath, + destExecuteObjectGseKey, + destFilePath) + ); + // 每个目标执行对象增加一条下载失败的日志到日志总Map中 + logService.addFileTaskLog( + stepInstance, + executionLogs, + targetExecuteObject, + logService.buildDownloadServiceFileTaskLogDTO( + stepInstance, + srcFile, + targetExecuteObject, + fileDest.getDestPath(), + FileDistStatusEnum.FAILED, + null, + null, + null, + errorMsg + ) + ); + analyseExecuteObjectTaskResult(errorCode, destExecuteObjectGseKey, getTimeOrDefault(startTime), + getTimeOrDefault(endTime), true); } - private long getEndTimeOrDefault(GSEFileTaskResult content) { - return (content != null && content.getEndTime() != null && content.getEndTime() > 0) ? - content.getEndTime() : System.currentTimeMillis(); + private long getTimeOrDefault(Long time) { + return time != null && time > 0 ? time : System.currentTimeMillis(); } - private void dealIpTaskFail(CopyFileRsp copyFileRsp, - Map executionLogs, - boolean isDownloadResult) { - GSEFileTaskResult taskResult = copyFileRsp.getGseFileTaskResult(); - - GSEFileTaskResult content = copyFileRsp.getGseFileTaskResult(); - long startTime = getStartTimeOrDefault(content); - long endTime = getEndTimeOrDefault(content); - if (isDownloadResult) { - String destAgentId = taskResult.getDestAgentId(); - recordDownloadFileFail(copyFileRsp, executionLogs); - analyseAgentTaskResult(copyFileRsp.getFinalErrorCode(), destAgentId, startTime, endTime, true); - } else { - // 上传源IP本身处理 - String sourceAgentId = taskResult.getSourceAgentId(); - analyseAgentTaskResult(copyFileRsp.getFinalErrorCode(), sourceAgentId, startTime, endTime, false); - - // 如果上传源失败，除了影响上传，还会影响到下载的目标IP - Set affectedTargetIps = new HashSet<>(); - dealUploadFail(copyFileRsp, executionLogs, affectedTargetIps); - for (String affectedTargetIp : affectedTargetIps) { - analyseAgentTaskResult(0, affectedTargetIp, startTime, endTime, true); - } - } - } /** - * 根据copyFileRsp内容填充executionLogs与affectIps + * 处理下载失败任务 * - * @param copyFileRsp GSE响应内容 - * @param executionLogs 执行日志总Map - * @param affectedTargetAgentIds 受影响的目标Agent集合 + * @param result 任务结果 + * @param executionLogs 执行日志总Map */ - private void dealUploadFail(CopyFileRsp copyFileRsp, Map executionLogs, - Set affectedTargetAgentIds) { - GSEFileTaskResult taskResult = copyFileRsp.getGseFileTaskResult(); - String sourceAgentId = taskResult.getSourceAgentId(); - HostDTO sourceHost = agentIdHostMap.get(sourceAgentId); - // 记录源IP单个文件上传任务的结束状态 - addFinishedFile(false, false, taskResult.getSourceAgentId(), taskResult.getTaskId()); - - String sourceFilePath = taskResult.getStandardSourceFilePath(); - String displayFilePath = sourceFileDisplayMap.get(sourceFilePath); - boolean isLocalUploadFile = sourceFilePath.startsWith(this.localUploadDir); - log.debug("StandardSourceFilePath: {}, localUploadDir: {}, isLocalUploadFile: {}, displayFilePath: {}", - sourceFilePath, this.localUploadDir, isLocalUploadFile, displayFilePath); + private void dealUploadTaskFail(JobAtomicFileTaskResult result, + Map executionLogs) { + AtomicFileTaskResultContent content = result.getResult().getContent(); + ExecuteObjectGseKey sourceExecuteObjectGseKey = content.getSourceExecuteObjectGseKey(); + JobFile srcFile = result.getSrcFile(); + Long startTime = getTimeOrDefault(content.getStartTime()); + Long endTime = getTimeOrDefault(content.getEndTime()); + // 记录源执行对象单个文件上传任务的结束状态 + addFinishedFile(false, false, sourceExecuteObjectGseKey, content.getTaskId()); // 增加一条上传源失败的上传日志 - addFileTaskLog(executionLogs, - new ServiceFileTaskLogDTO( - FileDistModeEnum.UPLOAD.getValue(), - null, - null, - null, - sourceHost.getHostId(), - sourceHost.toCloudIp(), - sourceHost.getDisplayIp(), - sourceFilePath, - displayFilePath, + logService.addFileTaskLog( + stepInstance, + executionLogs, + srcFile.getExecuteObject(), + logService.buildUploadServiceFileTaskLogDTO( + stepInstance, + srcFile, + FileDistStatusEnum.FAILED, null, - FileDistStatusEnum.FAILED.getValue(), - FileDistStatusEnum.FAILED.getName(), null, null, - copyFileRsp.getFinalErrorMsg()) + buildErrorLogContent(result.getResult())) ); + analyseExecuteObjectTaskResult(result.getResult().getErrorCode(), sourceExecuteObjectGseKey, startTime, + endTime, false); + + // 源失败了，会影响所有目标执行对象对应的agent上的download任务 + for (ExecuteObjectGseKey targetExecuteObjectGseKey : this.targetExecuteObjectGseKeys) { + FileDest fileDest = srcDestFileMap.get(srcFile); + dealDownloadTaskFail( + executionLogs, + sourceExecuteObjectGseKey, + content.getStandardSourceFilePath(), + targetExecuteObjectGseKey, + fileDest.getDestPath(), + result.getResult().getErrorCode(), + buildErrorLogContent(result.getResult()), + startTime, + endTime); + } + } - // 源失败了，会影响所有目标IP对应的agent上的download任务 - for (String targetAgentId : this.targetAgentIds) { - String destFilePath; - HostDTO targetHost = agentIdHostMap.get(targetAgentId); - if (isLocalUploadFile) { - destFilePath = this.sourceDestPathMap.get(displayFilePath); - } else { - destFilePath = - this.sourceDestPathMap.get( - taskResult.getSourceAgentId() + ":" + taskResult.getStandardSourceFilePath()); + private String buildErrorLogContent(AtomicFileTaskResult result) { + StringBuilder sb = new StringBuilder(); + sb.append(result.getErrorMsg()).append("."); + AtomicFileTaskResultContent content = result.getContent(); + if (content != null && content.getStatus() != null) { + sb.append(" Status: ").append(content.getStatus()); + if (StringUtils.isNotBlank(content.getStatusInfo())) { + sb.append(", StatusDesc: ").append(content.getStatusInfo()); } - // 记录目标IP单个文件下载任务的结束状态 - addFinishedFile(false, true, targetAgentId, - GSEFileTaskResult.buildTaskId(taskResult.getMode(), taskResult.getSourceAgentId(), - taskResult.getStandardSourceFilePath(), targetAgentId, destFilePath)); - // 每个目标IP增加一条下载失败的日志到日志总Map中 - addFileTaskLog(executionLogs, - new ServiceFileTaskLogDTO( - FileDistModeEnum.DOWNLOAD.getValue(), - targetHost.getHostId(), - targetHost.toCloudIp(), - taskResult.getStandardDestFilePath(), - sourceHost.getHostId(), - sourceHost.getIp(), - sourceHost.getDisplayIp(), - taskResult.getStandardSourceFilePath(), - taskResult.getStandardSourceFilePath() == null ? null : - sourceFileDisplayMap.get(taskResult.getStandardSourceFilePath()), - null, - FileDistStatusEnum.FAILED.getValue(), - FileDistStatusEnum.FAILED.getName(), - null, - null, - copyFileRsp.getFinalErrorMsg()) - ); - affectedTargetAgentIds.add(targetAgentId); } + return sb.toString(); } + /** - * 根据errorCode、fileNum、successNum更新successAgentIds状态集合与agentTask状态 + * 分析并设置执行对象任务的状态 * - * @param errorCode GSE错误码 - * @param agentId agentId - * @param fileNum 文件总数 - * @param successNum 成功分发的文件总数 - * @param isDownload 是否为下载结果 - * @param agentTask Agent任务 + * @param errorCode GSE错误码 + * @param executeObjectGseKey executeObjectGseKey + * @param fileNum 文件总数 + * @param successNum 成功分发的文件总数 + * @param isDownload 是否为下载结果 + * @param executeObjectTask 执行对象任务 */ - private void analyseAgentStatus(int errorCode, - String agentId, - int fileNum, - int successNum, - boolean isDownload, - AgentTaskDTO agentTask) { + private void analyseExecuteObjectTaskStatus(int errorCode, + ExecuteObjectGseKey executeObjectGseKey, + int fileNum, + int successNum, + boolean isDownload, + ExecuteObjectTask executeObjectTask) { + // 文件任务成功数=任务总数 if (successNum >= fileNum) { - // 每个文件都处理完了，才算任务完成执行 - agentTask.setStatus(AgentTaskStatusEnum.SUCCESS); - if (isDownload) { - this.successTargetAgentIds.add(agentId); + if (existNoExecutableSourceExecuteObject) { + // 如果包含了非法的源文件主机，即使GSE任务（已过滤非法主机)执行成功，那么对于这个主机来说，整体上任务状态是失败 + executeObjectTask.setStatus(ExecuteObjectTaskStatusEnum.FAILED); + } else { + executeObjectTask.setStatus(ExecuteObjectTaskStatusEnum.SUCCESS); + if (isDownload) { + this.successTargetExecuteObjectGseKeys.add(executeObjectGseKey); + } } } else { - AgentTaskStatusEnum agentTaskStatus = AgentTaskStatusEnum.FAILED; + ExecuteObjectTaskStatusEnum executeObjectTaskStatus = ExecuteObjectTaskStatusEnum.FAILED; if (errorCode != 0) { - agentTaskStatus = GseUtils.getStatusByGseErrorCode(errorCode); + executeObjectTaskStatus = GseUtils.getStatusByGseErrorCode(errorCode); } - agentTask.setStatus(agentTaskStatus); + executeObjectTask.setStatus(executeObjectTaskStatus); } } /** - * 分析结果，更新successAgentIds、notStartedAgentIds等状态集合，用于判定最终整体任务状态 + * 分析并判定最终整体任务状态 * - * @param errorCode GSE错误码 - * @param agentId agentId - * @param startTime 任务起始时间 - * @param endTime 任务终止时间 - * @param isDownloadResult 是否为下载结果 + * @param errorCode GSE错误码 + * @param executeObjectGseKey executeObjectGseKey + * @param startTime 任务起始时间 + * @param endTime 任务终止时间 + * @param isDownloadResult 是否为下载结果 */ - private void analyseAgentTaskResult(int errorCode, String agentId, long startTime, long endTime, - boolean isDownloadResult) { + private void analyseExecuteObjectTaskResult(int errorCode, + ExecuteObjectGseKey executeObjectGseKey, + long startTime, + long endTime, + boolean isDownloadResult) { int finishedNum; int fileNum; int successNum; if (isDownloadResult) { - finishedNum = this.finishedDownloadFileMap.get(agentId) == null ? 0 : - this.finishedDownloadFileMap.get(agentId).size(); - fileNum = this.fileDownloadTaskNumMap.get(agentId) == null ? 0 : this.fileDownloadTaskNumMap.get(agentId); - successNum = this.successDownloadFileMap.get(agentId) == null ? 0 : - this.successDownloadFileMap.get(agentId).size(); + finishedNum = this.finishedDownloadFileMap.get(executeObjectGseKey) == null ? 0 : + this.finishedDownloadFileMap.get(executeObjectGseKey).size(); + fileNum = this.fileDownloadTaskNumMap.get(executeObjectGseKey) == null ? 0 : + this.fileDownloadTaskNumMap.get(executeObjectGseKey); + successNum = this.successDownloadFileMap.get(executeObjectGseKey) == null ? 0 : + this.successDownloadFileMap.get(executeObjectGseKey).size(); } else { - finishedNum = this.finishedUploadFileMap.get(agentId) == null ? 0 : - this.finishedUploadFileMap.get(agentId).size(); - successNum = this.successUploadFileMap.get(agentId) == null ? 0 : - this.successUploadFileMap.get(agentId).size(); - fileNum = this.fileUploadTaskNumMap.get(agentId) == null ? 0 : this.fileUploadTaskNumMap.get(agentId); + finishedNum = this.finishedUploadFileMap.get(executeObjectGseKey) == null ? 0 : + this.finishedUploadFileMap.get(executeObjectGseKey).size(); + successNum = this.successUploadFileMap.get(executeObjectGseKey) == null ? 0 : + this.successUploadFileMap.get(executeObjectGseKey).size(); + fileNum = this.fileUploadTaskNumMap.get(executeObjectGseKey) == null ? + 0 : this.fileUploadTaskNumMap.get(executeObjectGseKey); } - AgentTaskDTO agentTask = getAgentTask(isDownloadResult, agentId); + ExecuteObjectTask executeObjectTask = getExecuteObjectTask(isDownloadResult, executeObjectGseKey); if (finishedNum >= fileNum) { - log.info("[{}] Ip analyse finished! ip: {}, finishedTaskNum: {}, expectedTaskNum: {}", - stepInstanceId, agentId, finishedNum, fileNum); - // 更新AgentTask结果 + log.info("[{}] Analyse Agent task finished! executeObjectGseKey: {}, finishedTaskNum: {}, " + + "expectedTaskNum: {}", gseTaskInfo, executeObjectGseKey, finishedNum, fileNum); + // 更新执行对象任务结果 if (isDownloadResult) { - dealTargetAgentFinish(agentId, startTime, endTime, agentTask); + dealTargetExecuteObjectFinish(executeObjectGseKey, startTime, endTime, executeObjectTask); } else { - dealUploadAgentFinished(agentId, startTime, endTime, agentTask); + dealUploadAgentFinished(executeObjectGseKey, startTime, endTime, executeObjectTask); } - analyseAgentStatus(errorCode, agentId, fileNum, successNum, isDownloadResult, agentTask); + analyseExecuteObjectTaskStatus(errorCode, executeObjectGseKey, fileNum, successNum, isDownloadResult, + executeObjectTask); } else { - agentTask.setStatus(AgentTaskStatusEnum.RUNNING); - this.notStartedTargetAgentIds.remove(agentId); + executeObjectTask.setStatus(ExecuteObjectTaskStatusEnum.RUNNING); } } /** - * 设置源agent任务结束状态 + * 设置源执行对象任务结束状态 * - * @param agentId agentId - * @param startTime 起始时间 - * @param endTime 终止时间 - * @param agentTask 日志 + * @param executeObjectGseKey executeObjectGseKey + * @param startTime 起始时间 + * @param endTime 终止时间 + * @param executeObjectTask 执行对象任务 */ - private void dealUploadAgentFinished(String agentId, Long startTime, Long endTime, AgentTaskDTO agentTask) { - log.info("[{}]: Deal source agent finished| agentId={}| startTime:{}, endTime:{}, agentTask:{}", - stepInstanceId, agentId, startTime, endTime, JsonUtils.toJsonWithoutSkippedFields(agentTask)); - - this.runningFileSourceAgentIds.remove(agentId); - this.notStartedFileSourceAgentIds.remove(agentId); - this.analyseFinishedSourceAgentIds.add(agentId); + private void dealUploadAgentFinished(ExecuteObjectGseKey executeObjectGseKey, + Long startTime, + Long endTime, + ExecuteObjectTask executeObjectTask) { + log.info("[{}]: Deal source agent finished| executeObjectGseKey={}| startTime:{}, endTime:{}, " + + "executeObjectTask:{}", + gseTaskInfo, executeObjectGseKey, startTime, endTime, + JsonUtils.toJsonWithoutSkippedFields(executeObjectTask)); + + this.notFinishedSourceExecuteObjectGseKeys.remove(executeObjectGseKey); + this.analyseFinishedSourceExecuteObjectGseKeys.add(executeObjectGseKey); if (endTime - startTime <= 0) { - agentTask.setTotalTime(100L); + executeObjectTask.setTotalTime(100L); } else { - agentTask.setTotalTime(endTime - startTime); + executeObjectTask.setTotalTime(endTime - startTime); } - agentTask.setStartTime(startTime); - agentTask.setEndTime(endTime); + executeObjectTask.setStartTime(startTime); + executeObjectTask.setEndTime(endTime); } - /* + /** * 从执行结果生成执行日志 */ - private void parseExecutionLog(CopyFileRsp copyFileRsp, Map executionLogs) { - GSEFileTaskResult taskResult = copyFileRsp.getGseFileTaskResult(); - if (null == taskResult) { - log.warn("gseFileTaskResult is null, please check field [content] of GSE resp"); - return; - } - Integer mode = taskResult.getMode(); + private void parseExecutionLog(JobAtomicFileTaskResult result, + Map executionLogs) { + AtomicFileTaskResultContent content = result.getResult().getContent(); + Integer mode = content.getMode(); + JobFile srcFile = result.getSrcFile(); boolean isDownloadResult = isDownloadResult(mode); - GSECode.AtomicErrorCode errorCode = GSECode.AtomicErrorCode.getErrorCode(copyFileRsp.getFinalErrorCode()); - String key = taskResult.getTaskId(); + FileDistStatusEnum status = parseFileTaskStatus(result.getResult(), isDownloadResult); + GSECode.AtomicErrorCode errorCode = GSECode.AtomicErrorCode.getErrorCode(result.getResult().getErrorCode()); + String key = content.getTaskId(); Integer process = processMap.computeIfAbsent(key, k -> -1); - if (errorCode == GSECode.AtomicErrorCode.RUNNING && process.equals(taskResult.getProcess())) { + if (errorCode == GSECode.AtomicErrorCode.RUNNING && process.equals(content.getProgress())) { return; } - processMap.put(key, taskResult.getProcess()); + // 由于GSE Bug,当状态为Finished的时候，偶现progress不为100的情况；为了避免用户理解歧义，统一以任务状态为准 + processMap.put(key, status == FileDistStatusEnum.FINISHED ? 100 : content.getProgress()); - StringBuilder logContent = new StringBuilder(); + String displayFilePath = isDownloadResult ? content.getStandardDestFilePath() : + srcFile.getDisplayFilePath(); + String fileSize = "--"; + String speed = ""; + String progressText = ""; - String filePath = isDownloadResult ? taskResult.getStandardDestFilePath() : - taskResult.getStandardSourceFilePath(); - String displayFilePath = buildDisplayFilePath(isDownloadResult, filePath); - if (filePath.endsWith("/") || filePath.endsWith("\\")) { + if (content.getSize() != null && content.getSize() > 0) { + fileSize = GseUtils.tranByteReadable(content.getSize()); + } + if (content.getSpeed() != null) { + speed = formatSpeed(content.getSpeed()) + " KB/s"; + } + if (content.getProgress() != null) { + progressText = content.getProgress() + "%"; + } + String logContentStr = buildFileLogContent(displayFilePath, fileSize, content.getStatus(), + content.getStatusInfo(), speed, progressText, result.getResult().getErrorMsg()); + + if (isDownloadResult) { + logService.addFileTaskLog( + stepInstance, + executionLogs, + result.getTargetExecuteObject(), + logService.buildDownloadServiceFileTaskLogDTO( + stepInstance, srcFile, result.getTargetExecuteObject(), content.getStandardDestFilePath(), status, + fileSize, speed, progressText, logContentStr)); + } else { + logService.addFileTaskLog( + stepInstance, + executionLogs, + result.getSourceExecuteObject(), + logService.buildUploadServiceFileTaskLogDTO( + stepInstance, srcFile, status, fileSize, speed, progressText, logContentStr)); + } + } + + private String buildFileLogContent(String displayFilePath, + String fileSize, + Integer status, + String statusDesc, + String speed, + String progress, + String detail) { + StringBuilder logContent = new StringBuilder(512); + if (displayFilePath.endsWith("/") || displayFilePath.endsWith("\\")) { // 传输的是目录，目录名以‘/’或‘\’结束 logContent.append("Directory: "); } else { @@ -917,143 +902,37 @@ private void parseExecutionLog(CopyFileRsp copyFileRsp, Map 0) { - // 兼容GSE不返回size的情况 - fileSize = GseUtils.tranByteReadable(taskResult.getSize()); + if (StringUtils.isNotBlank(fileSize)) { logContent.append(" FileSize: ").append(fileSize); } - if (taskResult.getSpeed() != null) { - speed = formatSpeed(taskResult.getSpeed()) + " KB/s"; - logContent.append(" Speed: ").append(speed); - } - if (taskResult.getProcess() != null) { - processText = taskResult.getProcess() + "%"; - logContent.append(" Progress: ").append(processText); + if (status != null) { + logContent.append(" Status: ").append(status); } - if (StringUtils.isNotEmpty(taskResult.getStatusDesc())) { - logContent.append(" StatusDesc: ").append(taskResult.getStatusDesc()); + if (StringUtils.isNotBlank(statusDesc)) { + logContent.append(" StatusDesc: ").append(statusDesc); } - if (StringUtils.isNotBlank(copyFileRsp.getFinalErrorMsg())) { - logContent.append(" Detail: ").append(copyFileRsp.getFinalErrorMsg()); + if (StringUtils.isNotBlank(speed)) { + logContent.append(" Speed: ").append(speed); } - String logContentStr = logContent.toString(); - - FileDistStatusEnum status = parseFileTaskStatus(copyFileRsp, isDownloadResult); - - HostDTO sourceHost = agentIdHostMap.get(taskResult.getSourceAgentId()); - if (isDownloadResult) { - HostDTO targetHost = agentIdHostMap.get(taskResult.getDestAgentId()); - ServiceFileTaskLogDTO downloadLog = initFileTaskLog(fileSize, status, speed, processText, logContentStr); - fillDownloadFileTaskLog(downloadLog, sourceHost, targetHost, taskResult); - addFileTaskLog(executionLogs, downloadLog); - } else { - ServiceFileTaskLogDTO uploadLog = initFileTaskLog(fileSize, status, speed, processText, logContentStr); - fillUploadFileTaskLog(uploadLog, sourceHost, filePath, displayFilePath); - addFileTaskLog(executionLogs, uploadLog); + if (StringUtils.isNotBlank(speed)) { + logContent.append(" Progress: ").append(progress); } - } - - /** - * 根据部分字段构造初始文件任务日志对象 - * - * @param fileSize 文件大小 - * @param status 任务状态 - * @param speed 传输速度 - * @param processText 进度信息 - * @param logContentStr 详细日志内容 - * @return 文件任务日志对象 - */ - private ServiceFileTaskLogDTO initFileTaskLog(String fileSize, - FileDistStatusEnum status, - String speed, - String processText, - String logContentStr) { - ServiceFileTaskLogDTO fileTaskLog = new ServiceFileTaskLogDTO(); - fileTaskLog.setSize(fileSize); - fileTaskLog.setStatus(status.getValue()); - fileTaskLog.setStatusDesc(status.getName()); - fileTaskLog.setSpeed(speed); - fileTaskLog.setProcess(processText); - fileTaskLog.setContent(logContentStr); - return fileTaskLog; - } - - /** - * 为下载类型的文件任务日志填充源、目标等相关信息 - * - * @param fileTaskLog 文件任务日志对象 - * @param sourceHost 源主机 - * @param targetHost 目标主机 - * @param taskResult 任务执行结果 - */ - private void fillDownloadFileTaskLog(ServiceFileTaskLogDTO fileTaskLog, - HostDTO sourceHost, - HostDTO targetHost, - GSEFileTaskResult taskResult) { - fileTaskLog.setMode(FileDistModeEnum.DOWNLOAD.getValue()); - fileTaskLog.setDestHostId(targetHost.getHostId()); - fileTaskLog.setDestIp(targetHost.toCloudIp()); - fileTaskLog.setDestFile(taskResult.getStandardDestFilePath()); - fileTaskLog.setSrcHostId(sourceHost.getHostId()); - fileTaskLog.setSrcIp(sourceHost.toCloudIp()); - fileTaskLog.setDisplaySrcIp(sourceHost.getDisplayIp()); - fileTaskLog.setSrcFile(taskResult.getStandardSourceFilePath()); - fileTaskLog.setDisplaySrcFile(taskResult.getStandardSourceFilePath() == null ? null : - sourceFileDisplayMap.get(taskResult.getStandardSourceFilePath())); - } - - /** - * 为上传类型的文件任务日志填充源相关信息 - * - * @param fileTaskLog 文件任务日志对象 - * @param sourceHost 源主机 - * @param srcFilePath 源文件路径 - * @param displaySrcFilePath 用于展示的源文件路径 - */ - private void fillUploadFileTaskLog(ServiceFileTaskLogDTO fileTaskLog, - HostDTO sourceHost, - String srcFilePath, - String displaySrcFilePath) { - fileTaskLog.setMode(FileDistModeEnum.UPLOAD.getValue()); - fileTaskLog.setSrcHostId(sourceHost.getHostId()); - fileTaskLog.setSrcIp(sourceHost.toCloudIp()); - fileTaskLog.setDisplaySrcIp(sourceHost.getDisplayIp()); - fileTaskLog.setSrcFile(srcFilePath); - fileTaskLog.setDisplaySrcFile(displaySrcFilePath); - } - - private String buildDisplayFilePath(boolean isDownloadResult, String originFilePath) { - String displayFilePath = originFilePath; - if (!isDownloadResult) { - displayFilePath = sourceFileDisplayMap.get(originFilePath); + if (StringUtils.isNotBlank(detail)) { + logContent.append(" Detail: ").append(detail); } - return displayFilePath; + return logContent.toString(); } - private FileDistStatusEnum parseFileTaskStatus(CopyFileRsp copyFileRsp, boolean isDownloadResult) { - GSEFileTaskResult taskResult = copyFileRsp.getGseFileTaskResult(); + private FileDistStatusEnum parseFileTaskStatus(AtomicFileTaskResult result, boolean isDownloadResult) { FileDistStatusEnum status; - if (copyFileRsp.getFinalErrorCode().equals(GSECode.AtomicErrorCode.RUNNING.getValue())) { + if (result.getErrorCode().equals(GSECode.AtomicErrorCode.RUNNING.getValue())) { if (isDownloadResult) { status = FileDistStatusEnum.DOWNLOADING; } else { status = FileDistStatusEnum.UPLOADING; } - } else if (copyFileRsp.getFinalErrorCode().equals(GSECode.AtomicErrorCode.FINISHED.getValue())) { - if (taskResult.getProcess() < 100) { - if (isDownloadResult) { - status = FileDistStatusEnum.DOWNLOADING; - } else { - status = FileDistStatusEnum.UPLOADING; - } - } else { - status = FileDistStatusEnum.FINISHED; - } - + } else if (result.getErrorCode().equals(GSECode.AtomicErrorCode.FINISHED.getValue())) { + status = FileDistStatusEnum.FINISHED; } else { status = FileDistStatusEnum.FAILED; } @@ -1069,58 +948,45 @@ private String formatSpeed(int speed) { return formatter.format(speed); } - - private void addFileTaskLog(Map hostLogs, ServiceFileTaskLogDTO fileTaskLog) { - boolean isDownloadResult = isDownloadResult(fileTaskLog.getMode()); - Long hostId = isDownloadResult ? fileTaskLog.getDestHostId() : fileTaskLog.getSrcHostId(); - String cloudIp = isDownloadResult ? fileTaskLog.getDestIp() : fileTaskLog.getSrcIp(); - ServiceHostLogDTO hostLog = hostLogs.get(hostId); - if (hostLog == null) { - hostLog = new ServiceHostLogDTO(); - hostLog.setStepInstanceId(stepInstanceId); - hostLog.setHostId(hostId); - hostLog.setIp(cloudIp); - hostLog.setBatch(stepInstance.getBatch()); - hostLog.setExecuteCount(stepInstance.getExecuteCount()); - hostLogs.put(hostId, hostLog); - } - hostLog.addFileTaskLog(fileTaskLog); - } - - private void writeFileTaskLogContent(Map executionLogs) { + private void writeFileTaskLogContent(Map executionLogs) { if (!executionLogs.isEmpty()) { - logService.writeFileLogs(taskInstance.getCreateTime(), new ArrayList<>(executionLogs.values())); + logService.writeFileLogs(taskInstance, new ArrayList<>(executionLogs.values())); } } /** * 向某个IP上传/下载文件的已结束状态Map、已成功状态Map中添加记录 * - * @param isSuccess 单个文件上传/下载任务是否成功 - * @param isDownloadMode 是否为下载 - * @param agentId bk_agent_id - * @param taskId 单个文件任务唯一Key - * @return 是否添加成功 + * @param isSuccess 单个文件上传/下载任务是否成功 + * @param isDownloadMode 是否为下载 + * @param executeObjectGseKey executeObjectGseKey + * @param taskId 单个文件任务唯一Key */ - private boolean addFinishedFile(boolean isSuccess, boolean isDownloadMode, String agentId, String taskId) { + private void addFinishedFile(boolean isSuccess, + boolean isDownloadMode, + ExecuteObjectGseKey executeObjectGseKey, + String taskId) { if (isDownloadMode) { - return addFinishedFile(isSuccess, agentId, taskId, finishedDownloadFileMap, successDownloadFileMap); + addFinishedFile(isSuccess, executeObjectGseKey, taskId, finishedDownloadFileMap, successDownloadFileMap); } else { - return addFinishedFile(isSuccess, agentId, taskId, finishedUploadFileMap, successUploadFileMap); + addFinishedFile(isSuccess, executeObjectGseKey, taskId, finishedUploadFileMap, successUploadFileMap); } } - private boolean addFinishedFile(boolean isSuccess, String agentId, String taskId, - Map> finishedFileMap, - Map> successFileMap) { - Set finishedFileSet = finishedFileMap.computeIfAbsent(agentId, k -> Sets.newHashSet()); - boolean isAdd = finishedFileSet.add(taskId); + private void addFinishedFile(boolean isSuccess, + ExecuteObjectGseKey executeObjectGseKey, + String taskId, + Map> finishedFileMap, + Map> successFileMap) { + Set finishedFileSet = finishedFileMap.computeIfAbsent( + executeObjectGseKey, k -> Sets.newHashSet()); + finishedFileSet.add(taskId); if (isSuccess) { - Set successFileSet = successFileMap.computeIfAbsent(agentId, k -> Sets.newHashSet()); + Set successFileSet = + successFileMap.computeIfAbsent(executeObjectGseKey, k -> Sets.newHashSet()); successFileSet.add(taskId); } - return isAdd; } @Override @@ -1143,4 +1009,42 @@ public String toString() { } return this.taskInfo; } + + @Getter + @Setter + @ToString + private static class JobAtomicFileTaskResult { + /** + * GSE 文件任务结果 + */ + private AtomicFileTaskResult result; + /** + * 源执行对象 + */ + private ExecuteObject sourceExecuteObject; + /** + * 目标执行对象 + */ + private ExecuteObject targetExecuteObject; + /** + * 源文件 + */ + private JobFile srcFile; + /** + * Agent 任务 + */ + private ExecuteObjectTask executeObjectTask; + + public JobAtomicFileTaskResult(AtomicFileTaskResult result, + ExecuteObject sourceExecuteObject, + ExecuteObject targetExecuteObject, + JobFile srcFile, + ExecuteObjectTask executeObjectTask) { + this.result = result; + this.sourceExecuteObject = sourceExecuteObject; + this.targetExecuteObject = targetExecuteObject; + this.srcFile = srcFile; + this.executeObjectTask = executeObjectTask; + } + } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/FileTaskResultHandleScheduleStrategy.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/FileTaskResultHandleScheduleStrategy.java index c3e4ae86b5..0aa340396f 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/FileTaskResultHandleScheduleStrategy.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/FileTaskResultHandleScheduleStrategy.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/Lifecycle.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/Lifecycle.java index 9dd47b8d76..064c72b62c 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/Lifecycle.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/Lifecycle.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ResultHandleManager.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ResultHandleManager.java index 109b27af2a..5c81d8d5ed 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ResultHandleManager.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ResultHandleManager.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,9 +24,11 @@ package com.tencent.bk.job.execute.engine.result; +import com.tencent.bk.job.execute.common.context.JobExecuteContextThreadLocalRepo; import com.tencent.bk.job.execute.common.exception.MessageHandlerUnavailableException; import com.tencent.bk.job.execute.common.ha.DestroyOrder; import com.tencent.bk.job.execute.config.JobExecuteConfig; +import com.tencent.bk.job.execute.engine.quota.limit.RunningJobKeepaliveManager; import com.tencent.bk.job.execute.engine.result.ha.ResultHandleLimiter; import com.tencent.bk.job.execute.engine.result.ha.ResultHandleTaskKeepaliveManager; import com.tencent.bk.job.execute.monitor.metrics.ExecuteMonitor; @@ -114,7 +116,7 @@ public class ResultHandleManager implements SmartLifecycle { /** * 新增消费者线程最小间隔时间 */ - private volatile long startConsumerMinInterval = 10000; + private final long startConsumerMinInterval = 10000; /** * 最近一次worker停止时间 */ @@ -122,7 +124,7 @@ public class ResultHandleManager implements SmartLifecycle { /** * 停止消费者线程最小间隔时间 */ - private volatile long stopConsumerMinInterval = 60000; + private final long stopConsumerMinInterval = 60000; /** * 任务结果处理引擎是否活动状态 */ @@ -133,18 +135,23 @@ public class ResultHandleManager implements SmartLifecycle { private volatile boolean running = false; private final ExecutorService shutdownExecutor; + private final RunningJobKeepaliveManager runningJobKeepaliveManager; + @Autowired - public ResultHandleManager(Tracer tracer, ExecuteMonitor counters, + public ResultHandleManager(Tracer tracer, + ExecuteMonitor counters, ResultHandleTaskKeepaliveManager resultHandleTaskKeepaliveManager, ResultHandleTaskSampler resultHandleTaskSampler, JobExecuteConfig jobExecuteConfig, - @Qualifier("shutdownExecutor") ExecutorService shutdownExecutor) { + @Qualifier("shutdownExecutor") ExecutorService shutdownExecutor, + RunningJobKeepaliveManager runningJobKeepaliveManager) { this.tracer = tracer; this.counters = counters; this.resultHandleTaskKeepaliveManager = resultHandleTaskKeepaliveManager; this.resultHandleTaskSampler = resultHandleTaskSampler; this.resultHandleLimiter = new ResultHandleLimiter(jobExecuteConfig.getResultHandleTasksLimit()); this.shutdownExecutor = shutdownExecutor; + this.runningJobKeepaliveManager = runningJobKeepaliveManager; } /** @@ -156,8 +163,16 @@ public void handleDeliveredTask(ContinuousScheduledTask task) { resultHandleLimiter.acquire(); log.info("Handle delivered task: {}", task); ScheduledContinuousResultHandleTask scheduleTask = - new ScheduledContinuousResultHandleTask(resultHandleTaskSampler, tracer, task, this, - resultHandleTaskKeepaliveManager, resultHandleLimiter); + new ScheduledContinuousResultHandleTask( + resultHandleTaskSampler, + tracer, + task, + this, + resultHandleTaskKeepaliveManager, + resultHandleLimiter, + runningJobKeepaliveManager, + JobExecuteContextThreadLocalRepo.get() + ); synchronized (lifecycleMonitor) { if (!isActive()) { log.warn("ResultHandleManager is not active, reject! task: {}", task); @@ -169,6 +184,7 @@ public void handleDeliveredTask(ContinuousScheduledTask task) { if (task instanceof AbstractResultHandleTask) { resultHandleTaskKeepaliveManager.addRunningTaskKeepaliveInfo(task.getTaskId()); } + runningJobKeepaliveManager.addKeepaliveTask(task.getTaskContext().getJobInstanceId()); this.tasksQueue.add(scheduleTask); if (task instanceof ScriptResultHandleTask) { ScriptResultHandleTask scriptResultHandleTask = (ScriptResultHandleTask) task; @@ -260,10 +276,11 @@ public void stop() { private void stopTasksGraceful() { log.info("Stop tasks graceful - start"); long start = System.currentTimeMillis(); - StopTaskCounter stopTaskCounter = StopTaskCounter.getInstance(); + StopTaskCounter stopTaskCounter = null; synchronized (lifecycleMonitor) { if (!this.scheduledTasks.isEmpty()) { log.info("Stop result handle tasks, size: {}, tasks: {}", scheduledTasks.size(), scheduledTasks); + stopTaskCounter = StopTaskCounter.getInstance(); stopTaskCounter.initCounter(scheduledTasks.keySet()); } for (ScheduledContinuousResultHandleTask task : scheduledTasks.values()) { @@ -271,7 +288,9 @@ private void stopTasksGraceful() { } } try { - stopTaskCounter.waitingForAllTasksDone(); + if (stopTaskCounter != null) { + stopTaskCounter.waitingForAllTasksDone(); + } } catch (Throwable e) { log.error("Stop tasks caught exception", e); } @@ -374,19 +393,22 @@ private static final class StopTask implements Runnable { @Override public void run() { - Span span = null; - try { - span = tracer1.nextSpan(task.getTraceContext()).name("stop-task"); + Span span = tracer1.nextSpan(task.getTraceContext()).name("stop-task"); + + try (Tracer.SpanInScope ignored = tracer1.withSpan(span.start())) { + JobExecuteContextThreadLocalRepo.set(task.getJobExecuteContext()); log.info("Begin to stop task, task: {}", task.getResultHandleTask()); task.getResultHandleTask().stop(); log.info("Stop task successfully, task: {}", task.getResultHandleTask()); } catch (Throwable e) { - String errorMsg = "Stop task caught exception, task: {}" + task; + span.error(e); + String errorMsg = "Stop task caught exception, task: " + task; log.warn(errorMsg, e); } finally { if (span != null) { span.end(); } + JobExecuteContextThreadLocalRepo.unset(); } } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ResultHandleTaskSampler.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ResultHandleTaskSampler.java index 23f8fb8b8d..1358570965 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ResultHandleTaskSampler.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ResultHandleTaskSampler.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ScheduleStrategy.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ScheduleStrategy.java index f9f4848952..0a100edf4d 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ScheduleStrategy.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ScheduleStrategy.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ScheduledContinuousResultHandleTask.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ScheduledContinuousResultHandleTask.java index 3bc592d455..e7a9001dc6 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ScheduledContinuousResultHandleTask.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ScheduledContinuousResultHandleTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,9 +24,13 @@ package com.tencent.bk.job.execute.engine.result; +import com.tencent.bk.job.execute.common.context.JobExecuteContext; +import com.tencent.bk.job.execute.common.context.JobExecuteContextThreadLocalRepo; +import com.tencent.bk.job.execute.engine.quota.limit.RunningJobKeepaliveManager; import com.tencent.bk.job.execute.engine.result.ha.ResultHandleLimiter; import com.tencent.bk.job.execute.engine.result.ha.ResultHandleTaskKeepaliveManager; import com.tencent.bk.job.execute.monitor.ExecuteMetricNames; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.springframework.cloud.sleuth.Span; import org.springframework.cloud.sleuth.Tracer; @@ -70,20 +74,33 @@ public class ScheduledContinuousResultHandleTask extends DelayedTask { */ private final Span parent; + private final RunningJobKeepaliveManager runningJobKeepaliveManager; + /** + * 作业执行上下文信息 + */ + @Getter + private final JobExecuteContext jobExecuteContext; + /** * ScheduledContinuousQueuedTask Constructor * - * @param sampler 采样器 - * @param tracer 日志调用链 - * @param task 任务 - * @param resultHandleManager resultHandleManager - * @param resultHandleLimiter 限流 + * @param sampler 采样器 + * @param tracer 日志调用链 + * @param task 任务 + * @param resultHandleManager resultHandleManager + * @param resultHandleLimiter 限流 + * @param resultHandleTaskKeepaliveManager resultHandleTaskKeepaliveManager + * @param runningJobKeepaliveManager runningJobKeepaliveManager + * @param jobExecuteContext 作业执行上下文 */ public ScheduledContinuousResultHandleTask(ResultHandleTaskSampler sampler, - Tracer tracer, ContinuousScheduledTask task, + Tracer tracer, + ContinuousScheduledTask task, ResultHandleManager resultHandleManager, ResultHandleTaskKeepaliveManager resultHandleTaskKeepaliveManager, - ResultHandleLimiter resultHandleLimiter) { + ResultHandleLimiter resultHandleLimiter, + RunningJobKeepaliveManager runningJobKeepaliveManager, + JobExecuteContext jobExecuteContext) { this.sampler = sampler; this.tracer = tracer; this.parent = tracer.currentSpan(); @@ -93,6 +110,8 @@ public ScheduledContinuousResultHandleTask(ResultHandleTaskSampler sampler, this.tasksQueue = resultHandleManager.getTasksQueue(); this.resultHandleTaskKeepaliveManager = resultHandleTaskKeepaliveManager; this.resultHandleLimiter = resultHandleLimiter; + this.runningJobKeepaliveManager = runningJobKeepaliveManager; + this.jobExecuteContext = jobExecuteContext; } private Span getChildSpan() { @@ -103,11 +122,13 @@ private Span getChildSpan() { public void execute() { Span span = getChildSpan(); try (Tracer.SpanInScope ignored = this.tracer.withSpan(span.start())) { + JobExecuteContextThreadLocalRepo.set(this.jobExecuteContext); doExecute(); } catch (Exception e) { span.error(e); throw e; } finally { + JobExecuteContextThreadLocalRepo.unset(); span.end(); } } @@ -142,8 +163,9 @@ public void doExecute() { } else { isDone = true; } - } catch (Exception | Error e) { + } catch (Throwable e) { status = "error"; + isDone = true; throw e; } finally { if (isDone) { @@ -156,12 +178,13 @@ public void doExecute() { resultHandleTaskKeepaliveManager.stopKeepaliveInfoTask(fileTask.getTaskId()); sampler.decrementFileTask(fileTask.getAppId()); } + runningJobKeepaliveManager.stopKeepaliveTask(task.getTaskContext().getJobInstanceId()); resultHandleManager.getScheduledTasks().remove(task.getTaskId()); } if (isExecutable) { long end = System.nanoTime(); sampler.getMeterRegistry().timer(ExecuteMetricNames.RESULT_HANDLE_TASK_SCHEDULE_PREFIX, - "task_type", task.getTaskType(), "status", status) + "task_type", task.getTaskType(), "status", status) .record(end - start, TimeUnit.NANOSECONDS); } if (!isReScheduled) { diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ScriptResultHandleTask.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ScriptResultHandleTask.java index 28a4de5425..2d0979edf8 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ScriptResultHandleTask.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ScriptResultHandleTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,45 +24,39 @@ package com.tencent.bk.job.execute.engine.result; -import com.tencent.bk.gse.taskapi.api_agent_task_rst; -import com.tencent.bk.gse.taskapi.api_query_task_info_v2; -import com.tencent.bk.gse.taskapi.api_task_detail_result; +import com.tencent.bk.job.common.gse.constants.GSECode; +import com.tencent.bk.job.common.gse.v2.model.ExecuteObjectGseKey; +import com.tencent.bk.job.common.gse.v2.model.GetExecuteScriptResultRequest; +import com.tencent.bk.job.common.gse.v2.model.ScriptExecuteObjectTaskResult; +import com.tencent.bk.job.common.gse.v2.model.ScriptTaskResult; import com.tencent.bk.job.common.model.dto.HostDTO; -import com.tencent.bk.job.common.util.BatchUtil; +import com.tencent.bk.job.common.util.CollectionUtil; import com.tencent.bk.job.common.util.date.DateUtils; -import com.tencent.bk.job.execute.common.exception.ReadTimeoutException; +import com.tencent.bk.job.common.util.file.FileSizeUtil; +import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.execute.config.JobExecuteConfig; import com.tencent.bk.job.execute.constants.VariableValueTypeEnum; -import com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum; -import com.tencent.bk.job.execute.engine.consts.GSECode; -import com.tencent.bk.job.execute.engine.evict.TaskEvictPolicyExecutor; -import com.tencent.bk.job.execute.engine.gse.GseRequestPrinter; -import com.tencent.bk.job.execute.engine.gse.GseRequestUtils; -import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; -import com.tencent.bk.job.execute.engine.model.GseLog; +import com.tencent.bk.job.execute.engine.EngineDependentServiceHolder; +import com.tencent.bk.job.execute.engine.consts.ExecuteObjectTaskStatusEnum; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; import com.tencent.bk.job.execute.engine.model.GseLogBatchPullResult; import com.tencent.bk.job.execute.engine.model.GseTaskExecuteResult; +import com.tencent.bk.job.execute.engine.model.GseTaskResult; import com.tencent.bk.job.execute.engine.model.LogPullProgress; -import com.tencent.bk.job.execute.engine.model.ScriptTaskLog; +import com.tencent.bk.job.execute.engine.model.ScriptGseTaskResult; import com.tencent.bk.job.execute.engine.model.TaskVariableDTO; import com.tencent.bk.job.execute.engine.model.TaskVariablesAnalyzeResult; -import com.tencent.bk.job.execute.engine.result.ha.ResultHandleTaskKeepaliveManager; import com.tencent.bk.job.execute.engine.util.GseUtils; -import com.tencent.bk.job.execute.model.AgentTaskDTO; +import com.tencent.bk.job.execute.model.ExecuteObjectTask; import com.tencent.bk.job.execute.model.GseTaskDTO; import com.tencent.bk.job.execute.model.HostVariableValuesDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.model.StepInstanceVariableValuesDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.model.VariableValueDTO; -import com.tencent.bk.job.execute.service.GseTaskService; -import com.tencent.bk.job.execute.service.LogService; -import com.tencent.bk.job.execute.service.ScriptAgentTaskService; -import com.tencent.bk.job.execute.service.StepInstanceService; -import com.tencent.bk.job.execute.service.StepInstanceVariableValueService; -import com.tencent.bk.job.execute.service.TaskInstanceService; -import com.tencent.bk.job.execute.service.TaskInstanceVariableService; -import com.tencent.bk.job.logsvr.model.service.ServiceScriptLogDTO; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; +import com.tencent.bk.job.execute.service.ScriptExecuteObjectTaskService; +import com.tencent.bk.job.logsvr.model.service.ServiceExecuteObjectScriptLogDTO; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.util.StopWatch; @@ -71,11 +65,9 @@ import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; -import java.util.HashSet; import java.util.LinkedList; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Collectors; @@ -83,31 +75,31 @@ * 脚本任务执行结果处理 */ @Slf4j -public class ScriptResultHandleTask extends AbstractResultHandleTask { +public class ScriptResultHandleTask extends AbstractResultHandleTask { /** * GSE日志查询支持的每一批次的最大Agent数目 */ - private static final int MAX_BATCH_SIZE = 1000; + private static final int MAX_QUERY_AGENT_TASK_SIZE = 1000; /** - * GSE日志查询支持的每一批次的最小Agent数目 + * GSE日志查询支持的每一批次的脚本执行输出日志长度(单位byte) */ - private static final int MIN_BATCH_SIZE = 10; + private long maxQueryContentSizeLimit; /** - * 每次批次的Agent数目 - 优先选择列表 + * GSE 每个脚本执行原子任务对应的最大日志大小，5MB */ - private static final int[] BATCH_SIZE_PRIORITY_ARRAY = new int[]{MAX_BATCH_SIZE, 100, MIN_BATCH_SIZE}; + private static final int MAX_GSE_ATOMIC_TASK_CONTENT_BYTES = 5 * 1024 * 1024; /** - * 脚本任务执行日志进度, Map + * 脚本任务执行日志进度 */ - private final Map logPullProgressMap = new HashMap<>(); + private final Map logPullProgressMap = new HashMap<>(); /** - * 保存命名空间参数，Map> + * 保存命名空间参数，Map> */ - private final Map> namespaceParamValues = new HashMap<>(); + private final Map> namespaceParamValues = new HashMap<>(); /** - * 保存可变全局参数，Map> + * 保存可变全局参数，Map> */ - private final Map> changeableGlobalParamValues = new HashMap<>(); + private final Map> changeableGlobalParamValues = new HashMap<>(); /** * 脚本任务结果处理调度策略 */ @@ -115,11 +107,7 @@ public class ScriptResultHandleTask extends AbstractResultHandleTask> pullAgentIdBatches = new LinkedList<>(); - /** - * 当前使用的批次大小 - */ - private volatile int currentBatchSize = MAX_BATCH_SIZE; + private List> pullExecuteObjectGseKeyBatches = new LinkedList<>(); /** * 每一轮拉取的批次序号 */ @@ -131,171 +119,129 @@ public class ScriptResultHandleTask extends AbstractResultHandleTask agentTaskMap, + Map executeObjectTaskMap, GseTaskDTO gseTask, - String requestId) { - super(taskInstanceService, - gseTaskService, - logService, - taskInstanceVariableService, - stepInstanceVariableValueService, - taskExecuteMQEventDispatcher, - resultHandleTaskKeepaliveManager, - taskEvictPolicyExecutor, - scriptAgentTaskService, - stepInstanceService, + String requestId, + List executeObjectTasks) { + super(engineDependentServiceHolder, + scriptExecuteObjectTaskService, taskInstance, stepInstance, taskVariablesAnalyzeResult, - agentTaskMap, + executeObjectTaskMap, gseTask, - requestId); - initLogPullProcess(agentTaskMap.values()); + requestId, + executeObjectTasks); + this.maxQueryContentSizeLimit = + FileSizeUtil.parseFileSizeBytes(jobExecuteConfig.getScriptTaskQueryContentSizeLimit()); + initLogPullProcess(executeObjectTaskMap.values()); } - private void initLogPullProcess(Collection agentTasks) { - agentTasks.forEach(agentTask -> { + private void initLogPullProcess(Collection executeObjectTasks) { + executeObjectTasks.forEach(executeObjectTask -> { + ExecuteObjectGseKey executeObjectGseKey = + executeObjectTask.getExecuteObject().toExecuteObjectGseKey(); LogPullProgress process = new LogPullProgress(); - process.setAgentId(agentTask.getAgentId()); - process.setByteOffset(agentTask.getScriptLogOffset()); - process.setMid(0); - logPullProgressMap.put(agentTask.getAgentId(), process); + process.setExecuteObjectGseKey(executeObjectGseKey); + process.setByteOffset(executeObjectTask.getScriptLogOffset()); + process.setAtomicTaskId(0); + logPullProgressMap.put(executeObjectGseKey, process); }); } @Override - GseLogBatchPullResult pullGseTaskResultInBatches() { - if (pullAgentIdBatches.isEmpty()) { - Set queryAgentIds = new HashSet<>(); - queryAgentIds.addAll(notStartedTargetAgentIds); - queryAgentIds.addAll(runningTargetAgentIds); - List queryAgentIdList = new ArrayList<>(queryAgentIds); - pullAgentIdBatches = BatchUtil.buildBatchList(queryAgentIdList, currentBatchSize); + GseLogBatchPullResult pullGseTaskResultInBatches() { + if (pullExecuteObjectGseKeyBatches.isEmpty()) { + List queryExecuteObjectGseKeyList + = new ArrayList<>(notFinishedTargetExecuteObjectGseKeys); + pullExecuteObjectGseKeyBatches = CollectionUtil.partitionList( + queryExecuteObjectGseKeyList, MAX_QUERY_AGENT_TASK_SIZE); } - return tryPullGseResultWithRetry(); - } - - private GseLogBatchPullResult tryPullGseResultWithRetry() { - List pullLogAgentIds = pullAgentIdBatches.get(pullResultBatchesIndex.get() - 1); - try { - api_task_detail_result detailRst = pullGseTaskResult(pullLogAgentIds); - boolean isLastBatch = pullResultBatchesIndex.get() == pullAgentIdBatches.size(); - GseLogBatchPullResult batchPullResult = new GseLogBatchPullResult<>(true, - isLastBatch, new ScriptTaskLog(detailRst), null); - if (isLastBatch) { - resetBatch(); - } else { - pullResultBatchesIndex.incrementAndGet(); - } - return batchPullResult; - } catch (ReadTimeoutException e) { - boolean isSuccess = tryReduceBatchSizeAndRebuildBatchList(); - if (isSuccess) { - log.info("Reduce batch size and rebuild batch list successfully, currentBatchSize: {}, batches: {}. " + - "Retry pull!", - this.currentBatchSize, this.pullAgentIdBatches); - return tryPullGseResultWithRetry(); - } else { - log.warn("Try pull gse log with min batch size, but fail!"); - return new GseLogBatchPullResult<>(false, true, null, "Pull gse task log timeout"); - } + List pullLogExecuteObjectGseKeys + = pullExecuteObjectGseKeyBatches.get(pullResultBatchesIndex.get() - 1); + ScriptTaskResult result = pullGseTaskResult(pullLogExecuteObjectGseKeys); + boolean isLastBatch = pullResultBatchesIndex.get() == pullExecuteObjectGseKeyBatches.size(); + GseLogBatchPullResult batchPullResult = new GseLogBatchPullResult<>( + isLastBatch, new ScriptGseTaskResult(result)); + if (isLastBatch) { + resetBatch(); + } else { + pullResultBatchesIndex.incrementAndGet(); } + return batchPullResult; } private void resetBatch() { pullResultBatchesIndex = new AtomicInteger(1); - pullAgentIdBatches.clear(); - } - - private api_task_detail_result pullGseTaskResult(List agentIds) { - api_query_task_info_v2 requestV2 = GseRequestUtils.buildScriptLogRequestV2(gseTask.getGseTaskId(), agentIds, - logPullProgressMap); - return GseRequestUtils.getScriptTaskDetailRst(stepInstanceId, requestV2); - } - - private boolean tryReduceBatchSizeAndRebuildBatchList() { - log.warn("Caught ReadTimeoutException when pull gse log, try to reduce batch size, currentBatchSize: {}", - currentBatchSize); - if (currentBatchSize <= MIN_BATCH_SIZE) { - return false; - } - currentBatchSize = getBatchSizeLessThanCurrentSize(); - List> newBatchList = new ArrayList<>(); - List leftAgentIds = new ArrayList<>(); - if (pullResultBatchesIndex.get() > 1) { - List> pullFinishedBatchList = pullAgentIdBatches.subList(0, pullResultBatchesIndex.get() - 1); - newBatchList.addAll(pullFinishedBatchList); - } - pullAgentIdBatches.subList(pullResultBatchesIndex.get() - 1, pullAgentIdBatches.size()) - .forEach(leftAgentIds::addAll); - newBatchList.addAll(BatchUtil.buildBatchList(leftAgentIds, currentBatchSize)); - pullAgentIdBatches = newBatchList; - return true; + pullExecuteObjectGseKeyBatches.clear(); } - private int getBatchSizeLessThanCurrentSize() { - for (int batchSize : BATCH_SIZE_PRIORITY_ARRAY) { - if (batchSize < currentBatchSize) { - return batchSize; + private ScriptTaskResult pullGseTaskResult(List executeObjectGseKeys) { + GetExecuteScriptResultRequest request = new GetExecuteScriptResultRequest(); + request.setGseV2Task(gseV2Task); + request.setTaskId(gseTask.getGseTaskId()); + + int executeObjectSize = executeObjectGseKeys.size(); + long limit = maxQueryContentSizeLimit / executeObjectSize; + // 如果计算出来的 limit 值大于 GSE 本身的单任务输出内容大小限制，不需要传入 limit + boolean enableLimitContentRequestParam = limit < MAX_GSE_ATOMIC_TASK_CONTENT_BYTES; + + executeObjectGseKeys.forEach(executeObjectGseKey -> { + LogPullProgress progress = logPullProgressMap.get(executeObjectGseKey); + if (progress == null) { + request.addAgentTaskQuery(executeObjectGseKey, 0, 0, + enableLimitContentRequestParam ? limit : null); + } else { + request.addAgentTaskQuery(executeObjectGseKey, progress.getAtomicTaskId(), progress.getByteOffset(), + enableLimitContentRequestParam ? limit : null); } - } - return currentBatchSize; + }); + + return gseClient.getExecuteScriptResult(request); } @Override - GseTaskExecuteResult analyseGseTaskResult(GseLog taskDetail) { - + GseTaskExecuteResult analyseGseTaskResult(GseTaskResult taskDetail) { + if (taskDetail == null || taskDetail.isEmptyResult()) { + log.info("[{}] Analyse gse task result, result is empty!", gseTaskInfo); + return analyseExecuteResult(); + } long currentTime = DateUtils.currentTimeMillis(); // 当前时间 - List scriptLogs = new ArrayList<>(); + List scriptLogs = new ArrayList<>(); StopWatch watch = new StopWatch("analyse-gse-script-task"); watch.start("analyse"); - for (api_agent_task_rst agentTaskResult : taskDetail.getGseLog().getResult()) { - log.info("[{}]: agentTaskResult={}", stepInstanceId, - GseRequestPrinter.printAgentTaskResult(agentTaskResult)); - - /*为了解决shell上下文传参的问题，在下发用户脚本的时候，实际上下下发两个脚本。第一个脚本是用户脚本，第二个脚本 - *是获取上下文参数的脚本。所以m_id=0的是用户脚本的执行日志，需要分析记录；m_id=1的，则是获取上下文参数 - *输出的日志内容，不需要记录，仅需要从日志分析提取上下文参数*/ - boolean isUserScriptResult = agentTaskResult.getAtomic_task_id() == 0; - String agentId = agentTaskResult.getGse_composite_id() + ":" + agentTaskResult.getIp(); - - // 该Agent已经日志分析结束，不要再分析 - if (this.analyseFinishedTargetAgentIds.contains(agentId)) { + for (ScriptExecuteObjectTaskResult executeObjectTaskResult : taskDetail.getResult().getResult()) { + ExecuteObjectGseKey executeObjectGseKey = executeObjectTaskResult.getExecuteObjectGseKey(); + if (!shouldAnalyse(executeObjectTaskResult)) { continue; } - AgentTaskDTO agentTask = targetAgentTasks.get(agentId); - if (agentTask == null) { + + ExecuteObjectTask executeObjectTask = targetExecuteObjectTasks.get(executeObjectGseKey); + if (executeObjectTask == null) { + log.warn("[{}] No execute object task found for executeObjectGseKey {}. result: {}", + gseTaskInfo, executeObjectGseKey, JsonUtils.toJson(executeObjectTaskResult)); continue; } + log.info("[{}]: Analyse execute object task result, result: {}", gseTaskInfo, executeObjectTaskResult); + + /*为了解决shell上下文传参的问题，在下发用户脚本的时候，实际上下发两个脚本。第一个脚本是用户脚本，第二个脚本 + *是获取上下文参数的脚本。所以m_id=0的是用户脚本的执行日志，需要分析记录；m_id=1的，则是获取上下文参数 + *输出的日志内容，不需要记录，仅需要从日志分析提取上下文参数*/ + boolean isUserScriptResult = executeObjectTaskResult.getAtomicTaskId() == 0; if (isUserScriptResult) { - addScriptLogsAndRefreshPullProgress(scriptLogs, agentTaskResult, agentId, agentTask, currentTime); + addScriptLogsAndRefreshPullProgress(scriptLogs, executeObjectTaskResult, + executeObjectGseKey, executeObjectTask, currentTime); } - analyseAgentResult(agentTaskResult, agentTask, agentId, isUserScriptResult, currentTime); + analyseAgentResult(executeObjectTaskResult, executeObjectTask, executeObjectGseKey, isUserScriptResult, + currentTime); } watch.stop(); @@ -303,10 +249,15 @@ GseTaskExecuteResult analyseGseTaskResult(GseLog taskDet saveScriptLogContent(scriptLogs); watch.stop(); - watch.start("saveGseAgentTasks"); - batchSaveChangedGseAgentTasks(targetAgentTasks.values()); + watch.start("saveChangedExecuteObjectTasks"); + batchSaveChangedExecuteObjectTasks(targetExecuteObjectTasks.values()); watch.stop(); + log.info("[{}] Analyse gse task result -> notFinishedTargetExecuteObjectGseKeys={}" + + ", analyseFinishedTargetExecuteObjectGseKeys={}", + this.gseTaskInfo, this.notFinishedTargetExecuteObjectGseKeys, + this.analyseFinishedTargetExecuteObjectGseKeys); + GseTaskExecuteResult rst = analyseExecuteResult(); if (!rst.getResultCode().equals(GseTaskExecuteResult.RESULT_CODE_RUNNING)) { watch.start("saveVariables"); @@ -315,67 +266,95 @@ GseTaskExecuteResult analyseGseTaskResult(GseLog taskDet } if (watch.getTotalTimeMillis() > 1000L) { - log.info("Analyse script gse task is slow, statistics: {}", watch.prettyPrint()); + log.info("[{}] Analyse script gse task is slow, statistics: {}", gseTaskInfo, watch.prettyPrint()); } return rst; } - private void addScriptLogsAndRefreshPullProgress(List logs, api_agent_task_rst agentTaskResult, - String agentId, AgentTaskDTO agentTask, long currentTime) { - HostDTO host = agentIdHostMap.get(agentTask.getAgentId()); - if (GSECode.AtomicErrorCode.getErrorCode(agentTaskResult.getBk_error_code()) == GSECode.AtomicErrorCode.ERROR) { - logs.add(logService.buildSystemScriptLog(host, agentTaskResult.getBk_error_msg(), - agentTask.getScriptLogOffset(), - currentTime)); + private boolean shouldAnalyse(ScriptExecuteObjectTaskResult executeObjectTaskResult) { + ExecuteObjectGseKey executeObjectGseKey = executeObjectTaskResult.getExecuteObjectGseKey(); + // 该Agent已经日志分析结束，不要再分析 + if (this.analyseFinishedTargetExecuteObjectGseKeys.contains(executeObjectGseKey)) { + return false; + } + if (!this.targetExecuteObjectGseKeys.contains(executeObjectGseKey)) { + log.warn("[{}] Unexpected target executeObjectGseKey {}. result: {}", gseTaskInfo, executeObjectGseKey, + JsonUtils.toJson(executeObjectTaskResult)); + return false; + } + return true; + } + + private void addScriptLogsAndRefreshPullProgress(List logs, + ScriptExecuteObjectTaskResult executeObjectTaskResult, + ExecuteObjectGseKey executeObjectGseKey, + ExecuteObjectTask executeObjectTask, + long currentTime) { + ExecuteObject executeObject = findExecuteObject(executeObjectGseKey); + if (GSECode.AtomicErrorCode.getErrorCode(executeObjectTaskResult.getErrorCode()) + == GSECode.AtomicErrorCode.ERROR) { + if (StringUtils.isNotEmpty(executeObjectTaskResult.getErrorMsg())) { + logs.add(logService.buildSystemScriptLog(stepInstance, executeObject, + executeObjectTaskResult.getErrorMsg(), executeObjectTask.getScriptLogOffset(), currentTime)); + } } else { - String content = agentTaskResult.getScreen(); + String content = executeObjectTaskResult.getScreen(); if (StringUtils.isEmpty(content)) { return; } - int offset = agentTask.getScriptLogOffset(); + int offset = executeObjectTask.getScriptLogOffset(); + int contentSizeBytes = 0; if (StringUtils.isNotEmpty(content)) { - int bytes = content.getBytes(StandardCharsets.UTF_8).length; - offset += bytes; - agentTask.setScriptLogOffset(offset); + contentSizeBytes = content.getBytes(StandardCharsets.UTF_8).length; + offset += contentSizeBytes; + executeObjectTask.setScriptLogOffset(offset); } - logs.add(new ServiceScriptLogDTO(host, offset, agentTaskResult.getScreen())); + logs.add(logService.buildScriptLog(stepInstance, executeObject, + executeObjectTaskResult.getScreen(), contentSizeBytes, offset)); } // 刷新日志拉取偏移量 - refreshPullLogProgress(agentTaskResult.getScreen(), agentId, agentTaskResult.getAtomic_task_id()); + refreshPullLogProgress(executeObjectTaskResult.getScreen(), executeObjectGseKey, + executeObjectTaskResult.getAtomicTaskId()); } - private void saveScriptLogContent(List logs) { - logService.batchWriteScriptLog(taskInstance.getCreateTime(), stepInstanceId, stepInstance.getExecuteCount(), + private ExecuteObject findExecuteObject(ExecuteObjectGseKey executeObjectGseKey) { + return targetExecuteObjectTasks.get(executeObjectGseKey).getExecuteObject(); + } + + private void saveScriptLogContent(List logs) { + logService.batchWriteScriptLog(taskInstance, stepInstanceId, stepInstance.getExecuteCount(), stepInstance.getBatch(), logs); } - private void analyseAgentResult(api_agent_task_rst agentResult, AgentTaskDTO agentTask, String agentId, - boolean isUserScriptResult, long currentTime) { - boolean isShellScript = (stepInstance.getScriptType().equals(ScriptTypeEnum.SHELL.getValue())); - if (agentTask.getStartTime() == null) { - agentTask.setStartTime(currentTime); + private void analyseAgentResult(ScriptExecuteObjectTaskResult executeObjectTaskResult, + ExecuteObjectTask executeObjectTask, + ExecuteObjectGseKey executeObjectGseKey, + boolean isUserScriptResult, + long currentTime) { + boolean isShellScript = stepInstance.getScriptType() == ScriptTypeEnum.SHELL; + if (executeObjectTask.getStartTime() == null) { + executeObjectTask.setStartTime(currentTime); } - agentTask.setErrorCode(agentResult.getBk_error_code()); - if (GSECode.AtomicErrorCode.getErrorCode(agentResult.getBk_error_code()) == GSECode.AtomicErrorCode.ERROR) { + executeObjectTask.setErrorCode(executeObjectTaskResult.getErrorCode()); + if (GSECode.AtomicErrorCode.getErrorCode(executeObjectTaskResult.getErrorCode()) + == GSECode.AtomicErrorCode.ERROR) { // 脚本执行失败 - dealAgentFinish(agentId, agentResult, agentTask); - agentTask.setStatus(GseUtils.getStatusByGseErrorCode(agentResult.getBk_error_code())); - } else if (GSECode.AtomicErrorCode.getErrorCode(agentResult.getBk_error_code()) + dealExecuteObjectFinish(executeObjectGseKey, executeObjectTaskResult, executeObjectTask); + executeObjectTask.setStatus(GseUtils.getStatusByGseErrorCode(executeObjectTaskResult.getErrorCode())); + } else if (GSECode.AtomicErrorCode.getErrorCode(executeObjectTaskResult.getErrorCode()) == GSECode.AtomicErrorCode.TERMINATE) { - dealAgentFinish(agentId, agentResult, agentTask); - agentTask.setStatus(AgentTaskStatusEnum.GSE_TASK_TERMINATE_SUCCESS); + dealExecuteObjectFinish(executeObjectGseKey, executeObjectTaskResult, executeObjectTask); + executeObjectTask.setStatus(ExecuteObjectTaskStatusEnum.GSE_TASK_TERMINATE_SUCCESS); this.isTerminatedSuccess = true; } else { // 分析GSE的返回状态 - GSECode.Status status = GSECode.Status.getStatus(agentResult.getStatus()); + GSECode.Status status = GSECode.Status.getStatus(executeObjectTaskResult.getStatus()); switch (status) { case UNSTART: // 0：原子任务已派发； case RUNNING: // 1：原子任务执行中； - notStartedTargetAgentIds.remove(agentId); - runningTargetAgentIds.add(agentId); - agentTask.setStatus(AgentTaskStatusEnum.RUNNING); + executeObjectTask.setStatus(ExecuteObjectTaskStatusEnum.RUNNING); break; case SUCCESS: if (isShellScript && isUserScriptResult) { @@ -383,60 +362,59 @@ private void analyseAgentResult(api_agent_task_rst agentResult, AgentTaskDTO age && (taskVariablesAnalyzeResult.isExistChangeableGlobalVar() || taskVariablesAnalyzeResult.isExistNamespaceVar())) { //对于包含云参或者上下文参数的任务，下发任务的时候包含了2个任务；第一个是执行用户脚本；第二个获取参数的值 - agentTask.setStatus(AgentTaskStatusEnum.RUNNING); - notStartedTargetAgentIds.remove(agentId); - runningTargetAgentIds.add(agentId); - refreshPullLogProgress("", agentId, 1); + executeObjectTask.setStatus(ExecuteObjectTaskStatusEnum.RUNNING); + refreshPullLogProgress("", executeObjectGseKey, 1); } else { //普通任务，拉取日志，设置为成功 - dealAgentFinish(agentId, agentResult, agentTask); - agentTask.setStatus(AgentTaskStatusEnum.SUCCESS); - if (this.targetAgentIds.contains(agentId)) { - successTargetAgentIds.add(agentId); + dealExecuteObjectFinish(executeObjectGseKey, executeObjectTaskResult, executeObjectTask); + executeObjectTask.setStatus(ExecuteObjectTaskStatusEnum.SUCCESS); + if (this.targetExecuteObjectGseKeys.contains(executeObjectGseKey)) { + successTargetExecuteObjectGseKeys.add(executeObjectGseKey); } } } else { //获取输出参数的任务执行完成，需要分析日志 - dealAgentFinish(agentId, agentResult, agentTask); - agentTask.setStatus(AgentTaskStatusEnum.SUCCESS); - if (this.targetAgentIds.contains(agentId)) { - successTargetAgentIds.add(agentId); + dealExecuteObjectFinish(executeObjectGseKey, executeObjectTaskResult, executeObjectTask); + executeObjectTask.setStatus(ExecuteObjectTaskStatusEnum.SUCCESS); + if (this.targetExecuteObjectGseKeys.contains(executeObjectGseKey)) { + successTargetExecuteObjectGseKeys.add(executeObjectGseKey); } - parseVariableValueFromResult(agentResult, agentId); + parseVariableValueFromResult(executeObjectTaskResult, executeObjectGseKey); } if (isUserScriptResult) { - agentTask.setTag(agentResult.getTag()); + executeObjectTask.setTag(executeObjectTaskResult.getTag()); } break; case TIMEOUT: - dealAgentFinish(agentId, agentResult, agentTask); - agentTask.setStatus(AgentTaskStatusEnum.SCRIPT_TIMEOUT); + dealExecuteObjectFinish(executeObjectGseKey, executeObjectTaskResult, executeObjectTask); + executeObjectTask.setStatus(ExecuteObjectTaskStatusEnum.SCRIPT_TIMEOUT); break; case DISCARD: - dealAgentFinish(agentId, agentResult, agentTask); - agentTask.setStatus(AgentTaskStatusEnum.SCRIPT_TERMINATE); + dealExecuteObjectFinish(executeObjectGseKey, executeObjectTaskResult, executeObjectTask); + executeObjectTask.setStatus(ExecuteObjectTaskStatusEnum.SCRIPT_TERMINATE); break; default: - dealAgentFinish(agentId, agentResult, agentTask); - int errCode = agentResult.getBk_error_code(); - int exitCode = getExitCode(agentResult.getExitcode()); + dealExecuteObjectFinish(executeObjectGseKey, executeObjectTaskResult, executeObjectTask); + int errCode = executeObjectTaskResult.getErrorCode(); + int exitCode = executeObjectTaskResult.getExitCode(); if (errCode == 0) { if (exitCode != 0) { - agentTask.setStatus(AgentTaskStatusEnum.SCRIPT_NOT_ZERO_EXIT_CODE); + executeObjectTask.setStatus(ExecuteObjectTaskStatusEnum.SCRIPT_NOT_ZERO_EXIT_CODE); } else { - agentTask.setStatus(AgentTaskStatusEnum.SCRIPT_FAILED); + executeObjectTask.setStatus(ExecuteObjectTaskStatusEnum.SCRIPT_FAILED); } - agentTask.setTag(agentResult.getTag()); + executeObjectTask.setTag(executeObjectTaskResult.getTag()); } break; } } } - private void parseVariableValueFromResult(api_agent_task_rst agentTaskResult, String agentId) { - if (agentTaskResult.getAtomic_task_id() == 1 - && agentTaskResult.getStatus() == GSECode.Status.SUCCESS.getValue()) { - String paramsContent = agentTaskResult.getScreen(); + private void parseVariableValueFromResult(ScriptExecuteObjectTaskResult executeObjectTaskResult, + ExecuteObjectGseKey executeObjectGseKey) { + if (executeObjectTaskResult.getAtomicTaskId() == 1 + && executeObjectTaskResult.getStatus() == GSECode.Status.SUCCESS.getValue()) { + String paramsContent = executeObjectTaskResult.getScreen(); if (!StringUtils.isEmpty(paramsContent)) { String[] varKeyAndValueStrArray = paramsContent.split("\n"); for (String varKeyAndValueStr : varKeyAndValueStrArray) { @@ -455,12 +433,12 @@ private void parseVariableValueFromResult(api_agent_task_rst agentTaskResult, St continue; } if (taskVariablesAnalyzeResult.isNamespaceVar(paramName)) { - Map param2Values = namespaceParamValues.computeIfAbsent(agentId, - k -> new HashMap<>()); + Map param2Values = namespaceParamValues.computeIfAbsent( + executeObjectGseKey, k -> new HashMap<>()); param2Values.put(paramName, parseShellEscapeValue(paramValue)); } else if (taskVariablesAnalyzeResult.isChangeableGlobalVar(paramName)) { - Map param2Values = changeableGlobalParamValues.computeIfAbsent(agentId, - k -> new HashMap<>()); + Map param2Values = changeableGlobalParamValues.computeIfAbsent( + executeObjectGseKey, k -> new HashMap<>()); param2Values.put(paramName, parseShellEscapeValue(paramValue)); } } @@ -482,7 +460,7 @@ private String parseShellEscapeValue(String value) { } private void saveStepInstanceVariables() { - boolean mayBeReassigned = (stepInstance.getScriptType().equals(ScriptTypeEnum.SHELL.getValue())) + boolean mayBeReassigned = (stepInstance.getScriptType() == ScriptTypeEnum.SHELL) && (taskVariablesAnalyzeResult.isExistChangeableGlobalVar() || taskVariablesAnalyzeResult.isExistNamespaceVar()); @@ -512,7 +490,7 @@ private List buildChangeableGlobalVarValues() { return null; } List varValues = null; - for (Map.Entry> entry : changeableGlobalParamValues.entrySet()) { + for (Map.Entry> entry : changeableGlobalParamValues.entrySet()) { // 如果存在多个主机，那么随机选择其中一个主机上面的参数值 varValues = toVariableValuesList(entry.getValue()); } @@ -524,14 +502,18 @@ private List buildNamespaceVariableValues() { return null; } List hostVariableValuesList = new ArrayList<>(); - for (Map.Entry> entry : namespaceParamValues.entrySet()) { + for (Map.Entry> entry : namespaceParamValues.entrySet()) { HostVariableValuesDTO hostVariableValues = new HostVariableValuesDTO(); List paramValues = toVariableValuesList(entry.getValue()); - AgentTaskDTO agentTask = targetAgentTasks.get(entry.getKey()); - hostVariableValues.setHostId(agentTask.getHostId()); - hostVariableValues.setAgentId(entry.getKey()); + ExecuteObject executeObject = findExecuteObject((entry.getKey())); + if (!executeObject.isHostExecuteObject()) { + continue; + } + HostDTO host = executeObject.getHost(); + hostVariableValues.setHostId(host.getHostId()); + hostVariableValues.setCloudIpv4(host.toCloudIp()); + hostVariableValues.setCloudIpv6(host.toCloudIpv6()); hostVariableValues.setValues(paramValues); - hostVariableValues.setIp(agentTask.getCloudIp()); hostVariableValuesList.add(hostVariableValues); } return hostVariableValuesList; @@ -549,9 +531,12 @@ private List toVariableValuesList(Map varNameV return variableValues; } - private void dealAgentFinish(String agentId, api_agent_task_rst agentTaskResult, AgentTaskDTO agentTask) { - dealTargetAgentFinish(agentId, agentTaskResult.getStart_time(), agentTaskResult.getEnd_time(), agentTask); - agentTask.setExitCode(getExitCode(agentTaskResult.getExitcode())); + private void dealExecuteObjectFinish(ExecuteObjectGseKey executeObjectGseKey, + ScriptExecuteObjectTaskResult executeObjectTaskResult, + ExecuteObjectTask executeObjectTask) { + dealTargetExecuteObjectFinish(executeObjectGseKey, executeObjectTaskResult.getStartTime(), + executeObjectTaskResult.getEndTime(), executeObjectTask); + executeObjectTask.setExitCode(getExitCode(executeObjectTaskResult.getExitCode())); } private int getExitCode(int exitCode) { @@ -565,51 +550,41 @@ private int getExitCode(int exitCode) { /** * 刷新拉取日志进度 * - * @param context 日志内容 - * @param agentId bk_agent_id - * @param mid gse任务的m_id + * @param context 日志内容 + * @param executeObjectGseKey executeObjectGseKey + * @param mid gse任务的m_id */ - private void refreshPullLogProgress(String context, String agentId, int mid) { + private void refreshPullLogProgress(String context, ExecuteObjectGseKey executeObjectGseKey, int mid) { int increase = 0; if (null != context) { increase = context.getBytes(StandardCharsets.UTF_8).length; } - LogPullProgress progress = logPullProgressMap.get(agentId); + LogPullProgress progress = logPullProgressMap.get(executeObjectGseKey); if (null == progress) { progress = new LogPullProgress(); - logPullProgressMap.put(agentId, progress); + logPullProgressMap.put(executeObjectGseKey, progress); } - int prevMid = progress.getMid(); + int prevMid = progress.getAtomicTaskId(); if (prevMid != mid) { //重置offset progress.setByteOffset(0); } else { progress.setByteOffset(progress.getByteOffset() + increase); } - progress.setAgentId(agentId); - progress.setMid(mid); + progress.setExecuteObjectGseKey(executeObjectGseKey); + progress.setAtomicTaskId(mid); } - /* + /** * 分析执行结果 + * * @return 任务执行结果 */ private GseTaskExecuteResult analyseExecuteResult() { GseTaskExecuteResult rst; - if (this.notStartedTargetAgentIds.isEmpty() && this.runningTargetAgentIds.isEmpty()) { - int targetAgentNum = this.targetAgentIds.size(); - int successTargetAgentNum = this.successTargetAgentIds.size(); - boolean isSuccess = !stepInstance.hasInvalidHost() && successTargetAgentNum == targetAgentNum; - if (isSuccess) { - rst = GseTaskExecuteResult.SUCCESS; - } else { - if (this.isTerminatedSuccess) { - rst = GseTaskExecuteResult.STOP_SUCCESS; - } else { - rst = GseTaskExecuteResult.FAILED; - } - } + if (isAllTargetExecuteObjectTasksDone()) { + rst = analyseFinishedExecuteResult(); } else { rst = GseTaskExecuteResult.RUNNING; } @@ -617,19 +592,18 @@ private GseTaskExecuteResult analyseExecuteResult() { } @Override - protected void saveFailInfoForUnfinishedAgentTask(AgentTaskStatusEnum status, String errorMsg) { - super.saveFailInfoForUnfinishedAgentTask(status, errorMsg); + protected void saveFailInfoForUnfinishedExecuteObjectTask(ExecuteObjectTaskStatusEnum status, String errorMsg) { + super.saveFailInfoForUnfinishedExecuteObjectTask(status, errorMsg); long endTime = System.currentTimeMillis(); - Set unfinishedAgentIds = new HashSet<>(); - unfinishedAgentIds.addAll(notStartedTargetAgentIds); - unfinishedAgentIds.addAll(runningTargetAgentIds); if (StringUtils.isNotEmpty(errorMsg)) { - List scriptLogs = unfinishedAgentIds.stream().map(agentId -> { - AgentTaskDTO agentTask = targetAgentTasks.get(agentId); - return logService.buildSystemScriptLog(agentTask.getHost(), errorMsg, agentTask.getScriptLogOffset(), - endTime); - }).collect(Collectors.toList()); - logService.batchWriteScriptLog(taskInstance.getCreateTime(), stepInstanceId, stepInstance.getExecuteCount(), + List scriptLogs = + notFinishedTargetExecuteObjectGseKeys.stream().map(executeObjectGseKey -> { + ExecuteObjectTask executeObjectTask = targetExecuteObjectTasks.get(executeObjectGseKey); + ExecuteObject executeObject = findExecuteObject(executeObjectGseKey); + return logService.buildSystemScriptLog(stepInstance, executeObject, errorMsg, + executeObjectTask.getScriptLogOffset(), endTime); + }).collect(Collectors.toList()); + logService.batchWriteScriptLog(taskInstance, stepInstanceId, stepInstance.getExecuteCount(), stepInstance.getBatch(), scriptLogs); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ScriptTaskResultHandleScheduleStrategy.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ScriptTaskResultHandleScheduleStrategy.java index a398457cd0..cfd2bb0ffb 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ScriptTaskResultHandleScheduleStrategy.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ScriptTaskResultHandleScheduleStrategy.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/StopTaskCounter.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/StopTaskCounter.java index 40e19dbcde..fccd1bb503 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/StopTaskCounter.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/StopTaskCounter.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/Task.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/Task.java index 08ec592c1a..edd9d029c1 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/Task.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/Task.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -43,4 +43,6 @@ public interface Task { default String getTaskType() { return "default"; } + + TaskContext getTaskContext(); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/TaskContext.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/TaskContext.java new file mode 100644 index 0000000000..00d02106b8 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/TaskContext.java @@ -0,0 +1,41 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.result; + +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * 任务上下文 + */ +@Data +@NoArgsConstructor +public class TaskContext { + private Long jobInstanceId; + + public TaskContext(Long jobInstanceId) { + this.jobInstanceId = jobInstanceId; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ha/NotAliveResultHandleTaskDetector.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ha/NotAliveResultHandleTaskDetector.java index f0dbc96a9c..efb5f35d91 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ha/NotAliveResultHandleTaskDetector.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ha/NotAliveResultHandleTaskDetector.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ha/ResultHandleLimiter.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ha/ResultHandleLimiter.java index 969ecf762a..9b2273fcc6 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ha/ResultHandleLimiter.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ha/ResultHandleLimiter.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ha/ResultHandleTaskKeepaliveManager.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ha/ResultHandleTaskKeepaliveManager.java index 456fe06fad..95775d27f9 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ha/ResultHandleTaskKeepaliveManager.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/result/ha/ResultHandleTaskKeepaliveManager.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/ExponentIncrementRollingExprPart.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/ExponentIncrementRollingExprPart.java index 31b92657b6..3019a62c99 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/ExponentIncrementRollingExprPart.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/ExponentIncrementRollingExprPart.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,12 +24,13 @@ package com.tencent.bk.job.execute.engine.rolling; -import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.execute.common.exception.RollingExprParseException; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; import lombok.Getter; import lombok.Setter; import lombok.ToString; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; import java.util.ArrayList; import java.util.List; @@ -75,16 +76,14 @@ public RollingExprPart parseExpr(String expr) throws RollingExprParseException { } @Override - public List compute(RollingServerBatchContext context) throws RollingExprParseException { - List candidateServers = context.getRemainedServers(); - // 上一批次的主机 - RollingServerBatch preServerBatch = context.getServerBatches().get(context.getServerBatches().size() - 1); - if (preServerBatch == null) { - log.warn("Invalid rolling expr part : {}, no preServerBatch", getExpr()); - throw new RollingExprParseException(); - } + public List compute(RollingExecuteObjectBatchContext context) throws RollingExprParseException { + List candidateExecuteObjects = context.getRemainedExecuteObjects(); + // 上一批次的执行对象 + RollingExecuteObjectBatch preBatch = CollectionUtils.isEmpty(context.getExecuteObjectBatches()) ? + null : context.getExecuteObjectBatches().get(context.getExecuteObjectBatches().size() - 1); - int currentBatchSize = preServerBatch.getServers().size() * exponent; - return new ArrayList<>(candidateServers.subList(0, Math.min(currentBatchSize, candidateServers.size()))); + int currentBatchSize = preBatch == null ? exponent : preBatch.getExecuteObjects().size() * exponent; + return new ArrayList<>(candidateExecuteObjects.subList( + 0, Math.min(currentBatchSize, candidateExecuteObjects.size()))); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/PercentRollingExprPart.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/PercentRollingExprPart.java index 3095e72468..036f8800e9 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/PercentRollingExprPart.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/PercentRollingExprPart.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,8 +24,8 @@ package com.tencent.bk.job.execute.engine.rolling; -import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.execute.common.exception.RollingExprParseException; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; import lombok.Getter; import lombok.Setter; import lombok.ToString; @@ -69,11 +69,11 @@ public RollingExprPart parseExpr(String expr) throws RollingExprParseException { } @Override - public List compute(RollingServerBatchContext context) throws RollingExprParseException { - List candidateServers = context.getRemainedServers(); - int total = context.getTotalServersSize(); + public List compute(RollingExecuteObjectBatchContext context) throws RollingExprParseException { + List candidateExecuteObjects = context.getRemainedExecuteObjects(); + int total = context.getTotalExecuteObjectSize(); // 批次大小，需要向上取整 int batchSize = (total * percent + 100 - 1) / 100; - return new ArrayList<>(candidateServers.subList(0, Math.min(batchSize, candidateServers.size()))); + return new ArrayList<>(candidateExecuteObjects.subList(0, Math.min(batchSize, candidateExecuteObjects.size()))); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/PlusIncrementRollingExprPart.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/PlusIncrementRollingExprPart.java index 30b66afc13..ec71a35abe 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/PlusIncrementRollingExprPart.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/PlusIncrementRollingExprPart.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,12 +24,13 @@ package com.tencent.bk.job.execute.engine.rolling; -import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.execute.common.exception.RollingExprParseException; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; import lombok.Getter; import lombok.Setter; import lombok.ToString; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; import java.util.ArrayList; import java.util.List; @@ -49,7 +50,7 @@ public class PlusIncrementRollingExprPart extends RollingExprPart { */ private static final String EXPR_REGEX = "^\\+(\\d+)$"; /** - * 每一批次递增的主机数量 + * 每一批次递增的执行对象数量 */ private int addend; @@ -75,16 +76,14 @@ public RollingExprPart parseExpr(String expr) throws RollingExprParseException { } @Override - public List compute(RollingServerBatchContext context) throws RollingExprParseException { - List candidateServers = context.getRemainedServers(); - // 上一批次的主机 - RollingServerBatch preServerBatch = context.getServerBatches().get(context.getServerBatches().size() - 1); - if (preServerBatch == null) { - log.warn("Invalid rolling expr part : {}, no preServerBatch", getExpr()); - throw new RollingExprParseException(); - } + public List compute(RollingExecuteObjectBatchContext context) throws RollingExprParseException { + List candidateExecuteObjects = context.getRemainedExecuteObjects(); + // 上一批次的执行对象 + RollingExecuteObjectBatch preBatch = CollectionUtils.isEmpty(context.getExecuteObjectBatches()) ? + null : context.getExecuteObjectBatches().get(context.getExecuteObjectBatches().size() - 1); - int currentBatchSize = preServerBatch.getServers().size() + addend; - return new ArrayList<>(candidateServers.subList(0, Math.min(currentBatchSize, candidateServers.size()))); + int currentBatchSize = preBatch == null ? addend : preBatch.getExecuteObjects().size() + addend; + return new ArrayList<>(candidateExecuteObjects.subList( + 0, Math.min(currentBatchSize, candidateExecuteObjects.size()))); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/QuantityRollingExprPart.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/QuantityRollingExprPart.java index 8f7a3295cd..ff5c0c4710 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/QuantityRollingExprPart.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/QuantityRollingExprPart.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,8 +24,8 @@ package com.tencent.bk.job.execute.engine.rolling; -import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.execute.common.exception.RollingExprParseException; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; import lombok.Getter; import lombok.Setter; import lombok.ToString; @@ -37,7 +37,7 @@ import java.util.regex.Pattern; /** - * 滚动执行子表达式-按照服务器数量解析 + * 滚动执行子表达式-按照执行对象数量解析 */ @Getter @Setter @@ -70,8 +70,9 @@ public RollingExprPart parseExpr(String expr) throws RollingExprParseException { } @Override - public List compute(RollingServerBatchContext context) throws RollingExprParseException { - List candidateServers = context.getRemainedServers(); - return new ArrayList<>(candidateServers.subList(0, Math.min(this.quantity, candidateServers.size()))); + public List compute(RollingExecuteObjectBatchContext context) throws RollingExprParseException { + List candidateExecuteObjects = context.getRemainedExecuteObjects(); + return new ArrayList<>(candidateExecuteObjects.subList( + 0, Math.min(this.quantity, candidateExecuteObjects.size()))); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/RollingBatchExecuteObjectsResolver.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/RollingBatchExecuteObjectsResolver.java new file mode 100644 index 0000000000..08692376e3 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/RollingBatchExecuteObjectsResolver.java @@ -0,0 +1,96 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.rolling; + +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.FailedPreconditionException; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; +import lombok.extern.slf4j.Slf4j; + +import java.util.List; + +/** + * 执行对象滚动分批Resolver + */ +@Slf4j +public class RollingBatchExecuteObjectsResolver { + + /** + * 允许的最大滚动批次 + */ + public static final int MAX_ALLOWED_ROLLING_BATCH_SIZE = 500; + + /** + * 滚动策略表达式 + */ + private final String rollingExpr; + + /** + * 滚动表达式解析上下文 + */ + private final RollingExecuteObjectBatchContext context; + + /** + * Constructor + * + * @param executeObjects 滚动执行的执行对象 + * @param rollingExpr 滚动表达式 + */ + public RollingBatchExecuteObjectsResolver(List executeObjects, String rollingExpr) { + this.context = new RollingExecuteObjectBatchContext(executeObjects); + this.rollingExpr = rollingExpr; + } + + /** + * 解析滚动表达式，滚动分批 + * + * @return 服务器分批情况 + */ + public List resolve() { + RollingExpr rollingExpr = new RollingExpr(this.rollingExpr); + while (context.hasRemainedExecuteObject()) { + if (context.getExecuteObjectBatches().size() > MAX_ALLOWED_ROLLING_BATCH_SIZE) { + log.warn("Batch {} size greater than {}", context.getExecuteObjectBatches().size(), + MAX_ALLOWED_ROLLING_BATCH_SIZE); + throw new FailedPreconditionException(ErrorCode.EXCEED_MAX_ALLOWED_BATCH_SIZE, + new Integer[]{MAX_ALLOWED_ROLLING_BATCH_SIZE}); + } + + context.increaseBatchCount(); + + RollingExprPart rollingExprPart = rollingExpr.nextRollingExprPart(context.getBatchCount()); + List executeObjectsOnBatch = rollingExprPart.compute(context); + context.removeResolvedServers(executeObjectsOnBatch); + RollingExecuteObjectBatch rollingExecuteObjectBatch = new RollingExecuteObjectBatch(); + rollingExecuteObjectBatch.setBatch(context.getBatchCount()); + rollingExecuteObjectBatch. setExecuteObjects(executeObjectsOnBatch); + rollingExecuteObjectBatch.setRollingExprPart(rollingExprPart); + context.addExecuteObjectBatch(rollingExecuteObjectBatch); + } + + return context.getExecuteObjectBatches(); + } + +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/RollingBatchServersResolver.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/RollingBatchServersResolver.java deleted file mode 100644 index c9c082f395..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/RollingBatchServersResolver.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.engine.rolling; - -import com.tencent.bk.job.common.constant.ErrorCode; -import com.tencent.bk.job.common.exception.FailedPreconditionException; -import com.tencent.bk.job.common.model.dto.HostDTO; -import lombok.extern.slf4j.Slf4j; - -import java.util.List; - -/** - * 服务器滚动分批Resolver - */ -@Slf4j -public class RollingBatchServersResolver { - - /** - * 允许的最大滚动批次 - */ - public static final int MAX_ALLOWED_ROLLING_BATCH_SIZE = 500; - - /** - * 滚动策略表达式 - */ - private final String rollingExpr; - - /** - * 滚动表达式解析上下文 - */ - private final RollingServerBatchContext context; - - /** - * Constructor - * - * @param servers 滚动执行的主机 - * @param rollingExpr 滚动表达式 - */ - public RollingBatchServersResolver(List servers, String rollingExpr) { - this.context = new RollingServerBatchContext(servers); - this.rollingExpr = rollingExpr; - } - - /** - * 解析滚动表达式，滚动分批 - * - * @return 服务器分批情况 - */ - public List resolve() { - RollingExpr rollingExpr = new RollingExpr(this.rollingExpr); - while (context.hasRemainedServer()) { - context.increaseBatchCount(); - RollingExprPart rollingExprPart = rollingExpr.nextRollingExprPart(context.getBatchCount()); - List serversOnBatch = rollingExprPart.compute(context); - context.removeResolvedServers(serversOnBatch); - RollingServerBatch rollingServerBatch = new RollingServerBatch(); - rollingServerBatch.setBatch(context.getBatchCount()); - rollingServerBatch.setServers(serversOnBatch); - rollingServerBatch.setRollingExprPart(rollingExprPart); - context.addServerBatch(rollingServerBatch); - } - - if (context.getServerBatches().size() > MAX_ALLOWED_ROLLING_BATCH_SIZE) { - log.warn("Batch {} size greater than {}", context.getServerBatches().size(), MAX_ALLOWED_ROLLING_BATCH_SIZE); - throw new FailedPreconditionException(ErrorCode.EXCEED_MAX_ALLOWED_BATCH_SIZE, - new Integer[]{MAX_ALLOWED_ROLLING_BATCH_SIZE}); - } - return context.getServerBatches(); - } - -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/RollingExecuteObjectBatch.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/RollingExecuteObjectBatch.java new file mode 100644 index 0000000000..e190dd4357 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/RollingExecuteObjectBatch.java @@ -0,0 +1,40 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.rolling; + +import com.tencent.bk.job.execute.engine.model.ExecuteObject; +import lombok.Data; + +import java.util.List; + +/** + * 执行对象滚动分批 + */ +@Data +public class RollingExecuteObjectBatch { + private RollingExprPart rollingExprPart; + private int batch; + private List executeObjects; +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/RollingExecuteObjectBatchContext.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/RollingExecuteObjectBatchContext.java new file mode 100644 index 0000000000..78919847b9 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/RollingExecuteObjectBatchContext.java @@ -0,0 +1,90 @@ +package com.tencent.bk.job.execute.engine.rolling; + +import com.tencent.bk.job.execute.engine.model.ExecuteObject; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * 执行对象计算滚动分批上下文 + */ +@Getter +@Setter +@ToString +public class RollingExecuteObjectBatchContext { + /** + * 需要分批的执行对象 + */ + private List executeObjects; + /** + * 未分批的执行对象 + */ + private List remainedExecuteObjects; + /** + * 需要分批的执行对象数量 + */ + private int totalExecuteObjectSize; + /** + * 分批数量 + */ + private int batchCount; + /** + * 分批结果 + */ + private List executeObjectBatches; + + /** + * Constructor + * + * @param executeObjects 所有参与滚动的执行对象 + */ + public RollingExecuteObjectBatchContext(List executeObjects) { + this.executeObjects = executeObjects; + this.remainedExecuteObjects = new ArrayList<>(this.executeObjects); + this.totalExecuteObjectSize = executeObjects.size(); + this.executeObjectBatches = new ArrayList<>(); + } + + /** + * 增加已经解析的分批数量 + */ + public void increaseBatchCount() { + this.batchCount++; + } + + /** + * 是否还有剩余执行对象没有被分批 + */ + public boolean hasRemainedExecuteObject() { + return this.remainedExecuteObjects.size() > 0; + } + + /** + * 移除已经解析过的执行对象 + * + * @param resolvedExecuteObjects 已经解析(分批)过的执行对象 + */ + public void removeResolvedServers(Collection resolvedExecuteObjects) { + if (resolvedExecuteObjects instanceof Set) { + this.remainedExecuteObjects.removeAll(resolvedExecuteObjects); + } else { + // 非 Set 集合，List.removeAll() 的实现有性能瓶颈 + this.remainedExecuteObjects.removeAll(new HashSet<>(resolvedExecuteObjects)); + } + } + + /** + * 新增一批滚动主机 + * + * @param rollingExecuteObjectBatch 一个滚动批次的主机 + */ + public void addExecuteObjectBatch(RollingExecuteObjectBatch rollingExecuteObjectBatch) { + this.executeObjectBatches.add(rollingExecuteObjectBatch); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/RollingExpr.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/RollingExpr.java index bf93d5f69f..3bee7e3a8a 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/RollingExpr.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/RollingExpr.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/RollingExprPart.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/RollingExprPart.java index e79c5d8ef9..ac21bbd645 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/RollingExprPart.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/RollingExprPart.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,8 +24,8 @@ package com.tencent.bk.job.execute.engine.rolling; -import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.execute.common.exception.RollingExprParseException; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; import lombok.Getter; import lombok.Setter; import lombok.ToString; @@ -53,11 +53,12 @@ public abstract class RollingExprPart { public abstract RollingExprPart parseExpr(String expr) throws RollingExprParseException; /** - * 根据滚动子表达式以及分批计算上下文，计算当前批次的主机 + * 根据滚动子表达式以及分批计算上下文，计算当前批次的执行对象 * * @param context 解析上下文 - * @return 当前批次的主机 + * @return 当前批次的执行对象 */ - public abstract List compute(RollingServerBatchContext context) throws RollingExprParseException; + public abstract List compute(RollingExecuteObjectBatchContext context) + throws RollingExprParseException; } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/RollingServerBatch.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/RollingServerBatch.java deleted file mode 100644 index b89ee6974f..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/RollingServerBatch.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.engine.rolling; - -import com.tencent.bk.job.common.model.dto.HostDTO; -import lombok.Data; - -import java.util.List; - -/** - * 滚动执行服务器分批 - */ -@Data -public class RollingServerBatch { - private RollingExprPart rollingExprPart; - private int batch; - private List servers; -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/RollingServerBatchContext.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/RollingServerBatchContext.java deleted file mode 100644 index 8d47563e07..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/rolling/RollingServerBatchContext.java +++ /dev/null @@ -1,83 +0,0 @@ -package com.tencent.bk.job.execute.engine.rolling; - -import com.tencent.bk.job.common.model.dto.HostDTO; -import lombok.Getter; -import lombok.Setter; -import lombok.ToString; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; - -/** - * 主机计算滚动分批上下文 - */ -@Getter -@Setter -@ToString -public class RollingServerBatchContext { - /** - * 需要分批的服务器 - */ - private List servers; - /** - * 未分批的服务器 - */ - private List remainedServers; - /** - * 需要分批的服务器数量 - */ - private int totalServersSize; - /** - * 分批数量 - */ - private int batchCount; - /** - * 分批结果 - */ - private List serverBatches; - - /** - * Constructor - * - * @param servers 所有参与滚动的主机 - */ - public RollingServerBatchContext(List servers) { - this.servers = servers; - this.remainedServers = new ArrayList<>(this.servers); - this.totalServersSize = servers.size(); - this.serverBatches = new ArrayList<>(); - } - - /** - * 增加已经解析的分批数量 - */ - public void increaseBatchCount() { - this.batchCount++; - } - - /** - * 是否还有剩余服务器没有被分批 - */ - public boolean hasRemainedServer() { - return this.remainedServers.size() > 0; - } - - /** - * 移除已经解析过的主机 - * - * @param resolvedServers 已经解析(分批)过的主机 - */ - public void removeResolvedServers(Collection resolvedServers) { - this.remainedServers.removeAll(resolvedServers); - } - - /** - * 新增一批滚动主机 - * - * @param rollingServerBatch 一个滚动批次的主机 - */ - public void addServerBatch(RollingServerBatch rollingServerBatch) { - this.serverBatches.add(rollingServerBatch); - } -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/Encryptor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/Encryptor.java index 356eb52a74..0ca1527e6a 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/Encryptor.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/Encryptor.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/GSEPullSleepUtil.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/GSEPullSleepUtil.java index a9e0cd86d1..fee5ed7a5c 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/GSEPullSleepUtil.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/GSEPullSleepUtil.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/GseUtils.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/GseUtils.java index d74ace741a..8be584d235 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/GseUtils.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/GseUtils.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,8 +24,8 @@ package com.tencent.bk.job.execute.engine.util; -import com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum; import com.tencent.bk.job.execute.engine.consts.Consts; +import com.tencent.bk.job.execute.engine.consts.ExecuteObjectTaskStatusEnum; /** * 处理 GSE 工具类 @@ -70,13 +70,13 @@ private static double changeUnit(double size) { /** * 根据gse ErrorCode返回 Status */ - public static AgentTaskStatusEnum getStatusByGseErrorCode(int gseErrorCode) { - AgentTaskStatusEnum status = Consts.GSE_ERROR_CODE_2_STATUS_MAP.get(gseErrorCode); + public static ExecuteObjectTaskStatusEnum getStatusByGseErrorCode(int gseErrorCode) { + ExecuteObjectTaskStatusEnum status = Consts.GSE_ERROR_CODE_2_STATUS_MAP.get(gseErrorCode); if (status == null) { - if (gseErrorCode > 0) { - status = AgentTaskStatusEnum.UNKNOWN_ERROR; + if (gseErrorCode == 0) { + status = ExecuteObjectTaskStatusEnum.SUCCESS; } else { - status = AgentTaskStatusEnum.SUCCESS; + status = ExecuteObjectTaskStatusEnum.UNKNOWN_ERROR; } } return status; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/IpHelper.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/IpHelper.java index 28b40876f0..66412c4fbf 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/IpHelper.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/IpHelper.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/JobSrcFileUtils.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/JobSrcFileUtils.java index 4d88f67b58..1e2c789be4 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/JobSrcFileUtils.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/JobSrcFileUtils.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,17 +25,17 @@ package com.tencent.bk.job.execute.engine.util; import com.google.common.collect.Sets; -import com.tencent.bk.job.common.model.dto.HostDTO; -import com.tencent.bk.job.common.util.file.PathUtil; -import com.tencent.bk.job.common.util.function.LambdasUtil; +import com.tencent.bk.job.common.util.FilePathUtils; import com.tencent.bk.job.execute.engine.consts.FileDirTypeConf; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; import com.tencent.bk.job.execute.engine.model.FileDest; import com.tencent.bk.job.execute.engine.model.JobFile; +import com.tencent.bk.job.execute.model.ExecuteTargetDTO; import com.tencent.bk.job.execute.model.FileDetailDTO; import com.tencent.bk.job.execute.model.FileSourceDTO; -import com.tencent.bk.job.execute.model.ServersDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; -import com.tencent.bk.job.manage.common.consts.task.TaskFileTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskFileTypeEnum; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; @@ -43,7 +43,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.function.Predicate; /** * 源文件工具类 @@ -57,27 +56,38 @@ public class JobSrcFileUtils { * @param targetFileName 文件分发到目标主机的对应名称 * @return 源文件路径与目标文件路径的映射关系 */ - public static Map buildSourceDestPathMapping(Set srcFiles, - String targetDir, - String targetFileName) { - Map sourceDestPathMap = new HashMap<>(); + public static Map buildSourceDestPathMapping(Set srcFiles, + String targetDir, + String targetFileName) { + Map sourceDestPathMap = new HashMap<>(); String standardTargetDir = FilePathUtils.standardizedDirPath(targetDir); long currentTime = System.currentTimeMillis(); for (JobFile srcFile : srcFiles) { - // 本地文件的源ip是本机ip，展开源文件IP地址宏采用"0.0.0.0" - String destDirPath = MacroUtil.resolveFileSrcIpMacro(standardTargetDir, - srcFile.getFileType() == TaskFileTypeEnum.LOCAL ? "0_0.0.0.0" : srcFile.getHost().toCloudIp()); + String destDirPath = resolveFileSrcIp(standardTargetDir, srcFile); destDirPath = MacroUtil.resolveDate(destDirPath, currentTime); addSourceDestPathMapping(sourceDestPathMap, srcFile, destDirPath, targetFileName); } return sourceDestPathMap; } - private static void addSourceDestPathMapping(Map sourceDestPathMap, + + private static String resolveFileSrcIp(String targetFilePath, JobFile srcFile) { + // 本地文件的源ip是本机ip，展开源文件IP地址宏采用"0.0.0.0" + String resolvedTargetPath = targetFilePath; + if (srcFile.getExecuteObject().isHostExecuteObject()) { + resolvedTargetPath = MacroUtil.resolveFileSrcIpMacro(targetFilePath, + srcFile.getFileType() == TaskFileTypeEnum.LOCAL ? "0_0.0.0.0" : + srcFile.getExecuteObject().getHost().getBkCloudId() + "_" + + srcFile.getExecuteObject().getHost().getPrimaryIp()); + } + return resolvedTargetPath; + } + + private static void addSourceDestPathMapping(Map sourceDestPathMap, JobFile sourceFile, String destDirPath, String destName) { - sourceDestPathMap.put(sourceFile.getUniqueKey(), buildFileDest(sourceFile, destDirPath, destName)); + sourceDestPathMap.put(sourceFile, buildFileDest(sourceFile, destDirPath, destName)); } private static FileDest buildFileDest(JobFile sourceFile, String destDirPath, String destName) { @@ -104,12 +114,11 @@ private static boolean isServerOrThirdFileSource(FileSourceDTO fileSource) { * 从步骤解析源文件，处理服务器文件、本地文件、第三方源文件的差异，统一为IP+Path信息 * * @param stepInstance 步骤 - * @param localHost job local host * @param jobStorageRootDir job共享存储根目录 * @return 多个要分发的源文件信息集合 */ - public static Set parseSendFileList(StepInstanceDTO stepInstance, HostDTO localHost, - String jobStorageRootDir) { + public static Set parseSrcFiles(StepInstanceDTO stepInstance, + String jobStorageRootDir) { Set sendFiles = Sets.newHashSet(); for (FileSourceDTO fileSource : stepInstance.getFileSourceList()) { List files = fileSource.getFiles(); @@ -124,30 +133,26 @@ public static Set parseSendFileList(StepInstanceDTO stepInstance, HostD Long accountId = fileSource.getAccountId(); String accountAlias = fileSource.getAccountAlias(); // 远程文件 - List sourceHosts = fileSource.getServers().getIpList(); - Set invalidHosts = stepInstance.getInvalidHosts(); for (FileDetailDTO file : files) { String filePath = StringUtils.isNotEmpty(file.getResolvedFilePath()) ? file.getResolvedFilePath() : file.getFilePath(); Pair fileNameAndPath = FilePathUtils.parseDirAndFileName(filePath); String dir = fileNameAndPath.getLeft(); String fileName = fileNameAndPath.getRight(); - Predicate predicate = LambdasUtil.not(invalidHosts::contains); - for (HostDTO sourceHost : sourceHosts) { - if (predicate.test(sourceHost)) { - // 第三方源文件的displayName不同 - if (isThirdFile) { - sendFiles.add(new JobFile(TaskFileTypeEnum.FILE_SOURCE, sourceHost, - file.getThirdFilePathWithFileSourceName(), - file.getThirdFilePathWithFileSourceName(), - dir, fileName, stepInstance.getAppId(), accountId, accountAlias)); - } else { - sendFiles.add(new JobFile(TaskFileTypeEnum.SERVER, sourceHost, filePath, filePath, dir, - fileName, stepInstance.getAppId(), accountId, accountAlias)); - } + List sourceExecuteObjects = fileSource.getServers().getExecuteObjectsCompatibly(); + for (ExecuteObject sourceExecuteObject : sourceExecuteObjects) { + // 第三方源文件的displayName不同 + if (isThirdFile) { + sendFiles.add(new JobFile(TaskFileTypeEnum.FILE_SOURCE, sourceExecuteObject, + file.getThirdFilePathWithFileSourceName(), + file.getThirdFilePathWithFileSourceName(), + dir, fileName, stepInstance.getAppId(), accountId, accountAlias)); + } else { + sendFiles.add(new JobFile(TaskFileTypeEnum.SERVER, sourceExecuteObject, + filePath, filePath, dir, fileName, stepInstance.getAppId(), accountId, + accountAlias)); } } - } } else if (fileSource.getFileType() == TaskFileTypeEnum.LOCAL.getType()) { // 本地文件 @@ -156,18 +161,15 @@ public static Set parseSendFileList(StepInstanceDTO stepInstance, HostD String dir = NFSUtils.getFileDir(jobStorageRootDir, FileDirTypeConf.UPLOAD_FILE_DIR) + fileNameAndPath.getLeft(); String fileName = fileNameAndPath.getRight(); - ServersDTO servers = fileSource.getServers(); - if (servers != null && servers.getIpList() != null && !servers.getIpList().isEmpty()) { - List ipList = servers.getIpList(); - for (HostDTO hostDTO : ipList) { - sendFiles.add(new JobFile(TaskFileTypeEnum.LOCAL, hostDTO, file.getFilePath(), dir, + ExecuteTargetDTO executeTarget = fileSource.getServers(); + if (executeTarget != null + && CollectionUtils.isNotEmpty(executeTarget.getExecuteObjectsCompatibly())) { + List executeObjects = executeTarget.getExecuteObjectsCompatibly(); + for (ExecuteObject executeObject : executeObjects) { + sendFiles.add(new JobFile(TaskFileTypeEnum.LOCAL, executeObject, file.getFilePath(), dir, fileName, "root", null, FilePathUtils.parseDirAndFileName(file.getFilePath()).getRight())); } - } else { - sendFiles.add(new JobFile(TaskFileTypeEnum.LOCAL, localHost, file.getFilePath(), dir, - fileName, "root", null, - FilePathUtils.parseDirAndFileName(file.getFilePath()).getRight())); } } } else if (fileSource.getFileType() == TaskFileTypeEnum.BASE64_FILE.getType()) { @@ -177,9 +179,9 @@ public static Set parseSendFileList(StepInstanceDTO stepInstance, HostD String dir = NFSUtils.getFileDir(jobStorageRootDir, FileDirTypeConf.UPLOAD_FILE_DIR) + fileNameAndPath.getLeft(); String fileName = fileNameAndPath.getRight(); - List ipList = fileSource.getServers().getIpList(); - for (HostDTO hostDTO : ipList) { - sendFiles.add(new JobFile(TaskFileTypeEnum.BASE64_FILE, hostDTO, file.getFilePath(), dir, + List executeObjects = fileSource.getServers().getExecuteObjectsCompatibly(); + for (ExecuteObject executeObject : executeObjects) { + sendFiles.add(new JobFile(TaskFileTypeEnum.BASE64_FILE, executeObject, file.getFilePath(), dir, fileName, "root", null, FilePathUtils.parseDirAndFileName(file.getFilePath()).getRight())); } @@ -189,24 +191,4 @@ public static Set parseSendFileList(StepInstanceDTO stepInstance, HostD return sendFiles; } - /** - * 构造源文件原始路径与显示路径的映射关系 - * - * @param sourceFiles 源文件 - * @param localUploadDir 本地上传文件根目录 - * @return 源文件原始路径与显示路径的映射关系 - */ - public static Map buildSourceFileDisplayMapping(Set sourceFiles, String localUploadDir) { - Map sourceFileDisplayMap = new HashMap<>(); - sourceFiles.forEach(sourceFile -> { - String standardPath = sourceFile.getStandardFilePath(); - if (sourceFile.getFileType() == TaskFileTypeEnum.LOCAL && !standardPath.startsWith(localUploadDir)) { - sourceFileDisplayMap.put(PathUtil.joinFilePath(localUploadDir, standardPath), - sourceFile.getDisplayFilePath()); - } else { - sourceFileDisplayMap.put(standardPath, sourceFile.getDisplayFilePath()); - } - }); - return sourceFileDisplayMap; - } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/MacroUtil.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/MacroUtil.java index e141d72f68..fe83c869ce 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/MacroUtil.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/MacroUtil.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,7 +25,6 @@ package com.tencent.bk.job.execute.engine.util; import com.tencent.bk.job.common.util.date.DateUtils; -import com.tencent.bk.job.execute.engine.gse.Strftime; import org.apache.commons.lang3.StringUtils; import java.time.ZoneId; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/NFSUtils.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/NFSUtils.java index b9888ee79c..a29cbdae56 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/NFSUtils.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/NFSUtils.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/RunningTaskCounter.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/RunningTaskCounter.java index 5f7a87d3c0..d28020a97c 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/RunningTaskCounter.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/RunningTaskCounter.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/gse/Strftime.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/Strftime.java similarity index 98% rename from src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/gse/Strftime.java rename to src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/Strftime.java index 879e7e0289..236bdb391c 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/gse/Strftime.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/Strftime.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.execute.engine.gse; +package com.tencent.bk.job.execute.engine.util; import java.text.SimpleDateFormat; import java.util.Date; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/TimeoutUtils.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/TimeoutUtils.java index d162d1c410..c87facdfd0 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/TimeoutUtils.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/util/TimeoutUtils.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/variable/JobBuildInVariableResolver.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/variable/JobBuildInVariableResolver.java index 376d153eee..371ac5b202 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/variable/JobBuildInVariableResolver.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/variable/JobBuildInVariableResolver.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/variable/JobLastHostsVariableResolver.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/variable/JobLastHostsVariableResolver.java index 5f31d59d8b..eff780208d 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/variable/JobLastHostsVariableResolver.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/variable/JobLastHostsVariableResolver.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,14 +26,13 @@ import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.execute.engine.consts.JobBuildInVariables; -import com.tencent.bk.job.execute.model.AgentTaskDTO; -import com.tencent.bk.job.execute.model.AgentTaskDetailDTO; +import com.tencent.bk.job.execute.model.ExecuteObjectTask; import com.tencent.bk.job.execute.model.StepInstanceDTO; -import com.tencent.bk.job.execute.service.FileAgentTaskService; -import com.tencent.bk.job.execute.service.ScriptAgentTaskService; -import com.tencent.bk.job.execute.service.TaskInstanceService; +import com.tencent.bk.job.execute.service.FileExecuteObjectTaskService; +import com.tencent.bk.job.execute.service.ScriptExecuteObjectTaskService; +import com.tencent.bk.job.execute.service.StepInstanceService; import com.tencent.bk.job.logsvr.consts.FileTaskModeEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskStepTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskStepTypeEnum; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -52,18 +51,18 @@ @Service @Slf4j public class JobLastHostsVariableResolver implements VariableResolver { - private final TaskInstanceService taskInstanceService; - private final ScriptAgentTaskService scriptAgentTaskService; - private final FileAgentTaskService fileAgentTaskService; + private final StepInstanceService stepInstanceService; + private final ScriptExecuteObjectTaskService scriptExecuteObjectTaskService; + private final FileExecuteObjectTaskService fileExecuteObjectTaskService; private final Set BUILD_IN_VARIABLES = new HashSet<>(); @Autowired - public JobLastHostsVariableResolver(TaskInstanceService taskInstanceService, - ScriptAgentTaskService scriptAgentTaskService, - FileAgentTaskService fileAgentTaskService) { - this.taskInstanceService = taskInstanceService; - this.scriptAgentTaskService = scriptAgentTaskService; - this.fileAgentTaskService = fileAgentTaskService; + public JobLastHostsVariableResolver(StepInstanceService stepInstanceService, + ScriptExecuteObjectTaskService scriptExecuteObjectTaskService, + FileExecuteObjectTaskService fileExecuteObjectTaskService) { + this.stepInstanceService = stepInstanceService; + this.scriptExecuteObjectTaskService = scriptExecuteObjectTaskService; + this.fileExecuteObjectTaskService = fileExecuteObjectTaskService; init(); } @@ -81,7 +80,7 @@ public boolean isMatch(String variable) { public String resolve(VariableResolveContext context, String variableName) { long taskInstanceId = context.getTaskInstanceId(); long stepInstanceId = context.getStepInstanceId(); - StepInstanceDTO preStepInstance = taskInstanceService.getPreExecutableStepInstance(taskInstanceId, + StepInstanceDTO preStepInstance = stepInstanceService.getPreExecutableStepInstance(taskInstanceId, stepInstanceId); if (preStepInstance == null) { log.info("Resolve value from latest executable step instance, but no pre step exist! taskInstanceId: {}, " + @@ -92,24 +91,24 @@ public String resolve(VariableResolveContext context, String variableName) { Set hosts = null; switch (variableName) { case JobBuildInVariables.JOB_LAST_ALL: - hosts = extractAllHosts(preStepInstance); + hosts = preStepInstance.extractAllHosts(); break; case JobBuildInVariables.JOB_LAST_SUCCESS: { - List agentTasks = listAgentTasks(preStepInstance); - if (CollectionUtils.isNotEmpty(agentTasks)) { - hosts = agentTasks.stream() - .filter(AgentTaskDTO::isSuccess) - .map(AgentTaskDTO::getHost) + List executeObjectTasks = listAgentTasks(preStepInstance); + if (CollectionUtils.isNotEmpty(executeObjectTasks)) { + hosts = executeObjectTasks.stream() + .filter(ExecuteObjectTask::isSuccess) + .map(task -> task.getExecuteObject().getHost()) .collect(Collectors.toSet()); } break; } case JobBuildInVariables.JOB_LAST_FAIL: { - List agentTasks = listAgentTasks(preStepInstance); - if (CollectionUtils.isNotEmpty(agentTasks)) { - hosts = agentTasks.stream() - .filter(not(AgentTaskDTO::isSuccess)) - .map(AgentTaskDTO::getHost) + List executeObjectTasks = listAgentTasks(preStepInstance); + if (CollectionUtils.isNotEmpty(executeObjectTasks)) { + hosts = executeObjectTasks.stream() + .filter(not(ExecuteObjectTask::isSuccess)) + .map(task -> task.getExecuteObject().getHost()) .collect(Collectors.toSet()); } break; @@ -122,13 +121,13 @@ public String resolve(VariableResolveContext context, String variableName) { return value; } - private List listAgentTasks(StepInstanceDTO stepInstance) { + private List listAgentTasks(StepInstanceDTO stepInstance) { TaskStepTypeEnum stepType = stepInstance.getStepType(); - List agentTasks = null; + List agentTasks = null; if (stepType == TaskStepTypeEnum.SCRIPT) { - agentTasks = scriptAgentTaskService.listAgentTaskDetail(stepInstance, stepInstance.getExecuteCount(), null); + agentTasks = scriptExecuteObjectTaskService.listTasks(stepInstance, stepInstance.getExecuteCount(), null); } else if (stepType == TaskStepTypeEnum.FILE) { - agentTasks = fileAgentTaskService.listAgentTaskDetail(stepInstance, stepInstance.getExecuteCount(), null); + agentTasks = fileExecuteObjectTaskService.listTasks(stepInstance, stepInstance.getExecuteCount(), null); if (CollectionUtils.isNotEmpty(agentTasks)) { agentTasks = agentTasks.stream() .filter(agentTask -> agentTask.getFileTaskMode() == FileTaskModeEnum.DOWNLOAD) @@ -137,24 +136,4 @@ private List listAgentTasks(StepInstanceDTO stepInstance) { } return agentTasks; } - - - private Set extractAllHosts(StepInstanceDTO stepInstance) { - Set hosts = new HashSet<>(); - if (CollectionUtils.isNotEmpty(stepInstance.getTargetServers().getIpList())) { - hosts.addAll(stepInstance.getTargetServers().getIpList()); - } - if (CollectionUtils.isNotEmpty(stepInstance.getTargetServers().getInvalidIpList())) { - hosts.addAll(stepInstance.getTargetServers().getInvalidIpList()); - } - if (CollectionUtils.isNotEmpty(stepInstance.getFileSourceList())) { - stepInstance.getFileSourceList().forEach(fileSource -> { - if (fileSource.getServers() != null - && CollectionUtils.isNotEmpty(fileSource.getServers().getIpList())) { - hosts.addAll(fileSource.getServers().getIpList()); - } - }); - } - return hosts; - } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/variable/JobNamespaceVariableResolver.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/variable/JobNamespaceVariableResolver.java index ca63a8dbf4..7f96dd6be6 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/variable/JobNamespaceVariableResolver.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/variable/JobNamespaceVariableResolver.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -60,12 +60,11 @@ public String resolve(VariableResolveContext context, String variableName) { Map> namespaceVarsValues = new HashMap<>(); List namespaceValues = stepInputVariables.getNamespaceParams(); namespaceValues.forEach(namespaceValue -> { - String ip = namespaceValue.getIp(); if (CollectionUtils.isNotEmpty(namespaceValue.getValues())) { namespaceValue.getValues().forEach(variableValue -> { Map ipAndValueMap = namespaceVarsValues.computeIfAbsent(variableValue.getName(), k -> new HashMap<>()); - ipAndValueMap.put(ip, variableValue.getValue()); + ipAndValueMap.put(namespaceValue.getPrimaryCloudIp(), variableValue.getValue()); }); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/variable/VariableResolveContext.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/variable/VariableResolveContext.java index 2b1acf171c..878c179666 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/variable/VariableResolveContext.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/variable/VariableResolveContext.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/variable/VariableResolveResult.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/variable/VariableResolveResult.java index bdb84e5774..abfafbb1a5 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/variable/VariableResolveResult.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/variable/VariableResolveResult.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/variable/VariableResolveUtils.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/variable/VariableResolveUtils.java index 06009afad0..1b2411d418 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/variable/VariableResolveUtils.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/variable/VariableResolveUtils.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -45,7 +45,7 @@ public static String formatHosts(Collection hosts) { return null; } StringJoiner joiner = new StringJoiner(","); - hosts.forEach(host -> joiner.add(host.toCloudIp())); + hosts.forEach(host -> joiner.add(host.getBkCloudId() + ":" + host.getPrimaryIp())); return joiner.toString(); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/variable/VariableResolver.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/variable/VariableResolver.java index ddc73967d0..58cc747eaf 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/variable/VariableResolver.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/engine/variable/VariableResolver.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/gse/GseApiExecutor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/gse/GseApiExecutor.java deleted file mode 100644 index 324ff84c34..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/gse/GseApiExecutor.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.gse; - -import com.tencent.bk.gse.taskapi.api_agent; -import com.tencent.bk.gse.taskapi.api_map_rsp; -import com.tencent.bk.gse.taskapi.api_process_base_info; -import com.tencent.bk.gse.taskapi.api_process_extra_info; -import com.tencent.bk.gse.taskapi.api_process_req; -import com.tencent.bk.job.common.model.dto.HostDTO; -import com.tencent.bk.job.execute.engine.gse.GseRequestUtils; -import com.tencent.bk.job.execute.engine.model.GseTaskResponse; -import com.tencent.bk.job.execute.gse.model.GseProcessInfoDTO; -import lombok.extern.slf4j.Slf4j; - -import java.util.ArrayList; -import java.util.List; -import java.util.stream.Collectors; - -@Slf4j -public class GseApiExecutor { - - private String uid; - - public GseApiExecutor(String uid) { - this.uid = uid; - } - - public GseTaskResponse processOperateProcess(List processInfoList, int reqType) { - List procReqList = new ArrayList<>(); - for (GseProcessInfoDTO processInfo : processInfoList) { - List agentList = GseRequestUtils.buildAgentList( - processInfo.getIpList().stream().map(HostDTO::toCloudIp).collect(Collectors.toList()), - processInfo.getUserName(), null); - - api_process_base_info processBaseInfo = createProcessBaseInfo(processInfo); - api_process_extra_info processExtraInfo = createProcessExtraInfo(processInfo); - - api_process_req procRequest = GseRequestUtils.buildProcessRequest(agentList, processBaseInfo, - processExtraInfo, reqType); - - procReqList.add(procRequest); - } - - return GseRequestUtils.sendProcessRequest(uid, procReqList, reqType); - } - - private api_process_extra_info createProcessExtraInfo(GseProcessInfoDTO processInfo) { - api_process_extra_info processExtraInfo = new api_process_extra_info(); - processExtraInfo.setType(processInfo.getType()); - processExtraInfo.setCpu_limit(processInfo.getCpuLmt()); - processExtraInfo.setMem_limit(processInfo.getMemLmt()); - processExtraInfo.setCycle_time(processInfo.getCycleTime()); - processExtraInfo.setInstance_num(processInfo.getInstanceNum()); - processExtraInfo.setStart_check_begin_time(processInfo.getStartCheckBeginTime()); - processExtraInfo.setStart_check_end_time(processInfo.getStartCheckEndTime()); - processExtraInfo.setOp_timeout(processInfo.getOpTimeOut()); - return processExtraInfo; - } - - private api_process_base_info createProcessBaseInfo(GseProcessInfoDTO processInfo) { - api_process_base_info processBaseInfo = new api_process_base_info(); - processBaseInfo.setProc_name(ifNull(processInfo.getProcName())); - processBaseInfo.setSetup_path(ifNull(processInfo.getSetupPath())); - processBaseInfo.setCfg_path(ifNull(processInfo.getCfgPath())); - processBaseInfo.setLog_path(ifNull(processInfo.getLogPath())); - processBaseInfo.setPid_path(ifNull(processInfo.getPidPath())); - processBaseInfo.setContact(ifNull(processInfo.getContact())); - processBaseInfo.setStart_cmd(ifNull(processInfo.getStartCmd())); - processBaseInfo.setStop_cmd(ifNull(processInfo.getStopCmd())); - processBaseInfo.setRestart_cmd(ifNull(processInfo.getRestartCmd())); - processBaseInfo.setReload_cmd(ifNull(processInfo.getReloadCmd())); - processBaseInfo.setKill_cmd(ifNull(processInfo.getKillCmd())); - processBaseInfo.setFunc_id(ifNull(processInfo.getFuncID())); - processBaseInfo.setInstance_id(ifNull(processInfo.getInstanceID())); - processBaseInfo.setValue_key(ifNull(processInfo.getValueKey())); - return processBaseInfo; - } - - private String ifNull(String str) { - if (str == null) { - return ""; - } - return str; - } - - public api_map_rsp getGetProcRst(String gseTaskId) { - return GseRequestUtils.getGetProcRst("", gseTaskId); - } -} - - - - - - - - - diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/gse/model/GseProcessInfoDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/gse/model/GseProcessInfoDTO.java deleted file mode 100644 index 7924a03cdf..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/gse/model/GseProcessInfoDTO.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.gse.model; - -import com.tencent.bk.job.common.model.dto.HostDTO; -import lombok.Getter; -import lombok.Setter; -import lombok.ToString; - -import java.util.List; - -@Getter -@Setter -@ToString -public class GseProcessInfoDTO { - private List ipList; - - private String userName; - - private String procName; // required - private String setupPath; // required - private String cfgPath; // required - private String logPath; // required - private String pidPath; // required - private String contact; // required - private String startCmd; // required - private String stopCmd; // required - private String restartCmd; // required - private String reloadCmd; // required - private String killCmd; // required - private String funcID; // required - private String instanceID; // required - private String valueKey; // required - - private int type; // required - private int cpuLmt; // optional - private int memLmt; // optional - private int cycleTime; // optional - private int instanceNum; // optional - private int startCheckBeginTime; // optional - private int startCheckEndTime; // optional - private int opTimeOut; // optional - -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/gse/model/ProcessManageTypeEnum.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/gse/model/ProcessManageTypeEnum.java deleted file mode 100644 index 65a0b2ead4..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/gse/model/ProcessManageTypeEnum.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.gse.model; - -/** - * 进程管理操作类型 - */ -public enum ProcessManageTypeEnum { - REGISTER(1, "注册托管进程"), UNREGISTER(2, "取消托管进程"); - - private final int value; - private final String name; - - ProcessManageTypeEnum(int val, String name) { - this.value = val; - this.name = name; - } - - public int getValue() { - return value; - } - - public String getName() { - return name; - } -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/gse/model/ProcessOperateTypeEnum.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/gse/model/ProcessOperateTypeEnum.java deleted file mode 100644 index f5dfe8e7d4..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/gse/model/ProcessOperateTypeEnum.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.gse.model; - -public enum ProcessOperateTypeEnum { - START(0, "启动进程"), STOP(1, "停止进程"), PROCESS_STATUS(2, "进程状态查询"), REGISTER_PROC(3, "注册托管进程"), - UNREGISTER_PROC(4, "取消托管进程"), RESTART(7, "重启进程"), RELOAD(8, "重新加载进程"), KILL(9, "杀死进程"); - - private final int value; - private final String name; - - ProcessOperateTypeEnum(int val, String name) { - this.value = val; - this.name = name; - } - - public int getValue() { - return value; - } - - public String getName() { - return name; - } -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/metrics/ExecuteMetricsConstants.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/metrics/ExecuteMetricsConstants.java index a19c0c5e6b..fce51226de 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/metrics/ExecuteMetricsConstants.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/metrics/ExecuteMetricsConstants.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -30,6 +30,19 @@ public class ExecuteMetricsConstants { * 任务启动指标 */ public static final String NAME_JOB_TASK_START = "job.task.start"; + /** + * 日志大小（字节） + */ + public static final String NAME_JOB_LOG_BYTES = "job.log.bytes"; + /** + * 任务执行结束后的回调指标 + */ + public static final String NAME_JOB_TASK_CALLBACK = "job.task.callback"; + /** + * 任务执行对象数量分布指标 + */ + public static final String NAME_JOB_TASK_EXECUTE_OBJECT_NUM = "job.task.executeObjectNum"; + // tag /** * 任务启动方式 @@ -51,6 +64,24 @@ public class ExecuteMetricsConstants { * 任务类型 */ public static final String TAG_KEY_TASK_TYPE = "task_type"; + /** + * Job业务ID + */ + public static final String TAG_KEY_APP_ID = "app_id"; + /** + * 定时任务ID + */ + public static final String TAG_KEY_CRON_TASK_ID = "cron_task_id"; + /** + * AppCode + */ + public static final String TAG_KEY_APP_CODE = "app_code"; + /** + * Result + */ + public static final String TAG_KEY_RESULT = "result"; + + /** * 任务类型：快速执行脚本 */ @@ -83,4 +114,36 @@ public class ExecuteMetricsConstants { * 任务启动状态：启动失败 */ public static final String TAG_VALUE_START_STATUS_FAILED = "failed"; + /** + * 任务回调结果：使用postJson回调成功 + */ + public static final String TAG_VALUE_RESULT_POST_JSON_SUCCESS = "post_json_success"; + /** + * 任务回调结果：使用postFormData回调成功 + */ + public static final String TAG_VALUE_RESULT_POST_FORM_DATA_SUCCESS = "post_form_data_success"; + /** + * 任务回调结果：回调失败 + */ + public static final String TAG_VALUE_RESULT_FAILED = "failed"; + /** + * 执行对象组成 + */ + public static final String TAG_KEY_EXECUTE_OBJECT_COMPOSITION = "execute_object_composition"; + /** + * 执行对象组成：无执行对象 + */ + public static final String TAG_VALUE_EXECUTE_OBJECT_COMPOSITION_NONE = "none"; + /** + * 执行对象组成：主机 + */ + public static final String TAG_VALUE_EXECUTE_OBJECT_COMPOSITION_HOST = "host"; + /** + * 执行对象组成：容器 + */ + public static final String TAG_VALUE_EXECUTE_OBJECT_COMPOSITION_CONTAINER = "container"; + /** + * 执行对象组成：混合 + */ + public static final String TAG_VALUE_EXECUTE_OBJECT_COMPOSITION_MIXED = "mixed"; } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/metrics/ExecuteObjectSampler.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/metrics/ExecuteObjectSampler.java new file mode 100644 index 0000000000..f7e4269a1e --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/metrics/ExecuteObjectSampler.java @@ -0,0 +1,40 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.metrics; + +import com.tencent.bk.job.execute.model.TaskInstanceDTO; +import com.tencent.bk.job.execute.model.TaskInstanceExecuteObjects; + +public interface ExecuteObjectSampler { + + /** + * 尝试记录执行对象相关指标，异常不抛出，内部做日志记录 + * + * @param taskInstance 任务实例对象 + * @param executeObjects 执行对象 + */ + void tryToRecordExecuteObjectMetrics(TaskInstanceDTO taskInstance, + TaskInstanceExecuteObjects executeObjects); +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/metrics/LogSampler.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/metrics/LogSampler.java new file mode 100644 index 0000000000..b0b6912733 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/metrics/LogSampler.java @@ -0,0 +1,37 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.metrics; + +import com.tencent.bk.job.logsvr.model.service.ServiceBatchSaveLogRequest; + +public interface LogSampler { + + /** + * 尝试记录日志大小指标，异常不抛出，内部做日志记录 + * + * @param request 保存日志请求 + */ + void tryToRecordLogSizeMetrics(ServiceBatchSaveLogRequest request); +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/metrics/TaskStartTagsContributor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/metrics/TaskStartTagsContributor.java index 5795331928..1b38ed42f3 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/metrics/TaskStartTagsContributor.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/metrics/TaskStartTagsContributor.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/metrics/impl/ExecuteObjectSamplerImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/metrics/impl/ExecuteObjectSamplerImpl.java new file mode 100644 index 0000000000..dd8f28714d --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/metrics/impl/ExecuteObjectSamplerImpl.java @@ -0,0 +1,148 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.metrics.impl; + +import com.tencent.bk.job.common.metrics.CommonMetricTags; +import com.tencent.bk.job.common.model.dto.ResourceScope; +import com.tencent.bk.job.execute.metrics.ExecuteMetricsConstants; +import com.tencent.bk.job.execute.metrics.ExecuteObjectSampler; +import com.tencent.bk.job.execute.model.TaskInstanceDTO; +import com.tencent.bk.job.execute.model.TaskInstanceExecuteObjects; +import com.tencent.bk.job.manage.GlobalAppScopeMappingService; +import io.micrometer.core.instrument.DistributionSummary; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tag; +import io.micrometer.core.instrument.Tags; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +/** + * 记录任务执行对象相关指标 + */ +@Slf4j +@Service +public class ExecuteObjectSamplerImpl implements ExecuteObjectSampler { + + private final MeterRegistry meterRegistry; + + @Autowired + public ExecuteObjectSamplerImpl(MeterRegistry meterRegistry) { + this.meterRegistry = meterRegistry; + } + + @Override + public void tryToRecordExecuteObjectMetrics(TaskInstanceDTO taskInstance, + TaskInstanceExecuteObjects executeObjects) { + try { + recordExecuteObjectMetrics(taskInstance, executeObjects); + } catch (Exception e) { + log.warn("tryToRecordExecuteObjectMetrics error", e); + } + } + + /** + * 记录任务执行对象指标 + * + * @param taskInstance 任务实例 + * @param executeObjects 任务执行对象 + */ + public void recordExecuteObjectMetrics(TaskInstanceDTO taskInstance, + TaskInstanceExecuteObjects executeObjects) { + int executeObjectNum = executeObjects.getExecuteObjectNum(); + Iterable tags = buildTags(taskInstance, executeObjects); + if (log.isDebugEnabled()) { + log.debug( + "tryToRecordExecuteObjectMetrics: appId={}, cronTaskId={}, appCode={}, executeObjectNum={}", + taskInstance.getAppId(), + taskInstance.getCronTaskId(), + taskInstance.getAppCode(), + executeObjectNum + ); + } + DistributionSummary.builder(ExecuteMetricsConstants.NAME_JOB_TASK_EXECUTE_OBJECT_NUM) + .tags(tags) + .register(meterRegistry) + .record(executeObjectNum); + } + + /** + * 构造资源范围标签值 + * + * @param resourceScope 资源范围 + * @return 资源范围标签值 + */ + private String buildResourceScopeTagValue(ResourceScope resourceScope) { + if (resourceScope == null) { + return "None"; + } + return resourceScope.getType() + ":" + resourceScope.getId(); + } + + /** + * 构造指标的标签 + * + * @param taskInstance 任务实例 + * @param executeObjects 任务执行对象 + * @return 指标标签列表 + */ + private Iterable buildTags(TaskInstanceDTO taskInstance, + TaskInstanceExecuteObjects executeObjects) { + ResourceScope resourceScope = GlobalAppScopeMappingService.get().getScopeByAppId(taskInstance.getAppId()); + Tag resourceScopeTag = Tag.of(CommonMetricTags.KEY_RESOURCE_SCOPE, buildResourceScopeTagValue(resourceScope)); + Tag cronTaskIdTag = Tag.of( + ExecuteMetricsConstants.TAG_KEY_CRON_TASK_ID, + String.valueOf(taskInstance.getCronTaskId()) + ); + Tag appCodeTag = Tag.of(ExecuteMetricsConstants.TAG_KEY_APP_CODE, String.valueOf(taskInstance.getAppCode())); + Tag compositionTag = buildExecuteObjectCompositionTag(executeObjects); + return Tags.of(resourceScopeTag, cronTaskIdTag, appCodeTag, compositionTag); + } + + /** + * 构造执行对象组成标签 + * + * @param executeObjects 任务执行对象 + * @return 执行对象组成标签 + */ + private Tag buildExecuteObjectCompositionTag(TaskInstanceExecuteObjects executeObjects) { + String executeObjectCompositionTagValue; + boolean hasHost = executeObjects.isContainsAnyHost(); + boolean hasContainer = executeObjects.isContainsAnyContainer(); + if (hasHost && hasContainer) { + executeObjectCompositionTagValue = ExecuteMetricsConstants.TAG_VALUE_EXECUTE_OBJECT_COMPOSITION_MIXED; + } else if (hasHost) { + executeObjectCompositionTagValue = ExecuteMetricsConstants.TAG_VALUE_EXECUTE_OBJECT_COMPOSITION_HOST; + } else if (hasContainer) { + executeObjectCompositionTagValue = ExecuteMetricsConstants.TAG_VALUE_EXECUTE_OBJECT_COMPOSITION_CONTAINER; + } else { + executeObjectCompositionTagValue = ExecuteMetricsConstants.TAG_VALUE_EXECUTE_OBJECT_COMPOSITION_NONE; + } + return Tag.of( + ExecuteMetricsConstants.TAG_KEY_EXECUTE_OBJECT_COMPOSITION, + executeObjectCompositionTagValue + ); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/metrics/impl/LogSamplerImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/metrics/impl/LogSamplerImpl.java new file mode 100644 index 0000000000..315dba4709 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/metrics/impl/LogSamplerImpl.java @@ -0,0 +1,143 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.metrics.impl; + +import com.tencent.bk.job.execute.metrics.ExecuteMetricsConstants; +import com.tencent.bk.job.execute.metrics.LogSampler; +import com.tencent.bk.job.logsvr.model.service.ServiceBatchSaveLogRequest; +import com.tencent.bk.job.logsvr.model.service.ServiceExecuteObjectLogDTO; +import com.tencent.bk.job.logsvr.model.service.ServiceExecuteObjectScriptLogDTO; +import com.tencent.bk.job.logsvr.model.service.ServiceFileTaskLogDTO; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tag; +import io.micrometer.core.instrument.Tags; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.nio.charset.StandardCharsets; +import java.util.List; + +/** + * 记录任务日志大小指标 + */ +@Slf4j +@Service +public class LogSamplerImpl implements LogSampler { + + private final MeterRegistry meterRegistry; + + @Autowired + public LogSamplerImpl(MeterRegistry meterRegistry) { + this.meterRegistry = meterRegistry; + } + + @Override + public void tryToRecordLogSizeMetrics(ServiceBatchSaveLogRequest request) { + try { + Iterable tags = buildTags(request); + long logBytes = calcLogsTotalBytes(request.getLogs()); + if (log.isDebugEnabled()) { + log.debug( + "tryToRecordLogSizeMetrics: appId={}, cronTaskId={}, appCode={}, logBytes={}", + request.getAppId(), + request.getCronTaskId(), + request.getAppCode(), + logBytes + ); + } + Counter.builder(ExecuteMetricsConstants.NAME_JOB_LOG_BYTES) + .tags(tags) + .register(meterRegistry) + .increment(logBytes); + } catch (Exception e) { + log.warn("tryToRecordLogSizeMetrics error", e); + } + } + + /** + * 构造指标的标签 + * + * @param request 保存日志请求 + * @return 指标标签列表 + */ + private Iterable buildTags(ServiceBatchSaveLogRequest request) { + Long appId = request.getAppId(); + Long cronTaskId = request.getCronTaskId(); + String appCode = request.getAppCode(); + return Tags.of( + Tag.of(ExecuteMetricsConstants.TAG_KEY_APP_ID, String.valueOf(appId)), + Tag.of(ExecuteMetricsConstants.TAG_KEY_CRON_TASK_ID, String.valueOf(cronTaskId)), + Tag.of(ExecuteMetricsConstants.TAG_KEY_APP_CODE, String.valueOf(appCode)) + ); + } + + /** + * 计算批量日志中所有日志的字节大小总和 + * + * @param logs 日志列表 + * @return 日志总字节大小 + */ + private long calcLogsTotalBytes(List logs) { + if (CollectionUtils.isEmpty(logs)) { + return 0; + } + return logs.stream() + .mapToLong(this::calcLogBytes) + .sum(); + } + + /** + * 计算单个日志的字节数 + * + * @param log 日志内容 + * @return 单个日志的字节数 + */ + private long calcLogBytes(ServiceExecuteObjectLogDTO log) { + if (log == null) { + return 0L; + } + ServiceExecuteObjectScriptLogDTO scriptLog = log.getScriptLog(); + // 脚本类型的日志，直接获取contentSizeBytes + if (scriptLog != null) { + return scriptLog.getContentSizeBytes(); + } + // 文件类型的日志，计算content字段的字节数并汇总 + List fileTaskLogs = log.getFileTaskLogs(); + if (CollectionUtils.isEmpty(fileTaskLogs)) { + return 0L; + } + long totalBytes = 0L; + for (ServiceFileTaskLogDTO fileTaskLog : fileTaskLogs) { + String content = fileTaskLog.getContent(); + if (content != null) { + totalBytes += content.getBytes(StandardCharsets.UTF_8).length; + } + } + return totalBytes; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AccountDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AccountDTO.java index 062ef6eaff..ab0b81138e 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AccountDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AccountDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,8 +24,8 @@ package com.tencent.bk.job.execute.model; -import com.tencent.bk.job.manage.common.consts.account.AccountCategoryEnum; -import com.tencent.bk.job.manage.common.consts.account.AccountTypeEnum; +import com.tencent.bk.job.common.constant.AccountCategoryEnum; +import com.tencent.bk.job.manage.api.common.constants.account.AccountTypeEnum; import lombok.Data; /** diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AgentTaskDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AgentTaskDTO.java deleted file mode 100644 index b2fff0a631..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AgentTaskDTO.java +++ /dev/null @@ -1,278 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.model; - -import com.tencent.bk.job.common.annotation.CompatibleImplementation; -import com.tencent.bk.job.common.model.dto.HostDTO; -import com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum; -import com.tencent.bk.job.logsvr.consts.FileTaskModeEnum; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; -import lombok.ToString; -import org.apache.commons.lang3.StringUtils; - -/** - * GSE Agent 任务 - */ -@Getter -@Setter -@ToString -@NoArgsConstructor -public class AgentTaskDTO { - /** - * 步骤实例ID - */ - private long stepInstanceId; - /** - * 步骤执行次数 - */ - private int executeCount; - /** - * Agent 任务对应的实际的步骤执行次数（重试场景，可能Agent任务并没有实际被执行) - */ - private Integer actualExecuteCount; - /** - * 滚动执行批次 - */ - private int batch; - /** - * GSE 任务ID - */ - private Long gseTaskId; - /** - * 主机ID - */ - private Long hostId; - /** - * Agent ID - */ - private String agentId; - /** - * 服务器云区域+IP - */ - @Deprecated - @CompatibleImplementation(name = "rolling_execute", explain = "兼容字段，后续AgentTask仅包含hostId,不再存储具体的IP数据", - version = "3.7.x") - private String cloudIp; - /** - * 展示给用户的服务器IP - */ - @Deprecated - @CompatibleImplementation(name = "rolling_execute", explain = "兼容字段，后续AgentTask仅包含hostId,不再存储具体的IP数据", - version = "3.7.x") - private String displayIp; - /** - * 任务状态 - */ - private AgentTaskStatusEnum status; - /** - * 任务开始时间 - */ - private Long startTime; - /** - * 任务结束时间 - */ - private Long endTime; - /** - * 耗时，毫秒 - */ - private Long totalTime; - /** - * GSE返回错误码 - */ - private int errorCode; - /** - * 脚本任务-执行程序退出码， 0 脚本执行成功，非 0 脚本执行失败 - */ - private Integer exitCode; - /** - * 脚本任务-用户自定义执行结果分组 - */ - private String tag = ""; - /** - * 脚本任务-日志偏移量。Job 从 GSE 根据 scriptLogOffset 增量拉取执行日志 - */ - private int scriptLogOffset; - /** - * 脚本任务-执行日志 - */ - private String scriptLogContent; - /** - * 文件任务类型 - */ - private FileTaskModeEnum fileTaskMode; - /** - * 结果是否发生变化 - */ - private volatile boolean changed; - - public AgentTaskDTO(long stepInstanceId, int executeCount, int batch, Long hostId, String agentId) { - this.stepInstanceId = stepInstanceId; - this.executeCount = executeCount; - this.batch = batch; - this.hostId = hostId; - this.agentId = agentId; - } - - public AgentTaskDTO(long stepInstanceId, int executeCount, int batch, FileTaskModeEnum fileTaskMode, - Long hostId, String agentId) { - this.stepInstanceId = stepInstanceId; - this.executeCount = executeCount; - this.batch = batch; - this.fileTaskMode = fileTaskMode; - this.hostId = hostId; - this.agentId = agentId; - } - - public AgentTaskDTO(AgentTaskDTO agentTask) { - this.stepInstanceId = agentTask.getStepInstanceId(); - this.executeCount = agentTask.getExecuteCount(); - this.actualExecuteCount = agentTask.getActualExecuteCount(); - this.batch = agentTask.getBatch(); - this.fileTaskMode = agentTask.getFileTaskMode(); - this.hostId = agentTask.getHostId(); - this.agentId = agentTask.getAgentId(); - this.cloudIp = agentTask.getCloudIp(); - this.displayIp = agentTask.getDisplayIp(); - this.status = agentTask.getStatus(); - this.startTime = agentTask.getStartTime(); - this.endTime = agentTask.getEndTime(); - this.totalTime = agentTask.getTotalTime(); - this.errorCode = agentTask.getErrorCode(); - this.exitCode = agentTask.getExitCode(); - this.tag = agentTask.getTag(); - this.scriptLogOffset = agentTask.getScriptLogOffset(); - this.scriptLogContent = agentTask.getScriptLogContent(); - this.fileTaskMode = agentTask.getFileTaskMode(); - this.gseTaskId = agentTask.getGseTaskId(); - this.changed = agentTask.isChanged(); - } - - public void setStatus(AgentTaskStatusEnum status) { - this.changed = true; - this.status = status; - } - - public void setStartTime(Long startTime) { - this.changed = true; - this.startTime = startTime; - } - - public void setEndTime(Long endTime) { - this.changed = true; - this.endTime = endTime; - } - - public void setTotalTime(Long totalTime) { - this.changed = true; - this.totalTime = totalTime; - } - - public void setErrorCode(int errorCode) { - this.changed = true; - this.errorCode = errorCode; - } - - public void setExitCode(Integer exitCode) { - this.changed = true; - this.exitCode = exitCode; - } - - public void setScriptLogOffset(int scriptLogOffset) { - this.changed = true; - this.scriptLogOffset = scriptLogOffset; - } - - /** - * 任务是否结束 - * - * @return 任务是否结束 - */ - public boolean isFinished() { - return status != AgentTaskStatusEnum.WAITING && status != AgentTaskStatusEnum.RUNNING; - } - - /** - * 计算任务运行时间 - */ - public void calculateTotalTime() { - if (this.endTime != null && this.startTime != null && this.endTime > this.startTime) { - this.totalTime = this.endTime - this.startTime; - } - } - - /** - * 重置任务状态数据 - */ - public void resetTaskInitialStatus() { - this.status = AgentTaskStatusEnum.WAITING; - this.startTime = null; - this.endTime = null; - this.totalTime = null; - - this.errorCode = 0; - this.scriptLogOffset = 0; - this.exitCode = null; - this.gseTaskId = null; - } - - /** - * 是否任务执行目标Agent - */ - public boolean isTarget() { - // 非文件分发文件源主机，目前来说都是目标主机 - return !(fileTaskMode != null && fileTaskMode == FileTaskModeEnum.UPLOAD); - } - - public HostDTO getHost() { - HostDTO host = null; - if (hostId != null) { - host = new HostDTO(); - host.setHostId(hostId); - host.setAgentId(agentId); - } else if (StringUtils.isNotEmpty(cloudIp)) { - host = HostDTO.fromCloudIp(cloudIp); - } - - return host; - } - - /** - * 任务是否执行成功 - */ - public boolean isSuccess() { - return AgentTaskStatusEnum.isSuccess(status); - } - - public String getAgentId() { - if (StringUtils.isNotBlank(agentId)) { - return agentId; - } else { - // 兼容没有agentId的历史数据 - return getCloudIp(); - } - } -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AgentTaskDetailDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AgentTaskDetailDTO.java deleted file mode 100644 index e183776964..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AgentTaskDetailDTO.java +++ /dev/null @@ -1,88 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.model; - -import com.tencent.bk.job.common.model.dto.HostDTO; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; -import lombok.ToString; -import org.apache.commons.lang3.StringUtils; - -/** - * GSE Agent 任务详情，包含主机的详细信息 - */ -@Getter -@Setter -@ToString -@NoArgsConstructor -public class AgentTaskDetailDTO extends AgentTaskDTO { - /** - * 服务器IP,不包含云区域 - */ - private String ip; - /** - * 云区域ID - */ - private Long bkCloudId; - /** - * 云区域名称 - */ - private String bkCloudName; - /** - * 展示给用户的IP - */ - private String displayIp; - - public AgentTaskDetailDTO(AgentTaskDTO agentTask) { - super(agentTask); - if (StringUtils.isNotEmpty(agentTask.getCloudIp())) { - HostDTO host = HostDTO.fromCloudIp(agentTask.getAgentId()); - this.ip = host.getIp(); - this.bkCloudId = host.getBkCloudId(); - } - } - - public String getDisplayIp() { - if (StringUtils.isNotEmpty(displayIp)) { - return displayIp; - } else { - return ip; - } - } - - public HostDTO getHost() { - HostDTO host = new HostDTO(); - host.setHostId(getHostId()); - host.setIp(getIp()); - host.setBkCloudId(getBkCloudId()); - host.setBkCloudName(getBkCloudName()); - host.setDisplayIp(getDisplayIp()); - host.setAgentId(getAgentId()); - return host; - } - - -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AgentTaskExecutionDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AgentTaskExecutionDTO.java deleted file mode 100644 index 35758273df..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AgentTaskExecutionDTO.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.model; - -import lombok.Data; - -/** - * Agent任务执行信息 - */ -@Data -public class AgentTaskExecutionDTO { - /** - * 云IP - */ - private String cloudIp; - /** - * 执行次数 - */ - private Integer executeCount; - /** - * Agent ip显示名称，展示给用户使用该ip - */ - private String displayIp; - - /** - * 云区域ID - */ - private Long cloudAreaId; - - /** - * 云区域名称 - */ - private String cloudAreaName; - - /** - * Agent任务执行状态 - */ - private Integer status; - /** - * 开始时间 - */ - private Long startTime; - /** - * 结束时间 - */ - private Long endTime; - /** - * 耗时,毫秒 - */ - private Long totalTime; - /** - * 脚本返回码 - */ - private Integer exitCode; - /** - * 脚本错误码 - */ - private Integer errorCode; - /** - * 脚本执行输出 - */ - private String tag; - - public void calculateTotalTime() { - if (this.endTime != null && this.startTime != null && this.endTime > this.startTime) { - this.totalTime = this.endTime - this.startTime; - } - } -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AgentTaskResultGroupBaseDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AgentTaskResultGroupBaseDTO.java deleted file mode 100644 index cc54c02624..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AgentTaskResultGroupBaseDTO.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.model; - -import com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.util.Objects; - -/** - * Agent任务执行结果分组 - */ -@Data -@NoArgsConstructor -public class AgentTaskResultGroupBaseDTO implements Comparable { - /** - * 任务状态 - * - * @see AgentTaskStatusEnum - */ - private Integer status; - /** - * 用户脚本输出的自定义分组tag - */ - private String tag; - /** - * Agent任务总数 - */ - private int totalAgentTasks; - - public AgentTaskResultGroupBaseDTO(Integer status, String tag) { - this.status = status; - this.tag = tag; - } - - public AgentTaskResultGroupBaseDTO(AgentTaskResultGroupBaseDTO baseResultGroup) { - this.status = baseResultGroup.getStatus(); - this.tag = baseResultGroup.getTag(); - this.totalAgentTasks = baseResultGroup.getTotalAgentTasks(); - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - AgentTaskResultGroupBaseDTO that = (AgentTaskResultGroupBaseDTO) o; - return status.equals(that.status) && - tagEquals(tag, that.tag); - } - - private boolean tagEquals(String thisTag, String thatTag) { - String tag1 = thisTag == null ? "" : thisTag; - String tag2 = thatTag == null ? "" : thatTag; - return tag1.equals(tag2); - } - - @Override - public int hashCode() { - if (this.tag == null) { - return Objects.hash(status, ""); - } else { - return Objects.hash(status, tag); - } - } - - @Override - public int compareTo(AgentTaskResultGroupBaseDTO that) { - int result = status.compareTo(that.status); - if (result != 0) { - return result; - } - String tag1 = this.tag == null ? "" : this.tag; - String tag2 = that.tag == null ? "" : that.tag; - return tag1.compareTo(tag2); - } -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AgentTaskResultGroupDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AgentTaskResultGroupDTO.java deleted file mode 100644 index 491947df7b..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AgentTaskResultGroupDTO.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.model; - -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; -import lombok.ToString; - -import java.util.List; - -/** - * Agent任务执行结果分组 - */ -@Getter -@Setter -@ToString -@NoArgsConstructor -public class AgentTaskResultGroupDTO extends AgentTaskResultGroupBaseDTO { - /** - * Agent任务 - */ - private List agentTasks; - - public AgentTaskResultGroupDTO(Integer status, String tag) { - super(status, tag); - } - - public AgentTaskResultGroupDTO(AgentTaskResultGroupBaseDTO baseResultGroup) { - super(baseResultGroup); - } -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AtomicFileTaskLog.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AtomicFileTaskLog.java new file mode 100644 index 0000000000..e02b02bb04 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/AtomicFileTaskLog.java @@ -0,0 +1,164 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; +import com.tencent.bk.job.logsvr.model.service.ServiceFileTaskLogDTO; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.function.Function; + +/** + * 文件分发执行日志 + */ +@Data +@NoArgsConstructor +public class AtomicFileTaskLog { + /** + * 文件任务ID + */ + @JsonProperty("taskId") + private String taskId; + + /** + * 文件分发类型，mode: upload-0/download-1 + */ + @JsonProperty("mode") + private Integer mode; + + /** + * 文件源-执行对象 + */ + private ExecuteObject srcExecuteObject; + + /** + * 分发目标-执行对象 + */ + private ExecuteObject destExecuteObject; + + /** + * 目标文件路径 + */ + private String destFile; + + /** + * 源文件类型 + */ + private Integer srcFileType; + + /** + * 源文件路径 - 真实路径 + */ + private String srcFile; + + /** + * 源文件路径 - 用于显示 + */ + private String displaySrcFile; + + /** + * 文件大小 + */ + private String size; + + /** + * 文件任务状态 + */ + private Integer status; + + /** + * 文件任务状态描述 + */ + private String statusDesc; + + /** + * 速度 + */ + private String speed; + + /** + * 进度 + */ + private String process; + + /** + * 日志内容 + */ + private String content; + + public AtomicFileTaskLog(Integer mode, + ExecuteObject destExecuteObject, + String destFile, + ExecuteObject srcExecuteObject, + Integer srcFileType, + String srcFile, + String displaySrcFile, + String size, + Integer status, + String statusDesc, + String speed, + String process, + String content) { + this.mode = mode; + this.destExecuteObject = destExecuteObject; + this.destFile = destFile; + this.srcExecuteObject = srcExecuteObject; + this.srcFileType = srcFileType; + this.srcFile = srcFile; + this.displaySrcFile = displaySrcFile; + this.size = size; + this.status = status; + this.statusDesc = statusDesc; + this.speed = speed; + this.process = process; + this.content = content; + } + + public static AtomicFileTaskLog fromServiceExecuteObjectLogDTO( + ServiceFileTaskLogDTO fileTaskLog, + Function srcExecuteObjectProvider, + Function destExecuteObjectProvider) { + AtomicFileTaskLog atomicFileTaskLog = new AtomicFileTaskLog(); + atomicFileTaskLog.setMode(fileTaskLog.getMode()); + atomicFileTaskLog.setSrcExecuteObject(srcExecuteObjectProvider.apply(fileTaskLog)); + atomicFileTaskLog.setSrcFile(fileTaskLog.getSrcFile()); + atomicFileTaskLog.setSrcFileType(fileTaskLog.getSrcFileType()); + atomicFileTaskLog.setDisplaySrcFile(fileTaskLog.getDisplaySrcFile()); + atomicFileTaskLog.setDestExecuteObject(destExecuteObjectProvider.apply(fileTaskLog)); + atomicFileTaskLog.setDestFile(fileTaskLog.getDestFile()); + atomicFileTaskLog.setProcess(fileTaskLog.getProcess()); + atomicFileTaskLog.setContent(fileTaskLog.getContent()); + atomicFileTaskLog.setSize(fileTaskLog.getSize()); + atomicFileTaskLog.setSpeed(fileTaskLog.getSpeed()); + atomicFileTaskLog.setStatus(fileTaskLog.getStatus()); + atomicFileTaskLog.setStatusDesc(fileTaskLog.getStatusDesc()); + atomicFileTaskLog.setTaskId(fileTaskLog.getTaskId()); + return atomicFileTaskLog; + } +} + + diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ConfirmStepInstanceDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ConfirmStepInstanceDTO.java index ead86acbc3..8435764531 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ConfirmStepInstanceDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ConfirmStepInstanceDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -34,6 +34,7 @@ @Setter @ToString public class ConfirmStepInstanceDTO { + private Long taskInstanceId; private Long stepInstanceId; private String confirmMessage; private String confirmReason; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/DangerousRecordDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/DangerousRecordDTO.java index 6c62320e33..1984ea5e26 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/DangerousRecordDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/DangerousRecordDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -65,7 +65,7 @@ public class DangerousRecordDTO { /** * 脚本语言 * - * @see com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum + * @see com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum */ private Integer scriptLanguage; /** diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/DynamicServerGroupDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/DynamicServerGroupDTO.java index 60b7850d52..19717764bc 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/DynamicServerGroupDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/DynamicServerGroupDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,7 @@ package com.tencent.bk.job.execute.model; +import com.tencent.bk.job.common.annotation.PersistenceObject; import com.tencent.bk.job.common.model.dto.HostDTO; import lombok.Data; import lombok.NoArgsConstructor; @@ -35,6 +36,7 @@ /** * 服务器动态分组 */ +@PersistenceObject @Data @NoArgsConstructor public class DynamicServerGroupDTO implements Cloneable { diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/DynamicServerTopoNodeDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/DynamicServerTopoNodeDTO.java index 8cce29ba31..11bc8a5064 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/DynamicServerTopoNodeDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/DynamicServerTopoNodeDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,7 @@ package com.tencent.bk.job.execute.model; +import com.tencent.bk.job.common.annotation.PersistenceObject; import com.tencent.bk.job.common.model.dto.HostDTO; import lombok.Data; import lombok.NoArgsConstructor; @@ -35,6 +36,7 @@ /** * 服务器动态分组 */ +@PersistenceObject @Data @NoArgsConstructor public class DynamicServerTopoNodeDTO implements Cloneable { @@ -80,4 +82,8 @@ public DynamicServerTopoNodeDTO clone() { } return cloneTopoNode; } + + public String getUniqueKey() { + return nodeType + "_" + topoNodeId; + } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ExecuteObjectCompositeKey.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ExecuteObjectCompositeKey.java new file mode 100644 index 0000000000..2db2042106 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ExecuteObjectCompositeKey.java @@ -0,0 +1,164 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model; + +import com.tencent.bk.job.common.constant.ExecuteObjectTypeEnum; +import lombok.Getter; +import lombok.Setter; + +import java.util.Objects; + +/** + * 执行对象复合 KEY + */ +@Getter +@Setter +public class ExecuteObjectCompositeKey { + /** + * 执行对象复合 KEY 的类型 + */ + private CompositeKeyType compositeKeyType; + /** + * 执行对象 ID + */ + private String executeObjectId; + /** + * 执行对象类型 + */ + private ExecuteObjectTypeEnum executeObjectType; + /** + * 执行对象对应资源的 ID(比如主机 ID/容器 ID) + */ + private Long resourceId; + /** + * 管控区域+ipv4 方式标识主机 + */ + private String cloudIp; + + + private ExecuteObjectCompositeKey() { + } + + + public static ExecuteObjectCompositeKey ofExecuteObjectId(String executeObjectId) { + ExecuteObjectCompositeKey key = new ExecuteObjectCompositeKey(); + key.setCompositeKeyType(CompositeKeyType.EXECUTE_OBJECT_ID); + key.setExecuteObjectId(executeObjectId); + return key; + } + + public static ExecuteObjectCompositeKey ofExecuteObjectResource( + ExecuteObjectTypeEnum executeObjectType, Long executeObjectResourceId) { + ExecuteObjectCompositeKey key = new ExecuteObjectCompositeKey(); + key.setCompositeKeyType(CompositeKeyType.RESOURCE_ID); + key.setExecuteObjectType(executeObjectType); + key.setResourceId(executeObjectResourceId); + return key; + } + + public static ExecuteObjectCompositeKey ofExecuteObjectResource( + ExecuteObjectTypeEnum executeObjectType, String executeObjectResourceId) { + ExecuteObjectCompositeKey key = new ExecuteObjectCompositeKey(); + key.setCompositeKeyType(CompositeKeyType.RESOURCE_ID); + key.setExecuteObjectType(executeObjectType); + key.setResourceId(Long.valueOf(executeObjectResourceId)); + return key; + } + + public static ExecuteObjectCompositeKey ofHostId(Long hostId) { + return ofExecuteObjectResource(ExecuteObjectTypeEnum.HOST, hostId); + } + + public static ExecuteObjectCompositeKey ofHostIp(String cloudIp) { + ExecuteObjectCompositeKey key = new ExecuteObjectCompositeKey(); + key.setCompositeKeyType(CompositeKeyType.HOST_CLOUD_IP); + key.setExecuteObjectType(ExecuteObjectTypeEnum.HOST); + key.setCloudIp(cloudIp); + return key; + } + + public static ExecuteObjectCompositeKey ofContainerId(Long containerId) { + return ofExecuteObjectResource(ExecuteObjectTypeEnum.CONTAINER, containerId); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ExecuteObjectCompositeKey that = (ExecuteObjectCompositeKey) o; + if (this.getCompositeKeyType() != that.getCompositeKeyType()) { + return false; + } + switch (this.getCompositeKeyType()) { + case EXECUTE_OBJECT_ID: + return this.getExecuteObjectId().equals(that.getExecuteObjectId()); + case RESOURCE_ID: + return this.getExecuteObjectType() == that.getExecuteObjectType() + && this.getResourceId().equals(that.getResourceId()); + case HOST_CLOUD_IP: + return this.getCloudIp().equals(that.getCloudIp()); + } + return false; + } + + @Override + public int hashCode() { + switch (this.getCompositeKeyType()) { + case EXECUTE_OBJECT_ID: + return Objects.hash(compositeKeyType, executeObjectId); + case RESOURCE_ID: + return Objects.hash(compositeKeyType, executeObjectType.getValue(), resourceId); + case HOST_CLOUD_IP: + return Objects.hash(compositeKeyType, cloudIp); + default: + return 0; + } + } + + public enum CompositeKeyType { + /** + * 执行对象ID 作为 KEY + */ + EXECUTE_OBJECT_ID(1), + /** + * 执行对象 <资源类型+资源ID> 作为 KEY + */ + RESOURCE_ID(2), + /** + * 主机类型的执行对象，使用<管控区域+ipv4> 作为 KEY + */ + HOST_CLOUD_IP(3); + + private final int value; + + CompositeKeyType(int value) { + this.value = value; + } + + public int getValue() { + return value; + } + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ExecuteObjectTask.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ExecuteObjectTask.java new file mode 100644 index 0000000000..8e86526db1 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ExecuteObjectTask.java @@ -0,0 +1,357 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model; + +import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.CompatibleType; +import com.tencent.bk.job.common.constant.ExecuteObjectTypeEnum; +import com.tencent.bk.job.execute.engine.consts.ExecuteObjectTaskStatusEnum; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; +import com.tencent.bk.job.logsvr.consts.FileTaskModeEnum; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; + +/** + * JOB执行对象任务 + */ +@ToString +@NoArgsConstructor +public class ExecuteObjectTask { + /** + * 执行对象任务 ID + */ + @Getter + @Setter + private Long id; + /** + * 作业实例ID + */ + @Setter + @Getter + private long taskInstanceId; + /** + * 步骤实例ID + */ + @Setter + @Getter + private long stepInstanceId; + /** + * 步骤执行次数 + */ + @Setter + @Getter + private int executeCount; + /** + * 任务对应的实际的步骤执行次数（重试场景，可能任务并没有实际被执行) + */ + @Setter + @Getter + private Integer actualExecuteCount; + /** + * 滚动执行批次 + */ + @Setter + @Getter + private int batch; + /** + * GSE 任务ID + */ + @Setter + @Getter + private Long gseTaskId; + /** + * 执行对象 ID + */ + @Setter + @Getter + private String executeObjectId; + /** + * 执行对象类型 + */ + @Setter + @Getter + private ExecuteObjectTypeEnum executeObjectType; + /** + * 执行对象 + */ + @Getter + private ExecuteObject executeObject; + /** + * 主机 ID + */ + @Deprecated + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA, + explain = "兼容老数据，数据失效后可删除") + private Long hostId; + /** + * 主机 agentId + */ + @Deprecated + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA, + explain = "兼容老数据，数据失效后可删除") + private String agentId; + /** + * 任务状态 + */ + @Getter + private ExecuteObjectTaskStatusEnum status; + /** + * 任务开始时间 + */ + @Getter + private Long startTime; + /** + * 任务结束时间 + */ + @Getter + private Long endTime; + /** + * 耗时，毫秒 + */ + @Getter + private Long totalTime; + /** + * GSE返回错误码 + */ + @Getter + private int errorCode; + /** + * 脚本任务-执行程序退出码， 0 脚本执行成功，非 0 脚本执行失败 + */ + @Getter + private Integer exitCode; + /** + * 脚本任务-用户自定义执行结果分组 + */ + @Setter + @Getter + private String tag = ""; + /** + * 脚本任务-日志偏移量。Job 从 GSE 根据 scriptLogOffset 增量拉取执行日志 + */ + @Getter + private int scriptLogOffset; + /** + * 脚本任务-执行日志 + */ + @Setter + @Getter + private String scriptLogContent; + /** + * 文件任务类型 + */ + @Setter + @Getter + private FileTaskModeEnum fileTaskMode; + /** + * 结果是否发生变化 + */ + @Setter + @Getter + private volatile boolean changed; + + public ExecuteObjectTask(long taskInstanceId, + long stepInstanceId, + int executeCount, + int batch, + ExecuteObjectTypeEnum executeObjectType, + String executeObjectId) { + this.taskInstanceId = taskInstanceId; + this.stepInstanceId = stepInstanceId; + this.executeCount = executeCount; + this.batch = batch; + this.executeObjectType = executeObjectType; + this.executeObjectId = executeObjectId; + } + + public ExecuteObjectTask(long taskInstanceId, + long stepInstanceId, + int executeCount, + int batch, + ExecuteObject executeObject) { + this.taskInstanceId = taskInstanceId; + this.stepInstanceId = stepInstanceId; + this.executeCount = executeCount; + this.batch = batch; + setExecuteObject(executeObject); + } + + public ExecuteObjectTask(long taskInstanceId, + long stepInstanceId, + int executeCount, + int batch, + FileTaskModeEnum fileTaskMode, + ExecuteObjectTypeEnum executeObjectType, + String executeObjectId) { + this.taskInstanceId = taskInstanceId; + this.stepInstanceId = stepInstanceId; + this.executeCount = executeCount; + this.batch = batch; + this.fileTaskMode = fileTaskMode; + this.executeObjectType = executeObjectType; + this.executeObjectId = executeObjectId; + } + + public ExecuteObjectTask(long taskInstanceId, + long stepInstanceId, + int executeCount, + int batch, + FileTaskModeEnum fileTaskMode, + ExecuteObject executeObject) { + this.taskInstanceId = taskInstanceId; + this.stepInstanceId = stepInstanceId; + this.executeCount = executeCount; + this.batch = batch; + this.fileTaskMode = fileTaskMode; + setExecuteObject(executeObject); + } + + public void setStatus(ExecuteObjectTaskStatusEnum status) { + this.changed = true; + this.status = status; + } + + public void setStartTime(Long startTime) { + this.changed = true; + this.startTime = startTime; + } + + public void setEndTime(Long endTime) { + this.changed = true; + this.endTime = endTime; + } + + public void setTotalTime(Long totalTime) { + this.changed = true; + this.totalTime = totalTime; + } + + public void setErrorCode(int errorCode) { + this.changed = true; + this.errorCode = errorCode; + } + + public void setExitCode(Integer exitCode) { + this.changed = true; + this.exitCode = exitCode; + } + + public void setScriptLogOffset(int scriptLogOffset) { + this.changed = true; + this.scriptLogOffset = scriptLogOffset; + } + + /** + * 任务是否结束 + * + * @return 任务是否结束 + */ + public boolean isFinished() { + return status != ExecuteObjectTaskStatusEnum.WAITING && status != ExecuteObjectTaskStatusEnum.RUNNING; + } + + /** + * 计算任务运行时间 + */ + public void calculateTotalTime() { + if (this.endTime != null && this.startTime != null && this.endTime > this.startTime) { + this.totalTime = this.endTime - this.startTime; + } + } + + /** + * 重置任务状态数据 + */ + public void resetTaskInitialStatus() { + this.status = ExecuteObjectTaskStatusEnum.WAITING; + this.startTime = null; + this.endTime = null; + this.totalTime = null; + + this.errorCode = 0; + this.scriptLogOffset = 0; + this.exitCode = null; + this.gseTaskId = null; + } + + /** + * 是否任务执行目标Agent + */ + public boolean isTarget() { + // 非文件分发文件源主机，目前来说都是目标主机 + return !(fileTaskMode != null && fileTaskMode == FileTaskModeEnum.UPLOAD); + } + + + /** + * 任务是否执行成功 + */ + public boolean isSuccess() { + return ExecuteObjectTaskStatusEnum.isSuccess(status); + } + + @Deprecated + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA, + explain = "兼容老数据，数据失效后可删除") + public Long getHostId() { + return hostId; + } + + @Deprecated + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA, + explain = "兼容老数据，数据失效后可删除") + public void setHostId(Long hostId) { + this.hostId = hostId; + } + + @Deprecated + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA, + explain = "兼容老数据，数据失效后可删除") + public String getAgentId() { + return agentId; + } + + @Deprecated + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA, + explain = "兼容老数据，数据失效后可删除") + public void setAgentId(String agentId) { + this.agentId = agentId; + } + + public void setExecuteObject(ExecuteObject executeObject) { + this.executeObject = executeObject; + if (executeObject != null) { + executeObjectType = executeObject.getType(); + executeObjectId = executeObject.getId(); + if (executeObject.isHostExecuteObject()) { + // 兼容老数据，发布完成后可删除 + hostId = executeObject.getHost().getHostId(); + agentId = executeObject.getHost().getAgentId(); + } + } + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ExecuteTargetDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ExecuteTargetDTO.java new file mode 100644 index 0000000000..982830e46c --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ExecuteTargetDTO.java @@ -0,0 +1,634 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.tencent.bk.job.common.annotation.PersistenceObject; +import com.tencent.bk.job.common.esb.model.job.EsbIpDTO; +import com.tencent.bk.job.common.esb.model.job.v3.EsbServerV3DTO; +import com.tencent.bk.job.common.gse.util.AgentUtils; +import com.tencent.bk.job.common.model.dto.Container; +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.common.model.openapi.v3.EsbCmdbTopoNodeDTO; +import com.tencent.bk.job.common.model.openapi.v3.EsbDynamicGroupDTO; +import com.tencent.bk.job.common.model.openapi.v4.OpenApiExecuteTargetDTO; +import com.tencent.bk.job.common.model.vo.ContainerVO; +import com.tencent.bk.job.common.model.vo.HostInfoVO; +import com.tencent.bk.job.common.model.vo.TaskExecuteObjectsInfoVO; +import com.tencent.bk.job.common.model.vo.TaskHostNodeVO; +import com.tencent.bk.job.common.model.vo.TaskTargetVO; +import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; +import com.tencent.bk.job.execute.model.inner.ServiceExecuteTargetDTO; +import com.tencent.bk.job.execute.util.label.selector.LabelSelectorParse; +import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * 任务执行目标 DTO + */ +@Data +@PersistenceObject +@JsonInclude(JsonInclude.Include.NON_EMPTY) +@Slf4j +public class ExecuteTargetDTO implements Cloneable { + /** + * 如果执行目标是通过全局变量-主机列表定义的，variable 表示变量 name + */ + private String variable; + /** + * 用户选择的主机列表（静态） + */ + private List staticIpList; + + /** + * 用户选择的执行对象列表（静态） + */ + private List staticContainerList; + + /** + * 主机动态分组列表 + */ + private List dynamicServerGroups; + + /** + * 主机拓扑节点 + */ + private List topoNodes; + + /** + * 主机动态分组、静态主机列表、动态主机topo的所有主机的集合。 + * + * @deprecated 使用 executeObjects 替换 + */ + @Deprecated + private List ipList; + + /** + * 容器过滤器 + */ + private List containerFilters; + + /** + * 执行对象列表(所有主机+容器） + */ + private List executeObjects; + + public ExecuteTargetDTO clone() { + ExecuteTargetDTO clone = new ExecuteTargetDTO(); + clone.setVariable(variable); + if (CollectionUtils.isNotEmpty(staticIpList)) { + List cloneStaticIpList = new ArrayList<>(staticIpList.size()); + staticIpList.forEach(staticIp -> cloneStaticIpList.add(staticIp.clone())); + clone.setStaticIpList(cloneStaticIpList); + } + if (CollectionUtils.isNotEmpty(dynamicServerGroups)) { + List cloneServerGroups = new ArrayList<>(dynamicServerGroups.size()); + dynamicServerGroups.forEach(serverGroup -> cloneServerGroups.add(serverGroup.clone())); + clone.setDynamicServerGroups(cloneServerGroups); + } + if (CollectionUtils.isNotEmpty(topoNodes)) { + clone.setTopoNodes(topoNodes); + } + if (CollectionUtils.isNotEmpty(staticContainerList)) { + List cloneContainerList = new ArrayList<>(staticContainerList.size()); + staticContainerList.forEach(container -> cloneContainerList.add(container.clone())); + clone.setStaticContainerList(cloneContainerList); + } + if (CollectionUtils.isNotEmpty(ipList)) { + List cloneIpList = new ArrayList<>(ipList.size()); + ipList.forEach(ip -> cloneIpList.add(ip.clone())); + clone.setIpList(cloneIpList); + } + if (CollectionUtils.isNotEmpty(containerFilters)) { + List cloneContainerFilters = new ArrayList<>(containerFilters.size()); + containerFilters.forEach(containerFilter -> cloneContainerFilters.add(containerFilter.clone())); + clone.setContainerFilters(cloneContainerFilters); + } + if (CollectionUtils.isNotEmpty(executeObjects)) { + List cloneExecuteObjectList = new ArrayList<>(executeObjects.size()); + executeObjects.forEach(executeObject -> cloneExecuteObjectList.add(executeObject.clone())); + clone.setExecuteObjects(cloneExecuteObjectList); + } + return clone; + } + + public ExecuteTargetDTO merge(ExecuteTargetDTO executeObjects) { + if (executeObjects == null) { + return this; + } + if (executeObjects.getStaticIpList() != null) { + if (this.staticIpList == null) { + this.staticIpList = new ArrayList<>(executeObjects.getStaticIpList()); + } else { + executeObjects.getStaticIpList().forEach(ipDTO -> { + if (!this.staticIpList.contains(ipDTO)) { + this.staticIpList.add(ipDTO); + } + }); + } + } + if (executeObjects.getTopoNodes() != null) { + if (this.topoNodes == null) { + this.topoNodes = new ArrayList<>(executeObjects.getTopoNodes()); + } else { + executeObjects.getTopoNodes().forEach(topoNode -> { + if (!this.topoNodes.contains(topoNode)) { + this.topoNodes.add(topoNode); + } + }); + } + } + if (executeObjects.getDynamicServerGroups() != null) { + if (this.dynamicServerGroups == null) { + this.dynamicServerGroups = new ArrayList<>(executeObjects.getDynamicServerGroups()); + } else { + executeObjects.getDynamicServerGroups().forEach(dynamicServerGroup -> { + if (!this.dynamicServerGroups.contains(dynamicServerGroup)) { + this.dynamicServerGroups.add(dynamicServerGroup); + } + }); + } + } + if (executeObjects.getStaticContainerList() != null) { + if (this.staticContainerList == null) { + this.staticContainerList = new ArrayList<>(executeObjects.getStaticContainerList()); + } else { + executeObjects.getStaticContainerList().forEach(container -> { + if (!this.staticContainerList.contains(container)) { + this.staticContainerList.add(container); + } + }); + } + } + return this; + } + + public void addStaticHosts(Collection hosts) { + if (staticIpList == null) { + staticIpList = new ArrayList<>(); + } + staticIpList.addAll(hosts); + } + + /** + * 执行对象是否为空 + */ + @JsonIgnore + public boolean isEmpty() { + return CollectionUtils.isEmpty(this.staticIpList) + && CollectionUtils.isEmpty(this.topoNodes) + && CollectionUtils.isEmpty(this.dynamicServerGroups) + && CollectionUtils.isEmpty(this.staticContainerList); + } + + /** + * 提取所有包含的主机 + * + * @return 主机列表 + */ + public List extractHosts() { + List hosts = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(staticIpList)) { + hosts.addAll(staticIpList); + } + if (CollectionUtils.isNotEmpty(dynamicServerGroups)) { + dynamicServerGroups.stream() + .filter(group -> CollectionUtils.isNotEmpty(group.getIpList())) + .forEach(group -> hosts.addAll(group.getIpList())); + } + if (CollectionUtils.isNotEmpty(topoNodes)) { + topoNodes.stream() + .filter(topoNode -> CollectionUtils.isNotEmpty(topoNode.getIpList())) + .forEach(topoNode -> hosts.addAll(topoNode.getIpList())); + } + return hosts.stream().distinct().collect(Collectors.toList()); + } + + public TaskTargetVO convertToTaskTargetVO() { + TaskTargetVO target = new TaskTargetVO(); + target.setVariable(variable); + + List executeObjects = getExecuteObjectsCompatibly(); + TaskExecuteObjectsInfoVO taskExecuteObjectsInfoVO = new TaskExecuteObjectsInfoVO(); + if (CollectionUtils.isNotEmpty(executeObjects)) { + // 主机 + List hostInfoVOS = executeObjects.stream() + .filter(ExecuteObject::isHostExecuteObject) + .map(ExecuteObject::getHost) + .map(host -> { + HostInfoVO hostInfoVO = host.toHostInfoVO(); + hostInfoVO.setAgentId(AgentUtils.displayAsRealAgentId(host.getAgentId())); + return hostInfoVO; + }) + .collect(Collectors.toList()); + if (CollectionUtils.isNotEmpty(hostInfoVOS)) { + taskExecuteObjectsInfoVO.setHostList(hostInfoVOS); + TaskHostNodeVO taskHostNodeVO = new TaskHostNodeVO(); + taskHostNodeVO.setHostList(hostInfoVOS); + // 发布兼容 + target.setHostNodeInfo(taskHostNodeVO); + } + + // 容器 + List containerVOs = executeObjects.stream() + .filter(ExecuteObject::isContainerExecuteObject) + .map(ExecuteObject::getContainer) + .map(Container::toContainerVO) + .collect(Collectors.toList()); + taskExecuteObjectsInfoVO.setContainerList(containerVOs); + } + target.setExecuteObjectsInfo(taskExecuteObjectsInfoVO); + + return target; + } + + /** + * 转换为 EsbServerV3DTO + */ + public EsbServerV3DTO toEsbServerV3DTO() { + EsbServerV3DTO esbServerV3DTO = new EsbServerV3DTO(); + esbServerV3DTO.setVariable(variable); + if (!CollectionUtils.isEmpty(staticIpList)) { + List ips = staticIpList.stream().map(EsbIpDTO::fromHost).collect(Collectors.toList()); + esbServerV3DTO.setIps(ips); + } + if (!CollectionUtils.isEmpty(topoNodes)) { + List esbTopoNodes = topoNodes.stream().map(topoNode -> { + EsbCmdbTopoNodeDTO esbCmdbTopoNodeDTO = new EsbCmdbTopoNodeDTO(); + esbCmdbTopoNodeDTO.setId(topoNode.getTopoNodeId()); + esbCmdbTopoNodeDTO.setNodeType(topoNode.getNodeType()); + return esbCmdbTopoNodeDTO; + }).collect(Collectors.toList()); + esbServerV3DTO.setTopoNodes(esbTopoNodes); + } + if (!CollectionUtils.isEmpty(dynamicServerGroups)) { + List dynamicGroups = dynamicServerGroups.stream().map(dynamicServerGroup -> { + EsbDynamicGroupDTO esbDynamicGroupDTO = new EsbDynamicGroupDTO(); + esbDynamicGroupDTO.setId(dynamicServerGroup.getGroupId()); + return esbDynamicGroupDTO; + }).collect(Collectors.toList()); + esbServerV3DTO.setDynamicGroups(dynamicGroups); + } + return esbServerV3DTO; + } + + public ExecuteObject findExecuteObjectByCompositeKey(ExecuteObjectCompositeKey executeObjectCompositeKey) { + ExecuteObjectCompositeKey.CompositeKeyType compositeKeyType = executeObjectCompositeKey.getCompositeKeyType(); + switch (compositeKeyType) { + case EXECUTE_OBJECT_ID: + return getExecuteObjectsCompatibly().stream() + .filter(executeObject -> + executeObjectCompositeKey.getExecuteObjectId().equals(executeObject.getId())) + .findFirst() + .orElse(null); + case RESOURCE_ID: + return getExecuteObjectsCompatibly().stream() + .filter(executeObject -> + executeObject.getType() == executeObjectCompositeKey.getExecuteObjectType() + && executeObject.getResourceId().equals(executeObjectCompositeKey.getResourceId())) + .findFirst() + .orElse(null); + case HOST_CLOUD_IP: + // 兼容使用 云区域+ip 的方式 + return getExecuteObjectsCompatibly().stream() + .filter(executeObject -> executeObject.isHostExecuteObject() + && executeObjectCompositeKey.getCloudIp().equals(executeObject.getHost().toCloudIp())) + .findFirst() + .orElse(null); + default: + throw new IllegalArgumentException("InvalidExecuteObjectCompositeKey"); + } + } + + public List findExecuteObjectByCompositeKeys( + Collection executeObjectCompositeKeys) { + if (CollectionUtils.isEmpty(executeObjectCompositeKeys)) { + return Collections.emptyList(); + } + ExecuteObjectCompositeKey anyKey = executeObjectCompositeKeys.stream().findFirst().orElse(null); + Objects.requireNonNull(anyKey); + + ExecuteObjectCompositeKey.CompositeKeyType compositeKeyType = anyKey.getCompositeKeyType(); + switch (compositeKeyType) { + case EXECUTE_OBJECT_ID: + List executeObjectIds = + executeObjectCompositeKeys.stream() + .map(ExecuteObjectCompositeKey::getExecuteObjectId) + .collect(Collectors.toList()); + return getExecuteObjectsCompatibly().stream() + .filter(executeObject -> executeObjectIds.contains(executeObject.getId())) + .collect(Collectors.toList()); + case RESOURCE_ID: + return getExecuteObjectsCompatibly().stream() + .filter(executeObject -> + executeObjectCompositeKeys.contains( + executeObject.toExecuteObjectCompositeKey( + ExecuteObjectCompositeKey.CompositeKeyType.RESOURCE_ID))) + .collect(Collectors.toList()); + case HOST_CLOUD_IP: + // 兼容使用 管控区域+ip 的方式 + return getExecuteObjectsCompatibly().stream() + .filter(executeObject -> + executeObject.isHostExecuteObject() && + executeObjectCompositeKeys.contains( + executeObject.toExecuteObjectCompositeKey( + ExecuteObjectCompositeKey.CompositeKeyType.HOST_CLOUD_IP))) + .collect(Collectors.toList()); + default: + throw new IllegalArgumentException("InvalidExecuteObjectCompositeKey"); + } + } + + /** + * 转换TaskTargetVO 为 ExecuteObjectsDTO + */ + public static ExecuteTargetDTO fromTaskTargetVO(TaskTargetVO target) { + + ExecuteTargetDTO executeTargetDTO = new ExecuteTargetDTO(); + executeTargetDTO.setVariable(target.getVariable()); + if (target.getExecuteObjectsInfoCompatibly() == null) { + return executeTargetDTO; + } + + TaskExecuteObjectsInfoVO taskExecuteObjectsInfoVO = target.getExecuteObjectsInfoCompatibly(); + + // 处理主机 + if (CollectionUtils.isNotEmpty(taskExecuteObjectsInfoVO.getHostList())) { + List hostList = new ArrayList<>(); + taskExecuteObjectsInfoVO.getHostList().forEach(host -> { + HostDTO targetHost = new HostDTO(); + if (host.getHostId() != null) { + targetHost.setHostId(host.getHostId()); + } + hostList.add(targetHost); + }); + executeTargetDTO.setStaticIpList(hostList); + } + if (CollectionUtils.isNotEmpty(taskExecuteObjectsInfoVO.getDynamicGroupList())) { + List dynamicServerGroups = new ArrayList<>(); + taskExecuteObjectsInfoVO.getDynamicGroupList().forEach( + group -> dynamicServerGroups.add(new DynamicServerGroupDTO(group.getId()))); + executeTargetDTO.setDynamicServerGroups(dynamicServerGroups); + } + if (CollectionUtils.isNotEmpty(taskExecuteObjectsInfoVO.getNodeList())) { + List topoNodes = new ArrayList<>(); + taskExecuteObjectsInfoVO.getNodeList().forEach( + topoNode -> topoNodes.add(new DynamicServerTopoNodeDTO(topoNode.getInstanceId(), + topoNode.getObjectId()))); + executeTargetDTO.setTopoNodes(topoNodes); + } + + // 处理容器 + if (CollectionUtils.isNotEmpty(taskExecuteObjectsInfoVO.getContainerList())) { + List containerList = new ArrayList<>(); + taskExecuteObjectsInfoVO.getContainerList().forEach(container -> { + Container targetContainer = new Container(); + if (container.getId() != null) { + targetContainer.setId(container.getId()); + } + containerList.add(targetContainer); + }); + executeTargetDTO.setStaticContainerList(containerList); + } + + return executeTargetDTO; + } + + /** + * 获取所有执行对象列表(兼容当前版本+历史版本数据） + */ + @JsonIgnore + public List getExecuteObjectsCompatibly() { + if (executeObjects != null) { + return executeObjects; + } else if (ipList != null) { + return ipList.stream().map(ExecuteObject::buildCompatibleExecuteObject).collect(Collectors.toList()); + } else { + return Collections.emptyList(); + } + } + + /** + * 计算所有执行对象数量(兼容当前版本+历史版本数据） + */ + @JsonIgnore + public int getExecuteObjectsCountCompatibly() { + if (executeObjects != null) { + return executeObjects.size(); + } else if (ipList != null) { + return ipList.size(); + } else { + return 0; + } + } + + /** + * 获取所有主机执行对象列表(兼容当前版本+历史版本数据） + */ + @JsonIgnore + public List getHostsCompatibly() { + if (executeObjects != null) { + return executeObjects.stream().filter(ExecuteObject::isHostExecuteObject) + .map(ExecuteObject::getHost).collect(Collectors.toList()); + } else if (ipList != null) { + return ipList; + } else { + return Collections.emptyList(); + } + } + + + /** + * 合并所有的执行对象 + * + * @param isSupportExecuteObjectFeature 是否支持执行对象特性 + */ + public void buildMergedExecuteObjects(boolean isSupportExecuteObjectFeature) { + if (isSupportExecuteObjectFeature) { + // 支持执行对象，写入 executeObjects 字段 + List executeObjects = new ArrayList<>(); + List hosts = extractHosts(); + if (CollectionUtils.isNotEmpty(hosts)) { + executeObjects.addAll(hosts.stream().map(ExecuteObject::new) + .collect(Collectors.toList())); + } + if (CollectionUtils.isNotEmpty(staticContainerList)) { + executeObjects.addAll(staticContainerList.stream().map(ExecuteObject::new) + .collect(Collectors.toList())); + } + if (CollectionUtils.isNotEmpty(containerFilters)) { + containerFilters.forEach(containerFilter -> { + if (CollectionUtils.isNotEmpty(containerFilter.getContainers())) { + executeObjects.addAll(containerFilter.getContainers().stream().map(ExecuteObject::new) + .collect(Collectors.toList())); + } + }); + } + this.executeObjects = executeObjects; + } else { + // 兼容方式，写入 ipList 字段 + this.ipList = extractHosts(); + } + } + + public static ExecuteTargetDTO buildFrom(OpenApiExecuteTargetDTO executeTarget) { + if (executeTarget == null) { + return null; + } + ExecuteTargetDTO executeTargetDTO = new ExecuteTargetDTO(); + + // 主机拓扑节点 + if (CollectionUtils.isNotEmpty(executeTarget.getHostTopoNodes())) { + List topoNodes = new ArrayList<>(); + executeTarget.getHostTopoNodes().forEach( + topoNode -> topoNodes.add(new DynamicServerTopoNodeDTO(topoNode.getId(), + topoNode.getNodeType()))); + executeTargetDTO.setTopoNodes(topoNodes); + } + + // 主机动态分组 + if (CollectionUtils.isNotEmpty(executeTarget.getHostDynamicGroups())) { + List dynamicServerGroups = new ArrayList<>(); + executeTarget.getHostDynamicGroups().forEach( + group -> dynamicServerGroups.add(new DynamicServerGroupDTO(group.getId()))); + executeTargetDTO.setDynamicServerGroups(dynamicServerGroups); + } + + // 静态主机列表 + if (CollectionUtils.isNotEmpty(executeTarget.getHosts())) { + executeTargetDTO.setStaticIpList( + executeTarget.getHosts().stream() + .map(host -> new HostDTO(host.getHostId(), host.getBkCloudId(), host.getIp())) + .collect(Collectors.toList())); + } + + // 容器过滤器 + if (CollectionUtils.isNotEmpty(executeTarget.getKubeContainerFilters())) { + executeTargetDTO.setContainerFilters(convertToKubeContainerFilter(executeTarget)); + } + + return executeTargetDTO; + } + + private static List convertToKubeContainerFilter(OpenApiExecuteTargetDTO executeTarget) { + List kubeContainerFilters = new ArrayList<>(); + + executeTarget.getKubeContainerFilters().forEach(originContainerFilter -> { + KubeContainerFilter containerFilter = new KubeContainerFilter(); + if (originContainerFilter.getClusterFilter() != null) { + KubeClusterFilter clusterFilter = new KubeClusterFilter(); + clusterFilter.setClusterUIDs(originContainerFilter.getClusterFilter().getClusterUIDs()); + containerFilter.setClusterFilter(clusterFilter); + } + if (originContainerFilter.getNamespaceFilter() != null) { + KubeNamespaceFilter namespaceFilter = new KubeNamespaceFilter(); + namespaceFilter.setNamespaces(originContainerFilter.getNamespaceFilter().getNamespaces()); + containerFilter.setNamespaceFilter(namespaceFilter); + } + if (originContainerFilter.getWorkloadFilter() != null) { + KubeWorkloadFilter workloadFilter = new KubeWorkloadFilter(); + workloadFilter.setKind(originContainerFilter.getWorkloadFilter().getKind()); + workloadFilter.setWorkloadNames(originContainerFilter.getWorkloadFilter().getWorkloadNames()); + containerFilter.setWorkloadFilter(workloadFilter); + } + if (originContainerFilter.getPodFilter() != null) { + KubePodFilter podFilter = new KubePodFilter(); + podFilter.setPodNames(originContainerFilter.getPodFilter().getPodNames()); + if (CollectionUtils.isNotEmpty(originContainerFilter.getPodFilter().getLabelSelector())) { + // 优先解析自定义的 Label Selector + podFilter.setLabelSelector( + originContainerFilter.getPodFilter().getLabelSelector() + .stream() + .map(labelSelectExpr -> new LabelSelectExprDTO( + labelSelectExpr.getKey(), + labelSelectExpr.getOperator(), + labelSelectExpr.getValues())) + .collect(Collectors.toList())); + + } else if (StringUtils.isNotBlank(originContainerFilter.getPodFilter().getLabelSelectorExpr())) { + // 解析 label selector 表达式 + List labelSelectExprList = LabelSelectorParse.parseToLabelSelectExprList( + originContainerFilter.getPodFilter().getLabelSelectorExpr()); + log.info("Parse kubernetes label selector expr, expr: {}, result: {}", + originContainerFilter.getPodFilter().getLabelSelectorExpr(), + JsonUtils.toJson(labelSelectExprList)); + podFilter.setLabelSelector(labelSelectExprList); + } + containerFilter.setPodFilter(podFilter); + } + if (originContainerFilter.getContainerPropFilter() != null) { + KubeContainerPropFilter containerPropFilter = new KubeContainerPropFilter(); + containerPropFilter.setContainerNames( + originContainerFilter.getContainerPropFilter().getContainerNames()); + containerFilter.setContainerPropFilter(containerPropFilter); + } + containerFilter.setEmptyFilter(originContainerFilter.isEmptyFilter()); + containerFilter.setFetchAnyOneContainer(originContainerFilter.isFetchAnyOneContainer()); + + kubeContainerFilters.add(containerFilter); + }); + + return kubeContainerFilters; + } + + /** + * 执行目标中是否包含容器执行对象 + */ + public boolean hasContainerExecuteObject() { + return CollectionUtils.isNotEmpty(staticContainerList) || CollectionUtils.isNotEmpty(containerFilters); + } + + /** + * 执行目标中是否包含主机执行对象 + */ + public boolean hasHostExecuteObject() { + return CollectionUtils.isNotEmpty(staticIpList) || CollectionUtils.isNotEmpty(dynamicServerGroups) + || CollectionUtils.isNotEmpty(topoNodes); + } + + public ServiceExecuteTargetDTO toServiceExecuteTargetDTO() { + ServiceExecuteTargetDTO serviceExecuteTargetDTO = new ServiceExecuteTargetDTO(); + serviceExecuteTargetDTO.setVariable(variable); + if (executeObjects != null) { + serviceExecuteTargetDTO.setExecuteObjects( + executeObjects.stream() + .map(ExecuteObject::toServiceExecuteObject) + .collect(Collectors.toList()) + ); + } + return serviceExecuteTargetDTO; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ExternalHostDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ExternalHostDTO.java new file mode 100644 index 0000000000..99af12cd5e --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ExternalHostDTO.java @@ -0,0 +1,49 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model; + +import com.tencent.bk.job.common.model.dto.HostDTO; +import lombok.Data; + +/** + * 使用集群外主机作为文件分发源机器时，用于接收配置中的主机信息 + */ +@Data +public class ExternalHostDTO { + + /** + * 云区域ID + */ + private Long bkCloudId; + + /** + * 主机IP - IPv4 + */ + private String ip; + + public HostDTO convertToHostDTO() { + return new HostDTO(bkCloudId, ip); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/FastTaskDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/FastTaskDTO.java index ce1c2fce37..3ae576f18e 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/FastTaskDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/FastTaskDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -45,6 +45,11 @@ public class FastTaskDTO { * 滚动配置 */ private StepRollingConfigDTO rollingConfig; + /** + * 是否启动任务 + */ + @Builder.Default + private Boolean startTask = true; /** * 是否滚动执行 diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/FileDetailDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/FileDetailDTO.java index c0dff5abc9..66704321eb 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/FileDetailDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/FileDetailDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,6 +25,7 @@ package com.tencent.bk.job.execute.model; import com.fasterxml.jackson.annotation.JsonInclude; +import com.tencent.bk.job.execute.model.inner.ServiceFileDetailDTO; import lombok.Data; import lombok.NoArgsConstructor; @@ -92,4 +93,10 @@ public FileDetailDTO clone() { fileDetailDTO.setResolvedFilePath(resolvedFilePath); return fileDetailDTO; } + + public ServiceFileDetailDTO toServiceFileDetailDTO() { + ServiceFileDetailDTO serviceFileDetailDTO = new ServiceFileDetailDTO(); + serviceFileDetailDTO.setFilePath(filePath); + return serviceFileDetailDTO; + } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/FileExecuteObjectLogContent.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/FileExecuteObjectLogContent.java new file mode 100644 index 0000000000..b0e9f8cd89 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/FileExecuteObjectLogContent.java @@ -0,0 +1,88 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model; + +import com.tencent.bk.job.execute.engine.model.ExecuteObject; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.apache.commons.lang3.StringUtils; + +import java.util.List; + +/** + * 执行对象对应的文件日志内容 + */ +@Data +@NoArgsConstructor +public class FileExecuteObjectLogContent { + /** + * 步骤实例ID + */ + private long stepInstanceId; + /** + * 执行次数 + */ + private int executeCount; + /** + * 目标执行对象 + */ + private ExecuteObject executeObject; + /** + * 日志内容 + */ + private String content; + /** + * 文件任务 + */ + private List fileTaskLogs; + /** + * 日志是否拉取完成 + */ + private boolean finished; + + public FileExecuteObjectLogContent(long stepInstanceId, + int executeCount, + ExecuteObject executeObject, + List fileTaskLogs, + boolean finished) { + this.stepInstanceId = stepInstanceId; + this.executeCount = executeCount; + this.executeObject = executeObject; + this.fileTaskLogs = fileTaskLogs; + this.finished = finished; + } + + public String getContent() { + if (StringUtils.isNotEmpty(content)) { + return this.content; + } + StringBuilder builder = new StringBuilder(); + if (this.fileTaskLogs == null) { + return null; + } + fileTaskLogs.forEach(fileTaskLog -> builder.append(fileTaskLog.getContent())); + return builder.toString(); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/FileIpLogContent.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/FileIpLogContent.java deleted file mode 100644 index a836caeff8..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/FileIpLogContent.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.model; - -import com.tencent.bk.job.logsvr.model.service.ServiceFileTaskLogDTO; -import lombok.Data; -import lombok.NoArgsConstructor; -import org.apache.commons.lang3.StringUtils; - -import java.util.List; - -/** - * 文件日志内容 - */ -@Data -@NoArgsConstructor -public class FileIpLogContent { - /** - * 步骤实例ID - */ - private long stepInstanceId; - /** - * 执行次数 - */ - private int executeCount; - /** - * 目标IP - */ - private String ip; - /** - * 日志内容 - */ - private String content; - /** - * 文件任务 - */ - private List fileTaskLogs; - /** - * 日志是否拉取完成 - */ - private boolean finished; - - public FileIpLogContent(long stepInstanceId, int executeCount, String ip, - List fileTaskLogs, boolean finished) { - this.stepInstanceId = stepInstanceId; - this.executeCount = executeCount; - this.ip = ip; - this.fileTaskLogs = fileTaskLogs; - this.finished = finished; - } - - public String getContent() { - if (StringUtils.isNotEmpty(content)) { - return this.content; - } - StringBuilder builder = new StringBuilder(); - if (this.fileTaskLogs == null) { - return null; - } - fileTaskLogs.forEach(fileTaskLog -> { - builder.append(fileTaskLog.getContent()); - }); - return builder.toString(); - } -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/FileSourceDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/FileSourceDTO.java index eaa939c875..bc69385b78 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/FileSourceDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/FileSourceDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,16 +26,21 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.annotation.PersistenceObject; +import com.tencent.bk.job.execute.model.inner.ServiceFileSourceDTO; import lombok.Data; +import org.apache.commons.collections4.CollectionUtils; import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; /** * 文件源 */ @Data @JsonInclude(JsonInclude.Include.NON_EMPTY) +@PersistenceObject public class FileSourceDTO implements Cloneable { /** @@ -72,7 +77,7 @@ public class FileSourceDTO implements Cloneable { * 文件源服务器 */ @JsonProperty("servers") - private ServersDTO servers; + private ExecuteTargetDTO servers; /** * 文件源ID @@ -104,4 +109,18 @@ public FileSourceDTO clone() { } return cloneFileSourceDTO; } + + public ServiceFileSourceDTO toServiceFileSourceDTO() { + ServiceFileSourceDTO serviceFileSourceDTO = new ServiceFileSourceDTO(); + serviceFileSourceDTO.setFileType(fileType); + if (CollectionUtils.isNotEmpty(files)) { + serviceFileSourceDTO.setFiles( + files.stream() + .map(FileDetailDTO::toServiceFileDetailDTO) + .collect(Collectors.toList()) + ); + } + serviceFileSourceDTO.setServers(servers.toServiceExecuteTargetDTO()); + return serviceFileSourceDTO; + } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/FileSourceTaskLogDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/FileSourceTaskLogDTO.java index 9d8dcec80f..ce00bc9c1d 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/FileSourceTaskLogDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/FileSourceTaskLogDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -31,6 +31,10 @@ */ @Data public class FileSourceTaskLogDTO { + private Long id; + + private Long taskInstanceId; + private Long stepInstanceId; private Integer executeCount; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/FileStepInstanceDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/FileStepInstanceDTO.java index 0e478f5758..921713a33d 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/FileStepInstanceDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/FileStepInstanceDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -34,15 +34,12 @@ @Setter @ToString public class FileStepInstanceDTO { + private Long taskInstanceId; private Long stepInstanceId; /** * 文件传输的源文件 */ private List fileSourceList; - /** - * 变量解析之后的文件传输的源文件 - */ - private List resolvedFileSourceList; /** * 文件传输的目标目录 */ diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/GseTaskDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/GseTaskDTO.java index 8e73890d68..3122f42003 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/GseTaskDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/GseTaskDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -37,6 +37,10 @@ public class GseTaskDTO { * GSE任务ID */ private Long id; + /** + * 作业实例ID + */ + private Long taskInstanceId; /** * 步骤实例ID */ @@ -69,14 +73,31 @@ public class GseTaskDTO { * GSE 任务ID */ private String gseTaskId; + /** + * GSE 任务的唯一名称，类似于ID的作用 + */ + private String taskUniqueName; - public GseTaskDTO(Long stepInstanceId, Integer executeCount, int batch) { + public GseTaskDTO(Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + int batch) { + this.taskInstanceId = taskInstanceId; this.stepInstanceId = stepInstanceId; this.executeCount = executeCount; this.batch = batch; } public String getTaskUniqueName() { - return "GseTask:" + id + ":" + stepInstanceId + ":" + executeCount + ":" + batch; + if (taskUniqueName != null) { + return taskUniqueName; + } else { + String taskName = "GseTask:" + id + ":" + stepInstanceId + ":" + executeCount; + if (batch > 0) { + taskName = taskName + ":" + batch; + } + taskUniqueName = taskName; + } + return taskUniqueName; } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/GseTaskSimpleDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/GseTaskSimpleDTO.java new file mode 100644 index 0000000000..7dca3102de --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/GseTaskSimpleDTO.java @@ -0,0 +1,57 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model; + +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * GSE 任务 + */ +@Data +@NoArgsConstructor +public class GseTaskSimpleDTO { + /** + * 作业实例ID + */ + private Long taskInstanceId; + /** + * 步骤实例ID + */ + private Long stepInstanceId; + /** + * 步骤执行次数 + */ + private Integer executeCount; + /** + * 滚动执行批次 + */ + private Integer batch; + + /** + * GSE 任务ID + */ + private String gseTaskId; +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/HostInstanceTopologyDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/HostInstanceTopologyDTO.java index aa30bca122..c01acedef5 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/HostInstanceTopologyDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/HostInstanceTopologyDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/HostVariableValuesDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/HostVariableValuesDTO.java index 59076784a9..b8bb2fedcd 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/HostVariableValuesDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/HostVariableValuesDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,31 +24,48 @@ package com.tencent.bk.job.execute.model; -import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.annotation.PersistenceObject; import lombok.Data; +import org.apache.commons.lang3.StringUtils; import java.util.List; /** * 作业-全局变量-命名空间-主机变量 */ +@PersistenceObject @Data public class HostVariableValuesDTO { /** - * 主机IP + * 主机云区域ID:IPv4 */ - @CompatibleImplementation(explain = "由于ip不再唯一，使用hostId/agentId替代该参数", version = "3.7.x") - private String ip; + @JsonProperty("ip") + private String cloudIpv4; /** - * 主机ID + * 主机云区域ID:ipv6 */ - private Long hostId; + @JsonProperty("ipv6") + private String cloudIpv6; /** - * bk_agent_id + * 主机ID */ - private String agentId; + @JsonProperty("hostId") + private Long hostId; /** * 变量值 */ + @JsonProperty("values") private List values; + + /** + * 获取主机的ip，优先返回ipv4 + * + * @return 主机ipv4/ipv6, ipv4 优先 + */ + @JsonIgnore + public String getPrimaryCloudIp() { + return StringUtils.isNotEmpty(cloudIpv4) ? cloudIpv4 : cloudIpv6; + } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/KubeClusterFilter.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/KubeClusterFilter.java new file mode 100644 index 0000000000..20ebcaf0b0 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/KubeClusterFilter.java @@ -0,0 +1,54 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model; + +import com.tencent.bk.job.common.annotation.PersistenceObject; +import lombok.Data; +import org.apache.commons.collections4.CollectionUtils; + +import java.util.ArrayList; +import java.util.List; + +/** + * 执行目标-容器选择过滤器-按集群过滤 + */ +@Data +@PersistenceObject +public class KubeClusterFilter implements Cloneable { + + /** + * k8s 集群 UID + */ + private List clusterUIDs; + + @Override + public KubeClusterFilter clone() { + KubeClusterFilter clone = new KubeClusterFilter(); + if (CollectionUtils.isNotEmpty(clusterUIDs)) { + clone.setClusterUIDs(new ArrayList<>(clusterUIDs)); + } + return clone; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/KubeContainerFilter.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/KubeContainerFilter.java new file mode 100644 index 0000000000..9da372de1a --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/KubeContainerFilter.java @@ -0,0 +1,118 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.tencent.bk.job.common.annotation.PersistenceObject; +import com.tencent.bk.job.common.model.dto.Container; +import lombok.Data; +import org.apache.commons.collections4.CollectionUtils; + +import java.util.ArrayList; +import java.util.List; + +/** + * 执行目标-容器选择过滤器 + */ +@Data +@PersistenceObject +public class KubeContainerFilter implements Cloneable { + + /** + * 集群过滤器 + */ + private KubeClusterFilter clusterFilter; + + /** + * namespace 过滤器 + */ + private KubeNamespaceFilter namespaceFilter; + + /** + * workload 过滤器 + */ + private KubeWorkloadFilter workloadFilter; + + /** + * pod 属性过滤器 + */ + private KubePodFilter podFilter; + + /** + * 容器属性过滤器 + */ + private KubeContainerPropFilter containerPropFilter; + + /** + * 标识一个没有设置任何条件的过滤器 + */ + private boolean emptyFilter; + + /** + * 是否从过滤结果集中选择任意一个容器作为执行对象（只有一个容器会被执行） + */ + private boolean fetchAnyOneContainer; + + /** + * 过滤之后的容器列表 + */ + private List containers; + + @Override + public KubeContainerFilter clone() { + KubeContainerFilter clone = new KubeContainerFilter(); + if (clusterFilter != null) { + clone.setClusterFilter(clusterFilter.clone()); + } + if (namespaceFilter != null) { + clone.setNamespaceFilter(namespaceFilter.clone()); + } + if (workloadFilter != null) { + clone.setWorkloadFilter(workloadFilter.clone()); + } + if (podFilter != null) { + clone.setPodFilter(podFilter.clone()); + } + if (containerPropFilter != null) { + clone.setContainerPropFilter(containerPropFilter.clone()); + } + clone.setFetchAnyOneContainer(fetchAnyOneContainer); + if (CollectionUtils.isNotEmpty(containers)) { + List cloneContainers = new ArrayList<>(containers.size()); + containers.forEach(container -> cloneContainers.add(container.clone())); + clone.setContainers(cloneContainers); + } + + return clone; + } + + /** + * 是否包含容器拓扑(cluster/namespace/workload)相关的 filter + */ + @JsonIgnore + public boolean hasKubeNodeFilter() { + return !isEmptyFilter() && (clusterFilter != null || namespaceFilter != null || workloadFilter != null); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/KubeContainerPropFilter.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/KubeContainerPropFilter.java new file mode 100644 index 0000000000..3d0b7da1fe --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/KubeContainerPropFilter.java @@ -0,0 +1,55 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model; + +import com.tencent.bk.job.common.annotation.PersistenceObject; +import lombok.Data; +import org.apache.commons.collections4.CollectionUtils; + +import java.util.ArrayList; +import java.util.List; + +/** + * 执行目标-容器选择过滤器-按容器属性过滤 + */ +@Data +@PersistenceObject +public class KubeContainerPropFilter implements Cloneable { + + /** + * k8s 容器名称列表 + */ + private List containerNames; + + @Override + public KubeContainerPropFilter clone() { + KubeContainerPropFilter clone = new KubeContainerPropFilter(); + if (CollectionUtils.isNotEmpty(containerNames)) { + clone.setContainerNames(new ArrayList<>(containerNames)); + } + return clone; + } + +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/KubeNamespaceFilter.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/KubeNamespaceFilter.java new file mode 100644 index 0000000000..c52f6d344a --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/KubeNamespaceFilter.java @@ -0,0 +1,54 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model; + +import com.tencent.bk.job.common.annotation.PersistenceObject; +import lombok.Data; +import org.apache.commons.collections4.CollectionUtils; + +import java.util.ArrayList; +import java.util.List; + +/** + * 执行目标-容器选择过滤器-按命名空间过滤 + */ +@Data +@PersistenceObject +public class KubeNamespaceFilter implements Cloneable { + + /** + * k8s 命名空间名称 + */ + private List namespaces; + + @Override + public KubeNamespaceFilter clone() { + KubeNamespaceFilter clone = new KubeNamespaceFilter(); + if (CollectionUtils.isNotEmpty(namespaces)) { + clone.setNamespaces(new ArrayList<>(namespaces)); + } + return clone; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/KubePodFilter.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/KubePodFilter.java new file mode 100644 index 0000000000..1cb475393d --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/KubePodFilter.java @@ -0,0 +1,71 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model; + +import com.tencent.bk.job.common.annotation.PersistenceObject; +import lombok.Data; +import org.apache.commons.collections4.CollectionUtils; + +import java.util.ArrayList; +import java.util.List; + +/** + * 执行目标-容器选择过滤器-按 POD 过滤 + */ +@Data +@PersistenceObject +public class KubePodFilter implements Cloneable { + + /** + * k8s pod 名称列表 + */ + private List podNames; + + /** + * label selector + */ + private List labelSelector; + + /** + * pod label selector expression + */ + private String labelSelectorExpr; + + @Override + public KubePodFilter clone() { + KubePodFilter clone = new KubePodFilter(); + if (CollectionUtils.isNotEmpty(podNames)) { + clone.setPodNames(new ArrayList<>(podNames)); + } + if (CollectionUtils.isNotEmpty(labelSelector)) { + List cloneLabelSelectExprList = new ArrayList<>(labelSelector.size()); + labelSelector.forEach(labelSelectExpr -> cloneLabelSelectExprList.add(labelSelectExpr.clone())); + clone.setLabelSelector(cloneLabelSelectExprList); + } + clone.setLabelSelectorExpr(labelSelectorExpr); + return clone; + } + +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/KubeWorkloadFilter.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/KubeWorkloadFilter.java new file mode 100644 index 0000000000..f6ac08f063 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/KubeWorkloadFilter.java @@ -0,0 +1,61 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model; + +import com.tencent.bk.job.common.annotation.PersistenceObject; +import lombok.Data; +import org.apache.commons.collections4.CollectionUtils; + +import java.util.ArrayList; +import java.util.List; + +/** + * 执行目标-容器选择过滤器-按 workload 过滤 + */ +@Data +@PersistenceObject +public class KubeWorkloadFilter implements Cloneable { + + /** + * workload 类型 + */ + private String kind; + + /** + * k8s workload 名称列表 + */ + private List workloadNames; + + @Override + public KubeWorkloadFilter clone() { + KubeWorkloadFilter clone = new KubeWorkloadFilter(); + clone.setKind(kind); + if (CollectionUtils.isNotEmpty(workloadNames)) { + clone.setWorkloadNames(new ArrayList<>(workloadNames)); + } + return clone; + } + +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/LabelSelectExprDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/LabelSelectExprDTO.java new file mode 100644 index 0000000000..55e7bba2d9 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/LabelSelectExprDTO.java @@ -0,0 +1,74 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model; + +import com.tencent.bk.job.common.annotation.PersistenceObject; +import com.tencent.bk.job.common.constant.LabelSelectorOperatorEnum; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.apache.commons.collections4.CollectionUtils; + +import java.util.ArrayList; +import java.util.List; + +/** + * Label selector 计算表达式 + */ +@Data +@NoArgsConstructor +@PersistenceObject +public class LabelSelectExprDTO implements Cloneable { + /** + * Label key + */ + private String key; + + /** + * 计算操作符 + **/ + private LabelSelectorOperatorEnum operator; + + /** + * Label values + */ + private List values; + + public LabelSelectExprDTO(String key, LabelSelectorOperatorEnum operator, List values) { + this.key = key; + this.operator = operator; + this.values = values; + } + + @Override + public LabelSelectExprDTO clone() { + LabelSelectExprDTO clone = new LabelSelectExprDTO(); + clone.setKey(key); + clone.setOperator(operator); + if (CollectionUtils.isNotEmpty(values)) { + clone.setValues(new ArrayList<>(values)); + } + return clone; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/LogExportJobInfoDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/LogExportJobInfoDTO.java index 75dde23f70..1cfdee72c8 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/LogExportJobInfoDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/LogExportJobInfoDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/NotificationConfig.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/NotificationConfig.java index dda6c341a0..79feeda8a8 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/NotificationConfig.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/NotificationConfig.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/NotifyDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/NotifyDTO.java index 8803aeeed2..e82bcb51d3 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/NotifyDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/NotifyDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/OperationLogDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/OperationLogDTO.java index c280311af6..bd9ec01e84 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/OperationLogDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/OperationLogDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ResultGroupBaseDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ResultGroupBaseDTO.java new file mode 100644 index 0000000000..d97c6d1ba1 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ResultGroupBaseDTO.java @@ -0,0 +1,103 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model; + +import com.tencent.bk.job.execute.engine.consts.ExecuteObjectTaskStatusEnum; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.Objects; + +/** + * 执行对象任务执行结果分组 + */ +@Data +@NoArgsConstructor +public class ResultGroupBaseDTO implements Comparable { + /** + * 任务状态 + * + * @see ExecuteObjectTaskStatusEnum + */ + private Integer status; + /** + * 用户脚本输出的自定义分组tag + */ + private String tag; + /** + * 执行对象任务总数 + */ + private int total; + + public ResultGroupBaseDTO(Integer status, String tag) { + this.status = status; + this.tag = tag; + } + + public ResultGroupBaseDTO(ResultGroupBaseDTO baseResultGroup) { + this.status = baseResultGroup.getStatus(); + this.tag = baseResultGroup.getTag(); + this.total = baseResultGroup.getTotal(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + ResultGroupBaseDTO that = (ResultGroupBaseDTO) o; + return status.equals(that.status) && + tagEquals(tag, that.tag); + } + + private boolean tagEquals(String thisTag, String thatTag) { + String tag1 = thisTag == null ? "" : thisTag; + String tag2 = thatTag == null ? "" : thatTag; + return tag1.equals(tag2); + } + + @Override + public int hashCode() { + if (this.tag == null) { + return Objects.hash(status, ""); + } else { + return Objects.hash(status, tag); + } + } + + @Override + public int compareTo(ResultGroupBaseDTO that) { + int result = status.compareTo(that.status); + if (result != 0) { + return result; + } + String tag1 = this.tag == null ? "" : this.tag; + String tag2 = that.tag == null ? "" : that.tag; + return tag1.compareTo(tag2); + } + + public String getGroupKey() { + return status + "_" + tag; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ResultGroupDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ResultGroupDTO.java new file mode 100644 index 0000000000..8eab49db1a --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ResultGroupDTO.java @@ -0,0 +1,54 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model; + +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; + +import java.util.List; + +/** + * 执行对象任务执行结果分组 + */ +@Getter +@Setter +@ToString +@NoArgsConstructor +public class ResultGroupDTO extends ResultGroupBaseDTO { + /** + * 执行对象任务 + */ + private List executeObjectTasks; + + public ResultGroupDTO(Integer status, String tag) { + super(status, tag); + } + + public ResultGroupDTO(ResultGroupBaseDTO baseResultGroup) { + super(baseResultGroup); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/RollingConfigDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/RollingConfigDTO.java index c01a5febc5..a91f363ff8 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/RollingConfigDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/RollingConfigDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/RollingStepBatchTaskDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/RollingStepBatchTaskDTO.java index 3c5d8e04af..77339f2069 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/RollingStepBatchTaskDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/RollingStepBatchTaskDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ScriptCheckItemDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ScriptCheckItemDTO.java index 4bbb769c87..25667654df 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ScriptCheckItemDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ScriptCheckItemDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,7 @@ package com.tencent.bk.job.execute.model; -import com.tencent.bk.job.manage.common.consts.script.ScriptCheckErrorLevelEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptCheckErrorLevelEnum; import lombok.Data; import lombok.NoArgsConstructor; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ScriptCheckResultDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ScriptCheckResultDTO.java index 7780f4e111..e1ba10c9f3 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ScriptCheckResultDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ScriptCheckResultDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ScriptExecuteObjectLogContent.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ScriptExecuteObjectLogContent.java new file mode 100644 index 0000000000..d1f2884a8a --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ScriptExecuteObjectLogContent.java @@ -0,0 +1,69 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model; + +import com.tencent.bk.job.execute.engine.model.ExecuteObject; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * 脚本日志内容 + */ +@Data +@NoArgsConstructor +public class ScriptExecuteObjectLogContent { + /** + * 步骤实例ID + */ + private long stepInstanceId; + /** + * 执行次数 + */ + private int executeCount; + /** + * 执行对象 + */ + private ExecuteObject executeObject; + /** + * 日志内容 + */ + private String content; + /** + * 日志是否拉取完成 + */ + private boolean finished; + + public ScriptExecuteObjectLogContent(long stepInstanceId, + int executeCount, + ExecuteObject executeObject, + String content, + boolean finished) { + this.stepInstanceId = stepInstanceId; + this.executeCount = executeCount; + this.executeObject = executeObject; + this.content = content; + this.finished = finished; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ScriptHostLogContent.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ScriptHostLogContent.java deleted file mode 100644 index 8751750724..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ScriptHostLogContent.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.model; - -import com.tencent.bk.job.common.annotation.CompatibleImplementation; -import lombok.Data; -import lombok.NoArgsConstructor; - -/** - * 脚本日志内容 - */ -@Data -@NoArgsConstructor -public class ScriptHostLogContent { - /** - * 步骤实例ID - */ - private long stepInstanceId; - /** - * 执行次数 - */ - private int executeCount; - /** - * 主机ID - */ - private Long hostId; - /** - * 目标IP - */ - @CompatibleImplementation(name = "rolling_execute", explain = "兼容字段，后续使用hostId替换", version = "3.7.x") - private String ip; - /** - * 日志内容 - */ - private String content; - /** - * 日志是否拉取完成 - */ - private boolean finished; - - public ScriptHostLogContent(long stepInstanceId, int executeCount, Long hostId, - String ip, String content, boolean finished) { - this.stepInstanceId = stepInstanceId; - this.executeCount = executeCount; - this.hostId = hostId; - this.ip = ip; - this.content = content; - this.finished = finished; - } -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ScriptStepInstanceDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ScriptStepInstanceDTO.java index 86d85b7929..8ad2b21a4e 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ScriptStepInstanceDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ScriptStepInstanceDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,7 @@ package com.tencent.bk.job.execute.model; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; import lombok.Getter; import lombok.Setter; import lombok.ToString; @@ -32,6 +33,10 @@ @Setter @ToString public class ScriptStepInstanceDTO { + /** + * 作业实例ID + */ + private Long taskInstanceId; /** * 步骤实例ID */ @@ -58,10 +63,8 @@ public class ScriptStepInstanceDTO { /** * 执行脚本的类型:1(shell脚本)、2(bat脚本)、3(perl脚本)、4(python脚本)、5(powershell脚本) - * - * @see com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum */ - private Integer scriptType; + private ScriptTypeEnum scriptType; /** * 执行脚本的执行参数 */ @@ -96,4 +99,8 @@ public class ScriptStepInstanceDTO { * 是否敏感参数 */ private boolean secureParam; + /** + * Windows解释器路径 + */ + private String windowsInterpreter; } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ServersDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ServersDTO.java deleted file mode 100644 index 5ec40badb9..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/ServersDTO.java +++ /dev/null @@ -1,210 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.model; - -import com.tencent.bk.job.common.annotation.PersistenceObject; -import com.tencent.bk.job.common.model.dto.HostDTO; -import lombok.Data; -import org.apache.commons.collections4.CollectionUtils; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.StringJoiner; - -@Data -@PersistenceObject -public class ServersDTO implements Cloneable { - /** - * 如果目标服务器是通过全局变量-主机列表定义的，variable 表示变量 name - */ - private String variable; - /** - * 用户选择的服务器ip列表（静态） - */ - private List staticIpList; - - /** - * 服务器动态分组列表 - */ - private List dynamicServerGroups; - - /** - * topo节点 - */ - private List topoNodes; - - /** - * 服务器（动态分组、静态ip、动态topo节点）等对应的所有ip的集合 - */ - private List ipList; - - /** - * 非法服务器 - */ - private List invalidIpList; - - /** - * 非法动态分组 - */ - private List invalidDynamicServerGroups; - - /** - * 非法topo节点 - */ - private List invalidTopoNodes; - - public static ServersDTO emptyInstance() { - ServersDTO serversDTO = new ServersDTO(); - serversDTO.setIpList(Collections.emptyList()); - serversDTO.setDynamicServerGroups(Collections.emptyList()); - serversDTO.setInvalidDynamicServerGroups(Collections.emptyList()); - serversDTO.setInvalidIpList(Collections.emptyList()); - serversDTO.setInvalidTopoNodes(Collections.emptyList()); - serversDTO.setStaticIpList(Collections.emptyList()); - serversDTO.setTopoNodes(Collections.emptyList()); - return serversDTO; - } - - public String buildIpListStr() { - if (ipList == null || ipList.isEmpty()) { - return null; - } - StringJoiner sj = new StringJoiner(","); - ipList.forEach(ipDTO -> sj.add(ipDTO.getBkCloudId() + ":" + ipDTO.getIp())); - return sj.toString(); - } - - public ServersDTO clone() { - ServersDTO cloneServersDTO = new ServersDTO(); - cloneServersDTO.setVariable(variable); - if (staticIpList != null) { - List cloneStaticIpList = new ArrayList<>(staticIpList.size()); - staticIpList.forEach(staticIp -> cloneStaticIpList.add(staticIp.clone())); - cloneServersDTO.setStaticIpList(cloneStaticIpList); - } - if (dynamicServerGroups != null) { - List cloneServerGroups = new ArrayList<>(dynamicServerGroups.size()); - dynamicServerGroups.forEach(serverGroup -> cloneServerGroups.add(serverGroup.clone())); - cloneServersDTO.setDynamicServerGroups(cloneServerGroups); - } - if (topoNodes != null) { - cloneServersDTO.setTopoNodes(topoNodes); - } - if (ipList != null) { - List cloneIpList = new ArrayList<>(ipList.size()); - ipList.forEach(ip -> cloneIpList.add(ip.clone())); - cloneServersDTO.setIpList(cloneIpList); - } - if (invalidIpList != null) { - List cloneIpList = new ArrayList<>(invalidIpList.size()); - invalidIpList.forEach(ip -> cloneIpList.add(ip.clone())); - cloneServersDTO.setInvalidIpList(cloneIpList); - } - if (invalidDynamicServerGroups != null) { - List cloneInvalidDynamicServerGroups = - new ArrayList<>(invalidDynamicServerGroups.size()); - invalidDynamicServerGroups.forEach(group -> cloneInvalidDynamicServerGroups.add(group.clone())); - cloneServersDTO.setInvalidDynamicServerGroups(cloneInvalidDynamicServerGroups); - } - if (invalidTopoNodes != null) { - List cloneInvalidDynamicServerGroups = new ArrayList<>(invalidTopoNodes.size()); - invalidTopoNodes.forEach(topo -> cloneInvalidDynamicServerGroups.add(topo.clone())); - cloneServersDTO.setInvalidTopoNodes(cloneInvalidDynamicServerGroups); - } - return cloneServersDTO; - } - - public ServersDTO merge(ServersDTO servers) { - if (servers == null) { - return this; - } - if (servers.getStaticIpList() != null) { - if (this.staticIpList == null) { - this.staticIpList = new ArrayList<>(servers.getStaticIpList()); - } else { - servers.getStaticIpList().forEach(ipDTO -> { - if (!this.staticIpList.contains(ipDTO)) { - this.staticIpList.add(ipDTO); - } - }); - } - } - if (servers.getTopoNodes() != null) { - if (this.topoNodes == null) { - this.topoNodes = new ArrayList<>(servers.getTopoNodes()); - } else { - servers.getTopoNodes().forEach(topoNode -> { - if (!this.topoNodes.contains(topoNode)) { - this.topoNodes.add(topoNode); - } - }); - } - } - if (servers.getDynamicServerGroups() != null) { - if (this.dynamicServerGroups == null) { - this.dynamicServerGroups = new ArrayList<>(servers.getDynamicServerGroups()); - } else { - servers.getDynamicServerGroups().forEach(dynamicServerGroup -> { - if (!this.dynamicServerGroups.contains(dynamicServerGroup)) { - this.dynamicServerGroups.add(dynamicServerGroup); - } - }); - } - } - return this; - } - - public void addStaticIps(Collection ips) { - if (staticIpList == null) { - staticIpList = new ArrayList<>(); - } - staticIpList.addAll(ips); - } - - /** - * 服务器是否为空 - */ - public boolean isEmpty() { - return CollectionUtils.isEmpty(this.staticIpList) - && CollectionUtils.isEmpty(this.topoNodes) - && CollectionUtils.isEmpty(this.dynamicServerGroups); - } - - public void addInvalidDynamicServerGroup(DynamicServerGroupDTO serverGroup) { - if (this.invalidDynamicServerGroups == null) { - this.invalidDynamicServerGroups = new ArrayList<>(); - } - this.invalidDynamicServerGroups.add(serverGroup); - } - - public void addInvalidTopoNodeDTO(DynamicServerTopoNodeDTO topoNode) { - if (this.invalidTopoNodes == null) { - this.invalidTopoNodes = new ArrayList<>(); - } - this.invalidTopoNodes.add(topoNode); - } -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepExecutionDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepExecutionDTO.java index 8019de4fdd..590b965a45 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepExecutionDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepExecutionDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepExecutionDetailDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepExecutionDetailDTO.java index 98e18a010a..414e4657ae 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepExecutionDetailDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepExecutionDetailDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,7 +27,7 @@ import com.tencent.bk.job.execute.common.constants.RunStatusEnum; import com.tencent.bk.job.execute.common.constants.StepRunModeEnum; import com.tencent.bk.job.execute.common.util.TaskCostCalculator; -import com.tencent.bk.job.manage.common.consts.task.TaskStepTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskStepTypeEnum; import lombok.Data; import lombok.NoArgsConstructor; @@ -46,7 +46,7 @@ public class StepExecutionDetailDTO { /** * 任务执行分组状态 */ - List resultGroups; + private List resultGroups; /** * 步骤实例ID */ diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepExecutionRecordDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepExecutionRecordDTO.java index fed39b2fb9..f4f6337283 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepExecutionRecordDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepExecutionRecordDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepExecutionResultQuery.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepExecutionResultQuery.java index a75ae5b025..5d01d557fd 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepExecutionResultQuery.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepExecutionResultQuery.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -30,7 +30,6 @@ import lombok.Setter; import lombok.ToString; import org.apache.commons.lang3.StringUtils; -import org.jooq.generated.tables.GseTaskIpLog; import java.util.HashMap; import java.util.Map; @@ -42,17 +41,17 @@ @Builder public class StepExecutionResultQuery { public static final String ORDER_FIELD_TOTAL_TIME = "totalTime"; - public static final String ORDER_FIELD_CLOUD_AREA_ID = "cloudAreaId"; public static final String ORDER_FIELD_EXIT_CODE = "exitCode"; private static final Map ORDER_FIELD_IN_DB = new HashMap<>(); static { - ORDER_FIELD_IN_DB.put(ORDER_FIELD_TOTAL_TIME, GseTaskIpLog.GSE_TASK_IP_LOG.TOTAL_TIME.getName()); - ORDER_FIELD_IN_DB.put(ORDER_FIELD_EXIT_CODE, GseTaskIpLog.GSE_TASK_IP_LOG.EXIT_CODE.getName()); - // 表中不包含cloud_area_id字段，可以使用IP达到相同的目的 - ORDER_FIELD_IN_DB.put(ORDER_FIELD_CLOUD_AREA_ID, GseTaskIpLog.GSE_TASK_IP_LOG.IP.getName()); + ORDER_FIELD_IN_DB.put(ORDER_FIELD_TOTAL_TIME, "total_time"); + ORDER_FIELD_IN_DB.put(ORDER_FIELD_EXIT_CODE, "exit_code"); } - + /** + * 作业实例ID + */ + private Long taskInstanceId; /** * 步骤实例ID */ @@ -88,7 +87,11 @@ public class StepExecutionResultQuery { /** * 执行结果分组下返回的最大任务数 */ - private Integer maxAgentTasksForResultGroup; + private Integer maxTasksForResultGroup; + /** + * 是否获取所有分组下的所有主机执行数据（默认为false，只获取第一个分组中的数据） + */ + private boolean fetchAllGroupData; /** * 排序字段 */ @@ -98,16 +101,15 @@ public class StepExecutionResultQuery { */ private Order order; - private Set matchHostIds; + private Set matchExecuteObjectCompositeKeys; - public boolean hasIpCondition() { + /** + * 是否包含按照执行对象过滤的条件。如果查询条件包含日志关键字、主机 ip 等，需要先根据条件查询到匹配的执行对象 + */ + public boolean hasExecuteObjectFilterCondition() { return StringUtils.isNotEmpty(logKeyword) || StringUtils.isNotEmpty(searchIp); } - public boolean hasBatchCondition() { - return batch != null && batch > 0; - } - public void transformOrderFieldToDbField() { if (orderField != null) { orderField = ORDER_FIELD_IN_DB.get(orderField); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepInstanceBaseDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepInstanceBaseDTO.java index 857b4ea94a..558d691dd3 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepInstanceBaseDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepInstanceBaseDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,12 +24,15 @@ package com.tencent.bk.job.execute.model; +import com.tencent.bk.job.common.constant.ExecuteObjectTypeEnum; +import com.tencent.bk.job.common.gse.util.AgentUtils; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskStepTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskStepTypeEnum; import lombok.Getter; import lombok.Setter; import lombok.ToString; +import org.apache.commons.collections4.CollectionUtils; import java.util.Set; @@ -70,7 +73,7 @@ public class StepInstanceBaseDTO { * * @see StepExecuteTypeEnum */ - protected Integer executeType; + protected StepExecuteTypeEnum executeType; /** * 步骤类型 */ @@ -104,13 +107,9 @@ public class StepInstanceBaseDTO { */ protected boolean ignoreError; /** - * 目标服务器 + * 执行目标 */ - protected ServersDTO targetServers; - /** - * 目标服务器 - */ - protected String ipList; + protected ExecuteTargetDTO targetExecuteObjects; /** * 不合法的服务器 */ @@ -132,6 +131,16 @@ public class StepInstanceBaseDTO { */ protected Long rollingConfigId; + /** + * 是否支持执行对象特性 + */ + private Boolean supportExecuteObject; + + /** + * 目标执行对象是否使用 GSE V2 Agent + */ + private Boolean targetGseV2Agent; + /** * 获取步骤类型 * @@ -141,12 +150,12 @@ public class StepInstanceBaseDTO { public TaskStepTypeEnum getStepType() { if (this.stepType == null) { if (executeType != null) { - if (executeType.equals(StepExecuteTypeEnum.EXECUTE_SCRIPT.getValue()) - || executeType.equals(StepExecuteTypeEnum.EXECUTE_SQL.getValue())) { + if (executeType == StepExecuteTypeEnum.EXECUTE_SCRIPT + || executeType == StepExecuteTypeEnum.EXECUTE_SQL) { this.stepType = TaskStepTypeEnum.SCRIPT; - } else if (executeType.equals(StepExecuteTypeEnum.SEND_FILE.getValue())) { + } else if (executeType == StepExecuteTypeEnum.SEND_FILE) { this.stepType = TaskStepTypeEnum.FILE; - } else if (executeType.equals(StepExecuteTypeEnum.MANUAL_CONFIRM.getValue())) { + } else if (executeType == StepExecuteTypeEnum.MANUAL_CONFIRM) { this.stepType = TaskStepTypeEnum.APPROVAL; } } @@ -175,12 +184,8 @@ public boolean isScriptStep() { return getStepType() == TaskStepTypeEnum.SCRIPT; } - public int getTargetServerTotalCount() { - if (this.targetServers != null && this.targetServers.getIpList() != null) { - return this.targetServers.getIpList().size(); - } else { - return 0; - } + public int getTargetExecuteObjectCount() { + return targetExecuteObjects.getExecuteObjectsCountCompatibly(); } /** @@ -203,4 +208,45 @@ public String getUniqueKey() { } return id + "_" + executeCount; } + + /** + * 执行目标是否是 GSE V2 Agent + */ + public boolean isTargetGseV2Agent() { + if (targetGseV2Agent != null) { + return targetGseV2Agent; + } + // 只需要判断任意一个即可，因为前置校验已经保证所有的主机的agentId全部都是V1或者V2 + this.targetGseV2Agent = this.targetExecuteObjects.getExecuteObjectsCompatibly().stream() + .anyMatch(executeObject -> { + if (executeObject.isHostExecuteObject()) { + return AgentUtils.isGseV2AgentId(executeObject.getHost().getAgentId()); + } else if (executeObject.isContainerExecuteObject()) { + // 只有 GSE V2 agent 才支持容器执行特性，这里这里必然是 true + return true; + } + return false; + }); + return this.targetGseV2Agent; + } + + /** + * 通过执行目标判断是否支持"执行对象特性" + */ + public boolean isSupportExecuteObjectFeature() { + if (supportExecuteObject == null) { + supportExecuteObject = CollectionUtils.isNotEmpty(targetExecuteObjects.getExecuteObjects()); + } + return supportExecuteObject; + + } + + public ExecuteObjectTypeEnum determineStepExecuteObjectType() { + return targetExecuteObjects.getExecuteObjectsCompatibly().get(0).getType(); + } + + public boolean isStepContainsExecuteObject() { + // 判断步骤是否包含执行对象 + return isScriptStep() || isFileStep(); + } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepInstanceDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepInstanceDTO.java index ee04059ba5..ac6a081dcd 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepInstanceDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepInstanceDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,16 +25,25 @@ package com.tencent.bk.job.execute.model; import com.tencent.bk.job.common.constant.DuplicateHandlerEnum; +import com.tencent.bk.job.common.model.dto.Container; import com.tencent.bk.job.common.model.dto.HostDTO; -import com.tencent.bk.job.common.util.function.LambdasUtil; +import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; +import com.tencent.bk.job.execute.model.inner.ServiceFileStepInstanceDTO; +import com.tencent.bk.job.execute.model.inner.ServiceScriptStepInstanceDTO; +import com.tencent.bk.job.execute.model.inner.ServiceStepInstanceDTO; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; import lombok.Getter; import lombok.Setter; import lombok.ToString; import org.apache.commons.collections4.CollectionUtils; +import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.function.Consumer; +import java.util.stream.Collectors; /** * 步骤实例 @@ -87,10 +96,8 @@ public class StepInstanceDTO extends StepInstanceBaseDTO { /** * 执行脚本的类型:1(shell脚本)、2(bat脚本)、3(perl脚本)、4(python脚本)、5(powershell脚本) - * - * @see com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum */ - private Integer scriptType; + private ScriptTypeEnum scriptType; /** * 执行脚本的执行参数 */ @@ -112,16 +119,16 @@ public class StepInstanceDTO extends StepInstanceBaseDTO { private Integer dbPort; private Integer dbType; private Long dbAccountId; + /** + * Windows解释器路径 + */ + private String windowsInterpreter; // --------------文件步骤字段--------------// /** * 文件传输的源文件 */ private List fileSourceList; - /** - * 变量解析之后的文件传输的源文件 - */ - private List resolvedFileSourceList; /** * 文件传输的目标目录 */ @@ -178,10 +185,10 @@ public StepInstanceDTO(StepInstanceBaseDTO stepInstanceBase) { this.totalTime = stepInstanceBase.totalTime; this.createTime = stepInstanceBase.createTime; this.ignoreError = stepInstanceBase.ignoreError; - this.targetServers = stepInstanceBase.targetServers; - this.ipList = stepInstanceBase.ipList; + this.targetExecuteObjects = stepInstanceBase.targetExecuteObjects; this.rollingConfigId = stepInstanceBase.rollingConfigId; this.batch = stepInstanceBase.getBatch(); + this.stepOrder = stepInstanceBase.getStepOrder(); } // -------------公共方法------------------// @@ -205,6 +212,7 @@ public void fillScriptStepInfo(ScriptStepInstanceDTO scriptStepInstance) { this.scriptSource = scriptStepInstance.getScriptSource(); this.resolvedScriptParam = scriptStepInstance.getResolvedScriptParam(); this.secureParam = scriptStepInstance.isSecureParam(); + this.windowsInterpreter = scriptStepInstance.getWindowsInterpreter(); } public void fillFileStepInfo(FileStepInstanceDTO fileStepInstance) { @@ -215,7 +223,6 @@ public void fillFileStepInfo(FileStepInstanceDTO fileStepInstance) { this.accountId = fileStepInstance.getAccountId(); this.account = fileStepInstance.getAccount(); this.fileSourceList = fileStepInstance.getFileSourceList(); - this.resolvedFileSourceList = fileStepInstance.getResolvedFileSourceList(); this.fileTargetPath = fileStepInstance.getFileTargetPath(); this.fileTargetName = fileStepInstance.getFileTargetName(); this.resolvedFileTargetPath = fileStepInstance.getResolvedFileTargetPath(); @@ -236,47 +243,114 @@ public void fillConfirmStepInfo(ConfirmStepInstanceDTO confirmStepInstance) { this.notifyChannels = confirmStepInstance.getNotifyChannels(); } + + public List extractStaticContainerList() { + if (targetExecuteObjects == null) { + return Collections.emptyList(); + } + + List containers = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(targetExecuteObjects.getStaticContainerList())) { + containers.addAll(targetExecuteObjects.getStaticContainerList()); + } + if (isFileStep()) { + for (FileSourceDTO fileSource : fileSourceList) { + ExecuteTargetDTO executeTargetDTO = fileSource.getServers(); + if (executeTargetDTO != null + && CollectionUtils.isNotEmpty(executeTargetDTO.getStaticContainerList())) { + containers.addAll(executeTargetDTO.getStaticContainerList()); + } + } + } + return containers; + } + + public Set extractAllHosts() { + if (targetExecuteObjects == null) { + return Collections.emptySet(); + } + + Set hosts = new HashSet<>(targetExecuteObjects.getHostsCompatibly()); + + if (CollectionUtils.isNotEmpty(fileSourceList)) { + fileSourceList.forEach(fileSource -> { + if (fileSource.getServers() != null) { + List fileSourceHosts = fileSource.getServers().getHostsCompatibly(); + if (CollectionUtils.isNotEmpty(fileSourceHosts)) { + hosts.addAll(fileSourceHosts); + } + } + }); + } + return hosts; + } + /** - * 返回步骤中不合法的主机 + * 步骤包含的执行对象 ExecuteObjectsDTO 的最终处理 * - * @return 不合法的主机 + * @param isSupportExecuteObjectFeature 是否执行执行对象特性 */ - public Set getInvalidHosts() { - Set invalidHosts = new HashSet<>(); - if (CollectionUtils.isNotEmpty(this.targetServers.getInvalidIpList())) { - invalidHosts.addAll(this.targetServers.getInvalidIpList()); + public void buildStepFinalExecuteObjects(boolean isSupportExecuteObjectFeature) { + if (targetExecuteObjects != null) { + targetExecuteObjects.buildMergedExecuteObjects(isSupportExecuteObjectFeature); } - if (isFileStep() && CollectionUtils.isNotEmpty(this.fileSourceList)) { - this.fileSourceList.stream().filter(LambdasUtil.not(FileSourceDTO::isLocalUpload)) - .forEach(fileSource -> { - ServersDTO fileSourceServers = fileSource.getServers(); - if (fileSourceServers != null && CollectionUtils.isNotEmpty(fileSourceServers.getInvalidIpList())) { - invalidHosts.addAll(fileSourceServers.getInvalidIpList()); - } - }); - + if (CollectionUtils.isNotEmpty(fileSourceList)) { + fileSourceList.forEach(fileSource -> { + if (fileSource.getServers() != null) { + fileSource.getServers().buildMergedExecuteObjects(isSupportExecuteObjectFeature); + } + }); } - return invalidHosts; } /** - * 判断步骤是否包含非法主机 + * 遍历步骤实例包含的所有执行目标（执行目标+文件分发源执行目标） + * + * @param consumer 消费者 */ - public boolean hasInvalidHost() { - boolean hasInvalidHost = false; - if (CollectionUtils.isNotEmpty(this.targetServers.getInvalidIpList())) { - return true; + public void forEachExecuteObjects(Consumer consumer) { + if (targetExecuteObjects != null) { + consumer.accept(targetExecuteObjects); + } + if (isFileStep()) { + for (FileSourceDTO fileSource : fileSourceList) { + ExecuteTargetDTO fileSourceExecuteObjects = fileSource.getServers(); + if (fileSourceExecuteObjects != null) { + consumer.accept(fileSourceExecuteObjects); + } + } } - if (isFileStep() && CollectionUtils.isNotEmpty(this.fileSourceList)) { - hasInvalidHost = this.fileSourceList.stream() - .filter(LambdasUtil.not(FileSourceDTO::isLocalUpload)) - .anyMatch(fileSource -> { - ServersDTO fileSourceServers = fileSource.getServers(); - return fileSourceServers != null - && CollectionUtils.isNotEmpty(fileSourceServers.getInvalidIpList()); - }); + } + public ServiceStepInstanceDTO toServiceStepInstanceDTO() { + ServiceStepInstanceDTO serviceStepInstanceDTO = new ServiceStepInstanceDTO(); + serviceStepInstanceDTO.setId(id); + serviceStepInstanceDTO.setName(name); + serviceStepInstanceDTO.setExecuteType(executeType.getValue()); + serviceStepInstanceDTO.setStatus(status.getValue()); + serviceStepInstanceDTO.setCreateTime(createTime); + if (executeType == StepExecuteTypeEnum.EXECUTE_SCRIPT || executeType == StepExecuteTypeEnum.EXECUTE_SQL) { + ServiceScriptStepInstanceDTO scriptStepInstance = new ServiceScriptStepInstanceDTO(); + scriptStepInstance.setStepInstanceId(id); + scriptStepInstance.setScriptType(scriptType.getValue()); + scriptStepInstance.setScriptContent(scriptContent); + scriptStepInstance.setScriptParam(scriptParam); + scriptStepInstance.setSecureParam(secureParam); + serviceStepInstanceDTO.setScriptStepInstance(scriptStepInstance); + } else if (executeType == StepExecuteTypeEnum.SEND_FILE) { + ServiceFileStepInstanceDTO fileStepInstance = new ServiceFileStepInstanceDTO(); + if (fileSourceList != null) { + fileStepInstance.setFileSourceList( + fileSourceList.stream() + .map(FileSourceDTO::toServiceFileSourceDTO) + .collect(Collectors.toList()) + ); + } + if (targetExecuteObjects != null) { + fileStepInstance.setTargetExecuteObjects(targetExecuteObjects.toServiceExecuteTargetDTO()); + } + serviceStepInstanceDTO.setFileStepInstance(fileStepInstance); } - return hasInvalidHost; + return serviceStepInstanceDTO; } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepInstanceRollingTaskDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepInstanceRollingTaskDTO.java index 92d3b1c7fe..3225fce7ec 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepInstanceRollingTaskDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepInstanceRollingTaskDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -37,6 +37,11 @@ public class StepInstanceRollingTaskDTO { */ private Long id; + /** + * 作业实例ID + */ + private Long taskInstanceId; + /** * 步骤实例ID */ diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepInstanceVariableValuesDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepInstanceVariableValuesDTO.java index 846609ab1a..46b2c7cc46 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepInstanceVariableValuesDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepInstanceVariableValuesDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,6 +27,8 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.annotation.PersistenceObject; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.execute.constants.VariableValueTypeEnum; import lombok.Data; @@ -36,9 +38,15 @@ /** * 步骤对应的变量值 */ +@PersistenceObject @Data @JsonInclude(JsonInclude.Include.NON_EMPTY) public class StepInstanceVariableValuesDTO { + /** + * 变量 ID + */ + @JsonProperty("id") + private Long id; /** * 步骤实例ID */ @@ -80,8 +88,10 @@ public class StepInstanceVariableValuesDTO { private Map globalParamsMap; /** - * Map> + * Map> */ @JsonIgnore - private Map> namespaceParamsMap; + private Map> namespaceParamsMap; + + } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepOperationDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepOperationDTO.java index 0ec995048b..fac8c3cebc 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepOperationDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepOperationDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -31,6 +31,10 @@ @Data @ToString public class StepOperationDTO { + /** + * 步骤实例ID + */ + private Long taskInstanceId; /** * 步骤实例ID */ diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepRollingConfigDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepRollingConfigDTO.java index 0080897aa8..b599fe5a4f 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepRollingConfigDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/StepRollingConfigDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskExecuteParam.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskExecuteParam.java index 8d64124c3c..e14f472928 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskExecuteParam.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskExecuteParam.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -78,4 +78,10 @@ public class TaskExecuteParam { * 是否跳过权限校验 */ private boolean skipAuth; + + /** + * 是否启动任务 + */ + @Builder.Default + private Boolean startTask = true; } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskExecuteResultDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskExecuteResultDTO.java index 3114060a10..190e17e3be 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskExecuteResultDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskExecuteResultDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskExecutionDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskExecutionDTO.java index a7e7155874..0481422daa 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskExecutionDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskExecutionDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskInfo.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskInfo.java index 818083b3e8..3cb9c21de9 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskInfo.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskInfo.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskInstanceDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskInstanceDTO.java index a978690772..26a605cfd3 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskInstanceDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskInstanceDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,13 +24,16 @@ package com.tencent.bk.job.execute.model; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; import com.tencent.bk.job.execute.common.constants.TaskStartupModeEnum; import com.tencent.bk.job.execute.common.constants.TaskTypeEnum; import com.tencent.bk.job.execute.engine.model.TaskVariableDTO; +import com.tencent.bk.job.manage.model.inner.ServiceTaskPlanDTO; import lombok.Data; import java.util.List; +import java.util.Set; /** * 执行作业实例 @@ -43,9 +46,14 @@ public class TaskInstanceDTO { private Long id; /** - * 执行作业id + * 执行方案id */ - private Long taskId; + private Long planId; + + /** + * 执行方案 + */ + private ServiceTaskPlanDTO plan; /** * 定时作业ID @@ -141,11 +149,16 @@ public class TaskInstanceDTO { */ private List variables; + /** + * 作业实例包含的所有主机 + */ + private Set allHosts; + /** * 判断是否为执行方案实例 */ public boolean isPlanInstance() { - return this.taskId > 0; + return this.planId > 0; } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskInstanceExecuteObjects.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskInstanceExecuteObjects.java new file mode 100644 index 0000000000..2997e96de5 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskInstanceExecuteObjects.java @@ -0,0 +1,175 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model; + +import com.tencent.bk.job.common.model.dto.Container; +import com.tencent.bk.job.common.model.dto.HostDTO; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; + +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * 作业实例中包含的执行对象 + */ +@Getter +@ToString +@Slf4j +public class TaskInstanceExecuteObjects { + + /** + * 当前作业实例是否包含主机执行对象 + */ + @Setter + private boolean containsAnyHost; + /** + * 当前作业实例是否包含容器执行对象 + */ + @Setter + private boolean containsAnyContainer; + + /** + * 合法的主机（在当前业务下) + */ + private List validHosts; + /** + * 不存在的主机 + */ + private List notExistHosts; + /** + * 在其他业务下的主机 + */ + private List notInAppHosts; + + /** + * 合法的容器（在当前业务下) + */ + private Set validContainers; + /** + * 不存在的容器ID列表 + */ + @Setter + private Set notExistContainerIds; + /** + * 主机白名单 + * key=hostId, value: 允许的操作列表 + */ + @Setter + private Map> whiteHostAllowActions; + /** + * 全量主机 Map - key: hostId + */ + private final Map hostsByHostId = new HashMap<>(); + /** + * 全量主机 Map - key: cloudIp + */ + private final Map hostsByCloudIp = new HashMap<>(); + + public void addContainers(Collection containers) { + if (validContainers == null) { + validContainers = new HashSet<>(); + } + validContainers.addAll(containers); + } + + public void addContainer(Container container) { + if (validContainers == null) { + validContainers = new HashSet<>(); + } + validContainers.add(container); + } + + public void setNotExistHosts(List notExistHosts) { + this.notExistHosts = notExistHosts; + putHostMap(notExistHosts); + } + + public void setNotInAppHosts(List notInAppHosts) { + this.notInAppHosts = notInAppHosts; + putHostMap(notInAppHosts); + } + + public void setValidHosts(List validHosts) { + this.validHosts = validHosts; + putHostMap(validHosts); + } + + private void putHostMap(List hosts) { + if (CollectionUtils.isNotEmpty(hosts)) { + hosts.forEach(host -> { + if (host.getHostId() != null) { + hostsByHostId.put(host.getHostId(), host); + } + if (host.toCloudIp() != null) { + hostsByCloudIp.put(host.toCloudIp(), host); + } + }); + } + } + + public HostDTO queryByHostKey(HostDTO host) { + if (host.getHostId() != null) { + return hostsByHostId.get(host.getHostId()); + } else if (host.toCloudIp() != null) { + return hostsByCloudIp.get(host.toCloudIp()); + } else { + log.info("Host not found, host: {}", host); + return null; + } + } + + /** + * 获取执行对象总数（同时包含有效、无效执行对象） + * + * @return 执行对象总数 + */ + public int getExecuteObjectNum() { + int num = 0; + if (CollectionUtils.isNotEmpty(validHosts)) { + num += validHosts.size(); + } + if (CollectionUtils.isNotEmpty(notExistHosts)) { + num += notExistHosts.size(); + } + if (CollectionUtils.isNotEmpty(notInAppHosts)) { + num += notInAppHosts.size(); + } + if (CollectionUtils.isNotEmpty(validContainers)) { + num += validContainers.size(); + } + if (CollectionUtils.isNotEmpty(notExistContainerIds)) { + num += notExistContainerIds.size(); + } + return num; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskInstanceQuery.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskInstanceQuery.java index 7cfe3f23c1..1173a882d9 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskInstanceQuery.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskInstanceQuery.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskInstanceRecordStateDO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskInstanceRecordStateDO.java new file mode 100644 index 0000000000..a82f5a8876 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskInstanceRecordStateDO.java @@ -0,0 +1,53 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.tencent.bk.job.common.constant.ArchiveFlag; +import lombok.Data; + +/** + * 作业实例数据状态 + */ +@Data +@JsonInclude(JsonInclude.Include.NON_EMPTY) +public class TaskInstanceRecordStateDO { + /** + * 数据归档状态 + * + * @see com.tencent.bk.job.common.constant.ArchiveFlag + */ + private int archiveFlag = ArchiveFlag.HOT.getValue(); + + /** + * 所属业务 ID + */ + private Long appId; + + /** + * 作业实例 ID + */ + private Long taskInstanceId; +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskNotifyDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskNotifyDTO.java index a3a98c9fc3..189ed1455b 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskNotifyDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/TaskNotifyDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,6 +26,7 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.tencent.bk.job.execute.common.constants.TaskStartupModeEnum; +import com.tencent.bk.job.execute.engine.listener.event.JobMessage; import lombok.Getter; import lombok.Setter; @@ -35,7 +36,7 @@ @Getter @Setter @JsonInclude(JsonInclude.Include.NON_EMPTY) -public class TaskNotifyDTO { +public class TaskNotifyDTO implements JobMessage { /** * 操作者 */ @@ -58,11 +59,15 @@ public class TaskNotifyDTO { * @see TaskStartupModeEnum */ private Integer startupMode; + /** + * 定时任务ID + */ + private Long cronTaskId; /** * 资源操作结果 * - * @see com.tencent.bk.job.manage.common.consts.notify.ExecuteStatusEnum + * @see com.tencent.bk.job.manage.api.common.constants.notify.ExecuteStatusEnum */ private Integer resourceExecuteStatus; /** diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/VariableValueDTO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/VariableValueDTO.java index 972a869aa6..a81a6d7a62 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/VariableValueDTO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/VariableValueDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,12 +26,14 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.annotation.PersistenceObject; import lombok.Data; import lombok.NoArgsConstructor; /** * 步骤变量值 */ +@PersistenceObject @Data @NoArgsConstructor @JsonInclude(JsonInclude.Include.NON_NULL) @@ -43,7 +45,7 @@ public class VariableValueDTO { @JsonProperty("type") private Integer type; @JsonProperty("serverValue") - private ServersDTO serverValue; + private ExecuteTargetDTO serverValue; public VariableValueDTO(String name, Integer type, String value) { this.name = name; @@ -51,9 +53,9 @@ public VariableValueDTO(String name, Integer type, String value) { this.type = type; } - public VariableValueDTO(String name, Integer type, ServersDTO serverValue) { + public VariableValueDTO(String name, Integer type, ExecuteTargetDTO executeTarget) { this.name = name; this.type = type; - this.serverValue = serverValue; + this.serverValue = executeTarget; } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/converter/TaskInstanceConverter.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/converter/TaskInstanceConverter.java index b6af8c673c..a4abd1c65a 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/converter/TaskInstanceConverter.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/converter/TaskInstanceConverter.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -61,7 +61,7 @@ public static TaskInstanceVO convertToTaskInstanceVO(TaskInstanceDTO taskInstanc taskInstanceVO.setType(taskInstanceDTO.getType()); taskInstanceVO.setTypeDesc( i18nService.getI18n(Objects.requireNonNull(TaskTypeEnum.valueOf(taskInstanceDTO.getType())).getI18nKey())); - taskInstanceVO.setTaskId(taskInstanceDTO.getTaskId()); + taskInstanceVO.setTaskId(taskInstanceDTO.getPlanId()); taskInstanceVO.setTemplateId(taskInstanceDTO.getTaskTemplateId()); taskInstanceVO.setDebugTask(taskInstanceDTO.isDebugTask()); taskInstanceVO.setTotalTime(taskInstanceDTO.getTotalTime()); @@ -88,7 +88,7 @@ public static ServiceTaskInstanceDTO convertToServiceTaskInstanceDTO(@NotNull Ta serviceTaskInstance.setStatusDesc( i18nService.getI18n(taskInstanceDTO.getStatus().getI18nKey())); serviceTaskInstance.setDebugTask(taskInstanceDTO.isDebugTask()); - serviceTaskInstance.setTaskId(taskInstanceDTO.getTaskId()); + serviceTaskInstance.setTaskId(taskInstanceDTO.getPlanId()); serviceTaskInstance.setTemplateId(taskInstanceDTO.getTaskTemplateId()); serviceTaskInstance.setStartTime(taskInstanceDTO.getStartTime()); serviceTaskInstance.setEndTime(taskInstanceDTO.getEndTime()); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/db/RollingConfigDetailDO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/db/RollingConfigDetailDO.java index 40d14b1d3a..6da7c67fc3 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/db/RollingConfigDetailDO.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/db/RollingConfigDetailDO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,12 +24,16 @@ package com.tencent.bk.job.execute.model.db; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; import com.tencent.bk.job.common.annotation.PersistenceObject; +import com.tencent.bk.job.common.constant.CompatibleType; import lombok.Data; import lombok.NoArgsConstructor; +import java.util.Collections; import java.util.List; import java.util.Map; @@ -77,7 +81,16 @@ public class RollingConfigDetailDO { * 目标服务器滚动分批 */ @JsonProperty("hostsBatchList") - private List hostsBatchList; + @Deprecated + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA, + explain = "兼容老数据，数据失效后可删除。使用 executeObjectsBatchList 替换") + private List hostsBatchList; + + /** + * 目标服务器滚动分批 + */ + @JsonProperty("executeObjectsBatchList") + private List executeObjectsBatchList; /** * 滚动总批次 @@ -99,4 +112,18 @@ public boolean isLastRollingStep(long stepInstanceId) { return this.includeStepInstanceIdList.get(includeStepInstanceIdList.size() - 1).equals(stepInstanceId); } + /** + * 获取执行对象滚动批次(兼容当前版本+历史版本) + */ + @JsonIgnore + public List getExecuteObjectsBatchListCompatibly() { + if (executeObjectsBatchList != null) { + return executeObjectsBatchList; + } else if (hostsBatchList != null) { + return hostsBatchList; + } else { + return Collections.emptyList(); + } + } + } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/db/RollingExecuteObjectsBatchDO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/db/RollingExecuteObjectsBatchDO.java new file mode 100644 index 0000000000..338a9599bb --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/db/RollingExecuteObjectsBatchDO.java @@ -0,0 +1,88 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.model.db; + +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.annotation.PersistenceObject; +import com.tencent.bk.job.common.constant.CompatibleType; +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +/** + * 滚动执行-执行对象分批 DO + */ +@Data +@JsonInclude(JsonInclude.Include.NON_EMPTY) +@NoArgsConstructor +@PersistenceObject +public class RollingExecuteObjectsBatchDO { + /** + * 滚动执行批次 + */ + private Integer batch; + /** + * 该批次的执行目标 + */ + @Deprecated + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA, + explain = "兼容老数据，数据失效后可删除。使用 executeObjects 替换") + private List hosts; + + private List executeObjects; + + public RollingExecuteObjectsBatchDO(Integer batch, List executeObjects) { + this.batch = batch; + boolean isSupportExecuteObject = executeObjects.get(0).isSupportExecuteObjectFeature(); + if (isSupportExecuteObject) { + this.executeObjects = executeObjects; + } else { + // 执行对象发布兼容 + this.hosts = executeObjects.stream().map(ExecuteObject::getHost).collect(Collectors.toList()); + } + } + + /** + * 获取所有执行对象列表(兼容当前版本+历史版本数据） + */ + @JsonIgnore + public List getExecuteObjectsCompatibly() { + if (executeObjects != null) { + return executeObjects; + } else if (hosts != null) { + return hosts.stream().map(ExecuteObject::buildCompatibleExecuteObject).collect(Collectors.toList()); + } else { + return Collections.emptyList(); + } + } + +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/db/RollingHostsBatchDO.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/db/RollingHostsBatchDO.java deleted file mode 100644 index 5ce2f6f94c..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/db/RollingHostsBatchDO.java +++ /dev/null @@ -1,56 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.model.db; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.tencent.bk.job.common.annotation.PersistenceObject; -import com.tencent.bk.job.common.model.dto.HostDTO; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.util.List; - -/** - * 滚动执行-主机分批 DO - */ -@Data -@JsonInclude(JsonInclude.Include.NON_EMPTY) -@NoArgsConstructor -@PersistenceObject -public class RollingHostsBatchDO { - /** - * 滚动执行批次 - */ - private Integer batch; - /** - * 该批次的目标服务器 - */ - private List hosts; - - public RollingHostsBatchDO(Integer batch, List hosts) { - this.batch = batch; - this.hosts = hosts; - } -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/JobCallbackDto.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/JobCallbackDto.java index ebf68097aa..fe41ce5ddd 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/JobCallbackDto.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/model/esb/JobCallbackDto.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/monitor/ExecuteMetricNames.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/monitor/ExecuteMetricNames.java index 3478ba43b1..ad7a7468b0 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/monitor/ExecuteMetricNames.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/monitor/ExecuteMetricNames.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/monitor/ExecuteMetricTags.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/monitor/ExecuteMetricTags.java index e07b06ea17..3de450e3d1 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/monitor/ExecuteMetricTags.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/monitor/ExecuteMetricTags.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/monitor/metrics/ExecuteMonitor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/monitor/metrics/ExecuteMonitor.java index 359b72781b..99a06eab68 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/monitor/metrics/ExecuteMonitor.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/monitor/metrics/ExecuteMonitor.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/monitor/metrics/GseResultHandleConsumersGauge.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/monitor/metrics/GseResultHandleConsumersGauge.java index 43f7d20998..48e959084d 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/monitor/metrics/GseResultHandleConsumersGauge.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/monitor/metrics/GseResultHandleConsumersGauge.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/monitor/metrics/GseTasksCounter.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/monitor/metrics/GseTasksCounter.java index f8bd90bb9f..7afb1d220d 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/monitor/metrics/GseTasksCounter.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/monitor/metrics/GseTasksCounter.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/monitor/metrics/GseTasksExceptionCounter.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/monitor/metrics/GseTasksExceptionCounter.java index 79c1152308..797e434cd8 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/monitor/metrics/GseTasksExceptionCounter.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/monitor/metrics/GseTasksExceptionCounter.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/monitor/metrics/NotAliveTasksCounter.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/monitor/metrics/NotAliveTasksCounter.java index b6ecc38733..d7db092e75 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/monitor/metrics/NotAliveTasksCounter.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/monitor/metrics/NotAliveTasksCounter.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/monitor/metrics/ResultHandleDelayedScheduleCounter.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/monitor/metrics/ResultHandleDelayedScheduleCounter.java index 38cbb02aea..2561348a39 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/monitor/metrics/ResultHandleDelayedScheduleCounter.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/monitor/metrics/ResultHandleDelayedScheduleCounter.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/monitor/metrics/ResultHandleWaitingScheduleTasksGauge.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/monitor/metrics/ResultHandleWaitingScheduleTasksGauge.java index e4bc50207c..3a1586d248 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/monitor/metrics/ResultHandleWaitingScheduleTasksGauge.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/monitor/metrics/ResultHandleWaitingScheduleTasksGauge.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/runner/InitArtifactoryDataRunner.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/runner/InitArtifactoryDataRunner.java index c344883aeb..bc03ed587e 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/runner/InitArtifactoryDataRunner.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/runner/InitArtifactoryDataRunner.java @@ -1,8 +1,8 @@ package com.tencent.bk.job.execute.runner; +import com.tencent.bk.job.common.artifactory.config.ArtifactoryConfig; import com.tencent.bk.job.common.artifactory.sdk.ArtifactoryHelper; import com.tencent.bk.job.common.constant.JobConstants; -import com.tencent.bk.job.execute.config.ArtifactoryConfig; import com.tencent.bk.job.execute.config.LogExportConfig; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; @@ -10,7 +10,7 @@ import org.springframework.stereotype.Component; @Slf4j -@Component +@Component("jobExecuteInitArtifactoryDataRunner") public class InitArtifactoryDataRunner implements CommandLineRunner { private final ArtifactoryConfig artifactoryConfig; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/AccountService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/AccountService.java index 6701310da0..6c4ebc6844 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/AccountService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/AccountService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,9 +24,9 @@ package com.tencent.bk.job.execute.service; +import com.tencent.bk.job.common.constant.AccountCategoryEnum; import com.tencent.bk.job.common.exception.ServiceException; import com.tencent.bk.job.execute.model.AccountDTO; -import com.tencent.bk.job.manage.common.consts.account.AccountCategoryEnum; /** * 账号服务 @@ -35,8 +35,8 @@ public interface AccountService { /** * 获取账号信息 * - * @param accountId - * @return + * @param accountId 账号ID + * @return 账号信息 */ AccountDTO getAccountById(Long accountId) throws ServiceException; @@ -53,9 +53,9 @@ public interface AccountService { /** * 根据业务ID和账号别名获取系统账号信息 * - * @param alias - * @param appId - * @return + * @param alias 账号别名 + * @param appId Job业务ID + * @return 账号信息 */ AccountDTO getSystemAccountByAlias(String alias, Long appId) throws ServiceException; @@ -65,7 +65,7 @@ public interface AccountService { * @param accountCategory 账号类型 * @param appId 业务ID * @param alias 账号别名 - * @return + * @return 账号信息 */ AccountDTO getAccountByAlias(AccountCategoryEnum accountCategory, Long appId, String alias) throws ServiceException; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/AgentService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/AgentService.java index 57d19862a1..01f42cf2a1 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/AgentService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/AgentService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,10 +25,13 @@ package com.tencent.bk.job.execute.service; import com.tencent.bk.job.common.model.dto.HostDTO; -import com.tencent.bk.job.execute.model.ServersDTO; +/** + * Agent 相关服务 + */ public interface AgentService { + /** + * 获取本地主机的Agent + */ HostDTO getLocalAgentHost(); - - ServersDTO getLocalServersDTO(); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/AgentTaskService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/AgentTaskService.java deleted file mode 100644 index da6e6120ce..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/AgentTaskService.java +++ /dev/null @@ -1,186 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.service; - -import com.tencent.bk.job.common.annotation.CompatibleImplementation; -import com.tencent.bk.job.common.constant.Order; -import com.tencent.bk.job.execute.model.AgentTaskDTO; -import com.tencent.bk.job.execute.model.AgentTaskDetailDTO; -import com.tencent.bk.job.execute.model.AgentTaskResultGroupBaseDTO; -import com.tencent.bk.job.execute.model.AgentTaskResultGroupDTO; -import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; - -import java.util.Collection; -import java.util.List; - -/** - * GSE Agent 任务 Service - */ -public interface AgentTaskService { - - /** - * 批量保存（insert/update) GSE Agent 任务 - * - * @param agentTasks GSE Agent 任务列表 - */ - void batchSaveAgentTasks(Collection agentTasks); - - /** - * 批量更新Agent任务 - * - * @param agentTasks Agent任务 - */ - void batchUpdateAgentTasks(Collection agentTasks); - - /** - * 获取执行成功的Agent任务数量 - * - * @param stepInstanceId 步骤实例ID - * @param executeCount 步骤执行次数 - * @return 执行成功的Agent数量 - */ - int getSuccessAgentTaskCount(long stepInstanceId, int executeCount); - - - /** - * 根据GSE任务ID获取Agent任务 - * - * @param gseTaskId GSE任务ID - * @return Agent任务 - */ - List listAgentTasksByGseTaskId(Long gseTaskId); - - /** - * 获取Agent任务 - * - * @param stepInstanceId 步骤实例ID - * @param executeCount 执行次数 - * @param batch 滚动执行批次；传入null或者0将忽略该参数 - * @return Agent任务 - */ - List listAgentTasks(Long stepInstanceId, - Integer executeCount, - Integer batch); - - /** - * 获取Agent任务详情并分组 - * - * @param stepInstance 步骤实例 - * @param executeCount 执行次数 - * @param batch 滚动执行批次；如果传入null或者0，将忽略该参数 - */ - List listAndGroupAgentTasks(StepInstanceBaseDTO stepInstance, - int executeCount, - Integer batch); - - /** - * 获取任务结果分组 - * - * @param stepInstanceId 步骤实例ID - * @param executeCount 执行次数 - * @param batch 滚动执行批次；传入null或者0将忽略该参数 - * @return Agent任务 - */ - List listResultGroups(long stepInstanceId, - int executeCount, - Integer batch); - - /** - * 根据执行结果查询Agent任务详情(排序、限制返回数量) - 包含主机详情 - * - * @param stepInstance 步骤实例 - * @param executeCount 执行次数 - * @param batch 滚动执行批次；如果传入null或者0，忽略该参数 - * @param status 任务状态 - * @param tag 用户自定义分组标签 - * @param limit 最大返回数量 - * @param orderField 排序字段 - * @param order 排序方式 - * @return Agent任务 - */ - List listAgentTaskDetailByResultGroup(StepInstanceBaseDTO stepInstance, - Integer executeCount, - Integer batch, - Integer status, - String tag, - Integer limit, - String orderField, - Order order); - - /** - * 根据结果分组获取Agent任务详情 - 包含主机详情 - * - * @param stepInstance 步骤实例 - * @param executeCount 执行次数 - * @param batch 滚动执行批次；如果传入null或者0，忽略该参数 - * @param status 任务状态 - * @param tag 用户自定义分组标签 - * @return Agent任务 - */ - List listAgentTaskDetailByResultGroup(StepInstanceBaseDTO stepInstance, - Integer executeCount, - Integer batch, - Integer status, - String tag); - - /** - * 获取Agent任务详情 - 包含主机详情 - * - * @param stepInstance 步骤实例 - * @param executeCount 执行次数 - * @param batch 滚动执行批次；传入null或者0将忽略该参数 - * @return Agent任务 - */ - List listAgentTaskDetail(StepInstanceBaseDTO stepInstance, - Integer executeCount, - Integer batch); - - /** - * 获取Agent任务实际执行成功的executeCount值(重试场景,兼容历史数据) - * - * @param stepInstanceId 步骤实例ID - * @param cloudIp 云区域+ip - * @return Agent任务实际执行成功的executeCount值 - */ - @CompatibleImplementation(name = "rolling_execution", explain = "兼容历史数据", version = "3.7.x") - int getActualSuccessExecuteCount(long stepInstanceId, String cloudIp); - - /** - * 批量更新AgentTask的字段 - * - * @param stepInstanceId 条件 - 步骤实例ID - * @param executeCount 条件 - 重试次数 - * @param batch 条件 - 滚动执行批次；传入null将忽略该条件 - * @param actualExecuteCount 值 - Agent任务实际执行的步骤重试次数；如果传入null，则不更新 - * @param gseTaskId 值 - Agent任务对应的GSE_TASK_ID；如果传入null，则不更新 - */ - void updateAgentTaskFields(long stepInstanceId, - int executeCount, - Integer batch, - Integer actualExecuteCount, - Long gseTaskId); - - -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ApplicationService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ApplicationService.java index cbbc7e2ff2..1ccd045431 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ApplicationService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ApplicationService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ArtifactoryLocalFileService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ArtifactoryLocalFileService.java index 9c1b8a7a81..8b56102cc0 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ArtifactoryLocalFileService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ArtifactoryLocalFileService.java @@ -1,15 +1,16 @@ package com.tencent.bk.job.execute.service; +import com.tencent.bk.job.common.artifactory.config.ArtifactoryConfig; import com.tencent.bk.job.common.artifactory.model.dto.NodeDTO; import com.tencent.bk.job.common.artifactory.sdk.ArtifactoryClient; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.exception.InternalException; import com.tencent.bk.job.common.util.file.PathUtil; -import com.tencent.bk.job.execute.config.ArtifactoryConfig; import com.tencent.bk.job.execute.config.LocalFileConfigForExecute; import com.tencent.bk.job.execute.model.FileDetailDTO; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; @Slf4j @@ -19,29 +20,29 @@ public class ArtifactoryLocalFileService { private final ArtifactoryConfig artifactoryConfig; private final LocalFileConfigForExecute localFileConfigForExecute; private final ArtifactoryClient artifactoryClient; + private final LocalFileConfigForExecute localFileConfig; @Autowired public ArtifactoryLocalFileService( - ArtifactoryConfig artifactoryConfig, LocalFileConfigForExecute localFileConfigForExecute, - ArtifactoryClient artifactoryClient + ArtifactoryConfig artifactoryConfig, + LocalFileConfigForExecute localFileConfigForExecute, + @Qualifier("jobArtifactoryClient") ArtifactoryClient artifactoryClient, + LocalFileConfigForExecute localFileConfig ) { this.artifactoryConfig = artifactoryConfig; this.localFileConfigForExecute = localFileConfigForExecute; this.artifactoryClient = artifactoryClient; + this.localFileConfig = localFileConfig; } public FileDetailDTO getFileDetailFromArtifactory(String filePath) { - String fullPath = PathUtil.joinFilePath( - artifactoryConfig.getArtifactoryJobProject() - + "/" + localFileConfigForExecute.getLocalUploadRepo(), - filePath - ); - NodeDTO nodeDTO = artifactoryClient.getFileNode(fullPath); + NodeDTO nodeDTO = getFileNodeAndHandleException(filePath); log.debug("nodeDTO={}", nodeDTO); if (nodeDTO == null) { throw new InternalException( + "local file not found in artifactory", ErrorCode.LOCAL_FILE_NOT_EXIST_IN_BACKEND, - new String[]{filePath} + new String[]{filePath, String.valueOf(localFileConfig.getExpireDays())} ); } FileDetailDTO fileDetailDTO = new FileDetailDTO(filePath); @@ -51,4 +52,28 @@ public FileDetailDTO getFileDetailFromArtifactory(String filePath) { return fileDetailDTO; } + private NodeDTO getFileNodeAndHandleException(String filePath) { + String fullPath = PathUtil.joinFilePath( + artifactoryConfig.getArtifactoryJobProject() + + "/" + localFileConfigForExecute.getLocalUploadRepo(), + filePath + ); + NodeDTO nodeDTO; + try { + nodeDTO = artifactoryClient.getFileNode(fullPath); + } catch (InternalException e) { + if (e.getErrorCode() == ErrorCode.CAN_NOT_FIND_NODE_IN_ARTIFACTORY) { + log.error("[TransferLocalFile] transfer fail, local file {} not in artifactory", filePath); + throw new InternalException( + "local file not found in artifactory", + ErrorCode.LOCAL_FILE_NOT_EXIST_IN_BACKEND, + new String[]{filePath, String.valueOf(localFileConfig.getExpireDays())} + ); + } else { + throw e; + } + } + return nodeDTO; + } + } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ContainerService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ContainerService.java new file mode 100644 index 0000000000..2879f398ab --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ContainerService.java @@ -0,0 +1,53 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.service; + +import com.tencent.bk.job.common.model.dto.Container; +import com.tencent.bk.job.execute.model.KubeContainerFilter; + +import java.util.Collection; +import java.util.List; + +/** + * 容器服务 + */ +public interface ContainerService { + /** + * 根据容器 ID 批量查询容器列表 + * + * @param appId Job 业务 ID + * @param ids 容器 ID 列表 + */ + List listContainerByIds(long appId, Collection ids); + + /** + * 根据容器过滤器查询容器列表 + * + * @param appId Job 业务 ID + * @param filter 容器过滤器 + * @return 查询到的容器列表 + */ + List listContainerByContainerFilter(long appId, KubeContainerFilter filter); +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/DangerousRecordService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/DangerousRecordService.java index 63ac169a4d..eebef81fe3 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/DangerousRecordService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/DangerousRecordService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/DangerousScriptCheckService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/DangerousScriptCheckService.java index c53fde62e7..5ece39f9f7 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/DangerousScriptCheckService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/DangerousScriptCheckService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,7 +26,7 @@ import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; import com.tencent.bk.job.manage.model.inner.ServiceScriptCheckResultItemDTO; import java.util.List; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ExecuteObjectTaskService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ExecuteObjectTaskService.java new file mode 100644 index 0000000000..1a9dbab3b1 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ExecuteObjectTaskService.java @@ -0,0 +1,165 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.service; + +import com.tencent.bk.job.common.constant.Order; +import com.tencent.bk.job.execute.model.ExecuteObjectTask; +import com.tencent.bk.job.execute.model.ResultGroupBaseDTO; +import com.tencent.bk.job.execute.model.ResultGroupDTO; +import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; + +import java.util.Collection; +import java.util.List; + +/** + * 执行对象任务 Service + */ +public interface ExecuteObjectTaskService { + + /** + * 批量保存（insert/update) 任务 + * + * @param tasks 任务列表 + */ + void batchSaveTasks(Collection tasks); + + /** + * 批量更新执行对象任务 + * + * @param tasks 执行对象任务 + */ + void batchUpdateTasks(Collection tasks); + + /** + * 获取执行成功的执行对象任务数量 + * + * @param taskInstanceId 作业实例 ID + * @param stepInstanceId 步骤实例ID + * @param executeCount 步骤执行次数 + * @return 执行成功的任务数量 + */ + int getSuccessTaskCount(Long taskInstanceId, long stepInstanceId, int executeCount); + + + /** + * 根据GSE任务ID获取执行对象任务 + * + * @param stepInstance 步骤实例 + * @param gseTaskId GSE任务ID + * @return 执行对象任务 + */ + List listTasksByGseTaskId(StepInstanceBaseDTO stepInstance, Long gseTaskId); + + /** + * 获取执行对象任务 + * + * @param stepInstance 步骤实例 + * @param executeCount 执行次数 + * @param batch 滚动执行批次；传入null或者0将忽略该参数 + * @return 执行对象任务 + */ + List listTasks(StepInstanceBaseDTO stepInstance, + Integer executeCount, + Integer batch); + + /** + * 获取执行对象任务详情并分组 + * + * @param stepInstance 步骤实例 + * @param executeCount 执行次数 + * @param batch 滚动执行批次；如果传入null或者0，将忽略该参数 + */ + List listAndGroupTasks(StepInstanceBaseDTO stepInstance, + int executeCount, + Integer batch); + + /** + * 获取任务结果分组 + * + * @param stepInstance 步骤实例 + * @param executeCount 执行次数 + * @param batch 滚动执行批次；传入null或者0将忽略该参数 + * @return 执行对象任务 + */ + List listResultGroups(StepInstanceBaseDTO stepInstance, + int executeCount, + Integer batch); + + /** + * 根据执行结果查询执行对象任务详情(排序、限制返回数量) - 包含主机详情 + * + * @param stepInstance 步骤实例 + * @param executeCount 执行次数 + * @param batch 滚动执行批次；如果传入null或者0，忽略该参数 + * @param status 任务状态 + * @param tag 用户自定义分组标签 + * @param limit 最大返回数量 + * @param orderField 排序字段 + * @param order 排序方式 + * @return 执行对象任务 + */ + List listTaskByResultGroup(StepInstanceBaseDTO stepInstance, + Integer executeCount, + Integer batch, + Integer status, + String tag, + Integer limit, + String orderField, + Order order); + + /** + * 根据结果分组获取执行对象任务详情 - 包含主机详情 + * + * @param stepInstance 步骤实例 + * @param executeCount 执行次数 + * @param batch 滚动执行批次；如果传入null或者0，忽略该参数 + * @param status 任务状态 + * @param tag 用户自定义分组标签 + * @return 执行对象任务 + */ + List listTaskByResultGroup(StepInstanceBaseDTO stepInstance, + Integer executeCount, + Integer batch, + Integer status, + String tag); + + + /** + * 批量更新任务的字段 + * + * @param stepInstance 条件 - 步骤实例 + * @param executeCount 条件 - 重试次数 + * @param batch 条件 - 滚动执行批次；传入null将忽略该条件 + * @param actualExecuteCount 值 - 执行对象任务实际执行的步骤重试次数；如果传入null，则不更新 + * @param gseTaskId 值 - 执行对象任务对应的GSE_TASK_ID；如果传入null，则不更新 + */ + void updateTaskFields(StepInstanceBaseDTO stepInstance, + int executeCount, + Integer batch, + Integer actualExecuteCount, + Long gseTaskId); + + +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ExternalAgentService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ExternalAgentService.java new file mode 100644 index 0000000000..1c44424c68 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ExternalAgentService.java @@ -0,0 +1,42 @@ +/* + * + * * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * * + * * Copyright (C) 2021 Tencent. All rights reserved. + * * + * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * * + * * License for BK-JOB蓝鲸智云作业平台: + * * -------------------------------------------------------------------- + * * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * * + * * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * * the Software. + * * + * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * * IN THE SOFTWARE. + * + */ + +package com.tencent.bk.job.execute.service; + +import com.tencent.bk.job.common.model.dto.HostDTO; + +/** + * 当且仅当启用了集群外机器作为文件分发的源时生效 + * 用于管理集群外的机器 + */ +public interface ExternalAgentService { + + /** + * 从已配置的集群外主机池中获取一台主机作为分发源 + * @return 可用的分发源主机 + */ + HostDTO getDistributeSourceHost(); +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/FileAgentTaskService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/FileAgentTaskService.java deleted file mode 100644 index 5d2c7c6bd2..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/FileAgentTaskService.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.service; - -import com.tencent.bk.job.common.model.dto.HostDTO; -import com.tencent.bk.job.execute.model.AgentTaskDTO; -import com.tencent.bk.job.execute.model.StepInstanceDTO; -import com.tencent.bk.job.logsvr.consts.FileTaskModeEnum; - -import java.util.List; - -/** - * GSE Agent 文件任务 Service - */ -public interface FileAgentTaskService extends AgentTaskService { - - /** - * 获取agent任务 - * - * @param stepInstanceId 步骤实例ID - * @param executeCount 执行次数 - * @param batch 滚动执行批次；传入null或者0将忽略该参数 - * @param fileTaskMode 文件分发任务模式;传入null表示忽略该过滤条件 - * @return agent任务 - */ - List listAgentTasks(Long stepInstanceId, - Integer executeCount, - Integer batch, - FileTaskModeEnum fileTaskMode); - - /** - * @param stepInstance 步骤实例 - * @param executeCount 执行次数 - * @param batch 滚动执行批次；传入null或者0将忽略该参数 - * @param fileTaskMode 文件分发任务模式 - * @param host 主机 - * @return Agent任务 - */ - AgentTaskDTO getAgentTaskByHost(StepInstanceDTO stepInstance, - Integer executeCount, - Integer batch, - FileTaskModeEnum fileTaskMode, - HostDTO host); -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/FileExecuteObjectTaskService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/FileExecuteObjectTaskService.java new file mode 100644 index 0000000000..9c9d0b4d40 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/FileExecuteObjectTaskService.java @@ -0,0 +1,66 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.service; + +import com.tencent.bk.job.execute.model.ExecuteObjectCompositeKey; +import com.tencent.bk.job.execute.model.ExecuteObjectTask; +import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; +import com.tencent.bk.job.logsvr.consts.FileTaskModeEnum; + +import java.util.List; + +/** + * GSE Agent 文件任务 Service + */ +public interface FileExecuteObjectTaskService extends ExecuteObjectTaskService { + + /** + * 获取执行对象任务 + * + * @param stepInstance 步骤实例 + * @param executeCount 执行次数 + * @param batch 滚动执行批次；传入null或者0将忽略该参数 + * @param fileTaskMode 文件分发任务模式;传入null表示忽略该过滤条件 + * @return 执行对象任务 + */ + List listTasks(StepInstanceBaseDTO stepInstance, + Integer executeCount, + Integer batch, + FileTaskModeEnum fileTaskMode); + + /** + * @param stepInstance 步骤实例 + * @param executeCount 执行次数 + * @param batch 滚动执行批次；传入null或者0将忽略该参数 + * @param fileTaskMode 文件分发任务模式 + * @param executeObjectCompositeKey 执行对象复合 KEY + * @return 执行对象任务 + */ + ExecuteObjectTask getTaskByExecuteObjectCompositeKey(StepInstanceBaseDTO stepInstance, + Integer executeCount, + Integer batch, + FileTaskModeEnum fileTaskMode, + ExecuteObjectCompositeKey executeObjectCompositeKey); +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/FileSourceTaskLogService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/FileSourceTaskLogService.java new file mode 100644 index 0000000000..3c414d43b4 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/FileSourceTaskLogService.java @@ -0,0 +1,43 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.service; + +import com.tencent.bk.job.execute.model.FileSourceTaskLogDTO; + +public interface FileSourceTaskLogService { + int addFileSourceTaskLog(FileSourceTaskLogDTO fileSourceTaskLog); + + int updateFileSourceTaskLog(FileSourceTaskLogDTO fileSourceTaskLog); + + FileSourceTaskLogDTO getFileSourceTaskLog(Long taskInstanceId, long stepInstanceId, int executeCount); + + FileSourceTaskLogDTO getFileSourceTaskLogByBatchTaskId(Long taskInstanceId, String fileSourceBatchTaskId); + + int updateTimeConsumingByBatchTaskId(Long taskInstanceId, + String fileSourceBatchTaskId, + Long startTime, + Long endTime, + Long totalTime); +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/FileSourceTaskService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/FileSourceTaskService.java index 8db3bc10b7..15c4a43026 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/FileSourceTaskService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/FileSourceTaskService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/GseTaskService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/GseTaskService.java index 6e53081c98..cfc3a4261c 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/GseTaskService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/GseTaskService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,6 +25,9 @@ package com.tencent.bk.job.execute.service; import com.tencent.bk.job.execute.model.GseTaskDTO; +import com.tencent.bk.job.execute.model.GseTaskSimpleDTO; + +import java.util.List; /** * GSE 任务 Service @@ -49,18 +52,40 @@ public interface GseTaskService { /** * 获取 GSE 任务 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 步骤执行次数 * @param batch 滚动执行批次 * @return GSE 任务 */ - GseTaskDTO getGseTask(long stepInstanceId, int executeCount, Integer batch); + GseTaskDTO getGseTask(Long taskInstanceId, long stepInstanceId, int executeCount, Integer batch); /** * 获取 GSE 任务 * - * @param gseTaskId GSE任务ID + * @param taskInstanceId 作业实例 ID + * @param gseTaskId GSE任务ID * @return GSE 任务 */ - GseTaskDTO getGseTask(long gseTaskId); + GseTaskDTO getGseTask(Long taskInstanceId, long gseTaskId); + + /** + * 获取 GSE 任务 + * + * @param gseTaskId GSE任务ID + * @return stepInstanceId + */ + GseTaskSimpleDTO getGseTaskSimpleInfo(String gseTaskId); + + /** + * 获取 GSE 任务列表 + * + * @param stepInstanceId 步骤实例ID + * @param executeCount 执行次数 + * @param batch 批次 + * @return GSE 任务列表 + */ + List listGseTaskSimpleInfo(Long stepInstanceId, + Integer executeCount, + Integer batch); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/HostService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/HostService.java index 4d0e01634f..51abbbcd63 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/HostService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/HostService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,6 +26,8 @@ import com.tencent.bk.job.common.cc.model.CcInstanceDTO; import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.execute.model.DynamicServerGroupDTO; +import com.tencent.bk.job.execute.model.DynamicServerTopoNodeDTO; import com.tencent.bk.job.manage.model.inner.ServiceHostDTO; import com.tencent.bk.job.manage.model.inner.ServiceListAppHostResultDTO; @@ -55,32 +57,23 @@ public interface HostService { ServiceHostDTO getHost(HostDTO host); /** - * 查询主机在白名单中允许的操作 + * 通过云区域ID与IPv6地址获取主机信息 * - * @param appId 业务ID - * @param host 主机 - * @return 允许的操作 - */ - List getHostAllowedAction(long appId, HostDTO host); - - /** - * 判断IP白名单是否配置该主机的对应操作 - * - * @param appId 业务ID - * @param cloudIp 主机IP - * @param action 操作 - * @return 是否配置规则 + * @param cloudAreaId 云区域ID + * @param ipv6 IPv6地址 + * @return 主机信息 */ - boolean isMatchWhiteIpRule(long appId, String cloudIp, String action); + ServiceHostDTO getHostByCloudIpv6(long cloudAreaId, String ipv6); /** * 获取业务下的主机列表 * - * @param appId Job业务ID - * @param hosts 主机列表 + * @param appId Job业务ID + * @param hosts 主机列表 + * @param refreshAgentId 是否重新刷新主机agentId * @return 主机信息 */ - ServiceListAppHostResultDTO batchGetAppHosts(Long appId, Collection hosts); + ServiceListAppHostResultDTO batchGetAppHosts(Long appId, Collection hosts, boolean refreshAgentId); /** * 获取动态分组主机 @@ -89,7 +82,19 @@ public interface HostService { * @param groupId 动态分组ID * @return 主机列表 */ - List getIpByDynamicGroupId(long appId, String groupId); + List getHostsByDynamicGroupId(long appId, String groupId); + + /** + * 批量获取动态分组主机，并根据动态分组ID对主机进行分组 + * + * @param appId 业务ID + * @param groupIds 动态分组ID列表 + * @return Map, key: 动态分组 value: 动态分组下的主机列表 + */ + Map> batchGetAndGroupHostsByDynamicGroup( + long appId, + Collection groupIds); + /** * 根据topo节点获取主机 @@ -98,13 +103,17 @@ public interface HostService { * @param ccInstances topo节点列表 * @return 主机列表 */ - List getIpByTopoNodes(long appId, List ccInstances); + List getHostsByTopoNodes(long appId, List ccInstances); /** - * 获取主机云区域名称 + * 根据topo节点获取主机并分组 * - * @param cloudAreaId 云区域ID - * @return 云区域名称 + * @param appId Job业务ID + * @param topoNodes topo节点列表 + * @return Map, key: topo 节点, value: 主机列表 */ - String getCloudAreaName(long cloudAreaId); + Map> getAndGroupHostsByTopoNodes( + long appId, + Collection topoNodes + ); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/LocalFileDistributeSourceHostProvisioner.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/LocalFileDistributeSourceHostProvisioner.java new file mode 100644 index 0000000000..b62b703601 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/LocalFileDistributeSourceHostProvisioner.java @@ -0,0 +1,40 @@ +/* + * + * * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * * + * * Copyright (C) 2021 Tencent. All rights reserved. + * * + * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * * + * * License for BK-JOB蓝鲸智云作业平台: + * * -------------------------------------------------------------------- + * * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * * + * * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * * the Software. + * * + * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * * IN THE SOFTWARE. + * + */ + +package com.tencent.bk.job.execute.service; + +import com.tencent.bk.job.common.model.dto.HostDTO; + +/** + * 分发本地文件时，根据配置使用job机器或者集群外机器作为源进行分发 + */ +public interface LocalFileDistributeSourceHostProvisioner { + + /** + * @return job-execute所在Node 信息 或者 集群外的机器信息 + */ + HostDTO getLocalFileDistributeSourceHost(); +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/LogExportService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/LogExportService.java index e354805c34..d5eaa03b7b 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/LogExportService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/LogExportService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,7 @@ package com.tencent.bk.job.execute.service; +import com.tencent.bk.job.common.constant.ExecuteObjectTypeEnum; import com.tencent.bk.job.execute.model.LogExportJobInfoDTO; /** @@ -31,8 +32,19 @@ */ public interface LogExportService { - LogExportJobInfoDTO packageLogFile(String username, Long appId, Long stepInstanceId, Long hostId, String ip, - int executeCount, String logFileDir, String logFileName, Boolean repackage); + LogExportJobInfoDTO packageLogFile(String username, + Long appId, + Long taskInstanceId, + Long stepInstanceId, + ExecuteObjectTypeEnum executeObjectType, + Long executeObjectResourceId, + int executeCount, + String logFileDir, + String logFileName, + Boolean repackage); - LogExportJobInfoDTO getExportInfo(Long appId, Long stepInstanceId, Long hostId, String ip); + LogExportJobInfoDTO getExportInfo(Long appId, + Long stepInstanceId, + ExecuteObjectTypeEnum executeObjectType, + Long executeObjectResourceId); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/LogService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/LogService.java index dd103242a1..a7d8bab9d3 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/LogService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/LogService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,16 +24,25 @@ package com.tencent.bk.job.execute.service; -import com.tencent.bk.job.common.exception.ServiceException; -import com.tencent.bk.job.common.model.dto.HostDTO; -import com.tencent.bk.job.execute.model.FileIpLogContent; -import com.tencent.bk.job.execute.model.ScriptHostLogContent; +import com.tencent.bk.job.execute.common.constants.FileDistStatusEnum; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; +import com.tencent.bk.job.execute.engine.model.JobFile; +import com.tencent.bk.job.execute.model.AtomicFileTaskLog; +import com.tencent.bk.job.execute.model.ExecuteObjectCompositeKey; +import com.tencent.bk.job.execute.model.ExecuteObjectTask; +import com.tencent.bk.job.execute.model.FileExecuteObjectLogContent; +import com.tencent.bk.job.execute.model.ScriptExecuteObjectLogContent; +import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; +import com.tencent.bk.job.execute.model.StepInstanceDTO; +import com.tencent.bk.job.execute.model.TaskInstanceDTO; +import com.tencent.bk.job.logsvr.consts.FileTaskModeEnum; +import com.tencent.bk.job.logsvr.model.service.ServiceExecuteObjectLogDTO; +import com.tencent.bk.job.logsvr.model.service.ServiceExecuteObjectScriptLogDTO; import com.tencent.bk.job.logsvr.model.service.ServiceFileTaskLogDTO; -import com.tencent.bk.job.logsvr.model.service.ServiceHostLogDTO; -import com.tencent.bk.job.logsvr.model.service.ServiceHostLogsDTO; -import com.tencent.bk.job.logsvr.model.service.ServiceScriptLogDTO; +import com.tencent.bk.job.manage.api.common.constants.task.TaskFileTypeEnum; import java.util.List; +import java.util.Map; /** * 日志服务 @@ -43,136 +52,292 @@ public interface LogService { /** * 构造job系统日志 * - * @param host 主机 + * @param stepInstance 步骤实例 + * @param executeObject 执行对象 * @param content 日志原始内容 * @param offset 日志偏移 - 字节 * @param logTimeInMillSeconds 日志时间 * @return 系统日志 */ - ServiceScriptLogDTO buildSystemScriptLog(HostDTO host, String content, int offset, Long logTimeInMillSeconds); + ServiceExecuteObjectScriptLogDTO buildSystemScriptLog(StepInstanceBaseDTO stepInstance, + ExecuteObject executeObject, + String content, + int offset, + Long logTimeInMillSeconds); + + /** + * 构造脚本日志 + * + * @param stepInstance 步骤实例 + * @param executeObject 执行对象 + * @param content 日志原始内容 + * @param contentSizeBytes 日志内容大小(单位byte) + * @param offset 日志偏移 - 字节 + * @return 系统日志 + */ + ServiceExecuteObjectScriptLogDTO buildScriptLog(StepInstanceBaseDTO stepInstance, + ExecuteObject executeObject, + String content, + int contentSizeBytes, + int offset); /** * 写脚本执行日志 * - * @param jobCreateTime 任务创建时间 + * @param taskInstance 任务实例 * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 * @param batch 滚动执行批次;非滚动步骤传入null * @param scriptLogs 脚本日志 */ - void batchWriteScriptLog(long jobCreateTime, long stepInstanceId, int executeCount, Integer batch, - List scriptLogs); + void batchWriteScriptLog(TaskInstanceDTO taskInstance, + long stepInstanceId, + int executeCount, + Integer batch, + List scriptLogs); /** - * 获取脚本执行日志 + * 根据执行对象获取脚本执行日志 * - * @param stepInstanceId 步骤实例 ID - * @param executeCount 执行次数 - * @param batch 滚动执行批次;非滚动步骤传入null - * @param host 主机 + * @param stepInstance 步骤实例 + * @param executeCount 执行次数 + * @param batch 滚动执行批次;非滚动步骤传入null + * @param key 执行对象复合 KEY + * @return 日志内容 + */ + ScriptExecuteObjectLogContent getScriptExecuteObjectLogContent(StepInstanceBaseDTO stepInstance, + int executeCount, + Integer batch, + ExecuteObjectCompositeKey key); + + /** + * 根据执行对象任务获取脚本执行日志 + * + * @param stepInstance 步骤实例 + * @param executeCount 执行次数 + * @param batch 滚动执行批次;非滚动步骤传入null + * @param executeObjectTask 执行对象任务 * @return 日志内容 */ - ScriptHostLogContent getScriptHostLogContent(long stepInstanceId, int executeCount, Integer batch, HostDTO host); + ScriptExecuteObjectLogContent getScriptExecuteObjectLogContent(StepInstanceBaseDTO stepInstance, + int executeCount, + Integer batch, + ExecuteObjectTask executeObjectTask); /** * 批量获取脚本执行日志 * - * @param jobCreateDateStr 作业创建时间 - * @param stepInstanceId 步骤实例 ID - * @param executeCount 执行次数 - * @param batch 滚动执行批次;非滚动步骤传入null - * @param hosts 主机列表,最大支持1000个 + * @param jobCreateDateStr 作业创建时间 + * @param stepInstance 步骤实例 + * @param executeCount 执行次数 + * @param batch 滚动执行批次;非滚动步骤传入null + * @param executeObjectCompositeKeys 执行对象复合 KEY 列表,最大支持1000个 * @return 日志内容 */ - List batchGetScriptHostLogContent(String jobCreateDateStr, long stepInstanceId, - int executeCount, Integer batch, List hosts); + List batchGetScriptExecuteObjectLogContent( + String jobCreateDateStr, + StepInstanceBaseDTO stepInstance, + int executeCount, + Integer batch, + List executeObjectCompositeKeys + ); /** - * 获取脚本执行日志 + * 根据执行对象获取脚本执行日志 * - * @param stepInstanceId 步骤实例 ID - * @param executeCount 执行次数 - * @param batch 滚动执行批次;非滚动步骤传入null - * @param host 主机 - * @param mode 文件传输模式 + * @param stepInstance 步骤实例 + * @param executeCount 执行次数 + * @param batch 滚动执行批次;非滚动步骤传入null + * @param executeObjectCompositeKey 执行对象复合 KEY + * @param mode 文件传输模式 * @return 日志内容 - * @throws ServiceException 异常 */ - FileIpLogContent getFileIpLogContent(long stepInstanceId, int executeCount, Integer batch, HostDTO host, - Integer mode); + FileExecuteObjectLogContent getFileExecuteObjectLogContent( + StepInstanceBaseDTO stepInstance, + int executeCount, + Integer batch, + ExecuteObjectCompositeKey executeObjectCompositeKey, + Integer mode + ); /** - * 获取脚本执行日志 + * 根据执行对象任务获取脚本执行日志 * + * @param stepInstance 步骤实例 + * @param executeCount 执行次数 + * @param batch 滚动执行批次;非滚动步骤传入null + * @param executeObjectTask 执行对象任务 + * @return 日志内容 + */ + FileExecuteObjectLogContent getFileExecuteObjectLogContent( + StepInstanceBaseDTO stepInstance, + int executeCount, + Integer batch, + ExecuteObjectTask executeObjectTask + ); + + /** + * 根据文件任务ID批量获取文件任务执行日志 + * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例 ID * @param executeCount 执行次数 * @param batch 滚动执行批次;非滚动步骤传入null * @param taskIds 文件任务ID列表 * @return 日志内容 */ - List getFileLogContentByTaskIds(long stepInstanceId, int executeCount, Integer batch, - List taskIds); + List getAtomicFileTaskLogByTaskIds(long taskInstanceId, + long stepInstanceId, + int executeCount, + Integer batch, + List taskIds); /** * 获取文件任务文件源日志 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例 ID * @param executeCount 执行次数 * @param batch 滚动执行批次;非滚动步骤传入null * @return 日志内容 */ - List batchGetFileSourceIpLogContent(long stepInstanceId, - int executeCount, - Integer batch); + List batchGetFileSourceExecuteObjectLogContent(long taskInstanceId, + long stepInstanceId, + int executeCount, + Integer batch); /** - * 获取文件任务文件源日志 + * 获取文件任务文件日志 * - * @param stepInstanceId 步骤实例 ID - * @param executeCount 执行次数 - * @param batch 滚动执行批次;非滚动步骤传入null - * @param hosts 主机列表 + * @param taskInstanceId 作业实例 ID + * @param stepInstanceId 步骤实例 ID + * @param executeCount 执行次数 + * @param batch 滚动执行批次;非滚动步骤传入null + * @param mode 文件分发任务模式;传入 null 该过滤条件不生效 + * @param executeObjectCompositeKeys 执行对象复合 KEY 列表;传入 null 该过滤条件不生效 * @return 日志内容 */ - ServiceHostLogsDTO batchGetFileIpLogContent(long stepInstanceId, - int executeCount, - Integer batch, - List hosts); + List batchGetFileExecuteObjectLogContent( + long taskInstanceId, + long stepInstanceId, + int executeCount, + Integer batch, + FileTaskModeEnum mode, + List executeObjectCompositeKeys); /** - * 根据关键字获取对应的ip + * 根据日志关键字获取对应的执行对象KEY * - * @param stepInstanceId 步骤实例ID - * @param executeCount 执行次数 - * @param batch 滚动执行批次;非滚动步骤传入null - * @param keyword 关键字 - * @return ips + * @param stepInstance 步骤实例 + * @param executeCount 执行次数 + * @param batch 滚动执行批次;非滚动步骤传入null + * @param keyword 关键字 + * @return 执行对象列表¬ */ - List getIpsByContentKeyword(long stepInstanceId, int executeCount, Integer batch, String keyword); + List getExecuteObjectsCompositeKeysByContentKeyword(StepInstanceBaseDTO stepInstance, + int executeCount, + Integer batch, + String keyword); /** - * 写日志 + * 写文件日志日志 - 指定时间 * - * @param jobCreateTime 任务创建时间 - * @param stepInstanceId 步骤实例ID - * @param executeCount 执行次数 - * @param batch 滚动执行批次;非滚动步骤传入null - * @param host 主机 - * @param executionLog 文件任务执行日志 + * @param taskInstance 任务实例 + * @param hostFileLogs 主机执行日志 * @param logTimeInMillSeconds 日志时间 */ - void writeFileLogWithTimestamp(long jobCreateTime, long stepInstanceId, int executeCount, Integer batch, - HostDTO host, ServiceHostLogDTO executionLog, - Long logTimeInMillSeconds); + void writeFileLogsWithTimestamp(TaskInstanceDTO taskInstance, + List hostFileLogs, + Long logTimeInMillSeconds); /** * 写文件日志日志 * - * @param jobCreateTime 任务创建时间 - * @param fileLogs 文件任务执行日志 + * @param taskInstance 任务实例 + * @param executeObjectLogs 文件任务执行日志 + */ + void writeFileLogs(TaskInstanceDTO taskInstance, List executeObjectLogs); + + /** + * 构造上传文件任务日志 + * + * @param stepInstance 步骤实例 + * @param srcFile 源文件 + * @param status 任务状态 + * @param size 源文件大小 + * @param speed 上传速度 + * @param process 进度 + * @param content 日志内容 + */ + ServiceFileTaskLogDTO buildUploadServiceFileTaskLogDTO(StepInstanceDTO stepInstance, + JobFile srcFile, + FileDistStatusEnum status, + String size, + String speed, + String process, + String content); + + /** + * 构造上传文件任务日志 + * + * @param stepInstance 步骤实例 + * @param fileType 源文件类型 + * @param srcFilePath 源文件路径 + * @param displaySrcFilePath 展示给用户的源文件路径 + * @param executeObject 上传源执行对象 + * @param status 任务状态 + * @param size 源文件大小 + * @param speed 上传速度 + * @param process 进度 + * @param content 日志内容 + */ + ServiceFileTaskLogDTO buildUploadServiceFileTaskLogDTO(StepInstanceDTO stepInstance, + TaskFileTypeEnum fileType, + String srcFilePath, + String displaySrcFilePath, + ExecuteObject executeObject, + FileDistStatusEnum status, + String size, + String speed, + String process, + String content); + + /** + * 构造下载文件任务日志 + * + * @param stepInstance 步骤实例 + * @param srcFile 源文件 + * @param targetExecuteObject 目标执行对象 + * @param targetPath 目标路径 + * @param status 任务状态 + * @param size 源文件大小 + * @param speed 下载速度 + * @param process 进度 + * @param content 日志内容 + */ + ServiceFileTaskLogDTO buildDownloadServiceFileTaskLogDTO(StepInstanceDTO stepInstance, + JobFile srcFile, + ExecuteObject targetExecuteObject, + String targetPath, + FileDistStatusEnum status, + String size, + String speed, + String process, + String content); + + /** + * 增加文件任务日志，并且按照执行对象的维度对文件任务日志进行分组 + * + * @param stepInstance 步骤实例 + * @param executeObjectLogs 执行对象日志列表 + * @param executeObject 文件任务对应的执行对象 + * @param fileTaskLog 单个文件任务日志 */ - void writeFileLogs(long jobCreateTime, List fileLogs); + void addFileTaskLog(StepInstanceBaseDTO stepInstance, + Map executeObjectLogs, + ExecuteObject executeObject, + ServiceFileTaskLogDTO fileTaskLog); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/NotifyService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/NotifyService.java index d67d027091..423be6e26a 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/NotifyService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/NotifyService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/RollingConfigService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/RollingConfigService.java index c9d9c39a48..2ca5b2925a 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/RollingConfigService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/RollingConfigService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,7 @@ package com.tencent.bk.job.execute.service; -import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; import com.tencent.bk.job.execute.model.FastTaskDTO; import com.tencent.bk.job.execute.model.RollingConfigDTO; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; @@ -36,13 +36,13 @@ */ public interface RollingConfigService { /** - * 根据滚动批次获取主机 + * 根据滚动批次获取执行对象 * * @param stepInstance 步骤实例 * @param batch 滚动执行批次 * @return 主机列表 */ - List getRollingServers(StepInstanceBaseDTO stepInstance, Integer batch); + List getRollingServers(StepInstanceBaseDTO stepInstance, Integer batch); /** * 保存快速执行作业滚动配置 @@ -52,5 +52,14 @@ public interface RollingConfigService { */ RollingConfigDTO saveRollingConfigForFastJob(FastTaskDTO fastTask); - RollingConfigDTO getRollingConfig(long rollingConfigId); + RollingConfigDTO getRollingConfig(Long taskInstanceId, long rollingConfigId); + + /** + * 任务是否启用了滚动执行 + * @param taskInstanceId 任务id + * @return boolean true启用，false未启用 + */ + boolean isTaskRollingEnabled(long taskInstanceId); + + long addRollingConfig(RollingConfigDTO rollingConfig); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ScriptAgentTaskService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ScriptAgentTaskService.java deleted file mode 100644 index 67cb8cfa34..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ScriptAgentTaskService.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.service; - -import com.tencent.bk.job.common.model.dto.HostDTO; -import com.tencent.bk.job.execute.model.AgentTaskDTO; -import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; - -/** - * GSE Agent 脚本任务 Service - */ -public interface ScriptAgentTaskService extends AgentTaskService { - /** - * 根据主机ID获取Agent任务 - * - * @param stepInstance 步骤实例 - * @param executeCount 执行次数 - * @param batch 滚动执行批次；传入null或者0将忽略该参数 - * @param host 主机 - * @return Agent任务 - */ - AgentTaskDTO getAgentTaskByHost(StepInstanceBaseDTO stepInstance, - Integer executeCount, - Integer batch, - HostDTO host); -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ScriptExecuteObjectTaskService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ScriptExecuteObjectTaskService.java new file mode 100644 index 0000000000..1a135dfe9b --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ScriptExecuteObjectTaskService.java @@ -0,0 +1,62 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.service; + +import com.tencent.bk.job.execute.model.ExecuteObjectCompositeKey; +import com.tencent.bk.job.execute.model.ExecuteObjectTask; +import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; + +import java.util.Collection; +import java.util.List; + +public interface ScriptExecuteObjectTaskService extends ExecuteObjectTaskService { + /** + * 根据执行对象复合 KEY 获取执行对象任务 + * + * @param stepInstance 步骤实例 + * @param executeCount 执行次数 + * @param batch 滚动执行批次；传入null或者0将忽略该参数 + * @param executeObjectCompositeKey 执行对象复合 KEY + * @return 执行对象任务 + */ + ExecuteObjectTask getTaskByExecuteObjectCompositeKey(StepInstanceBaseDTO stepInstance, + Integer executeCount, + Integer batch, + ExecuteObjectCompositeKey executeObjectCompositeKey); + + /** + * 根据执行对象ID批量获取执行对象任务 + * + * @param stepInstance 步骤实例 + * @param executeCount 执行次数 + * @param batch 滚动执行批次；传入null或者0将忽略该参数 + * @param executeObjectIds 执行对象ID列表 + * @return 执行对象任务列表 + */ + List getTaskByExecuteObjectIds(StepInstanceBaseDTO stepInstance, + Integer executeCount, + Integer batch, + Collection executeObjectIds); +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ScriptService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ScriptService.java index 49b66d6767..548daf298c 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ScriptService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ScriptService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceRollingTaskService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceRollingTaskService.java index 3bdce59ac7..508a34bcef 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceRollingTaskService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceRollingTaskService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -37,32 +37,36 @@ public interface StepInstanceRollingTaskService { /** * 查询步骤滚动任务 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 步骤执行次数 * @param batch 滚动批次 * @return 步骤滚动任务 */ - StepInstanceRollingTaskDTO queryRollingTask(long stepInstanceId, + StepInstanceRollingTaskDTO queryRollingTask(Long taskInstanceId, + long stepInstanceId, int executeCount, int batch); /** * 查询最新的步骤滚动任务 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 步骤执行次数 * @return 步骤滚动任务 */ - List listLatestRollingTasks(long stepInstanceId, int executeCount); + List listLatestRollingTasks(Long taskInstanceId, long stepInstanceId, int executeCount); /** * 根据批次查询滚动任务 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param batch 滚动批次 * @return 步骤滚动任务 */ - List listRollingTasksByBatch(long stepInstanceId, Integer batch); + List listRollingTasksByBatch(Long taskInstanceId, long stepInstanceId, Integer batch); /** * 保存步骤滚动任务 @@ -75,6 +79,7 @@ StepInstanceRollingTaskDTO queryRollingTask(long stepInstanceId, /** * 更新滚动任务 * + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 步骤执行次数 * @param batch 滚动执行批次 @@ -83,7 +88,8 @@ StepInstanceRollingTaskDTO queryRollingTask(long stepInstanceId, * @param endTime 任务结束时间；如果不更新传入null * @param totalTime 任务执行总时间；如果不更新传入null */ - void updateRollingTask(long stepInstanceId, + void updateRollingTask(Long taskInstanceId, + long stepInstanceId, int executeCount, int batch, RunStatusEnum status, @@ -92,4 +98,12 @@ void updateRollingTask(long stepInstanceId, Long totalTime); + /** + * 根据步骤实例ID查询滚动任务 + * + * @param taskInstanceId 作业实例 ID + * @param stepInstanceId 步骤实例ID + * @return 步骤滚动任务 + */ + List listRollingTasksByStep(Long taskInstanceId, long stepInstanceId); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceService.java index 2050800eab..6b101b92a4 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,9 +24,16 @@ package com.tencent.bk.job.execute.service; -import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.common.exception.NotFoundException; +import com.tencent.bk.job.execute.common.constants.RunStatusEnum; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; +import com.tencent.bk.job.execute.model.ExecuteObjectCompositeKey; +import com.tencent.bk.job.execute.model.FileSourceDTO; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; +import com.tencent.bk.job.execute.model.StepInstanceDTO; +import java.util.Collection; +import java.util.List; import java.util.Map; import java.util.function.Function; @@ -37,26 +44,20 @@ public interface StepInstanceService { /** * 更新步骤实例的当前滚动执行批次 * + * @param taskInstanceId 作业实例ID * @param stepInstanceId 步骤实例ID * @param batch 滚动执行批次 */ - void updateStepCurrentBatch(long stepInstanceId, int batch); - - /** - * 更新步骤实例的当前滚动执行批次 - * - * @param stepInstanceId 步骤实例ID - * @param executeCount 执行次数 - */ - void updateStepCurrentExecuteCount(long stepInstanceId, int executeCount); + void updateStepCurrentBatch(Long taskInstanceId, long stepInstanceId, int batch); /** * 更新步骤实例的滚动配置ID * + * @param taskInstanceId 作业实例ID * @param stepInstanceId 步骤实例ID * @param rollingConfigId 滚动配置ID */ - void updateStepRollingConfigId(long stepInstanceId, long rollingConfigId); + void updateStepRollingConfigId(Long taskInstanceId, long stepInstanceId, long rollingConfigId); /** * 获取下一个步骤实例 @@ -65,25 +66,239 @@ public interface StepInstanceService { * @param currentStepOrder 当前步骤的顺序 * @return 步骤实例；如果当前为最后一个步骤实例，那么返回null */ - StepInstanceBaseDTO getNextStepInstance(long taskInstanceId, int currentStepOrder); + StepInstanceBaseDTO getNextStepInstance(Long taskInstanceId, int currentStepOrder); /** - * 获取步骤包含的主机(源+目标) + * 获取步骤包含的执行对象(源+目标) * * @param stepInstance 步骤实例 * @return 步骤实例包含的主机列表 */ - Map computeStepHosts(StepInstanceBaseDTO stepInstance, - Function keyMapper); + Map computeStepExecuteObjects(StepInstanceBaseDTO stepInstance, + Function keyMapper); + + /** + * 保存步骤实例 + * + * @param stepInstance 步骤实例 + * @return 步骤实例ID + */ + long addStepInstance(StepInstanceDTO stepInstance); + + /** + * 获取作业实例下的所有步骤实例 + * + * @param taskInstanceId 作业实例 ID + * @return 步骤实例列表 + */ + List listBaseStepInstanceByTaskInstanceId(Long taskInstanceId); /** - * 获取步骤包含的主机(源+目标) + * 获取作业实例下的所有步骤实例(详细) * + * @param taskInstanceId 作业实例 ID + * @return 步骤实例列表 + */ + List listStepInstanceByTaskInstanceId(Long taskInstanceId); + + /** + * 获取步骤基本信息 + * + * @param taskInstanceId 作业实例ID * @param stepInstanceId 步骤实例ID - * @return 步骤实例包含的主机列表 + * @return 步骤基本信息 */ - Map computeStepHosts(long stepInstanceId, - Function keyMapper); + StepInstanceBaseDTO getBaseStepInstance(Long taskInstanceId, long stepInstanceId); + /** + * 获取步骤基本信息 + * + * @param stepInstanceId 步骤实例ID + * @return 步骤基本信息 + */ + StepInstanceBaseDTO getBaseStepInstanceById(long stepInstanceId); + /** + * 获取步骤基本信息 + * + * @param appId 业务 ID + * @param taskInstanceId 作业实例ID + * @param stepInstanceId 步骤实例ID + * @return 步骤基本信息 + */ + StepInstanceBaseDTO getBaseStepInstance(long appId, Long taskInstanceId, long stepInstanceId); + + /** + * 获取步骤基本信息 + * + * @param taskInstanceId 作业实例ID + * @param stepInstanceId 步骤实例ID + * @return 步骤基本信息 + */ + StepInstanceDTO getStepInstanceDetail(Long taskInstanceId, long stepInstanceId) throws NotFoundException; + + /** + * 获取步骤基本信息 + * + * @param appId 业务 ID + * @param taskInstanceId 作业实例ID + * @param stepInstanceId 步骤实例ID + * @return 步骤基本信息 + */ + StepInstanceDTO getStepInstanceDetail(long appId, + Long taskInstanceId, + long stepInstanceId) throws NotFoundException; + + /** + * 获取作业的第一个步骤实例 + * + * @param taskInstanceId 作业实例ID + * @return 作业第一个步骤实例 + */ + StepInstanceBaseDTO getFirstStepInstance(Long taskInstanceId); + + List getTaskStepIdList(Long taskInstanceId); + + void updateStepStatus(Long taskInstanceId, long stepInstanceId, int status); + + /** + * 重试步骤操作-重置步骤执行状态 + * + * @param taskInstanceId 作业实例ID + * @param stepInstanceId 步骤实例ID + */ + void resetStepExecuteInfoForRetry(Long taskInstanceId, long stepInstanceId); + + void resetStepStatus(Long taskInstanceId, long stepInstanceId); + + /** + * 更新步骤启动时间 - 仅当启动时间为空 + * + * @param taskInstanceId 作业实例ID + * @param stepInstanceId 步骤实例ID + * @param startTime 启动时间 + */ + void updateStepStartTimeIfNull(Long taskInstanceId, long stepInstanceId, Long startTime); + + void updateStepEndTime(Long taskInstanceId, long stepInstanceId, Long endTime); + + /** + * 步骤重试次数+1 + * + * @param taskInstanceId 作业实例ID + * @param stepInstanceId 步骤实例ID + */ + void addStepInstanceExecuteCount(Long taskInstanceId, long stepInstanceId); + + /** + * 更新步骤的执行信息 + * + * @param taskInstanceId 作业实例ID + * @param stepInstanceId 步骤实例ID + * @param status 步骤执行状态 + * @param startTime 开始时间 + * @param endTime 结束时间 + * @param totalTime 总耗时 + */ + void updateStepExecutionInfo(Long taskInstanceId, + long stepInstanceId, + RunStatusEnum status, + Long startTime, + Long endTime, + Long totalTime); + + + /** + * 更新解析之后的脚本参数 + * + * @param taskInstanceId 作业实例ID + * @param stepInstanceId 步骤实例ID + * @param isSecureParam 是否为敏感参数 + * @param resolvedScriptParam 解析之后的脚本参数 + */ + void updateResolvedScriptParam(Long taskInstanceId, + long stepInstanceId, + boolean isSecureParam, + String resolvedScriptParam); + + /** + * 更新变量解析之后的源文件 + * + * @param taskInstanceId 作业实例ID + * @param stepInstanceId 步骤实例ID + * @param resolvedFileSources 解析后的源文件信息 + */ + void updateResolvedSourceFile(Long taskInstanceId, long stepInstanceId, List resolvedFileSources); + + /** + * 更新变量解析之后的目标路径 + * + * @param taskInstanceId 作业实例ID + * @param stepInstanceId 步骤实例ID + * @param resolvedTargetPath 解析之后的目标路径 + */ + void updateResolvedTargetPath(Long taskInstanceId, long stepInstanceId, String resolvedTargetPath); + + /** + * 更新确认理由 + * + * @param taskInstanceId 作业实例ID + * @param stepInstanceId 步骤实例ID + * @param confirmReason 确认理由 + */ + void updateConfirmReason(Long taskInstanceId, long stepInstanceId, String confirmReason); + + /** + * 更新步骤操作人 + * + * @param taskInstanceId 作业实例ID + * @param stepInstanceId 步骤实例ID + * @param operator 操作人 + */ + void updateStepOperator(Long taskInstanceId, long stepInstanceId, String operator); + + /** + * 获取上一步骤实例(可执行的，不包含人工确认这种) + * + * @param taskInstanceId 任务实例ID + * @param stepInstanceId 当前步骤实例ID + * @return 上一步骤实例 + */ + StepInstanceDTO getPreExecutableStepInstance(Long taskInstanceId, long stepInstanceId); + + /** + * 获取步骤实例 ID 和步骤顺序的映射关系 + * + * @param taskInstanceId 任务实例ID + * @return 步骤实例 ID 和步骤顺序的映射关系 + */ + Map listStepInstanceIdAndStepOrderMapping(Long taskInstanceId); + + /** + * 根据 taskInstanceId 获取快速任务步骤实例详情 + * + * @param taskInstanceId 任务实例ID + * @return 步骤详情 + */ + StepInstanceDTO getStepInstanceByTaskInstanceId(Long taskInstanceId); + + /** + * 根据 executeObjectCompositeKey 获取步骤实例中的执行对象 + * + * @param stepInstance 步骤实例 + * @param executeObjectCompositeKey 执行对象复合 KEY + */ + ExecuteObject findExecuteObjectByCompositeKey(StepInstanceBaseDTO stepInstance, + ExecuteObjectCompositeKey executeObjectCompositeKey); + + /** + * 根据 executeObjectCompositeKeys 批量获取步骤实例中的执行对象 + * + * @param stepInstance 步骤实例 + * @param executeObjectCompositeKeys 执行对象复合 KEYS + */ + List findExecuteObjectByCompositeKeys( + StepInstanceBaseDTO stepInstance, + Collection executeObjectCompositeKeys + ); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceVariableValueService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceVariableValueService.java index bafd1b467f..5ae300701b 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceVariableValueService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/StepInstanceVariableValueService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,6 +25,7 @@ package com.tencent.bk.job.execute.service; import com.tencent.bk.job.execute.engine.model.TaskVariableDTO; +import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.StepInstanceVariableValuesDTO; import java.util.List; @@ -51,11 +52,10 @@ public interface StepInstanceVariableValueService { /** * 获取步骤输入参数 * - * @param taskInstanceId 作业实例ID - * @param stepInstanceId 当前步骤ID - * @param taskVariables 全局变量 + * @param stepInstance 当前步骤实例 + * @param taskVariables 全局变量初始值 * @return 变量值 */ - StepInstanceVariableValuesDTO computeInputStepInstanceVariableValues(long taskInstanceId, long stepInstanceId, + StepInstanceVariableValuesDTO computeInputStepInstanceVariableValues(StepInstanceBaseDTO stepInstance, List taskVariables); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskExecuteService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskExecuteService.java index d7ae6e40a2..e4cd89b2e2 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskExecuteService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskExecuteService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,6 @@ package com.tencent.bk.job.execute.service; -import com.tencent.bk.job.common.exception.ServiceException; import com.tencent.bk.job.execute.constants.TaskOperationEnum; import com.tencent.bk.job.execute.engine.model.TaskVariableDTO; import com.tencent.bk.job.execute.model.FastTaskDTO; @@ -42,17 +41,17 @@ public interface TaskExecuteService { * 快速执行作业 * * @param fastTask 快速执行作业 - * @return 作业实例 ID + * @return 作业实例 */ - Long executeFastTask(FastTaskDTO fastTask); + TaskInstanceDTO executeFastTask(FastTaskDTO fastTask); /** * 重做快速作业实例 * * @param fastTask 快速作业 - * @return 作业实例 ID + * @return 作业实例 */ - Long redoFastTask(FastTaskDTO fastTask); + TaskInstanceDTO redoFastTask(FastTaskDTO fastTask); /** * 启动作业 @@ -70,17 +69,17 @@ public interface TaskExecuteService { TaskInstanceDTO executeJobPlan(TaskExecuteParam executeParam); /** - * 创建重做作业实例 + * 重做作业 * * @param appId 业务 ID * @param taskInstanceId 作业实例 ID * @param operator 操作者 * @param executeVariableValues 全局变量 - * @return */ - TaskInstanceDTO createTaskInstanceForRedo(Long appId, Long taskInstanceId, String operator, - List executeVariableValues) - throws ServiceException; + TaskInstanceDTO redoJob(Long appId, + Long taskInstanceId, + String operator, + List executeVariableValues); /** * 步骤操作 @@ -89,9 +88,8 @@ TaskInstanceDTO createTaskInstanceForRedo(Long appId, Long taskInstanceId, Strin * @param operator 操作者 * @param stepOperation 步骤操作 * @return 执行次数 - * @throws ServiceException */ - Integer doStepOperation(Long appId, String operator, StepOperationDTO stepOperation) throws ServiceException; + Integer doStepOperation(Long appId, String operator, StepOperationDTO stepOperation); /** * 终止作业 @@ -99,9 +97,8 @@ TaskInstanceDTO createTaskInstanceForRedo(Long appId, Long taskInstanceId, Strin * @param username 操作者 * @param appId 业务ID * @param taskInstanceId 作业实例ID - * @throws ServiceException 终止操作异常的时候抛出 */ - void terminateJob(String username, Long appId, Long taskInstanceId) throws ServiceException; + void terminateJob(String username, Long appId, Long taskInstanceId); /** * 作业操作 @@ -110,15 +107,16 @@ TaskInstanceDTO createTaskInstanceForRedo(Long appId, Long taskInstanceId, Strin * @param operator 操作者 * @param taskInstanceId 作业实例 ID * @param operation 操作类型 - * @throws ServiceException */ - void doTaskOperation(Long appId, String operator, long taskInstanceId, - TaskOperationEnum operation) throws ServiceException; + void doTaskOperation(Long appId, + String operator, + long taskInstanceId, + TaskOperationEnum operation); /** * 作业执行方案执行鉴权 * * @param executeParam 作业执行参数 */ - void authExecuteJobPlan(TaskExecuteParam executeParam) throws ServiceException; + void authExecuteJobPlan(TaskExecuteParam executeParam); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskInstanceAccessProcessor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskInstanceAccessProcessor.java new file mode 100644 index 0000000000..6cd8f4b0f3 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskInstanceAccessProcessor.java @@ -0,0 +1,50 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.service; + +import com.tencent.bk.job.common.exception.NotFoundException; +import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +/** + * 查看作业实例前置处理 + */ +@Component +public class TaskInstanceAccessProcessor { + private final TaskInstanceService taskInstanceService; + + @Autowired + public TaskInstanceAccessProcessor(TaskInstanceService taskInstanceService) { + this.taskInstanceService = taskInstanceService; + } + + public void processBeforeAccess(String username, long appId, long taskInstanceId) + throws NotFoundException, PermissionDeniedException { + // 触发获取作业实例操作，会进行资源检查、鉴权、审计等操作 + taskInstanceService.getTaskInstance(username, appId, taskInstanceId); + } + +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskInstanceService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskInstanceService.java index 7acb09ae69..96e20cecfd 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskInstanceService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskInstanceService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,16 +24,11 @@ package com.tencent.bk.job.execute.service; +import com.tencent.bk.job.common.exception.NotFoundException; +import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; -import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; -import com.tencent.bk.job.execute.common.constants.TaskStartupModeEnum; -import com.tencent.bk.job.execute.common.constants.TaskTypeEnum; -import com.tencent.bk.job.execute.model.FileSourceDTO; -import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; -import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; import java.util.Collection; import java.util.List; @@ -45,67 +40,38 @@ public interface TaskInstanceService { long addTaskInstance(TaskInstanceDTO taskInstance); - TaskInstanceDTO getTaskInstance(long taskInstanceId); + TaskInstanceDTO getTaskInstance(long appId, long taskInstanceId) throws NotFoundException; - /** - * 保存步骤实例 - * - * @param stepInstance 步骤实例 - * @return 步骤实例ID - */ - long addStepInstance(StepInstanceDTO stepInstance); + TaskInstanceDTO getTaskInstance(long taskInstanceId) throws NotFoundException; + + TaskInstanceDTO getTaskInstance(String username, long appId, long taskInstanceId) + throws NotFoundException, PermissionDeniedException; /** * 获取作业实例详情-包含步骤信息和全局变量信息 * * @param taskInstanceId 作业实例 ID - * @return + * @return 作业实例 */ TaskInstanceDTO getTaskInstanceDetail(long taskInstanceId); - List listStepInstanceByTaskInstanceId(long taskInstanceId); - /** - * 获取步骤基本信息 - * - * @param stepInstanceId 步骤实例ID - * @return 步骤基本信息 - */ - StepInstanceBaseDTO getBaseStepInstance(long stepInstanceId); - - /** - * 获取步骤基本信息 - * - * @param stepInstanceId 步骤实例ID - * @return 步骤基本信息 - */ - StepInstanceDTO getStepInstanceDetail(long stepInstanceId); - - /** - * 获取作业的第一个步骤实例 + * 获取作业实例详情-包含步骤信息和全局变量信息 * - * @param taskInstanceId 作业实例ID - * @return 作业第一个步骤实例 + * @param username 用户名 + * @param appId 业务 ID + * @param taskInstanceId 作业实例 ID + * @return 作业实例 */ - StepInstanceBaseDTO getFirstStepInstance(long taskInstanceId); + TaskInstanceDTO getTaskInstanceDetail(String username, long appId, long taskInstanceId) + throws NotFoundException, PermissionDeniedException; void updateTaskStatus(long taskInstanceId, int status); - List getTaskStepIdList(long taskInstanceId); - void updateTaskCurrentStepId(long taskInstanceId, Long stepInstanceId); void resetTaskStatus(long taskInstanceId); - void updateStepStatus(long stepInstanceId, int status); - - /** - * 重试步骤操作-重置步骤执行状态 - * - * @param stepInstanceId 步骤实例ID - */ - void resetStepExecuteInfoForRetry(long stepInstanceId); - /** * 作业恢复执行-重置作业执行状态 * @@ -113,29 +79,6 @@ public interface TaskInstanceService { */ void resetTaskExecuteInfoForRetry(long taskInstanceId); - void resetStepStatus(long stepInstanceId); - - void updateStepStartTime(long stepInstanceId, Long startTime); - - /** - * 更新步骤启动时间 - 仅当启动时间为空 - * - * @param stepInstanceId 步骤实例ID - * @param startTime 启动时间 - */ - void updateStepStartTimeIfNull(long stepInstanceId, Long startTime); - - void updateStepEndTime(long stepInstanceId, Long endTime); - - /** - * 步骤重试次数+1 - * - * @param stepInstanceId 步骤实例ID - */ - void addStepInstanceExecuteCount(long stepInstanceId); - - void updateStepTotalTime(long stepInstanceId, long totalTime); - /** * 更新作业的执行信息 * @@ -146,87 +89,12 @@ public interface TaskInstanceService { * @param endTime 结束时间 * @param totalTime 总耗时 */ - void updateTaskExecutionInfo(long taskInstanceId, RunStatusEnum status, Long currentStepId, - Long startTime, Long endTime, Long totalTime); - - /** - * 更新步骤的执行信息 - * - * @param stepInstanceId 步骤实例ID - * @param status 步骤执行状态 - * @param startTime 开始时间 - * @param endTime 结束时间 - * @param totalTime 总耗时 - */ - void updateStepExecutionInfo(long stepInstanceId, RunStatusEnum status, - Long startTime, Long endTime, Long totalTime); - - - /** - * 更新解析之后的脚本参数 - * - * @param stepInstanceId 步骤实例ID - * @param resolvedScriptParam 解析之后的脚本参数 - */ - void updateResolvedScriptParam(long stepInstanceId, String resolvedScriptParam); - - /** - * 更新变量解析之后的源文件 - * - * @param stepInstanceId 步骤实例ID - * @param resolvedFileSources - */ - void updateResolvedSourceFile(long stepInstanceId, List resolvedFileSources); - - /** - * 更新变量解析之后的目标路径 - * - * @param stepInstanceId 步骤实例ID - * @param resolvedTargetPath 解析之后的目标路径 - */ - void updateResolvedTargetPath(long stepInstanceId, String resolvedTargetPath); - - /** - * 更新确认理由 - * - * @param stepInstanceId 步骤实例ID - * @param confirmReason 确认理由 - */ - void updateConfirmReason(long stepInstanceId, String confirmReason); - - /** - * 更新步骤操作人 - * - * @param stepInstanceId 步骤实例ID - * @param operator 操作人 - */ - void updateStepOperator(long stepInstanceId, String operator); - - /** - * 获取上一步骤实例(可执行的，不包含人工确认这种) - * - * @param taskInstanceId 任务实例ID - * @param stepInstanceId 当前步骤实例ID - * @return 上一步骤实例 - */ - StepInstanceDTO getPreExecutableStepInstance(long taskInstanceId, long stepInstanceId); - - /** - * 根据 taskInstanceId 获取快速任务步骤实例详情 - * - * @param taskInstanceId 任务实例ID - * @return 步骤详情 - */ - StepInstanceDTO getStepInstanceByTaskInstanceId(long taskInstanceId); - - Integer countTaskInstances(Long appId, Long minTotalTime, Long maxTotalTime, TaskStartupModeEnum taskStartupMode, - TaskTypeEnum taskType, List runStatusList, Long fromTime, Long toTime); - - Integer countStepInstances(Long appId, List stepIdList, StepExecuteTypeEnum stepExecuteType, - ScriptTypeEnum scriptType, RunStatusEnum runStatus, Long fromTime, Long toTime); - - Integer countFastPushFile(Long appId, Integer transferMode, Boolean localUpload, RunStatusEnum runStatus, - Long fromTime, Long toTime); + void updateTaskExecutionInfo(long taskInstanceId, + RunStatusEnum status, + Long currentStepId, + Long startTime, + Long endTime, + Long totalTime); List getJoinedAppIdList(); @@ -237,8 +105,9 @@ Integer countFastPushFile(Long appId, Integer transferMode, Boolean localUpload, /** * 保存作业实例与主机的关系，便于根据ip/ipv6检索作业实例 * + * @param appId 业务 ID * @param taskInstanceId 作业实例ID * @param hosts 主机列表 */ - void saveTaskInstanceHosts(long taskInstanceId, Collection hosts); + void saveTaskInstanceHosts(long appId, long taskInstanceId, Collection hosts); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskInstanceVariableService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskInstanceVariableService.java index 50afc1a32b..5e38fa6230 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskInstanceVariableService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskInstanceVariableService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskOperationLogService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskOperationLogService.java index fae4360820..abeeeae1cd 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskOperationLogService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskOperationLogService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskPlanService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskPlanService.java index 6e68c5868d..081965a12a 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskPlanService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskPlanService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskResultService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskResultService.java index 92ba34f03b..5a9ecdb1d7 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskResultService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TaskResultService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,10 +24,9 @@ package com.tencent.bk.job.execute.service; -import com.tencent.bk.job.common.exception.ServiceException; import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; -import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; import com.tencent.bk.job.execute.model.StepExecutionDetailDTO; import com.tencent.bk.job.execute.model.StepExecutionRecordDTO; import com.tencent.bk.job.execute.model.StepExecutionResultQuery; @@ -53,6 +52,7 @@ public interface TaskResultService { PageData listPageTaskInstance(TaskInstanceQuery taskQuery, BaseSearchCondition baseSearchCondition); + /** * 获取作业执行结果 * @@ -63,17 +63,6 @@ PageData listPageTaskInstance(TaskInstanceQuery taskQuery, */ TaskExecuteResultDTO getTaskExecutionResult(String username, Long appId, Long taskInstanceId); - /** - * 获取步骤执行详情 - * - * @param username 用户名 - * @param appId 业务ID - * @param taskInstanceId 作业实例ID - * @param query 查询条件 - * @return 步骤执行详情 - */ - StepExecutionDetailDTO getFastTaskStepExecutionResult(String username, Long appId, Long taskInstanceId, - StepExecutionResultQuery query); /** * 获取步骤执行详情 @@ -82,10 +71,8 @@ StepExecutionDetailDTO getFastTaskStepExecutionResult(String username, Long appI * @param appId 业务ID * @param query 查询条件 * @return 执行详情 - * @throws ServiceException */ - StepExecutionDetailDTO getStepExecutionResult(String username, Long appId, StepExecutionResultQuery query) - throws ServiceException; + StepExecutionDetailDTO getStepExecutionResult(String username, Long appId, StepExecutionResultQuery query); /** * 获取定时任务执行结果统计 @@ -98,10 +85,11 @@ Map getCronTaskExecuteResultStatis List cronTaskIdList); /** - * 根据执行结果分组获取主机信息 + * 根据执行结果分组获取执行对象信息 * * @param username 用户名 * @param appId 业务ID + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param executeCount 执行次数 * @param batch 滚动执行批次;如果传入null或0，忽略该参数 @@ -110,26 +98,29 @@ Map getCronTaskExecuteResultStatis * @param keyword 脚本日志关键字 * @return 主机列表 */ - List getHostsByResultType(String username, - Long appId, - Long stepInstanceId, - Integer batch, - Integer executeCount, - Integer resultType, - String tag, - String keyword); + List getExecuteObjectsByResultType(String username, + Long appId, + Long taskInstanceId, + Long stepInstanceId, + Integer batch, + Integer executeCount, + Integer resultType, + String tag, + String keyword); /** * 获取步骤执行历史 * * @param username 用户名 * @param appId 业务ID + * @param taskInstanceId 作业实例 ID * @param stepInstanceId 步骤实例ID * @param batch 滚动执行批次;如果传入，则会按照batch过滤步骤执行历史 * @return 执行历史 */ List listStepExecutionHistory(String username, Long appId, + Long taskInstanceId, Long stepInstanceId, Integer batch); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ThirdFileDistributeSourceHostProvisioner.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ThirdFileDistributeSourceHostProvisioner.java new file mode 100644 index 0000000000..12c5def39f --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/ThirdFileDistributeSourceHostProvisioner.java @@ -0,0 +1,40 @@ +/* + * + * * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * * + * * Copyright (C) 2021 Tencent. All rights reserved. + * * + * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * * + * * License for BK-JOB蓝鲸智云作业平台: + * * -------------------------------------------------------------------- + * * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * * + * * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * * the Software. + * * + * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * * IN THE SOFTWARE. + * + */ + +package com.tencent.bk.job.execute.service; + +import com.tencent.bk.job.common.model.dto.HostDTO; + +/** + * 分发文件源文件时，根据配置使用job机器或者集群外机器作为源进行分发 + */ +public interface ThirdFileDistributeSourceHostProvisioner { + + /** + * @return job-file-worker信息 或者 集群外的机器信息 + */ + HostDTO getThirdFileDistributeSourceHost(Long cloudId, String protocol, String ip); +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TopoService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TopoService.java index 70b727a883..aa9e9e1151 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TopoService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/TopoService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -36,9 +36,9 @@ public interface TopoService { /** * 批量获取topo节点的层级 * - * @param appId 业务ID + * @param bizId CMDB业务ID * @param topoNodes cmdb topo 节点列表 * @return topo节点层级 */ - List batchGetTopoNodeHierarchy(long appId, List topoNodes); + List batchGetTopoNodeHierarchy(long bizId, List topoNodes); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/AbstractAgentTaskServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/AbstractAgentTaskServiceImpl.java deleted file mode 100644 index f94265ce6a..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/AbstractAgentTaskServiceImpl.java +++ /dev/null @@ -1,80 +0,0 @@ -package com.tencent.bk.job.execute.service.impl; - -import com.tencent.bk.job.common.model.dto.HostDTO; -import com.tencent.bk.job.execute.model.AgentTaskDTO; -import com.tencent.bk.job.execute.model.AgentTaskDetailDTO; -import com.tencent.bk.job.execute.model.AgentTaskResultGroupDTO; -import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; -import com.tencent.bk.job.execute.service.AgentTaskService; -import com.tencent.bk.job.execute.service.HostService; -import com.tencent.bk.job.execute.service.StepInstanceService; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -/** - * Agent 任务Service的公共实现 - */ -public abstract class AbstractAgentTaskServiceImpl implements AgentTaskService { - private final StepInstanceService stepInstanceService; - - private final HostService hostService; - - public AbstractAgentTaskServiceImpl(StepInstanceService stepInstanceService, - HostService hostService) { - this.stepInstanceService = stepInstanceService; - this.hostService = hostService; - } - - protected final List fillHostDetail(StepInstanceBaseDTO stepInstance, - List agentTasks) { - if (CollectionUtils.isEmpty(agentTasks)) { - return Collections.emptyList(); - } - - List agentTaskDetailList; - // 历史版本AgentTask会包含ip信息,从当前版本开始AgentTask不会包含ip信息，需要从StepInstance反查 - boolean hasIpInfo = StringUtils.isNotEmpty(agentTasks.get(0).getCloudIp()); - if (!hasIpInfo) { - Map hosts = stepInstanceService.computeStepHosts(stepInstance, HostDTO::getHostId); - agentTaskDetailList = agentTasks.stream() - .map(agentTask -> { - AgentTaskDetailDTO agentTaskDetail = new AgentTaskDetailDTO(agentTask); - HostDTO host = hosts.get(agentTask.getHostId()); - agentTaskDetail.setCloudIp(host.toCloudIp()); - agentTaskDetail.setBkCloudId(host.getBkCloudId()); - agentTaskDetail.setIp(host.getIp()); - agentTaskDetail.setBkCloudName(hostService.getCloudAreaName(host.getBkCloudId())); - return agentTaskDetail; - }).collect(Collectors.toList()); - } else { - agentTaskDetailList = agentTasks.stream() - .map(agentTask -> { - AgentTaskDetailDTO agentTaskDetail = new AgentTaskDetailDTO(agentTask); - agentTaskDetail.setBkCloudName(hostService.getCloudAreaName(agentTaskDetail.getBkCloudId())); - return agentTaskDetail; - }).collect(Collectors.toList()); - } - return agentTaskDetailList; - } - - protected final List groupAgentTasks(List agentTasks) { - List resultGroups = new ArrayList<>(); - agentTasks.stream() - .collect(Collectors.groupingBy( - agentTask -> new AgentTaskResultGroupDTO(agentTask.getStatus().getValue(), agentTask.getTag()))) - .forEach((resultGroup, groupedAgentTasks) -> { - resultGroup.setTotalAgentTasks(groupedAgentTasks.size()); - resultGroup.setAgentTasks(groupedAgentTasks); - resultGroups.add(resultGroup); - }); - return resultGroups; - } - - -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/AbstractExecuteObjectTaskServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/AbstractExecuteObjectTaskServiceImpl.java new file mode 100644 index 0000000000..e5276aa265 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/AbstractExecuteObjectTaskServiceImpl.java @@ -0,0 +1,97 @@ +package com.tencent.bk.job.execute.service.impl; + +import com.tencent.bk.job.execute.engine.model.ExecuteObject; +import com.tencent.bk.job.execute.model.ExecuteObjectTask; +import com.tencent.bk.job.execute.model.ResultGroupDTO; +import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; +import com.tencent.bk.job.execute.service.ExecuteObjectTaskService; +import com.tencent.bk.job.execute.service.StepInstanceService; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * 执行对象任务公共实现 Service + */ +@Slf4j +public abstract class AbstractExecuteObjectTaskServiceImpl implements ExecuteObjectTaskService { + protected final StepInstanceService stepInstanceService; + + public AbstractExecuteObjectTaskServiceImpl(StepInstanceService stepInstanceService) { + this.stepInstanceService = stepInstanceService; + } + + protected final void fillExecuteObjectForExecuteObjectTasks( + StepInstanceBaseDTO stepInstance, + List tasks + ) { + + if (CollectionUtils.isEmpty(tasks)) { + return; + } + + if (stepInstance.isSupportExecuteObjectFeature()) { + Map executeObjectMap = + stepInstanceService.computeStepExecuteObjects(stepInstance, ExecuteObject::getId); + tasks.forEach(task -> { + ExecuteObject executeObject = executeObjectMap.get(task.getExecuteObjectId()); + if (executeObject == null) { + log.error("Can not find execute object for execute object task! stepInstanceId: {}, " + + "executeObjectId: {}", stepInstance.getId(), task.getExecuteObjectId()); + } + task.setExecuteObject(executeObject); + }); + } else { + // 兼容老版本不支持执行对象的数据 + Map hostExecuteObjects = stepInstanceService.computeStepExecuteObjects( + stepInstance, executeObject -> executeObject.getHost().getHostId()); + tasks.forEach(task -> { + ExecuteObject hostExecuteObject = hostExecuteObjects.get(task.getHostId()); + if (hostExecuteObject == null) { + log.error("Can not find host for execute object task! stepInstanceId: {}, " + + "hostId: {}", stepInstance.getId(), task.getHostId()); + } + task.setExecuteObject(hostExecuteObject); + }); + } + } + + protected final void fillExecuteObjectForExecuteObjectTask(StepInstanceBaseDTO stepInstance, + ExecuteObjectTask task) { + if (task != null) { + fillExecuteObjectForExecuteObjectTasks(stepInstance, Collections.singletonList(task)); + } + } + + protected final List groupTasks(List tasks) { + List resultGroups = new ArrayList<>(); + tasks.stream() + .collect(Collectors.groupingBy( + task -> new ResultGroupDTO(task.getStatus().getValue(), task.getTag()))) + .forEach((resultGroup, groupedExecuteObjectTasks) -> { + resultGroup.setTotal(groupedExecuteObjectTasks.size()); + resultGroup.setExecuteObjectTasks(groupedExecuteObjectTasks); + resultGroups.add(resultGroup); + }); + return resultGroups; + } + + /** + * 判断任务是否需要按照<执行对象>的方式保存到 DB + * + * @param tasks 需要被保存的任务 + */ + protected boolean isExecuteObjectSupported(Collection tasks) { + // 根据执行对象任务模型中的 executeObjectId 参数判断是否支持执行对象 + ExecuteObjectTask anyTask = tasks.stream().findAny().orElse(null); + return Objects.requireNonNull(anyTask).getExecuteObjectId() != null; + } + +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/AccountServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/AccountServiceImpl.java index 52896e1b0e..77b53723f7 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/AccountServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/AccountServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,15 +27,15 @@ import com.google.common.cache.CacheBuilder; import com.google.common.cache.CacheLoader; import com.google.common.cache.LoadingCache; +import com.tencent.bk.job.common.constant.AccountCategoryEnum; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.exception.InternalException; import com.tencent.bk.job.common.exception.ServiceException; import com.tencent.bk.job.common.model.InternalResponse; -import com.tencent.bk.job.execute.client.AccountResourceClient; import com.tencent.bk.job.execute.model.AccountDTO; import com.tencent.bk.job.execute.service.AccountService; -import com.tencent.bk.job.manage.common.consts.account.AccountCategoryEnum; -import com.tencent.bk.job.manage.common.consts.account.AccountTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.account.AccountTypeEnum; +import com.tencent.bk.job.manage.api.inner.ServiceAccountResource; import com.tencent.bk.job.manage.model.inner.ServiceAccountDTO; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -45,10 +45,10 @@ import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; -@Service +@Service("jobExecuteAccountService") @Slf4j public class AccountServiceImpl implements AccountService { - private final AccountResourceClient accountResourceClient; + private final ServiceAccountResource accountResource; private final LoadingCache accountCache = CacheBuilder.newBuilder() .maximumSize(10000).expireAfterWrite(1, TimeUnit.MINUTES). @@ -74,13 +74,13 @@ public AccountDTO load(String accountKey) { ); @Autowired - public AccountServiceImpl(AccountResourceClient accountResourceClient) { - this.accountResourceClient = accountResourceClient; + public AccountServiceImpl(ServiceAccountResource accountResource) { + this.accountResource = accountResource; } @Override public AccountDTO getAccountById(Long accountId) throws ServiceException { - InternalResponse resp = accountResourceClient.getAccountByAccountId(accountId); + InternalResponse resp = accountResource.getAccountByAccountId(accountId); if (!resp.isSuccess()) { log.warn("Get account by accountId:{} return fail resp", accountId); throw new InternalException(resp.getCode()); @@ -94,7 +94,7 @@ public AccountDTO getAccountById(Long accountId) throws ServiceException { @Override public AccountDTO getAccountByAccountName(Long appId, String accountName) throws ServiceException { - InternalResponse resp = accountResourceClient.getAccountByAccountName(appId, + InternalResponse resp = accountResource.getAccountByAccountName(appId, accountName); if (!resp.isSuccess()) { log.warn("Get accountName by appId:{}, accountName:{} return fail resp", appId, accountName); @@ -137,7 +137,7 @@ public AccountDTO getSystemAccountByAlias(String alias, Long appId) throws Servi @Override public AccountDTO getAccountByAlias(AccountCategoryEnum accountCategory, Long appId, String alias) throws ServiceException { - InternalResponse resp = accountResourceClient.getAccountByCategoryAndAliasInApp(appId, + InternalResponse resp = accountResource.getAccountByCategoryAndAliasInApp(appId, accountCategory.getValue(), alias); if (!resp.isSuccess()) { log.warn("Get account by category: {}, alias:{}, appId:{} return fail resp", accountCategory, alias, appId); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/AgentServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/AgentServiceImpl.java index 2edd57c81c..c70674cecf 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/AgentServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/AgentServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,66 +24,80 @@ package com.tencent.bk.job.execute.service.impl; -import com.tencent.bk.job.common.gse.service.QueryAgentStatusClient; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.tencent.bk.job.common.gse.service.AgentStateClient; +import com.tencent.bk.job.common.gse.service.model.HostAgentStateQuery; import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.common.util.ip.IpUtils; +import com.tencent.bk.job.execute.config.GseConfig; import com.tencent.bk.job.execute.engine.consts.Consts; -import com.tencent.bk.job.execute.model.ServersDTO; import com.tencent.bk.job.execute.service.AgentService; import com.tencent.bk.job.execute.service.HostService; import com.tencent.bk.job.manage.model.inner.ServiceHostDTO; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; import java.net.Inet4Address; import java.net.InetAddress; import java.net.NetworkInterface; import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; import java.util.Enumeration; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.StringJoiner; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; @Service @Slf4j public class AgentServiceImpl implements AgentService { - private final QueryAgentStatusClient queryAgentStatusClient; + private final AgentStateClient agentStateClient; private final HostService hostService; - private volatile HostDTO agentHost; + private final LoadingCache agentHostCache = CacheBuilder.newBuilder() + .maximumSize(1).expireAfterWrite(60, TimeUnit.SECONDS). + build(new CacheLoader() { + @SuppressWarnings("all") + @Override + public HostDTO load(String key) { + HostDTO agentHost = getAgentBindHost(); + log.info("Load agent host and save to cache:{}", agentHost); + return agentHost; + } + } + ); @Autowired - public AgentServiceImpl(QueryAgentStatusClient queryAgentStatusClient, + public AgentServiceImpl(@Qualifier(GseConfig.EXECUTE_BEAN_AGENT_STATE_CLIENT) + AgentStateClient agentStateClient, HostService hostService) { - this.queryAgentStatusClient = queryAgentStatusClient; + this.agentStateClient = agentStateClient; this.hostService = hostService; } @Override public HostDTO getLocalAgentHost() { - if (agentHost != null) { - return agentHost; + try { + String CACHE_KEY_AGENT_HOST = "agentHost"; + HostDTO host = agentHostCache.get(CACHE_KEY_AGENT_HOST); + return host == null ? null : host.clone(); + } catch (ExecutionException e) { + log.warn("Fail to load agentHost from cache, try to load directly", e); + return getAgentBindHost(); } - return getAgentBindHost(); - } - - @Override - public ServersDTO getLocalServersDTO() { - List hostDTOList = new ArrayList<>(); - hostDTOList.add(getLocalAgentHost()); - ServersDTO servers = new ServersDTO(); - servers.setStaticIpList(hostDTOList); - servers.setIpList(hostDTOList); - return servers; } private HostDTO getAgentBindHost() { log.info("Get local agent bind host!"); synchronized (this) { - if (agentHost != null) { - return agentHost; - } String physicalMachineMultiIp; String nodeIP = System.getenv("BK_JOB_NODE_IP"); if (StringUtils.isNotBlank(nodeIP)) { @@ -97,15 +111,120 @@ private HostDTO getAgentBindHost() { } physicalMachineMultiIp = sj.toString(); } - String agentBindIp = queryAgentStatusClient.getHostIpByAgentStatus(physicalMachineMultiIp, - Consts.DEFAULT_CLOUD_ID); - log.info("Local agent bind ip is {}", agentBindIp); - String agentBindCloudIp = Consts.DEFAULT_CLOUD_ID + ":" + agentBindIp; - ServiceHostDTO host = hostService.getHost(HostDTO.fromCloudIp(agentBindCloudIp)); - agentHost = HostDTO.fromHostIdAndCloudIp(host.getHostId(), agentBindCloudIp); - agentHost.setAgentId(agentHost.toCloudIp()); - return agentHost; + ServiceHostDTO host; + if (physicalMachineMultiIp.contains(":")) { + // IPv6地址 + // 首先转为完整无压缩格式 + physicalMachineMultiIp = IpUtils.getFullIpv6ByCompressedOne(physicalMachineMultiIp); + host = getServiceHostByMultiIpv6(physicalMachineMultiIp); + } else { + // IPv4地址 + host = getServiceHostByMultiIpv4(physicalMachineMultiIp); + } + if (host == null) { + log.error("Invalid host for ip: {}", physicalMachineMultiIp); + return null; + } + return ServiceHostDTO.toHostDTO(host); + } + } + + private ServiceHostDTO getServiceHostByMultiIpv4(String multiIpv4) { + List hostIps = buildHostIps((long) Consts.DEFAULT_CLOUD_ID, multiIpv4); + if (CollectionUtils.isEmpty(hostIps)) { + log.warn("Cannot find host by multiIpv4:{}", multiIpv4); + return null; + } + Map map = hostService.batchGetHosts(hostIps); + ServiceHostDTO aliveHost = findOneAliveHost(map.values()); + if (aliveHost == null) { + log.warn("Cannot find alive hosts, use first ip of {}", multiIpv4); + return map.get(hostIps.get(0)); + } + return aliveHost; + } + + @SuppressWarnings("SameParameterValue") + private List buildHostIps(Long cloudAreaId, String multiIpv4) { + if (StringUtils.isBlank(multiIpv4)) { + return Collections.emptyList(); + } + List hostIps = new ArrayList<>(); + String[] ipArr = multiIpv4.trim().split("[,;]"); + for (String ip : ipArr) { + hostIps.add(new HostDTO(cloudAreaId, ip)); + } + return hostIps; + } + + private ServiceHostDTO getServiceHostByMultiIpv6(String multiIpv6) { + if (StringUtils.isBlank(multiIpv6)) { + return null; + } + String[] ipv6Arr = multiIpv6.trim().split("[,;]"); + List hosts = new ArrayList<>(); + for (String ipv6 : ipv6Arr) { + hosts.add(hostService.getHostByCloudIpv6(Consts.DEFAULT_CLOUD_ID, ipv6)); + } + if (CollectionUtils.isEmpty(hosts)) { + log.warn("Cannot find host by multiIpv6:{}", multiIpv6); + return null; + } + ServiceHostDTO aliveHost = findOneAliveHost(hosts); + if (aliveHost == null) { + log.warn("Cannot find alive hosts, use first ip of {}", multiIpv6); + return hosts.get(0); + } + return aliveHost; + } + + /** + * 从多个主机中找出Agent存活的一个 + * + * @param serviceHosts 主机列表 + * @return Agent存活的第一个主机或者Null + */ + private ServiceHostDTO findOneAliveHost(Collection serviceHosts) { + if (CollectionUtils.isEmpty(serviceHosts)) { + return null; + } + List hostAgentStateQueryList = new ArrayList<>(serviceHosts.size()); + Map hostAgentStateQueryMap = new HashMap<>(serviceHosts.size()); + for (ServiceHostDTO serviceHost : serviceHosts) { + if (serviceHost == null) { + continue; + } + HostAgentStateQuery query = new HostAgentStateQuery(); + query.setHostId(serviceHost.getHostId()); + query.setBizId(serviceHost.getBizId()); + query.setCloudIp(serviceHost.getCloudIp()); + query.setAgentId(serviceHost.getAgentId()); + hostAgentStateQueryList.add(query); + hostAgentStateQueryMap.put(serviceHost.getHostIdOrCloudIp(), query); + } + Map agentStatusMap = agentStateClient.batchGetAgentAliveStatus(hostAgentStateQueryList); + + Map effectiveAgentIdToHostMap = new HashMap<>(); + serviceHosts.forEach(serviceHost -> { + HostAgentStateQuery hostAgentStateQuery = hostAgentStateQueryMap.get(serviceHost.getHostIdOrCloudIp()); + String effectiveAgentId = agentStateClient.getEffectiveAgentId(hostAgentStateQuery); + effectiveAgentIdToHostMap.put(effectiveAgentId, serviceHost); + } + ); + List aliveHosts = new ArrayList<>(); + agentStatusMap.forEach((effectiveAgentId, status) -> { + if (status != null && status) { + aliveHosts.add(effectiveAgentIdToHostMap.get(effectiveAgentId)); + } + }); + if (CollectionUtils.isEmpty(aliveHosts)) { + return null; + } else if (aliveHosts.size() > 1) { + ServiceHostDTO choosedHost = aliveHosts.get(0); + log.warn("{} aliveHosts found, use {}", aliveHosts.size(), choosedHost); + return choosedHost; } + return aliveHosts.get(0); } private Map getMachineIP() { diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ApplicationServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ApplicationServiceImpl.java index 101800e34e..51ccbdf286 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ApplicationServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ApplicationServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,9 +24,9 @@ package com.tencent.bk.job.execute.service.impl; -import com.tencent.bk.job.execute.client.ApplicationResourceClient; -import com.tencent.bk.job.execute.client.SyncResourceClient; import com.tencent.bk.job.execute.service.ApplicationService; +import com.tencent.bk.job.manage.api.inner.ServiceApplicationResource; +import com.tencent.bk.job.manage.api.inner.ServiceSyncResource; import com.tencent.bk.job.manage.model.inner.resp.ServiceApplicationDTO; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; @@ -36,32 +36,32 @@ import java.util.List; import java.util.stream.Collectors; -@Service +@Service("jobExecuteApplicationService") @Slf4j public class ApplicationServiceImpl implements ApplicationService { - private final ApplicationResourceClient applicationResourceClient; - private final SyncResourceClient syncResourceClient; + private final ServiceApplicationResource applicationResource; + private final ServiceSyncResource syncResource; @Autowired - public ApplicationServiceImpl(ApplicationResourceClient applicationResourceClient, - SyncResourceClient syncResourceClient) { - this.applicationResourceClient = applicationResourceClient; - this.syncResourceClient = syncResourceClient; + public ApplicationServiceImpl(ServiceApplicationResource applicationResource, + ServiceSyncResource syncResource) { + this.applicationResource = applicationResource; + this.syncResource = syncResource; } @Override public ServiceApplicationDTO getAppById(long appId) { - return applicationResourceClient.queryAppById(appId); + return applicationResource.queryAppById(appId); } @Override public ServiceApplicationDTO getAppByScope(String scopeType, String scopeId) { - return applicationResourceClient.queryAppByScope(scopeType, scopeId); + return applicationResource.queryAppByScope(scopeType, scopeId); } @Override public List listAllAppIds() { - List apps = syncResourceClient.listAllApps(); + List apps = syncResource.listAllApps(); if (apps == null) { return Collections.emptyList(); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/BizHostInfoQueryServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/BizHostInfoQueryServiceImpl.java new file mode 100644 index 0000000000..4032f85873 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/BizHostInfoQueryServiceImpl.java @@ -0,0 +1,51 @@ +package com.tencent.bk.job.execute.service.impl; + +import com.tencent.bk.job.common.gse.service.BizHostInfoQueryService; +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.common.util.CollectionUtil; +import com.tencent.bk.job.manage.api.inner.ServiceHostResource; +import com.tencent.bk.job.manage.model.inner.ServiceHostDTO; +import com.tencent.bk.job.manage.model.inner.request.ServiceBatchGetHostsReq; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Slf4j +@Service("jobExecuteBizHostInfoQueryService") +public class BizHostInfoQueryServiceImpl implements BizHostInfoQueryService { + + private final ServiceHostResource hostResource; + + @Autowired + public BizHostInfoQueryServiceImpl(ServiceHostResource hostResource) { + this.hostResource = hostResource; + } + + @Override + public Map queryBizIdsByHostId(Collection hostIds) { + List hostList = queryHosts(hostIds); + return CollectionUtil.convertToMap(hostList, ServiceHostDTO::getHostId, ServiceHostDTO::getBizId); + } + + @Override + public Map queryAgentIdsByHostId(Collection hostIds) { + List hostList = queryHosts(hostIds); + return CollectionUtil.convertToMap(hostList, ServiceHostDTO::getHostId, ServiceHostDTO::getAgentId); + } + + @Override + public Map queryCloudIpsByHostId(Collection hostIds) { + List hostList = queryHosts(hostIds); + return CollectionUtil.convertToMap(hostList, ServiceHostDTO::getHostId, ServiceHostDTO::getCloudIp); + } + + private List queryHosts(Collection hostIds) { + List hostDTOList = hostIds.stream().map(HostDTO::fromHostId).collect(Collectors.toList()); + return hostResource.batchGetHosts(new ServiceBatchGetHostsReq(hostDTOList)).getData(); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ContainerServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ContainerServiceImpl.java new file mode 100644 index 0000000000..64f1220a64 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ContainerServiceImpl.java @@ -0,0 +1,315 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.service.impl; + +import com.tencent.bk.job.common.cc.constants.KubeTopoNodeTypeEnum; +import com.tencent.bk.job.common.cc.model.container.ContainerDTO; +import com.tencent.bk.job.common.cc.model.container.ContainerDetailDTO; +import com.tencent.bk.job.common.cc.model.container.KubeClusterDTO; +import com.tencent.bk.job.common.cc.model.container.KubeNamespaceDTO; +import com.tencent.bk.job.common.cc.model.container.KubeNodeID; +import com.tencent.bk.job.common.cc.model.container.KubeWorkloadDTO; +import com.tencent.bk.job.common.cc.model.container.PodDTO; +import com.tencent.bk.job.common.cc.model.filter.BaseRuleDTO; +import com.tencent.bk.job.common.cc.model.filter.ComposeRuleDTO; +import com.tencent.bk.job.common.cc.model.filter.PropertyFilterDTO; +import com.tencent.bk.job.common.cc.model.filter.RuleConditionEnum; +import com.tencent.bk.job.common.cc.model.query.KubeClusterQuery; +import com.tencent.bk.job.common.cc.model.query.NamespaceQuery; +import com.tencent.bk.job.common.cc.model.query.WorkloadQuery; +import com.tencent.bk.job.common.cc.model.req.ListKubeContainerByTopoReq; +import com.tencent.bk.job.common.cc.sdk.BizCmdbClient; +import com.tencent.bk.job.common.constant.LabelSelectorOperatorEnum; +import com.tencent.bk.job.common.model.dto.Container; +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.common.model.dto.ResourceScope; +import com.tencent.bk.job.common.service.AppScopeMappingService; +import com.tencent.bk.job.common.util.RandomUtil; +import com.tencent.bk.job.execute.model.KubeClusterFilter; +import com.tencent.bk.job.execute.model.KubeContainerFilter; +import com.tencent.bk.job.execute.model.KubeContainerPropFilter; +import com.tencent.bk.job.execute.model.KubeNamespaceFilter; +import com.tencent.bk.job.execute.model.KubePodFilter; +import com.tencent.bk.job.execute.model.KubeWorkloadFilter; +import com.tencent.bk.job.execute.model.LabelSelectExprDTO; +import com.tencent.bk.job.execute.service.ContainerService; +import com.tencent.bk.job.execute.service.HostService; +import com.tencent.bk.job.manage.model.inner.ServiceListAppHostResultDTO; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Slf4j +@Service +public class ContainerServiceImpl implements ContainerService { + private final HostService hostService; + private final BizCmdbClient cmdbClient; + private final AppScopeMappingService appScopeMappingService; + + @Autowired + public ContainerServiceImpl(HostService hostService, BizCmdbClient cmdbClient, + AppScopeMappingService appScopeMappingService) { + this.hostService = hostService; + this.cmdbClient = cmdbClient; + this.appScopeMappingService = appScopeMappingService; + } + + @Override + public List listContainerByIds(long appId, Collection ids) { + ResourceScope resourceScope = appScopeMappingService.getScopeByAppId(appId); + List containerDetailList = + cmdbClient.listKubeContainerByIds(Long.parseLong(resourceScope.getId()), ids); + if (CollectionUtils.isEmpty(containerDetailList)) { + return Collections.emptyList(); + } + List containers = containerDetailList.stream() + .map(ContainerDetailDTO::toContainer).collect(Collectors.toList()); + + fillNodeHostInfo(appId, containers); + + return containers; + } + + private void fillNodeHostInfo(long appId, List containers) { + ServiceListAppHostResultDTO hostResult = + hostService.batchGetAppHosts( + appId, + containers.stream() + .map(container -> new HostDTO(container.getNodeHostId())) + .collect(Collectors.toList()), + false); + Map hostMap = hostResult.getValidHosts().stream() + .collect(Collectors.toMap(HostDTO::getHostId, host -> host, (oldValue, newValue) -> newValue)); + + containers.forEach(container -> { + HostDTO nodeHost = hostMap.get(container.getNodeHostId()); + if (nodeHost == null) { + log.error("Could not found node host for container: {}", container.getId()); + return; + } + container.setNodeAgentId(nodeHost.getAgentId()); + container.setNodeIp(nodeHost.getPrimaryIp()); + }); + } + + @Override + public List listContainerByContainerFilter(long appId, KubeContainerFilter filter) { + ListKubeContainerByTopoReq req = new ListKubeContainerByTopoReq(); + long bizId = convertToBizId(appId); + req.setBizId(bizId); + if (!filter.isEmptyFilter()) { + if (filter.hasKubeNodeFilter()) { + List kubeNodeIDS = computeKubeTopoNode(bizId, filter); + if (CollectionUtils.isEmpty(kubeNodeIDS)) { + // 如果根据条件查询，没有匹配的容器拓扑节点，无需进一步处理；直接返回空的容器列表 + return Collections.emptyList(); + } + req.setNodeIdList(kubeNodeIDS); + } + + if (filter.getPodFilter() != null) { + KubePodFilter kubePodFilter = filter.getPodFilter(); + + PropertyFilterDTO podPropFilter = new PropertyFilterDTO(); + podPropFilter.setCondition(RuleConditionEnum.AND.getCondition()); + + if (CollectionUtils.isNotEmpty(kubePodFilter.getPodNames())) { + podPropFilter.addRule(BaseRuleDTO.in(PodDTO.Fields.NAME, kubePodFilter.getPodNames())); + } + if (CollectionUtils.isNotEmpty(kubePodFilter.getLabelSelector())) { + ComposeRuleDTO labelsComposeRule = new ComposeRuleDTO(RuleConditionEnum.AND.getCondition()); + kubePodFilter.getLabelSelector().forEach( + labelSelectExpr -> labelsComposeRule.addRule(buildLabelFilterRule(labelSelectExpr))); + + podPropFilter.addRule(BaseRuleDTO.filterObject(PodDTO.Fields.LABELS, labelsComposeRule)); + } + + req.setPodFilter(podPropFilter); + } + + if (filter.getContainerPropFilter() != null) { + KubeContainerPropFilter containerPropFilter = filter.getContainerPropFilter(); + + PropertyFilterDTO containerFilter = new PropertyFilterDTO(); + containerFilter.setCondition(RuleConditionEnum.AND.getCondition()); + if (CollectionUtils.isNotEmpty(containerPropFilter.getContainerNames())) { + containerFilter.addRule(BaseRuleDTO.in(ContainerDTO.Fields.NAME, + containerPropFilter.getContainerNames())); + } + + req.setContainerFilter(containerFilter); + } + } + + List containerDetailList = cmdbClient.listKubeContainerByTopo(req); + if (CollectionUtils.isEmpty(containerDetailList)) { + return Collections.emptyList(); + } + + List containers = containerDetailList.stream() + .map(ContainerDetailDTO::toContainer).collect(Collectors.toList()); + + fillNodeHostInfo(appId, containers); + + if (filter.isFetchAnyOneContainer()) { + // 随机选择一个容器 + int index = RandomUtil.nextInt(containers.size()); + return Collections.singletonList(containers.get(index)); + } else { + return containers; + } + } + + private BaseRuleDTO buildLabelFilterRule(LabelSelectExprDTO labelSelectExpr) { + LabelSelectorOperatorEnum operator = labelSelectExpr.getOperator(); + switch (operator) { + case EQUALS: + return BaseRuleDTO.equals(labelSelectExpr.getKey(), labelSelectExpr.getValues().get(0)); + case NOT_EQUALS: + return BaseRuleDTO.notEquals(labelSelectExpr.getKey(), labelSelectExpr.getValues().get(0)); + case IN: + return BaseRuleDTO.in(labelSelectExpr.getKey(), labelSelectExpr.getValues()); + case NOT_IN: + return BaseRuleDTO.notIn(labelSelectExpr.getKey(), labelSelectExpr.getValues()); + case EXISTS: + return BaseRuleDTO.exists(labelSelectExpr.getKey()); + case NOT_EXISTS: + return BaseRuleDTO.notExists(labelSelectExpr.getKey()); + case LESS_THAN: + return BaseRuleDTO.lessThan(labelSelectExpr.getKey(), labelSelectExpr.getValues().get(0)); + case GREATER_THAN: + return BaseRuleDTO.greaterThan(labelSelectExpr.getKey(), labelSelectExpr.getValues().get(0)); + default: + throw new IllegalArgumentException("Invalid label select operator: " + operator); + } + } + + private Long convertToBizId(long appId) { + ResourceScope resourceScope = appScopeMappingService.getScopeByAppId(appId); + if (!resourceScope.isBiz()) { + throw new IllegalArgumentException("Invalid appId"); + } + return Long.parseLong(resourceScope.getId()); + } + + private List computeKubeTopoNode(long bizId, KubeContainerFilter filter) { + List kubeNodes; + if (filter.getWorkloadFilter() != null) { + kubeNodes = computeKubeWorkloadTopoNodes(bizId, filter.getClusterFilter(), filter.getNamespaceFilter(), + filter.getWorkloadFilter()); + } else if (filter.getNamespaceFilter() != null) { + kubeNodes = computeKubeNamespaceTopoNodes(bizId, filter.getClusterFilter(), filter.getNamespaceFilter()); + } else if (filter.getClusterFilter() != null) { + kubeNodes = computeKubeClusterTopoNodes(bizId, filter.getClusterFilter()); + } else { + throw new IllegalStateException("Invalid KubeContainerFilter for compute kube topo node"); + } + return kubeNodes; + } + + private List computeKubeWorkloadTopoNodes(long bizId, + KubeClusterFilter clusterFilter, + KubeNamespaceFilter namespaceFilter, + KubeWorkloadFilter workloadFilter) { + List matchClusters = null; + List matchNamespaces = null; + + if (clusterFilter != null) { + matchClusters = cmdbClient.listKubeClusters( + KubeClusterQuery.Builder.builder(bizId).bkClusterUIDs(clusterFilter.getClusterUIDs()).build()); + if (CollectionUtils.isEmpty(matchClusters)) { + return Collections.emptyList(); + } + } + + if (namespaceFilter != null && CollectionUtils.isNotEmpty(namespaceFilter.getNamespaces())) { + matchNamespaces = cmdbClient.listKubeNamespaces( + NamespaceQuery.Builder.builder(bizId).names(namespaceFilter.getNamespaces()).build()); + if (CollectionUtils.isEmpty(matchNamespaces)) { + return Collections.emptyList(); + } + } + + List workloads = cmdbClient.listKubeWorkloads( + WorkloadQuery.Builder + .builder(bizId, workloadFilter.getKind()) + .bkClusterIds(matchClusters == null ? null : + matchClusters.stream().map(KubeClusterDTO::getId).collect(Collectors.toList())) + .bkNamespaceIds(matchNamespaces == null ? null : + matchNamespaces.stream().map(KubeNamespaceDTO::getId).collect(Collectors.toList())) + .names(workloadFilter.getWorkloadNames()) + .build()); + + return workloads == null ? Collections.emptyList() : workloads.stream() + .map(workload -> new KubeNodeID(workload.getKind(), workload.getId())).collect(Collectors.toList()); + } + + private List computeKubeNamespaceTopoNodes(long bizId, + KubeClusterFilter clusterFilter, + KubeNamespaceFilter namespaceFilter) { + List matchClusters = null; + + if (clusterFilter != null) { + matchClusters = cmdbClient.listKubeClusters( + KubeClusterQuery.Builder.builder(bizId).bkClusterUIDs(clusterFilter.getClusterUIDs()).build()); + if (CollectionUtils.isEmpty(matchClusters)) { + return Collections.emptyList(); + } + } + + List namespaces = cmdbClient.listKubeNamespaces( + NamespaceQuery.Builder.builder(bizId) + .bkClusterIds(matchClusters == null ? null : + matchClusters.stream().map(KubeClusterDTO::getId).collect(Collectors.toList())) + .names(namespaceFilter.getNamespaces()) + .build()); + + + return namespaces == null ? Collections.emptyList() : namespaces.stream() + .map(namespace -> new KubeNodeID(KubeTopoNodeTypeEnum.NAMESPACE.getValue(), namespace.getId())) + .collect(Collectors.toList()); + } + + private List computeKubeClusterTopoNodes(long bizId, + KubeClusterFilter clusterFilter) { + + List clusters = cmdbClient.listKubeClusters( + KubeClusterQuery.Builder.builder(bizId) + .bkClusterUIDs(clusterFilter.getClusterUIDs()) + .build()); + + + return clusters == null ? Collections.emptyList() : clusters.stream() + .map(cluster -> new KubeNodeID(KubeTopoNodeTypeEnum.CLUSTER.getValue(), cluster.getId())) + .collect(Collectors.toList()); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/DangerousRecordServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/DangerousRecordServiceImpl.java index a0101d2120..341805ac9c 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/DangerousRecordServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/DangerousRecordServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/DangerousScriptCheckServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/DangerousScriptCheckServiceImpl.java index a05ec58ff6..eb7b8f16e4 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/DangerousScriptCheckServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/DangerousScriptCheckServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,7 +26,6 @@ import com.tencent.bk.job.common.i18n.service.MessageI18nService; import com.tencent.bk.job.common.model.InternalResponse; -import com.tencent.bk.job.execute.client.ScriptCheckResourceClient; import com.tencent.bk.job.execute.common.constants.TaskStartupModeEnum; import com.tencent.bk.job.execute.model.DangerousRecordDTO; import com.tencent.bk.job.execute.model.ScriptCheckItemDTO; @@ -36,9 +35,10 @@ import com.tencent.bk.job.execute.service.ApplicationService; import com.tencent.bk.job.execute.service.DangerousRecordService; import com.tencent.bk.job.execute.service.DangerousScriptCheckService; -import com.tencent.bk.job.manage.common.consts.RuleMatchHandleActionEnum; -import com.tencent.bk.job.manage.common.consts.script.ScriptCheckErrorLevelEnum; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.RuleMatchHandleActionEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptCheckErrorLevelEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; +import com.tencent.bk.job.manage.api.inner.ServiceCheckScriptResource; import com.tencent.bk.job.manage.model.inner.ServiceScriptCheckResultItemDTO; import com.tencent.bk.job.manage.model.inner.request.ServiceCheckScriptRequest; import com.tencent.bk.job.manage.model.inner.resp.ServiceApplicationDTO; @@ -56,17 +56,17 @@ @Slf4j @Service public class DangerousScriptCheckServiceImpl implements DangerousScriptCheckService { - private final ScriptCheckResourceClient scriptCheckResourceClient; + private final ServiceCheckScriptResource scriptCheckResource; private final MessageI18nService messageI18nService; private final DangerousRecordService dangerousRecordService; private final ApplicationService applicationService; @Autowired - public DangerousScriptCheckServiceImpl(ScriptCheckResourceClient scriptCheckResourceClient, + public DangerousScriptCheckServiceImpl(ServiceCheckScriptResource scriptCheckResource, MessageI18nService messageI18nService, DangerousRecordService dangerousRecordService, ApplicationService applicationService) { - this.scriptCheckResourceClient = scriptCheckResourceClient; + this.scriptCheckResource = scriptCheckResource; this.messageI18nService = messageI18nService; this.dangerousRecordService = dangerousRecordService; this.applicationService = applicationService; @@ -75,7 +75,7 @@ public DangerousScriptCheckServiceImpl(ScriptCheckResourceClient scriptCheckReso @Override public List check(ScriptTypeEnum scriptType, String content) { InternalResponse> response = - scriptCheckResourceClient.check(new ServiceCheckScriptRequest(content, scriptType.getValue())); + scriptCheckResource.check(new ServiceCheckScriptRequest(content, scriptType.getValue())); return response.isSuccess() ? response.getData() : Collections.emptyList(); } @@ -94,8 +94,8 @@ public String summaryDangerousScriptCheckResult(String stepName, List checkResultDescList = checkResults.stream().filter(checkResult -> ScriptCheckErrorLevelEnum.FATAL.getValue() == checkResult.getLevel()) - .map(checkResult -> buildScriptCheckResultDetail(stepName, checkResult)) - .collect(Collectors.toList()); + .map(checkResult -> buildScriptCheckResultDetail(stepName, checkResult)) + .collect(Collectors.toList()); if (CollectionUtils.isEmpty(checkResultDescList)) { return ""; } @@ -141,7 +141,7 @@ private DangerousRecordDTO buildDangerousRecord(TaskInstanceDTO taskInstance, St record.setClient("bk_job"); } record.setOperator(taskInstance.getOperator()); - record.setScriptLanguage(stepInstance.getScriptType()); + record.setScriptLanguage(stepInstance.getScriptType().getValue()); record.setScriptContent(stepInstance.getScriptContent()); List checkItems = new ArrayList<>(1); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ExecuteResourceNameQueryService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ExecuteResourceNameQueryService.java index 5cd98ae731..823b72586c 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ExecuteResourceNameQueryService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ExecuteResourceNameQueryService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -29,12 +29,12 @@ import com.tencent.bk.job.common.iam.service.ResourceNameQueryService; import com.tencent.bk.job.common.iam.util.IamUtil; import com.tencent.bk.job.common.service.AppScopeMappingService; -import com.tencent.bk.job.execute.client.TaskTemplateResourceClient; import com.tencent.bk.job.execute.model.AccountDTO; import com.tencent.bk.job.execute.service.AccountService; import com.tencent.bk.job.execute.service.ApplicationService; import com.tencent.bk.job.execute.service.ScriptService; import com.tencent.bk.job.execute.service.TaskPlanService; +import com.tencent.bk.job.manage.api.inner.ServiceTaskTemplateResource; import com.tencent.bk.job.manage.model.inner.ServiceScriptDTO; import com.tencent.bk.job.manage.model.inner.resp.ServiceApplicationDTO; import lombok.extern.slf4j.Slf4j; @@ -43,12 +43,12 @@ import org.springframework.stereotype.Service; @Slf4j -@Service +@Service("jobExecuteResourceNameQueryService") public class ExecuteResourceNameQueryService implements ResourceNameQueryService { private final ApplicationService applicationService; private final ScriptService scriptService; private final TaskPlanService taskPlanService; - private final TaskTemplateResourceClient taskTemplateService; + private final ServiceTaskTemplateResource taskTemplateResource; private final AccountService accountService; private final AppScopeMappingService appScopeMappingService; @@ -57,13 +57,13 @@ public class ExecuteResourceNameQueryService implements ResourceNameQueryService public ExecuteResourceNameQueryService(ApplicationService applicationService, ScriptService scriptService, TaskPlanService taskPlanService, - TaskTemplateResourceClient taskTemplateService, + ServiceTaskTemplateResource taskTemplateResource, AccountService accountService, AppScopeMappingService appScopeMappingService) { this.applicationService = applicationService; this.scriptService = scriptService; this.taskPlanService = taskPlanService; - this.taskTemplateService = taskTemplateService; + this.taskTemplateResource = taskTemplateResource; this.accountService = accountService; this.appScopeMappingService = appScopeMappingService; } @@ -85,7 +85,7 @@ public String getResourceName(ResourceTypeEnum resourceType, String resourceId) case TEMPLATE: long templateId = Long.parseLong(resourceId); if (templateId > 0) { - return taskTemplateService.getTemplateNameById(templateId).getData(); + return taskTemplateResource.getTemplateNameById(templateId).getData(); } break; case PLAN: diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ExternalAgentServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ExternalAgentServiceImpl.java new file mode 100644 index 0000000000..c150fdfbbb --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ExternalAgentServiceImpl.java @@ -0,0 +1,131 @@ +/* + * + * * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * * + * * Copyright (C) 2021 Tencent. All rights reserved. + * * + * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * * + * * License for BK-JOB蓝鲸智云作业平台: + * * -------------------------------------------------------------------- + * * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * * + * * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * * the Software. + * * + * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * * IN THE SOFTWARE. + * + */ + +package com.tencent.bk.job.execute.service.impl; + +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.DistributeFileFromExternalAgentException; +import com.tencent.bk.job.common.gse.service.AgentStateClient; +import com.tencent.bk.job.common.gse.service.model.HostAgentStateQuery; +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.execute.config.NFSExternalAgentHostConfig; +import com.tencent.bk.job.execute.model.ExternalHostDTO; +import com.tencent.bk.job.execute.service.ExternalAgentService; +import com.tencent.bk.job.execute.service.HostService; +import com.tencent.bk.job.manage.model.inner.ServiceHostDTO; +import lombok.extern.slf4j.Slf4j; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicLong; +import java.util.stream.Collectors; + + +/** + * 获取配置的外部有Gse agent的机器作为文件分发源 + */ +@Slf4j +public class ExternalAgentServiceImpl implements ExternalAgentService { + + private final HostService hostService; + private final AgentStateClient agentStateClient; + + private final AtomicLong roundRobinCnt = new AtomicLong(0); + private final List externalAgentHostPool = new ArrayList<>(); + + public ExternalAgentServiceImpl(NFSExternalAgentHostConfig externalAgentHostConfig, + HostService hostService, + AgentStateClient agentStateClient) { + this.hostService = hostService; + this.agentStateClient = agentStateClient; + initExternalHostsPool(externalAgentHostConfig); + } + + @Override + public HostDTO getDistributeSourceHost() { + Map cmdbHostMap = hostService.batchGetHosts(externalAgentHostPool); + List cmdbHostList = new ArrayList<>(); + externalAgentHostPool.forEach(hostDTO -> { + ServiceHostDTO serviceHostDTO = cmdbHostMap.get(hostDTO); + if (serviceHostDTO != null) { + cmdbHostList.add(serviceHostDTO); + } + }); + return ServiceHostDTO.toHostDTO(getOneAliveAgentHost(cmdbHostList)); + } + + private ServiceHostDTO getOneAliveAgentHost(List hosts) { + int hostCnt = hosts.size(); + List agentStateQueries = new ArrayList<>(); + hosts.forEach(host -> { + HostAgentStateQuery query = new HostAgentStateQuery(); + query.setHostId(host.getHostId()); + query.setBizId(host.getBizId()); + query.setCloudIp(host.getCloudIp()); + query.setAgentId(host.getAgentId()); + agentStateQueries.add(query); + }); + Map hostIpAliveStatusMap = agentStateClient.batchGetAgentAliveStatus(agentStateQueries); + ServiceHostDTO sourceHost = null; + // 同一个服务实例内通过RR获取分发的源主机，遍历配置的主机池直至找到一个 agent 状态正常的主机 + for (int i = 0; i < hostCnt; i++) { + // 从上次使用的主机的下一个开始获取 + int idx = (i + (int) roundRobinCnt.get()) % hostCnt; + ServiceHostDTO serviceHostDTO = hosts.get(idx); + if (Boolean.TRUE.equals(hostIpAliveStatusMap.get(serviceHostDTO.getAgentId()))) { + // 下一次获取，计数器从下一个开始 + roundRobinCnt.accumulateAndGet(i + 1, Long::sum); + sourceHost = serviceHostDTO; + break; + } + } + + if (sourceHost == null) { + String msg = "Distribute file use external agent, but no available host found, please check configuration"; + log.error(msg); + throw new DistributeFileFromExternalAgentException(msg, ErrorCode.INTERNAL_ERROR); + } + + log.info( + "Using external agent host:(cloudIp={}, hostId={}, agentId={}) to distribute file", + sourceHost.getCloudIp(), + sourceHost.getHostId(), + sourceHost.getAgentId() + ); + return sourceHost; + } + + private void initExternalHostsPool(NFSExternalAgentHostConfig configFromDeployment) { + externalAgentHostPool.addAll( + configFromDeployment.getHosts() + .stream() + .map(ExternalHostDTO::convertToHostDTO) + .collect(Collectors.toList()) + ); + } + +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileAgentTaskServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileAgentTaskServiceImpl.java deleted file mode 100644 index a199304d75..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileAgentTaskServiceImpl.java +++ /dev/null @@ -1,245 +0,0 @@ -package com.tencent.bk.job.execute.service.impl; - -import com.tencent.bk.job.common.constant.Order; -import com.tencent.bk.job.common.model.dto.HostDTO; -import com.tencent.bk.job.execute.dao.FileAgentTaskDAO; -import com.tencent.bk.job.execute.dao.GseTaskIpLogDAO; -import com.tencent.bk.job.execute.model.AgentTaskDTO; -import com.tencent.bk.job.execute.model.AgentTaskDetailDTO; -import com.tencent.bk.job.execute.model.AgentTaskResultGroupBaseDTO; -import com.tencent.bk.job.execute.model.AgentTaskResultGroupDTO; -import com.tencent.bk.job.execute.model.FileSourceDTO; -import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; -import com.tencent.bk.job.execute.model.StepInstanceDTO; -import com.tencent.bk.job.execute.service.FileAgentTaskService; -import com.tencent.bk.job.execute.service.HostService; -import com.tencent.bk.job.execute.service.StepInstanceService; -import com.tencent.bk.job.logsvr.consts.FileTaskModeEnum; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.stream.Collectors; - -@Service -@Slf4j -public class FileAgentTaskServiceImpl - extends AbstractAgentTaskServiceImpl - implements FileAgentTaskService { - - private final FileAgentTaskDAO fileAgentTaskDAO; - private final GseTaskIpLogDAO gseTaskIpLogDAO; - - @Autowired - public FileAgentTaskServiceImpl(FileAgentTaskDAO fileAgentTaskDAO, - StepInstanceService stepInstanceService, - HostService hostService, - GseTaskIpLogDAO gseTaskIpLogDAO) { - super(stepInstanceService, hostService); - this.fileAgentTaskDAO = fileAgentTaskDAO; - this.gseTaskIpLogDAO = gseTaskIpLogDAO; - } - - @Override - public void batchSaveAgentTasks(Collection agentTasks) { - if (CollectionUtils.isEmpty(agentTasks)) { - return; - } - fileAgentTaskDAO.batchSaveAgentTasks(agentTasks); - } - - @Override - public void batchUpdateAgentTasks(Collection agentTasks) { - if (CollectionUtils.isEmpty(agentTasks)) { - return; - } - fileAgentTaskDAO.batchUpdateAgentTasks(agentTasks); - } - - @Override - public int getSuccessAgentTaskCount(long stepInstanceId, int executeCount) { - if (isStepInstanceRecordExist(stepInstanceId)) { - return fileAgentTaskDAO.getSuccessAgentTaskCount(stepInstanceId, executeCount); - } else { - return gseTaskIpLogDAO.getSuccessAgentTaskCount(stepInstanceId, executeCount); - } - } - - @Override - public List listAndGroupAgentTasks(StepInstanceBaseDTO stepInstance, - int executeCount, - Integer batch) { - List resultGroups = new ArrayList<>(); - - List agentTasks = listAgentTasks(stepInstance.getId(), executeCount, batch, - FileTaskModeEnum.DOWNLOAD); - if (CollectionUtils.isEmpty(agentTasks)) { - return resultGroups; - } - - List agentTaskDetailList = fillHostDetail(stepInstance, agentTasks); - - resultGroups = groupAgentTasks(agentTaskDetailList); - - return resultGroups.stream().sorted().collect(Collectors.toList()); - } - - @Override - public List listResultGroups(long stepInstanceId, - int executeCount, - Integer batch) { - List resultGroups; - resultGroups = fileAgentTaskDAO.listResultGroups(stepInstanceId, executeCount, batch); - if (CollectionUtils.isEmpty(resultGroups)) { - // 兼容历史数据 - resultGroups = gseTaskIpLogDAO.listResultGroups(stepInstanceId, executeCount); - } - return resultGroups; - } - - @Override - public List listAgentTaskDetailByResultGroup(StepInstanceBaseDTO stepInstance, - Integer executeCount, - Integer batch, - Integer status, - String tag) { - List agentTasks = fileAgentTaskDAO.listAgentTaskByResultGroup(stepInstance.getId(), - executeCount, batch, status); - if (CollectionUtils.isEmpty(agentTasks)) { - // 兼容历史数据 - agentTasks = gseTaskIpLogDAO.listAgentTaskByResultGroup(stepInstance.getId(), executeCount, status, tag); - } - return fillHostDetail(stepInstance, agentTasks); - } - - @Override - public List listAgentTaskDetailByResultGroup(StepInstanceBaseDTO stepInstance, - Integer executeCount, - Integer batch, - Integer status, - String tag, - Integer limit, - String orderField, - Order order) { - List agentTasks = fileAgentTaskDAO.listAgentTaskByResultGroup(stepInstance.getId(), - executeCount, batch, status, limit, orderField, order); - if (CollectionUtils.isEmpty(agentTasks)) { - // 兼容历史数据 - agentTasks = gseTaskIpLogDAO.listAgentTaskByResultGroup(stepInstance.getId(), executeCount, status, tag, - limit, orderField, order); - } - return fillHostDetail(stepInstance, agentTasks); - } - - @Override - public List listAgentTasks(Long stepInstanceId, Integer executeCount, Integer batch, - FileTaskModeEnum fileTaskMode) { - List agentTasks = fileAgentTaskDAO.listAgentTasks(stepInstanceId, executeCount, batch, - fileTaskMode); - // 兼容历史数据 - if (CollectionUtils.isEmpty(agentTasks)) { - agentTasks = gseTaskIpLogDAO.listAgentTasks(stepInstanceId, executeCount); - if (CollectionUtils.isNotEmpty(agentTasks) && fileTaskMode != null) { - agentTasks = agentTasks.stream().filter(agentTask -> agentTask.getFileTaskMode() == fileTaskMode) - .collect(Collectors.toList()); - } - } - - return agentTasks; - } - - @Override - public List listAgentTasksByGseTaskId(Long gseTaskId) { - return fileAgentTaskDAO.listAgentTasksByGseTaskId(gseTaskId); - } - - @Override - public AgentTaskDTO getAgentTaskByHost(StepInstanceDTO stepInstance, Integer executeCount, Integer batch, - FileTaskModeEnum fileTaskMode, HostDTO host) { - AgentTaskDTO agentTask = null; - Long hostId = host.getHostId(); - if (hostId != null) { - // 根据hostId查询 - agentTask = fileAgentTaskDAO.getAgentTaskByHostId(stepInstance.getId(), executeCount, batch, - fileTaskMode, hostId); - } else if (StringUtils.isNotEmpty(host.toCloudIp())) { - // 根据ip查询的模式，有两种情况，数据可能在gse_file_agent_task/gse_task_ip_log表中，优先查询gse_file_agent_task - HostDTO queryHost = getStepHostByIp(stepInstance, host.toCloudIp()); - if (queryHost != null) { - agentTask = fileAgentTaskDAO.getAgentTaskByHostId(stepInstance.getId(), executeCount, batch, - fileTaskMode, queryHost.getHostId()); - } else { - // 根据ip查询gse_task_ip_log表中的数据 - agentTask = gseTaskIpLogDAO.getAgentTaskByIp(stepInstance.getId(), executeCount, host.toCloudIp()); - } - } - return agentTask; - } - - private HostDTO getStepHostByIp(StepInstanceDTO stepInstance, String cloudIp) { - HostDTO queryHost = stepInstance.getTargetServers().getIpList().stream() - .filter(targetHost -> cloudIp.equals(targetHost.toCloudIp())) - .findFirst() - .orElse(null); - if (queryHost == null) { - if (CollectionUtils.isNotEmpty(stepInstance.getResolvedFileSourceList())) { - for (FileSourceDTO fileSource : stepInstance.getResolvedFileSourceList()) { - if (fileSource.getServers() != null - && CollectionUtils.isNotEmpty(fileSource.getServers().getIpList())) { - queryHost = fileSource.getServers().getIpList().stream() - .filter(sourceHost -> cloudIp.equals(sourceHost.toCloudIp())) - .findFirst() - .orElse(null); - if (queryHost != null) { - break; - } - } - } - } - } - - return queryHost; - } - - @Override - public List listAgentTasks(Long stepInstanceId, Integer executeCount, Integer batch) { - List agentTasks = fileAgentTaskDAO.listAgentTasks(stepInstanceId, executeCount, batch, null); - if (CollectionUtils.isEmpty(agentTasks)) { - // 兼容历史数据 - agentTasks = gseTaskIpLogDAO.listAgentTasks(stepInstanceId, executeCount); - } - return agentTasks; - } - - @Override - public int getActualSuccessExecuteCount(long stepInstanceId, String cloudIp) { - // 兼容历史数据 - return gseTaskIpLogDAO.getActualSuccessExecuteCount(stepInstanceId, cloudIp); - } - - @Override - public List listAgentTaskDetail(StepInstanceBaseDTO stepInstance, - Integer executeCount, - Integer batch) { - List agentTasks = listAgentTasks(stepInstance.getId(), executeCount, batch); - return fillHostDetail(stepInstance, agentTasks); - } - - private boolean isStepInstanceRecordExist(long stepInstanceId) { - return fileAgentTaskDAO.isStepInstanceRecordExist(stepInstanceId); - } - - @Override - public void updateAgentTaskFields(long stepInstanceId, - int executeCount, - Integer batch, - Integer actualExecuteCount, - Long gseTaskId) { - fileAgentTaskDAO.updateAgentTaskFields(stepInstanceId, executeCount, batch, actualExecuteCount, gseTaskId); - } -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileExecuteObjectTaskServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileExecuteObjectTaskServiceImpl.java new file mode 100644 index 0000000000..4b40dbab27 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileExecuteObjectTaskServiceImpl.java @@ -0,0 +1,304 @@ +package com.tencent.bk.job.execute.service.impl; + +import com.tencent.bk.job.common.constant.Order; +import com.tencent.bk.job.common.util.BatchUtil; +import com.tencent.bk.job.execute.dao.FileAgentTaskDAO; +import com.tencent.bk.job.execute.dao.FileExecuteObjectTaskDAO; +import com.tencent.bk.job.execute.dao.common.IdGen; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; +import com.tencent.bk.job.execute.model.ExecuteObjectCompositeKey; +import com.tencent.bk.job.execute.model.ExecuteObjectTask; +import com.tencent.bk.job.execute.model.ResultGroupBaseDTO; +import com.tencent.bk.job.execute.model.ResultGroupDTO; +import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; +import com.tencent.bk.job.execute.service.FileExecuteObjectTaskService; +import com.tencent.bk.job.execute.service.StepInstanceService; +import com.tencent.bk.job.logsvr.consts.FileTaskModeEnum; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +@Service +@Slf4j +public class FileExecuteObjectTaskServiceImpl + extends AbstractExecuteObjectTaskServiceImpl + implements FileExecuteObjectTaskService { + + private final FileExecuteObjectTaskDAO fileExecuteObjectTaskDAO; + private final FileAgentTaskDAO fileAgentTaskDAO; + private final IdGen idGen; + + @Autowired + public FileExecuteObjectTaskServiceImpl(StepInstanceService stepInstanceService, + FileExecuteObjectTaskDAO fileExecuteObjectTaskDAO, + FileAgentTaskDAO fileAgentTaskDAO, + IdGen idGen) { + super(stepInstanceService); + this.fileAgentTaskDAO = fileAgentTaskDAO; + this.fileExecuteObjectTaskDAO = fileExecuteObjectTaskDAO; + this.idGen = idGen; + } + + @Override + public void batchSaveTasks(Collection tasks) { + if (CollectionUtils.isEmpty(tasks)) { + return; + } + tasks.forEach(task -> task.setId(idGen.genGseFileExecuteObjTaskId())); + + boolean executeObjectSupported = isExecuteObjectSupported(tasks); + + // 任务分批，避免大事务造成 db 主从延迟 + BatchUtil.executeBatch( + tasks, + 2000, + batchTasks -> executeSaveTasks(executeObjectSupported, batchTasks) + ); + } + + private void executeSaveTasks(boolean executeObjectSupported, + Collection tasks) { + if (executeObjectSupported) { + fileExecuteObjectTaskDAO.batchSaveTasks(tasks); + } else { + fileAgentTaskDAO.batchSaveAgentTasks(tasks); + } + } + + @Override + public void batchUpdateTasks(Collection tasks) { + if (CollectionUtils.isEmpty(tasks)) { + return; + } + + boolean executeObjectSupported = isExecuteObjectSupported(tasks); + // 任务分批，避免大事务造成 db 主从延迟 + BatchUtil.executeBatch( + tasks, + 2000, + batchTasks -> executeUpdateTasks(executeObjectSupported, batchTasks) + ); + } + + private void executeUpdateTasks(boolean executeObjectSupported, + Collection tasks) { + if (executeObjectSupported) { + fileExecuteObjectTaskDAO.batchUpdateTasks(tasks); + } else { + fileAgentTaskDAO.batchUpdateAgentTasks(tasks); + } + } + + @Override + public int getSuccessTaskCount(Long taskInstanceId, long stepInstanceId, int executeCount) { + if (isStepInstanceRecordExist(taskInstanceId, stepInstanceId)) { + return fileExecuteObjectTaskDAO.getSuccessTaskCount(taskInstanceId, stepInstanceId, executeCount); + } else { + return fileAgentTaskDAO.getSuccessAgentTaskCount(taskInstanceId, stepInstanceId, executeCount); + } + } + + @Override + public List listTasks(StepInstanceBaseDTO stepInstance, Integer executeCount, Integer batch) { + return listTasks(stepInstance, executeCount, batch, null); + } + + @Override + public List listTasksByGseTaskId(StepInstanceBaseDTO stepInstance, Long gseTaskId) { + List executeObjectTasks; + if (stepInstance.isSupportExecuteObjectFeature()) { + executeObjectTasks = fileExecuteObjectTaskDAO.listTasksByGseTaskId( + stepInstance.getTaskInstanceId(), gseTaskId); + } else { + // 兼容老版本数据 + executeObjectTasks = fileAgentTaskDAO.listAgentTasksByGseTaskId( + stepInstance.getTaskInstanceId(), gseTaskId); + } + + fillExecuteObjectForExecuteObjectTasks(stepInstance, executeObjectTasks); + return executeObjectTasks; + } + + @Override + public ExecuteObjectTask getTaskByExecuteObjectCompositeKey(StepInstanceBaseDTO stepInstance, + Integer executeCount, + Integer batch, + FileTaskModeEnum fileTaskMode, + ExecuteObjectCompositeKey executeObjectCompositeKey) { + ExecuteObject executeObject = + stepInstanceService.findExecuteObjectByCompositeKey(stepInstance, executeObjectCompositeKey); + if (executeObject == null) { + return null; + } + + long stepInstanceId = stepInstance.getId(); + ExecuteObjectTask executeObjectTask; + if (stepInstance.isSupportExecuteObjectFeature()) { + executeObjectTask = fileExecuteObjectTaskDAO.getTaskByExecuteObjectId( + stepInstance.getTaskInstanceId(), stepInstanceId, executeCount, + batch, fileTaskMode, executeObject.getId()); + } else { + // 兼容老版本不使用执行对象的场景(仅支持主机） + Long hostId = executeObject.getHost().getHostId(); + executeObjectTask = fileAgentTaskDAO.getAgentTaskByHostId(stepInstance.getTaskInstanceId(), + stepInstanceId, executeCount, batch, fileTaskMode, hostId); + } + executeObjectTask.setExecuteObject(executeObject); + return executeObjectTask; + } + + @Override + public List listAndGroupTasks(StepInstanceBaseDTO stepInstance, + int executeCount, + Integer batch) { + List resultGroups = new ArrayList<>(); + List executeObjectTasks = listTasks(stepInstance, executeCount, + batch, FileTaskModeEnum.DOWNLOAD); + if (CollectionUtils.isEmpty(executeObjectTasks)) { + return resultGroups; + } + + fillExecuteObjectForExecuteObjectTasks(stepInstance, executeObjectTasks); + + resultGroups = groupTasks(executeObjectTasks); + + return resultGroups.stream().sorted().collect(Collectors.toList()); + } + + @Override + public List listResultGroups(StepInstanceBaseDTO stepInstance, + int executeCount, + Integer batch) { + List resultGroups; + long stepInstanceId = stepInstance.getId(); + + if (stepInstance.isSupportExecuteObjectFeature()) { + resultGroups = fileExecuteObjectTaskDAO.listResultGroups( + stepInstance.getTaskInstanceId(), stepInstanceId, executeCount, batch); + } else { + // 兼容历史数据 + resultGroups = fileAgentTaskDAO.listResultGroups(stepInstance.getTaskInstanceId(), + stepInstanceId, executeCount, batch); + } + return resultGroups; + } + + @Override + public List listTaskByResultGroup(StepInstanceBaseDTO stepInstance, + Integer executeCount, + Integer batch, + Integer status, + String tag) { + List executeObjectTasks; + + if (stepInstance.isSupportExecuteObjectFeature()) { + executeObjectTasks = fileExecuteObjectTaskDAO.listTaskByResultGroup( + stepInstance.getTaskInstanceId(), stepInstance.getId(), executeCount, batch, status); + } else { + // 兼容历史数据 + executeObjectTasks = fileAgentTaskDAO.listAgentTaskByResultGroup( + stepInstance.getTaskInstanceId(), stepInstance.getId(), executeCount, batch, status); + } + + fillExecuteObjectForExecuteObjectTasks(stepInstance, executeObjectTasks); + return executeObjectTasks; + } + + + @Override + public List listTaskByResultGroup(StepInstanceBaseDTO stepInstance, + Integer executeCount, + Integer batch, + Integer status, + String tag, + Integer limit, + String orderField, + Order order) { + List executeObjectTasks; + + if (stepInstance.isSupportExecuteObjectFeature()) { + executeObjectTasks = fileExecuteObjectTaskDAO.listTaskByResultGroup( + stepInstance.getTaskInstanceId(), + stepInstance.getId(), + executeCount, + batch, + status, + limit, + orderField, + order + ); + } else { + // 兼容历史数据 + executeObjectTasks = fileAgentTaskDAO.listAgentTaskByResultGroup( + stepInstance.getTaskInstanceId(), + stepInstance.getId(), + executeCount, + batch, + status, + limit, + orderField, + order + ); + } + + fillExecuteObjectForExecuteObjectTasks(stepInstance, executeObjectTasks); + return executeObjectTasks; + } + + private boolean isStepInstanceRecordExist(Long taskInstanceId, long stepInstanceId) { + return fileExecuteObjectTaskDAO.isStepInstanceRecordExist(taskInstanceId, stepInstanceId); + } + + @Override + public void updateTaskFields(StepInstanceBaseDTO stepInstance, + int executeCount, + Integer batch, + Integer actualExecuteCount, + Long gseTaskId) { + if (stepInstance.isSupportExecuteObjectFeature()) { + fileExecuteObjectTaskDAO.updateTaskFields( + stepInstance.getTaskInstanceId(), + stepInstance.getId(), + executeCount, + batch, + actualExecuteCount, + gseTaskId + ); + } else { + // 兼容老版本方式 + fileAgentTaskDAO.updateAgentTaskFields( + stepInstance.getTaskInstanceId(), + stepInstance.getId(), + executeCount, + batch, + actualExecuteCount, + gseTaskId + ); + } + } + + @Override + public List listTasks(StepInstanceBaseDTO stepInstance, + Integer executeCount, + Integer batch, + FileTaskModeEnum fileTaskMode) { + List executeObjectTasks; + long stepInstanceId = stepInstance.getId(); + if (stepInstance.isSupportExecuteObjectFeature()) { + executeObjectTasks = fileExecuteObjectTaskDAO.listTasks( + stepInstance.getTaskInstanceId(), stepInstanceId, executeCount, batch, fileTaskMode); + } else { + // 兼容老版本数据 + executeObjectTasks = fileAgentTaskDAO.listAgentTasks(stepInstance.getTaskInstanceId(), + stepInstanceId, executeCount, batch, fileTaskMode); + } + fillExecuteObjectForExecuteObjectTasks(stepInstance, executeObjectTasks); + return executeObjectTasks; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileSourceTaskLogServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileSourceTaskLogServiceImpl.java new file mode 100644 index 0000000000..ece321048a --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileSourceTaskLogServiceImpl.java @@ -0,0 +1,82 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.service.impl; + +import com.tencent.bk.job.execute.dao.FileSourceTaskLogDAO; +import com.tencent.bk.job.execute.dao.common.IdGen; +import com.tencent.bk.job.execute.model.FileSourceTaskLogDTO; +import com.tencent.bk.job.execute.service.FileSourceTaskLogService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +@Service +@Slf4j +public class FileSourceTaskLogServiceImpl implements FileSourceTaskLogService { + + private final FileSourceTaskLogDAO fileSourceTaskLogDAO; + + private final IdGen idGen; + + public FileSourceTaskLogServiceImpl(FileSourceTaskLogDAO fileSourceTaskLogDAO, IdGen idGen) { + this.fileSourceTaskLogDAO = fileSourceTaskLogDAO; + this.idGen = idGen; + } + + @Override + public int addFileSourceTaskLog(FileSourceTaskLogDTO fileSourceTaskLog) { + fileSourceTaskLog.setId(idGen.genFileSourceTaskLogId()); + return fileSourceTaskLogDAO.insertFileSourceTaskLog(fileSourceTaskLog); + } + + @Override + public int updateFileSourceTaskLog(FileSourceTaskLogDTO fileSourceTaskLog) { + return fileSourceTaskLogDAO.updateFileSourceTaskLogByStepInstance(fileSourceTaskLog); + } + + @Override + public FileSourceTaskLogDTO getFileSourceTaskLog(Long taskInstanceId, long stepInstanceId, int executeCount) { + return fileSourceTaskLogDAO.getFileSourceTaskLog(taskInstanceId, stepInstanceId, executeCount); + } + + @Override + public FileSourceTaskLogDTO getFileSourceTaskLogByBatchTaskId(Long taskInstanceId, String fileSourceBatchTaskId) { + return fileSourceTaskLogDAO.getFileSourceTaskLogByBatchTaskId(taskInstanceId, fileSourceBatchTaskId); + } + + @Override + public int updateTimeConsumingByBatchTaskId(Long taskInstanceId, + String fileSourceBatchTaskId, + Long startTime, + Long endTime, + Long totalTime) { + return fileSourceTaskLogDAO.updateTimeConsumingByBatchTaskId( + taskInstanceId, + fileSourceBatchTaskId, + startTime, + endTime, + totalTime + ); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileWorkerHostProvisioner.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileWorkerHostProvisioner.java new file mode 100644 index 0000000000..e6389aa620 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/FileWorkerHostProvisioner.java @@ -0,0 +1,52 @@ +/* + * + * * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * * + * * Copyright (C) 2021 Tencent. All rights reserved. + * * + * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * * + * * License for BK-JOB蓝鲸智云作业平台: + * * -------------------------------------------------------------------- + * * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * * + * * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * * the Software. + * * + * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * * IN THE SOFTWARE. + * + */ + +package com.tencent.bk.job.execute.service.impl; + +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.execute.engine.prepare.third.FileWorkerHostService; +import com.tencent.bk.job.execute.service.ThirdFileDistributeSourceHostProvisioner; +import lombok.extern.slf4j.Slf4j; + +/** + * 使用job-file-worker所在Node作为文件源文件的分发源 + * 用于获取job-file-worker信息 + */ +@Slf4j +public class FileWorkerHostProvisioner implements ThirdFileDistributeSourceHostProvisioner { + + private final FileWorkerHostService fileWorkerHostService; + + public FileWorkerHostProvisioner(FileWorkerHostService fileWorkerHostService) { + this.fileWorkerHostService = fileWorkerHostService; + } + + @Override + public HostDTO getThirdFileDistributeSourceHost(Long cloudId, String protocol, String ip) { + log.debug("distribute third file from file worker host"); + return fileWorkerHostService.parseFileWorkerHostWithCache(cloudId, protocol, ip); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/GseTaskServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/GseTaskServiceImpl.java index f9814234c4..b01caf6fc3 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/GseTaskServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/GseTaskServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,25 +25,32 @@ package com.tencent.bk.job.execute.service.impl; import com.tencent.bk.job.execute.dao.GseTaskDAO; +import com.tencent.bk.job.execute.dao.common.IdGen; import com.tencent.bk.job.execute.model.GseTaskDTO; +import com.tencent.bk.job.execute.model.GseTaskSimpleDTO; import com.tencent.bk.job.execute.service.GseTaskService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import java.util.List; + @Service @Slf4j public class GseTaskServiceImpl implements GseTaskService { private final GseTaskDAO gseTaskDAO; + private final IdGen idGen; @Autowired - public GseTaskServiceImpl(GseTaskDAO gseTaskDAO) { + public GseTaskServiceImpl(GseTaskDAO gseTaskDAO, IdGen idGen) { this.gseTaskDAO = gseTaskDAO; + this.idGen = idGen; } @Override public Long saveGseTask(GseTaskDTO gseTask) { + gseTask.setId(idGen.genGseTaskId()); return gseTaskDAO.saveGseTask(gseTask); } @@ -53,12 +60,24 @@ public boolean updateGseTask(GseTaskDTO gseTask) { } @Override - public GseTaskDTO getGseTask(long stepInstanceId, int executeCount, Integer batch) { - return gseTaskDAO.getGseTask(stepInstanceId, executeCount, batch); + public GseTaskDTO getGseTask(Long taskInstanceId, long stepInstanceId, int executeCount, Integer batch) { + return gseTaskDAO.getGseTask(taskInstanceId, stepInstanceId, executeCount, batch); + } + + @Override + public GseTaskDTO getGseTask(Long taskInstanceId, long gseTaskId) { + return gseTaskDAO.getGseTask(taskInstanceId, gseTaskId); + } + + @Override + public GseTaskSimpleDTO getGseTaskSimpleInfo(String gseTaskId) { + return gseTaskDAO.getGseTaskSimpleInfo(gseTaskId); } @Override - public GseTaskDTO getGseTask(long gseTaskId) { - return gseTaskDAO.getGseTask(gseTaskId); + public List listGseTaskSimpleInfo(Long stepInstanceId, + Integer executeCount, + Integer batch) { + return gseTaskDAO.ListGseTaskSimpleInfo(stepInstanceId, executeCount, batch); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/HostServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/HostServiceImpl.java index 9db7353fc6..d903c15bd9 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/HostServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/HostServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,13 +24,9 @@ package com.tencent.bk.job.execute.service.impl; -import com.google.common.cache.CacheBuilder; -import com.google.common.cache.CacheLoader; -import com.google.common.cache.LoadingCache; -import com.tencent.bk.job.common.cc.model.CcCloudAreaInfoDTO; import com.tencent.bk.job.common.cc.model.CcCloudIdDTO; -import com.tencent.bk.job.common.cc.model.CcGroupHostPropDTO; import com.tencent.bk.job.common.cc.model.CcInstanceDTO; +import com.tencent.bk.job.common.cc.model.DynamicGroupHostPropDTO; import com.tencent.bk.job.common.cc.sdk.CmdbClientFactory; import com.tencent.bk.job.common.cc.sdk.IBizCmdbClient; import com.tencent.bk.job.common.model.InternalResponse; @@ -38,19 +34,21 @@ import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.model.dto.ResourceScope; import com.tencent.bk.job.common.service.AppScopeMappingService; -import com.tencent.bk.job.common.util.json.JsonUtils; -import com.tencent.bk.job.execute.client.ServiceHostResourceClient; -import com.tencent.bk.job.execute.client.WhiteIpResourceClient; +import com.tencent.bk.job.execute.model.DynamicServerGroupDTO; +import com.tencent.bk.job.execute.model.DynamicServerTopoNodeDTO; import com.tencent.bk.job.execute.service.HostService; +import com.tencent.bk.job.manage.api.inner.ServiceHostResource; import com.tencent.bk.job.manage.model.inner.ServiceHostDTO; import com.tencent.bk.job.manage.model.inner.ServiceListAppHostResultDTO; -import com.tencent.bk.job.manage.model.inner.ServiceWhiteIPInfo; +import com.tencent.bk.job.manage.model.inner.request.ServiceBatchGetAppHostsReq; import com.tencent.bk.job.manage.model.inner.request.ServiceBatchGetHostsReq; -import com.tencent.bk.job.manage.model.inner.request.ServiceCheckAppHostsReq; +import com.tencent.bk.job.manage.model.inner.request.ServiceGetHostsByCloudIpv6Req; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.tuple.ImmutablePair; +import org.apache.commons.lang3.tuple.Pair; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.scheduling.annotation.Scheduled; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; import java.util.ArrayList; @@ -59,54 +57,31 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; +import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; -@Service +@Service("jobExecuteHostService") @Slf4j public class HostServiceImpl implements HostService { - private final WhiteIpResourceClient whiteIpResourceClient; - private final ServiceHostResourceClient hostResourceClient; + private final ServiceHostResource hostResource; private final AppScopeMappingService appScopeMappingService; - - private volatile boolean isWhiteIpConfigLoaded = false; - private final Map whiteIpConfig = new ConcurrentHashMap<>(); - - private final LoadingCache cloudAreaNameCache = CacheBuilder.newBuilder() - .maximumSize(10000).expireAfterWrite(1, TimeUnit.HOURS). - build(new CacheLoader() { - @Override - public String load(Long cloudAreaId) { - IBizCmdbClient bizCmdbClient = CmdbClientFactory.getCmdbClient(); - List cloudAreaList = bizCmdbClient.getCloudAreaList(); - if (cloudAreaList == null || cloudAreaList.isEmpty()) { - log.warn("Get all cloud area return empty!"); - return "Unknown"; - } - log.info("Get all cloud area, result={}", JsonUtils.toJson(cloudAreaList)); - for (CcCloudAreaInfoDTO cloudArea : cloudAreaList) { - if (cloudArea.getId().equals(cloudAreaId)) { - return cloudArea.getName(); - } - } - log.info("No found cloud area for cloudAreaId:{}", cloudAreaId); - return "Unknown"; - } - } - ); + private final ExecutorService getHostsByTopoExecutor; @Autowired - public HostServiceImpl(WhiteIpResourceClient whiteIpResourceClient, - ServiceHostResourceClient hostResourceClient, - AppScopeMappingService appScopeMappingService) { - this.hostResourceClient = hostResourceClient; - this.whiteIpResourceClient = whiteIpResourceClient; + public HostServiceImpl(ServiceHostResource hostResource, + AppScopeMappingService appScopeMappingService, + @Qualifier("getHostsByTopoExecutor") ExecutorService getHostsByTopoExecutor) { + this.hostResource = hostResource; this.appScopeMappingService = appScopeMappingService; + this.getHostsByTopoExecutor = getHostsByTopoExecutor; } @Override public Map batchGetHosts(List hostIps) { - List hosts = hostResourceClient.batchGetHosts( + List hosts = hostResource.batchGetHosts( new ServiceBatchGetHostsReq(hostIps)).getData(); Map hostMap = new HashMap<>(); hosts.forEach(host -> hostMap.put(new HostDTO(host.getCloudAreaId(), host.getIp()), host)); @@ -115,7 +90,7 @@ public Map batchGetHosts(List hostIps) { @Override public ServiceHostDTO getHost(HostDTO host) { - List hosts = hostResourceClient.batchGetHosts( + List hosts = hostResource.batchGetHosts( new ServiceBatchGetHostsReq(Collections.singletonList(host))).getData(); if (CollectionUtils.isEmpty(hosts)) { return null; @@ -124,111 +99,87 @@ public ServiceHostDTO getHost(HostDTO host) { } @Override - public List getHostAllowedAction(long appId, HostDTO host) { - try { - InternalResponse> resp = whiteIpResourceClient.getWhiteIPActionScopes(appId, host.getIp(), - host.getBkCloudId()); - if (!resp.isSuccess()) { - log.warn("Get white ip action scopes return fail resp, appId:{}, host:{}", appId, - host.toCloudIp()); - return Collections.emptyList(); - } - log.info("Get white ip action scopes, appId:{}, host:{}, resp:{}", appId, host.toCloudIp(), resp); - return resp.getData(); - } catch (Exception e) { - log.warn("GetHostAllowedAction fail!", e); - return Collections.emptyList(); - } - } - - @Scheduled(cron = "0 * * * * ?") - public void syncWhiteIpConfig() { - log.info("Sync white ip config!"); - isWhiteIpConfigLoaded = true; - long start = System.currentTimeMillis(); - InternalResponse> resp = whiteIpResourceClient.listWhiteIPInfos(); - if (resp == null || !resp.isSuccess()) { - log.warn("Get all white ip config return fail resp!"); - return; - } - log.info("Sync white ip config, resp: {}", JsonUtils.toJson(resp)); - - List whiteIpInfos = resp.getData(); - whiteIpConfig.clear(); - whiteIpInfos.forEach(whiteIpInfo -> - whiteIpConfig.put(whiteIpInfo.getCloudId() + ":" + whiteIpInfo.getIp(), whiteIpInfo)); - - long cost = System.currentTimeMillis() - start; - if (cost > 1000L) { - log.warn("Sync white ip config is slow, cost: {}", cost); - } - log.info("Sync white ip config success!"); - } - - @Override - public boolean isMatchWhiteIpRule(long appId, String cloudIp, String action) { - try { - if (!isWhiteIpConfigLoaded) { - syncWhiteIpConfig(); - } - ServiceWhiteIPInfo whiteIpInfo = whiteIpConfig.get(cloudIp); - if (whiteIpInfo == null) { - return false; - } - if (whiteIpInfo.isForAllApp()) { - return CollectionUtils.isNotEmpty(whiteIpInfo.getAllAppActionScopeList()) - && whiteIpInfo.getAllAppActionScopeList().contains(action); - } else { - return whiteIpInfo.getAppIdActionScopeMap() != null - && whiteIpInfo.getAppIdActionScopeMap().get(appId) != null - && whiteIpInfo.getAppIdActionScopeMap().get(appId).contains(action); - } - } catch (Throwable e) { - log.warn("Get white ip config by host and action", e); - return false; + public ServiceHostDTO getHostByCloudIpv6(long cloudAreaId, String ipv6) { + List hosts = hostResource.getHostsByCloudIpv6( + new ServiceGetHostsByCloudIpv6Req(cloudAreaId, ipv6) + ).getData(); + if (CollectionUtils.isEmpty(hosts)) { + log.warn("Cannot find host by (cloudAreaId={}, ipv6={})", cloudAreaId, ipv6); + return null; + } else if (hosts.size() > 1) { + log.warn( + "Found {} host by (cloudAreaId={}, ipv6={}), use first one, hosts={}", + hosts.size(), + cloudAreaId, + ipv6, + hosts + ); } + return hosts.get(0); } @Override public ServiceListAppHostResultDTO batchGetAppHosts(Long appId, - Collection hosts) { + Collection hosts, + boolean refreshAgentId) { InternalResponse response = - hostResourceClient.batchGetAppHosts(appId, new ServiceCheckAppHostsReq(new ArrayList<>(hosts))); + hostResource.batchGetAppHosts(appId, + new ServiceBatchGetAppHostsReq(new ArrayList<>(hosts), refreshAgentId)); return response.getData(); } @Override - public List getIpByDynamicGroupId(long appId, String groupId) { + public List getHostsByDynamicGroupId(long appId, String groupId) { IBizCmdbClient bizCmdbClient = CmdbClientFactory.getCmdbClient(); ResourceScope resourceScope = appScopeMappingService.getScopeByAppId(appId); - List cmdbGroupHostList = + List cmdbGroupHostList = bizCmdbClient.getDynamicGroupIp(Long.parseLong(resourceScope.getId()), groupId); - List ips = new ArrayList<>(); + List hostList = new ArrayList<>(); if (cmdbGroupHostList == null || cmdbGroupHostList.isEmpty()) { - return ips; + return hostList; } - for (CcGroupHostPropDTO hostProp : cmdbGroupHostList) { + for (DynamicGroupHostPropDTO hostProp : cmdbGroupHostList) { List hostCloudIdList = hostProp.getCloudIdList(); if (hostCloudIdList == null || hostCloudIdList.isEmpty()) { - log.warn("Get ip by dynamic group id, cmdb return illegal host, skip it!appId={}, groupId={}, " + + log.warn("Get ip by dynamic group id, cmdb return illegal host, skip it!appId={}, " + + "groupId={}, " + "hostIp={}", appId, groupId, hostProp.getInnerIp()); continue; } CcCloudIdDTO hostCloudId = hostCloudIdList.get(0); if (hostCloudId == null) { - log.warn("Get ip by dynamic group id, cmdb return illegal host, skip it!appId={}, groupId={}, " + + log.warn("Get ip by dynamic group id, cmdb return illegal host, skip it!appId={}, " + + "groupId={}, " + "hostIp={}", appId, groupId, hostProp.getInnerIp()); continue; } - HostDTO ip = new HostDTO(hostCloudId.getInstanceId(), hostProp.getFirstIp()); - ips.add(ip); + HostDTO host = new HostDTO(); + host.setHostId(hostProp.getId()); + host.setBkCloudId(hostCloudId.getInstanceId()); + host.setIp(hostProp.getFirstIp()); + host.setIpv6(hostProp.getIpv6()); + host.setAgentId(hostProp.getAgentId()); + hostList.add(host); } - log.info("Get hosts by groupId, appId={}, groupId={}, hosts={}", appId, groupId, ips); - return ips; + log.info("Get hosts by groupId, appId={}, groupId={}, hosts={}", appId, groupId, hostList); + return hostList; } @Override - public List getIpByTopoNodes(long appId, List ccInstances) { + public Map> batchGetAndGroupHostsByDynamicGroup( + long appId, + Collection groups) { + if (CollectionUtils.isEmpty(groups)) { + return new HashMap<>(); + } + + Map> result = new HashMap<>(); + groups.forEach(group -> result.put(group, getHostsByDynamicGroupId(appId, group.getGroupId()))); + return result; + } + + @Override + public List getHostsByTopoNodes(long appId, List ccInstances) { IBizCmdbClient bizCmdbClient = CmdbClientFactory.getCmdbClient(); ResourceScope resourceScope = appScopeMappingService.getScopeByAppId(appId); long bizId = Long.parseLong(resourceScope.getId()); @@ -238,20 +189,94 @@ public List getIpByTopoNodes(long appId, List ccInstance return ips; } for (ApplicationHostDTO hostProp : appHosts) { - HostDTO ip = new HostDTO(hostProp.getCloudAreaId(), hostProp.getIp()); - ips.add(ip); + HostDTO host = new HostDTO(hostProp.getCloudAreaId(), hostProp.getIp()); + host.setHostId(hostProp.getHostId()); + host.setIpv6(hostProp.getIpv6()); + host.setAgentId(hostProp.getAgentId()); + ips.add(host); } log.info("Get hosts by cc topo nodes, appId={}, nodes={}, hosts={}", appId, ccInstances, ips); return ips; } @Override - public String getCloudAreaName(long cloudAreaId) { + public Map> getAndGroupHostsByTopoNodes( + long appId, + Collection topoNodes) { + + Map> result; + if (topoNodes.size() < 10) { + result = new HashMap<>(); + for (DynamicServerTopoNodeDTO topoNode : topoNodes) { + List topoHosts = getHostsByTopoNodes(appId, + Collections.singletonList(new CcInstanceDTO(topoNode.getNodeType(), + topoNode.getTopoNodeId()))); + topoNode.setIpList(topoHosts); + result.put(topoNode, topoHosts); + } + } else { + result = getTopoHostsConcurrent(appId, topoNodes); + } + return result; + } + + private Map> getTopoHostsConcurrent( + long appId, + Collection topoNodes) { + + Map> result = new HashMap<>(); + + CountDownLatch latch = new CountDownLatch(topoNodes.size()); + List>>> futures = + new ArrayList<>(topoNodes.size()); + for (DynamicServerTopoNodeDTO topoNode : topoNodes) { + futures.add(getHostsByTopoExecutor.submit(new GetTopoHostTask(appId, topoNode, latch))); + } + + try { + for (Future>> future : futures) { + Pair> topoAndHosts = future.get(); + for (DynamicServerTopoNodeDTO topoNode : topoNodes) { + if (topoNode.equals(topoAndHosts.getLeft())) { + result.put(topoNode, topoAndHosts.getRight()); + } + } + } + } catch (InterruptedException | ExecutionException e) { + log.error("Get topo hosts concurrent error", e); + } try { - return cloudAreaNameCache.get(cloudAreaId); - } catch (Exception e) { - log.warn("Fail to get cloud area name", e); - return "Unknown"; + latch.await(); + } catch (InterruptedException e) { + log.error("Get topo hosts concurrent error", e); + } + return result; + } + + + private class GetTopoHostTask implements Callable>> { + private final long appId; + private final DynamicServerTopoNodeDTO topoNode; + private final CountDownLatch latch; + + private GetTopoHostTask(long appId, DynamicServerTopoNodeDTO topoNode, CountDownLatch latch) { + this.appId = appId; + this.topoNode = topoNode; + this.latch = latch; + } + + @Override + public Pair> call() { + try { + List topoHosts = getHostsByTopoNodes(appId, + Collections.singletonList(new CcInstanceDTO(topoNode.getNodeType(), topoNode.getTopoNodeId()))); + return new ImmutablePair<>(topoNode, topoHosts); + } catch (Throwable e) { + log.warn("Get hosts by topo fail", e); + return new ImmutablePair<>(topoNode, Collections.emptyList()); + } finally { + latch.countDown(); + } } } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LocalAgentHostProvisioner.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LocalAgentHostProvisioner.java new file mode 100644 index 0000000000..2c57732e6d --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LocalAgentHostProvisioner.java @@ -0,0 +1,49 @@ +/* + * + * * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * * + * * Copyright (C) 2021 Tencent. All rights reserved. + * * + * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * * + * * License for BK-JOB蓝鲸智云作业平台: + * * -------------------------------------------------------------------- + * * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * * + * * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * * the Software. + * * + * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * * IN THE SOFTWARE. + * + */ + +package com.tencent.bk.job.execute.service.impl; + +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.execute.service.AgentService; +import com.tencent.bk.job.execute.service.LocalFileDistributeSourceHostProvisioner; + +/** + * 分发本地文件时，使用job-execute机器进行分发 + * 获取本机信息 + */ +public class LocalAgentHostProvisioner implements LocalFileDistributeSourceHostProvisioner { + + private final AgentService agentService; + + public LocalAgentHostProvisioner(AgentService agentService) { + this.agentService = agentService; + } + + @Override + public HostDTO getLocalFileDistributeSourceHost() { + return agentService.getLocalAgentHost(); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LocalFileExternalAgentHostProvisioner.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LocalFileExternalAgentHostProvisioner.java new file mode 100644 index 0000000000..765f5f5b3d --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LocalFileExternalAgentHostProvisioner.java @@ -0,0 +1,48 @@ +/* + * + * * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * * + * * Copyright (C) 2021 Tencent. All rights reserved. + * * + * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * * + * * License for BK-JOB蓝鲸智云作业平台: + * * -------------------------------------------------------------------- + * * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * * + * * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * * the Software. + * * + * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * * IN THE SOFTWARE. + * + */ + +package com.tencent.bk.job.execute.service.impl; + +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.execute.service.ExternalAgentService; +import com.tencent.bk.job.execute.service.LocalFileDistributeSourceHostProvisioner; + +/** + * 分发本地文件时，使用集群外机器作为源分发 + */ +public class LocalFileExternalAgentHostProvisioner implements LocalFileDistributeSourceHostProvisioner { + + private final ExternalAgentService externalAgentService; + + public LocalFileExternalAgentHostProvisioner(ExternalAgentService externalAgentService) { + this.externalAgentService = externalAgentService; + } + + @Override + public HostDTO getLocalFileDistributeSourceHost() { + return externalAgentService.getDistributeSourceHost(); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogExportServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogExportServiceImpl.java index 4054abf8d0..c38f8c4107 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogExportServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogExportServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,25 +24,28 @@ package com.tencent.bk.job.execute.service.impl; +import com.tencent.bk.job.common.artifactory.config.ArtifactoryConfig; import com.tencent.bk.job.common.artifactory.sdk.ArtifactoryClient; +import com.tencent.bk.job.common.constant.ExecuteObjectTypeEnum; import com.tencent.bk.job.common.constant.JobConstants; -import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.redis.util.LockUtils; -import com.tencent.bk.job.common.util.BatchUtil; -import com.tencent.bk.job.common.util.date.DateUtils; +import com.tencent.bk.job.common.util.CollectionUtil; +import com.tencent.bk.job.common.util.JobContextUtil; import com.tencent.bk.job.common.util.file.ZipUtil; import com.tencent.bk.job.common.util.json.JsonUtils; -import com.tencent.bk.job.execute.config.ArtifactoryConfig; import com.tencent.bk.job.execute.config.LogExportConfig; import com.tencent.bk.job.execute.constants.LogExportStatusEnum; -import com.tencent.bk.job.execute.model.AgentTaskDTO; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; +import com.tencent.bk.job.execute.model.ExecuteObjectCompositeKey; +import com.tencent.bk.job.execute.model.ExecuteObjectTask; import com.tencent.bk.job.execute.model.LogExportJobInfoDTO; -import com.tencent.bk.job.execute.model.ScriptHostLogContent; +import com.tencent.bk.job.execute.model.ScriptExecuteObjectLogContent; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.service.LogExportService; import com.tencent.bk.job.execute.service.LogService; -import com.tencent.bk.job.execute.service.ScriptAgentTaskService; -import com.tencent.bk.job.execute.service.TaskInstanceService; +import com.tencent.bk.job.execute.service.ScriptExecuteObjectTaskService; +import com.tencent.bk.job.execute.service.StepInstanceService; +import com.tencent.bk.job.logsvr.util.LogFieldUtil; import lombok.Data; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; @@ -56,19 +59,14 @@ import java.io.File; import java.io.PrintWriter; -import java.time.ZoneId; -import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.UUID; import java.util.concurrent.ExecutorService; +import java.util.stream.Collectors; -/** - * @since 19/1/2021 12:01 - */ @Slf4j @Service public class LogExportServiceImpl implements LogExportService { @@ -76,39 +74,48 @@ public class LogExportServiceImpl implements LogExportService { private final LogService logService; private final ExecutorService logExportExecutor; private final StringRedisTemplate redisTemplate; - private final TaskInstanceService taskInstanceService; + private final StepInstanceService stepInstanceService; private final ArtifactoryClient artifactoryClient; private final ArtifactoryConfig artifactoryConfig; private final LogExportConfig logExportConfig; - private final ScriptAgentTaskService scriptAgentTaskService; + private final ScriptExecuteObjectTaskService scriptExecuteObjectTaskService; @Autowired public LogExportServiceImpl(LogService logService, StringRedisTemplate redisTemplate, - TaskInstanceService taskInstanceService, - ArtifactoryClient artifactoryClient, + StepInstanceService stepInstanceService, + @Qualifier("jobArtifactoryClient") ArtifactoryClient artifactoryClient, ArtifactoryConfig artifactoryConfig, LogExportConfig logExportConfig, - ScriptAgentTaskService scriptAgentTaskService, + ScriptExecuteObjectTaskService scriptExecuteObjectTaskService, @Qualifier("logExportExecutor") ExecutorService logExportExecutor) { this.logService = logService; this.redisTemplate = redisTemplate; - this.taskInstanceService = taskInstanceService; + this.stepInstanceService = stepInstanceService; this.artifactoryClient = artifactoryClient; this.artifactoryConfig = artifactoryConfig; this.logExportConfig = logExportConfig; - this.scriptAgentTaskService = scriptAgentTaskService; + this.scriptExecuteObjectTaskService = scriptExecuteObjectTaskService; this.logExportExecutor = logExportExecutor; } @Override - public LogExportJobInfoDTO packageLogFile(String username, Long appId, Long stepInstanceId, Long hostId, - String cloudIp, int executeCount, - String logFileDir, String logFileName, Boolean repackage) { - log.debug("Package log file for {}|{}|{}|{}|{}|{}|{}|{}", username, appId, stepInstanceId, hostId, executeCount, - logFileDir, logFileName, repackage); + public LogExportJobInfoDTO packageLogFile(String username, + Long appId, + Long taskInstanceId, + Long stepInstanceId, + ExecuteObjectTypeEnum executeObjectType, + Long executeObjectResourceId, + int executeCount, + String logFileDir, + String logFileName, + Boolean repackage) { + log.info("Package log file for username={}|appId={}|stepInstanceId={}|executeObjectType={}" + + "|executeObjectResourceId={}|executeCount={}|logFileDir={}|logFileName={}|repackage={}", + username, appId, stepInstanceId, executeObjectType, executeObjectResourceId, executeCount, logFileDir, + logFileName, repackage); LogExportJobInfoDTO exportJobInfo = new LogExportJobInfoDTO(); - exportJobInfo.setJobKey(getExportJobKey(appId, stepInstanceId, hostId, cloudIp)); + exportJobInfo.setJobKey(getExportJobKey(appId, stepInstanceId, executeObjectType, executeObjectResourceId)); exportJobInfo.setStatus(LogExportStatusEnum.INIT); if (repackage) { @@ -118,34 +125,42 @@ public LogExportJobInfoDTO packageLogFile(String username, Long appId, Long step } saveExportInfo(exportJobInfo); - boolean isGetByHost = hostId != null || StringUtils.isNotBlank(cloudIp); + boolean isGetByExecuteObject = executeObjectResourceId != null; - if (isGetByHost) { - doPackage(exportJobInfo, stepInstanceId, hostId, cloudIp, executeCount, logFileDir, logFileName); + if (isGetByExecuteObject) { + doPackage(exportJobInfo, taskInstanceId, stepInstanceId, executeObjectType, executeObjectResourceId, + executeCount, logFileDir, logFileName); } else { + String requestId = JobContextUtil.getRequestId(); logExportExecutor.execute(() -> { - String requestId = UUID.randomUUID().toString(); - log.debug("Begin log package process|{}", requestId); + log.debug("Begin log package process |{}", stepInstanceId); try { - boolean lockResult = LockUtils.tryGetDistributedLock(exportJobInfo.getJobKey(), requestId, - 3600_000L); + boolean lockResult = LockUtils.tryGetDistributedLock(exportJobInfo.getJobKey(), + requestId, 3600_000L); if (lockResult) { - log.debug("Acquire lock success! Begin process!|{}", requestId); + log.debug("Acquire lock success! Begin process!|{}", stepInstanceId); exportJobInfo.setStatus(LogExportStatusEnum.PROCESSING); saveExportInfo(exportJobInfo); - doPackage(exportJobInfo, stepInstanceId, hostId, cloudIp, executeCount, logFileDir, - logFileName); + doPackage(exportJobInfo, taskInstanceId, stepInstanceId, executeObjectType, + executeObjectResourceId, executeCount, logFileDir, logFileName); } else { - log.error("Job already running!|{}|{}|{}|{}", requestId, appId, stepInstanceId, hostId); + log.error("Job already running!|appId={}|stepInstanceId={}", appId, stepInstanceId); } } catch (Exception e) { - log.error("Error while package log file!|{}|{}|{}|{}", requestId, stepInstanceId, hostId, - executeCount, e); + String msg = MessageFormatter.arrayFormat( + "Error while package log file!|stepInstanceId={}|executeCount={}", + new String[]{ + String.valueOf(stepInstanceId), + String.valueOf(executeCount) + } + ).getMessage(); + log.error(msg, e); markJobFailed(exportJobInfo); } finally { LockUtils.releaseDistributedLock(exportJobInfo.getJobKey(), requestId); - log.debug("Process finished!|{}", requestId); + log.debug("Process finished!|stepInstanceId={}|executeCount={}|logFileName={}", stepInstanceId, + executeCount, logFileName); } }); } @@ -158,8 +173,12 @@ private void markJobFailed(LogExportJobInfoDTO exportJobInfo) { } @Override - public LogExportJobInfoDTO getExportInfo(Long appId, Long stepInstanceId, Long hostId, String ip) { - return JsonUtils.fromJson(redisTemplate.opsForValue().get(getExportJobKey(appId, stepInstanceId, hostId, ip)), + public LogExportJobInfoDTO getExportInfo(Long appId, + Long stepInstanceId, + ExecuteObjectTypeEnum executeObjectType, + Long executeObjectResourceId) { + return JsonUtils.fromJson(redisTemplate.opsForValue().get( + getExportJobKey(appId, stepInstanceId, executeObjectType, executeObjectResourceId)), LogExportJobInfoDTO.class); } @@ -171,40 +190,43 @@ private void deleteExportInfo(String jobKey) { redisTemplate.delete(jobKey); } - private String getExportJobKey(Long appId, Long stepInstanceId, Long hostId, String cloudIp) { - String key = EXPORT_KEY_PREFIX + appId + ":" + stepInstanceId + ":"; - if (hostId != null) { - key = key + hostId; - } else { - key = key + cloudIp; + private String getExportJobKey(Long appId, + Long stepInstanceId, + ExecuteObjectTypeEnum executeObjectType, + Long executeObjectResourceId) { + String exportJobKey = EXPORT_KEY_PREFIX + appId + ":" + stepInstanceId; + if (executeObjectResourceId != null) { + exportJobKey += ":" + executeObjectType.getValue() + ":" + + executeObjectResourceId; } - return key; + return exportJobKey; } private void doPackage(LogExportJobInfoDTO exportJobInfo, + long taskInstanceId, long stepInstanceId, - Long hostId, - String cloudIp, + ExecuteObjectTypeEnum executeObjectType, + Long executeObjectResourceId, int executeCount, String logFileDir, String logFileName) { - StepInstanceBaseDTO stepInstance = taskInstanceService.getBaseStepInstance(stepInstanceId); - boolean isGetByHost = hostId != null || StringUtils.isNotBlank(cloudIp); + StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstance(taskInstanceId, stepInstanceId); File logFile = new File(logFileDir + logFileName); StopWatch watch = new StopWatch("exportJobLog"); - watch.start("getGseAgentTasks"); - List gseAgentTasks = getGseAgentTasks(stepInstance, executeCount, hostId, cloudIp); + watch.start("getExecuteObjectTasks"); + List executeObjectTasks = getExecuteObjectTasks(stepInstance, executeCount, + executeObjectType, executeObjectResourceId); watch.stop(); - if (gseAgentTasks == null || gseAgentTasks.isEmpty()) { + if (executeObjectTasks == null || executeObjectTasks.isEmpty()) { log.warn("Gse task ips are empty! stepInstanceId={}", stepInstanceId); markJobFailed(exportJobInfo); return; } watch.start("getLogContentAndWriteToFile"); - if (!getLogContentAndWriteToFile(stepInstance, gseAgentTasks, logFile, isGetByHost, exportJobInfo)) { + if (!getLogContentAndWriteToFile(stepInstance, executeObjectTasks, logFile, exportJobInfo)) { log.warn("Fail to getLogContentAndWriteToFile"); return; } @@ -217,49 +239,51 @@ private void doPackage(LogExportJobInfoDTO exportJobInfo, if (watch.getTotalTimeMillis() > 10000L) { log.info("Export job execution log is slow, cost: {}", watch.prettyPrint()); } + log.info("Package log success.|stepInstanceId={}|executeCount={}|logFileName={}", stepInstanceId, + executeCount, logFileName); } /** - * 根据hostId或CloudIP获取日志记录信息，hostId与CloudIP均为空则获取所有目标机器日志记录信息 + * 获取执行对象任务 * - * @param stepInstance 步骤实例 - * @param executeCount 重试次数 - * @param hostId 要获取日志记录的主机ID - * @param cloudIp 要获取日志记录的CloudIP + * @param stepInstance 步骤实例 + * @param executeCount 重试次数 + * @param executeObjectType 要获取日志记录的执行对象类型 + * @param executeObjectResourceId 要获取日志记录的执行对象资源 ID * @return 日志记录信息列表 */ - private List getGseAgentTasks(StepInstanceBaseDTO stepInstance, - int executeCount, - Long hostId, - String cloudIp) { - List gseAgentTasks = new ArrayList<>(); - boolean isGetByHost = hostId != null || StringUtils.isNotBlank(cloudIp); - if (isGetByHost) { - HostDTO host = HostDTO.fromHostIdAndCloudIp(hostId, cloudIp); - AgentTaskDTO agentTask = scriptAgentTaskService.getAgentTaskByHost(stepInstance, executeCount, null, - host); - if (agentTask != null) { - gseAgentTasks.add(agentTask); + private List getExecuteObjectTasks(StepInstanceBaseDTO stepInstance, + int executeCount, + ExecuteObjectTypeEnum executeObjectType, + Long executeObjectResourceId) { + List executeObjectTasks = new ArrayList<>(); + boolean isGetByExecuteObject = executeObjectResourceId != null; + if (isGetByExecuteObject) { + ExecuteObjectCompositeKey executeObjectCompositeKey = + ExecuteObjectCompositeKey.ofExecuteObjectResource(executeObjectType, executeObjectResourceId); + ExecuteObjectTask executeObjectTask = + scriptExecuteObjectTaskService.getTaskByExecuteObjectCompositeKey( + stepInstance, executeCount, null, executeObjectCompositeKey); + if (executeObjectTask != null) { + executeObjectTasks.add(executeObjectTask); } } else { - gseAgentTasks = scriptAgentTaskService.listAgentTasks(stepInstance.getId(), executeCount, null); + executeObjectTasks = scriptExecuteObjectTaskService.listTasks(stepInstance, executeCount, null); } - return gseAgentTasks; + return executeObjectTasks; } private boolean getLogContentAndWriteToFile(StepInstanceBaseDTO stepInstance, - List gseAgentTasks, + List executeObjectTasks, File logFile, - boolean isGetByHost, LogExportJobInfoDTO exportJobInfo) { - Collection querys = buildLogBatchQuery(stepInstance.getId(), gseAgentTasks); + Collection querys = buildLogBatchQuery(stepInstance.getId(), executeObjectTasks); - String jobCreateDate = DateUtils.formatUnixTimestamp(stepInstance.getCreateTime(), ChronoUnit.MILLIS, - "yyyy_MM_dd", ZoneId.of("UTC")); + String jobCreateDate = LogFieldUtil.buildJobCreateDate(stepInstance.getCreateTime()); try (PrintWriter out = new PrintWriter(logFile, "UTF-8")) { for (LogBatchQuery query : querys) { - for (List hosts : query.getHostBatches()) { - writeOneBatchIpLogs(out, jobCreateDate, stepInstance, query, hosts, isGetByHost); + for (List executeObjects : query.getExecuteObjectBatches()) { + writeOneBatchExecuteObjectLogs(out, jobCreateDate, stepInstance, query, executeObjects); } } out.flush(); @@ -272,25 +296,31 @@ private boolean getLogContentAndWriteToFile(StepInstanceBaseDTO stepInstance, } } - private void writeOneBatchIpLogs(PrintWriter out, - String jobCreateDate, - StepInstanceBaseDTO stepInstance, - LogBatchQuery query, - List hosts, - boolean isGetByHost) { - List scriptHostLogContentList = - logService.batchGetScriptHostLogContent(jobCreateDate, stepInstance.getId(), - query.getExecuteCount(), - null, hosts); - for (ScriptHostLogContent scriptHostLogContent : scriptHostLogContentList) { - if (scriptHostLogContent != null && StringUtils.isNotEmpty(scriptHostLogContent.getContent())) { - String[] logList = scriptHostLogContent.getContent().split("\n"); + private void writeOneBatchExecuteObjectLogs(PrintWriter out, + String jobCreateDate, + StepInstanceBaseDTO stepInstance, + LogBatchQuery query, + List executeObjects) { + List executeObjectQueryKeys; + if (stepInstance.isSupportExecuteObjectFeature()) { + executeObjectQueryKeys = executeObjects.stream() + .map(executeObject -> ExecuteObjectCompositeKey.ofExecuteObjectId(executeObject.getId())) + .collect(Collectors.toList()); + } else { + executeObjectQueryKeys = executeObjects.stream() + .map(executeObject -> ExecuteObjectCompositeKey.ofHostId(executeObject.getResourceId())) + .collect(Collectors.toList()); + } + List scriptExecuteObjectLogContentList = + logService.batchGetScriptExecuteObjectLogContent(jobCreateDate, stepInstance, + query.getExecuteCount(), null, executeObjectQueryKeys); + for (ScriptExecuteObjectLogContent scriptExecuteObjectLogContent : scriptExecuteObjectLogContentList) { + if (scriptExecuteObjectLogContent != null + && StringUtils.isNotEmpty(scriptExecuteObjectLogContent.getContent())) { + + String[] logList = scriptExecuteObjectLogContent.getContent().split("\n"); for (String log : logList) { - if (isGetByHost) { - out.println(log); - } else { - out.println(scriptHostLogContent.getIp() + " | " + log); - } + out.println(scriptExecuteObjectLogContent.getExecuteObject().getExecuteObjectName() + " | " + log); } } } @@ -342,18 +372,13 @@ private void uploadZipFileToArtifactoryIfNeeded(File zipFile, LogExportJobInfoDT } } - private Collection buildLogBatchQuery(long stepInstanceId, List agentTasks) { + private Collection buildLogBatchQuery(long stepInstanceId, + List executeObjectTasks) { Map batchQueryGroups = new HashMap<>(); - agentTasks.forEach(agentTask -> { - LogBatchQuery query = batchQueryGroups.computeIfAbsent(agentTask.getExecuteCount(), + executeObjectTasks.forEach(executeObjectTask -> { + LogBatchQuery query = batchQueryGroups.computeIfAbsent(executeObjectTask.getExecuteCount(), (executeCount) -> new LogBatchQuery(stepInstanceId, executeCount)); - HostDTO queryHost = null; - if (agentTask.getHostId() != null) { - queryHost = HostDTO.fromHostId(agentTask.getHostId()); - } else if (StringUtils.isNotEmpty(agentTask.getCloudIp())) { - queryHost = HostDTO.fromCloudIp(agentTask.getCloudIp()); - } - query.addHost(queryHost); + query.addExecuteObject(executeObjectTask.getExecuteObject()); }); batchQueryGroups.values().forEach(LogBatchQuery::batchHosts); return batchQueryGroups.values(); @@ -361,26 +386,26 @@ private Collection buildLogBatchQuery(long stepInstanceId, List hosts = new ArrayList<>(); - private List> hostBatches; + private List executeObjects = new ArrayList<>(); + private List> executeObjectBatches; LogBatchQuery(long stepInstanceId, int executeCount) { this.stepInstanceId = stepInstanceId; this.executeCount = executeCount; } - void addHost(HostDTO host) { - if (hosts == null) { - hosts = new ArrayList<>(); + void addExecuteObject(ExecuteObject executeObject) { + if (executeObjects == null) { + executeObjects = new ArrayList<>(); } - hosts.add(host); + executeObjects.add(executeObject); } void batchHosts() { - hostBatches = BatchUtil.buildBatchList(hosts, MAX_BATCH_IPS); + executeObjectBatches = CollectionUtil.partitionList(executeObjects, MAX_BATCH_SIZE); } } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogServiceImpl.java index 7647e6c621..4e1de427c8 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/LogServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,72 +25,99 @@ package com.tencent.bk.job.execute.service.impl; import com.tencent.bk.job.common.exception.InternalException; +import com.tencent.bk.job.common.gse.constants.FileDistModeEnum; import com.tencent.bk.job.common.model.InternalResponse; import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.util.date.DateUtils; -import com.tencent.bk.job.execute.client.LogServiceResourceClient; -import com.tencent.bk.job.execute.common.constants.FileDistModeEnum; +import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.execute.common.constants.FileDistStatusEnum; -import com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum; -import com.tencent.bk.job.execute.model.AgentTaskDTO; -import com.tencent.bk.job.execute.model.FileIpLogContent; -import com.tencent.bk.job.execute.model.ScriptHostLogContent; +import com.tencent.bk.job.execute.engine.consts.ExecuteObjectTaskStatusEnum; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; +import com.tencent.bk.job.execute.engine.model.JobFile; +import com.tencent.bk.job.execute.metrics.LogSampler; +import com.tencent.bk.job.execute.model.AtomicFileTaskLog; +import com.tencent.bk.job.execute.model.ExecuteObjectCompositeKey; +import com.tencent.bk.job.execute.model.ExecuteObjectTask; +import com.tencent.bk.job.execute.model.FileExecuteObjectLogContent; +import com.tencent.bk.job.execute.model.ScriptExecuteObjectLogContent; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; -import com.tencent.bk.job.execute.service.FileAgentTaskService; +import com.tencent.bk.job.execute.model.TaskInstanceDTO; +import com.tencent.bk.job.execute.service.FileExecuteObjectTaskService; import com.tencent.bk.job.execute.service.LogService; -import com.tencent.bk.job.execute.service.ScriptAgentTaskService; +import com.tencent.bk.job.execute.service.ScriptExecuteObjectTaskService; import com.tencent.bk.job.execute.service.StepInstanceService; -import com.tencent.bk.job.execute.service.TaskInstanceService; +import com.tencent.bk.job.logsvr.api.ServiceLogResource; import com.tencent.bk.job.logsvr.consts.FileTaskModeEnum; import com.tencent.bk.job.logsvr.consts.LogTypeEnum; import com.tencent.bk.job.logsvr.model.service.ServiceBatchSaveLogRequest; +import com.tencent.bk.job.logsvr.model.service.ServiceExecuteObjectLogDTO; +import com.tencent.bk.job.logsvr.model.service.ServiceExecuteObjectScriptLogDTO; import com.tencent.bk.job.logsvr.model.service.ServiceFileLogQueryRequest; import com.tencent.bk.job.logsvr.model.service.ServiceFileTaskLogDTO; -import com.tencent.bk.job.logsvr.model.service.ServiceHostLogDTO; -import com.tencent.bk.job.logsvr.model.service.ServiceHostLogsDTO; -import com.tencent.bk.job.logsvr.model.service.ServiceSaveLogRequest; -import com.tencent.bk.job.logsvr.model.service.ServiceScriptLogDTO; import com.tencent.bk.job.logsvr.model.service.ServiceScriptLogQueryRequest; +import com.tencent.bk.job.logsvr.util.LogFieldUtil; +import com.tencent.bk.job.manage.api.common.constants.task.TaskFileTypeEnum; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; +import javax.annotation.PostConstruct; import java.nio.charset.StandardCharsets; import java.time.ZoneId; import java.time.temporal.ChronoUnit; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.function.Function; import java.util.stream.Collectors; @Service @Slf4j +@SuppressWarnings("Duplicates") public class LogServiceImpl implements LogService { - private final LogServiceResourceClient logServiceResourceClient; - private final TaskInstanceService taskInstanceService; - private final ScriptAgentTaskService scriptAgentTaskService; - private final FileAgentTaskService fileAgentTaskService; + private final ServiceLogResource logResource; + private final ScriptExecuteObjectTaskService scriptExecuteObjectTaskService; + private final FileExecuteObjectTaskService fileExecuteObjectTaskService; private final StepInstanceService stepInstanceService; + private final LogSampler logSampler; + + + // 脚本日志阈值128MB，当批量保存脚本日志时，如果日志集合超过阈值，采用分批保存 + @Value("${job.execute.scriptLog.requestContentSizeThresholdMB:128}") + private int requestContentSizeThresholdMB; + + // 脚本日志阈值-字节 + private int requestContentSizeThreshold; @Autowired - public LogServiceImpl(LogServiceResourceClient logServiceResourceClient, - TaskInstanceService taskInstanceService, - ScriptAgentTaskService scriptAgentTaskService, - FileAgentTaskService fileAgentTaskService, - StepInstanceService stepInstanceService) { - this.logServiceResourceClient = logServiceResourceClient; - this.taskInstanceService = taskInstanceService; - this.scriptAgentTaskService = scriptAgentTaskService; - this.fileAgentTaskService = fileAgentTaskService; + public LogServiceImpl(ServiceLogResource logResource, + ScriptExecuteObjectTaskService scriptExecuteObjectTaskService, + FileExecuteObjectTaskService fileExecuteObjectTaskService, + StepInstanceService stepInstanceService, + LogSampler logSampler) { + this.logResource = logResource; + this.scriptExecuteObjectTaskService = scriptExecuteObjectTaskService; + this.fileExecuteObjectTaskService = fileExecuteObjectTaskService; this.stepInstanceService = stepInstanceService; + this.logSampler = logSampler; + } + + @PostConstruct + public void init() { + requestContentSizeThreshold = requestContentSizeThresholdMB * 1024 * 1024; } @Override - public ServiceScriptLogDTO buildSystemScriptLog(HostDTO host, String content, int offset, - Long logTimeInMillSeconds) { + public ServiceExecuteObjectScriptLogDTO buildSystemScriptLog(StepInstanceBaseDTO stepInstance, + ExecuteObject executeObject, + String content, + int offset, + Long logTimeInMillSeconds) { String logDateTime; if (logTimeInMillSeconds != null) { logDateTime = DateUtils.formatUnixTimestamp(logTimeInMillSeconds, ChronoUnit.MILLIS, @@ -100,26 +127,78 @@ public ServiceScriptLogDTO buildSystemScriptLog(HostDTO host, String content, in "yyyy-MM-dd HH:mm:ss", ZoneId.systemDefault()); } String logContentWithDateTime = "[" + logDateTime + "] " + content + "\n"; - int length = logContentWithDateTime.getBytes(StandardCharsets.UTF_8).length; - return new ServiceScriptLogDTO(host, offset + length, logContentWithDateTime); + return buildScriptLog(stepInstance, executeObject, logContentWithDateTime, + logContentWithDateTime.getBytes(StandardCharsets.UTF_8).length, offset); + } + + @Override + public ServiceExecuteObjectScriptLogDTO buildScriptLog(StepInstanceBaseDTO stepInstance, + ExecuteObject executeObject, + String content, + int contentSizeBytes, + int offset) { + if (stepInstance.isSupportExecuteObjectFeature()) { + return new ServiceExecuteObjectScriptLogDTO( + executeObject.getId(), content, contentSizeBytes, offset); + } else { + // 兼容历史版本使用 hostId 的方式；发布完成并全量切换到执行对象特性之后，可删除这里的兼容代码 + HostDTO host = executeObject.getHost(); + return new ServiceExecuteObjectScriptLogDTO(host.getHostId(), host.toCloudIp(), host.toCloudIpv6(), + content, contentSizeBytes, offset); + } } @Override - public void batchWriteScriptLog(long jobCreateTime, long stepInstanceId, int executeCount, Integer batch, - List scriptLogs) { + public void batchWriteScriptLog(TaskInstanceDTO taskInstance, + long stepInstanceId, + int executeCount, + Integer batch, + List scriptLogs) { if (CollectionUtils.isEmpty(scriptLogs)) { return; } - String jobCreateDate = DateUtils.formatUnixTimestamp(jobCreateTime, ChronoUnit.MILLIS, - "yyyy_MM_dd", ZoneId.of("UTC")); + String jobCreateDate = LogFieldUtil.buildJobCreateDate(taskInstance.getCreateTime()); ServiceBatchSaveLogRequest request = new ServiceBatchSaveLogRequest(); + request.setAppId(taskInstance.getAppId()); + request.setCronTaskId(taskInstance.getCronTaskId()); + request.setAppCode(taskInstance.getAppCode()); request.setJobCreateDate(jobCreateDate); request.setLogType(LogTypeEnum.SCRIPT.getValue()); - List logs = scriptLogs.stream() - .map(scriptLog -> buildServiceLogDTO(stepInstanceId, executeCount, batch, scriptLog)) - .collect(Collectors.toList()); - request.setLogs(logs); - InternalResponse resp = logServiceResourceClient.saveLogs(request); + + List logs = new ArrayList<>(); + int accumulatedSize = 0; + + for (ServiceExecuteObjectScriptLogDTO scriptLog : scriptLogs) { + ServiceExecuteObjectLogDTO executeObjectLog = + buildServiceExecuteObjectLogDTO(stepInstanceId, executeCount, batch, scriptLog); + logs.add(executeObjectLog); + accumulatedSize += scriptLog.getContentSizeBytes(); + + if (accumulatedSize > requestContentSizeThreshold) { + // 当达到阈值，保存当前积累的logs集合 + request.setLogs(logs); + saveLogs(request, stepInstanceId, executeCount, batch); + logs.clear(); + accumulatedSize = 0; + if (log.isDebugEnabled()) { + log.debug("The current script log is too large, exceeding {}, and should be saved in batches, " + + "stepInstanceId:{}, executeCount:{}, batch: {}", + requestContentSizeThreshold, stepInstanceId, executeCount, batch); + } + } + } + + if (!logs.isEmpty()) { + request.setLogs(logs); + saveLogs(request, stepInstanceId, executeCount, batch); + } + } + + private void saveLogs(ServiceBatchSaveLogRequest request, + long stepInstanceId, + int executeCount, + Integer batch) { + InternalResponse resp = saveLogsAndRecordLogSizeMetrics(request); if (!resp.isSuccess()) { log.error("Batch write log content fail, stepInstanceId:{}, executeCount:{}, batch: {}", stepInstanceId, executeCount, batch); @@ -127,323 +206,741 @@ public void batchWriteScriptLog(long jobCreateTime, long stepInstanceId, int exe } } - private ServiceHostLogDTO buildServiceLogDTO(long stepInstanceId, int executeCount, Integer batch, - ServiceScriptLogDTO scriptLog) { - ServiceHostLogDTO logDTO = new ServiceHostLogDTO(); + private ServiceExecuteObjectLogDTO buildServiceExecuteObjectLogDTO(long stepInstanceId, + int executeCount, + Integer batch, + ServiceExecuteObjectScriptLogDTO scriptLog) { + ServiceExecuteObjectLogDTO logDTO = new ServiceExecuteObjectLogDTO(); logDTO.setStepInstanceId(stepInstanceId); logDTO.setExecuteCount(executeCount); logDTO.setBatch(batch); + logDTO.setExecuteObjectId(scriptLog.getExecuteObjectId()); logDTO.setHostId(scriptLog.getHostId()); - logDTO.setIp(scriptLog.getCloudIp()); + logDTO.setCloudIp(scriptLog.getCloudIp()); + logDTO.setCloudIpv6(scriptLog.getCloudIpv6()); logDTO.setScriptLog(scriptLog); return logDTO; } @Override - public ScriptHostLogContent getScriptHostLogContent(long stepInstanceId, int executeCount, Integer batch, - HostDTO host) { - StepInstanceBaseDTO stepInstance = taskInstanceService.getBaseStepInstance(stepInstanceId); - // 如果存在重试，那么该ip可能是之前已经执行过的，查询日志的时候需要获取到对应的executeCount - int actualExecuteCount = executeCount; - AgentTaskDTO agentTask = scriptAgentTaskService.getAgentTaskByHost(stepInstance, executeCount, batch, host); - if (agentTask == null) { - return null; - } - - if (executeCount > 0 && agentTask.getActualExecuteCount() != null) { - actualExecuteCount = agentTask.getActualExecuteCount(); - } else if (agentTask.getStatus() == AgentTaskStatusEnum.LAST_SUCCESS) { - // 兼容历史数据 - actualExecuteCount = scriptAgentTaskService.getActualSuccessExecuteCount(stepInstanceId, host.toCloudIp()); - } + public ScriptExecuteObjectLogContent getScriptExecuteObjectLogContent( + StepInstanceBaseDTO stepInstance, + int executeCount, + Integer batch, + ExecuteObjectCompositeKey executeObjectCompositeKey + ) { + ExecuteObjectTask executeObjectTask = + scriptExecuteObjectTaskService.getTaskByExecuteObjectCompositeKey(stepInstance, + executeCount, batch, executeObjectCompositeKey); + return getScriptExecuteObjectLogContent(stepInstance, executeCount, batch, executeObjectTask); + } - String taskCreateDateStr = DateUtils.formatUnixTimestamp(stepInstance.getCreateTime(), ChronoUnit.MILLIS, - "yyyy_MM_dd", ZoneId.of("UTC")); - InternalResponse resp; - if (agentTask.getHostId() != null) { - resp = logServiceResourceClient.getScriptHostLogByHostId(taskCreateDateStr, - stepInstanceId, actualExecuteCount, agentTask.getHostId(), batch); - } else { - // 兼容ip查询 - resp = logServiceResourceClient.getScriptHostLogByIp(taskCreateDateStr, - stepInstanceId, actualExecuteCount, agentTask.getCloudIp(), batch); - } - if (!resp.isSuccess()) { - log.error("Get script log content by host error, stepInstanceId={}, executeCount={}, batch={}, host={}", - stepInstanceId, actualExecuteCount, batch, host); - throw new InternalException(resp.getCode()); - } - return convertToScriptIpLogContent(stepInstanceId, executeCount, resp.getData(), agentTask); + private String buildTaskCreateDateStr(StepInstanceBaseDTO stepInstance) { + return LogFieldUtil.buildJobCreateDate(stepInstance.getCreateTime()); } - private ScriptHostLogContent convertToScriptIpLogContent(long stepInstanceId, int executeCount, - ServiceHostLogDTO logDTO, AgentTaskDTO agentTask) { + private ScriptExecuteObjectLogContent convertToScriptExecuteObjectLogContent( + StepInstanceBaseDTO stepInstance, + int executeCount, + ServiceExecuteObjectLogDTO executeObjectLog, + ExecuteObjectTask executeObjectTask + ) { // 日志是否拉取完成 - boolean isFinished = agentTask.getStatus().isFinished(); - String scriptContent = logDTO != null && logDTO.getScriptLog() != null ? - logDTO.getScriptLog().getContent() : ""; - Long hostId = logDTO != null ? logDTO.getHostId() : null; - String ip = logDTO != null ? logDTO.getIp() : null; - return new ScriptHostLogContent(stepInstanceId, executeCount, hostId, ip, scriptContent, isFinished); + boolean isFinished = executeObjectTask.getStatus().isFinished(); + String scriptContent = executeObjectLog != null && executeObjectLog.getScriptLog() != null ? + executeObjectLog.getScriptLog().getContent() : ""; + return new ScriptExecuteObjectLogContent(stepInstance.getId(), executeCount, + executeObjectTask.getExecuteObject(), scriptContent, isFinished); } @Override - public List batchGetScriptHostLogContent(String jobCreateDateStr, - long stepInstanceId, - int executeCount, - Integer batch, - List hosts) { - - ServiceScriptLogQueryRequest query = new ServiceScriptLogQueryRequest(); - query.setBatch(batch); - - StepInstanceBaseDTO stepInstance = taskInstanceService.getBaseStepInstance(stepInstanceId); - if (isQueryByHostIdCondition(stepInstance)) { - query.setHostIds(buildHostIdQueryCondition(stepInstance, hosts)); - } else { - // 兼容历史数据的查询（使用ip，没有hostId) - query.setIps(hosts.stream().map(HostDTO::toCloudIp).distinct().collect(Collectors.toList())); + public ScriptExecuteObjectLogContent getScriptExecuteObjectLogContent(StepInstanceBaseDTO stepInstance, + int executeCount, + Integer batch, + ExecuteObjectTask executeObjectTask) { + if (executeObjectTask == null) { + return null; } + String taskCreateDateStr = buildTaskCreateDateStr(stepInstance); + long stepInstanceId = stepInstance.getId(); + InternalResponse resp; - InternalResponse> resp = - logServiceResourceClient.listScriptLogs(jobCreateDateStr, stepInstanceId, executeCount, query); - if (!resp.isSuccess()) { - log.error("Get script log content by ips error, stepInstanceId={}, executeCount={}, batch={}, ips={}", - stepInstanceId, executeCount, batch, hosts); - throw new InternalException(resp.getCode()); - } - if (CollectionUtils.isEmpty(resp.getData())) { - return Collections.emptyList(); + // 如果存在重试，那么该任务可能是之前已经执行过的，查询日志的时候需要获取到对应的executeCount + int actualExecuteCount = executeCount; + if (executeCount > 0 && executeObjectTask.getActualExecuteCount() != null) { + actualExecuteCount = executeObjectTask.getActualExecuteCount(); } - return resp.getData().stream().map(logDTO -> { - String scriptContent = logDTO.getScriptLog() != null ? - logDTO.getScriptLog().getContent() : ""; - return new ScriptHostLogContent(logDTO.getStepInstanceId(), logDTO.getExecuteCount(), logDTO.getHostId(), - logDTO.getIp(), scriptContent, true); - }).collect(Collectors.toList()); - - } - - private boolean isQueryByHostIdCondition(StepInstanceBaseDTO stepInstance) { - return stepInstance.getTargetServers().getIpList().stream() - .anyMatch(host -> host.getHostId() != null); - } - private List buildHostIdQueryCondition(StepInstanceBaseDTO stepInstance, List hosts) { - boolean hostIdParamNull = hosts.get(0).getHostId() == null; - List hostIds; - if (hostIdParamNull) { - // 需要把ip查询参数转换为基于hostId的查询参数 - Map ip2Hosts = stepInstanceService.computeStepHosts(stepInstance, HostDTO::toCloudIp); - hostIds = hosts.stream() - .map(host -> ip2Hosts.get(host.toCloudIp()).getHostId()) - .distinct() - .collect(Collectors.toList()); + ExecuteObject executeObject = executeObjectTask.getExecuteObject(); + if (stepInstance.isSupportExecuteObjectFeature()) { + resp = logResource.getScriptLogByExecuteObjectId(taskCreateDateStr, + stepInstanceId, actualExecuteCount, executeObject.getId(), batch); } else { - hostIds = hosts.stream() - .map(HostDTO::getHostId) - .distinct() - .collect(Collectors.toList()); + // 兼容hostId查询 + resp = logResource.getScriptHostLogByHostId(taskCreateDateStr, + stepInstanceId, actualExecuteCount, executeObject.getHost().getHostId(), batch); } - return hostIds; + return convertToScriptExecuteObjectLogContent(stepInstance, executeCount, resp.getData(), executeObjectTask); } @Override - public FileIpLogContent getFileIpLogContent(long stepInstanceId, int executeCount, Integer batch, HostDTO host, - Integer mode) { - StepInstanceDTO stepInstance = taskInstanceService.getStepInstanceDetail(stepInstanceId); - // 如果存在重试，那么该ip可能是之前已经执行过的，查询日志的时候需要获取到对应的executeCount - int actualExecuteCount = executeCount; - AgentTaskDTO agentTask = fileAgentTaskService.getAgentTaskByHost(stepInstance, executeCount, batch, - FileTaskModeEnum.getFileTaskMode(mode), host); - if (agentTask == null) { - return null; + public List batchGetScriptExecuteObjectLogContent( + String jobCreateDateStr, + StepInstanceBaseDTO stepInstance, + int executeCount, + Integer batch, + List executeObjectCompositeKeys + ) { + if (CollectionUtils.isEmpty(executeObjectCompositeKeys)) { + return Collections.emptyList(); } - if (executeCount > 0 && agentTask.getActualExecuteCount() != null) { - actualExecuteCount = agentTask.getActualExecuteCount(); - } else if (agentTask.getStatus() == AgentTaskStatusEnum.LAST_SUCCESS) { - // 兼容历史数据 - actualExecuteCount = scriptAgentTaskService.getActualSuccessExecuteCount(stepInstanceId, host.toCloudIp()); + List queryExecuteObjects = + stepInstanceService.findExecuteObjectByCompositeKeys(stepInstance, executeObjectCompositeKeys); + + List logList = queryScriptLogsFromLogsvr( + jobCreateDateStr, + stepInstance, + executeCount, + batch, + queryExecuteObjects + ); + if (CollectionUtils.isEmpty(logList)) { + return Collections.emptyList(); } - String taskCreateDateStr = DateUtils.formatUnixTimestamp(stepInstance.getCreateTime(), ChronoUnit.MILLIS, - "yyyy_MM_dd", ZoneId.of("UTC")); - InternalResponse resp; - if (agentTask.getHostId() != null) { - resp = logServiceResourceClient.getFileHostLogByHostId(taskCreateDateStr, - stepInstanceId, actualExecuteCount, agentTask.getHostId(), mode, batch); + if (stepInstance.isSupportExecuteObjectFeature()) { + Map executeObjectMap = queryExecuteObjects.stream() + .collect(Collectors.toMap(ExecuteObject::getId, executeObject -> executeObject, + (oldValue, newValue) -> newValue)); + return logList.stream().map(logDTO -> { + String scriptContent = logDTO.getScriptLog() != null ? logDTO.getScriptLog().getContent() : ""; + ExecuteObject executeObject = executeObjectMap.get(logDTO.getExecuteObjectId()); + if (executeObject == null) { + log.warn("Can not find log execute object by executeObjectId : {}", logDTO.getExecuteObjectId()); + if (log.isDebugEnabled()) { + log.debug("Log: {}", JsonUtils.toJson(logDTO)); + } + } + return new ScriptExecuteObjectLogContent(logDTO.getStepInstanceId(), logDTO.getExecuteCount(), + executeObject, scriptContent, true); + }).collect(Collectors.toList()); } else { - // 兼容ip查询 - resp = logServiceResourceClient.getFileHostLogByIp(taskCreateDateStr, - stepInstanceId, actualExecuteCount, agentTask.getCloudIp(), mode, batch); + Map executeObjectMap = queryExecuteObjects.stream() + .collect(Collectors.toMap( + executeObject -> executeObject.getHost().getHostId(), executeObject -> executeObject, + (oldValue, newValue) -> newValue)); + return logList.stream().map(logDTO -> { + String scriptContent = logDTO.getScriptLog() != null ? logDTO.getScriptLog().getContent() : ""; + ExecuteObject executeObject = executeObjectMap.get(logDTO.getHostId()); + if (executeObject == null) { + log.warn("Can not find log execute object by hostId : {}", logDTO.getHostId()); + if (log.isDebugEnabled()) { + log.debug("Log: {}", JsonUtils.toJson(logDTO)); + } + } + return new ScriptExecuteObjectLogContent(logDTO.getStepInstanceId(), logDTO.getExecuteCount(), + executeObject, scriptContent, true); + }).collect(Collectors.toList()); } + } + + private List queryScriptLogsFromLogsvr(String jobCreateDateStr, + StepInstanceBaseDTO stepInstance, + int executeCount, + Integer batch, + List queryExecuteObjects) { + List logList = new ArrayList<>(); + + List executeObjectTaskList = + scriptExecuteObjectTaskService.getTaskByExecuteObjectIds( + stepInstance, + executeCount, + batch, + queryExecuteObjects.stream().map(ExecuteObject::getId).collect(Collectors.toList()) + ); + long stepInstanceId = stepInstance.getId(); + // 根据真实执行次数对执行对象进行分组 + Map> actualExecuteCountObjMap = executeObjectTaskList.stream() + .collect(Collectors.groupingBy(ExecuteObjectTask::getActualExecuteCount)); + + for (Map.Entry> entry : actualExecuteCountObjMap.entrySet()) { + Integer actualExecuteCount = entry.getKey(); + List executeObjectTasks = entry.getValue(); + if (log.isDebugEnabled()) { + log.debug( + "actualExecuteCount={},executeObjectTasks.size={}", + actualExecuteCount, + executeObjectTasks.size() + ); + } + if (CollectionUtils.isEmpty(executeObjectTasks)) { + continue; + } + List executeObjectIds = executeObjectTasks.stream() + .map(ExecuteObjectTask::getExecuteObjectId) + .collect(Collectors.toList()); + List actualExecuteLogs = queryScriptLogsForExecuteOnce( + jobCreateDateStr, + stepInstanceId, + actualExecuteCount, + batch, + executeObjectIds + ); + logList.addAll(actualExecuteLogs); + } + return logList; + } + /** + * 查询单次执行对应的脚本日志 + * + * @param jobCreateDateStr 任务创建日期 + * @param stepInstanceId 步骤实例ID + * @param actualExecuteCount 实际执行次数 + * @param batch 批次 + * @param executeObjectIds 执行目标ID列表 + * @return 单次执行对应的脚本日志 + */ + private List queryScriptLogsForExecuteOnce(String jobCreateDateStr, + long stepInstanceId, + int actualExecuteCount, + Integer batch, + List executeObjectIds) { + ServiceScriptLogQueryRequest query = new ServiceScriptLogQueryRequest(); + query.setBatch(batch); + query.setExecuteObjectIds(executeObjectIds); + + InternalResponse> resp = logResource.listScriptExecuteObjectLogs( + jobCreateDateStr, + stepInstanceId, + actualExecuteCount, + query + ); if (!resp.isSuccess()) { - log.error("Get file log content by ip error, stepInstanceId={}, executeCount={}, batch={}, host={}", - stepInstanceId, actualExecuteCount, batch, host); + log.error( + "Get script log content by execute objects error, " + + "stepInstanceId={}, actualExecuteCount={}, batch={}, executeObjectIds: {}", + stepInstanceId, + actualExecuteCount, + batch, + executeObjectIds + ); throw new InternalException(resp.getCode()); } - List fileTaskLogs = (resp.getData() == null) ? null : resp.getData().getFileTaskLogs(); - AgentTaskStatusEnum agentTaskStatus = agentTask.getStatus(); - boolean isFinished = agentTaskStatus.isFinished() || isAllFileTasksFinished(fileTaskLogs); - return new FileIpLogContent(stepInstanceId, executeCount, null, fileTaskLogs, isFinished); + List logDTOList = resp.getData(); + if (log.isDebugEnabled()) { + log.debug( + "Get {} script log content by execute objects, " + + "stepInstanceId={}, actualExecuteCount={}, batch={}, executeObjectIds: {}", + logDTOList == null ? 0 : logDTOList.size(), + stepInstanceId, + actualExecuteCount, + batch, + executeObjectIds + ); + } + return logDTOList; + } + + @Override + public FileExecuteObjectLogContent getFileExecuteObjectLogContent( + StepInstanceBaseDTO stepInstance, + int executeCount, + Integer batch, + ExecuteObjectCompositeKey executeObjectCompositeKey, + Integer mode + ) { + ExecuteObjectTask executeObjectTask = fileExecuteObjectTaskService.getTaskByExecuteObjectCompositeKey( + stepInstance, executeCount, batch, FileTaskModeEnum.getFileTaskMode(mode), executeObjectCompositeKey); + return getFileExecuteObjectLogContent(stepInstance, executeCount, batch, executeObjectTask); } private boolean isAllFileTasksFinished(List fileTaskLogs) { if (CollectionUtils.isEmpty(fileTaskLogs)) { return false; } - return fileTaskLogs.stream().noneMatch(this::isFileTaskNotFinished); + return fileTaskLogs.stream().allMatch(this::isFileTaskFinished); } - private boolean isFileTaskNotFinished(ServiceFileTaskLogDTO fileTaskLog) { + private boolean isFileTaskFinished(ServiceFileTaskLogDTO fileTaskLog) { FileDistStatusEnum status = FileDistStatusEnum.getFileDistStatus(fileTaskLog.getStatus()); - if (status == null) { - return true; + return FileDistStatusEnum.isFinishedStatus(status); + } + + @Override + public FileExecuteObjectLogContent getFileExecuteObjectLogContent(StepInstanceBaseDTO stepInstance, + int executeCount, + Integer batch, + ExecuteObjectTask executeObjectTask) { + // 如果存在重试，那么该任务可能是之前已经执行过的，查询日志的时候需要获取到对应的executeCount + int actualExecuteCount = executeCount; + if (executeCount > 0 && executeObjectTask.getActualExecuteCount() != null) { + actualExecuteCount = executeObjectTask.getActualExecuteCount(); + } + + String taskCreateDateStr = buildTaskCreateDateStr(stepInstance); + long stepInstanceId = stepInstance.getId(); + InternalResponse resp; + ExecuteObject executeObject = executeObjectTask.getExecuteObject(); + if (stepInstance.isSupportExecuteObjectFeature()) { + resp = logResource.getFileLogByExecuteObjectId(taskCreateDateStr, stepInstanceId, actualExecuteCount, + executeObject.getId(), executeObjectTask.getFileTaskMode().getValue(), batch); + } else { + // 兼容hostId查询 + resp = logResource.getFileHostLogByHostId(taskCreateDateStr, stepInstanceId, actualExecuteCount, + executeObject.getHost().getHostId(), executeObjectTask.getFileTaskMode().getValue(), batch); + } + if (resp == null || resp.getData() == null) { + log.warn("Get file execute object log, api response is empty"); + return null; + } + List fileTaskLogs = resp.getData().getFileTaskLogs(); + if (CollectionUtils.isEmpty(fileTaskLogs)) { + log.warn("Get file execute object log, file task logs are empty"); + return null; } - return status == FileDistStatusEnum.DOWNLOADING || status == FileDistStatusEnum.UPLOADING - || status == FileDistStatusEnum.PULLING || status == FileDistStatusEnum.WAITING; + ExecuteObjectTaskStatusEnum executeObjectTaskStatus = executeObjectTask.getStatus(); + boolean isFinished = executeObjectTaskStatus.isFinished() || isAllFileTasksFinished(fileTaskLogs); + return new FileExecuteObjectLogContent(stepInstanceId, executeCount, executeObject, + batchConvertToAtomicFileTaskLog(stepInstance, fileTaskLogs), isFinished); } @Override - public List getFileLogContentByTaskIds(long stepInstanceId, int executeCount, Integer batch, - List taskIds) { - StepInstanceBaseDTO stepInstance = taskInstanceService.getBaseStepInstance(stepInstanceId); - String taskCreateDateStr = DateUtils.formatUnixTimestamp(stepInstance.getCreateTime(), ChronoUnit.MILLIS, - "yyyy_MM_dd", ZoneId.of("UTC")); - InternalResponse resp = logServiceResourceClient.listFileHostLogsByTaskIds( + public List getAtomicFileTaskLogByTaskIds(long taskInstanceId, + long stepInstanceId, + int executeCount, + Integer batch, + List taskIds) { + StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstance(taskInstanceId, stepInstanceId); + String taskCreateDateStr = buildTaskCreateDateStr(stepInstance); + InternalResponse> resp = logResource.listTaskFileLogsByTaskIds( taskCreateDateStr, stepInstanceId, executeCount, batch, taskIds); if (!resp.isSuccess()) { log.error("Get file log content by ids error, stepInstanceId={}, executeCount={}, batch={}, taskIds={}", stepInstanceId, executeCount, batch, taskIds); throw new InternalException(resp.getCode()); } - if (resp.getData() == null) { + List fileTaskLogs = resp.getData(); + return batchConvertToAtomicFileTaskLog(stepInstance, fileTaskLogs); + } + + private List batchConvertToAtomicFileTaskLog( + StepInstanceBaseDTO stepInstance, + List fileTaskLogs + ) { + if (CollectionUtils.isEmpty(fileTaskLogs)) { return Collections.emptyList(); } - return resp.getData().getFileTaskLogs(); + + if (stepInstance.isSupportExecuteObjectFeature()) { + Map executeObjectMap = + stepInstanceService.computeStepExecuteObjects(stepInstance, ExecuteObject::getId); + return batchConvertToAtomicFileTaskLog( + executeObjectMap, + fileTaskLogs, + ServiceFileTaskLogDTO::getSrcExecuteObjectId, + ServiceFileTaskLogDTO::getDestExecuteObjectId); + } else { + // 兼容老版本不支持执行对象的数据 + Map hostExecuteObjects = stepInstanceService.computeStepExecuteObjects( + stepInstance, executeObject -> executeObject.getHost().getHostId()); + return batchConvertToAtomicFileTaskLog( + hostExecuteObjects, + fileTaskLogs, + ServiceFileTaskLogDTO::getSrcHostId, + ServiceFileTaskLogDTO::getDestHostId); + } + } + + private List batchConvertToAtomicFileTaskLog( + Map executeObjectMap, + List fileTaskLogs, + Function srcKeyExtractor, + Function destKeyExtractor + ) { + if (CollectionUtils.isEmpty(fileTaskLogs)) { + return Collections.emptyList(); + } + + return fileTaskLogs.stream() + .map(fileTaskLog -> + AtomicFileTaskLog.fromServiceExecuteObjectLogDTO( + fileTaskLog, + fileTaskLogParam -> { + K srcKey = srcKeyExtractor.apply(fileTaskLogParam); + if (srcKey == null) { + return null; + } + ExecuteObject executeObject = executeObjectMap.get(srcKey); + if (executeObject == null) { + log.warn("Can not find src execute object by key : {}", srcKey); + } + return executeObject; + }, + fileTaskLogParam -> { + K destKey = destKeyExtractor.apply(fileTaskLog); + if (destKey == null) { + return null; + } + ExecuteObject executeObject = executeObjectMap.get(destKey); + if (executeObject == null) { + log.warn("Can not find dest execute object by key : {}", destKey); + } + return executeObject; + } + )) + .collect(Collectors.toList()); } @Override - public List batchGetFileSourceIpLogContent(long stepInstanceId, - int executeCount, - Integer batch) { - StepInstanceBaseDTO stepInstance = taskInstanceService.getBaseStepInstance(stepInstanceId); - String taskCreateDateStr = DateUtils.formatUnixTimestamp(stepInstance.getCreateTime(), ChronoUnit.MILLIS, - "yyyy_MM_dd", ZoneId.of("UTC")); - InternalResponse> resp = logServiceResourceClient.listFileHostLogs( - taskCreateDateStr, stepInstanceId, executeCount, batch, FileDistModeEnum.UPLOAD.getValue(), null, null); - if (!resp.isSuccess()) { - log.error("Get file source log content error, stepInstanceId={}, executeCount={}, batch={}", - stepInstanceId, executeCount, batch); + public List batchGetFileSourceExecuteObjectLogContent(long taskInstanceId, + long stepInstanceId, + int executeCount, + Integer batch) { + return batchGetFileExecuteObjectLogContent(taskInstanceId, stepInstanceId, executeCount, batch, + FileTaskModeEnum.UPLOAD, null); + } + + private List batchConvertToFileExecuteObjectLogContent( + StepInstanceDTO stepInstance, + List executeObjectLogs) { + + if (CollectionUtils.isEmpty(executeObjectLogs)) { return Collections.emptyList(); } - return resp.getData(); + if (stepInstance.isSupportExecuteObjectFeature()) { + Map executeObjectMap = + stepInstanceService.computeStepExecuteObjects(stepInstance, ExecuteObject::getId); + + return executeObjectLogs.stream() + .map(executeObjectLog -> { + FileExecuteObjectLogContent executeObjectLogContent = new FileExecuteObjectLogContent(); + executeObjectLogContent.setStepInstanceId(executeObjectLog.getStepInstanceId()); + executeObjectLogContent.setExecuteCount(executeObjectLog.getExecuteCount()); + executeObjectLogContent.setFileTaskLogs( + batchConvertToAtomicFileTaskLog( + executeObjectMap, + executeObjectLog.getFileTaskLogs(), + ServiceFileTaskLogDTO::getSrcExecuteObjectId, + ServiceFileTaskLogDTO::getDestExecuteObjectId)); + executeObjectLogContent.setExecuteObject( + executeObjectMap.get(executeObjectLog.getExecuteObjectId())); + return executeObjectLogContent; + }) + .collect(Collectors.toList()); + } else { + // 兼容老版本不支持执行对象的数据 + Map hostExecuteObjects = stepInstanceService.computeStepExecuteObjects( + stepInstance, executeObject -> executeObject.getHost().getHostId()); + return executeObjectLogs.stream() + .map(executeObjectLog -> { + FileExecuteObjectLogContent executeObjectLogContent = new FileExecuteObjectLogContent(); + executeObjectLogContent.setStepInstanceId(executeObjectLog.getStepInstanceId()); + executeObjectLogContent.setExecuteCount(executeObjectLog.getExecuteCount()); + executeObjectLogContent.setFileTaskLogs( + batchConvertToAtomicFileTaskLog( + hostExecuteObjects, + executeObjectLog.getFileTaskLogs(), + ServiceFileTaskLogDTO::getSrcHostId, + ServiceFileTaskLogDTO::getDestHostId)); + executeObjectLogContent.setExecuteObject( + hostExecuteObjects.get(executeObjectLog.getHostId())); + return executeObjectLogContent; + }) + .collect(Collectors.toList()); + } } @Override - public ServiceHostLogsDTO batchGetFileIpLogContent(long stepInstanceId, int executeCount, Integer batch, - List hosts) { - StepInstanceBaseDTO stepInstance = taskInstanceService.getBaseStepInstance(stepInstanceId); - String taskCreateDateStr = DateUtils.formatUnixTimestamp(stepInstance.getCreateTime(), ChronoUnit.MILLIS, - "yyyy_MM_dd", ZoneId.of("UTC")); + public List batchGetFileExecuteObjectLogContent( + long taskInstanceId, + long stepInstanceId, + int executeCount, + Integer batch, + FileTaskModeEnum mode, + List executeObjectCompositeKeys) { + + StepInstanceDTO stepInstance = stepInstanceService.getStepInstanceDetail(taskInstanceId, stepInstanceId); + String taskCreateDateStr = buildTaskCreateDateStr(stepInstance); ServiceFileLogQueryRequest request = new ServiceFileLogQueryRequest(); - request.setStepInstanceId(stepInstanceId); + request.setStepInstanceId(stepInstance.getId()); request.setExecuteCount(executeCount); request.setBatch(batch); request.setJobCreateDate(taskCreateDateStr); - - if (isQueryByHostIdCondition(stepInstance)) { - request.setHostIds(buildHostIdQueryCondition(stepInstance, hosts)); - } else { - request.setIps(hosts.stream().map(HostDTO::toCloudIp).distinct().collect(Collectors.toList())); + if (mode != null) { + request.setMode(mode.getValue()); } + if (CollectionUtils.isNotEmpty(executeObjectCompositeKeys)) { + List queryExecuteObjects = + stepInstanceService.findExecuteObjectByCompositeKeys(stepInstance, executeObjectCompositeKeys); + if (stepInstance.isSupportExecuteObjectFeature()) { + List executeObjectId = queryExecuteObjects.stream() + .map(ExecuteObject::getId).collect(Collectors.toList()); + request.setExecuteObjectIds(executeObjectId); + } else { + // 兼容 hostId 查询 + List hostIds = queryExecuteObjects.stream() + .map(executeObject -> executeObject.getHost().getHostId()).collect(Collectors.toList()); + request.setHostIds(hostIds); + } + } + - InternalResponse resp = logServiceResourceClient.listFileHostLogs(request); + InternalResponse> resp = logResource.listFileExecuteObjectLogs( + taskCreateDateStr, stepInstanceId, executeCount, request); if (!resp.isSuccess()) { - log.error("Get file log content error, request={}", request); - return null; + log.error("Get file log content error, stepInstanceId={}, executeCount={}, batch={}", + stepInstanceId, executeCount, batch); + return Collections.emptyList(); } - return resp.getData(); + + List executeObjectLogs = resp.getData(); + + return batchConvertToFileExecuteObjectLogContent(stepInstance, executeObjectLogs); } @Override - public List getIpsByContentKeyword(long stepInstanceId, int executeCount, Integer batch, - String keyword) { - StepInstanceBaseDTO stepInstance = taskInstanceService.getBaseStepInstance(stepInstanceId); - String taskCreateDateStr = DateUtils.formatUnixTimestamp(stepInstance.getCreateTime(), ChronoUnit.MILLIS, - "yyyy_MM_dd", ZoneId.of("UTC")); - InternalResponse> resp = logServiceResourceClient.getIpsByKeyword(taskCreateDateStr, - stepInstanceId, executeCount, batch, keyword); - - if (!resp.isSuccess()) { - log.error("Search ips by keyword error, stepInstanceId={}, executeCount={}, keyword={}", stepInstanceId, - executeCount, keyword); - throw new InternalException(resp.getCode()); + public List getExecuteObjectsCompositeKeysByContentKeyword( + StepInstanceBaseDTO stepInstance, + int executeCount, + Integer batch, + String keyword + ) { + String taskCreateDateStr = buildTaskCreateDateStr(stepInstance); + long stepInstanceId = stepInstance.getId(); + if (stepInstance.isSupportExecuteObjectFeature()) { + InternalResponse> resp = logResource.queryExecuteObjectsByLogKeyword(taskCreateDateStr, + stepInstanceId, executeCount, batch, keyword); + if (!resp.isSuccess()) { + log.error("Search execute object by keyword error, stepInstanceId={}, executeCount={}, keyword={}", + stepInstanceId, executeCount, keyword); + throw new InternalException(resp.getCode()); + } + List matchExecuteObjectIds = resp.getData(); + if (CollectionUtils.isEmpty(matchExecuteObjectIds)) { + return Collections.emptyList(); + } + return matchExecuteObjectIds.stream() + .map(ExecuteObjectCompositeKey::ofExecuteObjectId) + .collect(Collectors.toList()); + } else { + InternalResponse> resp = logResource.queryHostsByLogKeyword(taskCreateDateStr, + stepInstanceId, executeCount, batch, keyword); + if (!resp.isSuccess()) { + log.error("Search host by keyword error, stepInstanceId={}, executeCount={}, keyword={}", + stepInstanceId, executeCount, keyword); + throw new InternalException(resp.getCode()); + } + List matchHosts = resp.getData(); + if (CollectionUtils.isEmpty(matchHosts)) { + return Collections.emptyList(); + } + return matchHosts.stream() + .map(host -> ExecuteObjectCompositeKey.ofHostId(host.getHostId())) + .collect(Collectors.toList()); } - return resp.getData(); } - @Override - public void writeFileLogWithTimestamp(long jobCreateTime, - long stepInstanceId, - int executeCount, - Integer batch, - HostDTO host, - ServiceHostLogDTO executionLog, - Long logTimeInMillSeconds) { - - if (executionLog == null || CollectionUtils.isEmpty(executionLog.getFileTaskLogs())) { + public void writeFileLogsWithTimestamp(TaskInstanceDTO taskInstance, + List executeObjectFileLogs, + Long logTimeInMillSeconds) { + + if (CollectionUtils.isEmpty(executeObjectFileLogs)) { return; } + ServiceBatchSaveLogRequest request = new ServiceBatchSaveLogRequest(); + request.setAppId(taskInstance.getAppId()); + request.setCronTaskId(taskInstance.getCronTaskId()); + request.setAppCode(taskInstance.getAppCode()); + request.setJobCreateDate(DateUtils.formatUnixTimestamp( + taskInstance.getCreateTime(), + ChronoUnit.MILLIS, + "yyyy_MM_dd", + ZoneId.of("UTC") + )); + request.setLogType(LogTypeEnum.FILE.getValue()); + String logDateTime = "["; if (logTimeInMillSeconds != null) { - logDateTime += DateUtils.formatUnixTimestamp(logTimeInMillSeconds, ChronoUnit.MILLIS, "yyyy-MM-dd " + - "HH:mm:ss", ZoneId.systemDefault()); + logDateTime += DateUtils.formatUnixTimestamp(logTimeInMillSeconds, ChronoUnit.MILLIS, + "yyyy-MM-dd HH:mm:ss", ZoneId.systemDefault()); } else { - logDateTime += DateUtils.formatUnixTimestamp(System.currentTimeMillis(), ChronoUnit.MILLIS, "yyyy-MM-dd " + - "HH:mm:ss", ZoneId.systemDefault()); + logDateTime += DateUtils.formatUnixTimestamp(System.currentTimeMillis(), ChronoUnit.MILLIS, + "yyyy-MM-dd HH:mm:ss", ZoneId.systemDefault()); } logDateTime += "] "; - for (ServiceFileTaskLogDTO fileTaskLog : executionLog.getFileTaskLogs()) { - fileTaskLog.setContent(logDateTime + fileTaskLog.getContent() + "\n"); + for (ServiceExecuteObjectLogDTO executeObjectFileLog : executeObjectFileLogs) { + for (ServiceFileTaskLogDTO fileTaskLog : executeObjectFileLog.getFileTaskLogs()) { + if (StringUtils.isBlank(fileTaskLog.getContent())) { + continue; + } + fileTaskLog.setContent(logDateTime + fileTaskLog.getContent() + "\n"); + } + request.setLogs(executeObjectFileLogs); } - ServiceSaveLogRequest request = new ServiceSaveLogRequest(); - request.setLogType(LogTypeEnum.FILE.getValue()); - request.setStepInstanceId(stepInstanceId); - request.setExecuteCount(executeCount); - request.setBatch(batch); - request.setIp(host.getIp()); - request.setHostId(host.getHostId()); - request.setJobCreateDate(DateUtils.formatUnixTimestamp(jobCreateTime, ChronoUnit.MILLIS, "yyyy_MM_dd", - ZoneId.of("UTC"))); - request.setFileTaskLogs(executionLog.getFileTaskLogs()); - logServiceResourceClient.saveLog(request); + saveLogsAndRecordLogSizeMetrics(request); + } + + private InternalResponse saveLogsAndRecordLogSizeMetrics(ServiceBatchSaveLogRequest request) { + InternalResponse resp = logResource.saveLogs(request); + logSampler.tryToRecordLogSizeMetrics(request); + return resp; } @Override - public void writeFileLogs(long jobCreateTime, List fileLogs) { - if (CollectionUtils.isEmpty(fileLogs)) { - return; + public void writeFileLogs(TaskInstanceDTO taskInstance, List executeObjectLogs) { + writeFileLogsWithTimestamp(taskInstance, executeObjectLogs, System.currentTimeMillis()); + } + + @Override + public ServiceFileTaskLogDTO buildUploadServiceFileTaskLogDTO(StepInstanceDTO stepInstance, + JobFile srcFile, + FileDistStatusEnum status, + String size, + String speed, + String process, + String content) { + return buildUploadServiceFileTaskLogDTO( + stepInstance, + srcFile.getFileType(), + srcFile.getStandardFilePath(), + srcFile.getDisplayFilePath(), + srcFile.getExecuteObject(), + status, + size, + speed, + process, + content + ); + } + + @Override + public ServiceFileTaskLogDTO buildUploadServiceFileTaskLogDTO(StepInstanceDTO stepInstance, + TaskFileTypeEnum fileType, + String srcFilePath, + String displaySrcFilePath, + ExecuteObject executeObject, + FileDistStatusEnum status, + String size, + String speed, + String process, + String content) { + if (stepInstance.isSupportExecuteObjectFeature()) { + return new ServiceFileTaskLogDTO( + FileDistModeEnum.UPLOAD.getValue(), + null, + null, + executeObject.getId(), + fileType.getType(), + srcFilePath, + displaySrcFilePath, + size, + status.getValue(), + status.getName(), + speed, + process, + content + ); + } else { + HostDTO sourceHost = executeObject.getHost(); + return new ServiceFileTaskLogDTO( + FileDistModeEnum.UPLOAD.getValue(), + null, + null, + null, + null, + sourceHost.getHostId(), + sourceHost.toCloudIp(), + sourceHost.toCloudIpv6(), + fileType.getType(), + srcFilePath, + displaySrcFilePath, + size, + status.getValue(), + status.getName(), + speed, + process, + content + ); + } + } + + @Override + public ServiceFileTaskLogDTO buildDownloadServiceFileTaskLogDTO(StepInstanceDTO stepInstance, + JobFile srcFile, + ExecuteObject targetExecuteObject, + String targetPath, + FileDistStatusEnum status, + String size, + String speed, + String process, + String content) { + if (stepInstance.isSupportExecuteObjectFeature()) { + return new ServiceFileTaskLogDTO( + FileDistModeEnum.DOWNLOAD.getValue(), + targetExecuteObject.getId(), + targetPath, + srcFile.getExecuteObject().getId(), + srcFile.getFileType().getType(), + srcFile.getStandardFilePath(), + srcFile.getDisplayFilePath(), + size, + status.getValue(), + status.getName(), + speed, + process, + content + ); + } else { + HostDTO sourceHost = srcFile.getExecuteObject().getHost(); + HostDTO targetHost = targetExecuteObject.getHost(); + return new ServiceFileTaskLogDTO( + FileDistModeEnum.DOWNLOAD.getValue(), + targetHost.getHostId(), + targetHost.toCloudIp(), + targetHost.toCloudIpv6(), + targetPath, + sourceHost.getHostId(), + sourceHost.toCloudIp(), + sourceHost.toCloudIpv6(), + srcFile.getFileType().getType(), + srcFile.getStandardFilePath(), + srcFile.getDisplayFilePath(), + size, + status.getValue(), + status.getName(), + speed, + process, + content + ); } + } - String logDateTime = "[" + DateUtils.formatUnixTimestamp(System.currentTimeMillis(), ChronoUnit.MILLIS, - "yyyy-MM-dd HH:mm:ss", ZoneId.systemDefault()) + "]"; - fileLogs.forEach(fileLog -> { - if (CollectionUtils.isNotEmpty(fileLog.getFileTaskLogs())) { - fileLog.getFileTaskLogs().stream() - .filter(fileTaskLog -> StringUtils.isNotEmpty(fileTaskLog.getContent())) - .forEach(fileTaskLog -> fileTaskLog.setContent(logDateTime + fileTaskLog.getContent() + "\n")); + @Override + public void addFileTaskLog(StepInstanceBaseDTO stepInstance, + Map executeObjectLogs, + ExecuteObject executeObject, + ServiceFileTaskLogDTO fileTaskLog) { + + ExecuteObjectCompositeKey executeObjectCompositeKey = + executeObject.toExecuteObjectCompositeKey(ExecuteObjectCompositeKey.CompositeKeyType.RESOURCE_ID); + ServiceExecuteObjectLogDTO executeObjectLogDTO = executeObjectLogs.get(executeObjectCompositeKey); + if (executeObjectLogDTO == null) { + executeObjectLogDTO = new ServiceExecuteObjectLogDTO(); + executeObjectLogDTO.setStepInstanceId(stepInstance.getId()); + executeObjectLogDTO.setExecuteCount(stepInstance.getExecuteCount()); + executeObjectLogDTO.setBatch(stepInstance.getBatch()); + if (stepInstance.isSupportExecuteObjectFeature()) { + executeObjectLogDTO.setExecuteObjectId(executeObject.getId()); + } else { + executeObjectLogDTO.setHostId(executeObject.getHost().getHostId()); + executeObjectLogDTO.setCloudIp(executeObject.getHost().toCloudIp()); } - }); + executeObjectLogs.put(executeObjectCompositeKey, executeObjectLogDTO); + } - ServiceBatchSaveLogRequest request = new ServiceBatchSaveLogRequest(); - request.setJobCreateDate(DateUtils.formatUnixTimestamp(jobCreateTime, ChronoUnit.MILLIS, "yyyy_MM_dd", - ZoneId.of("UTC"))); - request.setLogs(fileLogs); - request.setLogType(LogTypeEnum.FILE.getValue()); - logServiceResourceClient.saveLogs(request); + executeObjectLogDTO.addFileTaskLog(fileTaskLog); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/NotifyServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/NotifyServiceImpl.java index 4ad1c9a8e4..dfc6a32a0c 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/NotifyServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/NotifyServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,13 +27,10 @@ import com.tencent.bk.job.common.i18n.locale.LocaleUtils; import com.tencent.bk.job.common.i18n.service.MessageI18nService; import com.tencent.bk.job.common.model.dto.UserRoleInfoDTO; +import com.tencent.bk.job.common.service.config.JobCommonConfig; import com.tencent.bk.job.common.util.JobContextUtil; import com.tencent.bk.job.common.util.date.DateUtils; -import com.tencent.bk.job.execute.client.ServiceNotificationResourceClient; -import com.tencent.bk.job.execute.client.ServiceUserResourceClient; -import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; import com.tencent.bk.job.execute.common.constants.TaskStartupModeEnum; -import com.tencent.bk.job.execute.config.JobExecuteConfig; import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; import com.tencent.bk.job.execute.model.NotifyDTO; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; @@ -41,15 +38,18 @@ import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.model.TaskNotifyDTO; import com.tencent.bk.job.execute.service.ApplicationService; -import com.tencent.bk.job.execute.service.FileAgentTaskService; +import com.tencent.bk.job.execute.service.FileExecuteObjectTaskService; import com.tencent.bk.job.execute.service.NotifyService; -import com.tencent.bk.job.execute.service.ScriptAgentTaskService; +import com.tencent.bk.job.execute.service.ScriptExecuteObjectTaskService; +import com.tencent.bk.job.execute.service.StepInstanceService; import com.tencent.bk.job.execute.service.TaskInstanceService; -import com.tencent.bk.job.manage.common.consts.notify.ExecuteStatusEnum; -import com.tencent.bk.job.manage.common.consts.notify.NotifyConsts; -import com.tencent.bk.job.manage.common.consts.notify.ResourceTypeEnum; -import com.tencent.bk.job.manage.common.consts.notify.TriggerTypeEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskStepTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.notify.ExecuteStatusEnum; +import com.tencent.bk.job.manage.api.common.constants.notify.NotifyConsts; +import com.tencent.bk.job.manage.api.common.constants.notify.ResourceTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.notify.TriggerTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskStepTypeEnum; +import com.tencent.bk.job.manage.api.inner.ServiceNotificationResource; +import com.tencent.bk.job.manage.api.inner.ServiceUserResource; import com.tencent.bk.job.manage.model.inner.ServiceNotificationTriggerDTO; import com.tencent.bk.job.manage.model.inner.ServiceTemplateNotificationDTO; import com.tencent.bk.job.manage.model.inner.ServiceTriggerTemplateNotificationDTO; @@ -76,7 +76,7 @@ import java.util.Set; @Slf4j -@Service +@Service("jobExecuteNotifyService") public class NotifyServiceImpl implements NotifyService { private static final Properties templatePropsEN = new Properties(); private static final Properties templatePropsZH = new Properties(); @@ -85,35 +85,39 @@ public class NotifyServiceImpl implements NotifyService { loadNotifyTemplate(); } - private final JobExecuteConfig jobExecuteConfig; - private final ServiceNotificationResourceClient notificationResourceClient; - private final ServiceUserResourceClient userResourceClient; + private final JobCommonConfig jobCommonConfig; + private final ServiceNotificationResource notificationResource; + private final ServiceUserResource userResource; private final ApplicationService applicationService; private final TaskInstanceService taskInstanceService; private final MessageI18nService i18nService; - private final ScriptAgentTaskService scriptAgentTaskService; - private final FileAgentTaskService fileAgentTaskService; + private final ScriptExecuteObjectTaskService scriptExecuteObjectTaskService; + + private final FileExecuteObjectTaskService fileExecuteObjectTaskService; private final TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher; + private final StepInstanceService stepInstanceService; @Autowired - public NotifyServiceImpl(JobExecuteConfig jobExecuteConfig, - ServiceNotificationResourceClient notificationResourceClient, - ServiceUserResourceClient userResourceClient, + public NotifyServiceImpl(JobCommonConfig jobCommonConfig, + ServiceNotificationResource notificationResource, + ServiceUserResource userResource, ApplicationService applicationService, TaskInstanceService taskInstanceService, MessageI18nService i18nService, - ScriptAgentTaskService scriptAgentTaskService, - FileAgentTaskService fileAgentTaskService, - TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher) { - this.jobExecuteConfig = jobExecuteConfig; - this.notificationResourceClient = notificationResourceClient; - this.userResourceClient = userResourceClient; + ScriptExecuteObjectTaskService scriptExecuteObjectTaskService, + FileExecuteObjectTaskService fileExecuteObjectTaskService, + TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher, + StepInstanceService stepInstanceService) { + this.jobCommonConfig = jobCommonConfig; + this.notificationResource = notificationResource; + this.userResource = userResource; this.applicationService = applicationService; this.taskInstanceService = taskInstanceService; this.i18nService = i18nService; - this.scriptAgentTaskService = scriptAgentTaskService; - this.fileAgentTaskService = fileAgentTaskService; + this.scriptExecuteObjectTaskService = scriptExecuteObjectTaskService; + this.fileExecuteObjectTaskService = fileExecuteObjectTaskService; this.taskExecuteMQEventDispatcher = taskExecuteMQEventDispatcher; + this.stepInstanceService = stepInstanceService; } private static void loadNotifyTemplate() { @@ -190,7 +194,7 @@ private ServiceNotificationTriggerDTO buildQueryTrigger(TaskNotifyDTO taskNotify trigger.setAppId(taskNotifyDTO.getAppId()); trigger.setTriggerUser(taskNotifyDTO.getOperator()); Integer startupMode = taskNotifyDTO.getStartupMode(); - if (startupMode.equals(TaskStartupModeEnum.NORMAL.getValue())) { + if (startupMode.equals(TaskStartupModeEnum.WEB.getValue())) { trigger.setTriggerType(TriggerTypeEnum.PAGE_EXECUTE.getType()); } else if (startupMode.equals(TaskStartupModeEnum.API.getValue())) { trigger.setTriggerType(TriggerTypeEnum.API_INVOKE.getType()); @@ -199,14 +203,16 @@ private ServiceNotificationTriggerDTO buildQueryTrigger(TaskNotifyDTO taskNotify } else { log.warn("Invalid startup mode!"); } + trigger.setResourceExecuteStatus(taskNotifyDTO.getResourceExecuteStatus()); trigger.setResourceType(taskNotifyDTO.getResourceType()); trigger.setResourceId(taskNotifyDTO.getResourceId()); + trigger.setCronTaskId(taskNotifyDTO.getCronTaskId()); return trigger; } private String buildJobExecuteDetailUrl(Long taskInstanceId) { - return jobExecuteConfig.getJobWebUrl() + "/api_execute/" + taskInstanceId; + return jobCommonConfig.getJobWebUrl() + "/api_execute/" + taskInstanceId; } @Override @@ -234,7 +240,7 @@ private void notifyCommon(TaskNotifyDTO taskNotifyDTO, ExecuteStatusEnum execute triggerTemplateNotificationDTO.setTriggerDTO(trigger); triggerTemplateNotificationDTO.setTemplateCode(getTemplateCodeByExecuteStatus(executeStatus)); triggerTemplateNotificationDTO.setVariablesMap(getTemplateVariablesMap(taskNotifyDTO, executeStatus)); - notificationResourceClient.triggerTemplateNotification(triggerTemplateNotificationDTO); + notificationResource.triggerTemplateNotification(triggerTemplateNotificationDTO); } private Map getTemplateVariablesMap(TaskNotifyDTO taskNotifyDTO, ExecuteStatusEnum executeStatus) { @@ -249,11 +255,12 @@ private Map getTemplateVariablesMap(TaskNotifyDTO taskNotifyDTO, TaskInstanceDTO taskInstanceDTO = taskInstanceService.getTaskInstance(taskNotifyDTO.getTaskInstanceId()); variablesMap.put("task.start_time", DateUtils.formatUnixTimestampWithZone(taskInstanceDTO.getStartTime(), ChronoUnit.MILLIS)); - List stepIdList = taskInstanceService.getTaskStepIdList(taskInstanceDTO.getId()); + List stepIdList = stepInstanceService.getTaskStepIdList(taskInstanceDTO.getId()); variablesMap.put("task.step.total_seq_cnt", "" + stepIdList.size()); long currentStepId = taskInstanceDTO.getCurrentStepInstanceId(); variablesMap.put("task.step.current_seq_id", "" + (stepIdList.indexOf(currentStepId) + 1)); - StepInstanceDTO stepInstanceDTO = taskInstanceService.getStepInstanceDetail(currentStepId); + StepInstanceDTO stepInstanceDTO = stepInstanceService.getStepInstanceDetail( + taskInstanceDTO.getId(), currentStepId); if (executeStatus == ExecuteStatusEnum.FAIL || executeStatus == ExecuteStatusEnum.SUCCESS) { if (stepInstanceDTO.getTotalTime() != null) { variablesMap.put("task.step.duration", "" + stepInstanceDTO.getTotalTime() / 1000.0); @@ -261,18 +268,24 @@ private Map getTemplateVariablesMap(TaskNotifyDTO taskNotifyDTO, if (taskInstanceDTO.getTotalTime() != null) { variablesMap.put("task.total_duration", "" + taskInstanceDTO.getTotalTime() / 1000.0); } - int totalTargetIpCount = stepInstanceDTO.getTargetServers().getIpList().size(); + int totalTargetIpCount = stepInstanceDTO.getTargetExecuteObjectCount(); if (executeStatus == ExecuteStatusEnum.SUCCESS) { variablesMap.put("task.step.success_cnt", "" + totalTargetIpCount); variablesMap.put("task.step.failed_cnt", "" + 0); } else { int successIpCount = 0; if (stepInstanceDTO.isScriptStep()) { - successIpCount = scriptAgentTaskService.getSuccessAgentTaskCount(stepInstanceDTO.getId(), - stepInstanceDTO.getExecuteCount()); + successIpCount = scriptExecuteObjectTaskService.getSuccessTaskCount( + stepInstanceDTO.getTaskInstanceId(), + stepInstanceDTO.getId(), + stepInstanceDTO.getExecuteCount() + ); } else if (stepInstanceDTO.isFileStep()) { - successIpCount = fileAgentTaskService.getSuccessAgentTaskCount(stepInstanceDTO.getId(), - stepInstanceDTO.getExecuteCount()); + successIpCount = fileExecuteObjectTaskService.getSuccessTaskCount( + stepInstanceDTO.getTaskInstanceId(), + stepInstanceDTO.getId(), + stepInstanceDTO.getExecuteCount() + ); } variablesMap.put("task.step.failed_cnt", String.valueOf(totalTargetIpCount - successIpCount)); variablesMap.put("task.step.success_cnt", String.valueOf(successIpCount)); @@ -299,11 +312,11 @@ private Map getTemplateVariablesMap(TaskNotifyDTO taskNotifyDTO, locale = Locale.ENGLISH; } if (taskNotifyDTO.getResourceType() == ResourceTypeEnum.SCRIPT.getType()) { - variablesMap.put("task.type", i18nService.getI18n("task.type.name.fast_execute_script", locale)); + variablesMap.put("task.type", i18nService.getI18n(locale, "task.type.name.fast_execute_script")); } else if (taskNotifyDTO.getResourceType() == ResourceTypeEnum.FILE.getType()) { - variablesMap.put("task.type", i18nService.getI18n("task.type.name.fast_push_file", locale)); + variablesMap.put("task.type", i18nService.getI18n(locale, "task.type.name.fast_push_file")); } else if (taskNotifyDTO.getResourceType() == ResourceTypeEnum.JOB.getType()) { - variablesMap.put("task.type", i18nService.getI18n("task.type.name.job", locale)); + variablesMap.put("task.type", i18nService.getI18n(locale, "task.type.name.job")); } else { variablesMap.put("task.type", "Unknown"); log.error("Not supported resourceType:{}", taskNotifyDTO.getResourceType()); @@ -324,7 +337,7 @@ private Map getTemplateVariablesMap(TaskNotifyDTO taskNotifyDTO, variablesMap.put("confirm.message", String.valueOf(taskNotifyDTO.getConfirmMessage())); variablesMap.put("task.step.confirm_info", String.valueOf(taskNotifyDTO.getConfirmMessage())); Set receiverSet = new HashSet<>(taskNotifyDTO.getNotifyDTO().getReceiverUsers()); - receiverSet.addAll(userResourceClient.getUsersByRoles( + receiverSet.addAll(userResource.getUsersByRoles( appId, taskNotifyDTO.getOperator(), taskNotifyDTO.getResourceType(), @@ -353,7 +366,7 @@ public void notifyTaskConfirm(TaskNotifyDTO taskNotifyDTO) { serviceTemplateNotificationDTO.setTemplateCode(NotifyConsts.NOTIFY_TEMPLATE_CODE_CONFIRMATION); Map variablesMap = getTemplateVariablesMap(taskNotifyDTO, ExecuteStatusEnum.READY); serviceTemplateNotificationDTO.setVariablesMap(variablesMap); - notificationResourceClient.sendTemplateNotification(serviceTemplateNotificationDTO); + notificationResource.sendTemplateNotification(serviceTemplateNotificationDTO); } } @@ -364,7 +377,7 @@ public void asyncSendMQFailTaskNotification(TaskInstanceDTO taskInstance, taskNotifyDTO.setResourceExecuteStatus(ExecuteStatusEnum.FAIL.getStatus()); taskNotifyDTO.setStepName(stepInstance.getName()); setResourceInfo(taskInstance, stepInstance, taskNotifyDTO); - taskExecuteMQEventDispatcher.asyncSendNotifyMsg(taskNotifyDTO); + taskExecuteMQEventDispatcher.dispatchNotifyMsg(taskNotifyDTO); } @Override @@ -374,13 +387,14 @@ public void asyncSendMQSuccessTaskNotification(TaskInstanceDTO taskInstance, taskNotifyDTO.setResourceExecuteStatus(ExecuteStatusEnum.SUCCESS.getStatus()); taskNotifyDTO.setCost(taskInstance.getTotalTime()); setResourceInfo(taskInstance, stepInstance, taskNotifyDTO); - taskExecuteMQEventDispatcher.asyncSendNotifyMsg(taskNotifyDTO); + taskExecuteMQEventDispatcher.dispatchNotifyMsg(taskNotifyDTO); } @Override public void asyncSendMQConfirmNotification(TaskInstanceDTO taskInstance, StepInstanceBaseDTO stepInstance) { - StepInstanceDTO stepInstanceDetail = taskInstanceService.getStepInstanceDetail(stepInstance.getId()); + StepInstanceDTO stepInstanceDetail = stepInstanceService.getStepInstanceDetail( + taskInstance.getId(), stepInstance.getId()); if (stepInstanceDetail == null) { log.warn("StepInstance is not exist, stepInstanceId: {}", stepInstance.getId()); return; @@ -395,21 +409,23 @@ public void asyncSendMQConfirmNotification(TaskInstanceDTO taskInstance, notifyDTO.setChannels(stepInstanceDetail.getNotifyChannels()); notifyDTO.setTriggerUser(stepInstance.getOperator()); taskNotifyDTO.setNotifyDTO(notifyDTO); - taskNotifyDTO.setResourceId(String.valueOf(taskInstance.getTaskId())); + taskNotifyDTO.setResourceId(String.valueOf(taskInstance.getPlanId())); taskNotifyDTO.setResourceType(ResourceTypeEnum.JOB.getType()); + taskNotifyDTO.setCronTaskId(taskInstance.getCronTaskId()); taskNotifyDTO.setOperator(stepInstance.getOperator()); - taskExecuteMQEventDispatcher.asyncSendNotifyMsg(taskNotifyDTO); + taskExecuteMQEventDispatcher.dispatchNotifyMsg(taskNotifyDTO); } private void setResourceInfo(TaskInstanceDTO taskInstance, StepInstanceBaseDTO stepInstance, TaskNotifyDTO taskNotifyDTO) { - Long taskPlanId = taskInstance.getTaskId(); + Long taskPlanId = taskInstance.getPlanId(); taskNotifyDTO.setResourceId(String.valueOf(taskPlanId)); + taskNotifyDTO.setCronTaskId(taskInstance.getCronTaskId()); if (taskPlanId == -1L) { - if (stepInstance.getExecuteType().equals(StepExecuteTypeEnum.EXECUTE_SCRIPT.getValue())) { + if (stepInstance.isScriptStep()) { taskNotifyDTO.setResourceType(ResourceTypeEnum.SCRIPT.getType()); - } else if (stepInstance.getExecuteType().equals(StepExecuteTypeEnum.SEND_FILE.getValue())) { + } else if (stepInstance.isFileStep()) { taskNotifyDTO.setResourceType(ResourceTypeEnum.FILE.getType()); } else { log.warn("notify resourceType not supported yet:{}, use Job", stepInstance.getExecuteType()); @@ -427,7 +443,7 @@ private TaskNotifyDTO buildCommonTaskNotification(TaskInstanceDTO taskInstance, taskNotifyDTO.setOperator(operator); taskNotifyDTO.setTaskInstanceId(taskInstance.getId()); taskNotifyDTO.setStartupMode(taskInstance.getStartupMode()); - taskNotifyDTO.setTaskId(taskInstance.getTaskId()); + taskNotifyDTO.setTaskId(taskInstance.getPlanId()); taskNotifyDTO.setTaskInstanceName(taskInstance.getName()); return taskNotifyDTO; } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/RollingConfigServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/RollingConfigServiceImpl.java index 690bbddcef..1435337e63 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/RollingConfigServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/RollingConfigServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,17 +27,18 @@ import com.google.common.collect.Lists; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.exception.InternalException; -import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.execute.dao.RollingConfigDAO; -import com.tencent.bk.job.execute.engine.rolling.RollingBatchServersResolver; -import com.tencent.bk.job.execute.engine.rolling.RollingServerBatch; +import com.tencent.bk.job.execute.dao.common.IdGen; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; +import com.tencent.bk.job.execute.engine.rolling.RollingBatchExecuteObjectsResolver; +import com.tencent.bk.job.execute.engine.rolling.RollingExecuteObjectBatch; import com.tencent.bk.job.execute.model.FastTaskDTO; import com.tencent.bk.job.execute.model.RollingConfigDTO; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.model.StepRollingConfigDTO; import com.tencent.bk.job.execute.model.db.RollingConfigDetailDO; -import com.tencent.bk.job.execute.model.db.RollingHostsBatchDO; +import com.tencent.bk.job.execute.model.db.RollingExecuteObjectsBatchDO; import com.tencent.bk.job.execute.model.db.StepRollingConfigDO; import com.tencent.bk.job.execute.service.RollingConfigService; import lombok.extern.slf4j.Slf4j; @@ -55,30 +56,38 @@ public class RollingConfigServiceImpl implements RollingConfigService { private final RollingConfigDAO rollingConfigDAO; + private final IdGen idGen; + @Autowired - public RollingConfigServiceImpl(RollingConfigDAO rollingConfigDAO) { + public RollingConfigServiceImpl(RollingConfigDAO rollingConfigDAO, IdGen idGen) { this.rollingConfigDAO = rollingConfigDAO; + this.idGen = idGen; } @Override - public List getRollingServers(StepInstanceBaseDTO stepInstance, Integer batch) { + public List getRollingServers(StepInstanceBaseDTO stepInstance, Integer batch) { long rollingConfigId = stepInstance.getRollingConfigId(); long stepInstanceId = stepInstance.getId(); RollingConfigDTO rollingConfig = - rollingConfigDAO.queryRollingConfigById(rollingConfigId); + rollingConfigDAO.queryRollingConfigById(stepInstance.getTaskInstanceId(), rollingConfigId); if (rollingConfig.isBatchRollingStep(stepInstanceId)) { if (batch == null || batch == 0) { // 忽略滚动批次，返回当前步骤的所有目标服务器 - return stepInstance.getTargetServers().getIpList(); + return stepInstance.getTargetExecuteObjects().getExecuteObjectsCompatibly(); } else { - return rollingConfig.getConfigDetail().getHostsBatchList() - .stream().filter(serverBatch -> serverBatch.getBatch().equals(batch)) - .findFirst().orElseThrow(() -> new InternalException(ErrorCode.INTERNAL_ERROR)).getHosts(); + return rollingConfig + .getConfigDetail() + .getExecuteObjectsBatchListCompatibly() + .stream() + .filter(serverBatch -> serverBatch.getBatch().equals(batch)) + .findFirst() + .orElseThrow(() -> new InternalException(ErrorCode.INTERNAL_ERROR)) + .getExecuteObjectsCompatibly(); } } else { - return stepInstance.getTargetServers().getIpList(); + return stepInstance.getTargetExecuteObjects().getExecuteObjectsCompatibly(); } } @@ -99,16 +108,19 @@ public RollingConfigDTO saveRollingConfigForFastJob(FastTaskDTO fastTask) { rollingConfigDetailDO.setMode(rollingConfig.getMode()); rollingConfigDetailDO.setExpr(rollingConfig.getExpr()); - RollingBatchServersResolver resolver = - new RollingBatchServersResolver(fastTask.getStepInstance().getTargetServers().getIpList(), + RollingBatchExecuteObjectsResolver resolver = + new RollingBatchExecuteObjectsResolver( + fastTask.getStepInstance().getTargetExecuteObjects().getExecuteObjectsCompatibly(), rollingConfig.getExpr()); - List serversBatchList = resolver.resolve(); - rollingConfigDetailDO.setHostsBatchList( - serversBatchList.stream() - .map(rollingServerBatch -> - new RollingHostsBatchDO(rollingServerBatch.getBatch(), rollingServerBatch.getServers())) + List executeObjectsBatchList = resolver.resolve(); + rollingConfigDetailDO.setExecuteObjectsBatchList( + executeObjectsBatchList.stream() + .map(rollingExecuteObjectBatch -> + new RollingExecuteObjectsBatchDO(rollingExecuteObjectBatch.getBatch(), + rollingExecuteObjectBatch.getExecuteObjects())) .collect(Collectors.toList())); - rollingConfigDetailDO.setTotalBatch(rollingConfigDetailDO.getHostsBatchList().size()); + + rollingConfigDetailDO.setTotalBatch(rollingConfigDetailDO.getExecuteObjectsBatchListCompatibly().size()); taskInstanceRollingConfig.setConfigDetail(rollingConfigDetailDO); rollingConfigDetailDO.setIncludeStepInstanceIdList(Lists.newArrayList(stepInstance.getId())); @@ -116,13 +128,24 @@ public RollingConfigDTO saveRollingConfigForFastJob(FastTaskDTO fastTask) { stepRollingConfigs.put(stepInstance.getId(), new StepRollingConfigDO(true)); rollingConfigDetailDO.setStepRollingConfigs(stepRollingConfigs); - Long rollingConfigId= rollingConfigDAO.saveRollingConfig(taskInstanceRollingConfig); + Long rollingConfigId = addRollingConfig(taskInstanceRollingConfig); taskInstanceRollingConfig.setId(rollingConfigId); return taskInstanceRollingConfig; } @Override - public RollingConfigDTO getRollingConfig(long rollingConfigId) { - return rollingConfigDAO.queryRollingConfigById(rollingConfigId); + public RollingConfigDTO getRollingConfig(Long taskInstanceId, long rollingConfigId) { + return rollingConfigDAO.queryRollingConfigById(taskInstanceId, rollingConfigId); + } + + @Override + public boolean isTaskRollingEnabled(long taskInstanceId) { + return rollingConfigDAO.existsRollingConfig(taskInstanceId); + } + + @Override + public long addRollingConfig(RollingConfigDTO rollingConfig) { + rollingConfig.setId(idGen.genRollingConfigId()); + return rollingConfigDAO.saveRollingConfig(rollingConfig); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ScriptAgentTaskServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ScriptAgentTaskServiceImpl.java deleted file mode 100644 index 0a9656d90b..0000000000 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ScriptAgentTaskServiceImpl.java +++ /dev/null @@ -1,206 +0,0 @@ -package com.tencent.bk.job.execute.service.impl; - -import com.tencent.bk.job.common.constant.Order; -import com.tencent.bk.job.common.model.dto.HostDTO; -import com.tencent.bk.job.execute.dao.GseTaskIpLogDAO; -import com.tencent.bk.job.execute.dao.ScriptAgentTaskDAO; -import com.tencent.bk.job.execute.model.AgentTaskDTO; -import com.tencent.bk.job.execute.model.AgentTaskDetailDTO; -import com.tencent.bk.job.execute.model.AgentTaskResultGroupBaseDTO; -import com.tencent.bk.job.execute.model.AgentTaskResultGroupDTO; -import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; -import com.tencent.bk.job.execute.service.HostService; -import com.tencent.bk.job.execute.service.ScriptAgentTaskService; -import com.tencent.bk.job.execute.service.StepInstanceService; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.stream.Collectors; - -@Service -@Slf4j -public class ScriptAgentTaskServiceImpl - extends AbstractAgentTaskServiceImpl - implements ScriptAgentTaskService { - - private final ScriptAgentTaskDAO scriptAgentTaskDAO; - private final GseTaskIpLogDAO gseTaskIpLogDAO; - - @Autowired - public ScriptAgentTaskServiceImpl(ScriptAgentTaskDAO scriptAgentTaskDAO, - StepInstanceService stepInstanceService, - HostService hostService, - GseTaskIpLogDAO gseTaskIpLogDAO) { - super(stepInstanceService, hostService); - this.scriptAgentTaskDAO = scriptAgentTaskDAO; - this.gseTaskIpLogDAO = gseTaskIpLogDAO; - } - - @Override - public void batchSaveAgentTasks(Collection agentTasks) { - if (CollectionUtils.isEmpty(agentTasks)) { - return; - } - scriptAgentTaskDAO.batchSaveAgentTasks(agentTasks); - } - - @Override - public void batchUpdateAgentTasks(Collection agentTasks) { - if (CollectionUtils.isEmpty(agentTasks)) { - return; - } - scriptAgentTaskDAO.batchUpdateAgentTasks(agentTasks); - } - - @Override - public int getSuccessAgentTaskCount(long stepInstanceId, int executeCount) { - if (isStepInstanceRecordExist(stepInstanceId)) { - return scriptAgentTaskDAO.getSuccessAgentTaskCount(stepInstanceId, executeCount); - } else { - return gseTaskIpLogDAO.getSuccessAgentTaskCount(stepInstanceId, executeCount); - } - } - - @Override - public List listAgentTasks(Long stepInstanceId, Integer executeCount, Integer batch) { - List agentTasks = scriptAgentTaskDAO.listAgentTasks(stepInstanceId, executeCount, batch); - if (CollectionUtils.isEmpty(agentTasks)) { - // 兼容历史数据 - agentTasks = gseTaskIpLogDAO.listAgentTasks(stepInstanceId, executeCount); - } - return agentTasks; - } - - @Override - public List listAgentTasksByGseTaskId(Long gseTaskId) { - return scriptAgentTaskDAO.listAgentTasksByGseTaskId(gseTaskId); - } - - @Override - public AgentTaskDTO getAgentTaskByHost(StepInstanceBaseDTO stepInstance, - Integer executeCount, - Integer batch, - HostDTO host) { - AgentTaskDTO agentTask = null; - Long hostId = host.getHostId(); - if (hostId != null) { - // 根据hostId查询 - agentTask = scriptAgentTaskDAO.getAgentTaskByHostId(stepInstance.getId(), executeCount, batch, hostId); - } else if (StringUtils.isNotEmpty(host.toCloudIp())) { - // 根据ip查询的模式，有两种情况，数据可能在gse_script_agent_task/gse_task_ip_log表中，优先查询gse_script_agent_task - HostDTO queryHost = getStepHostByIp(stepInstance, host.toCloudIp()); - if (queryHost != null && queryHost.getHostId() != null) { - agentTask = scriptAgentTaskDAO.getAgentTaskByHostId(stepInstance.getId(), executeCount, batch, - queryHost.getHostId()); - } else { - // 根据ip查询gse_task_ip_log表中的数据 - agentTask = gseTaskIpLogDAO.getAgentTaskByIp(stepInstance.getId(), executeCount, host.toCloudIp()); - } - } - return agentTask; - } - - private HostDTO getStepHostByIp(StepInstanceBaseDTO stepInstance, String cloudIp) { - return stepInstance.getTargetServers().getIpList().stream() - .filter(targetHost -> cloudIp.equals(targetHost.toCloudIp())) - .findFirst() - .orElse(null); - } - - @Override - public int getActualSuccessExecuteCount(long stepInstanceId, String cloudIp) { - // 兼容历史数据 - return gseTaskIpLogDAO.getActualSuccessExecuteCount(stepInstanceId, cloudIp); - } - - @Override - public List listAndGroupAgentTasks(StepInstanceBaseDTO stepInstance, - int executeCount, - Integer batch) { - List resultGroups = new ArrayList<>(); - List agentTasks = listAgentTasks(stepInstance.getId(), executeCount, batch); - if (CollectionUtils.isEmpty(agentTasks)) { - return resultGroups; - } - - List agentTaskDetailList = fillHostDetail(stepInstance, agentTasks); - resultGroups = groupAgentTasks(agentTaskDetailList); - - return resultGroups.stream().sorted().collect(Collectors.toList()); - } - - @Override - public List listResultGroups(long stepInstanceId, - int executeCount, - Integer batch) { - List resultGroups; - resultGroups = scriptAgentTaskDAO.listResultGroups(stepInstanceId, executeCount, batch); - if (CollectionUtils.isEmpty(resultGroups)) { - // 兼容历史数据 - resultGroups = gseTaskIpLogDAO.listResultGroups(stepInstanceId, executeCount); - } - return resultGroups; - } - - @Override - public List listAgentTaskDetailByResultGroup(StepInstanceBaseDTO stepInstance, - Integer executeCount, - Integer batch, - Integer status, - String tag) { - List agentTasks = scriptAgentTaskDAO.listAgentTaskByResultGroup(stepInstance.getId(), - executeCount, batch, status, tag); - if (CollectionUtils.isEmpty(agentTasks)) { - // 兼容历史数据 - agentTasks = gseTaskIpLogDAO.listAgentTaskByResultGroup(stepInstance.getId(), executeCount, status, tag); - } - return fillHostDetail(stepInstance, agentTasks); - } - - - @Override - public List listAgentTaskDetailByResultGroup(StepInstanceBaseDTO stepInstance, - Integer executeCount, - Integer batch, - Integer status, - String tag, - Integer limit, - String orderField, - Order order) { - List agentTasks = scriptAgentTaskDAO.listAgentTaskByResultGroup(stepInstance.getId(), - executeCount, batch, status, tag, limit, orderField, order); - if (CollectionUtils.isEmpty(agentTasks)) { - // 兼容历史数据 - agentTasks = gseTaskIpLogDAO.listAgentTaskByResultGroup(stepInstance.getId(), executeCount, status, tag, - limit, orderField, order); - } - return fillHostDetail(stepInstance, agentTasks); - } - - @Override - public List listAgentTaskDetail(StepInstanceBaseDTO stepInstance, - Integer executeCount, - Integer batch) { - List agentTasks = listAgentTasks(stepInstance.getId(), executeCount, batch); - return fillHostDetail(stepInstance, agentTasks); - } - - private boolean isStepInstanceRecordExist(long stepInstanceId) { - return scriptAgentTaskDAO.isStepInstanceRecordExist(stepInstanceId); - } - - @Override - public void updateAgentTaskFields(long stepInstanceId, - int executeCount, - Integer batch, - Integer actualExecuteCount, - Long gseTaskId) { - scriptAgentTaskDAO.updateAgentTaskFields(stepInstanceId, executeCount, batch, actualExecuteCount, gseTaskId); - } -} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ScriptExecuteObjectTaskServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ScriptExecuteObjectTaskServiceImpl.java new file mode 100644 index 0000000000..925973fc4b --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ScriptExecuteObjectTaskServiceImpl.java @@ -0,0 +1,323 @@ +package com.tencent.bk.job.execute.service.impl; + +import com.tencent.bk.job.common.constant.Order; +import com.tencent.bk.job.common.util.BatchUtil; +import com.tencent.bk.job.execute.dao.ScriptAgentTaskDAO; +import com.tencent.bk.job.execute.dao.ScriptExecuteObjectTaskDAO; +import com.tencent.bk.job.execute.dao.common.IdGen; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; +import com.tencent.bk.job.execute.model.ExecuteObjectCompositeKey; +import com.tencent.bk.job.execute.model.ExecuteObjectTask; +import com.tencent.bk.job.execute.model.ResultGroupBaseDTO; +import com.tencent.bk.job.execute.model.ResultGroupDTO; +import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; +import com.tencent.bk.job.execute.service.ScriptExecuteObjectTaskService; +import com.tencent.bk.job.execute.service.StepInstanceService; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +@Service +@Slf4j +public class ScriptExecuteObjectTaskServiceImpl + extends AbstractExecuteObjectTaskServiceImpl + implements ScriptExecuteObjectTaskService { + + private final ScriptExecuteObjectTaskDAO scriptExecuteObjectTaskDAO; + private final ScriptAgentTaskDAO scriptAgentTaskDAO; + private final IdGen idGen; + + @Autowired + public ScriptExecuteObjectTaskServiceImpl(StepInstanceService stepInstanceService, + ScriptExecuteObjectTaskDAO scriptExecuteObjectTaskDAO, + ScriptAgentTaskDAO scriptAgentTaskDAO, + IdGen idGen) { + super(stepInstanceService); + this.scriptExecuteObjectTaskDAO = scriptExecuteObjectTaskDAO; + this.scriptAgentTaskDAO = scriptAgentTaskDAO; + this.idGen = idGen; + } + + @Override + public void batchSaveTasks(Collection tasks) { + if (CollectionUtils.isEmpty(tasks)) { + return; + } + + tasks.forEach(task -> task.setId(idGen.genGseScriptExecuteObjTaskId())); + + boolean executeObjectSupported = isExecuteObjectSupported(tasks); + + // 任务分批，避免大事务造成 db 主从延迟 + BatchUtil.executeBatch( + tasks, + 2000, + batchTasks -> executeSaveTasks(executeObjectSupported, batchTasks) + ); + } + + private void executeSaveTasks(boolean executeObjectSupported, + Collection tasks) { + if (executeObjectSupported) { + scriptExecuteObjectTaskDAO.batchSaveTasks(tasks); + } else { + scriptAgentTaskDAO.batchSaveAgentTasks(tasks); + } + } + + @Override + public void batchUpdateTasks(Collection tasks) { + if (CollectionUtils.isEmpty(tasks)) { + return; + } + + boolean executeObjectSupported = isExecuteObjectSupported(tasks); + + // 任务分批，避免大事务造成 db 主从延迟 + BatchUtil.executeBatch( + tasks, + 2000, + batchTasks -> executeUpdateTasks(executeObjectSupported, batchTasks) + ); + } + + private void executeUpdateTasks(boolean executeObjectSupported, + Collection tasks) { + if (executeObjectSupported) { + scriptExecuteObjectTaskDAO.batchUpdateTasks(tasks); + } else { + scriptAgentTaskDAO.batchUpdateAgentTasks(tasks); + } + } + + @Override + public int getSuccessTaskCount(Long taskInstanceId, long stepInstanceId, int executeCount) { + if (isStepInstanceRecordExist(taskInstanceId, stepInstanceId)) { + return scriptExecuteObjectTaskDAO.getSuccessTaskCount(taskInstanceId, stepInstanceId, executeCount); + } else { + return scriptAgentTaskDAO.getSuccessAgentTaskCount(taskInstanceId, stepInstanceId, executeCount); + } + } + + @Override + public List listTasks(StepInstanceBaseDTO stepInstance, Integer executeCount, Integer batch) { + List executeObjectTasks; + long stepInstanceId = stepInstance.getId(); + if (stepInstance.isSupportExecuteObjectFeature()) { + executeObjectTasks = scriptExecuteObjectTaskDAO.listTasks(stepInstance.getTaskInstanceId(), + stepInstanceId, executeCount, batch); + } else { + // 兼容老版本数据 + executeObjectTasks = scriptAgentTaskDAO.listAgentTasks(stepInstance.getTaskInstanceId(), + stepInstanceId, executeCount, batch); + } + fillExecuteObjectForExecuteObjectTasks(stepInstance, executeObjectTasks); + return executeObjectTasks; + } + + @Override + public List listTasksByGseTaskId(StepInstanceBaseDTO stepInstance, Long gseTaskId) { + List executeObjectTasks; + + if (stepInstance.isSupportExecuteObjectFeature()) { + executeObjectTasks = scriptExecuteObjectTaskDAO.listTasksByGseTaskId( + stepInstance.getTaskInstanceId(), gseTaskId); + } else { + // 兼容老版本数据 + executeObjectTasks = scriptAgentTaskDAO.listAgentTasksByGseTaskId( + stepInstance.getTaskInstanceId(), gseTaskId); + } + fillExecuteObjectForExecuteObjectTasks(stepInstance, executeObjectTasks); + return executeObjectTasks; + } + + @Override + public ExecuteObjectTask getTaskByExecuteObjectCompositeKey(StepInstanceBaseDTO stepInstance, + Integer executeCount, + Integer batch, + ExecuteObjectCompositeKey executeObjectCompositeKey) { + ExecuteObject executeObject = + stepInstanceService.findExecuteObjectByCompositeKey(stepInstance, executeObjectCompositeKey); + if (executeObject == null) { + return null; + } + + long stepInstanceId = stepInstance.getId(); + ExecuteObjectTask executeObjectTask; + if (stepInstance.isSupportExecuteObjectFeature()) { + executeObjectTask = scriptExecuteObjectTaskDAO.getTaskByExecuteObjectId( + stepInstance.getTaskInstanceId(), + stepInstanceId, executeCount, + batch, + executeObject.getId() + ); + } else { + // 兼容老版本不使用执行对象的场景(仅支持主机） + Long hostId = executeObject.getHost().getHostId(); + executeObjectTask = scriptAgentTaskDAO.getAgentTaskByHostId( + stepInstance.getTaskInstanceId(), + stepInstanceId, + executeCount, + batch, + hostId + ); + } + fillExecuteObjectForExecuteObjectTask(stepInstance, executeObjectTask); + return executeObjectTask; + } + + @Override + public List getTaskByExecuteObjectIds(StepInstanceBaseDTO stepInstance, + Integer executeCount, + Integer batch, + Collection executeObjectIds) { + long stepInstanceId = stepInstance.getId(); + List executeObjectTaskList = scriptExecuteObjectTaskDAO.getTaskByExecuteObjectIds( + stepInstance.getTaskInstanceId(), + stepInstanceId, executeCount, + batch, + executeObjectIds + ); + fillExecuteObjectForExecuteObjectTasks(stepInstance, executeObjectTaskList); + return executeObjectTaskList; + } + + @Override + public List listAndGroupTasks(StepInstanceBaseDTO stepInstance, + int executeCount, + Integer batch) { + List resultGroups = new ArrayList<>(); + List executeObjectTasks = listTasks(stepInstance, executeCount, batch); + if (CollectionUtils.isEmpty(executeObjectTasks)) { + return resultGroups; + } + + fillExecuteObjectForExecuteObjectTasks(stepInstance, executeObjectTasks); + resultGroups = groupTasks(executeObjectTasks); + + return resultGroups.stream().sorted().collect(Collectors.toList()); + } + + @Override + public List listResultGroups(StepInstanceBaseDTO stepInstance, + int executeCount, + Integer batch) { + List resultGroups; + long stepInstanceId = stepInstance.getId(); + + if (stepInstance.isSupportExecuteObjectFeature()) { + resultGroups = scriptExecuteObjectTaskDAO.listResultGroups( + stepInstance.getTaskInstanceId(), + stepInstanceId, + executeCount, + batch + ); + } else { + // 兼容历史数据 + resultGroups = scriptAgentTaskDAO.listResultGroups(stepInstance.getTaskInstanceId(), + stepInstanceId, executeCount, batch); + } + return resultGroups; + } + + @Override + public List listTaskByResultGroup(StepInstanceBaseDTO stepInstance, + Integer executeCount, + Integer batch, + Integer status, + String tag) { + List executeObjectTasks; + + if (stepInstance.isSupportExecuteObjectFeature()) { + executeObjectTasks = scriptExecuteObjectTaskDAO.listTasksByResultGroup( + stepInstance.getTaskInstanceId(), + stepInstance.getId(), + executeCount, + batch, + status, + tag + ); + } else { + // 兼容历史数据 + executeObjectTasks = scriptAgentTaskDAO.listAgentTaskByResultGroup( + stepInstance.getTaskInstanceId(), + stepInstance.getId(), + executeCount, + batch, + status, + tag + ); + } + + fillExecuteObjectForExecuteObjectTasks(stepInstance, executeObjectTasks); + return executeObjectTasks; + } + + + public List listTaskByResultGroup(StepInstanceBaseDTO stepInstance, + Integer executeCount, + Integer batch, + Integer status, + String tag, + Integer limit, + String orderField, + Order order) { + List executeObjectTasks; + + if (stepInstance.isSupportExecuteObjectFeature()) { + executeObjectTasks = scriptExecuteObjectTaskDAO.listTasksByResultGroup( + stepInstance.getTaskInstanceId(), + stepInstance.getId(), + executeCount, + batch, + status, + tag, + limit, + orderField, + order + ); + } else { + // 兼容历史数据 + executeObjectTasks = scriptAgentTaskDAO.listAgentTaskByResultGroup( + stepInstance.getTaskInstanceId(), + stepInstance.getId(), + executeCount, + batch, + status, + tag, + limit, + orderField, + order + ); + } + + fillExecuteObjectForExecuteObjectTasks(stepInstance, executeObjectTasks); + return executeObjectTasks; + } + + private boolean isStepInstanceRecordExist(Long taskInstanceId, long stepInstanceId) { + return scriptExecuteObjectTaskDAO.isStepInstanceRecordExist(taskInstanceId, stepInstanceId); + } + + @Override + public void updateTaskFields(StepInstanceBaseDTO stepInstance, + int executeCount, + Integer batch, + Integer actualExecuteCount, + Long gseTaskId) { + if (stepInstance.isSupportExecuteObjectFeature()) { + scriptExecuteObjectTaskDAO.updateTaskFields(stepInstance.getTaskInstanceId(), stepInstance.getId(), + executeCount, batch, actualExecuteCount, gseTaskId); + } else { + // 兼容老版本方式 + scriptAgentTaskDAO.updateAgentTaskFields(stepInstance.getTaskInstanceId(), stepInstance.getId(), + executeCount, batch, actualExecuteCount, gseTaskId); + } + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ScriptServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ScriptServiceImpl.java index abcc6bf760..4aff4e52c2 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ScriptServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ScriptServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,25 +27,25 @@ import com.tencent.bk.job.common.exception.InternalException; import com.tencent.bk.job.common.exception.ServiceException; import com.tencent.bk.job.common.model.InternalResponse; -import com.tencent.bk.job.execute.client.ScriptResourceClient; import com.tencent.bk.job.execute.service.ScriptService; +import com.tencent.bk.job.manage.api.inner.ServiceScriptResource; import com.tencent.bk.job.manage.model.inner.ServiceScriptDTO; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; -@Service +@Service("jobExecuteScriptService") @Slf4j public class ScriptServiceImpl implements ScriptService { - private final ScriptResourceClient scriptResourceClient; + private final ServiceScriptResource scriptResource; - public ScriptServiceImpl(ScriptResourceClient scriptResourceClient) { - this.scriptResourceClient = scriptResourceClient; + public ScriptServiceImpl(ServiceScriptResource scriptResource) { + this.scriptResource = scriptResource; } @Override public ServiceScriptDTO getScriptByScriptVersionId(String username, long appId, long scriptVersionId) throws ServiceException { - InternalResponse resp = scriptResourceClient.getScriptByAppIdAndScriptVersionId(username, + InternalResponse resp = scriptResource.getScriptByAppIdAndScriptVersionId(username, appId, scriptVersionId); // if (!resp.isSuccess()) { // throw new ServiceException(resp.getCode(), resp.getErrorMsg()); @@ -55,7 +55,7 @@ public ServiceScriptDTO getScriptByScriptVersionId(String username, long appId, @Override public ServiceScriptDTO getScriptByScriptVersionId(long scriptVersionId) throws ServiceException { - InternalResponse resp = scriptResourceClient.getScriptByScriptVersionId(scriptVersionId); + InternalResponse resp = scriptResource.getScriptByScriptVersionId(scriptVersionId); if (!resp.isSuccess()) { throw new InternalException(resp.getCode()); } @@ -69,7 +69,7 @@ public ServiceScriptDTO getBasicScriptInfo(String scriptId) { @Override public ServiceScriptDTO getOnlineScriptVersion(String scriptId) { - InternalResponse resp = scriptResourceClient.getOnlineScriptVersion(scriptId); + InternalResponse resp = scriptResource.getOnlineScriptVersion(scriptId); if (!resp.isSuccess()) { throw new InternalException(resp.getCode()); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceRollingTaskServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceRollingTaskServiceImpl.java index d2cd2c3e56..ad9c615f62 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceRollingTaskServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceRollingTaskServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,6 +26,7 @@ import com.tencent.bk.job.execute.common.constants.RunStatusEnum; import com.tencent.bk.job.execute.dao.StepInstanceRollingTaskDAO; +import com.tencent.bk.job.execute.dao.common.IdGen; import com.tencent.bk.job.execute.model.StepInstanceRollingTaskDTO; import com.tencent.bk.job.execute.service.StepInstanceRollingTaskService; import lombok.extern.slf4j.Slf4j; @@ -44,23 +45,29 @@ public class StepInstanceRollingTaskServiceImpl implements StepInstanceRollingTaskService { private final StepInstanceRollingTaskDAO stepInstanceRollingTaskDAO; + private final IdGen idGen; @Autowired public StepInstanceRollingTaskServiceImpl( - StepInstanceRollingTaskDAO stepInstanceRollingTaskDAO) { + StepInstanceRollingTaskDAO stepInstanceRollingTaskDAO, IdGen idGen) { this.stepInstanceRollingTaskDAO = stepInstanceRollingTaskDAO; + this.idGen = idGen; } @Override - public StepInstanceRollingTaskDTO queryRollingTask(long stepInstanceId, - int executeCount, int batch) { - return stepInstanceRollingTaskDAO.queryRollingTask(stepInstanceId, executeCount, batch); + public StepInstanceRollingTaskDTO queryRollingTask(Long taskInstanceId, + long stepInstanceId, + int executeCount, + int batch) { + return stepInstanceRollingTaskDAO.queryRollingTask(taskInstanceId, stepInstanceId, executeCount, batch); } @Override - public List listLatestRollingTasks(long stepInstanceId, int executeCount) { + public List listLatestRollingTasks(Long taskInstanceId, + long stepInstanceId, + int executeCount) { List stepInstanceRollingTasks = - stepInstanceRollingTaskDAO.listRollingTasks(stepInstanceId, null, null); + stepInstanceRollingTaskDAO.listRollingTasks(taskInstanceId, stepInstanceId, null, null); if (CollectionUtils.isEmpty(stepInstanceRollingTasks)) { return stepInstanceRollingTasks; } @@ -92,25 +99,33 @@ public List listLatestRollingTasks(long stepInstance } @Override - public List listRollingTasksByBatch(long stepInstanceId, Integer batch) { - return stepInstanceRollingTaskDAO.listRollingTasks(stepInstanceId, null, batch); + public List listRollingTasksByBatch(Long taskInstanceId, + long stepInstanceId, + Integer batch) { + return stepInstanceRollingTaskDAO.listRollingTasks(taskInstanceId, stepInstanceId, null, batch); } @Override public long saveRollingTask(StepInstanceRollingTaskDTO rollingTask) { + rollingTask.setId(idGen.genStepInstanceRollingTaskId()); return stepInstanceRollingTaskDAO.saveRollingTask(rollingTask); } @Override - public void updateRollingTask(long stepInstanceId, + public void updateRollingTask(Long taskInstanceId, + long stepInstanceId, int executeCount, int batch, RunStatusEnum status, Long startTime, Long endTime, Long totalTime) { - stepInstanceRollingTaskDAO.updateRollingTask(stepInstanceId, executeCount, batch, status, startTime, - endTime, totalTime); + stepInstanceRollingTaskDAO.updateRollingTask(taskInstanceId, stepInstanceId, executeCount, batch, status, + startTime, endTime, totalTime); } + @Override + public List listRollingTasksByStep(Long taskInstanceId, long stepInstanceId) { + return stepInstanceRollingTaskDAO.listRollingTasks(taskInstanceId, stepInstanceId, null, null); + } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceServiceImpl.java index 6a5dd2b078..3c2b4cfbe0 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,67 +24,306 @@ package com.tencent.bk.job.execute.service.impl; -import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.InternalException; +import com.tencent.bk.job.common.exception.NotFoundException; +import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.execute.common.constants.RunStatusEnum; +import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; import com.tencent.bk.job.execute.dao.StepInstanceDAO; +import com.tencent.bk.job.execute.dao.common.IdGen; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; +import com.tencent.bk.job.execute.model.ExecuteObjectCompositeKey; import com.tencent.bk.job.execute.model.FileSourceDTO; import com.tencent.bk.job.execute.model.FileStepInstanceDTO; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.service.StepInstanceService; +import com.tencent.bk.job.manage.api.common.constants.task.TaskStepTypeEnum; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Function; import java.util.stream.Collectors; +import static com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum.EXECUTE_SCRIPT; +import static com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum.EXECUTE_SQL; +import static com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum.MANUAL_CONFIRM; +import static com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum.SEND_FILE; + @Slf4j @Service public class StepInstanceServiceImpl implements StepInstanceService { private final StepInstanceDAO stepInstanceDAO; + private final IdGen idGen; @Autowired - public StepInstanceServiceImpl(StepInstanceDAO stepInstanceDAO) { + public StepInstanceServiceImpl(StepInstanceDAO stepInstanceDAO, IdGen idGen) { this.stepInstanceDAO = stepInstanceDAO; + this.idGen = idGen; } @Override - public void updateStepCurrentBatch(long stepInstanceId, int batch) { - stepInstanceDAO.updateStepCurrentBatch(stepInstanceId, batch); + public void updateStepCurrentBatch(Long taskInstanceId, long stepInstanceId, int batch) { + stepInstanceDAO.updateStepCurrentBatch(taskInstanceId, stepInstanceId, batch); } @Override - public void updateStepCurrentExecuteCount(long stepInstanceId, int executeCount) { - stepInstanceDAO.updateStepCurrentExecuteCount(stepInstanceId, executeCount); + public void updateStepRollingConfigId(Long taskInstanceId, long stepInstanceId, long rollingConfigId) { + stepInstanceDAO.updateStepRollingConfigId(taskInstanceId, stepInstanceId, rollingConfigId); } @Override - public void updateStepRollingConfigId(long stepInstanceId, long rollingConfigId) { - stepInstanceDAO.updateStepRollingConfigId(stepInstanceId, rollingConfigId); + public StepInstanceBaseDTO getNextStepInstance(Long taskInstanceId, + int currentStepOrder) { + return stepInstanceDAO.getNextStepInstance(taskInstanceId, currentStepOrder); } @Override - public StepInstanceBaseDTO getNextStepInstance(long taskInstanceId, - int currentStepOrder) { - return stepInstanceDAO.getNextStepInstance(taskInstanceId, currentStepOrder); + public long addStepInstance(StepInstanceDTO stepInstance) { + stepInstance.setId(idGen.genStepInstanceId()); + long stepInstanceId = stepInstanceDAO.addStepInstanceBase(stepInstance); + if (stepInstanceId > 0) { + stepInstance.setId(stepInstanceId); + StepExecuteTypeEnum executeType = stepInstance.getExecuteType(); + if (executeType == EXECUTE_SQL || executeType == EXECUTE_SCRIPT) { + stepInstanceDAO.addScriptStepInstance(stepInstance); + } else if (executeType == SEND_FILE) { + stepInstanceDAO.addFileStepInstance(stepInstance); + } else if (executeType == MANUAL_CONFIRM) { + stepInstanceDAO.addConfirmStepInstance(stepInstance); + } + } + return stepInstanceId; + } + + private void fillStepInstanceDetail(StepInstanceDTO stepInstance) { + Long taskInstanceId = stepInstance.getTaskInstanceId(); + long stepInstanceId = stepInstance.getId(); + TaskStepTypeEnum stepType = stepInstance.getStepType(); + if (stepType == TaskStepTypeEnum.SCRIPT) { + stepInstance.fillScriptStepInfo(stepInstanceDAO.getScriptStepInstance(taskInstanceId, stepInstanceId)); + } else if (stepType == TaskStepTypeEnum.FILE) { + stepInstance.fillFileStepInfo(stepInstanceDAO.getFileStepInstance(taskInstanceId, stepInstanceId)); + } else if (stepType == TaskStepTypeEnum.APPROVAL) { + stepInstance.fillConfirmStepInfo(stepInstanceDAO.getConfirmStepInstance(taskInstanceId, stepInstanceId)); + } } @Override - public Map computeStepHosts(StepInstanceBaseDTO stepInstance, - Function keyMapper) { - Map hosts = - stepInstance.getTargetServers().getIpList().stream() - .collect(Collectors.toMap(keyMapper, host -> host)); + public List listBaseStepInstanceByTaskInstanceId(Long taskInstanceId) { + return stepInstanceDAO.listStepInstanceBaseByTaskInstanceId(taskInstanceId); + } + + @Override + public List listStepInstanceByTaskInstanceId(Long taskInstanceId) { + List stepInstanceList = listBaseStepInstanceByTaskInstanceId(taskInstanceId); + if (CollectionUtils.isEmpty(stepInstanceList)) { + return Collections.emptyList(); + } + return stepInstanceList.stream() + .map(baseStepInstance -> { + StepInstanceDTO stepInstance = new StepInstanceDTO(baseStepInstance); + fillStepInstanceDetail(stepInstance); + return stepInstance; + }).collect(Collectors.toList()); + } + + @Override + public StepInstanceBaseDTO getBaseStepInstance(Long taskInstanceId, long stepInstanceId) { + return stepInstanceDAO.getStepInstanceBase(taskInstanceId, stepInstanceId); + } + + @Override + public StepInstanceBaseDTO getBaseStepInstanceById(long stepInstanceId) { + return stepInstanceDAO.getStepInstanceBase(stepInstanceId); + } + + @Override + public StepInstanceBaseDTO getBaseStepInstance(long appId, Long taskInstanceId, long stepInstanceId) { + StepInstanceBaseDTO stepInstance = getBaseStepInstance(taskInstanceId, stepInstanceId); + if (stepInstance == null) { + log.warn("Step instance is not exist, taskInstanceId={}, stepInstanceId: {}", + taskInstanceId, stepInstanceId); + throw new NotFoundException(ErrorCode.STEP_INSTANCE_NOT_EXIST); + } + if (!stepInstance.getAppId().equals(appId)) { + log.warn("StepInstance:{} is not in app:{}", stepInstanceId, appId); + throw new NotFoundException(ErrorCode.STEP_INSTANCE_NOT_EXIST); + } + return stepInstance; + } + + @Override + public StepInstanceDTO getStepInstanceDetail(Long taskInstanceId, long stepInstanceId) throws NotFoundException { + StepInstanceBaseDTO stepInstanceBase = stepInstanceDAO.getStepInstanceBase(taskInstanceId, stepInstanceId); + if (stepInstanceBase == null) { + log.warn("StepInstance:{} not exist", stepInstanceId); + throw new NotFoundException(ErrorCode.STEP_INSTANCE_NOT_EXIST); + } + StepInstanceDTO stepInstance = new StepInstanceDTO(stepInstanceBase); + fillStepInstanceDetail(stepInstance); + return stepInstance; + } + + @Override + public StepInstanceDTO getStepInstanceDetail(long appId, Long taskInstanceId, + long stepInstanceId) throws NotFoundException { + StepInstanceDTO stepInstance = getStepInstanceDetail(taskInstanceId, stepInstanceId); + if (!stepInstance.getAppId().equals(appId)) { + log.warn("StepInstance:{} is not in app:{}", stepInstanceId, appId); + throw new NotFoundException(ErrorCode.STEP_INSTANCE_NOT_EXIST); + } + return stepInstance; + } + + @Override + public StepInstanceBaseDTO getFirstStepInstance(Long taskInstanceId) { + return stepInstanceDAO.getFirstStepInstanceBase(taskInstanceId); + } + + @Override + public List getTaskStepIdList(Long taskInstanceId) { + return stepInstanceDAO.getTaskStepInstanceIdList(taskInstanceId); + } + + @Override + public void updateStepStatus(Long taskInstanceId, long stepInstanceId, int status) { + stepInstanceDAO.updateStepStatus(taskInstanceId, stepInstanceId, status); + } + + @Override + public void resetStepExecuteInfoForRetry(Long taskInstanceId, long stepInstanceId) { + stepInstanceDAO.resetStepExecuteInfoForRetry(taskInstanceId, stepInstanceId); + } + + @Override + public void resetStepStatus(Long taskInstanceId, long stepInstanceId) { + stepInstanceDAO.resetStepStatus(taskInstanceId, stepInstanceId); + } + + @Override + public void updateStepStartTimeIfNull(Long taskInstanceId, long stepInstanceId, Long startTime) { + stepInstanceDAO.updateStepStartTimeIfNull(taskInstanceId, stepInstanceId, startTime); + } + + @Override + public void updateStepEndTime(Long taskInstanceId, long stepInstanceId, Long endTime) { + stepInstanceDAO.updateStepEndTime(taskInstanceId, stepInstanceId, endTime); + } + + @Override + public void addStepInstanceExecuteCount(Long taskInstanceId, long stepInstanceId) { + stepInstanceDAO.addStepInstanceExecuteCount(taskInstanceId, stepInstanceId); + } + + @Override + public void updateStepExecutionInfo(Long taskInstanceId, + long stepInstanceId, + RunStatusEnum status, + Long startTime, + Long endTime, + Long totalTime) { + stepInstanceDAO.updateStepExecutionInfo(taskInstanceId, stepInstanceId, status, startTime, endTime, totalTime); + } + + @Override + public void updateResolvedScriptParam(Long taskInstanceId, + long stepInstanceId, + boolean isSecureParam, + String resolvedScriptParam) { + stepInstanceDAO.updateResolvedScriptParam(taskInstanceId, stepInstanceId, isSecureParam, resolvedScriptParam); + } + + @Override + public void updateResolvedSourceFile(Long taskInstanceId, + long stepInstanceId, + List resolvedFileSources) { + if (log.isDebugEnabled()) { + log.debug("updateResolvedSourceFile={}", JsonUtils.toJson(resolvedFileSources)); + } + stepInstanceDAO.updateResolvedSourceFile(taskInstanceId, stepInstanceId, resolvedFileSources); + } + + @Override + public void updateResolvedTargetPath(Long taskInstanceId, long stepInstanceId, String resolvedTargetPath) { + stepInstanceDAO.updateResolvedTargetPath(taskInstanceId, stepInstanceId, resolvedTargetPath); + } + + @Override + public void updateConfirmReason(Long taskInstanceId, long stepInstanceId, String confirmReason) { + stepInstanceDAO.updateConfirmReason(taskInstanceId, stepInstanceId, confirmReason); + } + + @Override + public void updateStepOperator(Long taskInstanceId, long stepInstanceId, String operator) { + stepInstanceDAO.updateStepOperator(taskInstanceId, stepInstanceId, operator); + } + + @Override + public StepInstanceDTO getPreExecutableStepInstance(Long taskInstanceId, long stepInstanceId) { + StepInstanceBaseDTO currentStepInstance = stepInstanceDAO.getStepInstanceBase(taskInstanceId, stepInstanceId); + if (currentStepInstance == null) { + return null; + } + StepInstanceBaseDTO preStepInstance = stepInstanceDAO.getPreExecutableStepInstance(taskInstanceId, + currentStepInstance.getStepOrder()); + if (preStepInstance == null) { + return null; + } + StepInstanceDTO stepInstance = new StepInstanceDTO(preStepInstance); + fillStepInstanceDetail(stepInstance); + return stepInstance; + } + + @Override + public Map listStepInstanceIdAndStepOrderMapping(Long taskInstanceId) { + return stepInstanceDAO.listStepInstanceIdAndStepOrderMapping(taskInstanceId); + } + + @Override + public StepInstanceDTO getStepInstanceByTaskInstanceId(Long taskInstanceId) { + List stepInstanceList = + stepInstanceDAO.listStepInstanceBaseByTaskInstanceId(taskInstanceId); + if (CollectionUtils.isEmpty(stepInstanceList)) { + return null; + } + StepInstanceDTO stepInstance = new StepInstanceDTO(stepInstanceList.get(0)); + fillStepInstanceDetail(stepInstance); + return stepInstance; + } + + @Override + public Map computeStepExecuteObjects(StepInstanceBaseDTO stepInstance, + Function keyMapper) { + + Map executeObjects = new HashMap<>(); + stepInstance.getTargetExecuteObjects().getExecuteObjectsCompatibly() + .forEach(executeObject -> { + K key = keyMapper.apply(executeObject); + if (key != null) { + executeObjects.put(keyMapper.apply(executeObject), executeObject); + } + }); + if (stepInstance.isFileStep()) { List fileSourceList; if (stepInstance instanceof StepInstanceDTO) { fileSourceList = ((StepInstanceDTO) stepInstance).getFileSourceList(); } else { - FileStepInstanceDTO fileStepInstance = stepInstanceDAO.getFileStepInstance(stepInstance.getId()); + FileStepInstanceDTO fileStepInstance = stepInstanceDAO.getFileStepInstance( + stepInstance.getTaskInstanceId(), stepInstance.getId()); fileSourceList = fileStepInstance.getFileSourceList(); } @@ -92,20 +331,85 @@ public Map computeStepHosts(StepInstanceBaseDTO stepInstance, fileSourceList.forEach( fileSource -> { if (fileSource.getServers() != null - && CollectionUtils.isNotEmpty(fileSource.getServers().getIpList())) { - fileSource.getServers().getIpList().forEach(host -> hosts.put(keyMapper.apply(host), host)); + && CollectionUtils.isNotEmpty(fileSource.getServers().getExecuteObjectsCompatibly())) { + fileSource.getServers().getExecuteObjectsCompatibly() + .forEach(executeObject -> { + K key = keyMapper.apply(executeObject); + if (key != null) { + executeObjects.put(keyMapper.apply(executeObject), executeObject); + } + }); } }); } } - return hosts; + return executeObjects; } @Override - public Map computeStepHosts(long stepInstanceId, - Function keyMapper) { - StepInstanceBaseDTO stepInstance = stepInstanceDAO.getStepInstanceBase(stepInstanceId); - return computeStepHosts(stepInstance, keyMapper); + public ExecuteObject findExecuteObjectByCompositeKey(StepInstanceBaseDTO stepInstance, + ExecuteObjectCompositeKey executeObjectCompositeKey) { + if (stepInstance.isScriptStep()) { + return stepInstance.getTargetExecuteObjects().findExecuteObjectByCompositeKey(executeObjectCompositeKey); + } else if (stepInstance.isFileStep()) { + StepInstanceDTO fileStepInstance = castToStepInstanceDTO(stepInstance); + ExecuteObject executeObject = + fileStepInstance.getTargetExecuteObjects().findExecuteObjectByCompositeKey(executeObjectCompositeKey); + if (executeObject != null) { + return executeObject; + } + for (FileSourceDTO fileSource : fileStepInstance.getFileSourceList()) { + if (fileSource.getServers() == null) { + continue; + } + executeObject = fileSource.getServers().findExecuteObjectByCompositeKey(executeObjectCompositeKey); + if (executeObject != null) { + return executeObject; + } + } + log.warn("ExecuteObject not exist query by executeObjectCompositeKey: {}", executeObjectCompositeKey); + return null; + } else { + throw new InternalException("Not support method invoke for step", ErrorCode.INTERNAL_ERROR); + } + } + + private StepInstanceDTO castToStepInstanceDTO(StepInstanceBaseDTO stepInstance) { + if (stepInstance instanceof StepInstanceDTO) { + return (StepInstanceDTO) stepInstance; + } else { + return getStepInstanceDetail(stepInstance.getTaskInstanceId(), stepInstance.getId()); + } + } + + @Override + public List findExecuteObjectByCompositeKeys( + StepInstanceBaseDTO stepInstance, + Collection executeObjectCompositeKeys) { + if (stepInstance.isScriptStep()) { + return stepInstance.getTargetExecuteObjects().findExecuteObjectByCompositeKeys(executeObjectCompositeKeys); + } else if (stepInstance.isFileStep()) { + StepInstanceDTO fileStepInstance = castToStepInstanceDTO(stepInstance); + Set executeObjects = new HashSet<>(); + List matchTargetExecuteObjects = + fileStepInstance.getTargetExecuteObjects().findExecuteObjectByCompositeKeys(executeObjectCompositeKeys); + if (CollectionUtils.isNotEmpty(matchTargetExecuteObjects)) { + executeObjects.addAll(matchTargetExecuteObjects); + } + for (FileSourceDTO fileSource : fileStepInstance.getFileSourceList()) { + if (fileSource.getServers() == null) { + continue; + } + List fileSourceExecuteObjects = + fileSource.getServers().findExecuteObjectByCompositeKeys(executeObjectCompositeKeys); + if (CollectionUtils.isNotEmpty(fileSourceExecuteObjects)) { + executeObjects.addAll(fileSourceExecuteObjects); + } + } + return new ArrayList<>(executeObjects); + } else { + throw new InternalException("Not support method invoke for step", ErrorCode.INTERNAL_ERROR); + } } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceVariableValueServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceVariableValueServiceImpl.java index 1a2b8dbd62..e4f0808d58 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceVariableValueServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/StepInstanceVariableValueServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,19 +25,22 @@ package com.tencent.bk.job.execute.service.impl; import com.tencent.bk.job.common.constant.TaskVariableTypeEnum; -import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.common.util.ip.IpUtils; import com.tencent.bk.job.execute.dao.StepInstanceVariableDAO; +import com.tencent.bk.job.execute.dao.common.IdGen; import com.tencent.bk.job.execute.engine.model.TaskVariableDTO; import com.tencent.bk.job.execute.engine.model.TaskVariablesAnalyzeResult; import com.tencent.bk.job.execute.model.HostVariableValuesDTO; import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.StepInstanceVariableValuesDTO; import com.tencent.bk.job.execute.model.VariableValueDTO; +import com.tencent.bk.job.execute.service.StepInstanceService; import com.tencent.bk.job.execute.service.StepInstanceVariableValueService; -import com.tencent.bk.job.execute.service.TaskInstanceService; import com.tencent.bk.job.execute.service.TaskInstanceVariableService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -53,20 +56,24 @@ public class StepInstanceVariableValueServiceImpl implements StepInstanceVariableValueService { private final StepInstanceVariableDAO stepInstanceVariableDAO; - private final TaskInstanceService taskInstanceService; + private final StepInstanceService stepInstanceService; private final TaskInstanceVariableService taskInstanceVariableService; + private final IdGen idGen; @Autowired public StepInstanceVariableValueServiceImpl(StepInstanceVariableDAO stepInstanceVariableDAO, - TaskInstanceService taskInstanceService, - TaskInstanceVariableService taskInstanceVariableService) { + StepInstanceService stepInstanceService, + TaskInstanceVariableService taskInstanceVariableService, + IdGen idGen) { this.stepInstanceVariableDAO = stepInstanceVariableDAO; - this.taskInstanceService = taskInstanceService; + this.stepInstanceService = stepInstanceService; this.taskInstanceVariableService = taskInstanceVariableService; + this.idGen = idGen; } @Override public void saveVariableValues(StepInstanceVariableValuesDTO variableValues) { + variableValues.setId(idGen.genStepInstanceVariableId()); stepInstanceVariableDAO.saveVariableValues(variableValues); } @@ -74,7 +81,7 @@ public void saveVariableValues(StepInstanceVariableValuesDTO variableValues) { public List computeOutputVariableValuesForAllStep(long taskInstanceId) { List resultStepInstanceVariableValuesList = new ArrayList<>(); List stepInstanceList = - taskInstanceService.listStepInstanceByTaskInstanceId(taskInstanceId); + stepInstanceService.listBaseStepInstanceByTaskInstanceId(taskInstanceId); if (CollectionUtils.isEmpty(stepInstanceList)) { log.info("Step instance is empty! taskInstanceId: {}", taskInstanceId); return resultStepInstanceVariableValuesList; @@ -86,12 +93,10 @@ public List computeOutputVariableValuesForAllStep if (CollectionUtils.isEmpty(globalVars)) { return resultStepInstanceVariableValuesList; } - Map globalVarValueMap = new HashMap<>(); - initGlobalVarMap(globalVars, globalVarValueMap); - + Map globalVarValueMap = initGlobalVarMap(globalVars); stepInstanceList.forEach(stepInstance -> { - if (!StepExecuteTypeEnum.EXECUTE_SCRIPT.getValue().equals(stepInstance.getExecuteType())) { + if (!stepInstance.isScriptStep()) { return; } StepInstanceVariableValuesDTO resultStepInstanceVariableValues = new StepInstanceVariableValuesDTO(); @@ -120,28 +125,25 @@ public List computeOutputVariableValuesForAllStep } @Override - public StepInstanceVariableValuesDTO computeInputStepInstanceVariableValues(long taskInstanceId, - long stepInstanceId, + public StepInstanceVariableValuesDTO computeInputStepInstanceVariableValues(StepInstanceBaseDTO stepInstance, List taskVariables) { TaskVariablesAnalyzeResult variablesAnalyzeResult = new TaskVariablesAnalyzeResult(taskVariables); - StepInstanceVariableValuesDTO inputStepInstanceVariableValues = new StepInstanceVariableValuesDTO(); - inputStepInstanceVariableValues.setTaskInstanceId(taskInstanceId); - inputStepInstanceVariableValues.setStepInstanceId(stepInstanceId); if (!variablesAnalyzeResult.isExistAnyVar()) { - return inputStepInstanceVariableValues; + // 如果不存在任何变量，无需进一步处理 + return null; } + StepInstanceVariableValuesDTO inputStepInstanceVariableValues = new StepInstanceVariableValuesDTO(); + inputStepInstanceVariableValues.setTaskInstanceId(stepInstance.getTaskInstanceId()); + inputStepInstanceVariableValues.setStepInstanceId(stepInstance.getId()); - List namespaceVarValues = new ArrayList<>(); + // 初始化全局变量 List globalVarValues = new ArrayList<>(); - Map globalVarValueMap = new HashMap<>(); - Map> namespaceVarValueMap = new HashMap<>(); - inputStepInstanceVariableValues.setNamespaceParams(namespaceVarValues); - inputStepInstanceVariableValues.setNamespaceParamsMap(namespaceVarValueMap); inputStepInstanceVariableValues.setGlobalParams(globalVarValues); + // key: varName value: varValue + Map globalVarValueMap = initGlobalVarMap(taskVariables); inputStepInstanceVariableValues.setGlobalParamsMap(globalVarValueMap); - initGlobalVarMap(taskVariables, globalVarValueMap); - + // 如果作业只包含常量，由于变量值不可变，可以直接返回初始全局变量的值 if (variablesAnalyzeResult.isExistOnlyConstVar()) { if (!globalVarValueMap.isEmpty()) { globalVarValueMap.forEach((paramName, param) -> globalVarValues.add(param)); @@ -149,45 +151,102 @@ public StepInstanceVariableValuesDTO computeInputStepInstanceVariableValues(long return inputStepInstanceVariableValues; } - List stepInstanceVariableValuesList = - stepInstanceVariableDAO.listSortedPreStepOutputVariableValues(taskInstanceId, stepInstanceId); - if (CollectionUtils.isEmpty(stepInstanceVariableValuesList)) { + // 如果包含可变变量，那么需要获取前面所有步骤的输出变量值来进行处理 + List preStepInstanceVariableValuesList = + listPreStepOutputVariableValuesByStepOrder(stepInstance.getTaskInstanceId(), stepInstance.getStepOrder()); + if (CollectionUtils.isEmpty(preStepInstanceVariableValuesList)) { if (!globalVarValueMap.isEmpty()) { globalVarValueMap.forEach((paramName, param) -> globalVarValues.add(param)); } return inputStepInstanceVariableValues; } + // 按步骤执行先后顺序覆盖更新全局变量值 + updateGlobalVarValues(preStepInstanceVariableValuesList, globalVarValues, globalVarValueMap); + + // 处理命名空间变量 + if (variablesAnalyzeResult.isExistNamespaceVar()) { + updateNamespaceVarValues(inputStepInstanceVariableValues, preStepInstanceVariableValuesList); + } + + return inputStepInstanceVariableValues; + } + + private List listPreStepOutputVariableValuesByStepOrder(long taskInstanceId, + int currentStepOrder) { + // 获取全量步骤变量 + List stepInstanceVariableValuesList = + stepInstanceVariableDAO.listStepOutputVariableValuesByTaskInstanceId(taskInstanceId); + if (CollectionUtils.isEmpty(stepInstanceVariableValuesList)) { + return null; + } + Map stepInstanceIdAndOrderMap = stepInstanceService.listStepInstanceIdAndStepOrderMapping( + taskInstanceId); + + return stepInstanceVariableValuesList.stream() + .filter(var -> { + int stepOrder = stepInstanceIdAndOrderMap.get(var.getStepInstanceId()); + // 只保留前置步骤 + return stepOrder < currentStepOrder; + }).sorted((var1, var2) -> { + int stepOrderForVar1 = stepInstanceIdAndOrderMap.get(var1.getStepInstanceId()); + int stepOrderForVar2 = stepInstanceIdAndOrderMap.get(var2.getStepInstanceId()); + // 根据 stepOrder 升序排列 + return Integer.compare(stepOrderForVar1, stepOrderForVar2); + }).collect(Collectors.toList()); + } + + private void updateGlobalVarValues(List stepInstanceVariableValuesList, + List globalVarValues, + Map globalVarValueMap) { stepInstanceVariableValuesList.forEach(stepInstanceVariableValues -> { List stepGlobalParams = stepInstanceVariableValues.getGlobalParams(); if (CollectionUtils.isNotEmpty(stepGlobalParams)) { + // 覆盖全局变量初始值 stepGlobalParams.forEach(globalParam -> globalVarValueMap.put(globalParam.getName(), globalParam)); } - - if (variablesAnalyzeResult.isExistNamespaceVar()) { - List stepNamespaceParams = stepInstanceVariableValues.getNamespaceParams(); - if (CollectionUtils.isNotEmpty(stepNamespaceParams)) { - stepNamespaceParams.forEach(hostVariableValues -> { - if (CollectionUtils.isEmpty(hostVariableValues.getValues())) { - return; - } - Map hostVariables = namespaceVarValueMap.computeIfAbsent( - hostVariableValues.getIp(), k -> new HashMap<>()); - hostVariableValues.getValues().forEach(variable -> hostVariables.put(variable.getName(), - variable)); - }); - } - } }); - if (!globalVarValueMap.isEmpty()) { globalVarValueMap.forEach((paramName, param) -> globalVarValues.add(param)); } + } - if (variablesAnalyzeResult.isExistNamespaceVar()) { - namespaceVarValueMap.forEach((ip, param) -> { + private void updateNamespaceVarValues(StepInstanceVariableValuesDTO inputStepInstanceVariableValues, + List preStepInstanceVariableValuesList) { + if (CollectionUtils.isEmpty(preStepInstanceVariableValuesList)) { + return; + } + + List namespaceVarValues = new ArrayList<>(); + inputStepInstanceVariableValues.setNamespaceParams(namespaceVarValues); + Map> namespaceParamsMap = new HashMap<>(); + inputStepInstanceVariableValues.setNamespaceParamsMap(namespaceParamsMap); + + preStepInstanceVariableValuesList + .stream() + .filter(stepInstanceVariableValues -> CollectionUtils.isNotEmpty(stepInstanceVariableValues.getNamespaceParams())) + .forEach(stepInstanceVariableValues -> + stepInstanceVariableValues.getNamespaceParams().forEach(hostVar -> { + HostDTO host = new HostDTO(); + host.setHostId(hostVar.getHostId()); + if (StringUtils.isNotBlank(hostVar.getCloudIpv4())) { + host.setBkCloudId(IpUtils.extractBkCloudId(hostVar.getCloudIpv4())); + host.setIp(IpUtils.extractIp(hostVar.getCloudIpv4())); + } + if (StringUtils.isNotBlank(hostVar.getCloudIpv6())) { + host.setBkCloudId(IpUtils.extractBkCloudId(hostVar.getCloudIpv6())); + host.setIpv6(IpUtils.extractIp(hostVar.getCloudIpv6())); + } + Map hostVariables = namespaceParamsMap.computeIfAbsent(host, + k -> new HashMap<>()); + hostVar.getValues().forEach(variable -> hostVariables.put(variable.getName(), variable)); + })); + if (!namespaceParamsMap.isEmpty()) { + namespaceParamsMap.forEach((host, param) -> { HostVariableValuesDTO hostVariableValues = new HostVariableValuesDTO(); - hostVariableValues.setIp(ip); + hostVariableValues.setHostId(host.getHostId()); + hostVariableValues.setCloudIpv4(host.toCloudIp()); + hostVariableValues.setCloudIpv6(host.toCloudIpv6()); if (param != null && !param.isEmpty()) { List values = new ArrayList<>(); param.forEach((paramName, paramValue) -> values.add(paramValue)); @@ -196,21 +255,20 @@ public StepInstanceVariableValuesDTO computeInputStepInstanceVariableValues(long namespaceVarValues.add(hostVariableValues); }); } - - return inputStepInstanceVariableValues; } - private void initGlobalVarMap(List taskVariables, - Map globalVarValueMap) { + private Map initGlobalVarMap(List taskVariables) { + Map globalVarValueMap = new HashMap<>(); taskVariables.forEach(taskVariable -> { VariableValueDTO variableValue = new VariableValueDTO(); variableValue.setName(taskVariable.getName()); variableValue.setType(taskVariable.getType()); variableValue.setValue(taskVariable.getValue()); - variableValue.setServerValue(taskVariable.getTargetServers()); + variableValue.setServerValue(taskVariable.getExecuteTarget()); if (TaskVariableTypeEnum.valOf(taskVariable.getType()) != TaskVariableTypeEnum.NAMESPACE) { globalVarValueMap.put(variableValue.getName(), variableValue); } }); + return globalVarValueMap; } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java index 2274be3a9f..858a2f3335 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskExecuteServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,13 @@ package com.tencent.bk.job.execute.service.impl; -import com.tencent.bk.job.common.cc.model.CcInstanceDTO; +import com.tencent.bk.audit.context.ActionAuditContext; +import com.tencent.bk.audit.context.AuditContext; +import com.tencent.bk.audit.utils.AuditInstanceUtils; +import com.tencent.bk.job.common.audit.JobAuditAttributeNames; +import com.tencent.bk.job.common.audit.JobAuditExtendDataKeys; +import com.tencent.bk.job.common.audit.constants.EventContentConstants; +import com.tencent.bk.job.common.constant.AccountCategoryEnum; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.constant.TaskVariableTypeEnum; import com.tencent.bk.job.common.exception.AbortedException; @@ -33,22 +39,29 @@ import com.tencent.bk.job.common.exception.NotFoundException; import com.tencent.bk.job.common.exception.ResourceExhaustedException; import com.tencent.bk.job.common.exception.ServiceException; -import com.tencent.bk.job.common.gse.constants.AgentStatusEnum; -import com.tencent.bk.job.common.gse.service.QueryAgentStatusClient; +import com.tencent.bk.job.common.iam.constant.ActionId; +import com.tencent.bk.job.common.iam.constant.ResourceTypeId; import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; import com.tencent.bk.job.common.iam.model.AuthResult; import com.tencent.bk.job.common.model.InternalResponse; import com.tencent.bk.job.common.model.dto.AppResourceScope; import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.common.model.dto.ResourceScope; import com.tencent.bk.job.common.util.ArrayUtil; +import com.tencent.bk.job.common.util.DataSizeConverter; import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.execute.audit.ExecuteJobAuditEventBuilder; import com.tencent.bk.job.execute.auth.ExecuteAuthService; -import com.tencent.bk.job.execute.client.ServiceUserResourceClient; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; import com.tencent.bk.job.execute.common.constants.TaskStartupModeEnum; import com.tencent.bk.job.execute.common.constants.TaskTypeEnum; +import com.tencent.bk.job.execute.common.context.JobExecuteContext; +import com.tencent.bk.job.execute.common.context.JobExecuteContextThreadLocalRepo; +import com.tencent.bk.job.execute.common.context.JobInstanceContext; +import com.tencent.bk.job.execute.common.converter.StepTypeExecuteTypeConverter; +import com.tencent.bk.job.execute.common.exception.RunningJobQuotaLimitExceedException; import com.tencent.bk.job.execute.config.JobExecuteConfig; import com.tencent.bk.job.execute.constants.ScriptSourceEnum; import com.tencent.bk.job.execute.constants.StepOperationEnum; @@ -59,25 +72,26 @@ import com.tencent.bk.job.execute.engine.listener.event.StepEvent; import com.tencent.bk.job.execute.engine.listener.event.TaskExecuteMQEventDispatcher; import com.tencent.bk.job.execute.engine.model.TaskVariableDTO; +import com.tencent.bk.job.execute.engine.quota.limit.ResourceQuotaCheckResultEnum; +import com.tencent.bk.job.execute.engine.quota.limit.RunningJobResourceQuotaManager; import com.tencent.bk.job.execute.engine.util.TimeoutUtils; import com.tencent.bk.job.execute.model.AccountDTO; import com.tencent.bk.job.execute.model.DynamicServerGroupDTO; import com.tencent.bk.job.execute.model.DynamicServerTopoNodeDTO; +import com.tencent.bk.job.execute.model.ExecuteTargetDTO; import com.tencent.bk.job.execute.model.FastTaskDTO; import com.tencent.bk.job.execute.model.FileDetailDTO; import com.tencent.bk.job.execute.model.FileSourceDTO; import com.tencent.bk.job.execute.model.OperationLogDTO; import com.tencent.bk.job.execute.model.RollingConfigDTO; -import com.tencent.bk.job.execute.model.ServersDTO; -import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; import com.tencent.bk.job.execute.model.StepInstanceDTO; import com.tencent.bk.job.execute.model.StepOperationDTO; import com.tencent.bk.job.execute.model.TaskExecuteParam; import com.tencent.bk.job.execute.model.TaskInfo; import com.tencent.bk.job.execute.model.TaskInstanceDTO; +import com.tencent.bk.job.execute.model.TaskInstanceExecuteObjects; import com.tencent.bk.job.execute.service.AccountService; import com.tencent.bk.job.execute.service.DangerousScriptCheckService; -import com.tencent.bk.job.execute.service.HostService; import com.tencent.bk.job.execute.service.RollingConfigService; import com.tencent.bk.job.execute.service.ScriptService; import com.tencent.bk.job.execute.service.StepInstanceService; @@ -87,17 +101,18 @@ import com.tencent.bk.job.execute.service.TaskOperationLogService; import com.tencent.bk.job.execute.service.TaskPlanService; import com.tencent.bk.job.execute.util.LoggerFactory; -import com.tencent.bk.job.manage.common.consts.JobResourceStatusEnum; -import com.tencent.bk.job.manage.common.consts.account.AccountCategoryEnum; -import com.tencent.bk.job.manage.common.consts.notify.JobRoleEnum; -import com.tencent.bk.job.manage.common.consts.notify.ResourceTypeEnum; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskFileTypeEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskStepTypeEnum; -import com.tencent.bk.job.manage.common.consts.whiteip.ActionScopeEnum; +import com.tencent.bk.job.manage.GlobalAppScopeMappingService; +import com.tencent.bk.job.manage.api.common.constants.JobResourceStatusEnum; +import com.tencent.bk.job.manage.api.common.constants.notify.JobRoleEnum; +import com.tencent.bk.job.manage.api.common.constants.notify.ResourceTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskFileTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskStepTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.whiteip.ActionScopeEnum; +import com.tencent.bk.job.manage.api.inner.ServiceTaskTemplateResource; +import com.tencent.bk.job.manage.api.inner.ServiceUserResource; import com.tencent.bk.job.manage.model.inner.ServiceAccountDTO; import com.tencent.bk.job.manage.model.inner.ServiceHostInfoDTO; -import com.tencent.bk.job.manage.model.inner.ServiceListAppHostResultDTO; import com.tencent.bk.job.manage.model.inner.ServiceScriptCheckResultItemDTO; import com.tencent.bk.job.manage.model.inner.ServiceScriptDTO; import com.tencent.bk.job.manage.model.inner.ServiceTaskApprovalStepDTO; @@ -113,11 +128,9 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.ImmutablePair; import org.apache.commons.lang3.tuple.Pair; import org.slf4j.Logger; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; import org.springframework.util.StopWatch; @@ -130,11 +143,6 @@ import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.Callable; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Future; import java.util.stream.Collectors; import static com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum.EXECUTE_SCRIPT; @@ -150,18 +158,19 @@ public class TaskExecuteServiceImpl implements TaskExecuteService { private final TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher; private final TaskPlanService taskPlanService; private final TaskInstanceVariableService taskInstanceVariableService; - private final QueryAgentStatusClient queryAgentStatusClient; private final TaskOperationLogService taskOperationLogService; private final TaskInstanceService taskInstanceService; private final StepInstanceService stepInstanceService; - private final HostService hostService; - private final ServiceUserResourceClient userResource; + private final ServiceUserResource userResource; private final ExecuteAuthService executeAuthService; - private final ExecutorService getHostsByTopoExecutor; private final DangerousScriptCheckService dangerousScriptCheckService; private final RollingConfigService rollingConfigService; private final JobExecuteConfig jobExecuteConfig; private final TaskEvictPolicyExecutor taskEvictPolicyExecutor; + private final ServiceTaskTemplateResource taskTemplateResource; + private final TaskInstanceExecuteObjectProcessor taskInstanceExecuteObjectProcessor; + + private final RunningJobResourceQuotaManager runningJobResourceQuotaManager; private static final Logger TASK_MONITOR_LOGGER = LoggerFactory.TASK_MONITOR_LOGGER; @@ -171,116 +180,147 @@ public TaskExecuteServiceImpl(AccountService accountService, TaskExecuteMQEventDispatcher taskExecuteMQEventDispatcher, TaskPlanService taskPlanService, TaskInstanceVariableService taskInstanceVariableService, - QueryAgentStatusClient queryAgentStatusClient, TaskOperationLogService taskOperationLogService, ScriptService scriptService, StepInstanceService stepInstanceService, - HostService hostService, - ServiceUserResourceClient userResource, + ServiceUserResource userResource, ExecuteAuthService executeAuthService, - @Qualifier("getHostsByTopoExecutor") ExecutorService getHostsByTopoExecutor, DangerousScriptCheckService dangerousScriptCheckService, JobExecuteConfig jobExecuteConfig, TaskEvictPolicyExecutor taskEvictPolicyExecutor, - RollingConfigService rollingConfigService) { + RollingConfigService rollingConfigService, + ServiceTaskTemplateResource taskTemplateResource, + TaskInstanceExecuteObjectProcessor taskInstanceExecuteObjectProcessor, + RunningJobResourceQuotaManager runningJobResourceQuotaManager) { this.accountService = accountService; this.taskInstanceService = taskInstanceService; this.taskExecuteMQEventDispatcher = taskExecuteMQEventDispatcher; this.taskPlanService = taskPlanService; this.taskInstanceVariableService = taskInstanceVariableService; - this.queryAgentStatusClient = queryAgentStatusClient; this.taskOperationLogService = taskOperationLogService; this.scriptService = scriptService; this.stepInstanceService = stepInstanceService; - this.hostService = hostService; this.userResource = userResource; this.executeAuthService = executeAuthService; - this.getHostsByTopoExecutor = getHostsByTopoExecutor; this.dangerousScriptCheckService = dangerousScriptCheckService; this.rollingConfigService = rollingConfigService; this.jobExecuteConfig = jobExecuteConfig; this.taskEvictPolicyExecutor = taskEvictPolicyExecutor; + this.taskTemplateResource = taskTemplateResource; + this.taskInstanceExecuteObjectProcessor = taskInstanceExecuteObjectProcessor; + this.runningJobResourceQuotaManager = runningJobResourceQuotaManager; } @Override - public Long executeFastTask(FastTaskDTO fastTask) { + public TaskInstanceDTO executeFastTask(FastTaskDTO fastTask) { + // 设置脚本信息 + checkAndSetScript(fastTask.getTaskInstance(), fastTask.getStepInstance()); + + StepInstanceDTO stepInstance = fastTask.getStepInstance(); + + ActionAuditContext actionAuditContext; + if (stepInstance.isFileStep()) { + actionAuditContext = ActionAuditContext.builder(ActionId.QUICK_TRANSFER_FILE) + .setEventBuilder(ExecuteJobAuditEventBuilder.class) + .setContent(EventContentConstants.QUICK_TRANSFER_FILE) + .build(); + } else if (stepInstance.isScriptStep()) { + ScriptSourceEnum scriptSource = ScriptSourceEnum.getScriptSourceEnum(stepInstance.getScriptSource()); + if (scriptSource == ScriptSourceEnum.CUSTOM) { + actionAuditContext = ActionAuditContext.builder(ActionId.QUICK_EXECUTE_SCRIPT) + .setEventBuilder(ExecuteJobAuditEventBuilder.class) + .setContent(EventContentConstants.QUICK_EXECUTE_SCRIPT) + .build(); + } else if (scriptSource == ScriptSourceEnum.QUOTED_APP) { + actionAuditContext = ActionAuditContext.builder(ActionId.EXECUTE_SCRIPT) + .setEventBuilder(ExecuteJobAuditEventBuilder.class) + .setContent(EventContentConstants.EXECUTE_SCRIPT) + .build(); + } else if (scriptSource == ScriptSourceEnum.QUOTED_PUBLIC) { + actionAuditContext = ActionAuditContext.builder(ActionId.EXECUTE_PUBLIC_SCRIPT) + .setEventBuilder(ExecuteJobAuditEventBuilder.class) + .setContent(EventContentConstants.EXECUTE_PUBLIC_SCRIPT) + .build(); + } else { + actionAuditContext = ActionAuditContext.INVALID; + } + } else { + actionAuditContext = ActionAuditContext.INVALID; + } + + AuditContext.current().updateActionId(actionAuditContext.getActionId()); + + return actionAuditContext.wrapActionCallable(() -> executeFastTaskInternal(fastTask)).call(); + } + + private TaskInstanceDTO executeFastTaskInternal(FastTaskDTO fastTask) { log.info("Begin to execute fast task: {}", fastTask); + + long appId = fastTask.getTaskInstance().getAppId(); TaskInstanceDTO taskInstance = fastTask.getTaskInstance(); StepInstanceDTO stepInstance = fastTask.getStepInstance(); StopWatch watch = new StopWatch("executeFastTask"); - // 检查任务是否应当被驱逐 - checkTaskEvict(taskInstance); - standardizeStepDynamicGroupId(Collections.singletonList(stepInstance)); - adjustStepTimeout(stepInstance); try { - // 设置脚本信息 - watch.start("checkAndSetScriptInfoIfScriptTask"); - checkAndSetScriptInfoForFast(taskInstance, stepInstance); + watch.start("checkRunningJobQuoteLimit"); + // 检查正在执行的作业配额限制 + checkRunningJobQuotaLimit(appId, taskInstance.getAppCode()); watch.stop(); - // 设置账号信息 - watch.start("setAccountInfo"); - checkAndSetAccountInfo(stepInstance, taskInstance.getAppId()); - watch.stop(); + // 检查任务是否应当被驱逐 + checkTaskEvict(taskInstance); + + standardizeStepDynamicGroupId(Collections.singletonList(stepInstance)); + adjustStepTimeout(stepInstance); - // 获取主机列表 - watch.start("setServerInfoFastJob"); - setServerInfoFastJob(stepInstance); + // 设置账号信息 + watch.start("checkAndSetAccountInfo"); + checkAndSetAccountInfo(stepInstance, appId); watch.stop(); - //检查主机 - watch.start("checkHosts"); - checkAndSetHosts(Collections.singletonList(stepInstance)); + // 处理执行对象 + watch.start("processExecuteObjects"); + TaskInstanceExecuteObjects taskInstanceExecuteObjects = + taskInstanceExecuteObjectProcessor.processExecuteObjects(taskInstance, + Collections.singletonList(stepInstance), null); watch.stop(); - // 检查步骤约束 - watch.start("checkStepInstanceConstraint"); - checkStepInstanceConstraint(taskInstance, Collections.singletonList(stepInstance)); + // 检查步骤 + watch.start("checkStepInstance"); + checkStepInstance(taskInstance, Collections.singletonList(stepInstance)); watch.stop(); // 鉴权 watch.start("authFastExecute"); - authFastExecute(taskInstance, stepInstance); + authFastExecute(taskInstance, stepInstance, taskInstanceExecuteObjects.getWhiteHostAllowActions()); watch.stop(); - // 保存作业、步骤实例 - watch.start("saveInstance"); - Long taskInstanceId = taskInstanceService.addTaskInstance(taskInstance); - taskInstance.setId(taskInstanceId); - stepInstance.setTaskInstanceId(taskInstanceId); - stepInstance.setStepNum(1); - stepInstance.setStepOrder(1); - long stepInstanceId = taskInstanceService.addStepInstance(stepInstance); - stepInstance.setId(stepInstanceId); - watch.stop(); - - // 保存作业实例与主机的关系，优化根据主机检索作业执行历史的效率 - watch.start("saveTaskInstanceHosts"); - saveTaskInstanceHosts(taskInstanceId, Collections.singletonList(stepInstance)); - watch.stop(); - - // 保存滚动配置 - if (fastTask.isRollingEnabled()) { - watch.start("saveRollingConfig"); - RollingConfigDTO rollingConfig = rollingConfigService.saveRollingConfigForFastJob(fastTask); - long rollingConfigId = rollingConfig.getId(); - stepInstanceService.updateStepRollingConfigId(stepInstanceId, rollingConfigId); - watch.stop(); - } + // 保存作业 + saveTaskInstance(watch, fastTask, taskInstance, stepInstance); // 记录操作日志 watch.start("saveOperationLog"); taskOperationLogService.saveOperationLog(buildTaskOperationLog(taskInstance, taskInstance.getOperator(), - UserOperationEnum.START)); + fastTask.getStartTask() ? UserOperationEnum.START : UserOperationEnum.CREATE_JOB)); watch.stop(); // 启动作业 - watch.start("startJob"); - startTask(taskInstanceId); - watch.stop(); + if (fastTask.getStartTask()) { + watch.start("startJob"); + startTask(taskInstance.getId()); + watch.stop(); + } + + // 审计 + Set allHosts = taskInstanceExecuteObjectProcessor.extractHosts( + Collections.singletonList(stepInstance), null); + taskInstance.setAllHosts(allHosts); + auditFastJobExecute(taskInstance); + + // 日志记录容器执行对象的作业，用于统计、分析 + logContainerExecuteObjectJob(taskInstance, taskInstanceExecuteObjects); - return taskInstanceId; + return taskInstance; } finally { if (watch.isRunning()) { watch.stop(); @@ -291,10 +331,156 @@ public Long executeFastTask(FastTaskDTO fastTask) { } } - private void saveTaskInstanceHosts(long taskInstanceId, + /* + * 对于包含容器执行对象的作业，输出日志用于统计、分析（后续版本删除） + */ + private void logContainerExecuteObjectJob(TaskInstanceDTO taskInstance, + TaskInstanceExecuteObjects taskInstanceExecuteObjects) { + if (taskInstanceExecuteObjects.isContainsAnyContainer()) { + log.info("ContainerJobRecord -> resourceScope|{}|appCode|{}|jobInstanceId|{}|name|{}", + GlobalAppScopeMappingService.get().getScopeByAppId(taskInstance.getAppId()).toResourceScopeUniqueId(), + StringUtils.isNotEmpty(taskInstance.getAppCode()) ? taskInstance.getAppCode() : "None", + taskInstance.getId(), + taskInstance.getName() + ); + } + } + + /* + * 检查正在执行的作业配额限制，防止单个业务占用所有的执行引擎调度资源 + */ + private void checkRunningJobQuotaLimit(Long appId, String appCode) { + ResourceScope resourceScope = GlobalAppScopeMappingService.get().getScopeByAppId(appId); + ResourceQuotaCheckResultEnum checkResult = runningJobResourceQuotaManager.checkResourceQuotaLimit( + appCode, resourceScope); + switch (checkResult) { + case NO_LIMIT: + break; + case RESOURCE_SCOPE_LIMIT: + log.warn("ResourceQuotaLimit-runningJob exceed resource scope quota limit, resourceScope: {}", + resourceScope.toResourceScopeUniqueId()); + throw new RunningJobQuotaLimitExceedException(ErrorCode.RUNNING_JOB_EXCEED_RESOURCE_SCOPE_QUOTA_LIMIT); + case APP_LIMIT: + log.warn("ResourceQuotaLimit-runningJob exceed app quota limit, appCode: {}", appCode); + throw new RunningJobQuotaLimitExceedException(ErrorCode.RUNNING_JOB_EXCEED_APP_QUOTA_LIMIT); + case SYSTEM_LIMIT: + log.warn("ResourceQuotaLimit-runningJob exceed system quota limit, resourceScope: {}, appCode: {}", + resourceScope.toResourceScopeUniqueId(), appCode); + throw new RunningJobQuotaLimitExceedException(ErrorCode.RUNNING_JOB_EXCEED_SYSTEM_QUOTA_LIMIT); + } + } + + private void saveTaskInstance(StopWatch watch, + FastTaskDTO fastTask, + TaskInstanceDTO taskInstance, + StepInstanceDTO stepInstance) { + // 保存作业、步骤实例 + watch.start("saveInstance"); + long taskInstanceId = taskInstanceService.addTaskInstance(taskInstance); + taskInstance.setId(taskInstanceId); + + // 添加作业执行上下文，用于全局共享、传播上下文信息 + addJobInstanceContext(taskInstance); + + stepInstance.setTaskInstanceId(taskInstanceId); + stepInstance.setStepNum(1); + stepInstance.setStepOrder(1); + long stepInstanceId = stepInstanceService.addStepInstance(stepInstance); + stepInstance.setId(stepInstanceId); + taskInstance.setStepInstances(Collections.singletonList(stepInstance)); + watch.stop(); + + // 保存作业实例与主机的关系，优化根据主机检索作业执行历史的效率 + watch.start("saveTaskInstanceHosts"); + saveTaskInstanceHosts(taskInstance.getAppId(), taskInstanceId, Collections.singletonList(stepInstance)); + watch.stop(); + + // 保存滚动配置 + if (fastTask.isRollingEnabled()) { + watch.start("saveRollingConfig"); + RollingConfigDTO rollingConfig = rollingConfigService.saveRollingConfigForFastJob(fastTask); + long rollingConfigId = rollingConfig.getId(); + stepInstanceService.updateStepRollingConfigId(taskInstanceId, stepInstanceId, rollingConfigId); + watch.stop(); + } + + // 记录到当前正在运行的任务存储中，用于配额限制 + watch.start("addRunningJobResourceQuota"); + runningJobResourceQuotaManager.addJob( + taskInstance.getAppCode(), + GlobalAppScopeMappingService.get().getScopeByAppId(taskInstance.getAppId()), + taskInstanceId + ); + watch.stop(); + } + + private void addJobInstanceContext(TaskInstanceDTO taskInstance) { + JobExecuteContext jobExecuteContext = JobExecuteContextThreadLocalRepo.get(); + if (jobExecuteContext != null) { + JobInstanceContext jobInstanceContext = new JobInstanceContext(taskInstance.getId()); + jobExecuteContext.setJobInstanceContext(jobInstanceContext); + JobExecuteContextThreadLocalRepo.set(jobExecuteContext); + } + } + + private void auditFastJobExecute(TaskInstanceDTO taskInstance) { + addFastJobExecuteAuditInstance(taskInstance); + addJobInstanceInfoToExtendData(taskInstance); + } + + private void addFastJobExecuteAuditInstance(TaskInstanceDTO taskInstance) { + setHostAuditInstances(taskInstance.getAllHosts()); + // 快速执行任务，只有单个步骤 + StepInstanceDTO stepInstance = taskInstance.getStepInstances().get(0); + if (stepInstance.getScriptVersionId() != null) { + ActionAuditContext.current().addAttribute(JobAuditAttributeNames.SCRIPT_VERSION_ID, + stepInstance.getScriptVersionId()); + } + if (StringUtils.isNotBlank(stepInstance.getScriptName())) { + ActionAuditContext.current().addAttribute(JobAuditAttributeNames.SCRIPT_NAME, + stepInstance.getScriptName()); + } + } + + private void setHostAuditInstances(Collection hosts) { + ActionAuditContext.current() + .setInstanceIdList( + AuditInstanceUtils.mapInstanceList(hosts, host -> String.valueOf(host.getHostId())) + ) + .setInstanceNameList( + AuditInstanceUtils.mapInstanceList(hosts, HostDTO::getPrimaryIp) + ); + } + + private void auditJobPlanExecute(TaskInstanceDTO taskInstance) { + addExecuteJobPlanAuditInstance(taskInstance); + addJobInstanceInfoToExtendData(taskInstance); + } + + private void addExecuteJobPlanAuditInstance(TaskInstanceDTO taskInstance) { + setHostAuditInstances(taskInstance.getAllHosts()); + ActionAuditContext.current() + .addAttribute(JobAuditAttributeNames.PLAN_ID, taskInstance.getPlanId()) + .addAttribute(JobAuditAttributeNames.PLAN_NAME, taskInstance.getPlan().getName()); + } + + private void addJobInstanceInfoToExtendData(TaskInstanceDTO taskInstance) { + ActionAuditContext.current() + .addExtendData(JobAuditExtendDataKeys.JOB_INSTANCE_ID, taskInstance.getId()); + } + + private void saveTaskInstanceHosts(long appId, + long taskInstanceId, List stepInstanceList) { - Set stepHosts = extractHosts(stepInstanceList); - taskInstanceService.saveTaskInstanceHosts(taskInstanceId, stepHosts); + Set stepHosts = taskInstanceExecuteObjectProcessor.extractHosts(stepInstanceList, null); + saveTaskInstanceHosts(appId, taskInstanceId, stepHosts); + } + + private void saveTaskInstanceHosts(long appId, long taskInstanceId, Collection hosts) { + if (CollectionUtils.isEmpty(hosts)) { + return; + } + taskInstanceService.saveTaskInstanceHosts(appId, taskInstanceId, hosts); } private void checkTaskEvict(TaskInstanceDTO taskInstance) { @@ -314,7 +500,7 @@ private void adjustStepTimeout(StepInstanceDTO stepInstance) { private void checkAndSetAccountInfo(StepInstanceDTO stepInstance, Long appId) throws ServiceException { - if (stepInstance.getExecuteType().equals(EXECUTE_SQL.getValue())) { + if (stepInstance.getExecuteType() == EXECUTE_SQL) { checkAndSetDbAccountInfo(stepInstance); } else { checkAndSetOsAccountInfo(stepInstance, appId); @@ -403,13 +589,10 @@ private void checkAndSetDbAccountInfo(StepInstanceDTO stepInstance) { } } - private ServiceScriptDTO checkAndSetScriptInfoForFast( - TaskInstanceDTO taskInstance, - StepInstanceDTO stepInstance) throws ServiceException { + private void checkAndSetScript(TaskInstanceDTO taskInstance, StepInstanceDTO stepInstance) { long appId = taskInstance.getAppId(); ServiceScriptDTO script = null; - if (stepInstance.getExecuteType().equals(EXECUTE_SCRIPT.getValue()) - || stepInstance.getExecuteType().equals(EXECUTE_SQL.getValue())) { + if (stepInstance.isScriptStep()) { boolean isScriptSpecifiedById = false; Long scriptVersionId = stepInstance.getScriptVersionId(); if (scriptVersionId != null && scriptVersionId > 0) { @@ -431,7 +614,7 @@ private ServiceScriptDTO checkAndSetScriptInfoForFast( } stepInstance.setScriptContent(script.getContent()); stepInstance.setScriptName(script.getName()); - stepInstance.setScriptType(script.getType()); + stepInstance.setScriptType(ScriptTypeEnum.valOf(script.getType())); if (script.isPublicScript()) { stepInstance.setScriptSource(ScriptSourceEnum.QUOTED_PUBLIC.getValue()); } else { @@ -443,7 +626,6 @@ private ServiceScriptDTO checkAndSetScriptInfoForFast( } // 检查高危脚本 checkScriptMatchDangerousRule(taskInstance, stepInstance); - return script; } private void checkScriptExist(long appId, StepInstanceDTO stepInstance, ServiceScriptDTO script) { @@ -476,9 +658,10 @@ private void checkScriptMatchDangerousRule(TaskInstanceDTO taskInstance, StepIns if (!stepInstance.isScriptStep()) { return; } - ScriptTypeEnum scriptType = ScriptTypeEnum.valueOf(stepInstance.getScriptType()); + String content = stepInstance.getScriptContent(); - List checkResultItems = dangerousScriptCheckService.check(scriptType, content); + List checkResultItems = + dangerousScriptCheckService.check(stepInstance.getScriptType(), content); if (CollectionUtils.isNotEmpty(checkResultItems)) { String checkResultSummary = dangerousScriptCheckService.summaryDangerousScriptCheckResult(stepInstance.getName(), checkResultItems); @@ -498,14 +681,16 @@ private void batchCheckScriptMatchDangerousRule(TaskInstanceDTO taskInstance, stepInstanceList.forEach(stepInstance -> checkScriptMatchDangerousRule(taskInstance, stepInstance)); } - private void authFastExecute(TaskInstanceDTO taskInstance, StepInstanceDTO stepInstance) { + private void authFastExecute(TaskInstanceDTO taskInstance, + StepInstanceDTO stepInstance, + Map> whiteHostAllowActions) { AuthResult authResult; if (stepInstance.isScriptStep()) { // 鉴权脚本任务 - authResult = authExecuteScript(taskInstance, stepInstance); + authResult = authExecuteScript(taskInstance, stepInstance, whiteHostAllowActions); } else { // 鉴权文件任务 - authResult = authFileTransfer(taskInstance, stepInstance); + authResult = authFileTransfer(taskInstance, stepInstance, whiteHostAllowActions); } if (!authResult.isPass()) { @@ -513,45 +698,44 @@ private void authFastExecute(TaskInstanceDTO taskInstance, StepInstanceDTO stepI } } - private AuthResult authExecuteScript(TaskInstanceDTO taskInstance, StepInstanceDTO stepInstance) { + private AuthResult authExecuteScript(TaskInstanceDTO taskInstance, + StepInstanceDTO stepInstance, + Map> whiteHostAllowActions) { Long appId = taskInstance.getAppId(); String username = taskInstance.getOperator(); Long accountId = null; - if (StepExecuteTypeEnum.EXECUTE_SCRIPT.getValue().equals(stepInstance.getExecuteType())) { + if (StepExecuteTypeEnum.EXECUTE_SCRIPT == stepInstance.getExecuteType()) { accountId = stepInstance.getAccountId(); - } else if (StepExecuteTypeEnum.EXECUTE_SQL.getValue().equals(stepInstance.getExecuteType())) { + } else if (StepExecuteTypeEnum.EXECUTE_SQL == stepInstance.getExecuteType()) { accountId = stepInstance.getDbAccountId(); } if (accountId == null) { return AuthResult.fail(); } - AuthResult accountAuthResult = executeAuthService.authAccountExecutable(username, new AppResourceScope(appId) - , accountId); + AuthResult accountAuthResult = executeAuthService.authAccountExecutable(username, + new AppResourceScope(appId), accountId); AuthResult serverAuthResult; - ServersDTO servers = stepInstance.getTargetServers().clone(); - filterServerDoNotRequireAuth(appId, servers, ActionScopeEnum.SCRIPT_EXECUTE); - if (servers.isEmpty()) { - // 如果主机为空，无需对主机进行鉴权 + ExecuteTargetDTO executeObjects = stepInstance.getTargetExecuteObjects().clone(); + filterHostsDoNotRequireAuth(ActionScopeEnum.SCRIPT_EXECUTE, executeObjects, whiteHostAllowActions); + if (executeObjects.isEmpty()) { + // 如果执行对象为空，无需进一步鉴权 return accountAuthResult; } ScriptSourceEnum scriptSource = ScriptSourceEnum.getScriptSourceEnum(stepInstance.getScriptSource()); if (scriptSource == ScriptSourceEnum.CUSTOM) { // 快速执行脚本鉴权 - serverAuthResult = executeAuthService.authFastExecuteScript( - username, new AppResourceScope(appId), servers); + username, new AppResourceScope(appId), executeObjects); } else if (scriptSource == ScriptSourceEnum.QUOTED_APP) { - serverAuthResult = executeAuthService.authExecuteAppScript( username, new AppResourceScope(appId), stepInstance.getScriptId(), - stepInstance.getScriptName(), servers); + stepInstance.getScriptName(), executeObjects); } else if (scriptSource == ScriptSourceEnum.QUOTED_PUBLIC) { - serverAuthResult = executeAuthService.authExecutePublicScript( - username, new AppResourceScope(appId), stepInstance.getScriptId() - , stepInstance.getScriptName(), servers); + username, new AppResourceScope(appId), stepInstance.getScriptId(), + stepInstance.getScriptName(), executeObjects); } else { serverAuthResult = AuthResult.fail(); } @@ -559,45 +743,32 @@ username, new AppResourceScope(appId), stepInstance.getScriptId() return accountAuthResult.mergeAuthResult(serverAuthResult); } - /* - * 过滤掉白名单的机器/允许忽略的非法主机、topo、动态分组 + /** + * 过滤掉主机白名单的机器 */ - private void filterServerDoNotRequireAuth(long appId, ServersDTO servers, ActionScopeEnum action) { - - if (CollectionUtils.isNotEmpty(servers.getStaticIpList())) { - servers.setStaticIpList(servers.getStaticIpList().stream() - .filter(host -> { - boolean isWhiteIp = hostService.isMatchWhiteIpRule(appId, host.toCloudIp(), action.name()); - if (isWhiteIp) { - log.info("Host: {} is white ip, skip auth!", host.toCloudIp()); - } - return !isWhiteIp; - }) + private void filterHostsDoNotRequireAuth(ActionScopeEnum action, + ExecuteTargetDTO executeTarget, + Map> whiteHostAllowActions) { + if (whiteHostAllowActions == null || whiteHostAllowActions.isEmpty()) { + return; + } + if (CollectionUtils.isNotEmpty(executeTarget.getStaticIpList())) { + executeTarget.setStaticIpList(executeTarget.getStaticIpList().stream() .filter(host -> { - boolean isValidIp = - servers.getInvalidIpList() == null || !servers.getInvalidIpList().contains(host); - if (!isValidIp) { - log.info("Host: {} is invalid ip, skip auth!", host.toCloudIp()); + List allowedActions = whiteHostAllowActions.get(host.getHostId()); + boolean skipAuth = allowedActions != null && allowedActions.contains(action.name()); + if (skipAuth) { + log.info("Host: {} is white host, skip auth!", host.toStringBasic()); } - return isValidIp; + return !skipAuth; }) .collect(Collectors.toList())); } - if (CollectionUtils.isNotEmpty(servers.getDynamicServerGroups()) && - CollectionUtils.isNotEmpty(servers.getInvalidDynamicServerGroups())) { - servers.setDynamicServerGroups(servers.getDynamicServerGroups().stream() - .filter(group -> !servers.getInvalidDynamicServerGroups().contains(group)) - .collect(Collectors.toList())); - } - if (CollectionUtils.isNotEmpty(servers.getTopoNodes()) && - CollectionUtils.isNotEmpty(servers.getInvalidTopoNodes())) { - servers.setTopoNodes(servers.getTopoNodes().stream() - .filter(topoNode -> !servers.getInvalidTopoNodes().contains(topoNode)) - .collect(Collectors.toList())); - } } - private AuthResult authFileTransfer(TaskInstanceDTO taskInstance, StepInstanceDTO stepInstance) { + private AuthResult authFileTransfer(TaskInstanceDTO taskInstance, + StepInstanceDTO stepInstance, + Map> whiteHostAllowActions) { String username = taskInstance.getOperator(); Long appId = taskInstance.getAppId(); @@ -605,333 +776,66 @@ private AuthResult authFileTransfer(TaskInstanceDTO taskInstance, StepInstanceDT accounts.add(stepInstance.getAccountId()); stepInstance.getFileSourceList().stream() .filter(fileSource -> !fileSource.isLocalUpload() && fileSource.getAccountId() != null) - .forEach(fileSource -> { - accounts.add(fileSource.getAccountId()); - }); + .forEach(fileSource -> accounts.add(fileSource.getAccountId())); AuthResult accountAuthResult = executeAuthService.batchAuthAccountExecutable( username, new AppResourceScope(appId), accounts); - ServersDTO servers = stepInstance.getTargetServers().clone(); + ExecuteTargetDTO executeTarget = stepInstance.getTargetExecuteObjects().clone(); stepInstance.getFileSourceList().stream() .filter(fileSource -> !fileSource.isLocalUpload() && fileSource.getFileType() != TaskFileTypeEnum.BASE64_FILE.getType() && fileSource.getServers() != null) - .forEach(fileSource -> { - servers.merge(fileSource.getServers()); - }); - filterServerDoNotRequireAuth(appId, servers, ActionScopeEnum.FILE_DISTRIBUTION); - if (servers.isEmpty()) { + .forEach(fileSource -> executeTarget.merge(fileSource.getServers())); + filterHostsDoNotRequireAuth(ActionScopeEnum.FILE_DISTRIBUTION, executeTarget, whiteHostAllowActions); + if (executeTarget.isEmpty()) { // 如果主机为空，无需对主机进行权限 return accountAuthResult; } AuthResult serverAuthResult = executeAuthService.authFastPushFile( - username, new AppResourceScope(appId), servers); + username, new AppResourceScope(appId), executeTarget); return accountAuthResult.mergeAuthResult(serverAuthResult); } - private void setServerInfoFastJob(StepInstanceDTO stepInstance) { - ServersDTO targetServers = stepInstance.getTargetServers(); - acquireStaticIp(stepInstance.getAppId(), targetServers); - stepInstance.setIpList(convertToIpListStr(targetServers.getIpList())); - stepInstance.setTargetServers(targetServers); - setAgentStatus(targetServers.getIpList()); - - if (stepInstance.getExecuteType() == TaskStepTypeEnum.FILE.getValue()) { - List fileSources = stepInstance.getFileSourceList(); - for (FileSourceDTO fileSource : fileSources) { - ServersDTO servers = fileSource.getServers(); - if (servers != null && !fileSource.isLocalUpload()) { - // 服务器文件的处理 - acquireStaticIp(stepInstance.getAppId(), servers); - setAgentStatus(servers.getIpList()); - } - } - } - } - - private String convertToIpListStr(Collection ips) { - return StringUtils.join(ips.stream().map(ipDTO -> - ipDTO.getBkCloudId() + ":" + ipDTO.getIp()).collect(Collectors.toList()), ","); - } - - /** - * 设置主机信息并检查主机的合法性并设置主机信息 - * - * @param stepInstanceList 步骤列表 - * @throws ServiceException 如果包含不合法的主机，抛出异常 - */ - private void checkAndSetHosts(List stepInstanceList) throws ServiceException { - // 检查步骤引用的主机不为空 - stepInstanceList.forEach(this::checkStepInstanceHostNonEmpty); - - long appId = stepInstanceList.get(0).getAppId(); - Set stepHosts = extractHosts(stepInstanceList); - - if (stepHosts.isEmpty()) { + private void checkStepInstanceExecuteTargetNonEmpty(StepInstanceDTO stepInstance) { + if (!stepInstance.isStepContainsExecuteObject()) { return; } - - // 获取主机详情 - ServiceListAppHostResultDTO hosts = hostService.batchGetAppHosts(appId, stepHosts); - if (CollectionUtils.isNotEmpty(hosts.getNotExistHosts())) { - // 如果主机在cmdb不存在，直接报错 - throwHostInvalidException(hosts.getNotExistHosts(), appId); - } - - // 设置主机信息 - fillStepHostDetail(stepInstanceList, hosts); - - // 如果存在主机不在当前业务下，需要进一步判断 - if (CollectionUtils.isNotEmpty(hosts.getNotInAppHosts())) { - // 过滤掉不需要检查业务从属的主机 - List notInAppHosts = filterHostsDoNotNeedCheck(stepInstanceList, hosts.getNotInAppHosts()); - if (CollectionUtils.isNotEmpty(notInAppHosts)) { - // 检查是否在白名单配置 - notInAppHosts = checkHostsNotAllowedInWhiteIpConfig(appId, stepInstanceList, - notInAppHosts); - } - if (!notInAppHosts.isEmpty()) { - log.warn("Contains invalid host, invalidHost: {}", JsonUtils.toJson(notInAppHosts)); - throwHostInvalidException(notInAppHosts, appId); - } - } - } - - private List filterHostsDoNotNeedCheck(List stepInstanceList, - List notInAppHosts) { - Set needCheckHostIds = new HashSet<>(); - for (StepInstanceDTO stepInstance : stepInstanceList) { - if (!isStepContainsHostProps(stepInstance)) { - continue; - } - needCheckHostIds.addAll(stepInstance.getTargetServers().getIpList().stream() - .map(HostDTO::getHostId).collect(Collectors.toSet())); - if (stepInstance.getExecuteType().equals(SEND_FILE.getValue())) { - List fileSourceList = stepInstance.getFileSourceList(); - if (fileSourceList != null) { - for (FileSourceDTO fileSource : fileSourceList) { - if (fileSource.getFileType().equals(TaskFileTypeEnum.SERVER.getType())) { - // 远程文件分发需要校验文件源主机;其他类型不需要 - ServersDTO servers = fileSource.getServers(); - if (servers != null && servers.getIpList() != null) { - needCheckHostIds.addAll(servers.getIpList().stream() - .map(HostDTO::getHostId).collect(Collectors.toSet())); - } - } - } - } - } - } - - if (CollectionUtils.isNotEmpty(needCheckHostIds)) { - return notInAppHosts.stream().filter(host -> needCheckHostIds.contains(host.getHostId())) - .collect(Collectors.toList()); - } else { - return notInAppHosts; - } - } - - private void checkStepInstanceHostNonEmpty(StepInstanceDTO stepInstance) { - if (!isStepContainsHostProps(stepInstance)) { - return; - } - ServersDTO targetServers = stepInstance.getTargetServers(); - if (targetServers == null || CollectionUtils.isEmpty(targetServers.getIpList())) { - log.warn("Empty target server, stepInstanceId: {}", stepInstance.getId()); - throw new FailedPreconditionException(ErrorCode.SERVER_EMPTY); + ExecuteTargetDTO targetExecuteObjects = stepInstance.getTargetExecuteObjects(); + if (targetExecuteObjects == null + || CollectionUtils.isEmpty(targetExecuteObjects.getExecuteObjectsCompatibly())) { + log.warn("Empty target execute object! stepInstanceName: {}", stepInstance.getName()); + throw new FailedPreconditionException(ErrorCode.STEP_TARGET_EXECUTE_OBJECT_EMPTY, + new String[]{stepInstance.getName()}); } if (stepInstance.isFileStep()) { List fileSourceList = stepInstance.getFileSourceList(); for (FileSourceDTO fileSource : fileSourceList) { // 远程文件分发需要判断文件源主机是否为空 if (TaskFileTypeEnum.SERVER.getType() == fileSource.getFileType()) { - ServersDTO servers = fileSource.getServers(); - if (servers != null && CollectionUtils.isEmpty(servers.getIpList())) { - log.warn("Empty file source server, stepInstanceId: {}", stepInstance.getId()); - throw new FailedPreconditionException(ErrorCode.SERVER_EMPTY); - } - } - } - } - } - - private boolean isStepContainsHostProps(StepInstanceBaseDTO stepInstance) { - // 判断步骤是否包含主机信息 - return !stepInstance.getExecuteType().equals(MANUAL_CONFIRM.getValue()); - } - - private void fillStepHostDetail(List stepInstanceList, ServiceListAppHostResultDTO hosts) { - Map hostMap = new HashMap<>(); - if (CollectionUtils.isNotEmpty(hosts.getValidHosts())) { - hosts.getValidHosts().forEach(host -> { - hostMap.put("hostId:" + host.getHostId(), host); - hostMap.put("hostIp:" + host.toCloudIp(), host); - }); - } - if (CollectionUtils.isNotEmpty(hosts.getNotInAppHosts())) { - hosts.getNotInAppHosts().forEach(host -> { - hostMap.put("hostId:" + host.getHostId(), host); - hostMap.put("hostIp:" + host.toCloudIp(), host); - }); - } - - for (StepInstanceDTO stepInstance : stepInstanceList) { - if (!isStepContainsHostProps(stepInstance)) { - continue; - } - // 目标主机设置主机详情 - fillTargetHostDetail(stepInstance, hostMap); - // 文件源设置主机详情 - fillFileSourceHostDetail(stepInstance, hostMap); - } - } - - private void fillTargetHostDetail(StepInstanceDTO stepInstance, Map hostMap) { - if (CollectionUtils.isNotEmpty(stepInstance.getTargetServers().getStaticIpList())) { - stepInstance.getTargetServers().getStaticIpList() - .forEach(host -> fillHostDetail(host, hostMap)); - } - if (CollectionUtils.isNotEmpty(stepInstance.getTargetServers().getIpList())) { - stepInstance.getTargetServers().getIpList() - .forEach(host -> fillHostDetail(host, hostMap)); - } - } - - private void fillFileSourceHostDetail(StepInstanceDTO stepInstance, Map hostMap) { - if (stepInstance.getExecuteType().equals(SEND_FILE.getValue())) { - List fileSourceList = stepInstance.getFileSourceList(); - if (fileSourceList != null) { - for (FileSourceDTO fileSource : fileSourceList) { - ServersDTO servers = fileSource.getServers(); - if (servers != null) { - if (CollectionUtils.isNotEmpty(servers.getStaticIpList())) { - servers.getStaticIpList().forEach(host -> fillHostDetail(host, hostMap)); - } - if (CollectionUtils.isNotEmpty(servers.getIpList())) { - servers.getIpList().forEach(host -> fillHostDetail(host, hostMap)); - } + ExecuteTargetDTO executeTarget = fileSource.getServers(); + if (executeTarget == null + || CollectionUtils.isEmpty(executeTarget.getExecuteObjectsCompatibly())) { + log.warn("Empty file source server, stepInstanceName: {}", stepInstance.getName()); + throw new FailedPreconditionException(ErrorCode.STEP_SOURCE_EXECUTE_OBJECT_EMPTY, + new String[]{stepInstance.getName()}); } } } } } - private void fillHostDetail(HostDTO host, Map hostMap) { - if (host.getHostId() != null) { - HostDTO hostDetail = hostMap.get("hostId:" + host.getHostId()); - host.setBkCloudId(hostDetail.getBkCloudId()); - host.setIp(hostDetail.getIp()); - // 兼容没有agent_id的主机，按照与GSE的约定，按照{云区域ID:ip}的方式构造agent_id - host.setAgentId(host.toCloudIp()); - } else { - HostDTO hostDetail = hostMap.get("hostIp:" + host.toCloudIp()); - // 兼容没有agent_id的主机，按照与GSE的约定，按照{云区域ID:ip}的方式构造agent_id - host.setAgentId(host.toCloudIp()); - host.setHostId(hostDetail.getHostId()); - } + private void checkStepInstance(TaskInstanceDTO taskInstance, List stepInstanceList) { + // 检查步骤引用的执行对象不为空 + stepInstanceList.forEach(this::checkStepInstanceExecuteTargetNonEmpty); + // 检查步骤的GSE原子任务上限 + checkStepInstanceAtomicTasksLimit(taskInstance, stepInstanceList); } - private Set extractHosts(List stepInstanceList) { - Set stepHosts = new HashSet<>(); - for (StepInstanceDTO stepInstance : stepInstanceList) { - if (!isStepContainsHostProps(stepInstance)) { - continue; - } - stepHosts.addAll(stepInstance.getTargetServers().getIpList()); - if (stepInstance.getExecuteType().equals(SEND_FILE.getValue())) { - List fileSourceList = stepInstance.getFileSourceList(); - if (fileSourceList != null) { - for (FileSourceDTO fileSource : fileSourceList) { - ServersDTO servers = fileSource.getServers(); - if (servers != null && servers.getIpList() != null) { - stepHosts.addAll(servers.getIpList()); - } - } - } - } - } - return stepHosts; - } - - private List checkHostsNotAllowedInWhiteIpConfig(long appId, List stepInstanceList, - Collection unavailableHosts) { - Map> hostAllowActionsMap = new HashMap<>(); - for (HostDTO host : unavailableHosts) { - List allowActions = hostService.getHostAllowedAction(appId, host); - if (allowActions != null && !allowActions.isEmpty()) { - hostAllowActionsMap.put(host, allowActions); - } - } - if (hostAllowActionsMap.isEmpty()) { - log.warn("Hosts are not in the white-ip configuration, hosts:{}", unavailableHosts); - return new ArrayList<>(unavailableHosts); - } - - log.debug("Host allow actions:{}", hostAllowActionsMap); - // 如果配置了白名单，那么需要对主机支持的操作进行校验 - Map> hostBindActionsMap = getHostBindActions(stepInstanceList, unavailableHosts); - log.debug("Host bind actions:{}", hostBindActionsMap); - - return getHostsContainsNotAllowedAction(hostBindActionsMap, hostAllowActionsMap); - } - - private static List getHostsContainsNotAllowedAction(Map> hostBindActions, - Map> hostAllowedActions) { - List invalidHosts = new ArrayList<>(); - for (Map.Entry> binding : hostBindActions.entrySet()) { - HostDTO host = binding.getKey(); - if (!hostAllowedActions.containsKey(host) - || !hostAllowedActions.get(host).containsAll(binding.getValue())) { - invalidHosts.add(host); - } - } - return invalidHosts; - } - - private Map> getHostBindActions(List stepInstanceList, - Collection unavailableHosts) { - Map> hostBindActionsMap = new HashMap<>(); - for (HostDTO host : unavailableHosts) { - for (StepInstanceDTO stepInstance : stepInstanceList) { - if (stepInstance.getExecuteType().equals(MANUAL_CONFIRM.getValue())) { - continue; - } - if (stepInstance.getTargetServers().getIpList().contains(host)) { - if (stepInstance.getExecuteType().equals(EXECUTE_SCRIPT.getValue()) || - stepInstance.getExecuteType().equals(EXECUTE_SQL.getValue())) { - hostBindActionsMap.compute(host, (hostKey, actions) -> { - if (actions == null) { - actions = new HashSet<>(); - } - actions.add(ActionScopeEnum.SCRIPT_EXECUTE.name()); - return actions; - }); - } else if (stepInstance.getExecuteType().equals(SEND_FILE.getValue())) { - hostBindActionsMap.compute(host, (hostKey, actions) -> { - if (actions == null) { - actions = new HashSet<>(); - } - actions.add(ActionScopeEnum.FILE_DISTRIBUTION.name()); - return actions; - }); - } - } - } - } - return hostBindActionsMap; - } - - private void throwHostInvalidException(Collection unavailableHosts, long appId) { - String ipListStr = StringUtils.join(unavailableHosts.stream().map(HostDTO::getIp).collect(Collectors.toList()), - ","); - log.warn("The following hosts are invalid, appId:{}, ips={}", appId, ipListStr); - throw new FailedPreconditionException(ErrorCode.HOST_INVALID, new Object[]{ipListStr}); - } - - private void checkStepInstanceConstraint(TaskInstanceDTO taskInstance, List stepInstanceList) { + private void checkStepInstanceAtomicTasksLimit(TaskInstanceDTO taskInstance, + List stepInstanceList) { String appCode = taskInstance.getAppCode(); Long appId = taskInstance.getAppId(); String taskName = taskInstance.getName(); @@ -939,13 +843,13 @@ private void checkStepInstanceConstraint(TaskInstanceDTO taskInstance, List 10000) { + int targetExecuteObjectSize = stepInstance.getTargetExecuteObjectCount(); + if (targetExecuteObjectSize > 10000) { TASK_MONITOR_LOGGER.info("LargeTask|type:script|taskName:{}|appCode:{}|appId:{}|operator:{}" - + "|targetServerSize:{}", - taskName, appCode, appId, operator, targetServerSize); + + "|targetExecuteObjectSize:{}", + taskName, appCode, appId, operator, targetExecuteObjectSize); } - if (targetServerSize > jobExecuteConfig.getScriptTaskMaxTargetServer()) { + if (targetExecuteObjectSize > jobExecuteConfig.getScriptTaskMaxTargetServer()) { log.info("Reject large task|type:file|taskName:{}|appCode:{}|appId:{}|operator" + - ":{}|targetServerSize:{}|maxAllowedSize:{}", taskName, appCode, appId, operator, - targetServerSize, jobExecuteConfig.getScriptTaskMaxTargetServer()); + ":{}|targetExecuteObjectSize:{}|maxAllowedSize:{}", taskName, appCode, appId, operator, + targetExecuteObjectSize, jobExecuteConfig.getScriptTaskMaxTargetServer()); throw new ResourceExhaustedException(ErrorCode.SCRIPT_TASK_TARGET_SERVER_EXCEEDS_LIMIT, new Integer[]{jobExecuteConfig.getScriptTaskMaxTargetServer()}); } @@ -982,13 +886,13 @@ private void checkStepInstanceConstraint(TaskInstanceDTO taskInstance, List { + TaskInstanceDTO taskInstance = executeJobPlanInternal(watch, executeParam, taskInfo); + if (!plan.isDebugTask()) { + auditJobPlanExecute(taskInstance); + } else { + addJobInstanceInfoToExtendData(taskInstance); + } + return taskInstance; + }).call(); + } + private TaskInstanceDTO executeJobPlanInternal(StopWatch watch, TaskExecuteParam executeParam, TaskInfo taskInfo) { + try { + // 检查正在执行的作业配额限制 TaskInstanceDTO taskInstance = taskInfo.getTaskInstance(); + watch.start("checkRunningJobQuoteLimit"); + checkRunningJobQuotaLimit(taskInstance.getAppId(), taskInstance.getAppCode()); + watch.stop(); + + ServiceTaskPlanDTO jobPlan = taskInfo.getJobPlan(); + taskInstance.setPlan(jobPlan); List stepInstanceList = taskInfo.getStepInstances(); + taskInstance.setStepInstances(stepInstanceList); Map finalVariableValueMap = taskInfo.getVariables(); - ServiceTaskPlanDTO jobPlan = taskInfo.getJobPlan(); + // 调整超时时间 stepInstanceList.forEach(this::adjustStepTimeout); @@ -1025,19 +968,22 @@ public TaskInstanceDTO executeJobPlan(TaskExecuteParam executeParam) { batchCheckScriptMatchDangerousRule(taskInstance, stepInstanceList); watch.stop(); - // 检查主机合法性 - watch.start("checkHost"); - checkAndSetHosts(stepInstanceList); + // 处理执行对象 + watch.start("processExecuteObjects"); + TaskInstanceExecuteObjects taskInstanceExecuteObjects = + taskInstanceExecuteObjectProcessor.processExecuteObjects(taskInstance, stepInstanceList, + finalVariableValueMap.values()); watch.stop(); - // 检查步骤约束 - watch.start("checkStepInstanceConstraint"); - checkStepInstanceConstraint(taskInstance, stepInstanceList); + // 检查步骤 + watch.start("checkStepInstance"); + checkStepInstance(taskInstance, stepInstanceList); watch.stop(); if (!executeParam.isSkipAuth()) { watch.start("auth-execute-job"); - authExecuteJobPlan(executeParam.getOperator(), executeParam.getAppId(), jobPlan, stepInstanceList); + authExecuteJobPlan(executeParam.getOperator(), executeParam.getAppId(), jobPlan, stepInstanceList, + taskInstanceExecuteObjects.getWhiteHostAllowActions()); watch.stop(); } @@ -1046,20 +992,30 @@ public TaskInstanceDTO executeJobPlan(TaskExecuteParam executeParam) { saveTaskInstance(taskInstance, stepInstanceList, finalVariableValueMap); watch.stop(); + Set allHosts = taskInstanceExecuteObjectProcessor.extractHosts( + taskInstance.getStepInstances(), null); + taskInstance.setAllHosts(allHosts); + // 保存作业实例与主机的关系，优化根据主机检索作业执行历史的效率 watch.start("saveTaskInstanceHosts"); - saveTaskInstanceHosts(taskInstance.getId(), taskInstance.getStepInstances()); + saveTaskInstanceHosts(taskInstance.getAppId(), taskInstance.getId(), allHosts); watch.stop(); + // 记录操作日志 watch.start("saveOperationLog"); taskOperationLogService.saveOperationLog(buildTaskOperationLog(taskInstance, taskInstance.getOperator(), - UserOperationEnum.START)); + executeParam.getStartTask() ? UserOperationEnum.START : UserOperationEnum.CREATE_JOB)); watch.stop(); // 启动作业 - watch.start("startJob"); - startTask(taskInstance.getId()); - watch.stop(); + if (executeParam.getStartTask()) { + watch.start("startJob"); + startTask(taskInstance.getId()); + watch.stop(); + } + + // 日志记录容器执行对象的作业，用于统计、分析 + logContainerExecuteObjectJob(taskInstance, taskInstanceExecuteObjects); return taskInstance; } finally { @@ -1074,9 +1030,9 @@ public TaskInstanceDTO executeJobPlan(TaskExecuteParam executeParam) { private void standardizeStepDynamicGroupId(List stepInstanceList) { for (StepInstanceDTO stepInstance : stepInstanceList) { - if (stepInstance.getTargetServers() != null - && CollectionUtils.isNotEmpty(stepInstance.getTargetServers().getDynamicServerGroups())) { - standardizeServerDynamicGroupId(stepInstance.getTargetServers()); + if (stepInstance.getTargetExecuteObjects() != null + && CollectionUtils.isNotEmpty(stepInstance.getTargetExecuteObjects().getDynamicServerGroups())) { + standardizeServerDynamicGroupId(stepInstance.getTargetExecuteObjects()); } if (CollectionUtils.isNotEmpty(stepInstance.getFileSourceList())) { for (FileSourceDTO fileSource : stepInstance.getFileSourceList()) { @@ -1089,27 +1045,27 @@ private void standardizeStepDynamicGroupId(List stepInstanceLis } } - private void standardizeServerDynamicGroupId(ServersDTO servers) { - if (servers != null && CollectionUtils.isNotEmpty(servers.getDynamicServerGroups())) { - servers.getDynamicServerGroups().forEach(this::standardizeDynamicGroupId); + private void standardizeServerDynamicGroupId(ExecuteTargetDTO executeTarget) { + if (executeTarget != null && CollectionUtils.isNotEmpty(executeTarget.getDynamicServerGroups())) { + executeTarget.getDynamicServerGroups().forEach(this::standardizeDynamicGroupId); } } private void standardizeTaskVarDynamicGroupId(Collection variables) { if (CollectionUtils.isNotEmpty(variables)) { - variables.stream().filter(variable -> variable.getTargetServers() != null) - .forEach(variable -> standardizeServerDynamicGroupId(variable.getTargetServers())); + variables.stream().filter(variable -> variable.getExecuteTarget() != null) + .forEach(variable -> standardizeServerDynamicGroupId(variable.getExecuteTarget())); } } private void standardizeDynamicGroupId(DynamicServerGroupDTO dynamicGroup) { - // 移除动态分组ID中多余的appId(历史问题) - // appId:groupId - String[] appIdAndGroupId = dynamicGroup.getGroupId().split(":"); - if (appIdAndGroupId.length == 2) { + // 移除动态分组ID中多余的bizId(历史问题) + // bizId:groupId + String[] bizIdAndGroupId = dynamicGroup.getGroupId().split(":"); + if (bizIdAndGroupId.length == 2) { log.info("Found invalid dynamicGroupId, try to transform to standard format! dynamicGroupId: {}", dynamicGroup.getGroupId()); - dynamicGroup.setGroupId(appIdAndGroupId[1]); + dynamicGroup.setGroupId(bizIdAndGroupId[1]); } } @@ -1131,7 +1087,7 @@ private TaskInfo buildTaskInfoFromExecuteParam(TaskExecuteParam executeParam, St TaskInstanceDTO taskInstance = buildTaskInstanceForTask(executeParam, taskPlan); List planDefaultVariables = convertToCommonVariables(taskPlan.getVariableList()); - Map finalVariableValueMap = buildFinalTaskVariableValues(appId, planDefaultVariables + Map finalVariableValueMap = buildFinalTaskVariableValues(planDefaultVariables , executeParam.getExecuteVariableValues()); standardizeTaskVarDynamicGroupId(finalVariableValueMap.values()); log.info("Final variable={}", JsonUtils.toJson(finalVariableValueMap)); @@ -1144,8 +1100,8 @@ private TaskInfo buildTaskInfoFromExecuteParam(TaskExecuteParam executeParam, St List stepInstanceList = new ArrayList<>(); for (ServiceTaskStepDTO step : taskPlan.getStepList()) { StepExecuteTypeEnum executeType = getExecuteTypeFromTaskStepType(step); - StepInstanceDTO stepInstance = createCommonStepInstanceDTO(appId, operator, step.getId(), step.getName(), - executeType); + StepInstanceDTO stepInstance = createCommonStepInstanceDTO(appId, operator, step.getId(), + step.getName(), executeType); TaskStepTypeEnum stepType = TaskStepTypeEnum.valueOf(step.getType()); switch (stepType) { case SCRIPT: @@ -1167,39 +1123,28 @@ private TaskInfo buildTaskInfoFromExecuteParam(TaskExecuteParam executeParam, St } private void authExecuteJobPlan(String username, long appId, ServiceTaskPlanDTO plan, - List stepInstanceList) throws PermissionDeniedException { - boolean isDebugTask = plan.getDebugTask(); - ServersDTO authServers = new ServersDTO(); + List stepInstanceList, + Map> whiteHostAllowActions) throws PermissionDeniedException { + boolean needAuth = stepInstanceList.stream() + .anyMatch(stepInstance -> stepInstance.isScriptStep() || stepInstance.isFileStep()); + if (!needAuth) { + return; + } + + boolean isDebugTask = plan.isDebugTask(); + ExecuteTargetDTO authServers = new ExecuteTargetDTO(); Set accountIds = new HashSet<>(); for (StepInstanceDTO stepInstance : stepInstanceList) { if (!stepInstance.isScriptStep() && !stepInstance.isFileStep()) { continue; } - accountIds.add(stepInstance.getAccountId()); - if (stepInstance.isFileStep()) { - ServersDTO stepTargetServers = stepInstance.getTargetServers().clone(); - filterServerDoNotRequireAuth(appId, stepTargetServers, ActionScopeEnum.FILE_DISTRIBUTION); - authServers.merge(stepTargetServers); - if (!CollectionUtils.isEmpty(stepInstance.getFileSourceList())) { - stepInstance.getFileSourceList().stream().filter(fileSource -> !fileSource.isLocalUpload()) - .forEach(fileSource -> { - ServersDTO stepFileSourceServers = fileSource.getServers().clone(); - filterServerDoNotRequireAuth(appId, stepFileSourceServers, - ActionScopeEnum.FILE_DISTRIBUTION); - authServers.merge(stepFileSourceServers); - if (fileSource.getAccountId() != null) { - accountIds.add(fileSource.getAccountId()); - } - } - ); - } - } else if (stepInstance.isScriptStep()) { - ServersDTO stepTargetServers = stepInstance.getTargetServers().clone(); - filterServerDoNotRequireAuth(appId, stepTargetServers, ActionScopeEnum.SCRIPT_EXECUTE); - authServers.merge(stepTargetServers); - } + Pair> needAuthHostsAndAccounts = + extractNeedAuthHostsAndAccounts(stepInstance, whiteHostAllowActions); + authServers = needAuthHostsAndAccounts.getLeft(); + accountIds = needAuthHostsAndAccounts.getRight(); } + // 账号使用鉴权 AuthResult accountAuthResult = executeAuthService.batchAuthAccountExecutable( username, new AppResourceScope(appId), accountIds); @@ -1213,8 +1158,7 @@ private void authExecuteJobPlan(String username, long appId, ServiceTaskPlanDTO if (isDebugTask) { // 鉴权模板调试 serverAuthResult = executeAuthService.authDebugTemplate( - username, new AppResourceScope(appId), plan.getTaskTemplateId(), - authServers); + username, new AppResourceScope(appId), plan.getTaskTemplateId(), authServers); } else { // 鉴权执行方案的执行 serverAuthResult = executeAuthService.authExecutePlan( @@ -1229,14 +1173,60 @@ username, new AppResourceScope(appId), plan.getTaskTemplateId(), } } - private void authRedoJob(String username, long appId, TaskInstanceDTO taskInstance) { + private Pair> extractNeedAuthHostsAndAccounts + ( + StepInstanceDTO stepInstance, + Map> whiteHostAllowActions + ) { + + ExecuteTargetDTO authServers = new ExecuteTargetDTO(); + Set accountIds = new HashSet<>(); + accountIds.add(stepInstance.getAccountId()); + if (stepInstance.isFileStep()) { + ExecuteTargetDTO stepTargetServers = stepInstance.getTargetExecuteObjects().clone(); + filterHostsDoNotRequireAuth(ActionScopeEnum.FILE_DISTRIBUTION, stepTargetServers, + whiteHostAllowActions); + authServers.merge(stepTargetServers); + if (!CollectionUtils.isEmpty(stepInstance.getFileSourceList())) { + stepInstance.getFileSourceList().stream() + .filter(fileSource -> !fileSource.isLocalUpload()) + .forEach(fileSource -> { + if (fileSource.getServers() == null) { + return; + } + ExecuteTargetDTO stepFileSourceServers = fileSource.getServers().clone(); + filterHostsDoNotRequireAuth(ActionScopeEnum.FILE_DISTRIBUTION, stepFileSourceServers, + whiteHostAllowActions); + authServers.merge(stepFileSourceServers); + if (fileSource.getAccountId() != null) { + accountIds.add(fileSource.getAccountId()); + } + } + ); + } + } else if (stepInstance.isScriptStep()) { + ExecuteTargetDTO stepTargetServers = stepInstance.getTargetExecuteObjects().clone(); + filterHostsDoNotRequireAuth(ActionScopeEnum.SCRIPT_EXECUTE, stepTargetServers, whiteHostAllowActions); + authServers.merge(stepTargetServers); + } + return Pair.of(authServers, accountIds); + } + + private void authRedoJob(String username, long appId, TaskInstanceDTO taskInstance, + Map> whiteHostAllowActions) { Integer taskType = taskInstance.getType(); if (taskType.equals(TaskTypeEnum.NORMAL.getValue()) - && taskInstance.getTaskId() != null - && taskInstance.getTaskId() > 0) { + && taskInstance.getPlanId() != null + && taskInstance.getPlanId() > 0) { // 作业鉴权 - ServiceTaskPlanDTO serviceTaskPlanDTO = taskPlanService.getPlanById(appId, taskInstance.getTaskId()); - authExecuteJobPlan(username, appId, serviceTaskPlanDTO, taskInstance.getStepInstances()); + ServiceTaskPlanDTO serviceTaskPlanDTO = taskPlanService.getPlanById(appId, taskInstance.getPlanId()); + if (serviceTaskPlanDTO == null) { + log.warn("auth redo task instance for task, task plan is not exist.appId={}, planId={}", appId, + taskInstance.getPlanId()); + throw new NotFoundException(ErrorCode.TASK_PLAN_NOT_EXIST); + } + authExecuteJobPlan(username, appId, serviceTaskPlanDTO, taskInstance.getStepInstances(), + whiteHostAllowActions); } else if (taskType.equals(TaskTypeEnum.SCRIPT.getValue())) { // 快速执行脚本鉴权 StepInstanceDTO scriptStepInstance = taskInstance.getStepInstances().get(0); @@ -1247,11 +1237,11 @@ private void authRedoJob(String username, long appId, TaskInstanceDTO taskInstan scriptStepInstance.setScriptName(script.getName()); } } - authFastExecute(taskInstance, scriptStepInstance); + authFastExecute(taskInstance, scriptStepInstance, whiteHostAllowActions); } else if (taskType.equals(TaskTypeEnum.FILE.getValue())) { // 快速分发文件鉴权 StepInstanceDTO fileStepInstance = taskInstance.getStepInstances().get(0); - authFastExecute(taskInstance, fileStepInstance); + authFastExecute(taskInstance, fileStepInstance, whiteHostAllowActions); } else { log.warn("Auth fail because of invalid task type!"); throw new PermissionDeniedException(AuthResult.fail()); @@ -1263,7 +1253,7 @@ private StepExecuteTypeEnum getExecuteTypeFromTaskStepType(ServiceTaskStepDTO st TaskStepTypeEnum stepType = TaskStepTypeEnum.valueOf(step.getType()); switch (stepType) { case SCRIPT: - ScriptTypeEnum scriptType = ScriptTypeEnum.valueOf(step.getScriptStepInfo().getType()); + ScriptTypeEnum scriptType = ScriptTypeEnum.valOf(step.getScriptStepInfo().getType()); if (scriptType == ScriptTypeEnum.SQL) { executeType = EXECUTE_SQL; } else { @@ -1280,20 +1270,6 @@ private StepExecuteTypeEnum getExecuteTypeFromTaskStepType(ServiceTaskStepDTO st return executeType; } - private TaskStepTypeEnum getTaskStepTypeFromExecuteType(StepExecuteTypeEnum executeType) throws ServiceException { - TaskStepTypeEnum stepType; - if (executeType == EXECUTE_SCRIPT || executeType == EXECUTE_SQL) { - stepType = TaskStepTypeEnum.SCRIPT; - } else if (executeType == SEND_FILE) { - stepType = TaskStepTypeEnum.FILE; - } else if (executeType == MANUAL_CONFIRM) { - stepType = TaskStepTypeEnum.APPROVAL; - } else { - throw new InternalException(ErrorCode.INTERNAL_ERROR); - } - return stepType; - } - private TaskInstanceDTO buildTaskInstanceForTask(TaskExecuteParam executeParam, ServiceTaskPlanDTO taskPlan) { TaskInstanceDTO taskInstance = new TaskInstanceDTO(); taskInstance.setAppId(executeParam.getAppId()); @@ -1310,10 +1286,10 @@ private TaskInstanceDTO buildTaskInstanceForTask(TaskExecuteParam executeParam, String taskName = StringUtils.isBlank(executeParam.getTaskName()) ? taskPlan.getName() : executeParam.getTaskName(); taskInstance.setName(taskName); - taskInstance.setTaskId(taskPlan.getId()); + taskInstance.setPlanId(taskPlan.getId()); taskInstance.setTaskTemplateId(taskPlan.getTaskTemplateId()); taskInstance.setCurrentStepInstanceId(-1L); - taskInstance.setDebugTask(taskPlan.getDebugTask()); + taskInstance.setDebugTask(taskPlan.isDebugTask()); taskInstance.setCallbackUrl(executeParam.getCallbackUrl()); taskInstance.setAppCode(executeParam.getAppCode()); return taskInstance; @@ -1324,7 +1300,7 @@ private StepInstanceDTO createCommonStepInstanceDTO(long appId, String operator, StepInstanceDTO stepInstance = new StepInstanceDTO(); stepInstance.setStepId(stepId); stepInstance.setName(stepName); - stepInstance.setExecuteType(stepType.getValue()); + stepInstance.setExecuteType(stepType); stepInstance.setStatus(RunStatusEnum.BLANK); stepInstance.setOperator(operator); stepInstance.setAppId(appId); @@ -1333,9 +1309,10 @@ private StepInstanceDTO createCommonStepInstanceDTO(long appId, String operator, return stepInstance; } - public TaskInstanceDTO createTaskInstanceForRedo(Long appId, Long taskInstanceId, String operator, - List executeVariableValues) - throws ServiceException { + public TaskInstanceDTO redoJob(Long appId, + Long taskInstanceId, + String operator, + List executeVariableValues) { log.info("Create task instance for redo, appId={}, taskInstanceId={}, operator={}, variables={}", appId, taskInstanceId, operator, executeVariableValues); TaskInstanceDTO originTaskInstance = taskInstanceService.getTaskInstanceDetail(taskInstanceId); @@ -1347,7 +1324,7 @@ public TaskInstanceDTO createTaskInstanceForRedo(Long appId, Long taskInstanceId TaskInstanceDTO taskInstance = createTaskInstanceForRedo(originTaskInstance, operator); - Map finalVariableValueMap = buildFinalTaskVariableValues(appId, + Map finalVariableValueMap = buildFinalTaskVariableValues( originTaskInstance.getVariables(), executeVariableValues); log.info("Final variable={}", finalVariableValueMap); @@ -1358,10 +1335,10 @@ public TaskInstanceDTO createTaskInstanceForRedo(Long appId, Long taskInstanceId List stepInstanceList = new ArrayList<>(); for (StepInstanceDTO originStepInstance : originTaskInstance.getStepInstances()) { - StepExecuteTypeEnum executeType = StepExecuteTypeEnum.valueOf(originStepInstance.getExecuteType()); + StepExecuteTypeEnum executeType = originStepInstance.getExecuteType(); StepInstanceDTO stepInstance = createCommonStepInstanceDTO(appId, operator, originStepInstance.getStepId(), originStepInstance.getName(), executeType); - TaskStepTypeEnum stepType = getTaskStepTypeFromExecuteType(executeType); + TaskStepTypeEnum stepType = StepTypeExecuteTypeConverter.convertToStepType(executeType); switch (stepType) { case SCRIPT: parseScriptStepInstanceFromStepInstance(stepInstance, originStepInstance, finalVariableValueMap); @@ -1379,17 +1356,19 @@ public TaskInstanceDTO createTaskInstanceForRedo(Long appId, Long taskInstanceId // 检查高危脚本 batchCheckScriptMatchDangerousRule(taskInstance, stepInstanceList); - // 检查主机合法性 - checkAndSetHosts(stepInstanceList); + // 处理执行对象 + TaskInstanceExecuteObjects taskInstanceExecuteObjects = + taskInstanceExecuteObjectProcessor.processExecuteObjects(taskInstance, stepInstanceList, + finalVariableValueMap.values()); - // 检查步骤约束 - checkStepInstanceConstraint(taskInstance, stepInstanceList); + // 检查步骤 + checkStepInstance(taskInstance, stepInstanceList); - authRedoJob(operator, appId, originTaskInstance); + authRedoJob(operator, appId, originTaskInstance, taskInstanceExecuteObjects.getWhiteHostAllowActions()); saveTaskInstance(taskInstance, stepInstanceList, finalVariableValueMap); - saveTaskInstanceHosts(taskInstance.getId(), taskInstance.getStepInstances()); + saveTaskInstanceHosts(taskInstance.getAppId(), taskInstance.getId(), taskInstance.getStepInstances()); taskOperationLogService.saveOperationLog(buildTaskOperationLog(taskInstance, taskInstance.getOperator(), UserOperationEnum.START)); @@ -1404,21 +1383,22 @@ private TaskInstanceDTO createTaskInstanceForRedo(TaskInstanceDTO originTaskInst TaskInstanceDTO taskInstance = new TaskInstanceDTO(); taskInstance.setAppId(originTaskInstance.getAppId()); taskInstance.setType(originTaskInstance.getType()); - taskInstance.setStartupMode(TaskStartupModeEnum.NORMAL.getValue()); + taskInstance.setStartupMode(TaskStartupModeEnum.WEB.getValue()); taskInstance.setCronTaskId(-1L); taskInstance.setStatus(RunStatusEnum.BLANK); taskInstance.setCreateTime(DateUtils.currentTimeMillis()); taskInstance.setOperator(operator); taskInstance.setName(originTaskInstance.getName()); - taskInstance.setTaskId(originTaskInstance.getTaskId()); + taskInstance.setPlanId(originTaskInstance.getPlanId()); taskInstance.setTaskTemplateId(originTaskInstance.getTaskTemplateId()); taskInstance.setCurrentStepInstanceId(-1L); taskInstance.setDebugTask(false); return taskInstance; } - private TaskInstanceDTO saveTaskInstance(TaskInstanceDTO taskInstance, List stepInstances, - Map taskVariablesMap) throws ServiceException { + private void saveTaskInstance(TaskInstanceDTO taskInstance, + List stepInstances, + Map taskVariablesMap) { // 保存TaskInstance long newTaskInstanceId = taskInstanceService.addTaskInstance(taskInstance); taskInstance.setId(newTaskInstanceId); @@ -1429,7 +1409,7 @@ private TaskInstanceDTO saveTaskInstance(TaskInstanceDTO taskInstance, List convertToCommonVariables(List variables) { @@ -1458,7 +1444,7 @@ private List convertToCommonVariables(List convertToCommonVariables(List buildFinalTaskVariableValues( - long appId, List defaultVariableValues, List executeVariableValues ) throws ServiceException { @@ -1508,14 +1493,14 @@ private Map buildFinalTaskVariableValues( if (idKeyExecuteVariableValueMap.containsKey(defaultTaskVariable.getId())) { TaskVariableDTO executeVariableValue = idKeyExecuteVariableValueMap.get(defaultTaskVariable.getId()); - grantValueForVariable(appId, finalTaskVariable, executeVariableValue); + grantValueForVariable(finalTaskVariable, executeVariableValue); } else if (nameKeyExecuteVariableValueMap.containsKey(defaultTaskVariable.getName())) { TaskVariableDTO executeVariableValue = nameKeyExecuteVariableValueMap.get(defaultTaskVariable.getName()); - grantValueForVariable(appId, finalTaskVariable, executeVariableValue); + grantValueForVariable(finalTaskVariable, executeVariableValue); } else { // 否则，使用变量的默认值 - grantValueForVariable(appId, finalTaskVariable, defaultTaskVariable); + grantValueForVariable(finalTaskVariable, defaultTaskVariable); } finalVariableValueMap.put(finalTaskVariable.getName(), finalTaskVariable); } @@ -1523,17 +1508,9 @@ private Map buildFinalTaskVariableValues( return finalVariableValueMap; } - private void grantValueForVariable(long appId, TaskVariableDTO to, TaskVariableDTO from) { + private void grantValueForVariable(TaskVariableDTO to, TaskVariableDTO from) { if (TaskVariableTypeEnum.HOST_LIST.getType() == to.getType()) { - ServersDTO targetServers = from.getTargetServers(); - if (targetServers != null) { - // 动态-> 静态IP - acquireStaticIp(appId, targetServers); - if (targetServers.getIpList() != null && !targetServers.getIpList().isEmpty()) { - setAgentStatus(targetServers.getIpList()); - } - } - to.setTargetServers(targetServers); + to.setExecuteTarget(from.getExecuteTarget()); } else { to.setValue(from.getValue()); } @@ -1561,7 +1538,7 @@ private void parseScriptStepInstanceFromPlanStep(StepInstanceDTO stepInstance, S stepInstance.setTimeout(1000); } stepInstance.setSecureParam(scriptStepInfo.getSecureParam() != null && scriptStepInfo.getSecureParam()); - stepInstance.setScriptType(scriptStepInfo.getType()); + stepInstance.setScriptType(ScriptTypeEnum.valOf(scriptStepInfo.getType())); stepInstance.setScriptSource(scriptStepInfo.getScriptSource()); ServiceAccountDTO accountInfo = scriptStepInfo.getAccount(); @@ -1570,7 +1547,7 @@ private void parseScriptStepInstanceFromPlanStep(StepInstanceDTO stepInstance, S throw new NotFoundException(ErrorCode.ACCOUNT_NOT_EXIST); } - ScriptTypeEnum scriptType = ScriptTypeEnum.valueOf(scriptStepInfo.getType()); + ScriptTypeEnum scriptType = ScriptTypeEnum.valOf(scriptStepInfo.getType()); stepInstance.setAccountId(accountInfo.getId()); if (scriptType == ScriptTypeEnum.SQL) { stepInstance.setAccountId(accountInfo.getDbSystemAccount().getId()); @@ -1583,12 +1560,12 @@ private void parseScriptStepInstanceFromPlanStep(StepInstanceDTO stepInstance, S } else { stepInstance.setAccountId(accountInfo.getId()); stepInstance.setAccount(accountInfo.getAccount()); + stepInstance.setWindowsInterpreter(scriptStepInfo.getWindowsInterpreter()); } ServiceTaskTargetDTO target = scriptStepInfo.getExecuteTarget(); - ServersDTO targetServers = buildFinalTargetServers(stepInstance.getAppId(), target, variableValueMap); - stepInstance.setTargetServers(targetServers); - stepInstance.setIpList(convertToIpListStr(targetServers.getIpList())); + ExecuteTargetDTO targetServers = buildFinalTargetServers(target, variableValueMap); + stepInstance.setTargetExecuteObjects(targetServers); stepInstance.setIgnoreError(scriptStepInfo.getIgnoreError()); } @@ -1631,21 +1608,16 @@ private void parseFileStepInstanceFromPlanStep(StepInstanceDTO stepInstance, Ser fileSource.setAccountId(accountDTO.getId()); } ServiceTaskTargetDTO target = originFile.getExecuteTarget(); - ServersDTO targetServers = buildFinalTargetServers(stepInstance.getAppId(), target, variableValueMap); + ExecuteTargetDTO targetServers = buildFinalTargetServers(target, variableValueMap); fileSource.setServers(targetServers); List fileList = new ArrayList<>(); - originFile.getFileLocation().forEach(fileLocation -> { - fileList.add(new FileDetailDTO(fileLocation)); - }); + originFile.getFileLocation().forEach(fileLocation -> fileList.add(new FileDetailDTO(fileLocation))); fileSource.setFiles(fileList); } else if (originFile.getFileType() == TaskFileTypeEnum.FILE_SOURCE.getType()) { fileSource.setLocalUpload(false); - fileSource.setServers(ServersDTO.emptyInstance()); // 文件源文件只需要fileSourceId与文件路径 List fileList = new ArrayList<>(); - originFile.getFileLocation().forEach(fileLocation -> { - fileList.add(new FileDetailDTO(fileLocation)); - }); + originFile.getFileLocation().forEach(fileLocation -> fileList.add(new FileDetailDTO(fileLocation))); fileSource.setFiles(fileList); fileSource.setFileSourceId(originFile.getFileSourceId()); } @@ -1654,17 +1626,18 @@ private void parseFileStepInstanceFromPlanStep(StepInstanceDTO stepInstance, Ser stepInstance.setFileSourceList(fileSources); ServiceTaskTargetDTO target = fileStepInfo.getExecuteTarget(); - ServersDTO targetServers = buildFinalTargetServers(stepInstance.getAppId(), target, variableValueMap); - stepInstance.setTargetServers(targetServers); - stepInstance.setIpList(convertToIpListStr(targetServers.getIpList())); + ExecuteTargetDTO targetServers = buildFinalTargetServers(target, variableValueMap); + stepInstance.setTargetExecuteObjects(targetServers); if (fileStepInfo.getDownloadSpeedLimit() != null) { // MB->KB - stepInstance.setFileDownloadSpeedLimit(fileStepInfo.getDownloadSpeedLimit() << 10); + stepInstance.setFileDownloadSpeedLimit( + DataSizeConverter.convertMBToKB(fileStepInfo.getDownloadSpeedLimit())); } if (fileStepInfo.getUploadSpeedLimit() != null) { // MB->KB - stepInstance.setFileUploadSpeedLimit(fileStepInfo.getUploadSpeedLimit() << 10); + stepInstance.setFileUploadSpeedLimit( + DataSizeConverter.convertMBToKB(fileStepInfo.getUploadSpeedLimit())); } stepInstance.setTimeout(fileStepInfo.getTimeout()); @@ -1688,7 +1661,7 @@ private void parseScriptStepInstanceFromStepInstance(StepInstanceDTO stepInstanc stepInstance.setScriptType(originStepInstance.getScriptType()); stepInstance.setScriptSource(originStepInstance.getScriptSource()); - ScriptTypeEnum scriptType = ScriptTypeEnum.valueOf(originStepInstance.getScriptType()); + ScriptTypeEnum scriptType = originStepInstance.getScriptType(); stepInstance.setAccountId(originStepInstance.getAccountId()); if (scriptType == ScriptTypeEnum.SQL) { stepInstance.setAccountId(originStepInstance.getAccountId()); @@ -1703,12 +1676,13 @@ private void parseScriptStepInstanceFromStepInstance(StepInstanceDTO stepInstanc stepInstance.setAccount(originStepInstance.getAccount()); } - ServersDTO targetServers = buildFinalTargetServers(originStepInstance.getTargetServers(), variableValueMap); - stepInstance.setTargetServers(targetServers); - stepInstance.setIpList(convertToIpListStr(targetServers.getIpList())); + ExecuteTargetDTO targetServers = buildFinalTargetServers(originStepInstance.getTargetExecuteObjects(), + variableValueMap); + stepInstance.setTargetExecuteObjects(targetServers); } - private void parseFileStepInstanceFromStepInstance(StepInstanceDTO stepInstance, StepInstanceDTO originStepInstance, + private void parseFileStepInstanceFromStepInstance(StepInstanceDTO stepInstance, + StepInstanceDTO originStepInstance, Map variableValueMap) { stepInstance.setAccountId(originStepInstance.getAccountId()); stepInstance.setAccount(originStepInstance.getAccount()); @@ -1723,38 +1697,31 @@ private void parseFileStepInstanceFromStepInstance(StepInstanceDTO stepInstance, originStepInstance.getFileSourceList().forEach(fileSourceDTO -> { FileSourceDTO newFileSource = fileSourceDTO.clone(); // 重新解析源文件服务器信息 - ServersDTO targetServers = buildFinalTargetServers(newFileSource.getServers(), variableValueMap); + ExecuteTargetDTO targetServers = buildFinalTargetServers(newFileSource.getServers(), variableValueMap); newFileSource.setServers(targetServers); fileSourceList.add(newFileSource); }); stepInstance.setFileSourceList(fileSourceList); } - ServersDTO targetServers = buildFinalTargetServers(originStepInstance.getTargetServers(), variableValueMap); - stepInstance.setTargetServers(targetServers); - stepInstance.setIpList(convertToIpListStr(targetServers.getIpList())); + ExecuteTargetDTO targetServers = buildFinalTargetServers(originStepInstance.getTargetExecuteObjects(), + variableValueMap); + stepInstance.setTargetExecuteObjects(targetServers); } - private ServersDTO buildFinalTargetServers(@NotNull long appId, @NotNull ServiceTaskTargetDTO target, - @NotNull Map variableValueMap) + private ExecuteTargetDTO buildFinalTargetServers(@NotNull ServiceTaskTargetDTO target, + @NotNull Map variableValueMap) throws ServiceException { // 如果目标服务器使用主机变量，那么需要解析主机变量 if (StringUtils.isNotBlank(target.getVariable())) { return getServerValueFromVariable(target.getVariable(), variableValueMap); } else { - ServersDTO targetServers = convertToServersDTO(target); - acquireStaticIp(appId, targetServers); - if (targetServers.getIpList() == null || targetServers.getIpList().isEmpty()) { - log.warn("Target server variable host is empty.variable={}", target.getVariable()); - throw new FailedPreconditionException(ErrorCode.TASK_INSTANCE_RELATED_HOST_VAR_SERVER_EMPTY, - new String[]{target.getVariable()}); - } - setAgentStatus(targetServers.getIpList()); - return targetServers; + return convertToServersDTO(target); } } - private ServersDTO buildFinalTargetServers(ServersDTO target, Map variableValueMap) + private ExecuteTargetDTO buildFinalTargetServers(ExecuteTargetDTO target, + Map variableValueMap) throws ServiceException { // 如果目标服务器使用主机变量，那么需要解析主机变量 if (target != null && StringUtils.isNotBlank(target.getVariable())) { @@ -1764,8 +1731,8 @@ private ServersDTO buildFinalTargetServers(ServersDTO target, Map variableValueMap) + private ExecuteTargetDTO getServerValueFromVariable(@NotNull String hostVariableName, + @NotNull Map variableValueMap) throws ServiceException { TaskVariableDTO serverVariable = variableValueMap.get(hostVariableName); if (serverVariable == null) { @@ -1773,14 +1740,11 @@ private ServersDTO getServerValueFromVariable(@NotNull String hostVariableName, throw new FailedPreconditionException(ErrorCode.TASK_INSTANCE_RELATED_HOST_VAR_NOT_EXIST, new String[]{hostVariableName}); } - ServersDTO variableTargetServers = serverVariable.getTargetServers(); - if (variableTargetServers == null || CollectionUtils.isEmpty(variableTargetServers.getIpList())) { - log.warn("Target server variable host is empty.variable={}", hostVariableName); - throw new FailedPreconditionException(ErrorCode.TASK_INSTANCE_RELATED_HOST_VAR_SERVER_EMPTY, - new String[]{hostVariableName}); - } - ServersDTO targetServers = variableTargetServers.clone(); + if (serverVariable.getExecuteTarget() == null) { + return null; + } + ExecuteTargetDTO targetServers = serverVariable.getExecuteTarget().clone(); targetServers.setVariable(hostVariableName); return targetServers; } @@ -1802,19 +1766,20 @@ private void parseManualConfirmStepInstance(StepInstanceDTO stepInstance, StepIn } - private ServersDTO convertToServersDTO(ServiceTaskTargetDTO taskTarget) { + private ExecuteTargetDTO convertToServersDTO(ServiceTaskTargetDTO taskTarget) { if (taskTarget == null) { return null; } - ServersDTO servers = new ServersDTO(); + ExecuteTargetDTO executeTarget = new ExecuteTargetDTO(); ServiceTaskHostNodeDTO targetServers = taskTarget.getTargetServer(); List hostList = targetServers.getHostList(); if (hostList != null && !hostList.isEmpty()) { List staticIpList = new ArrayList<>(); for (ServiceHostInfoDTO hostInfo : hostList) { - staticIpList.add(new HostDTO(hostInfo.getCloudAreaId(), hostInfo.getIp())); + staticIpList.add( + new HostDTO(hostInfo.getHostId(), hostInfo.getCloudAreaId(), hostInfo.getIp())); } - servers.setStaticIpList(staticIpList); + executeTarget.setStaticIpList(staticIpList); } List groupIdList = targetServers.getDynamicGroupId(); @@ -1823,7 +1788,7 @@ private ServersDTO convertToServersDTO(ServiceTaskTargetDTO taskTarget) { for (String groupId : groupIdList) { groups.add(new DynamicServerGroupDTO(groupId)); } - servers.setDynamicServerGroups(groups); + executeTarget.setDynamicServerGroups(groups); } List topoNodeIdList = targetServers.getNodeInfoList(); @@ -1832,127 +1797,41 @@ private ServersDTO convertToServersDTO(ServiceTaskTargetDTO taskTarget) { for (ServiceTaskNodeInfoDTO topoNodeId : topoNodeIdList) { topoNodes.add(new DynamicServerTopoNodeDTO(topoNodeId.getId(), topoNodeId.getType())); } - servers.setTopoNodes(topoNodes); - } - return servers; - } - - private void acquireStaticIp(long appId, ServersDTO servers) throws ServiceException { - Set ipSet = new HashSet<>(); - List staticIps = servers.getStaticIpList(); - if (staticIps != null) { - ipSet.addAll(staticIps); - } - List dynamicServerGroups = servers.getDynamicServerGroups(); - if (dynamicServerGroups != null) { - for (DynamicServerGroupDTO group : dynamicServerGroups) { - List groupIps = hostService.getIpByDynamicGroupId(appId, group.getGroupId()); - if (CollectionUtils.isEmpty(groupIps)) { - servers.addInvalidDynamicServerGroup(group); - } else { - ipSet.addAll(groupIps); - group.setIpList(groupIps); - } - } - } - List topoNodes = servers.getTopoNodes(); - if (topoNodes != null && !topoNodes.isEmpty()) { - if (topoNodes.size() < 10) { - for (DynamicServerTopoNodeDTO topoNode : topoNodes) { - List topoIps = hostService.getIpByTopoNodes(appId, - Collections.singletonList(new CcInstanceDTO(topoNode.getNodeType(), topoNode.getTopoNodeId()))); - if (CollectionUtils.isEmpty(topoIps)) { - servers.addInvalidTopoNodeDTO(topoNode); - } else { - ipSet.addAll(topoIps); - } - } - } else { - getTopoHostsConcurrent(appId, topoNodes, servers, ipSet); - } - } - List ipList = new ArrayList<>(ipSet.size()); - ipList.addAll(ipSet); - servers.setIpList(ipList); - } - - - private void getTopoHostsConcurrent(long appId, List topoNodes, ServersDTO servers, - Set ipSet) { - log.info("Get topo hosts concurrent, topoNodes: {}", topoNodes); - CountDownLatch latch = new CountDownLatch(topoNodes.size()); - List>>> futures = new ArrayList<>(topoNodes.size()); - for (DynamicServerTopoNodeDTO topoNode : topoNodes) { - futures.add(getHostsByTopoExecutor.submit(new GetTopoHostTask(appId, topoNode, latch))); - } - - try { - for (Future>> future : futures) { - Pair> topoAndHosts = future.get(); - if (CollectionUtils.isEmpty(topoAndHosts.getRight())) { - servers.addInvalidTopoNodeDTO(topoAndHosts.getLeft()); - } else { - ipSet.addAll(topoAndHosts.getRight()); - } - } - } catch (InterruptedException | ExecutionException e) { - - } - try { - latch.await(); - } catch (InterruptedException e) { - - } - log.info("Get topo hosts success, servers: {}", servers); - } - - private void setAgentStatus(List ips) { - if (ips == null || ips.isEmpty()) { - return; - } - List ipList = new ArrayList<>(ips.size()); - for (HostDTO ip : ips) { - String fullIp = ip.toCloudIp(); - ipList.add(fullIp); - } - Map statusMap = queryAgentStatusClient.batchGetAgentStatus(ipList); - for (HostDTO ip : ips) { - String fullIp = ip.toCloudIp(); - ip.setAlive(statusMap.get(fullIp) == null ? - AgentStatusEnum.UNKNOWN.getValue() : statusMap.get(fullIp).status); + executeTarget.setTopoNodes(topoNodes); } + return executeTarget; } @Override - public Integer doStepOperation(Long appId, String operator, - StepOperationDTO stepOperation) throws ServiceException { + public Integer doStepOperation(Long appId, + String operator, + StepOperationDTO stepOperation) { long stepInstanceId = stepOperation.getStepInstanceId(); StepOperationEnum operation = stepOperation.getOperation(); + log.info("Operate step, appId:{}, stepInstanceId:{}, operator:{}, operation:{}", appId, stepInstanceId, operator, operation.getValue()); - StepInstanceDTO stepInstance = taskInstanceService.getStepInstanceDetail(stepInstanceId); - if (stepInstance == null) { - log.warn("Step instance {} is not exist", stepInstanceId); - throw new NotFoundException(ErrorCode.STEP_INSTANCE_NOT_EXIST); - } - if (!stepInstance.getAppId().equals(appId)) { - log.warn("Step instance {} is not in app:{}", stepInstance, appId); - throw new NotFoundException(ErrorCode.STEP_INSTANCE_NOT_EXIST); - } + + TaskInstanceDTO taskInstance = queryTaskInstanceAndCheckExist(appId, stepOperation.getTaskInstanceId()); + addJobInstanceContext(taskInstance); + + StepInstanceDTO stepInstance = queryStepInstanceAndCheckExist( + appId, stepOperation.getTaskInstanceId(), stepInstanceId); + int executeCount = stepInstance.getExecuteCount(); switch (operation) { case CONFIRM_CONTINUE: confirmContinue(stepInstance, operator, stepOperation.getConfirmReason()); break; case RETRY_FAIL_IP: - retryStepFail(stepInstance, operator); + retryStepFail(taskInstance, stepInstance, operator); executeCount++; break; case IGNORE_ERROR: - ignoreError(stepInstance, operator); + ignoreError(taskInstance, stepInstance, operator); break; case RETRY_ALL_IP: - retryStepAll(stepInstance, operator); + retryStepAll(taskInstance, stepInstance, operator); executeCount++; break; case CONFIRM_TERMINATE: @@ -1962,10 +1841,10 @@ public Integer doStepOperation(Long appId, String operator, confirmRestart(stepInstance, operator); break; case NEXT_STEP: - nextStep(stepInstance, operator); + nextStep(taskInstance, stepInstance, operator); break; case SKIP: - skipStep(stepInstance, operator); + skipStep(taskInstance, stepInstance, operator); break; case ROLLING_CONTINUE: continueRolling(stepInstance); @@ -1977,6 +1856,29 @@ public Integer doStepOperation(Long appId, String operator, return executeCount; } + private TaskInstanceDTO queryTaskInstanceAndCheckExist(long appId, long taskInstanceId) { + TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(taskInstanceId); + if (taskInstance == null || !taskInstance.getAppId().equals(appId)) { + log.warn("Task instance is not exist, appId:{}, taskInstanceId:{}", appId, taskInstance); + throw new NotFoundException(ErrorCode.TASK_INSTANCE_NOT_EXIST); + } + return taskInstance; + } + + private StepInstanceDTO queryStepInstanceAndCheckExist(long appId, long taskInstanceId, long stepInstanceId) { + StepInstanceDTO stepInstance = stepInstanceService.getStepInstanceDetail( + taskInstanceId, stepInstanceId); + if (stepInstance == null) { + log.warn("Step instance {} is not exist", stepInstanceId); + throw new NotFoundException(ErrorCode.STEP_INSTANCE_NOT_EXIST); + } + if (!stepInstance.getAppId().equals(appId)) { + log.warn("Step instance {} is not in app:{}", stepInstance, appId); + throw new NotFoundException(ErrorCode.STEP_INSTANCE_NOT_EXIST); + } + return stepInstance; + } + private void continueRolling(StepInstanceDTO stepInstance) { // 只有“等待用户”的滚动步骤可以继续滚动 if (stepInstance.getStatus() != RunStatusEnum.WAITING_USER) { @@ -1992,8 +1894,8 @@ private void continueRolling(StepInstanceDTO stepInstance) { // 需要同步设置任务状态为RUNNING，保证客户端可以在操作完之后立马获取到运行状态，开启同步刷新 taskInstanceService.updateTaskStatus(stepInstance.getTaskInstanceId(), RunStatusEnum.RUNNING.getValue()); - taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.startStep(stepInstance.getId(), - stepInstance.getBatch() + 1)); + taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.startStep(stepInstance.getTaskInstanceId(), + stepInstance.getId(), stepInstance.getBatch() + 1)); } private void confirmTerminate(StepInstanceDTO stepInstance, String operator, String reason) { @@ -2004,9 +1906,9 @@ private void confirmTerminate(StepInstanceDTO stepInstance, String operator, Str stepInstance.getId(), StepOperationEnum.CONFIRM_TERMINATE.name(), stepInstance.getStatus().name()); throw new FailedPreconditionException(ErrorCode.UNSUPPORTED_OPERATION); } - if (!stepInstance.getExecuteType().equals(MANUAL_CONFIRM.getValue())) { - log.warn("StepInstance:{} is not confirm step, Unsupported Operation:{}", stepInstance.getId(), "confirm" + - "-terminate"); + if (stepInstance.getExecuteType() != MANUAL_CONFIRM) { + log.warn("StepInstance:{} is not confirm step, Unsupported Operation:{}", stepInstance.getId(), + "confirm-terminate"); throw new FailedPreconditionException(ErrorCode.UNSUPPORTED_OPERATION); } checkConfirmUser(taskInstance, stepInstance, operator); @@ -2016,10 +1918,11 @@ private void confirmTerminate(StepInstanceDTO stepInstance, String operator, Str operationLog.getDetail().setConfirmReason(reason); taskOperationLogService.saveOperationLog(operationLog); - taskInstanceService.updateConfirmReason(stepInstance.getId(), reason); - taskInstanceService.updateStepOperator(stepInstance.getId(), operator); + stepInstanceService.updateConfirmReason(stepInstance.getTaskInstanceId(), stepInstance.getId(), reason); + stepInstanceService.updateStepOperator(stepInstance.getTaskInstanceId(), stepInstance.getId(), operator); - taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.confirmStepTerminate(stepInstance.getId())); + taskExecuteMQEventDispatcher.dispatchStepEvent( + StepEvent.confirmStepTerminate(stepInstance.getTaskInstanceId(), stepInstance.getId())); } private void confirmRestart(StepInstanceDTO stepInstance, String operator) { @@ -2029,49 +1932,73 @@ private void confirmRestart(StepInstanceDTO stepInstance, String operator) { stepInstance.getId(), StepOperationEnum.CONFIRM_RESTART.name(), stepInstance.getStatus().name()); throw new FailedPreconditionException(ErrorCode.UNSUPPORTED_OPERATION); } - if (!stepInstance.getExecuteType().equals(MANUAL_CONFIRM.getValue())) { - log.warn("StepInstance:{} is not confirm step, Unsupported Operation:{}", stepInstance.getId(), "confirm" + - "-restart"); + if (stepInstance.getExecuteType() != MANUAL_CONFIRM) { + log.warn("StepInstance:{} is not confirm step, Unsupported Operation:{}", stepInstance.getId(), + "confirm-restart"); throw new FailedPreconditionException(ErrorCode.UNSUPPORTED_OPERATION); } taskOperationLogService.saveOperationLog(buildCommonStepOperationLog(stepInstance, operator, UserOperationEnum.CONFIRM_RESTART)); - taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.confirmStepRestart(stepInstance.getId())); + taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.confirmStepRestart( + stepInstance.getTaskInstanceId(), stepInstance.getId())); } private void checkConfirmUser(TaskInstanceDTO taskInstance, StepInstanceDTO stepInstance, String operator) throws ServiceException { // 人工确认步骤，需要判断操作者 + // 判断指定确认人 if (CollectionUtils.isNotEmpty(stepInstance.getConfirmUsers()) && stepInstance.getConfirmUsers().contains(operator)) { return; } - Set confirmUsers = new HashSet<>(); + // 判断确认角色 if (stepInstance.getConfirmRoles() != null && !stepInstance.getConfirmRoles().isEmpty()) { - if (stepInstance.getConfirmRoles().contains(JobRoleEnum.JOB_RESOURCE_TRIGGER_USER.name())) { - confirmUsers.add(taskInstance.getOperator()); - } else { - Set roles = new HashSet<>(stepInstance.getConfirmRoles()); - // JOB_RESOURCE_TRIGGER_USER should remove - roles.remove(JobRoleEnum.JOB_RESOURCE_TRIGGER_USER.name()); - InternalResponse> resp = userResource.getUsersByRoles(stepInstance.getAppId(), operator, - ResourceTypeEnum.JOB.getType(), String.valueOf(taskInstance.getTaskId()), roles); - if (resp.isSuccess() && resp.getData() != null) { - confirmUsers.addAll(resp.getData()); - } + if (stepInstance.getConfirmRoles().contains(JobRoleEnum.JOB_RESOURCE_TRIGGER_USER.name()) + && taskInstance.getOperator().equals(operator)) { + return; + } + + Set confirmCmdbRoleUsers = getCmdbRoleUsers(taskInstance.getAppId(), operator, + String.valueOf(taskInstance.getPlanId()), stepInstance.getConfirmRoles()); + if (CollectionUtils.isEmpty(confirmCmdbRoleUsers) || !confirmCmdbRoleUsers.contains(operator)) { + log.warn("Confirm user is invalid, allowed confirmUsers: {}, confirmCmdbRoleUsers : {}, " + + "taskTrigger: {}, operator: {}", + stepInstance.getConfirmUsers(), confirmCmdbRoleUsers, taskInstance.getOperator(), operator); + throw new FailedPreconditionException(ErrorCode.NOT_IN_CONFIRM_USER_LIST); } } - if (confirmUsers.isEmpty() || !confirmUsers.contains(operator)) { - throw new FailedPreconditionException(ErrorCode.NOT_IN_CONFIRM_USER_LIST); + + } + + private Set getCmdbRoleUsers(Long appId, + String operator, + String resourceId, + Collection allRoles) { + Set cmdbRoles = getCmdbRoles(allRoles); + Set confirmRoleUsers = new HashSet<>(); + if (CollectionUtils.isNotEmpty(cmdbRoles)) { + InternalResponse> resp = userResource.getUsersByRoles(appId, operator, + ResourceTypeEnum.JOB.getType(), resourceId, cmdbRoles); + if (resp.isSuccess() && resp.getData() != null) { + confirmRoleUsers.addAll(resp.getData()); + } } + return confirmRoleUsers; + } + + private Set getCmdbRoles(Collection allRoles) { + // 不属于cmdb的角色，需要移除 + return allRoles.stream() + .filter(role -> !JobRoleEnum.isJobRole(role)) + .collect(Collectors.toSet()); } private void confirmContinue(StepInstanceDTO stepInstance, String operator, String reason) { // 只有"人工确认等待"，可以进行"确认继续"操作 - if (!stepInstance.getExecuteType().equals(MANUAL_CONFIRM.getValue())) { - log.warn("StepInstance:{} is not confirm-step, Unsupported Operation:{}", stepInstance.getId(), "confirm" + - "-continue"); + if (stepInstance.getExecuteType() != MANUAL_CONFIRM) { + log.warn("StepInstance:{} is not confirm-step, Unsupported Operation:{}", stepInstance.getId(), + "confirm-continue"); throw new FailedPreconditionException(ErrorCode.UNSUPPORTED_OPERATION); } if (RunStatusEnum.WAITING_USER != stepInstance.getStatus()) { @@ -2088,56 +2015,75 @@ private void confirmContinue(StepInstanceDTO stepInstance, String operator, Stri operationLog.getDetail().setConfirmReason(reason); taskOperationLogService.saveOperationLog(operationLog); - taskInstanceService.updateConfirmReason(stepInstance.getId(), reason); - taskInstanceService.updateStepOperator(stepInstance.getId(), operator); + stepInstanceService.updateConfirmReason(stepInstance.getTaskInstanceId(), stepInstance.getId(), reason); + stepInstanceService.updateStepOperator(stepInstance.getTaskInstanceId(), stepInstance.getId(), operator); // 需要同步设置任务状态为RUNNING，保证客户端可以在操作完之后立马获取到运行状态，开启同步刷新 taskInstanceService.updateTaskStatus(taskInstance.getId(), RunStatusEnum.RUNNING.getValue()); - taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.confirmStepContinue(stepInstance.getId())); + taskExecuteMQEventDispatcher.dispatchStepEvent( + StepEvent.confirmStepContinue(stepInstance.getTaskInstanceId(), stepInstance.getId())); } - private void nextStep(StepInstanceDTO stepInstance, String operator) { - TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(stepInstance.getTaskInstanceId()); + private void nextStep(TaskInstanceDTO taskInstance, StepInstanceDTO stepInstance, String operator) { // 只有"终止成功"状态的任务，可以直接进入下一步 if (RunStatusEnum.STOP_SUCCESS != stepInstance.getStatus()) { log.warn("Unsupported operation, stepInstanceId: {}, operation: {}, stepStatus: {}", stepInstance.getId(), StepOperationEnum.NEXT_STEP.name(), stepInstance.getStatus().name()); throw new FailedPreconditionException(ErrorCode.UNSUPPORTED_OPERATION); } + runningJobResourceQuotaManager.addJob( + taskInstance.getAppCode(), + GlobalAppScopeMappingService.get().getScopeByAppId(taskInstance.getAppId()), + taskInstance.getId() + ); taskOperationLogService.saveOperationLog(buildCommonStepOperationLog(stepInstance, operator, UserOperationEnum.NEXT_STEP)); // 需要同步设置任务状态为RUNNING，保证客户端可以在操作完之后立马获取到运行状态，开启同步刷新 taskInstanceService.updateTaskStatus(taskInstance.getId(), RunStatusEnum.RUNNING.getValue()); - taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.nextStep(stepInstance.getId())); + taskExecuteMQEventDispatcher.dispatchStepEvent( + StepEvent.nextStep(stepInstance.getTaskInstanceId(), stepInstance.getId())); } - private void retryStepFail(StepInstanceDTO stepInstance, String operator) { + private void retryStepFail(TaskInstanceDTO taskInstance, StepInstanceDTO stepInstance, String operator) { if (!isStepRetryable(stepInstance.getStatus())) { log.warn("Unsupported operation, stepInstanceId: {}, operation: {}, stepStatus: {}", stepInstance.getId(), StepOperationEnum.RETRY_FAIL_IP.name(), stepInstance.getStatus().name()); throw new FailedPreconditionException(ErrorCode.UNSUPPORTED_OPERATION); } + runningJobResourceQuotaManager.addJob( + taskInstance.getAppCode(), + GlobalAppScopeMappingService.get().getScopeByAppId(taskInstance.getAppId()), + taskInstance.getId() + ); // 需要同步设置任务状态为RUNNING，保证客户端可以在操作完之后立马获取到运行状态，开启同步刷新 taskInstanceService.updateTaskStatus(stepInstance.getTaskInstanceId(), RunStatusEnum.RUNNING.getValue()); - taskInstanceService.addStepInstanceExecuteCount(stepInstance.getId()); - taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.retryStepFail(stepInstance.getId())); + stepInstanceService.addStepInstanceExecuteCount( + stepInstance.getTaskInstanceId(), stepInstance.getId()); + taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.retryStepFail(stepInstance.getTaskInstanceId(), + stepInstance.getId())); OperationLogDTO operationLog = buildCommonStepOperationLog(stepInstance, operator, UserOperationEnum.RETRY_STEP_FAIL); taskOperationLogService.saveOperationLog(operationLog); } - private void retryStepAll(StepInstanceDTO stepInstance, String operator) { + private void retryStepAll(TaskInstanceDTO taskInstance, StepInstanceDTO stepInstance, String operator) { if (!isStepRetryable(stepInstance.getStatus())) { log.warn("Unsupported operation, stepInstanceId: {}, operation: {}, stepStatus: {}", stepInstance.getId(), StepOperationEnum.RETRY_ALL_IP.name(), stepInstance.getStatus().name()); throw new FailedPreconditionException(ErrorCode.UNSUPPORTED_OPERATION); } + runningJobResourceQuotaManager.addJob( + taskInstance.getAppCode(), + GlobalAppScopeMappingService.get().getScopeByAppId(taskInstance.getAppId()), + taskInstance.getId() + ); // 需要同步设置任务状态为RUNNING，保证客户端可以在操作完之后立马获取到运行状态，开启同步刷新 taskInstanceService.updateTaskStatus(stepInstance.getTaskInstanceId(), RunStatusEnum.RUNNING.getValue()); - taskInstanceService.addStepInstanceExecuteCount(stepInstance.getId()); - taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.retryStepAll(stepInstance.getId())); + stepInstanceService.addStepInstanceExecuteCount(stepInstance.getTaskInstanceId(), stepInstance.getId()); + taskExecuteMQEventDispatcher.dispatchStepEvent( + StepEvent.retryStepAll(stepInstance.getTaskInstanceId(), stepInstance.getId())); OperationLogDTO operationLog = buildCommonStepOperationLog(stepInstance, operator, UserOperationEnum.RETRY_STEP_ALL); taskOperationLogService.saveOperationLog(operationLog); @@ -2150,7 +2096,7 @@ private boolean isStepRetryable(RunStatusEnum stepStatus) { || stepStatus == RunStatusEnum.STOP_SUCCESS; } - private void ignoreError(StepInstanceDTO stepInstance, String operator) { + private void ignoreError(TaskInstanceDTO taskInstance, StepInstanceDTO stepInstance, String operator) { // 只有“执行失败”的作业可以忽略错误进入下一步 if (stepInstance.getStatus() != RunStatusEnum.FAIL && stepInstance.getStatus() != RunStatusEnum.ABNORMAL_STATE) { @@ -2158,22 +2104,34 @@ private void ignoreError(StepInstanceDTO stepInstance, String operator) { stepInstance.getId(), StepOperationEnum.IGNORE_ERROR.name(), stepInstance.getStatus().name()); throw new FailedPreconditionException(ErrorCode.UNSUPPORTED_OPERATION); } + runningJobResourceQuotaManager.addJob( + taskInstance.getAppCode(), + GlobalAppScopeMappingService.get().getScopeByAppId(taskInstance.getAppId()), + taskInstance.getId() + ); // 需要同步设置任务状态为RUNNING，保证客户端可以在操作完之后立马获取到运行状态，开启同步刷新 taskInstanceService.updateTaskStatus(stepInstance.getTaskInstanceId(), RunStatusEnum.RUNNING.getValue()); - taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.ignoreError(stepInstance.getId())); + taskExecuteMQEventDispatcher.dispatchStepEvent( + StepEvent.ignoreError(stepInstance.getTaskInstanceId(), stepInstance.getId())); OperationLogDTO operationLog = buildCommonStepOperationLog(stepInstance, operator, UserOperationEnum.IGNORE_ERROR); taskOperationLogService.saveOperationLog(operationLog); } - private void skipStep(StepInstanceDTO stepInstance, String operator) { + private void skipStep(TaskInstanceDTO taskInstance, StepInstanceDTO stepInstance, String operator) { // 只有“强制终止中”的作业可以跳过 if (stepInstance.getStatus() != RunStatusEnum.STOPPING) { log.warn("Unsupported operation, stepInstanceId: {}, operation: {}, stepStatus: {}", stepInstance.getId(), StepOperationEnum.SKIP.name(), stepInstance.getStatus().name()); throw new FailedPreconditionException(ErrorCode.UNSUPPORTED_OPERATION); } - taskExecuteMQEventDispatcher.dispatchStepEvent(StepEvent.skipStep(stepInstance.getId())); + runningJobResourceQuotaManager.addJob( + taskInstance.getAppCode(), + GlobalAppScopeMappingService.get().getScopeByAppId(taskInstance.getAppId()), + taskInstance.getId() + ); + taskExecuteMQEventDispatcher.dispatchStepEvent( + StepEvent.skipStep(stepInstance.getTaskInstanceId(), stepInstance.getId())); OperationLogDTO operationLog = buildCommonStepOperationLog(stepInstance, operator, UserOperationEnum.SKIP_STEP); taskOperationLogService.saveOperationLog(operationLog); } @@ -2212,34 +2170,50 @@ private OperationLogDTO buildTaskOperationLog(TaskInstanceDTO taskInstance, Stri } @Override - public void terminateJob(String username, Long appId, Long taskInstanceId) throws ServiceException { - TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(taskInstanceId); - if (taskInstance == null || !taskInstance.getAppId().equals(appId)) { - log.warn("Task instance is not exist, appId:{}, taskInstanceId:{}", appId, taskInstance); - throw new NotFoundException(ErrorCode.TASK_INSTANCE_NOT_EXIST); - } + public void terminateJob(String username, Long appId, Long taskInstanceId) { + TaskInstanceDTO taskInstance = queryTaskInstanceAndCheckExist(appId, taskInstanceId); + terminateJob(username, taskInstance); + } + + private void terminateJob(String operator, TaskInstanceDTO taskInstance) { if (RunStatusEnum.RUNNING != taskInstance.getStatus() && RunStatusEnum.WAITING_USER != taskInstance.getStatus()) { log.warn("TaskInstance:{} status is not running/waiting, should not terminate it!", taskInstance.getId()); throw new FailedPreconditionException(ErrorCode.UNSUPPORTED_OPERATION); } - if (RunStatusEnum.STOPPING == taskInstance.getStatus()) { - log.warn("TaskInstance:{} status is stopping now, should not terminate it!", taskInstance.getId()); - throw new FailedPreconditionException(ErrorCode.TASK_STOPPING_DO_NOT_REPEAT); + taskExecuteMQEventDispatcher.dispatchJobEvent(JobEvent.stopJob(taskInstance.getId())); + OperationLogDTO operationLog = buildTaskOperationLog(taskInstance, operator, UserOperationEnum.TERMINATE_JOB); + taskOperationLogService.saveOperationLog(operationLog); + } + + private void startJob(String operator, TaskInstanceDTO taskInstance) { + // 只有等待中的作业才可以启动 + if (RunStatusEnum.BLANK != taskInstance.getStatus()) { + log.warn("TaskInstance:{} status:{} Only task waiting for execution can be started!", + taskInstance.getId(), taskInstance.getStatus()); + throw new FailedPreconditionException(ErrorCode.UNSUPPORTED_OPERATION_REPEAT_START_JOB, + new Long[]{taskInstance.getId()}); } - taskExecuteMQEventDispatcher.dispatchJobEvent(JobEvent.stopJob(taskInstanceId)); - OperationLogDTO operationLog = buildTaskOperationLog(taskInstance, username, UserOperationEnum.TERMINATE_JOB); + startTask(taskInstance.getId()); + OperationLogDTO operationLog = buildTaskOperationLog(taskInstance, operator, UserOperationEnum.START); taskOperationLogService.saveOperationLog(operationLog); } @Override - public void doTaskOperation(Long appId, String operator, long taskInstanceId, - TaskOperationEnum operation) throws ServiceException { + public void doTaskOperation(Long appId, + String operator, + long taskInstanceId, + TaskOperationEnum operation) { log.info("Operate task instance, appId:{}, taskInstanceId:{}, operator:{}, operation:{}", appId, taskInstanceId, operator, operation.getValue()); + TaskInstanceDTO taskInstance = queryTaskInstanceAndCheckExist(appId, taskInstanceId); + addJobInstanceContext(taskInstance); switch (operation) { case TERMINATE_JOB: - terminateJob(operator, appId, taskInstanceId); + terminateJob(operator, taskInstance); + break; + case START_JOB: + startJob(operator, taskInstance); break; default: log.warn("Undefined task operation!"); @@ -2251,45 +2225,20 @@ public void doTaskOperation(Long appId, String operator, long taskInstanceId, public void authExecuteJobPlan(TaskExecuteParam executeParam) throws ServiceException { StopWatch watch = new StopWatch("authJobPlan"); TaskInfo taskInfo = buildTaskInfoFromExecuteParam(executeParam, watch); + TaskInstanceDTO taskInstance = taskInfo.getTaskInstance(); + List stepInstanceList = taskInfo.getStepInstances(); - // 检查主机合法性 - watch.start("checkHost"); - checkAndSetHosts(taskInfo.getStepInstances()); - watch.stop(); + TaskInstanceExecuteObjects taskInstanceExecuteObjects = + taskInstanceExecuteObjectProcessor.processExecuteObjects(taskInstance, stepInstanceList, + taskInfo.getVariables().values()); watch.start("auth-execute-job"); authExecuteJobPlan(executeParam.getOperator(), executeParam.getAppId(), taskInfo.getJobPlan(), - taskInfo.getStepInstances()); + taskInfo.getStepInstances(), taskInstanceExecuteObjects.getWhiteHostAllowActions()); watch.stop(); if (watch.getTotalTimeMillis() > 500) { log.warn("authJobPlan is slow, watcher: {}", watch.prettyPrint()); } } - - private class GetTopoHostTask implements Callable>> { - private final long appId; - private final DynamicServerTopoNodeDTO topoNode; - private final CountDownLatch latch; - - private GetTopoHostTask(long appId, DynamicServerTopoNodeDTO topoNode, CountDownLatch latch) { - this.appId = appId; - this.topoNode = topoNode; - this.latch = latch; - } - - @Override - public Pair> call() { - try { - List topoIps = hostService.getIpByTopoNodes(appId, - Collections.singletonList(new CcInstanceDTO(topoNode.getNodeType(), topoNode.getTopoNodeId()))); - return new ImmutablePair<>(topoNode, topoIps); - } catch (Throwable e) { - log.warn("Get hosts by topo fail", e); - return new ImmutablePair<>(topoNode, Collections.EMPTY_LIST); - } finally { - latch.countDown(); - } - } - } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskInstanceExecuteObjectProcessor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskInstanceExecuteObjectProcessor.java new file mode 100644 index 0000000000..e6054143e8 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskInstanceExecuteObjectProcessor.java @@ -0,0 +1,1041 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.service.impl; + +import com.tencent.bk.job.common.cc.model.container.KubeClusterDTO; +import com.tencent.bk.job.common.cc.model.container.KubeNamespaceDTO; +import com.tencent.bk.job.common.cc.model.query.KubeClusterQuery; +import com.tencent.bk.job.common.cc.model.query.NamespaceQuery; +import com.tencent.bk.job.common.cc.sdk.BizCmdbClient; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.constant.TaskVariableTypeEnum; +import com.tencent.bk.job.common.exception.FailedPreconditionException; +import com.tencent.bk.job.common.exception.NotImplementedException; +import com.tencent.bk.job.common.gse.constants.AgentAliveStatusEnum; +import com.tencent.bk.job.common.gse.constants.DefaultBeanNames; +import com.tencent.bk.job.common.gse.service.AgentStateClient; +import com.tencent.bk.job.common.gse.service.model.HostAgentStateQuery; +import com.tencent.bk.job.common.gse.util.AgentUtils; +import com.tencent.bk.job.common.gse.v2.model.resp.AgentState; +import com.tencent.bk.job.common.model.dto.Container; +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.common.model.dto.ResourceScope; +import com.tencent.bk.job.common.service.AppScopeMappingService; +import com.tencent.bk.job.common.service.toggle.strategy.JobInstanceAttrToggleStrategy; +import com.tencent.bk.job.common.util.ListUtil; +import com.tencent.bk.job.common.util.toggle.ToggleEvaluateContext; +import com.tencent.bk.job.common.util.toggle.ToggleStrategyContextParams; +import com.tencent.bk.job.common.util.toggle.feature.FeatureIdConstants; +import com.tencent.bk.job.common.util.toggle.feature.FeatureToggle; +import com.tencent.bk.job.execute.common.cache.WhiteHostCache; +import com.tencent.bk.job.execute.common.constants.TaskStartupModeEnum; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; +import com.tencent.bk.job.execute.engine.model.TaskVariableDTO; +import com.tencent.bk.job.execute.metrics.ExecuteObjectSampler; +import com.tencent.bk.job.execute.model.DynamicServerGroupDTO; +import com.tencent.bk.job.execute.model.DynamicServerTopoNodeDTO; +import com.tencent.bk.job.execute.model.ExecuteTargetDTO; +import com.tencent.bk.job.execute.model.FileSourceDTO; +import com.tencent.bk.job.execute.model.StepInstanceDTO; +import com.tencent.bk.job.execute.model.TaskInstanceDTO; +import com.tencent.bk.job.execute.model.TaskInstanceExecuteObjects; +import com.tencent.bk.job.execute.service.ApplicationService; +import com.tencent.bk.job.execute.service.ContainerService; +import com.tencent.bk.job.execute.service.HostService; +import com.tencent.bk.job.manage.api.common.constants.task.TaskFileTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskStepTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.whiteip.ActionScopeEnum; +import com.tencent.bk.job.manage.model.inner.ServiceListAppHostResultDTO; +import com.tencent.bk.job.manage.model.inner.resp.ServiceApplicationDTO; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Service; +import org.springframework.util.StopWatch; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; + +import static com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum.SEND_FILE; + +/** + * 作业实例执行对象处理器 + */ +@Slf4j +@Service +public class TaskInstanceExecuteObjectProcessor { + + private final HostService hostService; + private final ApplicationService applicationService; + private final ContainerService containerService; + private final AppScopeMappingService appScopeMappingService; + private final WhiteHostCache whiteHostCache; + private final AgentStateClient preferV2AgentStateClient; + private final BizCmdbClient bizCmdbClient; + private final ExecuteObjectSampler executeObjectSampler; + + public TaskInstanceExecuteObjectProcessor(HostService hostService, + ApplicationService applicationService, + ContainerService containerService, + AppScopeMappingService appScopeMappingService, + WhiteHostCache whiteHostCache, + @Qualifier(DefaultBeanNames.PREFER_V2_AGENT_STATE_CLIENT) + AgentStateClient preferV2AgentStateClient, + BizCmdbClient bizCmdbClient, + ExecuteObjectSampler executeObjectSampler) { + this.hostService = hostService; + this.applicationService = applicationService; + this.containerService = containerService; + this.appScopeMappingService = appScopeMappingService; + this.whiteHostCache = whiteHostCache; + this.preferV2AgentStateClient = preferV2AgentStateClient; + this.bizCmdbClient = bizCmdbClient; + this.executeObjectSampler = executeObjectSampler; + } + + /** + * 处理作业实例的执行对象（获取、设置主机、容器等执行对象），并返回所有作业实例中包含的执行执行对象（分类） + * + * @param taskInstance 作业实例 + * @param stepInstanceList 步骤实例列表 + * @param variables 作业全局变量 + * @return 作业实例中包含的执行对象 + */ + public TaskInstanceExecuteObjects processExecuteObjects(TaskInstanceDTO taskInstance, + List stepInstanceList, + Collection variables) { + + StopWatch watch = new StopWatch("processExecuteObjects"); + boolean hasContainer; + TaskInstanceExecuteObjects taskInstanceExecuteObjects = new TaskInstanceExecuteObjects(); + try { + hasContainer = isJobHasContainerExecuteObject(stepInstanceList, variables); + + if (hasContainer && !isContainerExecuteFeatureEnabled(taskInstance.getAppId())) { + // 如果资源空间不支持容器执行（比如业务集不支持容器执行），或者该资源空间未在容器执行特性灰度列表，需要返回错误信息 + throw new NotImplementedException( + "ContainerExecute is not support", ErrorCode.NOT_SUPPORT_FEATURE); + } + + long appId = taskInstance.getAppId(); + // 获取执行对象 + watch.start("acquireAndSetExecuteObjects"); + // 获取并设置主机执行对象 + acquireAndSetHosts(taskInstanceExecuteObjects, taskInstance, stepInstanceList, variables); + // 获取并设置容器执行对象 + acquireAndSetContainers(taskInstanceExecuteObjects, taskInstance, stepInstanceList); + boolean isSupportExecuteObjectFeature = isSupportExecuteObjectFeature(taskInstance); + // 合并所有执行对象 + mergeExecuteObjects(stepInstanceList, variables, isSupportExecuteObjectFeature); + // 检查执行对象是否合法 + checkExecuteObjectExist(taskInstance, stepInstanceList, taskInstanceExecuteObjects); + watch.stop(); + + // 如果包含主机执行对象，需要获取主机白名单 + if (taskInstanceExecuteObjects.isContainsAnyHost()) { + watch.start("getHostAllowedActions"); + taskInstanceExecuteObjects.setWhiteHostAllowActions( + getHostAllowedActions( + appId, + ListUtil.union(taskInstanceExecuteObjects.getValidHosts(), + taskInstanceExecuteObjects.getNotInAppHosts()))); + watch.stop(); + } + + // 检查执行对象是否可用 + watch.start("checkExecuteObjectAccessible"); + checkExecuteObjectAccessible(taskInstance, stepInstanceList, taskInstanceExecuteObjects); + watch.stop(); + + return taskInstanceExecuteObjects; + } finally { + // 记录作业执行对象相关指标 + executeObjectSampler.tryToRecordExecuteObjectMetrics( + taskInstance, + taskInstanceExecuteObjects + ); + if (watch.isRunning()) { + watch.stop(); + } + if (watch.getTotalTimeMillis() > 1000) { + log.warn("ProcessExecuteObjects is slow, taskInfo: {}", watch.prettyPrint()); + } + } + } + + private boolean isContainerExecuteFeatureEnabled(long appId) { + ResourceScope resourceScope = appScopeMappingService.getScopeByAppId(appId); + if (resourceScope.isBizSet()) { + // 业务集不支持容器执行 + return false; + } + return FeatureToggle.checkFeature( + FeatureIdConstants.FEATURE_CONTAINER_EXECUTE, + ToggleEvaluateContext.builder() + .addContextParam( + ToggleStrategyContextParams.CTX_PARAM_RESOURCE_SCOPE, + appScopeMappingService.getScopeByAppId(appId) + ) + ); + } + + private boolean isJobHasContainerExecuteObject(List stepInstanceList, + Collection variables) { + return checkExecuteObjectExists(stepInstanceList, variables, + ExecuteTargetDTO::hasContainerExecuteObject); + } + + /** + * 检查步骤、全局变量中是否存在某种类型的执行对象 + * + * @param stepInstanceList 作业步骤实例列表 + * @param variables 作业全局变量列表 + * @param executeObjectExistsFunction 检查是否存在函数 + */ + private boolean checkExecuteObjectExists(List stepInstanceList, + Collection variables, + Function executeObjectExistsFunction) { + boolean checkResult = stepInstanceList.stream() + .anyMatch(stepInstance -> + (stepInstance.getTargetExecuteObjects() != null && + executeObjectExistsFunction.apply(stepInstance.getTargetExecuteObjects()) || + CollectionUtils.isNotEmpty(stepInstance.getFileSourceList()) && + stepInstance.getFileSourceList().stream() + .anyMatch(fileSource -> fileSource.getServers() != null && + executeObjectExistsFunction.apply(fileSource.getServers())))); + if (checkResult) { + return true; + } + + if (CollectionUtils.isNotEmpty(variables)) { + checkResult = variables.stream() + .anyMatch(variable -> + variable.getExecuteTarget() != null + && executeObjectExistsFunction.apply(variable.getExecuteTarget())); + } + return checkResult; + } + + private void acquireAndSetHosts(TaskInstanceExecuteObjects taskInstanceExecuteObjects, + TaskInstanceDTO taskInstance, + List stepInstances, + Collection variables) { + StopWatch watch = new StopWatch("AcquireAndSetHosts"); + try { + long appId = taskInstance.getAppId(); + + // 提取动态分组/topo节点 + Set groups = new HashSet<>(); + Set topoNodes = new HashSet<>(); + stepInstances.forEach(stepInstance -> extractDynamicGroupsAndTopoNodes(stepInstance, groups, topoNodes)); + if (CollectionUtils.isNotEmpty(variables)) { + variables.forEach(variable -> { + if (TaskVariableTypeEnum.HOST_LIST.getType() == variable.getType()) { + extractDynamicGroupsAndTopoNodes(variable.getExecuteTarget(), groups, topoNodes); + } + }); + } + + // 获取动态分组的主机并设置 + fillDynamicGroupHosts(watch, appId, groups, stepInstances, variables); + + // 获取topo节点的主机并设置 + fillTopoNodeHosts(watch, appId, topoNodes, stepInstances, variables); + + // 提取作业包含的主机列表 + watch.start("extractHosts"); + Set queryHosts = extractHosts(stepInstances, variables); + watch.stop(); + + if (CollectionUtils.isEmpty(queryHosts)) { + return; + } + + taskInstanceExecuteObjects.setContainsAnyHost(true); + + watch.start("batchGetAppHosts"); + ServiceListAppHostResultDTO queryHostsResult = hostService.batchGetAppHosts(appId, queryHosts, + needRefreshHostBkAgentId(taskInstance)); + watch.stop(); + + taskInstanceExecuteObjects.setValidHosts(queryHostsResult.getValidHosts()); + taskInstanceExecuteObjects.setNotExistHosts(queryHostsResult.getNotExistHosts()); + taskInstanceExecuteObjects.setNotInAppHosts(queryHostsResult.getNotInAppHosts()); + + watch.start("fillTaskInstanceHostDetail"); + fillTaskInstanceHostDetail(taskInstance, stepInstances, variables, taskInstanceExecuteObjects); + watch.stop(); + } finally { + if (watch.isRunning()) { + watch.stop(); + } + if (watch.getTotalTimeMillis() > 1000) { + log.warn("AcquireAndSetHosts slow, watch: {}", watch.prettyPrint()); + } + } + + } + + private void extractDynamicGroupsAndTopoNodes(StepInstanceDTO stepInstance, + Set groups, + Set topoNodes) { + extractDynamicGroupsAndTopoNodes(stepInstance.getTargetExecuteObjects(), groups, topoNodes); + if (stepInstance.isFileStep()) { + List fileSources = stepInstance.getFileSourceList(); + for (FileSourceDTO fileSource : fileSources) { + ExecuteTargetDTO executeTarget = fileSource.getServers(); + if (executeTarget != null && !fileSource.isLocalUpload()) { + // 服务器文件的处理 + extractDynamicGroupsAndTopoNodes(executeTarget, groups, topoNodes); + } + } + } + } + + private void extractDynamicGroupsAndTopoNodes(ExecuteTargetDTO executeTarget, + Set groups, + Set topoNodes) { + if (executeTarget == null) { + return; + } + if (CollectionUtils.isNotEmpty(executeTarget.getDynamicServerGroups())) { + groups.addAll(executeTarget.getDynamicServerGroups()); + } + if (CollectionUtils.isNotEmpty(executeTarget.getTopoNodes())) { + topoNodes.addAll(executeTarget.getTopoNodes()); + } + } + + private void fillDynamicGroupHosts(StopWatch watch, + long appId, + Set groups, + List stepInstances, + Collection variables) { + if (CollectionUtils.isEmpty(groups)) { + return; + } + // 获取动态分组的主机并设置 + watch.start("fillDynamicGroupHosts"); + Map> dynamicGroupHosts = + hostService.batchGetAndGroupHostsByDynamicGroup(appId, groups); + stepInstances.forEach(stepInstance -> { + setHostsForDynamicGroup(stepInstance.getTargetExecuteObjects(), dynamicGroupHosts); + if (stepInstance.isFileStep()) { + List fileSources = stepInstance.getFileSourceList(); + for (FileSourceDTO fileSource : fileSources) { + ExecuteTargetDTO executeTarget = fileSource.getServers(); + if (executeTarget != null && !fileSource.isLocalUpload()) { + // 服务器文件的处理 + setHostsForDynamicGroup(executeTarget, dynamicGroupHosts); + } + } + } + }); + if (CollectionUtils.isNotEmpty(variables)) { + variables.forEach(variable -> { + if (TaskVariableTypeEnum.HOST_LIST.getType() == variable.getType()) { + setHostsForDynamicGroup(variable.getExecuteTarget(), dynamicGroupHosts); + } + }); + } + watch.stop(); + } + + private void setHostsForDynamicGroup(ExecuteTargetDTO executeTarget, + Map> groups) { + if (executeTarget != null && CollectionUtils.isNotEmpty(executeTarget.getDynamicServerGroups())) { + executeTarget.getDynamicServerGroups().forEach(group -> group.setIpList(groups.get(group))); + } + } + + private void fillTopoNodeHosts(StopWatch watch, + long appId, + Set topoNodes, + List stepInstances, + Collection variables) { + if (CollectionUtils.isEmpty(topoNodes)) { + return; + } + watch.start("fillTopoNodeHosts"); + Map> topoNodeHosts = + hostService.getAndGroupHostsByTopoNodes(appId, topoNodes); + stepInstances.forEach(stepInstance -> { + setHostsForTopoNode(stepInstance.getTargetExecuteObjects(), topoNodeHosts); + if (stepInstance.isFileStep()) { + List fileSources = stepInstance.getFileSourceList(); + for (FileSourceDTO fileSource : fileSources) { + ExecuteTargetDTO executeTarget = fileSource.getServers(); + if (executeTarget != null && !fileSource.isLocalUpload()) { + // 服务器文件的处理 + setHostsForTopoNode(executeTarget, topoNodeHosts); + } + } + } + }); + if (CollectionUtils.isNotEmpty(variables)) { + variables.forEach(variable -> { + if (TaskVariableTypeEnum.HOST_LIST.getType() == variable.getType()) { + setHostsForTopoNode(variable.getExecuteTarget(), topoNodeHosts); + } + }); + } + watch.stop(); + } + + private void setHostsForTopoNode(ExecuteTargetDTO executeTarget, + Map> topoNodes) { + if (executeTarget != null && CollectionUtils.isNotEmpty(executeTarget.getTopoNodes())) { + executeTarget.getTopoNodes().forEach(topoNode -> topoNode.setIpList(topoNodes.get(topoNode))); + } + } + + private void fillTaskInstanceHostDetail(TaskInstanceDTO taskInstance, + List stepInstanceList, + Collection variables, + TaskInstanceExecuteObjects taskInstanceExecuteObjects) { + + fillHostAgent(taskInstance, taskInstanceExecuteObjects); + + for (StepInstanceDTO stepInstance : stepInstanceList) { + if (!stepInstance.isStepContainsExecuteObject()) { + continue; + } + // 目标主机设置主机详情 + fillTargetHostDetail(stepInstance, taskInstanceExecuteObjects); + // 文件源设置主机详情 + fillFileSourceHostDetail(stepInstance, taskInstanceExecuteObjects); + } + + if (CollectionUtils.isNotEmpty(variables)) { + variables.forEach(variable -> { + if (variable.getType() == TaskVariableTypeEnum.HOST_LIST.getType()) { + fillHostsDetail(variable.getExecuteTarget(), taskInstanceExecuteObjects); + } + }); + } + } + + private void fillHostAgent(TaskInstanceDTO taskInstance, TaskInstanceExecuteObjects taskInstanceExecuteObjects) { + boolean isUsingGseV2 = isUsingGseV2(taskInstance, + ListUtil.union(taskInstanceExecuteObjects.getValidHosts(), taskInstanceExecuteObjects.getNotInAppHosts())); + /* + * 后续下发任务给GSE会根据agentId路由请求到GSE1.0/2.0。如果要使用GSE2.0，那么直接使用原始bk_agent_id;如果要使用GSE1.0, + * 按照{云区域ID:ip}的方式构造agent_id + */ + Set invalidAgentIdHosts = new HashSet<>(); + + if (CollectionUtils.isNotEmpty(taskInstanceExecuteObjects.getValidHosts())) { + taskInstanceExecuteObjects.getValidHosts() + .forEach(host -> setHostAgentId(isUsingGseV2, host, invalidAgentIdHosts)); + } + + if (CollectionUtils.isNotEmpty(taskInstanceExecuteObjects.getNotInAppHosts())) { + taskInstanceExecuteObjects.getNotInAppHosts() + .forEach(host -> setHostAgentId(isUsingGseV2, host, invalidAgentIdHosts)); + } + + if (CollectionUtils.isNotEmpty(invalidAgentIdHosts)) { + // 如果存在主机没有agentID，不影响影响整个任务的执行。所以这里仅输出日志，不拦截整个任务的执行。后续执行代码会处理`主机没有agentId`的情况 + log.warn("Contains invalid agent id host, appId: {}, isUsingGseV2: {}, invalidHosts: {}", + taskInstance.getAppId(), isUsingGseV2, invalidAgentIdHosts); + } + + setAgentStatus(taskInstanceExecuteObjects.getValidHosts(), isUsingGseV2); + setAgentStatus(taskInstanceExecuteObjects.getNotInAppHosts(), isUsingGseV2); + } + + private void setHostAgentId(boolean isUsingGseV2, HostDTO host, Set invalidAgentIdHosts) { + // 如果对接GSE1.0,使用云区域+ipv4构造agentId + if (!isUsingGseV2) { + host.setAgentId(host.toCloudIp()); + } + if (StringUtils.isBlank(host.getAgentId())) { + invalidAgentIdHosts.add(host); + } + } + + private void fillTargetHostDetail(StepInstanceDTO stepInstance, + TaskInstanceExecuteObjects taskInstanceExecuteObjects) { + fillHostsDetail(stepInstance.getTargetExecuteObjects(), taskInstanceExecuteObjects); + } + + private void fillFileSourceHostDetail(StepInstanceDTO stepInstance, + TaskInstanceExecuteObjects taskInstanceExecuteObjects) { + if (stepInstance.getExecuteType() == SEND_FILE) { + List fileSourceList = stepInstance.getFileSourceList(); + if (fileSourceList != null) { + for (FileSourceDTO fileSource : fileSourceList) { + fillHostsDetail(fileSource.getServers(), taskInstanceExecuteObjects); + } + } + } + } + + private void fillHostsDetail(ExecuteTargetDTO executeTargetDTO, + TaskInstanceExecuteObjects taskInstanceExecuteObjects) { + if (executeTargetDTO != null) { + fillHostsDetail(executeTargetDTO.getStaticIpList(), taskInstanceExecuteObjects); + if (CollectionUtils.isNotEmpty(executeTargetDTO.getDynamicServerGroups())) { + executeTargetDTO.getDynamicServerGroups() + .forEach(group -> fillHostsDetail(group.getIpList(), taskInstanceExecuteObjects)); + } + if (CollectionUtils.isNotEmpty(executeTargetDTO.getTopoNodes())) { + executeTargetDTO.getTopoNodes().forEach( + topoNode -> fillHostsDetail(topoNode.getIpList(), taskInstanceExecuteObjects)); + } + } + } + + private void fillHostsDetail(Collection hosts, TaskInstanceExecuteObjects taskInstanceExecuteObjects) { + if (CollectionUtils.isNotEmpty(hosts)) { + hosts.forEach(host -> host.updateByHost(taskInstanceExecuteObjects.queryByHostKey(host))); + } + } + + private void setAgentStatus(List hosts, boolean isUsingGseV2) { + if (CollectionUtils.isEmpty(hosts)) { + return; + } + long start = System.currentTimeMillis(); + + List hostAgentStateQueryList = new ArrayList<>(hosts.size()); + Map hostAgentStateQueryMap = new HashMap<>(hosts.size()); + hosts.stream() + .filter(host -> isAgentIdValid(host, isUsingGseV2)) + .forEach(host -> { + HostAgentStateQuery hostAgentStateQuery = HostAgentStateQuery.from(host); + hostAgentStateQueryList.add(hostAgentStateQuery); + hostAgentStateQueryMap.put(host, hostAgentStateQuery); + }); + + // 此处用于记录下发任务时的Agent状态快照数据，因此使用最终真实下发任务的agentId获取状态 + Map agentStateMap = preferV2AgentStateClient.batchGetAgentState(hostAgentStateQueryList); + + for (HostDTO host : hosts) { + HostAgentStateQuery hostAgentStateQuery = hostAgentStateQueryMap.get(host); + if (hostAgentStateQuery == null) { + host.setAlive(AgentAliveStatusEnum.NOT_ALIVE.getStatusValue()); + continue; + } + + String effectiveAgentId = preferV2AgentStateClient.getEffectiveAgentId(hostAgentStateQuery); + if (StringUtils.isEmpty(effectiveAgentId)) { + host.setAlive(AgentAliveStatusEnum.NOT_ALIVE.getStatusValue()); + continue; + } + AgentState agentState = agentStateMap.get(effectiveAgentId); + if (agentState != null) { + host.setAlive(AgentAliveStatusEnum.fromAgentState(agentState).getStatusValue()); + } else { + host.setAlive(AgentAliveStatusEnum.NOT_ALIVE.getStatusValue()); + } + } + + long cost = System.currentTimeMillis() - start; + if (cost > 1000) { + log.warn("SetAgentStatus slow, hostSize: {}, cost:{} ms", hosts.size(), cost); + } + } + + private boolean isAgentIdValid(HostDTO host, boolean isUsingGseV2) { + return isUsingGseV2 ? AgentUtils.isGseV2AgentId(host.getAgentId()) : + AgentUtils.isGseV1AgentId(host.getAgentId()); + } + + private void acquireAndSetContainers(TaskInstanceExecuteObjects taskInstanceExecuteObjects, + TaskInstanceDTO taskInstance, + List stepInstances) { + + // 根据静态容器列表方式获取并设置容器执行对象 + acquireAndSetContainersByStaticContainerList(taskInstanceExecuteObjects, + taskInstance, stepInstances); + + // 根据 ContainerFilter 方式获取并设置容器执行对象 + acquireAndSetContainersByContainerFilters(taskInstanceExecuteObjects, + taskInstance, stepInstances); + + taskInstanceExecuteObjects.setContainsAnyContainer( + CollectionUtils.isNotEmpty(taskInstanceExecuteObjects.getValidContainers())); + + // 增加容器 topo 信息（集群 UID，集群名称、命名空间名称等) + fillContainerTopoInfo(taskInstance.getAppId(), taskInstanceExecuteObjects.getValidContainers(), stepInstances); + } + + private void fillContainerTopoInfo(long appId, + Collection containers, + List stepInstances) { + if (CollectionUtils.isEmpty(containers)) { + return; + } + long bizId = Long.parseLong(appScopeMappingService.getScopeByAppId(appId).getId()); + // 从 cmdb 获取集群信息 + List ccKubeClusterIds = + containers.stream().map(Container::getClusterId).distinct().collect(Collectors.toList()); + List clusters = + bizCmdbClient.listKubeClusters(KubeClusterQuery.Builder.builder(bizId).ids(ccKubeClusterIds).build()); + Map clusterMap = clusters.stream().collect( + Collectors.toMap(KubeClusterDTO::getId, cluster -> cluster)); + + // 从 cmdb 获取命名空间信息 + List ccKubeNamespaceIds = + containers.stream().map(Container::getNamespaceId).distinct().collect(Collectors.toList()); + List namespaces = + bizCmdbClient.listKubeNamespaces(NamespaceQuery.Builder.builder(bizId).ids(ccKubeNamespaceIds).build()); + Map namespaceMap = namespaces.stream().collect( + Collectors.toMap(KubeNamespaceDTO::getId, namespace -> namespace)); + + // 填充 cluster、 namespace 信息 + for (StepInstanceDTO stepInstance : stepInstances) { + stepInstance.forEachExecuteObjects(executeObjects -> { + if (CollectionUtils.isNotEmpty(executeObjects.getContainerFilters())) { + executeObjects.getContainerFilters().forEach(containerFilter -> { + if (CollectionUtils.isNotEmpty(containerFilter.getContainers())) { + containerFilter.getContainers().forEach( + container -> addTopoDetail(container, clusterMap, namespaceMap)); + } + }); + } + if (CollectionUtils.isNotEmpty(executeObjects.getStaticContainerList())) { + executeObjects.getStaticContainerList().forEach( + container -> addTopoDetail(container, clusterMap, namespaceMap)); + } + }); + } + } + + private void addTopoDetail(Container container, + Map clusterMap, + Map namespaceMap) { + KubeClusterDTO cluster = clusterMap.get(container.getClusterId()); + container.setClusterName(cluster.getName()); + container.setClusterUID(cluster.getUid()); + + KubeNamespaceDTO namespace = namespaceMap.get(container.getNamespaceId()); + container.setNamespace(namespace.getName()); + } + + private void acquireAndSetContainersByStaticContainerList(TaskInstanceExecuteObjects taskInstanceExecuteObjects, + TaskInstanceDTO taskInstance, + List stepInstances) { + Set queryContainerIds = new HashSet<>(); + for (StepInstanceDTO stepInstance : stepInstances) { + queryContainerIds.addAll( + stepInstance.extractStaticContainerList().stream() + .map(Container::getId) + .collect(Collectors.toList())); + } + if (CollectionUtils.isNotEmpty(queryContainerIds)) { + List containers = containerService.listContainerByIds( + taskInstance.getAppId(), queryContainerIds); + if (CollectionUtils.isNotEmpty(containers)) { + fillTaskInstanceContainerDetail(taskInstanceExecuteObjects, stepInstances, + containers.stream().collect( + Collectors.toMap(Container::getId, container -> container, (oldValue, newValue) -> newValue))); + } + } + } + + private void acquireAndSetContainersByContainerFilters(TaskInstanceExecuteObjects taskInstanceExecuteObjects, + TaskInstanceDTO taskInstance, + List stepInstances) { + for (StepInstanceDTO stepInstance : stepInstances) { + stepInstance.forEachExecuteObjects(executeObjects -> { + if (CollectionUtils.isNotEmpty(executeObjects.getContainerFilters())) { + executeObjects.getContainerFilters().forEach(containerFilter -> { + List filteredContainers = + containerService.listContainerByContainerFilter(taskInstance.getAppId(), containerFilter); + if (CollectionUtils.isNotEmpty(filteredContainers)) { + taskInstanceExecuteObjects.addContainers(filteredContainers); + containerFilter.setContainers(filteredContainers); + } + }); + } + }); + } + } + + private void fillTaskInstanceContainerDetail(TaskInstanceExecuteObjects taskInstanceExecuteObjects, + List stepInstanceList, + Map containerMap) { + Set notExistContainerIds = new HashSet<>(); + taskInstanceExecuteObjects.setNotExistContainerIds(notExistContainerIds); + + for (StepInstanceDTO stepInstance : stepInstanceList) { + stepInstance.forEachExecuteObjects(executeObjects -> { + if (CollectionUtils.isNotEmpty(executeObjects.getStaticContainerList())) { + executeObjects.getStaticContainerList() + .forEach(container -> { + Container containDetail = containerMap.get(container.getId()); + if (containDetail == null) { + notExistContainerIds.add(container.getId()); + return; + } + taskInstanceExecuteObjects.addContainer(containDetail); + container.updatePropsByContainer(containDetail); + }); + } + }); + } + } + + private void mergeExecuteObjects(List stepInstanceList, + Collection variables, + boolean isSupportExecuteObjectFeature) { + stepInstanceList.forEach(stepInstance -> + stepInstance.buildStepFinalExecuteObjects(isSupportExecuteObjectFeature)); + if (CollectionUtils.isNotEmpty(variables)) { + variables.forEach(variable -> { + if (TaskVariableTypeEnum.HOST_LIST.getType() == variable.getType() + && variable.getExecuteTarget() != null) { + variable.getExecuteTarget().buildMergedExecuteObjects(isSupportExecuteObjectFeature); + } + }); + } + } + + private void checkExecuteObjectExist(TaskInstanceDTO taskInstance, + List stepInstanceList, + TaskInstanceExecuteObjects taskInstanceExecuteObjects) { + List invalidExecuteObjects = new ArrayList<>(); + + // 处理主机执行对象 + if (CollectionUtils.isNotEmpty(taskInstanceExecuteObjects.getNotExistHosts())) { + if (shouldIgnoreInvalidHost(taskInstance)) { + if (taskInstanceExecuteObjects.getNotExistHosts().stream().anyMatch(host -> host.getHostId() == null)) { + // 由于历史原因，部分定时任务使用了管控区域ID:Ipv4 作为主机 ID，并且这部分主机已经不存在于 cmdb，所以无法 + // 正确获取到对应的 hostId，会导致后续报错；所以这里直接对外抛出错误，不再继续兼容处理 + invalidExecuteObjects.addAll(taskInstanceExecuteObjects.getNotExistHosts().stream() + .map(this::printHostIdOrIp).collect(Collectors.toList())); + } else { + // 忽略主机不存在错误，并标识执行对象的 invalid 属性为 true + markExecuteObjectInvalid(stepInstanceList, taskInstanceExecuteObjects.getNotExistHosts()); + } + } else { + invalidExecuteObjects.addAll(taskInstanceExecuteObjects.getNotExistHosts().stream() + .map(this::printHostIdOrIp).collect(Collectors.toList())); + } + } + + // 处理容器执行对象 + if (CollectionUtils.isNotEmpty(taskInstanceExecuteObjects.getNotExistContainerIds())) { + invalidExecuteObjects.addAll( + taskInstanceExecuteObjects.getNotExistContainerIds().stream() + .map(containerId -> "(container_id:" + containerId + ")") + .collect(Collectors.toList())); + } + + if (CollectionUtils.isNotEmpty(invalidExecuteObjects)) { + String executeObjectStr = StringUtils.join(invalidExecuteObjects, ","); + log.warn("The following execute object are not exist, invalidExecuteObjects={}", + invalidExecuteObjects); + throw new FailedPreconditionException(ErrorCode.EXECUTE_OBJECT_NOT_EXIST, + new Object[]{invalidExecuteObjects.size(), executeObjectStr}); + } + } + + private void markExecuteObjectInvalid(List stepInstanceList, + List invalidHost) { + for (StepInstanceDTO stepInstance : stepInstanceList) { + if (!stepInstance.isStepContainsExecuteObject()) { + continue; + } + // 检查目标主机 + stepInstance.getTargetExecuteObjects().getExecuteObjectsCompatibly().stream() + .filter(ExecuteObject::isHostExecuteObject) + .forEach(executeObject -> { + if (invalidHost.contains(executeObject.getHost())) { + executeObject.setInvalid(true); + } + }); + // 如果是文件分发任务，检查文件源 + if (stepInstance.isFileStep()) { + List fileSourceList = stepInstance.getFileSourceList(); + if (CollectionUtils.isEmpty(fileSourceList)) { + return; + } + for (FileSourceDTO fileSource : fileSourceList) { + // 远程文件分发需要校验文件源主机;其他类型不需要 + if (fileSource.getFileType().equals(TaskFileTypeEnum.SERVER.getType())) { + ExecuteTargetDTO executeTarget = fileSource.getServers(); + if (executeTarget == null || + CollectionUtils.isEmpty(executeTarget.getExecuteObjectsCompatibly())) { + continue; + } + executeTarget.getExecuteObjectsCompatibly().stream() + .filter(ExecuteObject::isHostExecuteObject) + .forEach(executeObject -> { + if (invalidHost.contains(executeObject.getHost())) { + executeObject.setInvalid(true); + } + }); + } + } + } + } + } + + private boolean shouldIgnoreInvalidHost(TaskInstanceDTO taskInstance) { + // 定时任务忽略非法主机，继续执行 + return TaskStartupModeEnum.getStartupMode(taskInstance.getStartupMode()) == TaskStartupModeEnum.CRON; + } + + private void throwHostInvalidException(Long appId, Collection invalidHosts) { + ServiceApplicationDTO application = applicationService.getAppById(appId); + String appName = application.getName(); + String hostListStr = StringUtils.join(invalidHosts.stream() + .map(this::printHostIdOrIp).collect(Collectors.toList()), ","); + log.warn("The following hosts are invalid, hosts={}", invalidHosts); + throw new FailedPreconditionException(ErrorCode.HOST_INVALID, + new Object[]{appName, invalidHosts.size(), hostListStr}); + } + + /** + * 判断执行对象是否可以被当前作业使用 + * + * @param taskInstance 作业实例 + * @param stepInstanceList 作业步骤列表 + * @param taskInstanceExecuteObjects 作业实例中包含的执行对象 + */ + private void checkExecuteObjectAccessible(TaskInstanceDTO taskInstance, + List stepInstanceList, + TaskInstanceExecuteObjects taskInstanceExecuteObjects) { + if (CollectionUtils.isEmpty(taskInstanceExecuteObjects.getNotInAppHosts())) { + return; + } + Map> whileHostAllowActions = taskInstanceExecuteObjects.getWhiteHostAllowActions(); + log.info("Contains hosts not in app, check white host config. notInAppHosts: {}, whileHostAllowActions: {}", + taskInstanceExecuteObjects.getNotInAppHosts(), whileHostAllowActions); + Map notInAppHostMap = taskInstanceExecuteObjects.getNotInAppHosts().stream() + .collect(Collectors.toMap(HostDTO::getHostId, host -> host, (host1, host2) -> host2)); + + // 非法的主机 + boolean shouldIgnoreInvalidHost = shouldIgnoreInvalidHost(taskInstance); + Set invalidHosts = new HashSet<>(); + for (StepInstanceDTO stepInstance : stepInstanceList) { + if (!stepInstance.isStepContainsExecuteObject()) { + continue; + } + TaskStepTypeEnum stepType = stepInstance.getStepType(); + // 检查目标主机 + stepInstance.getTargetExecuteObjects().getExecuteObjectsCompatibly().stream() + .filter(ExecuteObject::isHostExecuteObject) + .forEach(executeObject -> { + if (isHostUnAccessible(stepType, executeObject.getHost(), notInAppHostMap, whileHostAllowActions)) { + if (shouldIgnoreInvalidHost) { + executeObject.setInvalid(true); + } else { + invalidHosts.add(executeObject.getHost()); + } + } + }); + // 如果是文件分发任务，检查文件源 + checkFileSourceHostAccessible(invalidHosts, stepInstance, stepType, notInAppHostMap, + whileHostAllowActions, shouldIgnoreInvalidHost); + } + + if (CollectionUtils.isNotEmpty(invalidHosts)) { + // 检查是否在白名单配置 + log.warn("Found hosts not in target app: {}!", taskInstance.getAppId()); + throwHostInvalidException(taskInstance.getAppId(), invalidHosts); + } + } + + private void checkFileSourceHostAccessible(Set invalidHosts, + StepInstanceDTO stepInstance, + TaskStepTypeEnum stepType, + Map notInAppHostMap, + Map> whileHostAllowActions, + boolean ignoreInvalidHost) { + if (!stepInstance.isFileStep()) { + return; + } + List fileSourceList = stepInstance.getFileSourceList(); + if (CollectionUtils.isEmpty(fileSourceList)) { + return; + } + for (FileSourceDTO fileSource : fileSourceList) { + // 远程文件分发需要校验文件源主机;其他类型不需要 + if (fileSource.getFileType().equals(TaskFileTypeEnum.SERVER.getType())) { + ExecuteTargetDTO executeTarget = fileSource.getServers(); + if (executeTarget == null || CollectionUtils.isEmpty(executeTarget.getExecuteObjectsCompatibly())) { + continue; + } + executeTarget.getExecuteObjectsCompatibly().stream() + .filter(ExecuteObject::isHostExecuteObject) + .forEach(executeObject -> { + if (isHostUnAccessible(stepType, executeObject.getHost(), + notInAppHostMap, whileHostAllowActions)) { + if (ignoreInvalidHost) { + executeObject.setInvalid(true); + } else { + invalidHosts.add(executeObject.getHost()); + } + } + }); + } + } + } + + private boolean isHostUnAccessible(TaskStepTypeEnum stepType, + HostDTO host, + Map notInAppHostMap, + Map> whileHostAllowActions) { + long hostId = host.getHostId(); + if (!notInAppHostMap.containsKey(host.getHostId())) { + // 主机在当前业务下，可以使用 + return false; + } + // 如果主机不在当前业务下，需要判断主机白名单 + if (whileHostAllowActions == null || whileHostAllowActions.isEmpty()) { + return true; + } + List allowActions = whileHostAllowActions.get(hostId); + String actionScope = (stepType == TaskStepTypeEnum.SCRIPT ? + ActionScopeEnum.SCRIPT_EXECUTE.name() : + (stepType == TaskStepTypeEnum.FILE ? ActionScopeEnum.FILE_DISTRIBUTION.name() : "")); + return CollectionUtils.isEmpty(allowActions) || !allowActions.contains(actionScope); + } + + private String printHostIdOrIp(HostDTO host) { + if (StringUtils.isNotBlank(host.getPrimaryIp())) { + // 优先使用ip，可读性更好 + return "(ip:" + host.getPrimaryIp() + ")"; + } else { + return "(host_id:" + host.getHostId() + ")"; + } + } + + private boolean isSupportExecuteObjectFeature(TaskInstanceDTO taskInstance) { + ToggleEvaluateContext featureExecutionContext = + ToggleEvaluateContext.builder() + .addContextParam(ToggleStrategyContextParams.CTX_PARAM_RESOURCE_SCOPE, + appScopeMappingService.getScopeByAppId(taskInstance.getAppId())); + + boolean featureEnabled = FeatureToggle.checkFeature( + FeatureIdConstants.FEATURE_EXECUTE_OBJECT, + featureExecutionContext + ); + log.info("Check feature: {}, result: {}", FeatureIdConstants.FEATURE_EXECUTE_OBJECT, featureEnabled); + return featureEnabled; + } + + /** + * @param appId 业务ID + * @param hosts 主机列表 + * @return key=hostId, value: 允许的操作列表 + */ + private Map> getHostAllowedActions(long appId, Collection hosts) { + Map> hostAllowActionsMap = new HashMap<>(); + if (CollectionUtils.isEmpty(hosts)) { + return hostAllowActionsMap; + } + for (HostDTO host : hosts) { + List allowActions = whiteHostCache.getHostAllowedAction(appId, host.getHostId()); + if (CollectionUtils.isNotEmpty(allowActions)) { + hostAllowActionsMap.put(host.getHostId(), allowActions); + } + } + return hostAllowActionsMap; + } + + private boolean needRefreshHostBkAgentId(TaskInstanceDTO taskInstance) { + /* + * tmp: GSE Agent v1/v2 兼容期间特殊逻辑, 对于节点管理安装Agent插件的请求需要实时获取bk_agent_id。等后续只对接GSE V2 之后， + * 此处代码可删除。 https://github.com/Tencent/bk-job/issues/1542 + */ + return taskInstance.getStartupMode() == TaskStartupModeEnum.API.getValue() + && StringUtils.isNotEmpty(taskInstance.getAppCode()) + && (StringUtils.equals(taskInstance.getAppCode(), "bkc-nodeman") + || StringUtils.equals(taskInstance.getAppCode(), "bk_nodeman")); + } + + private boolean isUsingGseV2(TaskInstanceDTO taskInstance, Collection taskInstanceHosts) { + // 初始化Job任务灰度对接 GSE2.0 上下文 + ToggleEvaluateContext featureExecutionContext = + ToggleEvaluateContext.builder() + .addContextParam(ToggleStrategyContextParams.CTX_PARAM_RESOURCE_SCOPE, + appScopeMappingService.getScopeByAppId(taskInstance.getAppId())) + .addContextParam(JobInstanceAttrToggleStrategy.CTX_PARAM_IS_ANY_GSE_V2_AGENT_AVAILABLE, + () -> taskInstanceHosts.stream().anyMatch(host -> AgentUtils.isGseV2AgentId(host.getAgentId()))) + .addContextParam(JobInstanceAttrToggleStrategy.CTX_PARAM_IS_ALL_GSE_V2_AGENT_AVAILABLE, + () -> taskInstanceHosts.stream().allMatch( + host -> AgentUtils.isGseV2AgentId(host.getAgentId()))) + .addContextParam(JobInstanceAttrToggleStrategy.CTX_PARAM_STARTUP_MODE, + () -> TaskStartupModeEnum.getStartupMode(taskInstance.getStartupMode()).getName()) + .addContextParam(JobInstanceAttrToggleStrategy.CTX_PARAM_OPERATOR, taskInstance::getOperator); + + boolean isUsingGseV2 = FeatureToggle.checkFeature( + FeatureIdConstants.FEATURE_GSE_V2, + featureExecutionContext + ); + log.info("Use gse version {}", isUsingGseV2 ? "v2" : "v1"); + return isUsingGseV2; + } + + /** + * 提取步骤和全局变量中包含的所有主机 + * + * @param stepInstanceList 步骤实例列表 + * @param variables 作业全局变量列表 + * @return 所有主机 + */ + public Set extractHosts(List stepInstanceList, + Collection variables) { + Set hosts = new HashSet<>(); + for (StepInstanceDTO stepInstance : stepInstanceList) { + if (!stepInstance.isStepContainsExecuteObject()) { + continue; + } + if (stepInstance.getTargetExecuteObjects() != null) { + hosts.addAll(stepInstance.getTargetExecuteObjects().extractHosts()); + } + if (stepInstance.getExecuteType() == SEND_FILE) { + List fileSourceList = stepInstance.getFileSourceList(); + if (fileSourceList != null) { + for (FileSourceDTO fileSource : fileSourceList) { + if (fileSource.getServers() != null) { + hosts.addAll(fileSource.getServers().extractHosts()); + } + } + } + } + } + if (CollectionUtils.isNotEmpty(variables)) { + variables.stream() + .filter(variable -> variable.getType() == TaskVariableTypeEnum.HOST_LIST.getType() + && variable.getExecuteTarget() != null) + .forEach(variable -> hosts.addAll(variable.getExecuteTarget().extractHosts())); + } + return hosts; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskInstanceServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskInstanceServiceImpl.java index c7b203cf61..e5b48ae565 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskInstanceServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskInstanceServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,147 +24,151 @@ package com.tencent.bk.job.execute.service.impl; -import com.tencent.bk.job.common.constant.NotExistPathHandlerEnum; +import com.tencent.bk.audit.annotations.ActionAuditRecord; +import com.tencent.bk.audit.annotations.AuditInstanceRecord; +import com.tencent.bk.audit.context.ActionAuditContext; +import com.tencent.bk.job.common.audit.constants.EventContentConstants; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.NotFoundException; +import com.tencent.bk.job.common.iam.constant.ActionId; +import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; +import com.tencent.bk.job.common.model.dto.AppResourceScope; import com.tencent.bk.job.common.model.dto.HostDTO; -import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.execute.auth.ExecuteAuthService; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; -import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; -import com.tencent.bk.job.execute.common.constants.TaskStartupModeEnum; -import com.tencent.bk.job.execute.common.constants.TaskTypeEnum; -import com.tencent.bk.job.execute.common.converter.StepTypeExecuteTypeConverter; -import com.tencent.bk.job.execute.dao.StepInstanceDAO; +import com.tencent.bk.job.execute.common.context.JobExecuteContext; +import com.tencent.bk.job.execute.common.context.JobExecuteContextThreadLocalRepo; +import com.tencent.bk.job.execute.common.context.JobInstanceContext; import com.tencent.bk.job.execute.dao.TaskInstanceDAO; -import com.tencent.bk.job.execute.engine.model.TaskVariableDTO; -import com.tencent.bk.job.execute.model.FileSourceDTO; -import com.tencent.bk.job.execute.model.StepInstanceBaseDTO; -import com.tencent.bk.job.execute.model.StepInstanceDTO; +import com.tencent.bk.job.execute.dao.common.IdGen; import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.service.ApplicationService; +import com.tencent.bk.job.execute.service.StepInstanceService; import com.tencent.bk.job.execute.service.TaskInstanceService; import com.tencent.bk.job.execute.service.TaskInstanceVariableService; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskStepTypeEnum; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import java.util.ArrayList; import java.util.Collection; import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; - -import static com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum.EXECUTE_SCRIPT; -import static com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum.EXECUTE_SQL; -import static com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum.MANUAL_CONFIRM; -import static com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum.SEND_FILE; @Slf4j @Service public class TaskInstanceServiceImpl implements TaskInstanceService { private final ApplicationService applicationService; - private final StepInstanceDAO stepInstanceDAO; private final TaskInstanceDAO taskInstanceDAO; private final TaskInstanceVariableService taskInstanceVariableService; + private final ExecuteAuthService executeAuthService; + private final StepInstanceService stepInstanceService; + + private final IdGen idGen; @Autowired public TaskInstanceServiceImpl(ApplicationService applicationService, - StepInstanceDAO stepInstanceDAO, TaskInstanceDAO taskInstanceDAO, - TaskInstanceVariableService taskInstanceVariableService) { + TaskInstanceVariableService taskInstanceVariableService, + ExecuteAuthService executeAuthService, + StepInstanceService stepInstanceService, + IdGen idGen) { this.applicationService = applicationService; - this.stepInstanceDAO = stepInstanceDAO; + this.stepInstanceService = stepInstanceService; this.taskInstanceDAO = taskInstanceDAO; this.taskInstanceVariableService = taskInstanceVariableService; + this.executeAuthService = executeAuthService; + this.idGen = idGen; } @Override public long addTaskInstance(TaskInstanceDTO taskInstance) { + taskInstance.setId(idGen.genTaskInstanceId()); return taskInstanceDAO.addTaskInstance(taskInstance); } @Override - public TaskInstanceDTO getTaskInstance(long taskInstanceId) { - return taskInstanceDAO.getTaskInstance(taskInstanceId); + public TaskInstanceDTO getTaskInstance(long appId, long taskInstanceId) throws NotFoundException { + TaskInstanceDTO taskInstance = getTaskInstance(taskInstanceId); + if (!taskInstance.getAppId().equals(appId)) { + log.warn("Task instance is not in application, taskInstanceId={}, appId={}", taskInstanceId, appId); + throw new NotFoundException(ErrorCode.TASK_INSTANCE_NOT_EXIST); + } + return taskInstance; } @Override - public long addStepInstance(StepInstanceDTO stepInstance) { - long stepInstanceId = stepInstanceDAO.addStepInstanceBase(stepInstance); - if (stepInstanceId > 0) { - stepInstance.setId(stepInstanceId); - Integer stepType = stepInstance.getExecuteType(); - if (stepType.equals(EXECUTE_SQL.getValue()) || stepType.equals(EXECUTE_SCRIPT.getValue())) { - stepInstanceDAO.addScriptStepInstance(stepInstance); - } else if (stepType.equals(SEND_FILE.getValue())) { - stepInstanceDAO.addFileStepInstance(stepInstance); - } else if (stepType.equals(MANUAL_CONFIRM.getValue())) { - stepInstanceDAO.addConfirmStepInstance(stepInstance); - } + public TaskInstanceDTO getTaskInstance(long taskInstanceId) throws NotFoundException { + TaskInstanceDTO taskInstance = taskInstanceDAO.getTaskInstance(taskInstanceId); + if (taskInstance == null) { + log.warn("Task instance is not exist, taskInstanceId={}", taskInstanceId); + throw new NotFoundException(ErrorCode.TASK_INSTANCE_NOT_EXIST); } - return stepInstanceId; + return taskInstance; } - public TaskInstanceDTO getTaskInstanceDetail(long taskInstanceId) { + @Override + @ActionAuditRecord( + actionId = ActionId.VIEW_HISTORY, + instance = @AuditInstanceRecord( + instanceIds = "#taskInstanceId", + instanceNames = "#$?.name" + ), + content = EventContentConstants.VIEW_JOB_INSTANCE + ) + public TaskInstanceDTO getTaskInstance(String username, long appId, long taskInstanceId) + throws NotFoundException, PermissionDeniedException { + TaskInstanceDTO taskInstance = getTaskInstance(taskInstanceId); - if (taskInstance == null) { - return null; - } - List baseStepInstanceList = - stepInstanceDAO.listStepInstanceBaseByTaskInstanceId(taskInstanceId); - if (baseStepInstanceList == null || baseStepInstanceList.isEmpty()) { - return taskInstance; + checkTaskInstanceExist(appId, taskInstanceId, taskInstance); + auditAndAuthViewTaskInstance(username, taskInstance); + JobExecuteContext jobExecuteContext = JobExecuteContextThreadLocalRepo.get(); + if (jobExecuteContext != null) { + JobInstanceContext jobInstanceContext = new JobInstanceContext(); + jobInstanceContext.setTaskInstanceId(taskInstanceId); } - List stepInstanceList = new ArrayList<>(); - for (StepInstanceBaseDTO baseStepInstance : baseStepInstanceList) { - StepInstanceDTO stepInstance = new StepInstanceDTO(baseStepInstance); - fillStepInstanceDetail(stepInstance); - stepInstanceList.add(stepInstance); - } - taskInstance.setStepInstances(stepInstanceList); - List taskVariables = taskInstanceVariableService.getByTaskInstanceId(taskInstanceId); - taskInstance.setVariables(taskVariables); return taskInstance; } - private void fillStepInstanceDetail(StepInstanceDTO stepInstance) { - long stepInstanceId = stepInstance.getId(); - Integer stepType = StepTypeExecuteTypeConverter.convertToStepType(stepInstance.getExecuteType()); - if (stepType.equals(TaskStepTypeEnum.SCRIPT.getValue())) { - stepInstance.fillScriptStepInfo(stepInstanceDAO.getScriptStepInstance(stepInstanceId)); - } else if (stepType.equals(TaskStepTypeEnum.FILE.getValue())) { - stepInstance.fillFileStepInfo(stepInstanceDAO.getFileStepInstance(stepInstanceId)); - } else if (stepType.equals(TaskStepTypeEnum.APPROVAL.getValue())) { - stepInstance.fillConfirmStepInfo(stepInstanceDAO.getConfirmStepInstance(stepInstanceId)); + private void checkTaskInstanceExist(long appId, long taskInstanceId, TaskInstanceDTO taskInstance) { + if (taskInstance == null) { + log.warn("Task instance is not exist, taskInstanceId={}", taskInstanceId); + throw new NotFoundException(ErrorCode.TASK_INSTANCE_NOT_EXIST); + } + if (!taskInstance.getAppId().equals(appId)) { + log.warn("Task instance is not in application, taskInstanceId={}, appId={}", taskInstanceId, appId); + throw new NotFoundException(ErrorCode.TASK_INSTANCE_NOT_EXIST); } } - @Override - public List listStepInstanceByTaskInstanceId(long taskInstanceId) { - return stepInstanceDAO.listStepInstanceBaseByTaskInstanceId(taskInstanceId); + private void auditAndAuthViewTaskInstance(String username, + TaskInstanceDTO taskInstance) { + // 审计 + ActionAuditContext.current() + .setInstanceId(String.valueOf(taskInstance.getId())) + .setInstanceName(taskInstance.getName()); + + // 鉴权 + executeAuthService.authViewTaskInstance(username, new AppResourceScope(taskInstance.getAppId()), taskInstance); } @Override - public StepInstanceBaseDTO getBaseStepInstance(long stepInstanceId) { - return stepInstanceDAO.getStepInstanceBase(stepInstanceId); + public TaskInstanceDTO getTaskInstanceDetail(long taskInstanceId) { + TaskInstanceDTO taskInstance = getTaskInstance(taskInstanceId); + fillStepAndVariable(taskInstance); + return taskInstance; } - @Override - public StepInstanceDTO getStepInstanceDetail(long stepInstanceId) { - StepInstanceBaseDTO stepInstanceBase = stepInstanceDAO.getStepInstanceBase(stepInstanceId); - if (stepInstanceBase == null) { - return null; - } - StepInstanceDTO stepInstance = new StepInstanceDTO(stepInstanceBase); - fillStepInstanceDetail(stepInstance); - return stepInstance; + private void fillStepAndVariable(TaskInstanceDTO taskInstance) { + taskInstance.setStepInstances(stepInstanceService.listStepInstanceByTaskInstanceId(taskInstance.getId())); + taskInstance.setVariables(taskInstanceVariableService.getByTaskInstanceId(taskInstance.getId())); } @Override - public StepInstanceBaseDTO getFirstStepInstance(long taskInstanceId) { - return stepInstanceDAO.getFirstStepInstanceBase(taskInstanceId); + public TaskInstanceDTO getTaskInstanceDetail(String username, long appId, long taskInstanceId) + throws NotFoundException, PermissionDeniedException { + TaskInstanceDTO taskInstance = getTaskInstance(username, appId, taskInstanceId); + fillStepAndVariable(taskInstance); + return taskInstance; } @Override @@ -172,11 +176,6 @@ public void updateTaskStatus(long taskInstanceId, int status) { taskInstanceDAO.updateTaskStatus(taskInstanceId, status); } - @Override - public List getTaskStepIdList(long taskInstanceId) { - return taskInstanceDAO.getTaskStepInstanceIdList(taskInstanceId); - } - @Override public void updateTaskCurrentStepId(long taskInstanceId, Long stepInstanceId) { taskInstanceDAO.updateTaskCurrentStepId(taskInstanceId, stepInstanceId); @@ -187,197 +186,19 @@ public void resetTaskStatus(long taskInstanceId) { taskInstanceDAO.resetTaskStatus(taskInstanceId); } - @Override - public void updateStepStatus(long stepInstanceId, int status) { - stepInstanceDAO.updateStepStatus(stepInstanceId, status); - } - - @Override - public void resetStepExecuteInfoForRetry(long stepInstanceId) { - stepInstanceDAO.resetStepExecuteInfoForRetry(stepInstanceId); - } - @Override public void resetTaskExecuteInfoForRetry(long taskInstanceId) { taskInstanceDAO.resetTaskExecuteInfoForRetry(taskInstanceId); } @Override - public void resetStepStatus(long stepInstanceId) { - stepInstanceDAO.resetStepStatus(stepInstanceId); - } - - @Override - public void updateStepStartTime(long stepInstanceId, Long startTime) { - stepInstanceDAO.updateStepStartTime(stepInstanceId, startTime); - } - - @Override - public void updateStepStartTimeIfNull(long stepInstanceId, Long startTime) { - stepInstanceDAO.updateStepStartTimeIfNull(stepInstanceId, startTime); - } - - @Override - public void updateStepEndTime(long stepInstanceId, Long endTime) { - stepInstanceDAO.updateStepEndTime(stepInstanceId, endTime); - } - - @Override - public void addStepInstanceExecuteCount(long stepInstanceId) { - stepInstanceDAO.addStepInstanceExecuteCount(stepInstanceId); - } - - @Override - public void updateStepTotalTime(long stepInstanceId, long totalTime) { - stepInstanceDAO.updateStepTotalTime(stepInstanceId, totalTime); - } - - @Override - public void updateTaskExecutionInfo(long taskInstanceId, RunStatusEnum status, Long currentStepId, Long startTime - , Long endTime, Long totalTime) { - taskInstanceDAO.updateTaskExecutionInfo(taskInstanceId, status, currentStepId, startTime, endTime, totalTime); - } - - @Override - public void updateStepExecutionInfo(long stepInstanceId, RunStatusEnum status, Long startTime, Long endTime, + public void updateTaskExecutionInfo(long taskInstanceId, + RunStatusEnum status, + Long currentStepId, + Long startTime, + Long endTime, Long totalTime) { - stepInstanceDAO.updateStepExecutionInfo(stepInstanceId, status, startTime, endTime, totalTime); - } - - @Override - public void updateResolvedScriptParam(long stepInstanceId, String resolvedScriptParam) { - stepInstanceDAO.updateResolvedScriptParam(stepInstanceId, resolvedScriptParam); - } - - @Override - public void updateResolvedSourceFile(long stepInstanceId, List resolvedFileSources) { - if (log.isDebugEnabled()) { - log.debug("updateResolvedSourceFile={}", JsonUtils.toJson(resolvedFileSources)); - } - stepInstanceDAO.updateResolvedSourceFile(stepInstanceId, resolvedFileSources); - } - - @Override - public void updateResolvedTargetPath(long stepInstanceId, String resolvedTargetPath) { - stepInstanceDAO.updateResolvedTargetPath(stepInstanceId, resolvedTargetPath); - } - - @Override - public void updateConfirmReason(long stepInstanceId, String confirmReason) { - stepInstanceDAO.updateConfirmReason(stepInstanceId, confirmReason); - } - - @Override - public void updateStepOperator(long stepInstanceId, String operator) { - stepInstanceDAO.updateStepOperator(stepInstanceId, operator); - } - - @Override - public StepInstanceDTO getPreExecutableStepInstance(long taskInstanceId, long stepInstanceId) { - StepInstanceBaseDTO preStepInstance = stepInstanceDAO.getPreExecutableStepInstance(taskInstanceId, - stepInstanceId); - if (preStepInstance == null) { - return null; - } - StepInstanceDTO stepInstance = new StepInstanceDTO(preStepInstance); - fillStepInstanceDetail(stepInstance); - return stepInstance; - } - - @Override - public StepInstanceDTO getStepInstanceByTaskInstanceId(long taskInstanceId) { - List stepInstanceList = - stepInstanceDAO.listStepInstanceBaseByTaskInstanceId(taskInstanceId); - if (CollectionUtils.isEmpty(stepInstanceList)) { - return null; - } - StepInstanceDTO stepInstance = new StepInstanceDTO(stepInstanceList.get(0)); - fillStepInstanceDetail(stepInstance); - return stepInstance; - } - - @Override - public Integer countTaskInstances(Long appId, Long minTotalTime, Long maxTotalTime, - TaskStartupModeEnum taskStartupMode, TaskTypeEnum taskType, - List runStatusList, Long fromTime, Long toTime) { - return taskInstanceDAO.countTaskInstances(appId, minTotalTime, maxTotalTime, taskStartupMode, taskType, - runStatusList, fromTime, toTime); - } - - @Override - public Integer countStepInstances(Long appId, List stepIdList, StepExecuteTypeEnum stepExecuteType, - ScriptTypeEnum scriptType, RunStatusEnum runStatus, Long fromTime, Long toTime) { - return stepInstanceDAO.count(appId, stepIdList, stepExecuteType, scriptType, runStatus, fromTime, toTime); - } - - @Override - public Integer countFastPushFile(Long appId, Integer transferMode, Boolean localUpload, RunStatusEnum runStatus, - Long fromTime, Long toTime) { - if (localUpload == null) { - // 不区分本地与服务器文件 - if (transferMode == null) { - // 不区分模式 - return stepInstanceDAO.countFastPushFile(appId, null, null, null, null, runStatus, fromTime, toTime); - } else if (transferMode == 1) { - // 严谨模式 - return stepInstanceDAO.countFastPushFile(appId, null, null, NotExistPathHandlerEnum.STEP_FAIL, null, - runStatus, fromTime, toTime); - } else if (transferMode == 2) { - // 强制模式 - Integer count = 0; - count += stepInstanceDAO.countFastPushFile(appId, null, null, NotExistPathHandlerEnum.CREATE_DIR, - null, runStatus, fromTime, toTime); - count += stepInstanceDAO.countFastPushFile(appId, null, null, null, true, runStatus, fromTime, toTime); - return count; - } else { - log.warn("Unexpected transferMode={}", transferMode); - return 0; - } - } else { - // 区分本地与服务器文件 - // 查出所有记录 - List> recordList = new ArrayList<>(); - if (transferMode == null) { - // 不区分模式 - recordList.addAll(stepInstanceDAO.listFastPushFileSource(appId, null, null, null, null, runStatus, - fromTime, toTime)); - } else if (transferMode == 1) { - // 严谨模式 - recordList.addAll(stepInstanceDAO.listFastPushFileSource(appId, null, null, - NotExistPathHandlerEnum.STEP_FAIL, null, runStatus, fromTime, toTime)); - } else if (transferMode == 2) { - // 强制模式 - recordList.addAll(stepInstanceDAO.listFastPushFileSource(appId, null, null, - NotExistPathHandlerEnum.CREATE_DIR, null, runStatus, fromTime, toTime)); - recordList.addAll(stepInstanceDAO.listFastPushFileSource(appId, null, null, null, true, runStatus, - fromTime, toTime)); - } else { - log.warn("Unexpected transferMode={}", transferMode); - return 0; - } - // 筛选本地文件与服务器文件 - int localCount = 0; - int serverCount = 0; - for (List fileSourceDTOList : recordList) { - Set localFlagSet = - fileSourceDTOList.parallelStream().map(FileSourceDTO::isLocalUpload).collect(Collectors.toSet()); - if (localFlagSet.size() == 2) { - localCount += 1; - serverCount += 1; - } else if (localFlagSet.size() == 1) { - if (localFlagSet.iterator().next()) { - localCount += 1; - } else { - serverCount += 1; - } - } - } - if (localUpload) { - return localCount; - } else { - return serverCount; - } - } + taskInstanceDAO.updateTaskExecutionInfo(taskInstanceId, status, currentStepId, startTime, endTime, totalTime); } @Override @@ -397,8 +218,9 @@ public List listTaskInstanceId(Long appId, Long fromTime, Long toTime, int } @Override - public void saveTaskInstanceHosts(long taskInstanceId, + public void saveTaskInstanceHosts(long appId, + long taskInstanceId, Collection hosts) { - taskInstanceDAO.saveTaskInstanceHosts(taskInstanceId, hosts); + taskInstanceDAO.saveTaskInstanceHosts(appId, taskInstanceId, hosts); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskInstanceVariableServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskInstanceVariableServiceImpl.java index 2095a5841e..b141958005 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskInstanceVariableServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskInstanceVariableServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,8 +27,9 @@ import com.tencent.bk.job.common.constant.TaskVariableTypeEnum; import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.execute.dao.TaskInstanceVariableDAO; +import com.tencent.bk.job.execute.dao.common.IdGen; import com.tencent.bk.job.execute.engine.model.TaskVariableDTO; -import com.tencent.bk.job.execute.model.ServersDTO; +import com.tencent.bk.job.execute.model.ExecuteTargetDTO; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -39,10 +40,12 @@ @Slf4j public class TaskInstanceVariableServiceImpl implements com.tencent.bk.job.execute.service.TaskInstanceVariableService { private final TaskInstanceVariableDAO taskInstanceVariableDAO; + private final IdGen idGen; @Autowired - public TaskInstanceVariableServiceImpl(TaskInstanceVariableDAO taskInstanceVariableDAO) { + public TaskInstanceVariableServiceImpl(TaskInstanceVariableDAO taskInstanceVariableDAO, IdGen idGen) { this.taskInstanceVariableDAO = taskInstanceVariableDAO; + this.idGen = idGen; } @Override @@ -52,7 +55,8 @@ public List getByTaskInstanceId(long taskInstanceId) { for (TaskVariableDTO taskVariable : taskVarList) { if (taskVariable.getType() == TaskVariableTypeEnum.HOST_LIST.getType() && taskVariable.getValue() != null) { - taskVariable.setTargetServers(JsonUtils.fromJson(taskVariable.getValue(), ServersDTO.class)); + taskVariable.setExecuteTarget( + JsonUtils.fromJson(taskVariable.getValue(), ExecuteTargetDTO.class)); } } } @@ -76,9 +80,10 @@ public void saveTaskInstanceVariables(List taskVarList) { } for (TaskVariableDTO taskVariable : taskVarList) { if (taskVariable.getType() == TaskVariableTypeEnum.HOST_LIST.getType() - && taskVariable.getTargetServers() != null) { - taskVariable.setValue(JsonUtils.toJson(taskVariable.getTargetServers())); + && taskVariable.getExecuteTarget() != null) { + taskVariable.setValue(JsonUtils.toJson(taskVariable.getExecuteTarget())); } + taskVariable.setId(idGen.genTaskInstanceVariableId()); } taskInstanceVariableDAO.saveTaskInstanceVariables(taskVarList); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskOperationLogServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskOperationLogServiceImpl.java index cf5600410d..4bff94096c 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskOperationLogServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskOperationLogServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,6 +25,7 @@ package com.tencent.bk.job.execute.service.impl; import com.tencent.bk.job.execute.dao.OperationLogDAO; +import com.tencent.bk.job.execute.dao.common.IdGen; import com.tencent.bk.job.execute.model.OperationLogDTO; import com.tencent.bk.job.execute.service.TaskOperationLogService; import org.springframework.beans.factory.annotation.Autowired; @@ -35,14 +36,17 @@ @Service public class TaskOperationLogServiceImpl implements TaskOperationLogService { private final OperationLogDAO operationLogDAO; + private final IdGen idGen; @Autowired - public TaskOperationLogServiceImpl(OperationLogDAO operationLogDAO) { + public TaskOperationLogServiceImpl(OperationLogDAO operationLogDAO, IdGen idGen) { this.operationLogDAO = operationLogDAO; + this.idGen = idGen; } @Override public long saveOperationLog(OperationLogDTO operationLog) { + operationLog.setId(idGen.genOperationLogId()); return operationLogDAO.saveOperationLog(operationLog); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskPlanServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskPlanServiceImpl.java index cb695fcd90..e691a41da1 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskPlanServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskPlanServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,27 +26,27 @@ import com.tencent.bk.job.common.model.InternalResponse; import com.tencent.bk.job.common.util.json.JsonUtils; -import com.tencent.bk.job.execute.client.TaskPlanResourceClient; import com.tencent.bk.job.execute.service.TaskPlanService; +import com.tencent.bk.job.manage.api.inner.ServiceTaskPlanResource; import com.tencent.bk.job.manage.model.inner.ServiceTaskPlanDTO; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -@Service +@Service("jobExecuteTaskPlanService") @Slf4j public class TaskPlanServiceImpl implements TaskPlanService { - private final TaskPlanResourceClient taskPlanResourceClient; + private final ServiceTaskPlanResource taskPlanResource; @Autowired - public TaskPlanServiceImpl(TaskPlanResourceClient taskPlanResourceClient) { - this.taskPlanResourceClient = taskPlanResourceClient; + public TaskPlanServiceImpl(ServiceTaskPlanResource taskPlanResource) { + this.taskPlanResource = taskPlanResource; } @Override public ServiceTaskPlanDTO getPlanById(Long appId, Long planId) { try { - InternalResponse resp = taskPlanResourceClient.getPlanById(appId, planId, false); + InternalResponse resp = taskPlanResource.getPlanById(appId, planId, false); log.info("Get plan by id, appId={}, planId={}, result={}", appId, planId, JsonUtils.toJson(resp)); return resp.getData(); } catch (Throwable e) { @@ -59,7 +59,7 @@ public ServiceTaskPlanDTO getPlanById(Long appId, Long planId) { @Override public String getPlanName(Long planId) { try { - InternalResponse resp = taskPlanResourceClient.getPlanName(planId); + InternalResponse resp = taskPlanResource.getPlanName(planId); return resp.getData(); } catch (Throwable e) { String errorMsg = "Get plan name by id caught exception, planId=" + planId; @@ -71,7 +71,7 @@ public String getPlanName(Long planId) { @Override public Long getPlanAppId(Long planId) { try { - InternalResponse resp = taskPlanResourceClient.getPlanAppId(planId); + InternalResponse resp = taskPlanResource.getPlanAppId(planId); log.info("Get planAppId by id, planId={}, result={}", planId, JsonUtils.toJson(resp)); return resp.getData(); } catch (Throwable e) { diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskResultServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskResultServiceImpl.java index b237d3acfc..bc9ca299e9 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskResultServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TaskResultServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,31 +28,27 @@ import com.tencent.bk.job.common.constant.Order; import com.tencent.bk.job.common.exception.FailedPreconditionException; import com.tencent.bk.job.common.exception.NotFoundException; -import com.tencent.bk.job.common.exception.ServiceException; import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; -import com.tencent.bk.job.common.iam.model.AuthResult; import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; -import com.tencent.bk.job.common.model.dto.AppResourceScope; import com.tencent.bk.job.common.model.dto.HostDTO; -import com.tencent.bk.job.execute.auth.ExecuteAuthService; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; import com.tencent.bk.job.execute.common.constants.StepExecuteTypeEnum; import com.tencent.bk.job.execute.common.constants.StepRunModeEnum; import com.tencent.bk.job.execute.common.converter.StepTypeExecuteTypeConverter; import com.tencent.bk.job.execute.common.util.TaskCostCalculator; import com.tencent.bk.job.execute.constants.UserOperationEnum; -import com.tencent.bk.job.execute.dao.FileSourceTaskLogDAO; import com.tencent.bk.job.execute.dao.StepInstanceDAO; import com.tencent.bk.job.execute.dao.TaskInstanceDAO; -import com.tencent.bk.job.execute.engine.consts.AgentTaskStatusEnum; -import com.tencent.bk.job.execute.model.AgentTaskDTO; -import com.tencent.bk.job.execute.model.AgentTaskDetailDTO; -import com.tencent.bk.job.execute.model.AgentTaskResultGroupBaseDTO; -import com.tencent.bk.job.execute.model.AgentTaskResultGroupDTO; +import com.tencent.bk.job.execute.engine.consts.ExecuteObjectTaskStatusEnum; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; import com.tencent.bk.job.execute.model.ConfirmStepInstanceDTO; +import com.tencent.bk.job.execute.model.ExecuteObjectCompositeKey; +import com.tencent.bk.job.execute.model.ExecuteObjectTask; import com.tencent.bk.job.execute.model.FileSourceTaskLogDTO; import com.tencent.bk.job.execute.model.OperationLogDTO; +import com.tencent.bk.job.execute.model.ResultGroupBaseDTO; +import com.tencent.bk.job.execute.model.ResultGroupDTO; import com.tencent.bk.job.execute.model.RollingConfigDTO; import com.tencent.bk.job.execute.model.StepExecutionDTO; import com.tencent.bk.job.execute.model.StepExecutionDetailDTO; @@ -66,12 +62,14 @@ import com.tencent.bk.job.execute.model.TaskInstanceQuery; import com.tencent.bk.job.execute.model.inner.CronTaskExecuteResult; import com.tencent.bk.job.execute.model.inner.ServiceCronTaskExecuteResultStatistics; -import com.tencent.bk.job.execute.service.FileAgentTaskService; -import com.tencent.bk.job.execute.service.HostService; +import com.tencent.bk.job.execute.service.FileExecuteObjectTaskService; +import com.tencent.bk.job.execute.service.FileSourceTaskLogService; import com.tencent.bk.job.execute.service.LogService; import com.tencent.bk.job.execute.service.RollingConfigService; -import com.tencent.bk.job.execute.service.ScriptAgentTaskService; +import com.tencent.bk.job.execute.service.ScriptExecuteObjectTaskService; import com.tencent.bk.job.execute.service.StepInstanceRollingTaskService; +import com.tencent.bk.job.execute.service.StepInstanceService; +import com.tencent.bk.job.execute.service.TaskInstanceAccessProcessor; import com.tencent.bk.job.execute.service.TaskInstanceService; import com.tencent.bk.job.execute.service.TaskOperationLogService; import com.tencent.bk.job.execute.service.TaskResultService; @@ -89,6 +87,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.stream.Collectors; @@ -103,49 +102,54 @@ public class TaskResultServiceImpl implements TaskResultService { private final TaskInstanceDAO taskInstanceDAO; private final StepInstanceDAO stepInstanceDAO; private final TaskInstanceService taskInstanceService; - private final FileSourceTaskLogDAO fileSourceTaskLogDAO; - private final ScriptAgentTaskService scriptAgentTaskService; - private final FileAgentTaskService fileAgentTaskService; + private final FileSourceTaskLogService fileSourceTaskLogService; + private final ScriptExecuteObjectTaskService scriptExecuteObjectTaskService; + private final FileExecuteObjectTaskService fileExecuteObjectTaskService; private final LogService logService; - private final ExecuteAuthService executeAuthService; private final TaskOperationLogService operationLogService; private final RollingConfigService rollingConfigService; private final StepInstanceRollingTaskService stepInstanceRollingTaskService; - private final HostService hostService; + private final TaskInstanceAccessProcessor taskInstanceAccessProcessor; + private final StepInstanceService stepInstanceService; @Autowired public TaskResultServiceImpl(TaskInstanceDAO taskInstanceDAO, StepInstanceDAO stepInstanceDAO, TaskInstanceService taskInstanceService, - FileSourceTaskLogDAO fileSourceTaskLogDAO, - ScriptAgentTaskService scriptAgentTaskService, - FileAgentTaskService fileAgentTaskService, + FileSourceTaskLogService fileSourceTaskLogService, + ScriptExecuteObjectTaskService scriptExecuteObjectTaskService, + FileExecuteObjectTaskService fileExecuteObjectTaskService, LogService logService, - ExecuteAuthService executeAuthService, TaskOperationLogService operationLogService, RollingConfigService rollingConfigService, StepInstanceRollingTaskService stepInstanceRollingTaskService, - HostService hostService) { + TaskInstanceAccessProcessor taskInstanceAccessProcessor, + StepInstanceService stepInstanceService) { this.taskInstanceDAO = taskInstanceDAO; this.stepInstanceDAO = stepInstanceDAO; this.taskInstanceService = taskInstanceService; - this.fileSourceTaskLogDAO = fileSourceTaskLogDAO; - this.scriptAgentTaskService = scriptAgentTaskService; - this.fileAgentTaskService = fileAgentTaskService; + this.fileSourceTaskLogService = fileSourceTaskLogService; + this.scriptExecuteObjectTaskService = scriptExecuteObjectTaskService; + this.fileExecuteObjectTaskService = fileExecuteObjectTaskService; this.logService = logService; - this.executeAuthService = executeAuthService; this.operationLogService = operationLogService; this.rollingConfigService = rollingConfigService; this.stepInstanceRollingTaskService = stepInstanceRollingTaskService; - this.hostService = hostService; + this.taskInstanceAccessProcessor = taskInstanceAccessProcessor; + this.stepInstanceService = stepInstanceService; } @Override public PageData listPageTaskInstance(TaskInstanceQuery taskQuery, BaseSearchCondition baseSearchCondition) { PageData pageData = taskInstanceDAO.listPageTaskInstance(taskQuery, baseSearchCondition); - if (pageData != null && pageData.getData() != null && !pageData.getData().isEmpty()) { - pageData.getData().forEach(taskInstanceDTO -> { + computeTotalTime(pageData.getData()); + return pageData; + } + + private void computeTotalTime(List pageData) { + if (CollectionUtils.isNotEmpty(pageData)) { + pageData.forEach(taskInstanceDTO -> { if (taskInstanceDTO.getTotalTime() == null) { RunStatusEnum status = taskInstanceDTO.getStatus(); if (status == RunStatusEnum.RUNNING || status == RunStatusEnum.WAITING_USER @@ -156,23 +160,13 @@ public PageData listPageTaskInstance(TaskInstanceQuery taskQuer } }); } - return pageData; } @Override - public TaskExecuteResultDTO getTaskExecutionResult(String username, Long appId, - Long taskInstanceId) throws ServiceException { - TaskInstanceDTO taskInstance = taskInstanceDAO.getTaskInstance(taskInstanceId); - if (taskInstance == null) { - log.warn("Task instance is not exist, taskInstanceId={}", taskInstanceId); - throw new NotFoundException(ErrorCode.TASK_INSTANCE_NOT_EXIST); - } - if (!taskInstance.getAppId().equals(appId)) { - log.warn("Task instance is not in application, taskInstanceId={}, appId={}", taskInstanceId, appId); - throw new NotFoundException(ErrorCode.TASK_INSTANCE_NOT_EXIST); - } - - authViewTaskInstance(username, appId, taskInstance); + public TaskExecuteResultDTO getTaskExecutionResult(String username, + Long appId, + Long taskInstanceId) { + TaskInstanceDTO taskInstance = taskInstanceService.getTaskInstance(username, appId, taskInstanceId); TaskExecutionDTO taskExecution = buildTaskExecutionDTO(taskInstance); @@ -201,7 +195,7 @@ private TaskExecutionDTO buildTaskExecutionDTO(TaskInstanceDTO taskInstance) { taskInstance.getEndTime(), taskInstance.getTotalTime())); taskExecution.setStartTime(taskInstance.getStartTime()); taskExecution.setEndTime(taskInstance.getEndTime()); - taskExecution.setTaskId(taskInstance.getTaskId()); + taskExecution.setTaskId(taskInstance.getPlanId()); taskExecution.setTaskTemplateId(taskInstance.getTaskTemplateId()); taskExecution.setDebugTask(taskInstance.isDebugTask()); taskExecution.setCurrentStepInstanceId(taskInstance.getCurrentStepInstanceId()); @@ -214,15 +208,16 @@ private StepExecutionDTO buildStepExecutionDTO(StepInstanceBaseDTO stepInstance) stepExecution.setName(stepInstance.getName()); stepExecution.setExecuteCount(stepInstance.getExecuteCount()); stepExecution.setStatus(stepInstance.getStatus().getValue()); - stepExecution.setType(StepTypeExecuteTypeConverter.convertToStepType(stepInstance.getExecuteType())); + stepExecution.setType(StepTypeExecuteTypeConverter.convertToStepType(stepInstance.getExecuteType()).getValue()); stepExecution.setStartTime(stepInstance.getStartTime()); stepExecution.setEndTime(stepInstance.getEndTime()); stepExecution.setOperator(stepInstance.getOperator()); stepExecution.setTotalTime(TaskCostCalculator.calculate(stepInstance.getStartTime(), stepInstance.getEndTime(), stepInstance.getTotalTime())); stepExecution.setLastStep(stepInstance.isLastStep()); - if (stepInstance.getExecuteType().equals(StepExecuteTypeEnum.MANUAL_CONFIRM.getValue())) { - ConfirmStepInstanceDTO confirmStepInstance = stepInstanceDAO.getConfirmStepInstance(stepInstance.getId()); + if (stepInstance.getExecuteType() == StepExecuteTypeEnum.MANUAL_CONFIRM) { + ConfirmStepInstanceDTO confirmStepInstance = stepInstanceDAO.getConfirmStepInstance( + stepInstance.getTaskInstanceId(), stepInstance.getId()); if (confirmStepInstance != null) { stepExecution.setConfirmMessage(confirmStepInstance.getConfirmMessage()); stepExecution.setConfirmReason(confirmStepInstance.getConfirmReason()); @@ -234,29 +229,6 @@ private StepExecutionDTO buildStepExecutionDTO(StepInstanceBaseDTO stepInstance) return stepExecution; } - private void authViewTaskInstance(String username, Long appId, TaskInstanceDTO taskInstance) { - - AuthResult authResult = executeAuthService.authViewTaskInstance( - username, new AppResourceScope(appId), taskInstance); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } - } - - private void authViewStepInstance(String username, Long appId, StepInstanceBaseDTO stepInstance) { - String operator = stepInstance.getOperator(); - if (username.equals(operator)) { - return; - } - - AuthResult authResult = executeAuthService.authViewTaskInstance( - username, new AppResourceScope(appId), - stepInstance.getTaskInstanceId()); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } - } - /** * 加入文件源文件拉取所使用的时间 * @@ -265,45 +237,29 @@ username, new AppResourceScope(appId), */ private void involveFileSourceTaskLog(StepExecutionDetailDTO stepExecutionDetail, FileSourceTaskLogDTO fileSourceTaskLog) { - List resultGroups = stepExecutionDetail.getResultGroups(); - for (AgentTaskResultGroupDTO resultGroup : resultGroups) { + List resultGroups = stepExecutionDetail.getResultGroups(); + for (ResultGroupDTO resultGroup : resultGroups) { if (resultGroup == null) { continue; } - List agentTaskExecutionDetailList = resultGroup.getAgentTasks(); - if (agentTaskExecutionDetailList == null) { + List executeObjectTasks = resultGroup.getExecuteObjectTasks(); + if (executeObjectTasks == null) { continue; } - for (AgentTaskDTO agentTaskDetail : agentTaskExecutionDetailList) { - if (agentTaskDetail == null) { + for (ExecuteObjectTask executeObjectTask : executeObjectTasks) { + if (executeObjectTask == null) { continue; } - agentTaskDetail.setStartTime(fileSourceTaskLog.getStartTime()); - if (agentTaskDetail.getEndTime() == null || agentTaskDetail.getEndTime() == 0) { - agentTaskDetail.setEndTime(stepExecutionDetail.getEndTime()); + executeObjectTask.setStartTime(fileSourceTaskLog.getStartTime()); + if (executeObjectTask.getEndTime() == null || executeObjectTask.getEndTime() == 0) { + executeObjectTask.setEndTime(stepExecutionDetail.getEndTime()); } - agentTaskDetail.calculateTotalTime(); + executeObjectTask.calculateTotalTime(); } } } - private StepInstanceBaseDTO checkGetStepExecutionDetail(String username, long appId, long stepInstanceId) { - StepInstanceBaseDTO stepInstance = taskInstanceService.getBaseStepInstance(stepInstanceId); - if (stepInstance == null) { - log.warn("Step instance is not exist, stepInstanceId={}", stepInstanceId); - throw new NotFoundException(ErrorCode.STEP_INSTANCE_NOT_EXIST); - } - - authViewStepInstance(username, appId, stepInstance); - - if (stepInstance.getExecuteType().equals(StepExecuteTypeEnum.MANUAL_CONFIRM.getValue())) { - log.warn("Manual confirm step does not support get-step-detail operation"); - throw new FailedPreconditionException(ErrorCode.UNSUPPORTED_OPERATION); - } - return stepInstance; - } - - private boolean isMatchResultGroup(AgentTaskResultGroupDTO resultGroup, Integer resultType, String tag) { + private boolean isMatchResultGroup(ResultGroupDTO resultGroup, Integer resultType, String tag) { String matchTag = tag == null ? "" : tag; return resultType.equals(resultGroup.getStatus()) && matchTag.equals(resultGroup.getTag()); } @@ -320,45 +276,39 @@ private List getLimitedSizedList(List list, Integer maxSize) { } } - @Override - public StepExecutionDetailDTO getFastTaskStepExecutionResult(String username, Long appId, Long taskInstanceId, - StepExecutionResultQuery query) { - List stepInstanceList = - stepInstanceDAO.listStepInstanceBaseByTaskInstanceId(taskInstanceId); - StepInstanceBaseDTO stepInstance = stepInstanceList.get(0); - if (stepInstance == null) { - log.warn("Step instance is not exist, taskInstanceId={}", taskInstanceId); - throw new NotFoundException(ErrorCode.STEP_INSTANCE_NOT_EXIST); - } - if (!stepInstance.getAppId().equals(appId)) { - log.warn("Step instance is not in application, stepInstanceId={}, appId={}", stepInstance.getId(), appId); - throw new NotFoundException(ErrorCode.STEP_INSTANCE_NOT_EXIST); - } + private void preProcessViewStepExecutionResult(String username, Long appId, StepInstanceBaseDTO stepInstance) + throws NotFoundException, PermissionDeniedException { + // 查看步骤执行结果预处理，包括鉴权、审计等 + taskInstanceAccessProcessor.processBeforeAccess(username, appId, stepInstance.getTaskInstanceId()); - return getStepExecutionResult(username, appId, query); + if (stepInstance.getExecuteType() == StepExecuteTypeEnum.MANUAL_CONFIRM) { + log.warn("Manual confirm step does not support get-step-detail operation"); + throw new FailedPreconditionException(ErrorCode.UNSUPPORTED_OPERATION); + } } - private void setAgentTasksForSpecifiedResultType(List resultGroups, - Integer status, - String tag, - List agentTasksForResultType) { - for (AgentTaskResultGroupDTO resultGroup : resultGroups) { + private void setExecuteObjectTasksForSpecifiedResultType(List resultGroups, + Integer status, + String tag, + List executeObjectTasksForResultType) { + for (ResultGroupDTO resultGroup : resultGroups) { if (status.equals(resultGroup.getStatus()) && ( (StringUtils.isEmpty(tag) ? StringUtils.isEmpty(resultGroup.getTag()) : tag.equals(resultGroup.getTag())))) { - resultGroup.setAgentTasks(agentTasksForResultType); + resultGroup.setExecuteObjectTasks(executeObjectTasksForResultType); } } } - private void setAgentTasksForAnyResultType(List resultGroups, + private void setAgentTasksForAnyResultType(List resultGroups, StepInstanceBaseDTO stepInstance, int executeCount, Integer batch, - Integer maxAgentTasksForResultGroup) { + Integer maxAgentTasksForResultGroup, + boolean fetchAllGroupData) { boolean isAgentTaskSet = false; - for (AgentTaskResultGroupDTO resultGroup : resultGroups) { - if (!isAgentTaskSet) { + for (ResultGroupDTO resultGroup : resultGroups) { + if (!isAgentTaskSet || fetchAllGroupData) { isAgentTaskSet = fillAgentTasksForResultGroup(resultGroup, stepInstance, executeCount, batch, resultGroup.getStatus(), resultGroup.getTag(), maxAgentTasksForResultGroup); } else { @@ -367,19 +317,19 @@ private void setAgentTasksForAnyResultType(List resultG } } - private boolean fillAgentTasksForResultGroup(AgentTaskResultGroupDTO resultGroup, + private boolean fillAgentTasksForResultGroup(ResultGroupDTO resultGroup, StepInstanceBaseDTO stepInstance, int executeCount, Integer batch, Integer status, String tag, Integer maxAgentTasksForResultGroup) { - List agentTasks = listAgentTaskByResultGroup(stepInstance, + List executeObjectTasks = listExecuteObjectTaskByResultGroup(stepInstance, executeCount, batch, status, tag, maxAgentTasksForResultGroup, null, null); - if (CollectionUtils.isEmpty(agentTasks)) { + if (CollectionUtils.isEmpty(executeObjectTasks)) { return false; } - resultGroup.setAgentTasks(agentTasks); + resultGroup.setExecuteObjectTasks(executeObjectTasks); return true; } @@ -402,48 +352,45 @@ private StepExecutionDetailDTO buildNotStartStepExecutionResult(StepInstanceBase StepExecutionDetailDTO stepExecuteDetail = new StepExecutionDetailDTO(stepInstance); stepExecuteDetail.setExecuteCount(queryExecuteCount); - List resultGroups = new ArrayList<>(); - AgentTaskResultGroupDTO resultGroup = new AgentTaskResultGroupDTO(); - resultGroup.setStatus(AgentTaskStatusEnum.WAITING.getValue()); + List resultGroups = new ArrayList<>(); + ResultGroupDTO resultGroup = new ResultGroupDTO(); + resultGroup.setStatus(ExecuteObjectTaskStatusEnum.WAITING.getValue()); resultGroup.setTag(null); - List targetHosts = filterTargetHostsByBatch(stepInstance, batch); + List targetExecuteObjects = filterTargetExecuteObjectsByBatch(stepInstance, batch); - List agentTasks = new ArrayList<>(); - // 如果需要根据IP过滤，那么需要重新计算Agent任务总数 + List executeObjectTasks = new ArrayList<>(); + // 如果需要根据IP过滤，那么需要重新计算执行对象任务总数 boolean fuzzyFilterByIp = StringUtils.isNotEmpty(fuzzySearchIp); - int agentTaskSize = targetHosts.size(); + int executeObjectTaskSize = targetExecuteObjects.size(); if (fuzzyFilterByIp) { - agentTaskSize = (int) targetHosts.stream() - .filter(ipDTO -> ipDTO.getIp().contains(fuzzySearchIp)).count(); + executeObjectTaskSize = (int) targetExecuteObjects.stream() + .filter(executeObject -> isMatchByIp(executeObject, fuzzySearchIp)).count(); } - resultGroup.setTotalAgentTasks(agentTaskSize); - - if (CollectionUtils.isNotEmpty(targetHosts)) { - int maxAgentTasks = (maxAgentTasksForResultGroup != null ? - Math.min(maxAgentTasksForResultGroup, targetHosts.size()) : targetHosts.size()); - for (HostDTO targetHost : targetHosts) { - String ip = targetHost.getIp(); - if (fuzzyFilterByIp && !ip.contains(fuzzySearchIp)) { + resultGroup.setTotal(executeObjectTaskSize); + + if (CollectionUtils.isNotEmpty(targetExecuteObjects)) { + int maxExecuteObjectTasks = (maxAgentTasksForResultGroup != null ? + Math.min(maxAgentTasksForResultGroup, targetExecuteObjects.size()) : targetExecuteObjects.size()); + for (ExecuteObject targetExecuteObject : targetExecuteObjects) { + if (fuzzyFilterByIp && !isMatchByIp(targetExecuteObject, fuzzySearchIp)) { // 如果需要根据IP过滤，那么过滤掉不匹配的任务 continue; } - if (maxAgentTasks-- > 0) { - AgentTaskDetailDTO agentTask = new AgentTaskDetailDTO(); - agentTask.setHostId(targetHost.getHostId()); - agentTask.setIp(targetHost.getIp()); - agentTask.setBkCloudId(targetHost.getBkCloudId()); - agentTask.setBkCloudName(hostService.getCloudAreaName(targetHost.getBkCloudId())); - agentTask.setStatus(AgentTaskStatusEnum.WAITING); - agentTask.setTag(null); - agentTask.setErrorCode(0); - agentTask.setExitCode(0); - agentTask.setTotalTime(0L); - agentTasks.add(agentTask); + if (maxExecuteObjectTasks-- > 0) { + ExecuteObjectTask executeObjectTask = new ExecuteObjectTask(); + executeObjectTask.setExecuteObject(targetExecuteObject); + executeObjectTask.setExecuteObjectId(targetExecuteObject.getId()); + executeObjectTask.setStatus(ExecuteObjectTaskStatusEnum.WAITING); + executeObjectTask.setTag(null); + executeObjectTask.setErrorCode(0); + executeObjectTask.setExitCode(0); + executeObjectTask.setTotalTime(0L); + executeObjectTasks.add(executeObjectTask); } } } - resultGroup.setAgentTasks(agentTasks); + resultGroup.setExecuteObjectTasks(executeObjectTasks); resultGroups.add(resultGroup); stepExecuteDetail.setResultGroups(resultGroups); @@ -456,26 +403,30 @@ private StepExecutionDetailDTO buildNotStartStepExecutionResult(StepInstanceBase return stepExecuteDetail; } - private List filterTargetHostsByBatch(StepInstanceBaseDTO stepInstance, Integer batch) { - List targetServers; + private List filterTargetExecuteObjectsByBatch(StepInstanceBaseDTO stepInstance, Integer batch) { + List executeObjects; if (stepInstance.isRollingStep()) { - targetServers = rollingConfigService.getRollingServers(stepInstance, batch); + executeObjects = rollingConfigService.getRollingServers(stepInstance, batch); } else { - targetServers = stepInstance.getTargetServers().getIpList(); + executeObjects = stepInstance.getTargetExecuteObjects().getExecuteObjectsCompatibly(); } - return targetServers; + return executeObjects; } @Override - public StepExecutionDetailDTO getStepExecutionResult(String username, Long appId, - StepExecutionResultQuery query) throws ServiceException { + public StepExecutionDetailDTO getStepExecutionResult(String username, + Long appId, + StepExecutionResultQuery query) { StopWatch watch = new StopWatch("getStepExecutionResult"); try { Long stepInstanceId = query.getStepInstanceId(); - watch.start("checkGetStepExecutionDetail"); - StepInstanceBaseDTO stepInstance = checkGetStepExecutionDetail(username, appId, stepInstanceId); + watch.start("getAndCheckStepInstance"); + StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstance( + appId, query.getTaskInstanceId(), stepInstanceId); + preProcessViewStepExecutionResult(username, appId, stepInstance); + int queryExecuteCount = query.getExecuteCount() == null ? stepInstance.getExecuteCount() : query.getExecuteCount(); query.setExecuteCount(queryExecuteCount); @@ -487,12 +438,12 @@ public StepExecutionDetailDTO getStepExecutionResult(String username, Long appId if (stepInstance.getStatus() == RunStatusEnum.BLANK) { // 步骤未启动，AgentTask数据还未在DB初始化，构造初始任务结果 return buildNotStartStepExecutionResult(stepInstance, queryExecuteCount, query.getBatch(), - query.getMaxAgentTasksForResultGroup(), query.getSearchIp()); + query.getMaxTasksForResultGroup(), query.getSearchIp()); } StepExecutionDetailDTO stepExecutionDetail; - // 如果步骤的目标服务器数量<100,或者通过IP匹配的方式过滤agent任务，为了提升性能，直接全量从DB查询数据，在内存进行处理 - if ((stepInstance.getTargetServerTotalCount() <= 100) || query.hasIpCondition()) { + // 如果步骤的目标服务器数量<100,或者通过IP匹配的方式过滤执行对象任务，为了提升性能，直接全量从DB查询数据，在内存进行处理 + if ((stepInstance.getTargetExecuteObjectCount() <= 100) || query.hasExecuteObjectFilterCondition()) { stepExecutionDetail = loadAllTasksFromDBAndBuildExecutionResultInMemory(watch, stepInstance, query); } else { stepExecutionDetail = filterAndSortExecutionResultInDB(watch, stepInstance, query); @@ -508,8 +459,12 @@ public StepExecutionDetailDTO getStepExecutionResult(String username, Long appId if (stepInstance.isFileStep()) { watch.start("involveFileSourceTaskLog"); - FileSourceTaskLogDTO fileSourceTaskLog = fileSourceTaskLogDAO.getFileSourceTaskLog(stepInstance.getId(), - queryExecuteCount); + FileSourceTaskLogDTO fileSourceTaskLog = + fileSourceTaskLogService.getFileSourceTaskLog( + stepInstance.getTaskInstanceId(), + stepInstance.getId(), + queryExecuteCount + ); if (fileSourceTaskLog != null) { involveFileSourceTaskLog(stepExecutionDetail, fileSourceTaskLog); } @@ -532,36 +487,40 @@ private StepExecutionDetailDTO loadAllTasksFromDBAndBuildExecutionResultInMemory StepInstanceBaseDTO stepInstance, StepExecutionResultQuery query) { try { - if (query.hasIpCondition()) { - watch.start("getMatchIps"); - Set matchHostIds = getMatchHostIdsByIp(stepInstance, query); - if (CollectionUtils.isEmpty(matchHostIds)) { + StepExecutionDetailDTO executeDetail = new StepExecutionDetailDTO(stepInstance); + executeDetail.setExecuteCount(query.getExecuteCount()); + + if (query.hasExecuteObjectFilterCondition()) { + watch.start("getMatchExecuteObjectCompositeKeys"); + Set matchKeys = getMatchExecuteObjectCompositeKeys(stepInstance, query); + if (CollectionUtils.isEmpty(matchKeys)) { watch.stop(); - return buildExecutionDetailWhenTaskAreEmpty(stepInstance, query.getBatch()); + executeDetail.setResultGroups(buildEmptyResultGroups(stepInstance, query.getBatch())); + return executeDetail; } else { - query.setMatchHostIds(matchHostIds); + query.setMatchExecuteObjectCompositeKeys(matchKeys); } watch.stop(); } watch.start("loadAllTasksFromDbAndGroup"); - List resultGroups = listAndGroupAgentTasks(stepInstance, + List resultGroups = listAndGroupExecuteObjectTasks(stepInstance, query.getExecuteCount(), query.getBatch()); - if (CollectionUtils.isNotEmpty(query.getMatchHostIds())) { - filterAgentTasksByMatchIp(resultGroups, query.getMatchHostIds()); + if (CollectionUtils.isNotEmpty(query.getMatchExecuteObjectCompositeKeys())) { + filterExecuteObjectTasksByMatchKeys(resultGroups, query.getMatchExecuteObjectCompositeKeys()); } - removeAgentTasksForNotSpecifiedResultGroup(resultGroups, query.getStatus(), query.getTag()); + if (!query.isFetchAllGroupData()) { + removeAgentTasksForNotSpecifiedResultGroup(resultGroups, query.getStatus(), query.getTag()); + } watch.stop(); watch.start("sortAndLimitTasks"); - sortAgentTasksAndLimitSize(resultGroups, query); + sortExecuteObjectTasksAndLimitSize(resultGroups, query); watch.stop(); - StepExecutionDetailDTO executeDetail = new StepExecutionDetailDTO(stepInstance); - executeDetail.setExecuteCount(query.getExecuteCount()); executeDetail.setResultGroups(resultGroups); return executeDetail; @@ -573,35 +532,47 @@ private StepExecutionDetailDTO loadAllTasksFromDBAndBuildExecutionResultInMemory } - private List listAndGroupAgentTasks(StepInstanceBaseDTO stepInstance, - int executeCount, - Integer batch) { - List resultGroups = null; + private List listAndGroupExecuteObjectTasks(StepInstanceBaseDTO stepInstance, + int executeCount, + Integer batch) { + List resultGroups = null; if (stepInstance.isScriptStep()) { - resultGroups = scriptAgentTaskService.listAndGroupAgentTasks(stepInstance, executeCount, batch); + resultGroups = scriptExecuteObjectTaskService.listAndGroupTasks(stepInstance, executeCount, batch); } else if (stepInstance.isFileStep()) { - resultGroups = fileAgentTaskService.listAndGroupAgentTasks(stepInstance, executeCount, batch); + resultGroups = fileExecuteObjectTaskService.listAndGroupTasks(stepInstance, executeCount, batch); } return resultGroups; + } + private List listResultGroups(StepInstanceBaseDTO stepInstance, + int executeCount, + Integer batch) { + List resultGroups = null; + if (stepInstance.isScriptStep()) { + resultGroups = scriptExecuteObjectTaskService.listResultGroups(stepInstance, executeCount, batch); + } else if (stepInstance.isFileStep()) { + resultGroups = fileExecuteObjectTaskService.listResultGroups(stepInstance, executeCount, batch); + } + return resultGroups; } - private void sortAgentTasksAndLimitSize(List resultGroups, - StepExecutionResultQuery query) { + + private void sortExecuteObjectTasksAndLimitSize(List resultGroups, + StepExecutionResultQuery query) { resultGroups.stream() - .filter(resultGroup -> CollectionUtils.isNotEmpty(resultGroup.getAgentTasks())) + .filter(resultGroup -> CollectionUtils.isNotEmpty(resultGroup.getExecuteObjectTasks())) .forEach(resultGroup -> { // 排序 if (StringUtils.isNotEmpty(query.getOrderField())) { - List agentTasks = resultGroup.getAgentTasks(); + List executeObjectTasks = resultGroup.getExecuteObjectTasks(); if (StepExecutionResultQuery.ORDER_FIELD_TOTAL_TIME.equals(query.getOrderField())) { - agentTasks.sort(Comparator.comparingLong(task -> task.getTotalTime() == null ? 0L : + executeObjectTasks.sort(Comparator.comparingLong(task -> task.getTotalTime() == null ? 0L : task.getTotalTime())); if (query.getOrder() == DESCENDING) { - Collections.reverse(agentTasks); + Collections.reverse(executeObjectTasks); } } else if (StepExecutionResultQuery.ORDER_FIELD_EXIT_CODE.equals(query.getOrderField())) { - agentTasks.sort((o1, o2) -> { + executeObjectTasks.sort((o1, o2) -> { if (o1.getExitCode() != null && o2.getExitCode() != null) { if (o1.getExitCode().equals(o2.getExitCode())) { return 0; @@ -617,88 +588,98 @@ private void sortAgentTasksAndLimitSize(List resultGrou } }); if (query.getOrder() == DESCENDING) { - Collections.reverse(agentTasks); + Collections.reverse(executeObjectTasks); } } } // 截断 - if (query.getMaxAgentTasksForResultGroup() != null) { - resultGroup.setAgentTasks( - getLimitedSizedList(resultGroup.getAgentTasks(), query.getMaxAgentTasksForResultGroup())); + if (query.getMaxTasksForResultGroup() != null) { + resultGroup.setExecuteObjectTasks( + getLimitedSizedList(resultGroup.getExecuteObjectTasks(), + query.getMaxTasksForResultGroup())); } }); } - private Set getMatchHostIdsByIp(StepInstanceBaseDTO stepInstance, StepExecutionResultQuery query) { - long stepInstanceId = query.getStepInstanceId(); - int executeCount = query.getExecuteCount(); - - Set matchHostIdsByLogKeywordSearch = null; + private Set getMatchExecuteObjectCompositeKeys(StepInstanceBaseDTO stepInstance, + StepExecutionResultQuery query) { + List matchKeysByLogKeywordSearch = null; if (StringUtils.isNotBlank(query.getLogKeyword())) { - List matchHosts = getHostsByLogContentKeyword(stepInstanceId, executeCount, - query.getBatch(), query.getLogKeyword()); - if (CollectionUtils.isNotEmpty(matchHosts)) { - matchHostIdsByLogKeywordSearch = - matchHosts.stream().map(HostDTO::getHostId).collect(Collectors.toSet()); - } + matchKeysByLogKeywordSearch = getExecuteObjectCompositeKeysByLogContentKeyword(stepInstance, + query.getExecuteCount(), query.getBatch(), query.getLogKeyword()); } - Set matchHostIdsByIpSearch = null; + List matchKeysByIpSearch = null; if (StringUtils.isNotBlank(query.getSearchIp())) { - List matchHosts = fuzzySearchHostsByIp(stepInstance, query.getSearchIp()); - if (CollectionUtils.isNotEmpty(matchHosts)) { - matchHostIdsByIpSearch = matchHosts.stream().map(HostDTO::getHostId).collect(Collectors.toSet()); - } + matchKeysByIpSearch = fuzzySearchHostsByIp(stepInstance, query.getSearchIp()); } - if (matchHostIdsByLogKeywordSearch != null && matchHostIdsByIpSearch != null) { - return new HashSet<>(CollectionUtils.intersection(matchHostIdsByLogKeywordSearch, matchHostIdsByIpSearch)); - } else if (matchHostIdsByLogKeywordSearch != null) { - return matchHostIdsByLogKeywordSearch; - } else if (matchHostIdsByIpSearch != null) { - return matchHostIdsByIpSearch; + if (matchKeysByLogKeywordSearch != null && matchKeysByIpSearch != null) { + return new HashSet<>(CollectionUtils.intersection(matchKeysByLogKeywordSearch, matchKeysByIpSearch)); + } else if (matchKeysByLogKeywordSearch != null) { + return new HashSet<>(matchKeysByLogKeywordSearch); + } else if (matchKeysByIpSearch != null) { + return new HashSet<>(matchKeysByIpSearch); } else { return Collections.emptySet(); } } - private StepExecutionDetailDTO buildExecutionDetailWhenTaskAreEmpty(StepInstanceBaseDTO stepInstance, - Integer batch) { - List resultGroups = listAndGroupAgentTasks(stepInstance, + private List buildEmptyResultGroups(StepInstanceBaseDTO stepInstance, + Integer batch) { + List resultGroups = listResultGroups(stepInstance, stepInstance.getExecuteCount(), batch); - StepExecutionDetailDTO executeDetail = new StepExecutionDetailDTO(stepInstance); - executeDetail.setResultGroups(resultGroups); - return executeDetail; + return resultGroups.stream().map(resultGroupBase -> { + ResultGroupDTO resultGroup = new ResultGroupDTO(resultGroupBase); + resultGroup.setTotal(0); + return resultGroup; + }).collect(Collectors.toList()); } - private void filterAgentTasksByMatchIp(List resultGroups, Set matchHostIds) { - for (AgentTaskResultGroupDTO resultGroup : resultGroups) { - List agentTasks = resultGroup.getAgentTasks(); - if (CollectionUtils.isNotEmpty(agentTasks)) { - agentTasks = agentTasks.stream() - .filter(agentTask -> matchHostIds.contains(agentTask.getHostId())) + private void filterExecuteObjectTasksByMatchKeys(List resultGroups, + Set matchExecuteObjectCompositeKeys) { + ExecuteObjectCompositeKey.CompositeKeyType compositeKeyType = + getCompositeKeyType(matchExecuteObjectCompositeKeys); + for (ResultGroupDTO resultGroup : resultGroups) { + List executeObjectTasks = resultGroup.getExecuteObjectTasks(); + if (CollectionUtils.isNotEmpty(executeObjectTasks)) { + executeObjectTasks = executeObjectTasks.stream() + .filter(executeObjectTask -> isExecuteObjectMatch(executeObjectTask.getExecuteObject(), + matchExecuteObjectCompositeKeys, compositeKeyType)) .collect(Collectors.toList()); - resultGroup.setAgentTasks(agentTasks); - resultGroup.setTotalAgentTasks(agentTasks.size()); + resultGroup.setExecuteObjectTasks(executeObjectTasks); + resultGroup.setTotal(executeObjectTasks.size()); } } } - private void removeAgentTasksForNotSpecifiedResultGroup(List resultGroups, Integer status, + private ExecuteObjectCompositeKey.CompositeKeyType getCompositeKeyType( + Set matchExecuteObjectCompositeKeys) { + return Objects.requireNonNull(matchExecuteObjectCompositeKeys.stream().findFirst().orElse(null)) + .getCompositeKeyType(); + } + + private boolean isExecuteObjectMatch(ExecuteObject executeObject, + Set matchExecuteObjectCompositeKeys, + ExecuteObjectCompositeKey.CompositeKeyType compositeKeyType) { + return matchExecuteObjectCompositeKeys.contains(executeObject.toExecuteObjectCompositeKey(compositeKeyType)); + } + + private void removeAgentTasksForNotSpecifiedResultGroup(List resultGroups, Integer status, String tag) { - if (status != null && resultGroups.contains(new AgentTaskResultGroupDTO(status, tag))) { + if (status != null && resultGroups.contains(new ResultGroupDTO(status, tag))) { resultGroups.forEach(resultGroup -> { if (!isMatchResultGroup(resultGroup, status, tag)) { - resultGroup.setAgentTasks(Collections.emptyList()); + resultGroup.setExecuteObjectTasks(Collections.emptyList()); } }); } else { int i = 0; - for (AgentTaskResultGroupDTO resultGroup : resultGroups) { + for (ResultGroupDTO resultGroup : resultGroups) { i++; if (i != 1) { - resultGroup.setAgentTasks(Collections.emptyList()); + resultGroup.setExecuteObjectTasks(Collections.emptyList()); } } } @@ -716,27 +697,27 @@ private StepExecutionDetailDTO filterAndSortExecutionResultInDB(StopWatch watch, executeDetail.setExecuteCount(queryExecuteCount); watch.start("getBaseResultGroups"); - List baseResultGroups = listBaseResultGroups(stepInstance, + List baseResultGroups = listBaseResultGroups(stepInstance, queryExecuteCount, query.getBatch()); watch.stop(); - watch.start("setAgentTasks"); - List resultGroups = baseResultGroups.stream() - .map(AgentTaskResultGroupDTO::new) + watch.start("setExecuteObjectTasks"); + List resultGroups = baseResultGroups.stream() + .map(ResultGroupDTO::new) .collect(Collectors.toList()); if (status != null) { - List tasks = listAgentTaskByResultGroup(stepInstance, queryExecuteCount, - query.getBatch(), status, tag, query.getMaxAgentTasksForResultGroup(), query.getOrderField(), + List tasks = listExecuteObjectTaskByResultGroup(stepInstance, queryExecuteCount, + query.getBatch(), status, tag, query.getMaxTasksForResultGroup(), query.getOrderField(), query.getOrder()); if (CollectionUtils.isNotEmpty(tasks)) { - setAgentTasksForSpecifiedResultType(resultGroups, status, tag, tasks); + setExecuteObjectTasksForSpecifiedResultType(resultGroups, status, tag, tasks); } else { setAgentTasksForAnyResultType(resultGroups, stepInstance, queryExecuteCount, - query.getBatch(), query.getMaxAgentTasksForResultGroup()); + query.getBatch(), query.getMaxTasksForResultGroup(), query.isFetchAllGroupData()); } } else { setAgentTasksForAnyResultType(resultGroups, stepInstance, queryExecuteCount, - query.getBatch(), query.getMaxAgentTasksForResultGroup()); + query.getBatch(), query.getMaxTasksForResultGroup(), query.isFetchAllGroupData()); } watch.stop(); @@ -745,48 +726,48 @@ private StepExecutionDetailDTO filterAndSortExecutionResultInDB(StopWatch watch, return executeDetail; } - private List listBaseResultGroups(StepInstanceBaseDTO stepInstance, - int executeCount, - Integer batch) { - List resultGroups = null; + private List listBaseResultGroups(StepInstanceBaseDTO stepInstance, + int executeCount, + Integer batch) { + List resultGroups = null; if (stepInstance.isScriptStep()) { - resultGroups = scriptAgentTaskService.listResultGroups(stepInstance.getId(), executeCount, batch); + resultGroups = scriptExecuteObjectTaskService.listResultGroups(stepInstance, executeCount, batch); } else if (stepInstance.isFileStep()) { - resultGroups = fileAgentTaskService.listResultGroups(stepInstance.getId(), executeCount, batch); + resultGroups = fileExecuteObjectTaskService.listResultGroups(stepInstance, executeCount, batch); } return resultGroups; } - private List listAgentTaskByResultGroup(StepInstanceBaseDTO stepInstance, - int queryExecuteCount, - Integer batch, - Integer status, - String tag, - Integer maxAgentTasksForResultGroup, - String orderField, - Order order) { - List tasks = null; + private List listExecuteObjectTaskByResultGroup(StepInstanceBaseDTO stepInstance, + int queryExecuteCount, + Integer batch, + Integer status, + String tag, + Integer maxAgentTasksForResultGroup, + String orderField, + Order order) { + List tasks = null; if (stepInstance.isScriptStep()) { - tasks = scriptAgentTaskService.listAgentTaskDetailByResultGroup(stepInstance, queryExecuteCount, + tasks = scriptExecuteObjectTaskService.listTaskByResultGroup(stepInstance, queryExecuteCount, batch, status, tag, maxAgentTasksForResultGroup, orderField, order); } else if (stepInstance.isFileStep()) { - tasks = fileAgentTaskService.listAgentTaskDetailByResultGroup(stepInstance, queryExecuteCount, + tasks = fileExecuteObjectTaskService.listTaskByResultGroup(stepInstance, queryExecuteCount, batch, status, tag, maxAgentTasksForResultGroup, orderField, order); } return tasks; } - private List listAgentTaskByResultGroup(StepInstanceBaseDTO stepInstance, - int queryExecuteCount, - Integer batch, - Integer status, - String tag) { - List tasks = null; + private List listExecuteObjectTaskByResultGroup(StepInstanceBaseDTO stepInstance, + int queryExecuteCount, + Integer batch, + Integer status, + String tag) { + List tasks = null; if (stepInstance.isScriptStep()) { - tasks = scriptAgentTaskService.listAgentTaskDetailByResultGroup(stepInstance, queryExecuteCount, + tasks = scriptExecuteObjectTaskService.listTaskByResultGroup(stepInstance, queryExecuteCount, batch, status, tag); } else if (stepInstance.isFileStep()) { - tasks = fileAgentTaskService.listAgentTaskDetailByResultGroup(stepInstance, queryExecuteCount, + tasks = fileExecuteObjectTaskService.listTaskByResultGroup(stepInstance, queryExecuteCount, batch, status, tag); } return tasks; @@ -795,15 +776,16 @@ private List listAgentTaskByResultGroup(StepInstanceBaseDTO private void setRollingInfoForStep(StepInstanceBaseDTO stepInstance, StepExecutionDetailDTO stepExecutionDetail) { RollingConfigDTO rollingConfig = - rollingConfigService.getRollingConfig(stepInstance.getRollingConfigId()); + rollingConfigService.getRollingConfig(stepInstance.getTaskInstanceId(), stepInstance.getRollingConfigId()); Map latestStepInstanceRollingTasks = stepInstanceRollingTaskService.listLatestRollingTasks( + stepInstance.getTaskInstanceId(), stepExecutionDetail.getStepInstanceId(), stepExecutionDetail.getExecuteCount()) .stream() .collect(Collectors.toMap(StepInstanceRollingTaskDTO::getBatch, - stepInstanceRollingTask -> stepInstanceRollingTask)); + stepInstanceRollingTask -> stepInstanceRollingTask, (oldValue, newValue) -> newValue)); // 如果滚动任务还未调度，那么需要在结果中补充 int totalBatch = rollingConfig.getConfigDetail().getTotalBatch(); @@ -833,20 +815,47 @@ private void setRollingInfoForStep(StepInstanceBaseDTO stepInstance, StepRunModeEnum.ROLLING_IN_BATCH : StepRunModeEnum.ROLLING_ALL); } - private List getHostsByLogContentKeyword(long stepInstanceId, int executeCount, Integer batch, - String keyword) { - String searchKey = keyword.replaceAll("'", "").replaceAll("\\$", "") - .replaceAll("&", "").replaceAll("\\$", "") - .replaceAll("\\|", "").replaceAll("`", "") - .replaceAll(";", ""); + private List getExecuteObjectCompositeKeysByLogContentKeyword( + StepInstanceBaseDTO stepInstance, + int executeCount, + Integer batch, + String keyword + ) { + return logService.getExecuteObjectsCompositeKeysByContentKeyword(stepInstance, executeCount, batch, keyword); + } - return logService.getIpsByContentKeyword(stepInstanceId, executeCount, batch, searchKey); + private List fuzzySearchHostsByIp(StepInstanceBaseDTO stepInstance, String searchIp) { + List matchExecuteObjects = + stepInstance.getTargetExecuteObjects().getExecuteObjectsCompatibly().stream() + .filter(executeObject -> isMatchByIp(executeObject, searchIp)) + .collect(Collectors.toList()); + if (CollectionUtils.isEmpty(matchExecuteObjects)) { + return null; + } + if (stepInstance.isSupportExecuteObjectFeature()) { + return matchExecuteObjects.stream() + .map(executeObject -> ExecuteObjectCompositeKey.ofExecuteObjectId(executeObject.getId())) + .collect(Collectors.toList()); + } else { + return matchExecuteObjects.stream() + .map(executeObject -> ExecuteObjectCompositeKey.ofHostId(executeObject.getResourceId())) + .collect(Collectors.toList()); + } } - private List fuzzySearchHostsByIp(StepInstanceBaseDTO stepInstance, String searchIp) { - return stepInstance.getTargetServers().getIpList().stream() - .filter(host -> host.getIp().contains(searchIp)) - .collect(Collectors.toList()); + private boolean isMatchByIp(ExecuteObject executeObject, String searchIp) { + boolean isMatch = false; + if (!executeObject.isHostExecuteObject()) { + return false; + } + HostDTO host = executeObject.getHost(); + if (StringUtils.isNotBlank(host.getIp())) { + isMatch = host.getIp().contains(searchIp); + } + if (!isMatch && StringUtils.isNotBlank(host.getIpv6())) { + isMatch = host.getIpv6().contains(searchIp); + } + return isMatch; } @Override @@ -905,7 +914,7 @@ private List convertToCronTaskExecuteResult(List { CronTaskExecuteResult cronTaskExecuteResult = new CronTaskExecuteResult(); cronTaskExecuteResult.setCronTaskId(taskInstance.getCronTaskId()); - cronTaskExecuteResult.setPlanId(taskInstance.getTaskId()); + cronTaskExecuteResult.setPlanId(taskInstance.getPlanId()); cronTaskExecuteResult.setStatus(taskInstance.getStatus().getValue()); cronTaskExecuteResult.setExecuteTime(taskInstance.getCreateTime()); return cronTaskExecuteResult; @@ -913,60 +922,61 @@ private List convertToCronTaskExecuteResult(List getHostsByResultType(String username, - Long appId, - Long stepInstanceId, - Integer executeCount, - Integer batch, - Integer resultType, - String tag, - String keyword) { - StepInstanceBaseDTO stepInstance = checkGetStepExecutionDetail(username, appId, stepInstanceId); - - if (!stepInstance.getAppId().equals(appId)) { - log.warn("Step instance is not in application, stepInstanceId={}, appId={}", stepInstanceId, appId); - throw new NotFoundException(ErrorCode.STEP_INSTANCE_NOT_EXIST); - } + public List getExecuteObjectsByResultType(String username, + Long appId, + Long taskInstanceId, + Long stepInstanceId, + Integer executeCount, + Integer batch, + Integer resultType, + String tag, + String keyword) { + StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstance( + appId, taskInstanceId, stepInstanceId); + preProcessViewStepExecutionResult(username, appId, stepInstance); StepExecutionResultQuery query = StepExecutionResultQuery.builder() + .taskInstanceId(taskInstanceId) .stepInstanceId(stepInstanceId) .executeCount(executeCount) .batch(batch) .logKeyword(keyword) .build(); - Set matchHostIds = null; + Set matchExecuteObjectCompositeKeys = null; boolean filterByKeyword = StringUtils.isNotEmpty(keyword); if (filterByKeyword) { - matchHostIds = getMatchHostIdsByIp(stepInstance, query); - if (CollectionUtils.isEmpty(matchHostIds)) { + matchExecuteObjectCompositeKeys = getMatchExecuteObjectCompositeKeys(stepInstance, query); + if (CollectionUtils.isEmpty(matchExecuteObjectCompositeKeys)) { return Collections.emptyList(); } } if (stepInstance.getStatus() == RunStatusEnum.BLANK) { - // 步骤未启动，AgentTask数据还未在DB初始化，构造初始任务结果 - return filterTargetHostsByBatch(stepInstance, query.getBatch()); + // 步骤未启动，ExecuteObjectTask数据还未在DB初始化，构造初始任务结果 + return filterTargetExecuteObjectsByBatch(stepInstance, query.getBatch()); } - List agentTaskGroupByResultType = listAgentTaskByResultGroup(stepInstance, + List executeObjectTaskGroupByResultType = listExecuteObjectTaskByResultGroup(stepInstance, executeCount, batch, resultType, tag); - if (CollectionUtils.isEmpty(agentTaskGroupByResultType)) { + if (CollectionUtils.isEmpty(executeObjectTaskGroupByResultType)) { return Collections.emptyList(); } - List hosts = agentTaskGroupByResultType.stream() - .map(AgentTaskDetailDTO::getHost) + List executeObjects = executeObjectTaskGroupByResultType.stream() + .map(ExecuteObjectTask::getExecuteObject) .collect(Collectors.toList()); - if (filterByKeyword && CollectionUtils.isNotEmpty(matchHostIds)) { - List finalHosts = new ArrayList<>(); - for (HostDTO host : hosts) { - if (matchHostIds.contains(host.getHostId())) { - finalHosts.add(host); + if (filterByKeyword && CollectionUtils.isNotEmpty(matchExecuteObjectCompositeKeys)) { + ExecuteObjectCompositeKey.CompositeKeyType compositeKeyType = + getCompositeKeyType(matchExecuteObjectCompositeKeys); + List finalExecuteObject = new ArrayList<>(); + for (ExecuteObject executeObject : executeObjects) { + if (isExecuteObjectMatch(executeObject, matchExecuteObjectCompositeKeys, compositeKeyType)) { + finalExecuteObject.add(executeObject); } } - return finalHosts; + return finalExecuteObject; } else { - return hosts; + return executeObjects; } } @@ -974,9 +984,12 @@ public List getHostsByResultType(String username, @Override public List listStepExecutionHistory(String username, Long appId, + Long taskInstanceId, Long stepInstanceId, Integer batch) { - StepInstanceBaseDTO stepInstance = checkGetStepExecutionDetail(username, appId, stepInstanceId); + StepInstanceBaseDTO stepInstance = stepInstanceService.getBaseStepInstance( + appId, taskInstanceId, stepInstanceId); + preProcessViewStepExecutionResult(username, appId, stepInstance); // 步骤没有重试执行过 if (stepInstance.getExecuteCount() == 0) { @@ -993,7 +1006,7 @@ public List listStepExecutionHistory(String username, records = queryStepRetryRecords(stepInstance); } else { // 获取滚动任务维度的重试记录 - records = queryStepRollingTaskRetryRecords(stepInstanceId, batch); + records = queryStepRollingTaskRetryRecords(stepInstance.getTaskInstanceId(), stepInstanceId, batch); } records.sort(Comparator.comparingInt(StepExecutionRecordDTO::getRetryCount).reversed()); @@ -1033,10 +1046,12 @@ private List queryStepRetryRecords(StepInstanceBaseDTO s return records; } - private List queryStepRollingTaskRetryRecords(long stepInstanceId, int batch) { + private List queryStepRollingTaskRetryRecords(Long taskInstanceId, + long stepInstanceId, + int batch) { List records = new ArrayList<>(); List rollingTasks = - stepInstanceRollingTaskService.listRollingTasksByBatch(stepInstanceId, batch); + stepInstanceRollingTaskService.listRollingTasksByBatch(taskInstanceId, stepInstanceId, batch); rollingTasks.forEach(rollingTask -> { StepExecutionRecordDTO record = new StepExecutionRecordDTO(); record.setStepInstanceId(stepInstanceId); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ThirdFileExternalAgentHostProvisioner.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ThirdFileExternalAgentHostProvisioner.java new file mode 100644 index 0000000000..e6d4ec15a5 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/ThirdFileExternalAgentHostProvisioner.java @@ -0,0 +1,51 @@ +/* + * + * * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * * + * * Copyright (C) 2021 Tencent. All rights reserved. + * * + * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * * + * * License for BK-JOB蓝鲸智云作业平台: + * * -------------------------------------------------------------------- + * * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * * + * * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * * the Software. + * * + * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * * IN THE SOFTWARE. + * + */ + +package com.tencent.bk.job.execute.service.impl; + +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.execute.service.ExternalAgentService; +import com.tencent.bk.job.execute.service.ThirdFileDistributeSourceHostProvisioner; +import lombok.extern.slf4j.Slf4j; + +/** + * 使用外部机器作为文件源文件的分发源 + */ +@Slf4j +public class ThirdFileExternalAgentHostProvisioner implements ThirdFileDistributeSourceHostProvisioner { + + private final ExternalAgentService externalAgentService; + + public ThirdFileExternalAgentHostProvisioner(ExternalAgentService externalAgentService) { + this.externalAgentService = externalAgentService; + } + + @Override + public HostDTO getThirdFileDistributeSourceHost(Long cloudId, String protocol, String ip) { + log.debug("distribute third file from external agent host"); + return externalAgentService.getDistributeSourceHost(); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TopoServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TopoServiceImpl.java index 7b0cf06f8a..1d61ec524c 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TopoServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/service/impl/TopoServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,7 @@ package com.tencent.bk.job.execute.service.impl; +import com.google.common.collect.Lists; import com.tencent.bk.job.common.cc.model.InstanceTopologyDTO; import com.tencent.bk.job.common.cc.model.req.GetTopoNodePathReq; import com.tencent.bk.job.common.cc.sdk.BizCmdbClient; @@ -34,6 +35,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -49,9 +51,21 @@ public TopoServiceImpl(BizCmdbClient bizCmdbClient) { } @Override - public List batchGetTopoNodeHierarchy(long appId, List topoNodes) { + public List batchGetTopoNodeHierarchy(long bizId, List topoNodes) { + // CMDB接口限制每次最多查询1000个拓扑节点 + int batchSize = 1000; + List> topoNodeSubList = Lists.partition(topoNodes, batchSize); + List hierarchyNodes = new ArrayList<>(); + for (List subList : topoNodeSubList) { + hierarchyNodes.addAll(batchGetTopoNodePathWithoutLimit(bizId, subList)); + } + return hierarchyNodes; + } + + private List batchGetTopoNodePathWithoutLimit(long bizId, + List topoNodes) { GetTopoNodePathReq req = new GetTopoNodePathReq(); - req.setBizId(appId); + req.setBizId(bizId); topoNodes.forEach(topoNode -> req.add(topoNode.getNodeType(), topoNode.getTopoNodeId())); List hierarchyNodes = bizCmdbClient.getTopoInstancePath(req); log.debug("Get topo node hierarchy, req:{}, result:{}", req, hierarchyNodes); diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/statistics/ClearExpiredStatisticsTask.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/statistics/ClearExpiredStatisticsTask.java index 5cb8f4b2c4..57faf4b1b1 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/statistics/ClearExpiredStatisticsTask.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/statistics/ClearExpiredStatisticsTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,7 @@ package com.tencent.bk.job.execute.statistics; -import com.tencent.bk.job.common.statistics.consts.StatisticsConstants; +import com.tencent.bk.job.analysis.api.consts.StatisticsConstants; import com.tencent.bk.job.common.util.TimeUtil; import com.tencent.bk.job.execute.config.StatisticConfig; import com.tencent.bk.job.execute.dao.StatisticsDAO; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/statistics/StatisticsFlushThread.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/statistics/StatisticsFlushThread.java index 5c47b570fc..7adcef1ae0 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/statistics/StatisticsFlushThread.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/statistics/StatisticsFlushThread.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,21 +24,9 @@ package com.tencent.bk.job.execute.statistics; -import com.tencent.bk.job.common.util.ObjectWrapper; +import com.tencent.bk.job.execute.dao.StatisticsDAO; import lombok.extern.slf4j.Slf4j; -import lombok.val; -import org.jooq.Condition; -import org.jooq.DSLContext; -import org.jooq.Record2; -import org.jooq.Result; -import org.jooq.conf.ParamType; -import org.jooq.exception.DataAccessException; -import org.jooq.generated.tables.Statistics; -import org.jooq.impl.DSL; -import org.jooq.types.ULong; -import java.util.ArrayList; -import java.util.List; import java.util.Map; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.atomic.AtomicInteger; @@ -46,100 +34,23 @@ @Slf4j public class StatisticsFlushThread extends Thread { - private static final Statistics tableStatistics = Statistics.STATISTICS; - private final DSLContext dslContext; + private final StatisticsDAO statisticsDAO; private volatile LinkedBlockingQueue>> flushQueue; - public StatisticsFlushThread(DSLContext dslContext, LinkedBlockingQueue>> flushQueue) { - this.dslContext = dslContext; + public StatisticsFlushThread(StatisticsDAO statisticsDAO, + LinkedBlockingQueue>> flushQueue) { + this.statisticsDAO = statisticsDAO; this.flushQueue = flushQueue; } private void flushIncrementMap(Map> incrementMap) { incrementMap.forEach((dateStr, metricsMap) -> { metricsMap.forEach((statisticsKey, value) -> { - final Long appId = statisticsKey.getAppId(); - final String resource = statisticsKey.getResource(); - final String dimension = statisticsKey.getDimension(); - final String dimensionValue = statisticsKey.getDimensionValue(); - final String date = dateStr; final Integer incrementValue = value.get(); - final ObjectWrapper>> recordsWrapper = new ObjectWrapper<>(null); - AtomicInteger affectedRows = new AtomicInteger(0); + int affectRows; do { - dslContext.transaction(configuration -> { - DSLContext context = DSL.using(configuration); - List conditions = new ArrayList<>(); - conditions.add(tableStatistics.APP_ID.eq(appId)); - conditions.add(tableStatistics.RESOURCE.eq(resource)); - conditions.add(tableStatistics.DIMENSION.eq(dimension)); - conditions.add(tableStatistics.DIMENSION_VALUE.eq(dimensionValue)); - conditions.add(tableStatistics.DATE.eq(date)); - try { - Long oldValue = 0L; - Long id; - val selectQuery = context.select(tableStatistics.ID, tableStatistics.VALUE) - .from(tableStatistics) - .where(conditions) - .forUpdate(); - log.debug("selectQuery=" + selectQuery.getSQL(ParamType.INLINED)); - Result> records = selectQuery.fetch(); - recordsWrapper.set(records); - if (records.isEmpty()) { - log.debug("records is empty"); - // 记录不存在，先插入 - val query = context.insertInto(tableStatistics, - tableStatistics.ID, - tableStatistics.APP_ID, - tableStatistics.RESOURCE, - tableStatistics.DIMENSION, - tableStatistics.DIMENSION_VALUE, - tableStatistics.DATE, - tableStatistics.VALUE, - tableStatistics.CREATE_TIME, - tableStatistics.LAST_MODIFY_TIME - ).values( - null, - appId, - resource, - dimension, - dimensionValue, - date, - "0", - ULong.valueOf(System.currentTimeMillis()), - ULong.valueOf(System.currentTimeMillis()) - ).returning(tableStatistics.ID); - id = query.fetchOne().getId(); - } else { - if (records.size() > 1) { - log.warn("more than 1 records, statisticsKey:{}", statisticsKey); - } - id = records.get(0).get(tableStatistics.ID); - oldValue = Long.parseLong(records.get(0).get(tableStatistics.VALUE)); - } - // 更新 - log.debug("Update record {} from {} to {}", id, oldValue, (oldValue + incrementValue)); - affectedRows.set(context.update(tableStatistics) - .set(tableStatistics.VALUE, "" + (oldValue + incrementValue)) - .where(tableStatistics.ID.eq(id)) - .and(tableStatistics.VALUE.eq("" + oldValue)) - .execute()); - if (affectedRows.get() == 0) { - log.debug("Record {} updated by other thread just now, retry", statisticsKey); - } - } catch (DataAccessException dataAccessException) { - if (dataAccessException.getMessage().contains("Duplicate entry")) { - //多个实例并发插入导致的主键冲突，忽略，改为更新 - } else { - log.warn("dataAccessException when update/create", dataAccessException); - } - } catch (Throwable t) { - log.info("May fail to lock", t); - } - log.debug("records={},affectedRows={}", recordsWrapper.get(), affectedRows); - }); - } while (recordsWrapper.get() == null || affectedRows.get() == 0); + affectRows = statisticsDAO.increaseStatisticValue(dateStr, statisticsKey, incrementValue); + } while (affectRows == 0); }); }); } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/statistics/StatisticsKey.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/statistics/StatisticsKey.java index 2e7987af43..a11a2681c1 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/statistics/StatisticsKey.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/statistics/StatisticsKey.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/statistics/StatisticsService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/statistics/StatisticsService.java index cb90224586..7ec5711671 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/statistics/StatisticsService.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/statistics/StatisticsService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,7 @@ package com.tencent.bk.job.execute.statistics; -import com.tencent.bk.job.common.statistics.model.dto.StatisticsDTO; +import com.tencent.bk.job.analysis.api.dto.StatisticsDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; import java.util.List; diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/statistics/StatisticsServiceImpl.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/statistics/StatisticsServiceImpl.java index b2e9c2117b..7eb5da0af0 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/statistics/StatisticsServiceImpl.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/statistics/StatisticsServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,9 +24,9 @@ package com.tencent.bk.job.execute.statistics; +import com.tencent.bk.job.analysis.api.consts.StatisticsConstants; +import com.tencent.bk.job.analysis.api.dto.StatisticsDTO; import com.tencent.bk.job.common.constant.NotExistPathHandlerEnum; -import com.tencent.bk.job.common.statistics.consts.StatisticsConstants; -import com.tencent.bk.job.common.statistics.model.dto.StatisticsDTO; import com.tencent.bk.job.common.util.TimeUtil; import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.execute.common.constants.RunStatusEnum; @@ -38,13 +38,14 @@ import com.tencent.bk.job.execute.model.FileStepInstanceDTO; import com.tencent.bk.job.execute.model.TaskInstanceDTO; import com.tencent.bk.job.execute.service.ApplicationService; +import com.tencent.bk.job.execute.service.RollingConfigService; import com.tencent.bk.job.execute.service.TaskInstanceService; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; import lombok.extern.slf4j.Slf4j; -import org.jooq.DSLContext; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Service; +import org.springframework.util.StopWatch; import javax.annotation.PostConstruct; import java.time.LocalDateTime; @@ -63,9 +64,9 @@ public class StatisticsServiceImpl implements StatisticsService { private final TaskInstanceService taskInstanceService; private final ApplicationService applicationService; private final StepInstanceDAO stepInstanceDAO; - private final DSLContext dslContext; private final StatisticsDAO statisticsDAO; private final StatisticConfig statisticConfig; + private final RollingConfigService rollingConfigService; private volatile Map> incrementMap = new ConcurrentHashMap<>(); private volatile LinkedBlockingQueue>> flushQueue = new LinkedBlockingQueue<>(600); @@ -75,26 +76,26 @@ public StatisticsServiceImpl( TaskInstanceService taskInstanceService, ApplicationService applicationService, StepInstanceDAO stepInstanceDAO, - DSLContext dslContext, StatisticsDAO statisticsDAO, - StatisticConfig statisticConfig + StatisticConfig statisticConfig, + RollingConfigService rollingConfigService ) { this.taskInstanceService = taskInstanceService; this.applicationService = applicationService; this.stepInstanceDAO = stepInstanceDAO; - this.dslContext = dslContext; this.statisticsDAO = statisticsDAO; this.statisticConfig = statisticConfig; + this.rollingConfigService = rollingConfigService; } @PostConstruct public void init() { - StatisticsFlushThread flushThread = new StatisticsFlushThread(dslContext, flushQueue); + StatisticsFlushThread flushThread = new StatisticsFlushThread(statisticsDAO, flushQueue); flushThread.setName("flushThread"); flushThread.start(); } - public void updateExecutedTaskCount( + private void updateExecutedTaskCount( TaskInstanceDTO taskInstanceDTO, Map metricsMap ) { @@ -108,15 +109,15 @@ public void updateExecutedTaskCount( log.debug("executedTaskCount={}", executedTaskCountValue); } - public void updateExecutedTaskByStartupMode(TaskInstanceDTO taskInstanceDTO, - Map metricsMap) { + private void updateExecutedTaskByStartupMode(TaskInstanceDTO taskInstanceDTO, + Map metricsMap) { // 按渠道统计 Integer startupMode = taskInstanceDTO.getStartupMode(); if (startupMode == null) { log.warn("startupMode is null, ignore"); return; } - if (startupMode == TaskStartupModeEnum.NORMAL.getValue()) { + if (startupMode == TaskStartupModeEnum.WEB.getValue()) { StatisticsKey keyStartupNormalTaskCount = new StatisticsKey(taskInstanceDTO.getAppId(), StatisticsConstants.RESOURCE_EXECUTED_TASK, StatisticsConstants.DIMENSION_TASK_STARTUP_MODE, StatisticsConstants.DIMENSION_VALUE_TASK_STARTUP_MODE_NORMAL); @@ -145,8 +146,8 @@ public void updateExecutedTaskByStartupMode(TaskInstanceDTO taskInstanceDTO, } } - public void updateExecutedTaskByType(TaskInstanceDTO taskInstanceDTO, - Map metricsMap) { + private void updateExecutedTaskByType(TaskInstanceDTO taskInstanceDTO, + Map metricsMap) { // 按类型统计 Integer type = taskInstanceDTO.getType(); if (type == null) { @@ -161,6 +162,16 @@ public void updateExecutedTaskByType(TaskInstanceDTO taskInstanceDTO, statisticsKey -> new AtomicInteger(0)); int taskTypeNormalTaskCountValue = taskTypeNormalTaskCount.incrementAndGet(); log.debug("taskTypeNormalTaskCount={}", taskTypeNormalTaskCountValue); + // 滚动执行 + if (rollingConfigService.isTaskRollingEnabled(taskInstanceDTO.getId())) { + StatisticsKey keyRollingTaskNormalTaskCount = new StatisticsKey(taskInstanceDTO.getAppId(), + StatisticsConstants.RESOURCE_ROLLING_TASK, StatisticsConstants.DIMENSION_TASK_TYPE, + StatisticsConstants.DIMENSION_VALUE_TASK_TYPE_EXECUTE_TASK); + AtomicInteger rollingTaskNormalTaskCount = metricsMap.computeIfAbsent(keyRollingTaskNormalTaskCount, + statisticsKey -> new AtomicInteger(0)); + int rollingTaskNormalTaskCountValue = rollingTaskNormalTaskCount.incrementAndGet(); + log.debug("rollingTaskNormalTaskCount={}", rollingTaskNormalTaskCountValue); + } } else if (type.equals(TaskTypeEnum.SCRIPT.getValue())) { StatisticsKey keyTaskTypeScriptTaskCount = new StatisticsKey(taskInstanceDTO.getAppId(), StatisticsConstants.RESOURCE_EXECUTED_TASK, StatisticsConstants.DIMENSION_TASK_TYPE, @@ -169,6 +180,16 @@ public void updateExecutedTaskByType(TaskInstanceDTO taskInstanceDTO, statisticsKey -> new AtomicInteger(0)); int taskTypeScriptTaskCountValue = taskTypeScriptTaskCount.incrementAndGet(); log.debug("taskTypeScriptTaskCount={}", taskTypeScriptTaskCountValue); + // 滚动执行 + if (rollingConfigService.isTaskRollingEnabled(taskInstanceDTO.getId())) { + StatisticsKey keyRollingTaskScriptTaskCount = new StatisticsKey(taskInstanceDTO.getAppId(), + StatisticsConstants.RESOURCE_ROLLING_TASK, StatisticsConstants.DIMENSION_TASK_TYPE, + StatisticsConstants.DIMENSION_VALUE_TASK_TYPE_FAST_EXECUTE_SCRIPT); + AtomicInteger rollingTaskScriptTaskCount = metricsMap.computeIfAbsent(keyRollingTaskScriptTaskCount, + statisticsKey -> new AtomicInteger(0)); + int rollingTaskScriptTaskCountValue = rollingTaskScriptTaskCount.incrementAndGet(); + log.debug("rollingTaskScriptTaskCount={}", rollingTaskScriptTaskCountValue); + } } else if (type.equals(TaskTypeEnum.FILE.getValue())) { StatisticsKey keyTaskTypeFileTaskCount = new StatisticsKey(taskInstanceDTO.getAppId(), StatisticsConstants.RESOURCE_EXECUTED_TASK, StatisticsConstants.DIMENSION_TASK_TYPE, @@ -177,13 +198,23 @@ public void updateExecutedTaskByType(TaskInstanceDTO taskInstanceDTO, statisticsKey -> new AtomicInteger(0)); int taskTypeFileTaskCountValue = taskTypeFileTaskCount.incrementAndGet(); log.debug("taskTypeFileTaskCount={}", taskTypeFileTaskCountValue); + // 滚动执行 + if (rollingConfigService.isTaskRollingEnabled(taskInstanceDTO.getId())) { + StatisticsKey keyRollingTaskFileTaskCount = new StatisticsKey(taskInstanceDTO.getAppId(), + StatisticsConstants.RESOURCE_ROLLING_TASK, StatisticsConstants.DIMENSION_TASK_TYPE, + StatisticsConstants.DIMENSION_VALUE_TASK_TYPE_FAST_PUSH_FILE); + AtomicInteger rollingTaskFileTaskCount = metricsMap.computeIfAbsent(keyRollingTaskFileTaskCount, + statisticsKey -> new AtomicInteger(0)); + int rollingTaskFileTaskCountValue = rollingTaskFileTaskCount.incrementAndGet(); + log.debug("rollingTaskFileTaskCount={}", rollingTaskFileTaskCountValue); + } } else { log.warn("do not support type {}, ignore", type); } } - public void updateExecutedFastScriptStatistics(TaskInstanceDTO taskInstanceDTO, - Map metricsMap) { + private void updateExecutedFastScriptStatistics(TaskInstanceDTO taskInstanceDTO, + Map metricsMap) { Integer type = taskInstanceDTO.getType(); if (type != null && type.equals(TaskTypeEnum.SCRIPT.getValue())) { // 查StepInstance @@ -193,7 +224,7 @@ public void updateExecutedFastScriptStatistics(TaskInstanceDTO taskInstanceDTO, return; } // 查StepInstanceScript - Byte scriptType = stepInstanceDAO.getScriptTypeByStepInstanceId(stepInstanceId); + Byte scriptType = stepInstanceDAO.getScriptTypeByStepInstanceId(taskInstanceDTO.getId(), stepInstanceId); // 更新统计数据 // 快速执行脚本：按脚本类型统计 String scriptTypeName = ScriptTypeEnum.getName(scriptType.intValue()); @@ -207,8 +238,8 @@ public void updateExecutedFastScriptStatistics(TaskInstanceDTO taskInstanceDTO, } } - public void updateExecutedFastFileStatistics(TaskInstanceDTO taskInstanceDTO, - Map metricsMap) { + private void updateExecutedFastFileStatistics(TaskInstanceDTO taskInstanceDTO, + Map metricsMap) { Integer type = taskInstanceDTO.getType(); if (type != null && type.equals(TaskTypeEnum.FILE.getValue())) { // 查StepInstance @@ -218,7 +249,8 @@ public void updateExecutedFastFileStatistics(TaskInstanceDTO taskInstanceDTO, return; } // 查StepInstanceFile - FileStepInstanceDTO fileStepInstanceDTO = stepInstanceDAO.getFileStepInstance(stepInstanceId); + FileStepInstanceDTO fileStepInstanceDTO = stepInstanceDAO.getFileStepInstance( + taskInstanceDTO.getId(), stepInstanceId); // 更新统计数据 // 快速分发文件：按传输模式统计 Integer notExistPathHandler = fileStepInstanceDTO.getNotExistPathHandler(); @@ -245,7 +277,7 @@ public void updateExecutedFastFileStatistics(TaskInstanceDTO taskInstanceDTO, } } - public void updateFailedTaskCount(TaskInstanceDTO taskInstanceDTO, Map metricsMap) { + private void updateFailedTaskCount(TaskInstanceDTO taskInstanceDTO, Map metricsMap) { // 累计执行失败次数统计 if (taskInstanceDTO.getStatus() == RunStatusEnum.FAIL) { StatisticsKey keyFailedTaskCount = new StatisticsKey(taskInstanceDTO.getAppId(), @@ -255,11 +287,21 @@ public void updateFailedTaskCount(TaskInstanceDTO taskInstanceDTO, Map new AtomicInteger(0)); int failedTaskCountValue = failedTaskCount.incrementAndGet(); log.debug("failedTaskCount={}", failedTaskCountValue); + // 滚动执行 + if (rollingConfigService.isTaskRollingEnabled(taskInstanceDTO.getId())) { + StatisticsKey keyFailedRollingTaskCount = new StatisticsKey(taskInstanceDTO.getAppId(), + StatisticsConstants.RESOURCE_ROLLING_FAILED_TASK, StatisticsConstants.DIMENSION_TIME_UNIT, + StatisticsConstants.DIMENSION_VALUE_TIME_UNIT_DAY); + AtomicInteger failedRollingTaskCount = metricsMap.computeIfAbsent(keyFailedRollingTaskCount, + statisticsKey -> new AtomicInteger(0)); + int failedRollingTaskCountValue = failedRollingTaskCount.incrementAndGet(); + log.debug("failedRollingTaskCount={}", failedRollingTaskCountValue); + } } } - public void updateFastScriptCountByStatus(TaskInstanceDTO taskInstanceDTO, - Map metricsMap) { + private void updateFastScriptCountByStatus(TaskInstanceDTO taskInstanceDTO, + Map metricsMap) { // 快速脚本按状态统计 Integer type = taskInstanceDTO.getType(); if (type != null && type.equals(TaskTypeEnum.SCRIPT.getValue())) { @@ -295,8 +337,8 @@ public void updateFastScriptCountByStatus(TaskInstanceDTO taskInstanceDTO, } } - public void updateFastFileCountByStatus(TaskInstanceDTO taskInstanceDTO, - Map metricsMap) { + private void updateFastFileCountByStatus(TaskInstanceDTO taskInstanceDTO, + Map metricsMap) { // 快速文件按状态统计 Integer type = taskInstanceDTO.getType(); if (type != null && type.equals(TaskTypeEnum.FILE.getValue())) { @@ -332,8 +374,8 @@ public void updateFastFileCountByStatus(TaskInstanceDTO taskInstanceDTO, } } - public void updateExecutedTaskByTimeConsuming(TaskInstanceDTO taskInstanceDTO, - Map metricsMap) { + private void updateExecutedTaskByTimeConsuming(TaskInstanceDTO taskInstanceDTO, + Map metricsMap) { // 按执行耗时统计 Long totalTime = taskInstanceDTO.getTotalTime(); if (totalTime == null) { @@ -372,22 +414,53 @@ public void updateExecutedTaskByTimeConsuming(TaskInstanceDTO taskInstanceDTO, @Override public void updateStartJobStatistics(TaskInstanceDTO taskInstanceDTO) { - String createDateStr = DateUtils.getDateStrFromUnixTimeMills(taskInstanceDTO.getCreateTime()); - synchronized (writeLock) { - Map metricsMap = incrementMap.computeIfAbsent(createDateStr, - dateStr -> new ConcurrentHashMap<>()); - // 触发时间当天的数据统计 - // 累计任务执行次数统计 - updateExecutedTaskCount(taskInstanceDTO, metricsMap); - // 按渠道统计 - updateExecutedTaskByStartupMode(taskInstanceDTO, metricsMap); - // 按类型统计 - updateExecutedTaskByType(taskInstanceDTO, metricsMap); - // 快速执行脚本：按脚本类型统计 - updateExecutedFastScriptStatistics(taskInstanceDTO, metricsMap); - // 快速分发文件：按传输模式统计 - updateExecutedFastFileStatistics(taskInstanceDTO, metricsMap); + StopWatch watch = new StopWatch("updateStartJobStatistics"); + try { + String createDateStr = DateUtils.getDateStrFromUnixTimeMills(taskInstanceDTO.getCreateTime()); + watch.start("getWriteLock"); + synchronized (writeLock) { + watch.stop(); + + Map metricsMap = incrementMap.computeIfAbsent(createDateStr, + dateStr -> new ConcurrentHashMap<>()); + // 触发时间当天的数据统计 + // 累计任务执行次数统计 + watch.start("updateExecutedTaskCount"); + updateExecutedTaskCount(taskInstanceDTO, metricsMap); + watch.stop(); + + // 按渠道统计 + watch.start("updateExecutedTaskByStartupMode"); + updateExecutedTaskByStartupMode(taskInstanceDTO, metricsMap); + watch.stop(); + + // 按类型统计 + watch.start("updateExecutedTaskByType"); + updateExecutedTaskByType(taskInstanceDTO, metricsMap); + watch.stop(); + + // 快速执行脚本：按脚本类型统计 + watch.start("updateExecutedFastScriptStatistics"); + updateExecutedFastScriptStatistics(taskInstanceDTO, metricsMap); + watch.stop(); + + // 快速分发文件：按传输模式统计 + watch.start("updateExecutedFastFileStatistics"); + updateExecutedFastFileStatistics(taskInstanceDTO, metricsMap); + watch.stop(); + } + } catch (Throwable ignoreException) { + // 捕获所有异常，不影响主流程 + log.error("UpdateStartJobStatistics exception", ignoreException); + } finally { + if (watch.isRunning()) { + watch.stop(); + } + if (watch.getTotalTimeMillis() > 100) { + log.warn("UpdateStartJobStatistics slow, watch: {}", watch.prettyPrint()); + } } + } @Override @@ -455,7 +528,6 @@ public Boolean triggerStatistics(List dateList) { updateStartJobStatistics(taskInstance); } offset += limit; - Thread.sleep(2000); } while (taskInstanceIds.size() == limit); } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/statistics/consts/StatisticsActionEnum.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/statistics/consts/StatisticsActionEnum.java index 58d5ea8df9..0c74c87e00 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/statistics/consts/StatisticsActionEnum.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/statistics/consts/StatisticsActionEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/statistics/message/TaskStatisticsProcessor.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/statistics/message/TaskStatisticsProcessor.java index cb9733f5a3..3dcf9f3195 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/statistics/message/TaskStatisticsProcessor.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/statistics/message/TaskStatisticsProcessor.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/statistics/model/TaskStatisticsCmd.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/statistics/model/TaskStatisticsCmd.java index dccf4dc0c7..77809f641b 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/statistics/model/TaskStatisticsCmd.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/statistics/model/TaskStatisticsCmd.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/task/LocalTmpFileCleanTask.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/task/LocalTmpFileCleanTask.java index 7a7ad738ef..7b183a2d3b 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/task/LocalTmpFileCleanTask.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/task/LocalTmpFileCleanTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,9 +25,11 @@ package com.tencent.bk.job.execute.task; import com.tencent.bk.job.common.constant.JobConstants; +import com.tencent.bk.job.common.util.StringUtil; +import com.tencent.bk.job.common.util.file.FileUtil; import com.tencent.bk.job.common.util.file.PathUtil; +import com.tencent.bk.job.execute.config.FileDistributeConfig; import com.tencent.bk.job.execute.config.LocalFileConfigForExecute; -import com.tencent.bk.job.execute.config.StorageSystemConfig; import com.tencent.bk.job.execute.constants.Consts; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; @@ -44,20 +46,20 @@ public class LocalTmpFileCleanTask { private final LocalFileConfigForExecute localFileConfigForExecute; - private final StorageSystemConfig storageSystemConfig; + private final FileDistributeConfig fileDistributeConfig; public LocalTmpFileCleanTask( LocalFileConfigForExecute localFileConfigForExecute, - StorageSystemConfig storageSystemConfig) { + FileDistributeConfig fileDistributeConfig) { this.localFileConfigForExecute = localFileConfigForExecute; - this.storageSystemConfig = storageSystemConfig; + this.fileDistributeConfig = fileDistributeConfig; } public void execute() { // 使用制品库作为存储后端时才清理下载产生的临时文件 if (JobConstants.FILE_STORAGE_BACKEND_ARTIFACTORY .equals(localFileConfigForExecute.getStorageBackend())) { - cleanLocalTmpFile(); + cleanLocalTmpFileAndDir(); } else { // 使用本地NFS作为存储后端时在job-manage中筛选被引用文件并清理NFS，此处无需清理 log.info( @@ -68,27 +70,109 @@ public void execute() { } /** - * 清理下载到本地的临时文件 + * 清理下载到本地的临时文件及目录 */ - private void cleanLocalTmpFile() { - log.info("begin to cleanLocalTmpFile downloaded from artifactory"); + private void cleanLocalTmpFileAndDir() { + log.info("begin to cleanLocalTmpFileAndDir downloaded from artifactory"); // 单个任务最长执行时间为1天，清理一天之前的所有临时文件 Date thresholdDate = new Date( System.currentTimeMillis() - 3600 * 24 * 1000 ); String localFileDirPath = PathUtil.joinFilePath( - storageSystemConfig.getJobStorageRootPath(), Consts.LOCAL_FILE_DIR_NAME + fileDistributeConfig.getJobDistributeRootPath(), Consts.LOCAL_FILE_DIR_NAME ); - Iterator fileIterator = FileUtils.iterateFiles( - new File(localFileDirPath), - new AgeFileFilter(thresholdDate), - TrueFileFilter.TRUE - ); - while (fileIterator.hasNext()) { - File aFile = fileIterator.next(); - log.info("Delete local tmp file {}", aFile.getPath()); - FileUtils.deleteQuietly(aFile); + + File localFileDir = new File(localFileDirPath); + if (!localFileDir.exists() || !localFileDir.isDirectory()) { + log.warn("Local tmp file directory does not exist or is not a directory: {}", localFileDirPath); + log.info("cleanLocalTmpFile finished"); + return; + } + + File[] files = localFileDir.listFiles(); + if (files != null && files.length > 0) { + Iterator fileIterator = FileUtils.iterateFilesAndDirs( + localFileDir, + new AgeFileFilter(thresholdDate), + TrueFileFilter.TRUE + ); + while (fileIterator.hasNext()) { + File aFile = fileIterator.next(); + // 清理存量遗留的空目录 + if (aFile.isDirectory()) { + clearEmptyDirAndParent(aFile); + continue; + } + // 清理文件 + boolean fileDeleted = FileUtils.deleteQuietly(aFile); + if (!fileDeleted) { + log.warn("Fail to delete tmp file {}, ignore", aFile.getAbsolutePath()); + continue; + } + // 同时清理两级无用的父目录 + File parentDirFile = aFile.getParentFile(); + boolean parentDirDeleted = false; + boolean grandParentDirDeleted = false; + if (parentDirFile != null) { + parentDirDeleted = deleteEmptyDirIfNotDisTributeRoot(parentDirFile); + File grandParentDirFile = parentDirFile.getParentFile(); + if (grandParentDirFile != null) { + grandParentDirDeleted = deleteEmptyDirIfNotDisTributeRoot(grandParentDirFile); + } + } + log.info( + "Local tmp file {} deleted, parentDirDeleted={}, grandParentDirDeleted={}", + aFile.getPath(), + parentDirDeleted, + grandParentDirDeleted + ); + } + } else { + log.warn("Local tmp file directory is empty: {}", localFileDirPath); } log.info("cleanLocalTmpFile finished"); } + + /** + * 清理非分发根目录的空目录及其父目录 + */ + private void clearEmptyDirAndParent(File dirFile) { + boolean dirDeleted = deleteEmptyDirIfNotDisTributeRoot(dirFile); + if (!dirDeleted) { + return; + } + File parentDirFile = dirFile.getParentFile(); + boolean parentDirDeleted = false; + if (parentDirFile != null) { + parentDirDeleted = deleteEmptyDirIfNotDisTributeRoot(parentDirFile); + } + log.info("Local tmp dir {} deleted, parentDirDeleted={}", dirFile.getAbsolutePath(), parentDirDeleted); + } + + /** + * 清理非分发根目录的空目录 + * + * @param dirFile 目录文件 + * @return 是否删除了空目录 + */ + private boolean deleteEmptyDirIfNotDisTributeRoot(File dirFile) { + if (!isJobDisTributeRootPath(dirFile.getAbsolutePath())) { + return FileUtil.deleteEmptyDirectory(dirFile); + } + return false; + } + + /** + * 判断路径是否指向分发根目录（忽略反斜杠后缀） + * + * @param path 路径 + * @return 路径是否指向分发根目录 + */ + private boolean isJobDisTributeRootPath(String path) { + if (path == null) { + return false; + } + String jobDisTributeRootPath = StringUtil.removeSuffix(fileDistributeConfig.getJobDistributeRootPath(), "/"); + return jobDisTributeRootPath.equals(StringUtil.removeSuffix(path, "/")); + } } diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/task/LogExportFileCleanTask.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/task/LogExportFileCleanTask.java index ebd743a59a..6c9a15ad6c 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/task/LogExportFileCleanTask.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/task/LogExportFileCleanTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,15 +24,15 @@ package com.tencent.bk.job.execute.task; +import com.tencent.bk.job.common.artifactory.config.ArtifactoryConfig; import com.tencent.bk.job.common.artifactory.model.dto.NodeDTO; import com.tencent.bk.job.common.artifactory.model.dto.PageData; import com.tencent.bk.job.common.artifactory.sdk.ArtifactoryClient; import com.tencent.bk.job.common.constant.JobConstants; import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.common.util.file.PathUtil; -import com.tencent.bk.job.execute.config.ArtifactoryConfig; import com.tencent.bk.job.execute.config.LogExportConfig; -import com.tencent.bk.job.execute.config.StorageSystemConfig; +import com.tencent.bk.job.execute.config.StorageConfig; import com.tencent.bk.job.execute.constants.Consts; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; @@ -41,6 +41,7 @@ import org.apache.commons.io.filefilter.TrueFileFilter; import org.slf4j.helpers.FormattingTuple; import org.slf4j.helpers.MessageFormatter; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Component; import java.io.File; @@ -58,16 +59,16 @@ public class LogExportFileCleanTask { private final LogExportConfig logExportConfig; private final ArtifactoryConfig artifactoryConfig; - private final StorageSystemConfig storageSystemConfig; + private final StorageConfig storageConfig; private final ArtifactoryClient artifactoryClient; public LogExportFileCleanTask(LogExportConfig logExportConfig, ArtifactoryConfig artifactoryConfig, - StorageSystemConfig storageSystemConfig, - ArtifactoryClient artifactoryClient) { + StorageConfig storageConfig, + @Qualifier("jobArtifactoryClient") ArtifactoryClient artifactoryClient) { this.logExportConfig = logExportConfig; this.artifactoryConfig = artifactoryConfig; - this.storageSystemConfig = storageSystemConfig; + this.storageConfig = storageConfig; this.artifactoryClient = artifactoryClient; } @@ -160,7 +161,7 @@ private void cleanLocalLogExportFile() { System.currentTimeMillis() - logExportConfig.getArtifactoryFileExpireDays() * 3600 * 24 * 1000 ); String logExportFileDirPath = PathUtil.joinFilePath( - storageSystemConfig.getJobStorageRootPath(), Consts.LOG_EXPORT_DIR_NAME + storageConfig.getJobStorageRootPath(), Consts.LOG_EXPORT_DIR_NAME ); Iterator fileIterator = FileUtils.iterateFiles( new File(logExportFileDirPath), diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/task/ScheduledTasks.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/task/ScheduledTasks.java index cc427c7009..f43a722ea3 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/task/ScheduledTasks.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/task/ScheduledTasks.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -34,7 +34,7 @@ * 增加调度需要注意到ScheduleConfig中更新线程池配置 */ @Slf4j -@Component +@Component("jobExecuteScheduledTasks") @EnableScheduling public class ScheduledTasks { diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/util/ContextExecutorService.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/util/ContextExecutorService.java new file mode 100644 index 0000000000..87fffd0d22 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/util/ContextExecutorService.java @@ -0,0 +1,150 @@ +package com.tencent.bk.job.execute.util; + +import com.tencent.bk.job.execute.common.context.JobExecuteContext; +import com.tencent.bk.job.execute.common.context.JobExecuteContextThreadLocalRepo; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; + +/** + * 支持上下文传播的 ExecutorService + */ +public class ContextExecutorService implements ExecutorService { + + private final ExecutorService delegate; + + private ContextExecutorService(ExecutorService delegate) { + this.delegate = delegate; + } + + public static ContextExecutorService wrap(ExecutorService delegate) { + return new ContextExecutorService(delegate); + } + + + public Future submit(Callable task) { + return this.delegate.submit(ContextCallable.wrap(task)); + } + + public Future submit(Runnable task, T result) { + return this.delegate.submit(ContextRunnable.wrap(task), result); + } + + public Future submit(Runnable task) { + return this.delegate.submit(ContextRunnable.wrap(task)); + } + + public List> invokeAll(Collection> tasks) throws InterruptedException { + return this.delegate.invokeAll(ContextCallable.wrap(tasks)); + } + + public List> invokeAll(Collection> tasks, long timeout, + TimeUnit unit) throws InterruptedException { + return this.delegate.invokeAll(ContextCallable.wrap(tasks), timeout, unit); + } + + public T invokeAny(Collection> tasks) throws InterruptedException, ExecutionException { + return this.delegate.invokeAny(ContextCallable.wrap(tasks)); + } + + public T invokeAny(Collection> tasks, long timeout, + TimeUnit unit) throws InterruptedException, ExecutionException, TimeoutException { + return this.delegate.invokeAny(ContextCallable.wrap(tasks), timeout, unit); + } + + public void execute(Runnable command) { + this.delegate.execute(ContextRunnable.wrap(command)); + } + + public final void shutdown() { + this.delegate.shutdown(); + } + + public final List shutdownNow() { + return this.delegate.shutdownNow(); + } + + public final boolean isShutdown() { + return this.delegate.isShutdown(); + } + + public final boolean isTerminated() { + return this.delegate.isTerminated(); + } + + public final boolean awaitTermination(long timeout, TimeUnit unit) throws InterruptedException { + return this.delegate.awaitTermination(timeout, unit); + } + + private static class ContextCallable implements Callable { + + private final JobExecuteContext context; + private final Callable delegate; + + + private ContextCallable(Callable delegate) { + this.context = JobExecuteContextThreadLocalRepo.get(); + this.delegate = delegate; + } + + public static ContextCallable wrap(Callable delegate) { + return new ContextCallable<>(delegate); + } + + public static List> wrap(Collection> delegates) { + List> contextCallables = new ArrayList<>(delegates.size()); + delegates.forEach(delegate -> contextCallables.add(new ContextCallable<>(delegate))); + return contextCallables; + } + + @Override + public T call() throws Exception { + try { + JobExecuteContextThreadLocalRepo.set(context); + return delegate.call(); + } finally { + JobExecuteContextThreadLocalRepo.unset(); + } + } + } + + private static class ContextRunnable implements Runnable { + + private final JobExecuteContext context; + private final Runnable delegate; + + + private ContextRunnable(Runnable delegate) { + this.context = JobExecuteContextThreadLocalRepo.get(); + this.delegate = delegate; + } + + public static ContextRunnable wrap(Runnable delegate) { + return new ContextRunnable(delegate); + } + + public static List wrap(Collection delegates) { + List contextRunnableList = new ArrayList<>(delegates.size()); + delegates.forEach(delegate -> contextRunnableList.add(new ContextRunnable(delegate))); + return contextRunnableList; + } + + @Override + public void run() { + try { + JobExecuteContextThreadLocalRepo.set(context); + delegate.run(); + } finally { + JobExecuteContextThreadLocalRepo.unset(); + } + } + } +} + diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/util/ExecuteObjectCompositeKeyUtils.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/util/ExecuteObjectCompositeKeyUtils.java new file mode 100644 index 0000000000..9e0beac93c --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/util/ExecuteObjectCompositeKeyUtils.java @@ -0,0 +1,86 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.util; + +import com.tencent.bk.job.common.constant.ExecuteObjectTypeEnum; +import com.tencent.bk.job.common.model.openapi.v4.OpenApiExecuteObjectDTO; +import com.tencent.bk.job.common.model.openapi.v4.OpenApiHostDTO; +import com.tencent.bk.job.common.util.ip.IpUtils; +import com.tencent.bk.job.execute.model.ExecuteObjectCompositeKey; +import org.apache.commons.collections4.CollectionUtils; + +import java.util.List; +import java.util.stream.Collectors; + +public class ExecuteObjectCompositeKeyUtils { + + public static List fromEsbHostParams(List hostIds, + List openApiHostDTOList) { + if (CollectionUtils.isNotEmpty(hostIds)) { + // hostId方式作为主机标识 + return hostIds.stream() + .map(ExecuteObjectCompositeKey::ofHostId) + .collect(Collectors.toList()); + } else if (CollectionUtils.isNotEmpty(openApiHostDTOList)) { + return openApiHostDTOList.stream() + .map(openApiHostDTO -> { + if (openApiHostDTO.getHostId() != null) { + // hostId方式作为主机标识 + return ExecuteObjectCompositeKey.ofHostId(openApiHostDTO.getHostId()); + } else { + // 管控区域+ip方式作为主机标识 + return ExecuteObjectCompositeKey.ofHostIp( + IpUtils.buildCloudIp(openApiHostDTO.getBkCloudId(), openApiHostDTO.getIp())); + } + } + ).collect(Collectors.toList()); + } else { + throw new IllegalArgumentException("Invalid host params"); + } + } + + + public static ExecuteObjectCompositeKey fromHostParam(Long hostId, Long bkCloudId, String ip) { + if (hostId != null) { + // hostId方式作为主机标识 + return ExecuteObjectCompositeKey.ofHostId(hostId); + } else if (bkCloudId != null && ip != null) { + // 管控区域+ip方式作为主机标识 + return ExecuteObjectCompositeKey.ofHostIp(IpUtils.buildCloudIp(bkCloudId, ip)); + } else { + throw new IllegalArgumentException("Invalid host params"); + } + } + + public static List fromOpenApiExecuteObjectDTOList( + List executeObjects) { + + return executeObjects.stream() + .map(executeObject -> + ExecuteObjectCompositeKey.ofExecuteObjectResource( + ExecuteObjectTypeEnum.valOf(executeObject.getType()), executeObject.getResourceId())) + .collect(Collectors.toList()); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/util/FileTransferModeUtil.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/util/FileTransferModeUtil.java new file mode 100644 index 0000000000..b1f6c23a76 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/util/FileTransferModeUtil.java @@ -0,0 +1,48 @@ +package com.tencent.bk.job.execute.util; + +import com.tencent.bk.job.common.constant.DuplicateHandlerEnum; +import com.tencent.bk.job.common.constant.NotExistPathHandlerEnum; +import com.tencent.bk.job.execute.common.constants.FileTransferModeEnum; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +import static com.tencent.bk.job.common.constant.DuplicateHandlerEnum.GROUP_BY_DATE_AND_IP; +import static com.tencent.bk.job.common.constant.DuplicateHandlerEnum.GROUP_BY_IP; +import static com.tencent.bk.job.common.constant.DuplicateHandlerEnum.OVERWRITE; +import static com.tencent.bk.job.common.constant.NotExistPathHandlerEnum.CREATE_DIR; +import static com.tencent.bk.job.common.constant.NotExistPathHandlerEnum.STEP_FAIL; + +@Slf4j +@Service +public class FileTransferModeUtil { + + public static FileTransferModeEnum getTransferMode(Integer duplicateHandler, Integer notExistPathHandler) { + return getTransferMode( + DuplicateHandlerEnum.valueOf(duplicateHandler), + NotExistPathHandlerEnum.valueOf(notExistPathHandler) + ); + } + + private static FileTransferModeEnum getTransferMode(DuplicateHandlerEnum duplicateHandlerEnum, + NotExistPathHandlerEnum notExistPathHandlerEnum) { + if (duplicateHandlerEnum == null) { + // 默认覆盖 + duplicateHandlerEnum = OVERWRITE; + } + if (notExistPathHandlerEnum == null) { + // 默认直接创建 + notExistPathHandlerEnum = CREATE_DIR; + } + if (OVERWRITE == duplicateHandlerEnum && STEP_FAIL == notExistPathHandlerEnum) { + return FileTransferModeEnum.STRICT; + } else if (OVERWRITE == duplicateHandlerEnum && CREATE_DIR == notExistPathHandlerEnum) { + return FileTransferModeEnum.FORCE; + } else if (GROUP_BY_IP == duplicateHandlerEnum && CREATE_DIR == notExistPathHandlerEnum) { + return FileTransferModeEnum.SAFETY_IP_PREFIX; + } else if (GROUP_BY_DATE_AND_IP == duplicateHandlerEnum && CREATE_DIR == notExistPathHandlerEnum) { + return FileTransferModeEnum.SAFETY_DATE_PREFIX; + } else { + return FileTransferModeEnum.STRICT; + } + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/util/LoggerFactory.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/util/LoggerFactory.java index 44531c4c1e..484fb3550d 100644 --- a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/util/LoggerFactory.java +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/util/LoggerFactory.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/util/label/selector/LabelSelectorParse.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/util/label/selector/LabelSelectorParse.java new file mode 100644 index 0000000000..16db65ef51 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/util/label/selector/LabelSelectorParse.java @@ -0,0 +1,86 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.util.label.selector; + +import com.tencent.bk.job.common.constant.LabelSelectorOperatorEnum; +import com.tencent.bk.job.execute.model.LabelSelectExprDTO; +import org.apache.commons.collections4.CollectionUtils; + +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Kubernetes label selector 运算表达式解析。参考 Kubernetes 官方开源代码： + *

+ * https://github.com/kubernetes/kubernetes/blob/master/staging/src/k8s.io/apimachinery/pkg/labels/selector.go + */ +public class LabelSelectorParse { + + public static List parse(String selector) { + Parser parser = new Parser(selector); + return parser.parse(); + } + + public static List parseToLabelSelectExprList(String selector) { + Parser parser = new Parser(selector); + List requirements = parser.parse(); + + if (CollectionUtils.isEmpty(requirements)) { + return Collections.emptyList(); + } + + return requirements.stream() + .map(requirement -> new LabelSelectExprDTO( + requirement.getKey(), + mapToLabelSelectorOperatorEnum(requirement.getOperator()), + requirement.getValues())) + .collect(Collectors.toList()); + } + + private static LabelSelectorOperatorEnum mapToLabelSelectorOperatorEnum(Operator operator) { + switch (operator) { + case Equals: + case DoubleEquals: + return LabelSelectorOperatorEnum.EQUALS; + case In: + return LabelSelectorOperatorEnum.IN; + case NotIn: + return LabelSelectorOperatorEnum.NOT_IN; + case Exists: + return LabelSelectorOperatorEnum.EXISTS; + case DoesNotExist: + return LabelSelectorOperatorEnum.NOT_EXISTS; + case NotEquals: + return LabelSelectorOperatorEnum.NOT_EQUALS; + case GreaterThan: + return LabelSelectorOperatorEnum.GREATER_THAN; + case LessThan: + return LabelSelectorOperatorEnum.LESS_THAN; + default: + throw new IllegalArgumentException("Invalid operator: " + operator); + } + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/util/label/selector/LabelSelectorParseException.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/util/label/selector/LabelSelectorParseException.java new file mode 100644 index 0000000000..671ceefbd9 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/util/label/selector/LabelSelectorParseException.java @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.util.label.selector; + +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.InternalException; + +/** + * Label selector 解析异常 + */ +public class LabelSelectorParseException extends InternalException { + + public LabelSelectorParseException(String message) { + super(message, null, ErrorCode.INVALID_LABEL_SELECTOR); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/util/label/selector/LabelValidator.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/util/label/selector/LabelValidator.java new file mode 100644 index 0000000000..546c5a74c5 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/util/label/selector/LabelValidator.java @@ -0,0 +1,170 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.util.label.selector; + +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Pattern; + +public class LabelValidator { + + private static final String dns1123LabelFmt = "[a-z0-9]([-a-z0-9]*[a-z0-9])?"; + private static final String dns1123SubdomainFmt = dns1123LabelFmt + "(\\." + dns1123LabelFmt + ")*"; + private static final String dns1123SubdomainErrorMsg = "a lowercase RFC 1123 subdomain must consist of lower case" + + " alphanumeric characters, '-' or '.', and must start and end with an alphanumeric character"; + private static final int DNS1123SubdomainMaxLength = 253; + private static final Pattern dns1123SubdomainRegexp = Pattern.compile("^" + dns1123SubdomainFmt + "$"); + private static final int qualifiedNameMaxLength = 63; + private static final String qnameCharFmt = "[a-zA-Z0-9]"; + private static final String qnameExtCharFmt = "[-A-Za-z0-9_.]"; + private static final String qualifiedNameFmt = "(" + qnameCharFmt + qnameExtCharFmt + "*)?" + qnameCharFmt; + private static final String qualifiedNameErrMsg = "must consist of alphanumeric characters, '-', '_' or '.', and " + + "must start and end with an alphanumeric character"; + private static final Pattern qualifiedNameRegexp = Pattern.compile("^" + qualifiedNameFmt + "$"); + + + private static final int LabelValueMaxLength = 63; + private static final String labelValueFmt = "(" + qualifiedNameFmt + ")?"; + private static final Pattern labelValueRegexp = Pattern.compile("^" + labelValueFmt + "$"); + private static final String labelValueErrMsg = "a valid label must be an empty string or consist of alphanumeric " + + "characters, '-', '_' or '.', and must start and end with an alphanumeric character"; + + /** + * 校验 label key,并返回校验错误信息 + * + * @param labelKey label key + * @return 校验错误信息 + */ + public static List validateLabelKey(String labelKey) { + return isQualifiedName(labelKey); + } + + /** + * IsQualifiedName tests whether the value passed is what Kubernetes calls a + * "qualified name". This is a format used in various places throughout the + * system. If the value is not valid, a list of error strings is returned. + * Otherwise an empty list (or nil) is returned. + */ + private static List isQualifiedName(String value) { + List errs = new ArrayList<>(); + String[] parts = value.split("/"); + String name; + switch (parts.length) { + case 1: + name = parts[0]; + break; + case 2: + String prefix = parts[0]; + name = parts[1]; + if (prefix.length() == 0) { + errs.add("prefix part must be " + emptyError()); + } else { + List msgs = isDNS1123Subdomain(prefix); + if (msgs.size() != 0) { + errs.addAll(prefixEach(msgs, "prefix part ")); + } + } + break; + default: + errs.add("a qualified name " + regexError(qualifiedNameErrMsg, qualifiedNameFmt, "MyName", "my.name", + "123-abc") + + " with an optional DNS subdomain prefix and '/' (e.g. 'example.com/MyName')"); + return errs; + } + if (name.length() == 0) { + errs.add("name part " + emptyError()); + } else if (name.length() > qualifiedNameMaxLength) { + errs.add("name part " + maxLenError(qualifiedNameMaxLength)); + } + if (!qualifiedNameRegexp.matcher(name).matches()) { + errs.add("name part " + regexError(qualifiedNameErrMsg, qualifiedNameFmt, "MyName", "my.name", "123-abc")); + } + return errs; + } + + + private static List isDNS1123Subdomain(String value) { + List errs = new ArrayList<>(); + if (value.length() > DNS1123SubdomainMaxLength) { + errs.add(maxLenError(DNS1123SubdomainMaxLength)); + } + if (!dns1123SubdomainRegexp.matcher(value).matches()) { + errs.add(regexError(dns1123SubdomainErrorMsg, dns1123SubdomainFmt, "example.com")); + } + return errs; + } + + private static String maxLenError(int length) { + return String.format("must be no more than %d characters", length); + } + + private static String regexError(String msg, String fmt, String... examples) { + if (examples.length == 0) { + return msg + " (regex used for validation is '" + fmt + "')"; + } + msg += " (e.g. "; + StringBuilder msgBuilder = new StringBuilder(msg); + for (int i = 0; i < examples.length; i++) { + if (i > 0) { + msgBuilder.append(" or "); + } + msgBuilder.append("'").append(examples[i]).append("', "); + } + msg = msgBuilder.toString(); + msg += "regex used for validation is '" + fmt + "')"; + return msg; + } + + private static String emptyError() { + return "must be non-empty"; + } + + private static List prefixEach(List msgs, String prefix) { + for (int i = 0; i < msgs.size(); i++) { + msgs.set(i, prefix + msgs.get(i)); + } + return msgs; + } + + /** + * 校验 label value,并返回校验错误信息 + * + * @param value label value + * @return 校验错误信息 + */ + public static List validateLabelValue(String value) { + List errs = new ArrayList<>(); + + if (value.length() > LabelValueMaxLength) { + errs.add(maxLenError(LabelValueMaxLength)); + } + + if (!labelValueRegexp.matcher(value).matches()) { + errs.add(regexError(labelValueErrMsg, labelValueFmt, "MyValue", "my_value", "12345")); + } + + return errs; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/util/label/selector/Lexer.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/util/label/selector/Lexer.java new file mode 100644 index 0000000000..30a0a2233b --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/util/label/selector/Lexer.java @@ -0,0 +1,171 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.util.label.selector; + +/** + * Lexer represents the Lexer struct for label selector. + * It contains necessary information to tokenize the input string + */ +public class Lexer { + /** + * s stores the string to be tokenized + */ + private final String s; + /** + * pos is the position currently tokenized + */ + private int pos; + + public Lexer(String s) { + this.s = s; + this.pos = 0; + } + + /** + * read returns the character currently lexed + * increment the position and check the buffer overflow + */ + private char read() { + char ch = 0; + if (pos < s.length()) { + ch = s.charAt(pos); + pos++; + } + return ch; + } + + /** + * unread 'undoes' the last read character + */ + private void unread() { + pos--; + } + + /** + * isWhitespace returns true if the rune is a space, tab, or newline + */ + private boolean isWhitespace(char ch) { + return ch == ' ' || ch == '\t' || ch == '\r' || ch == '\n'; + } + + /** + * isSpecialSymbol detects if the character ch can be an operator + */ + private boolean isSpecialSymbol(char ch) { + switch (ch) { + case '=': + case '!': + case '(': + case ')': + case ',': + case '>': + case '<': + return true; + } + return false; + } + + public ScannedItem lex() { + char ch = skipWhiteSpaces(read()); + if (ch == 0) { + return new ScannedItem(Token.EndOfStringToken, ""); + } + if (isSpecialSymbol(ch)) { + unread(); + return scanSpecialSymbol(); + } else { + unread(); + return scanIDOrKeyword(); + } + } + + /** + * scanIDOrKeyword scans string to recognize literal token (for example 'in') or an identifier. + */ + private ScannedItem scanIDOrKeyword() { + StringBuilder buffer = new StringBuilder(); + while (true) { + char ch = read(); + if (ch == 0) { + break; + } + if (isSpecialSymbol(ch) || isWhitespace(ch)) { + unread(); + break; + } else { + buffer.append(ch); + } + } + String s = buffer.toString(); + if (Token.string2token.containsKey(s)) { + return new ScannedItem(Token.string2token.get(s), s); + } + return new ScannedItem(Token.IdentifierToken, s); + } + + /** + * scanSpecialSymbol scans string starting with special symbol. + * special symbol identify non literal operators. "!=", "==", "=" + */ + private ScannedItem scanSpecialSymbol() { + ScannedItem lastScannedItem = new ScannedItem(); + StringBuilder buffer = new StringBuilder(); + while (true) { + char ch = read(); + if (ch == 0) { + break; + } + if (isSpecialSymbol(ch)) { + buffer.append(ch); + if (Token.string2token.containsKey(buffer.toString())) { + lastScannedItem = new ScannedItem(Token.string2token.get(buffer.toString()), buffer.toString()); + } else if (lastScannedItem.getToken() != null) { + unread(); + break; + } + } else { + unread(); + break; + } + } + if (lastScannedItem.getToken() == null) { + return new ScannedItem(Token.ErrorToken, ""); + } + return lastScannedItem; + } + + /** + * skipWhiteSpaces consumes all blank characters + * returning the first non blank character + */ + private char skipWhiteSpaces(char ch) { + while (true) { + if (!isWhitespace(ch)) { + return ch; + } + ch = read(); + } + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/util/label/selector/Operator.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/util/label/selector/Operator.java new file mode 100644 index 0000000000..641a16845b --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/util/label/selector/Operator.java @@ -0,0 +1,68 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.util.label.selector; + +import java.util.Arrays; +import java.util.List; +import java.util.stream.Collectors; + +/** + * Label selector 操作符 + */ +public enum Operator { + DoesNotExist("!"), + Equals("="), + DoubleEquals("=="), + In("in"), + NotEquals("!="), + NotIn("notin"), + Exists("exists"), + GreaterThan("gt"), + LessThan("lt"); + + private final String symbol; + + Operator(String symbol) { + this.symbol = symbol; + } + + public String getSymbol() { + return symbol; + } + + public static Operator valOf(String operator) { + for (Operator operatorEnum : values()) { + if (operatorEnum.getSymbol().equals(operator)) { + return operatorEnum; + } + } + throw new IllegalArgumentException("No Operator constant: " + operator); + } + + public static List allOperators() { + return Arrays.stream(values()).map(Operator::getSymbol).collect(Collectors.toList()); + } + +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/util/label/selector/Parser.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/util/label/selector/Parser.java new file mode 100644 index 0000000000..c6648ac8a1 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/util/label/selector/Parser.java @@ -0,0 +1,403 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.util.label.selector; + +import com.tencent.bk.job.common.validation.FieldError; +import com.tencent.bk.job.common.validation.FieldErrors; +import com.tencent.bk.job.common.validation.Path; +import lombok.Data; +import org.apache.commons.collections4.CollectionUtils; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +import static com.tencent.bk.job.execute.util.label.selector.ParserContext.Values; + +/** + * Kubernetes label selector 解析 + */ +public class Parser { + private final Lexer lexer; + private final List scannedItems; + private int position; + private final Path rootPath; + + public Parser(String selector) { + this.lexer = new Lexer(selector); + this.scannedItems = new ArrayList<>(); + this.position = 0; + rootPath = Path.newPath("labelSelector"); + } + + /** + * scan runs through the input string and stores the ScannedItem in an array + * Parser can now lookahead and consume the tokens + */ + private void scan() { + while (true) { + ScannedItem scannedItem = lexer.lex(); + scannedItems.add(scannedItem); + if (scannedItem.getToken() == Token.EndOfStringToken) { + break; + } + } + } + + /** + * lookahead func returns the current token and string. No increment of current position + */ + private ScannedItem lookahead(ParserContext context) { + Token token = scannedItems.get(position).getToken(); + String literal = scannedItems.get(position).getLiteral(); + if (context == Values) { + switch (token) { + case InToken: + case NotInToken: + token = Token.IdentifierToken; + } + } + return new ScannedItem(token, literal); + } + + /** + * consume returns current token and string. Increments the position + */ + private ScannedItem consume(ParserContext context) { + position++; + Token token = scannedItems.get(position - 1).getToken(); + String literal = scannedItems.get(position - 1).getLiteral(); + if (context == Values) { + switch (token) { + case InToken: + case NotInToken: + token = Token.IdentifierToken; + } + } + return new ScannedItem(token, literal); + } + + public List parse() throws LabelSelectorParseException { + scan(); + + List requirements = new ArrayList<>(); + while (true) { + ScannedItem scannedItem = lookahead(Values); + Token token = scannedItem.getToken(); + switch (token) { + case IdentifierToken: + case DoesNotExistToken: + Requirement requirement = parseRequirement(); + requirements.add(requirement); + ScannedItem nextScannedItem = consume(Values); + Token nextToken = nextScannedItem.getToken(); + switch (nextToken) { + case EndOfStringToken: + return requirements; + case CommaToken: + ScannedItem nextScannedItem2 = lookahead(Values); + Token nextToken2 = nextScannedItem2.getToken(); + if (nextToken2 != Token.IdentifierToken && nextToken2 != Token.DoesNotExistToken) { + throw new LabelSelectorParseException( + "found '" + nextScannedItem2.getLiteral() + "', expected: identifier after ','"); + } + break; + default: + throw new LabelSelectorParseException("found '" + nextScannedItem.getLiteral() + + "', expected: ',' or 'end of string'"); + } + break; + case EndOfStringToken: + return requirements; + default: + throw new LabelSelectorParseException("found '" + scannedItems.get(position).getLiteral() + + "', expected: !, identifier, or 'end of string'"); + } + } + } + + private Requirement parseRequirement() throws LabelSelectorParseException { + KeyAndOperator keyAndOperator = parseKeyAndInferOperator(); + String key = keyAndOperator.getKey(); + Operator operator = keyAndOperator.getOperator(); + + if (operator == Operator.Exists || operator == Operator.DoesNotExist) { + return newRequirement(key, operator, null); + } + + Operator op = parseOperator(); + List values; + switch (op) { + case In: + case NotIn: + values = parseValues(); + break; + case Equals: + case DoubleEquals: + case NotEquals: + case GreaterThan: + case LessThan: + values = parseExactValue(); + break; + default: + throw new LabelSelectorParseException("found '" + scannedItems.get(position).getLiteral() + + "', expected: " + Arrays.stream(Operator.values()) + .map(Operator::getSymbol).collect(Collectors.joining(", "))); + } + return newRequirement(key, op, values); + } + + private Requirement newRequirement(String key, Operator op, List vals) { + FieldErrors fieldErrors = new FieldErrors(); + + List labelKeyValidateErrors = LabelValidator.validateLabelKey(key); + if (CollectionUtils.isNotEmpty(labelKeyValidateErrors)) { + fieldErrors.add(FieldError.invalid(rootPath.child("key"), vals, + String.join("; ", labelKeyValidateErrors))); + } + + Path valuePath = rootPath.child("values"); + switch (op) { + case In: + case NotIn: + if (vals.size() == 0) { + fieldErrors.add(FieldError.invalid(valuePath, vals, + "for 'in', 'notin' operators, values set can't be empty")); + } + break; + case Equals: + case DoubleEquals: + case NotEquals: + if (vals.size() != 1) { + fieldErrors.add(FieldError.invalid(valuePath, vals, + "exact-match compatibility requires one single value")); + } + break; + case Exists: + case DoesNotExist: + if (CollectionUtils.isNotEmpty(vals)) { + fieldErrors.add(FieldError.invalid(valuePath, vals, + "values set must be empty for exists and does not exist")); + } + break; + case GreaterThan: + case LessThan: + if (vals.size() != 1) { + fieldErrors.add(FieldError.invalid(valuePath, vals, + "for 'Gt', 'Lt' operators, exactly one value is required")); + } + for (int i = 0; i < vals.size(); i++) { + String val = vals.get(i); + try { + Long.parseLong(val); + } catch (NumberFormatException e) { + fieldErrors.add(FieldError.invalid(valuePath.index(i), val, + "for 'Gt', 'Lt' operators, the value must be an integer")); + } + } + break; + default: + fieldErrors.add(FieldError.notSupported(rootPath.child("operator"), op, + Operator.allOperators())); + } + + if (CollectionUtils.isNotEmpty(vals)) { + for (int i = 0; i < vals.size(); i++) { + String val = vals.get(i); + + List valueErrors = LabelValidator.validateLabelValue(val); + if (CollectionUtils.isNotEmpty(valueErrors)) { + fieldErrors.add(FieldError.invalid(valuePath.index(i), val, + String.join("; ", valueErrors))); + } + } + } + + if (fieldErrors.hasError()) { + throw new LabelSelectorParseException("Validate label selector fail, errors:" + fieldErrors.toString()); + } + return new Requirement(key, op, vals); + } + + private Operator parseOperator() throws LabelSelectorParseException { + ScannedItem scannedItem = consume(ParserContext.KeyAndOperator); + Token token = scannedItem.getToken(); + switch (token) { + case InToken: + return Operator.In; + case EqualsToken: + return Operator.Equals; + case DoubleEqualsToken: + return Operator.DoubleEquals; + case GreaterThanToken: + return Operator.GreaterThan; + case LessThanToken: + return Operator.LessThan; + case NotInToken: + return Operator.NotIn; + case NotEqualsToken: + return Operator.NotEquals; + default: + throw new LabelSelectorParseException("found '" + scannedItem.getLiteral() + + "', expected: " + Arrays.stream(Operator.values()) + .map(Operator::getSymbol).collect(Collectors.joining(", "))); + } + } + + private List parseValues() throws LabelSelectorParseException { + ScannedItem scannedItem = consume(Values); + Token token = scannedItem.getToken(); + if (token != Token.OpenParToken) { + throw new LabelSelectorParseException( + "found '" + scannedItem.getLiteral() + "' expected: '('"); + } + scannedItem = lookahead(Values); + token = scannedItem.getToken(); + switch (token) { + case IdentifierToken: + case CommaToken: + List values = parseIdentifiersList(); + token = consume(Values).getToken(); + if (token != Token.ClosedParToken) { + throw new LabelSelectorParseException( + "found '" + scannedItem.getLiteral() + "', expected: ')'"); + } + return values; + case ClosedParToken: + consume(Values); + return Collections.emptyList(); + default: + throw new LabelSelectorParseException( + "found '" + scannedItems.get(position).getLiteral() + "', expected: ',', ')' or identifier"); + } + } + + private List parseIdentifiersList() throws LabelSelectorParseException { + List values = new ArrayList<>(); + while (true) { + Token token = consume(Values).getToken(); + switch (token) { + case IdentifierToken: + values.add(scannedItems.get(position - 1).getLiteral()); + token = lookahead(Values).getToken(); + switch (token) { + case CommaToken: + continue; + case ClosedParToken: + return values; + default: + throw new LabelSelectorParseException("found '" + scannedItems.get(position).getLiteral() + + "', expected: ',' or ')'"); + } + case CommaToken: + if (values.isEmpty()) { + values.add(""); + } + Token token2 = lookahead(Values).getToken(); + if (token2 == Token.ClosedParToken) { + values.add(""); + return values; + } + if (token2 == Token.CommaToken) { + consume(Values); + values.add(""); + } + break; + default: + return values; + } + } + } + + private List parseExactValue() throws LabelSelectorParseException { + List values = new ArrayList<>(); + Token token = lookahead(Values).getToken(); + if (token == Token.EndOfStringToken || token == Token.CommaToken) { + values.add(""); + return values; + } + token = consume(Values).getToken(); + if (token == Token.IdentifierToken) { + values.add(scannedItems.get(position - 1).getLiteral()); + return values; + } + throw new LabelSelectorParseException("found '" + scannedItems.get(position).getLiteral() + + "', expected: identifier"); + } + + + /** + * parseKeyAndInferOperator parses literals. + * in case of no operator '!, in, notin, ==, =, !=' are found + * the 'exists' operator is inferred + */ + private KeyAndOperator parseKeyAndInferOperator() throws LabelSelectorParseException { + Operator operator = null; + ScannedItem scannedItem = consume(Values); + Token tok = scannedItem.getToken(); + String literal = scannedItem.getLiteral(); + + if (tok == Token.DoesNotExistToken) { + operator = Operator.DoesNotExist; + scannedItem = consume(Values); + tok = scannedItem.getToken(); + literal = scannedItem.getLiteral(); + } + + if (tok != Token.IdentifierToken) { + throw new LabelSelectorParseException(String.format("found '%s', expected: identifier", literal)); + } + + List labelKeyValidateErrors = LabelValidator.validateLabelKey(literal); + if (CollectionUtils.isNotEmpty(labelKeyValidateErrors)) { + throw new LabelSelectorParseException("Invalid label key, errors:" + + String.join("; ", labelKeyValidateErrors)); + } + + ScannedItem lookaheadScanItem = lookahead(Values); + Token t = lookaheadScanItem.getToken(); + + if (t == Token.EndOfStringToken || t == Token.CommaToken) { + if (operator != Operator.DoesNotExist) { + operator = Operator.Exists; + } + } + + return new KeyAndOperator(literal, operator); + } + + @Data + private static class KeyAndOperator { + private String key; + private Operator operator; + + public KeyAndOperator(String key, Operator operator) { + this.key = key; + this.operator = operator; + } + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/util/label/selector/ParserContext.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/util/label/selector/ParserContext.java new file mode 100644 index 0000000000..b78e1e7582 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/util/label/selector/ParserContext.java @@ -0,0 +1,36 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.util.label.selector; + +/** + * ParserContext represents context during parsing: + * some literal for example 'in' and 'notin' can be + * recognized as operator for example 'x in (a)' but + * it can be recognized as value for example 'value in (in)' + */ +public enum ParserContext { + KeyAndOperator, + Values +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/util/label/selector/Requirement.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/util/label/selector/Requirement.java new file mode 100644 index 0000000000..02d6867dcd --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/util/label/selector/Requirement.java @@ -0,0 +1,153 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.util.label.selector; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * Requirement contains values, a key, and an operator that relates the key and values. + */ +public class Requirement { + private final String key; + private final Operator operator; + private final List values; + + public Requirement(String key, Operator operator, List values) { + this.key = key; + this.operator = operator; + this.values = values; + } + + public String getKey() { + return key; + } + + public Operator getOperator() { + return operator; + } + + public List getValues() { + return values; + } + + public boolean matches(Map labels) { + if (!labels.containsKey(key)) { + return false; + } + String value = labels.get(key); + switch (operator) { + case In: + case Equals: + case DoubleEquals: + return values.contains(value); + case NotIn: + case NotEquals: + return !values.contains(value); + case Exists: + return true; + case GreaterThan: + case LessThan: + if (!values.isEmpty()) { + long labelValue = Long.parseLong(value); + long requirementValue = Long.parseLong(values.iterator().next()); + return (operator == Operator.GreaterThan && labelValue > requirementValue) || + (operator == Operator.LessThan && labelValue < requirementValue); + } + return false; + default: + return false; + } + } + + @Override + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(key); + switch (operator) { + case Equals: + sb.append("="); + break; + case DoubleEquals: + sb.append("=="); + break; + case NotEquals: + sb.append("!="); + break; + case In: + sb.append(" in "); + break; + case NotIn: + sb.append(" notin "); + break; + case GreaterThan: + sb.append(">"); + break; + case LessThan: + sb.append("<"); + break; + case Exists: + case DoesNotExist: + return sb.toString(); + } + if (operator == Operator.In || operator == Operator.NotIn) { + sb.append("("); + } + List sortedValues = new ArrayList<>(values); + Collections.sort(sortedValues); + sb.append(String.join(",", sortedValues)); + if (operator == Operator.In || operator == Operator.NotIn) { + sb.append(")"); + } + return sb.toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + Requirement that = (Requirement) o; + return key.equals(that.key) && + operator == that.operator && listEquals(values, that.values); + } + + private boolean listEquals(List thisList, List thatList) { + if (thisList == null && thatList == null) { + return true; + } else if (thisList != null && thatList != null) { + return thisList.equals(thatList); + } else { + return false; + } + } + + @Override + public int hashCode() { + return Objects.hash(key, operator, values); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/util/label/selector/ScannedItem.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/util/label/selector/ScannedItem.java new file mode 100644 index 0000000000..b1c6330ce1 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/util/label/selector/ScannedItem.java @@ -0,0 +1,51 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.util.label.selector; + +/** + * ScannedItem contains the Token and the literal produced by the lexer. + */ +public class ScannedItem { + private final Token token; + private final String literal; + + public ScannedItem() { + this.token = null; + this.literal = null; + } + + public ScannedItem(Token token, String literal) { + this.token = token; + this.literal = literal; + } + + public Token getToken() { + return token; + } + + public String getLiteral() { + return literal; + } +} diff --git a/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/util/label/selector/Token.java b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/util/label/selector/Token.java new file mode 100644 index 0000000000..abf1d172e2 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/java/com/tencent/bk/job/execute/util/label/selector/Token.java @@ -0,0 +1,62 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.util.label.selector; + +import java.util.HashMap; +import java.util.Map; + +/** + * lexical token + */ +public enum Token { + ErrorToken, + EndOfStringToken, + ClosedParToken, + CommaToken, + DoesNotExistToken, + DoubleEqualsToken, + EqualsToken, + GreaterThanToken, + IdentifierToken, + InToken, + LessThanToken, + NotEqualsToken, + NotInToken, + OpenParToken; + + public static final Map string2token = new HashMap() {{ + put(")", ClosedParToken); + put(",", CommaToken); + put("!", DoesNotExistToken); + put("==", DoubleEqualsToken); + put("=", EqualsToken); + put(">", GreaterThanToken); + put("in", InToken); + put("<", LessThanToken); + put("!=", NotEqualsToken); + put("notin", NotInToken); + put("(", OpenParToken); + }}; +} diff --git a/src/backend/job-execute/service-job-execute/src/main/resources/i18n/message.properties b/src/backend/job-execute/service-job-execute/src/main/resources/i18n/message.properties new file mode 100644 index 0000000000..6832ae0a07 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/resources/i18n/message.properties @@ -0,0 +1,120 @@ +# +# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. +# +# Copyright (C) 2021 Tencent. All rights reserved. +# +# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. +# +# License for BK-JOB蓝鲸智云作业平台: +# -------------------------------------------------------------------- +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF +# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +task.startup.mode.normal=页面执行 +task.startup.mode.api=API 调用 +task.startup.mode.cron=定时执行 + +task.instance.type.normal=作业执行 +task.instance.type.script=脚本执行 +task.instance.type.file=文件分发 + +task.run.status.blank=等待执行 +task.run.status.running=正在执行 +task.run.status.success=执行成功 +task.run.status.fail=执行失败 +task.run.status.skipped=跳过 +task.run.status.ignore_error=忽略错误 +task.run.status.waiting_user=等待确认 +task.run.status.terminated=手动结束 +task.run.status.abnormal_state=状态异常 +task.run.status.stopping=强制终止中 +task.run.status.stop_success=强制终止成功 +task.run.status.confirm_terminated=确认终止 +task.run.status.abandoned=被丢弃 +task.run.status.rolling_waiting=滚动等待 + +agent.task.status.unknown_error=未知错误 +agent.task.status.agent_error=Agent 异常 +agent.task.status.last_success=上次已成功 +agent.task.status.waiting=等待执行 +agent.task.status.running=正在执行 +agent.task.status.success=执行成功 +agent.task.status.failed=任务失败 +agent.task.status.submit_failed=任务下发失败 +agent.task.status.task_timeout=任务超时 +agent.task.status.log_error=任务日志错误 +agent.task.status.gse_script_timeout=脚本日志超时 +agent.task.status.gse_file_timeout=文件日志超时 +agent.task.status.script_failed=脚本执行失败 +agent.task.status.script_timeout=脚本执行超时 +agent.task.status.script_terminate=脚本执行被终止 +agent.task.status.script_not_zero_exit_code=脚本返回码非零 +agent.task.status.copyfile_failed=文件传输失败 +agent.task.status.copyfile_source_file_not_exist=源文件不存在 +agent.task.status.file_error_unclassified=文件任务未分类错误 +agent.task.status.gse_timeout=执行任务超时 +agent.task.status.gse_agent_error=Agent 异常 +agent.task.status.gse_user_error=用户名不存在 +agent.task.status.gse_user_pwd_error=用户密码不正确 +agent.task.status.gse_file_error=文件获取失败 +agent.task.status.gse_file_size_exceed=文件超出限制 +agent.task.status.gse_file_task_error=文件传输错误 +agent.task.status.gse_task_error=任务执行出错 +agent.task.status.gse_task_terminate_success=任务强制终止成功 +agent.task.status.gse_task_terminate_failed=任务强制终止失败 +agent.task.status.invalid_execute_object=无效执行对象 +agent.task.status.unknown=未知 +agent.task.status.agent_not_installed=Agent 未安装 + +user.operation.retry_step_fail=失败重试 +user.operation.ignore_error=忽略错误 +user.operation.skip_step=手动跳过 +user.operation.terminate_job=强制终止 +user.operation.retry_step_all=全部重试 +user.operation.start=启动作业 +user.operation.confirm_continue=人工确认 +user.operation.confirm_terminate=人工确认 +user.operation.confirm_restart=人工确认 +user.operation.next_step=进入下一步 + +user.operation.detail.start.web=用户从页面点击启动 +user.operation.detail.start.cron=由定时任务调度启动 +user.operation.detail.start.api=由 {0} 调用 API 启动 +user.operation.detail.confirm.terminate=用户已确认中止任务，理由：{0} +user.operation.detail.confirm.continue=用户已确认继续执行，理由：{0} +user.operation.detail.confirm.restart=用户重新发起确认 +user.operation.detail.retry_step_all=点击查看重试前的执行详情 +user.operation.detail.retry_step_fail=点击查看重试前的执行详情 +user.operation.detail.ignore_error=已忽略执行失败的目标，任务继续执行 +user.operation.detail.next_step=用户已确认直接进入下一步 +user.operation.detail.terminate_job=强制终止 + +task.variable.type.name.string=字符串 +task.variable.type.name.namespace=命名空间 +task.variable.type.name.host_list=主机列表 +task.variable.type.name.cipher=密文 +task.variable.type.name.associative_array=关联数组 +task.variable.type.name.index_array=索引数组 + +task.step.type.name.script=执行脚本 +task.step.type.name.file=分发文件 +task.step.type.name.manual_confirm=人工确认 + +task.type.name.fast_execute_script=快速执行脚本 +task.type.name.fast_execute_sql=快速执行SQL +task.type.name.fast_push_file=快速分发文件 +task.type.name.job=执行方案 + +script.check.result.detect.dangerous.script=高危脚本 diff --git a/src/backend/job-execute/service-job-execute/src/main/resources/i18n/message_en.properties b/src/backend/job-execute/service-job-execute/src/main/resources/i18n/message_en.properties new file mode 100644 index 0000000000..ff1c11fd43 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/resources/i18n/message_en.properties @@ -0,0 +1,120 @@ +# +# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. +# +# Copyright (C) 2021 Tencent. All rights reserved. +# +# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. +# +# License for BK-JOB蓝鲸智云作业平台: +# -------------------------------------------------------------------- +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF +# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +task.startup.mode.normal=Web UI +task.startup.mode.api=API +task.startup.mode.cron=Cron + +task.instance.type.normal=Job Execution +task.instance.type.script=Script Execution +task.instance.type.file=File Transfer + +task.run.status.blank=Pending +task.run.status.running=Running +task.run.status.success=Success +task.run.status.fail=Failed +task.run.status.skipped=Skipped +task.run.status.ignore_error=Ignore Error +task.run.status.waiting_user=Waiting +task.run.status.terminated=Terminated +task.run.status.abnormal_state=Abnormal +task.run.status.stopping=Terminating +task.run.status.stop_success=Terminate Success +task.run.status.confirm_terminated=Termination Confirmed +task.run.status.abandoned=Abandoned +task.run.status.rolling_waiting=Rolling Waiting + +agent.task.status.unknown_error=Unknown Error +agent.task.status.agent_error=Abnormal Agent +agent.task.status.last_success=Successful Last Time +agent.task.status.waiting=Waiting +agent.task.status.running=Running +agent.task.status.success=Success +agent.task.status.failed=Failed +agent.task.status.submit_failed=Task send failed +agent.task.status.task_timeout=Task Timeout +agent.task.status.log_error=Error in Log +agent.task.status.gse_script_timeout=Script Log Timeout +agent.task.status.gse_file_timeout=File Log Timeout +agent.task.status.script_failed=Script Execution Failed +agent.task.status.script_timeout=Script Execution Timeout +agent.task.status.script_terminate=Script Execution Terminated +agent.task.status.script_not_zero_exit_code=Non-zero Exit Code +agent.task.status.copyfile_failed=File Transmission Failed +agent.task.status.copyfile_source_file_not_exist=Source File Not Exists +agent.task.status.file_error_unclassified=Unclassified File Failure +agent.task.status.gse_timeout=File Transmission Timeout +agent.task.status.gse_agent_error=Abnormal Agent +agent.task.status.gse_user_error=User not exist +agent.task.status.gse_user_pwd_error=User password incorrect +agent.task.status.gse_file_error=File pull failed +agent.task.status.gse_file_size_exceed=File size exceeds limit +agent.task.status.gse_file_task_error=File Transmission Error +agent.task.status.gse_task_error=Execution Error +agent.task.status.gse_task_terminate_success=Force Terminated +agent.task.status.gse_task_terminate_failed=Force Terminate Failed +agent.task.status.invalid_execute_object=Invalid Execute Object +agent.task.status.unknown=Unknown +agent.task.status.agent_not_installed=Agent not installed + +user.operation.retry_step_fail=Failed-host Retry +user.operation.ignore_error=Ignore Error +user.operation.skip_step=Skip +user.operation.terminate_job=Force Terminate +user.operation.retry_step_all=Retry All +user.operation.start=Launch +user.operation.confirm_continue=Confirm +user.operation.confirm_terminate=Confirm +user.operation.confirm_restart=Confirm +user.operation.next_step=Next step + +user.operation.detail.start.web=User launch it by Web UI. +user.operation.detail.start.cron=Launched by Cron Service. +user.operation.detail.start.api=Launched by {0} call API. +user.operation.detail.confirm.terminate=User has confirmed to refute the task, reason: {0}. +user.operation.detail.confirm.continue=User has confirmed to continue execution, reason: {0}. +user.operation.detail.confirm.restart=User re-initiates confirmation. +user.operation.detail.retry_step_all=Click here to view the execution details before retrying. +user.operation.detail.retry_step_fail=Click here to view the execution details before retrying. +user.operation.detail.ignore_error=Ignored Failed-host, and continues. +user.operation.detail.next_step=User has confirmed to go directly to the next step. +user.operation.detail.terminate_job=Task has been force terminated. + +task.variable.type.name.string=String +task.variable.type.name.namespace=Namespace +task.variable.type.name.host_list=Host +task.variable.type.name.cipher=Cipher +task.variable.type.name.associative_array=Associative array +task.variable.type.name.index_array=Index array + +task.step.type.name.script=Script Execution +task.step.type.name.file=File Transfer +task.step.type.name.manual_confirm=User Confirmation + +task.type.name.fast_execute_script=Script Execution +task.type.name.fast_execute_sql=SQL Script +task.type.name.fast_push_file=File Transfer +task.type.name.job=Job + +script.check.result.detect.dangerous.script=Dangerous script diff --git a/src/backend/job-execute/service-job-execute/src/main/resources/i18n/message_en_US.properties b/src/backend/job-execute/service-job-execute/src/main/resources/i18n/message_en_US.properties new file mode 100644 index 0000000000..ff1c11fd43 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/resources/i18n/message_en_US.properties @@ -0,0 +1,120 @@ +# +# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. +# +# Copyright (C) 2021 Tencent. All rights reserved. +# +# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. +# +# License for BK-JOB蓝鲸智云作业平台: +# -------------------------------------------------------------------- +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF +# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +task.startup.mode.normal=Web UI +task.startup.mode.api=API +task.startup.mode.cron=Cron + +task.instance.type.normal=Job Execution +task.instance.type.script=Script Execution +task.instance.type.file=File Transfer + +task.run.status.blank=Pending +task.run.status.running=Running +task.run.status.success=Success +task.run.status.fail=Failed +task.run.status.skipped=Skipped +task.run.status.ignore_error=Ignore Error +task.run.status.waiting_user=Waiting +task.run.status.terminated=Terminated +task.run.status.abnormal_state=Abnormal +task.run.status.stopping=Terminating +task.run.status.stop_success=Terminate Success +task.run.status.confirm_terminated=Termination Confirmed +task.run.status.abandoned=Abandoned +task.run.status.rolling_waiting=Rolling Waiting + +agent.task.status.unknown_error=Unknown Error +agent.task.status.agent_error=Abnormal Agent +agent.task.status.last_success=Successful Last Time +agent.task.status.waiting=Waiting +agent.task.status.running=Running +agent.task.status.success=Success +agent.task.status.failed=Failed +agent.task.status.submit_failed=Task send failed +agent.task.status.task_timeout=Task Timeout +agent.task.status.log_error=Error in Log +agent.task.status.gse_script_timeout=Script Log Timeout +agent.task.status.gse_file_timeout=File Log Timeout +agent.task.status.script_failed=Script Execution Failed +agent.task.status.script_timeout=Script Execution Timeout +agent.task.status.script_terminate=Script Execution Terminated +agent.task.status.script_not_zero_exit_code=Non-zero Exit Code +agent.task.status.copyfile_failed=File Transmission Failed +agent.task.status.copyfile_source_file_not_exist=Source File Not Exists +agent.task.status.file_error_unclassified=Unclassified File Failure +agent.task.status.gse_timeout=File Transmission Timeout +agent.task.status.gse_agent_error=Abnormal Agent +agent.task.status.gse_user_error=User not exist +agent.task.status.gse_user_pwd_error=User password incorrect +agent.task.status.gse_file_error=File pull failed +agent.task.status.gse_file_size_exceed=File size exceeds limit +agent.task.status.gse_file_task_error=File Transmission Error +agent.task.status.gse_task_error=Execution Error +agent.task.status.gse_task_terminate_success=Force Terminated +agent.task.status.gse_task_terminate_failed=Force Terminate Failed +agent.task.status.invalid_execute_object=Invalid Execute Object +agent.task.status.unknown=Unknown +agent.task.status.agent_not_installed=Agent not installed + +user.operation.retry_step_fail=Failed-host Retry +user.operation.ignore_error=Ignore Error +user.operation.skip_step=Skip +user.operation.terminate_job=Force Terminate +user.operation.retry_step_all=Retry All +user.operation.start=Launch +user.operation.confirm_continue=Confirm +user.operation.confirm_terminate=Confirm +user.operation.confirm_restart=Confirm +user.operation.next_step=Next step + +user.operation.detail.start.web=User launch it by Web UI. +user.operation.detail.start.cron=Launched by Cron Service. +user.operation.detail.start.api=Launched by {0} call API. +user.operation.detail.confirm.terminate=User has confirmed to refute the task, reason: {0}. +user.operation.detail.confirm.continue=User has confirmed to continue execution, reason: {0}. +user.operation.detail.confirm.restart=User re-initiates confirmation. +user.operation.detail.retry_step_all=Click here to view the execution details before retrying. +user.operation.detail.retry_step_fail=Click here to view the execution details before retrying. +user.operation.detail.ignore_error=Ignored Failed-host, and continues. +user.operation.detail.next_step=User has confirmed to go directly to the next step. +user.operation.detail.terminate_job=Task has been force terminated. + +task.variable.type.name.string=String +task.variable.type.name.namespace=Namespace +task.variable.type.name.host_list=Host +task.variable.type.name.cipher=Cipher +task.variable.type.name.associative_array=Associative array +task.variable.type.name.index_array=Index array + +task.step.type.name.script=Script Execution +task.step.type.name.file=File Transfer +task.step.type.name.manual_confirm=User Confirmation + +task.type.name.fast_execute_script=Script Execution +task.type.name.fast_execute_sql=SQL Script +task.type.name.fast_push_file=File Transfer +task.type.name.job=Job + +script.check.result.detect.dangerous.script=Dangerous script diff --git a/src/backend/job-execute/service-job-execute/src/main/resources/i18n/message_zh.properties b/src/backend/job-execute/service-job-execute/src/main/resources/i18n/message_zh.properties new file mode 100644 index 0000000000..6832ae0a07 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/resources/i18n/message_zh.properties @@ -0,0 +1,120 @@ +# +# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. +# +# Copyright (C) 2021 Tencent. All rights reserved. +# +# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. +# +# License for BK-JOB蓝鲸智云作业平台: +# -------------------------------------------------------------------- +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF +# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +task.startup.mode.normal=页面执行 +task.startup.mode.api=API 调用 +task.startup.mode.cron=定时执行 + +task.instance.type.normal=作业执行 +task.instance.type.script=脚本执行 +task.instance.type.file=文件分发 + +task.run.status.blank=等待执行 +task.run.status.running=正在执行 +task.run.status.success=执行成功 +task.run.status.fail=执行失败 +task.run.status.skipped=跳过 +task.run.status.ignore_error=忽略错误 +task.run.status.waiting_user=等待确认 +task.run.status.terminated=手动结束 +task.run.status.abnormal_state=状态异常 +task.run.status.stopping=强制终止中 +task.run.status.stop_success=强制终止成功 +task.run.status.confirm_terminated=确认终止 +task.run.status.abandoned=被丢弃 +task.run.status.rolling_waiting=滚动等待 + +agent.task.status.unknown_error=未知错误 +agent.task.status.agent_error=Agent 异常 +agent.task.status.last_success=上次已成功 +agent.task.status.waiting=等待执行 +agent.task.status.running=正在执行 +agent.task.status.success=执行成功 +agent.task.status.failed=任务失败 +agent.task.status.submit_failed=任务下发失败 +agent.task.status.task_timeout=任务超时 +agent.task.status.log_error=任务日志错误 +agent.task.status.gse_script_timeout=脚本日志超时 +agent.task.status.gse_file_timeout=文件日志超时 +agent.task.status.script_failed=脚本执行失败 +agent.task.status.script_timeout=脚本执行超时 +agent.task.status.script_terminate=脚本执行被终止 +agent.task.status.script_not_zero_exit_code=脚本返回码非零 +agent.task.status.copyfile_failed=文件传输失败 +agent.task.status.copyfile_source_file_not_exist=源文件不存在 +agent.task.status.file_error_unclassified=文件任务未分类错误 +agent.task.status.gse_timeout=执行任务超时 +agent.task.status.gse_agent_error=Agent 异常 +agent.task.status.gse_user_error=用户名不存在 +agent.task.status.gse_user_pwd_error=用户密码不正确 +agent.task.status.gse_file_error=文件获取失败 +agent.task.status.gse_file_size_exceed=文件超出限制 +agent.task.status.gse_file_task_error=文件传输错误 +agent.task.status.gse_task_error=任务执行出错 +agent.task.status.gse_task_terminate_success=任务强制终止成功 +agent.task.status.gse_task_terminate_failed=任务强制终止失败 +agent.task.status.invalid_execute_object=无效执行对象 +agent.task.status.unknown=未知 +agent.task.status.agent_not_installed=Agent 未安装 + +user.operation.retry_step_fail=失败重试 +user.operation.ignore_error=忽略错误 +user.operation.skip_step=手动跳过 +user.operation.terminate_job=强制终止 +user.operation.retry_step_all=全部重试 +user.operation.start=启动作业 +user.operation.confirm_continue=人工确认 +user.operation.confirm_terminate=人工确认 +user.operation.confirm_restart=人工确认 +user.operation.next_step=进入下一步 + +user.operation.detail.start.web=用户从页面点击启动 +user.operation.detail.start.cron=由定时任务调度启动 +user.operation.detail.start.api=由 {0} 调用 API 启动 +user.operation.detail.confirm.terminate=用户已确认中止任务，理由：{0} +user.operation.detail.confirm.continue=用户已确认继续执行，理由：{0} +user.operation.detail.confirm.restart=用户重新发起确认 +user.operation.detail.retry_step_all=点击查看重试前的执行详情 +user.operation.detail.retry_step_fail=点击查看重试前的执行详情 +user.operation.detail.ignore_error=已忽略执行失败的目标，任务继续执行 +user.operation.detail.next_step=用户已确认直接进入下一步 +user.operation.detail.terminate_job=强制终止 + +task.variable.type.name.string=字符串 +task.variable.type.name.namespace=命名空间 +task.variable.type.name.host_list=主机列表 +task.variable.type.name.cipher=密文 +task.variable.type.name.associative_array=关联数组 +task.variable.type.name.index_array=索引数组 + +task.step.type.name.script=执行脚本 +task.step.type.name.file=分发文件 +task.step.type.name.manual_confirm=人工确认 + +task.type.name.fast_execute_script=快速执行脚本 +task.type.name.fast_execute_sql=快速执行SQL +task.type.name.fast_push_file=快速分发文件 +task.type.name.job=执行方案 + +script.check.result.detect.dangerous.script=高危脚本 diff --git a/src/backend/job-execute/service-job-execute/src/main/resources/i18n/message_zh_CN.properties b/src/backend/job-execute/service-job-execute/src/main/resources/i18n/message_zh_CN.properties new file mode 100644 index 0000000000..6832ae0a07 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/resources/i18n/message_zh_CN.properties @@ -0,0 +1,120 @@ +# +# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. +# +# Copyright (C) 2021 Tencent. All rights reserved. +# +# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. +# +# License for BK-JOB蓝鲸智云作业平台: +# -------------------------------------------------------------------- +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF +# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +task.startup.mode.normal=页面执行 +task.startup.mode.api=API 调用 +task.startup.mode.cron=定时执行 + +task.instance.type.normal=作业执行 +task.instance.type.script=脚本执行 +task.instance.type.file=文件分发 + +task.run.status.blank=等待执行 +task.run.status.running=正在执行 +task.run.status.success=执行成功 +task.run.status.fail=执行失败 +task.run.status.skipped=跳过 +task.run.status.ignore_error=忽略错误 +task.run.status.waiting_user=等待确认 +task.run.status.terminated=手动结束 +task.run.status.abnormal_state=状态异常 +task.run.status.stopping=强制终止中 +task.run.status.stop_success=强制终止成功 +task.run.status.confirm_terminated=确认终止 +task.run.status.abandoned=被丢弃 +task.run.status.rolling_waiting=滚动等待 + +agent.task.status.unknown_error=未知错误 +agent.task.status.agent_error=Agent 异常 +agent.task.status.last_success=上次已成功 +agent.task.status.waiting=等待执行 +agent.task.status.running=正在执行 +agent.task.status.success=执行成功 +agent.task.status.failed=任务失败 +agent.task.status.submit_failed=任务下发失败 +agent.task.status.task_timeout=任务超时 +agent.task.status.log_error=任务日志错误 +agent.task.status.gse_script_timeout=脚本日志超时 +agent.task.status.gse_file_timeout=文件日志超时 +agent.task.status.script_failed=脚本执行失败 +agent.task.status.script_timeout=脚本执行超时 +agent.task.status.script_terminate=脚本执行被终止 +agent.task.status.script_not_zero_exit_code=脚本返回码非零 +agent.task.status.copyfile_failed=文件传输失败 +agent.task.status.copyfile_source_file_not_exist=源文件不存在 +agent.task.status.file_error_unclassified=文件任务未分类错误 +agent.task.status.gse_timeout=执行任务超时 +agent.task.status.gse_agent_error=Agent 异常 +agent.task.status.gse_user_error=用户名不存在 +agent.task.status.gse_user_pwd_error=用户密码不正确 +agent.task.status.gse_file_error=文件获取失败 +agent.task.status.gse_file_size_exceed=文件超出限制 +agent.task.status.gse_file_task_error=文件传输错误 +agent.task.status.gse_task_error=任务执行出错 +agent.task.status.gse_task_terminate_success=任务强制终止成功 +agent.task.status.gse_task_terminate_failed=任务强制终止失败 +agent.task.status.invalid_execute_object=无效执行对象 +agent.task.status.unknown=未知 +agent.task.status.agent_not_installed=Agent 未安装 + +user.operation.retry_step_fail=失败重试 +user.operation.ignore_error=忽略错误 +user.operation.skip_step=手动跳过 +user.operation.terminate_job=强制终止 +user.operation.retry_step_all=全部重试 +user.operation.start=启动作业 +user.operation.confirm_continue=人工确认 +user.operation.confirm_terminate=人工确认 +user.operation.confirm_restart=人工确认 +user.operation.next_step=进入下一步 + +user.operation.detail.start.web=用户从页面点击启动 +user.operation.detail.start.cron=由定时任务调度启动 +user.operation.detail.start.api=由 {0} 调用 API 启动 +user.operation.detail.confirm.terminate=用户已确认中止任务，理由：{0} +user.operation.detail.confirm.continue=用户已确认继续执行，理由：{0} +user.operation.detail.confirm.restart=用户重新发起确认 +user.operation.detail.retry_step_all=点击查看重试前的执行详情 +user.operation.detail.retry_step_fail=点击查看重试前的执行详情 +user.operation.detail.ignore_error=已忽略执行失败的目标，任务继续执行 +user.operation.detail.next_step=用户已确认直接进入下一步 +user.operation.detail.terminate_job=强制终止 + +task.variable.type.name.string=字符串 +task.variable.type.name.namespace=命名空间 +task.variable.type.name.host_list=主机列表 +task.variable.type.name.cipher=密文 +task.variable.type.name.associative_array=关联数组 +task.variable.type.name.index_array=索引数组 + +task.step.type.name.script=执行脚本 +task.step.type.name.file=分发文件 +task.step.type.name.manual_confirm=人工确认 + +task.type.name.fast_execute_script=快速执行脚本 +task.type.name.fast_execute_sql=快速执行SQL +task.type.name.fast_push_file=快速分发文件 +task.type.name.job=执行方案 + +script.check.result.detect.dangerous.script=高危脚本 diff --git a/src/backend/job-execute/service-job-execute/src/main/resources/lua/add_running_job.lua b/src/backend/job-execute/service-job-execute/src/main/resources/lua/add_running_job.lua new file mode 100644 index 0000000000..ea0b375479 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/resources/lua/add_running_job.lua @@ -0,0 +1,16 @@ +local system_running_job_zset_key = KEYS[1] +local resource_scope_job_count_hash_key = KEYS[2] +local app_job_count_hash_key = KEYS[3] +local job_id = ARGV[1] +local resource_scope = ARGV[2] +local app_code = ARGV[3] +local job_create_time = tonumber(ARGV[4]) + +local add_result = redis.call('zadd', system_running_job_zset_key, 'NX', job_create_time, job_id) +if add_result == 1 then + redis.call('hincrby', resource_scope_job_count_hash_key, resource_scope, 1) + + if app_code ~= "None" then + redis.call('hincrby', app_job_count_hash_key, app_code, 1) + end +end diff --git a/src/backend/job-execute/service-job-execute/src/main/resources/lua/check_running_job_quota_limit.lua b/src/backend/job-execute/service-job-execute/src/main/resources/lua/check_running_job_quota_limit.lua new file mode 100644 index 0000000000..2f7dd05e8f --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/resources/lua/check_running_job_quota_limit.lua @@ -0,0 +1,33 @@ +local system_running_job_zset_key = KEYS[1] +local resource_scope_job_count_hash_key = KEYS[2] +local app_job_count_hash_key = KEYS[3] +local resource_scope = ARGV[1] +local app_code = ARGV[2] + +local system_limit = tonumber(ARGV[3]) +if system_limit > 0 then + local system_count = tonumber(redis.call('zcard', system_running_job_zset_key) or "0") + if system_count >= system_limit then + return "system_quota_limit" + end +end + +local resource_scope_limit = tonumber(ARGV[4]) +if resource_scope_limit > 0 then + local resource_scope_count = tonumber(redis.call('hget', resource_scope_job_count_hash_key, resource_scope) or "0") + if resource_scope_count >= resource_scope_limit then + return "resource_scope_quota_limit" + end +end + +if app_code ~= "None" then + local app_limit = tonumber(ARGV[5]) + if app_limit > 0 then + local app_count = tonumber(redis.call('hget', app_job_count_hash_key, app_code) or "0") + if app_count >= app_limit then + return "app_quota_limit" + end + end +end + +return "no_limit" diff --git a/src/backend/job-execute/service-job-execute/src/main/resources/lua/remove_running_job.lua b/src/backend/job-execute/service-job-execute/src/main/resources/lua/remove_running_job.lua new file mode 100644 index 0000000000..4dae867b61 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/main/resources/lua/remove_running_job.lua @@ -0,0 +1,15 @@ +local system_running_job_zset_key = KEYS[1] +local resource_scope_job_count_hash_key = KEYS[2] +local app_job_count_hash_key = KEYS[3] +local job_id = ARGV[1] +local resource_scope = ARGV[2] +local app_code = ARGV[3] + +local del_result = redis.call('zrem', system_running_job_zset_key, job_id) +if del_result == 1 then + redis.call('hincrby', resource_scope_job_count_hash_key, resource_scope, -1) + + if app_code ~= "None" then + redis.call('hincrby', app_job_count_hash_key, app_code, -1) + end +end diff --git a/src/backend/job-execute/service-job-execute/src/main/resources/notification-template/notification_en.properties b/src/backend/job-execute/service-job-execute/src/main/resources/notification-template/notification_en.properties index 99afe7ca31..d40b8635cd 100644 --- a/src/backend/job-execute/service-job-execute/src/main/resources/notification-template/notification_en.properties +++ b/src/backend/job-execute/service-job-execute/src/main/resources/notification-template/notification_en.properties @@ -1,11 +1,11 @@ # -# Tencent is pleased to support the open source community by making BK-JOB����������ҵƽ̨ available. +# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. # -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +# Copyright (C) 2021 Tencent. All rights reserved. # -# BK-JOB����������ҵƽ̨ is licensed under the MIT License. +# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. # -# License for BK-JOB����������ҵƽ̨: +# License for BK-JOB蓝鲸智云作业平台: # -------------------------------------------------------------------- # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated # documentation files (the "Software"), to deal in the Software without restriction, including without limitation diff --git a/src/backend/job-execute/service-job-execute/src/main/resources/notification-template/notification_zh.properties b/src/backend/job-execute/service-job-execute/src/main/resources/notification-template/notification_zh.properties index 3279a6245b..15a773ea3e 100644 --- a/src/backend/job-execute/service-job-execute/src/main/resources/notification-template/notification_zh.properties +++ b/src/backend/job-execute/service-job-execute/src/main/resources/notification-template/notification_zh.properties @@ -1,7 +1,7 @@ # # Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. # -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +# Copyright (C) 2021 Tencent. All rights reserved. # # BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. # @@ -44,11 +44,11 @@ task.waiting.msg.content.template.common=任务名称：\u0020{task.name}\n\ 查看详情：\u0020{task.detail.url} ### 消息通知短信模板 task.fail.msg.title.template.sms=【蓝鲸作业平台】任务执行失败通知 -task.fail.msg.content.template.sms=您的业务[{task.app.name}]当前有一个任务:{task.name}(\u0020{task.detail.url}\u0020)执行失败，请尽快查看。【蓝鲸作业平台】 +task.fail.msg.content.template.sms=您的业务[{task.app.name}]当前有一个任务：{task.name}(\u0020{task.detail.url}\u0020)执行失败，请尽快查看。【蓝鲸作业平台】 task.success.msg.title.template.sms=【蓝鲸作业平台】任务执行成功通知 -task.success.msg.content.template.sms=您的业务[{task.app.name}]当前有一个任务:{task.name}(\u0020{task.detail.url}\u0020)已执行成功，请尽快查看。【蓝鲸作业平台】 +task.success.msg.content.template.sms=您的业务[{task.app.name}]当前有一个任务：{task.name}(\u0020{task.detail.url}\u0020)已执行成功，请尽快查看。【蓝鲸作业平台】 task.waiting.msg.title.template.sms=【蓝鲸作业平台】 -task.waiting.msg.content.template.sms=您的业务[{task.app.name}]当前有一个待确认的作业任务:{task.name}(\u0020{task.detail.url}\u0020)需要处理，请尽快操作。【蓝鲸作业平台】 +task.waiting.msg.content.template.sms=您的业务[{task.app.name}]当前有一个待确认的作业任务：{task.name}(\u0020{task.detail.url}\u0020)需要处理，请尽快操作。【蓝鲸作业平台】 ### 消息通知微信模板 task.fail.msg.title.template.weixin=【蓝鲸作业平台】任务执行失败通知 task.fail.msg.content.template.weixin=任务名称：\u0020{task.name}\n\ diff --git a/src/backend/job-execute/service-job-execute/src/main/resources/sqltask/mysql_exec_template.sh b/src/backend/job-execute/service-job-execute/src/main/resources/sqltask/mysql_exec_template.sh index 29b40e936e..9ee2e8251f 100644 --- a/src/backend/job-execute/service-job-execute/src/main/resources/sqltask/mysql_exec_template.sh +++ b/src/backend/job-execute/service-job-execute/src/main/resources/sqltask/mysql_exec_template.sh @@ -38,7 +38,7 @@ timeSec=`date +'%s'` LOG=/tmp/result_${SCRIPT_NAME}_${timeSec}.log if [ ${PORT} -gt 0 ]; then - HOST_AND_PORT=`netstat -ntl | grep ":${PORT} " | awk '{print $4}'` + HOST_AND_PORT=`netstat -ntl | grep ":${PORT} " | awk '{print $4}' | head -n 1` HOST=${HOST_AND_PORT%:${PORT}} if [ ${HOST} = '0.0.0.0' -o ${HOST} = '::' ];then HOST='127.0.0.1' diff --git a/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/common/gse/util/K8sUtilsTest.java b/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/common/gse/util/K8sUtilsTest.java new file mode 100644 index 0000000000..9f0f424909 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/common/gse/util/K8sUtilsTest.java @@ -0,0 +1,50 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.common.gse.util; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class K8sUtilsTest { + + @Test + void removeContainerIdType() { + assertEquals( + K8sUtils.removeContainerIdType("docker://1c5c1c9e3faf4295889b3974dc3f2eb4a8f77e5f0de2bac16ef6c76f7e67b6cb"), + "1c5c1c9e3faf4295889b3974dc3f2eb4a8f77e5f0de2bac16ef6c76f7e67b6cb" + ); + + assertEquals( + K8sUtils.removeContainerIdType("1c5c1c9e3faf4295889b3974dc3f2eb4a8f77e5f0de2bac16ef6c76f7e67b6cb"), + "1c5c1c9e3faf4295889b3974dc3f2eb4a8f77e5f0de2bac16ef6c76f7e67b6cb" + ); + + assertEquals( + K8sUtils.removeContainerIdType("17b6cb"), + "17b6cb" + ); + } +} diff --git a/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/common/util/VariableValueResolverTest.java b/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/common/util/VariableValueResolverTest.java index a7fd2b81d5..02e7bb9275 100644 --- a/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/common/util/VariableValueResolverTest.java +++ b/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/common/util/VariableValueResolverTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/common/util/label/selector/K8sLabelSelectorTest.java b/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/common/util/label/selector/K8sLabelSelectorTest.java new file mode 100644 index 0000000000..5fdcc2ab60 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/common/util/label/selector/K8sLabelSelectorTest.java @@ -0,0 +1,182 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.common.util.label.selector; + +import com.tencent.bk.job.execute.util.label.selector.LabelSelectorParse; +import com.tencent.bk.job.execute.util.label.selector.LabelSelectorParseException; +import com.tencent.bk.job.execute.util.label.selector.Operator; +import com.tencent.bk.job.execute.util.label.selector.Requirement; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +public class K8sLabelSelectorTest { + + @Test + void testSelectorParse() { + String[] testGoodStrings = { + "x=a,y=b,z=c", + "", + "x!=a,y=b", + "x=", + "x= ", + "x=,z= ", + "x= ,z= ", + "!x", + "x>1", + "x>1,z<5", + }; + String[] testBadStrings = { + "x=a||y=b", + "x==a==b", + "!x=a", + "x LabelSelectorParse.parse(test)); + } + for (String test : testBadStrings) { + assertThrows(LabelSelectorParseException.class, () -> LabelSelectorParse.parse(test)); + } + } + + @Test + void testParseToRequirements() { + expectMatch(LabelSelectorParse.parse("app=nginx"), + Requirements.build(newRequirement("app", Operator.Equals, "nginx"))); + expectMatch(LabelSelectorParse.parse("app==nginx"), + Requirements.build(newRequirement("app", Operator.DoubleEquals, "nginx"))); + expectMatch(LabelSelectorParse.parse("version==1.0"), + Requirements.build(newRequirement("version", Operator.DoubleEquals, "1.0"))); + expectMatch(LabelSelectorParse.parse("version=1.0"), + Requirements.build(newRequirement("version", Operator.Equals, "1.0"))); + + + expectMatch(LabelSelectorParse.parse("app!=nginx"), + Requirements.build(newRequirement("app", Operator.NotEquals, "nginx"))); + + expectMatch(LabelSelectorParse.parse("app in (nginx, redis)"), + Requirements.build(newRequirement("app", Operator.In, new String[]{"nginx", "redis"}))); + expectMatch(LabelSelectorParse.parse("version in (1.0, 2.0)"), + Requirements.build(newRequirement("version", Operator.In, new String[]{"1.0", "2.0"}))); + + expectMatch(LabelSelectorParse.parse("app notin (nginx, redis)"), + Requirements.build(newRequirement("app", Operator.NotIn, new String[]{"nginx", "redis"}))); + + expectMatch(LabelSelectorParse.parse("app"), + Requirements.build(newRequirement("app", Operator.Exists))); + expectMatch(LabelSelectorParse.parse("release-version"), + Requirements.build(newRequirement("release-version", Operator.Exists))); + + expectMatch(LabelSelectorParse.parse("!app"), + Requirements.build(newRequirement("app", Operator.DoesNotExist))); + + expectMatch(LabelSelectorParse.parse("version > 10"), + Requirements.build(newRequirement("version", Operator.GreaterThan, "10"))); + expectMatch(LabelSelectorParse.parse("version < 10"), + Requirements.build(newRequirement("version", Operator.LessThan, "10"))); + + expectMatch(LabelSelectorParse.parse("app=nginx, tier=frontend"), + Requirements.build( + newRequirement("app", Operator.Equals, "nginx"), + newRequirement("tier", Operator.Equals, "frontend")) + ); + expectMatch(LabelSelectorParse.parse("env=production,version!=1.0"), + Requirements.build( + newRequirement("env", Operator.Equals, "production"), + newRequirement("version", Operator.NotEquals, "1.0")) + ); + expectMatch(LabelSelectorParse.parse("app in (nginx, redis),tier=backend"), + Requirements.build( + newRequirement("app", Operator.In, new String[]{"nginx", "redis"}), + newRequirement("tier", Operator.Equals, "backend")) + ); + expectMatch(LabelSelectorParse.parse("!beta-version,app!=mysql"), + Requirements.build( + newRequirement("beta-version", Operator.DoesNotExist), + newRequirement("app", Operator.NotEquals, "mysql")) + ); + + expectMatch(LabelSelectorParse.parse("val in (in, notin)"), + Requirements.build( + newRequirement("val", Operator.In, new String[]{"in", "notin"})) + ); + } + + private Requirement newRequirement(String key, Operator op, String val) { + return new Requirement(key, op, Collections.singletonList(val)); + } + + private Requirement newRequirement(String key, Operator op) { + return new Requirement(key, op, null); + } + + private Requirement newRequirement(String key, Operator op, String[] vals) { + return new Requirement(key, op, Arrays.asList(vals)); + } + + private void expectMatch(List actualRequirementList, + Requirements expectRequirements) { + actualRequirementList.sort(new ByKey()); + expectRequirements.requirements.sort(new ByKey()); + assertEquals(actualRequirementList.size(), expectRequirements.requirements.size()); + for (int i = 0; i < actualRequirementList.size(); i++) { + Requirement actual = actualRequirementList.get(i); + Requirement expect = expectRequirements.requirements.get(i); + assertEquals(actual, expect); + } + } + + private static class Requirements { + private final List requirements = new ArrayList<>(); + + public static Requirements build(Requirement... requirementArgs) { + Requirements requirements = new Requirements(); + for (Requirement requirement : requirementArgs) { + requirements.add(requirement); + } + return requirements; + } + + public void add(Requirement requirement) { + requirements.add(requirement); + } + } + + private static class ByKey implements Comparator { + @Override + public int compare(Requirement o1, Requirement o2) { + return o1.getKey().compareTo(o2.getKey()); + } + } +} diff --git a/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/evict/TaskEvictPolicyTest.java b/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/evict/TaskEvictPolicyTest.java index 61575664cf..4a0bad6dd9 100644 --- a/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/evict/TaskEvictPolicyTest.java +++ b/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/evict/TaskEvictPolicyTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/rolling/ExponentIncrementRollingExprPartTest.java b/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/rolling/ExponentIncrementRollingExprPartTest.java index 2541733187..8390bca085 100644 --- a/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/rolling/ExponentIncrementRollingExprPartTest.java +++ b/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/rolling/ExponentIncrementRollingExprPartTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,6 +26,7 @@ import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.execute.common.exception.RollingExprParseException; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -64,41 +65,43 @@ void parseValidExpr() { RollingExprPart rollingExprPart = ROLLING_EXPR_PART.parseExpr("*2"); assertThat(rollingExprPart).isNotNull(); assertThat(rollingExprPart).isInstanceOf(ExponentIncrementRollingExprPart.class); - ExponentIncrementRollingExprPart exponentIncrementRollingExprPart = (ExponentIncrementRollingExprPart) rollingExprPart; + ExponentIncrementRollingExprPart exponentIncrementRollingExprPart = + (ExponentIncrementRollingExprPart) rollingExprPart; assertThat(exponentIncrementRollingExprPart.getExpr()).isEqualTo("*2"); assertThat(exponentIncrementRollingExprPart.getExponent()).isEqualTo(2); } @Test void compute() { - List rollingServers = new ArrayList<>(); - rollingServers.add(new HostDTO(0L, "127.0.0.1")); - rollingServers.add(new HostDTO(0L, "127.0.0.2")); - rollingServers.add(new HostDTO(0L, "127.0.0.3")); - rollingServers.add(new HostDTO(0L, "127.0.0.4")); - rollingServers.add(new HostDTO(0L, "127.0.0.5")); - RollingServerBatchContext context = new RollingServerBatchContext(rollingServers); + List rollingExecuteObjects = new ArrayList<>(); + rollingExecuteObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(1L))); + rollingExecuteObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(2L))); + rollingExecuteObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(3L))); + rollingExecuteObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(4L))); + rollingExecuteObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(5L))); + RollingExecuteObjectBatchContext context = new RollingExecuteObjectBatchContext(rollingExecuteObjects); - List remainingServers = new ArrayList<>(); - remainingServers.add(new HostDTO(0L, "127.0.0.2")); - remainingServers.add(new HostDTO(0L, "127.0.0.3")); - remainingServers.add(new HostDTO(0L, "127.0.0.4")); - remainingServers.add(new HostDTO(0L, "127.0.0.5")); - context.setRemainedServers(remainingServers); + List remainingExecuteObjects = new ArrayList<>(); + remainingExecuteObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(2L))); + remainingExecuteObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(3L))); + remainingExecuteObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(4L))); + remainingExecuteObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(5L))); + context.setRemainedExecuteObjects(remainingExecuteObjects); - RollingServerBatch preRollingServerBatch = new RollingServerBatch(); - preRollingServerBatch.setBatch(1); - preRollingServerBatch.setServers(Collections.singletonList(new HostDTO(0L, "127.0.0.1"))); - context.addServerBatch(preRollingServerBatch); + RollingExecuteObjectBatch preRollingExecuteObjectBatch = new RollingExecuteObjectBatch(); + preRollingExecuteObjectBatch.setBatch(1); + preRollingExecuteObjectBatch.setExecuteObjects( + Collections.singletonList(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(1L)))); + context.addExecuteObjectBatch(preRollingExecuteObjectBatch); context.setBatchCount(1); ExponentIncrementRollingExprPart exponentIncrementRollingExprPart = (ExponentIncrementRollingExprPart) ROLLING_EXPR_PART.parseExpr("*3"); - List serversOnBatch = exponentIncrementRollingExprPart.compute(context); - assertThat(serversOnBatch).containsSequence( - new HostDTO(0L, "127.0.0.2"), - new HostDTO(0L, "127.0.0.3"), - new HostDTO(0L, "127.0.0.4") + List executeObjectsOnBatch = exponentIncrementRollingExprPart.compute(context); + assertThat(executeObjectsOnBatch).containsSequence( + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(2L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(3L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(4L)) ); } } diff --git a/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/rolling/PercentRollingExprPartTest.java b/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/rolling/PercentRollingExprPartTest.java index 5070866360..16b19a50a7 100644 --- a/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/rolling/PercentRollingExprPartTest.java +++ b/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/rolling/PercentRollingExprPartTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,6 +26,7 @@ import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.execute.common.exception.RollingExprParseException; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; @@ -72,63 +73,63 @@ void parseValidExpr() { } @Nested - @DisplayName("验证根据百分比计算服务器分批大小") + @DisplayName("验证根据百分比计算执行对象分批大小") class ComputeTest { @Test @DisplayName("验证100%表达式") void compute() { - List rollingServers = new ArrayList<>(); - rollingServers.add(new HostDTO(0L, "127.0.0.1")); - rollingServers.add(new HostDTO(0L, "127.0.0.2")); - rollingServers.add(new HostDTO(0L, "127.0.0.3")); + List rollingExecuteObjects = new ArrayList<>(); + rollingExecuteObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(1L))); + rollingExecuteObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(2L))); + rollingExecuteObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(3L))); PercentRollingExprPart percentRollingExprPart = (PercentRollingExprPart) PERCENT_ROLLING_EXPR_PART.parseExpr("100%"); - RollingServerBatchContext context = new RollingServerBatchContext(rollingServers); - List serversOnBatch = percentRollingExprPart.compute(context); - assertThat(serversOnBatch).hasSize(3); - assertThat(serversOnBatch).containsSequence( - new HostDTO(0L, "127.0.0.1"), - new HostDTO(0L, "127.0.0.2"), - new HostDTO(0L, "127.0.0.3") + RollingExecuteObjectBatchContext context = new RollingExecuteObjectBatchContext(rollingExecuteObjects); + List executeObjectsOnBatch = percentRollingExprPart.compute(context); + assertThat(executeObjectsOnBatch).hasSize(3); + assertThat(executeObjectsOnBatch).containsSequence( + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(1L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(2L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(3L)) ); } @Test @DisplayName("验证批次计算向上取整") void testCeil() { - List rollingServers = new ArrayList<>(); - rollingServers.add(new HostDTO(0L, "127.0.0.1")); - rollingServers.add(new HostDTO(0L, "127.0.0.2")); - rollingServers.add(new HostDTO(0L, "127.0.0.3")); - rollingServers.add(new HostDTO(0L, "127.0.0.4")); - rollingServers.add(new HostDTO(0L, "127.0.0.5")); - RollingServerBatchContext context = new RollingServerBatchContext(rollingServers); + List rollingExecuteObjects = new ArrayList<>(); + rollingExecuteObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(1L))); + rollingExecuteObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(2L))); + rollingExecuteObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(3L))); + rollingExecuteObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(4L))); + rollingExecuteObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(5L))); + RollingExecuteObjectBatchContext context = new RollingExecuteObjectBatchContext(rollingExecuteObjects); PercentRollingExprPart percentRollingExprPart = (PercentRollingExprPart) PERCENT_ROLLING_EXPR_PART.parseExpr("10%"); - List serversOnBatch = percentRollingExprPart.compute(context); - assertThat(serversOnBatch).hasSize(1); - assertThat(serversOnBatch).containsSequence( - new HostDTO(0L, "127.0.0.1") + List executeObjectsOnBatch = percentRollingExprPart.compute(context); + assertThat(executeObjectsOnBatch).hasSize(1); + assertThat(executeObjectsOnBatch).containsSequence( + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(1L)) ); percentRollingExprPart = (PercentRollingExprPart) PERCENT_ROLLING_EXPR_PART.parseExpr("20%"); - serversOnBatch = percentRollingExprPart.compute(context); - assertThat(serversOnBatch).hasSize(1); - assertThat(serversOnBatch).containsSequence( - new HostDTO(0L, "127.0.0.1") + executeObjectsOnBatch = percentRollingExprPart.compute(context); + assertThat(executeObjectsOnBatch).hasSize(1); + assertThat(executeObjectsOnBatch).containsSequence( + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(1L)) ); percentRollingExprPart = (PercentRollingExprPart) PERCENT_ROLLING_EXPR_PART.parseExpr("30%"); - serversOnBatch = percentRollingExprPart.compute(context); - assertThat(serversOnBatch).hasSize(2); - assertThat(serversOnBatch).containsSequence( - new HostDTO(0L, "127.0.0.1"), - new HostDTO(0L, "127.0.0.2") + executeObjectsOnBatch = percentRollingExprPart.compute(context); + assertThat(executeObjectsOnBatch).hasSize(2); + assertThat(executeObjectsOnBatch).containsSequence( + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(1L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(2L)) ); } } diff --git a/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/rolling/PlusIncrementRollingExprPartTest.java b/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/rolling/PlusIncrementRollingExprPartTest.java index 54cf205608..c3bdd9e8fd 100644 --- a/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/rolling/PlusIncrementRollingExprPartTest.java +++ b/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/rolling/PlusIncrementRollingExprPartTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,6 +26,7 @@ import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.execute.common.exception.RollingExprParseException; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -78,35 +79,36 @@ void parseValidExpr() { @Test void compute() { - List rollingServers = new ArrayList<>(); - rollingServers.add(new HostDTO(0L, "127.0.0.1")); - rollingServers.add(new HostDTO(0L, "127.0.0.2")); - rollingServers.add(new HostDTO(0L, "127.0.0.3")); - rollingServers.add(new HostDTO(0L, "127.0.0.4")); - rollingServers.add(new HostDTO(0L, "127.0.0.5")); - RollingServerBatchContext context = new RollingServerBatchContext(rollingServers); - - List remainingServers = new ArrayList<>(); - remainingServers.add(new HostDTO(0L, "127.0.0.2")); - remainingServers.add(new HostDTO(0L, "127.0.0.3")); - remainingServers.add(new HostDTO(0L, "127.0.0.4")); - remainingServers.add(new HostDTO(0L, "127.0.0.5")); - context.setRemainedServers(remainingServers); - - RollingServerBatch preRollingServerBatch = new RollingServerBatch(); - preRollingServerBatch.setBatch(1); - preRollingServerBatch.setServers(Collections.singletonList(new HostDTO(0L, "127.0.0.1"))); - context.addServerBatch(preRollingServerBatch); + List rollingExecuteObjects = new ArrayList<>(); + rollingExecuteObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(1L))); + rollingExecuteObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(2L))); + rollingExecuteObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(3L))); + rollingExecuteObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(4L))); + rollingExecuteObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(5L))); + RollingExecuteObjectBatchContext context = new RollingExecuteObjectBatchContext(rollingExecuteObjects); + + List remainingExecuteObjects = new ArrayList<>(); + remainingExecuteObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(2L))); + remainingExecuteObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(3L))); + remainingExecuteObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(4L))); + remainingExecuteObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(5L))); + context.setRemainedExecuteObjects(remainingExecuteObjects); + + RollingExecuteObjectBatch preRollingExecuteObjectBatch = new RollingExecuteObjectBatch(); + preRollingExecuteObjectBatch.setBatch(1); + preRollingExecuteObjectBatch.setExecuteObjects( + Collections.singletonList(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(1L)))); + context.addExecuteObjectBatch(preRollingExecuteObjectBatch); context.setBatchCount(1); PlusIncrementRollingExprPart plusIncrementRollingExprPart = (PlusIncrementRollingExprPart) ROLLING_EXPR_PART.parseExpr("+3"); - List serversOnBatch = plusIncrementRollingExprPart.compute(context); - assertThat(serversOnBatch).containsSequence( - new HostDTO(0L, "127.0.0.2"), - new HostDTO(0L, "127.0.0.3"), - new HostDTO(0L, "127.0.0.4"), - new HostDTO(0L, "127.0.0.5") + List executeObjectsOnBatch = plusIncrementRollingExprPart.compute(context); + assertThat(executeObjectsOnBatch).containsSequence( + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(2L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(3L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(4L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(5L)) ); } } diff --git a/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/rolling/QuantityRollingExprPartTest.java b/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/rolling/QuantityRollingExprPartTest.java index b88a2113b2..3f7263f66d 100644 --- a/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/rolling/QuantityRollingExprPartTest.java +++ b/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/rolling/QuantityRollingExprPartTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,6 +26,7 @@ import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.execute.common.exception.RollingExprParseException; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; import org.junit.jupiter.api.Test; import java.util.ArrayList; @@ -70,33 +71,33 @@ void parseValidExpr() { @Test void compute() { - List rollingServers = new ArrayList<>(); - rollingServers.add(new HostDTO(0L, "127.0.0.1")); - rollingServers.add(new HostDTO(0L, "127.0.0.2")); - rollingServers.add(new HostDTO(0L, "127.0.0.3")); - rollingServers.add(new HostDTO(0L, "127.0.0.4")); - rollingServers.add(new HostDTO(0L, "127.0.0.5")); - RollingServerBatchContext context = new RollingServerBatchContext(rollingServers); + List rollingExecuteObjects = new ArrayList<>(); + rollingExecuteObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(1L))); + rollingExecuteObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(2L))); + rollingExecuteObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(3L))); + rollingExecuteObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(4L))); + rollingExecuteObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(5L))); + RollingExecuteObjectBatchContext context = new RollingExecuteObjectBatchContext(rollingExecuteObjects); QuantityRollingExprPart quantityRollingExprPart = (QuantityRollingExprPart) QUANTITY_ROLLING_EXPR_PART.parseExpr("2"); - List serversOnBatch = quantityRollingExprPart.compute(context); - assertThat(serversOnBatch).hasSize(2); - assertThat(serversOnBatch).containsSequence( - new HostDTO(0L, "127.0.0.1"), - new HostDTO(0L, "127.0.0.2") + List executeObjectsOnBatch = quantityRollingExprPart.compute(context); + assertThat(executeObjectsOnBatch).hasSize(2); + assertThat(executeObjectsOnBatch).containsSequence( + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(1L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(2L)) ); quantityRollingExprPart = (QuantityRollingExprPart) QUANTITY_ROLLING_EXPR_PART.parseExpr("10"); - serversOnBatch = quantityRollingExprPart.compute(context); - assertThat(serversOnBatch).hasSize(5); - assertThat(serversOnBatch).containsSequence( - new HostDTO(0L, "127.0.0.1"), - new HostDTO(0L, "127.0.0.2"), - new HostDTO(0L, "127.0.0.3"), - new HostDTO(0L, "127.0.0.4"), - new HostDTO(0L, "127.0.0.5") + executeObjectsOnBatch = quantityRollingExprPart.compute(context); + assertThat(executeObjectsOnBatch).hasSize(5); + assertThat(executeObjectsOnBatch).containsSequence( + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(1L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(2L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(3L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(4L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(5L)) ); } } diff --git a/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/rolling/RollingExecuteObjectBatchResolverTest.java b/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/rolling/RollingExecuteObjectBatchResolverTest.java new file mode 100644 index 0000000000..24bfb81959 --- /dev/null +++ b/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/rolling/RollingExecuteObjectBatchResolverTest.java @@ -0,0 +1,464 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.execute.engine.rolling; + +import com.tencent.bk.job.common.exception.FailedPreconditionException; +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.execute.engine.model.ExecuteObject; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Nested; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.Timeout; + +import java.util.ArrayList; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatExceptionOfType; + +class RollingExecuteObjectBatchResolverTest { + + @Test + @DisplayName("计算滚动批次 - 一个子表达式，按百分比分批") + void resolveForSinglePercentExpr() { + List executeObjects = new ArrayList<>(); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(1L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(2L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(3L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(4L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(5L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(6L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(7L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(8L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(9L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(10L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(11L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(12L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(13L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(14L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(15L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(16L))); + + RollingBatchExecuteObjectsResolver context = new RollingBatchExecuteObjectsResolver(executeObjects, "25%"); + List executeObjectBatchList = context.resolve(); + assertThat(executeObjectBatchList).hasSize(4); + + assertThat(executeObjectBatchList.get(0).getBatch()).isEqualTo(1); + assertThat(executeObjectBatchList.get(0).getExecuteObjects()).containsSequence( + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(1L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(2L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(3L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(4L)) + ); + assertThat(executeObjectBatchList.get(0).getRollingExprPart().getExpr()).isEqualTo("25%"); + + assertThat(executeObjectBatchList.get(1).getBatch()).isEqualTo(2); + assertThat(executeObjectBatchList.get(1).getExecuteObjects()).containsSequence( + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(5L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(6L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(7L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(8L)) + ); + assertThat(executeObjectBatchList.get(1).getRollingExprPart().getExpr()).isEqualTo("25%"); + + assertThat(executeObjectBatchList.get(2).getBatch()).isEqualTo(3); + assertThat(executeObjectBatchList.get(2).getExecuteObjects()).containsSequence( + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(9L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(10L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(11L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(12L)) + ); + assertThat(executeObjectBatchList.get(2).getRollingExprPart().getExpr()).isEqualTo("25%"); + + assertThat(executeObjectBatchList.get(3).getBatch()).isEqualTo(4); + assertThat(executeObjectBatchList.get(3).getExecuteObjects()).containsSequence( + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(13L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(14L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(15L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(16L)) + ); + assertThat(executeObjectBatchList.get(3).getRollingExprPart().getExpr()).isEqualTo("25%"); + } + + @Test + @DisplayName("计算滚动批次 - 多个子表达式，按百分比分批") + void resolveForMultiPercentExpr() { + List executeObjects = new ArrayList<>(); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(1L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(2L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(3L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(4L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(5L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(6L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(7L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(8L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(9L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(10L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(11L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(12L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(13L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(14L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(15L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(16L))); + RollingBatchExecuteObjectsResolver context = new RollingBatchExecuteObjectsResolver(executeObjects, "10% 30%"); + List executeObjectBatchList = context.resolve(); + assertThat(executeObjectBatchList).hasSize(4); + + assertThat(executeObjectBatchList.get(0).getBatch()).isEqualTo(1); + assertThat(executeObjectBatchList.get(0).getExecuteObjects()).containsSequence( + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(1L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(2L)) + ); + assertThat(executeObjectBatchList.get(0).getRollingExprPart().getExpr()).isEqualTo("10%"); + + assertThat(executeObjectBatchList.get(1).getBatch()).isEqualTo(2); + assertThat(executeObjectBatchList.get(1).getExecuteObjects()).containsSequence( + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(3L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(4L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(5L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(6L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(7L)) + ); + assertThat(executeObjectBatchList.get(1).getRollingExprPart().getExpr()).isEqualTo("30%"); + + assertThat(executeObjectBatchList.get(2).getBatch()).isEqualTo(3); + assertThat(executeObjectBatchList.get(2).getExecuteObjects()).containsSequence( + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(8L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(9L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(10L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(11L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(12L)) + ); + assertThat(executeObjectBatchList.get(2).getRollingExprPart().getExpr()).isEqualTo("30%"); + + assertThat(executeObjectBatchList.get(3).getBatch()).isEqualTo(4); + assertThat(executeObjectBatchList.get(3).getExecuteObjects()).containsSequence( + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(13L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(14L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(15L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(16L)) + ); + assertThat(executeObjectBatchList.get(3).getRollingExprPart().getExpr()).isEqualTo("30%"); + } + + @Test + @DisplayName("计算滚动批次 - 一个子表达式，按数量分批") + void resolveForSingleQuantityExpr() { + List executeObjects = new ArrayList<>(); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(1L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(2L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(3L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(4L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(5L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(6L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(7L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(8L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(9L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(10L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(11L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(12L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(13L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(14L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(15L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(16L))); + RollingBatchExecuteObjectsResolver context = new RollingBatchExecuteObjectsResolver(executeObjects, "10"); + List executeObjectBatchList = context.resolve(); + assertThat(executeObjectBatchList).hasSize(2); + + assertThat(executeObjectBatchList.get(0).getBatch()).isEqualTo(1); + assertThat(executeObjectBatchList.get(0).getExecuteObjects()).containsSequence( + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(1L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(2L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(3L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(4L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(5L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(6L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(7L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(8L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(9L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(10L)) + ); + assertThat(executeObjectBatchList.get(0).getRollingExprPart().getExpr()).isEqualTo("10"); + + assertThat(executeObjectBatchList.get(1).getBatch()).isEqualTo(2); + assertThat(executeObjectBatchList.get(1).getExecuteObjects()).containsSequence( + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(11L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(12L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(13L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(14L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(15L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(16L)) + ); + assertThat(executeObjectBatchList.get(1).getRollingExprPart().getExpr()).isEqualTo("10"); + } + + @Test + @DisplayName("计算滚动批次 - 多个子表达式，按数量分批") + void resolveForMultiQuantityExpr() { + List executeObjects = new ArrayList<>(); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(1L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(2L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(3L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(4L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(5L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(6L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(7L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(8L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(9L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(10L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(11L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(12L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(13L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(14L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(15L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(16L))); + RollingBatchExecuteObjectsResolver context = new RollingBatchExecuteObjectsResolver(executeObjects, "1 5"); + List executeObjectBatchList = context.resolve(); + assertThat(executeObjectBatchList).hasSize(4); + + assertThat(executeObjectBatchList.get(0).getBatch()).isEqualTo(1); + assertThat(executeObjectBatchList.get(0).getExecuteObjects()).containsSequence( + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(1L)) + ); + assertThat(executeObjectBatchList.get(0).getRollingExprPart().getExpr()).isEqualTo("1"); + + assertThat(executeObjectBatchList.get(1).getBatch()).isEqualTo(2); + assertThat(executeObjectBatchList.get(1).getExecuteObjects()).containsSequence( + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(2L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(3L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(4L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(5L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(6L)) + ); + assertThat(executeObjectBatchList.get(1).getRollingExprPart().getExpr()).isEqualTo("5"); + + assertThat(executeObjectBatchList.get(2).getBatch()).isEqualTo(3); + assertThat(executeObjectBatchList.get(2).getExecuteObjects()).containsSequence( + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(7L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(8L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(9L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(10L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(11L)) + ); + assertThat(executeObjectBatchList.get(2).getRollingExprPart().getExpr()).isEqualTo("5"); + + assertThat(executeObjectBatchList.get(3).getBatch()).isEqualTo(4); + assertThat(executeObjectBatchList.get(3).getExecuteObjects()).containsSequence( + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(12L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(13L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(14L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(15L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(16L)) + ); + assertThat(executeObjectBatchList.get(3).getRollingExprPart().getExpr()).isEqualTo("5"); + } + + @Test + @DisplayName("计算滚动批次 - 混合表达式，按数量和百分比分批") + void resolveForMixedExpr() { + List executeObjects = new ArrayList<>(); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(1L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(2L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(3L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(4L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(5L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(6L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(7L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(8L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(9L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(10L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(11L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(12L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(13L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(14L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(15L))); + RollingBatchExecuteObjectsResolver context = new RollingBatchExecuteObjectsResolver(executeObjects, "1 30%"); + List executeObjectBatchList = context.resolve(); + assertThat(executeObjectBatchList).hasSize(4); + + assertThat(executeObjectBatchList.get(0).getBatch()).isEqualTo(1); + assertThat(executeObjectBatchList.get(0).getExecuteObjects()).containsSequence( + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(1L)) + ); + assertThat(executeObjectBatchList.get(0).getRollingExprPart().getExpr()).isEqualTo("1"); + + assertThat(executeObjectBatchList.get(1).getBatch()).isEqualTo(2); + assertThat(executeObjectBatchList.get(1).getExecuteObjects()).containsSequence( + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(2L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(3L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(4L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(5L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(6L)) + ); + assertThat(executeObjectBatchList.get(1).getRollingExprPart().getExpr()).isEqualTo("30%"); + + assertThat(executeObjectBatchList.get(2).getBatch()).isEqualTo(3); + assertThat(executeObjectBatchList.get(2).getExecuteObjects()).containsSequence( + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(7L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(8L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(9L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(10L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(11L)) + ); + assertThat(executeObjectBatchList.get(2).getRollingExprPart().getExpr()).isEqualTo("30%"); + + assertThat(executeObjectBatchList.get(3).getBatch()).isEqualTo(4); + assertThat(executeObjectBatchList.get(3).getExecuteObjects()).containsSequence( + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(12L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(13L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(14L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(15L)) + + ); + assertThat(executeObjectBatchList.get(3).getRollingExprPart().getExpr()).isEqualTo("30%"); + } + + + @Test + @DisplayName("计算滚动批次 - 最后一批包含所有") + void resolveForAllRemainedExpr() { + List executeObjects = new ArrayList<>(); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(1L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(2L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(3L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(4L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(5L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(6L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(7L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(8L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(9L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(10L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(11L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(12L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(13L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(14L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(15L))); + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(16L))); + RollingBatchExecuteObjectsResolver context = new RollingBatchExecuteObjectsResolver(executeObjects, "1 30% " + + "100%"); + List executeObjectBatchList = context.resolve(); + assertThat(executeObjectBatchList).hasSize(3); + + assertThat(executeObjectBatchList.get(0).getBatch()).isEqualTo(1); + assertThat(executeObjectBatchList.get(0).getExecuteObjects()).containsSequence( + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(1L)) + ); + assertThat(executeObjectBatchList.get(0).getRollingExprPart().getExpr()).isEqualTo("1"); + + assertThat(executeObjectBatchList.get(1).getBatch()).isEqualTo(2); + assertThat(executeObjectBatchList.get(1).getExecuteObjects()).containsSequence( + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(2L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(3L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(4L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(5L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(6L)) + ); + assertThat(executeObjectBatchList.get(1).getRollingExprPart().getExpr()).isEqualTo("30%"); + + assertThat(executeObjectBatchList.get(2).getBatch()).isEqualTo(3); + assertThat(executeObjectBatchList.get(2).getExecuteObjects()).containsSequence( + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(7L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(8L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(9L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(10L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(11L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(12L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(13L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(14L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(15L)), + ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId(16L)) + ); + assertThat(executeObjectBatchList.get(2).getRollingExprPart().getExpr()).isEqualTo("100%"); + } + + @Test + @DisplayName("计算滚动批次 - 超过最大允许批次 500") + void resolveExceedMaxBatchSize() { + List executeObjects = new ArrayList<>(); + for (int i = 1; i <= 1000; i++) { + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId((long) i))); + } + RollingBatchExecuteObjectsResolver context = new RollingBatchExecuteObjectsResolver(executeObjects, "1"); + assertThatExceptionOfType(FailedPreconditionException.class).isThrownBy( + context::resolve + ); + } + + @Nested + @DisplayName("计算滚动批次性能测试") + class RollingResolvePerformanceTest { + + @Test + @DisplayName("计算滚动批次 - 按数量分批，性能测试") + @Timeout(1) + void resolveQuantityExpr() { + List executeObjects = new ArrayList<>(); + for (int i = 1; i <= 50000; i++) { + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId((long) i))); + } + RollingBatchExecuteObjectsResolver context = new RollingBatchExecuteObjectsResolver(executeObjects, "500"); + List executeObjectBatchList = context.resolve(); + assertThat(executeObjectBatchList).hasSize(100); + } + + @Test + @DisplayName("计算滚动批次 - 按比例分批，性能测试") + @Timeout(1) + void resolvePercentExpr() { + List executeObjects = new ArrayList<>(); + for (int i = 1; i <= 50000; i++) { + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId((long) i))); + } + RollingBatchExecuteObjectsResolver context = new RollingBatchExecuteObjectsResolver(executeObjects, "1%"); + List executeObjectBatchList = context.resolve(); + assertThat(executeObjectBatchList).hasSize(100); + } + + @Test + @DisplayName("计算滚动批次 - 按加法递增分批，性能测试") + @Timeout(1) + void resolvePlusIncrementRollingExpr() { + List executeObjects = new ArrayList<>(); + for (int i = 1; i <= 50000; i++) { + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId((long) i))); + } + RollingBatchExecuteObjectsResolver context = new RollingBatchExecuteObjectsResolver(executeObjects, "+10"); + List executeObjectBatchList = context.resolve(); + assertThat(executeObjectBatchList).hasSize(100); + } + + @Test + @DisplayName("计算滚动批次 - 按指数递增分批，性能测试") + @Timeout(1) + void resolveExponentIncrRollingExpr() { + List executeObjects = new ArrayList<>(); + for (int i = 1; i <= 50000; i++) { + executeObjects.add(ExecuteObject.buildCompatibleExecuteObject(HostDTO.fromHostId((long) i))); + } + RollingBatchExecuteObjectsResolver context = new RollingBatchExecuteObjectsResolver(executeObjects, "*2"); + context.resolve(); + } + } +} diff --git a/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/rolling/RollingExprTest.java b/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/rolling/RollingExprTest.java index 740d5dcf00..7b3c04bb1f 100644 --- a/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/rolling/RollingExprTest.java +++ b/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/rolling/RollingExprTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/rolling/RollingServerBatchResolverTest.java b/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/rolling/RollingServerBatchResolverTest.java deleted file mode 100644 index 7ff654e1a7..0000000000 --- a/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/rolling/RollingServerBatchResolverTest.java +++ /dev/null @@ -1,388 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.execute.engine.rolling; - -import com.tencent.bk.job.common.model.dto.HostDTO; -import org.junit.jupiter.api.DisplayName; -import org.junit.jupiter.api.Test; - -import java.util.ArrayList; -import java.util.List; - -import static org.assertj.core.api.Assertions.assertThat; - -class RollingServerBatchResolverTest { - - @Test - @DisplayName("计算滚动批次 - 一个子表达式，按百分比分批") - void resolveForSinglePercentExpr() { - List servers = new ArrayList<>(); - servers.add(new HostDTO(0L, "127.0.0.1")); - servers.add(new HostDTO(0L, "127.0.0.2")); - servers.add(new HostDTO(0L, "127.0.0.3")); - servers.add(new HostDTO(0L, "127.0.0.4")); - servers.add(new HostDTO(0L, "127.0.0.5")); - servers.add(new HostDTO(0L, "127.0.0.6")); - servers.add(new HostDTO(0L, "127.0.0.7")); - servers.add(new HostDTO(0L, "127.0.0.8")); - servers.add(new HostDTO(0L, "127.0.0.9")); - servers.add(new HostDTO(0L, "127.0.0.10")); - servers.add(new HostDTO(0L, "127.0.0.11")); - servers.add(new HostDTO(0L, "127.0.0.12")); - servers.add(new HostDTO(0L, "127.0.0.13")); - servers.add(new HostDTO(0L, "127.0.0.14")); - servers.add(new HostDTO(0L, "127.0.0.15")); - servers.add(new HostDTO(0L, "127.0.0.16")); - RollingBatchServersResolver context = new RollingBatchServersResolver(servers, "25%"); - List serverBatchList = context.resolve(); - assertThat(serverBatchList).hasSize(4); - - assertThat(serverBatchList.get(0).getBatch()).isEqualTo(1); - assertThat(serverBatchList.get(0).getServers()).containsSequence( - new HostDTO(0L, "127.0.0.1"), - new HostDTO(0L, "127.0.0.2"), - new HostDTO(0L, "127.0.0.3"), - new HostDTO(0L, "127.0.0.4") - ); - assertThat(serverBatchList.get(0).getRollingExprPart().getExpr()).isEqualTo("25%"); - - assertThat(serverBatchList.get(1).getBatch()).isEqualTo(2); - assertThat(serverBatchList.get(1).getServers()).containsSequence( - new HostDTO(0L, "127.0.0.5"), - new HostDTO(0L, "127.0.0.6"), - new HostDTO(0L, "127.0.0.7"), - new HostDTO(0L, "127.0.0.8") - ); - assertThat(serverBatchList.get(1).getRollingExprPart().getExpr()).isEqualTo("25%"); - - assertThat(serverBatchList.get(2).getBatch()).isEqualTo(3); - assertThat(serverBatchList.get(2).getServers()).containsSequence( - new HostDTO(0L, "127.0.0.9"), - new HostDTO(0L, "127.0.0.10"), - new HostDTO(0L, "127.0.0.11"), - new HostDTO(0L, "127.0.0.12") - ); - assertThat(serverBatchList.get(2).getRollingExprPart().getExpr()).isEqualTo("25%"); - - assertThat(serverBatchList.get(3).getBatch()).isEqualTo(4); - assertThat(serverBatchList.get(3).getServers()).containsSequence( - new HostDTO(0L, "127.0.0.13"), - new HostDTO(0L, "127.0.0.14"), - new HostDTO(0L, "127.0.0.15"), - new HostDTO(0L, "127.0.0.16") - ); - assertThat(serverBatchList.get(3).getRollingExprPart().getExpr()).isEqualTo("25%"); - } - - @Test - @DisplayName("计算滚动批次 - 多个子表达式，按百分比分批") - void resolveForMultiPercentExpr() { - List servers = new ArrayList<>(); - servers.add(new HostDTO(0L, "127.0.0.1")); - servers.add(new HostDTO(0L, "127.0.0.2")); - servers.add(new HostDTO(0L, "127.0.0.3")); - servers.add(new HostDTO(0L, "127.0.0.4")); - servers.add(new HostDTO(0L, "127.0.0.5")); - servers.add(new HostDTO(0L, "127.0.0.6")); - servers.add(new HostDTO(0L, "127.0.0.7")); - servers.add(new HostDTO(0L, "127.0.0.8")); - servers.add(new HostDTO(0L, "127.0.0.9")); - servers.add(new HostDTO(0L, "127.0.0.10")); - servers.add(new HostDTO(0L, "127.0.0.11")); - servers.add(new HostDTO(0L, "127.0.0.12")); - servers.add(new HostDTO(0L, "127.0.0.13")); - servers.add(new HostDTO(0L, "127.0.0.14")); - servers.add(new HostDTO(0L, "127.0.0.15")); - servers.add(new HostDTO(0L, "127.0.0.16")); - RollingBatchServersResolver context = new RollingBatchServersResolver(servers, "10% 30%"); - List serverBatchList = context.resolve(); - assertThat(serverBatchList).hasSize(4); - - assertThat(serverBatchList.get(0).getBatch()).isEqualTo(1); - assertThat(serverBatchList.get(0).getServers()).containsSequence( - new HostDTO(0L, "127.0.0.1"), - new HostDTO(0L, "127.0.0.2") - ); - assertThat(serverBatchList.get(0).getRollingExprPart().getExpr()).isEqualTo("10%"); - - assertThat(serverBatchList.get(1).getBatch()).isEqualTo(2); - assertThat(serverBatchList.get(1).getServers()).containsSequence( - new HostDTO(0L, "127.0.0.3"), - new HostDTO(0L, "127.0.0.4"), - new HostDTO(0L, "127.0.0.5"), - new HostDTO(0L, "127.0.0.6"), - new HostDTO(0L, "127.0.0.7") - ); - assertThat(serverBatchList.get(1).getRollingExprPart().getExpr()).isEqualTo("30%"); - - assertThat(serverBatchList.get(2).getBatch()).isEqualTo(3); - assertThat(serverBatchList.get(2).getServers()).containsSequence( - new HostDTO(0L, "127.0.0.8"), - new HostDTO(0L, "127.0.0.9"), - new HostDTO(0L, "127.0.0.10"), - new HostDTO(0L, "127.0.0.11"), - new HostDTO(0L, "127.0.0.12") - ); - assertThat(serverBatchList.get(2).getRollingExprPart().getExpr()).isEqualTo("30%"); - - assertThat(serverBatchList.get(3).getBatch()).isEqualTo(4); - assertThat(serverBatchList.get(3).getServers()).containsSequence( - new HostDTO(0L, "127.0.0.13"), - new HostDTO(0L, "127.0.0.14"), - new HostDTO(0L, "127.0.0.15"), - new HostDTO(0L, "127.0.0.16") - ); - assertThat(serverBatchList.get(3).getRollingExprPart().getExpr()).isEqualTo("30%"); - } - - @Test - @DisplayName("计算滚动批次 - 一个子表达式，按数量分批") - void resolveForSingleQuantityExpr() { - List servers = new ArrayList<>(); - servers.add(new HostDTO(0L, "127.0.0.1")); - servers.add(new HostDTO(0L, "127.0.0.2")); - servers.add(new HostDTO(0L, "127.0.0.3")); - servers.add(new HostDTO(0L, "127.0.0.4")); - servers.add(new HostDTO(0L, "127.0.0.5")); - servers.add(new HostDTO(0L, "127.0.0.6")); - servers.add(new HostDTO(0L, "127.0.0.7")); - servers.add(new HostDTO(0L, "127.0.0.8")); - servers.add(new HostDTO(0L, "127.0.0.9")); - servers.add(new HostDTO(0L, "127.0.0.10")); - servers.add(new HostDTO(0L, "127.0.0.11")); - servers.add(new HostDTO(0L, "127.0.0.12")); - servers.add(new HostDTO(0L, "127.0.0.13")); - servers.add(new HostDTO(0L, "127.0.0.14")); - servers.add(new HostDTO(0L, "127.0.0.15")); - servers.add(new HostDTO(0L, "127.0.0.16")); - RollingBatchServersResolver context = new RollingBatchServersResolver(servers, "10"); - List serverBatchList = context.resolve(); - assertThat(serverBatchList).hasSize(2); - - assertThat(serverBatchList.get(0).getBatch()).isEqualTo(1); - assertThat(serverBatchList.get(0).getServers()).containsSequence( - new HostDTO(0L, "127.0.0.1"), - new HostDTO(0L, "127.0.0.2"), - new HostDTO(0L, "127.0.0.3"), - new HostDTO(0L, "127.0.0.4"), - new HostDTO(0L, "127.0.0.5"), - new HostDTO(0L, "127.0.0.6"), - new HostDTO(0L, "127.0.0.7"), - new HostDTO(0L, "127.0.0.8"), - new HostDTO(0L, "127.0.0.9"), - new HostDTO(0L, "127.0.0.10") - ); - assertThat(serverBatchList.get(0).getRollingExprPart().getExpr()).isEqualTo("10"); - - assertThat(serverBatchList.get(1).getBatch()).isEqualTo(2); - assertThat(serverBatchList.get(1).getServers()).containsSequence( - new HostDTO(0L, "127.0.0.11"), - new HostDTO(0L, "127.0.0.12"), - new HostDTO(0L, "127.0.0.13"), - new HostDTO(0L, "127.0.0.14"), - new HostDTO(0L, "127.0.0.15"), - new HostDTO(0L, "127.0.0.16") - ); - assertThat(serverBatchList.get(1).getRollingExprPart().getExpr()).isEqualTo("10"); - } - - @Test - @DisplayName("计算滚动批次 - 多个子表达式，按数量分批") - void resolveForMultiQuantityExpr() { - List servers = new ArrayList<>(); - servers.add(new HostDTO(0L, "127.0.0.1")); - servers.add(new HostDTO(0L, "127.0.0.2")); - servers.add(new HostDTO(0L, "127.0.0.3")); - servers.add(new HostDTO(0L, "127.0.0.4")); - servers.add(new HostDTO(0L, "127.0.0.5")); - servers.add(new HostDTO(0L, "127.0.0.6")); - servers.add(new HostDTO(0L, "127.0.0.7")); - servers.add(new HostDTO(0L, "127.0.0.8")); - servers.add(new HostDTO(0L, "127.0.0.9")); - servers.add(new HostDTO(0L, "127.0.0.10")); - servers.add(new HostDTO(0L, "127.0.0.11")); - servers.add(new HostDTO(0L, "127.0.0.12")); - servers.add(new HostDTO(0L, "127.0.0.13")); - servers.add(new HostDTO(0L, "127.0.0.14")); - servers.add(new HostDTO(0L, "127.0.0.15")); - servers.add(new HostDTO(0L, "127.0.0.16")); - RollingBatchServersResolver context = new RollingBatchServersResolver(servers, "1 5"); - List serverBatchList = context.resolve(); - assertThat(serverBatchList).hasSize(4); - - assertThat(serverBatchList.get(0).getBatch()).isEqualTo(1); - assertThat(serverBatchList.get(0).getServers()).containsSequence( - new HostDTO(0L, "127.0.0.1") - ); - assertThat(serverBatchList.get(0).getRollingExprPart().getExpr()).isEqualTo("1"); - - assertThat(serverBatchList.get(1).getBatch()).isEqualTo(2); - assertThat(serverBatchList.get(1).getServers()).containsSequence( - new HostDTO(0L, "127.0.0.2"), - new HostDTO(0L, "127.0.0.3"), - new HostDTO(0L, "127.0.0.4"), - new HostDTO(0L, "127.0.0.5"), - new HostDTO(0L, "127.0.0.6") - ); - assertThat(serverBatchList.get(1).getRollingExprPart().getExpr()).isEqualTo("5"); - - assertThat(serverBatchList.get(2).getBatch()).isEqualTo(3); - assertThat(serverBatchList.get(2).getServers()).containsSequence( - new HostDTO(0L, "127.0.0.7"), - new HostDTO(0L, "127.0.0.8"), - new HostDTO(0L, "127.0.0.9"), - new HostDTO(0L, "127.0.0.10"), - new HostDTO(0L, "127.0.0.11") - ); - assertThat(serverBatchList.get(2).getRollingExprPart().getExpr()).isEqualTo("5"); - - assertThat(serverBatchList.get(3).getBatch()).isEqualTo(4); - assertThat(serverBatchList.get(3).getServers()).containsSequence( - new HostDTO(0L, "127.0.0.12"), - new HostDTO(0L, "127.0.0.13"), - new HostDTO(0L, "127.0.0.14"), - new HostDTO(0L, "127.0.0.15"), - new HostDTO(0L, "127.0.0.16") - ); - assertThat(serverBatchList.get(3).getRollingExprPart().getExpr()).isEqualTo("5"); - } - - @Test - @DisplayName("计算滚动批次 - 混合表达式，按数量和百分比分批") - void resolveForMixedExpr() { - List servers = new ArrayList<>(); - servers.add(new HostDTO(0L, "127.0.0.1")); - servers.add(new HostDTO(0L, "127.0.0.2")); - servers.add(new HostDTO(0L, "127.0.0.3")); - servers.add(new HostDTO(0L, "127.0.0.4")); - servers.add(new HostDTO(0L, "127.0.0.5")); - servers.add(new HostDTO(0L, "127.0.0.6")); - servers.add(new HostDTO(0L, "127.0.0.7")); - servers.add(new HostDTO(0L, "127.0.0.8")); - servers.add(new HostDTO(0L, "127.0.0.9")); - servers.add(new HostDTO(0L, "127.0.0.10")); - servers.add(new HostDTO(0L, "127.0.0.11")); - servers.add(new HostDTO(0L, "127.0.0.12")); - servers.add(new HostDTO(0L, "127.0.0.13")); - servers.add(new HostDTO(0L, "127.0.0.14")); - servers.add(new HostDTO(0L, "127.0.0.15")); - RollingBatchServersResolver context = new RollingBatchServersResolver(servers, "1 30%"); - List serverBatchList = context.resolve(); - assertThat(serverBatchList).hasSize(4); - - assertThat(serverBatchList.get(0).getBatch()).isEqualTo(1); - assertThat(serverBatchList.get(0).getServers()).containsSequence( - new HostDTO(0L, "127.0.0.1") - ); - assertThat(serverBatchList.get(0).getRollingExprPart().getExpr()).isEqualTo("1"); - - assertThat(serverBatchList.get(1).getBatch()).isEqualTo(2); - assertThat(serverBatchList.get(1).getServers()).containsSequence( - new HostDTO(0L, "127.0.0.2"), - new HostDTO(0L, "127.0.0.3"), - new HostDTO(0L, "127.0.0.4"), - new HostDTO(0L, "127.0.0.5"), - new HostDTO(0L, "127.0.0.6") - ); - assertThat(serverBatchList.get(1).getRollingExprPart().getExpr()).isEqualTo("30%"); - - assertThat(serverBatchList.get(2).getBatch()).isEqualTo(3); - assertThat(serverBatchList.get(2).getServers()).containsSequence( - new HostDTO(0L, "127.0.0.7"), - new HostDTO(0L, "127.0.0.8"), - new HostDTO(0L, "127.0.0.9"), - new HostDTO(0L, "127.0.0.10"), - new HostDTO(0L, "127.0.0.11") - ); - assertThat(serverBatchList.get(2).getRollingExprPart().getExpr()).isEqualTo("30%"); - - assertThat(serverBatchList.get(3).getBatch()).isEqualTo(4); - assertThat(serverBatchList.get(3).getServers()).containsSequence( - new HostDTO(0L, "127.0.0.12"), - new HostDTO(0L, "127.0.0.13"), - new HostDTO(0L, "127.0.0.14"), - new HostDTO(0L, "127.0.0.15") - - ); - assertThat(serverBatchList.get(3).getRollingExprPart().getExpr()).isEqualTo("30%"); - } - - - @Test - @DisplayName("计算滚动批次 - 最后一批包含所有") - void resolveForAllRemainedExpr() { - List servers = new ArrayList<>(); - servers.add(new HostDTO(0L, "127.0.0.1")); - servers.add(new HostDTO(0L, "127.0.0.2")); - servers.add(new HostDTO(0L, "127.0.0.3")); - servers.add(new HostDTO(0L, "127.0.0.4")); - servers.add(new HostDTO(0L, "127.0.0.5")); - servers.add(new HostDTO(0L, "127.0.0.6")); - servers.add(new HostDTO(0L, "127.0.0.7")); - servers.add(new HostDTO(0L, "127.0.0.8")); - servers.add(new HostDTO(0L, "127.0.0.9")); - servers.add(new HostDTO(0L, "127.0.0.10")); - servers.add(new HostDTO(0L, "127.0.0.11")); - servers.add(new HostDTO(0L, "127.0.0.12")); - servers.add(new HostDTO(0L, "127.0.0.13")); - servers.add(new HostDTO(0L, "127.0.0.14")); - servers.add(new HostDTO(0L, "127.0.0.15")); - servers.add(new HostDTO(0L, "127.0.0.16")); - RollingBatchServersResolver context = new RollingBatchServersResolver(servers, "1 30% 100%"); - List serverBatchList = context.resolve(); - assertThat(serverBatchList).hasSize(3); - - assertThat(serverBatchList.get(0).getBatch()).isEqualTo(1); - assertThat(serverBatchList.get(0).getServers()).containsSequence( - new HostDTO(0L, "127.0.0.1") - ); - assertThat(serverBatchList.get(0).getRollingExprPart().getExpr()).isEqualTo("1"); - - assertThat(serverBatchList.get(1).getBatch()).isEqualTo(2); - assertThat(serverBatchList.get(1).getServers()).containsSequence( - new HostDTO(0L, "127.0.0.2"), - new HostDTO(0L, "127.0.0.3"), - new HostDTO(0L, "127.0.0.4"), - new HostDTO(0L, "127.0.0.5"), - new HostDTO(0L, "127.0.0.6") - ); - assertThat(serverBatchList.get(1).getRollingExprPart().getExpr()).isEqualTo("30%"); - - assertThat(serverBatchList.get(2).getBatch()).isEqualTo(3); - assertThat(serverBatchList.get(2).getServers()).containsSequence( - new HostDTO(0L, "127.0.0.7"), - new HostDTO(0L, "127.0.0.8"), - new HostDTO(0L, "127.0.0.9"), - new HostDTO(0L, "127.0.0.10"), - new HostDTO(0L, "127.0.0.11"), - new HostDTO(0L, "127.0.0.12"), - new HostDTO(0L, "127.0.0.13"), - new HostDTO(0L, "127.0.0.14"), - new HostDTO(0L, "127.0.0.15"), - new HostDTO(0L, "127.0.0.16") - ); - assertThat(serverBatchList.get(2).getRollingExprPart().getExpr()).isEqualTo("100%"); - } -} diff --git a/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/util/FilePathUtilsTest.java b/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/util/FilePathUtilsTest.java index d67b4598d6..5840a457fd 100644 --- a/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/util/FilePathUtilsTest.java +++ b/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/util/FilePathUtilsTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,7 @@ package com.tencent.bk.job.execute.engine.util; +import com.tencent.bk.job.common.util.FilePathUtils; import org.apache.commons.lang3.tuple.Pair; import org.junit.jupiter.api.Test; diff --git a/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/util/MacroUtilTest.java b/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/util/MacroUtilTest.java index b7b15bf4ce..e834529be7 100644 --- a/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/util/MacroUtilTest.java +++ b/src/backend/job-execute/service-job-execute/src/test/java/com/tencent/bk/job/execute/engine/util/MacroUtilTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway-worker/build.gradle b/src/backend/job-file-gateway/api-job-file-gateway-worker/build.gradle index 39c9557a83..1344b0f9fd 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway-worker/build.gradle +++ b/src/backend/job-file-gateway/api-job-file-gateway-worker/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/api/remote/RemoteFileWorkerResource.java b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/api/remote/RemoteFileWorkerResource.java index c705a45b40..0b2086e57b 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/api/remote/RemoteFileWorkerResource.java +++ b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/api/remote/RemoteFileWorkerResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,7 @@ package com.tencent.bk.job.file_gateway.api.remote; +import com.tencent.bk.job.common.annotation.RemoteAPI; import com.tencent.bk.job.common.model.Response; import com.tencent.bk.job.file_gateway.model.req.inner.HeartBeatReq; import com.tencent.bk.job.file_gateway.model.req.inner.OffLineAndReDispatchReq; @@ -41,6 +42,7 @@ @Api(tags = {"job-file-gateway:remote:FileWorker"}) @RequestMapping("/remote/fileWorker") @RestController +@RemoteAPI public interface RemoteFileWorkerResource { @ApiOperation(value = "Worker心跳", produces = "application/json") diff --git a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/consts/FileSourceInfoConsts.java b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/consts/FileSourceInfoConsts.java new file mode 100644 index 0000000000..f3e5c0a3eb --- /dev/null +++ b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/consts/FileSourceInfoConsts.java @@ -0,0 +1,34 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file_gateway.consts; + +/** + * 文件源信息相关常量 + */ +public class FileSourceInfoConsts { + + // Key: 蓝鲸制品库根地址 + public static final String KEY_BK_ARTIFACTORY_BASE_URL = "base_url"; +} diff --git a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/consts/FileSourceTypeEnum.java b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/consts/FileSourceTypeEnum.java new file mode 100644 index 0000000000..92ca1a8fd8 --- /dev/null +++ b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/consts/FileSourceTypeEnum.java @@ -0,0 +1,40 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file_gateway.consts; + +public enum FileSourceTypeEnum { + TENCENT_CLOUD_COS, + BLUEKING_ARTIFACTORY; + + /** + * 根据文件源类型编码判断是否为蓝鲸制品库文件源 + * + * @param fileSourceTypeCode 文件源类型编码 + * @return 布尔值 + */ + public static boolean isBlueKingArtifactory(String fileSourceTypeCode) { + return BLUEKING_ARTIFACTORY.name().equals(fileSourceTypeCode); + } +} diff --git a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/consts/TaskCommandEnum.java b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/consts/TaskCommandEnum.java index 40e8882cd9..0ece178136 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/consts/TaskCommandEnum.java +++ b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/consts/TaskCommandEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/consts/TaskStatusEnum.java b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/consts/TaskStatusEnum.java index 7966c2c1a9..b1dd3fd925 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/consts/TaskStatusEnum.java +++ b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/consts/TaskStatusEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/consts/WorkerActionTypeEnum.java b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/consts/WorkerActionTypeEnum.java index 0abf099a23..6ed38e2481 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/consts/WorkerActionTypeEnum.java +++ b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/consts/WorkerActionTypeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/dto/FileTaskProgressDTO.java b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/dto/FileTaskProgressDTO.java new file mode 100644 index 0000000000..411b85b04d --- /dev/null +++ b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/dto/FileTaskProgressDTO.java @@ -0,0 +1,88 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file_gateway.model.dto; + +import com.tencent.bk.job.file_gateway.consts.TaskStatusEnum; +import com.tencent.bk.job.file_gateway.model.req.inner.UpdateFileSourceTaskReq; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; + +/** + * 单个文件任务进度信息 + */ +@NoArgsConstructor +@Getter +@Setter +public class FileTaskProgressDTO { + /** + * ID + */ + private String fileSourceTaskId; + /** + * 文件路径（含bucketName） + */ + private String filePath; + /** + * 文件下载到机器上的真实路径 + */ + private String downloadPath; + /** + * 任务文件状态 + */ + private TaskStatusEnum status; + /** + * 文件大小（字节） + */ + private Long fileSize; + /** + * 速度 + */ + private String speed; + /** + * 进度 + */ + private Integer progress; + /** + * 日志内容 + */ + private String content; + + public static FileTaskProgressDTO fromUpdateFileSourceTaskReq(UpdateFileSourceTaskReq req) { + if (req == null) { + return null; + } + FileTaskProgressDTO fileTaskProgressDTO = new FileTaskProgressDTO(); + fileTaskProgressDTO.setFileSourceTaskId(req.getFileSourceTaskId()); + fileTaskProgressDTO.setFilePath(req.getFilePath()); + fileTaskProgressDTO.setDownloadPath(req.getDownloadPath()); + fileTaskProgressDTO.setStatus(req.getStatus()); + fileTaskProgressDTO.setFileSize(req.getFileSize()); + fileTaskProgressDTO.setSpeed(req.getSpeed()); + fileTaskProgressDTO.setProgress(req.getProgress()); + fileTaskProgressDTO.setContent(req.getContent()); + return fileTaskProgressDTO; + } +} diff --git a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/req/common/ExecuteActionReq.java b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/req/common/ExecuteActionReq.java index 1ecb460cdf..6914d55f64 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/req/common/ExecuteActionReq.java +++ b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/req/common/ExecuteActionReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/req/common/FileSourceMetaData.java b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/req/common/FileSourceMetaData.java index a3ada753fd..dcf707f497 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/req/common/FileSourceMetaData.java +++ b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/req/common/FileSourceMetaData.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -34,6 +34,7 @@ public class FileSourceMetaData { private String storageTypeCode; private String fileSourceTypeCode; private String name; + private Boolean enabled; private String iconPath; private String iconBase64; private List staticParamList; diff --git a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/req/common/FileSourceStaticParam.java b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/req/common/FileSourceStaticParam.java index 0fd0cf6221..4e38956751 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/req/common/FileSourceStaticParam.java +++ b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/req/common/FileSourceStaticParam.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/req/common/FileWorkerConfig.java b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/req/common/FileWorkerConfig.java index d6a57d50c8..fa89e612d9 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/req/common/FileWorkerConfig.java +++ b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/req/common/FileWorkerConfig.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/req/inner/HeartBeatReq.java b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/req/inner/HeartBeatReq.java index e93ef43d16..5cc8125d59 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/req/inner/HeartBeatReq.java +++ b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/req/inner/HeartBeatReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,7 @@ package com.tencent.bk.job.file_gateway.model.req.inner; +import com.tencent.bk.job.common.util.json.SkipLogFields; import com.tencent.bk.job.file_gateway.model.req.common.FileWorkerConfig; import io.swagger.annotations.ApiModelProperty; import lombok.AllArgsConstructor; @@ -44,6 +45,7 @@ public class HeartBeatReq { List tagList; @ApiModelProperty(value = "业务ID", required = true) Long appId; + @SkipLogFields(value = "token") @ApiModelProperty(value = "密钥", required = true) String token; @ApiModelProperty(value = "访问worker使用的host", required = true) @@ -52,6 +54,8 @@ public class HeartBeatReq { Integer accessPort; @ApiModelProperty(value = "worker所在云区域Id", required = true) Long cloudAreaId; + @ApiModelProperty(value = "worker的内网IP协议", required = true) + String innerIpProtocol; @ApiModelProperty(value = "worker的内网IP", required = true) String innerIp; @ApiModelProperty(value = "能力标签列表") @@ -82,6 +86,7 @@ public String toString() { ", accessHost='" + accessHost + '\'' + ", accessPort=" + accessPort + ", cloudAreaId=" + cloudAreaId + + ", innerIpProtocol='" + innerIpProtocol + '\'' + ", innerIp='" + innerIp + '\'' + ", abilityTagList=" + abilityTagList + ", cpuOverload=" + cpuOverload + diff --git a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/req/inner/OffLineAndReDispatchReq.java b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/req/inner/OffLineAndReDispatchReq.java index b091d4ed6c..adc07d094d 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/req/inner/OffLineAndReDispatchReq.java +++ b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/req/inner/OffLineAndReDispatchReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/req/inner/UpdateFileSourceTaskReq.java b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/req/inner/UpdateFileSourceTaskReq.java index 0c628ebe2f..5ccba2cb96 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/req/inner/UpdateFileSourceTaskReq.java +++ b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/req/inner/UpdateFileSourceTaskReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/CheckBoxEnable.java b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/CheckBoxEnable.java index a89bfc0ea1..34986ee662 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/CheckBoxEnable.java +++ b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/CheckBoxEnable.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/CompositeExpression.java b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/CompositeExpression.java index fa7c6c049b..7a3abfdfd7 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/CompositeExpression.java +++ b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/CompositeExpression.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/Expression.java b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/Expression.java index a4d39f3c89..a37342ce78 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/Expression.java +++ b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/Expression.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/FileLinkAction.java b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/FileLinkAction.java index 7f3e600421..f4f5f39d3a 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/FileLinkAction.java +++ b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/FileLinkAction.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/FileNodesDTO.java b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/FileNodesDTO.java index 203dc4a662..2041a5da23 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/FileNodesDTO.java +++ b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/FileNodesDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/FileNodesVO.java b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/FileNodesVO.java index c4a4606ca4..f5c211daf4 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/FileNodesVO.java +++ b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/FileNodesVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/FileTreeNodeButtonGroupProperty.java b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/FileTreeNodeButtonGroupProperty.java index 964576a4a5..fdbb72ee11 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/FileTreeNodeButtonGroupProperty.java +++ b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/FileTreeNodeButtonGroupProperty.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/FileTreeNodeCheckBoxProperty.java b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/FileTreeNodeCheckBoxProperty.java index 05dde5d178..dfc8bd2171 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/FileTreeNodeCheckBoxProperty.java +++ b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/FileTreeNodeCheckBoxProperty.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/FileTreeNodeDef.java b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/FileTreeNodeDef.java index fe1b834dd3..1aa4a0bf1b 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/FileTreeNodeDef.java +++ b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/FileTreeNodeDef.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/FileTreeNodeProperty.java b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/FileTreeNodeProperty.java index 31fd4071d3..29a3bce7ef 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/FileTreeNodeProperty.java +++ b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/FileTreeNodeProperty.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/FileTreeNodeTextProperty.java b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/FileTreeNodeTextProperty.java index 2756623711..245a86301a 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/FileTreeNodeTextProperty.java +++ b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/FileTreeNodeTextProperty.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/FileVO.java b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/FileVO.java index 95833205b6..e6d837e82f 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/FileVO.java +++ b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/FileVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/OperationEnum.java b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/OperationEnum.java index 119aeace4a..03d56fe314 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/OperationEnum.java +++ b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/OperationEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/PropertyAction.java b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/PropertyAction.java index b051ea2734..93d67b417f 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/PropertyAction.java +++ b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/PropertyAction.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/SimpleFileSourceVO.java b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/SimpleFileSourceVO.java index c1677ed42d..7b727cf03e 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/SimpleFileSourceVO.java +++ b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/SimpleFileSourceVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/WorkerAction.java b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/WorkerAction.java index 143939228c..674666807e 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/WorkerAction.java +++ b/src/backend/job-file-gateway/api-job-file-gateway-worker/src/main/java/com/tencent/bk/job/file_gateway/model/resp/common/WorkerAction.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway/build.gradle b/src/backend/job-file-gateway/api-job-file-gateway/build.gradle index 31ef32d34d..3fbffd3bb6 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway/build.gradle +++ b/src/backend/job-file-gateway/api-job-file-gateway/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/esb/EsbFileSourceV3Resource.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/esb/EsbFileSourceV3Resource.java index c70f08a808..ac09044124 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/esb/EsbFileSourceV3Resource.java +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/esb/EsbFileSourceV3Resource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,13 +25,19 @@ package com.tencent.bk.job.file_gateway.api.esb; import com.tencent.bk.job.common.annotation.EsbAPI; +import com.tencent.bk.job.common.constant.JobCommonHeaders; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.file_gateway.model.req.esb.v3.EsbCreateOrUpdateFileSourceV3Req; +import com.tencent.bk.job.file_gateway.model.req.esb.v3.EsbGetFileSourceDetailV3Req; import com.tencent.bk.job.file_gateway.model.resp.esb.v3.EsbFileSourceSimpleInfoV3DTO; +import com.tencent.bk.job.file_gateway.model.resp.esb.v3.EsbFileSourceV3DTO; import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; /** @@ -44,6 +50,8 @@ public interface EsbFileSourceV3Resource { @PostMapping("/create_file_source") EsbResp createFileSource( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbCreateOrUpdateFileSourceV3Req req @@ -51,9 +59,33 @@ EsbResp createFileSource( @PostMapping("/update_file_source") EsbResp updateFileSource( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbCreateOrUpdateFileSourceV3Req req ); + @GetMapping("/get_file_source_detail") + EsbResp getFileSourceDetail( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, + @RequestParam(value = "bk_biz_id", required = false) Long bizId, + @RequestParam(value = "bk_scope_type", required = false) String scopeType, + @RequestParam(value = "bk_scope_id", required = false) String scopeId, + @RequestParam(value = "code") String code); + + /** + * 获取文件源详情 + * + * @param req 查询请求 + * @return 文件源详情 + */ + @PostMapping("/get_file_source_detail") + EsbResp getFileSourceDetailUsingPost( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, + @RequestBody + @Validated + EsbGetFileSourceDetailV3Req req); } diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/iam/IamFileSourceCallbackResource.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/iam/IamFileSourceCallbackResource.java index f9e65a52e2..028115e428 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/iam/IamFileSourceCallbackResource.java +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/iam/IamFileSourceCallbackResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/inner/ServiceFileSourceResource.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/inner/ServiceFileSourceResource.java index 52f6c27c29..41201cc2b9 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/inner/ServiceFileSourceResource.java +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/inner/ServiceFileSourceResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,24 +24,39 @@ package com.tencent.bk.job.file_gateway.api.inner; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; import com.tencent.bk.job.common.annotation.InternalAPI; +import com.tencent.bk.job.common.constant.CompatibleType; import com.tencent.bk.job.common.model.InternalResponse; +import com.tentent.bk.job.common.api.feign.annotation.SmartFeignClient; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; @Api(tags = {"job-file-gateway:service:FileSource"}) -@RequestMapping("/service/fileSource/") -@RestController +@SmartFeignClient(value = "job-file-gateway", contextId = "fileSourceResource") @InternalAPI public interface ServiceFileSourceResource { @ApiOperation(value = "获取文件源ID", produces = "application/json") - @GetMapping("getFileSourceIdByCode/codes/{code}") + @GetMapping("/service/app/{appId}/fileSource/getFileSourceIdByCode/codes/{code}") InternalResponse getFileSourceIdByCode( + @ApiParam(value = "Job业务ID", required = true) @PathVariable("appId") Long appId, @ApiParam(value = "文件源标识", required = true) @PathVariable("code") String code); + + @Deprecated + @CompatibleImplementation(name = "fileSourceId", deprecatedVersion = "3.9.x", type = CompatibleType.DEPLOY, + explain = "文件源标识仅在appId下唯一，发布完成后可删除") + @ApiOperation(value = "获取文件源ID", produces = "application/json") + @GetMapping("/service/fileSource/getFileSourceIdByCode/codes/{code}") + InternalResponse getFileSourceIdByCode( + @ApiParam(value = "文件源标识", required = true) @PathVariable("code") String code); + + @ApiOperation(value = "判断是否存在文件源引用了指定凭证", produces = "application/json") + @GetMapping("/service/app/{appId}/fileSource/existsFileSourceUsingCredential/credentialIds/{credentialId}") + InternalResponse existsFileSourceUsingCredential( + @ApiParam(value = "Job业务ID", required = true) @PathVariable("appId") Long appId, + @ApiParam(value = "凭证ID", required = true) @PathVariable("credentialId") String credentialId); } diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/inner/ServiceFileSourceTaskResource.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/inner/ServiceFileSourceTaskResource.java index baca063393..80d5175c76 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/inner/ServiceFileSourceTaskResource.java +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/inner/ServiceFileSourceTaskResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,16 +26,12 @@ import com.tencent.bk.job.common.annotation.InternalAPI; import com.tencent.bk.job.common.model.InternalResponse; -import com.tencent.bk.job.file_gateway.model.req.inner.ClearBatchTaskFilesReq; import com.tencent.bk.job.file_gateway.model.req.inner.ClearTaskFilesReq; import com.tencent.bk.job.file_gateway.model.req.inner.FileSourceBatchDownloadTaskReq; -import com.tencent.bk.job.file_gateway.model.req.inner.FileSourceDownloadTaskReq; import com.tencent.bk.job.file_gateway.model.req.inner.StopBatchTaskReq; -import com.tencent.bk.job.file_gateway.model.req.inner.StopTaskReq; import com.tencent.bk.job.file_gateway.model.resp.inner.BatchTaskInfoDTO; import com.tencent.bk.job.file_gateway.model.resp.inner.BatchTaskStatusDTO; -import com.tencent.bk.job.file_gateway.model.resp.inner.FileSourceTaskStatusDTO; -import com.tencent.bk.job.file_gateway.model.resp.inner.TaskInfoDTO; +import com.tentent.bk.job.common.api.feign.annotation.SmartFeignClient; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; @@ -44,93 +40,49 @@ import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestHeader; -import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; @Api(tags = {"job-file-gateway:service:FileSourceTask"}) -@RequestMapping("/service/fileSource/filetask") -@RestController +@SmartFeignClient(value = "job-file-gateway", contextId = "fileSourceTaskResource") @InternalAPI public interface ServiceFileSourceTaskResource { - // 直接转发至FileWorker的请求，URL子路径保持一致 - @ApiOperation(value = "创建并启动文件下载任务", produces = "application/json") - @PostMapping("/downloadFiles/start") - InternalResponse startFileSourceDownloadTask( - @ApiParam("用户名") - @RequestHeader("username") - String username, - @ApiParam("文件源下载任务请求") - @RequestBody FileSourceDownloadTaskReq req - ); - @ApiOperation(value = "清理任务已下载的文件", produces = "application/json") - @PostMapping("/downloadFiles/stop") - InternalResponse stopTasks( - @ApiParam("文件源下载任务请求") - @RequestBody StopTaskReq req - ); - - @ApiOperation(value = "清理任务已下载的文件", produces = "application/json") - @PostMapping("/clearFiles") + @PostMapping("/service/fileSource/filetask/clearFiles") InternalResponse clearTaskFiles( @ApiParam("文件源下载任务请求") @RequestBody ClearTaskFilesReq req ); - // 文件网关自有资源请求 - @ApiOperation(value = "获取文件任务状态", produces = "application/json") - @GetMapping("/taskIds/{taskId}/status") - InternalResponse getFileSourceTaskStatusAndLogs( - @ApiParam("任务Id") - @PathVariable("taskId") - String taskId, - @ApiParam("日志开始位置") - @RequestParam(value = "logStart", required = false) - Long logStart, - @ApiParam("获取日志数量") - @RequestParam(value = "logLength", required = false) - Long logLength - ); - @ApiOperation(value = "创建并启动批量文件下载任务", produces = "application/json") - @PostMapping("/batch/downloadFiles/start") + @PostMapping("/service/fileSource/filetask/batch/downloadFiles/start") InternalResponse startFileSourceBatchDownloadTask( @ApiParam("用户名") @RequestHeader("username") - String username, + String username, @ApiParam("文件源下载任务请求") @RequestBody FileSourceBatchDownloadTaskReq req ); @ApiOperation(value = "清理批量任务已下载的文件", produces = "application/json") - @PostMapping("/batch/downloadFiles/stop") + @PostMapping("/service/fileSource/filetask/batch/downloadFiles/stop") InternalResponse stopBatchTasks( @ApiParam("文件源下载任务请求") @RequestBody StopBatchTaskReq req ); - @ApiOperation(value = "清理批量任务已下载的文件", produces = "application/json") - @PostMapping("/batch/clearFiles") - InternalResponse clearBatchTaskFiles( - @ApiParam("文件源下载任务请求") - @RequestBody ClearBatchTaskFilesReq req - ); - - // 文件网关自有资源请求 @ApiOperation(value = "获取文件批量任务状态", produces = "application/json") - @GetMapping("/batch/batchTaskIds/{batchTaskId}/status") + @GetMapping("/service/fileSource/filetask/batch/batchTaskIds/{batchTaskId}/status") InternalResponse getBatchTaskStatusAndLogs( @ApiParam("任务Id") @PathVariable("batchTaskId") - String batchTaskId, + String batchTaskId, @ApiParam("日志开始位置") @RequestParam(value = "logStart", required = false) - Long logStart, + Long logStart, @ApiParam("获取日志数量") @RequestParam(value = "logLength", required = false) - Long logLength + Long logLength ); } diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/op/FileSourceWhiteInfoOpResource.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/op/FileSourceWhiteInfoOpResource.java new file mode 100644 index 0000000000..65e31f9ba5 --- /dev/null +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/op/FileSourceWhiteInfoOpResource.java @@ -0,0 +1,82 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file_gateway.api.op; + +import com.tencent.bk.job.common.model.Response; +import com.tencent.bk.job.file_gateway.model.req.op.AddBkArtifactoryWhiteBaseUrlReq; +import com.tencent.bk.job.file_gateway.model.req.op.BatchDeleteFileSourceWhiteInfoReq; +import com.tencent.bk.job.file_gateway.model.resp.op.FileSourceWhiteInfoVO; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; + + +@Api(tags = {"job-file-gateway:OP:文件源白名单OP管理接口"}) +@RequestMapping("/op/fileSourceWhiteInfo") +@RestController +public interface FileSourceWhiteInfoOpResource { + + @ApiOperation(value = "添加蓝鲸制品库地址白名单", produces = "application/json") + @PostMapping("/addBkArtifactoryWhiteBaseUrl") + Response addBkArtifactoryWhiteBaseUrl( + @ApiParam("用户名") + @RequestHeader("username") + String username, + @RequestBody + AddBkArtifactoryWhiteBaseUrlReq req + ); + + @ApiOperation(value = "查询白名单列表", produces = "application/json") + @GetMapping("/list") + Response> list( + @ApiParam("用户名") + @RequestHeader("username") + String username, + @RequestParam(value = "start", required = false, defaultValue = "0") + Integer start, + @RequestParam(value = "length", required = false, defaultValue = "10") + Integer length + ); + + @ApiOperation(value = "批量删除白名单", produces = "application/json") + @DeleteMapping("/batchDelete") + Response batchDelete( + @ApiParam("用户名") + @RequestHeader("username") + String username, + @RequestBody + BatchDeleteFileSourceWhiteInfoReq req + ); +} diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/op/OpResource.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/op/OpResource.java index ff5dddb423..38972d9047 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/op/OpResource.java +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/op/OpResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/web/WebFileResource.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/web/WebFileResource.java index 0e3e96d69d..eaf2362a9f 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/web/WebFileResource.java +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/web/WebFileResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/web/WebFileSourceResource.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/web/WebFileSourceResource.java index d68ebfd884..7c2ff6d165 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/web/WebFileSourceResource.java +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/web/WebFileSourceResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -34,6 +34,7 @@ import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; +import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; @@ -79,7 +80,7 @@ Response checkAlias( @ApiOperation(value = "新增文件源", produces = "application/json") @PostMapping("") - Response saveFileSource( + Response saveFileSource( @ApiParam("用户名，网关自动传入") @RequestHeader("username") String username, @@ -93,13 +94,14 @@ Response saveFileSource( @PathVariable(value = "scopeId") String scopeId, @ApiParam(value = "创建文件源请求") + @Validated @RequestBody FileSourceCreateUpdateReq fileSourceCreateUpdateReq ); @ApiOperation(value = "更新文件源", produces = "application/json") - @PutMapping("") - Response updateFileSource( + @PutMapping("/{id}") + Response updateFileSource( @ApiParam("用户名，网关自动传入") @RequestHeader("username") String username, @@ -112,7 +114,11 @@ Response updateFileSource( @ApiParam(value = "资源范围ID", required = true) @PathVariable(value = "scopeId") String scopeId, + @ApiParam(value = "文件源 ID", required = true) + @PathVariable("id") + Integer id, @ApiParam(value = "更新文件源请求") + @Validated @RequestBody FileSourceCreateUpdateReq fileSourceCreateUpdateReq ); diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/web/WebFileSourceTypeResource.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/web/WebFileSourceTypeResource.java index db7f580af8..7fc1208d6d 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/web/WebFileSourceTypeResource.java +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/web/WebFileSourceTypeResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/web/WebFileWorkerResource.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/web/WebFileWorkerResource.java index cb44041929..d0c25ec592 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/web/WebFileWorkerResource.java +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/web/WebFileWorkerResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/web/WebPermissionResource.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/web/WebPermissionResource.java index 27843f458c..32fd481ef2 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/web/WebPermissionResource.java +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/web/WebPermissionResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -40,7 +40,7 @@ import org.springframework.web.bind.annotation.RestController; @Api(tags = {"job-file-gateway:web:Permission"}) -@RequestMapping("/web/permission/") +@RequestMapping("/web/permission/fileGateway") @RestController @WebAPI public interface WebPermissionResource { diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/consts/FileGatewayConsts.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/consts/FileGatewayConsts.java index 2616cfccf8..084dfecff8 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/consts/FileGatewayConsts.java +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/consts/FileGatewayConsts.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/consts/FileSourceStatusEnum.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/consts/FileSourceStatusEnum.java index 7432637178..36332385fe 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/consts/FileSourceStatusEnum.java +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/consts/FileSourceStatusEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/consts/FileSourceWhiteInfoTypeConsts.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/consts/FileSourceWhiteInfoTypeConsts.java new file mode 100644 index 0000000000..566889cf4d --- /dev/null +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/consts/FileSourceWhiteInfoTypeConsts.java @@ -0,0 +1,34 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file_gateway.consts; + +/** + * 文件源白名单信息常量 + */ +public class FileSourceWhiteInfoTypeConsts { + + // 允许访问的蓝鲸制品库根地址 + public static final String BK_ARTIFACTORY_BASE_URL = "bk-artifactory.base_url"; +} diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/consts/FileWorkerOnlineStatusEnum.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/consts/FileWorkerOnlineStatusEnum.java index 396295845b..f7dcf5676d 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/consts/FileWorkerOnlineStatusEnum.java +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/consts/FileWorkerOnlineStatusEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/consts/WorkerSelectModeEnum.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/consts/WorkerSelectModeEnum.java index 90d078ec2e..25f16d0d57 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/consts/WorkerSelectModeEnum.java +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/consts/WorkerSelectModeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/consts/WorkerSelectScopeEnum.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/consts/WorkerSelectScopeEnum.java index f6b4bd402d..db175b2152 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/consts/WorkerSelectScopeEnum.java +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/consts/WorkerSelectScopeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/esb/v3/EsbCreateOrUpdateFileSourceV3Req.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/esb/v3/EsbCreateOrUpdateFileSourceV3Req.java index 52178bdf74..c5e0f6c32b 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/esb/v3/EsbCreateOrUpdateFileSourceV3Req.java +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/esb/v3/EsbCreateOrUpdateFileSourceV3Req.java @@ -1,8 +1,13 @@ package com.tencent.bk.job.file_gateway.model.req.esb.v3; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.tencent.bk.job.common.esb.model.EsbAppScopeReq; +import com.tencent.bk.job.common.validation.NotBlankField; import com.tencent.bk.job.common.validation.NotContainSpecialChar; +import com.tencent.bk.job.file_gateway.consts.FileSourceInfoConsts; +import com.tencent.bk.job.file_gateway.consts.FileSourceTypeEnum; +import com.tencent.bk.job.file_gateway.validate.ValidFileSourceInfo; import io.swagger.annotations.ApiModelProperty; import lombok.Data; import lombok.EqualsAndHashCode; @@ -18,18 +23,21 @@ public class EsbCreateOrUpdateFileSourceV3Req extends EsbAppScopeReq { * 文件源Code */ @ApiModelProperty(value = "文件源Code") + @NotBlankField(fieldName = "code") @NotContainSpecialChar(fieldName = "code") private String code; /** * 文件源别名 */ @ApiModelProperty(value = "文件源名称") + @NotBlankField(fieldName = "alias") @NotContainSpecialChar(fieldName = "alias") private String alias; /** * 文件源类型 */ @ApiModelProperty(value = "文件源类型") + @NotBlankField(fieldName = "type") @NotContainSpecialChar(fieldName = "type") private String type; @@ -38,12 +46,14 @@ public class EsbCreateOrUpdateFileSourceV3Req extends EsbAppScopeReq { */ @ApiModelProperty(value = "文件源信息Map") @JsonProperty(value = "access_params") + @ValidFileSourceInfo private Map accessParams = new HashMap<>(); /** * 文件源凭证Id */ @ApiModelProperty(value = "文件源凭证Id") @JsonProperty(value = "credential_id") + @NotBlankField(fieldName = "credential_id") @NotContainSpecialChar(fieldName = "credential_id") private String credentialId; /** @@ -52,4 +62,24 @@ public class EsbCreateOrUpdateFileSourceV3Req extends EsbAppScopeReq { @ApiModelProperty(value = "文件前缀：后台自动生成UUID传${UUID}，自定义字符串直接传") @JsonProperty(value = "file_prefix") private String filePrefix = ""; + + /** + * 判断是否为蓝鲸制品库类型的文件源 + * + * @return 布尔值 + */ + @JsonIgnore + public boolean isBlueKingArtifactoryType() { + return FileSourceTypeEnum.isBlueKingArtifactory(type); + } + + /** + * 获取蓝鲸制品库根地址 + * + * @return 蓝鲸制品库根地址 + */ + @JsonIgnore + public String getBkArtifactoryBaseUrl() { + return (String) accessParams.get(FileSourceInfoConsts.KEY_BK_ARTIFACTORY_BASE_URL); + } } diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/esb/v3/EsbGetFileSourceDetailV3Req.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/esb/v3/EsbGetFileSourceDetailV3Req.java new file mode 100644 index 0000000000..44ea316d1a --- /dev/null +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/esb/v3/EsbGetFileSourceDetailV3Req.java @@ -0,0 +1,21 @@ +package com.tencent.bk.job.file_gateway.model.req.esb.v3; + +import com.tencent.bk.job.common.esb.model.EsbAppScopeReq; +import com.tencent.bk.job.common.validation.NotBlankField; +import com.tencent.bk.job.common.validation.NotContainSpecialChar; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.EqualsAndHashCode; + +@EqualsAndHashCode(callSuper = true) +@Data +public class EsbGetFileSourceDetailV3Req extends EsbAppScopeReq { + + /** + * 文件源Code + */ + @ApiModelProperty(value = "文件源Code") + @NotBlankField(fieldName = "code") + @NotContainSpecialChar(fieldName = "code") + private String code; +} diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/inner/ClearBatchTaskFilesReq.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/inner/ClearBatchTaskFilesReq.java index 29451c853f..5b60bbbe3f 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/inner/ClearBatchTaskFilesReq.java +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/inner/ClearBatchTaskFilesReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/inner/ClearTaskFilesReq.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/inner/ClearTaskFilesReq.java index cbed6ce17e..c94711709d 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/inner/ClearTaskFilesReq.java +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/inner/ClearTaskFilesReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/inner/FileSourceBatchDownloadTaskReq.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/inner/FileSourceBatchDownloadTaskReq.java index a5f6364da5..22367de4a0 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/inner/FileSourceBatchDownloadTaskReq.java +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/inner/FileSourceBatchDownloadTaskReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/inner/FileSourceDownloadTaskReq.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/inner/FileSourceDownloadTaskReq.java index 6ab3058bfc..2a085db1cc 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/inner/FileSourceDownloadTaskReq.java +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/inner/FileSourceDownloadTaskReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/inner/FileSourceTaskContent.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/inner/FileSourceTaskContent.java index 419b654d62..c1d8383ed6 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/inner/FileSourceTaskContent.java +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/inner/FileSourceTaskContent.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -30,6 +30,7 @@ import lombok.NoArgsConstructor; import java.util.List; +import java.util.stream.Collectors; /** * 单个文件源的文件任务内容 @@ -42,4 +43,13 @@ public class FileSourceTaskContent { Integer fileSourceId; @ApiModelProperty(value = "文件路径列表") List filePathList; + + /** + * 获取去重后的文件路径列表 + * + * @return 去重后的文件路径列表 + */ + public List acquireUniqueFilePathList() { + return filePathList.stream().distinct().collect(Collectors.toList()); + } } diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/inner/StopBatchTaskReq.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/inner/StopBatchTaskReq.java index aa51d16ba3..d9ded451b0 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/inner/StopBatchTaskReq.java +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/inner/StopBatchTaskReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/inner/StopTaskReq.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/inner/StopTaskReq.java index 803a859558..4bb0b6ebe8 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/inner/StopTaskReq.java +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/inner/StopTaskReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/op/AddBkArtifactoryWhiteBaseUrlReq.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/op/AddBkArtifactoryWhiteBaseUrlReq.java new file mode 100644 index 0000000000..f5498b5d40 --- /dev/null +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/op/AddBkArtifactoryWhiteBaseUrlReq.java @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file_gateway.model.req.op; + + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +@Data +@ApiModel("添加文件源蓝鲸制品库地址白名单请求内容") +public class AddBkArtifactoryWhiteBaseUrlReq { + + /** + * 根地址 + */ + @ApiModelProperty(value = "蓝鲸制品库根地址") + private String baseUrl; + + /** + * 备注 + */ + @ApiModelProperty(value = "备注") + private String remark; +} diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/op/BatchDeleteFileSourceWhiteInfoReq.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/op/BatchDeleteFileSourceWhiteInfoReq.java new file mode 100644 index 0000000000..267fb85bef --- /dev/null +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/op/BatchDeleteFileSourceWhiteInfoReq.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file_gateway.model.req.op; + + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.util.List; + +@Data +@ApiModel("删除文件源蓝鲸制品库地址白名单请求内容") +public class BatchDeleteFileSourceWhiteInfoReq { + + /** + * 需要删除的id列表 + */ + @ApiModelProperty(value = "需要删除的id列表") + private List idList; + +} diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/web/FileSourceCreateUpdateReq.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/web/FileSourceCreateUpdateReq.java index b61365befa..d99dad6fa7 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/web/FileSourceCreateUpdateReq.java +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/web/FileSourceCreateUpdateReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,7 +25,12 @@ package com.tencent.bk.job.file_gateway.model.req.web; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.tencent.bk.job.common.model.dto.ResourceScope; +import com.tencent.bk.job.common.validation.NotBlankField; +import com.tencent.bk.job.file_gateway.consts.FileSourceInfoConsts; +import com.tencent.bk.job.file_gateway.consts.FileSourceTypeEnum; +import com.tencent.bk.job.file_gateway.validate.ValidFileSourceInfo; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; @@ -37,28 +42,31 @@ @ApiModel("文件源创建、更新请求") public class FileSourceCreateUpdateReq { - @ApiModelProperty(value = "ID,更新文件源的时候需要传入，新建文件源不需要", required = false) - private Integer id; /** * 文件源Code */ @ApiModelProperty(value = "文件源Code", required = true) + @NotBlankField(fieldName = "code") private String code; /** * 文件源别名 */ @ApiModelProperty(value = "文件源名称", required = true) + @NotBlankField(fieldName = "alias") private String alias; @ApiModelProperty(value = "存储类型", required = true) + @NotBlankField(fieldName = "storageType") private String storageType; @ApiModelProperty(value = "文件源类型Code", required = true) + @NotBlankField(fieldName = "fileSourceTypeCode") private String fileSourceTypeCode; /** * 文件源信息Map */ @ApiModelProperty(value = "文件源信息Map") + @ValidFileSourceInfo private Map fileSourceInfoMap; /** * 是否为公共文件源 @@ -79,6 +87,7 @@ public class FileSourceCreateUpdateReq { * 文件源凭证Id */ @ApiModelProperty(value = "文件源凭证Id", required = true) + @NotBlankField(fieldName = "credentialId") private String credentialId; /** * 文件前缀 @@ -89,15 +98,37 @@ public class FileSourceCreateUpdateReq { * 接入点选择范围 */ @ApiModelProperty(value = "接入点选择范围:APP/PUBLIC/ALL，分别为业务私有接入点/公共接入点/全部", required = true) + @NotBlankField(fieldName = "workerSelectScope") private String workerSelectScope; /** * 接入点选择模式 */ @ApiModelProperty(value = "接入点选择模式：AUTO/MANUAL，分别为自动/手动", required = true) + @NotBlankField(fieldName = "workerSelectMode") private String workerSelectMode; /** * 接入点Id */ @ApiModelProperty(value = "接入点Id，手动选择时传入，自动选择不传") private Long workerId; + + /** + * 判断是否为蓝鲸制品库类型的文件源 + * + * @return 布尔值 + */ + @JsonIgnore + public boolean isBlueKingArtifactoryType() { + return FileSourceTypeEnum.isBlueKingArtifactory(fileSourceTypeCode); + } + + /** + * 获取蓝鲸制品库根地址 + * + * @return 蓝鲸制品库根地址 + */ + @JsonIgnore + public String getBkArtifactoryBaseUrl() { + return (String) fileSourceInfoMap.get(FileSourceInfoConsts.KEY_BK_ARTIFACTORY_BASE_URL); + } } diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/web/OperationPermissionReq.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/web/OperationPermissionReq.java index ed13d83a7e..43c77e1a22 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/web/OperationPermissionReq.java +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/web/OperationPermissionReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/web/UploadBucketFilesReq.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/web/UploadBucketFilesReq.java index 3f31dcb8a6..ff6b0917b9 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/web/UploadBucketFilesReq.java +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/req/web/UploadBucketFilesReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/esb/v3/EsbFileSourceSimpleInfoV3DTO.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/esb/v3/EsbFileSourceSimpleInfoV3DTO.java index aee7669c92..e694b18081 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/esb/v3/EsbFileSourceSimpleInfoV3DTO.java +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/esb/v3/EsbFileSourceSimpleInfoV3DTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/esb/v3/EsbFileSourceV3DTO.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/esb/v3/EsbFileSourceV3DTO.java new file mode 100644 index 0000000000..5cb07cf480 --- /dev/null +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/esb/v3/EsbFileSourceV3DTO.java @@ -0,0 +1,111 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file_gateway.model.resp.esb.v3; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; +import com.tencent.bk.job.common.esb.model.EsbAppScopeDTO; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + +@EqualsAndHashCode(callSuper = true) +@Data +@NoArgsConstructor +public class EsbFileSourceV3DTO extends EsbAppScopeDTO { + /** + * id + */ + @JsonPropertyDescription("File source id") + private Integer id; + /** + * 文件源标识 + */ + @JsonPropertyDescription("File source code") + private String code; + + /** + * 文件源别名 + */ + @JsonPropertyDescription("File source alias") + private String alias; + /** + * 状态 + */ + @JsonPropertyDescription("File source status") + private Integer status; + /** + * 文件源类型code + */ + @JsonProperty("file_source_type_code") + @JsonPropertyDescription("File source type code") + private String fileSourceTypeCode; + + /** + * 是否为公共文件源 + */ + @JsonProperty("is_public") + @JsonPropertyDescription("Is public file source") + private boolean publicFlag; + + /** + * 凭证Id + */ + @JsonPropertyDescription("File source credential id") + @JsonProperty("credential_id") + private String credentialId; + + /** + * 是否启用 + */ + @JsonPropertyDescription("Is file source enabled") + private Boolean enable; + + /** + * 创建人 + */ + @JsonPropertyDescription("Creator") + private String creator; + /** + * 创建时间 + */ + @JsonProperty("create_time") + @JsonPropertyDescription("Create time") + private Long createTime; + /** + * 更新人 + */ + @JsonProperty("last_modify_user") + @JsonPropertyDescription("Last modify user") + private String lastModifyUser; + + /** + * 更新时间 + */ + @JsonProperty("last_modify_time") + @JsonPropertyDescription("Last modify time") + private Long lastModifyTime; + +} diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/inner/BatchTaskInfoDTO.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/inner/BatchTaskInfoDTO.java index 5d4b695884..8932a5f4f7 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/inner/BatchTaskInfoDTO.java +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/inner/BatchTaskInfoDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/inner/BatchTaskStatusDTO.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/inner/BatchTaskStatusDTO.java index 85619f0202..1c16f28ae4 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/inner/BatchTaskStatusDTO.java +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/inner/BatchTaskStatusDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/inner/FileLogPieceDTO.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/inner/FileLogPieceDTO.java index f058c27856..be00df748c 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/inner/FileLogPieceDTO.java +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/inner/FileLogPieceDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/inner/FileSourceTaskStatusDTO.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/inner/FileSourceTaskStatusDTO.java index 697c4e5184..b5fc2e8ace 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/inner/FileSourceTaskStatusDTO.java +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/inner/FileSourceTaskStatusDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -43,6 +43,7 @@ public class FileSourceTaskStatusDTO { Byte status; String message; Long cloudId; + String ipProtocol; String ip; Boolean fileCleared; /** diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/inner/ServiceFileSourceDTO.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/inner/ServiceFileSourceDTO.java index 4092e7fd38..527f019607 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/inner/ServiceFileSourceDTO.java +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/inner/ServiceFileSourceDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/inner/ServiceFileSourceTypeDTO.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/inner/ServiceFileSourceTypeDTO.java index a449559666..0348a6de66 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/inner/ServiceFileSourceTypeDTO.java +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/inner/ServiceFileSourceTypeDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/inner/TaskInfoDTO.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/inner/TaskInfoDTO.java index 984af60df1..b2861d02c3 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/inner/TaskInfoDTO.java +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/inner/TaskInfoDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -35,6 +35,9 @@ public class TaskInfoDTO { String taskId; String fileSourceName; boolean fileSourcePublic; + Long workerId; + String workerAccessHost; Long cloudId; + String ipProtocol; String ip; } diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/inner/ThirdFileSourceTaskLogDTO.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/inner/ThirdFileSourceTaskLogDTO.java index 58bbaa4409..04638716c2 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/inner/ThirdFileSourceTaskLogDTO.java +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/inner/ThirdFileSourceTaskLogDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -29,13 +29,17 @@ import io.swagger.annotations.ApiModelProperty; import lombok.Data; +import java.io.Serializable; import java.util.ArrayList; import java.util.List; @ApiModel("第三方文件源文件下载任务日志") @Data @JsonInclude(JsonInclude.Include.NON_EMPTY) -public class ThirdFileSourceTaskLogDTO { +public class ThirdFileSourceTaskLogDTO implements Serializable { + + private static final long serialVersionUID = 1L; + /** * 文件源下载任务ID */ diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/op/FileSourceWhiteInfoVO.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/op/FileSourceWhiteInfoVO.java new file mode 100644 index 0000000000..29161f7540 --- /dev/null +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/op/FileSourceWhiteInfoVO.java @@ -0,0 +1,62 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file_gateway.model.resp.op; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * 文件源白名单信息 + */ +@Data +@NoArgsConstructor +@AllArgsConstructor +public class FileSourceWhiteInfoVO { + /** + * id + */ + private Integer id; + /** + * 类型 + */ + private String type; + /** + * 内容 + */ + private String content; + /** + * 备注 + */ + private String remark; + /** + * 创建人 + */ + private String creator; + /** + * 创建时间 + */ + private String createTime; +} diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/web/BaseFileWorkerVO.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/web/BaseFileWorkerVO.java index f7faa4876b..cd557a1f73 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/web/BaseFileWorkerVO.java +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/web/BaseFileWorkerVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/web/BucketVO.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/web/BucketVO.java index 4808672083..13dd487094 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/web/BucketVO.java +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/web/BucketVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/web/FileSourceTypeVO.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/web/FileSourceTypeVO.java index 17c14c25c5..4e21183306 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/web/FileSourceTypeVO.java +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/web/FileSourceTypeVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/web/FileSourceVO.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/web/FileSourceVO.java index 3ad0e193cb..0737d38094 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/web/FileSourceVO.java +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/web/FileSourceVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/web/FileWorkerVO.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/web/FileWorkerVO.java index bc0264cda4..44d688d04a 100644 --- a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/web/FileWorkerVO.java +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/resp/web/FileWorkerVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -60,11 +60,6 @@ public class FileWorkerVO { */ @ApiModelProperty("描述") private String description; - /** - * 密钥 - */ - @ApiModelProperty("密钥") - private String token; /** * 所在机器云区域Id */ diff --git a/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/validate/ValidFileSourceInfo.java b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/validate/ValidFileSourceInfo.java new file mode 100644 index 0000000000..f88c997bca --- /dev/null +++ b/src/backend/job-file-gateway/api-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/validate/ValidFileSourceInfo.java @@ -0,0 +1,90 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file_gateway.validate; + +import com.tencent.bk.job.common.util.StringUtil; +import com.tencent.bk.job.file_gateway.consts.FileSourceInfoConsts; +import org.apache.commons.lang3.StringUtils; + +import javax.validation.Constraint; +import javax.validation.ConstraintValidator; +import javax.validation.ConstraintValidatorContext; +import javax.validation.Payload; +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.Target; +import java.util.Map; + +import static java.lang.annotation.ElementType.ANNOTATION_TYPE; +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.ElementType.TYPE_USE; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +/** + * spring validation java 文件源信息合法校验 + */ +@Target({FIELD, METHOD, PARAMETER, ANNOTATION_TYPE, TYPE_USE}) +@Constraint(validatedBy = ValidFileSourceInfo.Validator.class) +@Documented +@Retention(RUNTIME) +public @interface ValidFileSourceInfo { + + String message() default "{validation.constraints.BkRepoBaseUrlInvalid.message}"; + + Class[] groups() default {}; + + Class[] payload() default {}; + + class Validator implements ConstraintValidator> { + + @Override + public boolean isValid(Map value, ConstraintValidatorContext constraintValidatorContext) { + if (value == null || value.isEmpty()) { + return true; + } + Object baseUrlObj = value.get(FileSourceInfoConsts.KEY_BK_ARTIFACTORY_BASE_URL); + if (!(baseUrlObj instanceof String)) { + return false; + } + String baseUrl = (String) baseUrlObj; + if (StringUtils.isBlank(baseUrl)) { + return false; + } + // 限制文件源根地址只能是 http(s)://domain 形式 + if (!isHttpOrHttpsUrl(baseUrl)) { + return false; + } + String urlWithoutScheme = StringUtil.removeHttpOrHttpsSchemeOfUrl(baseUrl); + return !urlWithoutScheme.contains("/"); + } + + private boolean isHttpOrHttpsUrl(String url) { + return url.startsWith("http://") || url.startsWith("https://"); + } + + } +} diff --git a/src/backend/job-file-gateway/boot-job-file-gateway/build.gradle b/src/backend/job-file-gateway/boot-job-file-gateway/build.gradle index 9708e91c0d..088fe01019 100644 --- a/src/backend/job-file-gateway/boot-job-file-gateway/build.gradle +++ b/src/backend/job-file-gateway/boot-job-file-gateway/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,11 +28,10 @@ dependencies { api project(":commons:common-redis") api project(":job-file-gateway:service-job-file-gateway") implementation 'org.springframework.boot:spring-boot-starter' - implementation 'org.springframework.boot:spring-boot-starter-jdbc' implementation 'org.springframework.cloud:spring-cloud-starter-bootstrap' implementation 'org.springframework:spring-webmvc' implementation(group: 'org.springframework.boot', name: 'spring-boot-starter-data-redis') - runtimeOnly('mysql:mysql-connector-java') + runtimeOnly('com.mysql:mysql-connector-j') } springBoot { getMainClass().set("com.tencent.bk.job.file_gateway.JobFileGatewayBootApplication") @@ -44,8 +43,7 @@ task renameArtifacts(type: Copy) { destinationDir file('build/libs/') rename "boot-job-file-gateway-${version}.jar", "job-file-gateway-${version}.jar" } -renameArtifacts.dependsOn jar -assemble.dependsOn renameArtifacts +renameArtifacts.dependsOn assemble task copyToLatestJar(type: Copy) { group = "local" @@ -57,3 +55,4 @@ task copyToLatestJar(type: Copy) { copyToLatestJar.dependsOn assemble apply from: "$rootDir/task_job_package.gradle" +copyToRelease.dependsOn renameArtifacts diff --git a/src/backend/job-file-gateway/boot-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/JobFileGatewayBootApplication.java b/src/backend/job-file-gateway/boot-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/JobFileGatewayBootApplication.java index be284ada6f..44fd620376 100644 --- a/src/backend/job-file-gateway/boot-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/JobFileGatewayBootApplication.java +++ b/src/backend/job-file-gateway/boot-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/JobFileGatewayBootApplication.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,22 +24,27 @@ package com.tencent.bk.job.file_gateway; -import com.tencent.bk.job.common.config.FeatureToggleConfig; +import com.tencent.bk.job.common.service.boot.JobBootApplication; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration; -import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.boot.autoconfigure.availability.ApplicationAvailabilityAutoConfiguration; +import org.springframework.boot.autoconfigure.jooq.JooqAutoConfiguration; import org.springframework.cache.annotation.EnableCaching; import org.springframework.cloud.openfeign.EnableFeignClients; import org.springframework.scheduling.annotation.EnableScheduling; @Slf4j -@SpringBootApplication(scanBasePackages = "com.tencent.bk.job", exclude = {RedisAutoConfiguration.class}) +@JobBootApplication( + scanBasePackages = "com.tencent.bk.job.file_gateway", + exclude = {JooqAutoConfiguration.class, ApplicationAvailabilityAutoConfiguration.class}, + excludeName = {"org.springframework.cloud.kubernetes.client.discovery.KubernetesDiscoveryClientAutoConfiguration"}) @EnableCaching -@EnableFeignClients +@EnableFeignClients( + basePackages = { + "com.tencent.bk.job.manage.api" + } +) @EnableScheduling -@EnableConfigurationProperties({FeatureToggleConfig.class}) public class JobFileGatewayBootApplication { public static void main(String[] args) { diff --git a/src/backend/job-file-gateway/boot-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/config/DbConfig.java b/src/backend/job-file-gateway/boot-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/config/DbConfig.java deleted file mode 100644 index d1620998c1..0000000000 --- a/src/backend/job-file-gateway/boot-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/config/DbConfig.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.file_gateway.config; - -import org.jooq.ConnectionProvider; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; -import org.jooq.impl.DataSourceConnectionProvider; -import org.jooq.impl.DefaultConfiguration; -import org.jooq.impl.DefaultDSLContext; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.boot.jdbc.DataSourceBuilder; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.DependsOn; -import org.springframework.context.annotation.Primary; -import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.jdbc.datasource.DataSourceTransactionManager; -import org.springframework.transaction.annotation.EnableTransactionManagement; - -import javax.sql.DataSource; - -@Configuration -@EnableTransactionManagement -public class DbConfig { - @Qualifier("job-file-gateway-data-source") - @Primary - @Bean(name = "job-file-gateway-data-source") - @ConfigurationProperties(prefix = "spring.datasource.job-file-gateway") - public DataSource dataSource() { - return DataSourceBuilder.create().build(); - } - - @Qualifier("transactionManager") - @Bean(name = "transactionManager") - @DependsOn("job-file-gateway-data-source") - @Primary - public DataSourceTransactionManager transactionManager( - @Qualifier("job-file-gateway-data-source") DataSource dataSource) { - return new DataSourceTransactionManager(dataSource); - } - - @Qualifier("job-file-gateway-jdbc-template") - @Bean(name = "job-file-gateway-jdbc-template") - public JdbcTemplate jdbcTemplate(@Qualifier("job-file-gateway-data-source") DataSource dataSource) { - return new JdbcTemplate(dataSource); - } - - @Qualifier("job-file-gateway-dsl-context") - @Bean(name = "job-file-gateway-dsl-context") - public DSLContext dslContext( - @Qualifier("job-file-gateway-jooq-conf") org.jooq.Configuration configuration) { - return new DefaultDSLContext(configuration); - } - - @Qualifier("job-file-gateway-jooq-conf") - @Bean(name = "job-file-gateway-jooq-conf") - public org.jooq.Configuration - jooqConf( - @Qualifier("job-file-gateway-conn-provider") ConnectionProvider connectionProvider) { - return new DefaultConfiguration().derive(connectionProvider).derive(SQLDialect.MYSQL); - } - - @Qualifier("job-file-gateway-conn-provider") - @Bean(name = "job-file-gateway-conn-provider") - public ConnectionProvider connectionProvider(@Qualifier("job-file-gateway-data-source") DataSource dataSource) { - return new DataSourceConnectionProvider(dataSource); - } - -} diff --git a/src/backend/job-file-gateway/boot-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/config/InterceptorConfiguration.java b/src/backend/job-file-gateway/boot-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/config/InterceptorConfiguration.java deleted file mode 100644 index 0a4aacc858..0000000000 --- a/src/backend/job-file-gateway/boot-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/config/InterceptorConfiguration.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.file_gateway.config; - -import com.tencent.bk.job.common.iam.interceptor.AuthAppInterceptor; -import com.tencent.bk.job.common.web.interceptor.EsbApiLogInterceptor; -import com.tencent.bk.job.common.web.interceptor.EsbReqRewriteInterceptor; -import com.tencent.bk.job.common.web.interceptor.JobCommonInterceptor; -import com.tencent.bk.job.common.web.interceptor.ServiceSecurityInterceptor; -import com.tencent.bk.job.file_gateway.interceptor.UriPermissionInterceptor; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Configuration; -import org.springframework.web.servlet.config.annotation.InterceptorRegistry; -import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; - -@Configuration -public class InterceptorConfiguration implements WebMvcConfigurer { - - private final JobCommonInterceptor jobCommonInterceptor; - private final AuthAppInterceptor authAppInterceptor; - private final EsbApiLogInterceptor esbApiLogInterceptor; - private final EsbReqRewriteInterceptor esbReqRewriteInterceptor; - private final ServiceSecurityInterceptor serviceSecurityInterceptor; - private final UriPermissionInterceptor uriPermissionInterceptor; - - @Autowired - public InterceptorConfiguration( - JobCommonInterceptor jobCommonInterceptor, - AuthAppInterceptor authAppInterceptor, - EsbApiLogInterceptor esbApiLogInterceptor, - EsbReqRewriteInterceptor esbReqRewriteInterceptor, - ServiceSecurityInterceptor serviceSecurityInterceptor, - UriPermissionInterceptor uriPermissionInterceptor - ) { - this.jobCommonInterceptor = jobCommonInterceptor; - this.authAppInterceptor = authAppInterceptor; - this.esbApiLogInterceptor = esbApiLogInterceptor; - this.esbReqRewriteInterceptor = esbReqRewriteInterceptor; - this.serviceSecurityInterceptor = serviceSecurityInterceptor; - this.uriPermissionInterceptor = uriPermissionInterceptor; - } - - @Override - public void addInterceptors(InterceptorRegistry registry) { - // 注册拦截器 - registry.addInterceptor(serviceSecurityInterceptor).addPathPatterns("/**").order(0); - registry.addInterceptor(jobCommonInterceptor).addPathPatterns("/**").order(10); - registry.addInterceptor(uriPermissionInterceptor) - .addPathPatterns(uriPermissionInterceptor.getControlUriPatternsList()).order(30); - registry.addInterceptor(esbApiLogInterceptor).addPathPatterns("/esb/api/**").order(40); - registry.addInterceptor(esbReqRewriteInterceptor).addPathPatterns("/esb/api/**").order(50); - registry.addInterceptor(authAppInterceptor).addPathPatterns("/web/**", "/esb/api/**").order(60); - } -} diff --git a/src/backend/job-file-gateway/boot-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/config/JobFileGatewayConfiguration.java b/src/backend/job-file-gateway/boot-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/config/JobFileGatewayConfiguration.java new file mode 100644 index 0000000000..2259742aa9 --- /dev/null +++ b/src/backend/job-file-gateway/boot-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/config/JobFileGatewayConfiguration.java @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file_gateway.config; + +import com.tencent.bk.job.common.service.AppScopeMappingService; +import com.tencent.bk.job.common.web.interceptor.AppResourceScopeInterceptor; +import com.tencent.bk.job.manage.AppScopeMappingServiceImpl; +import com.tencent.bk.job.manage.api.inner.ServiceApplicationResource; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class JobFileGatewayConfiguration { + @Bean + AppScopeMappingService appScopeMappingService(ServiceApplicationResource applicationResource) { + return new AppScopeMappingServiceImpl(applicationResource); + } + + @Bean + public AppResourceScopeInterceptor appResourceScopeInterceptor(AppScopeMappingService appScopeMappingService) { + return new AppResourceScopeInterceptor(appScopeMappingService); + } +} diff --git a/src/backend/job-file-gateway/boot-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/config/ScheduleConfig.java b/src/backend/job-file-gateway/boot-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/config/ScheduleConfig.java deleted file mode 100644 index 6061675bbd..0000000000 --- a/src/backend/job-file-gateway/boot-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/config/ScheduleConfig.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.file_gateway.config; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.context.annotation.Configuration; -import org.springframework.scheduling.annotation.SchedulingConfigurer; -import org.springframework.scheduling.config.ScheduledTaskRegistrar; - -import java.util.concurrent.ScheduledThreadPoolExecutor; - -/** - * @Description - * @Date 2020/2/24 - * @Version 1.0 - */ - -//设定一个长度5的定时任务线程池 -@Slf4j -@Configuration -public class ScheduleConfig implements SchedulingConfigurer { - @Override - public void configureTasks(ScheduledTaskRegistrar taskRegistrar) { - taskRegistrar.setScheduler(new ScheduledThreadPoolExecutor(5, (r, executor) -> log.error( - "ScheduledThreadPoolExecutor rejected a runnable"))); - } -} diff --git a/src/backend/job-file-gateway/boot-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/config/SwaggerConfig.java b/src/backend/job-file-gateway/boot-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/config/SwaggerConfig.java index 673df114f9..67c096aca4 100644 --- a/src/backend/job-file-gateway/boot-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/config/SwaggerConfig.java +++ b/src/backend/job-file-gateway/boot-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/config/SwaggerConfig.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,35 +24,40 @@ package com.tencent.bk.job.file_gateway.config; +import com.tencent.bk.job.common.service.config.JobCommonConfig; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Profile; import springfox.documentation.builders.PathSelectors; import springfox.documentation.builders.RequestHandlerSelectors; +import springfox.documentation.oas.annotations.EnableOpenApi; import springfox.documentation.spi.DocumentationType; import springfox.documentation.spring.web.plugins.Docket; -import springfox.documentation.swagger2.annotations.EnableSwagger2; import java.util.Arrays; import java.util.HashSet; +/** + * Swagger 配置 + */ @Configuration -@EnableSwagger2 -@Profile({"dev", "test", "local"}) +@EnableOpenApi +@Profile({"dev", "local"}) public class SwaggerConfig { - private final BkConfig bkConfig; + private final JobCommonConfig jobCommonConfig; @Autowired - public SwaggerConfig(BkConfig bkConfig) { - this.bkConfig = bkConfig; + public SwaggerConfig(JobCommonConfig jobCommonConfig) { + this.jobCommonConfig = jobCommonConfig; } @Bean public Docket api() { - return new Docket(DocumentationType.SWAGGER_2) - .host(bkConfig.getSwaggerUrl() + "/job-file-gateway") + return new Docket(DocumentationType.OAS_30) + .host(jobCommonConfig.getSwaggerUrl()) + .pathMapping("job-file-gateway") .protocols(new HashSet<>(Arrays.asList("http", "https"))) .select() .apis(RequestHandlerSelectors.basePackage("com.tencent.bk.job.file_gateway.api")) diff --git a/src/backend/job-file-gateway/boot-job-file-gateway/src/main/resources/application.yml b/src/backend/job-file-gateway/boot-job-file-gateway/src/main/resources/application.yml index 3e3c4fe8d6..c7d109b591 100644 --- a/src/backend/job-file-gateway/boot-job-file-gateway/src/main/resources/application.yml +++ b/src/backend/job-file-gateway/boot-job-file-gateway/src/main/resources/application.yml @@ -1,14 +1,22 @@ spring: profiles: active: prod -ribbon: - ServerListRefreshInterval: 3000 - + mvc: + pathmatch: + matching-strategy: ant_path_matcher management: + health: + livenessstate: + enabled: true + readinessstate: + enabled: true + # 使用spring cloud stream rabbitmq, 禁用掉springboot的rabbitmq health检查 + rabbit: + enabled: false endpoints: web: exposure: - include: shutdown,health,configprops,env,beans,conditions,loggers,metrics,mappings,prometheus,scheduledtasks,info,refresh + include: shutdown,health,configprops,env,beans,conditions,loggers,metrics,mappings,prometheus,scheduledtasks,info,refresh,busrefresh,bindings base-path: /actuator enabled-by-default: false endpoint: @@ -16,6 +24,7 @@ management: enabled: true health: enabled: true + show-details: when_authorized configprops: enabled: true env: @@ -38,6 +47,8 @@ management: enabled: true refresh: enabled: true + busrefresh: + enabled: true metrics: tags: application: ${spring.application.name} diff --git a/src/backend/job-file-gateway/boot-job-file-gateway/src/main/resources/bootstrap.yml b/src/backend/job-file-gateway/boot-job-file-gateway/src/main/resources/bootstrap.yml index 809540ddb0..ac908dea30 100644 --- a/src/backend/job-file-gateway/boot-job-file-gateway/src/main/resources/bootstrap.yml +++ b/src/backend/job-file-gateway/boot-job-file-gateway/src/main/resources/bootstrap.yml @@ -11,3 +11,5 @@ spring: name: ${JOB_APPLICATION_CONFIGMAP_NAME} - namespace: ${KUBERNETES_NAMESPACE} name: ${JOB_COMMON_CONFIGMAP_NAME} + - namespace: ${KUBERNETES_NAMESPACE} + name: ${JOB_COMMON_REFRESHABLE_CONFIGMAP_NAME} diff --git a/src/backend/job-file-gateway/boot-job-file-gateway/src/main/resources/logback-spring.xml b/src/backend/job-file-gateway/boot-job-file-gateway/src/main/resources/logback-spring.xml new file mode 100644 index 0000000000..33a7ab0a9e --- /dev/null +++ b/src/backend/job-file-gateway/boot-job-file-gateway/src/main/resources/logback-spring.xml @@ -0,0 +1,13 @@ + + + logback + + + + + + + + + + diff --git a/src/backend/job-file-gateway/boot-job-file-gateway/src/main/resources/logback/logback-app-props.xml b/src/backend/job-file-gateway/boot-job-file-gateway/src/main/resources/logback/logback-app-props.xml deleted file mode 100644 index 3782587094..0000000000 --- a/src/backend/job-file-gateway/boot-job-file-gateway/src/main/resources/logback/logback-app-props.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - diff --git a/src/backend/job-file-gateway/build.gradle b/src/backend/job-file-gateway/build.gradle index 60722b91f1..3bd96667fe 100644 --- a/src/backend/job-file-gateway/build.gradle +++ b/src/backend/job-file-gateway/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/model-job-file-gateway/build.gradle b/src/backend/job-file-gateway/model-job-file-gateway/build.gradle index 53cd347e49..8006176375 100644 --- a/src/backend/job-file-gateway/model-job-file-gateway/build.gradle +++ b/src/backend/job-file-gateway/model-job-file-gateway/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/service-job-file-gateway/build.gradle b/src/backend/job-file-gateway/service-job-file-gateway/build.gradle index 46fed4fdea..07486b109e 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/build.gradle +++ b/src/backend/job-file-gateway/service-job-file-gateway/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,6 +25,7 @@ dependencies { api project(":commons:common-web") api project(":commons:common-redis") + api project(":commons:common-mysql") api project(":job-manage:api-job-manage") api project(":job-file-worker-sdk:api-job-file-worker-sdk") api project(":job-file-gateway:api-job-file-gateway") @@ -34,4 +35,5 @@ dependencies { implementation "org.springframework.cloud:spring-cloud-starter-sleuth" implementation('org.springframework.cloud:spring-cloud-starter-openfeign') implementation 'org.apache.httpcomponents:httpclient' + implementation 'org.springframework.boot:spring-boot-starter-jdbc' } diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/esb/EsbFileSourceV3ResourceImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/esb/EsbFileSourceV3ResourceImpl.java index a8f3e14c10..4744b0feaf 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/esb/EsbFileSourceV3ResourceImpl.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/esb/EsbFileSourceV3ResourceImpl.java @@ -1,21 +1,25 @@ package com.tencent.bk.job.file_gateway.api.esb; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.audit.annotations.AuditRequestBody; import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.esb.metrics.EsbApiTimed; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.common.exception.FailedPreconditionException; import com.tencent.bk.job.common.exception.InvalidParamException; -import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; -import com.tencent.bk.job.common.iam.model.AuthResult; -import com.tencent.bk.job.common.model.dto.AppResourceScope; +import com.tencent.bk.job.common.iam.constant.ActionId; +import com.tencent.bk.job.common.metrics.CommonMetricNames; import com.tencent.bk.job.common.service.AppScopeMappingService; -import com.tencent.bk.job.file_gateway.auth.FileSourceAuthService; import com.tencent.bk.job.file_gateway.consts.WorkerSelectModeEnum; import com.tencent.bk.job.file_gateway.consts.WorkerSelectScopeEnum; import com.tencent.bk.job.file_gateway.model.dto.FileSourceDTO; import com.tencent.bk.job.file_gateway.model.dto.FileSourceTypeDTO; import com.tencent.bk.job.file_gateway.model.req.esb.v3.EsbCreateOrUpdateFileSourceV3Req; +import com.tencent.bk.job.file_gateway.model.req.esb.v3.EsbGetFileSourceDetailV3Req; import com.tencent.bk.job.file_gateway.model.resp.esb.v3.EsbFileSourceSimpleInfoV3DTO; +import com.tencent.bk.job.file_gateway.model.resp.esb.v3.EsbFileSourceV3DTO; import com.tencent.bk.job.file_gateway.service.FileSourceService; +import com.tencent.bk.job.file_gateway.service.validation.FileSourceValidateService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -27,47 +31,76 @@ @Slf4j public class EsbFileSourceV3ResourceImpl implements EsbFileSourceV3Resource { - private final FileSourceAuthService fileSourceAuthService; private final FileSourceService fileSourceService; private final AppScopeMappingService appScopeMappingService; + private final FileSourceValidateService fileSourceValidateService; @Autowired - public EsbFileSourceV3ResourceImpl(FileSourceAuthService fileSourceAuthService, - FileSourceService fileSourceService, - AppScopeMappingService appScopeMappingService) { - this.fileSourceAuthService = fileSourceAuthService; + public EsbFileSourceV3ResourceImpl(FileSourceService fileSourceService, + AppScopeMappingService appScopeMappingService, + FileSourceValidateService fileSourceValidateService) { this.fileSourceService = fileSourceService; this.appScopeMappingService = appScopeMappingService; + this.fileSourceValidateService = fileSourceValidateService; } @Override - public EsbResp createFileSource(EsbCreateOrUpdateFileSourceV3Req req) { - req.fillAppResourceScope(appScopeMappingService); + @AuditEntry(actionId = ActionId.CREATE_FILE_SOURCE) + @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "create_file_source"}) + public EsbResp createFileSource( + String username, + String appCode, + @AuditRequestBody EsbCreateOrUpdateFileSourceV3Req req) { Long appId = req.getAppId(); - String username = req.getUserName(); - checkCreateFileSourcePermission(username, req.getAppResourceScope()); checkCreateParam(req); - FileSourceDTO fileSourceDTO = buildFileSourceDTO(req.getUserName(), appId, null, req); - Integer fileSourceId = fileSourceService.saveFileSource(appId, fileSourceDTO); - boolean registerResult = fileSourceAuthService.registerFileSource( - username, fileSourceId, fileSourceDTO.getAlias()); - if (!registerResult) { - log.warn("Fail to register file_source to iam:({},{})", fileSourceId, fileSourceDTO.getAlias()); - } - return EsbResp.buildSuccessResp(new EsbFileSourceSimpleInfoV3DTO(fileSourceId)); + FileSourceDTO fileSourceDTO = buildFileSourceDTO(username, appId, null, req); + FileSourceDTO createdFileSource = fileSourceService.saveFileSource(username, appId, fileSourceDTO); + return EsbResp.buildSuccessResp(new EsbFileSourceSimpleInfoV3DTO(createdFileSource.getId())); } @Override - public EsbResp updateFileSource(EsbCreateOrUpdateFileSourceV3Req req) { - req.fillAppResourceScope(appScopeMappingService); + @AuditEntry(actionId = ActionId.MANAGE_FILE_SOURCE) + @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "update_file_source"}) + public EsbResp updateFileSource( + String username, + String appCode, + @AuditRequestBody EsbCreateOrUpdateFileSourceV3Req req) { Integer id = checkUpdateParamAndGetId(req); Long appId = req.getAppId(); - String username = req.getUserName(); - checkManageFileSourcePermission(username, req.getAppResourceScope(), id); - FileSourceDTO fileSourceDTO = buildFileSourceDTO(req.getUserName(), appId, id, req); - int affectedNum = fileSourceService.updateFileSourceById(appId, fileSourceDTO); - log.info("{} fileSource updated", affectedNum); - return EsbResp.buildSuccessResp(new EsbFileSourceSimpleInfoV3DTO(id)); + FileSourceDTO fileSourceDTO = buildFileSourceDTO(username, appId, id, req); + FileSourceDTO updateFileSource = fileSourceService.updateFileSourceById( + username, appId, fileSourceDTO); + return EsbResp.buildSuccessResp(new EsbFileSourceSimpleInfoV3DTO(updateFileSource.getId())); + } + + @Override + @AuditEntry(actionId = ActionId.VIEW_FILE_SOURCE) + @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "get_file_source_detail"}) + public EsbResp getFileSourceDetail( + String username, + String appCode, + Long bizId, + String scopeType, + String scopeId, + String code) { + EsbGetFileSourceDetailV3Req req = new EsbGetFileSourceDetailV3Req(); + req.setBizId(bizId); + req.setScopeType(scopeType); + req.setScopeId(scopeId); + req.setCode(code); + req.fillAppResourceScope(appScopeMappingService); + return getFileSourceDetailUsingPost(username, appCode, req); + } + + @Override + @AuditEntry(actionId = ActionId.VIEW_FILE_SOURCE) + @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "get_file_source_detail"}) + public EsbResp getFileSourceDetailUsingPost( + String username, + String appCode, + @AuditRequestBody EsbGetFileSourceDetailV3Req req) { + FileSourceDTO fileSourceDTO = fileSourceService.getFileSourceByCode(req.getAppId(), req.getCode()); + return EsbResp.buildSuccessResp(FileSourceDTO.toEsbFileSourceV3DTO(fileSourceDTO)); } private void checkCommonParam(EsbCreateOrUpdateFileSourceV3Req req) { @@ -84,21 +117,17 @@ private void checkCommonParam(EsbCreateOrUpdateFileSourceV3Req req) { private void checkCreateParam(EsbCreateOrUpdateFileSourceV3Req req) { String code = req.getCode(); - if (StringUtils.isBlank(code)) { - throw new InvalidParamException(ErrorCode.MISSING_PARAM_WITH_PARAM_NAME, - new String[]{"code"}); - } FileSourceTypeDTO fileSourceTypeDTO = fileSourceService.getFileSourceTypeByCode( req.getType() ); if (fileSourceTypeDTO == null) { - throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME, - new String[]{"type"}); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME, new String[]{"type"}); } if (fileSourceService.existsCode(req.getAppId(), code)) { throw new FailedPreconditionException(ErrorCode.FILE_SOURCE_CODE_ALREADY_EXISTS, new String[]{code}); } checkCommonParam(req); + checkBkArtifactoryBaseUrlIfNeed(req); } private Integer checkUpdateParamAndGetId(EsbCreateOrUpdateFileSourceV3Req req) { @@ -111,18 +140,22 @@ private Integer checkUpdateParamAndGetId(EsbCreateOrUpdateFileSourceV3Req req) { if (!fileSourceService.existsFileSource(appId, id)) { throw new FailedPreconditionException(ErrorCode.FILE_SOURCE_ID_NOT_IN_BIZ, new String[]{id.toString()}); } - if (StringUtils.isNotBlank(req.getType())) { - FileSourceTypeDTO fileSourceTypeDTO = fileSourceService.getFileSourceTypeByCode( - req.getType() - ); - if (fileSourceTypeDTO == null) { - throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME, - new String[]{"type"}); - } + FileSourceTypeDTO fileSourceTypeDTO = fileSourceService.getFileSourceTypeByCode(req.getType()); + if (fileSourceTypeDTO == null) { + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME, + new String[]{"type"}); } + checkBkArtifactoryBaseUrlIfNeed(req); return id; } + private void checkBkArtifactoryBaseUrlIfNeed(EsbCreateOrUpdateFileSourceV3Req req) { + if (req.isBlueKingArtifactoryType()) { + // 制品库类型的文件源需要校验根地址 + fileSourceValidateService.checkBkArtifactoryBaseUrl(req.getBkArtifactoryBaseUrl()); + } + } + private FileSourceDTO buildFileSourceDTO(String username, Long appId, Integer id, @@ -155,22 +188,4 @@ private FileSourceDTO buildFileSourceDTO(String username, fileSourceDTO.setLastModifyTime(System.currentTimeMillis()); return fileSourceDTO; } - - public void checkCreateFileSourcePermission(String username, AppResourceScope appResourceScope) { - // 需要拥有在业务下创建文件源的权限 - AuthResult authResult = fileSourceAuthService.authCreateFileSource(username, appResourceScope); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } - } - - public void checkManageFileSourcePermission(String username, AppResourceScope appResourceScope, - Integer fileSourceId) { - // 需要拥有在业务下管理某个具体文件源的权限 - AuthResult authResult = fileSourceAuthService.authManageFileSource(username, appResourceScope, - fileSourceId, null); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } - } } diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/iam/IamFileSourceCallbackResourceImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/iam/IamFileSourceCallbackResourceImpl.java index 4a1c0eea71..e3cb002e19 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/iam/IamFileSourceCallbackResourceImpl.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/iam/IamFileSourceCallbackResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,7 @@ package com.tencent.bk.job.file_gateway.api.iam; +import com.tencent.bk.audit.utils.json.JsonSchemaUtils; import com.tencent.bk.job.common.iam.constant.ResourceTypeId; import com.tencent.bk.job.common.iam.service.BaseIamCallbackService; import com.tencent.bk.job.common.iam.util.IamRespUtil; @@ -32,17 +33,20 @@ import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.file_gateway.model.dto.FileSourceBasicInfoDTO; import com.tencent.bk.job.file_gateway.model.dto.FileSourceDTO; +import com.tencent.bk.job.file_gateway.model.resp.esb.v3.EsbFileSourceV3DTO; import com.tencent.bk.job.file_gateway.service.FileSourceService; import com.tencent.bk.sdk.iam.dto.PathInfoDTO; import com.tencent.bk.sdk.iam.dto.callback.request.CallbackRequestDTO; import com.tencent.bk.sdk.iam.dto.callback.request.IamSearchCondition; import com.tencent.bk.sdk.iam.dto.callback.response.CallbackBaseResponseDTO; import com.tencent.bk.sdk.iam.dto.callback.response.FetchInstanceInfoResponseDTO; +import com.tencent.bk.sdk.iam.dto.callback.response.FetchResourceTypeSchemaResponseDTO; import com.tencent.bk.sdk.iam.dto.callback.response.InstanceInfoDTO; import com.tencent.bk.sdk.iam.dto.callback.response.ListInstanceResponseDTO; import com.tencent.bk.sdk.iam.dto.callback.response.SearchInstanceResponseDTO; import lombok.Data; import lombok.extern.slf4j.Slf4j; +import org.slf4j.helpers.MessageFormatter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RestController; @@ -103,7 +107,7 @@ private FileSourceSearchCondition getSearchCondition(CallbackRequestDTO callback List idStrList = searchCondition.getIdList(); List fileSourceIdList = null; if (idStrList != null) { - fileSourceIdList = idStrList.parallelStream().map(Integer::parseInt).collect(Collectors.toList()); + fileSourceIdList = idStrList.stream().map(Integer::parseInt).collect(Collectors.toList()); } int start = searchCondition.getStart().intValue(); @@ -199,7 +203,11 @@ protected CallbackBaseResponseDTO fetchInstanceResp( Integer id = Integer.parseInt(instanceId); fileSourceIdList.add(id); } catch (NumberFormatException e) { - log.error("Parse fileSource id failed!|{}", instanceId, e); + String msg = MessageFormatter.format( + "Parse fileSource id failed!|{}", + instanceId + ).getMessage(); + log.error(msg, e); } } List fileSourceBasicInfoDTOList = @@ -238,4 +246,12 @@ protected CallbackBaseResponseDTO fetchInstanceResp( public CallbackBaseResponseDTO callback(CallbackRequestDTO callbackRequest) { return baseCallback(callbackRequest); } + + @Override + protected FetchResourceTypeSchemaResponseDTO fetchResourceTypeSchemaResp( + CallbackRequestDTO callbackRequest) { + FetchResourceTypeSchemaResponseDTO resp = new FetchResourceTypeSchemaResponseDTO(); + resp.setData(JsonSchemaUtils.generateJsonSchema(EsbFileSourceV3DTO.class)); + return resp; + } } diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/inner/ServiceFileSourceResourceImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/inner/ServiceFileSourceResourceImpl.java index 95685a6d69..a598c4e023 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/inner/ServiceFileSourceResourceImpl.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/inner/ServiceFileSourceResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -45,6 +45,15 @@ public ServiceFileSourceResourceImpl(FileSourceService fileSourceService) { this.fileSourceService = fileSourceService; } + @Override + public InternalResponse getFileSourceIdByCode(Long appId, String code) { + FileSourceDTO fileSourceDTO = fileSourceService.getFileSourceByCode(appId, code); + if (null == fileSourceDTO) { + throw new NotFoundException(ErrorCode.FAIL_TO_FIND_FILE_SOURCE_BY_CODE, new String[]{code}); + } + return InternalResponse.buildSuccessResp(fileSourceDTO.getId()); + } + @Override public InternalResponse getFileSourceIdByCode(String code) { FileSourceDTO fileSourceDTO = fileSourceService.getFileSourceByCode(code); @@ -53,4 +62,10 @@ public InternalResponse getFileSourceIdByCode(String code) { } return InternalResponse.buildSuccessResp(fileSourceDTO.getId()); } + + @Override + public InternalResponse existsFileSourceUsingCredential(Long appId, String credentialId) { + boolean result = fileSourceService.existsFileSourceUsingCredential(appId, credentialId); + return InternalResponse.buildSuccessResp(result); + } } diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/inner/ServiceFileSourceTaskResourceImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/inner/ServiceFileSourceTaskResourceImpl.java index bbfda90faf..249ede3b3a 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/inner/ServiceFileSourceTaskResourceImpl.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/inner/ServiceFileSourceTaskResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,16 +25,11 @@ package com.tencent.bk.job.file_gateway.api.inner; import com.tencent.bk.job.common.model.InternalResponse; -import com.tencent.bk.job.file_gateway.model.req.inner.ClearBatchTaskFilesReq; import com.tencent.bk.job.file_gateway.model.req.inner.ClearTaskFilesReq; import com.tencent.bk.job.file_gateway.model.req.inner.FileSourceBatchDownloadTaskReq; -import com.tencent.bk.job.file_gateway.model.req.inner.FileSourceDownloadTaskReq; import com.tencent.bk.job.file_gateway.model.req.inner.StopBatchTaskReq; -import com.tencent.bk.job.file_gateway.model.req.inner.StopTaskReq; import com.tencent.bk.job.file_gateway.model.resp.inner.BatchTaskInfoDTO; import com.tencent.bk.job.file_gateway.model.resp.inner.BatchTaskStatusDTO; -import com.tencent.bk.job.file_gateway.model.resp.inner.FileSourceTaskStatusDTO; -import com.tencent.bk.job.file_gateway.model.resp.inner.TaskInfoDTO; import com.tencent.bk.job.file_gateway.service.BatchTaskService; import com.tencent.bk.job.file_gateway.service.FileSourceTaskService; import lombok.extern.slf4j.Slf4j; @@ -56,31 +51,6 @@ public ServiceFileSourceTaskResourceImpl(FileSourceTaskService fileSourceTaskSer this.batchTaskService = batchTaskService; } - @Override - public InternalResponse startFileSourceDownloadTask(String username, FileSourceDownloadTaskReq req) { - return InternalResponse.buildSuccessResp(fileSourceTaskService.startFileSourceDownloadTask(username, - req.getAppId(), req.getStepInstanceId(), req.getExecuteCount(), null, req.getFileSourceId(), - req.getFilePathList())); - } - - @Override - public InternalResponse stopTasks(StopTaskReq req) { - return InternalResponse.buildSuccessResp(fileSourceTaskService.stopTasks(req.getTaskIdList())); - } - - @Override - public InternalResponse getFileSourceTaskStatusAndLogs(String taskId, Long logStart, - Long logLength) { - if (logStart == null || logStart < 0) { - logStart = 0L; - } - if (logLength == null || logLength <= 0) { - logLength = -1L; - } - return InternalResponse.buildSuccessResp(fileSourceTaskService.getFileSourceTaskStatusAndLogs(taskId, - logStart, logLength)); - } - @Override public InternalResponse clearTaskFiles(ClearTaskFilesReq req) { return InternalResponse.buildSuccessResp(fileSourceTaskService.clearTaskFiles(req.getTaskIdList())); @@ -88,9 +58,16 @@ public InternalResponse clearTaskFiles(ClearTaskFilesReq req) { @Override public InternalResponse startFileSourceBatchDownloadTask(String username, - FileSourceBatchDownloadTaskReq req) { - return InternalResponse.buildSuccessResp(batchTaskService.startFileSourceBatchDownloadTask(username, - req.getAppId(), req.getStepInstanceId(), req.getExecuteCount(), req.getFileSourceTaskList())); + FileSourceBatchDownloadTaskReq req) { + return InternalResponse.buildSuccessResp( + batchTaskService.startFileSourceBatchDownloadTask( + username, + req.getAppId(), + req.getStepInstanceId(), + req.getExecuteCount(), + req.getFileSourceTaskList() + ) + ); } @Override @@ -99,20 +76,21 @@ public InternalResponse stopBatchTasks(StopBatchTaskReq req) { } @Override - public InternalResponse getBatchTaskStatusAndLogs(String batchTaskId, Long logStart, - Long logLength) { + public InternalResponse getBatchTaskStatusAndLogs(String batchTaskId, + Long logStart, + Long logLength) { if (logStart == null || logStart < 0) { logStart = 0L; } if (logLength == null || logLength <= 0) { logLength = -1L; } - return InternalResponse.buildSuccessResp(batchTaskService.getBatchTaskStatusAndLogs(batchTaskId, logStart, - logLength)); - } - - @Override - public InternalResponse clearBatchTaskFiles(ClearBatchTaskFilesReq req) { - return InternalResponse.buildSuccessResp(batchTaskService.clearBatchTaskFiles(req.getBatchTaskIdList())); + return InternalResponse.buildSuccessResp( + batchTaskService.getBatchTaskStatusAndLogs( + batchTaskId, + logStart, + logLength + ) + ); } } diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/op/FileSourceWhiteInfoOpResourceImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/op/FileSourceWhiteInfoOpResourceImpl.java new file mode 100644 index 0000000000..7652cdb64f --- /dev/null +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/op/FileSourceWhiteInfoOpResourceImpl.java @@ -0,0 +1,82 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file_gateway.api.op; + +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.model.Response; +import com.tencent.bk.job.file_gateway.consts.FileSourceWhiteInfoTypeConsts; +import com.tencent.bk.job.file_gateway.dao.filesource.FileSourceWhiteInfoDAO; +import com.tencent.bk.job.file_gateway.model.dto.FileSourceWhiteInfoDTO; +import com.tencent.bk.job.file_gateway.model.req.op.AddBkArtifactoryWhiteBaseUrlReq; +import com.tencent.bk.job.file_gateway.model.req.op.BatchDeleteFileSourceWhiteInfoReq; +import com.tencent.bk.job.file_gateway.model.resp.op.FileSourceWhiteInfoVO; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; +import java.util.stream.Collectors; + + +@Slf4j +@RestController("jobFileGatewayFileSourceWhiteInfoOpResourceImpl") +public class FileSourceWhiteInfoOpResourceImpl implements FileSourceWhiteInfoOpResource { + + private final FileSourceWhiteInfoDAO fileSourceWhiteInfoDAO; + + @Autowired + public FileSourceWhiteInfoOpResourceImpl(FileSourceWhiteInfoDAO fileSourceWhiteInfoDAO) { + this.fileSourceWhiteInfoDAO = fileSourceWhiteInfoDAO; + } + + @Override + public Response addBkArtifactoryWhiteBaseUrl(String username, AddBkArtifactoryWhiteBaseUrlReq req) { + if(fileSourceWhiteInfoDAO.exists(FileSourceWhiteInfoTypeConsts.BK_ARTIFACTORY_BASE_URL, req.getBaseUrl())){ + return Response.buildCommonFailResp(ErrorCode.FILE_SOURCE_WHITE_INFO_ALREADY_EXISTS); + } + FileSourceWhiteInfoDTO fileSourceWhiteInfoDTO = new FileSourceWhiteInfoDTO(); + fileSourceWhiteInfoDTO.setType(FileSourceWhiteInfoTypeConsts.BK_ARTIFACTORY_BASE_URL); + fileSourceWhiteInfoDTO.setContent(req.getBaseUrl()); + fileSourceWhiteInfoDTO.setRemark(req.getRemark()); + fileSourceWhiteInfoDTO.setCreator(username); + Integer id = fileSourceWhiteInfoDAO.insert(fileSourceWhiteInfoDTO); + return Response.buildSuccessResp(id); + } + + @Override + public Response> list(String username, Integer start, Integer length) { + List fileSourceWhiteInfoDTOList = fileSourceWhiteInfoDAO.list(start, length); + List resultList = fileSourceWhiteInfoDTOList.stream() + .map(FileSourceWhiteInfoDTO::toFileSourceWhiteInfoVO) + .collect(Collectors.toList()); + return Response.buildSuccessResp(resultList); + } + + @Override + public Response batchDelete(String username, BatchDeleteFileSourceWhiteInfoReq req) { + int affectedNum = fileSourceWhiteInfoDAO.delete(req.getIdList()); + return Response.buildSuccessResp(affectedNum); + } +} diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/op/OpResourceImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/op/OpResourceImpl.java index 423cc87727..c267cd60fb 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/op/OpResourceImpl.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/op/OpResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,14 +25,14 @@ package com.tencent.bk.job.file_gateway.api.op; import com.tencent.bk.job.common.model.Response; -import com.tencent.bk.job.file_gateway.service.ReDispatchService; +import com.tencent.bk.job.file_gateway.service.dispatch.ReDispatchService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RestController; @Slf4j -@RestController +@RestController("jobFileGatewayOpResourceImpl") public class OpResourceImpl implements OpResource { private final ReDispatchService reDispatchService; @@ -44,6 +44,6 @@ public OpResourceImpl(ReDispatchService reDispatchService) { @Override public Response getReDispatchThreadsNum(String username) { - return Response.buildSuccessResp(reDispatchService.getReDispatchThreadsNum(username)); + return Response.buildSuccessResp(reDispatchService.getReDispatchThreadsNum()); } } diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/remote/RemoteFileWorkerResourceImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/remote/RemoteFileWorkerResourceImpl.java index b8286e6710..2a384bc4bf 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/remote/RemoteFileWorkerResourceImpl.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/remote/RemoteFileWorkerResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,14 +25,14 @@ package com.tencent.bk.job.file_gateway.api.remote; import com.tencent.bk.job.common.model.Response; -import com.tencent.bk.job.file_gateway.consts.TaskStatusEnum; +import com.tencent.bk.job.file_gateway.model.dto.FileTaskProgressDTO; import com.tencent.bk.job.file_gateway.model.dto.FileWorkerDTO; import com.tencent.bk.job.file_gateway.model.req.inner.HeartBeatReq; import com.tencent.bk.job.file_gateway.model.req.inner.OffLineAndReDispatchReq; import com.tencent.bk.job.file_gateway.model.req.inner.UpdateFileSourceTaskReq; import com.tencent.bk.job.file_gateway.service.FileSourceTaskService; import com.tencent.bk.job.file_gateway.service.FileWorkerService; -import com.tencent.bk.job.file_gateway.service.ReDispatchService; +import com.tencent.bk.job.file_gateway.service.dispatch.ReDispatchService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RestController; @@ -65,17 +65,10 @@ public Response heartBeat(HeartBeatReq heartBeatReq) { @Override public Response updateFileSourceTask(UpdateFileSourceTaskReq updateFileSourceTaskReq) { - log.debug("Input=({})", updateFileSourceTaskReq); - String taskId = updateFileSourceTaskReq.getFileSourceTaskId(); - String filePath = updateFileSourceTaskReq.getFilePath(); - String downloadPath = updateFileSourceTaskReq.getDownloadPath(); - Long fileSize = updateFileSourceTaskReq.getFileSize(); - String speed = updateFileSourceTaskReq.getSpeed(); - Integer progress = updateFileSourceTaskReq.getProgress(); - String content = updateFileSourceTaskReq.getContent(); - TaskStatusEnum status = updateFileSourceTaskReq.getStatus(); - return Response.buildSuccessResp(fileSourceTaskService.updateFileSourceTask(taskId, filePath, - downloadPath, fileSize, speed, progress, content, status)); + log.info("updateFileSourceTaskReq=({})", updateFileSourceTaskReq); + FileTaskProgressDTO fileTaskProgressDTO = + FileTaskProgressDTO.fromUpdateFileSourceTaskReq(updateFileSourceTaskReq); + return Response.buildSuccessResp(fileSourceTaskService.updateFileSourceTask(fileTaskProgressDTO)); } @Override diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/web/WebFileResourceImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/web/WebFileResourceImpl.java index 8542618e20..326c64b456 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/web/WebFileResourceImpl.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/web/WebFileResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/web/WebFileSourceResourceImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/web/WebFileSourceResourceImpl.java index f1f2878fc6..4ccc01d4bc 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/web/WebFileSourceResourceImpl.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/web/WebFileSourceResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,11 +24,12 @@ package com.tencent.bk.job.file_gateway.api.web; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.audit.annotations.AuditRequestBody; import com.tencent.bk.job.common.constant.ErrorCode; -import com.tencent.bk.job.common.exception.FailedPreconditionException; import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.exception.ServiceException; -import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; +import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.iam.model.AuthResult; import com.tencent.bk.job.common.model.PageData; import com.tencent.bk.job.common.model.Response; @@ -42,9 +43,9 @@ import com.tencent.bk.job.file_gateway.model.req.web.FileSourceCreateUpdateReq; import com.tencent.bk.job.file_gateway.model.resp.web.FileSourceVO; import com.tencent.bk.job.file_gateway.service.FileSourceService; +import com.tencent.bk.job.file_gateway.service.validation.FileSourceValidateService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Pair; -import org.jooq.tools.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RestController; @@ -61,53 +62,24 @@ public class WebFileSourceResourceImpl implements WebFileSourceResource { private final FileSourceService fileSourceService; private final FileSourceAuthService fileSourceAuthService; private final AppScopeMappingService appScopeMappingService; + private final FileSourceValidateService fileSourceValidateService; @Autowired public WebFileSourceResourceImpl( FileSourceService fileSourceService, FileSourceAuthService fileSourceAuthService, - AppScopeMappingService appScopeMappingService) { + AppScopeMappingService appScopeMappingService, + FileSourceValidateService fileSourceValidateService) { this.fileSourceService = fileSourceService; this.fileSourceAuthService = fileSourceAuthService; this.appScopeMappingService = appScopeMappingService; + this.fileSourceValidateService = fileSourceValidateService; } - private void checkCodeBlank(String code) { - if (StringUtils.isBlank(code)) { - throw new InvalidParamException(ErrorCode.MISSING_PARAM_WITH_PARAM_NAME, new String[]{"code"}); - } - } - - private void confirmIdExists(Integer id) { - if (id == null || id <= 0) { - throw new InvalidParamException(ErrorCode.MISSING_OR_ILLEGAL_PARAM_WITH_PARAM_NAME, new String[]{"id"}); - } - } - - private void checkParam(Long appId, FileSourceCreateUpdateReq fileSourceCreateUpdateReq, boolean forCreate) { - Integer id = fileSourceCreateUpdateReq.getId(); - String code = fileSourceCreateUpdateReq.getCode(); - checkCodeBlank(code); - if (forCreate) { - // 创建 - if (fileSourceService.existsCode(appId, code)) { - throw new FailedPreconditionException( - ErrorCode.FILE_SOURCE_CODE_ALREADY_EXISTS, - new String[]{code} - ); - } - } else { - // 更新 - confirmIdExists(id); - if (fileSourceService.existsCodeExceptId(appId, code, id)) { - throw new FailedPreconditionException( - ErrorCode.FILE_SOURCE_CODE_ALREADY_EXISTS, - new String[]{code} - ); - } - } - if (StringUtils.isBlank(fileSourceCreateUpdateReq.getCredentialId())) { - throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME, new String[]{"credentialId"}); + private void checkParamSecurity(FileSourceCreateUpdateReq fileSourceCreateUpdateReq) { + if(fileSourceCreateUpdateReq.isBlueKingArtifactoryType()){ + // 制品库类型的文件源需要校验根地址 + fileSourceValidateService.checkBkArtifactoryBaseUrl(fileSourceCreateUpdateReq.getBkArtifactoryBaseUrl()); } } @@ -123,94 +95,74 @@ public Response checkAlias(String username, } @Override - public Response saveFileSource(String username, - AppResourceScope appResourceScope, - String scopeType, - String scopeId, - FileSourceCreateUpdateReq fileSourceCreateUpdateReq) { + @AuditEntry(actionId = ActionId.CREATE_FILE_SOURCE) + public Response saveFileSource( + String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + @AuditRequestBody FileSourceCreateUpdateReq req) { try { Long appId = appResourceScope.getAppId(); - AuthResult authResult = checkCreateFileSourcePermission(username, appResourceScope); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } - - checkParam(appId, fileSourceCreateUpdateReq, true); - FileSourceDTO fileSourceDTO = buildFileSourceDTO(username, appId, fileSourceCreateUpdateReq); - Integer fileSourceId = fileSourceService.saveFileSource(appId, fileSourceDTO); - boolean registerResult = fileSourceAuthService.registerFileSource( - username, fileSourceId, fileSourceDTO.getAlias()); - if (!registerResult) { - log.warn("Fail to register file_source to iam:({},{})", fileSourceId, fileSourceDTO.getAlias()); - } - return Response.buildSuccessResp(fileSourceId); + checkParamSecurity(req); + FileSourceDTO fileSourceDTO = buildFileSourceDTO(username, appId, null, req); + FileSourceDTO createdFileSource = fileSourceService.saveFileSource(username, appId, fileSourceDTO); + return Response.buildSuccessResp(FileSourceDTO.toVO(createdFileSource)); } catch (ServiceException e) { return Response.buildCommonFailResp(e.getErrorCode(), e.getErrorParams()); } } @Override - public Response updateFileSource(String username, - AppResourceScope appResourceScope, - String scopeType, - String scopeId, - FileSourceCreateUpdateReq fileSourceCreateUpdateReq) { + @AuditEntry(actionId = ActionId.MANAGE_FILE_SOURCE) + public Response updateFileSource( + String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + Integer id, + @AuditRequestBody FileSourceCreateUpdateReq req) { Long appId = appResourceScope.getAppId(); - log.info("Input=({},{},{})", username, appId, fileSourceCreateUpdateReq); - FileSourceDTO fileSourceDTO = buildFileSourceDTO(username, appId, fileSourceCreateUpdateReq); - AuthResult authResult = checkManageFileSourcePermission(username, appResourceScope, fileSourceDTO.getId()); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } - checkParam(appId, fileSourceCreateUpdateReq, false); - return Response.buildSuccessResp(fileSourceService.updateFileSourceById(appId, fileSourceDTO)); + log.info("Input=({},{},{})", username, appId, req); + FileSourceDTO fileSourceDTO = buildFileSourceDTO(username, appId, id, req); + checkParamSecurity(req); + + FileSourceDTO updateFileSource = fileSourceService.updateFileSourceById(username, appId, fileSourceDTO); + return Response.buildSuccessResp(FileSourceDTO.toVO(updateFileSource)); } @Override + @AuditEntry(actionId = ActionId.MANAGE_FILE_SOURCE) public Response deleteFileSource(String username, AppResourceScope appResourceScope, String scopeType, String scopeId, Integer id) { - Long appId = appResourceScope.getAppId(); - AuthResult authResult = checkManageFileSourcePermission(username, appResourceScope, id); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } - - return Response.buildSuccessResp(fileSourceService.deleteFileSourceById(appId, id)); + return Response.buildSuccessResp(fileSourceService.deleteFileSourceById(username, + appResourceScope.getAppId(), id)); } @Override + @AuditEntry(actionId = ActionId.MANAGE_FILE_SOURCE) public Response enableFileSource(String username, AppResourceScope appResourceScope, String scopeType, String scopeId, Integer id, Boolean enableFlag) { - Long appId = appResourceScope.getAppId(); - AuthResult authResult = checkManageFileSourcePermission(username, appResourceScope, id); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } - - return Response.buildSuccessResp(fileSourceService.enableFileSourceById(username, appId, id, - enableFlag)); + return Response.buildSuccessResp(fileSourceService.enableFileSourceById(username, + appResourceScope.getAppId(), id, enableFlag)); } @Override + @AuditEntry(actionId = ActionId.VIEW_FILE_SOURCE) public Response getFileSourceDetail(String username, AppResourceScope appResourceScope, String scopeType, String scopeId, Integer id) { - - AuthResult authResult = checkViewFileSourcePermission(username, appResourceScope, id); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } return Response.buildSuccessResp( - FileSourceDTO.toVO(fileSourceService.getFileSourceById(appResourceScope.getAppId(), id))); + FileSourceDTO.toVO(fileSourceService.getFileSourceById(username, appResourceScope.getAppId(), id))); } @Override @@ -230,7 +182,7 @@ public Response> listAvailableFileSource(String username, PageData pageData = new PageData<>(); Integer count = fileSourceService.countAvailableFileSource(appId, credentialId, alias); List resultList = fileSourceService.listAvailableFileSource(appId, credentialId, alias, start, - pageSize).parallelStream().map(FileSourceDTO::toVO).collect(Collectors.toList()); + pageSize).stream().map(FileSourceDTO::toVO).collect(Collectors.toList()); pageData.setTotal((long) count); pageData.setData(resultList); pageData.setStart(start); @@ -256,7 +208,7 @@ public Response> listWorkTableFileSource(String username, PageData pageData = new PageData<>(); Integer count = fileSourceService.countWorkTableFileSource(appId, credentialId, alias); List resultList = fileSourceService.listWorkTableFileSource(appId, credentialId, alias, start, - pageSize).parallelStream().map(FileSourceDTO::toVO).collect(Collectors.toList()); + pageSize).stream().map(FileSourceDTO::toVO).collect(Collectors.toList()); pageData.setTotal((long) count); pageData.setData(resultList); pageData.setStart(start); @@ -276,11 +228,11 @@ public Response> getFileSourceParams(String username return Response.buildSuccessResp(fileSourceService.getFileSourceParams(appId, fileSourceTypeCode)); } - private FileSourceDTO buildFileSourceDTO(String username, Long appId, + private FileSourceDTO buildFileSourceDTO(String username, Long appId, Integer fileSourceId, FileSourceCreateUpdateReq fileSourceCreateUpdateReq) { FileSourceDTO fileSourceDTO = new FileSourceDTO(); fileSourceDTO.setAppId(appId); - fileSourceDTO.setId(fileSourceCreateUpdateReq.getId()); + fileSourceDTO.setId(fileSourceId); fileSourceDTO.setCode(fileSourceCreateUpdateReq.getCode()); fileSourceDTO.setAlias(fileSourceCreateUpdateReq.getAlias()); fileSourceDTO.setStatus(null); @@ -356,7 +308,7 @@ private void addAvailableFileSourcePermissionData(String username, AppResourceSc private void addPermissionData(String username, AppResourceScope appResourceScope, PageData fileSourceVOPageData) { List fileSourceVOList = fileSourceVOPageData.getData(); - List currentAppFileSourceIdList = fileSourceVOList.parallelStream() + List currentAppFileSourceIdList = fileSourceVOList.stream() .map(FileSourceVO::getId) .collect(Collectors.toList()); // 添加权限数据 diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/web/WebFileSourceTypeResourceImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/web/WebFileSourceTypeResourceImpl.java index 5fb7e47de8..0a15d89cc8 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/web/WebFileSourceTypeResourceImpl.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/web/WebFileSourceTypeResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -57,7 +57,7 @@ public Response> listFileSourceType(String username, String scopeId, String storageType) { List resultList = - fileSourceService.listUniqueFileSourceType(storageType).parallelStream() + fileSourceService.listUniqueFileSourceType(storageType).stream() .map(FileSourceTypeDTO::toVO) .collect(Collectors.toList()); return Response.buildSuccessResp(resultList); diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/web/WebFileWorkerResourceImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/web/WebFileWorkerResourceImpl.java index 017e987ce0..bd5c173d44 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/web/WebFileWorkerResourceImpl.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/web/WebFileWorkerResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -62,7 +62,7 @@ public Response> listFileWorker(String username, return Response.buildSuccessResp(Collections.emptyList()); } List fileWorkerVOList = - fileWorkerDTOList.parallelStream().map(FileWorkerDTO::toVO).collect(Collectors.toList()); + fileWorkerDTOList.stream().map(FileWorkerDTO::toVO).collect(Collectors.toList()); return Response.buildSuccessResp(fileWorkerVOList); } } diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/web/WebPermissionResourceImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/web/WebPermissionResourceImpl.java index 9586bce246..b0ce7ed95e 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/web/WebPermissionResourceImpl.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/api/web/WebPermissionResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -37,7 +37,7 @@ import org.springframework.web.bind.annotation.RestController; @Slf4j -@RestController +@RestController("jobFileGatewayWebPermissionResourceImpl") public class WebPermissionResourceImpl implements WebPermissionResource { private final WebAuthService webAuthService; diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/auth/FileSourceAuthService.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/auth/FileSourceAuthService.java index 00879898ec..2cbd3ba28c 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/auth/FileSourceAuthService.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/auth/FileSourceAuthService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/auth/impl/FileSourceAuthServiceImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/auth/impl/FileSourceAuthServiceImpl.java index a5c18f4d85..437b25ff52 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/auth/impl/FileSourceAuthServiceImpl.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/auth/impl/FileSourceAuthServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -102,8 +102,8 @@ public List batchAuthViewFileSource(String username, List fileSourceIdList) { List allowedIdList = appAuthService.batchAuth(username, ActionId.VIEW_FILE_SOURCE, appResourceScope, ResourceTypeEnum.FILE_SOURCE, - fileSourceIdList.parallelStream().map(Object::toString).collect(Collectors.toList())); - return allowedIdList.parallelStream().map(Integer::valueOf).collect(Collectors.toList()); + fileSourceIdList.stream().map(Object::toString).collect(Collectors.toList())); + return allowedIdList.stream().map(Integer::valueOf).collect(Collectors.toList()); } @Override @@ -112,8 +112,8 @@ public List batchAuthManageFileSource(String username, List fileSourceIdList) { List allowedIdList = appAuthService.batchAuth(username, ActionId.MANAGE_FILE_SOURCE, appResourceScope, ResourceTypeEnum.FILE_SOURCE, - fileSourceIdList.parallelStream().map(Object::toString).collect(Collectors.toList())); - return allowedIdList.parallelStream().map(Integer::valueOf).collect(Collectors.toList()); + fileSourceIdList.stream().map(Object::toString).collect(Collectors.toList())); + return allowedIdList.stream().map(Integer::valueOf).collect(Collectors.toList()); } @Override diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/client/ServiceApplicationResourceClient.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/client/ServiceApplicationResourceClient.java deleted file mode 100644 index 450c9cee49..0000000000 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/client/ServiceApplicationResourceClient.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.file_gateway.client; - -import com.tencent.bk.job.manage.api.inner.ServiceApplicationResource; -import org.springframework.cloud.openfeign.FeignClient; - -/** - * 业务服务远程调用客户端 - */ -@FeignClient(value = "job-manage", contextId = "job-manage-app") -public interface ServiceApplicationResourceClient extends ServiceApplicationResource { -} diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/client/ServiceCredentialResourceClient.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/client/ServiceCredentialResourceClient.java deleted file mode 100644 index 143f36d2e2..0000000000 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/client/ServiceCredentialResourceClient.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.file_gateway.client; - -import com.tencent.bk.job.manage.api.inner.ServiceCredentialResource; -import org.springframework.cloud.openfeign.FeignClient; - -/** - * 业务服务远程调用客户端 - */ -@FeignClient(value = "job-manage", contextId = "job-manage-credential") -public interface ServiceCredentialResourceClient extends ServiceCredentialResource { -} diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/config/ArtifactoryConfig.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/config/ArtifactoryConfig.java new file mode 100644 index 0000000000..c7d5ddda46 --- /dev/null +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/config/ArtifactoryConfig.java @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file_gateway.config; + +import lombok.Data; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Configuration; + +@Data +@Configuration +public class ArtifactoryConfig { + + @Value("${artifactory.base-url:}") + private String artifactoryBaseUrl; + +} diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/config/BkConfig.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/config/BkConfig.java deleted file mode 100644 index d9640fadac..0000000000 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/config/BkConfig.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.file_gateway.config; - -import lombok.Getter; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Configuration; - -@Getter -@Configuration -public class BkConfig { - - @Value("${swagger.url:swagger.job.com}") - private String swaggerUrl; - -} diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/config/DbConfig.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/config/DbConfig.java new file mode 100644 index 0000000000..721e84aece --- /dev/null +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/config/DbConfig.java @@ -0,0 +1,102 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file_gateway.config; + +import com.tencent.bk.job.common.mysql.util.JooqConfigurationUtil; +import org.jooq.ConnectionProvider; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; +import org.jooq.impl.DataSourceConnectionProvider; +import org.jooq.impl.DefaultConfiguration; +import org.jooq.impl.DefaultDSLContext; +import org.jooq.impl.DefaultExecuteListenerProvider; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.jdbc.DataSourceBuilder; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.DependsOn; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.DataSourceTransactionManager; +import org.springframework.jdbc.datasource.TransactionAwareDataSourceProxy; +import org.springframework.transaction.annotation.EnableTransactionManagement; + +import javax.sql.DataSource; + +@Configuration(value = "jobFileGatewayDbConfig") +@EnableTransactionManagement +public class DbConfig { + @Qualifier("job-file-gateway-data-source") + @Bean(name = "job-file-gateway-data-source") + @ConfigurationProperties(prefix = "spring.datasource.job-file-gateway") + public DataSource dataSource() { + return DataSourceBuilder.create().build(); + } + + @Qualifier("jobFileGatewayTransactionManager") + @Bean(name = "jobFileGatewayTransactionManager") + @DependsOn("job-file-gateway-data-source") + public DataSourceTransactionManager transactionManager( + @Qualifier("job-file-gateway-data-source") DataSource dataSource) { + return new DataSourceTransactionManager(dataSource); + } + + @Qualifier("job-file-gateway-jdbc-template") + @Bean(name = "job-file-gateway-jdbc-template") + public JdbcTemplate jdbcTemplate(@Qualifier("job-file-gateway-data-source") DataSource dataSource) { + return new JdbcTemplate(dataSource); + } + + @Qualifier("job-file-gateway-dsl-context") + @Bean(name = "job-file-gateway-dsl-context") + public DSLContext dslContext( + @Qualifier("job-file-gateway-jooq-conf") org.jooq.Configuration configuration) { + return new DefaultDSLContext(configuration); + } + + @Qualifier("job-file-gateway-jooq-conf") + @Bean(name = "job-file-gateway-jooq-conf") + public org.jooq.Configuration + jooqConf( + @Qualifier("job-file-gateway-conn-provider") ConnectionProvider connectionProvider, + DefaultExecuteListenerProvider jooqExecuteListenerProvider + ) { + return JooqConfigurationUtil.getConfiguration(connectionProvider, jooqExecuteListenerProvider); + } + + @Qualifier("job-file-gateway-conn-provider") + @Bean(name = "job-file-gateway-conn-provider") + public ConnectionProvider connectionProvider( + @Qualifier("jobFileGatewayTransactionAwareDataSource") DataSource dataSource) { + return new DataSourceConnectionProvider(dataSource); + } + + @Qualifier("jobFileGatewayTransactionAwareDataSource") + @Bean(name = "jobFileGatewayTransactionAwareDataSource") + public TransactionAwareDataSourceProxy + transactionAwareDataSourceProxy(@Qualifier("job-file-gateway-data-source") DataSource dataSource) { + return new TransactionAwareDataSourceProxy(dataSource); + } +} diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/config/FileGatewayConfig.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/config/FileGatewayConfig.java index a3170bd99e..f78ea65bac 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/config/FileGatewayConfig.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/config/FileGatewayConfig.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/config/JobFileGatewayAutoConfiguration.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/config/JobFileGatewayAutoConfiguration.java index d521661381..da308edcbb 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/config/JobFileGatewayAutoConfiguration.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/config/JobFileGatewayAutoConfiguration.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,11 +24,8 @@ package com.tencent.bk.job.file_gateway.config; -import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.common.util.http.JobHttpClient; import com.tencent.bk.job.common.util.http.JobHttpClientImpl; -import com.tencent.bk.job.file_gateway.client.ServiceApplicationResourceClient; -import com.tencent.bk.job.manage.AppScopeMappingServiceImpl; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.web.client.RestTemplate; @@ -36,11 +33,6 @@ @Configuration public class JobFileGatewayAutoConfiguration { - @Bean - AppScopeMappingService appScopeMappingService(ServiceApplicationResourceClient applicationResource) { - return new AppScopeMappingServiceImpl(applicationResource); - } - @Bean JobHttpClient jobHttpClient(RestTemplate restTemplate) { return new JobHttpClientImpl(restTemplate); diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/config/RestTemplateConfig.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/config/RestTemplateConfig.java index c7562cdd9a..95081b5147 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/config/RestTemplateConfig.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/config/RestTemplateConfig.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,12 +26,19 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.http.client.OkHttp3ClientHttpRequestFactory; import org.springframework.web.client.RestTemplate; @Configuration public class RestTemplateConfig { @Bean public RestTemplate restTemplate() { - return new RestTemplate(); + RestTemplate restTemplate = new RestTemplate(); + OkHttp3ClientHttpRequestFactory requestFactory = new OkHttp3ClientHttpRequestFactory(); + requestFactory.setConnectTimeout(5000); + requestFactory.setReadTimeout(15000); + requestFactory.setWriteTimeout(15000); + restTemplate.setRequestFactory(requestFactory); + return restTemplate; } } diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/config/ScheduleConfig.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/config/ScheduleConfig.java new file mode 100644 index 0000000000..1c638c0d5f --- /dev/null +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/config/ScheduleConfig.java @@ -0,0 +1,49 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file_gateway.config; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.annotation.SchedulingConfigurer; +import org.springframework.scheduling.config.ScheduledTaskRegistrar; + +import java.util.concurrent.ScheduledThreadPoolExecutor; + +/** + * @Description + * @Date 2020/2/24 + * @Version 1.0 + */ + +//设定一个长度5的定时任务线程池 +@Slf4j +@Configuration +public class ScheduleConfig implements SchedulingConfigurer { + @Override + public void configureTasks(ScheduledTaskRegistrar taskRegistrar) { + taskRegistrar.setScheduler(new ScheduledThreadPoolExecutor(5, (r, executor) -> log.error( + "ScheduledThreadPoolExecutor rejected a runnable"))); + } +} diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/FileSourceBatchTaskDAO.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/FileSourceBatchTaskDAO.java index 4b6fdb5b14..49504ef3e6 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/FileSourceBatchTaskDAO.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/FileSourceBatchTaskDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,23 +25,14 @@ package com.tencent.bk.job.file_gateway.dao.filesource; import com.tencent.bk.job.file_gateway.model.dto.FileSourceBatchTaskDTO; -import org.jooq.DSLContext; - -import java.util.List; public interface FileSourceBatchTaskDAO { - String insertFileSourceBatchTask(DSLContext dslContext, FileSourceBatchTaskDTO fileSourceBatchTaskDTO); - - int updateFileSourceBatchTask(DSLContext dslContext, FileSourceBatchTaskDTO fileSourceBatchTaskDTO); - - int updateFileClearStatus(DSLContext dslContext, List taskIdList, boolean fileCleared); + String insertFileSourceBatchTask(FileSourceBatchTaskDTO fileSourceBatchTaskDTO); - int deleteFileSourceBatchTaskById(DSLContext dslContext, String id); + int updateFileSourceBatchTask(FileSourceBatchTaskDTO fileSourceBatchTaskDTO); - FileSourceBatchTaskDTO getFileSourceBatchTaskById(DSLContext dslContext, String id); + FileSourceBatchTaskDTO getBatchTaskById(String id); - Long countFileSourceBatchTasks(DSLContext dslContext, Long appId); + FileSourceBatchTaskDTO getBatchTaskByIdForUpdate(String id); - List listFileSourceBatchTasks(DSLContext dslContext, Long appId, Integer start, - Integer pageSize); } diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/FileSourceDAO.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/FileSourceDAO.java index d120c51052..bcecab22da 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/FileSourceDAO.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/FileSourceDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,47 +24,53 @@ package com.tencent.bk.job.file_gateway.dao.filesource; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.CompatibleType; import com.tencent.bk.job.file_gateway.model.dto.FileSourceBasicInfoDTO; import com.tencent.bk.job.file_gateway.model.dto.FileSourceDTO; -import org.jooq.DSLContext; import java.util.Collection; import java.util.List; public interface FileSourceDAO { - Integer insertFileSource(DSLContext dslContext, FileSourceDTO fileSourceDTO); + Integer insertFileSource(FileSourceDTO fileSourceDTO); - int updateFileSource(DSLContext dslContext, FileSourceDTO fileSourceDTO); + int updateFileSource(FileSourceDTO fileSourceDTO); - int updateFileSourceStatus(DSLContext dslContext, Integer fileSourceId, Integer status); + int updateFileSourceStatus(Integer fileSourceId, Integer status); - int deleteFileSourceById(DSLContext dslContext, Integer id); + int deleteFileSourceById(Integer id); - int enableFileSourceById(DSLContext dslContext, String username, Long appId, Integer id, Boolean enableFlag); + int enableFileSourceById(String username, Long appId, Integer id, Boolean enableFlag); - FileSourceDTO getFileSourceById(DSLContext dslContext, Integer id); + FileSourceDTO getFileSourceById(Integer id); - List listFileSourceByIds(DSLContext dslContext, Collection ids); + List listFileSourceByIds(Collection ids); - FileSourceDTO getFileSourceByCode(DSLContext dslContext, String code); + @Deprecated + @CompatibleImplementation(name = "fileSourceId", deprecatedVersion = "3.9.x", type = CompatibleType.DEPLOY, + explain = "文件源标识仅在appId下唯一，发布完成后可删除") + FileSourceDTO getFileSourceByCode(String code); - Integer countAvailableLikeFileSource(DSLContext dslContext, Long appId, String credentialId, String alias); + FileSourceDTO getFileSourceByCode(Long appId, String code); - Integer countFileSource(DSLContext dslContext, Long appId, String credentialId, String alias); + Integer countAvailableLikeFileSource(Long appId, String credentialId, String alias); - Integer countWorkTableFileSource(DSLContext dslContext, Long appId, String credentialId, String alias); + Integer countFileSource(Long appId, String credentialId, String alias); - Integer countWorkTableFileSource(DSLContext dslContext, List appIdList, List idList); + Integer countWorkTableFileSource(Long appId, String credentialId, String alias); - Boolean checkFileSourceExists(DSLContext dslContext, Long appId, String alias); + Integer countWorkTableFileSource(List appIdList, List idList); - List listAvailableFileSource(DSLContext dslContext, Long appId, String credentialId, String alias, + Boolean checkFileSourceExists(Long appId, String alias); + + List listAvailableFileSource(Long appId, String credentialId, String alias, Integer start, Integer pageSize); - List listWorkTableFileSource(DSLContext dslContext, Long appId, String credentialId, String alias, + List listWorkTableFileSource(Long appId, String credentialId, String alias, Integer start, Integer pageSize); - List listWorkTableFileSource(DSLContext dslContext, List appIdList, List idList, + List listWorkTableFileSource(List appIdList, List idList, Integer start, Integer pageSize); boolean existsCode(Long appId, String code); @@ -73,5 +79,7 @@ List listWorkTableFileSource(DSLContext dslContext, List ap boolean existsFileSource(Long appId, Integer id); + boolean existsFileSourceUsingCredential(Long appId, String credentialId); + Integer getFileSourceIdByCode(Long appId, String code); } diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/FileSourceTaskDAO.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/FileSourceTaskDAO.java index ea878b0a36..a1ada11e91 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/FileSourceTaskDAO.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/FileSourceTaskDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,32 +25,23 @@ package com.tencent.bk.job.file_gateway.dao.filesource; import com.tencent.bk.job.file_gateway.model.dto.FileSourceTaskDTO; -import org.jooq.DSLContext; -import java.util.Collection; import java.util.List; public interface FileSourceTaskDAO { - String insertFileSourceTask(DSLContext dslContext, FileSourceTaskDTO fileSourceTaskDTO); + String insertFileSourceTask(FileSourceTaskDTO fileSourceTaskDTO); - int updateFileSourceTask(DSLContext dslContext, FileSourceTaskDTO fileSourceTaskDTO); + int updateFileSourceTask(FileSourceTaskDTO fileSourceTaskDTO); - int updateFileClearStatus(DSLContext dslContext, List taskIdList, boolean fileCleared); + int updateFileClearStatus(List taskIdList, boolean fileCleared); - int deleteById(DSLContext dslContext, String id); + int deleteById(String id); - FileSourceTaskDTO getFileSourceTaskById(DSLContext dslContext, String id); + FileSourceTaskDTO getFileSourceTaskById(String id); - Long countFileSourceTasks(DSLContext dslContext, Long appId); + FileSourceTaskDTO getFileSourceTaskByIdForUpdate(String id); - Long countFileSourceTasksByBatchTaskId(DSLContext dslContext, String batchTaskId, Byte status); - - List listFileSourceTasks(DSLContext dslContext, Long appId, Integer start, Integer pageSize); - - List listTimeoutTasks(DSLContext dslContext, Long expireTimeMills, Collection statusSet, - Integer start, Integer pageSize); + Long countFileSourceTasksByBatchTaskId(String batchTaskId, Byte status); List listByBatchTaskId(String batchTaskId); - - int deleteByBatchTaskId(DSLContext dslContext, String batchTaskId); } diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/FileSourceTypeDAO.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/FileSourceTypeDAO.java index 0cf9482ffb..0126797056 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/FileSourceTypeDAO.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/FileSourceTypeDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,20 +25,16 @@ package com.tencent.bk.job.file_gateway.dao.filesource; import com.tencent.bk.job.file_gateway.model.dto.FileSourceTypeDTO; -import org.jooq.DSLContext; +import java.util.Collection; import java.util.List; public interface FileSourceTypeDAO { - Integer upsertByWorker(DSLContext dslContext, FileSourceTypeDTO fileSourceTypeDTO); + Integer batchInsert(List fileSourceTypeList); - Integer insert(DSLContext dslContext, FileSourceTypeDTO fileSourceTypeDTO); + int batchUpdate(List fileSourceTypeList); - int update(DSLContext dslContext, FileSourceTypeDTO fileSourceTypeDTO); - - int deleteById(DSLContext dslContext, Integer id); - - FileSourceTypeDTO get(DSLContext dslContext, Long workerId, String storageType, String code); + int batchDeleteByIds(Collection ids); FileSourceTypeDTO getById(Integer id); @@ -46,9 +42,9 @@ public interface FileSourceTypeDAO { List listByCode(String code); - List listByCodeOrderByVersion(String code); + List listByWorkerId(Long workerId); - List listOrderByVersion(DSLContext dslContext, String storageType); + List listByCodeOrderByVersion(String code); - List list(DSLContext dslContext, String storageType); + List listEnabledTypeOrderByVersion(String storageType); } diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/FileSourceWhiteInfoDAO.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/FileSourceWhiteInfoDAO.java new file mode 100644 index 0000000000..5eabb48c8d --- /dev/null +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/FileSourceWhiteInfoDAO.java @@ -0,0 +1,41 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file_gateway.dao.filesource; + +import com.tencent.bk.job.file_gateway.model.dto.FileSourceWhiteInfoDTO; + +import java.util.Collection; +import java.util.List; + +public interface FileSourceWhiteInfoDAO { + Integer insert(FileSourceWhiteInfoDTO fileSourceWhiteInfoDTO); + + List list(Integer offset, Integer limit); + + boolean exists(String type, String content); + + int delete(Collection ids); + +} diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/FileTaskDAO.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/FileTaskDAO.java index b373f9b7a1..909febf3f2 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/FileTaskDAO.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/FileTaskDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,32 +25,29 @@ package com.tencent.bk.job.file_gateway.dao.filesource; import com.tencent.bk.job.file_gateway.model.dto.FileTaskDTO; -import org.jooq.DSLContext; import java.util.Collection; import java.util.List; public interface FileTaskDAO { - Long insertFileTask(DSLContext dslContext, FileTaskDTO fileTaskDTO); + Long insertFileTask(FileTaskDTO fileTaskDTO); - int updateFileTask(DSLContext dslContext, FileTaskDTO fileTaskDTO); + int updateFileTask(FileTaskDTO fileTaskDTO); - int resetFileTasks(DSLContext dslContext, String fileSourceTaskId); + int resetFileTasks(String fileSourceTaskId); - int deleteFileTaskById(DSLContext dslContext, Long id); + int deleteFileTaskByFileSourceTaskId(String fileSourceTaskId); - int deleteFileTaskByFileSourceTaskId(DSLContext dslContext, String fileSourceTaskId); + FileTaskDTO getFileTaskByIdForUpdate(Long id); - FileTaskDTO getFileTaskById(DSLContext dslContext, Long id); - - FileTaskDTO getOneFileTask(DSLContext dslContext, String fileSourceTaskId, String filePath); - - Long countFileTasks(DSLContext dslContext, String fileSourceTaskId); + FileTaskDTO getOneFileTask(String fileSourceTaskId, String filePath); List listFileTasks(String fileSourceTaskId, Integer start, Integer pageSize); - List listTimeoutFileSourceTaskIds(DSLContext dslContext, Long expireTimeMills, Collection statusSet - , Integer start, Integer pageSize); + List listTimeoutFileSourceTaskIds(Long startTimeMills, + Long endTimeMills, + Collection statusSet, Integer start, + Integer pageSize); List listFileTasks(String fileSourceTaskId); diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/FileWorkerAbilityDAO.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/FileWorkerAbilityDAO.java index 7dbb0e5f93..b575844dc9 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/FileWorkerAbilityDAO.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/FileWorkerAbilityDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/FileWorkerDAO.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/FileWorkerDAO.java index 18e3d3e649..f924d9f6bc 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/FileWorkerDAO.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/FileWorkerDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/FileWorkerTagDAO.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/FileWorkerTagDAO.java index 4b2d9e8a62..2c492a6d06 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/FileWorkerTagDAO.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/FileWorkerTagDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/BaseDAOImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/BaseDAOImpl.java deleted file mode 100644 index a0255407a2..0000000000 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/BaseDAOImpl.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.file_gateway.dao.filesource.impl; - -import lombok.extern.slf4j.Slf4j; -import org.jooq.Record; -import org.jooq.Result; -import org.jooq.ResultQuery; -import org.jooq.SelectLimitStep; -import org.jooq.conf.ParamType; - -import java.util.Collections; -import java.util.List; - -/** - * 抽取通用查询方法 - */ -@Slf4j -public class BaseDAOImpl { - public List listPage(SelectLimitStep query, - Integer start, - Integer pageSize, - RecordDTOConverter converter) { - Result records = null; - if (start == null || start < 0) { - start = 0; - } - if (pageSize == null || pageSize < 0) { - pageSize = -1; - } - ResultQuery finalQuery = query; - String sql = ""; - try { - if (pageSize >= 0) { - finalQuery = query.limit(start, pageSize); - } - sql = finalQuery.getSQL(ParamType.INLINED); - log.debug("SQL={}", sql); - records = finalQuery.fetch(); - } catch (Exception e) { - log.error("error SQL={}", sql, e); - } - if (records == null || records.isEmpty()) { - return Collections.emptyList(); - } else { - return records.map(converter::convert); - } - } -} diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/FileSourceBatchTaskDAOImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/FileSourceBatchTaskDAOImpl.java index c5ab60fe4e..9ec2e5933c 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/FileSourceBatchTaskDAOImpl.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/FileSourceBatchTaskDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,25 +26,23 @@ import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.exception.InternalException; +import com.tencent.bk.job.common.mysql.dao.BaseDAOImpl; import com.tencent.bk.job.common.util.JobUUID; import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.file_gateway.dao.filesource.FileSourceBatchTaskDAO; import com.tencent.bk.job.file_gateway.dao.filesource.FileSourceTaskDAO; import com.tencent.bk.job.file_gateway.model.dto.FileSourceBatchTaskDTO; import com.tencent.bk.job.file_gateway.model.dto.FileSourceTaskDTO; +import com.tencent.bk.job.file_gateway.model.tables.FileSourceBatchTask; +import com.tencent.bk.job.file_gateway.model.tables.records.FileSourceBatchTaskRecord; import lombok.extern.slf4j.Slf4j; import lombok.val; -import org.jooq.Condition; import org.jooq.DSLContext; import org.jooq.conf.ParamType; -import org.jooq.generated.tables.FileSourceBatchTask; -import org.jooq.generated.tables.records.FileSourceBatchTaskRecord; -import org.jooq.impl.DSL; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; -import java.util.ArrayList; -import java.util.Collection; import java.util.List; @Slf4j @@ -52,10 +50,13 @@ public class FileSourceBatchTaskDAOImpl extends BaseDAOImpl implements FileSourceBatchTaskDAO { private static final FileSourceBatchTask defaultTable = FileSourceBatchTask.FILE_SOURCE_BATCH_TASK; + private final DSLContext dslContext; private final FileSourceTaskDAO fileSourceTaskDAO; @Autowired - public FileSourceBatchTaskDAOImpl(FileSourceTaskDAO fileSourceTaskDAO) { + public FileSourceBatchTaskDAOImpl(@Qualifier("job-file-gateway-dsl-context") DSLContext dslContext, + FileSourceTaskDAO fileSourceTaskDAO) { + this.dslContext = dslContext; this.fileSourceTaskDAO = fileSourceTaskDAO; } @@ -66,7 +67,7 @@ private void setDefaultValue(FileSourceBatchTaskDTO fileSourceBatchTaskDTO) { } @Override - public String insertFileSourceBatchTask(DSLContext dslContext, FileSourceBatchTaskDTO fileSourceBatchTaskDTO) { + public String insertFileSourceBatchTask(FileSourceBatchTaskDTO fileSourceBatchTaskDTO) { setDefaultValue(fileSourceBatchTaskDTO); String id = JobUUID.getUUID(); val query = dslContext.insertInto(defaultTable, @@ -103,7 +104,7 @@ public String insertFileSourceBatchTask(DSLContext dslContext, FileSourceBatchTa for (FileSourceTaskDTO fileSourceTaskDTO : fileSourceTaskDTOList) { fileSourceTaskDTO.setBatchTaskId(id); // 插入FileSourceTask - fileSourceTaskDAO.insertFileSourceTask(dslContext, fileSourceTaskDTO); + fileSourceTaskDAO.insertFileSourceTask(fileSourceTaskDTO); } fileSourceBatchTaskDTO.setFileSourceTaskList(fileSourceTaskDTOList); return id; @@ -114,7 +115,7 @@ public String insertFileSourceBatchTask(DSLContext dslContext, FileSourceBatchTa } @Override - public int updateFileSourceBatchTask(DSLContext dslContext, FileSourceBatchTaskDTO fileSourceBatchTaskDTO) { + public int updateFileSourceBatchTask(FileSourceBatchTaskDTO fileSourceBatchTaskDTO) { val query = dslContext.update(defaultTable) .set(defaultTable.APP_ID, fileSourceBatchTaskDTO.getAppId()) .set(defaultTable.STATUS, fileSourceBatchTaskDTO.getStatus()) @@ -130,29 +131,7 @@ public int updateFileSourceBatchTask(DSLContext dslContext, FileSourceBatchTaskD } @Override - public int updateFileClearStatus(DSLContext dslContext, List taskIdList, boolean fileCleared) { - val query = dslContext.update(defaultTable) - .set(defaultTable.FILE_CLEARED, fileCleared) - .set(defaultTable.LAST_MODIFY_TIME, System.currentTimeMillis()) - .where(defaultTable.ID.in(taskIdList)); - val sql = query.getSQL(ParamType.INLINED); - try { - return query.execute(); - } catch (Exception e) { - log.error(sql); - throw e; - } - } - - @Override - public int deleteFileSourceBatchTaskById(DSLContext dslContext, String id) { - return dslContext.deleteFrom(defaultTable).where( - defaultTable.ID.eq(id) - ).execute(); - } - - @Override - public FileSourceBatchTaskDTO getFileSourceBatchTaskById(DSLContext dslContext, String id) { + public FileSourceBatchTaskDTO getBatchTaskById(String id) { val record = dslContext.selectFrom(defaultTable).where( defaultTable.ID.eq(id) ).fetchOne(); @@ -164,33 +143,15 @@ val record = dslContext.selectFrom(defaultTable).where( } @Override - public Long countFileSourceBatchTasks(DSLContext dslContext, Long appId) { - List conditions = new ArrayList<>(); - if (appId != null) { - conditions.add(defaultTable.APP_ID.eq(appId)); - } - return countFileSourceBatchTasksByConditions(dslContext, conditions); - } - - public Long countFileSourceBatchTasksByConditions(DSLContext dslContext, Collection conditions) { - val query = dslContext.select( - DSL.countDistinct(defaultTable.ID) - ).from(defaultTable) - .where(conditions); - return query.fetchOne(0, Long.class); - } - - @Override - public List listFileSourceBatchTasks(DSLContext dslContext, Long appId, Integer start, - Integer pageSize) { - List conditions = new ArrayList<>(); - if (appId != null) { - conditions.add(defaultTable.APP_ID.eq(appId)); + public FileSourceBatchTaskDTO getBatchTaskByIdForUpdate(String id) { + val record = dslContext.selectFrom(defaultTable).where( + defaultTable.ID.eq(id) + ).forUpdate().fetchOne(); + if (record == null) { + return null; + } else { + return convertRecordToDto(record); } - val query = dslContext.selectFrom(defaultTable) - .where(conditions) - .orderBy(defaultTable.LAST_MODIFY_TIME.desc()); - return listPage(query, start, pageSize, this::convertRecordToDto); } private FileSourceBatchTaskDTO convertRecordToDto(FileSourceBatchTaskRecord record) { diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/FileSourceDAOImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/FileSourceDAOImpl.java index 40f4e485cc..f770990652 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/FileSourceDAOImpl.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/FileSourceDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,11 +25,14 @@ package com.tencent.bk.job.file_gateway.dao.filesource.impl; import com.fasterxml.jackson.core.type.TypeReference; +import com.tencent.bk.job.common.mysql.dao.BaseDAOImpl; import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.file_gateway.dao.filesource.FileSourceDAO; import com.tencent.bk.job.file_gateway.dao.filesource.FileSourceTypeDAO; import com.tencent.bk.job.file_gateway.model.dto.FileSourceBasicInfoDTO; import com.tencent.bk.job.file_gateway.model.dto.FileSourceDTO; +import com.tencent.bk.job.file_gateway.model.tables.FileSource; +import com.tencent.bk.job.file_gateway.model.tables.FileSourceShare; import com.tencent.bk.job.file_gateway.util.JooqTypeUtil; import io.micrometer.core.instrument.util.StringUtils; import lombok.extern.slf4j.Slf4j; @@ -40,8 +43,6 @@ import org.jooq.DSLContext; import org.jooq.Record; import org.jooq.conf.ParamType; -import org.jooq.generated.tables.FileSource; -import org.jooq.generated.tables.FileSourceShare; import org.jooq.impl.DSL; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -58,13 +59,13 @@ public class FileSourceDAOImpl extends BaseDAOImpl implements FileSourceDAO { private static final FileSource defaultTable = FileSource.FILE_SOURCE; private static final FileSourceShare tableFileSourceShare = FileSourceShare.FILE_SOURCE_SHARE; - private final DSLContext defaultContext; + private final DSLContext dslContext; private final FileSourceTypeDAO fileSourceTypeDAO; @Autowired - public FileSourceDAOImpl(@Qualifier("job-file-gateway-dsl-context") DSLContext defaultContext, + public FileSourceDAOImpl(@Qualifier("job-file-gateway-dsl-context") DSLContext dslContext, FileSourceTypeDAO fileSourceTypeDAO) { - this.defaultContext = defaultContext; + this.dslContext = dslContext; this.fileSourceTypeDAO = fileSourceTypeDAO; } @@ -75,7 +76,7 @@ private void setDefaultValue(FileSourceDTO fileSourceDTO) { } @Override - public Integer insertFileSource(DSLContext dslContext, FileSourceDTO fileSourceDTO) { + public Integer insertFileSource(FileSourceDTO fileSourceDTO) { setDefaultValue(fileSourceDTO); val insertFileSourceQuery = dslContext.insertInto(defaultTable, defaultTable.ID, @@ -124,7 +125,7 @@ public Integer insertFileSource(DSLContext dslContext, FileSourceDTO fileSourceD val record = insertFileSourceQuery.fetchOne(); assert record != null; Integer fileSourceId = record.getId(); - saveFileSourceShareInfo(dslContext, fileSourceId, fileSourceDTO); + saveFileSourceShareInfo(fileSourceId, fileSourceDTO); return fileSourceId; } catch (Exception e) { log.error(sql); @@ -132,7 +133,7 @@ assert record != null; } } - private List getSharedAppIdList(DSLContext dslContext, Long appId, Integer fileSourceId) { + private List getSharedAppIdList(Long appId, Integer fileSourceId) { val records = dslContext.select(tableFileSourceShare.APP_ID) .from(tableFileSourceShare) .where(tableFileSourceShare.FILE_SOURCE_ID.eq(fileSourceId)) @@ -141,7 +142,7 @@ private List getSharedAppIdList(DSLContext dslContext, Long appId, Integer return records.map(record -> record.get(tableFileSourceShare.APP_ID)); } - private void saveFileSourceShareInfo(DSLContext dslContext, Integer fileSourceId, FileSourceDTO fileSourceDTO) { + private void saveFileSourceShareInfo(Integer fileSourceId, FileSourceDTO fileSourceDTO) { String sql = null; try { List sharedAppIdList = fileSourceDTO.getSharedAppIdList(); @@ -195,10 +196,12 @@ private void saveFileSourceShareInfo(DSLContext dslContext, Integer fileSourceId } @Override - public int updateFileSource(DSLContext dslContext, FileSourceDTO fileSourceDTO) { + public int updateFileSource(FileSourceDTO fileSourceDTO) { val query = dslContext.update(defaultTable); var updateSetStep = query.set(defaultTable.APP_ID, fileSourceDTO.getAppId()); - updateSetStep = updateSetStep.set(defaultTable.CODE, fileSourceDTO.getCode()); + if (StringUtils.isNotBlank(fileSourceDTO.getCode())) { + updateSetStep = updateSetStep.set(defaultTable.CODE, fileSourceDTO.getCode()); + } if (StringUtils.isNotBlank(fileSourceDTO.getAlias())) { updateSetStep = updateSetStep.set(defaultTable.ALIAS, fileSourceDTO.getAlias()); } @@ -242,7 +245,7 @@ public int updateFileSource(DSLContext dslContext, FileSourceDTO fileSourceDTO) val finalStep = updateSetStep.where(defaultTable.ID.eq(fileSourceDTO.getId())); val sql = finalStep.getSQL(ParamType.INLINED); try { - saveFileSourceShareInfo(dslContext, fileSourceDTO.getId(), fileSourceDTO); + saveFileSourceShareInfo(fileSourceDTO.getId(), fileSourceDTO); return finalStep.execute(); } catch (Exception e) { log.error(sql); @@ -251,7 +254,7 @@ public int updateFileSource(DSLContext dslContext, FileSourceDTO fileSourceDTO) } @Override - public int updateFileSourceStatus(DSLContext dslContext, Integer fileSourceId, Integer status) { + public int updateFileSourceStatus(Integer fileSourceId, Integer status) { val query = dslContext.update(defaultTable) .set(defaultTable.STATUS, JooqTypeUtil.convertToByte(status)) .where(defaultTable.ID.eq(fileSourceId)); @@ -266,14 +269,14 @@ public int updateFileSourceStatus(DSLContext dslContext, Integer fileSourceId, I @Override - public int deleteFileSourceById(DSLContext dslContext, Integer id) { + public int deleteFileSourceById(Integer id) { return dslContext.deleteFrom(defaultTable).where( defaultTable.ID.eq(id) ).execute(); } @Override - public int enableFileSourceById(DSLContext dslContext, String username, Long appId, Integer id, + public int enableFileSourceById(String username, Long appId, Integer id, Boolean enableFlag) { val query = dslContext.update(defaultTable) .set(defaultTable.ENABLE, enableFlag) @@ -290,7 +293,7 @@ public int enableFileSourceById(DSLContext dslContext, String username, Long app } @Override - public FileSourceDTO getFileSourceById(DSLContext dslContext, Integer id) { + public FileSourceDTO getFileSourceById(Integer id) { val record = dslContext.selectFrom(defaultTable).where( defaultTable.ID.eq(id) ).fetchOne(); @@ -302,7 +305,7 @@ val record = dslContext.selectFrom(defaultTable).where( } @Override - public List listFileSourceByIds(DSLContext dslContext, Collection ids) { + public List listFileSourceByIds(Collection ids) { val records = dslContext.selectFrom(defaultTable).where( defaultTable.ID.in(ids) ).fetch(); @@ -310,10 +313,23 @@ public List listFileSourceByIds(DSLContext dslContext, C } @Override - public FileSourceDTO getFileSourceByCode(DSLContext dslContext, String code) { - val record = dslContext.selectFrom(defaultTable).where( - defaultTable.CODE.eq(code) - ).fetchOne(); + public FileSourceDTO getFileSourceByCode(String code) { + val record = dslContext.selectFrom(defaultTable) + .where(defaultTable.CODE.eq(code)) + .fetchOne(); + if (record == null) { + return null; + } else { + return convertRecordToDto(record); + } + } + + @Override + public FileSourceDTO getFileSourceByCode(Long appId, String code) { + val record = dslContext.selectFrom(defaultTable) + .where(defaultTable.CODE.eq(code)) + .and(defaultTable.APP_ID.eq(appId)) + .fetchOne(); if (record == null) { return null; } else { @@ -352,18 +368,18 @@ private Collection genAvailableConditions(Long appId, String credenti } @Override - public Integer countFileSource(DSLContext dslContext, Long appId, String credentialId, String alias) { + public Integer countFileSource(Long appId, String credentialId, String alias) { Collection conditions = genAvailableConditions(appId, credentialId, alias); - return countFileSourcesByConditions(dslContext, conditions); + return countFileSourcesByConditions(conditions); } @Override - public Integer countAvailableLikeFileSource(DSLContext dslContext, Long appId, String credentialId, String alias) { + public Integer countAvailableLikeFileSource(Long appId, String credentialId, String alias) { Collection conditions = genAvailableLikeConditions(appId, credentialId, alias); - return countAvailableFileSourcesByConditions(dslContext, conditions); + return countAvailableFileSourcesByConditions(conditions); } - public Integer countFileSourcesByConditions(DSLContext dslContext, Collection conditions) { + public Integer countFileSourcesByConditions(Collection conditions) { val query = dslContext.select( DSL.count(defaultTable.ID) ).from(defaultTable) @@ -371,7 +387,7 @@ public Integer countFileSourcesByConditions(DSLContext dslContext, Collection conditions) { + public Integer countAvailableFileSourcesByConditions(Collection conditions) { val query = dslContext.select( DSL.countDistinct(defaultTable.ID) ).from(defaultTable) @@ -382,7 +398,7 @@ public Integer countAvailableFileSourcesByConditions(DSLContext dslContext, Coll } @Override - public Boolean checkFileSourceExists(DSLContext dslContext, Long appId, String alias) { + public Boolean checkFileSourceExists(Long appId, String alias) { List conditions = new ArrayList<>(); if (appId != null) { conditions.add(defaultTable.APP_ID.eq(appId)); @@ -390,22 +406,22 @@ public Boolean checkFileSourceExists(DSLContext dslContext, Long appId, String a if (StringUtils.isNotBlank(alias)) { conditions.add(defaultTable.ALIAS.eq(alias)); } - return countWorkTableFileSourcesByConditions(dslContext, conditions) > 0; + return countWorkTableFileSourcesByConditions(conditions) > 0; } @Override - public Integer countWorkTableFileSource(DSLContext dslContext, Long appId, String credentialId, String alias) { + public Integer countWorkTableFileSource(Long appId, String credentialId, String alias) { Collection conditions = genWorkTableConditions(appId, credentialId, alias); - return countWorkTableFileSourcesByConditions(dslContext, conditions); + return countWorkTableFileSourcesByConditions(conditions); } @Override - public Integer countWorkTableFileSource(DSLContext dslContext, List appIdList, List idList) { + public Integer countWorkTableFileSource(List appIdList, List idList) { Collection conditions = genWorkTableConditions(appIdList, idList); - return countWorkTableFileSourcesByConditions(dslContext, conditions); + return countWorkTableFileSourcesByConditions(conditions); } - public Integer countWorkTableFileSourcesByConditions(DSLContext dslContext, Collection conditions) { + public Integer countWorkTableFileSourcesByConditions(Collection conditions) { val query = dslContext.select( DSL.countDistinct(defaultTable.ID) ).from(defaultTable) @@ -414,10 +430,10 @@ public Integer countWorkTableFileSourcesByConditions(DSLContext dslContext, Coll } @Override - public List listAvailableFileSource(DSLContext dslContext, Long appId, String credentialId, String + public List listAvailableFileSource(Long appId, String credentialId, String alias, Integer start, Integer pageSize) { Collection conditions = genAvailableLikeConditions(appId, credentialId, alias); - return listFileSourceByShareConditions(dslContext, conditions, start, pageSize); + return listFileSourceByShareConditions(conditions, start, pageSize); } private Collection genWorkTableConditions(Long appId, String credentialId, String alias) { @@ -446,22 +462,22 @@ private Collection genWorkTableConditions(List appIdList, List< } @Override - public List listWorkTableFileSource(DSLContext dslContext, Long appId, String credentialId, + public List listWorkTableFileSource(Long appId, String credentialId, String alias, Integer start, Integer pageSize) { Collection conditions = genWorkTableConditions(appId, credentialId, alias); - return listFileSourceByConditions(dslContext, conditions, start, pageSize); + return listFileSourceByConditions(conditions, start, pageSize); } @Override - public List listWorkTableFileSource(DSLContext dslContext, List appIdList, + public List listWorkTableFileSource(List appIdList, List idList, Integer start, Integer pageSize) { Collection conditions = genWorkTableConditions(appIdList, idList); - return listFileSourceByConditions(dslContext, conditions, start, pageSize); + return listFileSourceByConditions(conditions, start, pageSize); } @Override public boolean existsCode(Long appId, String code) { - val query = defaultContext.selectZero().from(defaultTable) + val query = dslContext.selectZero().from(defaultTable) .where(defaultTable.APP_ID.eq(appId)) .and(defaultTable.CODE.eq(code)) .limit(1); @@ -470,7 +486,7 @@ public boolean existsCode(Long appId, String code) { @Override public boolean existsCodeExceptId(Long appId, String code, Integer exceptId) { - val query = defaultContext.selectZero().from(defaultTable) + val query = dslContext.selectZero().from(defaultTable) .where(defaultTable.APP_ID.eq(appId)) .and(defaultTable.CODE.eq(code)) .and(defaultTable.ID.notEqual(exceptId)) @@ -480,37 +496,38 @@ public boolean existsCodeExceptId(Long appId, String code, Integer exceptId) { @Override public boolean existsFileSource(Long appId, Integer id) { - val query = defaultContext.selectZero().from(defaultTable) + val query = dslContext.selectZero().from(defaultTable) .where(defaultTable.APP_ID.eq(appId)) .and(defaultTable.ID.eq(id)) .limit(1); return query.fetch().size() > 0; } + @Override + public boolean existsFileSourceUsingCredential(Long appId, String credentialId) { + Collection conditions = new ArrayList<>(); + conditions.add(defaultTable.APP_ID.eq(appId)); + conditions.add(defaultTable.CREDENTIAL_ID.eq(credentialId)); + return dslContext.fetchExists(defaultTable, conditions); + } + @Override public Integer getFileSourceIdByCode(Long appId, String code) { List conditions = new ArrayList<>(); - if (appId != null) { - conditions.add(defaultTable.APP_ID.eq(appId)); - } - if (code != null) { - conditions.add(defaultTable.CODE.eq(code)); - } - val query = defaultContext.select( + conditions.add(defaultTable.APP_ID.eq(appId)); + conditions.add(defaultTable.CODE.eq(code)); + val query = dslContext.select( defaultTable.ID ).from(defaultTable) .where(conditions); - val result = query.fetch(); - if (result.size() > 0) { - if (result.size() > 1) { - log.warn("{} records found when get id by code, use first one", result.size()); - } - return result.get(0).get(defaultTable.ID); + val result = query.fetchOne(); + if (result != null) { + return result.get(defaultTable.ID); } return null; } - private List listFileSourceByConditions(DSLContext dslContext, Collection conditions, + private List listFileSourceByConditions(Collection conditions, Integer start, Integer pageSize) { val query = dslContext.select( defaultTable.ID, @@ -540,8 +557,8 @@ private List listFileSourceByConditions(DSLContext dslContext, Co return listPage(query, start, pageSize, this::convertRecordToDto); } - private List listFileSourceByShareConditions(DSLContext dslContext, - Collection conditions, Integer start, + private List listFileSourceByShareConditions(Collection conditions, + Integer start, Integer pageSize) { val query = dslContext.select( defaultTable.ID, @@ -584,7 +601,7 @@ private FileSourceDTO convertRecordToDto(Record record) { fileSourceDTO.setStatus(JooqTypeUtil.convertToInt(record.get(defaultTable.STATUS))); fileSourceDTO.setFileSourceType(fileSourceTypeDAO.getByCode(record.get(defaultTable.TYPE))); fileSourceDTO.setPublicFlag(record.get(defaultTable.PUBLIC)); - fileSourceDTO.setSharedAppIdList(getSharedAppIdList(defaultContext, appId, id)); + fileSourceDTO.setSharedAppIdList(getSharedAppIdList(appId, id)); fileSourceDTO.setShareToAllApp(record.get(defaultTable.SHARE_TO_ALL_APP)); fileSourceDTO.setCredentialId(record.get(defaultTable.CREDENTIAL_ID)); fileSourceDTO.setFilePrefix(record.get(defaultTable.FILE_PREFIX)); diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/FileSourceTaskDAOImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/FileSourceTaskDAOImpl.java index e8ffbd0a56..00e0c9a384 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/FileSourceTaskDAOImpl.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/FileSourceTaskDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,22 +26,24 @@ import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.exception.InternalException; +import com.tencent.bk.job.common.mysql.dao.BaseDAOImpl; import com.tencent.bk.job.common.util.JobUUID; import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.file_gateway.dao.filesource.FileSourceTaskDAO; import com.tencent.bk.job.file_gateway.dao.filesource.FileTaskDAO; import com.tencent.bk.job.file_gateway.model.dto.FileSourceTaskDTO; import com.tencent.bk.job.file_gateway.model.dto.FileTaskDTO; +import com.tencent.bk.job.file_gateway.model.tables.FileSourceTask; +import com.tencent.bk.job.file_gateway.model.tables.records.FileSourceTaskRecord; import lombok.extern.slf4j.Slf4j; import lombok.val; import org.apache.commons.lang3.StringUtils; import org.jooq.Condition; import org.jooq.DSLContext; import org.jooq.conf.ParamType; -import org.jooq.generated.tables.FileSourceTask; -import org.jooq.generated.tables.records.FileSourceTaskRecord; import org.jooq.impl.DSL; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; import java.util.ArrayList; @@ -54,12 +56,13 @@ public class FileSourceTaskDAOImpl extends BaseDAOImpl implements FileSourceTask private static final FileSourceTask defaultTable = FileSourceTask.FILE_SOURCE_TASK; private final FileTaskDAO fileTaskDAO; - private final DSLContext defaultContext; + private final DSLContext dslContext; @Autowired - public FileSourceTaskDAOImpl(FileTaskDAO fileTaskDAO, DSLContext dslContext) { + public FileSourceTaskDAOImpl(FileTaskDAO fileTaskDAO, + @Qualifier("job-file-gateway-dsl-context") DSLContext dslContext) { this.fileTaskDAO = fileTaskDAO; - this.defaultContext = dslContext; + this.dslContext = dslContext; } private void setDefaultValue(FileSourceTaskDTO fileSourceTaskDTO) { @@ -69,7 +72,7 @@ private void setDefaultValue(FileSourceTaskDTO fileSourceTaskDTO) { } @Override - public String insertFileSourceTask(DSLContext dslContext, FileSourceTaskDTO fileSourceTaskDTO) { + public String insertFileSourceTask(FileSourceTaskDTO fileSourceTaskDTO) { setDefaultValue(fileSourceTaskDTO); String id = fileSourceTaskDTO.getId(); if (id == null) { @@ -111,7 +114,8 @@ public String insertFileSourceTask(DSLContext dslContext, FileSourceTaskDTO file for (FileTaskDTO fileTaskDTO : fileTaskDTOList) { fileTaskDTO.setFileSourceTaskId(id); // 插入FileTask - fileTaskDAO.insertFileTask(dslContext, fileTaskDTO); + Long fileTaskId = fileTaskDAO.insertFileTask(fileTaskDTO); + log.debug("{} inserted, id={}", fileTaskDTO, fileTaskId); } fileSourceTaskDTO.setFileTaskList(fileTaskDTOList); return id; @@ -122,7 +126,7 @@ public String insertFileSourceTask(DSLContext dslContext, FileSourceTaskDTO file } @Override - public int updateFileSourceTask(DSLContext dslContext, FileSourceTaskDTO fileSourceTaskDTO) { + public int updateFileSourceTask(FileSourceTaskDTO fileSourceTaskDTO) { val query = dslContext.update(defaultTable) .set(defaultTable.BATCH_TASK_ID, fileSourceTaskDTO.getBatchTaskId()) .set(defaultTable.APP_ID, fileSourceTaskDTO.getAppId()) @@ -141,7 +145,7 @@ public int updateFileSourceTask(DSLContext dslContext, FileSourceTaskDTO fileSou } @Override - public int updateFileClearStatus(DSLContext dslContext, List taskIdList, boolean fileCleared) { + public int updateFileClearStatus(List taskIdList, boolean fileCleared) { val query = dslContext.update(defaultTable) .set(defaultTable.FILE_CLEARED, fileCleared) .set(defaultTable.LAST_MODIFY_TIME, System.currentTimeMillis()) @@ -156,14 +160,14 @@ public int updateFileClearStatus(DSLContext dslContext, List taskIdList, } @Override - public int deleteById(DSLContext dslContext, String id) { + public int deleteById(String id) { return dslContext.deleteFrom(defaultTable).where( defaultTable.ID.eq(id) ).execute(); } @Override - public FileSourceTaskDTO getFileSourceTaskById(DSLContext dslContext, String id) { + public FileSourceTaskDTO getFileSourceTaskById(String id) { val record = dslContext.selectFrom(defaultTable).where( defaultTable.ID.eq(id) ).fetchOne(); @@ -175,16 +179,19 @@ val record = dslContext.selectFrom(defaultTable).where( } @Override - public Long countFileSourceTasks(DSLContext dslContext, Long appId) { - List conditions = new ArrayList<>(); - if (appId != null) { - conditions.add(defaultTable.APP_ID.eq(appId)); + public FileSourceTaskDTO getFileSourceTaskByIdForUpdate(String id) { + val record = dslContext.selectFrom(defaultTable).where( + defaultTable.ID.eq(id) + ).forUpdate().fetchOne(); + if (record == null) { + return null; + } else { + return convertRecordToDto(record); } - return countFileSourceTasksByConditions(dslContext, conditions); } @Override - public Long countFileSourceTasksByBatchTaskId(DSLContext dslContext, String batchTaskId, Byte status) { + public Long countFileSourceTasksByBatchTaskId(String batchTaskId, Byte status) { List conditions = new ArrayList<>(); if (batchTaskId != null) { conditions.add(defaultTable.BATCH_TASK_ID.eq(batchTaskId)); @@ -192,10 +199,10 @@ public Long countFileSourceTasksByBatchTaskId(DSLContext dslContext, String batc if (status != null) { conditions.add(defaultTable.STATUS.eq(status)); } - return countFileSourceTasksByConditions(dslContext, conditions); + return countFileSourceTasksByConditions(conditions); } - public Long countFileSourceTasksByConditions(DSLContext dslContext, Collection conditions) { + public Long countFileSourceTasksByConditions(Collection conditions) { val query = dslContext.select( DSL.countDistinct(defaultTable.ID) ).from(defaultTable) @@ -203,7 +210,7 @@ public Long countFileSourceTasksByConditions(DSLContext dslContext, Collection listByConditions(DSLContext dslContext, Collection conditions, + public List listByConditions(Collection conditions, Integer start, Integer pageSize) { val query = dslContext.selectFrom(defaultTable) .where(conditions) @@ -211,43 +218,13 @@ public List listByConditions(DSLContext dslContext, Collectio return listPage(query, start, pageSize, this::convertRecordToDto); } - @Override - public List listFileSourceTasks(DSLContext dslContext, Long appId, Integer start, - Integer pageSize) { - List conditions = new ArrayList<>(); - if (appId != null) { - conditions.add(defaultTable.APP_ID.eq(appId)); - } - return listByConditions(dslContext, conditions, start, pageSize); - } - - @Override - public List listTimeoutTasks(DSLContext dslContext, Long expireTimeMills, - Collection statusSet, Integer start, Integer pageSize) { - List conditions = new ArrayList<>(); - if (expireTimeMills != null && expireTimeMills > 0) { - conditions.add(defaultTable.LAST_MODIFY_TIME.le(System.currentTimeMillis() - expireTimeMills)); - } - if (statusSet != null && !statusSet.isEmpty()) { - conditions.add(defaultTable.STATUS.in(statusSet)); - } - return listByConditions(defaultContext, conditions, start, pageSize); - } - @Override public List listByBatchTaskId(String batchTaskId) { List conditions = new ArrayList<>(); if (StringUtils.isNotBlank(batchTaskId)) { conditions.add(defaultTable.BATCH_TASK_ID.eq(batchTaskId)); } - return listByConditions(defaultContext, conditions, null, null); - } - - @Override - public int deleteByBatchTaskId(DSLContext dslContext, String batchTaskId) { - return dslContext.deleteFrom(defaultTable).where( - defaultTable.BATCH_TASK_ID.eq(batchTaskId) - ).execute(); + return listByConditions(conditions, null, null); } private FileSourceTaskDTO convertRecordToDto(FileSourceTaskRecord record) { diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/FileSourceTypeDAOImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/FileSourceTypeDAOImpl.java index 56d0a8bc3f..cc7d1c4ebd 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/FileSourceTypeDAOImpl.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/FileSourceTypeDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,22 +26,25 @@ import com.tencent.bk.job.file_gateway.dao.filesource.FileSourceTypeDAO; import com.tencent.bk.job.file_gateway.model.dto.FileSourceTypeDTO; +import com.tencent.bk.job.file_gateway.model.tables.FileSourceType; +import com.tencent.bk.job.file_gateway.model.tables.FileWorker; import lombok.extern.slf4j.Slf4j; import lombok.val; +import org.apache.commons.collections4.CollectionUtils; +import org.jooq.BatchBindStep; import org.jooq.Condition; import org.jooq.DSLContext; +import org.jooq.Query; import org.jooq.Record; -import org.jooq.conf.ParamType; -import org.jooq.generated.tables.FileSourceType; -import org.jooq.generated.tables.FileWorker; -import org.jooq.impl.DSL; +import org.jooq.TableField; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; @Slf4j @Repository @@ -50,120 +53,111 @@ public class FileSourceTypeDAOImpl implements FileSourceTypeDAO { private static final FileSourceType defaultTable = FileSourceType.FILE_SOURCE_TYPE; private static final FileWorker tableFileWorker = FileWorker.FILE_WORKER; private final DSLContext dslContext; + private static final TableField[] ALL_FIELDS = { + defaultTable.ID, + defaultTable.WORKER_ID, + defaultTable.STORAGE_TYPE, + defaultTable.CODE, + defaultTable.NAME, + defaultTable.ENABLED, + defaultTable.ICON, + defaultTable.LAST_MODIFY_USER, + defaultTable.LAST_MODIFY_TIME + }; @Autowired - public FileSourceTypeDAOImpl(DSLContext dslContext) { + public FileSourceTypeDAOImpl(@Qualifier("job-file-gateway-dsl-context") DSLContext dslContext) { this.dslContext = dslContext; } - private void setDefaultValue(FileSourceTypeDTO fileSourceTypeDTO) { - // - } - - @Override - public Integer upsertByWorker(DSLContext dslContext, FileSourceTypeDTO fileSourceTypeDTO) { - AtomicInteger result = new AtomicInteger(-1); - dslContext.transaction(configuration -> { - DSLContext context = DSL.using(configuration); - FileSourceTypeDTO oldFileSourceTypeDTO = get(context, fileSourceTypeDTO.getWorkerId(), - fileSourceTypeDTO.getStorageType(), fileSourceTypeDTO.getCode()); - if (oldFileSourceTypeDTO == null) { - // 插入 - result.set(insert(context, fileSourceTypeDTO)); - } else { - // 更新 - fileSourceTypeDTO.setId(oldFileSourceTypeDTO.getId()); - result.set(update(context, fileSourceTypeDTO)); - } - }); - return result.get(); - } - @Override - public Integer insert(DSLContext dslContext, FileSourceTypeDTO fileSourceTypeDTO) { - setDefaultValue(fileSourceTypeDTO); - val query = dslContext.insertInto(defaultTable, + public Integer batchInsert(List fileSourceTypeList) { + if (CollectionUtils.isEmpty(fileSourceTypeList)) { + return 0; + } + val insertQuery = dslContext.insertInto(defaultTable, defaultTable.ID, defaultTable.WORKER_ID, defaultTable.STORAGE_TYPE, defaultTable.CODE, defaultTable.NAME, + defaultTable.ENABLED, defaultTable.ICON, defaultTable.LAST_MODIFY_USER, defaultTable.LAST_MODIFY_TIME ).values( (Integer) null, - fileSourceTypeDTO.getWorkerId(), - fileSourceTypeDTO.getStorageType(), - fileSourceTypeDTO.getCode(), - fileSourceTypeDTO.getName(), - fileSourceTypeDTO.getIcon(), - fileSourceTypeDTO.getLastModifier(), - fileSourceTypeDTO.getLastModifyTime() - ).returning(defaultTable.ID); - val sql = query.getSQL(ParamType.INLINED); - try { - return query.fetchOne().getId(); - } catch (Exception e) { - log.error(sql); - throw e; + null, + null, + null, + null, + null, + null, + null, + null + ); + BatchBindStep batchQuery = dslContext.batch(insertQuery); + for (FileSourceTypeDTO fileSourceTypeDTO : fileSourceTypeList) { + batchQuery = batchQuery.bind( + null, + fileSourceTypeDTO.getWorkerId(), + fileSourceTypeDTO.getStorageType(), + fileSourceTypeDTO.getCode(), + fileSourceTypeDTO.getName(), + fileSourceTypeDTO.getEnabled(), + fileSourceTypeDTO.getIcon(), + fileSourceTypeDTO.getLastModifier(), + fileSourceTypeDTO.getLastModifyTime() + ); + } + int[] results = batchQuery.execute(); + int affectedRowNum = 0; + for (int result : results) { + affectedRowNum += result; } + return affectedRowNum; } - @Override - public int update(DSLContext dslContext, FileSourceTypeDTO fileSourceTypeDTO) { - val query = dslContext.update(defaultTable) + private Query buildUpdateByIdQuery(FileSourceTypeDTO fileSourceTypeDTO) { + return dslContext.update(defaultTable) .set(defaultTable.WORKER_ID, fileSourceTypeDTO.getWorkerId()) .set(defaultTable.STORAGE_TYPE, fileSourceTypeDTO.getStorageType()) .set(defaultTable.CODE, fileSourceTypeDTO.getCode()) .set(defaultTable.NAME, fileSourceTypeDTO.getName()) + .set(defaultTable.ENABLED, fileSourceTypeDTO.getEnabled()) .set(defaultTable.ICON, fileSourceTypeDTO.getIcon()) .set(defaultTable.LAST_MODIFY_USER, fileSourceTypeDTO.getLastModifier()) .set(defaultTable.LAST_MODIFY_TIME, System.currentTimeMillis()) .where(defaultTable.ID.eq(fileSourceTypeDTO.getId())); - val sql = query.getSQL(ParamType.INLINED); - try { - return query.execute(); - } catch (Exception e) { - log.error(sql); - throw e; - } } - @Override - public int deleteById(DSLContext dslContext, Integer id) { - return dslContext.deleteFrom(defaultTable).where( - defaultTable.ID.eq(id) - ).execute(); + public int batchUpdate(List fileSourceTypeList) { + if (CollectionUtils.isEmpty(fileSourceTypeList)) { + return 0; + } + List queryList = new ArrayList<>(); + for (FileSourceTypeDTO fileSourceTypeDTO : fileSourceTypeList) { + queryList.add(buildUpdateByIdQuery(fileSourceTypeDTO)); + } + int[] results = dslContext.batch(queryList).execute(); + int affectedNum = 0; + for (int result : results) { + affectedNum += result; + } + return affectedNum; } @Override - public FileSourceTypeDTO get(DSLContext dslContext, Long workerId, String storageType, String code) { - val record = dslContext.selectFrom(defaultTable) - .where(defaultTable.WORKER_ID.eq(workerId)) - .and(defaultTable.STORAGE_TYPE.eq(storageType)) - .and(defaultTable.CODE.eq(code)) - .fetchOne(); - if (record == null) { - return null; - } else { - return convertRecordToDto(record); - } + public int batchDeleteByIds(Collection ids) { + return dslContext.deleteFrom(defaultTable).where( + defaultTable.ID.in(ids) + ).execute(); } @Override public FileSourceTypeDTO getById(Integer id) { - val record = dslContext.select( - defaultTable.ID, - defaultTable.WORKER_ID, - defaultTable.STORAGE_TYPE, - defaultTable.CODE, - defaultTable.NAME, - defaultTable.ICON, - defaultTable.LAST_MODIFY_USER, - defaultTable.LAST_MODIFY_TIME - ) + val record = dslContext.select(ALL_FIELDS) .from(defaultTable) .where(defaultTable.ID.eq(id) ).fetchOne(); @@ -183,20 +177,24 @@ public FileSourceTypeDTO getByCode(String code) { @Override public List listByCode(String code) { - val records = dslContext.select( - defaultTable.ID, - defaultTable.WORKER_ID, - defaultTable.STORAGE_TYPE, - defaultTable.CODE, - defaultTable.NAME, - defaultTable.ICON, - defaultTable.LAST_MODIFY_USER, - defaultTable.LAST_MODIFY_TIME - ) + val records = dslContext.select(ALL_FIELDS) .from(defaultTable).where( defaultTable.CODE.eq(code) ).fetch(); - if (records == null || records.isEmpty()) { + if (records.isEmpty()) { + return Collections.emptyList(); + } else { + return records.map(this::convertRecordToDto); + } + } + + @Override + public List listByWorkerId(Long workerId) { + val records = dslContext.select(ALL_FIELDS) + .from(defaultTable).where( + defaultTable.WORKER_ID.eq(workerId) + ).fetch(); + if (records.isEmpty()) { return Collections.emptyList(); } else { return records.map(this::convertRecordToDto); @@ -205,23 +203,14 @@ public List listByCode(String code) { @Override public List listByCodeOrderByVersion(String code) { - val records = dslContext.select( - defaultTable.ID, - defaultTable.WORKER_ID, - defaultTable.STORAGE_TYPE, - defaultTable.CODE, - defaultTable.NAME, - defaultTable.ICON, - defaultTable.LAST_MODIFY_USER, - defaultTable.LAST_MODIFY_TIME - ) + val records = dslContext.select(ALL_FIELDS) .from(defaultTable) .join(tableFileWorker) .on(defaultTable.WORKER_ID.eq(tableFileWorker.ID)) .where(defaultTable.CODE.eq(code)) .orderBy(tableFileWorker.VERSION.desc(), tableFileWorker.LAST_MODIFY_TIME.desc()) .fetch(); - if (records == null || records.isEmpty()) { + if (records.isEmpty()) { return Collections.emptyList(); } else { return records.map(this::convertRecordToDto); @@ -229,52 +218,20 @@ public List listByCodeOrderByVersion(String code) { } @Override - public List listOrderByVersion(DSLContext dslContext, String storageType) { + public List listEnabledTypeOrderByVersion(String storageType) { List conditions = new ArrayList<>(); + conditions.add(defaultTable.ENABLED.equal(true)); if (storageType != null) { conditions.add(defaultTable.STORAGE_TYPE.equal(storageType)); } - val records = dslContext.select( - defaultTable.ID, - defaultTable.WORKER_ID, - defaultTable.STORAGE_TYPE, - defaultTable.CODE, - defaultTable.NAME, - defaultTable.ICON, - defaultTable.LAST_MODIFY_USER, - defaultTable.LAST_MODIFY_TIME - ) + val records = dslContext.select(ALL_FIELDS) .from(defaultTable) .join(tableFileWorker) .on(defaultTable.WORKER_ID.eq(tableFileWorker.ID)) .where(conditions) .orderBy(tableFileWorker.VERSION.desc(), tableFileWorker.LAST_MODIFY_TIME.desc()) .fetch(); - if (records == null || records.isEmpty()) { - return Collections.emptyList(); - } else { - return records.map(this::convertRecordToDto); - } - } - - @Override - public List list(DSLContext dslContext, String storageType) { - List conditions = new ArrayList<>(); - if (storageType != null) { - conditions.add(defaultTable.STORAGE_TYPE.equal(storageType)); - } - val records = dslContext.select( - defaultTable.ID, - defaultTable.WORKER_ID, - defaultTable.STORAGE_TYPE, - defaultTable.CODE, - defaultTable.NAME, - defaultTable.ICON, - defaultTable.LAST_MODIFY_USER, - defaultTable.LAST_MODIFY_TIME - ) - .from(defaultTable).where(conditions).orderBy(defaultTable.LAST_MODIFY_TIME.desc()).fetch(); - if (records == null || records.isEmpty()) { + if (records.isEmpty()) { return Collections.emptyList(); } else { return records.map(this::convertRecordToDto); @@ -288,6 +245,7 @@ private FileSourceTypeDTO convertRecordToDto(Record record) { record.get(defaultTable.STORAGE_TYPE), record.get(defaultTable.CODE), record.get(defaultTable.NAME), + record.get(defaultTable.ENABLED), record.get(defaultTable.ICON), record.get(defaultTable.LAST_MODIFY_USER), record.get(defaultTable.LAST_MODIFY_TIME) diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/FileSourceWhiteInfoDAOImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/FileSourceWhiteInfoDAOImpl.java new file mode 100644 index 0000000000..997ebc40a9 --- /dev/null +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/FileSourceWhiteInfoDAOImpl.java @@ -0,0 +1,122 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file_gateway.dao.filesource.impl; + +import com.tencent.bk.job.file_gateway.dao.filesource.FileSourceWhiteInfoDAO; +import com.tencent.bk.job.file_gateway.model.dto.FileSourceWhiteInfoDTO; +import com.tencent.bk.job.file_gateway.model.tables.FileSourceWhiteInfo; +import lombok.val; +import org.jooq.Condition; +import org.jooq.DSLContext; +import org.jooq.Record; +import org.jooq.Result; +import org.jooq.TableField; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Repository; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +@Repository +public class FileSourceWhiteInfoDAOImpl implements FileSourceWhiteInfoDAO { + + private static final FileSourceWhiteInfo defaultTable = FileSourceWhiteInfo.FILE_SOURCE_WHITE_INFO; + private final DSLContext dslContext; + private static final TableField[] ALL_FIELDS = { + defaultTable.ID, + defaultTable.TYPE, + defaultTable.CONTENT, + defaultTable.REMARK, + defaultTable.CREATOR, + defaultTable.CREATE_TIME + }; + + @Autowired + public FileSourceWhiteInfoDAOImpl(@Qualifier("job-file-gateway-dsl-context") DSLContext dslContext) { + this.dslContext = dslContext; + } + + @Override + public Integer insert(FileSourceWhiteInfoDTO fileSourceWhiteInfoDTO) { + val query = dslContext.insertInto(defaultTable, + defaultTable.ID, + defaultTable.TYPE, + defaultTable.CONTENT, + defaultTable.REMARK, + defaultTable.CREATOR + ).values( + null, + fileSourceWhiteInfoDTO.getType(), + fileSourceWhiteInfoDTO.getContent(), + fileSourceWhiteInfoDTO.getRemark(), + fileSourceWhiteInfoDTO.getCreator() + ).returning(defaultTable.ID); + val record = query.fetchOne(); + assert record != null; + return record.getId(); + } + + @Override + public List list(Integer offset, Integer limit) { + val query = dslContext.select(ALL_FIELDS) + .from(defaultTable); + Result records; + if (offset != null && offset >= 0 && limit != null && limit > 0) { + records = query.limit(offset, limit).fetch(); + } else { + records = query.fetch(); + } + return records.map(this::convert); + } + + @Override + public boolean exists(String type, String content) { + List conditions = new ArrayList<>(2); + conditions.add(defaultTable.TYPE.eq(type)); + conditions.add(defaultTable.CONTENT.eq(content)); + return dslContext.fetchExists(defaultTable, conditions); + } + + @Override + public int delete(Collection ids) { + return dslContext.deleteFrom(defaultTable) + .where(defaultTable.ID.in(ids)) + .execute(); + } + + private FileSourceWhiteInfoDTO convert(Record record) { + return new FileSourceWhiteInfoDTO( + record.get(defaultTable.ID), + record.get(defaultTable.TYPE), + record.get(defaultTable.CONTENT), + record.get(defaultTable.REMARK), + record.get(defaultTable.CREATOR), + record.get(defaultTable.CREATE_TIME) + ); + } +} diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/FileTaskDAOImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/FileTaskDAOImpl.java index b807346025..dbcc699f72 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/FileTaskDAOImpl.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/FileTaskDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,9 +24,12 @@ package com.tencent.bk.job.file_gateway.dao.filesource.impl; +import com.tencent.bk.job.common.mysql.dao.BaseDAOImpl; import com.tencent.bk.job.file_gateway.consts.TaskStatusEnum; import com.tencent.bk.job.file_gateway.dao.filesource.FileTaskDAO; import com.tencent.bk.job.file_gateway.model.dto.FileTaskDTO; +import com.tencent.bk.job.file_gateway.model.tables.FileTask; +import com.tencent.bk.job.file_gateway.model.tables.records.FileTaskRecord; import lombok.extern.slf4j.Slf4j; import lombok.val; import org.jooq.Condition; @@ -34,10 +37,9 @@ import org.jooq.Record1; import org.jooq.Result; import org.jooq.conf.ParamType; -import org.jooq.generated.tables.FileTask; -import org.jooq.generated.tables.records.FileTaskRecord; import org.jooq.impl.DSL; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; import java.util.ArrayList; @@ -54,12 +56,12 @@ public class FileTaskDAOImpl extends BaseDAOImpl implements FileTaskDAO { private final DSLContext dslContext; @Autowired - public FileTaskDAOImpl(DSLContext dslContext) { + public FileTaskDAOImpl(@Qualifier("job-file-gateway-dsl-context") DSLContext dslContext) { this.dslContext = dslContext; } @Override - public Long insertFileTask(DSLContext dslContext, FileTaskDTO fileTaskDTO) { + public Long insertFileTask(FileTaskDTO fileTaskDTO) { val query = dslContext.insertInto(defaultTable, defaultTable.FILE_SOURCE_TASK_ID, defaultTable.FILE_PATH, @@ -89,7 +91,7 @@ public Long insertFileTask(DSLContext dslContext, FileTaskDTO fileTaskDTO) { } @Override - public int updateFileTask(DSLContext dslContext, FileTaskDTO fileTaskDTO) { + public int updateFileTask(FileTaskDTO fileTaskDTO) { val query = dslContext.update(defaultTable) .set(defaultTable.FILE_SOURCE_TASK_ID, fileTaskDTO.getFileSourceTaskId()) .set(defaultTable.FILE_PATH, fileTaskDTO.getFilePath()) @@ -110,7 +112,7 @@ public int updateFileTask(DSLContext dslContext, FileTaskDTO fileTaskDTO) { } @Override - public int resetFileTasks(DSLContext dslContext, String fileSourceTaskId) { + public int resetFileTasks(String fileSourceTaskId) { val query = dslContext.update(defaultTable) .set(defaultTable.DOWNLOAD_PATH, (String) null) .set(defaultTable.STATUS, TaskStatusEnum.INIT.getStatus()) @@ -126,26 +128,20 @@ public int resetFileTasks(DSLContext dslContext, String fileSourceTaskId) { } } - - @Override - public int deleteFileTaskById(DSLContext dslContext, Long id) { - return dslContext.deleteFrom(defaultTable).where( - defaultTable.ID.eq(id) - ).execute(); - } - @Override - public int deleteFileTaskByFileSourceTaskId(DSLContext dslContext, String fileSourceTaskId) { + public int deleteFileTaskByFileSourceTaskId(String fileSourceTaskId) { return dslContext.deleteFrom(defaultTable).where( defaultTable.FILE_SOURCE_TASK_ID.eq(fileSourceTaskId) ).execute(); } @Override - public FileTaskDTO getFileTaskById(DSLContext dslContext, Long id) { + public FileTaskDTO getFileTaskByIdForUpdate(Long id) { + List conditions = new ArrayList<>(); + conditions.add(defaultTable.ID.eq(id)); val record = dslContext.selectFrom(defaultTable).where( - defaultTable.ID.eq(id) - ).fetchOne(); + conditions + ).forUpdate().fetchOne(); if (record == null) { return null; } else { @@ -154,14 +150,10 @@ val record = dslContext.selectFrom(defaultTable).where( } @Override - public FileTaskDTO getOneFileTask(DSLContext dslContext, String fileSourceTaskId, String filePath) { + public FileTaskDTO getOneFileTask(String fileSourceTaskId, String filePath) { List conditions = new ArrayList<>(); - if (fileSourceTaskId != null) { - conditions.add(defaultTable.FILE_SOURCE_TASK_ID.eq(fileSourceTaskId)); - } - if (filePath != null) { - conditions.add(defaultTable.FILE_PATH.eq(filePath)); - } + conditions.add(defaultTable.FILE_SOURCE_TASK_ID.eq(fileSourceTaskId)); + conditions.add(defaultTable.FILE_PATH.eq(filePath)); val record = dslContext.selectFrom(defaultTable).where( conditions ).fetchOne(); @@ -172,16 +164,7 @@ val record = dslContext.selectFrom(defaultTable).where( } } - @Override - public Long countFileTasks(DSLContext dslContext, String fileSourceTaskId) { - List conditions = new ArrayList<>(); - if (fileSourceTaskId != null) { - conditions.add(defaultTable.FILE_SOURCE_TASK_ID.eq(fileSourceTaskId)); - } - return countFileTasksByConditions(dslContext, conditions); - } - - public Long countFileTasksByConditions(DSLContext dslContext, Collection conditions) { + public Long countFileTasksByConditions(Collection conditions) { val query = dslContext.select( DSL.countDistinct(defaultTable.ID) ).from(defaultTable) @@ -202,11 +185,17 @@ public List listFileTasks(String fileSourceTaskId, Integer start, I } @Override - public List listTimeoutFileSourceTaskIds(DSLContext dslContext, Long expireTimeMills, - Collection statusSet, Integer start, Integer pageSize) { + public List listTimeoutFileSourceTaskIds(Long startTimeMills, + Long endTimeMills, + Collection statusSet, + Integer start, + Integer pageSize) { List conditions = new ArrayList<>(); - if (expireTimeMills != null) { - conditions.add(defaultTable.LAST_MODIFY_TIME.le(System.currentTimeMillis() - expireTimeMills)); + if (startTimeMills != null) { + conditions.add(defaultTable.LAST_MODIFY_TIME.greaterOrEqual(startTimeMills)); + } + if (endTimeMills != null) { + conditions.add(defaultTable.LAST_MODIFY_TIME.lessOrEqual(endTimeMills)); } if (statusSet != null && !statusSet.isEmpty()) { conditions.add(defaultTable.STATUS.in(statusSet)); @@ -219,7 +208,7 @@ public List listTimeoutFileSourceTaskIds(DSLContext dslContext, Long exp } else { records = query.fetch(); } - return records.parallelStream().map(record -> record.get(defaultTable.FILE_SOURCE_TASK_ID)).collect(Collectors.toList()); + return records.stream().map(record -> record.get(defaultTable.FILE_SOURCE_TASK_ID)).collect(Collectors.toList()); } @Override @@ -236,7 +225,7 @@ public Long countFileTask(String fileSourceTaskId, Byte status) { if (status != null) { conditions.add(defaultTable.STATUS.eq(status)); } - return countFileTasksByConditions(dslContext, conditions); + return countFileTasksByConditions(conditions); } private FileTaskDTO convertRecordToDto(FileTaskRecord record) { diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/FileWorkerAbilityDAOImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/FileWorkerAbilityDAOImpl.java index 3bc704c7a5..feff939bff 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/FileWorkerAbilityDAOImpl.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/FileWorkerAbilityDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,14 +26,15 @@ import com.tencent.bk.job.file_gateway.dao.filesource.FileWorkerAbilityDAO; import com.tencent.bk.job.file_gateway.model.dto.WorkerAbilityDTO; +import com.tencent.bk.job.file_gateway.model.tables.FileWorkerAbility; import lombok.extern.slf4j.Slf4j; import lombok.val; import org.apache.commons.collections4.CollectionUtils; import org.jooq.BatchBindStep; import org.jooq.DSLContext; import org.jooq.Record; -import org.jooq.generated.tables.FileWorkerAbility; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; import java.util.Collection; @@ -47,7 +48,7 @@ public class FileWorkerAbilityDAOImpl implements FileWorkerAbilityDAO { private final DSLContext defaultContext; @Autowired - public FileWorkerAbilityDAOImpl(DSLContext dslContext) { + public FileWorkerAbilityDAOImpl(@Qualifier("job-file-gateway-dsl-context") DSLContext dslContext) { this.defaultContext = dslContext; } diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/FileWorkerDAOImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/FileWorkerDAOImpl.java index 89ad22153b..c757832e07 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/FileWorkerDAOImpl.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/dao/filesource/impl/FileWorkerDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -32,6 +32,9 @@ import com.tencent.bk.job.file_gateway.model.dto.FileWorkerDTO; import com.tencent.bk.job.file_gateway.model.dto.WorkerAbilityDTO; import com.tencent.bk.job.file_gateway.model.dto.WorkerTagDTO; +import com.tencent.bk.job.file_gateway.model.tables.FileWorker; +import com.tencent.bk.job.file_gateway.model.tables.FileWorkerAbility; +import com.tencent.bk.job.file_gateway.model.tables.records.FileWorkerRecord; import com.tencent.bk.job.file_gateway.util.JooqTypeUtil; import lombok.extern.slf4j.Slf4j; import lombok.val; @@ -39,17 +42,14 @@ import org.jooq.Condition; import org.jooq.DSLContext; import org.jooq.Record; -import org.jooq.Record22; import org.jooq.Result; import org.jooq.UpdateConditionStep; import org.jooq.UpdateSetFirstStep; import org.jooq.UpdateSetMoreStep; import org.jooq.conf.ParamType; -import org.jooq.generated.tables.FileWorker; -import org.jooq.generated.tables.FileWorkerAbility; -import org.jooq.generated.tables.records.FileWorkerRecord; import org.jooq.impl.DSL; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; import java.util.ArrayList; @@ -74,6 +74,7 @@ public class FileWorkerDAOImpl implements FileWorkerDAO { public static final String KEY_FILE_WORKER_ACCESS_HOST = "accessHost"; public static final String KEY_FILE_WORKER_ACCESS_PORT = "accessPort"; public static final String KEY_FILE_WORKER_CLOUD_AREA_ID = "cloudAreaId"; + public static final String KEY_FILE_WORKER_INNER_IP_PROTOCOL = "innerIpProtocol"; public static final String KEY_FILE_WORKER_INNER_IP = "innerIp"; public static final String KEY_FILE_WORKER_CPU_OVERLOAD = "cpuOverload"; public static final String KEY_FILE_WORKER_MEM_RATE = "memRate"; @@ -95,7 +96,7 @@ public class FileWorkerDAOImpl implements FileWorkerDAO { private final FileWorkerTagDAO fileWorkerTagDAO; @Autowired - public FileWorkerDAOImpl(DSLContext dslContext, + public FileWorkerDAOImpl(@Qualifier("job-file-gateway-dsl-context") DSLContext dslContext, FileWorkerAbilityDAO fileWorkerAbilityDAO, FileWorkerTagDAO fileWorkerTagDAO) { this.defaultContext = dslContext; @@ -120,6 +121,7 @@ public Long insertFileWorker(FileWorkerDTO fileWorkerDTO) { defaultTable.ACCESS_HOST, defaultTable.ACCESS_PORT, defaultTable.CLOUD_AREA_ID, + defaultTable.INNER_IP_PROTOCOL, defaultTable.INNER_IP, defaultTable.CPU_OVERLOAD, defaultTable.MEM_RATE, @@ -142,6 +144,7 @@ public Long insertFileWorker(FileWorkerDTO fileWorkerDTO) { fileWorkerDTO.getAccessHost(), fileWorkerDTO.getAccessPort(), fileWorkerDTO.getCloudAreaId(), + fileWorkerDTO.getInnerIpProtocol(), fileWorkerDTO.getInnerIp(), JooqTypeUtil.convertToDouble(fileWorkerDTO.getCpuOverload()), JooqTypeUtil.convertToDouble(fileWorkerDTO.getMemRate()), @@ -184,6 +187,12 @@ public int updateFileWorker(FileWorkerDTO fileWorkerDTO) { if (fileWorkerDTO.getCloudAreaId() != null) { updateSetMoreStep = updateSetMoreStep.set(defaultTable.CLOUD_AREA_ID, fileWorkerDTO.getCloudAreaId()); } + if (fileWorkerDTO.getInnerIpProtocol() != null) { + updateSetMoreStep = updateSetMoreStep.set( + defaultTable.INNER_IP_PROTOCOL, + fileWorkerDTO.getInnerIpProtocol() + ); + } if (fileWorkerDTO.getInnerIp() != null) { updateSetMoreStep = updateSetMoreStep.set(defaultTable.INNER_IP, fileWorkerDTO.getInnerIp()); } @@ -425,8 +434,7 @@ public List listFileWorkersByConditions(Collection con if (conditions == null) { conditions = new ArrayList<>(); } - Result> records = null; + Result records = null; val query = defaultContext.select( defaultTable.ID.as(KEY_FILE_WORKER_ID), defaultTable.APP_ID.as(KEY_FILE_WORKER_APP_ID), @@ -436,6 +444,7 @@ public List listFileWorkersByConditions(Collection con defaultTable.ACCESS_HOST.as(KEY_FILE_WORKER_ACCESS_HOST), defaultTable.ACCESS_PORT.as(KEY_FILE_WORKER_ACCESS_PORT), defaultTable.CLOUD_AREA_ID.as(KEY_FILE_WORKER_CLOUD_AREA_ID), + defaultTable.INNER_IP_PROTOCOL.as(KEY_FILE_WORKER_INNER_IP_PROTOCOL), defaultTable.INNER_IP.as(KEY_FILE_WORKER_INNER_IP), defaultTable.CPU_OVERLOAD.as(KEY_FILE_WORKER_CPU_OVERLOAD), defaultTable.MEM_RATE.as(KEY_FILE_WORKER_MEM_RATE), @@ -481,6 +490,7 @@ private FileWorkerDTO convertRecordToDto(Record record, List abilityTagL fileWorkerDTO.setAccessHost(record.get(defaultTable.ACCESS_HOST)); fileWorkerDTO.setAccessPort(record.get(defaultTable.ACCESS_PORT)); fileWorkerDTO.setCloudAreaId(record.get(defaultTable.CLOUD_AREA_ID)); + fileWorkerDTO.setInnerIpProtocol(record.get(defaultTable.INNER_IP_PROTOCOL)); fileWorkerDTO.setInnerIp(record.get(defaultTable.INNER_IP)); fileWorkerDTO.setCpuOverload(JooqTypeUtil.convertToFloat(record.get(defaultTable.CPU_OVERLOAD))); fileWorkerDTO.setMemRate(JooqTypeUtil.convertToFloat((record.get(defaultTable.MEM_RATE)))); @@ -499,8 +509,7 @@ private FileWorkerDTO convertRecordToDto(Record record, List abilityTagL return fileWorkerDTO; } - private FileWorkerDTO convertRecordToDto(Record22 record) { + private FileWorkerDTO convertRecordToDto(Record record) { FileWorkerDTO fileWorkerDTO = new FileWorkerDTO(); fileWorkerDTO.setId((Long) (record.get(KEY_FILE_WORKER_ID))); fileWorkerDTO.setAppId((Long) (record.get(KEY_FILE_WORKER_APP_ID))); @@ -510,6 +519,7 @@ private FileWorkerDTO convertRecordToDto(Record22 getControlUriPatternsList() { - return Collections.emptyList(); - } - - @Override - public boolean preHandle(HttpServletRequest request, - @NonNull HttpServletResponse response, - @NonNull Object handler) { - String username = JobContextUtil.getUsername(); - String uri = request.getRequestURI(); - log.info("PermissionControlInterceptor.preHandle:username=" + username + ", uri=" + uri + ", " + - "controlUriPatterns=" + getControlUriPatternsList()); - return true; - } - - @Override - public void afterCompletion(@NonNull HttpServletRequest request, - @NonNull HttpServletResponse response, - @NonNull Object handler, - Exception ex) { - } -} diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/metrics/MetricsConstants.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/metrics/MetricsConstants.java index 0fe559c43b..223383cc62 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/metrics/MetricsConstants.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/metrics/MetricsConstants.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -31,12 +31,20 @@ public class MetricsConstants { public static final String NAME_FILE_WORKER_ONLINE_NUM = "fileWorker.online.num"; public static final String NAME_FILE_WORKER_RESPONSE_TIME = "fileWorker.response.time"; public static final String NAME_FILE_GATEWAY_DISPATCH_TIME = "fileGateway.dispatch.time"; + public static final String NAME_FILE_GATEWAY_REDISPATCH_TIME = "fileGateway.reDispatch.time"; // tag public static final String TAG_KEY_MODULE = "module"; + public static final String TAG_KEY_REQUEST_SOURCE = "requestSource"; + public static final String TAG_KEY_DISPATCH_RESULT = "dispatchResult"; + public static final String TAG_KEY_APP_ID = "appId"; // value public static final String TAG_VALUE_MODULE_FILE_WORKER = "fileWorker"; public static final String TAG_VALUE_MODULE_FILE_GATEWAY = "fileGateway"; + public static final String TAG_VALUE_DISPATCH_RESULT_TRUE = "true"; + public static final String TAG_VALUE_DISPATCH_RESULT_FALSE = "false"; + public static final String TAG_VALUE_REDISPATCH_STATUS_SUCCESS = "success"; + public static final String TAG_VALUE_REDISPATCH_STATUS_ERROR = "error"; } diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/dto/FileSourceBasicInfoDTO.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/dto/FileSourceBasicInfoDTO.java index 9f0a00c34b..972b602cb2 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/dto/FileSourceBasicInfoDTO.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/dto/FileSourceBasicInfoDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/dto/FileSourceBatchTaskDTO.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/dto/FileSourceBatchTaskDTO.java index 5389bdf930..713011c0fe 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/dto/FileSourceBatchTaskDTO.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/dto/FileSourceBatchTaskDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/dto/FileSourceDTO.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/dto/FileSourceDTO.java index 85e40efe01..ad092fac86 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/dto/FileSourceDTO.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/dto/FileSourceDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -30,6 +30,7 @@ import com.tencent.bk.job.common.util.ApplicationContextRegister; import com.tencent.bk.job.common.util.json.LongTimestampSerializer; import com.tencent.bk.job.file_gateway.model.resp.common.SimpleFileSourceVO; +import com.tencent.bk.job.file_gateway.model.resp.esb.v3.EsbFileSourceV3DTO; import com.tencent.bk.job.file_gateway.model.resp.web.FileSourceVO; import lombok.AllArgsConstructor; import lombok.Data; @@ -91,7 +92,7 @@ public class FileSourceDTO { */ private Boolean shareToAllApp; /** - * 凭据Id + * 凭证Id */ private String credentialId; /** @@ -149,6 +150,32 @@ public static SimpleFileSourceVO toSimpleFileSourceVO(FileSourceDTO fileSourceDT return fileSourceVO; } + public static EsbFileSourceV3DTO toEsbFileSourceV3DTO(FileSourceDTO fileSourceDTO) { + if (fileSourceDTO == null) { + return null; + } + EsbFileSourceV3DTO fileSource = new EsbFileSourceV3DTO(); + fileSource.setId(fileSourceDTO.getId()); + AppScopeMappingService appScopeMappingService = + ApplicationContextRegister.getBean(AppScopeMappingService.class); + ResourceScope resourceScope = appScopeMappingService.getScopeByAppId(fileSourceDTO.getAppId()); + fileSource.setScopeType(resourceScope.getType().getValue()); + fileSource.setScopeId(resourceScope.getId()); + fileSource.setCode(fileSourceDTO.getCode()); + fileSource.setAlias(fileSourceDTO.getAlias()); + fileSource.setCredentialId(fileSourceDTO.getCredentialId()); + fileSource.setEnable(fileSourceDTO.getEnable()); + fileSource.setFileSourceTypeCode(fileSourceDTO.getFileSourceType() != null ? + fileSourceDTO.getFileSourceType().getCode() : null); + fileSource.setPublicFlag(fileSourceDTO.getPublicFlag()); + fileSource.setStatus(fileSourceDTO.getStatus()); + fileSource.setCreateTime(fileSourceDTO.getCreateTime()); + fileSource.setCreator(fileSourceDTO.getCreator()); + fileSource.setLastModifyTime(fileSourceDTO.getLastModifyTime()); + fileSource.setLastModifyUser(fileSourceDTO.getLastModifyUser()); + return fileSource; + } + public static FileSourceVO toVO(FileSourceDTO fileSourceDTO) { if (fileSourceDTO == null) { return null; @@ -199,4 +226,8 @@ public static FileSourceVO toVO(FileSourceDTO fileSourceDTO) { fileSourceVO.setLastModifyTime(fileSourceDTO.getLastModifyTime()); return fileSourceVO; } + + public String getBasicDesc() { + return "(id=" + id + ", appId=" + appId + ", code=" + code + ", alias=" + alias + ", enable=" + enable + ")"; + } } diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/dto/FileSourceTaskDTO.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/dto/FileSourceTaskDTO.java index c85ec8868b..77e3fe4921 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/dto/FileSourceTaskDTO.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/dto/FileSourceTaskDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/dto/FileSourceTypeDTO.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/dto/FileSourceTypeDTO.java index 3f5cbbca0f..bcc8add902 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/dto/FileSourceTypeDTO.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/dto/FileSourceTypeDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -60,6 +60,10 @@ public class FileSourceTypeDTO { * 文件源类型名称 */ private String name; + /** + * 文件源类型是否启用 + */ + private Boolean enabled; /** * 文件源类型图标Base64编码 */ diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/dto/FileSourceWhiteInfoDTO.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/dto/FileSourceWhiteInfoDTO.java new file mode 100644 index 0000000000..b80ca46fb1 --- /dev/null +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/dto/FileSourceWhiteInfoDTO.java @@ -0,0 +1,77 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file_gateway.model.dto; + +import com.tencent.bk.job.common.util.TimeUtil; +import com.tencent.bk.job.file_gateway.model.resp.op.FileSourceWhiteInfoVO; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.time.LocalDateTime; + +/** + * 文件源白名单信息 + */ +@Data +@NoArgsConstructor +@AllArgsConstructor +public class FileSourceWhiteInfoDTO { + /** + * id + */ + private Integer id; + /** + * 类型 + */ + private String type; + /** + * 内容 + */ + private String content; + /** + * 备注 + */ + private String remark; + /** + * 创建人 + */ + private String creator; + /** + * 创建时间 + */ + private LocalDateTime createTime; + + public FileSourceWhiteInfoVO toFileSourceWhiteInfoVO() { + FileSourceWhiteInfoVO fileSourceWhiteInfoVO = new FileSourceWhiteInfoVO(); + fileSourceWhiteInfoVO.setId(this.id); + fileSourceWhiteInfoVO.setType(this.type); + fileSourceWhiteInfoVO.setContent(this.content); + fileSourceWhiteInfoVO.setRemark(this.remark); + fileSourceWhiteInfoVO.setCreator(this.creator); + fileSourceWhiteInfoVO.setCreateTime(TimeUtil.getTimeStr(this.createTime, TimeUtil.DEFAULT_TIME_FORMAT)); + return fileSourceWhiteInfoVO; + } +} diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/dto/FileTaskDTO.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/dto/FileTaskDTO.java index f42723ee0d..b3ee4e7aba 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/dto/FileTaskDTO.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/dto/FileTaskDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/dto/FileWorkerDTO.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/dto/FileWorkerDTO.java index c8fa2d8820..f93058bb45 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/dto/FileWorkerDTO.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/dto/FileWorkerDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -30,6 +30,7 @@ import com.tencent.bk.job.common.util.ApplicationContextRegister; import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.common.util.json.LongTimestampSerializer; +import com.tencent.bk.job.common.util.json.SkipLogFields; import com.tencent.bk.job.file_gateway.consts.FileGatewayConsts; import com.tencent.bk.job.file_gateway.model.req.common.FileSourceMetaData; import com.tencent.bk.job.file_gateway.model.req.common.FileWorkerConfig; @@ -72,6 +73,7 @@ public class FileWorkerDTO { /** * 密钥 */ + @SkipLogFields private String token; /** * 访问Host @@ -85,6 +87,10 @@ public class FileWorkerDTO { * 所在机器云区域Id */ private Long cloudAreaId; + /** + * 内网IP协议，取值：v4/v6 + */ + private String innerIpProtocol; /** * 内网IP */ @@ -167,6 +173,7 @@ public static FileWorkerDTO fromReq(HeartBeatReq heartBeatReq) { fileWorkerDTO.setAccessHost(heartBeatReq.getAccessHost()); fileWorkerDTO.setAccessPort(heartBeatReq.getAccessPort()); fileWorkerDTO.setCloudAreaId(heartBeatReq.getCloudAreaId()); + fileWorkerDTO.setInnerIpProtocol(heartBeatReq.getInnerIpProtocol()); fileWorkerDTO.setInnerIp(heartBeatReq.getInnerIp()); FileWorkerConfig fileWorkerConfig = heartBeatReq.getFileWorkerConfig(); List fileSourceMetaDataList = fileWorkerConfig.getFileSourceMetaDataList(); @@ -208,7 +215,6 @@ public FileWorkerVO toVO() { fileWorkerVO.setName(name); fileWorkerVO.setDescription(description); - fileWorkerVO.setToken(token); fileWorkerVO.setCloudAreaId(cloudAreaId); fileWorkerVO.setInnerIp(innerIp); fileWorkerVO.setAbilityTagList(abilityTagList); @@ -258,4 +264,12 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(id); } + + public String getBasicDesc() { + return "(id=" + id + ", appId=" + appId + ", name=" + name + ")"; + } + + public String getCloudIp() { + return cloudAreaId + ":" + innerIp; + } } diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/dto/WorkerAbilityDTO.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/dto/WorkerAbilityDTO.java index e4d84ba011..dcac869c7e 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/dto/WorkerAbilityDTO.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/dto/WorkerAbilityDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/dto/WorkerTagDTO.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/dto/WorkerTagDTO.java index fc0750f080..55f3b7fa5d 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/dto/WorkerTagDTO.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/model/dto/WorkerTagDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/AbilityTagService.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/AbilityTagService.java index 78087fdc78..070f6515dd 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/AbilityTagService.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/AbilityTagService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/BatchTaskService.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/BatchTaskService.java index 680eb61f45..1e22aa4ea8 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/BatchTaskService.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/BatchTaskService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -31,7 +31,9 @@ import java.util.List; public interface BatchTaskService { - BatchTaskInfoDTO startFileSourceBatchDownloadTask(String username, Long appId, Long stepInstanceId, + BatchTaskInfoDTO startFileSourceBatchDownloadTask(String username, + Long appId, + Long stepInstanceId, Integer executeCount, List fileSourceTaskList); @@ -39,5 +41,4 @@ BatchTaskInfoDTO startFileSourceBatchDownloadTask(String username, Long appId, L BatchTaskStatusDTO getBatchTaskStatusAndLogs(String batchTaskId, Long logStart, Long logLength); - Integer clearBatchTaskFiles(List batchTaskIdList); } diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/CredentialService.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/CredentialService.java index 5dd296a433..5669d483e8 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/CredentialService.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/CredentialService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -29,7 +29,7 @@ public interface CredentialService { /** - * 获取凭据 + * 获取凭证 * * @param id * @return diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/FileService.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/FileService.java index f6de456397..ef93c1ded0 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/FileService.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/FileService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/FileSourceService.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/FileSourceService.java index a42e50ad2c..dc5e863739 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/FileSourceService.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/FileSourceService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,8 @@ package com.tencent.bk.job.file_gateway.service; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.CompatibleType; import com.tencent.bk.job.file_gateway.model.dto.FileSourceBasicInfoDTO; import com.tencent.bk.job.file_gateway.model.dto.FileSourceDTO; import com.tencent.bk.job.file_gateway.model.dto.FileSourceTypeDTO; @@ -68,9 +70,9 @@ List listWorkTableFileSource( Integer pageSize ); - Integer saveFileSource(Long appId, FileSourceDTO fileSourceDTO); + FileSourceDTO saveFileSource(String username, Long appId, FileSourceDTO fileSourceDTO); - int updateFileSourceById(Long appId, FileSourceDTO fileSourceDTO); + FileSourceDTO updateFileSourceById(String username, Long appId, FileSourceDTO fileSourceDTO); int updateFileSourceStatus(Integer fileSourceId, Integer status); @@ -78,18 +80,25 @@ List listWorkTableFileSource( FileSourceTypeDTO getFileSourceTypeByCode(String code); - Integer deleteFileSourceById(Long appId, Integer id); + Integer deleteFileSourceById(String username, Long appId, Integer id); Boolean enableFileSourceById(String username, Long appId, Integer id, Boolean enableFlag); + FileSourceDTO getFileSourceById(String username, Long appId, Integer id); + FileSourceDTO getFileSourceById(Long appId, Integer id); FileSourceDTO getFileSourceById(Integer id); List listFileSourceByIds(Collection ids); + @Deprecated + @CompatibleImplementation(name = "fileSourceId", deprecatedVersion = "3.9.x", type = CompatibleType.DEPLOY, + explain = "文件源标识仅在appId下唯一，发布完成后可删除") FileSourceDTO getFileSourceByCode(String code); + FileSourceDTO getFileSourceByCode(Long appId, String code); + List getFileSourceParams(Long appId, String fileSourceTypeCode); Boolean checkFileSourceAlias(Long appId, String alias, Integer fileSourceId); @@ -100,5 +109,7 @@ List listWorkTableFileSource( boolean existsFileSource(Long appId, Integer id); + boolean existsFileSourceUsingCredential(Long appId, String credentialId); + Integer getFileSourceIdByCode(Long appId, String code); } diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/FileSourceTaskService.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/FileSourceTaskService.java index 8f567b8298..3633fef118 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/FileSourceTaskService.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/FileSourceTaskService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,23 +24,32 @@ package com.tencent.bk.job.file_gateway.service; -import com.tencent.bk.job.file_gateway.consts.TaskStatusEnum; import com.tencent.bk.job.file_gateway.model.dto.FileSourceTaskDTO; +import com.tencent.bk.job.file_gateway.model.dto.FileTaskProgressDTO; import com.tencent.bk.job.file_gateway.model.resp.inner.FileSourceTaskStatusDTO; import com.tencent.bk.job.file_gateway.model.resp.inner.TaskInfoDTO; import java.util.List; public interface FileSourceTaskService { - TaskInfoDTO startFileSourceDownloadTask(String username, Long appId, Long stepInstanceId, Integer executeCount, - String batchTaskId, Integer fileSourceId, List filePathList); + TaskInfoDTO startFileSourceDownloadTask(String username, + Long appId, + Long stepInstanceId, + Integer executeCount, + String batchTaskId, + Integer fileSourceId, + List filePathList); - TaskInfoDTO startFileSourceDownloadTaskWithId(String username, Long appId, Long stepInstanceId, - Integer executeCount, String batchTaskId, Integer fileSourceId, - List filePathList, String fileSourceTaskId); + TaskInfoDTO startFileSourceDownloadTaskWithId(String username, + Long appId, + Long stepInstanceId, + Integer executeCount, + String batchTaskId, + Integer fileSourceId, + List filePathList, + String fileSourceTaskId); - String updateFileSourceTask(String taskId, String filePath, String downloadPath, Long fileSize, String speed, - Integer progress, String content, TaskStatusEnum status); + String updateFileSourceTask(FileTaskProgressDTO fileTaskProgressDTO); FileSourceTaskStatusDTO getFileSourceTaskStatusAndLogs(String taskId, Long logStart, Long logLength); diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/FileSourceTaskUpdateService.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/FileSourceTaskUpdateService.java new file mode 100644 index 0000000000..f48d12f799 --- /dev/null +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/FileSourceTaskUpdateService.java @@ -0,0 +1,36 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file_gateway.service; + +import com.tencent.bk.job.file_gateway.model.dto.FileTaskProgressDTO; + +public interface FileSourceTaskUpdateService { + + String updateFileSourceTask(String batchTaskId, + String fileSourceTaskId, + Long fileTaskId, + FileTaskProgressDTO fileTaskProgressDTO); + +} diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/FileTaskService.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/FileTaskService.java index 2ff42487cc..947eabb80c 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/FileTaskService.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/FileTaskService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/FileWorkerService.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/FileWorkerService.java index 77e59e5dc6..2725ba73ed 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/FileWorkerService.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/FileWorkerService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/RetryPolicyFileSourceTaskService.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/RetryPolicyFileSourceTaskService.java new file mode 100644 index 0000000000..48ca7fd172 --- /dev/null +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/RetryPolicyFileSourceTaskService.java @@ -0,0 +1,39 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file_gateway.service; + +import com.tencent.bk.job.file_gateway.model.resp.inner.TaskInfoDTO; + +import java.util.List; + +public interface RetryPolicyFileSourceTaskService { + TaskInfoDTO startFileSourceDownloadTask(String username, + Long appId, + Long stepInstanceId, + Integer executeCount, + String batchTaskId, + Integer fileSourceId, + List filePathList); +} diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/context/TaskContext.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/context/TaskContext.java index 5b8ef6de59..0b51aa7be1 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/context/TaskContext.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/context/TaskContext.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/context/impl/DefaultTaskContext.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/context/impl/DefaultTaskContext.java index 55810419ca..966aadb6d8 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/context/impl/DefaultTaskContext.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/context/impl/DefaultTaskContext.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/context/impl/FileSourceTaskRetryContext.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/context/impl/FileSourceTaskRetryContext.java new file mode 100644 index 0000000000..a06f514294 --- /dev/null +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/context/impl/FileSourceTaskRetryContext.java @@ -0,0 +1,37 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file_gateway.service.context.impl; + +import lombok.AllArgsConstructor; +import lombok.Data; + +@Data +@AllArgsConstructor +public class FileSourceTaskRetryContext { + /** + * 重试前业务逻辑代码抛出的异常 + */ + private Exception exception; +} diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/DispatchService.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/dispatch/DispatchService.java similarity index 91% rename from src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/DispatchService.java rename to src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/dispatch/DispatchService.java index a8b0e4ad10..86c09f1fb6 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/DispatchService.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/dispatch/DispatchService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.file_gateway.service; +package com.tencent.bk.job.file_gateway.service.dispatch; import com.tencent.bk.job.file_gateway.model.dto.FileSourceDTO; import com.tencent.bk.job.file_gateway.model.dto.FileWorkerDTO; @@ -36,7 +36,8 @@ public interface DispatchService { * 根据文件源找到一个最适合的FileWorker * * @param fileSourceDTO 文件源对象 + * @param requestSource 请求来源 * @return 选中的对接文件源的FileWorker对象 */ - FileWorkerDTO findBestFileWorker(FileSourceDTO fileSourceDTO); + FileWorkerDTO findBestFileWorker(FileSourceDTO fileSourceDTO, String requestSource); } diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/ReDispatchService.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/dispatch/ReDispatchService.java similarity index 90% rename from src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/ReDispatchService.java rename to src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/dispatch/ReDispatchService.java index e96ef1bb91..b45e532bc9 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/ReDispatchService.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/dispatch/ReDispatchService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.file_gateway.service; +package com.tencent.bk.job.file_gateway.service.dispatch; import java.util.List; @@ -39,6 +39,6 @@ List reDispatchByWorker( boolean reDispatchByGateway(String fileSourceTaskId, Long initDelayMills, Long intervalMills); - Integer getReDispatchThreadsNum(String username); + Integer getReDispatchThreadsNum(); } diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/dispatch/ReDispatchTaskService.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/dispatch/ReDispatchTaskService.java new file mode 100644 index 0000000000..6ac01fd58e --- /dev/null +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/dispatch/ReDispatchTaskService.java @@ -0,0 +1,31 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file_gateway.service.dispatch; + +import com.tencent.bk.job.file_gateway.model.resp.inner.TaskInfoDTO; + +public interface ReDispatchTaskService { + TaskInfoDTO reDispatchFileSourceTask(String fileSourceTaskId); +} diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/DispatchServiceImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/dispatch/impl/DispatchServiceImpl.java similarity index 79% rename from src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/DispatchServiceImpl.java rename to src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/dispatch/impl/DispatchServiceImpl.java index c37ef9752d..0ed094b007 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/DispatchServiceImpl.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/dispatch/impl/DispatchServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.file_gateway.service.impl; +package com.tencent.bk.job.file_gateway.service.dispatch.impl; import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.file_gateway.consts.WorkerSelectModeEnum; @@ -32,18 +32,20 @@ import com.tencent.bk.job.file_gateway.model.dto.FileSourceDTO; import com.tencent.bk.job.file_gateway.model.dto.FileWorkerDTO; import com.tencent.bk.job.file_gateway.service.AbilityTagService; -import com.tencent.bk.job.file_gateway.service.DispatchService; import com.tencent.bk.job.file_gateway.service.FileWorkerService; +import com.tencent.bk.job.file_gateway.service.dispatch.DispatchService; +import com.tencent.bk.job.file_gateway.service.impl.WorkerIdsCondition; import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tag; import io.micrometer.core.instrument.Timer; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import java.util.ArrayList; -import java.util.Comparator; import java.util.List; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicLong; import java.util.stream.Collectors; @Slf4j @@ -55,6 +57,8 @@ public class DispatchServiceImpl implements DispatchService { private final FileWorkerDAO fileWorkerDAO; private final FileWorkerService fileWorkerService; + private final AtomicLong roundRobinCount = new AtomicLong(0); + @Autowired public DispatchServiceImpl(FileWorkerDAO fileWorkerDAO, AbilityTagService abilityTagService, @@ -142,24 +146,50 @@ private List getFileWorkerByScopeAndAbilityTag(Long appId, } @Override - public FileWorkerDTO findBestFileWorker(FileSourceDTO fileSourceDTO) { + public FileWorkerDTO findBestFileWorker(FileSourceDTO fileSourceDTO, String requestSource) { Timer.Sample sample = Timer.start(meterRegistry); - FileWorkerDTO fileWorkerDTO = findBestFileWorkerIndeed(fileSourceDTO); - long nanoSeconds = sample.stop(meterRegistry.timer(MetricsConstants.NAME_FILE_GATEWAY_DISPATCH_TIME, - MetricsConstants.TAG_KEY_MODULE, MetricsConstants.TAG_VALUE_MODULE_FILE_GATEWAY)); - long millis = TimeUnit.NANOSECONDS.toMillis(nanoSeconds); - if (millis > 2000) { - log.warn("Dispatch time over 2000ms, fileSourceDTO={}", JsonUtils.toJson(fileSourceDTO)); + FileWorkerDTO fileWorkerDTO = null; + try { + fileWorkerDTO = findBestFileWorkerIndeed(fileSourceDTO); + } catch (Exception e) { + log.warn("Fail to findBestFileWorker", e); + } finally { + long nanoSeconds = sample.stop( + meterRegistry.timer( + MetricsConstants.NAME_FILE_GATEWAY_DISPATCH_TIME, + buildDispatchResult(fileWorkerDTO, requestSource) + ) + ); + long millis = TimeUnit.NANOSECONDS.toMillis(nanoSeconds); + if (millis > 2000) { + log.warn("SLOW: Dispatch time over 2000ms, fileSourceDTO={}", fileSourceDTO.getBasicDesc()); + } } return fileWorkerDTO; } + private Iterable buildDispatchResult(FileWorkerDTO fileWorkerDTO, String requestSource) { + List tagList = new ArrayList<>(); + tagList.add(Tag.of(MetricsConstants.TAG_KEY_MODULE, MetricsConstants.TAG_VALUE_MODULE_FILE_GATEWAY)); + tagList.add(Tag.of(MetricsConstants.TAG_KEY_REQUEST_SOURCE, requestSource)); + if (fileWorkerDTO == null) { + tagList.add( + Tag.of(MetricsConstants.TAG_KEY_DISPATCH_RESULT, MetricsConstants.TAG_VALUE_DISPATCH_RESULT_FALSE) + ); + } else { + tagList.add( + Tag.of(MetricsConstants.TAG_KEY_DISPATCH_RESULT, MetricsConstants.TAG_VALUE_DISPATCH_RESULT_TRUE) + ); + } + return tagList; + } + private FileWorkerDTO findWorkerByAuto(FileSourceDTO fileSourceDTO) { FileWorkerDTO fileWorkerDTO = null; String workerSelectScope = fileSourceDTO.getWorkerSelectScope(); List abilityTagList = abilityTagService.getAbilityTagList(fileSourceDTO); List fileWorkerDTOList; - if (abilityTagList == null || abilityTagList.size() == 0) { + if (abilityTagList == null || abilityTagList.isEmpty()) { // 无能力标签要求，任选一个FileWorker fileWorkerDTOList = getFileWorkerByScope(fileSourceDTO.getAppId(), workerSelectScope); if (fileWorkerDTOList.isEmpty()) { @@ -193,15 +223,14 @@ private FileWorkerDTO findWorkerByAuto(FileSourceDTO fileSourceDTO) { return null; } // 在线状态过滤 - fileWorkerDTOList = fileWorkerDTOList.parallelStream().filter(tmpFileWorkerDTO -> { + fileWorkerDTOList = fileWorkerDTOList.stream().filter(tmpFileWorkerDTO -> { Byte onlineStatus = tmpFileWorkerDTO.getOnlineStatus(); return onlineStatus != null && onlineStatus.intValue() == 1; }).collect(Collectors.toList()); if (!fileWorkerDTOList.isEmpty()) { - // 按策略调度：内存占用最小 - fileWorkerDTOList.sort(Comparator.comparing(FileWorkerDTO::getMemRate)); - log.debug("ordered fileWorkerDTOList:{}", fileWorkerDTOList); - fileWorkerDTO = fileWorkerDTOList.get(0); + // 按策略调度：RoundRobin + int index = (int) (roundRobinCount.getAndIncrement() % fileWorkerDTOList.size()); + fileWorkerDTO = fileWorkerDTOList.get(index); } else { log.error("Cannot find available file worker, abilityTagList={}", abilityTagList); } @@ -211,7 +240,7 @@ private FileWorkerDTO findWorkerByAuto(FileSourceDTO fileSourceDTO) { private FileWorkerDTO findBestFileWorkerIndeed(FileSourceDTO fileSourceDTO) { String mode = fileSourceDTO.getWorkerSelectMode(); FileWorkerDTO fileWorkerDTO; - log.info("select worker with mode={},fileSourceDTO={}", mode, JsonUtils.toJson(fileSourceDTO)); + log.info("select worker with mode={},fileSourceDTO={}", mode, fileSourceDTO.getBasicDesc()); if (WorkerSelectModeEnum.MANUAL.name().equals(mode)) { Long workerId = fileSourceDTO.getWorkerId(); fileWorkerDTO = fileWorkerDAO.getFileWorkerById(workerId); @@ -225,10 +254,13 @@ private FileWorkerDTO findBestFileWorkerIndeed(FileSourceDTO fileSourceDTO) { throw new RuntimeException(String.format("workerSelectMode %s not supported yet", mode)); } if (fileWorkerDTO != null) { - log.info("FileSource ({},{}) choose worker:{}", fileSourceDTO.getCode(), fileSourceDTO.getAlias(), - JsonUtils.toJson(fileWorkerDTO.toBaseVO())); + log.info( + "FileSource {} choose worker:{}", + fileSourceDTO.getBasicDesc(), + JsonUtils.toJson(fileWorkerDTO.toBaseVO()) + ); } else { - log.info("FileSource ({},{}) can not find worker", fileSourceDTO.getCode(), fileSourceDTO.getAlias()); + log.info("FileSource {} can not find worker", fileSourceDTO.getBasicDesc()); } return fileWorkerDTO; } diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/dispatch/impl/ReDispatchServiceImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/dispatch/impl/ReDispatchServiceImpl.java new file mode 100644 index 0000000000..87a95756c9 --- /dev/null +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/dispatch/impl/ReDispatchServiceImpl.java @@ -0,0 +1,144 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file_gateway.service.dispatch.impl; + +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.InternalException; +import com.tencent.bk.job.file_gateway.model.dto.FileSourceTaskDTO; +import com.tencent.bk.job.file_gateway.model.dto.FileWorkerDTO; +import com.tencent.bk.job.file_gateway.service.FileSourceTaskService; +import com.tencent.bk.job.file_gateway.service.FileWorkerService; +import com.tencent.bk.job.file_gateway.service.dispatch.ReDispatchService; +import com.tencent.bk.job.file_gateway.service.dispatch.ReDispatchTaskService; +import lombok.extern.slf4j.Slf4j; +import org.slf4j.helpers.FormattingTuple; +import org.slf4j.helpers.MessageFormatter; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.Collections; +import java.util.List; +import java.util.Timer; + +@Slf4j +@Service +public class ReDispatchServiceImpl implements ReDispatchService { + + private final FileWorkerService fileWorkerService; + private final FileSourceTaskService fileSourceTaskService; + private final ReDispatchTaskService reDispatchTaskService; + // 最多使用50线程进行重调度 + private final int MAX_THREAD_NUM_REDISPATCH = 50; + + @Autowired + public ReDispatchServiceImpl( + FileWorkerService fileWorkerService, + FileSourceTaskService fileSourceTaskService, + ReDispatchTaskService reDispatchTaskService + ) { + this.fileWorkerService = fileWorkerService; + this.fileSourceTaskService = fileSourceTaskService; + this.reDispatchTaskService = reDispatchTaskService; + } + + @Override + public List reDispatchByWorker( + String accessHost, + Integer accessPort, + List taskIdList, + Long initDelayMills, + Long intervalMills + ) { + FileWorkerDTO fileWorkerDTO = fileWorkerService.getFileWorker(accessHost, accessPort); + if (fileWorkerDTO == null) { + FormattingTuple msg = MessageFormatter.format( + "Fail to find file-worker by accessHost:{} accessPort:{}", accessHost, accessPort + ); + log.warn(msg.getMessage()); + throw new InternalException( + ErrorCode.FILE_WORKER_NOT_FOUND, + new String[]{ + "accessHost:" + accessHost + ",accessPort:" + accessPort, + } + ); + } + Long workerId = fileWorkerDTO.getId(); + log.debug("worker {} apply to reDispatch tasks:{}, initDelayMills={}, intervalMills={}", workerId, taskIdList + , initDelayMills, intervalMills); + // 1.立即下线Worker + int affectedWorkerNum = fileWorkerService.offLine(workerId); + log.info("{} worker state changed to offline", affectedWorkerNum); + // 2.任务延时重调度 + for (String taskId : taskIdList) { + if (getReDispatchThreadsNum() >= MAX_THREAD_NUM_REDISPATCH) { + log.warn("reDispatch thread reach MAX_NUM:{}, do not reDispatch {}", MAX_THREAD_NUM_REDISPATCH, taskId); + continue; + } + Timer timer = new Timer(); + ReDispatchTask reDispatchTask = buildReDispatchTask(taskId, intervalMills); + timer.schedule(reDispatchTask, initDelayMills); + } + return taskIdList; + } + + @Override + public boolean reDispatchByGateway(String fileSourceTaskId, Long initDelayMills, Long intervalMills) { + // 1.尝试通知Worker主动取消该任务 + FileSourceTaskDTO fileSourceTaskDTO = fileSourceTaskService.getFileSourceTaskById(fileSourceTaskId); + if (fileSourceTaskDTO == null) { + log.warn("task not exist, ignore, id={}", fileSourceTaskId); + return false; + } + try { + fileSourceTaskService.recallTasks(Collections.singletonList(fileSourceTaskId)); + } catch (Throwable t) { + log.warn("Fail to recallTask:{}", fileSourceTaskId, t); + } + // 2.重调度 + if (getReDispatchThreadsNum() >= MAX_THREAD_NUM_REDISPATCH) { + log.warn("reDispatch thread reach MAX_NUM:{}, do not reDispatch {}", MAX_THREAD_NUM_REDISPATCH, + fileSourceTaskId); + return false; + } + Timer timer = new Timer(); + ReDispatchTask reDispatchTask = buildReDispatchTask(fileSourceTaskId, intervalMills); + timer.schedule(reDispatchTask, initDelayMills); + return true; + } + + private ReDispatchTask buildReDispatchTask(String fileSourceTaskId, Long intervalMills) { + return new ReDispatchTask( + reDispatchTaskService, + fileSourceTaskId, + intervalMills + ); + } + + @Override + public Integer getReDispatchThreadsNum() { + return ReDispatchTask.getReDispatchThreadsNum(); + } + +} diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/dispatch/impl/ReDispatchTask.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/dispatch/impl/ReDispatchTask.java new file mode 100644 index 0000000000..2b8d8e08b6 --- /dev/null +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/dispatch/impl/ReDispatchTask.java @@ -0,0 +1,106 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file_gateway.service.dispatch.impl; + +import com.tencent.bk.job.common.util.ThreadUtils; +import com.tencent.bk.job.file_gateway.model.resp.inner.TaskInfoDTO; +import com.tencent.bk.job.file_gateway.service.dispatch.ReDispatchTaskService; +import lombok.extern.slf4j.Slf4j; +import org.slf4j.helpers.MessageFormatter; + +import java.util.HashSet; +import java.util.Set; +import java.util.TimerTask; +import java.util.concurrent.atomic.AtomicInteger; + +@Slf4j +public class ReDispatchTask extends TimerTask { + private static final AtomicInteger reDispatchThreadNum = new AtomicInteger(0); + private static final Set reDispatchingTaskIds = new HashSet<>(); + private final ReDispatchTaskService reDispatchTaskService; + private final String fileSourceTaskId; + private final Long intervalMills; + + ReDispatchTask(ReDispatchTaskService reDispatchTaskService, + String fileSourceTaskId, + Long intervalMills) { + this.reDispatchTaskService = reDispatchTaskService; + this.fileSourceTaskId = fileSourceTaskId; + this.intervalMills = intervalMills; + } + + public static Integer getReDispatchThreadsNum() { + return reDispatchThreadNum.get(); + } + + @Override + public void run() { + synchronized (reDispatchingTaskIds) { + if (reDispatchingTaskIds.contains(fileSourceTaskId)) { + log.info("task {} already in reDispatching, ignore", fileSourceTaskId); + return; + } + reDispatchingTaskIds.add(fileSourceTaskId); + reDispatchThreadNum.incrementAndGet(); + } + reDispatchTaskWithRetry(); + synchronized (reDispatchingTaskIds) { + reDispatchingTaskIds.remove(fileSourceTaskId); + reDispatchThreadNum.decrementAndGet(); + } + } + + /** + * 对文件源任务进行重调度，失败时进行重试 + */ + private void reDispatchTaskWithRetry() { + boolean reDispatchSuccess = false; + int retryCount = 0; + int maxRetryCount = 3; + while (!reDispatchSuccess && retryCount < maxRetryCount) { + try { + TaskInfoDTO taskInfoDTO = reDispatchTaskService.reDispatchFileSourceTask(fileSourceTaskId); + reDispatchSuccess = true; + log.debug("reDispatch result of {}:{}", fileSourceTaskId, taskInfoDTO); + } catch (Exception e) { + retryCount += 1; + String message = MessageFormatter.format( + "Fail to redispatch task {}, wait {}ms to retry {}", + new Object[]{ + fileSourceTaskId, + intervalMills, + retryCount + } + ).getMessage(); + if (retryCount < maxRetryCount) { + log.info(message); + ThreadUtils.sleep(intervalMills); + } else { + log.error(message, e); + } + } + } + } +} diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/dispatch/impl/ReDispatchTaskServiceImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/dispatch/impl/ReDispatchTaskServiceImpl.java new file mode 100644 index 0000000000..0927d77171 --- /dev/null +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/dispatch/impl/ReDispatchTaskServiceImpl.java @@ -0,0 +1,142 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file_gateway.service.dispatch.impl; + +import com.tencent.bk.job.common.mysql.JobTransactional; +import com.tencent.bk.job.file_gateway.metrics.MetricsConstants; +import com.tencent.bk.job.file_gateway.model.dto.FileSourceTaskDTO; +import com.tencent.bk.job.file_gateway.model.dto.FileTaskDTO; +import com.tencent.bk.job.file_gateway.model.resp.inner.TaskInfoDTO; +import com.tencent.bk.job.file_gateway.service.FileSourceTaskService; +import com.tencent.bk.job.file_gateway.service.FileTaskService; +import com.tencent.bk.job.file_gateway.service.dispatch.ReDispatchTaskService; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tag; +import io.micrometer.core.instrument.Timer; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; + +@Slf4j +@Service +public class ReDispatchTaskServiceImpl implements ReDispatchTaskService { + + private final FileSourceTaskService fileSourceTaskService; + private final FileTaskService fileTaskService; + private final MeterRegistry meterRegistry; + + @Autowired + public ReDispatchTaskServiceImpl(FileSourceTaskService fileSourceTaskService, + FileTaskService fileTaskService, + MeterRegistry meterRegistry) { + this.fileSourceTaskService = fileSourceTaskService; + this.fileTaskService = fileTaskService; + this.meterRegistry = meterRegistry; + } + + /** + * 对文件源任务进行重调度，过程中开启事务保证数据一致性 + * + * @param fileSourceTaskId 文件源任务ID + * @return 重调度结果 + */ + @Override + @JobTransactional(transactionManager = "jobFileGatewayTransactionManager") + public TaskInfoDTO reDispatchFileSourceTask(String fileSourceTaskId) { + long startTime = System.currentTimeMillis(); + FileSourceTaskDTO fileSourceTaskDTO = fileSourceTaskService.getFileSourceTaskById(fileSourceTaskId); + String reDispatchStatus = null; + try { + TaskInfoDTO taskInfoDTO = doReDispatchFileSourceTask(fileSourceTaskDTO); + reDispatchStatus = MetricsConstants.TAG_VALUE_REDISPATCH_STATUS_SUCCESS; + return taskInfoDTO; + } catch (Exception e) { + reDispatchStatus = MetricsConstants.TAG_VALUE_REDISPATCH_STATUS_ERROR; + throw e; + } finally { + long timeConsumingMills = System.currentTimeMillis() - startTime; + recordReDispatchCost(timeConsumingMills, buildDispatchTags(fileSourceTaskDTO.getAppId(), reDispatchStatus)); + } + } + + private Iterable buildDispatchTags(Long appId, String reDispatchStatus) { + List tagList = new ArrayList<>(); + tagList.add(Tag.of(MetricsConstants.TAG_KEY_MODULE, MetricsConstants.TAG_VALUE_MODULE_FILE_GATEWAY)); + tagList.add(Tag.of(MetricsConstants.TAG_KEY_APP_ID, String.valueOf(appId))); + tagList.add(Tag.of(MetricsConstants.TAG_KEY_DISPATCH_RESULT, reDispatchStatus)); + return tagList; + } + + private TaskInfoDTO doReDispatchFileSourceTask(FileSourceTaskDTO fileSourceTaskDTO) { + String fileSourceTaskId = fileSourceTaskDTO.getId(); + Long oldFileWorkerId = fileSourceTaskDTO.getFileWorkerId(); + List fileTaskDTOList = fileTaskService.listFileTasks(fileSourceTaskId); + List filePathList = + fileTaskDTOList.stream().map(FileTaskDTO::getFilePath).collect(Collectors.toList()); + // 1.删除现有子任务 + int deletedTaskNum = fileTaskService.deleteTasks(fileSourceTaskId); + // 2.删除现有FileSourceTask任务 + int deletedFileSourceTaskNum = fileSourceTaskService.deleteFileSourceTaskById(fileSourceTaskId); + // 3.重新派发任务 + TaskInfoDTO taskInfoDTO = fileSourceTaskService.startFileSourceDownloadTaskWithId( + fileSourceTaskDTO.getCreator(), + fileSourceTaskDTO.getAppId(), + fileSourceTaskDTO.getStepInstanceId(), + fileSourceTaskDTO.getExecuteCount(), + fileSourceTaskDTO.getBatchTaskId(), + fileSourceTaskDTO.getFileSourceId(), + filePathList, + fileSourceTaskId + ); + log.info( + "FileSourceTask(id={}, oldFileWorkerId={}) reDispatched to worker(id={},accessHost={})," + + " [ {} fileTask, {} fileSourceTask] deleted and re-inserted", + fileSourceTaskId, + oldFileWorkerId, + taskInfoDTO.getWorkerId(), + taskInfoDTO.getWorkerAccessHost(), + deletedTaskNum, + deletedFileSourceTaskNum + ); + return taskInfoDTO; + } + + private void recordReDispatchCost(long timeConsumingMillis, Iterable tags) { + Timer.builder(MetricsConstants.NAME_FILE_GATEWAY_REDISPATCH_TIME) + .description("ReDispatch FileSourceTask Cost") + .tags(tags) + .publishPercentileHistogram(true) + .minimumExpectedValue(Duration.ofMillis(10)) + .maximumExpectedValue(Duration.ofSeconds(60L)) + .register(meterRegistry) + .record(timeConsumingMillis, TimeUnit.MILLISECONDS); + } +} diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/AbilityTagServiceImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/AbilityTagServiceImpl.java index 3eb4ea0315..2a4c5bc6d6 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/AbilityTagServiceImpl.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/AbilityTagServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/BatchTaskServiceImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/BatchTaskServiceImpl.java index f03bdaf23a..5f1b00c744 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/BatchTaskServiceImpl.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/BatchTaskServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -38,8 +38,8 @@ import com.tencent.bk.job.file_gateway.model.resp.inner.TaskInfoDTO; import com.tencent.bk.job.file_gateway.service.BatchTaskService; import com.tencent.bk.job.file_gateway.service.FileSourceTaskService; +import com.tencent.bk.job.file_gateway.service.RetryPolicyFileSourceTaskService; import lombok.extern.slf4j.Slf4j; -import org.jooq.DSLContext; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -52,22 +52,25 @@ public class BatchTaskServiceImpl implements BatchTaskService { private final FileSourceTaskService fileSourceTaskService; + private final RetryPolicyFileSourceTaskService retryPolicyFileSourceTaskService; private final FileSourceBatchTaskDAO fileSourceBatchTaskDAO; - private final DSLContext dslContext; private final FileSourceTaskDAO fileSourceTaskDAO; @Autowired public BatchTaskServiceImpl(FileSourceTaskService fileSourceTaskService, - FileSourceBatchTaskDAO fileSourceBatchTaskDAO, DSLContext dslContext, + RetryPolicyFileSourceTaskService retryPolicyFileSourceTaskService, + FileSourceBatchTaskDAO fileSourceBatchTaskDAO, FileSourceTaskDAO fileSourceTaskDAO) { this.fileSourceTaskService = fileSourceTaskService; + this.retryPolicyFileSourceTaskService = retryPolicyFileSourceTaskService; this.fileSourceBatchTaskDAO = fileSourceBatchTaskDAO; - this.dslContext = dslContext; this.fileSourceTaskDAO = fileSourceTaskDAO; } @Override - public BatchTaskInfoDTO startFileSourceBatchDownloadTask(String username, Long appId, Long stepInstanceId, + public BatchTaskInfoDTO startFileSourceBatchDownloadTask(String username, + Long appId, + Long stepInstanceId, Integer executeCount, List fileSourceTaskList) { BatchTaskInfoDTO batchTaskInfoDTO = new BatchTaskInfoDTO(); @@ -78,13 +81,19 @@ public BatchTaskInfoDTO startFileSourceBatchDownloadTask(String username, Long a fileSourceBatchTaskDTO.setStepInstanceId(stepInstanceId); fileSourceBatchTaskDTO.setExecuteCount(executeCount); fileSourceBatchTaskDTO.setStatus(TaskStatusEnum.INIT.getStatus()); - String batchTaskId = fileSourceBatchTaskDAO.insertFileSourceBatchTask(dslContext, fileSourceBatchTaskDTO); + String batchTaskId = fileSourceBatchTaskDAO.insertFileSourceBatchTask(fileSourceBatchTaskDTO); batchTaskInfoDTO.setBatchTaskId(batchTaskId); List taskInfoDTOList = new ArrayList<>(); for (FileSourceTaskContent fileSourceTaskContent : fileSourceTaskList) { - TaskInfoDTO taskInfoDTO = fileSourceTaskService.startFileSourceDownloadTask(username, appId, - stepInstanceId, executeCount, batchTaskId, fileSourceTaskContent.getFileSourceId(), - fileSourceTaskContent.getFilePathList()); + TaskInfoDTO taskInfoDTO = retryPolicyFileSourceTaskService.startFileSourceDownloadTask( + username, + appId, + stepInstanceId, + executeCount, + batchTaskId, + fileSourceTaskContent.getFileSourceId(), + fileSourceTaskContent.acquireUniqueFilePathList() + ); taskInfoDTOList.add(taskInfoDTO); } batchTaskInfoDTO.setTaskInfoList(taskInfoDTOList); @@ -95,7 +104,7 @@ public List getFileSourceTaskIdListByBatch(List batchTaskIdList) List fileSourceTaskIdList = new ArrayList<>(); for (String batchTaskId : batchTaskIdList) { List fileSourceTaskDTOList = fileSourceTaskDAO.listByBatchTaskId(batchTaskId); - fileSourceTaskIdList.addAll(fileSourceTaskDTOList.parallelStream() + fileSourceTaskIdList.addAll(fileSourceTaskDTOList.stream() .map(FileSourceTaskDTO::getId).collect(Collectors.toList())); } return fileSourceTaskIdList; @@ -111,8 +120,7 @@ public Integer stopBatchTasks(List batchTaskIdList) { public BatchTaskStatusDTO getBatchTaskStatusAndLogs(String batchTaskId, Long logStart, Long logLength) { BatchTaskStatusDTO batchTaskStatusDTO = new BatchTaskStatusDTO(); batchTaskStatusDTO.setBatchTaskId(batchTaskId); - FileSourceBatchTaskDTO fileSourceBatchTaskDTO = fileSourceBatchTaskDAO.getFileSourceBatchTaskById(dslContext, - batchTaskId); + FileSourceBatchTaskDTO fileSourceBatchTaskDTO = fileSourceBatchTaskDAO.getBatchTaskById(batchTaskId); if (fileSourceBatchTaskDTO == null) { throw new InternalException(ErrorCode.INTERNAL_ERROR); } @@ -127,10 +135,4 @@ public BatchTaskStatusDTO getBatchTaskStatusAndLogs(String batchTaskId, Long log batchTaskStatusDTO.setFileSourceTaskStatusInfoList(fileSourceTaskStatusInfoList); return batchTaskStatusDTO; } - - @Override - public Integer clearBatchTaskFiles(List batchTaskIdList) { - List fileSourceTaskIdList = getFileSourceTaskIdListByBatch(batchTaskIdList); - return fileSourceTaskService.clearTaskFiles(fileSourceTaskIdList); - } } diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/CredentialServiceImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/CredentialServiceImpl.java index 493e4a6c0c..53b8c996b4 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/CredentialServiceImpl.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/CredentialServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,10 +25,10 @@ package com.tencent.bk.job.file_gateway.service.impl; import com.tencent.bk.job.common.model.InternalResponse; +import com.tencent.bk.job.common.model.dto.CommonCredential; import com.tencent.bk.job.common.util.json.JsonUtils; -import com.tencent.bk.job.file_gateway.client.ServiceCredentialResourceClient; import com.tencent.bk.job.file_gateway.service.CredentialService; -import com.tencent.bk.job.common.model.dto.CommonCredential; +import com.tencent.bk.job.manage.api.inner.ServiceCredentialResource; import com.tencent.bk.job.manage.model.inner.resp.ServiceCredentialDTO; import lombok.extern.slf4j.Slf4j; import org.slf4j.helpers.FormattingTuple; @@ -37,19 +37,19 @@ import org.springframework.stereotype.Service; @Slf4j -@Service +@Service("jobFileGatewayCredentialServiceImpl") public class CredentialServiceImpl implements CredentialService { - ServiceCredentialResourceClient credentialService; + private final ServiceCredentialResource credentialResource; @Autowired - public CredentialServiceImpl(ServiceCredentialResourceClient credentialService) { - this.credentialService = credentialService; + public CredentialServiceImpl(ServiceCredentialResource credentialResource) { + this.credentialResource = credentialResource; } @Override public CommonCredential getCredentialById(Long appId, String id) { - InternalResponse credentialWebResponse = credentialService.getCredentialById(appId, + InternalResponse credentialWebResponse = credentialResource.getCredentialById(appId, id); ServiceCredentialDTO credentialDTO = credentialWebResponse.getData(); if (credentialDTO == null) { diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/FileServiceImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/FileServiceImpl.java index 15878bfd7e..4a13f80924 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/FileServiceImpl.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/FileServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -37,11 +37,12 @@ import com.tencent.bk.job.file_gateway.model.req.common.ExecuteActionReq; import com.tencent.bk.job.file_gateway.model.resp.common.FileNodesDTO; import com.tencent.bk.job.file_gateway.model.resp.common.FileNodesVO; -import com.tencent.bk.job.file_gateway.service.DispatchService; import com.tencent.bk.job.file_gateway.service.FileService; import com.tencent.bk.job.file_gateway.service.FileSourceService; +import com.tencent.bk.job.file_gateway.service.dispatch.DispatchService; import com.tencent.bk.job.file_gateway.service.remote.FileSourceReqGenService; import lombok.extern.slf4j.Slf4j; +import org.slf4j.helpers.MessageFormatter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -66,21 +67,21 @@ public FileServiceImpl(FileSourceService fileSourceService, this.jobHttpClient = jobHttpClient; } - private FileWorkerDTO getFileWorker(FileSourceDTO fileSourceDTO) { + private FileWorkerDTO getFileWorker(FileSourceDTO fileSourceDTO, String requestSource) { if (fileSourceDTO == null) { throw new InternalException(ErrorCode.FILE_SOURCE_NOT_EXIST); } - return dispatchService.findBestFileWorker(fileSourceDTO); + return dispatchService.findBestFileWorker(fileSourceDTO, requestSource); } @Override public boolean isFileAvailable(String username, Long appId, Integer fileSourceId) { FileSourceDTO fileSourceDTO = fileSourceService.getFileSourceById(appId, fileSourceId); - FileWorkerDTO fileWorkerDTO = getFileWorker(fileSourceDTO); + FileWorkerDTO fileWorkerDTO = getFileWorker(fileSourceDTO, "isFileAvailable"); if (fileWorkerDTO == null) { throw new InternalException(ErrorCode.CAN_NOT_FIND_AVAILABLE_FILE_WORKER); } - log.info("choose file worker:" + fileWorkerDTO); + log.info("choose file worker:" + fileWorkerDTO.getBasicDesc()); // 访问文件Worker接口，拿到available状态信息 HttpReq req = fileSourceReqGenService.genFileAvailableReq(appId, fileWorkerDTO, fileSourceDTO); String respStr; @@ -102,11 +103,11 @@ public FileNodesVO listFileNode(String username, Long appId, Integer fileSourceI if (name == null) name = ""; final String finalName = name; FileSourceDTO fileSourceDTO = fileSourceService.getFileSourceById(appId, fileSourceId); - FileWorkerDTO fileWorkerDTO = getFileWorker(fileSourceDTO); + FileWorkerDTO fileWorkerDTO = getFileWorker(fileSourceDTO, "listFileNode"); if (fileWorkerDTO == null) { throw new InternalException(ErrorCode.CAN_NOT_FIND_AVAILABLE_FILE_WORKER); } - log.info("choose file worker:" + fileWorkerDTO); + log.info("choose file worker:" + fileWorkerDTO.getBasicDesc()); // 访问文件Worker接口，拿到FileNode信息 HttpReq req = fileSourceReqGenService.genListFileNodeReq(appId, path, finalName, start, pageSize, fileWorkerDTO, fileSourceDTO); @@ -128,11 +129,14 @@ public FileNodesVO listFileNode(String username, Long appId, Integer fileSourceI @Override public Boolean executeAction(String username, Long appId, Integer fileSourceId, ExecuteActionReq executeActionReq) { FileSourceDTO fileSourceDTO = fileSourceService.getFileSourceById(appId, fileSourceId); - FileWorkerDTO fileWorkerDTO = getFileWorker(fileSourceDTO); + FileWorkerDTO fileWorkerDTO = getFileWorker( + fileSourceDTO, + "executeAction" + executeActionReq.getActionCode() + ); if (fileWorkerDTO == null) { throw new InternalException(ErrorCode.CAN_NOT_FIND_AVAILABLE_FILE_WORKER); } - log.info("choose file worker:" + fileWorkerDTO); + log.info("choose file worker:" + fileWorkerDTO.getBasicDesc()); HttpReq req = fileSourceReqGenService.genExecuteActionReq(appId, executeActionReq.getActionCode(), executeActionReq.getParams(), fileWorkerDTO, fileSourceDTO); String respStr; @@ -162,7 +166,11 @@ private FileNodesDTO parseFileNodesDTO(String respStr) { resp = JsonUtils.fromJson(respStr, new TypeReference>() { }); } catch (Exception e) { - log.error("Fail to parse bucket from response={}", respStr, e); + String msg = MessageFormatter.format( + "Fail to parse bucket from response={}", + respStr + ).getMessage(); + log.error(msg, e); throw new InternalException(e.getMessage(), ErrorCode.FAIL_TO_REQUEST_FILE_WORKER_LIST_FILE_NODE); } if (resp.isSuccess()) { diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/FileSourceServiceImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/FileSourceServiceImpl.java index 1ac428c10f..8eea761f2b 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/FileSourceServiceImpl.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/FileSourceServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,9 +25,21 @@ package com.tencent.bk.job.file_gateway.service.impl; import com.fasterxml.jackson.core.type.TypeReference; +import com.tencent.bk.audit.annotations.ActionAuditRecord; +import com.tencent.bk.audit.annotations.AuditInstanceRecord; +import com.tencent.bk.audit.context.ActionAuditContext; +import com.tencent.bk.job.common.audit.JobAuditAttributeNames; +import com.tencent.bk.job.common.audit.constants.EventContentConstants; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.exception.AlreadyExistsException; +import com.tencent.bk.job.common.exception.FailedPreconditionException; +import com.tencent.bk.job.common.exception.NotFoundException; +import com.tencent.bk.job.common.iam.constant.ActionId; +import com.tencent.bk.job.common.iam.constant.ResourceTypeId; +import com.tencent.bk.job.common.model.dto.AppResourceScope; +import com.tencent.bk.job.common.mysql.JobTransactional; import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.file_gateway.auth.FileSourceAuthService; import com.tencent.bk.job.file_gateway.dao.filesource.FileSourceDAO; import com.tencent.bk.job.file_gateway.dao.filesource.FileSourceTypeDAO; import com.tencent.bk.job.file_gateway.dao.filesource.FileWorkerDAO; @@ -39,8 +51,8 @@ import com.tencent.bk.job.file_gateway.model.req.common.FileSourceStaticParam; import com.tencent.bk.job.file_gateway.model.req.common.FileWorkerConfig; import com.tencent.bk.job.file_gateway.service.FileSourceService; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -import org.jooq.DSLContext; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -51,26 +63,28 @@ import java.util.Set; @Service +@Slf4j public class FileSourceServiceImpl implements FileSourceService { - private DSLContext dslContext; - private FileSourceTypeDAO fileSourceTypeDAO; - private FileSourceDAO fileSourceDAO; - private FileWorkerDAO fileWorkerDAO; + private final FileSourceTypeDAO fileSourceTypeDAO; + private final FileSourceDAO fileSourceDAO; + private final FileWorkerDAO fileWorkerDAO; + private final FileSourceAuthService fileSourceAuthService; @Autowired - public FileSourceServiceImpl(DSLContext dslContext, FileSourceTypeDAO fileSourceTypeDAO, - FileSourceDAO fileSourceDAO, FileWorkerDAO fileWorkerDAO) { - this.dslContext = dslContext; + public FileSourceServiceImpl(FileSourceTypeDAO fileSourceTypeDAO, + FileSourceDAO fileSourceDAO, + FileWorkerDAO fileWorkerDAO, + FileSourceAuthService fileSourceAuthService) { this.fileSourceTypeDAO = fileSourceTypeDAO; this.fileSourceDAO = fileSourceDAO; this.fileWorkerDAO = fileWorkerDAO; + this.fileSourceAuthService = fileSourceAuthService; } @Override public List listUniqueFileSourceType(String storageType) { - List fileSourceTypeDTOList = fileSourceTypeDAO.listOrderByVersion(dslContext - , storageType); + List fileSourceTypeDTOList = fileSourceTypeDAO.listEnabledTypeOrderByVersion(storageType); Set codeSet = new HashSet<>(); List resultList = new ArrayList<>(); // 多个Worker提供的同一种文件源能力只取版本最高的任意一个作为标准 @@ -85,54 +99,131 @@ public List listUniqueFileSourceType(String storageType) { @Override public Integer countAvailableFileSource(Long appId, String credentialId, String alias) { - return fileSourceDAO.countAvailableLikeFileSource(dslContext, appId, credentialId, alias); + return fileSourceDAO.countAvailableLikeFileSource(appId, credentialId, alias); } @Override public Integer countWorkTableFileSource(Long appId, String credentialId, String alias) { - return fileSourceDAO.countWorkTableFileSource(dslContext, appId, credentialId, alias); + return fileSourceDAO.countWorkTableFileSource(appId, credentialId, alias); } @Override public Integer countWorkTableFileSource(List appIdList, List idList) { - return fileSourceDAO.countWorkTableFileSource(dslContext, appIdList, idList); + return fileSourceDAO.countWorkTableFileSource(appIdList, idList); } @Override public List listAvailableFileSource(Long appId, String credentialId, String alias, Integer start, Integer pageSize) { - return fileSourceDAO.listAvailableFileSource(dslContext, appId, credentialId, alias, start, pageSize); + return fileSourceDAO.listAvailableFileSource(appId, credentialId, alias, start, pageSize); } @Override public List listWorkTableFileSource(Long appId, String credentialId, String alias, Integer start, Integer pageSize) { - return fileSourceDAO.listWorkTableFileSource(dslContext, appId, credentialId, alias, start, pageSize); + return fileSourceDAO.listWorkTableFileSource(appId, credentialId, alias, start, pageSize); } @Override public List listWorkTableFileSource(List appIdList, List idList, Integer start, Integer pageSize) { - return fileSourceDAO.listWorkTableFileSource(dslContext, appIdList, idList, start, pageSize); + return fileSourceDAO.listWorkTableFileSource(appIdList, idList, start, pageSize); } @Override - public Integer saveFileSource(Long appId, FileSourceDTO fileSourceDTO) { - if (fileSourceDAO.checkFileSourceExists(dslContext, fileSourceDTO.getAppId(), fileSourceDTO.getAlias())) { + @ActionAuditRecord( + actionId = ActionId.CREATE_FILE_SOURCE, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.FILE_SOURCE, + instanceIds = "#$?.id", + instanceNames = "#fileSource?.alias" + ), + content = EventContentConstants.CREATE_FILE_SOURCE + ) + @JobTransactional( + transactionManager = "jobFileGatewayTransactionManager", + rollbackFor = {Exception.class, Error.class} + ) + public FileSourceDTO saveFileSource(String username, Long appId, FileSourceDTO fileSource) { + authCreate(username, appId); + + if (existsCode(appId, fileSource.getCode())) { + throw new FailedPreconditionException( + ErrorCode.FILE_SOURCE_CODE_ALREADY_EXISTS, + new String[]{fileSource.getCode()} + ); + } + + if (fileSourceDAO.checkFileSourceExists(fileSource.getAppId(), fileSource.getAlias())) { throw new AlreadyExistsException(ErrorCode.FILE_SOURCE_ALIAS_ALREADY_EXISTS, - new String[]{fileSourceDTO.getAlias()}); + new String[]{fileSource.getAlias()}); } - return fileSourceDAO.insertFileSource(dslContext, fileSourceDTO); + + Integer id = fileSourceDAO.insertFileSource(fileSource); + fileSource.setId(id); + + boolean registerResult = fileSourceAuthService.registerFileSource( + username, fileSource.getId(), fileSource.getAlias()); + if (!registerResult) { + log.warn("Fail to register file_source to iam:({},{})", fileSource.getId(), fileSource.getAlias()); + } + return getFileSourceById(id); + } + + private void authView(String username, long appId, int fileSourceId) { + fileSourceAuthService.authViewFileSource(username, new AppResourceScope(appId), fileSourceId, null) + .denyIfNoPermission(); + } + + private void authCreate(String username, long appId) { + fileSourceAuthService.authCreateFileSource(username, new AppResourceScope(appId)).denyIfNoPermission(); + } + + private void authManage(String username, long appId, int fileSourceId) { + fileSourceAuthService.authManageFileSource(username, new AppResourceScope(appId), + fileSourceId, null).denyIfNoPermission(); } @Override - public int updateFileSourceById(Long appId, FileSourceDTO fileSourceDTO) { - return fileSourceDAO.updateFileSource(dslContext, fileSourceDTO); + @ActionAuditRecord( + actionId = ActionId.MANAGE_FILE_SOURCE, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.FILE_SOURCE, + instanceIds = "#fileSource?.id", + instanceNames = "$?.alias" + ), + content = EventContentConstants.EDIT_FILE_SOURCE + ) + public FileSourceDTO updateFileSourceById(String username, Long appId, FileSourceDTO fileSource) { + authManage(username, appId, fileSource.getId()); + + if (existsCodeExceptId(appId, fileSource.getCode(), fileSource.getId())) { + throw new FailedPreconditionException( + ErrorCode.FILE_SOURCE_CODE_ALREADY_EXISTS, + new String[]{fileSource.getCode()} + ); + } + + FileSourceDTO originFileSource = getFileSourceById(fileSource.getId()); + if (originFileSource == null) { + throw new NotFoundException(ErrorCode.FILE_SOURCE_NOT_EXIST); + } + + fileSourceDAO.updateFileSource(fileSource); + + FileSourceDTO updateFileSource = getFileSourceById(fileSource.getId()); + + // 审计 + ActionAuditContext.current() + .setOriginInstance(FileSourceDTO.toEsbFileSourceV3DTO(fileSource)) + .setInstance(FileSourceDTO.toEsbFileSourceV3DTO(updateFileSource)); + + return updateFileSource; } @Override public int updateFileSourceStatus(Integer fileSourceId, Integer status) { - return fileSourceDAO.updateFileSourceStatus(dslContext, fileSourceId, status); + return fileSourceDAO.updateFileSourceStatus(fileSourceId, status); } @Override @@ -146,33 +237,88 @@ public FileSourceTypeDTO getFileSourceTypeByCode(String code) { } @Override - public Integer deleteFileSourceById(Long appId, Integer id) { - return fileSourceDAO.deleteFileSourceById(dslContext, id); + @ActionAuditRecord( + actionId = ActionId.MANAGE_FILE_SOURCE, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.FILE_SOURCE, + instanceIds = "#id" + ), + content = EventContentConstants.DELETE_FILE_SOURCE + ) + public Integer deleteFileSourceById(String username, Long appId, Integer id) { + authManage(username, appId, id); + + FileSourceDTO fileSource = getFileSourceById(id); + if (fileSource == null) { + throw new NotFoundException(ErrorCode.FILE_SOURCE_NOT_EXIST); + } + ActionAuditContext.current().setInstanceName(fileSource.getAlias()); + + return fileSourceDAO.deleteFileSourceById(id); } @Override + @ActionAuditRecord( + actionId = ActionId.MANAGE_FILE_SOURCE, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.FILE_SOURCE, + instanceIds = "#id" + ), + content = EventContentConstants.SWITCH_FILE_SOURCE_STATUS + ) public Boolean enableFileSourceById(String username, Long appId, Integer id, Boolean enableFlag) { - return fileSourceDAO.enableFileSourceById(dslContext, username, appId, id, enableFlag) == 1; + authManage(username, appId, id); + FileSourceDTO fileSource = getFileSourceById(id); + if (fileSource == null) { + throw new NotFoundException(ErrorCode.FILE_SOURCE_NOT_EXIST); + } + + // 审计 + ActionAuditContext.current() + .setInstanceName(fileSource.getAlias()) + .addAttribute(JobAuditAttributeNames.OPERATION, enableFlag ? "Switch on" : "Switch off"); + + return fileSourceDAO.enableFileSourceById(username, appId, id, enableFlag) == 1; + } + + @Override + @ActionAuditRecord( + actionId = ActionId.VIEW_FILE_SOURCE, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.FILE_SOURCE, + instanceIds = "#id", + instanceNames = "#$?.alias" + ), + content = EventContentConstants.VIEW_FILE_SOURCE + ) + public FileSourceDTO getFileSourceById(String username, Long appId, Integer id) { + authView(username, appId, id); + return getFileSourceById(appId, id); } @Override public FileSourceDTO getFileSourceById(Long appId, Integer id) { - return fileSourceDAO.getFileSourceById(dslContext, id); + return fileSourceDAO.getFileSourceById(id); } @Override public FileSourceDTO getFileSourceById(Integer id) { - return fileSourceDAO.getFileSourceById(dslContext, id); + return fileSourceDAO.getFileSourceById(id); } @Override public List listFileSourceByIds(Collection ids) { - return fileSourceDAO.listFileSourceByIds(dslContext, ids); + return fileSourceDAO.listFileSourceByIds(ids); } @Override public FileSourceDTO getFileSourceByCode(String code) { - return fileSourceDAO.getFileSourceByCode(dslContext, code); + return fileSourceDAO.getFileSourceByCode(code); + } + + @Override + public FileSourceDTO getFileSourceByCode(Long appId, String code) { + return fileSourceDAO.getFileSourceByCode(appId, code); } private Long chooseAvailableWorker(String fileSourceTypeCode) { @@ -203,7 +349,7 @@ public List getFileSourceParams(Long appId, String fileSo @Override public Boolean checkFileSourceAlias(Long appId, String alias, Integer fileSourceId) { - int count = fileSourceDAO.countFileSource(dslContext, appId, null, alias); + int count = fileSourceDAO.countFileSource(appId, null, alias); if (count == 0) { return true; } else { @@ -231,6 +377,11 @@ public boolean existsFileSource(Long appId, Integer id) { return fileSourceDAO.existsFileSource(appId, id); } + @Override + public boolean existsFileSourceUsingCredential(Long appId, String credentialId) { + return fileSourceDAO.existsFileSourceUsingCredential(appId, credentialId); + } + @Override public Integer getFileSourceIdByCode(Long appId, String code) { return fileSourceDAO.getFileSourceIdByCode(appId, code); diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/FileSourceTaskServiceImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/FileSourceTaskServiceImpl.java index 25d17f4255..4f4b8d3b23 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/FileSourceTaskServiceImpl.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/FileSourceTaskServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,16 +27,11 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.exception.InternalException; -import com.tencent.bk.job.common.exception.NotFoundException; import com.tencent.bk.job.common.exception.ServiceException; import com.tencent.bk.job.common.model.Response; import com.tencent.bk.job.common.model.http.HttpReq; -import com.tencent.bk.job.common.util.ArrayUtil; -import com.tencent.bk.job.common.util.file.FileSizeUtil; -import com.tencent.bk.job.common.util.file.PathUtil; import com.tencent.bk.job.common.util.http.JobHttpClient; import com.tencent.bk.job.common.util.json.JsonUtils; -import com.tencent.bk.job.execute.common.constants.FileDistStatusEnum; import com.tencent.bk.job.file_gateway.consts.TaskCommandEnum; import com.tencent.bk.job.file_gateway.consts.TaskStatusEnum; import com.tencent.bk.job.file_gateway.dao.filesource.FileSourceDAO; @@ -46,26 +41,24 @@ import com.tencent.bk.job.file_gateway.model.dto.FileSourceDTO; import com.tencent.bk.job.file_gateway.model.dto.FileSourceTaskDTO; import com.tencent.bk.job.file_gateway.model.dto.FileTaskDTO; +import com.tencent.bk.job.file_gateway.model.dto.FileTaskProgressDTO; import com.tencent.bk.job.file_gateway.model.dto.FileWorkerDTO; -import com.tencent.bk.job.file_gateway.model.resp.inner.FileLogPieceDTO; import com.tencent.bk.job.file_gateway.model.resp.inner.FileSourceTaskStatusDTO; import com.tencent.bk.job.file_gateway.model.resp.inner.TaskInfoDTO; import com.tencent.bk.job.file_gateway.model.resp.inner.ThirdFileSourceTaskLogDTO; -import com.tencent.bk.job.file_gateway.service.DispatchService; import com.tencent.bk.job.file_gateway.service.FileSourceTaskService; -import com.tencent.bk.job.file_gateway.service.context.TaskContext; -import com.tencent.bk.job.file_gateway.service.context.impl.DefaultTaskContext; -import com.tencent.bk.job.file_gateway.service.listener.FileTaskStatusChangeListener; +import com.tencent.bk.job.file_gateway.service.FileSourceTaskUpdateService; +import com.tencent.bk.job.file_gateway.service.dispatch.DispatchService; import com.tencent.bk.job.file_gateway.service.remote.FileSourceTaskReqGenService; import lombok.extern.slf4j.Slf4j; -import org.jooq.DSLContext; +import org.slf4j.helpers.MessageFormatter; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.stereotype.Service; import java.util.ArrayList; import java.util.Collections; -import java.util.Date; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -77,26 +70,27 @@ public class FileSourceTaskServiceImpl implements FileSourceTaskService { public static final String PREFIX_REDIS_TASK_LOG = "job:file-gateway:taskLog:"; - private final DSLContext dslContext; + private final FileSourceTaskUpdateService fileSourceTaskUpdateService; private final FileSourceTaskDAO fileSourceTaskDAO; private final FileTaskDAO fileTaskDAO; private final FileWorkerDAO fileworkerDAO; private final FileSourceDAO fileSourceDAO; private final DispatchService dispatchService; private final FileSourceTaskReqGenService fileSourceTaskReqGenService; - private final RedisTemplate redisTemplate; + private final RedisTemplate redisTemplate; private final JobHttpClient jobHttpClient; - private final List fileTaskStatusChangeListenerList = new ArrayList<>(); @Autowired - public FileSourceTaskServiceImpl(DSLContext dslContext, FileSourceTaskDAO fileSourceTaskDAO, - FileTaskDAO fileTaskDAO, FileWorkerDAO fileworkerDAO, - FileSourceDAO fileSourceDAO, DispatchService dispatchService, + public FileSourceTaskServiceImpl(FileSourceTaskUpdateService fileSourceTaskUpdateService, + FileSourceTaskDAO fileSourceTaskDAO, + FileTaskDAO fileTaskDAO, + FileWorkerDAO fileworkerDAO, + FileSourceDAO fileSourceDAO, + DispatchService dispatchService, FileSourceTaskReqGenService fileSourceTaskReqGenService, - RedisTemplate redisTemplate, - FileTaskStatusChangeListener fileTaskStatusChangeListener, + @Qualifier("jsonRedisTemplate") RedisTemplate redisTemplate, JobHttpClient jobHttpClient) { - this.dslContext = dslContext; + this.fileSourceTaskUpdateService = fileSourceTaskUpdateService; this.fileSourceTaskDAO = fileSourceTaskDAO; this.fileTaskDAO = fileTaskDAO; this.fileworkerDAO = fileworkerDAO; @@ -105,37 +99,111 @@ public FileSourceTaskServiceImpl(DSLContext dslContext, FileSourceTaskDAO fileSo this.fileSourceTaskReqGenService = fileSourceTaskReqGenService; this.redisTemplate = redisTemplate; this.jobHttpClient = jobHttpClient; - addFileTaskStatusChangeListener(fileTaskStatusChangeListener); - } - - public void addFileTaskStatusChangeListener(FileTaskStatusChangeListener listener) { - this.fileTaskStatusChangeListenerList.add(listener); } @Override - public TaskInfoDTO startFileSourceDownloadTask(String username, Long appId, Long stepInstanceId, - Integer executeCount, String batchTaskId, Integer fileSourceId, + public TaskInfoDTO startFileSourceDownloadTask(String username, + Long appId, + Long stepInstanceId, + Integer executeCount, + String batchTaskId, + Integer fileSourceId, List filePathList) { - return startFileSourceDownloadTaskWithId(username, appId, stepInstanceId, executeCount, batchTaskId, - fileSourceId, filePathList, null); + return startFileSourceDownloadTaskWithId( + username, + appId, + stepInstanceId, + executeCount, + batchTaskId, + fileSourceId, + filePathList, + null + ); } - public TaskInfoDTO startFileSourceDownloadTaskWithId(String username, Long appId, Long stepInstanceId, - Integer executeCount, String batchTaskId, - Integer fileSourceId, List filePathList, + public TaskInfoDTO startFileSourceDownloadTaskWithId(String username, + Long appId, + Long stepInstanceId, + Integer executeCount, + String batchTaskId, + Integer fileSourceId, + List filePathList, String fileSourceTaskId) { - log.info("Input=({},{},{},{},{},{},{})", username, appId, stepInstanceId, executeCount, batchTaskId, - fileSourceId, filePathList); - FileSourceDTO fileSourceDTO = fileSourceDAO.getFileSourceById(dslContext, fileSourceId); + log.info( + "startFileSourceDownloadTaskWithId, input=({},{},{},{},{},{},{})", + username, + appId, + stepInstanceId, + executeCount, + batchTaskId, + fileSourceId, + filePathList + ); + FileSourceDTO fileSourceDTO = fileSourceDAO.getFileSourceById(fileSourceId); if (fileSourceDTO == null) { throw new RuntimeException("FileSource not exist, fileSourceId=" + fileSourceId.toString()); } - FileWorkerDTO fileWorkerDTO = dispatchService.findBestFileWorker(fileSourceDTO); + FileWorkerDTO fileWorkerDTO = dispatchService.findBestFileWorker( + fileSourceDTO, + "DownloadTask(appId=" + appId + ")" + ); if (fileWorkerDTO == null) { throw new RuntimeException(String.format("Cannot match fileWorker for FileSourceTask,appId=%d," + "stepInstanceId=%d,fileSourceId=%d,filePathList=%s", appId, stepInstanceId, fileSourceId, filePathList.toString())); } + FileSourceTaskDTO fileSourceTaskDTO = saveFileSourceTask( + username, + appId, + stepInstanceId, + executeCount, + batchTaskId, + fileSourceId, + filePathList, + fileSourceTaskId, + fileWorkerDTO.getId() + ); + fileSourceTaskId = fileSourceTaskDTO.getId(); + try { + // 分发文件任务 + HttpReq req = fileSourceTaskReqGenService.genDownloadFilesReq(appId, fileWorkerDTO, fileSourceDTO, + fileSourceTaskDTO); + postHttpReq(req); + } catch (Exception e) { + String msg = MessageFormatter.format( + "Fail to dispatch FileSourceTask={}", + JsonUtils.toJson(fileSourceTaskDTO) + ).getMessage(); + log.error(msg, e); + // 清理DB中的任务数据便于外层重试 + clearSavedFileSourceTask(fileSourceTaskId); + throw new InternalException( + e, + ErrorCode.FAIL_TO_REQUEST_FILE_WORKER_START_FILE_SOURCE_DOWNLOAD_TASK, + new String[]{e.getMessage()} + ); + } + return new TaskInfoDTO( + fileSourceTaskId, + fileSourceDTO.getAlias(), + fileSourceDTO.getPublicFlag(), + fileWorkerDTO.getId(), + fileWorkerDTO.getAccessHost(), + fileWorkerDTO.getCloudAreaId(), + fileWorkerDTO.getInnerIpProtocol(), + fileWorkerDTO.getInnerIp() + ); + } + + private FileSourceTaskDTO saveFileSourceTask(String username, + Long appId, + Long stepInstanceId, + Integer executeCount, + String batchTaskId, + Integer fileSourceId, + List filePathList, + String fileSourceTaskId, + Long fileWorkerId) { FileSourceTaskDTO fileSourceTaskDTO = new FileSourceTaskDTO(); fileSourceTaskDTO.setId(fileSourceTaskId); fileSourceTaskDTO.setBatchTaskId(batchTaskId); @@ -145,7 +213,7 @@ public TaskInfoDTO startFileSourceDownloadTaskWithId(String username, Long appId fileSourceTaskDTO.setStepInstanceId(stepInstanceId); fileSourceTaskDTO.setExecuteCount(executeCount); fileSourceTaskDTO.setFileSourceId(fileSourceId); - fileSourceTaskDTO.setFileWorkerId(fileWorkerDTO.getId()); + fileSourceTaskDTO.setFileWorkerId(fileWorkerId); fileSourceTaskDTO.setStatus(TaskStatusEnum.INIT.getStatus()); List fileTaskDTOList = new ArrayList<>(); for (String filePath : filePathList) { @@ -161,141 +229,58 @@ public TaskInfoDTO startFileSourceDownloadTaskWithId(String username, Long appId fileTaskDTOList.add(fileTaskDTO); } fileSourceTaskDTO.setFileTaskList(fileTaskDTOList); - try { - fileSourceTaskId = fileSourceTaskDAO.insertFileSourceTask(dslContext, fileSourceTaskDTO); - fileSourceTaskDTO.setId(fileSourceTaskId); - // 分发文件任务 - HttpReq req = fileSourceTaskReqGenService.genDownloadFilesReq(appId, fileWorkerDTO, fileSourceDTO, - fileSourceTaskDTO); - postHttpReq(req); - } catch (Exception e) { - log.error("Fail to dispatch FileSourceTask={}", JsonUtils.toJson(fileSourceTaskDTO), e); - // 更新任务状态为启动失败 - fileSourceTaskDTO.setStatus(TaskStatusEnum.DISPATCH_FAILED.getStatus()); - int affectedCount = fileSourceTaskDAO.updateFileSourceTask(dslContext, fileSourceTaskDTO); - if (affectedCount != 1) { - log.error("Fail to update status of FileSourceTask={}", JsonUtils.toJson(fileSourceTaskDTO)); - } - throw new InternalException(e, ErrorCode.FAIL_TO_REQUEST_FILE_WORKER_START_FILE_SOURCE_DOWNLOAD_TASK, - new String[]{e.getMessage()}); - } - return new TaskInfoDTO(fileSourceTaskId, fileSourceDTO.getAlias(), fileSourceDTO.getPublicFlag(), - fileWorkerDTO.getCloudAreaId(), fileWorkerDTO.getInnerIp()); - } - - private void writeLog(FileSourceTaskDTO fileSourceTaskDTO, - FileWorkerDTO fileWorkerDTO, - String filePath, - Long fileSize, - String speed, - Integer progress, - String content) { - String taskId = fileSourceTaskDTO.getId(); - String fileSizeStr = FileSizeUtil.getFileSizeStr(fileSize); - ThirdFileSourceTaskLogDTO thirdFileSourceTaskLog = new ThirdFileSourceTaskLogDTO(); - String sourceIp = fileWorkerDTO.getCloudAreaId().toString() + ":" + fileWorkerDTO.getInnerIp(); - thirdFileSourceTaskLog.setIp(sourceIp); - // 追加文件源名称 - // 日志定位坐标：（文件源，文件路径），需要区分不同文件源下相同文件路径的日志 - FileSourceDTO fileSourceDTO = fileSourceDAO.getFileSourceById(dslContext, fileSourceTaskDTO.getFileSourceId()); - if (fileSourceDTO == null) { - throw new NotFoundException(ErrorCode.FILE_SOURCE_NOT_EXIST, - ArrayUtil.toArray("fileSourceId:" + fileSourceTaskDTO.getFileSourceId())); - } - String filePathWithSourceAlias = PathUtil.joinFilePath(fileSourceDTO.getAlias(), filePath); - List fileLogPieceList = new ArrayList<>(); - FileLogPieceDTO fileLogPiece = new FileLogPieceDTO(); - fileLogPiece.setContent("FileName: " + filePathWithSourceAlias + " FileSize: " + fileSizeStr + " " + - "Speed: " + speed + " Progress: " + progress + "%" + " Detail: " + content); - fileLogPiece.setDisplaySrcFile(filePathWithSourceAlias); - fileLogPiece.setProcess("" + progress + "%"); - fileLogPiece.setSize(fileSizeStr); - fileLogPiece.setSrcIp(sourceIp); - fileLogPiece.setStatus(FileDistStatusEnum.PULLING.getValue()); - fileLogPiece.setStatusDesc(FileDistStatusEnum.PULLING.getName()); - fileLogPieceList.add(fileLogPiece); - thirdFileSourceTaskLog.setFileTaskLogs(fileLogPieceList); - // 写入Redis - redisTemplate.opsForList().rightPush(PREFIX_REDIS_TASK_LOG + taskId, thirdFileSourceTaskLog); - // 一小时后过期 - redisTemplate.expireAt(PREFIX_REDIS_TASK_LOG + taskId, new Date(System.currentTimeMillis() + 3600 * 1000)); + fileSourceTaskId = fileSourceTaskDAO.insertFileSourceTask(fileSourceTaskDTO); + fileSourceTaskDTO.setId(fileSourceTaskId); + return fileSourceTaskDTO; } - private void notifyFileTaskStatusChangeListeners(FileTaskDTO fileTaskDTO, FileSourceTaskDTO fileSourceTaskDTO, - FileWorkerDTO fileWorkerDTO, TaskStatusEnum previousStatus, - TaskStatusEnum currentStatus) { - TaskContext context = new DefaultTaskContext(fileTaskDTO, fileSourceTaskDTO, fileWorkerDTO); - if (!fileTaskStatusChangeListenerList.isEmpty()) { - boolean stop; - for (FileTaskStatusChangeListener listener : fileTaskStatusChangeListenerList) { - stop = listener.onStatusChange(context, previousStatus, currentStatus); - if (stop) break; - } - } + private void clearSavedFileSourceTask(String fileSourceTaskId) { + // 1.删除子任务 + int deletedTaskNum = fileTaskDAO.deleteFileTaskByFileSourceTaskId(fileSourceTaskId); + // 2.删除FileSourceTask任务 + int deletedFileSourceTaskNum = deleteFileSourceTaskById(fileSourceTaskId); + log.info( + "{} fileTask {} fileSourceTask deleted, fileSourceTaskId={}", + deletedTaskNum, + deletedFileSourceTaskNum, + fileSourceTaskId + ); } @Override - public String updateFileSourceTask(String taskId, String filePath, String downloadPath, Long fileSize, - String speed, Integer progress, String content, TaskStatusEnum status) { - FileTaskDTO fileTaskDTO = fileTaskDAO.getOneFileTask(dslContext, taskId, filePath); + public String updateFileSourceTask(FileTaskProgressDTO fileTaskProgressDTO) { + String fileSourceTaskId = fileTaskProgressDTO.getFileSourceTaskId(); + String filePath = fileTaskProgressDTO.getFilePath(); + FileTaskDTO fileTaskDTO = fileTaskDAO.getOneFileTask( + fileSourceTaskId, + filePath + ); if (fileTaskDTO == null) { - log.error("Cannot find fileTaskDTO by taskId {} filePath {}", taskId, filePath); + log.error( + "Cannot find fileTaskDTO by taskId {} filePath {}", + fileTaskProgressDTO.getFileSourceTaskId(), + fileTaskProgressDTO.getFilePath() + ); return null; } - TaskStatusEnum previousStatus = TaskStatusEnum.valueOf(fileTaskDTO.getStatus()); - fileTaskDTO.setDownloadPath(downloadPath); - FileSourceTaskDTO fileSourceTaskDTO = fileSourceTaskDAO.getFileSourceTaskById(dslContext, taskId); + FileSourceTaskDTO fileSourceTaskDTO = fileSourceTaskDAO.getFileSourceTaskById(fileSourceTaskId); if (fileSourceTaskDTO == null) { - log.error("Cannot find fileSourceTaskDTO by taskId {} filePath {}", taskId, filePath); + log.error("Cannot find fileSourceTaskDTO by taskId {} filePath {}", fileSourceTaskId, filePath); return null; } - FileWorkerDTO fileWorkerDTO = fileworkerDAO.getFileWorkerById(fileSourceTaskDTO.getFileWorkerId()); - if (status == TaskStatusEnum.RUNNING) { - // 已处于结束态的任务不再接受状态更新 - if (!fileTaskDTO.isDone()) { - fileTaskDTO.setProgress(progress); - fileTaskDTO.setFileSize(fileSize); - fileTaskDTO.setStatus(TaskStatusEnum.RUNNING.getStatus()); - fileTaskDAO.updateFileTask(dslContext, fileTaskDTO); - logUpdatedTaskStatus(taskId, filePath, progress, status); - } else { - log.info("fileTask {} already done, do not update to running", taskId); - } - } else if (status == TaskStatusEnum.SUCCESS) { - fileTaskDTO.setProgress(100); - fileTaskDTO.setStatus(TaskStatusEnum.SUCCESS.getStatus()); - fileTaskDAO.updateFileTask(dslContext, fileTaskDTO); - logUpdatedTaskStatus(taskId, filePath, progress, status); - } else if (status == TaskStatusEnum.FAILED) { - fileTaskDTO.setProgress(progress); - fileTaskDTO.setStatus(TaskStatusEnum.FAILED.getStatus()); - fileTaskDAO.updateFileTask(dslContext, fileTaskDTO); - logUpdatedTaskStatus(taskId, filePath, progress, status); - } else if (status == TaskStatusEnum.STOPPED) { - fileTaskDTO.setProgress(progress); - fileTaskDTO.setStatus(TaskStatusEnum.STOPPED.getStatus()); - fileTaskDAO.updateFileTask(dslContext, fileTaskDTO); - logUpdatedTaskStatus(taskId, filePath, progress, status); - } else { - log.warn("fileTask {} unknown status:{}", taskId, status); - } - // 通知关注者 - if (status != previousStatus) { - notifyFileTaskStatusChangeListeners(fileTaskDTO, fileSourceTaskDTO, fileWorkerDTO, - TaskStatusEnum.valueOf(fileTaskDTO.getStatus()), status); - } - // 进度上报 - writeLog(fileSourceTaskDTO, fileWorkerDTO, filePath, fileSize, speed, progress, content); - return taskId; - } - - private void logUpdatedTaskStatus(String taskId, String filePath, Integer progress, TaskStatusEnum status) { - log.info("updated fileTask:{},{},{},{}", taskId, filePath, progress, status.name()); + String batchTaskId = fileSourceTaskDTO.getBatchTaskId(); + Long fileTaskId = fileTaskDTO.getId(); + return fileSourceTaskUpdateService.updateFileSourceTask( + batchTaskId, + fileSourceTaskId, + fileTaskId, + fileTaskProgressDTO + ); } @Override public FileSourceTaskStatusDTO getFileSourceTaskStatusAndLogs(String taskId, Long logStart, Long logLength) { - FileSourceTaskDTO fileSourceTaskDTO = fileSourceTaskDAO.getFileSourceTaskById(dslContext, taskId); + FileSourceTaskDTO fileSourceTaskDTO = fileSourceTaskDAO.getFileSourceTaskById(taskId); if (fileSourceTaskDTO == null) { throw new RuntimeException("FileSourceTask not exist, fileTaskId=" + taskId); } @@ -315,6 +300,7 @@ public FileSourceTaskStatusDTO getFileSourceTaskStatusAndLogs(String taskId, Lon }); fileSourceTaskStatusDTO.setMessage(messageBuilder.toString()); fileSourceTaskStatusDTO.setCloudId(fileWorkerDTO.getCloudAreaId()); + fileSourceTaskStatusDTO.setIpProtocol(fileWorkerDTO.getInnerIpProtocol()); fileSourceTaskStatusDTO.setIp(fileWorkerDTO.getInnerIp()); fileSourceTaskStatusDTO.setFileCleared(fileSourceTaskDTO.getFileCleared()); Map filePathMap = new HashMap<>(); @@ -339,10 +325,13 @@ public FileSourceTaskStatusDTO getFileSourceTaskStatusAndLogs(String taskId, Lon if (logLength < 0) { logEnd = logSize; } - log.debug("logStart={},logEnd={}", logStart, logEnd); + log.info("taskId={},logStart={},logEnd={},logSize={}", taskId, logStart, logEnd, logSize); List logDTOList = null; - List logObjList = redisTemplate.opsForList().range(PREFIX_REDIS_TASK_LOG + taskId, logStart, - logEnd); + List logObjList = redisTemplate.opsForList().range( + PREFIX_REDIS_TASK_LOG + taskId, + logStart, + logEnd + ); if (logObjList != null) { logDTOList = logObjList.stream().map(obj -> (ThirdFileSourceTaskLogDTO) obj).collect(Collectors.toList()); } @@ -383,7 +372,7 @@ private Integer stopTasksWithCommand(List taskIdList, TaskCommandEnum co // 将taskId按照FileWorker分组 Map> workerTaskMap = new HashMap<>(); for (String taskId : taskIdList) { - FileSourceTaskDTO fileSourceTaskDTO = fileSourceTaskDAO.getFileSourceTaskById(dslContext, taskId); + FileSourceTaskDTO fileSourceTaskDTO = fileSourceTaskDAO.getFileSourceTaskById(taskId); if (fileSourceTaskDTO == null || fileSourceTaskDTO.isDone() || fileSourceTaskDTO.getFileCleared()) { // 任务已达终止态/已被清理 continue; @@ -423,7 +412,7 @@ public Integer clearTaskFiles(List taskIdList) { // 将taskId按照FileWorker分组 Map> workerTaskMap = new HashMap<>(); for (String taskId : taskIdList) { - FileSourceTaskDTO fileSourceTaskDTO = fileSourceTaskDAO.getFileSourceTaskById(dslContext, taskId); + FileSourceTaskDTO fileSourceTaskDTO = fileSourceTaskDAO.getFileSourceTaskById(taskId); if (fileSourceTaskDTO == null || fileSourceTaskDTO.getFileCleared()) { // 任务已被清理 continue; @@ -436,7 +425,8 @@ public Integer clearTaskFiles(List taskIdList) { addTaskIdToWorkerTaskMap(workerTaskMap, fileWorkerDTO, taskId); targetTaskIdList.add(taskId); } - fileSourceTaskDAO.updateFileClearStatus(dslContext, targetTaskIdList, true); + int affectedRowNum = fileSourceTaskDAO.updateFileClearStatus(targetTaskIdList, true); + log.info("{}/{} taskFile set clear status true", affectedRowNum, targetTaskIdList.size()); int allCount = 0; // 逐个Worker清理文件 for (Map.Entry> entry : workerTaskMap.entrySet()) { @@ -461,12 +451,12 @@ public Integer clearTaskFiles(List taskIdList) { @Override public FileSourceTaskDTO getFileSourceTaskById(String id) { - return fileSourceTaskDAO.getFileSourceTaskById(dslContext, id); + return fileSourceTaskDAO.getFileSourceTaskById(id); } @Override public Integer deleteFileSourceTaskById(String id) { - return fileSourceTaskDAO.deleteById(dslContext, id); + return fileSourceTaskDAO.deleteById(id); } private Integer parseInteger(String respStr) { diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/FileSourceTaskUpdateServiceImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/FileSourceTaskUpdateServiceImpl.java new file mode 100644 index 0000000000..2045d4db10 --- /dev/null +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/FileSourceTaskUpdateServiceImpl.java @@ -0,0 +1,248 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file_gateway.service.impl; + +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.NotFoundException; +import com.tencent.bk.job.common.mysql.JobTransactional; +import com.tencent.bk.job.common.util.ArrayUtil; +import com.tencent.bk.job.common.util.file.FileSizeUtil; +import com.tencent.bk.job.common.util.file.PathUtil; +import com.tencent.bk.job.execute.common.constants.FileDistStatusEnum; +import com.tencent.bk.job.file_gateway.consts.TaskStatusEnum; +import com.tencent.bk.job.file_gateway.dao.filesource.FileSourceBatchTaskDAO; +import com.tencent.bk.job.file_gateway.dao.filesource.FileSourceDAO; +import com.tencent.bk.job.file_gateway.dao.filesource.FileSourceTaskDAO; +import com.tencent.bk.job.file_gateway.dao.filesource.FileTaskDAO; +import com.tencent.bk.job.file_gateway.dao.filesource.FileWorkerDAO; +import com.tencent.bk.job.file_gateway.model.dto.FileSourceBatchTaskDTO; +import com.tencent.bk.job.file_gateway.model.dto.FileSourceDTO; +import com.tencent.bk.job.file_gateway.model.dto.FileSourceTaskDTO; +import com.tencent.bk.job.file_gateway.model.dto.FileTaskDTO; +import com.tencent.bk.job.file_gateway.model.dto.FileTaskProgressDTO; +import com.tencent.bk.job.file_gateway.model.dto.FileWorkerDTO; +import com.tencent.bk.job.file_gateway.model.resp.inner.FileLogPieceDTO; +import com.tencent.bk.job.file_gateway.model.resp.inner.ThirdFileSourceTaskLogDTO; +import com.tencent.bk.job.file_gateway.service.FileSourceTaskUpdateService; +import com.tencent.bk.job.file_gateway.service.context.TaskContext; +import com.tencent.bk.job.file_gateway.service.context.impl.DefaultTaskContext; +import com.tencent.bk.job.file_gateway.service.listener.FileTaskStatusChangeListener; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +@Slf4j +@Service +public class FileSourceTaskUpdateServiceImpl implements FileSourceTaskUpdateService { + + public static final String PREFIX_REDIS_TASK_LOG = "job:file-gateway:taskLog:"; + + private final FileSourceBatchTaskDAO fileSourceBatchTaskDAO; + private final FileSourceTaskDAO fileSourceTaskDAO; + private final FileTaskDAO fileTaskDAO; + private final FileWorkerDAO fileworkerDAO; + private final FileSourceDAO fileSourceDAO; + private final RedisTemplate redisTemplate; + private final List fileTaskStatusChangeListenerList = new ArrayList<>(); + + @Autowired + public FileSourceTaskUpdateServiceImpl(FileSourceBatchTaskDAO fileSourceBatchTaskDAO, + FileSourceTaskDAO fileSourceTaskDAO, + FileTaskDAO fileTaskDAO, + FileWorkerDAO fileworkerDAO, + FileSourceDAO fileSourceDAO, + @Qualifier("jsonRedisTemplate") RedisTemplate redisTemplate, + FileTaskStatusChangeListener fileTaskStatusChangeListener) { + this.fileSourceBatchTaskDAO = fileSourceBatchTaskDAO; + this.fileSourceTaskDAO = fileSourceTaskDAO; + this.fileTaskDAO = fileTaskDAO; + this.fileworkerDAO = fileworkerDAO; + this.fileSourceDAO = fileSourceDAO; + this.redisTemplate = redisTemplate; + addFileTaskStatusChangeListener(fileTaskStatusChangeListener); + } + + public void addFileTaskStatusChangeListener(FileTaskStatusChangeListener listener) { + this.fileTaskStatusChangeListenerList.add(listener); + } + + @Override + @JobTransactional(transactionManager = "jobFileGatewayTransactionManager") + public String updateFileSourceTask(String batchTaskId, + String fileSourceTaskId, + Long fileTaskId, + FileTaskProgressDTO fileTaskProgressDTO) { + // 开启事务后立即加排它锁，保证读取到其他事务已提交的数据 + FileSourceBatchTaskDTO fileSourceBatchTaskDTO = + fileSourceBatchTaskDAO.getBatchTaskByIdForUpdate(batchTaskId); + // 查出加锁后的最新数据 + FileSourceTaskDTO fileSourceTaskDTO = fileSourceTaskDAO.getFileSourceTaskByIdForUpdate(fileSourceTaskId); + FileTaskDTO fileTaskDTO = fileTaskDAO.getFileTaskByIdForUpdate(fileTaskId); + if (log.isDebugEnabled()) { + log.debug("fileTaskDTO={}", fileTaskDTO); + } + fileTaskDTO.setDownloadPath(fileTaskProgressDTO.getDownloadPath()); + + String filePath = fileTaskProgressDTO.getFilePath(); + TaskStatusEnum previousStatus = TaskStatusEnum.valueOf(fileTaskDTO.getStatus()); + TaskStatusEnum status = fileTaskProgressDTO.getStatus(); + Integer progress = fileTaskProgressDTO.getProgress(); + Long fileSize = fileTaskProgressDTO.getFileSize(); + if (fileSourceBatchTaskDTO == null) { + log.error("Cannot find fileSourceBatchTaskDTO by batchTaskId {} fileSourceTaskId {} filePath {}", + fileSourceTaskDTO.getBatchTaskId(), + fileSourceTaskId, + filePath + ); + return null; + } + FileWorkerDTO fileWorkerDTO = fileworkerDAO.getFileWorkerById(fileSourceTaskDTO.getFileWorkerId()); + int affectedRowNum = -1; + if (status == TaskStatusEnum.RUNNING) { + // 已处于结束态的任务不再接受状态更新 + if (!fileTaskDTO.isDone()) { + fileTaskDTO.setProgress(progress); + fileTaskDTO.setFileSize(fileSize); + fileTaskDTO.setStatus(TaskStatusEnum.RUNNING.getStatus()); + affectedRowNum = fileTaskDAO.updateFileTask(fileTaskDTO); + logUpdatedTaskStatus(fileSourceTaskId, filePath, progress, status); + } else { + log.info("fileTask {} already done, do not update to running", fileSourceTaskId); + } + } else if (status == TaskStatusEnum.SUCCESS) { + fileTaskDTO.setProgress(100); + fileTaskDTO.setStatus(TaskStatusEnum.SUCCESS.getStatus()); + affectedRowNum = fileTaskDAO.updateFileTask(fileTaskDTO); + logUpdatedTaskStatus(fileSourceTaskId, filePath, progress, status); + } else if (status == TaskStatusEnum.FAILED) { + fileTaskDTO.setProgress(progress); + fileTaskDTO.setStatus(TaskStatusEnum.FAILED.getStatus()); + affectedRowNum = fileTaskDAO.updateFileTask(fileTaskDTO); + logUpdatedTaskStatus(fileSourceTaskId, filePath, progress, status); + } else if (status == TaskStatusEnum.STOPPED) { + fileTaskDTO.setProgress(progress); + fileTaskDTO.setStatus(TaskStatusEnum.STOPPED.getStatus()); + affectedRowNum = fileTaskDAO.updateFileTask(fileTaskDTO); + logUpdatedTaskStatus(fileSourceTaskId, filePath, progress, status); + } else { + log.warn("fileTask {} unknown status:{}", fileSourceTaskId, status); + } + if (affectedRowNum != -1) { + log.info("{} updated, affectedRowNum={}", fileTaskDTO, affectedRowNum); + } + // 通知关注者 + if (status != previousStatus) { + notifyFileTaskStatusChangeListeners(fileTaskDTO, fileSourceTaskDTO, fileWorkerDTO, previousStatus, status); + } + // 进度上报 + writeLog(fileSourceTaskDTO, fileWorkerDTO, fileTaskProgressDTO); + return fileSourceTaskId; + } + + private void notifyFileTaskStatusChangeListeners(FileTaskDTO fileTaskDTO, + FileSourceTaskDTO fileSourceTaskDTO, + FileWorkerDTO fileWorkerDTO, + TaskStatusEnum previousStatus, + TaskStatusEnum currentStatus) { + TaskContext context = new DefaultTaskContext(fileTaskDTO, fileSourceTaskDTO, fileWorkerDTO); + if (!fileTaskStatusChangeListenerList.isEmpty()) { + boolean stop; + for (FileTaskStatusChangeListener listener : fileTaskStatusChangeListenerList) { + stop = listener.onStatusChange(context, previousStatus, currentStatus); + if (stop) break; + } + } + } + + private void logUpdatedTaskStatus(String taskId, String filePath, Integer progress, TaskStatusEnum status) { + log.info("updated fileTask:{},{},{},{}", taskId, filePath, progress, status.name()); + } + + private void writeLog(FileSourceTaskDTO fileSourceTaskDTO, + FileWorkerDTO fileWorkerDTO, + FileTaskProgressDTO fileTaskProgressDTO) { + String taskId = fileSourceTaskDTO.getId(); + String fileSizeStr = FileSizeUtil.getFileSizeStr(fileTaskProgressDTO.getFileSize()); + ThirdFileSourceTaskLogDTO thirdFileSourceTaskLog = new ThirdFileSourceTaskLogDTO(); + String sourceCloudIp = fileWorkerDTO.getCloudIp(); + thirdFileSourceTaskLog.setIp(sourceCloudIp); + // 追加文件源名称 + // 日志定位坐标：（文件源，文件路径），需要区分不同文件源下相同文件路径的日志 + FileSourceDTO fileSourceDTO = fileSourceDAO.getFileSourceById(fileSourceTaskDTO.getFileSourceId()); + if (fileSourceDTO == null) { + throw new NotFoundException(ErrorCode.FILE_SOURCE_NOT_EXIST, + ArrayUtil.toArray("fileSourceId:" + fileSourceTaskDTO.getFileSourceId())); + } + String filePathWithSourceAlias = PathUtil.joinFilePath( + fileSourceDTO.getAlias(), + fileTaskProgressDTO.getFilePath() + ); + List fileLogPieceList = new ArrayList<>(); + FileLogPieceDTO fileLogPiece = new FileLogPieceDTO(); + fileLogPiece.setContent(buildFileLogContent(fileTaskProgressDTO, filePathWithSourceAlias, fileSizeStr)); + fileLogPiece.setDisplaySrcFile(filePathWithSourceAlias); + fileLogPiece.setProcess(buildProcessStr(fileTaskProgressDTO)); + fileLogPiece.setSize(fileSizeStr); + fileLogPiece.setSrcIp(sourceCloudIp); + fileLogPiece.setStatus(FileDistStatusEnum.PULLING.getValue()); + fileLogPiece.setStatusDesc(FileDistStatusEnum.PULLING.getName()); + fileLogPieceList.add(fileLogPiece); + thirdFileSourceTaskLog.setFileTaskLogs(fileLogPieceList); + // 写入Redis + redisTemplate.opsForList().rightPush(PREFIX_REDIS_TASK_LOG + taskId, thirdFileSourceTaskLog); + // 一小时后过期 + redisTemplate.expireAt(PREFIX_REDIS_TASK_LOG + taskId, new Date(System.currentTimeMillis() + 3600 * 1000)); + } + + @SuppressWarnings("StringBufferReplaceableByString") + private String buildFileLogContent(FileTaskProgressDTO fileTaskProgressDTO, + String filePathWithSourceAlias, + String fileSizeStr) { + StringBuilder sb = new StringBuilder(); + sb.append("FileName: "); + sb.append(filePathWithSourceAlias); + sb.append(" FileSize: "); + sb.append(fileSizeStr); + sb.append(" "); + sb.append("Speed: "); + sb.append(fileTaskProgressDTO.getSpeed()); + sb.append(" Progress: "); + sb.append(fileTaskProgressDTO.getProgress()); + sb.append("% Detail: "); + sb.append(fileTaskProgressDTO.getContent()); + return sb.toString(); + } + + private String buildProcessStr(FileTaskProgressDTO fileTaskProgressDTO) { + return "" + fileTaskProgressDTO.getProgress() + "%"; + } + +} diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/FileTaskServiceImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/FileTaskServiceImpl.java index 15c23a718f..fb8f6aa291 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/FileTaskServiceImpl.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/FileTaskServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,7 +28,6 @@ import com.tencent.bk.job.file_gateway.model.dto.FileTaskDTO; import com.tencent.bk.job.file_gateway.service.FileTaskService; import lombok.extern.slf4j.Slf4j; -import org.jooq.DSLContext; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -39,12 +38,10 @@ public class FileTaskServiceImpl implements FileTaskService { private final FileTaskDAO fileTaskDAO; - private final DSLContext dslContext; @Autowired - public FileTaskServiceImpl(FileTaskDAO fileTaskDAO, DSLContext dslContext) { + public FileTaskServiceImpl(FileTaskDAO fileTaskDAO) { this.fileTaskDAO = fileTaskDAO; - this.dslContext = dslContext; } @Override @@ -54,11 +51,12 @@ public List listFileTasks(String fileSourceTaskId) { @Override public void resetTasks(String fileSourceTaskId) { - fileTaskDAO.resetFileTasks(dslContext, fileSourceTaskId); + int affectedRowNum = fileTaskDAO.resetFileTasks(fileSourceTaskId); + log.info("fileSourceTask {} has been reset, {} fileTask updated", fileSourceTaskId, affectedRowNum); } @Override public int deleteTasks(String fileSourceTaskId) { - return fileTaskDAO.deleteFileTaskByFileSourceTaskId(dslContext, fileSourceTaskId); + return fileTaskDAO.deleteFileTaskByFileSourceTaskId(fileSourceTaskId); } } diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/FileWorkerServiceImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/FileWorkerServiceImpl.java index 17e289977e..dcc875df9f 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/FileWorkerServiceImpl.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/FileWorkerServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,6 +25,7 @@ package com.tencent.bk.job.file_gateway.service.impl; import com.fasterxml.jackson.core.type.TypeReference; +import com.tencent.bk.job.common.mysql.JobTransactional; import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.file_gateway.config.FileGatewayConfig; import com.tencent.bk.job.file_gateway.consts.WorkerSelectScopeEnum; @@ -39,30 +40,30 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; -import org.jooq.DSLContext; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Set; @Slf4j @Service public class FileWorkerServiceImpl implements FileWorkerService { - private final DSLContext dslContext; private final FileWorkerDAO fileWorkerDAO; private final FileSourceTypeDAO fileSourceTypeDAO; private final FileWorkerTagDAO fileWorkerTagDAO; private final FileGatewayConfig fileGatewayConfig; @Autowired - public FileWorkerServiceImpl(DSLContext dslContext, - FileWorkerDAO fileWorkerDAO, + public FileWorkerServiceImpl(FileWorkerDAO fileWorkerDAO, FileSourceTypeDAO fileSourceTypeDAO, FileWorkerTagDAO fileWorkerTagDAO, FileGatewayConfig fileGatewayConfig) { - this.dslContext = dslContext; this.fileWorkerDAO = fileWorkerDAO; this.fileSourceTypeDAO = fileSourceTypeDAO; this.fileWorkerTagDAO = fileWorkerTagDAO; @@ -70,6 +71,7 @@ public FileWorkerServiceImpl(DSLContext dslContext, } @Override + @JobTransactional(transactionManager = "jobFileGatewayTransactionManager") public Long heartBeat(FileWorkerDTO fileWorkerDTO) { Long id; String configStr = fileWorkerDTO.getConfigStr(); @@ -98,18 +100,67 @@ public Long heartBeat(FileWorkerDTO fileWorkerDTO) { fileSourceTypeDTO.setWorkerId(id); fileSourceTypeDTO.setCode(fileSourceMetaData.getFileSourceTypeCode()); fileSourceTypeDTO.setName(fileSourceMetaData.getName()); + fileSourceTypeDTO.setEnabled(fileSourceMetaData.getEnabled()); fileSourceTypeDTO.setIcon(fileSourceMetaData.getIconBase64()); fileSourceTypeDTO.setStorageType(fileSourceMetaData.getStorageTypeCode()); fileSourceTypeDTO.setLastModifier("Worker_" + id); fileSourceTypeDTOList.add(fileSourceTypeDTO); } - for (FileSourceTypeDTO fileSourceTypeDTO : fileSourceTypeDTOList) { - fileSourceTypeDAO.upsertByWorker(dslContext, fileSourceTypeDTO); - } + updateWorkerFileSourceType(id, fileSourceTypeDTOList); } return id; } + private void updateWorkerFileSourceType(Long workerId, List newFileSourceTypeList) { + List currentFileSourceTypeList = fileSourceTypeDAO.listByWorkerId(workerId); + Map codeTypeMap = new HashMap<>(); + for (FileSourceTypeDTO fileSourceTypeDTO : currentFileSourceTypeList) { + if (!codeTypeMap.containsKey(fileSourceTypeDTO.getCode())) { + codeTypeMap.put(fileSourceTypeDTO.getCode(), fileSourceTypeDTO); + } + } + Set newFileSourceCodes = new HashSet<>(); + List updateList = new ArrayList<>(); + List insertList = new ArrayList<>(); + for (FileSourceTypeDTO fileSourceTypeDTO : newFileSourceTypeList) { + String code = fileSourceTypeDTO.getCode(); + newFileSourceCodes.add(code); + if (codeTypeMap.containsKey(code)) { + fileSourceTypeDTO.setId(codeTypeMap.get(code).getId()); + updateList.add(fileSourceTypeDTO); + } else { + insertList.add(fileSourceTypeDTO); + } + } + Set currentFileSourceCodes = new HashSet<>(); + List deleteIdList = new ArrayList<>(); + for (FileSourceTypeDTO fileSourceTypeDTO : currentFileSourceTypeList) { + String code = fileSourceTypeDTO.getCode(); + if (!newFileSourceCodes.contains(code) || currentFileSourceCodes.contains(code)) { + deleteIdList.add(fileSourceTypeDTO.getId()); + } + currentFileSourceCodes.add(code); + } + applyDataChanges(insertList, updateList, deleteIdList); + } + + private void applyDataChanges(List insertList, + List updateList, + List deleteIdList) { + if (!deleteIdList.isEmpty()) { + int deletedRowNum = fileSourceTypeDAO.batchDeleteByIds(deleteIdList); + log.info("{} fileSourceType deleted, deleteIdList={}", deletedRowNum, deleteIdList); + } + if (!insertList.isEmpty()) { + int insertedRowNum = fileSourceTypeDAO.batchInsert(insertList); + log.info("{} fileSourceType inserted", insertedRowNum); + } + if (!updateList.isEmpty()) { + int updatedRowNum = fileSourceTypeDAO.batchUpdate(updateList); + log.debug("{} fileSourceType updated", updatedRowNum); + } + } + private Long saveFileWorker(FileWorkerDTO fileWorkerDTO) { return fileWorkerDAO.insertFileWorker(fileWorkerDTO); } diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/MeasureServiceImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/MeasureServiceImpl.java index 6adc5c095a..1f669c8201 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/MeasureServiceImpl.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/MeasureServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -35,7 +35,7 @@ import java.util.Collections; @Slf4j -@Service +@Service("jobFileGatewayMeasureService") public class MeasureServiceImpl { @Autowired diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/ReDispatchServiceImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/ReDispatchServiceImpl.java deleted file mode 100644 index ceff0843ba..0000000000 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/ReDispatchServiceImpl.java +++ /dev/null @@ -1,245 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.file_gateway.service.impl; - -import com.tencent.bk.job.common.constant.ErrorCode; -import com.tencent.bk.job.common.exception.InternalException; -import com.tencent.bk.job.file_gateway.model.dto.FileSourceDTO; -import com.tencent.bk.job.file_gateway.model.dto.FileSourceTaskDTO; -import com.tencent.bk.job.file_gateway.model.dto.FileTaskDTO; -import com.tencent.bk.job.file_gateway.model.dto.FileWorkerDTO; -import com.tencent.bk.job.file_gateway.model.resp.inner.TaskInfoDTO; -import com.tencent.bk.job.file_gateway.service.DispatchService; -import com.tencent.bk.job.file_gateway.service.FileSourceService; -import com.tencent.bk.job.file_gateway.service.FileSourceTaskService; -import com.tencent.bk.job.file_gateway.service.FileTaskService; -import com.tencent.bk.job.file_gateway.service.FileWorkerService; -import com.tencent.bk.job.file_gateway.service.ReDispatchService; -import lombok.extern.slf4j.Slf4j; -import org.slf4j.helpers.FormattingTuple; -import org.slf4j.helpers.MessageFormatter; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.Timer; -import java.util.TimerTask; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.stream.Collectors; - -@Slf4j -@Service -public class ReDispatchServiceImpl implements ReDispatchService { - - private final DispatchService dispatchService; - private final FileWorkerService fileWorkerService; - private final FileSourceService fileSourceService; - private final FileSourceTaskService fileSourceTaskService; - private final FileTaskService fileTaskService; - // 最多使用50线程进行重调度 - private final int MAX_THREAD_NUM_REDISPATCH = 50; - private final AtomicInteger reDispatchThreadNum = new AtomicInteger(0); - private final Set reDispatchingTaskIds = new HashSet<>(); - - @Autowired - public ReDispatchServiceImpl( - DispatchService dispatchService, - FileWorkerService fileWorkerService, - FileSourceService fileSourceService, - FileSourceTaskService fileSourceTaskService, - FileTaskService fileTaskService - ) { - this.dispatchService = dispatchService; - this.fileWorkerService = fileWorkerService; - this.fileSourceService = fileSourceService; - this.fileSourceTaskService = fileSourceTaskService; - this.fileTaskService = fileTaskService; - } - - @Override - public List reDispatchByWorker( - String accessHost, - Integer accessPort, - List taskIdList, - Long initDelayMills, - Long intervalMills - ) { - FileWorkerDTO fileWorkerDTO = fileWorkerService.getFileWorker(accessHost, accessPort); - if (fileWorkerDTO == null) { - FormattingTuple msg = MessageFormatter.format( - "Fail to find file-worker by accessHost:{} accessPort:{}", accessHost, accessPort - ); - log.warn(msg.getMessage()); - throw new InternalException( - ErrorCode.FILE_WORKER_NOT_FOUND, - new String[]{ - "accessHost:" + accessHost + ",accessPort:" + accessPort, - } - ); - } - Long workerId = fileWorkerDTO.getId(); - log.debug("worker {} apply to reDispatch tasks:{}, initDelayMills={}, intervalMills={}", workerId, taskIdList - , initDelayMills, intervalMills); - // 1.立即下线Worker - int affectedWorkerNum = fileWorkerService.offLine(workerId); - log.info("{} worker state changed to offline", affectedWorkerNum); - // 2.任务延时重调度 - for (String taskId : taskIdList) { - if (reDispatchThreadNum.get() >= MAX_THREAD_NUM_REDISPATCH) { - log.warn("reDispatch thread reach MAX_NUM:{}, do not reDispatch {}", MAX_THREAD_NUM_REDISPATCH, taskId); - } else { - Timer timer = new Timer(); - timer.schedule(new ReDispatchTask(taskId, intervalMills), initDelayMills); - } - } - return taskIdList; - } - - @Override - public boolean reDispatchByGateway(String fileSourceTaskId, Long initDelayMills, Long intervalMills) { - // 1.尝试通知Worker主动取消该任务 - FileSourceTaskDTO fileSourceTaskDTO = fileSourceTaskService.getFileSourceTaskById(fileSourceTaskId); - if (fileSourceTaskDTO == null) { - log.warn("task not exist, ignore, id={}", fileSourceTaskId); - return false; - } - try { - fileSourceTaskService.recallTasks(Collections.singletonList(fileSourceTaskId)); - } catch (Throwable t) { - log.warn("Fail to recallTask:{}", fileSourceTaskId, t); - } - // 2.重调度 - if (reDispatchThreadNum.get() >= MAX_THREAD_NUM_REDISPATCH) { - log.warn("reDispatch thread reach MAX_NUM:{}, do not reDispatch {}", MAX_THREAD_NUM_REDISPATCH, - fileSourceTaskId); - return false; - } else { - Timer timer = new Timer(); - timer.schedule(new ReDispatchTask(fileSourceTaskId, intervalMills), initDelayMills); - return true; - } - } - - @Override - public Integer getReDispatchThreadsNum(String username) { - return reDispatchThreadNum.get(); - } - - class ReDispatchTask extends TimerTask { - private final String fileSourceTaskId; - private final Long intervalMills; - - ReDispatchTask(String fileSourceTaskId, Long intervalMills) { - this.fileSourceTaskId = fileSourceTaskId; - this.intervalMills = intervalMills; - } - - @Override - public void run() { - synchronized (reDispatchingTaskIds) { - if (reDispatchingTaskIds.contains(fileSourceTaskId)) { - log.info("task {} already in reDispatching, ignore", fileSourceTaskId); - return; - } - reDispatchingTaskIds.add(fileSourceTaskId); - } - boolean reDispatchSuccess = false; - int retryCount = 0; - try { - reDispatchThreadNum.incrementAndGet(); - log.debug("taskId={}", fileSourceTaskId); - FileSourceTaskDTO fileSourceTaskDTO = fileSourceTaskService.getFileSourceTaskById(fileSourceTaskId); - log.debug("fileSourceTaskDTO={}", fileSourceTaskDTO); - if (fileSourceTaskDTO == null) { - log.warn("Cannot find fileSourceTaskDTO by id {}", fileSourceTaskId); - return; - } - List fileTaskDTOList = fileTaskService.listFileTasks(fileSourceTaskId); - log.debug("fileTaskDTOList={}", fileTaskDTOList); - List filePathList = - fileTaskDTOList.parallelStream().map(FileTaskDTO::getFilePath).collect(Collectors.toList()); - - FileSourceDTO fileSourceDTO = fileSourceService.getFileSourceById(fileSourceTaskDTO.getFileSourceId()); - while (!reDispatchSuccess && retryCount < 100) { - // 1.删除现有子任务 - log.debug("delete fileTasks of fileSourceTask {}", fileSourceTaskId); - fileTaskService.deleteTasks(fileSourceTaskId); - // 2.删除现有FileSourceTask任务 - fileSourceTaskService.deleteFileSourceTaskById(fileSourceTaskId); - log.debug("delete fileSourceTask {}", fileSourceTaskId); - FileWorkerDTO fileWorkerDTO = dispatchService.findBestFileWorker(fileSourceDTO); - log.debug("found bestWorker:{}", fileSourceDTO); - if (fileWorkerDTO != null) { - // 3.重新派发任务 - try { - TaskInfoDTO taskInfoDTO = - fileSourceTaskService.startFileSourceDownloadTaskWithId( - fileSourceTaskDTO.getCreator(), - fileSourceTaskDTO.getAppId(), - fileSourceTaskDTO.getStepInstanceId(), - fileSourceTaskDTO.getExecuteCount(), - fileSourceTaskDTO.getBatchTaskId(), - fileSourceTaskDTO.getFileSourceId(), - filePathList, - fileSourceTaskId - ); - reDispatchSuccess = true; - log.info("reDispatch result of {}:{}", fileSourceTaskId, taskInfoDTO); - } catch (Exception e) { - retryCount += 1; - log.info("Fail to redispatch task {}, wait {}ms to retry {}", fileSourceTaskId, - intervalMills, retryCount); - try { - Thread.sleep(intervalMills); - } catch (InterruptedException interruptedException) { - log.error("redispatch wait interrupted", e); - } - } - } else { - // 3.暂时没有合适的FileWorker，延时等待 - try { - retryCount += 1; - log.info("No suitable worker to redispatch task {}, wait {}ms to retry {}", - fileSourceTaskId, intervalMills, retryCount); - Thread.sleep(intervalMills); - } catch (InterruptedException e) { - log.error("redispatch wait interrupted", e); - } - } - } - } catch (Throwable t) { - log.error("ReDispatchTask fail", t); - } finally { - synchronized (reDispatchingTaskIds) { - reDispatchingTaskIds.remove(fileSourceTaskId); - } - reDispatchThreadNum.decrementAndGet(); - } - } - } -} diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/ResourceNameQueryServiceImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/ResourceNameQueryServiceImpl.java index 8cf7b8cb54..99e9bfe43e 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/ResourceNameQueryServiceImpl.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/ResourceNameQueryServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -30,37 +30,37 @@ import com.tencent.bk.job.common.iam.service.ResourceNameQueryService; import com.tencent.bk.job.common.iam.util.IamUtil; import com.tencent.bk.job.common.service.AppScopeMappingService; -import com.tencent.bk.job.file_gateway.client.ServiceApplicationResourceClient; import com.tencent.bk.job.file_gateway.model.dto.FileSourceDTO; import com.tencent.bk.job.file_gateway.service.FileSourceService; +import com.tencent.bk.job.manage.api.inner.ServiceApplicationResource; import com.tencent.bk.job.manage.model.inner.resp.ServiceApplicationDTO; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @Slf4j -@Service("ResourceNameQueryService") +@Service("jobFileGatewayResourceNameQueryService") public class ResourceNameQueryServiceImpl implements ResourceNameQueryService { private final FileSourceService fileSourceService; - private final ServiceApplicationResourceClient applicationResourceClient; + private final ServiceApplicationResource applicationResource; private final AppScopeMappingService appScopeMappingService; @Autowired public ResourceNameQueryServiceImpl(AuthService authService, AppAuthService appAuthService, FileSourceService fileSourceService, - ServiceApplicationResourceClient applicationResourceClient, + ServiceApplicationResource applicationResource, AppScopeMappingService appScopeMappingService) { this.fileSourceService = fileSourceService; - this.applicationResourceClient = applicationResourceClient; + this.applicationResource = applicationResource; this.appScopeMappingService = appScopeMappingService; authService.setResourceNameQueryService(this); appAuthService.setResourceNameQueryService(this); } private String getAppName(long appId) { - ServiceApplicationDTO serviceApplicationDTO = applicationResourceClient.queryAppById(appId); + ServiceApplicationDTO serviceApplicationDTO = applicationResource.queryAppById(appId); if (serviceApplicationDTO == null) return null; return serviceApplicationDTO.getName(); } diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/RetryPolicyFileSourceTaskServiceImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/RetryPolicyFileSourceTaskServiceImpl.java new file mode 100644 index 0000000000..21f4a21dea --- /dev/null +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/RetryPolicyFileSourceTaskServiceImpl.java @@ -0,0 +1,95 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file_gateway.service.impl; + +import com.tencent.bk.job.file_gateway.model.resp.inner.TaskInfoDTO; +import com.tencent.bk.job.file_gateway.service.FileSourceTaskService; +import com.tencent.bk.job.file_gateway.service.RetryPolicyFileSourceTaskService; +import com.tencent.bk.job.file_gateway.service.context.impl.FileSourceTaskRetryContext; +import com.tencent.bk.job.file_gateway.service.retry.FileSourceTaskRetryPolicy; +import com.tencent.bk.job.file_gateway.service.retry.impl.ExceptionRetryPolicy; +import lombok.extern.slf4j.Slf4j; +import org.slf4j.helpers.MessageFormatter; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.List; + +@Slf4j +@Service +public class RetryPolicyFileSourceTaskServiceImpl implements RetryPolicyFileSourceTaskService { + + private final FileSourceTaskService fileSourceTaskService; + private final FileSourceTaskRetryPolicy retryPolicy = new ExceptionRetryPolicy(3, 5000); + + @Autowired + public RetryPolicyFileSourceTaskServiceImpl(FileSourceTaskService fileSourceTaskService) { + this.fileSourceTaskService = fileSourceTaskService; + } + + @Override + public TaskInfoDTO startFileSourceDownloadTask(String username, + Long appId, + Long stepInstanceId, + Integer executeCount, + String batchTaskId, + Integer fileSourceId, + List filePathList) { + int retryCount = 0; + boolean shouldRetry; + do { + try { + return fileSourceTaskService.startFileSourceDownloadTask( + username, + appId, + stepInstanceId, + executeCount, + batchTaskId, + fileSourceId, + filePathList + ); + } catch (Exception e) { + retryCount += 1; + FileSourceTaskRetryContext retryContext = new FileSourceTaskRetryContext(e); + shouldRetry = retryPolicy.shouldRetry(retryContext, retryCount); + if (shouldRetry) { + String msg = MessageFormatter.arrayFormat( + "Fail to startFileSourceDownloadTask, stepInstanceId={}, " + + "executeCount={}, batchTaskId={}, retry {}", + new Object[]{ + stepInstanceId, + executeCount, + batchTaskId, + retryCount + } + ).getMessage(); + log.info(msg, e); + } else { + throw e; + } + } + } while (true); + } +} diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/WorkerIdsCondition.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/WorkerIdsCondition.java index 1fcf175c0b..03fccac978 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/WorkerIdsCondition.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/impl/WorkerIdsCondition.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/listener/FileSourceTaskStatusChangeListener.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/listener/FileSourceTaskStatusChangeListener.java index 5e5d1f85f8..1ce671c746 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/listener/FileSourceTaskStatusChangeListener.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/listener/FileSourceTaskStatusChangeListener.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -29,12 +29,12 @@ public interface FileSourceTaskStatusChangeListener { /** - * Invoked when FileSourceTask status change + * 在单个文件源任务状态改变时被调用 * - * @param context - * @param previousStatus - * @param currentStatus - * @return true if need to stop event dispatch to later listeners + * @param context 任务上下文 + * @param previousStatus 上一个状态 + * @param currentStatus 当前状态 + * @return true 如果需要终止事件向后续的Listener传递则返回true */ boolean onStatusChange(TaskContext context, TaskStatusEnum previousStatus, TaskStatusEnum currentStatus); } diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/listener/FileTaskStatusChangeListener.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/listener/FileTaskStatusChangeListener.java index 87f1d0a3df..70e4b133e4 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/listener/FileTaskStatusChangeListener.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/listener/FileTaskStatusChangeListener.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -29,12 +29,12 @@ public interface FileTaskStatusChangeListener { /** - * Invoked when FileTask status change + * 在单个文件任务状态改变时被调用 * - * @param context - * @param previousStatus - * @param currentStatus - * @return true if need to stop event dispatch to later listeners + * @param context 任务上下文 + * @param previousStatus 上一个状态 + * @param currentStatus 当前状态 + * @return true 如果需要终止事件向后续的Listener传递则返回true */ boolean onStatusChange(TaskContext context, TaskStatusEnum previousStatus, TaskStatusEnum currentStatus); } diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/listener/impl/BatchTaskStatusUpdater.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/listener/impl/BatchTaskStatusUpdater.java index 92fd34b5e7..3ed1d27171 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/listener/impl/BatchTaskStatusUpdater.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/listener/impl/BatchTaskStatusUpdater.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -33,7 +33,6 @@ import com.tencent.bk.job.file_gateway.service.listener.FileSourceTaskStatusChangeListener; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -import org.jooq.DSLContext; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -41,14 +40,12 @@ @Service public class BatchTaskStatusUpdater implements FileSourceTaskStatusChangeListener { - private final DSLContext dslContext; private final FileSourceTaskDAO fileSourceTaskDAO; private final FileSourceBatchTaskDAO fileSourceBatchTaskDAO; @Autowired - public BatchTaskStatusUpdater(DSLContext dslContext, FileSourceTaskDAO fileSourceTaskDAO, + public BatchTaskStatusUpdater(FileSourceTaskDAO fileSourceTaskDAO, FileSourceBatchTaskDAO fileSourceBatchTaskDAO) { - this.dslContext = dslContext; this.fileSourceTaskDAO = fileSourceTaskDAO; this.fileSourceBatchTaskDAO = fileSourceBatchTaskDAO; } @@ -57,8 +54,7 @@ public BatchTaskStatusUpdater(DSLContext dslContext, FileSourceTaskDAO fileSourc public boolean onStatusChange(TaskContext context, TaskStatusEnum previousStatus, TaskStatusEnum currentStatus) { FileSourceTaskDTO fileSourceTaskDTO = context.getFileSourceTaskDTO(); String batchTaskId = fileSourceTaskDTO.getBatchTaskId(); - FileSourceBatchTaskDTO fileSourceBatchTaskDTO = fileSourceBatchTaskDAO.getFileSourceBatchTaskById(dslContext, - batchTaskId); + FileSourceBatchTaskDTO fileSourceBatchTaskDTO = fileSourceBatchTaskDAO.getBatchTaskById(batchTaskId); if (StringUtils.isBlank(batchTaskId)) { return false; } @@ -69,14 +65,13 @@ public boolean onStatusChange(TaskContext context, TaskStatusEnum previousStatus } if (TaskStatusEnum.SUCCESS.equals(currentStatus)) { //检查批量任务是否成功 - if (fileSourceTaskDAO.countFileSourceTasksByBatchTaskId( - dslContext, batchTaskId, null) + if (fileSourceTaskDAO.countFileSourceTasksByBatchTaskId(batchTaskId, null) .equals(fileSourceTaskDAO.countFileSourceTasksByBatchTaskId( - dslContext, batchTaskId, TaskStatusEnum.SUCCESS.getStatus()))) { + batchTaskId, TaskStatusEnum.SUCCESS.getStatus()))) { // 批量任务成功 fileSourceBatchTaskDTO.setStatus(TaskStatusEnum.SUCCESS.getStatus()); - fileSourceBatchTaskDAO.updateFileSourceBatchTask(dslContext, fileSourceBatchTaskDTO); - logUpdatedBatchTaskStatus(batchTaskId, TaskStatusEnum.SUCCESS); + int affectedNum = fileSourceBatchTaskDAO.updateFileSourceBatchTask(fileSourceBatchTaskDTO); + logUpdatedBatchTaskStatus(batchTaskId, TaskStatusEnum.SUCCESS, affectedNum); } else { // 主任务尚未成功 log.info("task {} done, batchTask not finished yet", fileSourceTaskDTO.getId()); @@ -84,18 +79,18 @@ public boolean onStatusChange(TaskContext context, TaskStatusEnum previousStatus } else if (TaskStatusEnum.FAILED.equals(currentStatus)) { // 批量任务失败 fileSourceBatchTaskDTO.setStatus(TaskStatusEnum.FAILED.getStatus()); - fileSourceBatchTaskDAO.updateFileSourceBatchTask(dslContext, fileSourceBatchTaskDTO); - logUpdatedBatchTaskStatus(batchTaskId, TaskStatusEnum.FAILED); + int affectedNum = fileSourceBatchTaskDAO.updateFileSourceBatchTask(fileSourceBatchTaskDTO); + logUpdatedBatchTaskStatus(batchTaskId, TaskStatusEnum.FAILED, affectedNum); } else if (TaskStatusEnum.STOPPED.equals(currentStatus)) { // 批量任务停止 fileSourceBatchTaskDTO.setStatus(TaskStatusEnum.STOPPED.getStatus()); - fileSourceBatchTaskDAO.updateFileSourceBatchTask(dslContext, fileSourceBatchTaskDTO); - logUpdatedBatchTaskStatus(batchTaskId, TaskStatusEnum.STOPPED); + int affectedNum = fileSourceBatchTaskDAO.updateFileSourceBatchTask(fileSourceBatchTaskDTO); + logUpdatedBatchTaskStatus(batchTaskId, TaskStatusEnum.STOPPED, affectedNum); } return false; } - private void logUpdatedBatchTaskStatus(String batchTaskId, TaskStatusEnum status) { - log.info("updated batchTask:{},{}", batchTaskId, status.name()); + private void logUpdatedBatchTaskStatus(String batchTaskId, TaskStatusEnum status, int affectedNum) { + log.info("updated batchTask:{},{}, affectedNum={}", batchTaskId, status.name(), affectedNum); } } diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/listener/impl/FileSourceTaskStatusUpdater.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/listener/impl/FileSourceTaskStatusUpdater.java index f720b2bb25..7e503caf22 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/listener/impl/FileSourceTaskStatusUpdater.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/listener/impl/FileSourceTaskStatusUpdater.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -35,7 +35,6 @@ import com.tencent.bk.job.file_gateway.service.listener.FileSourceTaskStatusChangeListener; import com.tencent.bk.job.file_gateway.service.listener.FileTaskStatusChangeListener; import lombok.extern.slf4j.Slf4j; -import org.jooq.DSLContext; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -46,15 +45,14 @@ @Service public class FileSourceTaskStatusUpdater implements FileTaskStatusChangeListener { - private final DSLContext dslContext; private final FileSourceTaskDAO fileSourceTaskDAO; private final FileTaskDAO fileTaskDAO; private final List fileSourceTaskStatusChangeListenerList = new ArrayList<>(); @Autowired - public FileSourceTaskStatusUpdater(DSLContext dslContext, FileSourceTaskDAO fileSourceTaskDAO, - FileTaskDAO fileTaskDAO, FileSourceTaskStatusChangeListener listener) { - this.dslContext = dslContext; + public FileSourceTaskStatusUpdater(FileSourceTaskDAO fileSourceTaskDAO, + FileTaskDAO fileTaskDAO, + FileSourceTaskStatusChangeListener listener) { this.fileSourceTaskDAO = fileSourceTaskDAO; this.fileTaskDAO = fileTaskDAO; this.addFileTaskStatusChangeListener(listener); @@ -88,18 +86,18 @@ public boolean onStatusChange(TaskContext context, TaskStatusEnum previousStatus if (!fileSourceTaskDTO.isDone()) { // 主任务RUNNING fileSourceTaskDTO.setStatus(TaskStatusEnum.RUNNING.getStatus()); - fileSourceTaskDAO.updateFileSourceTask(dslContext, fileSourceTaskDTO); + fileSourceTaskDAO.updateFileSourceTask(fileSourceTaskDTO); logUpdatedFileSourceTaskStatus(fileSourceTaskId, TaskStatusEnum.RUNNING); } else { log.info("fileSourceTask {} already done, do not update to running", fileSourceTaskId); } } else if (TaskStatusEnum.SUCCESS.equals(currentStatus)) { //检查主任务是否成功 - if (fileTaskDAO.countFileTask(fileSourceTaskId, null).equals(fileTaskDAO.countFileTask(fileSourceTaskId, - TaskStatusEnum.SUCCESS.getStatus()))) { + if (fileTaskDAO.countFileTask(fileSourceTaskId, null).equals( + fileTaskDAO.countFileTask(fileSourceTaskId, TaskStatusEnum.SUCCESS.getStatus()))) { // 主任务成功 fileSourceTaskDTO.setStatus(TaskStatusEnum.SUCCESS.getStatus()); - fileSourceTaskDAO.updateFileSourceTask(dslContext, fileSourceTaskDTO); + fileSourceTaskDAO.updateFileSourceTask(fileSourceTaskDTO); logUpdatedFileSourceTaskStatus(fileSourceTaskId, TaskStatusEnum.SUCCESS); } else { // 主任务尚未成功 @@ -108,12 +106,12 @@ public boolean onStatusChange(TaskContext context, TaskStatusEnum previousStatus } else if (TaskStatusEnum.FAILED.equals(currentStatus)) { // 主任务失败 fileSourceTaskDTO.setStatus(TaskStatusEnum.FAILED.getStatus()); - fileSourceTaskDAO.updateFileSourceTask(dslContext, fileSourceTaskDTO); + fileSourceTaskDAO.updateFileSourceTask(fileSourceTaskDTO); logUpdatedFileSourceTaskStatus(fileSourceTaskId, TaskStatusEnum.FAILED); } else if (TaskStatusEnum.STOPPED.equals(currentStatus)) { // 主任务停止 fileSourceTaskDTO.setStatus(TaskStatusEnum.STOPPED.getStatus()); - fileSourceTaskDAO.updateFileSourceTask(dslContext, fileSourceTaskDTO); + fileSourceTaskDAO.updateFileSourceTask(fileSourceTaskDTO); logUpdatedFileSourceTaskStatus(fileSourceTaskId, TaskStatusEnum.STOPPED); } // 通知其他关注事件的Listener diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/remote/FileSourceReqGenService.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/remote/FileSourceReqGenService.java index 85988d99a7..0a93d5aabc 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/remote/FileSourceReqGenService.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/remote/FileSourceReqGenService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/remote/FileSourceTaskReqGenService.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/remote/FileSourceTaskReqGenService.java index 82afc12725..ed39e34317 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/remote/FileSourceTaskReqGenService.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/remote/FileSourceTaskReqGenService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/remote/impl/BaseRemoteFileReqGenServiceImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/remote/impl/BaseRemoteFileReqGenServiceImpl.java index 335d07da0f..b4ea90403e 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/remote/impl/BaseRemoteFileReqGenServiceImpl.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/remote/impl/BaseRemoteFileReqGenServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,25 +24,34 @@ package com.tencent.bk.job.file_gateway.service.remote.impl; +import com.tencent.bk.job.common.jwt.JwtManager; +import com.tencent.bk.job.common.model.dto.CommonCredential; import com.tencent.bk.job.common.model.http.HttpReq; +import com.tencent.bk.job.common.security.consts.JwtConsts; import com.tencent.bk.job.common.util.http.HttpReqGenUtil; import com.tencent.bk.job.file.worker.model.req.BaseReq; import com.tencent.bk.job.file_gateway.model.dto.FileSourceDTO; import com.tencent.bk.job.file_gateway.model.dto.FileWorkerDTO; import com.tencent.bk.job.file_gateway.service.CredentialService; -import com.tencent.bk.job.common.model.dto.CommonCredential; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.apache.http.Header; +import org.apache.http.message.BasicHeader; import org.springframework.stereotype.Service; +import java.util.ArrayList; +import java.util.List; + @Slf4j @Service public class BaseRemoteFileReqGenServiceImpl { protected final CredentialService credentialService; + private final JwtManager jwtManager; - public BaseRemoteFileReqGenServiceImpl(CredentialService credentialService) { + public BaseRemoteFileReqGenServiceImpl(CredentialService credentialService, JwtManager jwtManager) { this.credentialService = credentialService; + this.jwtManager = jwtManager; } protected String getCompleteUrl(FileWorkerDTO fileWorkerDTO, String url) { @@ -51,8 +60,10 @@ protected String getCompleteUrl(FileWorkerDTO fileWorkerDTO, String url) { return "http://" + host + ":" + port.toString() + "/worker/api" + url; } - protected String fillBaseReqGetUrl(BaseReq req, FileWorkerDTO fileWorkerDTO, - FileSourceDTO fileSourceDTO, String url) { + protected String fillBaseReqGetUrl(BaseReq req, + FileWorkerDTO fileWorkerDTO, + FileSourceDTO fileSourceDTO, + String url) { String completeUrl = getCompleteUrl(fileWorkerDTO, url); String credentialId = fileSourceDTO.getCredentialId(); CommonCredential commonCredential = null; @@ -64,7 +75,7 @@ protected String fillBaseReqGetUrl(BaseReq req, FileWorkerDTO fileWorkerDTO, req.setCredential(commonCredential); log.debug("Credential of id {} is {}", credentialId, commonCredential); } else if (StringUtils.isNotBlank(credentialId)) { - log.warn("Cannot find credential by id {}, fileSource={}", credentialId, fileSourceDTO); + log.warn("Cannot find credential by id {}, fileSource={}", credentialId, fileSourceDTO.getBasicDesc()); } req.setFileSourceTypeCode(fileSourceDTO.getFileSourceType().getCode()); req.setFileSourceInfoMap(fileSourceDTO.getFileSourceInfoMap()); @@ -72,6 +83,9 @@ protected String fillBaseReqGetUrl(BaseReq req, FileWorkerDTO fileWorkerDTO, } protected HttpReq genRemoteFileReq(String url, Object body) { - return HttpReqGenUtil.genSimpleJsonReq(url, body); + String token = jwtManager.getToken(); + List
headerList = new ArrayList<>(); + headerList.add(new BasicHeader(JwtConsts.HEADER_KEY_SERVICE_JWT_TOKEN, token)); + return HttpReqGenUtil.genSimpleJsonReq(url, headerList, body); } } diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/remote/impl/FileSourceReqGenServiceImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/remote/impl/FileSourceReqGenServiceImpl.java index f7a2307e77..19423525ef 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/remote/impl/FileSourceReqGenServiceImpl.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/remote/impl/FileSourceReqGenServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,9 @@ package com.tencent.bk.job.file_gateway.service.remote.impl; +import com.tencent.bk.job.common.jwt.JwtManager; import com.tencent.bk.job.common.model.http.HttpReq; +import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.file.worker.model.req.BaseReq; import com.tencent.bk.job.file.worker.model.req.ExecuteActionReq; import com.tencent.bk.job.file.worker.model.req.ListFileNodeReq; @@ -32,24 +34,26 @@ import com.tencent.bk.job.file_gateway.model.dto.FileWorkerDTO; import com.tencent.bk.job.file_gateway.service.CredentialService; import com.tencent.bk.job.file_gateway.service.remote.FileSourceReqGenService; +import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import java.util.Map; - +@Slf4j @Service public class FileSourceReqGenServiceImpl extends BaseRemoteFileReqGenServiceImpl implements FileSourceReqGenService { @Autowired - public FileSourceReqGenServiceImpl(CredentialService credentialService) { - super(credentialService); + public FileSourceReqGenServiceImpl(CredentialService credentialService, JwtManager jwtManager) { + super(credentialService, jwtManager); } @Override public HttpReq genFileAvailableReq(Long appId, FileWorkerDTO fileWorkerDTO, FileSourceDTO fileSourceDTO) { BaseReq req = new BaseReq(); String url = fillBaseReqGetUrl(req, fileWorkerDTO, fileSourceDTO, "/file/available"); + log.info("genFileAvailableReq: url={},req={}", url, JsonUtils.toJsonWithoutSkippedFields(req)); return genRemoteFileReq(url, req); } @@ -62,6 +66,7 @@ public HttpReq genListFileNodeReq(Long appId, String path, String name, Integer req.setName(name); req.setStart(start); req.setPageSize(pageSize); + log.info("genListFileNodeReq: url={},req={}", url, JsonUtils.toJsonWithoutSkippedFields(req)); return genRemoteFileReq(url, req); } @@ -72,6 +77,7 @@ public HttpReq genExecuteActionReq(Long appId, String actionCode, Map ta ClearTaskFilesReq req = new ClearTaskFilesReq(taskIdList); String url = getCompleteUrl(fileWorkerDTO, "/filetask/clearFiles"); req.setTaskIdList(taskIdList); + log.info("genClearTaskFilesReq: url={},req={}", url, JsonUtils.toJsonWithoutSkippedFields(req)); return genRemoteFileReq(url, req); } @@ -85,6 +89,7 @@ public HttpReq genStopTasksReq(FileWorkerDTO fileWorkerDTO, List taskIdL String url = getCompleteUrl(fileWorkerDTO, "/filetask/downloadFiles/stop"); req.setTaskIdList(taskIdList); req.setTaskCommand(command.getValue()); + log.info("genStopTasksReq: url={},req={}", url, JsonUtils.toJsonWithoutSkippedFields(req)); return genRemoteFileReq(url, req); } } diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/retry/FileSourceTaskRetryPolicy.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/retry/FileSourceTaskRetryPolicy.java new file mode 100644 index 0000000000..69ffbfae2e --- /dev/null +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/retry/FileSourceTaskRetryPolicy.java @@ -0,0 +1,34 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file_gateway.service.retry; + +import com.tencent.bk.job.file_gateway.service.context.impl.FileSourceTaskRetryContext; + +/** + * 第三方文件源任务重试策略接口 + */ +public interface FileSourceTaskRetryPolicy { + boolean shouldRetry(FileSourceTaskRetryContext context, int retryCount); +} diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/retry/impl/ExceptionRetryPolicy.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/retry/impl/ExceptionRetryPolicy.java new file mode 100644 index 0000000000..a3f0b5ffbf --- /dev/null +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/retry/impl/ExceptionRetryPolicy.java @@ -0,0 +1,94 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file_gateway.service.retry.impl; + +import com.tencent.bk.job.common.exception.InternalException; +import com.tencent.bk.job.common.util.ThreadUtils; +import com.tencent.bk.job.file_gateway.service.context.impl.FileSourceTaskRetryContext; +import com.tencent.bk.job.file_gateway.service.retry.FileSourceTaskRetryPolicy; +import org.springframework.web.client.ResourceAccessException; + +import java.net.SocketTimeoutException; +import java.net.UnknownHostException; + +/** + * 针对由于某些异常导致的任务启动失败重试策略 + * 当前支持的异常：UnknownHostException、SocketTimeoutException: connect timed out + */ +public class ExceptionRetryPolicy implements FileSourceTaskRetryPolicy { + + /** + * 最大重试次数 + */ + private final int maxRetryCount; + /** + * 重试前应当休眠的毫秒数 + */ + private final int sleepMillsBeforeRetryCount; + + public ExceptionRetryPolicy(int maxRetryCount, int sleepMillsBeforeRetryCount) { + this.maxRetryCount = maxRetryCount; + this.sleepMillsBeforeRetryCount = sleepMillsBeforeRetryCount; + } + + @Override + public boolean shouldRetry(FileSourceTaskRetryContext context, int retryCount) { + // 超出重试次数后不再重试 + if (retryCount > maxRetryCount) { + return false; + } + // 各层次异常检查 + Exception exception = context.getException(); + if (!(exception instanceof InternalException)) { + return false; + } + Throwable cause = exception.getCause(); + if (!(cause instanceof ResourceAccessException)) { + return false; + } + Throwable innerCause = cause.getCause(); + if (isTargetThrowable(innerCause)) { + ThreadUtils.sleep(sleepMillsBeforeRetryCount); + return true; + } + return false; + } + + private boolean isTargetThrowable(Throwable t) { + return isUnknownHostException(t) || isConnectTimeoutException(t); + } + + private boolean isUnknownHostException(Throwable t) { + return t instanceof UnknownHostException; + } + + private boolean isConnectTimeoutException(Throwable t) { + if (!(t instanceof SocketTimeoutException)) { + return false; + } + String message = t.getMessage(); + return message != null && message.equalsIgnoreCase("connect timed out"); + } +} diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/validation/FileSourceValidateService.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/validation/FileSourceValidateService.java new file mode 100644 index 0000000000..e2c4594bb4 --- /dev/null +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/validation/FileSourceValidateService.java @@ -0,0 +1,34 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file_gateway.service.validation; + +public interface FileSourceValidateService { + /** + * 检查对接的蓝鲸制品库根地址是否合法 + * + * @param baseUrl 根地址 + */ + void checkBkArtifactoryBaseUrl(String baseUrl); +} diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/validation/impl/FileSourceValidateServiceImpl.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/validation/impl/FileSourceValidateServiceImpl.java new file mode 100644 index 0000000000..3da6523415 --- /dev/null +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/service/validation/impl/FileSourceValidateServiceImpl.java @@ -0,0 +1,86 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file_gateway.service.validation.impl; + +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.InvalidParamException; +import com.tencent.bk.job.common.util.StringUtil; +import com.tencent.bk.job.file_gateway.config.ArtifactoryConfig; +import com.tencent.bk.job.file_gateway.consts.FileSourceWhiteInfoTypeConsts; +import com.tencent.bk.job.file_gateway.dao.filesource.FileSourceWhiteInfoDAO; +import com.tencent.bk.job.file_gateway.service.validation.FileSourceValidateService; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +@Service +public class FileSourceValidateServiceImpl implements FileSourceValidateService { + + private final ArtifactoryConfig artifactoryConfig; + private final FileSourceWhiteInfoDAO fileSourceWhiteInfoDAO; + + @Autowired + public FileSourceValidateServiceImpl(ArtifactoryConfig artifactoryConfig, + FileSourceWhiteInfoDAO fileSourceWhiteInfoDAO) { + this.artifactoryConfig = artifactoryConfig; + this.fileSourceWhiteInfoDAO = fileSourceWhiteInfoDAO; + } + + @Override + public void checkBkArtifactoryBaseUrl(String baseUrl) { + // 1.默认允许对接当前环境的蓝鲸制品库或其子域名 + if (isUrlOrChildUrlOfCurrentEnv(baseUrl)) { + return; + } + // 2.对接其他环境蓝鲸制品库需要添加白名单 + boolean existsWhiteInfo = fileSourceWhiteInfoDAO.exists( + FileSourceWhiteInfoTypeConsts.BK_ARTIFACTORY_BASE_URL, + baseUrl + ); + if (!existsWhiteInfo) { + throw new InvalidParamException(ErrorCode.BK_ARTIFACTORY_BASE_URL_INVALID); + } + } + + /** + * 判断URL是否为当前环境制品库的地址或子域名地址 + * + * @param url 目标URL + * @return 布尔值 + */ + private boolean isUrlOrChildUrlOfCurrentEnv(String url) { + String baseUrlOfCurrentEnv = artifactoryConfig.getArtifactoryBaseUrl(); + if (baseUrlOfCurrentEnv.equals(url)) { + return true; + } + String urlWithoutScheme = StringUtil.removeHttpOrHttpsSchemeOfUrl(baseUrlOfCurrentEnv); + int indexOfPath = urlWithoutScheme.indexOf("/"); + String domain = urlWithoutScheme; + if (indexOfPath != -1) { + domain = urlWithoutScheme.substring(0, indexOfPath); + } + String suffix = "." + domain.trim(); + return url.endsWith(suffix); + } +} diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/task/ScheduledTasks.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/task/ScheduledTasks.java index 7d2da1b163..f694f3b2b1 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/task/ScheduledTasks.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/task/ScheduledTasks.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -34,7 +34,7 @@ import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; -@Component +@Component("jobFileGatewayScheduledTasks") @EnableScheduling public class ScheduledTasks { @@ -80,9 +80,9 @@ public void refreshWorkerOnlineStatus() { } /** - * 任务超时重调度：1s/次 + * 任务超时重调度：10s/次 */ - @Scheduled(fixedDelay = 1000L, initialDelay = 3 * 1000L) + @Scheduled(fixedDelay = 10000L, initialDelay = 3 * 1000L) public void reDispatchTimeoutFileSourceTask() { logger.info(Thread.currentThread().getId() + ":reDispatchTimeoutFileSourceTask start"); try { diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/task/dispatch/ReDispatchTimeoutTask.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/task/dispatch/ReDispatchTimeoutTask.java index c01bd0c0f4..97199bcf23 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/task/dispatch/ReDispatchTimeoutTask.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/task/dispatch/ReDispatchTimeoutTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,14 +24,20 @@ package com.tencent.bk.job.file_gateway.task.dispatch; +import com.tencent.bk.job.common.redis.util.HeartBeatRedisLock; +import com.tencent.bk.job.common.redis.util.HeartBeatRedisLockConfig; +import com.tencent.bk.job.common.redis.util.LockResult; +import com.tencent.bk.job.common.util.TimeUtil; +import com.tencent.bk.job.common.util.ip.IpUtils; import com.tencent.bk.job.file_gateway.consts.TaskStatusEnum; import com.tencent.bk.job.file_gateway.dao.filesource.FileTaskDAO; -import com.tencent.bk.job.file_gateway.service.ReDispatchService; +import com.tencent.bk.job.file_gateway.service.dispatch.ReDispatchService; import lombok.extern.slf4j.Slf4j; -import org.jooq.DSLContext; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; +import org.springframework.data.redis.core.RedisTemplate; import org.springframework.stereotype.Service; +import org.springframework.util.StopWatch; import java.util.List; @@ -39,41 +45,131 @@ @Service public class ReDispatchTimeoutTask { - private final DSLContext dslContext; + private static final String machineIp = IpUtils.getFirstMachineIP(); + private static final String REDIS_KEY_REDISPATCH_TASK_RUNNING_MACHINE = + "file-gateway:reDispatch-task-running-machine"; private final FileTaskDAO fileTaskDAO; private final ReDispatchService reDispatchService; + private final RedisTemplate redisTemplate; - @Value("${job.file-gateway.task.timeout.reDispatch.enabled:true}") - private final boolean enableTimeoutRedispatch = true; + @Value("${job.file-gateway.reDispatch.timeoutTask.enabled:true}") + private boolean reDispatchTimeoutTaskEnabled = true; + + @Value("${job.file-gateway.reDispatch.timeoutTask.timeoutSeconds:10}") + private int reDispatchTaskTimeoutSeconds = 10; @Autowired - public ReDispatchTimeoutTask(DSLContext dslContext, - FileTaskDAO fileTaskDAO, - ReDispatchService reDispatchService) { - this.dslContext = dslContext; + public ReDispatchTimeoutTask(FileTaskDAO fileTaskDAO, + ReDispatchService reDispatchService, + RedisTemplate redisTemplate) { this.fileTaskDAO = fileTaskDAO; this.reDispatchService = reDispatchService; + this.redisTemplate = redisTemplate; } public void run() { - if (!enableTimeoutRedispatch) { - log.info("Timeout task reDispatch not enabled, you can config it in configuration file by set job" + - ".file-gateway.task.timeout.reDispatch.enable=true"); + if (!reDispatchTimeoutTaskEnabled) { + log.info("reDispatch timeout task not enabled, you can config it in configuration file by set job" + + ".file-gateway.reDispatch.timeoutTask.enabled=true"); + return; + } + // 分布式唯一性保证 + HeartBeatRedisLockConfig config = HeartBeatRedisLockConfig.getDefault(); + config.setHeartBeatThreadName("reDispatchTaskRedisKeyHeartBeatThread"); + HeartBeatRedisLock lock = new HeartBeatRedisLock( + redisTemplate, + REDIS_KEY_REDISPATCH_TASK_RUNNING_MACHINE, + machineIp, + config + ); + LockResult lockResult = lock.lock(); + if (!lockResult.isLockGotten()) { + log.info( + "lock {} gotten by another machine: {}, return", + REDIS_KEY_REDISPATCH_TASK_RUNNING_MACHINE, + lockResult.getLockValue() + ); + return; + } + try { + reDispatchFileSourceTasks(); + } finally { + lockResult.tryToRelease(); } - // 找出未结束且长时间无响应的任务，10s无响应且未结束的任务就应当被重调度了 - long fileSourceTaskStatusExpireTimeMills = 10 * 1000L; + } + + private void reDispatchFileSourceTasks() { + StopWatch watch = new StopWatch("reDispatchFileSourceTasks"); + watch.start("listTimeoutFileSourceTaskIds"); + // 找出未结束且长时间无响应的任务，无响应且未结束的任务就应当被重调度了 + + long intervalStart = computeReDispatchIntervalStart(); + long intervalEnd = computeReDispatchIntervalEnd(); List timeoutFileSourceTaskIdList = fileTaskDAO.listTimeoutFileSourceTaskIds( - dslContext, - fileSourceTaskStatusExpireTimeMills, + intervalStart, + intervalEnd, TaskStatusEnum.getRunningStatusSet(), 0, -1 ); + watch.stop(); + if (timeoutFileSourceTaskIdList.isEmpty()) { + log.info("no fileSourceTask need to be reDispatch"); + return; + } + log.info( + "find {} fileSourceTask between [{},{}] to reDispatch: {}", + timeoutFileSourceTaskIdList.size(), + TimeUtil.formatTime(intervalStart), + TimeUtil.formatTime(intervalEnd), + timeoutFileSourceTaskIdList + ); + watch.start("reDispatch Tasks"); // 进行超时重调度 for (String fileSourceTaskId : timeoutFileSourceTaskIdList) { - log.info("reDispatch fileSourceTask by timeout:{}", fileSourceTaskId); boolean result = reDispatchService.reDispatchByGateway(fileSourceTaskId, 0L, 5000L); - log.info("result={}", result); + log.info( + "reDispatch fileSourceTask by timeout({}s):{}, result={}", + reDispatchTaskTimeoutSeconds, + fileSourceTaskId, + result + ); } + watch.stop(); + if (watch.getTotalTimeSeconds() > 10) { + log.warn( + "SLOW: reDispatched {} fileSourceTask, timeConsuming:{}", + timeoutFileSourceTaskIdList.size(), + watch.prettyPrint() + ); + } else { + log.info( + "reDispatched {} fileSourceTask, timeConsuming: {}s", + timeoutFileSourceTaskIdList.size(), + watch.getTotalTimeSeconds() + ); + } + } + + /** + * 计算重调度区间开始时间 + * + * @return 重调度区间开始时间(ms) + */ + private long computeReDispatchIntervalStart() { + // 只对最近半小时内的任务进行重调度 + long reDispatchStartIntervalMills = 30 * 60 * 1000L; + return System.currentTimeMillis() - reDispatchStartIntervalMills; + } + + /** + * 计算重调度区间结束时间 + * + * @return 重调度区间结束时间(ms) + */ + private long computeReDispatchIntervalEnd() { + // 对已经超时未更新状态的任务进行重调度 + long fileSourceTaskStatusExpireTimeMills = reDispatchTaskTimeoutSeconds * 1000L; + return System.currentTimeMillis() - fileSourceTaskStatusExpireTimeMills; } } diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/task/filesource/FileSourceStatusUpdateTask.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/task/filesource/FileSourceStatusUpdateTask.java index 3a3f879659..a971042214 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/task/filesource/FileSourceStatusUpdateTask.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/task/filesource/FileSourceStatusUpdateTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,9 +27,9 @@ import com.tencent.bk.job.file_gateway.consts.FileSourceStatusEnum; import com.tencent.bk.job.file_gateway.model.dto.FileSourceDTO; import com.tencent.bk.job.file_gateway.model.dto.FileWorkerDTO; -import com.tencent.bk.job.file_gateway.service.DispatchService; import com.tencent.bk.job.file_gateway.service.FileService; import com.tencent.bk.job.file_gateway.service.FileSourceService; +import com.tencent.bk.job.file_gateway.service.dispatch.DispatchService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -65,7 +65,9 @@ public void run() { pageSize ); for (FileSourceDTO fileSourceDTO : fileSourceDTOList) { - FileWorkerDTO fileWorkerDTO = dispatchService.findBestFileWorker(fileSourceDTO); + FileWorkerDTO fileWorkerDTO = dispatchService.findBestFileWorker( + fileSourceDTO, "FileSourceStatusUpdateTask" + ); int status; if (fileWorkerDTO == null) { log.info( diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/task/worker/WorkerOnlineStatusUpdateTask.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/task/worker/WorkerOnlineStatusUpdateTask.java index d77862f015..ad541f10f5 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/task/worker/WorkerOnlineStatusUpdateTask.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/task/worker/WorkerOnlineStatusUpdateTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/util/JooqTypeUtil.java b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/util/JooqTypeUtil.java index 5ea4b2e578..a51e25fbeb 100644 --- a/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/util/JooqTypeUtil.java +++ b/src/backend/job-file-gateway/service-job-file-gateway/src/main/java/com/tencent/bk/job/file_gateway/util/JooqTypeUtil.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/build.gradle b/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/build.gradle index 0bad34da0e..3e0556c860 100644 --- a/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/build.gradle +++ b/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,7 @@ dependencies { api project(':commons:common') + api project(":commons:common-api") api project(':job-file-gateway:api-job-file-gateway-worker') implementation "org.springframework:spring-web" implementation "javax.ws.rs:javax.ws.rs-api" diff --git a/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/FileResource.java b/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/FileResource.java index 6b42f0ad77..02f05f8424 100644 --- a/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/FileResource.java +++ b/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/FileResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,7 @@ package com.tencent.bk.job.file.worker.api; -import com.tencent.bk.job.common.annotation.InternalAPI; +import com.tencent.bk.job.common.annotation.WorkerAPI; import com.tencent.bk.job.common.model.InternalResponse; import com.tencent.bk.job.file.worker.model.req.BaseReq; import com.tencent.bk.job.file.worker.model.req.ExecuteActionReq; @@ -42,7 +42,7 @@ @Api(tags = {"job-file-worker:api:File"}) @RequestMapping("/worker/api/file") @RestController -@InternalAPI +@WorkerAPI public interface FileResource { @ApiOperation(value = "测试文件源是否可用", produces = "application/json") diff --git a/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/FileTaskResource.java b/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/FileTaskResource.java index fcec53d992..de509d2d7f 100644 --- a/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/FileTaskResource.java +++ b/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/FileTaskResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,7 @@ package com.tencent.bk.job.file.worker.api; -import com.tencent.bk.job.common.annotation.InternalAPI; +import com.tencent.bk.job.common.annotation.WorkerAPI; import com.tencent.bk.job.common.model.Response; import com.tencent.bk.job.file.worker.model.req.ClearTaskFilesReq; import com.tencent.bk.job.file.worker.model.req.DownloadFilesTaskReq; @@ -40,7 +40,7 @@ @Api(tags = {"job-file-worker:api:FileTask"}) @RequestMapping("/worker/api/filetask") @RestController -@InternalAPI +@WorkerAPI public interface FileTaskResource { // 子路径与gateway转发请求子路径保持一致 diff --git a/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/OpResource.java b/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/OpResource.java index 6fbe32e996..b116d82f48 100644 --- a/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/OpResource.java +++ b/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/OpResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,7 @@ package com.tencent.bk.job.file.worker.api; -import com.tencent.bk.job.common.annotation.InternalAPI; +import com.tencent.bk.job.common.annotation.WorkerAPI; import com.tencent.bk.job.common.model.Response; import com.tencent.bk.job.file.worker.model.req.WorkerOffLineReq; import io.swagger.annotations.Api; @@ -41,7 +41,7 @@ @Api(tags = {"job-file-worker:api:OP"}) @RequestMapping("/worker/api/op") @RestController -@InternalAPI +@WorkerAPI public interface OpResource { @ApiOperation(value = "Worker下线", produces = "application/json") diff --git a/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/model/FileMetaData.java b/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/model/FileMetaData.java index a29be68b97..4583ad0788 100644 --- a/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/model/FileMetaData.java +++ b/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/model/FileMetaData.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/model/req/BaseReq.java b/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/model/req/BaseReq.java index 9893a362b5..892d5f5e02 100644 --- a/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/model/req/BaseReq.java +++ b/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/model/req/BaseReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,8 +24,8 @@ package com.tencent.bk.job.file.worker.model.req; -import com.tencent.bk.job.common.util.json.SkipLogFields; import com.tencent.bk.job.common.model.dto.CommonCredential; +import com.tencent.bk.job.common.util.json.SkipLogFields; import lombok.Data; import lombok.EqualsAndHashCode; @@ -38,7 +38,7 @@ public class BaseReq { // 文件源类型编码 String fileSourceTypeCode; - // 凭据信息 + // 凭证信息 @SkipLogFields CommonCredential credential; diff --git a/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/model/req/ClearTaskFilesReq.java b/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/model/req/ClearTaskFilesReq.java index 747b0fad51..6542622970 100644 --- a/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/model/req/ClearTaskFilesReq.java +++ b/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/model/req/ClearTaskFilesReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/model/req/DeleteBucketFileReq.java b/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/model/req/DeleteBucketFileReq.java index 68557fbb90..b9b80f0e3a 100644 --- a/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/model/req/DeleteBucketFileReq.java +++ b/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/model/req/DeleteBucketFileReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/model/req/DownloadFilesTaskReq.java b/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/model/req/DownloadFilesTaskReq.java index 055ada4803..94cb882186 100644 --- a/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/model/req/DownloadFilesTaskReq.java +++ b/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/model/req/DownloadFilesTaskReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/model/req/ExecuteActionReq.java b/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/model/req/ExecuteActionReq.java index e32780d792..f75917f3ab 100644 --- a/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/model/req/ExecuteActionReq.java +++ b/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/model/req/ExecuteActionReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/model/req/ListFileNodeReq.java b/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/model/req/ListFileNodeReq.java index 3257cce07d..91e9a851a7 100644 --- a/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/model/req/ListFileNodeReq.java +++ b/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/model/req/ListFileNodeReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/model/req/StopTasksReq.java b/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/model/req/StopTasksReq.java index d865e79487..e2d52f611e 100644 --- a/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/model/req/StopTasksReq.java +++ b/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/model/req/StopTasksReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/model/req/WorkerOffLineReq.java b/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/model/req/WorkerOffLineReq.java index 1d11361e1d..b5fcb50c52 100644 --- a/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/model/req/WorkerOffLineReq.java +++ b/src/backend/job-file-worker-sdk/api-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/model/req/WorkerOffLineReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-worker-sdk/build.gradle b/src/backend/job-file-worker-sdk/build.gradle index 0214002ba8..d2f4a3833d 100644 --- a/src/backend/job-file-worker-sdk/build.gradle +++ b/src/backend/job-file-worker-sdk/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -41,8 +41,6 @@ subprojects { implementation 'org.springframework.boot:spring-boot-starter-actuator' implementation 'ch.qos.logback:logback-core' implementation 'ch.qos.logback:logback-classic' - implementation 'io.springfox:springfox-swagger2' - implementation 'io.springfox:springfox-swagger-ui' implementation "org.hibernate.validator:hibernate-validator" testImplementation("org.springframework.boot:spring-boot-starter-test") testImplementation 'org.junit.jupiter:junit-jupiter' diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/build.gradle b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/build.gradle index 2200207fae..24ca8f2e6a 100644 --- a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/build.gradle +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -23,8 +23,11 @@ */ dependencies { + api project(":commons:common-log") + api project(":commons:common-task") api project(":commons:common-i18n") api project(":commons:common-otel") + api project(":commons:common-security") api project(":job-file-worker-sdk:api-job-file-worker-sdk") api "org.springframework.boot:spring-boot-starter-web" api "ch.qos.logback:logback-core" @@ -35,8 +38,7 @@ dependencies { api group: 'org.apache.thrift', name: 'libthrift' api "commons-io:commons-io" api "commons-codec:commons-codec" - api 'io.springfox:springfox-swagger2' - api 'io.springfox:springfox-swagger-ui' + api 'io.springfox:springfox-boot-starter' api 'net.coobird:thumbnailator:0.4.14' testImplementation 'org.springframework.boot:spring-boot-starter-test' testImplementation 'org.apache.commons:commons-lang3' diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/FileResourceImpl.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/FileResourceImpl.java index c857281e8f..7308ba92f8 100644 --- a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/FileResourceImpl.java +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/FileResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/FileTaskResourceImpl.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/FileTaskResourceImpl.java index 29f0258c7d..b174ebb6d9 100644 --- a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/FileTaskResourceImpl.java +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/FileTaskResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,13 +26,13 @@ import com.tencent.bk.job.common.model.Response; import com.tencent.bk.job.common.util.json.JsonUtils; -import com.tencent.bk.job.file.worker.cos.service.FileTaskService; -import com.tencent.bk.job.file.worker.cos.service.RemoteClient; -import com.tencent.bk.job.file.worker.cos.service.ThreadCommandBus; import com.tencent.bk.job.file.worker.model.req.BaseReq; import com.tencent.bk.job.file.worker.model.req.ClearTaskFilesReq; import com.tencent.bk.job.file.worker.model.req.DownloadFilesTaskReq; import com.tencent.bk.job.file.worker.model.req.StopTasksReq; +import com.tencent.bk.job.file.worker.service.FileTaskService; +import com.tencent.bk.job.file.worker.service.RemoteClient; +import com.tencent.bk.job.file.worker.service.ThreadCommandBus; import com.tencent.bk.job.file_gateway.consts.TaskCommandEnum; import lombok.extern.slf4j.Slf4j; @@ -71,8 +71,7 @@ public Response stopTasks(StopTasksReq req) { @Override public Response clearFiles(ClearTaskFilesReq req) { List taskIdList = req.getTaskIdList(); - Integer count = 0; - count = fileTaskService.clearTaskFilesAtOnce(taskIdList); + Integer count = fileTaskService.clearTaskFilesAtOnce(taskIdList); return Response.buildSuccessResp(count); } } diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/IFileResource.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/IFileResource.java index dbc6831e53..90161420dc 100644 --- a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/IFileResource.java +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/IFileResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,10 +25,10 @@ package com.tencent.bk.job.file.worker.api; import com.tencent.bk.job.common.model.InternalResponse; -import com.tencent.bk.job.file.worker.cos.service.RemoteClient; import com.tencent.bk.job.file.worker.model.req.BaseReq; import com.tencent.bk.job.file.worker.model.req.ExecuteActionReq; import com.tencent.bk.job.file.worker.model.req.ListFileNodeReq; +import com.tencent.bk.job.file.worker.service.RemoteClient; import com.tencent.bk.job.file_gateway.model.resp.common.FileNodesDTO; public interface IFileResource { diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/OpResourceImpl.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/OpResourceImpl.java index 9d6f77aac0..89d68da7da 100644 --- a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/OpResourceImpl.java +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/OpResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,8 +25,8 @@ package com.tencent.bk.job.file.worker.api; import com.tencent.bk.job.common.model.Response; -import com.tencent.bk.job.file.worker.cos.service.OpService; import com.tencent.bk.job.file.worker.model.req.WorkerOffLineReq; +import com.tencent.bk.job.file.worker.service.OpService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RestController; @@ -34,7 +34,7 @@ import java.util.List; @Slf4j -@RestController +@RestController("jobFileWorkerOpResourceImpl") public class OpResourceImpl implements OpResource { private final OpService opService; diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/RemoteClientAccess.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/RemoteClientAccess.java index dd6d9b7390..caba75b8d6 100644 --- a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/RemoteClientAccess.java +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/api/RemoteClientAccess.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,8 +24,8 @@ package com.tencent.bk.job.file.worker.api; -import com.tencent.bk.job.file.worker.cos.service.RemoteClient; import com.tencent.bk.job.file.worker.model.req.BaseReq; +import com.tencent.bk.job.file.worker.service.RemoteClient; public interface RemoteClientAccess { RemoteClient getRemoteClient(BaseReq req); diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/ApplicationReadyListener.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/ApplicationReadyListener.java index 0e66134715..6678c81e70 100644 --- a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/ApplicationReadyListener.java +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/ApplicationReadyListener.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,8 @@ package com.tencent.bk.job.file.worker.config; -import com.tencent.bk.job.file.worker.task.heartbeat.HeartBeatTask; +import com.tencent.bk.job.file.worker.state.event.WorkerEvent; +import com.tencent.bk.job.file.worker.state.event.WorkerEventService; import lombok.extern.slf4j.Slf4j; import org.springframework.boot.context.event.ApplicationReadyEvent; import org.springframework.context.ApplicationListener; @@ -36,12 +37,12 @@ public class ApplicationReadyListener implements ApplicationListener { private final WorkerConfig workerConfig; - private final HeartBeatTask heartBeatTask; + private final WorkerEventService workerEventService; public ApplicationReadyListener(WorkerConfig workerConfig, - HeartBeatTask heartBeatTask) { + WorkerEventService workerEventService) { this.workerConfig = workerConfig; - this.heartBeatTask = heartBeatTask; + this.workerEventService = workerEventService; } @SuppressWarnings("NullableProblems") @@ -56,7 +57,7 @@ public void onApplicationEvent(ApplicationReadyEvent event) { log.info("created JobFileWorker workspace:" + wsDirFile.getAbsolutePath()); } } - // 2.启动后立即上报一次心跳 - new Thread(heartBeatTask::run).start(); + // 2.启动后等待自身可被外界访问 + workerEventService.commitWorkerEvent(WorkerEvent.waitAccessReady()); } } diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/ApplicationReadyListenerConfig.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/ApplicationReadyListenerConfig.java index 3a6865e79d..ad985033b4 100644 --- a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/ApplicationReadyListenerConfig.java +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/ApplicationReadyListenerConfig.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,17 +24,12 @@ package com.tencent.bk.job.file.worker.config; -import com.tencent.bk.job.file.worker.task.heartbeat.HeartBeatTask; +import com.tencent.bk.job.file.worker.state.event.WorkerEventService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; -/** - * @Description - * @Date 2020/2/24 - * @Version 1.0 - */ @Slf4j @Configuration @@ -42,9 +37,9 @@ public class ApplicationReadyListenerConfig { @Bean public ApplicationReadyListener applicationReadyListener(@Autowired WorkerConfig workerConfig, - @Autowired HeartBeatTask heartBeatTask) { + @Autowired WorkerEventService workerEventService) { log.info("applicationReadyListener inited"); - return new ApplicationReadyListener(workerConfig, heartBeatTask); + return new ApplicationReadyListener(workerConfig, workerEventService); } } diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/ExecutorConfiguration.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/ExecutorConfiguration.java index 0d763d5b42..042f9c24d7 100644 --- a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/ExecutorConfiguration.java +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/ExecutorConfiguration.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,8 @@ package com.tencent.bk.job.file.worker.config; +import com.tencent.bk.job.common.WatchableThreadPoolExecutor; +import io.micrometer.core.instrument.MeterRegistry; import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -33,30 +35,34 @@ import java.util.concurrent.TimeUnit; @Slf4j -@Configuration +@Configuration(value = "jobFileWorkerExecutorConfig") public class ExecutorConfiguration { @Bean("fileTaskExecutor") - public ThreadPoolExecutor fileTaskExecutor() { - return new ThreadPoolExecutor( - 20, - 50, + public ThreadPoolExecutor fileTaskExecutor(MeterRegistry meterRegistry) { + return new WatchableThreadPoolExecutor( + meterRegistry, + "fileTaskExecutor", + 100, + 100, 1, TimeUnit.MINUTES, - new LinkedBlockingQueue<>(1000), (r, executor) -> { + new LinkedBlockingQueue<>(10000), (r, executor) -> { log.error("fileTaskWorkerPool rejected a task, use current thread now, plz add more threads"); r.run(); }); } @Bean("watchingTaskExecutor") - public ThreadPoolExecutor watchingTaskExecutor() { - return new ThreadPoolExecutor( - 20, - 50, + public ThreadPoolExecutor watchingTaskExecutor(MeterRegistry meterRegistry) { + return new WatchableThreadPoolExecutor( + meterRegistry, + "watchingTaskExecutor", + 100, + 100, 1, TimeUnit.MINUTES - , new LinkedBlockingQueue<>(1000), (r, executor) -> + , new LinkedBlockingQueue<>(10000), (r, executor) -> log.error("watchingTaskExecutor rejected a task, ignore, plz add more threads") ); } diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/GracefulShutdown.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/GracefulShutdown.java index 23a92cb4f2..4894d09ed3 100644 --- a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/GracefulShutdown.java +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/GracefulShutdown.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,9 @@ package com.tencent.bk.job.file.worker.config; -import com.tencent.bk.job.file.worker.cos.service.OpService; +import com.tencent.bk.job.common.util.ThreadUtils; +import com.tencent.bk.job.file.worker.service.OpService; +import com.tencent.bk.job.file.worker.state.WorkerStateMachine; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; @@ -39,13 +41,15 @@ public class GracefulShutdown implements ApplicationListener { private final OpService opService; + private final WorkerStateMachine workerStateMachine; @Value("${app.shutdownTimeout:30}") int shutdownTimeout = 30; @Autowired - public GracefulShutdown(OpService opService) { + public GracefulShutdown(OpService opService, WorkerStateMachine workerStateMachine) { this.opService = opService; + this.workerStateMachine = workerStateMachine; } @Override @@ -54,6 +58,16 @@ public void onApplicationEvent(ContextClosedEvent event) { List runningTaskIdList = opService.offLine(); log.info("worker apply to offLine, {} tasks to be reDispatched are {}", runningTaskIdList.size(), runningTaskIdList); + long waitStart = System.currentTimeMillis(); + long maxWaitMills = 5000; + long waitMills; + do { + // 1.等待Worker主动下线完成 + ThreadUtils.sleep(100); + waitMills = System.currentTimeMillis() - waitStart; + } while (!workerStateMachine.isWorkerOffLineIncludeFail() && waitMills < maxWaitMills); + // 2.等待File-Gateway内存中存量已调度请求完成 + ThreadUtils.sleep(3000); + log.info("Worker offLine done"); } - } diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/InterceptorConfig.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/InterceptorConfig.java new file mode 100644 index 0000000000..339caf3936 --- /dev/null +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/InterceptorConfig.java @@ -0,0 +1,54 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file.worker.config; + +import com.tencent.bk.job.common.jwt.JwtManager; +import com.tencent.bk.job.common.security.annotation.ConditionalOnSecurityEnabled; +import com.tencent.bk.job.file.worker.interceptor.FileWorkerSecurityInterceptor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + + +/** + * 拦截器配置 + */ +@Slf4j +@Configuration(proxyBeanMethods = false) +public class InterceptorConfig { + + @Bean + @ConditionalOnSecurityEnabled + public FileWorkerSecurityInterceptor fileWorkerSecurityInterceptor(JwtManager jwtManager) { + log.info("fileWorkerSecurityInterceptor inited"); + return new FileWorkerSecurityInterceptor(jwtManager); + } + + @Bean + @ConditionalOnSecurityEnabled + InterceptorConfigurer webInterceptorAutoRegister(FileWorkerSecurityInterceptor interceptor) { + return new InterceptorConfigurer(interceptor); + } +} diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/InterceptorConfigurer.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/InterceptorConfigurer.java new file mode 100644 index 0000000000..70596e2c63 --- /dev/null +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/InterceptorConfigurer.java @@ -0,0 +1,59 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file.worker.config; + +import com.tencent.bk.job.common.annotation.JobInterceptor; +import com.tencent.bk.job.file.worker.interceptor.FileWorkerSecurityInterceptor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.web.servlet.config.annotation.InterceptorRegistry; +import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; + + +/** + * 拦截器注册 + */ +@Slf4j +public class InterceptorConfigurer implements WebMvcConfigurer { + + private final FileWorkerSecurityInterceptor interceptor; + + public InterceptorConfigurer(FileWorkerSecurityInterceptor interceptor) { + this.interceptor = interceptor; + } + + @Override + public void addInterceptors(InterceptorRegistry registry) { + JobInterceptor jobInterceptor = interceptor.getClass().getAnnotation(JobInterceptor.class); + if (jobInterceptor != null) { + log.info("Add job interceptor: {}, pathPatterns: {}, order: {}", + interceptor.getClass().getName(), + jobInterceptor.pathPatterns(), + jobInterceptor.order()); + registry.addInterceptor(interceptor) + .addPathPatterns(jobInterceptor.pathPatterns()) + .order(jobInterceptor.order()); + } + } +} diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/JobFileWorkerAutoConfiguration.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/JobFileWorkerAutoConfiguration.java index a494c5e90b..2b5690f4fc 100644 --- a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/JobFileWorkerAutoConfiguration.java +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/JobFileWorkerAutoConfiguration.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/RestTemplateConfig.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/RestTemplateConfig.java index 7e49b1a2aa..ee6b8d8345 100644 --- a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/RestTemplateConfig.java +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/RestTemplateConfig.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,12 +26,19 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; +import org.springframework.http.client.OkHttp3ClientHttpRequestFactory; import org.springframework.web.client.RestTemplate; @Configuration public class RestTemplateConfig { @Bean public RestTemplate restTemplate() { - return new RestTemplate(); + RestTemplate restTemplate = new RestTemplate(); + OkHttp3ClientHttpRequestFactory requestFactory = new OkHttp3ClientHttpRequestFactory(); + requestFactory.setConnectTimeout(5000); + requestFactory.setReadTimeout(15000); + requestFactory.setWriteTimeout(15000); + restTemplate.setRequestFactory(requestFactory); + return restTemplate; } } diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/ScheduleConfig.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/ScheduleConfig.java index 596a24f4e6..d1ed732c66 100644 --- a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/ScheduleConfig.java +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/ScheduleConfig.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/SwaggerAdapterConfig.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/SwaggerAdapterConfig.java new file mode 100644 index 0000000000..d95ad89c7f --- /dev/null +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/SwaggerAdapterConfig.java @@ -0,0 +1,85 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file.worker.config; + +import org.springframework.boot.actuate.autoconfigure.endpoint.web.CorsEndpointProperties; +import org.springframework.boot.actuate.autoconfigure.endpoint.web.WebEndpointProperties; +import org.springframework.boot.actuate.autoconfigure.web.server.ManagementPortType; +import org.springframework.boot.actuate.endpoint.ExposableEndpoint; +import org.springframework.boot.actuate.endpoint.web.EndpointLinksResolver; +import org.springframework.boot.actuate.endpoint.web.EndpointMapping; +import org.springframework.boot.actuate.endpoint.web.EndpointMediaTypes; +import org.springframework.boot.actuate.endpoint.web.ExposableWebEndpoint; +import org.springframework.boot.actuate.endpoint.web.WebEndpointsSupplier; +import org.springframework.boot.actuate.endpoint.web.annotation.ControllerEndpointsSupplier; +import org.springframework.boot.actuate.endpoint.web.annotation.ServletEndpointsSupplier; +import org.springframework.boot.actuate.endpoint.web.servlet.WebMvcEndpointHandlerMapping; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.env.Environment; +import org.springframework.util.StringUtils; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +@Configuration +public class SwaggerAdapterConfig { + + /** + * 增加如下配置可解决Spring Boot 6.x 与Swagger 不兼容问题 + * + *

参考官方解决方案: https://github.com/spring-projects/spring-boot/wiki/Spring-Boot-2 + * .6-Release-Notes#pathpattern-based-path-matching-strategy-for-spring-mvc

+ **/ + @Bean + public WebMvcEndpointHandlerMapping webEndpointServletHandlerMapping(WebEndpointsSupplier webEndpointsSupplier, + ServletEndpointsSupplier servletEndpointsSupplier, + ControllerEndpointsSupplier controllerEndpointsSupplier, + EndpointMediaTypes endpointMediaTypes, + CorsEndpointProperties corsProperties, + WebEndpointProperties webEndpointProperties, + Environment environment) { + List> allEndpoints = new ArrayList<>(); + Collection webEndpoints = webEndpointsSupplier.getEndpoints(); + allEndpoints.addAll(webEndpoints); + allEndpoints.addAll(servletEndpointsSupplier.getEndpoints()); + allEndpoints.addAll(controllerEndpointsSupplier.getEndpoints()); + String basePath = webEndpointProperties.getBasePath(); + EndpointMapping endpointMapping = new EndpointMapping(basePath); + boolean shouldRegisterLinksMapping = this.shouldRegisterLinksMapping(webEndpointProperties, environment, + basePath); + return new WebMvcEndpointHandlerMapping(endpointMapping, webEndpoints, endpointMediaTypes, + corsProperties.toCorsConfiguration(), new EndpointLinksResolver(allEndpoints, basePath), + shouldRegisterLinksMapping, null); + } + + private boolean shouldRegisterLinksMapping(WebEndpointProperties webEndpointProperties, + Environment environment, + String basePath) { + return webEndpointProperties.getDiscovery().isEnabled() + && (StringUtils.hasText(basePath) || ManagementPortType.get(environment).equals(ManagementPortType.DIFFERENT)); + } +} diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/SwaggerConfig.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/SwaggerConfig.java index f0dcf4bf3a..4be23a3cde 100644 --- a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/SwaggerConfig.java +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/SwaggerConfig.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,35 +24,40 @@ package com.tencent.bk.job.file.worker.config; +import com.tencent.bk.job.common.service.config.JobCommonConfig; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Profile; import springfox.documentation.builders.PathSelectors; import springfox.documentation.builders.RequestHandlerSelectors; +import springfox.documentation.oas.annotations.EnableOpenApi; import springfox.documentation.spi.DocumentationType; import springfox.documentation.spring.web.plugins.Docket; -import springfox.documentation.swagger2.annotations.EnableSwagger2; import java.util.Arrays; import java.util.HashSet; +/** + * Swagger 配置 + */ @Configuration -@EnableSwagger2 -@Profile({"dev", "test", "local"}) +@EnableOpenApi +@Profile({"dev", "local"}) public class SwaggerConfig { - private final WorkerConfig workerConfig; + private final JobCommonConfig jobCommonConfig; @Autowired - public SwaggerConfig(WorkerConfig workerConfig) { - this.workerConfig = workerConfig; + public SwaggerConfig(JobCommonConfig jobCommonConfig) { + this.jobCommonConfig = jobCommonConfig; } @Bean public Docket api() { - return new Docket(DocumentationType.SWAGGER_2) - .host(workerConfig.getSwaggerUrl() + "/job-file-worker") + return new Docket(DocumentationType.OAS_30) + .host(jobCommonConfig.getSwaggerUrl()) + .pathMapping("job-file-worker") .protocols(new HashSet<>(Arrays.asList("http", "https"))) .select() .apis(RequestHandlerSelectors.basePackage("com.tencent.bk.job.file.worker.api")) diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/WorkerConfig.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/WorkerConfig.java index fcd322417d..e521202a78 100644 --- a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/WorkerConfig.java +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/config/WorkerConfig.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -36,9 +36,6 @@ @Component public class WorkerConfig { - @Value("${swagger.url:swagger.job.com}") - private String swaggerUrl; - @Value("${job.file-worker.version:0.0.1}") private String version; @@ -54,6 +51,9 @@ public class WorkerConfig { @Value("${job.file-worker.app-id:-1}") private Long appId; + @Value("${job.file-worker.enabled-file-source-types:}") + private String enabledFileSourceTypeStr; + @Value("${job.file-worker.ability-tags:}") private String abilityTagStr; @@ -66,6 +66,9 @@ public class WorkerConfig { @Value("${job.file-worker.cloud-area-id:0}") private Long cloudAreaId; + @Value("${job.file-worker.inner-ip-protocol:}") + private String innerIpProtocol; + @Value("${job.file-worker.inner-ip:}") private String innerIp; diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/EnvironmentService.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/EnvironmentService.java deleted file mode 100644 index e0779d272b..0000000000 --- a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/EnvironmentService.java +++ /dev/null @@ -1,85 +0,0 @@ -package com.tencent.bk.job.file.worker.cos.service; - -import com.tencent.bk.job.common.constant.JobConstants; -import com.tencent.bk.job.common.util.ip.IpUtils; -import com.tencent.bk.job.file.worker.config.WorkerConfig; -import io.micrometer.core.instrument.util.StringUtils; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.BeansException; -import org.springframework.context.ApplicationContext; -import org.springframework.context.ApplicationContextAware; -import org.springframework.stereotype.Service; - -@Slf4j -@Service -public class EnvironmentService implements ApplicationContextAware { - - private ApplicationContext applicationContext; - private final WorkerConfig workerConfig; - - public EnvironmentService(WorkerConfig workerConfig) { - this.workerConfig = workerConfig; - } - - @Override - public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { - this.applicationContext = applicationContext; - } - - public boolean isInK8s() { - String[] profileArr = applicationContext.getEnvironment().getActiveProfiles(); - for (String profile : profileArr) { - if (JobConstants.PROFILE_KUBERNETES.equals(profile)) { - return true; - } - } - return false; - } - - private String getAccessHostInK8s() { - String podName = System.getenv("BK_JOB_POD_NAME"); - String fileWorkerServiceName = System.getenv("BK_JOB_FILE_WORKER_SERVICE_NAME"); - String accessHost = podName + "." + fileWorkerServiceName; - log.debug("accessHost={}", accessHost); - return accessHost; - } - - private String getInnerIpInK8s() { - String nodeIP = System.getenv("BK_JOB_NODE_IP"); - String podIp = IpUtils.getFirstMachineIP(); - log.info("nodeIP={}", nodeIP); - if (!StringUtils.isBlank(nodeIP)) { - return nodeIP; - } else { - return podIp; - } - } - - public String getAccessHost() { - String accessHost; - if (isInK8s()) { - accessHost = getAccessHostInK8s(); - } else { - accessHost = workerConfig.getAccessHost(); - } - if (StringUtils.isBlank(accessHost)) { - accessHost = IpUtils.getFirstMachineIP(); - log.debug("accessHost is blank, use first machine ip"); - } - return accessHost; - } - - public String getInnerIp() { - String innerIp; - if (isInK8s()) { - innerIp = getInnerIpInK8s(); - } else { - innerIp = workerConfig.getInnerIp(); - } - if (StringUtils.isBlank(innerIp)) { - innerIp = IpUtils.getFirstMachineIP(); - log.debug("innerIp is blank, use first machine ip"); - } - return innerIp; - } -} diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/FileProgressWatchingTask.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/FileProgressWatchingTask.java deleted file mode 100644 index 98a5b6792c..0000000000 --- a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/FileProgressWatchingTask.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.file.worker.cos.service; - -import com.tencent.bk.job.common.util.file.PathUtil; -import lombok.extern.slf4j.Slf4j; - -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; - -@Slf4j -class FileProgressWatchingTask extends Thread { - - String taskId; - String filePath; - String downloadFileDir; - AtomicLong fileSize; - AtomicInteger speed; - AtomicInteger process; - TaskReporter taskReporter; - FileProgressWatchingTaskEventListener watchingTaskEventListener; - volatile boolean runFlag = true; - - public FileProgressWatchingTask(String taskId, String filePath, String downloadFileDir, AtomicLong fileSize, - AtomicInteger speed, AtomicInteger process, TaskReporter taskReporter, - FileProgressWatchingTaskEventListener watchingTaskEventListener) { - this.taskId = taskId; - this.filePath = filePath; - this.downloadFileDir = downloadFileDir; - this.fileSize = fileSize; - this.speed = speed; - this.process = process; - this.taskReporter = taskReporter; - this.watchingTaskEventListener = watchingTaskEventListener; - } - - public void stopWatching() { - this.runFlag = false; - } - - @Override - public void run() { - String fileTaskKey = taskId + "_" + filePath; - try { - String downloadPath = PathUtil.joinFilePath(downloadFileDir, taskId + "/" + filePath); - while (runFlag) { - try { - taskReporter.reportFileDownloadProgress(taskId, filePath, downloadPath, fileSize.get(), - speed.get(), process.get()); - } catch (Throwable t) { - log.error("Fail to reportFileDownloadProgress of file:{}", filePath, t); - } - if (runFlag) { - sleep(1000); - } - } - } catch (InterruptedException e) { - log.info("watching interrupted", e); - } finally { - if (watchingTaskEventListener != null) { - watchingTaskEventListener.onWatchingTaskFinally(fileTaskKey); - } - } - log.debug("end watching:{}", fileTaskKey); - } - - public interface FileProgressWatchingTaskEventListener { - void onWatchingTaskFinally(String fileTaskKey); - } -} diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/FileTaskService.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/FileTaskService.java deleted file mode 100644 index 91a5bd7031..0000000000 --- a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/FileTaskService.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.file.worker.cos.service; - -import com.tencent.bk.job.common.util.file.PathUtil; -import com.tencent.bk.job.file.worker.config.WorkerConfig; -import lombok.extern.slf4j.Slf4j; -import org.apache.tomcat.util.http.fileupload.FileUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.stereotype.Service; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Enumeration; -import java.util.List; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Future; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; - -@Slf4j -@Service -public class FileTaskService { - - private final ThreadPoolExecutor fileTaskExecutor; - private final ThreadPoolExecutor watchingTaskExecutor; - private static final ConcurrentHashMap> fileTaskMap = new ConcurrentHashMap<>(); - private static final ConcurrentHashMap> watchingTaskMap = new ConcurrentHashMap<>(); - private final WorkerConfig workerConfig; - private final TaskReporter taskReporter; - - @Autowired - public FileTaskService(@Qualifier("fileTaskExecutor") ThreadPoolExecutor fileTaskExecutor, - @Qualifier("watchingTaskExecutor") ThreadPoolExecutor watchingTaskExecutor, - WorkerConfig workerConfig, - TaskReporter taskReporter) { - this.fileTaskExecutor = fileTaskExecutor; - this.watchingTaskExecutor = watchingTaskExecutor; - this.workerConfig = workerConfig; - this.taskReporter = taskReporter; - } - - public List getAllTaskIdList() { - List runningTaskIdList = new ArrayList<>(); - Enumeration keys = fileTaskMap.keys(); - while (keys.hasMoreElements()) { - runningTaskIdList.add(keys.nextElement().split("_")[0]); - } - return runningTaskIdList; - } - - public Integer downloadFiles(RemoteClient client, String taskId, List filePathList, String filePrefix) { - for (String filePath : filePathList) { - String fileTaskKey = taskId + "_" + filePath; - AtomicLong fileSize = new AtomicLong(0L); - AtomicInteger speed = new AtomicInteger(0); - AtomicInteger process = new AtomicInteger(0); - FileProgressWatchingTask progressWatchingTask = new FileProgressWatchingTask(taskId, filePath, - workerConfig.getWorkspaceDirPath(), fileSize, speed, process, taskReporter, watchingTaskMap::remove); - DownloadFileTask downloadFileTask = new DownloadFileTask(client, taskId, filePath, - workerConfig.getWorkspaceDirPath(), filePrefix, fileSize, speed, process, progressWatchingTask, - taskReporter, tmpfileTaskKey -> { - fileTaskMap.remove(tmpfileTaskKey); - ThreadCommandBus.destroyCommandQueue(tmpfileTaskKey); - }); - Future fileTaskFuture = fileTaskExecutor.submit(downloadFileTask); - Future watchingTaskFuture = watchingTaskExecutor.submit(progressWatchingTask); - fileTaskMap.put(fileTaskKey, fileTaskFuture); - watchingTaskMap.put(fileTaskKey, watchingTaskFuture); - } - return filePathList.size(); - } - - public Integer clearTaskFilesAtOnce(List taskIdList) { - int count = 0; - for (String taskId : taskIdList) { - String deleteDirPath = taskId; - try { - deleteDirPath = PathUtil.joinFilePath(workerConfig.getWorkspaceDirPath(), taskId); - FileUtils.deleteDirectory(new File(deleteDirPath)); - count += 1; - } catch (IOException e) { - log.warn("Fail to delete dir:{}", deleteDirPath, e); - } - } - return count; - } - - public Integer stopTasksAtOnce(List taskIdList, ThreadCommandBus.Command command) { - int allStoppedFileCount = 0; - Enumeration keys = fileTaskMap.keys(); - List keyList = new ArrayList<>(); - while (keys.hasMoreElements()) { - String key = keys.nextElement(); - keyList.add(key); - } - for (String taskId : taskIdList) { - List stopKeyList = new ArrayList<>(); - for (String key : keyList) { - if (key.startsWith(taskId)) { - stopKeyList.add(key); - } - } - for (String stopKey : stopKeyList) { - log.info("try to stop {}", stopKey); - Future fileTaskFuture = fileTaskMap.get(stopKey); - if (fileTaskFuture != null && !fileTaskFuture.isDone()) { - ThreadCommandBus.sendCommand(stopKey, command); - fileTaskFuture.cancel(true); - allStoppedFileCount += 1; - } else { - log.info("task {} already done, stop too late", stopKey); - } - } - } - return allStoppedFileCount; - } -} diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/OpService.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/OpService.java deleted file mode 100644 index dcc1e414e6..0000000000 --- a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/OpService.java +++ /dev/null @@ -1,113 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.file.worker.cos.service; - -import com.tencent.bk.job.common.model.http.HttpReq; -import com.tencent.bk.job.common.util.http.ExtHttpHelper; -import com.tencent.bk.job.common.util.http.HttpHelperFactory; -import com.tencent.bk.job.common.util.http.HttpReqGenUtil; -import com.tencent.bk.job.common.util.json.JsonUtils; -import com.tencent.bk.job.file.worker.config.WorkerConfig; -import com.tencent.bk.job.file.worker.task.heartbeat.HeartBeatTask; -import com.tencent.bk.job.file_gateway.consts.TaskCommandEnum; -import com.tencent.bk.job.file_gateway.model.req.inner.OffLineAndReDispatchReq; -import lombok.extern.slf4j.Slf4j; -import org.slf4j.helpers.FormattingTuple; -import org.slf4j.helpers.MessageFormatter; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import java.util.List; - -@Slf4j -@Service -public class OpService { - - private final ExtHttpHelper httpHelper = HttpHelperFactory.getDefaultHttpHelper(); - private final WorkerConfig workerConfig; - private final FileTaskService fileTaskService; - private final GatewayInfoService gatewayInfoService; - private final EnvironmentService environmentService; - private final TaskReporter taskReporter; - - @Autowired - public OpService(WorkerConfig workerConfig, FileTaskService fileTaskService, - GatewayInfoService gatewayInfoService, EnvironmentService environmentService, - TaskReporter taskReporter) { - this.workerConfig = workerConfig; - this.fileTaskService = fileTaskService; - this.gatewayInfoService = gatewayInfoService; - this.environmentService = environmentService; - this.taskReporter = taskReporter; - } - - public List offLine() { - List runningTaskIdList = fileTaskService.getAllTaskIdList(); - // 停止心跳 - HeartBeatTask.stopHeartBeat(); - // 调网关接口下线自己 - String url = gatewayInfoService.getWorkerOffLineUrl(); - OffLineAndReDispatchReq offLineReq = new OffLineAndReDispatchReq(); - offLineReq.setAccessHost(environmentService.getAccessHost()); - offLineReq.setAccessPort(workerConfig.getAccessPort()); - offLineReq.setAppId(workerConfig.getAppId()); - offLineReq.setToken(workerConfig.getToken()); - offLineReq.setTaskIdList(runningTaskIdList); - offLineReq.setInitDelayMills(3000L); - offLineReq.setIntervalMills(3000L); - HttpReq req = HttpReqGenUtil.genSimpleJsonReq(url, offLineReq); - String respStr; - try { - log.info( - "url={},body={},headers={}", - url, - JsonUtils.toJsonWithoutSkippedFields(req.getBody()), - JsonUtils.toJson(req.getHeaders()) - ); - respStr = httpHelper.post(url, req.getBody(), req.getHeaders()); - log.info(String.format("respStr=%s", respStr)); - // 停止任务 - Integer allStoppedFileCount = fileTaskService.stopTasksAtOnce(runningTaskIdList, - new ThreadCommandBus.Command(TaskCommandEnum.STOP_QUIETLY, null)); - log.info("{} file tasks stopped", allStoppedFileCount); - taskReporter.reportWorkerOffLine(runningTaskIdList, "FileWorker offline"); - } catch (Exception e) { - FormattingTuple msg = MessageFormatter.arrayFormat( - "Fail to request file-gateway,url={},body={},headers={}", - new String[]{ - url, - JsonUtils.toJsonWithoutSkippedFields(req.getBody()), - JsonUtils.toJson(req.getHeaders()) - } - ); - log.error(msg.getMessage(), e); - } - return runningTaskIdList; - } - - public List taskList() { - return fileTaskService.getAllTaskIdList(); - } -} diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/TaskReporter.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/TaskReporter.java deleted file mode 100644 index ad82a43807..0000000000 --- a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/TaskReporter.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.file.worker.cos.service; - -import java.util.List; - -public interface TaskReporter { - - void reportFileDownloadStart(String taskId, String filePath, String downloadPath); - - void reportFileDownloadProgress(String taskId, String filePath, String downloadPath, Long fileSize, Integer speed - , Integer progress); - - void reportFileDownloadProgressWithContent(String taskId, String filePath, String downloadPath, Long fileSize, Integer speed - , Integer progress, String content); - - void reportFileDownloadSuccess(String taskId, String filePath, String downloadPath, Long fileSize, Integer speed, - Integer progress); - - void reportFileDownloadFailure(String taskId, String filePath, String downloadPath); - - void reportFileDownloadStopped(String taskId, String filePath, String downloadPath, Long fileSize, - Integer progress); - - void reportFileDownloadFailure(String taskId, String filePath, String downloadPath, String content); - - void reportWorkerOffLine(List taskIdList, String content); - -} diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/TaskReporterImpl.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/TaskReporterImpl.java deleted file mode 100644 index cc4f2e51cb..0000000000 --- a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/TaskReporterImpl.java +++ /dev/null @@ -1,156 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.file.worker.cos.service; - -import com.tencent.bk.job.common.model.http.HttpReq; -import com.tencent.bk.job.common.util.http.ExtHttpHelper; -import com.tencent.bk.job.common.util.http.HttpHelperFactory; -import com.tencent.bk.job.common.util.http.HttpReqGenUtil; -import com.tencent.bk.job.common.util.json.JsonUtils; -import com.tencent.bk.job.file_gateway.consts.TaskStatusEnum; -import com.tencent.bk.job.file_gateway.model.req.inner.UpdateFileSourceTaskReq; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import java.util.List; - -@Slf4j -@Service -public class TaskReporterImpl implements TaskReporter { - - private final ExtHttpHelper httpHelper = HttpHelperFactory.getDefaultHttpHelper(); - private final GatewayInfoService gatewayInfoService; - - @Autowired - public TaskReporterImpl(GatewayInfoService gatewayInfoService) { - this.gatewayInfoService = gatewayInfoService; - } - - public void reportFileDownloadStart(String taskId, String filePath, String downloadPath) { - UpdateFileSourceTaskReq req = new UpdateFileSourceTaskReq(); - req.setFileSourceTaskId(taskId); - req.setFilePath(filePath); - req.setDownloadPath(downloadPath); - req.setStatus(TaskStatusEnum.RUNNING); - req.setSpeed("0 KB/s"); - req.setContent("Start pulling"); - req.setProgress(0); - reportTaskStatus(req); - } - - public void reportFileDownloadProgress(String taskId, String filePath, String downloadPath, Long fileSize, - Integer speed, Integer progress) { - UpdateFileSourceTaskReq req = new UpdateFileSourceTaskReq(); - req.setFileSourceTaskId(taskId); - req.setFilePath(filePath); - req.setDownloadPath(downloadPath); - req.setStatus(TaskStatusEnum.RUNNING); - req.setFileSize(fileSize); - req.setSpeed(speed + " KB/s"); - req.setContent("Pulling OSS file..."); - req.setProgress(progress); - reportTaskStatus(req); - } - - public void reportFileDownloadProgressWithContent(String taskId, String filePath, String downloadPath, Long fileSize, - Integer speed, Integer progress, String content) { - UpdateFileSourceTaskReq req = new UpdateFileSourceTaskReq(); - req.setFileSourceTaskId(taskId); - req.setFilePath(filePath); - req.setDownloadPath(downloadPath); - req.setStatus(TaskStatusEnum.RUNNING); - req.setFileSize(fileSize); - req.setSpeed(speed + " KB/s"); - req.setContent(content); - req.setProgress(progress); - reportTaskStatus(req); - } - - public void reportFileDownloadSuccess(String taskId, String filePath, String downloadPath, Long fileSize, - Integer speed, Integer progress) { - UpdateFileSourceTaskReq req = new UpdateFileSourceTaskReq(); - req.setFileSourceTaskId(taskId); - req.setFilePath(filePath); - req.setDownloadPath(downloadPath); - req.setStatus(TaskStatusEnum.SUCCESS); - req.setFileSize(fileSize); - req.setSpeed(speed + " KB/s"); - req.setContent("Pulling finished"); - req.setProgress(progress); - reportTaskStatus(req); - } - - public void reportFileDownloadFailure(String taskId, String filePath, String downloadPath) { - reportFileDownloadFailure(taskId, filePath, downloadPath, "Pulling failed"); - } - - @Override - public void reportFileDownloadStopped(String taskId, String filePath, String downloadPath, Long fileSize, - Integer progress) { - UpdateFileSourceTaskReq req = new UpdateFileSourceTaskReq(); - req.setFileSourceTaskId(taskId); - req.setFilePath(filePath); - req.setDownloadPath(downloadPath); - req.setStatus(TaskStatusEnum.STOPPED); - req.setFileSize(fileSize); - req.setSpeed("--"); - req.setContent("Pulling stopped"); - req.setProgress(progress); - reportTaskStatus(req); - } - - public void reportFileDownloadFailure(String taskId, String filePath, String downloadPath, String content) { - UpdateFileSourceTaskReq req = new UpdateFileSourceTaskReq(); - req.setFileSourceTaskId(taskId); - req.setFilePath(filePath); - req.setDownloadPath(downloadPath); - req.setStatus(TaskStatusEnum.FAILED); - req.setSpeed("--"); - req.setContent(content); - req.setProgress(0); - reportTaskStatus(req); - } - - @Override - public void reportWorkerOffLine(List taskIdList, String content) { - // TODO - } - - - public void reportTaskStatus(UpdateFileSourceTaskReq updateFileSourceTaskReq) { - String url = gatewayInfoService.getReportTaskStatusUrl(); - HttpReq req = HttpReqGenUtil.genSimpleJsonReq(url, updateFileSourceTaskReq); - String respStr = null; - try { - log.info(String.format("url=%s,body=%s,headers=%s", url, req.getBody(), - JsonUtils.toJson(req.getHeaders()))); - respStr = httpHelper.post(url, req.getBody(), req.getHeaders()); - log.info(String.format("respStr=%s", respStr)); - } catch (Exception e) { - log.error("Fail to request file-gateway:", e); - } - } -} diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/exception/handler/WorkerExceptionControllerAdvice.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/exception/handler/WorkerExceptionControllerAdvice.java new file mode 100644 index 0000000000..7d7b989c29 --- /dev/null +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/exception/handler/WorkerExceptionControllerAdvice.java @@ -0,0 +1,33 @@ +package com.tencent.bk.job.file.worker.exception.handler; + +import com.tencent.bk.job.common.annotation.WorkerAPI; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.model.InternalResponse; +import com.tencent.bk.job.common.model.error.ErrorType; +import lombok.extern.slf4j.Slf4j; +import org.springframework.http.HttpStatus; +import org.springframework.web.bind.annotation.ControllerAdvice; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.ResponseBody; +import org.springframework.web.bind.annotation.ResponseStatus; +import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler; + +import javax.servlet.http.HttpServletRequest; + +/** + * 处理Worker接口异常 + */ +@ControllerAdvice(annotations = {WorkerAPI.class}) +@Slf4j +public class WorkerExceptionControllerAdvice extends ResponseEntityExceptionHandler { + + @ExceptionHandler(Throwable.class) + @ResponseBody + @ResponseStatus(HttpStatus.INTERNAL_SERVER_ERROR) + InternalResponse handleException(HttpServletRequest request, Throwable ex) { + String exceptionInfo = "Handle Exception, uri: " + request.getRequestURI(); + log.error(exceptionInfo, ex); + return InternalResponse.buildCommonFailResp(ErrorType.INTERNAL, ErrorCode.INTERNAL_ERROR); + } + +} diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/interceptor/FileWorkerSecurityInterceptor.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/interceptor/FileWorkerSecurityInterceptor.java new file mode 100644 index 0000000000..4945f52e30 --- /dev/null +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/interceptor/FileWorkerSecurityInterceptor.java @@ -0,0 +1,74 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file.worker.interceptor; + +import com.tencent.bk.job.common.annotation.JobInterceptor; +import com.tencent.bk.job.common.constant.InterceptorOrder; +import com.tencent.bk.job.common.jwt.JwtManager; +import com.tencent.bk.job.common.security.consts.JwtConsts; +import com.tencent.bk.job.common.security.exception.ServiceNoAuthException; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.web.servlet.handler.HandlerInterceptorAdapter; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/** + * File-Worker请求认证拦截器 + */ +@Slf4j +@JobInterceptor(pathPatterns = "/**", order = InterceptorOrder.Init.CHECK_VALID) +public class FileWorkerSecurityInterceptor extends HandlerInterceptorAdapter { + + private final JwtManager jwtManager; + + public FileWorkerSecurityInterceptor(JwtManager jwtManager) { + this.jwtManager = jwtManager; + } + + @Override + public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) { + if (shouldFilter(request)) { + String jwt = request.getHeader(JwtConsts.HEADER_KEY_SERVICE_JWT_TOKEN); + if (StringUtils.isEmpty(jwt)) { + log.error("Invalid request, jwt is empty! url: {}", request.getRequestURI()); + throw new ServiceNoAuthException(); + } + boolean checkResult = jwtManager.verifyJwt(jwt); + if (!checkResult) { + log.error("Invalid request, jwt is invalid or expired! url: {}", request.getRequestURI()); + throw new ServiceNoAuthException(); + } + } + return true; + } + + public boolean shouldFilter(HttpServletRequest request) { + String uri = request.getServletPath(); + // 拦截所有Worker API请求 + return uri.startsWith("/worker/api/"); + } +} diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/DownloadFileTask.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/service/DownloadFileTask.java similarity index 91% rename from src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/DownloadFileTask.java rename to src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/service/DownloadFileTask.java index bfbd697ec4..f5fa5fb65f 100644 --- a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/DownloadFileTask.java +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/service/DownloadFileTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,15 +22,16 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.file.worker.cos.service; +package com.tencent.bk.job.file.worker.service; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.exception.InternalException; import com.tencent.bk.job.common.exception.ServiceException; -import com.tencent.bk.job.common.util.FileUtil; +import com.tencent.bk.job.common.util.file.FileUtil; import com.tencent.bk.job.common.util.file.PathUtil; import com.tencent.bk.job.file.worker.model.FileMetaData; import com.tencent.bk.job.file_gateway.consts.TaskCommandEnum; +import lombok.Builder; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Pair; import org.apache.http.client.methods.HttpRequestBase; @@ -52,14 +53,22 @@ class DownloadFileTask extends Thread { AtomicLong fileSize; AtomicInteger speed; AtomicInteger process; + FileProgressWatchingTask watchingTask; TaskReporter taskReporter; DownloadFileTaskEventListener taskEventListener; - FileProgressWatchingTask watchingTask; - public DownloadFileTask(RemoteClient remoteClient, String taskId, String filePath, String downloadFileDir, - String filePrefix, AtomicLong fileSize, AtomicInteger speed, AtomicInteger process, + @Builder + public DownloadFileTask(RemoteClient remoteClient, + String taskId, + String filePath, + String downloadFileDir, + String filePrefix, + AtomicLong fileSize, + AtomicInteger speed, + AtomicInteger process, FileProgressWatchingTask watchingTask, - TaskReporter taskReporter, DownloadFileTaskEventListener taskEventListener) { + TaskReporter taskReporter, + DownloadFileTaskEventListener taskEventListener) { this.remoteClient = remoteClient; this.taskId = taskId; this.filePath = filePath; @@ -94,8 +103,8 @@ public void downloadFileToLocal( InputStream ins = null; HttpRequestBase req = null; try { - String fileMd5 = ""; - String currentMd5 = ""; + String fileMd5; + String currentMd5; int count = 0; boolean downloadSuccess = false; do { diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/service/EnvironmentService.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/service/EnvironmentService.java new file mode 100644 index 0000000000..b189961aa1 --- /dev/null +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/service/EnvironmentService.java @@ -0,0 +1,125 @@ +package com.tencent.bk.job.file.worker.service; + +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.constant.JobConstants; +import com.tencent.bk.job.common.exception.IncorrectConfigException; +import com.tencent.bk.job.common.util.ip.IpUtils; +import com.tencent.bk.job.file.worker.config.WorkerConfig; +import io.micrometer.core.instrument.util.StringUtils; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.Pair; +import org.springframework.beans.BeansException; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.stereotype.Service; + +@Slf4j +@Service +public class EnvironmentService implements ApplicationContextAware { + + private ApplicationContext applicationContext; + private final WorkerConfig workerConfig; + + public EnvironmentService(WorkerConfig workerConfig) { + this.workerConfig = workerConfig; + } + + @Override + public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { + this.applicationContext = applicationContext; + } + + public boolean isInK8s() { + String[] profileArr = applicationContext.getEnvironment().getActiveProfiles(); + for (String profile : profileArr) { + if (JobConstants.PROFILE_KUBERNETES.equals(profile)) { + return true; + } + } + return false; + } + + private String getAccessHostInK8s() { + String podName = System.getenv("BK_JOB_POD_NAME"); + if (StringUtils.isBlank(podName)) { + String message = "ENV BK_JOB_POD_NAME cannot be blank!"; + throw new IncorrectConfigException( + message, + ErrorCode.INVALID_CONFIG, + new String[]{"ENV:BK_JOB_POD_NAME"} + ); + } + String fileWorkerServiceName = System.getenv("BK_JOB_FILE_WORKER_SERVICE_NAME"); + if (StringUtils.isBlank(fileWorkerServiceName)) { + String message = "ENV BK_JOB_FILE_WORKER_SERVICE_NAME cannot be blank!"; + throw new IncorrectConfigException( + message, + ErrorCode.INVALID_CONFIG, + new String[]{"ENV:BK_JOB_FILE_WORKER_SERVICE_NAME"} + ); + } + String accessHost = podName + "." + fileWorkerServiceName; + log.debug("accessHost={}", accessHost); + return accessHost; + } + + private String getInnerIpInK8s() { + String nodeIP = System.getenv("BK_JOB_NODE_IP"); + String podIp = IpUtils.getFirstMachineIP(); + log.info("nodeIP={}", nodeIP); + if (!StringUtils.isBlank(nodeIP)) { + return nodeIP; + } else { + return podIp; + } + } + + public String getAccessHost() { + String accessHost; + if (isInK8s()) { + accessHost = getAccessHostInK8s(); + } else { + accessHost = workerConfig.getAccessHost(); + } + if (StringUtils.isBlank(accessHost)) { + accessHost = IpUtils.getFirstMachineIP(); + log.debug("accessHost is blank, use first machine ip"); + } + return accessHost; + } + + public Pair getInnerProtocolAndIp() { + String innerIpProtocol; + String innerIp; + if (isInK8s()) { + innerIp = getInnerIpInK8s(); + } else { + innerIp = workerConfig.getInnerIp().trim(); + } + if (StringUtils.isBlank(innerIp)) { + innerIp = IpUtils.getFirstMachineIP(); + log.debug("innerIp is blank, use first machine ip"); + } + String protocolInferredByIp = IpUtils.inferProtocolByIp(innerIp); + if (StringUtils.isNotBlank(workerConfig.getInnerIpProtocol())) { + innerIpProtocol = workerConfig.getInnerIpProtocol().trim(); + if (!protocolInferredByIp.equalsIgnoreCase(innerIpProtocol)) { + log.warn( + "innerIp({}) protocol specified by job.file-worker.inner-ip-protocol={} " + + "not match protocolInferredByIp({}), use {}", + innerIp, + innerIpProtocol, + protocolInferredByIp, + innerIpProtocol + ); + } + } else { + innerIpProtocol = protocolInferredByIp; + } + // 将IPv6地址转为完整无压缩格式 + if (IpUtils.PROTOCOL_IP_V6.equalsIgnoreCase(innerIpProtocol)) { + innerIp = IpUtils.getFullIpv6ByCompressedOne(innerIp); + } + return Pair.of(innerIpProtocol, innerIp); + } +} diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/service/FileProgressWatchingTask.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/service/FileProgressWatchingTask.java new file mode 100644 index 0000000000..04c4f254f7 --- /dev/null +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/service/FileProgressWatchingTask.java @@ -0,0 +1,100 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file.worker.service; + +import com.tencent.bk.job.common.util.file.PathUtil; +import lombok.Builder; +import lombok.extern.slf4j.Slf4j; + +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +@Slf4j +class FileProgressWatchingTask extends Thread { + + private final String taskId; + private final String filePath; + private final String downloadFileDir; + private final AtomicLong fileSize; + private final AtomicInteger speed; + private final AtomicInteger process; + private final TaskReporter taskReporter; + private final FileProgressWatchingTaskEventListener watchingTaskEventListener; + volatile boolean runFlag = true; + + @Builder + public FileProgressWatchingTask(String taskId, + String filePath, + String downloadFileDir, + AtomicLong fileSize, + AtomicInteger speed, + AtomicInteger process, + TaskReporter taskReporter, + FileProgressWatchingTaskEventListener watchingTaskEventListener) { + this.taskId = taskId; + this.filePath = filePath; + this.downloadFileDir = downloadFileDir; + this.fileSize = fileSize; + this.speed = speed; + this.process = process; + this.taskReporter = taskReporter; + this.watchingTaskEventListener = watchingTaskEventListener; + } + + public void stopWatching() { + log.info("stop watching: taskId={}, filePath={}", taskId, filePath); + this.runFlag = false; + } + + @SuppressWarnings("BusyWait") + @Override + public void run() { + String fileTaskKey = taskId + "_" + filePath; + try { + String downloadPath = PathUtil.joinFilePath(downloadFileDir, taskId + "/" + filePath); + while (runFlag) { + try { + taskReporter.reportFileDownloadProgress(taskId, filePath, downloadPath, fileSize.get(), + speed.get(), process.get()); + } catch (Throwable t) { + log.error("Fail to reportFileDownloadProgress of file:{}", filePath, t); + } + if (runFlag) { + sleep(1000); + } + } + } catch (InterruptedException ignore) { + } finally { + if (watchingTaskEventListener != null) { + watchingTaskEventListener.onWatchingTaskFinally(fileTaskKey); + } + } + log.info("end watching:{}", fileTaskKey); + } + + public interface FileProgressWatchingTaskEventListener { + void onWatchingTaskFinally(String fileTaskKey); + } +} diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/service/FileTaskService.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/service/FileTaskService.java new file mode 100644 index 0000000000..7b7f1fe8c3 --- /dev/null +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/service/FileTaskService.java @@ -0,0 +1,178 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file.worker.service; + +import com.tencent.bk.job.common.util.file.PathUtil; +import com.tencent.bk.job.file.worker.config.WorkerConfig; +import lombok.extern.slf4j.Slf4j; +import org.apache.tomcat.util.http.fileupload.FileUtils; +import org.slf4j.helpers.MessageFormatter; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Service; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Enumeration; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Future; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +@Slf4j +@Service +public class FileTaskService { + + private final ThreadPoolExecutor fileTaskExecutor; + private final ThreadPoolExecutor watchingTaskExecutor; + private static final ConcurrentHashMap> fileTaskMap = new ConcurrentHashMap<>(); + private static final ConcurrentHashMap> watchingTaskMap = new ConcurrentHashMap<>(); + private final WorkerConfig workerConfig; + private final TaskReporter taskReporter; + + @Autowired + public FileTaskService(@Qualifier("fileTaskExecutor") ThreadPoolExecutor fileTaskExecutor, + @Qualifier("watchingTaskExecutor") ThreadPoolExecutor watchingTaskExecutor, + WorkerConfig workerConfig, + TaskReporter taskReporter) { + this.fileTaskExecutor = fileTaskExecutor; + this.watchingTaskExecutor = watchingTaskExecutor; + this.workerConfig = workerConfig; + this.taskReporter = taskReporter; + } + + public List getAllTaskIdList() { + List runningTaskIdList = new ArrayList<>(); + Enumeration keys = fileTaskMap.keys(); + while (keys.hasMoreElements()) { + runningTaskIdList.add(keys.nextElement().split("_")[0]); + } + return runningTaskIdList; + } + + public Integer downloadFiles(RemoteClient client, String taskId, List filePathList, String filePrefix) { + for (String filePath : filePathList) { + String fileTaskKey = taskId + "_" + filePath; + AtomicLong fileSize = new AtomicLong(0L); + AtomicInteger speed = new AtomicInteger(0); + AtomicInteger process = new AtomicInteger(0); + FileProgressWatchingTask progressWatchingTask = FileProgressWatchingTask.builder() + .taskId(taskId) + .filePath(filePath) + .downloadFileDir(workerConfig.getWorkspaceDirPath()) + .fileSize(fileSize) + .speed(speed) + .process(process) + .taskReporter(taskReporter) + .watchingTaskEventListener(pFileTaskKey -> { + Future future = watchingTaskMap.get(pFileTaskKey); + if (future != null) { + future.cancel(true); + watchingTaskMap.remove(pFileTaskKey); + } + }) + .build(); + DownloadFileTask downloadFileTask = DownloadFileTask.builder() + .remoteClient(client) + .taskId(taskId) + .filePath(filePath) + .downloadFileDir(workerConfig.getWorkspaceDirPath()) + .filePrefix(filePrefix) + .fileSize(fileSize) + .speed(speed) + .process(process) + .watchingTask(progressWatchingTask) + .taskReporter(taskReporter) + .taskEventListener(tmpFileTaskKey -> { + fileTaskMap.remove(tmpFileTaskKey); + ThreadCommandBus.destroyCommandQueue(tmpFileTaskKey); + }) + .build(); + Future fileTaskFuture = fileTaskExecutor.submit(downloadFileTask); + Future watchingTaskFuture = watchingTaskExecutor.submit(progressWatchingTask); + fileTaskMap.put(fileTaskKey, fileTaskFuture); + watchingTaskMap.put(fileTaskKey, watchingTaskFuture); + } + return filePathList.size(); + } + + public Integer clearTaskFilesAtOnce(List taskIdList) { + int count = 0; + for (String taskId : taskIdList) { + String deleteDirPath = taskId; + try { + deleteDirPath = PathUtil.joinFilePath(workerConfig.getWorkspaceDirPath(), taskId); + FileUtils.deleteDirectory(new File(deleteDirPath)); + count += 1; + } catch (IOException e) { + String msg = MessageFormatter.format( + "Fail to delete dir:{}", + deleteDirPath + ).getMessage(); + log.warn(msg, e); + } + } + return count; + } + + public Integer stopTasksAtOnce(List taskIdList, ThreadCommandBus.Command command) { + int allStoppedFileCount = 0; + Enumeration keys = fileTaskMap.keys(); + List keyList = new ArrayList<>(); + while (keys.hasMoreElements()) { + String key = keys.nextElement(); + keyList.add(key); + } + for (String taskId : taskIdList) { + List stopKeyList = new ArrayList<>(); + for (String key : keyList) { + if (key.startsWith(taskId)) { + stopKeyList.add(key); + } + } + for (String stopKey : stopKeyList) { + log.info("try to stop {}", stopKey); + Future fileTaskFuture = fileTaskMap.get(stopKey); + if (fileTaskFuture != null && !fileTaskFuture.isDone()) { + ThreadCommandBus.sendCommand(stopKey, command); + fileTaskFuture.cancel(true); + allStoppedFileCount += 1; + } else { + log.info("fileTask {} already done, stop too late", stopKey); + } + Future watchingTaskFuture = watchingTaskMap.get(stopKey); + if (watchingTaskFuture != null && !watchingTaskFuture.isDone()) { + watchingTaskFuture.cancel(true); + } else { + log.info("watchingTask {} already done, stop too late", stopKey); + } + } + } + return allStoppedFileCount; + } +} diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/GatewayInfoService.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/service/GatewayInfoService.java similarity index 94% rename from src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/GatewayInfoService.java rename to src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/service/GatewayInfoService.java index 259b83a088..12c4fb3d10 100644 --- a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/GatewayInfoService.java +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/service/GatewayInfoService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.file.worker.cos.service; +package com.tencent.bk.job.file.worker.service; import com.tencent.bk.job.file.worker.config.WorkerConfig; import lombok.extern.slf4j.Slf4j; diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/service/JwtTokenService.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/service/JwtTokenService.java new file mode 100644 index 0000000000..ec9a53000a --- /dev/null +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/service/JwtTokenService.java @@ -0,0 +1,60 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file.worker.service; + +import com.tencent.bk.job.common.jwt.JwtManager; +import com.tencent.bk.job.common.security.consts.JwtConsts; +import lombok.extern.slf4j.Slf4j; +import org.apache.http.Header; +import org.apache.http.message.BasicHeader; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.List; + +@Slf4j +@Service +public class JwtTokenService { + + private final JwtManager jwtManager; + + @Autowired + public JwtTokenService(JwtManager jwtManager) { + this.jwtManager = jwtManager; + } + + /** + * 获取jwt token的header列表 + * + * @return jwt token的header列表 + */ + public List
getJwtTokenHeaders() { + String token = jwtManager.getToken(); + List
headerList = new ArrayList<>(); + headerList.add(new BasicHeader(JwtConsts.HEADER_KEY_SERVICE_JWT_TOKEN, token)); + return headerList; + } +} diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/MetaDataService.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/service/MetaDataService.java similarity index 83% rename from src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/MetaDataService.java rename to src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/service/MetaDataService.java index 7f1be611d6..1712c15f41 100644 --- a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/MetaDataService.java +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/service/MetaDataService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,19 +22,23 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.file.worker.cos.service; +package com.tencent.bk.job.file.worker.service; import com.fasterxml.jackson.core.type.TypeReference; +import com.google.common.collect.Sets; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.util.Base64Util; import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.file.worker.config.WorkerConfig; import com.tencent.bk.job.file_gateway.model.req.common.FileSourceMetaData; import com.tencent.bk.job.file_gateway.model.req.common.FileWorkerConfig; import com.tencent.bk.job.file_gateway.model.resp.common.FileTreeNodeDef; import lombok.extern.slf4j.Slf4j; import net.coobird.thumbnailator.Thumbnails; import org.apache.commons.lang3.StringUtils; +import org.slf4j.helpers.MessageFormatter; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import javax.imageio.ImageIO; @@ -46,11 +50,27 @@ import java.io.InputStreamReader; import java.nio.charset.StandardCharsets; import java.util.List; +import java.util.Set; @Slf4j @Service public class MetaDataService { + private final Set enabledFileSourceTypeCodes; + + @Autowired + public MetaDataService(WorkerConfig workerConfig) { + this.enabledFileSourceTypeCodes = parseEnabledFileSourceType(workerConfig.getEnabledFileSourceTypeStr()); + } + + private Set parseEnabledFileSourceType(String fileSourceStr) { + if (StringUtils.isBlank(fileSourceStr)) { + return null; + } + fileSourceStr = fileSourceStr.trim().replace(" ", ""); + return Sets.newHashSet(fileSourceStr.split(",")); + } + private String loadResizedBase64ImageFromResource(String path) { InputStream ins = this.getClass().getClassLoader().getResourceAsStream(path); if (ins == null) { @@ -74,7 +94,11 @@ private String loadResizedBase64ImageFromResource(String path) { base64Str = base64Str.replace("\n", ""); return "data:image/" + suffix + ";base64," + base64Str; } catch (IOException e) { - log.warn("Fail to read and encode image from path:{}", path, e); + String msg = MessageFormatter.format( + "Fail to read and encode image from path:{}", + path + ).getMessage(); + log.warn(msg, e); } return null; } @@ -106,6 +130,13 @@ public FileWorkerConfig getFileWorkerConfig() { String jsonStr = jsonStrBuilder.toString(); FileWorkerConfig fileWorkerConfig = JsonUtils.fromJson(jsonStr, new TypeReference() { }); + if (enabledFileSourceTypeCodes != null) { + fileWorkerConfig.getFileSourceMetaDataList().forEach(metaData -> + metaData.setEnabled(metaData.getEnabled() + && enabledFileSourceTypeCodes.contains(metaData.getFileSourceTypeCode()) + ) + ); + } parseFileSourceIcon(fileWorkerConfig); return fileWorkerConfig; } catch (IOException e) { diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/service/OpService.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/service/OpService.java new file mode 100644 index 0000000000..8e665a8e1f --- /dev/null +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/service/OpService.java @@ -0,0 +1,132 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file.worker.service; + +import com.tencent.bk.job.common.constant.HttpMethodEnum; +import com.tencent.bk.job.common.model.http.HttpReq; +import com.tencent.bk.job.common.util.http.HttpHelper; +import com.tencent.bk.job.common.util.http.HttpHelperFactory; +import com.tencent.bk.job.common.util.http.HttpReqGenUtil; +import com.tencent.bk.job.common.util.http.HttpRequest; +import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.file.worker.config.WorkerConfig; +import com.tencent.bk.job.file.worker.state.event.WorkerEvent; +import com.tencent.bk.job.file.worker.state.event.WorkerEventService; +import com.tencent.bk.job.file_gateway.consts.TaskCommandEnum; +import com.tencent.bk.job.file_gateway.model.req.inner.OffLineAndReDispatchReq; +import lombok.extern.slf4j.Slf4j; +import org.slf4j.helpers.FormattingTuple; +import org.slf4j.helpers.MessageFormatter; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.List; + +@Slf4j +@Service +public class OpService { + + private final HttpHelper httpHelper = HttpHelperFactory.getDefaultHttpHelper(); + private final WorkerConfig workerConfig; + private final FileTaskService fileTaskService; + private final GatewayInfoService gatewayInfoService; + private final EnvironmentService environmentService; + private final TaskReporter taskReporter; + private final WorkerEventService workerEventService; + private final JwtTokenService jwtTokenService; + + @Autowired + public OpService(WorkerConfig workerConfig, + FileTaskService fileTaskService, + GatewayInfoService gatewayInfoService, + EnvironmentService environmentService, + TaskReporter taskReporter, + WorkerEventService workerEventService, + JwtTokenService jwtTokenService) { + this.workerConfig = workerConfig; + this.fileTaskService = fileTaskService; + this.gatewayInfoService = gatewayInfoService; + this.environmentService = environmentService; + this.taskReporter = taskReporter; + this.workerEventService = workerEventService; + this.jwtTokenService = jwtTokenService; + } + + public List offLine() { + List runningTaskIdList = fileTaskService.getAllTaskIdList(); + workerEventService.commitWorkerEvent(WorkerEvent.offLine()); + return runningTaskIdList; + } + + public List doOffLine() { + List runningTaskIdList = fileTaskService.getAllTaskIdList(); + // 调网关接口下线自己 + String url = gatewayInfoService.getWorkerOffLineUrl(); + OffLineAndReDispatchReq offLineReq = new OffLineAndReDispatchReq(); + offLineReq.setAccessHost(environmentService.getAccessHost()); + offLineReq.setAccessPort(workerConfig.getAccessPort()); + offLineReq.setAppId(workerConfig.getAppId()); + offLineReq.setToken(workerConfig.getToken()); + offLineReq.setTaskIdList(runningTaskIdList); + offLineReq.setInitDelayMills(3000L); + offLineReq.setIntervalMills(3000L); + log.info("offLine: url={},body={}", url, JsonUtils.toJsonWithoutSkippedFields(offLineReq)); + HttpReq req = HttpReqGenUtil.genSimpleJsonReq( + url, + jwtTokenService.getJwtTokenHeaders(), + offLineReq + ); + String respStr; + try { + respStr = httpHelper.requestForSuccessResp( + HttpRequest.builder(HttpMethodEnum.POST, url) + .setStringEntity(req.getBody()) + .setHeaders(req.getHeaders()) + .build()) + .getEntity(); + log.info(String.format("respStr=%s", respStr)); + // 停止任务 + Integer allStoppedFileCount = fileTaskService.stopTasksAtOnce(runningTaskIdList, + new ThreadCommandBus.Command(TaskCommandEnum.STOP_QUIETLY, null)); + log.info("{} file tasks stopped", allStoppedFileCount); + taskReporter.reportWorkerOffLine(runningTaskIdList, "FileWorker offline"); + } catch (Exception e) { + FormattingTuple msg = MessageFormatter.arrayFormat( + "Fail to request file-gateway,url={},body={},headers={}", + new String[]{ + url, + JsonUtils.toJsonWithoutSkippedFields(offLineReq), + JsonUtils.toJson(req.getHeaders()) + } + ); + log.error(msg.getMessage(), e); + } + return runningTaskIdList; + } + + public List taskList() { + return fileTaskService.getAllTaskIdList(); + } +} diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/RemoteClient.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/service/RemoteClient.java similarity index 93% rename from src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/RemoteClient.java rename to src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/service/RemoteClient.java index 01b34824ce..65d1863c4a 100644 --- a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/RemoteClient.java +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/service/RemoteClient.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.file.worker.cos.service; +package com.tencent.bk.job.file.worker.service; import com.tencent.bk.job.common.exception.ServiceException; import com.tencent.bk.job.file.worker.model.FileMetaData; diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/service/TaskReporter.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/service/TaskReporter.java new file mode 100644 index 0000000000..b6882ccd1b --- /dev/null +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/service/TaskReporter.java @@ -0,0 +1,67 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file.worker.service; + +import java.util.List; + +public interface TaskReporter { + + void reportFileDownloadStart(String taskId, String filePath, String downloadPath); + + void reportFileDownloadProgress(String taskId, + String filePath, + String downloadPath, + Long fileSize, + Integer speed, + Integer progress); + + void reportFileDownloadProgressWithContent(String taskId, + String filePath, + String downloadPath, + Long fileSize, + Integer speed, + Integer progress, + String content); + + void reportFileDownloadSuccess(String taskId, + String filePath, + String downloadPath, + Long fileSize, + Integer speed, + Integer progress); + + void reportFileDownloadFailure(String taskId, String filePath, String downloadPath); + + void reportFileDownloadStopped(String taskId, + String filePath, + String downloadPath, + Long fileSize, + Integer progress); + + void reportFileDownloadFailure(String taskId, String filePath, String downloadPath, String content); + + void reportWorkerOffLine(List taskIdList, String content); + +} diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/service/TaskReporterImpl.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/service/TaskReporterImpl.java new file mode 100644 index 0000000000..6188b631f3 --- /dev/null +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/service/TaskReporterImpl.java @@ -0,0 +1,175 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file.worker.service; + +import com.tencent.bk.job.common.model.http.HttpReq; +import com.tencent.bk.job.common.util.http.HttpReqGenUtil; +import com.tencent.bk.job.common.util.http.JobHttpClient; +import com.tencent.bk.job.file_gateway.consts.TaskStatusEnum; +import com.tencent.bk.job.file_gateway.model.req.inner.UpdateFileSourceTaskReq; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.List; + +@Slf4j +@Service +public class TaskReporterImpl implements TaskReporter { + + private final GatewayInfoService gatewayInfoService; + private final JobHttpClient jobHttpClient; + private final JwtTokenService jwtTokenService; + + @Autowired + public TaskReporterImpl(GatewayInfoService gatewayInfoService, + JobHttpClient jobHttpClient, + JwtTokenService jwtTokenService) { + this.gatewayInfoService = gatewayInfoService; + this.jobHttpClient = jobHttpClient; + this.jwtTokenService = jwtTokenService; + } + + public void reportFileDownloadStart(String taskId, String filePath, String downloadPath) { + UpdateFileSourceTaskReq req = new UpdateFileSourceTaskReq(); + req.setFileSourceTaskId(taskId); + req.setFilePath(filePath); + req.setDownloadPath(downloadPath); + req.setStatus(TaskStatusEnum.RUNNING); + req.setSpeed("0 KB/s"); + req.setContent("Start pulling"); + req.setProgress(0); + reportTaskStatus(req); + } + + public void reportFileDownloadProgress(String taskId, + String filePath, + String downloadPath, + Long fileSize, + Integer speed, + Integer progress) { + UpdateFileSourceTaskReq req = new UpdateFileSourceTaskReq(); + req.setFileSourceTaskId(taskId); + req.setFilePath(filePath); + req.setDownloadPath(downloadPath); + req.setStatus(TaskStatusEnum.RUNNING); + req.setFileSize(fileSize); + req.setSpeed(speed + " KB/s"); + req.setContent("Pulling OSS file..."); + req.setProgress(progress); + reportTaskStatus(req); + } + + public void reportFileDownloadProgressWithContent(String taskId, + String filePath, + String downloadPath, + Long fileSize, + Integer speed, + Integer progress, + String content) { + UpdateFileSourceTaskReq req = new UpdateFileSourceTaskReq(); + req.setFileSourceTaskId(taskId); + req.setFilePath(filePath); + req.setDownloadPath(downloadPath); + req.setStatus(TaskStatusEnum.RUNNING); + req.setFileSize(fileSize); + req.setSpeed(speed + " KB/s"); + req.setContent(content); + req.setProgress(progress); + reportTaskStatus(req); + } + + public void reportFileDownloadSuccess(String taskId, + String filePath, + String downloadPath, + Long fileSize, + Integer speed, + Integer progress) { + UpdateFileSourceTaskReq req = new UpdateFileSourceTaskReq(); + req.setFileSourceTaskId(taskId); + req.setFilePath(filePath); + req.setDownloadPath(downloadPath); + req.setStatus(TaskStatusEnum.SUCCESS); + req.setFileSize(fileSize); + req.setSpeed(speed + " KB/s"); + req.setContent("Pulling finished"); + req.setProgress(progress); + reportTaskStatus(req); + } + + public void reportFileDownloadFailure(String taskId, String filePath, String downloadPath) { + reportFileDownloadFailure(taskId, filePath, downloadPath, "Pulling failed"); + } + + @Override + public void reportFileDownloadStopped(String taskId, + String filePath, + String downloadPath, + Long fileSize, + Integer progress) { + UpdateFileSourceTaskReq req = new UpdateFileSourceTaskReq(); + req.setFileSourceTaskId(taskId); + req.setFilePath(filePath); + req.setDownloadPath(downloadPath); + req.setStatus(TaskStatusEnum.STOPPED); + req.setFileSize(fileSize); + req.setSpeed("--"); + req.setContent("Pulling stopped"); + req.setProgress(progress); + reportTaskStatus(req); + } + + public void reportFileDownloadFailure(String taskId, String filePath, String downloadPath, String content) { + UpdateFileSourceTaskReq req = new UpdateFileSourceTaskReq(); + req.setFileSourceTaskId(taskId); + req.setFilePath(filePath); + req.setDownloadPath(downloadPath); + req.setStatus(TaskStatusEnum.FAILED); + req.setSpeed("--"); + req.setContent(content); + req.setProgress(0); + reportTaskStatus(req); + } + + @Override + public void reportWorkerOffLine(List taskIdList, String content) { + // TODO + } + + public void reportTaskStatus(UpdateFileSourceTaskReq updateFileSourceTaskReq) { + String url = gatewayInfoService.getReportTaskStatusUrl(); + HttpReq req = HttpReqGenUtil.genSimpleJsonReq( + url, + jwtTokenService.getJwtTokenHeaders(), + updateFileSourceTaskReq + ); + try { + log.info("url={},body={}", url, req.getBody()); + jobHttpClient.post(req); + } catch (Exception e) { + log.error("Fail to request file-gateway:", e); + } + } +} diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/ThreadCommandBus.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/service/ThreadCommandBus.java similarity index 95% rename from src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/ThreadCommandBus.java rename to src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/service/ThreadCommandBus.java index 6e155b9fff..7dc8bda707 100644 --- a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/cos/service/ThreadCommandBus.java +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/service/ThreadCommandBus.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.file.worker.cos.service; +package com.tencent.bk.job.file.worker.service; import com.tencent.bk.job.file_gateway.consts.TaskCommandEnum; import lombok.AllArgsConstructor; @@ -37,7 +37,6 @@ @Slf4j public class ThreadCommandBus { - private static final ConcurrentHashMap> map = new ConcurrentHashMap<>(); public static void destroyCommandQueue(String key) { diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/state/WorkerStateEnum.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/state/WorkerStateEnum.java new file mode 100644 index 0000000000..10e9a745a4 --- /dev/null +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/state/WorkerStateEnum.java @@ -0,0 +1,71 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file.worker.state; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonValue; + +/** + * File-Worker状态枚举值 + */ +public enum WorkerStateEnum { + STARTING(1, "启动中"), + WAIT_ACCESS_READY(2, "等待自身可被外界访问"), + HEART_BEATING(3, "心跳中"), + HEART_BEAT_WAIT(4, "等待下一次心跳中"), + RUNNING(5, "运行中"), + OFFLINE_ING(6, "下线中"), + OFFLINE_FAILED(7, "下线失败"), + OFFLINE(8, "已下线"); + + /** + * 状态值 + */ + @JsonValue + private final int state; + /** + * 状态描述 + */ + private final String description; + + WorkerStateEnum(int state, String description) { + this.state = state; + this.description = description; + } + + @JsonCreator(mode = JsonCreator.Mode.DELEGATING) + public static WorkerStateEnum valOf(int state) { + for (WorkerStateEnum workerState : values()) { + if (workerState.state == state) { + return workerState; + } + } + return null; + } + + public int getValue() { + return state; + } +} diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/state/WorkerStateMachine.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/state/WorkerStateMachine.java new file mode 100644 index 0000000000..6b3cd44b3a --- /dev/null +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/state/WorkerStateMachine.java @@ -0,0 +1,87 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file.worker.state; + +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cloud.sleuth.Tracer; +import org.springframework.stereotype.Component; + +/** + * File-Worker状态机，管理Worker状态流转 + */ +@Getter +@Slf4j +@Component +public class WorkerStateMachine { + + private WorkerStateEnum workerState = WorkerStateEnum.STARTING; + + @Autowired + public WorkerStateMachine(Tracer tracer) { + } + + public void setWorkerState(WorkerStateEnum workerState) { + log.info("state change: {} -> {}", this.workerState.name(), workerState.name()); + this.workerState = workerState; + } + + public void waitAccessReady() { + setWorkerState(WorkerStateEnum.WAIT_ACCESS_READY); + } + + public void accessReady() { + setWorkerState(WorkerStateEnum.HEART_BEAT_WAIT); + } + + public void heartBeatStart() { + setWorkerState(WorkerStateEnum.HEART_BEATING); + } + + public void heartBeatSuccess() { + setWorkerState(WorkerStateEnum.RUNNING); + } + + public void heartBeatFailed() { + setWorkerState(WorkerStateEnum.HEART_BEAT_WAIT); + } + + public void offlineStart() { + setWorkerState(WorkerStateEnum.OFFLINE_ING); + } + + public void offlineFailed() { + setWorkerState(WorkerStateEnum.OFFLINE_FAILED); + } + + public void offlineSuccess() { + setWorkerState(WorkerStateEnum.OFFLINE); + } + + public boolean isWorkerOffLineIncludeFail() { + return workerState == WorkerStateEnum.OFFLINE || workerState == WorkerStateEnum.OFFLINE_FAILED; + } +} diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/state/event/WorkerActionEnum.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/state/event/WorkerActionEnum.java new file mode 100644 index 0000000000..f517ff6eaa --- /dev/null +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/state/event/WorkerActionEnum.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file.worker.state.event; + + +/** + * Worker动作 + */ +public enum WorkerActionEnum { + /** + * 等待外界访问路径准备好 + */ + WAIT_ACCESS_READY, + /** + * 定时心跳 + */ + HEART_BEAT, + /** + * 下线 + */ + OFF_LINE; +} diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/state/event/WorkerEvent.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/state/event/WorkerEvent.java new file mode 100644 index 0000000000..87c38a0339 --- /dev/null +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/state/event/WorkerEvent.java @@ -0,0 +1,69 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file.worker.state.event; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.tencent.bk.job.common.event.Event; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; + +import java.time.LocalDateTime; + +@Getter +@Setter +@ToString +@NoArgsConstructor +@JsonInclude(JsonInclude.Include.NON_NULL) +public class WorkerEvent extends Event { + /** + * Worker动作 + * + * @see WorkerActionEnum + */ + private WorkerActionEnum action; + + public static WorkerEvent waitAccessReady() { + WorkerEvent workerEvent = new WorkerEvent(); + workerEvent.setAction(WorkerActionEnum.WAIT_ACCESS_READY); + workerEvent.setTime(LocalDateTime.now()); + return workerEvent; + } + + public static WorkerEvent heartBeat() { + WorkerEvent workerEvent = new WorkerEvent(); + workerEvent.setAction(WorkerActionEnum.HEART_BEAT); + workerEvent.setTime(LocalDateTime.now()); + return workerEvent; + } + + public static WorkerEvent offLine() { + WorkerEvent workerEvent = new WorkerEvent(); + workerEvent.setAction(WorkerActionEnum.OFF_LINE); + workerEvent.setTime(LocalDateTime.now()); + return workerEvent; + } +} diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/state/event/WorkerEventDispatcher.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/state/event/WorkerEventDispatcher.java new file mode 100644 index 0000000000..b8b3f40195 --- /dev/null +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/state/event/WorkerEventDispatcher.java @@ -0,0 +1,107 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file.worker.state.event; + +import com.tencent.bk.job.common.tracing.util.SpanUtil; +import com.tencent.bk.job.file.worker.state.event.handler.DefaultEventHandler; +import com.tencent.bk.job.file.worker.state.event.handler.EventHandler; +import com.tencent.bk.job.file.worker.state.event.handler.HeartBeatEventHandler; +import com.tencent.bk.job.file.worker.state.event.handler.OffLineEventHandler; +import com.tencent.bk.job.file.worker.state.event.handler.WaitAccessEventHandler; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cloud.sleuth.Span; +import org.springframework.cloud.sleuth.Tracer; +import org.springframework.stereotype.Component; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.BlockingQueue; + +@Slf4j +@Component +public class WorkerEventDispatcher extends Thread { + + @SuppressWarnings("FieldCanBeLocal") + private boolean enabled = true; + /** + * 日志调用链tracer + */ + private final Tracer tracer; + private BlockingQueue eventQueue; + private static final Map handlerMap = new HashMap<>(); + private static final EventHandler defaultHandler = new DefaultEventHandler(); + + @Autowired + public WorkerEventDispatcher(Tracer tracer, + WaitAccessEventHandler waitAccessEventHandler, + HeartBeatEventHandler heartBeatEventHandler, + OffLineEventHandler offLineEventHandler) { + this.tracer = tracer; + handlerMap.put(WorkerActionEnum.WAIT_ACCESS_READY, waitAccessEventHandler); + handlerMap.put(WorkerActionEnum.HEART_BEAT, heartBeatEventHandler); + handlerMap.put(WorkerActionEnum.OFF_LINE, offLineEventHandler); + } + + public void initQueue(BlockingQueue eventQueue) { + this.eventQueue = eventQueue; + } + + @Override + public void run() { + while (enabled) { + WorkerEvent event; + try { + event = eventQueue.take(); + dispatchEventWithTrace(event); + } catch (InterruptedException e) { + log.warn("queue.take interrupted", e); + } catch (Throwable t) { + log.error("Fail to handleEventWithTrace", t); + } + } + } + + private void dispatchEventWithTrace(WorkerEvent event) { + Span span = buildSpan(event); + try (Tracer.SpanInScope ignored = this.tracer.withSpan(span.start())) { + dispatchEvent(event); + } catch (Throwable t) { + span.error(t); + log.warn("Fail to handlePropChangeEvent:" + event, t); + } finally { + span.end(); + } + } + + private void dispatchEvent(WorkerEvent event) { + EventHandler handler = handlerMap.getOrDefault(event.getAction(), defaultHandler); + handler.handleEvent(event); + } + + private Span buildSpan(WorkerEvent event) { + return SpanUtil.buildNewSpan(this.tracer, event.getAction().name()); + } +} diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/state/event/WorkerEventService.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/state/event/WorkerEventService.java new file mode 100644 index 0000000000..7058e9a32a --- /dev/null +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/state/event/WorkerEventService.java @@ -0,0 +1,56 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file.worker.state.event; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; + +/** + * File-Worker生命周期事件服务，用于接收上层业务逻辑触发的事件 + */ +@Slf4j +@Service +public class WorkerEventService { + + private final BlockingQueue eventQueue = new LinkedBlockingQueue<>(100); + + @Autowired + public WorkerEventService(WorkerEventDispatcher workerEventDispatcher) { + workerEventDispatcher.initQueue(eventQueue); + workerEventDispatcher.start(); + } + + public void commitWorkerEvent(WorkerEvent event) { + boolean result = eventQueue.add(event); + if (!result) { + log.warn("Fail to add event to queue:{}, ignore", event); + } + } + +} diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/state/event/handler/DefaultEventHandler.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/state/event/handler/DefaultEventHandler.java new file mode 100644 index 0000000000..839d53f92e --- /dev/null +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/state/event/handler/DefaultEventHandler.java @@ -0,0 +1,39 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file.worker.state.event.handler; + +import com.tencent.bk.job.file.worker.state.event.WorkerEvent; +import lombok.extern.slf4j.Slf4j; + +/** + * 默认事件处理器，仅对事件做日志记录 + */ +@Slf4j +public class DefaultEventHandler implements EventHandler { + @Override + public void handleEvent(WorkerEvent event) { + log.warn("No handler specified for event:{}, ignore", event); + } +} diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/state/event/handler/EventHandler.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/state/event/handler/EventHandler.java new file mode 100644 index 0000000000..48678d1a52 --- /dev/null +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/state/event/handler/EventHandler.java @@ -0,0 +1,31 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file.worker.state.event.handler; + +import com.tencent.bk.job.file.worker.state.event.WorkerEvent; + +public interface EventHandler { + void handleEvent(WorkerEvent event); +} diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/state/event/handler/HealthResult.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/state/event/handler/HealthResult.java new file mode 100644 index 0000000000..6950d1b9dc --- /dev/null +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/state/event/handler/HealthResult.java @@ -0,0 +1,41 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file.worker.state.event.handler; + +import lombok.Data; + +import java.util.List; + +@Data +public class HealthResult { + /** + * 健康状态 + */ + private String status; + /** + * 健康指标分组 + */ + private List groups; +} diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/state/event/handler/HeartBeatEventHandler.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/state/event/handler/HeartBeatEventHandler.java new file mode 100644 index 0000000000..ecd5a56be7 --- /dev/null +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/state/event/handler/HeartBeatEventHandler.java @@ -0,0 +1,96 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file.worker.state.event.handler; + +import com.tencent.bk.job.file.worker.state.WorkerStateEnum; +import com.tencent.bk.job.file.worker.state.WorkerStateMachine; +import com.tencent.bk.job.file.worker.state.event.WorkerEvent; +import com.tencent.bk.job.file.worker.state.event.WorkerEventService; +import com.tencent.bk.job.file.worker.task.heartbeat.HeartBeatTask; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; + +/** + * 心跳事件处理器，用于向File-Gateway上报Worker状态信息 + */ +@Slf4j +@Component +public class HeartBeatEventHandler implements EventHandler { + + private final WorkerEventService workerEventService; + private final WorkerStateMachine workerStateMachine; + private final HeartBeatTask heartBeatTask; + + @Autowired + public HeartBeatEventHandler(@Lazy WorkerEventService workerEventService, + WorkerStateMachine workerStateMachine, + HeartBeatTask heartBeatTask) { + this.workerEventService = workerEventService; + this.workerStateMachine = workerStateMachine; + this.heartBeatTask = heartBeatTask; + } + + @Override + public void handleEvent(WorkerEvent event) { + WorkerStateEnum workerState = workerStateMachine.getWorkerState(); + switch (workerState) { + case STARTING: + workerEventService.commitWorkerEvent(WorkerEvent.waitAccessReady()); + break; + case WAIT_ACCESS_READY: + log.info("wait access ready, ignore current event:{}", event); + break; + case HEART_BEAT_WAIT: + case RUNNING: + heartBeat(); + break; + default: + log.info("currentState:{}, heartBeat condition not satisfy, ignore", workerState); + break; + } + } + + private Long lastSuccessHeartBeatTime = null; + + private void heartBeat() { + workerStateMachine.heartBeatStart(); + try { + // 如果上一次成功的心跳在10s内发生，则忽略本次心跳 + if (lastSuccessHeartBeatTime != null && System.currentTimeMillis() - lastSuccessHeartBeatTime < 10_000L) { + log.info("lastSuccessHeartBeat finish with 10s, ignore current heartBeat"); + workerStateMachine.heartBeatSuccess(); + } else { + heartBeatTask.doHeartBeat(); + workerStateMachine.heartBeatSuccess(); + lastSuccessHeartBeatTime = System.currentTimeMillis(); + } + } catch (Throwable t) { + log.warn("Fail to heartBeat", t); + workerStateMachine.heartBeatFailed(); + } + } +} diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/state/event/handler/OffLineEventHandler.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/state/event/handler/OffLineEventHandler.java new file mode 100644 index 0000000000..ea550dc116 --- /dev/null +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/state/event/handler/OffLineEventHandler.java @@ -0,0 +1,81 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file.worker.state.event.handler; + +import com.tencent.bk.job.file.worker.service.OpService; +import com.tencent.bk.job.file.worker.state.WorkerStateEnum; +import com.tencent.bk.job.file.worker.state.WorkerStateMachine; +import com.tencent.bk.job.file.worker.state.event.WorkerEvent; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; + +/** + * 下线事件处理器 + */ +@Slf4j +@Component +public class OffLineEventHandler implements EventHandler { + + private final WorkerStateMachine workerStateMachine; + private final OpService opService; + + @Autowired + public OffLineEventHandler(WorkerStateMachine workerStateMachine, + @Lazy OpService opService) { + this.workerStateMachine = workerStateMachine; + this.opService = opService; + } + + @Override + public void handleEvent(WorkerEvent event) { + WorkerStateEnum workerState = workerStateMachine.getWorkerState(); + switch (workerState) { + case RUNNING: + case HEART_BEAT_WAIT: + case OFFLINE_FAILED: + offLine(); + break; + case OFFLINE_ING: + log.info("last offLine action is executing, ignore current one"); + break; + default: + log.info("currentState:{}, offLine condition not satisfy, ignore", workerState); + break; + } + } + + private void offLine() { + workerStateMachine.offlineStart(); + try { + opService.doOffLine(); + workerStateMachine.offlineSuccess(); + } catch (Throwable t) { + log.warn("Fail to offLine", t); + workerStateMachine.offlineFailed(); + } + } +} diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/state/event/handler/WaitAccessEventHandler.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/state/event/handler/WaitAccessEventHandler.java new file mode 100644 index 0000000000..dfb5e6600f --- /dev/null +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/state/event/handler/WaitAccessEventHandler.java @@ -0,0 +1,131 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.file.worker.state.event.handler; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.tencent.bk.job.common.model.http.HttpReq; +import com.tencent.bk.job.common.util.ThreadUtils; +import com.tencent.bk.job.common.util.http.HttpReqGenUtil; +import com.tencent.bk.job.common.util.http.JobHttpClient; +import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.file.worker.config.WorkerConfig; +import com.tencent.bk.job.file.worker.service.EnvironmentService; +import com.tencent.bk.job.file.worker.state.WorkerStateEnum; +import com.tencent.bk.job.file.worker.state.WorkerStateMachine; +import com.tencent.bk.job.file.worker.state.event.WorkerEvent; +import com.tencent.bk.job.file.worker.state.event.WorkerEventService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Component; + +/** + * 等待Worker可被外界访问的事件处理器，实现检查与等待逻辑 + */ +@Slf4j +@Component +public class WaitAccessEventHandler implements EventHandler { + + private final WorkerEventService workerEventService; + private final WorkerStateMachine workerStateMachine; + private final JobHttpClient jobHttpClient; + private final String checkAccessUrl; + + @Autowired + public WaitAccessEventHandler(@Lazy WorkerEventService workerEventService, + WorkerStateMachine workerStateMachine, + JobHttpClient jobHttpClient, + WorkerConfig workerConfig, + EnvironmentService environmentService) { + this.workerEventService = workerEventService; + this.workerStateMachine = workerStateMachine; + this.jobHttpClient = jobHttpClient; + this.checkAccessUrl = buildCheckAccessUrl(environmentService.getAccessHost(), workerConfig.getAccessPort()); + } + + @SuppressWarnings("HttpUrlsUsage") + private String buildCheckAccessUrl(String accessHost, Integer accessPort) { + return "http://" + accessHost + ":" + accessPort + "/actuator/health"; + } + + @Override + public void handleEvent(WorkerEvent event) { + WorkerStateEnum workerState = workerStateMachine.getWorkerState(); + switch (workerState) { + case STARTING: + case WAIT_ACCESS_READY: + workerStateMachine.waitAccessReady(); + waitAccessReady(); + break; + default: + log.info("currentState:{}, waitAccessReady condition not satisfy, ignore", workerState); + break; + } + } + + public void waitAccessReady() { + boolean accessReady = checkAccess(); + if (accessReady) { + // 1.状态切换 + workerStateMachine.accessReady(); + // 2.自身可被外界访问后立即触发心跳 + workerEventService.commitWorkerEvent(WorkerEvent.heartBeat()); + } else { + // 3.检查失败，状态不变，继续检查 + workerEventService.commitWorkerEvent(WorkerEvent.waitAccessReady()); + } + } + + private boolean checkAccess() { + boolean accessReady = false; + int maxCheckNum = 300; + int checkNum = 0; + int errorNum = 0; + do { + try { + checkNum += 1; + log.info("CheckAccess: url={}", checkAccessUrl); + HttpReq req = HttpReqGenUtil.genUrlGetReq(checkAccessUrl); + String respStr = jobHttpClient.get(req); + HealthResult healthResult = JsonUtils.fromJson(respStr, new TypeReference() { + }); + String status = healthResult.getStatus(); + if (status != null && status.equalsIgnoreCase("UP")) { + accessReady = true; + } + } catch (Throwable t) { + errorNum += 1; + if (errorNum % 10 == 0) { + log.info("Fail to checkAccess", t); + } + } + if (!accessReady && checkNum < maxCheckNum) { + log.info("Access not ready, checkNum={}, wait 1s", checkNum); + ThreadUtils.sleep(1000); + } + } while (!accessReady && checkNum < maxCheckNum); + return accessReady; + } +} diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/task/ScheduledTasks.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/task/ScheduledTasks.java index c0770dd455..2e9c256be1 100644 --- a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/task/ScheduledTasks.java +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/task/ScheduledTasks.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,8 +24,9 @@ package com.tencent.bk.job.file.worker.task; +import com.tencent.bk.job.file.worker.state.event.WorkerEvent; +import com.tencent.bk.job.file.worker.state.event.WorkerEventService; import com.tencent.bk.job.file.worker.task.clear.ClearFileTask; -import com.tencent.bk.job.file.worker.task.heartbeat.HeartBeatTask; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -33,18 +34,18 @@ import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; -@Component +@Component("jobFileWorkerScheduledTasks") @EnableScheduling public class ScheduledTasks { private static final Logger logger = LoggerFactory.getLogger(ScheduledTasks.class); - private final HeartBeatTask heartBeatTask; + private final WorkerEventService workerEventService; private final ClearFileTask clearFileTask; @Autowired - public ScheduledTasks(HeartBeatTask heartBeatTask, ClearFileTask clearFileTask) { - this.heartBeatTask = heartBeatTask; + public ScheduledTasks(WorkerEventService workerEventService, ClearFileTask clearFileTask) { + this.workerEventService = workerEventService; this.clearFileTask = clearFileTask; } @@ -81,9 +82,9 @@ public void checkVolumeAndClear() { public void heartBeat() { logger.info(Thread.currentThread().getId() + ":heartBeat start"); try { - heartBeatTask.run(); + workerEventService.commitWorkerEvent(WorkerEvent.heartBeat()); } catch (Exception e) { - logger.error("heartBeatTask fail", e); + logger.error("commit heartBeat event fail", e); } } } diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/task/clear/ClearFileTask.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/task/clear/ClearFileTask.java index 5a3d699e4c..980405f4d7 100644 --- a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/task/clear/ClearFileTask.java +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/task/clear/ClearFileTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,11 +25,10 @@ package com.tencent.bk.job.file.worker.task.clear; import com.tencent.bk.job.common.util.date.DateUtils; -import com.tencent.bk.job.common.util.file.FileSizeUtil; +import com.tencent.bk.job.common.util.file.FileUtil; import com.tencent.bk.job.file.worker.config.WorkerConfig; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; -import org.slf4j.helpers.FormattingTuple; import org.slf4j.helpers.MessageFormatter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -37,13 +36,6 @@ import java.io.File; import java.io.IOException; import java.sql.Timestamp; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Comparator; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; @Slf4j @Service @@ -80,44 +72,17 @@ public void clearExpiredFile() { try { FileUtils.deleteDirectory(taskDirFile); } catch (IOException e) { - log.warn("Fail to delete {}", taskDirFile.getAbsolutePath(), e); + String msg = MessageFormatter.format( + "Fail to delete {}", + taskDirFile.getAbsolutePath() + ).getMessage(); + log.warn(msg, e); } } } log.info("clearTask end"); } - /** - * 删除文件，若删除失败则记录路径 - * - * @param file 文件 - * @param failedPathSet 删除失败时用于记录路径的Set - * @return 是否删除成功 - */ - private boolean deleteFileAndRecordIfFail(File file, Set failedPathSet) { - try { - FileUtils.deleteQuietly(file); - return true; - } catch (Exception e) { - failedPathSet.add(file.getAbsolutePath()); - FormattingTuple message = MessageFormatter.format( - "Fail to delete file {}", - file.getAbsolutePath() - ); - log.warn(message.getMessage(), e); - return false; - } - } - - private void showVolumeUsage(String path, long maxSizeBytes, long currentSize) { - log.info( - "{},currentSize={},maxSizeBytes={}", - path, - FileSizeUtil.getFileSizeStr(currentSize), - FileSizeUtil.getFileSizeStr(maxSizeBytes) - ); - } - /** * 检查磁盘容量并清理最旧的文件 */ @@ -139,45 +104,10 @@ public void checkVolumeAndClear() { private void doCheckVolumeAndClear() { long maxSizeBytes = workerConfig.getMaxSizeGB() * 1024L * 1024L * 1024L; - File workDirFile = new File(workerConfig.getWorkspaceDirPath()); - long currentSize = FileUtils.sizeOfDirectory(workDirFile); - if (log.isDebugEnabled()) { - showVolumeUsage(workDirFile.getAbsolutePath(), maxSizeBytes, currentSize); - } - File[] files = workDirFile.listFiles(); - if (files == null || files.length == 0) return; - List fileList = new ArrayList<>(Arrays.asList(files)); - fileList.sort(Comparator.comparingLong(File::lastModified)); - // 记录删除失败的文件，下次不再列出 - Set deleteFailedFilePathSet = new HashSet<>(); - int count = 0; - while (currentSize > maxSizeBytes) { - if (fileList.isEmpty()) { - // 上一次拿到的文件列表已删完，空间依然超限，说明删除过程中又新产生了许多文件，重新列出 - files = workDirFile.listFiles(); - if (files == null || files.length == 0) return; - fileList.addAll(Arrays.stream(files) - .filter(file -> !deleteFailedFilePathSet.contains(file.getAbsolutePath())) - .collect(Collectors.toList()) - ); - fileList.sort(Comparator.comparingLong(File::lastModified)); - } - if (fileList.isEmpty()) { - log.warn("volume still overlimit after clear, deleteFailedFilePathSet={}", deleteFailedFilePathSet); - return; - } - File oldestFile = fileList.remove(0); - if (deleteFileAndRecordIfFail(oldestFile, deleteFailedFilePathSet)) { - count += 1; - log.info("delete file {} because of volume overlimit", oldestFile.getAbsolutePath()); - } - currentSize = FileUtils.sizeOfDirectory(workDirFile); - showVolumeUsage(workDirFile.getAbsolutePath(), maxSizeBytes, currentSize); - } - if (log.isDebugEnabled()) { - log.debug("{} files deleted because of volume overlimit", count); - } else if (count > 0) { - log.info("{} files deleted because of volume overlimit", count); + int count = FileUtil.checkVolumeAndClearOldestFiles(maxSizeBytes, workerConfig.getWorkspaceDirPath()); + if (count > 0) { + log.info("{} file cleared", count); } } + } diff --git a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/task/heartbeat/HeartBeatTask.java b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/task/heartbeat/HeartBeatTask.java index f6bdde220a..0bc7d9f2a7 100644 --- a/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/task/heartbeat/HeartBeatTask.java +++ b/src/backend/job-file-worker-sdk/service-job-file-worker-sdk/src/main/java/com/tencent/bk/job/file/worker/task/heartbeat/HeartBeatTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,13 +27,16 @@ import com.tencent.bk.job.common.model.http.HttpReq; import com.tencent.bk.job.common.util.http.HttpReqGenUtil; import com.tencent.bk.job.common.util.http.JobHttpClient; +import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.common.util.machine.MachineUtil; import com.tencent.bk.job.file.worker.config.WorkerConfig; -import com.tencent.bk.job.file.worker.cos.service.EnvironmentService; -import com.tencent.bk.job.file.worker.cos.service.GatewayInfoService; -import com.tencent.bk.job.file.worker.cos.service.MetaDataService; +import com.tencent.bk.job.file.worker.service.EnvironmentService; +import com.tencent.bk.job.file.worker.service.GatewayInfoService; +import com.tencent.bk.job.file.worker.service.JwtTokenService; +import com.tencent.bk.job.file.worker.service.MetaDataService; import com.tencent.bk.job.file_gateway.model.req.inner.HeartBeatReq; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.tuple.Pair; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -43,32 +46,29 @@ @Service public class HeartBeatTask { - public static volatile boolean runFlag = true; - private final JobHttpClient jobHttpClient; private final WorkerConfig workerConfig; private final GatewayInfoService gatewayInfoService; private final MetaDataService metaDataService; private final EnvironmentService environmentService; + private final JwtTokenService jwtTokenService; @Autowired public HeartBeatTask(JobHttpClient jobHttpClient, WorkerConfig workerConfig, GatewayInfoService gatewayInfoService, MetaDataService metaDataService, - EnvironmentService environmentService) { + EnvironmentService environmentService, + JwtTokenService jwtTokenService) { this.jobHttpClient = jobHttpClient; this.workerConfig = workerConfig; this.gatewayInfoService = gatewayInfoService; this.metaDataService = metaDataService; this.environmentService = environmentService; + this.jwtTokenService = jwtTokenService; } - public static void stopHeartBeat() { - runFlag = false; - } - - private HeartBeatReq getWorkerInfo() { + private HeartBeatReq getHeartBeatReq() { HeartBeatReq heartBeatReq = new HeartBeatReq(); heartBeatReq.setName(workerConfig.getName()); heartBeatReq.setTagList(workerConfig.getTagList()); @@ -77,7 +77,9 @@ private HeartBeatReq getWorkerInfo() { // 二进制部署环境与K8s环境差异处理 heartBeatReq.setAccessHost(environmentService.getAccessHost()); - heartBeatReq.setInnerIp(environmentService.getInnerIp()); + Pair protocolAndIpPair = environmentService.getInnerProtocolAndIp(); + heartBeatReq.setInnerIpProtocol(protocolAndIpPair.getLeft()); + heartBeatReq.setInnerIp(protocolAndIpPair.getRight()); heartBeatReq.setAccessPort(workerConfig.getAccessPort()); heartBeatReq.setCloudAreaId(workerConfig.getCloudAreaId()); @@ -97,13 +99,15 @@ private HeartBeatReq getWorkerInfo() { return heartBeatReq; } - public void run() { - if (!runFlag) { - log.info("HeartBeat closed, ignore"); - return; - } + public void doHeartBeat() { String url = gatewayInfoService.getHeartBeatUrl(); - HttpReq req = HttpReqGenUtil.genSimpleJsonReq(url, getWorkerInfo()); + HeartBeatReq heartBeatReq = getHeartBeatReq(); + log.info("HeartBeat: url={},body={}", url, JsonUtils.toJsonWithoutSkippedFields(heartBeatReq)); + HttpReq req = HttpReqGenUtil.genSimpleJsonReq( + url, + jwtTokenService.getJwtTokenHeaders(), + heartBeatReq + ); jobHttpClient.post(req); } } diff --git a/src/backend/job-file-worker/api-job-file-worker/build.gradle b/src/backend/job-file-worker/api-job-file-worker/build.gradle index 6640dfe3ac..1ce00cd344 100644 --- a/src/backend/job-file-worker/api-job-file-worker/build.gradle +++ b/src/backend/job-file-worker/api-job-file-worker/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-worker/api-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/model/BucketDTO.java b/src/backend/job-file-worker/api-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/model/BucketDTO.java index 0f70472a03..db4792dee1 100644 --- a/src/backend/job-file-worker/api-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/model/BucketDTO.java +++ b/src/backend/job-file-worker/api-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/model/BucketDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-worker/api-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/model/FileDTO.java b/src/backend/job-file-worker/api-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/model/FileDTO.java index 77dd7bc937..cc717458ed 100644 --- a/src/backend/job-file-worker/api-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/model/FileDTO.java +++ b/src/backend/job-file-worker/api-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/model/FileDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-worker/boot-job-file-worker/build.gradle b/src/backend/job-file-worker/boot-job-file-worker/build.gradle index a848dba51f..f7c2dbceed 100644 --- a/src/backend/job-file-worker/boot-job-file-worker/build.gradle +++ b/src/backend/job-file-worker/boot-job-file-worker/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -21,15 +21,11 @@ * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS * IN THE SOFTWARE. */ -def assemblyMode = System.getProperty("assemblyMode") -def k8s = assemblyMode == "k8s" -println("assembly mode: $assemblyMode") - apply plugin: 'org.springframework.boot' apply plugin: 'io.spring.dependency-management' dependencies { - if (k8s) { - println("Compile with kubernetes mode") + if (rootProject.ext.k8sMode) { + println("Compile module[boot-job-file-worker] with kubernetes mode") api 'org.springframework.cloud:spring-cloud-starter-kubernetes-client-all' } implementation project(":job-file-worker:service-job-file-worker") @@ -48,8 +44,7 @@ task renameArtifacts(type: Copy) { destinationDir file('build/libs/') rename "boot-job-file-worker-${version}.jar", "job-file-worker-${version}.jar" } -renameArtifacts.dependsOn jar -assemble.dependsOn renameArtifacts +renameArtifacts.dependsOn assemble task copyToLatestJar(type: Copy) { group = "local" @@ -61,3 +56,4 @@ task copyToLatestJar(type: Copy) { copyToLatestJar.dependsOn assemble apply from: "$rootDir/task_job_package.gradle" +copyToRelease.dependsOn renameArtifacts diff --git a/src/backend/job-file-worker/boot-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/JobFileWorkerBootApplication.java b/src/backend/job-file-worker/boot-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/JobFileWorkerBootApplication.java index f64587f9a0..9617e0bccd 100644 --- a/src/backend/job-file-worker/boot-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/JobFileWorkerBootApplication.java +++ b/src/backend/job-file-worker/boot-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/JobFileWorkerBootApplication.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,11 +26,13 @@ import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration; +import org.springframework.boot.autoconfigure.jooq.JooqAutoConfiguration; import org.springframework.cache.annotation.EnableCaching; import org.springframework.scheduling.annotation.EnableScheduling; -@SpringBootApplication(scanBasePackages = "com.tencent.bk.job", exclude = {RedisAutoConfiguration.class}) +@SpringBootApplication( + scanBasePackages = "com.tencent.bk.job.file.worker", + exclude = {JooqAutoConfiguration.class}) @EnableCaching @EnableScheduling public class JobFileWorkerBootApplication { diff --git a/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/FileWorkerConfig.json b/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/FileWorkerConfig.json index 8a721a2fdb..dc223ca591 100644 --- a/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/FileWorkerConfig.json +++ b/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/FileWorkerConfig.json @@ -4,8 +4,9 @@ { "storageTypeCode": "OSS", "fileSourceTypeCode": "TENCENT_CLOUD_COS", - "name": "腾讯云COS", - "iconPath": "tencent_cloud.png", + "name": "内部COS", + "enabled": true, + "iconPath": "internal_cos.png", "staticParamList": [ { "name": "end_point_domain", @@ -167,6 +168,7 @@ "storageTypeCode": "OSS", "fileSourceTypeCode": "BLUEKING_ARTIFACTORY", "name": "蓝鲸制品库", + "enabled": true, "iconPath": "bk_artifactory.png", "staticParamList": [ { diff --git a/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/application.yml b/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/application.yml index f323171ad8..aa1e348b6a 100644 --- a/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/application.yml +++ b/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/application.yml @@ -5,6 +5,9 @@ spring: multipart: max-file-size: 5GB max-request-size: 5GB + mvc: + pathmatch: + matching-strategy: ant_path_matcher management: endpoints: @@ -48,3 +51,7 @@ server: charset: UTF-8 enabled: true force: true + +job: + file-worker: + enabled-file-source-types: BLUEKING_ARTIFACTORY diff --git a/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/bootstrap.yml b/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/bootstrap.yml index 67fbda41df..5e9815400c 100644 --- a/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/bootstrap.yml +++ b/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/bootstrap.yml @@ -11,6 +11,8 @@ spring: name: ${JOB_APPLICATION_CONFIGMAP_NAME} - namespace: ${KUBERNETES_NAMESPACE} name: ${JOB_COMMON_CONFIGMAP_NAME} + - namespace: ${KUBERNETES_NAMESPACE} + name: ${JOB_COMMON_REFRESHABLE_CONFIGMAP_NAME} logging: pattern: level: "%5p [${spring.application.name:},%X{traceId:-},%X{spanId:-}]" diff --git a/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/i18n/message.properties b/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/i18n/message.properties index e028c76106..5414b99b69 100644 --- a/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/i18n/message.properties +++ b/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/i18n/message.properties @@ -1,11 +1,11 @@ # -# Tencent is pleased to support the open source community by making BK-JOB����������ҵƽ̨ available. +# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. # -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +# Copyright (C) 2021 Tencent. All rights reserved. # -# BK-JOB����������ҵƽ̨ is licensed under the MIT License. +# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. # -# License for BK-JOB����������ҵƽ̨: +# License for BK-JOB蓝鲸智云作业平台: # -------------------------------------------------------------------- # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated # documentation files (the "Software"), to deal in the Software without restriction, including without limitation @@ -21,4 +21,3 @@ # CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # - diff --git a/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/i18n/message_en.properties b/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/i18n/message_en.properties index e028c76106..5414b99b69 100644 --- a/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/i18n/message_en.properties +++ b/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/i18n/message_en.properties @@ -1,11 +1,11 @@ # -# Tencent is pleased to support the open source community by making BK-JOB����������ҵƽ̨ available. +# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. # -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +# Copyright (C) 2021 Tencent. All rights reserved. # -# BK-JOB����������ҵƽ̨ is licensed under the MIT License. +# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. # -# License for BK-JOB����������ҵƽ̨: +# License for BK-JOB蓝鲸智云作业平台: # -------------------------------------------------------------------- # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated # documentation files (the "Software"), to deal in the Software without restriction, including without limitation @@ -21,4 +21,3 @@ # CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # - diff --git a/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/i18n/message_en_US.properties b/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/i18n/message_en_US.properties index e028c76106..5414b99b69 100644 --- a/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/i18n/message_en_US.properties +++ b/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/i18n/message_en_US.properties @@ -1,11 +1,11 @@ # -# Tencent is pleased to support the open source community by making BK-JOB����������ҵƽ̨ available. +# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. # -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +# Copyright (C) 2021 Tencent. All rights reserved. # -# BK-JOB����������ҵƽ̨ is licensed under the MIT License. +# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. # -# License for BK-JOB����������ҵƽ̨: +# License for BK-JOB蓝鲸智云作业平台: # -------------------------------------------------------------------- # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated # documentation files (the "Software"), to deal in the Software without restriction, including without limitation @@ -21,4 +21,3 @@ # CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # - diff --git a/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/i18n/message_zh.properties b/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/i18n/message_zh.properties index e028c76106..5414b99b69 100644 --- a/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/i18n/message_zh.properties +++ b/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/i18n/message_zh.properties @@ -1,11 +1,11 @@ # -# Tencent is pleased to support the open source community by making BK-JOB����������ҵƽ̨ available. +# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. # -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +# Copyright (C) 2021 Tencent. All rights reserved. # -# BK-JOB����������ҵƽ̨ is licensed under the MIT License. +# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. # -# License for BK-JOB����������ҵƽ̨: +# License for BK-JOB蓝鲸智云作业平台: # -------------------------------------------------------------------- # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated # documentation files (the "Software"), to deal in the Software without restriction, including without limitation @@ -21,4 +21,3 @@ # CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # - diff --git a/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/i18n/message_zh_CN.properties b/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/i18n/message_zh_CN.properties index e028c76106..5414b99b69 100644 --- a/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/i18n/message_zh_CN.properties +++ b/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/i18n/message_zh_CN.properties @@ -1,11 +1,11 @@ # -# Tencent is pleased to support the open source community by making BK-JOB����������ҵƽ̨ available. +# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. # -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +# Copyright (C) 2021 Tencent. All rights reserved. # -# BK-JOB����������ҵƽ̨ is licensed under the MIT License. +# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. # -# License for BK-JOB����������ҵƽ̨: +# License for BK-JOB蓝鲸智云作业平台: # -------------------------------------------------------------------- # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated # documentation files (the "Software"), to deal in the Software without restriction, including without limitation @@ -21,4 +21,3 @@ # CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # - diff --git a/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/tencent_cloud.png b/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/internal_cos.png similarity index 100% rename from src/backend/job-file-worker/boot-job-file-worker/src/main/resources/tencent_cloud.png rename to src/backend/job-file-worker/boot-job-file-worker/src/main/resources/internal_cos.png diff --git a/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/logback-spring.xml b/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/logback-spring.xml index 22debb699b..6c2789268a 100644 --- a/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/logback-spring.xml +++ b/src/backend/job-file-worker/boot-job-file-worker/src/main/resources/logback-spring.xml @@ -1,112 +1,13 @@ logback - - + - - - + + + - - ${FILE_WORKER_LOG_FILE} - - ${FILE_WORKER_LOG_FILE}-%d{yyyyMMdd_HH}.%i.log - 1GB - 120 - 10GB - - - ${FILE_LOG_PATTERN} - UTF-8 - - - - ${ERROR_LOG_FILE} - - ${ERROR_LOG_FILE}-%d{yyyy-MM-dd}.%i.log - 1GB - 120 - 10GB - - - ${FILE_LOG_PATTERN} - UTF-8 - - - - - ${FILE_LOG_PATTERN} - utf8 - - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/backend/job-file-worker/build.gradle b/src/backend/job-file-worker/build.gradle index 84eb60d981..eb8b1eff8b 100644 --- a/src/backend/job-file-worker/build.gradle +++ b/src/backend/job-file-worker/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-worker/service-job-file-worker/build.gradle b/src/backend/job-file-worker/service-job-file-worker/build.gradle index edff962efa..4794379c70 100644 --- a/src/backend/job-file-worker/service-job-file-worker/build.gradle +++ b/src/backend/job-file-worker/service-job-file-worker/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -23,7 +23,7 @@ */ dependencies { - api(files("libs/innercos-java-sdk-5.6.9.jar")) + api(files("libs/innercos-java-sdk-5.6.10.jar")) api project(":commons:artifactory-sdk") api project(":job-file-worker:api-job-file-worker") api project(":job-file-worker-sdk:api-job-file-worker-sdk") @@ -37,8 +37,7 @@ dependencies { implementation group: 'org.apache.thrift', name: 'libthrift' implementation "commons-io:commons-io" implementation "commons-codec:commons-codec" - implementation 'io.springfox:springfox-swagger2' - implementation 'io.springfox:springfox-swagger-ui' + implementation 'io.springfox:springfox-boot-starter' implementation 'joda-time:joda-time' implementation 'org.bouncycastle:bcprov-jdk15on' implementation 'io.micrometer:micrometer-registry-prometheus' diff --git a/src/backend/job-file-worker/service-job-file-worker/libs/innercos-java-sdk-5.6.10.jar b/src/backend/job-file-worker/service-job-file-worker/libs/innercos-java-sdk-5.6.10.jar new file mode 100644 index 0000000000..f182540904 Binary files /dev/null and b/src/backend/job-file-worker/service-job-file-worker/libs/innercos-java-sdk-5.6.10.jar differ diff --git a/src/backend/job-file-worker/service-job-file-worker/libs/innercos-java-sdk-5.6.9.jar b/src/backend/job-file-worker/service-job-file-worker/libs/innercos-java-sdk-5.6.9.jar deleted file mode 100644 index 35f92780b6..0000000000 Binary files a/src/backend/job-file-worker/service-job-file-worker/libs/innercos-java-sdk-5.6.9.jar and /dev/null differ diff --git a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/api/FileResourceProxy.java b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/api/FileResourceProxy.java index 404e8431b3..74af090acd 100644 --- a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/api/FileResourceProxy.java +++ b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/api/FileResourceProxy.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,11 +27,11 @@ import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.model.InternalResponse; -import com.tencent.bk.job.file.worker.consts.FileSourceTypeEnum; -import com.tencent.bk.job.file.worker.cos.service.RemoteClient; +import com.tencent.bk.job.file_gateway.consts.FileSourceTypeEnum; import com.tencent.bk.job.file.worker.model.req.BaseReq; import com.tencent.bk.job.file.worker.model.req.ExecuteActionReq; import com.tencent.bk.job.file.worker.model.req.ListFileNodeReq; +import com.tencent.bk.job.file.worker.service.RemoteClient; import com.tencent.bk.job.file_gateway.model.resp.common.FileNodesDTO; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; @@ -60,7 +60,10 @@ public IFileResource chooseFileResource(BaseReq req) { } else if (FileSourceTypeEnum.BLUEKING_ARTIFACTORY.name().equals(req.getFileSourceTypeCode())) { return artifactoryFileResource; } else { - throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME, new String[]{"fileSourceTypeCode"}); + throw new InvalidParamException( + ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME, + new String[]{"fileSourceTypeCode"} + ); } } diff --git a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/api/FileTaskResourceProxy.java b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/api/FileTaskResourceProxy.java index 9817169b0a..b717049d2b 100644 --- a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/api/FileTaskResourceProxy.java +++ b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/api/FileTaskResourceProxy.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,9 +24,9 @@ package com.tencent.bk.job.file.worker.api; -import com.tencent.bk.job.file.worker.cos.service.FileTaskService; -import com.tencent.bk.job.file.worker.cos.service.RemoteClient; import com.tencent.bk.job.file.worker.model.req.BaseReq; +import com.tencent.bk.job.file.worker.service.FileTaskService; +import com.tencent.bk.job.file.worker.service.RemoteClient; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RestController; diff --git a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/artifactory/consts/ArtifactoryActionCodeEnum.java b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/artifactory/consts/ArtifactoryActionCodeEnum.java index 22b68bc7c9..7e2e072558 100644 --- a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/artifactory/consts/ArtifactoryActionCodeEnum.java +++ b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/artifactory/consts/ArtifactoryActionCodeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/artifactory/consts/ArtifactoryNodeTypeEnum.java b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/artifactory/consts/ArtifactoryNodeTypeEnum.java index 9f54e41874..f8f9ade07e 100644 --- a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/artifactory/consts/ArtifactoryNodeTypeEnum.java +++ b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/artifactory/consts/ArtifactoryNodeTypeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/artifactory/impl/ArtifactoryFileResourceImpl.java b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/artifactory/impl/ArtifactoryFileResourceImpl.java index 8a82b7a82d..4bc9546f14 100644 --- a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/artifactory/impl/ArtifactoryFileResourceImpl.java +++ b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/artifactory/impl/ArtifactoryFileResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -40,11 +40,11 @@ import com.tencent.bk.job.file.worker.artifactory.consts.ArtifactoryNodeTypeEnum; import com.tencent.bk.job.file.worker.artifactory.service.ArtifactoryBaseService; import com.tencent.bk.job.file.worker.artifactory.service.ArtifactoryRemoteClient; -import com.tencent.bk.job.file.worker.cos.service.MetaDataService; -import com.tencent.bk.job.file.worker.cos.service.RemoteClient; import com.tencent.bk.job.file.worker.model.req.BaseReq; import com.tencent.bk.job.file.worker.model.req.ExecuteActionReq; import com.tencent.bk.job.file.worker.model.req.ListFileNodeReq; +import com.tencent.bk.job.file.worker.service.MetaDataService; +import com.tencent.bk.job.file.worker.service.RemoteClient; import com.tencent.bk.job.file_gateway.model.resp.common.FileNodesDTO; import com.tencent.bk.job.file_gateway.model.resp.common.FileTreeNodeDef; import io.micrometer.core.instrument.util.StringUtils; @@ -98,7 +98,7 @@ private void fillProjectFileNodesDTO(FileNodesDTO fileNodesDTO, ListFileNodeReq ArtifactoryRemoteClient client = baseService.getArtifactoryClientFromBaseReq(req); List projectDTOList = client.listProject(); // 按名称搜索 - projectDTOList = projectDTOList.parallelStream().filter(projectDTO -> { + projectDTOList = projectDTOList.stream().filter(projectDTO -> { String displayName = projectDTO.getDisplayName(); String name = req.getName(); if (null == displayName) { @@ -126,7 +126,7 @@ private void fillProjectFileNodesDTO(FileNodesDTO fileNodesDTO, ListFileNodeReq mappedPageData.setStart(pageData.getStart()); mappedPageData.setPageSize(pageData.getPageSize()); mappedPageData.setTotal(pageData.getTotal()); - mappedPageData.setData(pageData.getData().parallelStream().map(projectDTO -> { + mappedPageData.setData(pageData.getData().stream().map(projectDTO -> { Map map = new HashMap<>(); map.put("name", projectDTO.getName()); map.put("displayName", projectDTO.getDisplayName()); @@ -179,7 +179,7 @@ private void fillRepoFileNodesDTO(FileNodesDTO fileNodesDTO, ListFileNodeReq req } }); // 字段映射 - List> repoMapList = repoList.parallelStream().map(repoDTO -> { + List> repoMapList = repoList.stream().map(repoDTO -> { Map map = new HashMap<>(); map.put("projectId", repoDTO.getProjectId()); map.put("name", repoDTO.getName()); @@ -258,7 +258,7 @@ private void fillNodeFileNodesDTO(FileNodesDTO fileNodesDTO, ListFileNodeReq req mappedPageData.setStart((pageData.getPageNumber() - 1) * pageData.getPageSize()); mappedPageData.setPageSize(pageData.getPageSize()); mappedPageData.setTotal(pageData.getTotalRecords()); - mappedPageData.setData(pageData.getRecords().parallelStream().map(nodeDTO -> { + mappedPageData.setData(pageData.getRecords().stream().map(nodeDTO -> { Map map = new HashMap<>(); map.put("projectId", nodeDTO.getProjectId()); map.put("repoName", nodeDTO.getRepoName()); diff --git a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/artifactory/service/ArtifactoryBaseService.java b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/artifactory/service/ArtifactoryBaseService.java index d1aace8649..9d72c3e773 100644 --- a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/artifactory/service/ArtifactoryBaseService.java +++ b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/artifactory/service/ArtifactoryBaseService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,8 +24,9 @@ package com.tencent.bk.job.file.worker.artifactory.service; -import com.tencent.bk.job.file.worker.model.req.BaseReq; import com.tencent.bk.job.common.model.dto.CommonCredential; +import com.tencent.bk.job.file.worker.model.req.BaseReq; +import com.tencent.bk.job.file_gateway.consts.FileSourceInfoConsts; import io.micrometer.core.instrument.MeterRegistry; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -45,7 +46,11 @@ public ArtifactoryBaseService(MeterRegistry meterRegistry) { public ArtifactoryRemoteClient getArtifactoryClientFromBaseReq(BaseReq req) { CommonCredential credential = req.getCredential(); Map fileSourceInfoMap = req.getFileSourceInfoMap(); - return new ArtifactoryRemoteClient((String) fileSourceInfoMap.get("base_url"), credential.getUsername(), - credential.getPassword(), meterRegistry); + return new ArtifactoryRemoteClient( + (String) fileSourceInfoMap.get(FileSourceInfoConsts.KEY_BK_ARTIFACTORY_BASE_URL), + credential.getUsername(), + credential.getPassword(), + meterRegistry + ); } } diff --git a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/artifactory/service/ArtifactoryRemoteClient.java b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/artifactory/service/ArtifactoryRemoteClient.java index 1b0ec79002..407836cc0a 100644 --- a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/artifactory/service/ArtifactoryRemoteClient.java +++ b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/artifactory/service/ArtifactoryRemoteClient.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,8 +27,8 @@ import com.tencent.bk.job.common.artifactory.model.dto.NodeDTO; import com.tencent.bk.job.common.artifactory.sdk.ArtifactoryClient; import com.tencent.bk.job.common.exception.ServiceException; -import com.tencent.bk.job.file.worker.cos.service.RemoteClient; import com.tencent.bk.job.file.worker.model.FileMetaData; +import com.tencent.bk.job.file.worker.service.RemoteClient; import io.micrometer.core.instrument.MeterRegistry; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Pair; diff --git a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/consts/FileSourceTypeEnum.java b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/consts/FileSourceTypeEnum.java deleted file mode 100644 index 74730b4070..0000000000 --- a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/consts/FileSourceTypeEnum.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.file.worker.consts; - -public enum FileSourceTypeEnum { - TENCENT_CLOUD_COS, - BLUEKING_ARTIFACTORY -} diff --git a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/InnerCOSCustomInfo.java b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/InnerCOSCustomInfo.java index 33d57e0c4f..144b9b3b9c 100644 --- a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/InnerCOSCustomInfo.java +++ b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/InnerCOSCustomInfo.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/JobTencentInnerCOSClient.java b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/JobTencentInnerCOSClient.java index 38c72e69bb..aa1ebf700b 100644 --- a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/JobTencentInnerCOSClient.java +++ b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/JobTencentInnerCOSClient.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/TencentInnerCOSUtil.java b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/TencentInnerCOSUtil.java index 3dfb92beae..4120a5daa4 100644 --- a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/TencentInnerCOSUtil.java +++ b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/TencentInnerCOSUtil.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/consts/COSActionCodeEnum.java b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/consts/COSActionCodeEnum.java index cb334ea102..7926e9583e 100644 --- a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/consts/COSActionCodeEnum.java +++ b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/consts/COSActionCodeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/consts/COSNodeTypeEnum.java b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/consts/COSNodeTypeEnum.java index cbf9c4602f..c0ca0ad896 100644 --- a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/consts/COSNodeTypeEnum.java +++ b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/consts/COSNodeTypeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/impl/COSFileResourceImpl.java b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/impl/COSFileResourceImpl.java index b45d3b59ee..f610b85b2c 100644 --- a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/impl/COSFileResourceImpl.java +++ b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/impl/COSFileResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -35,15 +35,15 @@ import com.tencent.bk.job.file.worker.cos.JobTencentInnerCOSClient; import com.tencent.bk.job.file.worker.cos.consts.COSActionCodeEnum; import com.tencent.bk.job.file.worker.cos.consts.COSNodeTypeEnum; -import com.tencent.bk.job.file.worker.cos.service.COSBaseService; -import com.tencent.bk.job.file.worker.cos.service.COSRemoteClient; -import com.tencent.bk.job.file.worker.cos.service.MetaDataService; -import com.tencent.bk.job.file.worker.cos.service.RemoteClient; import com.tencent.bk.job.file.worker.model.BucketDTO; import com.tencent.bk.job.file.worker.model.FileDTO; import com.tencent.bk.job.file.worker.model.req.BaseReq; import com.tencent.bk.job.file.worker.model.req.ExecuteActionReq; import com.tencent.bk.job.file.worker.model.req.ListFileNodeReq; +import com.tencent.bk.job.file.worker.service.COSBaseService; +import com.tencent.bk.job.file.worker.service.COSRemoteClient; +import com.tencent.bk.job.file.worker.service.MetaDataService; +import com.tencent.bk.job.file.worker.service.RemoteClient; import com.tencent.bk.job.file_gateway.model.resp.common.FileNodesDTO; import com.tencent.bk.job.file_gateway.model.resp.common.FileTreeNodeDef; import com.tencent.cos.model.Bucket; @@ -51,6 +51,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; +import org.slf4j.helpers.MessageFormatter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -79,7 +80,7 @@ public List listBucket(ListFileNodeReq req) { JobTencentInnerCOSClient jobTencentInnerCOSClient = cosBaseService.getCOSClientFromBaseReq(req); List bucketList = jobTencentInnerCOSClient.listBuckets(); // 根据name搜索 - bucketList = bucketList.parallelStream().filter( + bucketList = bucketList.stream().filter( bucketDTO -> { String name = req.getName(); String bucketName = bucketDTO.getName(); @@ -139,7 +140,11 @@ public Boolean deleteBucket(String bucketName, BaseReq req) { jobTencentInnerCOSClient.deleteBucket(bucketName); return true; } catch (Exception e) { - log.error("Fail to delete bucket {}", bucketName, e); + String msg = MessageFormatter.format( + "Fail to delete bucket {}", + bucketName + ).getMessage(); + log.error(msg, e); throw new InvalidParamException(e.getMessage(), ErrorCode.FAIL_TO_REQUEST_THIRD_FILE_SOURCE_DELETE_BUCKET); } } @@ -152,7 +157,14 @@ public Boolean deleteBucketFile(String bucketName, String key, BaseReq req) { jobTencentInnerCOSClient.deleteObject(bucketName, key); return true; } catch (Exception e) { - log.error("Fail to delete bucket {} file:{}", bucketName, key, e); + String msg = MessageFormatter.arrayFormat( + "Fail to delete bucket {} file:{}", + new String[]{ + bucketName, + key + } + ).getMessage(); + log.error(msg, e); throw new InvalidParamException(e.getMessage(), ErrorCode.FAIL_TO_REQUEST_THIRD_FILE_SOURCE_DELETE_OBJECT); } } @@ -269,7 +281,7 @@ private void fillBucketFileNodesDTO(FileNodesDTO fileNodesDTO, ListFileNodeReq r List bucketDTOList = listBucket(req); // 排序：创建时间降序 bucketDTOList.sort((o1, o2) -> o2.getCreateDate().compareTo(o1.getCreateDate())); - List> mapData = bucketDTOList.parallelStream().map(bucketDTO -> { + List> mapData = bucketDTOList.stream().map(bucketDTO -> { Map map = new HashMap<>(); map.put("name", bucketDTO.getName()); map.put("type", bucketDTO.getXCosAcl()); @@ -317,7 +329,7 @@ private void fillFileFileNodesDTO(FileNodesDTO fileNodesDTO, ListFileNodeReq req fileDTOList.addAll(dirList); fileDTOList.addAll(fileList); // 分页 - List> mapData = fileDTOList.parallelStream().map(fileDTO -> { + List> mapData = fileDTOList.stream().map(fileDTO -> { Map map = new HashMap<>(); String fileName = fileDTO.getKey(); map.put("name", fileName); diff --git a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/service/COSBaseService.java b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/service/COSBaseService.java similarity index 94% rename from src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/service/COSBaseService.java rename to src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/service/COSBaseService.java index 5b7450c13c..458db575cb 100644 --- a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/service/COSBaseService.java +++ b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/service/COSBaseService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,11 +22,11 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.file.worker.cos.service; +package com.tencent.bk.job.file.worker.service; +import com.tencent.bk.job.common.model.dto.CommonCredential; import com.tencent.bk.job.file.worker.cos.JobTencentInnerCOSClient; import com.tencent.bk.job.file.worker.model.req.BaseReq; -import com.tencent.bk.job.common.model.dto.CommonCredential; import lombok.extern.slf4j.Slf4j; import org.springframework.stereotype.Service; diff --git a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/service/COSRemoteClient.java b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/service/COSRemoteClient.java similarity index 95% rename from src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/service/COSRemoteClient.java rename to src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/service/COSRemoteClient.java index 4e1d15878b..261a5ec196 100644 --- a/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/cos/service/COSRemoteClient.java +++ b/src/backend/job-file-worker/service-job-file-worker/src/main/java/com/tencent/bk/job/file/worker/service/COSRemoteClient.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.file.worker.cos.service; +package com.tencent.bk.job.file.worker.service; import com.tencent.bk.job.file.worker.cos.JobTencentInnerCOSClient; import com.tencent.bk.job.file.worker.model.FileMetaData; diff --git a/src/backend/job-gateway/build.gradle b/src/backend/job-gateway/build.gradle index 6e067486f7..39740ef8ea 100644 --- a/src/backend/job-gateway/build.gradle +++ b/src/backend/job-gateway/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -36,14 +36,14 @@ dependencies { api project(':commons:common') api project(":commons:common-i18n") api project(":commons:common-security") + api project(":commons:esb-sdk") api project(":commons:paas-sdk") api project(":commons:common-service") implementation("org.springframework.boot:spring-boot-autoconfigure") implementation 'org.springframework:spring-webflux' implementation 'org.springframework.cloud:spring-cloud-starter-bootstrap' implementation 'org.springframework.cloud:spring-cloud-starter-gateway' - // CVE-2022-22947 - implementation 'org.springframework.cloud:spring-cloud-gateway-server:3.0.7' + implementation 'org.springframework.cloud:spring-cloud-gateway-server' implementation 'redis.clients:jedis' implementation 'commons-io:commons-io' implementation 'org.springframework.boot:spring-boot-starter-validation' diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/JobGatewayBootApplication.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/JobGatewayBootApplication.java index 1aba88aa35..db6e6935bb 100644 --- a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/JobGatewayBootApplication.java +++ b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/JobGatewayBootApplication.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,39 +24,48 @@ package com.tencent.bk.job.gateway; +import com.tencent.bk.job.common.service.boot.JobBootApplication; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.availability.ApplicationAvailabilityAutoConfiguration; +import org.springframework.boot.autoconfigure.web.embedded.NettyWebServerFactoryCustomizer; import org.springframework.boot.web.embedded.netty.NettyReactiveWebServerFactory; -import org.springframework.boot.web.reactive.server.ReactiveWebServerFactory; import org.springframework.boot.web.server.WebServer; -import org.springframework.cloud.client.discovery.EnableDiscoveryClient; +import org.springframework.cloud.openfeign.EnableFeignClients; import org.springframework.http.server.reactive.HttpHandler; import javax.annotation.PostConstruct; import javax.annotation.PreDestroy; /** - * @date 2019/09/20 + * Job Gateway Spring Boot Application */ -@SpringBootApplication(scanBasePackages = "com.tencent.bk.job") -@EnableDiscoveryClient +@JobBootApplication(scanBasePackages = "com.tencent.bk.job.gateway", + exclude = {ApplicationAvailabilityAutoConfiguration.class}, + excludeName = {"org.springframework.cloud.kubernetes.client.discovery.KubernetesDiscoveryClientAutoConfiguration"}) @Slf4j +@EnableFeignClients public class JobGatewayBootApplication { - private HttpHandler httpHandler; + private final HttpHandler httpHandler; private WebServer httpWebServer; + private final NettyWebServerFactoryCustomizer nettyWebServerFactoryCustomizer; + @Value("${server.http.enabled}") private Boolean httpEnabled; @Value("${server.http.port:}") private Integer httpPort; - public JobGatewayBootApplication(@Autowired HttpHandler httpHandler) { + public JobGatewayBootApplication(@Autowired + HttpHandler httpHandler, + @Autowired(required = false) + NettyWebServerFactoryCustomizer nettyWebServerFactoryCustomizer) { this.httpHandler = httpHandler; + this.nettyWebServerFactoryCustomizer = nettyWebServerFactoryCustomizer; } public static void main(String[] args) { @@ -66,7 +75,10 @@ public static void main(String[] args) { @PostConstruct public void startHttpWebServer() { if (httpEnabled && httpPort != null) { - ReactiveWebServerFactory factory = new NettyReactiveWebServerFactory(httpPort); + NettyReactiveWebServerFactory factory = new NettyReactiveWebServerFactory(httpPort); + if (nettyWebServerFactoryCustomizer != null) { + nettyWebServerFactoryCustomizer.customize(factory); + } this.httpWebServer = factory.getWebServer(this.httpHandler); this.httpWebServer.start(); } diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/common/consts/EsbLangHeader.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/common/consts/EsbLangHeader.java index 7e3378cd19..c1d159013f 100644 --- a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/common/consts/EsbLangHeader.java +++ b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/common/consts/EsbLangHeader.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/common/consts/WebLangCookie.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/common/consts/WebLangCookie.java index 6873326b1b..11b20ce77c 100644 --- a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/common/consts/WebLangCookie.java +++ b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/common/consts/WebLangCookie.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/common/util/UrlUtil.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/common/util/UrlUtil.java index 4542d9f9e0..5011438295 100644 --- a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/common/util/UrlUtil.java +++ b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/common/util/UrlUtil.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/config/ActuatorSecurityConfig.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/config/ActuatorSecurityConfig.java index 701d8cc51c..abe1c86a42 100644 --- a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/config/ActuatorSecurityConfig.java +++ b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/config/ActuatorSecurityConfig.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/config/BkConfig.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/config/BkConfig.java deleted file mode 100644 index 16c1e3ab0c..0000000000 --- a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/config/BkConfig.java +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.gateway.config; - -import lombok.Getter; -import lombok.Setter; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Configuration; - -@Getter -@Setter -@Configuration -public class BkConfig { - //---------------------------- Esb Config --------------------------------- - /** - * 是否对接ESB测试环境,内部环境使用 - */ - @Value("${esb.use.test.env:false}") - private boolean useEsbTestEnv; - /** - * 应用code - */ - @Value("${app.code}") - private String appCode; - - /** - * ESB App Secret - */ - @Value("${app.secret}") - private String appSecret; - - /** - * ESB service url - */ - @Value("${esb.service.url:}") - private String esbUrl; - - //---------------------------- Login Config --------------------------------- - /** - * 蓝鲸标准的登录url - */ - @Value("${paas.login.url:}") - private String loginUrl; - - /** - * 是否使用第三方登录系统 - */ - @Value("${paas.login.custom.enabled:false}") - private boolean customPaasLoginEnabled; - - /** - * 第三方登录系统用户token的cookie名称 - */ - @Value("${paas.login.custom.token-name:bk_token}") - private String customLoginToken; - - /** - * 第三方登录系统登录url - */ - @Value("${paas.login.custom.login-url:}") - private String customLoginUrl; - - /** - * 第三方登录系统API url，用于根据token获取用户信息 - */ - @Value("${paas.login.custom.api-url:}") - private String customLoginApiUrl; - - - //---------------------------- Job Config --------------------------------- - @Value("${job.edition:ee}") - private String jobEdition; - - /** - * 作业平台web访问地址，可配置多个，用","分隔 - */ - @Value("${job.web.url:}") - private String jobWebUrl; - - @Value("${swagger.url:swagger.job.com}") - private String swaggerUrl; - - //---------------------------- License Config --------------------------------- - /** - * license 校验服务url - */ - @Value("${license.enabled:false}") - private boolean enableLicenseValidate; - /** - * license 校验服务url - */ - @Value("${license.service.url:}") - private String licenseCheckServiceUrl; - - /** - * license 校验服务url - */ - @Value("${license.file:}") - private String licenseFilePath; - - /** - * license 校验执行次数 - */ - @Value("${license.check.retry-times:3}") - private Integer licenseCheckRetryTimes; - - /** - * license 校验重试间隔，单位秒 - */ - @Value("${license.check.retry-interval:1}") - private Integer licenseCheckRetryInterval; - - -} diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/config/BkGatewayConfig.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/config/BkGatewayConfig.java new file mode 100644 index 0000000000..e0fa0f93a0 --- /dev/null +++ b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/config/BkGatewayConfig.java @@ -0,0 +1,54 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.gateway.config; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Configuration; + +@Getter +@Setter +@Configuration +public class BkGatewayConfig { + + /** + * 蓝鲸网关认证 + */ + @Value("${job.bkApiGateway.enabled:true}") + private boolean enabled; + + /** + * jwtPublicKey获取策略，重试：retry, 终止启动：abort + */ + @Value("${job.bkApiGateway.jwtPublicKey.get.failPolicy:retry}") + private String jwtPublicKeyFailPolicy; + + /** + * 网关名称 + */ + @Value("${job.bkApiGateway.gatewayName: bk-job}") + private String gatewayName; +} diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/config/CorsConfig.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/config/CorsConfig.java index 0207ce875f..ab66d4f917 100644 --- a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/config/CorsConfig.java +++ b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/config/CorsConfig.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,9 +24,9 @@ package com.tencent.bk.job.gateway.config; +import com.tencent.bk.job.common.service.config.JobCommonConfig; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.web.cors.CorsConfiguration; @@ -67,11 +67,11 @@ private static String buildAllowOriginFromUrl(String url) { } @Bean - public CorsWebFilter corsFilter(@Autowired BkConfig bkConfig) { + public CorsWebFilter corsFilter(JobCommonConfig jobCommonConfig) { UrlBasedCorsConfigurationSource source = new UrlBasedCorsConfigurationSource(new PathPatternParser()); List allowOrigins = new ArrayList<>(); - if (StringUtils.isNotBlank(bkConfig.getJobWebUrl())) { - for (String webUrl : bkConfig.getJobWebUrl().split(",")) { + if (StringUtils.isNotBlank(jobCommonConfig.getJobWebUrl())) { + for (String webUrl : jobCommonConfig.getJobWebUrl().split(",")) { if (StringUtils.isNotBlank(webUrl)) { String allowOrigin = buildAllowOriginFromUrl(webUrl.trim()); if (StringUtils.isNotBlank(allowOrigin)) { diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/config/GlobalExceptionConfig.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/config/GlobalExceptionConfig.java index 715f6bc93f..e3493b660e 100644 --- a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/config/GlobalExceptionConfig.java +++ b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/config/GlobalExceptionConfig.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/config/Ipv6ReactiveLoadBalancerClientConfiguration.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/config/Ipv6ReactiveLoadBalancerClientConfiguration.java new file mode 100644 index 0000000000..813d484c0a --- /dev/null +++ b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/config/Ipv6ReactiveLoadBalancerClientConfiguration.java @@ -0,0 +1,124 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.gateway.config; + +import org.apache.commons.lang3.StringUtils; +import org.springframework.cloud.client.ServiceInstance; +import org.springframework.cloud.client.loadbalancer.LoadBalancerProperties; +import org.springframework.cloud.client.loadbalancer.LoadBalancerUriTools; +import org.springframework.cloud.gateway.config.GatewayLoadBalancerProperties; +import org.springframework.cloud.gateway.filter.ReactiveLoadBalancerClientFilter; +import org.springframework.cloud.loadbalancer.support.LoadBalancerClientFactory; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import java.net.URI; +import java.util.Map; + +/** + * 用于修复spring-cloud-gateway在IPv6环境下的负载均衡路由报错问题 + */ +@Configuration +public class Ipv6ReactiveLoadBalancerClientConfiguration { + + @Bean + public ReactiveLoadBalancerClientFilter ipv6GatewayLoadBalancerClientFilter( + LoadBalancerClientFactory clientFactory, + GatewayLoadBalancerProperties properties, + LoadBalancerProperties loadBalancerProperties + ) { + return new Ipv6ReactiveLoadBalancerClientFilter(clientFactory, properties, loadBalancerProperties); + } + + static class Ipv6ReactiveLoadBalancerClientFilter extends ReactiveLoadBalancerClientFilter { + + Ipv6ReactiveLoadBalancerClientFilter( + LoadBalancerClientFactory clientFactory, + GatewayLoadBalancerProperties properties, + LoadBalancerProperties loadBalancerProperties) { + super(clientFactory, properties, loadBalancerProperties); + } + + @Override + protected URI reconstructURI(ServiceInstance serviceInstance, URI original) { + return LoadBalancerUriTools.reconstructURI( + new Ipv6CapableDelegatingServiceInstance(serviceInstance), + original + ); + } + + } + + static class Ipv6CapableDelegatingServiceInstance implements ServiceInstance { + + final ServiceInstance delegate; + + Ipv6CapableDelegatingServiceInstance(ServiceInstance delegate) { + this.delegate = delegate; + } + + @Override + public String getServiceId() { + return delegate.getServiceId(); + } + + @Override + public String getHost() { + return getAvailableIpv6Host(delegate.getHost()); + } + + private String getAvailableIpv6Host(String host) { + if (StringUtils.isNotBlank(host) && host.contains(":") && !host.startsWith("[")) { + return "[" + host + "]"; + } + return host; + } + + @Override + public int getPort() { + return delegate.getPort(); + } + + @Override + public boolean isSecure() { + return delegate.isSecure(); + } + + @Override + public URI getUri() { + return delegate.getUri(); + } + + @Override + public Map getMetadata() { + return delegate.getMetadata(); + } + + @Override + public String getScheme() { + return delegate.getScheme(); + } + } +} diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/config/LicenseConfig.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/config/LicenseConfig.java new file mode 100644 index 0000000000..d526464ce8 --- /dev/null +++ b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/config/LicenseConfig.java @@ -0,0 +1,65 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.gateway.config; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.Configuration; + +@Getter +@Setter +@Configuration +public class LicenseConfig { + /** + * license 校验服务url + */ + @Value("${license.enabled:false}") + private boolean enableLicenseValidate; + /** + * license 校验服务url + */ + @Value("${license.service.url:}") + private String licenseCheckServiceUrl; + + /** + * license 校验服务url + */ + @Value("${license.file:}") + private String licenseFilePath; + + /** + * license 校验执行次数 + */ + @Value("${license.check.retry-times:3}") + private Integer licenseCheckRetryTimes; + + /** + * license 校验重试间隔，单位秒 + */ + @Value("${license.check.retry-interval:1}") + private Integer licenseCheckRetryInterval; + +} diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/config/LocaleSupportConfig.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/config/LocaleSupportConfig.java index b081a902b3..3831a221a7 100644 --- a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/config/LocaleSupportConfig.java +++ b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/config/LocaleSupportConfig.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/config/RestConfig.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/config/RestConfig.java index acf8959497..db4750193e 100644 --- a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/config/RestConfig.java +++ b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/config/RestConfig.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,7 @@ package com.tencent.bk.job.gateway.config; +import lombok.extern.slf4j.Slf4j; import org.apache.http.config.Registry; import org.apache.http.config.RegistryBuilder; import org.apache.http.conn.socket.ConnectionSocketFactory; @@ -47,6 +48,7 @@ import java.security.cert.X509Certificate; import java.util.List; +@Slf4j @Configuration public class RestConfig { @Bean @@ -61,8 +63,12 @@ public RestTemplate restTemplate() { try { SSLContextBuilder builder = new SSLContextBuilder(); builder.loadTrustMaterial(null, (X509Certificate[] x509Certificates, String s) -> true); - SSLConnectionSocketFactory socketFactory = new SSLConnectionSocketFactory(builder.build(), new String[]{ - "SSLv2Hello", "SSLv3", "TLSv1", "TLSv1.2"}, null, NoopHostnameVerifier.INSTANCE); + SSLConnectionSocketFactory socketFactory = new SSLConnectionSocketFactory( + builder.build(), + null, + null, + NoopHostnameVerifier.INSTANCE + ); Registry registry = RegistryBuilder.create() .register("http", new PlainConnectionSocketFactory()) .register("https", socketFactory).build(); @@ -72,9 +78,8 @@ public RestTemplate restTemplate() { HttpClients.custom().setSSLSocketFactory(socketFactory).setConnectionManager(phccm) .setConnectionManagerShared(true).build(); factory.setHttpClient(httpClient); - } catch (Exception e) { - + log.error("Fail to init httpClient", e); } RestTemplate restTemplate = new RestTemplate(factory); diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/config/RouteConfig.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/config/RouteConfig.java index 88a8d0ee7f..b63fdc7f10 100644 --- a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/config/RouteConfig.java +++ b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/config/RouteConfig.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -46,6 +46,7 @@ import java.util.List; +import static com.tencent.bk.job.common.i18n.locale.LocaleUtils.BLUEKING_LANG_HEADER; import static org.springframework.web.reactive.function.server.RequestPredicates.GET; @Slf4j @@ -95,6 +96,7 @@ Mono getUserByBkToken(ServerRequest request) { String tokenCookieName = loginService.getCookieNameForToken(); List cookieList = request.headers().header("cookie"); + String lang = request.headers().firstHeader(BLUEKING_LANG_HEADER); List bkTokenList = RequestUtil.getCookieValuesFromCookies(cookieList, tokenCookieName); if (CollectionUtils.isEmpty(bkTokenList)) { @@ -107,7 +109,7 @@ Mono getUserByBkToken(ServerRequest request) { BkUserDTO user = null; // 遍历所有传入token找出当前环境的 for (String bkToken : bkTokenList) { - user = loginService.getUser(bkToken); + user = loginService.getUser(bkToken, lang); if (user != null) { break; } diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/config/SdkClientConfig.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/config/SdkClientConfig.java deleted file mode 100644 index 9533ecc21d..0000000000 --- a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/config/SdkClientConfig.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.gateway.config; - -import com.tencent.bk.job.common.paas.login.CustomLoginClient; -import com.tencent.bk.job.common.paas.login.EELoginClient; -import com.tencent.bk.job.common.paas.login.ILoginClient; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; - -@Configuration -@Slf4j -public class SdkClientConfig { - @Bean - @ConditionalOnProperty(value = "paas.login.custom.enabled", havingValue = "true") - public ILoginClient innerLoginClient(@Autowired BkConfig bkConfig) { - log.info("Init custom login client"); - return new CustomLoginClient(bkConfig.getCustomLoginApiUrl()); - } - - @Bean - @ConditionalOnProperty(value = "paas.login.custom.enabled", havingValue = "false", matchIfMissing = true) - @Primary - public ILoginClient standardLoginClient(@Autowired BkConfig bkConfig) { - log.info("Init standard login client"); - return new EELoginClient(bkConfig.getEsbUrl(), bkConfig.getAppCode(), bkConfig.getAppSecret(), - bkConfig.isUseEsbTestEnv()); - } -} diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/config/SwaggerConfig.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/config/SwaggerConfig.java index e4a658af48..24c3577c34 100644 --- a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/config/SwaggerConfig.java +++ b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/config/SwaggerConfig.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,34 +24,40 @@ package com.tencent.bk.job.gateway.config; +import com.tencent.bk.job.common.service.config.JobCommonConfig; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Profile; import springfox.documentation.builders.PathSelectors; import springfox.documentation.builders.RequestHandlerSelectors; +import springfox.documentation.oas.annotations.EnableOpenApi; import springfox.documentation.spi.DocumentationType; import springfox.documentation.spring.web.plugins.Docket; import java.util.Arrays; import java.util.HashSet; +/** + * Swagger 配置 + */ @Configuration -//@EnableSwagger2 -@Profile({"dev", "test", "local"}) +@EnableOpenApi +@Profile({"dev", "local"}) public class SwaggerConfig { - private final BkConfig bkConfig; + private final JobCommonConfig jobCommonConfig; @Autowired - public SwaggerConfig(BkConfig bkConfig) { - this.bkConfig = bkConfig; + public SwaggerConfig(JobCommonConfig jobCommonConfig) { + this.jobCommonConfig = jobCommonConfig; } @Bean public Docket api() { - return new Docket(DocumentationType.SWAGGER_2) - .host(bkConfig.getSwaggerUrl() + "/job-gateway") + return new Docket(DocumentationType.OAS_30) + .host(jobCommonConfig.getSwaggerUrl()) + .pathMapping("job-gateway") .protocols(new HashSet<>(Arrays.asList("http", "https"))) .select() .apis(RequestHandlerSelectors.basePackage("com.tencent.bk.job.gateway.api")) diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/LogoutGatewayFilterFactory.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/LogoutGatewayFilterFactory.java index bf5b237f94..21a89d8f7e 100644 --- a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/LogoutGatewayFilterFactory.java +++ b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/LogoutGatewayFilterFactory.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/esb/AddEsbLangHeaderGatewayFilterFactory.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/esb/AddEsbLangHeaderGatewayFilterFactory.java index 7a0682533a..9184ead92c 100644 --- a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/esb/AddEsbLangHeaderGatewayFilterFactory.java +++ b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/esb/AddEsbLangHeaderGatewayFilterFactory.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/esb/CheckEsbJwtGatewayFilterFactory.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/esb/CheckEsbJwtGatewayFilterFactory.java deleted file mode 100644 index 216aff2410..0000000000 --- a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/esb/CheckEsbJwtGatewayFilterFactory.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.gateway.filter.esb; - -import com.tencent.bk.job.common.constant.JobCommonHeaders; -import com.tencent.bk.job.common.util.RequestUtil; -import com.tencent.bk.job.gateway.model.esb.EsbJwtInfo; -import com.tencent.bk.job.gateway.service.EsbJwtService; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.cloud.gateway.filter.GatewayFilter; -import org.springframework.cloud.gateway.filter.factory.AbstractGatewayFilterFactory; -import org.springframework.http.HttpStatus; -import org.springframework.http.server.reactive.ServerHttpRequest; -import org.springframework.http.server.reactive.ServerHttpResponse; -import org.springframework.stereotype.Component; - -/** - * ESB JWT 解析与校验，用于确认ESB-API调用方式来自于ESB - */ -@Slf4j -@Component -public class CheckEsbJwtGatewayFilterFactory - extends AbstractGatewayFilterFactory { - private EsbJwtService esbJwtService; - - @Autowired - public CheckEsbJwtGatewayFilterFactory(EsbJwtService esbJwtService) { - super(Config.class); - this.esbJwtService = esbJwtService; - } - - @Override - public GatewayFilter apply(Config config) { - return (exchange, chain) -> { - ServerHttpResponse response = exchange.getResponse(); - ServerHttpRequest request = exchange.getRequest(); - String token = RequestUtil.getHeaderValue(request, JobCommonHeaders.BK_GATEWAY_JWT); - if (StringUtils.isEmpty(token)) { - log.warn("Esb token is empty!"); - response.setStatusCode(HttpStatus.UNAUTHORIZED); - return response.setComplete(); - } - EsbJwtInfo authInfo = esbJwtService.extractFromJwt(token); - if (authInfo == null) { - log.warn("Untrusted esb request, request-id:{}", RequestUtil.getHeaderValue(request, - JobCommonHeaders.BK_GATEWAY_REQUEST_ID)); - response.setStatusCode(HttpStatus.UNAUTHORIZED); - return response.setComplete(); - } - - // set app code header - request.mutate().header(JobCommonHeaders.APP_CODE, new String[]{authInfo.getAppCode()}).build(); - request.mutate().header(JobCommonHeaders.USERNAME, new String[]{authInfo.getUsername()}).build(); - return chain.filter(exchange.mutate().request(request).build()); - }; - } - - static class Config { - - } - -} diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/esb/CheckOpenApiJwtGatewayFilterFactory.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/esb/CheckOpenApiJwtGatewayFilterFactory.java new file mode 100644 index 0000000000..26cef53808 --- /dev/null +++ b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/esb/CheckOpenApiJwtGatewayFilterFactory.java @@ -0,0 +1,155 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.gateway.filter.esb; + +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.constant.JobCommonHeaders; +import com.tencent.bk.job.common.crypto.util.RSAUtils; +import com.tencent.bk.job.common.esb.model.EsbResp; +import com.tencent.bk.job.common.security.autoconfigure.ServiceSecurityProperties; +import com.tencent.bk.job.common.service.SpringProfile; +import com.tencent.bk.job.common.util.JobContextUtil; +import com.tencent.bk.job.common.util.RequestUtil; +import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.gateway.model.esb.EsbJwtInfo; +import com.tencent.bk.job.gateway.service.OpenApiJwtService; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.cloud.gateway.filter.GatewayFilter; +import org.springframework.cloud.gateway.filter.factory.AbstractGatewayFilterFactory; +import org.springframework.core.io.buffer.DataBuffer; +import org.springframework.http.HttpStatus; +import org.springframework.http.server.reactive.ServerHttpRequest; +import org.springframework.http.server.reactive.ServerHttpResponse; +import org.springframework.stereotype.Component; +import reactor.core.publisher.Mono; + +import java.nio.charset.StandardCharsets; + +/** + * OPEN API JWT 解析与校验 + */ +@Slf4j +@Component +public class CheckOpenApiJwtGatewayFilterFactory + extends AbstractGatewayFilterFactory { + private final OpenApiJwtService openApiJwtService; + private final SpringProfile springProfile; + private final ServiceSecurityProperties securityProperties; + + @Autowired + public CheckOpenApiJwtGatewayFilterFactory(OpenApiJwtService openApiJwtService, + SpringProfile springProfile, + ServiceSecurityProperties securityProperties) { + super(Config.class); + this.openApiJwtService = openApiJwtService; + this.springProfile = springProfile; + this.securityProperties = securityProperties; + } + + @Override + public GatewayFilter apply(Config config) { + return (exchange, chain) -> { + ServerHttpResponse response = exchange.getResponse(); + ServerHttpRequest request = exchange.getRequest(); + + String requestFrom = RequestUtil.getHeaderValue(request, JobCommonHeaders.BK_GATEWAY_FROM); + if (log.isDebugEnabled()) { + log.debug("Open api request from : {}", + StringUtils.isNotEmpty(requestFrom) ? requestFrom : "bk-job-esb"); + } + JobContextUtil.setRequestFrom(requestFrom); + String token = RequestUtil.getHeaderValue(request, JobCommonHeaders.BK_GATEWAY_JWT); + if (StringUtils.isEmpty(token)) { + log.warn("Jwt token is empty! requestFrom={}", requestFrom); + response.setStatusCode(HttpStatus.UNAUTHORIZED); + return response.setComplete(); + } + + EsbJwtInfo authInfo; + if (isOpenApiTestActive(request)) { + // 如果是 OpenApi 测试请求，使用 Job 的 JWT 认证方式，不使用 ESB JWT（避免依赖 ESB) + authInfo = openApiJwtService.extractFromJwt(token, + RSAUtils.getPublicKey(securityProperties.getPublicKeyBase64())); + } else { + authInfo = openApiJwtService.extractFromJwt(token); + } + + if (authInfo == null) { + log.warn("Untrusted esb request, request-id:{}", RequestUtil.getHeaderValue(request, + JobCommonHeaders.BK_GATEWAY_REQUEST_ID)); + response.setStatusCode(HttpStatus.UNAUTHORIZED); + return response.setComplete(); + } + + // 缺少用户信息 + if (StringUtils.isEmpty(authInfo.getUsername())) { + logAuthInfo(request, authInfo); + return buildResponse(response, ErrorCode.MISSING_USER_INFO); + } + + // 缺少AppCode + if (StringUtils.isEmpty(authInfo.getAppCode())) { + logAuthInfo(request, authInfo); + return buildResponse(response, ErrorCode.MISSING_APP_CODE); + } + + // set app code header + request.mutate().header(JobCommonHeaders.APP_CODE, new String[]{authInfo.getAppCode()}).build(); + request.mutate().header(JobCommonHeaders.USERNAME, new String[]{authInfo.getUsername()}).build(); + return chain.filter(exchange.mutate().request(request).build()); + }; + } + + private void logAuthInfo(ServerHttpRequest request, EsbJwtInfo authInfo) { + log.warn( + "Untrusted open api request, request-id:{}, authInfo: {}", + RequestUtil.getHeaderValue(request, JobCommonHeaders.BK_GATEWAY_REQUEST_ID), + authInfo + ); + } + + private Mono buildResponse(ServerHttpResponse response, int errorCode) { + response.setStatusCode(HttpStatus.UNAUTHORIZED); + EsbResp resp = EsbResp.buildCommonFailResp(errorCode); + String respStr = JsonUtils.toJson(resp); + DataBuffer buffer = response.bufferFactory().wrap(respStr.getBytes(StandardCharsets.UTF_8)); + return response.writeWith(Mono.just(buffer)); + } + + private boolean isOpenApiTestActive(ServerHttpRequest request) { + if (!springProfile.isProfileActive("openApiTestEnv")) { + return false; + } + String value = RequestUtil.getHeaderValue(request, "X-JOB-OPENAPI-TEST"); + return StringUtils.isNotEmpty(value) && value.equalsIgnoreCase("true"); + } + + static class Config { + + } + +} diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/esb/RecordEsbAccessLogGatewayFilterFactory.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/esb/RecordEsbAccessLogGatewayFilterFactory.java index 604b7c5983..000b991191 100644 --- a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/esb/RecordEsbAccessLogGatewayFilterFactory.java +++ b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/esb/RecordEsbAccessLogGatewayFilterFactory.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/esb/RecordIamAccessLogGatewayFilterFactory.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/esb/RecordIamAccessLogGatewayFilterFactory.java index 6388c86539..cdc9f96eed 100644 --- a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/esb/RecordIamAccessLogGatewayFilterFactory.java +++ b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/esb/RecordIamAccessLogGatewayFilterFactory.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/global/AddJwtHeaderGlobalFilter.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/global/AddJwtHeaderGlobalFilter.java index cb7e6e0a3f..4029133489 100644 --- a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/global/AddJwtHeaderGlobalFilter.java +++ b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/global/AddJwtHeaderGlobalFilter.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,8 @@ package com.tencent.bk.job.gateway.filter.global; -import com.tencent.bk.job.common.util.jwt.JwtManager; +import com.tencent.bk.job.common.jwt.JwtManager; +import com.tencent.bk.job.common.security.consts.JwtConsts; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.cloud.gateway.filter.GatewayFilterChain; @@ -50,7 +51,7 @@ public AddJwtHeaderGlobalFilter(JwtManager jwtManager) { public Mono filter(ServerWebExchange exchange, GatewayFilterChain chain) { String jwt = jwtManager.getToken(); ServerHttpRequest request = exchange.getRequest(); - request.mutate().header("x-job-auth-token", jwt).build(); + request.mutate().header(JwtConsts.HEADER_KEY_SERVICE_JWT_TOKEN, jwt).build(); return chain.filter(exchange.mutate().request(request).build()); } diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/global/AddTraceResponseHeaderGlobalFilter.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/global/AddTraceResponseHeaderGlobalFilter.java index ed28ea100f..44d489ca97 100644 --- a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/global/AddTraceResponseHeaderGlobalFilter.java +++ b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/global/AddTraceResponseHeaderGlobalFilter.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/remote/AuthorizeRemoteGatewayFilterFactory.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/remote/AuthorizeRemoteGatewayFilterFactory.java index 5666f0f996..037d480bbb 100644 --- a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/remote/AuthorizeRemoteGatewayFilterFactory.java +++ b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/remote/AuthorizeRemoteGatewayFilterFactory.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,7 @@ package com.tencent.bk.job.gateway.filter.remote; -import com.tencent.bk.job.gateway.config.BkConfig; +import com.tencent.bk.job.gateway.config.LicenseConfig; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.cloud.gateway.filter.GatewayFilter; @@ -40,10 +40,10 @@ public class AuthorizeRemoteGatewayFilterFactory extends AbstractGatewayFilterFactory { - private final BkConfig bkConfig; + private final LicenseConfig bkConfig; @Autowired - public AuthorizeRemoteGatewayFilterFactory(BkConfig bkConfig) { + public AuthorizeRemoteGatewayFilterFactory(LicenseConfig bkConfig) { super(Config.class); this.bkConfig = bkConfig; } diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/web/AddWebLangHeaderGatewayFilterFactory.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/web/AddWebLangHeaderGatewayFilterFactory.java index 72b117b61d..5bb7442b14 100644 --- a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/web/AddWebLangHeaderGatewayFilterFactory.java +++ b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/web/AddWebLangHeaderGatewayFilterFactory.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -35,6 +35,7 @@ import org.springframework.http.server.reactive.ServerHttpRequest; import org.springframework.stereotype.Component; +import static com.tencent.bk.job.common.i18n.locale.LocaleUtils.BLUEKING_LANG_HEADER; import static com.tencent.bk.job.common.i18n.locale.LocaleUtils.COMMON_LANG_HEADER; /** @@ -54,7 +55,7 @@ public AddWebLangHeaderGatewayFilterFactory() { public GatewayFilter apply(Config config) { return (exchange, chain) -> { ServerHttpRequest request = exchange.getRequest(); - String webLangCookieValue = RequestUtil.getCookieValue(request, "blueking_language"); + String webLangCookieValue = RequestUtil.getCookieValue(request, BLUEKING_LANG_HEADER); String commonLang = LocaleUtils.LANG_ZH_CN; if (!StringUtils.isEmpty(webLangCookieValue)) { if (webLangCookieValue.equalsIgnoreCase(WebLangCookie.EN)) { diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/web/AuthorizeGatewayFilterFactory.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/web/AuthorizeGatewayFilterFactory.java index faf7edff43..e3f6b2d24c 100644 --- a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/web/AuthorizeGatewayFilterFactory.java +++ b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/web/AuthorizeGatewayFilterFactory.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,8 +24,14 @@ package com.tencent.bk.job.gateway.filter.web; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.InternalUserManageException; +import com.tencent.bk.job.common.i18n.locale.LocaleUtils; +import com.tencent.bk.job.common.model.Response; import com.tencent.bk.job.common.model.dto.BkUserDTO; +import com.tencent.bk.job.common.paas.exception.AppPermissionDeniedException; import com.tencent.bk.job.common.util.RequestUtil; +import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.gateway.config.LoginExemptionConfig; import com.tencent.bk.job.gateway.web.service.LoginService; import lombok.extern.slf4j.Slf4j; @@ -33,14 +39,21 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.cloud.gateway.filter.GatewayFilter; import org.springframework.cloud.gateway.filter.factory.AbstractGatewayFilterFactory; +import org.springframework.context.i18n.LocaleContextHolder; +import org.springframework.core.io.buffer.DataBuffer; import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; import org.springframework.http.server.reactive.ServerHttpRequest; import org.springframework.http.server.reactive.ServerHttpResponse; import org.springframework.stereotype.Component; import org.springframework.util.CollectionUtils; +import reactor.core.publisher.Mono; +import java.nio.charset.StandardCharsets; import java.util.List; +import static com.tencent.bk.job.common.i18n.locale.LocaleUtils.BLUEKING_LANG_HEADER; + /** * 用户token校验 */ @@ -74,6 +87,12 @@ private GatewayFilter getLoginFilter() { ServerHttpResponse response = exchange.getResponse(); String tokenCookieName = loginService.getCookieNameForToken(); List bkTokenList = RequestUtil.getCookieValuesFromHeader(request, tokenCookieName); + String lang = RequestUtil.getCookieValue(request, BLUEKING_LANG_HEADER); + if (StringUtils.isBlank(lang)) { + lang = LocaleUtils.LANG_EN; + log.warn("Cannot find blueking_language in cookie, use en"); + } + LocaleContextHolder.setLocale(LocaleUtils.getLocale(lang), true); if (CollectionUtils.isEmpty(bkTokenList)) { log.warn("Fail to parse token from headers, please check"); String bkToken = RequestUtil.getCookieValue(request, tokenCookieName); @@ -87,12 +106,15 @@ private GatewayFilter getLoginFilter() { response.getHeaders().add("x-login-url", loginService.getLoginRedirectUrl()); return response.setComplete(); } - BkUserDTO user = null; - // 遍历所有传入token找出当前环境的 - for (String bkToken : bkTokenList) { - user = loginService.getUser(bkToken); - if (user != null) { - break; + BkUserDTO user; + try { + user = getUserByTokenList(bkTokenList, lang); + } catch (InternalUserManageException e) { + Throwable cause = e.getCause(); + if (cause instanceof AppPermissionDeniedException) { + return getUserAccessAppForbiddenResp(response, cause.getMessage()); + } else { + throw e; } } if (user == null) { @@ -108,6 +130,29 @@ private GatewayFilter getLoginFilter() { }; } + private Mono getUserAccessAppForbiddenResp(ServerHttpResponse response, String data) { + Response resp = new Response<>(ErrorCode.USER_ACCESS_APP_FORBIDDEN, data); + response.setStatusCode(HttpStatus.FORBIDDEN); + String body = JsonUtils.toJson(resp); + byte[] bodyBytes = body.getBytes(StandardCharsets.UTF_8); + DataBuffer dataBuffer = response.bufferFactory().wrap(bodyBytes); + response.getHeaders().setContentLength(bodyBytes.length); + response.getHeaders().setContentType(MediaType.APPLICATION_JSON); + response.writeWith(Mono.just(dataBuffer)).subscribe(); + return response.setComplete(); + } + + private BkUserDTO getUserByTokenList(List bkTokenList, String lang) { + // 遍历所有传入token找出当前环境的 + for (String bkToken : bkTokenList) { + BkUserDTO user = loginService.getUser(bkToken, lang); + if (user != null) { + return user; + } + } + return null; + } + @Override public GatewayFilter apply(Config config) { if (loginExemptionConfig.isEnableLoginExemption()) { diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/web/CsrfCheckGatewayFilterFactory.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/web/CsrfCheckGatewayFilterFactory.java index b48ab61679..884363b0d3 100644 --- a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/web/CsrfCheckGatewayFilterFactory.java +++ b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/filter/web/CsrfCheckGatewayFilterFactory.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,9 +24,9 @@ package com.tencent.bk.job.gateway.filter.web; +import com.tencent.bk.job.common.service.config.JobCommonConfig; import com.tencent.bk.job.common.util.RequestUtil; import com.tencent.bk.job.gateway.common.util.UrlUtil; -import com.tencent.bk.job.gateway.config.BkConfig; import com.tencent.bk.job.gateway.web.service.LoginService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; @@ -56,14 +56,14 @@ public class CsrfCheckGatewayFilterFactory extends AbstractGatewayFilterFactory< private static final String COOKIE_CSRF_KEY_NAME = "job_csrf_key"; private static final String HEADER_CSRF_TOKEN_NAME = "X-CSRF-Token"; - private final BkConfig bkConfig; + private final JobCommonConfig jobCommonConfig; private final LoginService loginService; private String domain = ""; @Autowired - public CsrfCheckGatewayFilterFactory(BkConfig bkConfig, LoginService loginService) { + public CsrfCheckGatewayFilterFactory(JobCommonConfig jobCommonConfig, LoginService loginService) { super(Config.class); - this.bkConfig = bkConfig; + this.jobCommonConfig = jobCommonConfig; this.loginService = loginService; } @@ -121,7 +121,7 @@ private void addCsrfKeyCookie(ServerHttpResponse response) { String cookieValue = createCsrfKey(); ResponseCookie responseCookie = ResponseCookie.from(COOKIE_CSRF_KEY_NAME, cookieValue).path("/").domain(getDomain()) - .httpOnly(false).maxAge(Duration.ofDays(7)).build(); + .httpOnly(false).maxAge(Duration.ofDays(7)).build(); response.addCookie(responseCookie); } @@ -129,7 +129,7 @@ private String getDomain() { if (StringUtils.isNotBlank(this.domain)) { return this.domain; } - String webUrl = bkConfig.getJobWebUrl(); + String webUrl = jobCommonConfig.getJobWebUrl(); if (StringUtils.isEmpty(webUrl)) { throw new IllegalArgumentException("job.web.url"); } diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/i18n/JobLangHeaderLocaleContextResolver.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/i18n/JobLangHeaderLocaleContextResolver.java index e485a249b5..e65a31e959 100644 --- a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/i18n/JobLangHeaderLocaleContextResolver.java +++ b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/i18n/JobLangHeaderLocaleContextResolver.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/model/LicenseCheckResultDTO.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/model/LicenseCheckResultDTO.java index c9d52a1586..3f9ede56a8 100644 --- a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/model/LicenseCheckResultDTO.java +++ b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/model/LicenseCheckResultDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/model/esb/EsbJwtInfo.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/model/esb/EsbJwtInfo.java index e61d5631f3..f05e5dd2bd 100644 --- a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/model/esb/EsbJwtInfo.java +++ b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/model/esb/EsbJwtInfo.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/model/esb/EsbPublicKeyDTO.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/model/esb/EsbPublicKeyDTO.java index aaf9a52f04..eccf458982 100644 --- a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/model/esb/EsbPublicKeyDTO.java +++ b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/model/esb/EsbPublicKeyDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/model/response/VersionLogVO.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/model/response/VersionLogVO.java index d2b46a660c..ef00ef34d5 100644 --- a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/model/response/VersionLogVO.java +++ b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/model/response/VersionLogVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/predicate/JobEsbV2PathRoutePredicateFactory.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/predicate/JobEsbV2PathRoutePredicateFactory.java index 4c4d83ad91..5cf33a7ba3 100644 --- a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/predicate/JobEsbV2PathRoutePredicateFactory.java +++ b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/predicate/JobEsbV2PathRoutePredicateFactory.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/runner/LicenseCheckRunner.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/runner/LicenseCheckRunner.java index 138f173d38..3fbce13359 100644 --- a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/runner/LicenseCheckRunner.java +++ b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/runner/LicenseCheckRunner.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,7 +26,7 @@ import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.util.json.JsonUtils; -import com.tencent.bk.job.gateway.config.BkConfig; +import com.tencent.bk.job.gateway.config.LicenseConfig; import com.tencent.bk.job.gateway.model.LicenseCheckResultDTO; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; @@ -65,11 +65,11 @@ public class LicenseCheckRunner implements CommandLineRunner, ApplicationContext private ApplicationContext context; - private final BkConfig bkConfig; + private final LicenseConfig bkConfig; private CloseableHttpClient httpClient; - public LicenseCheckRunner(BkConfig bkConfig) { + public LicenseCheckRunner(LicenseConfig bkConfig) { this.bkConfig = bkConfig; } diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/service/EsbJwtPublicKeyService.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/service/EsbJwtPublicKeyService.java deleted file mode 100644 index 52ad53a212..0000000000 --- a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/service/EsbJwtPublicKeyService.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.gateway.service; - -/** - * ESB JWT 公钥获取服务 - */ -public interface EsbJwtPublicKeyService { - /** - * 获取ESB JWT 公钥 - * - * @return - */ - String getEsbJWTPublicKey(); -} diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/service/EsbJwtService.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/service/EsbJwtService.java deleted file mode 100644 index 2b56458d1d..0000000000 --- a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/service/EsbJwtService.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.gateway.service; - -import com.tencent.bk.job.gateway.model.esb.EsbJwtInfo; - -public interface EsbJwtService { - - /** - * 从jwt提取数据 - * - * @param token token - * @return - */ - EsbJwtInfo extractFromJwt(String token); -} diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/service/OpenApiJwtPublicKeyService.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/service/OpenApiJwtPublicKeyService.java new file mode 100644 index 0000000000..ded8ed818c --- /dev/null +++ b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/service/OpenApiJwtPublicKeyService.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.gateway.service; + +/** + * OPEN API JWT 公钥获取服务 + */ +public interface OpenApiJwtPublicKeyService { + /** + * 获取ESB JWT 公钥 + * + * @return + */ + String getEsbJWTPublicKey(); + + /** + * 获取bkApiGateway JWT 公钥 + * + * @return + */ + String getBkApiGatewayJWTPublicKey(); +} diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/service/OpenApiJwtService.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/service/OpenApiJwtService.java new file mode 100644 index 0000000000..8af5376e66 --- /dev/null +++ b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/service/OpenApiJwtService.java @@ -0,0 +1,49 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.gateway.service; + +import com.tencent.bk.job.gateway.model.esb.EsbJwtInfo; + +import java.security.PublicKey; + +public interface OpenApiJwtService { + + /** + * 从jwt提取数据 + * + * @param token token + * @return 解析 JWT 结果 + */ + EsbJwtInfo extractFromJwt(String token); + + /** + * 从jwt提取数据 + * + * @param token token + * @param publicKey 公钥 + * @return 解析 JWT 结果 + */ + EsbJwtInfo extractFromJwt(String token, PublicKey publicKey); +} diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/service/impl/EsbJwtPublicKeyServiceImpl.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/service/impl/EsbJwtPublicKeyServiceImpl.java deleted file mode 100644 index 9cb80db4d0..0000000000 --- a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/service/impl/EsbJwtPublicKeyServiceImpl.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.gateway.service.impl; - -import com.tencent.bk.job.common.constant.ErrorCode; -import com.tencent.bk.job.common.esb.model.EsbResp; -import com.tencent.bk.job.gateway.config.BkConfig; -import com.tencent.bk.job.gateway.model.esb.EsbPublicKeyDTO; -import com.tencent.bk.job.gateway.service.EsbJwtPublicKeyService; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.core.ParameterizedTypeReference; -import org.springframework.http.HttpMethod; -import org.springframework.stereotype.Service; -import org.springframework.web.client.RestTemplate; - -import java.util.HashMap; -import java.util.Map; - -@Service -@Slf4j -public class EsbJwtPublicKeyServiceImpl implements EsbJwtPublicKeyService { - - private BkConfig bkConfig; - private RestTemplate restTemplate; - private volatile String publicKey; - - @Autowired - public EsbJwtPublicKeyServiceImpl(BkConfig bkConfig, RestTemplate restTemplate) { - this.bkConfig = bkConfig; - this.restTemplate = restTemplate; - } - - @Override - public String getEsbJWTPublicKey() { - if (StringUtils.isNotEmpty(publicKey)) { - return publicKey; - } - String url = getEsbUrl() + "api/c/compapi/v2/esb/get_api_public_key?bk_app_code={bk_app_code}&bk_app_secret" + - "={bk_app_secret}&bk_username=admin"; - Map variables = new HashMap<>(); - variables.put("bk_app_code", bkConfig.getAppCode()); - variables.put("bk_app_secret", bkConfig.getAppSecret()); - EsbResp resp = restTemplate.exchange(url, HttpMethod.GET, null, - new ParameterizedTypeReference>() { - }, variables).getBody(); - log.info("Get esb jwt public key, resp: {}", resp); - if (resp == null || !resp.getCode().equals(ErrorCode.RESULT_OK) || resp.getData() == null) { - log.error("Get esb jwt public key fail!"); - throw new RuntimeException("Get esb jwt public key fail"); - } - String publicKey = resp.getData().getPublicKey(); - log.info("Get esb public key success, public key : {}", publicKey); - this.publicKey = publicKey; - return publicKey; - } - - private String getEsbUrl() { - String esbUrl = bkConfig.getEsbUrl(); - if (StringUtils.isEmpty(esbUrl)) { - throw new RuntimeException("Illegal esb url!"); - } - if (!esbUrl.endsWith("/")) { - esbUrl = esbUrl + "/"; - } - return esbUrl; - } -} diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/service/impl/EsbJwtServiceImpl.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/service/impl/EsbJwtServiceImpl.java deleted file mode 100644 index 88e4d12b59..0000000000 --- a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/service/impl/EsbJwtServiceImpl.java +++ /dev/null @@ -1,196 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.gateway.service.impl; - -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; -import com.tencent.bk.job.common.util.ThreadUtils; -import com.tencent.bk.job.gateway.model.esb.EsbJwtInfo; -import com.tencent.bk.job.gateway.service.EsbJwtPublicKeyService; -import com.tencent.bk.job.gateway.service.EsbJwtService; -import io.jsonwebtoken.Claims; -import io.jsonwebtoken.Jwts; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.bouncycastle.util.io.pem.PemObject; -import org.bouncycastle.util.io.pem.PemReader; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; - -import java.io.IOException; -import java.io.StringReader; -import java.security.KeyFactory; -import java.security.NoSuchAlgorithmException; -import java.security.PublicKey; -import java.security.spec.InvalidKeySpecException; -import java.security.spec.X509EncodedKeySpec; -import java.time.Instant; -import java.util.Date; -import java.util.LinkedHashMap; -import java.util.concurrent.TimeUnit; - -@Service -@Slf4j -public class EsbJwtServiceImpl implements EsbJwtService { - private final EsbJwtPublicKeyService esbJwtPublicKeyService; - private PublicKey publicKey; - private final Cache tokenCache = CacheBuilder.newBuilder() - .maximumSize(99999).expireAfterWrite(30, TimeUnit.SECONDS).build(); - - @Autowired - public EsbJwtServiceImpl(EsbJwtPublicKeyService esbJwtPublicKeyService) { - this.esbJwtPublicKeyService = esbJwtPublicKeyService; - getPublicKeyOrRetryInBackground(); - } - - private void getPublicKeyOrRetryInBackground() { - boolean publicKeyGotten = tryToGetAndCachePublicKeyOnce(); - if (publicKeyGotten) { - return; - } - Thread esbPublicKeyGetter = new Thread(() -> { - boolean keyGotten; - int retryCount = 0; - int sleepMillsOnce = 5000; - // 最多重试3天 - int maxRetryCount = 3 * 24 * 3600 / 5; - do { - log.warn("esbJwtPublicKey not gotten, retry {} after 5s", ++retryCount); - ThreadUtils.sleep(sleepMillsOnce); - keyGotten = tryToGetAndCachePublicKeyOnce(); - } while (!keyGotten && retryCount <= maxRetryCount); - if (!keyGotten) { - log.error("esbJwtPublicKey not gotten after {} retry (3 days), plz check esb", maxRetryCount); - } - }); - esbPublicKeyGetter.setDaemon(true); - esbPublicKeyGetter.setName("esbPublicKeyGetter"); - esbPublicKeyGetter.start(); - } - - private boolean tryToGetAndCachePublicKeyOnce() { - try { - String esbJwtPublicKey = esbJwtPublicKeyService.getEsbJWTPublicKey(); - if (StringUtils.isEmpty(esbJwtPublicKey)) { - log.error("Esb jwt public key is not configured!"); - return false; - } - this.publicKey = buildPublicKey(esbJwtPublicKey); - return true; - } catch (Throwable e) { - // Catch all exception - log.error("Build esb jwt public key caught error!", e); - return false; - } - } - - private PublicKey buildPublicKey(String pemContent) - throws NoSuchAlgorithmException, InvalidKeySpecException, IOException { - PemReader pemReader = new PemReader(new StringReader(pemContent)); - PemObject pemObject = pemReader.readPemObject(); - if (pemObject == null) { - log.error("Esb public key pem is illegal!"); - return null; - } - java.security.Security.addProvider(new org.bouncycastle.jce.provider.BouncyCastleProvider()); - X509EncodedKeySpec pubKeySpec = new X509EncodedKeySpec(pemObject.getContent()); - KeyFactory keyFactory = KeyFactory.getInstance("RSA"); - - return keyFactory.generatePublic(pubKeySpec); - } - - @Override - public EsbJwtInfo extractFromJwt(String token) { - long start = System.currentTimeMillis(); - EsbJwtInfo cacheJwtInfo = tokenCache.getIfPresent(token); - if (cacheJwtInfo != null) { - Long tokenExpireAt = cacheJwtInfo.getTokenExpireAt(); - // 如果未超时 - if (tokenExpireAt > Instant.now().getEpochSecond()) { - return cacheJwtInfo; - } - } - - EsbJwtInfo esbJwtInfo; - try { - Claims claims = Jwts.parser() - .setSigningKey(publicKey) - .parseClaimsJws(token) - .getBody(); - - String appCode = ""; - if (claims.get("app") != null) { - LinkedHashMap appProps = claims.get("app", LinkedHashMap.class); - if (appProps == null) { - log.warn("Invalid JWT token, app is null!"); - return null; - } - boolean isVerified = appProps.get("verified") != null && (boolean) appProps.get("verified"); - appCode = (String) appProps.get("app_code"); - if (StringUtils.isEmpty(appCode)) { - appCode = (String) appProps.get("bk_app_code"); - } - if (!isVerified || StringUtils.isEmpty(appCode)) { - log.warn("App code not verified or empty, isVerified:{}, jwtAppCode:{}", isVerified, appCode); - return null; - } - } - - String username = ""; - if (claims.get("user") != null) { - LinkedHashMap userProps = claims.get("user", LinkedHashMap.class); - if (userProps == null) { - log.warn("Invalid JWT token, user is null!"); - return null; - } - username = (String) userProps.get("username"); - if (StringUtils.isEmpty(username)) { - username = (String) userProps.get("bk_username"); - } - if (StringUtils.isEmpty(username)) { - log.warn("Username is empty!"); - return null; - } - } - Date expireAt = claims.get("exp", Date.class); - if (expireAt == null) { - log.warn("Invalid JWT token, exp is null!"); - return null; - } - esbJwtInfo = new EsbJwtInfo(expireAt.getTime(), username, appCode); - tokenCache.put(token, esbJwtInfo); - } catch (Exception e) { - log.warn("Verify jwt caught exception", e); - return null; - } finally { - long cost = System.currentTimeMillis() - start; - if (cost > 10) { - log.warn("Verify jwt cost too much, cost:{}", cost); - } - } - return esbJwtInfo; - } - -} diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/service/impl/OpenApiJwtPublicKeyServiceImpl.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/service/impl/OpenApiJwtPublicKeyServiceImpl.java new file mode 100644 index 0000000000..a7950b6593 --- /dev/null +++ b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/service/impl/OpenApiJwtPublicKeyServiceImpl.java @@ -0,0 +1,149 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.gateway.service.impl; + +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.esb.config.AppProperties; +import com.tencent.bk.job.common.esb.config.BkApiGatewayProperties; +import com.tencent.bk.job.common.esb.config.EsbProperties; +import com.tencent.bk.job.common.esb.model.EsbResp; +import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.gateway.config.BkGatewayConfig; +import com.tencent.bk.job.gateway.model.esb.EsbPublicKeyDTO; +import com.tencent.bk.job.gateway.service.OpenApiJwtPublicKeyService; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.core.ParameterizedTypeReference; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.stereotype.Service; +import org.springframework.web.client.RestTemplate; + +import java.util.HashMap; +import java.util.Map; + +@Service +@Slf4j +public class OpenApiJwtPublicKeyServiceImpl implements OpenApiJwtPublicKeyService { + + private final AppProperties appProperties; + private final EsbProperties esbProperties; + private final RestTemplate restTemplate; + private volatile String esbJwtPublicKey; + private volatile String bkApiGatewayPublicKey; + private final BkApiGatewayProperties bkApiGatewayProperties; + private final BkGatewayConfig bkApiGatewayConfig; + + private static final String URI_BK_APIGW_JWT_PUBLIC_KEY = "/api/v1/apis/{api_name}/public_key/"; + + @Autowired + public OpenApiJwtPublicKeyServiceImpl(AppProperties appProperties, + EsbProperties esbProperties, + RestTemplate restTemplate, + BkApiGatewayProperties bkApiGatewayProperties, + BkGatewayConfig bkApiGatewayConfig) { + this.appProperties = appProperties; + this.esbProperties = esbProperties; + this.restTemplate = restTemplate; + this.bkApiGatewayProperties = bkApiGatewayProperties; + this.bkApiGatewayConfig = bkApiGatewayConfig; + } + + @Override + public String getEsbJWTPublicKey() { + if (StringUtils.isNotEmpty(esbJwtPublicKey)) { + return esbJwtPublicKey; + } + String url = getEsbUrl() + "api/c/compapi/v2/esb/get_api_public_key?bk_app_code={bk_app_code}&bk_app_secret" + + "={bk_app_secret}&bk_username=admin"; + Map variables = new HashMap<>(); + variables.put("bk_app_code", appProperties.getCode()); + variables.put("bk_app_secret", appProperties.getSecret()); + EsbResp resp = restTemplate.exchange(url, HttpMethod.GET, null, + new ParameterizedTypeReference>() { + }, variables).getBody(); + log.info("Get esb jwt public key, resp: {}", resp); + if (resp == null || !resp.getCode().equals(ErrorCode.RESULT_OK) || resp.getData() == null) { + log.error("Get esb jwt public key fail!"); + throw new RuntimeException("Get esb jwt public key fail"); + } + String esbJwtPublicKey = resp.getData().getPublicKey(); + this.esbJwtPublicKey = esbJwtPublicKey; + return esbJwtPublicKey; + } + + @Override + public String getBkApiGatewayJWTPublicKey() { + if (StringUtils.isNotEmpty(bkApiGatewayPublicKey)) { + return bkApiGatewayPublicKey; + } + String url = getBkApiGatewayUrl() + URI_BK_APIGW_JWT_PUBLIC_KEY.replace("{api_name}", + bkApiGatewayConfig.getGatewayName()); + Map authInfo = new HashMap<>(); + authInfo.put("bk_app_code", appProperties.getCode()); + authInfo.put("bk_app_secret", appProperties.getSecret()); + HttpHeaders headers = new HttpHeaders(); + headers.add("X-Bkapi-Authorization", JsonUtils.toJson(authInfo)); + EsbResp resp = restTemplate.exchange( + url, + HttpMethod.GET, + new HttpEntity<>(null, headers), + new ParameterizedTypeReference>() { + } + ).getBody(); + + log.info("Get bkApiGateway jwt public key, resp: {}", resp); + if (resp == null || !resp.getCode().equals(ErrorCode.RESULT_OK) || resp.getData() == null) { + log.error("Get bkApiGateway jwt public key fail!"); + throw new RuntimeException("Get gateway jwt public key fail"); + } + this.bkApiGatewayPublicKey = resp.getData().getPublicKey(); + return bkApiGatewayPublicKey; + } + + private String getEsbUrl() { + String esbUrl = esbProperties.getService().getUrl(); + if (StringUtils.isEmpty(esbUrl)) { + throw new RuntimeException("Illegal esb url!"); + } + if (!esbUrl.endsWith("/")) { + esbUrl = esbUrl + "/"; + } + return esbUrl; + } + + private String getBkApiGatewayUrl() { + String bkApiGatewayUrl = bkApiGatewayProperties.getBkApiGateway().getUrl(); + if (StringUtils.isEmpty(bkApiGatewayUrl)) { + throw new RuntimeException("Illegal gateway url!"); + } + if (!bkApiGatewayUrl.endsWith("/")) { + bkApiGatewayUrl = bkApiGatewayUrl + "/"; + } + return bkApiGatewayUrl + "prod/"; + } +} diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/service/impl/OpenApiJwtServiceImpl.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/service/impl/OpenApiJwtServiceImpl.java new file mode 100644 index 0000000000..a2bc758a4c --- /dev/null +++ b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/service/impl/OpenApiJwtServiceImpl.java @@ -0,0 +1,252 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.gateway.service.impl; + +import com.google.common.cache.Cache; +import com.google.common.cache.CacheBuilder; +import com.tencent.bk.job.common.exception.InternalException; +import com.tencent.bk.job.common.util.JobContextUtil; +import com.tencent.bk.job.common.util.ThreadUtils; +import com.tencent.bk.job.gateway.config.BkGatewayConfig; +import com.tencent.bk.job.gateway.model.esb.EsbJwtInfo; +import com.tencent.bk.job.gateway.service.OpenApiJwtPublicKeyService; +import com.tencent.bk.job.gateway.service.OpenApiJwtService; +import io.jsonwebtoken.Claims; +import io.jsonwebtoken.Jwts; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.bouncycastle.util.io.pem.PemObject; +import org.bouncycastle.util.io.pem.PemReader; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.io.StringReader; +import java.security.KeyFactory; +import java.security.NoSuchAlgorithmException; +import java.security.PublicKey; +import java.security.spec.InvalidKeySpecException; +import java.security.spec.X509EncodedKeySpec; +import java.time.Instant; +import java.util.Date; +import java.util.LinkedHashMap; +import java.util.concurrent.TimeUnit; + +@Service +@Slf4j +public class OpenApiJwtServiceImpl implements OpenApiJwtService { + private final OpenApiJwtPublicKeyService openApiJwtPublicKeyService; + private volatile PublicKey esbJwtPublicKey; + private volatile PublicKey bkApiGatewayJwtPublicKey; + private final BkGatewayConfig bkApiGatewayConfig; + + /** + * 蓝鲸网关请求标识 + */ + private static final String REQUEST_FROM_BK_API_GW = "bk-job-apigw"; + + private final Cache tokenCache = CacheBuilder.newBuilder() + .maximumSize(99999).expireAfterWrite(30, TimeUnit.SECONDS).build(); + + @Autowired + public OpenApiJwtServiceImpl(OpenApiJwtPublicKeyService openApiJwtPublicKeyService, + BkGatewayConfig bkApiGatewayConfig) { + this.openApiJwtPublicKeyService = openApiJwtPublicKeyService; + this.bkApiGatewayConfig = bkApiGatewayConfig; + getJwtPublicKeyByPolicy(); + } + + private void getJwtPublicKeyByPolicy() { + boolean publicKeyGotten = tryToGetAndCachePublicKeyOnce(); + if (publicKeyGotten) { + log.info("Get and cache bkApiGateway/esb public key success"); + } else if ("abort".equalsIgnoreCase(bkApiGatewayConfig.getJwtPublicKeyFailPolicy())) { + throw new InternalException("Failed to get jwt public key, abort policy triggered"); + } else if ("retry".equalsIgnoreCase(bkApiGatewayConfig.getJwtPublicKeyFailPolicy())) { + getJwtPublicKeyWithBackgroundRetry(); + } else { + throw new InternalException("Illegal jwt public key get fail policy"); + } + } + + private void getJwtPublicKeyWithBackgroundRetry() { + Thread openApiPublicKeyGetter = new Thread(() -> { + boolean keyGotten; + int retryCount = 0; + int sleepMillsOnce = 5000; + // 最多重试3天 + int maxRetryCount = 3 * 24 * 3600 / 5; + do { + log.warn("Gateway public key not gotten, retry {} after 5s", ++retryCount); + ThreadUtils.sleep(sleepMillsOnce); + keyGotten = tryToGetAndCachePublicKeyOnce(); + } while (!keyGotten && retryCount <= maxRetryCount); + if (!keyGotten) { + log.error("Gateway public key not gotten after {} retry (3 days), plz check esb", maxRetryCount); + } + }); + openApiPublicKeyGetter.setDaemon(true); + openApiPublicKeyGetter.setName("gatewayPublicKeyGetter"); + openApiPublicKeyGetter.start(); + } + + private boolean tryToGetAndCachePublicKeyOnce() { + try { + if (this.esbJwtPublicKey == null) { + String esbJwtPublicKey = openApiJwtPublicKeyService.getEsbJWTPublicKey(); + if (StringUtils.isEmpty(esbJwtPublicKey)) { + log.error("Esb jwt public key is not configured!"); + return false; + } + this.esbJwtPublicKey = buildPublicKey(esbJwtPublicKey); + log.info("Init esb jwt public key success"); + } + + if (this.bkApiGatewayJwtPublicKey == null && bkApiGatewayConfig.isEnabled()) { + String bkApiGatewayJwtPublicKey = openApiJwtPublicKeyService.getBkApiGatewayJWTPublicKey(); + if (StringUtils.isEmpty(bkApiGatewayJwtPublicKey)) { + log.error("BkApiGateway jwt public key is not configured!"); + return false; + } + this.bkApiGatewayJwtPublicKey = buildPublicKey(bkApiGatewayJwtPublicKey); + log.info("Init bkApiGateway jwt public key success"); + } + return true; + } catch (Throwable e) { + // Catch all exception + log.error("Build jwt public key caught error!", e); + return false; + } + } + + private PublicKey buildPublicKey(String pemContent) + throws NoSuchAlgorithmException, InvalidKeySpecException, IOException { + PemReader pemReader = new PemReader(new StringReader(pemContent)); + PemObject pemObject = pemReader.readPemObject(); + if (pemObject == null) { + log.error("Public key pem is illegal!"); + return null; + } + java.security.Security.addProvider(new org.bouncycastle.jce.provider.BouncyCastleProvider()); + X509EncodedKeySpec pubKeySpec = new X509EncodedKeySpec(pemObject.getContent()); + KeyFactory keyFactory = KeyFactory.getInstance("RSA"); + + return keyFactory.generatePublic(pubKeySpec); + } + + @Override + public EsbJwtInfo extractFromJwt(String token) { + if (requestFromApiGw()) { + log.debug("Extract bkApiGateway jwt"); + return extractFromJwt(token, this.bkApiGatewayJwtPublicKey); + } else { + log.debug("Extract esb jwt"); + return extractFromJwt(token, this.esbJwtPublicKey); + } + } + + @Override + public EsbJwtInfo extractFromJwt(String token, PublicKey publicKey) { + long start = System.currentTimeMillis(); + EsbJwtInfo cacheJwtInfo = tokenCache.getIfPresent(token); + if (cacheJwtInfo != null) { + Long tokenExpireAt = cacheJwtInfo.getTokenExpireAt(); + // 如果未超时 + if (tokenExpireAt > Instant.now().getEpochSecond()) { + return cacheJwtInfo; + } + } + + EsbJwtInfo esbJwtInfo; + try { + Claims claims = Jwts.parser() + .setSigningKey(publicKey) + .parseClaimsJws(token) + .getBody(); + String appCode = ""; + if (claims.get("app") != null) { + LinkedHashMap appProps = claims.get("app", LinkedHashMap.class); + if (appProps == null) { + log.warn("Invalid JWT token, app is null!"); + return null; + } + boolean isVerified = appProps.get("verified") != null && (boolean) appProps.get("verified"); + appCode = (String) appProps.get("app_code"); + if (StringUtils.isEmpty(appCode)) { + appCode = (String) appProps.get("bk_app_code"); + } + if (!isVerified || StringUtils.isEmpty(appCode)) { + log.warn("App code not verified or empty, isVerified:{}, jwtAppCode:{}", isVerified, appCode); + return null; + } + } + + String username = ""; + if (claims.get("user") != null) { + LinkedHashMap userProps = claims.get("user", LinkedHashMap.class); + if (userProps == null) { + log.warn("Invalid JWT token, user is null!"); + return null; + } + username = (String) userProps.get("username"); + if (StringUtils.isEmpty(username)) { + username = (String) userProps.get("bk_username"); + } + if (StringUtils.isEmpty(username)) { + log.warn("Username is empty!"); + return null; + } + } + Date expireAt = claims.get("exp", Date.class); + if (expireAt == null) { + log.warn("Invalid JWT token, exp is null!"); + return null; + } + esbJwtInfo = new EsbJwtInfo(expireAt.getTime(), username, appCode); + tokenCache.put(token, esbJwtInfo); + } catch (Exception e) { + log.warn("Verify jwt caught exception", e); + if (log.isDebugEnabled()) { + log.debug("Parse jwt error, token: {}", token); + } + return null; + } finally { + long cost = System.currentTimeMillis() - start; + if (cost > 10) { + log.warn("Verify jwt cost too much, cost:{}", cost); + } + } + return esbJwtInfo; + } + + // 请求是否来自蓝鲸网关 + private boolean requestFromApiGw() { + String requestFrom = JobContextUtil.getRequestFrom(); + if (bkApiGatewayConfig.isEnabled() && StringUtils.equals(requestFrom, REQUEST_FROM_BK_API_GW)) { + return true; + } + return false; + } +} diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/web/service/LoginService.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/web/service/LoginService.java index 5931d66f90..c6e1a31034 100644 --- a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/web/service/LoginService.java +++ b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/web/service/LoginService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -33,7 +33,7 @@ public interface LoginService { /** * 使用户失效 * - * @param bkToken + * @param bkToken 用户Token */ void deleteUser(String bkToken); @@ -41,9 +41,10 @@ public interface LoginService { * 根据token获取用户信息 * * @param bkToken 用户token + * @param lang 语言 * @return 用户信息 */ - BkUserDTO getUser(String bkToken); + BkUserDTO getUser(String bkToken, String lang); /** * 获取登录跳转url diff --git a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/web/service/impl/LoginServiceImpl.java b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/web/service/impl/LoginServiceImpl.java index 9b8bd8ac35..a48dd450f7 100644 --- a/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/web/service/impl/LoginServiceImpl.java +++ b/src/backend/job-gateway/src/main/java/com/tencent/bk/job/gateway/web/service/impl/LoginServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -31,11 +31,12 @@ import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.exception.InternalException; import com.tencent.bk.job.common.model.dto.BkUserDTO; +import com.tencent.bk.job.common.paas.config.LoginConfiguration; import com.tencent.bk.job.common.paas.login.ILoginClient; -import com.tencent.bk.job.gateway.config.BkConfig; import com.tencent.bk.job.gateway.web.service.LoginService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.jetbrains.annotations.NotNull; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -46,41 +47,38 @@ @Slf4j @Service public class LoginServiceImpl implements LoginService { - private final BkConfig bkConfig; + private final LoginConfiguration loginConfig; private final String tokenName; private final String loginUrl; private final ILoginClient loginClient; - private LoadingCache> onlineUserCache = CacheBuilder.newBuilder() - .maximumSize(200).expireAfterWrite(10, TimeUnit.SECONDS).build( + private final LoadingCache> onlineUserCache = CacheBuilder.newBuilder() + .maximumSize(2000).expireAfterWrite(10, TimeUnit.SECONDS).build( new CacheLoader>() { @Override - public Optional load(String bkToken) throws Exception { - try { - BkUserDTO userDto = loginClient.getUserInfoByToken(bkToken); - return Optional.ofNullable(userDto); - } catch (Exception e) { - return Optional.empty(); - } + public Optional load(@NotNull String bkToken) { + BkUserDTO userDto = loginClient.getUserInfoByToken(bkToken); + return Optional.ofNullable(userDto); } } ); @Autowired - public LoginServiceImpl(BkConfig bkConfig, ILoginClient loginClient) { - this.bkConfig = bkConfig; + public LoginServiceImpl(LoginConfiguration loginConfig, + ILoginClient loginClient) { + this.loginConfig = loginConfig; this.loginClient = loginClient; this.loginUrl = getLoginUrlProp(); - this.tokenName = bkConfig.isCustomPaasLoginEnabled() ? bkConfig.getCustomLoginToken() : "bk_token"; + this.tokenName = loginConfig.isCustomPaasLoginEnabled() ? loginConfig.getCustomLoginToken() : "bk_token"; log.info("Init login service, customLoginEnabled:{}, loginClient:{}, loginUrl:{}, tokenName:{}", - bkConfig.isCustomPaasLoginEnabled(), loginClient.getClass(), loginUrl, tokenName); + loginConfig.isCustomPaasLoginEnabled(), loginClient.getClass(), loginUrl, tokenName); } private String getLoginUrlProp() { String loginUrl; - if (bkConfig.isCustomPaasLoginEnabled()) { - loginUrl = bkConfig.getCustomLoginUrl(); + if (loginConfig.isCustomPaasLoginEnabled()) { + loginUrl = loginConfig.getCustomLoginUrl(); } else { - loginUrl = bkConfig.getLoginUrl(); + loginUrl = loginConfig.getLoginUrl(); } if (!loginUrl.endsWith("?")) { loginUrl = loginUrl + "?"; @@ -96,12 +94,12 @@ public void deleteUser(String bkToken) { onlineUserCache.invalidate(bkToken); } - @Override - public BkUserDTO getUser(String bkToken) { + public BkUserDTO getUser(String bkToken, String bkLang) { if (StringUtils.isBlank(bkToken)) { return null; } + try { Optional userDto = onlineUserCache.get(bkToken); return userDto.orElse(null); diff --git a/src/backend/job-gateway/src/main/resources/application.yml b/src/backend/job-gateway/src/main/resources/application.yml index 54e8455799..3f4bca9d0c 100644 --- a/src/backend/job-gateway/src/main/resources/application.yml +++ b/src/backend/job-gateway/src/main/resources/application.yml @@ -14,214 +14,17 @@ spring: locator: enabled: false lower-case-service-id: true - routes: - - id: job-logout - uri: lb://job-gateway - predicates: - - Path= /logout - filters: - - Logout - - id: job-manage-web - uri: lb://job-manage - predicates: - - Path= /job-manage/web/** - filters: - - Authorize - - CsrfCheck - - StripPrefix=1 - - AddWebLangHeader - - id: job-crontab-web - uri: lb://job-crontab - predicates: - - Path= /job-crontab/web/** - filters: - - Authorize - - CsrfCheck - - StripPrefix=1 - - AddWebLangHeader - - id: job-execute-web - uri: lb://job-execute - predicates: - - Path= /job-execute/web/** - filters: - - Authorize - - CsrfCheck - - StripPrefix=1 - - AddWebLangHeader - - id: job-backup-web - uri: lb://job-backup - predicates: - - Path= /job-backup/web/** - filters: - - Authorize - - CsrfCheck - - StripPrefix=1 - - AddWebLangHeader - - id: job-file-gateway-web - uri: lb://job-file-gateway - predicates: - - Path= /job-file-gateway/web/** - filters: - - Authorize - - CsrfCheck - - StripPrefix=1 - - AddWebLangHeader - - id: job-ticket-web - uri: lb://job-manage - predicates: - - Path= /job-ticket/web/** - filters: - - Authorize - - CsrfCheck - - StripPrefix=1 - - AddWebLangHeader - - id: job-analysis-web - uri: lb://job-analysis - predicates: - - Path= /job-analysis/web/** - filters: - - Authorize - - CsrfCheck - - StripPrefix=1 - - AddWebLangHeader - - - id: job-file-gateway-remote - uri: lb://job-file-gateway - predicates: - - Path= /job-file-gateway/remote/** - filters: - - AuthorizeRemote - - StripPrefix=1 - - - id: job-manage-esb - uri: lb://job-manage - predicates: - - JobEsbV2Path=/api/job/v2/job-manage/{api_name} - filters: - - CheckEsbJwt - - SetPath=/esb/api/v2/{api_name} - - RecordEsbAccessLog - - AddEsbLangHeader - - - id: job-manage-esb-v3 - uri: lb://job-manage - predicates: - - Path=/api/job/v3/job-manage/{api_name} - filters: - - CheckEsbJwt - - SetPath=/esb/api/v3/{api_name} - - RecordEsbAccessLog - - AddEsbLangHeader - - - id: job-crontab-esb - uri: lb://job-crontab - predicates: - - JobEsbV2Path=/api/job/v2/job-crontab/{api_name} - filters: - - CheckEsbJwt - - SetPath=/esb/api/v2/{api_name} - - RecordEsbAccessLog - - AddEsbLangHeader - - - id: job-crontab-esb-v3 - uri: lb://job-crontab - predicates: - - JobEsbV2Path=/api/job/v3/job-crontab/{api_name} - filters: - - CheckEsbJwt - - SetPath=/esb/api/v3/{api_name} - - RecordEsbAccessLog - - AddEsbLangHeader - - - id: job-execute-esb - uri: lb://job-execute - predicates: - - JobEsbV2Path=/api/job/v2/job-execute/{api_name} - filters: - - CheckEsbJwt - - SetPath=/esb/api/v2/{api_name} - - RecordEsbAccessLog - - AddEsbLangHeader - - - id: job-execute-esb-v3 - uri: lb://job-execute - predicates: - - JobEsbV2Path=/api/job/v3/job-execute/{api_name} - filters: - - CheckEsbJwt - - SetPath=/esb/api/v3/{api_name} - - RecordEsbAccessLog - - AddEsbLangHeader - - - id: job-file-gateway-esb-v3 - uri: lb://job-file-gateway - predicates: - - JobEsbV2Path=/api/job/v3/job-file-gateway/{api_name} - filters: - - CheckEsbJwt - - SetPath=/esb/api/v3/{api_name} - - RecordEsbAccessLog - - AddEsbLangHeader - - - id: job-file-gateway-iam - uri: lb://job-file-gateway - predicates: - - Path=/iam/api/v1/resources/file_source - filters: - - RecordIamAccessLog - - AddEsbLangHeader - - - id: job-ticket-iam - uri: lb://job-manage - predicates: - - Path=/iam/api/v1/resources/ticket - filters: - - RecordIamAccessLog - - AddEsbLangHeader - - - id: job-analysis-iam - uri: lb://job-analysis - predicates: - - Path=/iam/api/v1/resources/dashboard_view - filters: - - RecordIamAccessLog - - AddEsbLangHeader - - - id: job-manage-iam - uri: lb://job-manage - predicates: - - Path=/iam/api/v1/resources/script,/iam/api/v1/resources/task/template,/iam/api/v1/resources/task/plan,/iam/api/v1/resources/account,/iam/api/v1/resources/whitelist,/iam/api/v1/resources/script/public,/iam/api/v1/resources/tag - filters: - - RecordIamAccessLog - - AddEsbLangHeader - - - id: job-crontab-iam - uri: lb://job-crontab - predicates: - - Path=/iam/api/v1/resources/cron/job - filters: - - RecordIamAccessLog - - AddEsbLangHeader - - - id: job-execute-iam - uri: lb://job-execute - predicates: - - Path=/iam/api/v1/resources/execute/record - filters: - - RecordIamAccessLog - - AddEsbLangHeader server: shutdown: graceful - -ribbon: - ServerListRefreshInterval: 3000 - management: health: livenessstate: enabled: true readinessstate: enabled: true + # 使用spring cloud stream rabbitmq, 禁用掉springboot的rabbitmq health检查 + rabbit: + enabled: false server: port: 19876 ssl: @@ -229,7 +32,7 @@ management: endpoints: web: exposure: - include: health,configprops,env,beans,conditions,loggers,metrics,mappings,prometheus,scheduledtasks,info,refresh + include: health,configprops,env,beans,conditions,loggers,metrics,mappings,prometheus,scheduledtasks,info,refresh,busrefresh,bindings base-path: /actuator enabled-by-default: false endpoint: @@ -260,6 +63,8 @@ management: enabled: true refresh: enabled: true + busrefresh: + enabled: true job: gateway: diff --git a/src/backend/job-gateway/src/main/resources/bootstrap.yml b/src/backend/job-gateway/src/main/resources/bootstrap.yml index 55ccc70423..d4d7b4b07a 100644 --- a/src/backend/job-gateway/src/main/resources/bootstrap.yml +++ b/src/backend/job-gateway/src/main/resources/bootstrap.yml @@ -11,3 +11,5 @@ spring: name: ${JOB_APPLICATION_CONFIGMAP_NAME} - namespace: ${KUBERNETES_NAMESPACE} name: ${JOB_COMMON_CONFIGMAP_NAME} + - namespace: ${KUBERNETES_NAMESPACE} + name: ${JOB_COMMON_REFRESHABLE_CONFIGMAP_NAME} diff --git a/src/backend/job-gateway/src/main/resources/i18n/message.properties b/src/backend/job-gateway/src/main/resources/i18n/message.properties index e028c76106..5414b99b69 100644 --- a/src/backend/job-gateway/src/main/resources/i18n/message.properties +++ b/src/backend/job-gateway/src/main/resources/i18n/message.properties @@ -1,11 +1,11 @@ # -# Tencent is pleased to support the open source community by making BK-JOB����������ҵƽ̨ available. +# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. # -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +# Copyright (C) 2021 Tencent. All rights reserved. # -# BK-JOB����������ҵƽ̨ is licensed under the MIT License. +# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. # -# License for BK-JOB����������ҵƽ̨: +# License for BK-JOB蓝鲸智云作业平台: # -------------------------------------------------------------------- # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated # documentation files (the "Software"), to deal in the Software without restriction, including without limitation @@ -21,4 +21,3 @@ # CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # - diff --git a/src/backend/job-gateway/src/main/resources/i18n/message_en.properties b/src/backend/job-gateway/src/main/resources/i18n/message_en.properties index e028c76106..5414b99b69 100644 --- a/src/backend/job-gateway/src/main/resources/i18n/message_en.properties +++ b/src/backend/job-gateway/src/main/resources/i18n/message_en.properties @@ -1,11 +1,11 @@ # -# Tencent is pleased to support the open source community by making BK-JOB����������ҵƽ̨ available. +# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. # -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +# Copyright (C) 2021 Tencent. All rights reserved. # -# BK-JOB����������ҵƽ̨ is licensed under the MIT License. +# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. # -# License for BK-JOB����������ҵƽ̨: +# License for BK-JOB蓝鲸智云作业平台: # -------------------------------------------------------------------- # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated # documentation files (the "Software"), to deal in the Software without restriction, including without limitation @@ -21,4 +21,3 @@ # CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # - diff --git a/src/backend/job-gateway/src/main/resources/i18n/message_en_US.properties b/src/backend/job-gateway/src/main/resources/i18n/message_en_US.properties index e028c76106..5414b99b69 100644 --- a/src/backend/job-gateway/src/main/resources/i18n/message_en_US.properties +++ b/src/backend/job-gateway/src/main/resources/i18n/message_en_US.properties @@ -1,11 +1,11 @@ # -# Tencent is pleased to support the open source community by making BK-JOB����������ҵƽ̨ available. +# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. # -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +# Copyright (C) 2021 Tencent. All rights reserved. # -# BK-JOB����������ҵƽ̨ is licensed under the MIT License. +# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. # -# License for BK-JOB����������ҵƽ̨: +# License for BK-JOB蓝鲸智云作业平台: # -------------------------------------------------------------------- # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated # documentation files (the "Software"), to deal in the Software without restriction, including without limitation @@ -21,4 +21,3 @@ # CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # - diff --git a/src/backend/job-gateway/src/main/resources/i18n/message_zh.properties b/src/backend/job-gateway/src/main/resources/i18n/message_zh.properties index e028c76106..5414b99b69 100644 --- a/src/backend/job-gateway/src/main/resources/i18n/message_zh.properties +++ b/src/backend/job-gateway/src/main/resources/i18n/message_zh.properties @@ -1,11 +1,11 @@ # -# Tencent is pleased to support the open source community by making BK-JOB����������ҵƽ̨ available. +# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. # -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +# Copyright (C) 2021 Tencent. All rights reserved. # -# BK-JOB����������ҵƽ̨ is licensed under the MIT License. +# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. # -# License for BK-JOB����������ҵƽ̨: +# License for BK-JOB蓝鲸智云作业平台: # -------------------------------------------------------------------- # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated # documentation files (the "Software"), to deal in the Software without restriction, including without limitation @@ -21,4 +21,3 @@ # CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # - diff --git a/src/backend/job-gateway/src/main/resources/i18n/message_zh_CN.properties b/src/backend/job-gateway/src/main/resources/i18n/message_zh_CN.properties index e028c76106..5414b99b69 100644 --- a/src/backend/job-gateway/src/main/resources/i18n/message_zh_CN.properties +++ b/src/backend/job-gateway/src/main/resources/i18n/message_zh_CN.properties @@ -1,11 +1,11 @@ # -# Tencent is pleased to support the open source community by making BK-JOB����������ҵƽ̨ available. +# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. # -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +# Copyright (C) 2021 Tencent. All rights reserved. # -# BK-JOB����������ҵƽ̨ is licensed under the MIT License. +# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. # -# License for BK-JOB����������ҵƽ̨: +# License for BK-JOB蓝鲸智云作业平台: # -------------------------------------------------------------------- # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated # documentation files (the "Software"), to deal in the Software without restriction, including without limitation @@ -21,4 +21,3 @@ # CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS # IN THE SOFTWARE. # - diff --git a/src/backend/job-gateway/src/main/resources/logback-spring.xml b/src/backend/job-gateway/src/main/resources/logback-spring.xml new file mode 100644 index 0000000000..ef1ad01fc8 --- /dev/null +++ b/src/backend/job-gateway/src/main/resources/logback-spring.xml @@ -0,0 +1,117 @@ + + + logback + + + + + + + + + + + + + + ${ACCESS_LOG} + + ${ACCESS_LOG}-%d{yyyy-MM-dd}.log.%i + 1GB + ${LOG_KEEP_DAYS} + ${CLEAN_HISTORY_ON_START} + + + [%X{traceId}|%X{spanId}]%msg%n + UTF-8 + + + + + ${ESB_ACCESS_LOG} + + ${ESB_ACCESS_LOG}-%d{yyyy-MM-dd}.log.%i + 1GB + ${LOG_KEEP_DAYS} + ${CLEAN_HISTORY_ON_START} + + + [%X{traceId}|%X{spanId}]%msg%n + UTF-8 + + + + + ${IAM_ACCESS_LOG} + + ${IAM_ACCESS_LOG}-%d{yyyy-MM-dd}.log.%i + 1GB + ${LOG_KEEP_DAYS} + ${CLEAN_HISTORY_ON_START} + + + [%X{traceId}|%X{spanId}]%msg%n + UTF-8 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/backend/job-gateway/src/main/resources/logback/logback-app-props.xml b/src/backend/job-gateway/src/main/resources/logback/logback-app-props.xml deleted file mode 100644 index e890aed4ec..0000000000 --- a/src/backend/job-gateway/src/main/resources/logback/logback-app-props.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - diff --git a/src/backend/job-gateway/src/main/resources/logback/logback-app.xml b/src/backend/job-gateway/src/main/resources/logback/logback-app.xml deleted file mode 100644 index a6b5d8546e..0000000000 --- a/src/backend/job-gateway/src/main/resources/logback/logback-app.xml +++ /dev/null @@ -1,103 +0,0 @@ - - - - - - - ${ACCESS_LOG} - - ${ACCESS_LOG}-%d{yyyy-MM-dd}.log.%i - 1GB - 15 - - - [%X{traceId}|%X{spanId}]%msg%n - UTF-8 - - - - - ${ESB_ACCESS_LOG} - - ${ESB_ACCESS_LOG}-%d{yyyy-MM-dd}.log.%i - 1GB - 15 - - - [%X{traceId}|%X{spanId}]%msg%n - UTF-8 - - - - - ${IAM_ACCESS_LOG} - - ${IAM_ACCESS_LOG}-%d{yyyy-MM-dd}.log.%i - 1GB - 15 - - - [%X{traceId}|%X{spanId}]%msg%n - UTF-8 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/backend/job-gateway/src/test/java/com/tencent/bk/job/gateway/common/util/RequestUtilTest.java b/src/backend/job-gateway/src/test/java/com/tencent/bk/job/gateway/common/util/RequestUtilTest.java index 8570589b03..41e4f69f6c 100644 --- a/src/backend/job-gateway/src/test/java/com/tencent/bk/job/gateway/common/util/RequestUtilTest.java +++ b/src/backend/job-gateway/src/test/java/com/tencent/bk/job/gateway/common/util/RequestUtilTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-gateway/src/test/java/com/tencent/bk/job/gateway/common/util/UrlUtilTest.java b/src/backend/job-gateway/src/test/java/com/tencent/bk/job/gateway/common/util/UrlUtilTest.java index 2443e5b26f..1e59c40de8 100644 --- a/src/backend/job-gateway/src/test/java/com/tencent/bk/job/gateway/common/util/UrlUtilTest.java +++ b/src/backend/job-gateway/src/test/java/com/tencent/bk/job/gateway/common/util/UrlUtilTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-gateway/src/test/java/com/tencent/bk/job/gateway/filter/CsrfCheckGatewayFilterFactoryTest.java b/src/backend/job-gateway/src/test/java/com/tencent/bk/job/gateway/filter/CsrfCheckGatewayFilterFactoryTest.java index 872ad28205..87f8756f74 100644 --- a/src/backend/job-gateway/src/test/java/com/tencent/bk/job/gateway/filter/CsrfCheckGatewayFilterFactoryTest.java +++ b/src/backend/job-gateway/src/test/java/com/tencent/bk/job/gateway/filter/CsrfCheckGatewayFilterFactoryTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,7 @@ package com.tencent.bk.job.gateway.filter; -import com.tencent.bk.job.gateway.config.BkConfig; +import com.tencent.bk.job.common.service.config.JobCommonConfig; import com.tencent.bk.job.gateway.filter.web.CsrfCheckGatewayFilterFactory; import com.tencent.bk.job.gateway.web.service.LoginService; import org.junit.jupiter.api.AfterEach; @@ -57,10 +57,10 @@ public class CsrfCheckGatewayFilterFactoryTest { @BeforeEach public void init() { - BkConfig bkConfig = mock(BkConfig.class); - when(bkConfig.getJobWebUrl()).thenReturn("http://jobv3.com"); + JobCommonConfig jobCommonConfig = mock(JobCommonConfig.class); + when(jobCommonConfig.getJobWebUrl()).thenReturn("http://jobv3.com"); LoginService loginService = mock(LoginService.class); - factory = new CsrfCheckGatewayFilterFactory(bkConfig, loginService); + factory = new CsrfCheckGatewayFilterFactory(jobCommonConfig, loginService); } @AfterEach diff --git a/src/backend/job-logsvr/api-job-logsvr/build.gradle b/src/backend/job-logsvr/api-job-logsvr/build.gradle index 60380239fd..a1a9fee605 100644 --- a/src/backend/job-logsvr/api-job-logsvr/build.gradle +++ b/src/backend/job-logsvr/api-job-logsvr/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/api/ServiceLogResource.java b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/api/ServiceLogResource.java index e5c2eb128c..39a22df392 100644 --- a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/api/ServiceLogResource.java +++ b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/api/ServiceLogResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,15 +26,16 @@ import com.tencent.bk.job.common.annotation.CompatibleImplementation; import com.tencent.bk.job.common.annotation.InternalAPI; +import com.tencent.bk.job.common.constant.CompatibleType; import com.tencent.bk.job.common.model.InternalResponse; import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.logsvr.model.service.ServiceBatchSaveLogRequest; +import com.tencent.bk.job.logsvr.model.service.ServiceExecuteObjectLogDTO; +import com.tencent.bk.job.logsvr.model.service.ServiceExecuteObjectLogsDTO; import com.tencent.bk.job.logsvr.model.service.ServiceFileLogQueryRequest; import com.tencent.bk.job.logsvr.model.service.ServiceFileTaskLogDTO; -import com.tencent.bk.job.logsvr.model.service.ServiceHostLogDTO; -import com.tencent.bk.job.logsvr.model.service.ServiceHostLogsDTO; -import com.tencent.bk.job.logsvr.model.service.ServiceSaveLogRequest; import com.tencent.bk.job.logsvr.model.service.ServiceScriptLogQueryRequest; +import com.tentent.bk.job.common.api.feign.annotation.SmartFeignClient; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; @@ -42,9 +43,7 @@ import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; import java.util.List; @@ -52,69 +51,62 @@ * 执行日志服务 */ @Api(tags = {"Log"}) -@RequestMapping("/service/log") -@RestController +@SmartFeignClient(value = "job-logsvr", contextId = "logResource") @InternalAPI public interface ServiceLogResource { - /** - * 保存执行日志 - * - * @param request 保存日志请求 - */ - @ApiOperation("保存执行日志") - @PostMapping - InternalResponse saveLog( - @ApiParam("保存日志请求报文") - @RequestBody ServiceSaveLogRequest request - ); - /** * 批量保存执行日志 * * @param request 保存日志请求 */ @ApiOperation("批量保存执行日志") - @PostMapping("/batch") + @PostMapping("/service/log/batch") InternalResponse saveLogs( @ApiParam("批量保存日志请求报文") @RequestBody ServiceBatchSaveLogRequest request ); - @CompatibleImplementation(name = "rolling_execute", explain = "兼容API,后续使用hostId查询", version = "3.7.x") - @ApiOperation("根据目标服务器IP获取脚本任务对应的执行日志") - @GetMapping(value = {"/script/jobCreateDate/{jobCreateDate}/step/{stepInstanceId}/retry/{executeCount}/ip/{ip}"}) - InternalResponse getScriptHostLogByIp( + @ApiOperation("根据目标主机ID获取脚本任务对应的执行日志") + @GetMapping(value = { + "/service/log/script/jobCreateDate/{jobCreateDate}/step/{stepInstanceId}/retry/{executeCount}/hostId/{hostId}" + }) + @Deprecated + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA, + explain = "兼容历史数据使用 hostId 的查询方式") + InternalResponse getScriptHostLogByHostId( @ApiParam("作业创建时间") @PathVariable("jobCreateDate") String jobCreateDate, @ApiParam("步骤实例ID") @PathVariable("stepInstanceId") Long stepInstanceId, @ApiParam("执行次数") @PathVariable("executeCount") Integer executeCount, - @ApiParam("ip") - @PathVariable("ip") String ip, + @ApiParam("hostId") + @PathVariable("hostId") Long hostId, @ApiParam("滚动执行批次，非滚动任务传入null") @RequestParam(value = "batch", required = false) Integer batch); - @ApiOperation("根据目标主机ID获取脚本任务对应的执行日志") + @ApiOperation("根据目标执行对象ID获取脚本任务对应的执行日志") @GetMapping(value = { - "/script/jobCreateDate/{jobCreateDate}/step/{stepInstanceId}/retry/{executeCount}/hostId/{hostId}" + "/service/log/script/jobCreateDate/{jobCreateDate}/step/{stepInstanceId}/retry/{executeCount}/executeObject" + + "/{executeObjectId}" }) - InternalResponse getScriptHostLogByHostId( + InternalResponse getScriptLogByExecuteObjectId( @ApiParam("作业创建时间") @PathVariable("jobCreateDate") String jobCreateDate, @ApiParam("步骤实例ID") @PathVariable("stepInstanceId") Long stepInstanceId, @ApiParam("执行次数") @PathVariable("executeCount") Integer executeCount, - @ApiParam("hostId") - @PathVariable("hostId") Long hostId, + @ApiParam("executeObjectId") + @PathVariable("executeObjectId") String executeObjectId, @ApiParam("滚动执行批次，非滚动任务传入null") @RequestParam(value = "batch", required = false) Integer batch); @ApiOperation("批量获取脚本任务对应的执行日志") - @PostMapping(value = {"/script/jobCreateDate/{jobCreateDate}/step/{stepInstanceId}/retry/{executeCount}"}) - InternalResponse> listScriptLogs( + @PostMapping(value = {"/service/log/script/jobCreateDate/{jobCreateDate}/step/{stepInstanceId}/retry" + + "/{executeCount}"}) + InternalResponse> listScriptExecuteObjectLogs( @ApiParam("作业创建时间") @PathVariable("jobCreateDate") String jobCreateDate, @ApiParam("步骤实例ID") @@ -125,43 +117,52 @@ InternalResponse> listScriptLogs( @RequestBody ServiceScriptLogQueryRequest query ); - @CompatibleImplementation(name = "rolling_execute", explain = "兼容API,后续使用hostId查询", version = "3.7.x") - @ApiOperation("按照IP获取文件任务对应的执行日志") - @GetMapping(value = {"/file/jobCreateDate/{jobCreateDate}/step/{stepInstanceId}/retry/{executeCount}/ip/{ip}"}) - InternalResponse getFileHostLogByIp( + @Deprecated + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA, + explain = "兼容历史数据使用 hostId 的查询方式") + @ApiOperation("按照hostId获取文件任务对应的执行日志") + @GetMapping(value = { + "/service/log/file/jobCreateDate/{jobCreateDate}/step/{stepInstanceId}/retry/{executeCount}/hostId/{hostId}" + }) + InternalResponse getFileHostLogByHostId( @ApiParam("作业创建时间") @PathVariable("jobCreateDate") String jobCreateDate, @ApiParam("步骤实例ID") @PathVariable("stepInstanceId") Long stepInstanceId, @ApiParam("执行次数") @PathVariable("executeCount") Integer executeCount, - @ApiParam("ip") - @PathVariable("ip") String ip, + @ApiParam("hostId") + @PathVariable("hostId") Long hostId, @ApiParam("分发模式,0:upload,1:download") @RequestParam(value = "mode", required = false) Integer mode, @ApiParam("滚动执行批次，非滚动任务传入null") @RequestParam(value = "batch", required = false) Integer batch); - @ApiOperation("按照hostId获取文件任务对应的执行日志") + @ApiOperation("按照执行对象ID获取文件任务对应的执行日志") @GetMapping(value = { - "/file/jobCreateDate/{jobCreateDate}/step/{stepInstanceId}/retry/{executeCount}/hostId/{hostId}" + "/service/log/file/jobCreateDate/{jobCreateDate}/step/{stepInstanceId}/retry/{executeCount}/executeObject" + + "/{executeObjectId}" }) - InternalResponse getFileHostLogByHostId( + InternalResponse getFileLogByExecuteObjectId( @ApiParam("作业创建时间") @PathVariable("jobCreateDate") String jobCreateDate, @ApiParam("步骤实例ID") @PathVariable("stepInstanceId") Long stepInstanceId, @ApiParam("执行次数") @PathVariable("executeCount") Integer executeCount, - @ApiParam("hostId") - @PathVariable("hostId") Long hostId, + @ApiParam("executeObjectId") + @PathVariable("executeObjectId") String executeObjectId, @ApiParam("分发模式,0:upload,1:download") @RequestParam(value = "mode", required = false) Integer mode, @ApiParam("滚动执行批次，非滚动任务传入null") @RequestParam(value = "batch", required = false) Integer batch); + + @Deprecated + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.DEPLOY, + explain = "使用 listFileExecuteObjectLogs() 替换，发布完成后可删除") @ApiOperation("获取文件任务对应的执行日志") - @GetMapping("/file/jobCreateDate/{jobCreateDate}/step/{stepInstanceId}/retry/{executeCount}") + @GetMapping("/service/log/file/jobCreateDate/{jobCreateDate}/step/{stepInstanceId}/retry/{executeCount}") InternalResponse> listFileHostLogs( @ApiParam("作业创建时间") @PathVariable("jobCreateDate") String jobCreateDate, @@ -179,9 +180,13 @@ InternalResponse> listFileHostLogs( @RequestParam(value = "hostId", required = false) Long hostId ); - @ApiOperation("获取文件任务agent对应的执行日志") - @PostMapping("/file/jobCreateDate/{jobCreateDate}/step/{stepInstanceId}/retry/{executeCount}/queryByTaskIds") - InternalResponse listFileHostLogsByTaskIds( + @ApiOperation("获取文件任务对应的执行日志") + @PostMapping( + "/service/log/file/jobCreateDate/{jobCreateDate}/step/{stepInstanceId}/retry/{executeCount}/queryByTaskIds") + @Deprecated + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.DEPLOY, + explain = "返回的协议内容有问题，发布完成后可删除") + InternalResponse listFileLogsByTaskIds( @ApiParam("作业创建时间") @PathVariable("jobCreateDate") String jobCreateDate, @ApiParam("步骤实例ID") @@ -190,13 +195,47 @@ InternalResponse listFileHostLogsByTaskIds( @PathVariable("executeCount") Integer executeCount, @ApiParam("滚动执行批次，非滚动任务传入null") @RequestParam(value = "batch", required = false) Integer batch, - @ApiParam("文件任务ID列表，多个任务ID以;分隔") + @ApiParam("文件任务ID列表") + @RequestBody List taskIds + ); + + @ApiOperation("根据任务 ID 批量获取文件任务对应的执行日志") + @PostMapping( + "/service/log/file/jobCreateDate/{jobCreateDate}/step/{stepInstanceId}/retry/{executeCount}" + + "/listTaskFileLogsByTaskIds") + InternalResponse> listTaskFileLogsByTaskIds( + @ApiParam("作业创建时间") + @PathVariable("jobCreateDate") String jobCreateDate, + @ApiParam("步骤实例ID") + @PathVariable("stepInstanceId") Long stepInstanceId, + @ApiParam("执行次数") + @PathVariable("executeCount") Integer executeCount, + @ApiParam("滚动执行批次，非滚动任务传入null") + @RequestParam(value = "batch", required = false) Integer batch, + @ApiParam("文件任务ID列表") @RequestBody List taskIds ); @ApiOperation("获取文件任务对应的执行日志") @PostMapping("/file") - InternalResponse listFileHostLogs(@RequestBody ServiceFileLogQueryRequest request); + @Deprecated + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.DEPLOY, + explain = "发布之后不再被使用，可删除") + InternalResponse listFileHostLogs(@RequestBody ServiceFileLogQueryRequest request); + + @ApiOperation("批量获取文件任务对应的执行日志") + @PostMapping( + value = {"/service/log/file/jobCreateDate/{jobCreateDate}/step/{stepInstanceId}/retry/{executeCount}"}) + InternalResponse> listFileExecuteObjectLogs( + @ApiParam("作业创建时间") + @PathVariable("jobCreateDate") String jobCreateDate, + @ApiParam("步骤实例ID") + @PathVariable("stepInstanceId") Long stepInstanceId, + @ApiParam("执行次数") + @PathVariable("executeCount") Integer executeCount, + @ApiParam("查询请求") + @RequestBody ServiceFileLogQueryRequest request + ); /** * 返回日志内容包含关键字的主机ip @@ -207,9 +246,35 @@ InternalResponse listFileHostLogsByTaskIds( * @param keyword 查询关键字 * @return ip */ + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA) + @Deprecated @ApiOperation("根据脚本任务日志关键字获取对应的ip") - @GetMapping("/keywordMatch/jobCreateDate/{jobCreateDate}/step/{stepInstanceId}/retry/{executeCount}") - InternalResponse> getIpsByKeyword( + @GetMapping("/service/log/keywordMatch/jobCreateDate/{jobCreateDate}/step/{stepInstanceId}/retry/{executeCount}") + InternalResponse> queryHostsByLogKeyword( + @ApiParam("作业创建时间") + @PathVariable("jobCreateDate") String jobCreateDate, + @ApiParam("步骤实例ID") + @PathVariable("stepInstanceId") Long stepInstanceId, + @ApiParam("执行次数") + @PathVariable("executeCount") Integer executeCount, + @ApiParam("滚动执行批次，非滚动任务传入null") + @RequestParam(value = "batch", required = false) Integer batch, + @ApiParam("关键字") + @RequestParam("keyword") String keyword + ); + + /** + * 返回日志内容包含关键字的执行对象 ID 集合 + * + * @param jobCreateDate 创建时间 + * @param stepInstanceId 步骤实例ID + * @param executeCount 执行次数 + * @param keyword 查询关键字 + * @return 执行对象 ID 集合 + */ + @ApiOperation("根据脚本任务日志关键字获取对应的执行对象ID集合") + @GetMapping("/service/log/jobCreateDate/{jobCreateDate}/step/{stepInstanceId}/retry/{executeCount}/executeObject") + InternalResponse> queryExecuteObjectsByLogKeyword( @ApiParam("作业创建时间") @PathVariable("jobCreateDate") String jobCreateDate, @ApiParam("步骤实例ID") diff --git a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/consts/FileTaskModeEnum.java b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/consts/FileTaskModeEnum.java index 2abbcbdae3..5e8a02db1b 100644 --- a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/consts/FileTaskModeEnum.java +++ b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/consts/FileTaskModeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/consts/LogTypeEnum.java b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/consts/LogTypeEnum.java index f1115560a8..21da8d3382 100644 --- a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/consts/LogTypeEnum.java +++ b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/consts/LogTypeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceBatchSaveLogRequest.java b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceBatchSaveLogRequest.java index 8dc19c19a9..264d8ce7b9 100644 --- a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceBatchSaveLogRequest.java +++ b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceBatchSaveLogRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -36,6 +36,16 @@ @ApiModel("批量保存执行日志请求") @Data public class ServiceBatchSaveLogRequest { + + @ApiModelProperty(value = "Job业务ID", required = true) + private Long appId; + + @ApiModelProperty(value = "定时任务ID", required = true) + private Long cronTaskId; + + @ApiModelProperty(value = "API调用的作业，调用方appCode", required = true) + private String appCode; + /** * 作业实例创建时间 */ @@ -46,7 +56,7 @@ public class ServiceBatchSaveLogRequest { * 执行日志 */ @ApiModelProperty(value = "执行日志", required = true) - private List logs; + private List logs; /** * 日志类型 diff --git a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceExecuteObjectLogDTO.java b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceExecuteObjectLogDTO.java new file mode 100644 index 0000000000..d01c108638 --- /dev/null +++ b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceExecuteObjectLogDTO.java @@ -0,0 +1,111 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.logsvr.model.service; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.CompatibleType; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +@ApiModel("执行对象执行日志") +@Data +@JsonInclude(JsonInclude.Include.NON_EMPTY) +public class ServiceExecuteObjectLogDTO { + /** + * 作业步骤实例ID + */ + @ApiModelProperty("步骤实例ID") + private Long stepInstanceId; + + /** + * 执行次数 + */ + @ApiModelProperty("执行次数") + private Integer executeCount; + + /** + * 滚动执行批次 + */ + @ApiModelProperty("滚动执行批次") + private Integer batch; + + /** + * 云区域ID:ipv4 + */ + @ApiModelProperty(value = "云区域ID:ipv4") + @JsonProperty("ip") + @Deprecated + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA) + private String cloudIp; + + /** + * 云区域ID:ipv6 + */ + @ApiModelProperty(value = "云区域ID:ipv6") + @JsonProperty("ipv6") + @Deprecated + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA) + private String cloudIpv6; + + /** + * 主机ID + */ + @Deprecated + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA) + @ApiModelProperty(value = "主机ID") + private Long hostId; + + /** + * 执行对象ID + */ + @ApiModelProperty("执行对象ID") + private String executeObjectId; + + /** + * 脚本日志内容 + */ + @ApiModelProperty(value = "脚本日志内容，保存日志的时候需要传入,查询日志的时候该字段无效") + @JsonProperty("scriptLog") + private ServiceExecuteObjectScriptLogDTO scriptLog; + + /** + * 文件任务执行日志 + */ + @JsonProperty("fileTaskLogs") + private List fileTaskLogs; + + public void addFileTaskLog(ServiceFileTaskLogDTO fileTaskDetailLog) { + if (fileTaskLogs == null) { + fileTaskLogs = new ArrayList<>(); + } + fileTaskLogs.add(fileTaskDetailLog); + } +} diff --git a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceExecuteObjectLogsDTO.java b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceExecuteObjectLogsDTO.java new file mode 100644 index 0000000000..2b5d937e05 --- /dev/null +++ b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceExecuteObjectLogsDTO.java @@ -0,0 +1,61 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.logsvr.model.service; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.CompatibleType; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.util.List; + +@ApiModel("执行对象执行日志-批量") +@Data +@JsonInclude(JsonInclude.Include.NON_EMPTY) +@CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.DEPLOY) +public class ServiceExecuteObjectLogsDTO { + /** + * 作业步骤实例ID + */ + @ApiModelProperty("步骤实例ID") + private Long stepInstanceId; + + /** + * 执行次数 + */ + @ApiModelProperty("执行次数") + private Integer executeCount; + + /** + * 执行对象执行日志列表 + */ + @ApiModelProperty(value = "执行对象执行日志列表") + @JsonProperty("ipLogs") + private List ipLogs; + +} diff --git a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceExecuteObjectScriptLogDTO.java b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceExecuteObjectScriptLogDTO.java new file mode 100644 index 0000000000..57b78b52d4 --- /dev/null +++ b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceExecuteObjectScriptLogDTO.java @@ -0,0 +1,138 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.logsvr.model.service; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.CompatibleType; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * 执行对象对应的脚本日志 + */ +@Data +@JsonInclude(JsonInclude.Include.NON_NULL) +@NoArgsConstructor +@ApiModel("执行对象对应的脚本日志") +public class ServiceExecuteObjectScriptLogDTO { + /** + * 执行对象ID + */ + @ApiModelProperty("执行对象ID") + private String executeObjectId; + + /** + * 主机ID + */ + @ApiModelProperty("主机ID") + @Deprecated + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA) + private Long hostId; + + /** + * 主机ipv4,格式: 云区域ID:IPv4 + */ + @ApiModelProperty("主机ipv4,格式: 云区域ID:IPv4") + @JsonProperty("ip") + @Deprecated + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA) + private String cloudIp; + + /** + * 主机ipv6,格式: 云区域ID:IPv6 + */ + @ApiModelProperty("主机ipv6,格式: 云区域ID:IPv6") + @JsonProperty("ipv6") + @Deprecated + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA) + private String cloudIpv6; + + /** + * 日志偏移 - 字节 + */ + @ApiModelProperty("日志偏移 - 字节") + private Integer offset; + + /** + * 日志大小 - 字节 + */ + @ApiModelProperty("日志大小 - 字节") + private Integer contentSizeBytes = 0; + + /** + * 日志内容 + */ + @ApiModelProperty("日志内容") + private String content; + + + /** + * Constructor + * + * @param hostId 主机hostId + * @param cloudIp 主机ipv4,格式: 云区域ID:IPv4 + * @param cloudIpv6 主机ipv6,格式: 云区域ID:IPv6 + * @param content 日志内容 + * @param contentSizeBytes 日志内容大小(单位byte) + * @param offset 日志偏移量 + */ + @Deprecated + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA) + public ServiceExecuteObjectScriptLogDTO(Long hostId, + String cloudIp, + String cloudIpv6, + String content, + int contentSizeBytes, + int offset) { + this.hostId = hostId; + this.cloudIp = cloudIp; + this.cloudIpv6 = cloudIpv6; + this.content = content; + this.contentSizeBytes = contentSizeBytes; + this.offset = offset; + } + + /** + * Constructor + * + * @param executeObjectId 执行对象 ID + * @param content 日志内容 + * @param contentSizeBytes 日志内容大小(单位byte) + * @param offset 日志偏移量 + */ + public ServiceExecuteObjectScriptLogDTO(String executeObjectId, + String content, + int contentSizeBytes, + int offset) { + this.executeObjectId = executeObjectId; + this.content = content; + this.contentSizeBytes = contentSizeBytes; + this.offset = offset; + } +} diff --git a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceFileLogQueryRequest.java b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceFileLogQueryRequest.java index e4b1faca67..5bdb34d885 100644 --- a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceFileLogQueryRequest.java +++ b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceFileLogQueryRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,6 +25,7 @@ package com.tencent.bk.job.logsvr.model.service; import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.CompatibleType; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; @@ -50,12 +51,16 @@ public class ServiceFileLogQueryRequest { @ApiModelProperty(value = "滚动执行批次") private Integer batch; - @ApiModelProperty("主机ID列表") + /** + * 查询的主机ID列表 + */ + @ApiModelProperty("查询的主机ID列表") + @Deprecated + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA) private List hostIds; - @ApiModelProperty("IP列表;如果hostIds参数不为空，那么忽略ips参数") - @CompatibleImplementation(name = "rolling_execute", explain = "兼容字段，后续用hostIds替换", version = "3.7.x") - private List ips; + @ApiModelProperty("查询的执行对象ID列表") + private List executeObjectIds; /** * @see com.tencent.bk.job.logsvr.consts.FileTaskModeEnum diff --git a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceFileTaskLogDTO.java b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceFileTaskLogDTO.java index 945a09ecdd..cf0de329e9 100644 --- a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceFileTaskLogDTO.java +++ b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceFileTaskLogDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,10 +26,11 @@ import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.CompatibleType; import io.swagger.annotations.ApiModel; import lombok.Data; import lombok.NoArgsConstructor; -import org.apache.commons.lang3.StringUtils; /** * 文件分发执行日志 @@ -52,23 +53,35 @@ public class ServiceFileTaskLogDTO { private Integer mode; /** - * 目标IP + * 目标主机（云区域ID:IPv4) */ + @Deprecated + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA) @JsonProperty("destIp") private String destIp; /** - * 用于展示的目标IP + * 目标主机（云区域ID:IPv6) */ - @JsonProperty("displayDestIp") - private String displayDestIp; + @Deprecated + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA) + @JsonProperty("destIpv6") + private String destIpv6; /** * 目标主机ID */ + @Deprecated + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA) @JsonProperty("destHostId") private Long destHostId; + /** + * 目标执行对象ID + */ + @JsonProperty("destExecuteObjectId") + private String destExecuteObjectId; + /** * 目标文件路径 */ @@ -76,22 +89,40 @@ public class ServiceFileTaskLogDTO { private String destFile; /** - * 文件源IP + * 文件源主机(云区域ID:IPv4) */ + @Deprecated + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA) @JsonProperty("srcIp") private String srcIp; + /** + * 文件源主机(云区域ID:IPv6) + */ + @Deprecated + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA) + @JsonProperty("srcIpv6") + private String srcIpv6; + /** * 文件源主机ID */ + @Deprecated + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA) @JsonProperty("srcHostId") private Long srcHostId; /** - * 文件源IP - 显示 + * 文件源执行对象ID + */ + @JsonProperty("srcExecuteObjectId") + private String srcExecuteObjectId; + + /** + * 源文件类型 */ - @JsonProperty("displaySrcIp") - private String displaySrcIp; + @JsonProperty("srcFileType") + private Integer srcFileType; /** * 源文件路径 - 真实路径 @@ -141,13 +172,45 @@ public class ServiceFileTaskLogDTO { @JsonProperty("content") private String content; + public ServiceFileTaskLogDTO(Integer mode, + String destExecuteObjectId, + String destFile, + String srcExecuteObjectId, + Integer srcFileType, + String srcFile, + String displaySrcFile, + String size, + Integer status, + String statusDesc, + String speed, + String process, + String content) { + this.mode = mode; + this.destExecuteObjectId = destExecuteObjectId; + this.destFile = destFile; + this.srcExecuteObjectId = srcExecuteObjectId; + this.srcFileType = srcFileType; + this.srcFile = srcFile; + this.displaySrcFile = displaySrcFile; + this.size = size; + this.status = status; + this.statusDesc = statusDesc; + this.speed = speed; + this.process = process; + this.content = content; + } + + @Deprecated + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA) public ServiceFileTaskLogDTO(Integer mode, Long destHostId, String destIp, + String destIpv6, String destFile, Long srcHostId, String srcIp, - String displaySrcIp, + String srcIpv6, + Integer srcFileType, String srcFile, String displaySrcFile, String size, @@ -159,10 +222,12 @@ public ServiceFileTaskLogDTO(Integer mode, this.mode = mode; this.destHostId = destHostId; this.destIp = destIp; + this.destIpv6 = destIpv6; this.destFile = destFile; this.srcHostId = srcHostId; this.srcIp = srcIp; - this.displaySrcIp = displaySrcIp; + this.srcIpv6 = srcIpv6; + this.srcFileType = srcFileType; this.srcFile = srcFile; this.displaySrcFile = displaySrcFile; this.size = size; @@ -172,14 +237,6 @@ public ServiceFileTaskLogDTO(Integer mode, this.process = process; this.content = content; } - - public String getDisplaySrcIp() { - if (StringUtils.isNotEmpty(this.displaySrcIp)) { - return this.displaySrcIp; - } else { - return this.srcIp; - } - } } diff --git a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceHostLogDTO.java b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceHostLogDTO.java deleted file mode 100644 index ccfc492889..0000000000 --- a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceHostLogDTO.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.logsvr.model.service; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.tencent.bk.job.common.annotation.CompatibleImplementation; -import io.swagger.annotations.ApiModel; -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; - -import java.util.ArrayList; -import java.util.List; - -@ApiModel("主机执行日志") -@Data -@JsonInclude(JsonInclude.Include.NON_EMPTY) -public class ServiceHostLogDTO { - /** - * 作业步骤实例ID - */ - @ApiModelProperty("步骤实例ID") - private Long stepInstanceId; - - /** - * 执行次数 - */ - @ApiModelProperty("执行次数") - private Integer executeCount; - - /** - * 滚动执行批次 - */ - @ApiModelProperty("滚动执行批次") - private Integer batch; - - /** - * ip - */ - @CompatibleImplementation(name = "rolling_execute", explain = "兼容字段，后续用hostId替换", version = "3.7.x") - @ApiModelProperty(value = "ip, 兼容参数, 如果存在hostId那么忽略ip参数") - private String ip; - - /** - * 主机ID - */ - @ApiModelProperty(value = "主机ID") - private Long hostId; - - /** - * 脚本日志内容 - */ - @ApiModelProperty(value = "脚本日志内容，保存日志的时候需要传入,查询日志的时候该字段无效") - @JsonProperty("scriptLog") - private ServiceScriptLogDTO scriptLog; - - /** - * 文件任务执行日志 - */ - private List fileTaskLogs; - - public void addFileTaskLog(ServiceFileTaskLogDTO fileTaskDetailLog) { - if (fileTaskLogs == null) { - fileTaskLogs = new ArrayList<>(); - } - fileTaskLogs.add(fileTaskDetailLog); - } -} diff --git a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceHostLogsDTO.java b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceHostLogsDTO.java deleted file mode 100644 index c6c5501956..0000000000 --- a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceHostLogsDTO.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.logsvr.model.service; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; -import io.swagger.annotations.ApiModel; -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; - -import java.util.List; - -@ApiModel("主机执行日志-批量") -@Data -@JsonInclude(JsonInclude.Include.NON_EMPTY) -public class ServiceHostLogsDTO { - /** - * 作业步骤实例ID - */ - @ApiModelProperty("步骤实例ID") - private Long stepInstanceId; - - /** - * 执行次数 - */ - @ApiModelProperty("执行次数") - private Integer executeCount; - - /** - * 主机执行日志列表 - */ - @ApiModelProperty(value = "主机执行日志") - @JsonProperty("ipLogs") - private List ipLogs; - -} diff --git a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceSaveLogRequest.java b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceSaveLogRequest.java deleted file mode 100644 index 7d791ff0bf..0000000000 --- a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceSaveLogRequest.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.logsvr.model.service; - -import com.fasterxml.jackson.annotation.JsonProperty; -import com.tencent.bk.job.common.annotation.CompatibleImplementation; -import io.swagger.annotations.ApiModel; -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; - -import java.util.List; - -@ApiModel("执行日志保存请求") -@Data -public class ServiceSaveLogRequest { - /** - * 作业实例创建时间 - */ - @ApiModelProperty(value = "作业实例创建时间，格式为yyyy_MM_dd", required = true) - private String jobCreateDate; - - /** - * 作业步骤实例ID - */ - @ApiModelProperty(value = "步骤实例ID", required = true) - private Long stepInstanceId; - - /** - * ip - */ - @CompatibleImplementation(name = "rolling_execute", explain = "兼容参数,由于IP不再唯一，后续使用hostId参数替换", - version = "3.7.x") - @ApiModelProperty(value = "ip, 兼容参数, 如果存在hostId那么忽略ip参数") - private String ip; - - /** - * 主机ID - */ - @ApiModelProperty(value = "主机ID") - private Long hostId; - - /** - * 执行次数 - */ - @ApiModelProperty(value = "执行次数", required = true) - private Integer executeCount; - - /** - * 滚动执行批次 - */ - @ApiModelProperty(value = "滚动批次") - private Integer batch; - - /** - * 脚本日志内容 - */ - @ApiModelProperty(value = "脚本日志内容") - @JsonProperty("scriptLog") - private ServiceScriptLogDTO scriptLog; - - /** - * 文件日志 - */ - @ApiModelProperty(value = "文件任务日志") - private List fileTaskLogs; - - /** - * 日志类型 - * - * @see com.tencent.bk.job.logsvr.consts.LogTypeEnum - */ - @ApiModelProperty(value = "日志类型,1-script;2-file") - private Integer logType; -} diff --git a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceScriptLogDTO.java b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceScriptLogDTO.java deleted file mode 100644 index d00f0e0620..0000000000 --- a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceScriptLogDTO.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.logsvr.model.service; - -import com.fasterxml.jackson.annotation.JsonInclude; -import com.tencent.bk.job.common.annotation.CompatibleImplementation; -import com.tencent.bk.job.common.model.dto.HostDTO; -import io.swagger.annotations.ApiModel; -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; -import lombok.NoArgsConstructor; - -/** - * 脚本日志 - */ -@Data -@JsonInclude(JsonInclude.Include.NON_NULL) -@NoArgsConstructor -@ApiModel("脚本日志") -public class ServiceScriptLogDTO { - /** - * 主机ID - */ - @ApiModelProperty("主机ID") - private Long hostId; - - /** - * 云区域+IP - */ - @CompatibleImplementation(name = "rolling_execute", explain = "兼容字段，后续使用hostId替换", version = "3.7.x") - @ApiModelProperty("云区域+IP") - private String cloudIp; - - /** - * 日志偏移 - 字节 - */ - @ApiModelProperty("日志偏移 - 字节") - private Integer offset; - - /** - * 日志内容 - */ - @ApiModelProperty("日志内容") - private String content; - - public ServiceScriptLogDTO(HostDTO host, Integer offset, String content) { - this.hostId = host.getHostId(); - if (host.getIp() != null && host.getBkCloudId() != null) { - this.cloudIp = host.toCloudIp(); - } - this.offset = offset; - this.content = content; - } - - public ServiceScriptLogDTO(Long hostId, String cloudIp, String content) { - this.hostId = hostId; - this.cloudIp = cloudIp; - this.content = content; - } -} diff --git a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceScriptLogQueryRequest.java b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceScriptLogQueryRequest.java index e4c477c7d5..4bd5911d32 100644 --- a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceScriptLogQueryRequest.java +++ b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/service/ServiceScriptLogQueryRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,6 +25,7 @@ package com.tencent.bk.job.logsvr.model.service; import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.CompatibleType; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; @@ -55,13 +56,14 @@ public class ServiceScriptLogQueryRequest { @ApiModelProperty(value = "滚动执行批次，可能为null") private Integer batch; - @ApiModelProperty("主机IP列表;兼容参数,如果hostIds参数不为空，那么忽略ips参数") - @CompatibleImplementation(name = "rolling_execute", explain = "兼容字段，后续用hostIds替换", version = "3.7.x") - private List ips; - /** * 查询的主机ID列表 */ @ApiModelProperty("查询的主机ID列表") + @Deprecated + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA) private List hostIds; + + @ApiModelProperty("查询的执行对象ID列表") + private List executeObjectIds; } diff --git a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/util/CollectionNameUtil.java b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/util/CollectionNameUtil.java new file mode 100644 index 0000000000..ce1f6087c0 --- /dev/null +++ b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/util/CollectionNameUtil.java @@ -0,0 +1,114 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.logsvr.util; + +import com.tencent.bk.job.logsvr.consts.LogTypeEnum; + +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * 集合名称工具类 + */ +public class CollectionNameUtil { + /** + * 集合名称前缀 + */ + private static final String COLLECTION_PREFIX = "job_log"; + + /** + * 集合名各部分之间的分隔符 + */ + private static final String COLLECTION_NAME_SEPARATOR = "_"; + + /** + * 脚本执行日志类型 + */ + private static final String TYPE_SCRIPT = "script"; + + /** + * 文件分发日志类型 + */ + private static final String TYPE_FILE = "file"; + + /** + * 正则：提取集合名称中的日期部分 + */ + private static final Pattern LOG_COLLECTION_NAME_PATTERN = Pattern.compile( + COLLECTION_PREFIX + "_(?:script|file)_(\\d{4}_\\d{2}_\\d{2})" + ); + + + /** + * 构建日志集合名 + * + * @param jobCreateDate 作业创建日期，格式为 yyyy_MM_dd + * @param logType 日志类型枚举 + * @return 集合名称，如：job_log_script_2025_01_01 + */ + public static String buildLogCollectionName(String jobCreateDate, LogTypeEnum logType) { + return COLLECTION_PREFIX + COLLECTION_NAME_SEPARATOR + + getLogTypeName(logType) + COLLECTION_NAME_SEPARATOR + + jobCreateDate; + } + + /** + * 获取类型名称 + */ + public static String getLogTypeName(LogTypeEnum logType) { + if (logType == LogTypeEnum.SCRIPT) { + return TYPE_SCRIPT; + } else if (logType == LogTypeEnum.FILE) { + return TYPE_FILE; + } else { + throw new IllegalArgumentException("Invalid logType: " + logType); + } + } + + /** + * 构建日志集合名称中不带日期的部分（下划线结尾），如：job_log_script_ + * + * @param logType 日志类型 + * @return 集合名前缀字符串 + */ + public static String buildCollectionNamePrefix(LogTypeEnum logType) { + return COLLECTION_PREFIX + COLLECTION_NAME_SEPARATOR + + getLogTypeName(logType) + COLLECTION_NAME_SEPARATOR; + } + + /** + * 从日志集合名中提取日期部分 + * + * @param collectionName MongoDB集合名，如：job_log_script_2025_01_01 + * @return String 日期字符串，如：2025_01_01 + */ + public static String collectionNameToDateStr(String collectionName) { + Matcher m = LOG_COLLECTION_NAME_PATTERN.matcher(collectionName); + if (!m.matches()) { + return null; + } + return m.group(1); + } +} diff --git a/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/util/LogFieldUtil.java b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/util/LogFieldUtil.java new file mode 100644 index 0000000000..e56d1658cd --- /dev/null +++ b/src/backend/job-logsvr/api-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/util/LogFieldUtil.java @@ -0,0 +1,46 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.logsvr.util; + +import com.tencent.bk.job.common.util.date.DateUtils; + +import java.time.ZoneId; +import java.time.temporal.ChronoUnit; + +/** + * 日志字段工具类 + */ +public class LogFieldUtil { + + /** + * 根据任务创建时间获取创建日期，格式为：yyyy_MM_dd + * + * @param createTime 任务创建时间 + * @return 创建日期 + */ + public static String buildJobCreateDate(Long createTime) { + return DateUtils.formatUnixTimestamp(createTime, ChronoUnit.MILLIS, "yyyy_MM_dd", ZoneId.of("UTC")); + } +} diff --git a/src/backend/job-logsvr/boot-job-logsvr/build.gradle b/src/backend/job-logsvr/boot-job-logsvr/build.gradle index 9d53bc8833..7642cf3020 100644 --- a/src/backend/job-logsvr/boot-job-logsvr/build.gradle +++ b/src/backend/job-logsvr/boot-job-logsvr/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -30,6 +30,7 @@ dependencies { implementation 'org.springframework.cloud:spring-cloud-starter-bootstrap' testImplementation 'org.springframework.boot:spring-boot-starter-test' testImplementation 'de.flapdoodle.embed:de.flapdoodle.embed.mongo' + testImplementation 'de.flapdoodle.embed:de.flapdoodle.embed.mongo.spring26x' } springBoot { getMainClass().set("com.tencent.bk.job.logsvr.JobLogBootApplication") @@ -41,8 +42,7 @@ task renameArtifacts(type: Copy) { destinationDir file('build/libs/') rename "boot-job-logsvr-${version}.jar", "job-logsvr-${version}.jar" } -renameArtifacts.dependsOn jar -assemble.dependsOn renameArtifacts +renameArtifacts.dependsOn assemble task copyToLatestJar(type: Copy) { group = "local" @@ -53,3 +53,4 @@ task copyToLatestJar(type: Copy) { } copyToLatestJar.dependsOn assemble apply from: "$rootDir/task_job_package.gradle" +copyToRelease.dependsOn renameArtifacts diff --git a/src/backend/job-logsvr/boot-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/JobLogBootApplication.java b/src/backend/job-logsvr/boot-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/JobLogBootApplication.java index 8cb2684798..fa9f7b6053 100644 --- a/src/backend/job-logsvr/boot-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/JobLogBootApplication.java +++ b/src/backend/job-logsvr/boot-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/JobLogBootApplication.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,14 +24,16 @@ package com.tencent.bk.job.logsvr; +import com.tencent.bk.job.common.service.boot.JobBootApplication; import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.autoconfigure.availability.ApplicationAvailabilityAutoConfiguration; +import org.springframework.boot.autoconfigure.jooq.JooqAutoConfiguration; import org.springframework.cloud.openfeign.EnableFeignClients; -/** - * @date 2019/09/20 - */ -@SpringBootApplication(scanBasePackages = "com.tencent.bk.job") +@JobBootApplication( + scanBasePackages = "com.tencent.bk.job.logsvr", + exclude = {JooqAutoConfiguration.class, ApplicationAvailabilityAutoConfiguration.class}, + excludeName = {"org.springframework.cloud.kubernetes.client.discovery.KubernetesDiscoveryClientAutoConfiguration"}) @EnableFeignClients public class JobLogBootApplication { public static void main(String[] args) { diff --git a/src/backend/job-logsvr/boot-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/config/InterceptorConfiguration.java b/src/backend/job-logsvr/boot-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/config/InterceptorConfiguration.java deleted file mode 100644 index f287e4e87d..0000000000 --- a/src/backend/job-logsvr/boot-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/config/InterceptorConfiguration.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.logsvr.config; - -import com.tencent.bk.job.common.web.interceptor.ServiceSecurityInterceptor; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Configuration; -import org.springframework.web.servlet.config.annotation.InterceptorRegistry; -import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; - -@Configuration -public class InterceptorConfiguration implements WebMvcConfigurer { - private final ServiceSecurityInterceptor serviceSecurityInterceptor; - - @Autowired - public InterceptorConfiguration(ServiceSecurityInterceptor serviceSecurityInterceptor) { - this.serviceSecurityInterceptor = serviceSecurityInterceptor; - } - - @Override - public void addInterceptors(InterceptorRegistry registry) { - // 注册拦截器 - registry.addInterceptor(serviceSecurityInterceptor).addPathPatterns("/**").order(0); - } - -} diff --git a/src/backend/job-logsvr/boot-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/config/MongoDBConfig.java b/src/backend/job-logsvr/boot-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/config/MongoDBConfig.java deleted file mode 100644 index 68370c19a8..0000000000 --- a/src/backend/job-logsvr/boot-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/config/MongoDBConfig.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.logsvr.config; - -import com.mongodb.Block; -import com.mongodb.ReadConcern; -import com.mongodb.ReadPreference; -import com.mongodb.WriteConcern; -import com.mongodb.connection.ConnectionPoolSettings; -import com.mongodb.connection.SocketSettings; -import lombok.extern.slf4j.Slf4j; -import org.springframework.boot.autoconfigure.mongo.MongoClientSettingsBuilderCustomizer; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.data.mongodb.core.WriteConcernResolver; - -import java.util.concurrent.TimeUnit; - -@Configuration -@Slf4j -public class MongoDBConfig { - @Bean - public MongoClientSettingsBuilderCustomizer mongoClientCustomizer() { - log.info("Init MongoClientSettingsBuilderCustomizer"); - Block socketSettings = builder -> builder.connectTimeout(15, TimeUnit.SECONDS) - .readTimeout(60, TimeUnit.SECONDS); - Block connectionPoolSettings = builder -> builder.minSize(200).maxSize(500) - .maxConnectionLifeTime(0, TimeUnit.SECONDS) - .maxConnectionIdleTime(0, TimeUnit.SECONDS); - return clientSettingsBuilder -> clientSettingsBuilder.writeConcern(WriteConcern.W1) - .readConcern(ReadConcern.LOCAL) - .readPreference(ReadPreference.primaryPreferred()) - .applyToSocketSettings(socketSettings) - .applyToConnectionPoolSettings(connectionPoolSettings); - - } - - @Bean - public WriteConcernResolver writeConcernResolver() { - return action -> { - log.info("Using Write Concern of Unacknowledged"); - return WriteConcern.W1; - }; - } -} diff --git a/src/backend/job-logsvr/boot-job-logsvr/src/main/resources/application.yml b/src/backend/job-logsvr/boot-job-logsvr/src/main/resources/application.yml index 39fcad002a..6f393470a8 100644 --- a/src/backend/job-logsvr/boot-job-logsvr/src/main/resources/application.yml +++ b/src/backend/job-logsvr/boot-job-logsvr/src/main/resources/application.yml @@ -9,25 +9,27 @@ spring: name: ${job.security.actuator.user.name} password: ${job.security.actuator.user.password} roles: ENDPOINT_ADMIN + mvc: + pathmatch: + matching-strategy: ant_path_matcher job: security: service: private-key-base64: ${job.security.private-key-base64} public-key-base64: ${job.security.public-key-base64} - -ribbon: - ServerListRefreshInterval: 3000 - management: health: livenessstate: enabled: true readinessstate: enabled: true + # 使用spring cloud stream rabbitmq, 禁用掉springboot的rabbitmq health检查 + rabbit: + enabled: false endpoints: web: exposure: - include: health,configprops,env,beans,conditions,loggers,metrics,mappings,prometheus,scheduledtasks,info,refresh + include: health,configprops,env,beans,conditions,loggers,metrics,mappings,prometheus,scheduledtasks,info,refresh,busrefresh,bindings base-path: /actuator enabled-by-default: false endpoint: @@ -58,6 +60,8 @@ management: enabled: true refresh: enabled: true + busrefresh: + enabled: true server: port: 19806 servlet: diff --git a/src/backend/job-logsvr/boot-job-logsvr/src/main/resources/bootstrap.yml b/src/backend/job-logsvr/boot-job-logsvr/src/main/resources/bootstrap.yml index ddeac107d9..7def625845 100644 --- a/src/backend/job-logsvr/boot-job-logsvr/src/main/resources/bootstrap.yml +++ b/src/backend/job-logsvr/boot-job-logsvr/src/main/resources/bootstrap.yml @@ -11,3 +11,5 @@ spring: name: ${JOB_APPLICATION_CONFIGMAP_NAME} - namespace: ${KUBERNETES_NAMESPACE} name: ${JOB_COMMON_CONFIGMAP_NAME} + - namespace: ${KUBERNETES_NAMESPACE} + name: ${JOB_COMMON_REFRESHABLE_CONFIGMAP_NAME} diff --git a/src/backend/job-logsvr/boot-job-logsvr/src/main/resources/i18n/message.properties b/src/backend/job-logsvr/boot-job-logsvr/src/main/resources/i18n/message.properties deleted file mode 100644 index e028c76106..0000000000 --- a/src/backend/job-logsvr/boot-job-logsvr/src/main/resources/i18n/message.properties +++ /dev/null @@ -1,24 +0,0 @@ -# -# Tencent is pleased to support the open source community by making BK-JOB����������ҵƽ̨ available. -# -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. -# -# BK-JOB����������ҵƽ̨ is licensed under the MIT License. -# -# License for BK-JOB����������ҵƽ̨: -# -------------------------------------------------------------------- -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF -# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - diff --git a/src/backend/job-logsvr/boot-job-logsvr/src/main/resources/i18n/message_en.properties b/src/backend/job-logsvr/boot-job-logsvr/src/main/resources/i18n/message_en.properties deleted file mode 100644 index e028c76106..0000000000 --- a/src/backend/job-logsvr/boot-job-logsvr/src/main/resources/i18n/message_en.properties +++ /dev/null @@ -1,24 +0,0 @@ -# -# Tencent is pleased to support the open source community by making BK-JOB����������ҵƽ̨ available. -# -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. -# -# BK-JOB����������ҵƽ̨ is licensed under the MIT License. -# -# License for BK-JOB����������ҵƽ̨: -# -------------------------------------------------------------------- -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF -# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - diff --git a/src/backend/job-logsvr/boot-job-logsvr/src/main/resources/i18n/message_en_US.properties b/src/backend/job-logsvr/boot-job-logsvr/src/main/resources/i18n/message_en_US.properties deleted file mode 100644 index e028c76106..0000000000 --- a/src/backend/job-logsvr/boot-job-logsvr/src/main/resources/i18n/message_en_US.properties +++ /dev/null @@ -1,24 +0,0 @@ -# -# Tencent is pleased to support the open source community by making BK-JOB����������ҵƽ̨ available. -# -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. -# -# BK-JOB����������ҵƽ̨ is licensed under the MIT License. -# -# License for BK-JOB����������ҵƽ̨: -# -------------------------------------------------------------------- -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF -# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - diff --git a/src/backend/job-logsvr/boot-job-logsvr/src/main/resources/i18n/message_zh.properties b/src/backend/job-logsvr/boot-job-logsvr/src/main/resources/i18n/message_zh.properties deleted file mode 100644 index e028c76106..0000000000 --- a/src/backend/job-logsvr/boot-job-logsvr/src/main/resources/i18n/message_zh.properties +++ /dev/null @@ -1,24 +0,0 @@ -# -# Tencent is pleased to support the open source community by making BK-JOB����������ҵƽ̨ available. -# -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. -# -# BK-JOB����������ҵƽ̨ is licensed under the MIT License. -# -# License for BK-JOB����������ҵƽ̨: -# -------------------------------------------------------------------- -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF -# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - diff --git a/src/backend/job-logsvr/boot-job-logsvr/src/main/resources/i18n/message_zh_CN.properties b/src/backend/job-logsvr/boot-job-logsvr/src/main/resources/i18n/message_zh_CN.properties deleted file mode 100644 index e028c76106..0000000000 --- a/src/backend/job-logsvr/boot-job-logsvr/src/main/resources/i18n/message_zh_CN.properties +++ /dev/null @@ -1,24 +0,0 @@ -# -# Tencent is pleased to support the open source community by making BK-JOB����������ҵƽ̨ available. -# -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. -# -# BK-JOB����������ҵƽ̨ is licensed under the MIT License. -# -# License for BK-JOB����������ҵƽ̨: -# -------------------------------------------------------------------- -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF -# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - diff --git a/src/backend/job-logsvr/boot-job-logsvr/src/main/resources/logback-spring.xml b/src/backend/job-logsvr/boot-job-logsvr/src/main/resources/logback-spring.xml new file mode 100644 index 0000000000..ea8880550c --- /dev/null +++ b/src/backend/job-logsvr/boot-job-logsvr/src/main/resources/logback-spring.xml @@ -0,0 +1,13 @@ + + + logback + + + + + + + + + + diff --git a/src/backend/job-logsvr/boot-job-logsvr/src/main/resources/logback/logback-app-props.xml b/src/backend/job-logsvr/boot-job-logsvr/src/main/resources/logback/logback-app-props.xml deleted file mode 100644 index 352f606720..0000000000 --- a/src/backend/job-logsvr/boot-job-logsvr/src/main/resources/logback/logback-app-props.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - diff --git a/src/backend/job-logsvr/boot-job-logsvr/src/test/java/com/tencent/bk/job/logsvr/service/impl/LogServiceImplIntegrationTest.java b/src/backend/job-logsvr/boot-job-logsvr/src/test/java/com/tencent/bk/job/logsvr/service/impl/LogServiceImplIntegrationTest.java index 0731fa11f3..ca7b226017 100644 --- a/src/backend/job-logsvr/boot-job-logsvr/src/test/java/com/tencent/bk/job/logsvr/service/impl/LogServiceImplIntegrationTest.java +++ b/src/backend/job-logsvr/boot-job-logsvr/src/test/java/com/tencent/bk/job/logsvr/service/impl/LogServiceImplIntegrationTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,13 +24,14 @@ package com.tencent.bk.job.logsvr.service.impl; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.logsvr.consts.FileTaskModeEnum; import com.tencent.bk.job.logsvr.consts.LogTypeEnum; import com.tencent.bk.job.logsvr.model.FileLogQuery; import com.tencent.bk.job.logsvr.model.FileTaskLogDoc; import com.tencent.bk.job.logsvr.model.ScriptLogQuery; import com.tencent.bk.job.logsvr.model.ScriptTaskLogDoc; -import com.tencent.bk.job.logsvr.model.TaskHostLog; +import com.tencent.bk.job.logsvr.model.TaskExecuteObjectLog; import com.tencent.bk.job.logsvr.mongo.FileLogsCollectionLoader; import com.tencent.bk.job.logsvr.mongo.LogCollectionFactory; import com.tencent.bk.job.logsvr.mongo.LogCollectionLoaderFactory; @@ -60,31 +61,187 @@ public class LogServiceImplIntegrationTest { @Autowired private LogService logService; + @Nested + @DisplayName("测试查询脚本日志") + class GetScriptLogTest { + @Test + @DisplayName("测试通过关键字查询IP - 关键字包含特殊字符") + void testGetIpsBykeyword() { + String jobCreateDate = "2023_03_21"; + long stepInstanceId = 1L; + int executeCount = 1; + int batch = 1; + + String content = "str1 ~`!@#$%^&*()_+-=;,./?\\|':><{}"; + String ip = "0:127.0.0.1"; + String ipv6 = "0:::1"; + long hostId = 1L; + TaskExecuteObjectLog taskExecuteObjectLog1 = new TaskExecuteObjectLog(); + taskExecuteObjectLog1.setStepInstanceId(stepInstanceId); + taskExecuteObjectLog1.setExecuteCount(executeCount); + taskExecuteObjectLog1.setBatch(batch); + taskExecuteObjectLog1.setJobCreateDate(jobCreateDate); + ScriptTaskLogDoc scriptTaskLog1 = new ScriptTaskLogDoc(stepInstanceId, executeCount, batch, hostId, ip, + ipv6, content, 10); + taskExecuteObjectLog1.setScriptTaskLog(scriptTaskLog1); + taskExecuteObjectLog1.setLogType(LogTypeEnum.SCRIPT.getValue()); + logService.saveLog(taskExecuteObjectLog1); + content = "str2 ?|':><{}"; + ip = "0:127.0.0.2"; + ipv6 = "0:::2"; + hostId = 2L; + TaskExecuteObjectLog taskExecuteObjectLog2 = new TaskExecuteObjectLog(); + taskExecuteObjectLog2.setStepInstanceId(stepInstanceId); + taskExecuteObjectLog2.setExecuteCount(executeCount); + taskExecuteObjectLog2.setBatch(batch); + taskExecuteObjectLog2.setJobCreateDate(jobCreateDate); + ScriptTaskLogDoc scriptTaskLog2 = new ScriptTaskLogDoc(stepInstanceId, executeCount, batch, hostId, ip, + ipv6, content, 10); + taskExecuteObjectLog2.setScriptTaskLog(scriptTaskLog2); + taskExecuteObjectLog2.setLogType(LogTypeEnum.SCRIPT.getValue()); + logService.saveLog(taskExecuteObjectLog2); + + String keyword = "str"; + List hostDTOS = logService.getHostsByKeyword(jobCreateDate, stepInstanceId, executeCount, + batch, keyword); + assertThat(hostDTOS).hasSize(2); + assertThat(hostDTOS).extracting("hostId").containsOnly(1L, 2L); + keyword = "job"; + hostDTOS = logService.getHostsByKeyword(jobCreateDate, stepInstanceId, executeCount,batch, keyword); + assertThat(hostDTOS).hasSize(0); + keyword = "~"; + hostDTOS = logService.getHostsByKeyword(jobCreateDate, stepInstanceId, executeCount,batch, keyword); + assertThat(hostDTOS).hasSize(1); + assertThat(hostDTOS).extracting("hostId").containsOnly(1L); + keyword = "`"; + hostDTOS = logService.getHostsByKeyword(jobCreateDate, stepInstanceId, executeCount, batch, keyword); + assertThat(hostDTOS).hasSize(1); + assertThat(hostDTOS).extracting("hostId").containsOnly(1L); + keyword = "!"; + hostDTOS = logService.getHostsByKeyword(jobCreateDate, stepInstanceId, executeCount, batch, keyword); + assertThat(hostDTOS).hasSize(1); + assertThat(hostDTOS).extracting("hostId").containsOnly(1L); + keyword = "@"; + hostDTOS = logService.getHostsByKeyword(jobCreateDate, stepInstanceId, executeCount, batch, keyword); + assertThat(hostDTOS).hasSize(1); + assertThat(hostDTOS).extracting("hostId").containsOnly(1L); + keyword = "#"; + hostDTOS = logService.getHostsByKeyword(jobCreateDate, stepInstanceId, executeCount, batch, keyword); + assertThat(hostDTOS).hasSize(1); + assertThat(hostDTOS).extracting("hostId").containsOnly(1L); + keyword = "$"; + hostDTOS = logService.getHostsByKeyword(jobCreateDate, stepInstanceId, executeCount, batch, keyword); + assertThat(hostDTOS).hasSize(1); + assertThat(hostDTOS).extracting("hostId").containsOnly(1L); + keyword = "%"; + hostDTOS = logService.getHostsByKeyword(jobCreateDate, stepInstanceId, executeCount, batch, keyword); + assertThat(hostDTOS).hasSize(1); + assertThat(hostDTOS).extracting("hostId").containsOnly(1L); + keyword = "^"; + hostDTOS = logService.getHostsByKeyword(jobCreateDate, stepInstanceId, executeCount, batch, keyword); + assertThat(hostDTOS).hasSize(1); + assertThat(hostDTOS).extracting("hostId").containsOnly(1L); + keyword = "&"; + hostDTOS = logService.getHostsByKeyword(jobCreateDate, stepInstanceId, executeCount, batch, keyword); + assertThat(hostDTOS).hasSize(1); + assertThat(hostDTOS).extracting("hostId").containsOnly(1L); + keyword = "*"; + hostDTOS = logService.getHostsByKeyword(jobCreateDate, stepInstanceId, executeCount, batch, keyword); + assertThat(hostDTOS).hasSize(1); + assertThat(hostDTOS).extracting("hostId").containsOnly(1L); + keyword = "("; + hostDTOS = logService.getHostsByKeyword(jobCreateDate, stepInstanceId, executeCount, batch, keyword); + assertThat(hostDTOS).hasSize(1); + assertThat(hostDTOS).extracting("hostId").containsOnly(1L); + keyword = ")"; + hostDTOS = logService.getHostsByKeyword(jobCreateDate, stepInstanceId, executeCount, batch, keyword); + assertThat(hostDTOS).hasSize(1); + assertThat(hostDTOS).extracting("hostId").containsOnly(1L); + keyword = "_"; + hostDTOS = logService.getHostsByKeyword(jobCreateDate, stepInstanceId, executeCount, batch, keyword); + assertThat(hostDTOS).hasSize(1); + assertThat(hostDTOS).extracting("hostId").containsOnly(1L); + keyword = "+"; + hostDTOS = logService.getHostsByKeyword(jobCreateDate, stepInstanceId, executeCount, batch, keyword); + assertThat(hostDTOS).hasSize(1); + assertThat(hostDTOS).extracting("hostId").containsOnly(1L); + keyword = "="; + hostDTOS = logService.getHostsByKeyword(jobCreateDate, stepInstanceId, executeCount, batch, keyword); + assertThat(hostDTOS).hasSize(1); + assertThat(hostDTOS).extracting("hostId").containsOnly(1L); + keyword = ";"; + hostDTOS = logService.getHostsByKeyword(jobCreateDate, stepInstanceId, executeCount, batch, keyword); + assertThat(hostDTOS).hasSize(1); + assertThat(hostDTOS).extracting("hostId").containsOnly(1L); + keyword = ","; + hostDTOS = logService.getHostsByKeyword(jobCreateDate, stepInstanceId, executeCount, batch, keyword); + assertThat(hostDTOS).hasSize(1); + assertThat(hostDTOS).extracting("hostId").containsOnly(1L); + keyword = "."; + hostDTOS = logService.getHostsByKeyword(jobCreateDate, stepInstanceId, executeCount, batch, keyword); + assertThat(hostDTOS).hasSize(1); + assertThat(hostDTOS).extracting("hostId").containsOnly(1L); + keyword = "/"; + hostDTOS = logService.getHostsByKeyword(jobCreateDate, stepInstanceId, executeCount, batch, keyword); + assertThat(hostDTOS).hasSize(1); + assertThat(hostDTOS).extracting("hostId").containsOnly(1L); + keyword = "?"; + hostDTOS = logService.getHostsByKeyword(jobCreateDate, stepInstanceId, executeCount, batch, keyword); + assertThat(hostDTOS).hasSize(2); + assertThat(hostDTOS).extracting("hostId").containsOnly(1L, 2L); + keyword = "\\"; + hostDTOS = logService.getHostsByKeyword(jobCreateDate, stepInstanceId, executeCount, batch, keyword); + assertThat(hostDTOS).hasSize(1); + assertThat(hostDTOS).extracting("hostId").containsOnly(1L); + keyword = ":"; + hostDTOS = logService.getHostsByKeyword(jobCreateDate, stepInstanceId, executeCount, batch, keyword); + assertThat(hostDTOS).hasSize(2); + assertThat(hostDTOS).extracting("hostId").containsOnly(1L, 2L); + keyword = "<"; + hostDTOS = logService.getHostsByKeyword(jobCreateDate, stepInstanceId, executeCount, batch, keyword); + assertThat(hostDTOS).hasSize(2); + assertThat(hostDTOS).extracting("hostId").containsOnly(1L, 2L); + keyword = ">"; + hostDTOS = logService.getHostsByKeyword(jobCreateDate, stepInstanceId, executeCount, batch, keyword); + assertThat(hostDTOS).hasSize(2); + assertThat(hostDTOS).extracting("hostId").containsOnly(1L, 2L); + keyword = "{"; + hostDTOS = logService.getHostsByKeyword(jobCreateDate, stepInstanceId, executeCount, batch, keyword); + assertThat(hostDTOS).hasSize(2); + assertThat(hostDTOS).extracting("hostId").containsOnly(1L, 2L); + keyword = "}"; + hostDTOS = logService.getHostsByKeyword(jobCreateDate, stepInstanceId, executeCount, batch, keyword); + assertThat(hostDTOS).hasSize(2); + assertThat(hostDTOS).extracting("hostId").containsOnly(1L, 2L); + } + } + + @Nested @DisplayName("测试保存脚本日志") class SaveScriptLogTest { @Test - @DisplayName("测试保存脚本日志 - 使用hostId") + @DisplayName("测试保存脚本日志") void testSaveScriptLogV2() { - TaskHostLog taskHostLog1 = new TaskHostLog(); - taskHostLog1.setStepInstanceId(1L); - taskHostLog1.setExecuteCount(0); - taskHostLog1.setBatch(1); - taskHostLog1.setJobCreateDate("2020_07_29"); - ScriptTaskLogDoc scriptTaskLog1 = new ScriptTaskLogDoc(1L, 0, 1, 101L, null, "hello", 10); - taskHostLog1.setScriptTaskLog(scriptTaskLog1); - taskHostLog1.setLogType(LogTypeEnum.SCRIPT.getValue()); - logService.saveLog(taskHostLog1); + TaskExecuteObjectLog taskExecuteObjectLog1 = new TaskExecuteObjectLog(); + taskExecuteObjectLog1.setStepInstanceId(1L); + taskExecuteObjectLog1.setExecuteCount(0); + taskExecuteObjectLog1.setBatch(1); + taskExecuteObjectLog1.setJobCreateDate("2020_07_29"); + ScriptTaskLogDoc scriptTaskLog1 = new ScriptTaskLogDoc(1L, 0, 1, 101L, "0:127.0.0.1", "0:::1", "hello", 10); + taskExecuteObjectLog1.setScriptTaskLog(scriptTaskLog1); + taskExecuteObjectLog1.setLogType(LogTypeEnum.SCRIPT.getValue()); + logService.saveLog(taskExecuteObjectLog1); - TaskHostLog taskHostLog2 = new TaskHostLog(); - taskHostLog2.setStepInstanceId(1L); - taskHostLog2.setExecuteCount(0); - taskHostLog2.setBatch(1); - taskHostLog2.setJobCreateDate("2020_07_29"); - ScriptTaskLogDoc scriptTaskLog2 = new ScriptTaskLogDoc(1L, 0, 1, 101L, null, "world", 20); - taskHostLog2.setScriptTaskLog(scriptTaskLog2); - taskHostLog2.setLogType(LogTypeEnum.SCRIPT.getValue()); - logService.saveLog(taskHostLog2); + TaskExecuteObjectLog taskExecuteObjectLog2 = new TaskExecuteObjectLog(); + taskExecuteObjectLog2.setStepInstanceId(1L); + taskExecuteObjectLog2.setExecuteCount(0); + taskExecuteObjectLog2.setBatch(1); + taskExecuteObjectLog2.setJobCreateDate("2020_07_29"); + ScriptTaskLogDoc scriptTaskLog2 = new ScriptTaskLogDoc(1L, 0, 1, 101L, "0:127.0.0.1", "0:::1", "world", 20); + taskExecuteObjectLog2.setScriptTaskLog(scriptTaskLog2); + taskExecuteObjectLog2.setLogType(LogTypeEnum.SCRIPT.getValue()); + logService.saveLog(taskExecuteObjectLog2); ScriptLogQuery searchRequest = new ScriptLogQuery(); searchRequest.setStepInstanceId(1L); @@ -92,32 +249,34 @@ void testSaveScriptLogV2() { searchRequest.setBatch(1); searchRequest.setJobCreateDate("2020_07_29"); searchRequest.setHostIds(Collections.singletonList(101L)); - List result = logService.listScriptLogs(searchRequest); + List result = logService.listScriptLogs(searchRequest); assertThat(result).hasSize(1); - TaskHostLog hostLog = result.get(0); + TaskExecuteObjectLog hostLog = result.get(0); assertThat(hostLog.getStepInstanceId()).isEqualTo(1L); assertThat(hostLog.getExecuteCount()).isEqualTo(0); assertThat(hostLog.getBatch()).isEqualTo(1); assertThat(hostLog.getHostId()).isEqualTo(101L); + assertThat(hostLog.getIp()).isEqualTo("0:127.0.0.1"); + assertThat(hostLog.getIpv6()).isEqualTo("0:::1"); assertThat(hostLog.getScriptContent()).isEqualTo("helloworld"); - TaskHostLog taskHostLog3 = new TaskHostLog(); - taskHostLog3.setStepInstanceId(2L); - taskHostLog3.setExecuteCount(0); - taskHostLog3.setBatch(null); - taskHostLog3.setJobCreateDate("2020_07_29"); - ScriptTaskLogDoc scriptTaskLog3 = new ScriptTaskLogDoc(2L, 0, null, 101L, null, "abc", 20); - taskHostLog3.setScriptTaskLog(scriptTaskLog3); - taskHostLog3.setLogType(LogTypeEnum.SCRIPT.getValue()); - logService.saveLog(taskHostLog3); + TaskExecuteObjectLog taskExecuteObjectLog3 = new TaskExecuteObjectLog(); + taskExecuteObjectLog3.setStepInstanceId(2L); + taskExecuteObjectLog3.setExecuteCount(0); + taskExecuteObjectLog3.setBatch(null); + taskExecuteObjectLog3.setJobCreateDate("2020_07_29"); + ScriptTaskLogDoc scriptTaskLog3 = new ScriptTaskLogDoc(2L, 0, null, 101L, "0:127.0.0.1", "0:::1", "abc", + 20); + taskExecuteObjectLog3.setScriptTaskLog(scriptTaskLog3); + taskExecuteObjectLog3.setLogType(LogTypeEnum.SCRIPT.getValue()); + logService.saveLog(taskExecuteObjectLog3); searchRequest = new ScriptLogQuery(); searchRequest.setStepInstanceId(2L); searchRequest.setExecuteCount(0); searchRequest.setBatch(null); searchRequest.setJobCreateDate("2020_07_29"); - searchRequest.setIps(Collections.singletonList(null)); searchRequest.setHostIds(Collections.singletonList(101L)); result = logService.listScriptLogs(searchRequest); assertThat(result).hasSize(1); @@ -126,72 +285,8 @@ void testSaveScriptLogV2() { assertThat(hostLog.getExecuteCount()).isEqualTo(0); assertThat(hostLog.getBatch()).isEqualTo(null); assertThat(hostLog.getHostId()).isEqualTo(101L); - assertThat(hostLog.getScriptContent()).isEqualTo("abc"); - } - - @Test - @DisplayName("测试保存脚本日志 - 使用ip") - void testSaveScriptLogV1() { - TaskHostLog taskHostLog1 = new TaskHostLog(); - taskHostLog1.setStepInstanceId(1L); - taskHostLog1.setExecuteCount(0); - taskHostLog1.setBatch(1); - taskHostLog1.setJobCreateDate("2020_07_29"); - taskHostLog1.setIp("0:127.0.0.1"); - ScriptTaskLogDoc scriptTaskLog1 = new ScriptTaskLogDoc(1L, 0, 1, null, "0:127.0.0.1", "hello", 10); - taskHostLog1.setScriptTaskLog(scriptTaskLog1); - taskHostLog1.setLogType(LogTypeEnum.SCRIPT.getValue()); - logService.saveLog(taskHostLog1); - - TaskHostLog taskHostLog2 = new TaskHostLog(); - taskHostLog2.setStepInstanceId(1L); - taskHostLog2.setExecuteCount(0); - taskHostLog2.setBatch(1); - taskHostLog2.setJobCreateDate("2020_07_29"); - ScriptTaskLogDoc scriptTaskLog2 = new ScriptTaskLogDoc(1L, 0, 1, null, "0:127.0.0.1", "world", 20); - taskHostLog2.setScriptTaskLog(scriptTaskLog2); - taskHostLog2.setLogType(LogTypeEnum.SCRIPT.getValue()); - logService.saveLog(taskHostLog2); - - ScriptLogQuery searchRequest = new ScriptLogQuery(); - searchRequest.setStepInstanceId(1L); - searchRequest.setExecuteCount(0); - searchRequest.setBatch(1); - searchRequest.setJobCreateDate("2020_07_29"); - searchRequest.setIps(Collections.singletonList("0:127.0.0.1")); - List result = logService.listScriptLogs(searchRequest); - assertThat(result).hasSize(1); - TaskHostLog hostLog = result.get(0); - assertThat(hostLog.getStepInstanceId()).isEqualTo(1L); - assertThat(hostLog.getExecuteCount()).isEqualTo(0); - assertThat(hostLog.getBatch()).isEqualTo(1); - assertThat(hostLog.getIp()).isEqualTo("0:127.0.0.1"); - assertThat(hostLog.getScriptContent()).isEqualTo("helloworld"); - - - TaskHostLog taskHostLog3 = new TaskHostLog(); - taskHostLog3.setStepInstanceId(2L); - taskHostLog3.setExecuteCount(0); - taskHostLog3.setBatch(null); - taskHostLog3.setJobCreateDate("2020_07_29"); - ScriptTaskLogDoc scriptTaskLog3 = new ScriptTaskLogDoc(2L, 0, null, null, "0:127.0.0.1", "abc", 20); - taskHostLog3.setScriptTaskLog(scriptTaskLog3); - taskHostLog3.setLogType(LogTypeEnum.SCRIPT.getValue()); - logService.saveLog(taskHostLog3); - - searchRequest = new ScriptLogQuery(); - searchRequest.setStepInstanceId(2L); - searchRequest.setExecuteCount(0); - searchRequest.setBatch(null); - searchRequest.setJobCreateDate("2020_07_29"); - searchRequest.setIps(Collections.singletonList("0:127.0.0.1")); - result = logService.listScriptLogs(searchRequest); - assertThat(result).hasSize(1); - hostLog = result.get(0); - assertThat(hostLog.getStepInstanceId()).isEqualTo(2L); - assertThat(hostLog.getExecuteCount()).isEqualTo(0); - assertThat(hostLog.getBatch()).isEqualTo(null); assertThat(hostLog.getIp()).isEqualTo("0:127.0.0.1"); + assertThat(hostLog.getIpv6()).isEqualTo("0:::1"); assertThat(hostLog.getScriptContent()).isEqualTo("abc"); } } @@ -199,101 +294,22 @@ void testSaveScriptLogV1() { @Nested @DisplayName("测试保存文件分发日志") class SaveFileLogTest { - @Test - @DisplayName("测试保存文件分发日志 - 使用IP") - void testSaveFileLog1V1() { - FileTaskLogDoc fileTaskLog1 = buildFileTaskDetailLog( - FileTaskModeEnum.DOWNLOAD.getValue(), - null, - "0:127.0.0.1", - "/tmp/1.log", - "/tmp/1.log", - null, - "0:127.0.0.2", - "/tmp/2.log", - 3, - "Downloading", - "100KB/S", - "100MB", - "50%", - "[2020-07-30 11:00:00] Downloading...\n"); - - List fileTaskLogList = new ArrayList<>(); - fileTaskLogList.add(fileTaskLog1); - TaskHostLog taskHostLog = buildFileTaskHostLog(1L, 0, "2020_07_29", null, - "0:127.0.0.2", fileTaskLogList); - logService.saveLog(taskHostLog); - - FileLogQuery searchRequest = FileLogQuery.builder() - .stepInstanceId(1L) - .executeCount(0) - .jobCreateDate("2020_07_29") - .ips(Collections.singletonList("0:127.0.0.2")) - .build(); - List fileLogDocs = logService.listFileLogs(searchRequest); - assertThat(fileLogDocs).hasSize(1); - FileTaskLogDoc resultFileTaskLog1 = fileLogDocs.get(0); - assertThat(resultFileTaskLog1.getMode()).isEqualTo(FileTaskModeEnum.DOWNLOAD.getValue()); - assertThat(resultFileTaskLog1.getSrcIp()).isEqualTo("0:127.0.0.1"); - assertThat(resultFileTaskLog1.getSrcFile()).isEqualTo("/tmp/1.log"); - assertThat(resultFileTaskLog1.getIp()).isEqualTo("0:127.0.0.2"); - assertThat(resultFileTaskLog1.getDestFile()).isEqualTo("/tmp/2.log"); - assertThat(resultFileTaskLog1.getStatus()).isEqualTo(3); - assertThat(resultFileTaskLog1.getStatusDesc()).isEqualTo("Downloading"); - assertThat(resultFileTaskLog1.getSize()).isEqualTo("100MB"); - assertThat(resultFileTaskLog1.getSpeed()).isEqualTo("100KB/S"); - assertThat(resultFileTaskLog1.getProcess()).isEqualTo("50%"); - assertThat(resultFileTaskLog1.getContent()).isEqualTo("[2020-07-30 11:00:00] Downloading...\n"); - - // 再次插入日志，验证日志更新场景 - fileTaskLog1 = buildFileTaskDetailLog( - FileTaskModeEnum.DOWNLOAD.getValue(), - null, - "0:127.0.0.1", - "/tmp/1.log", - "/tmp/1.log", - null, - "0:127.0.0.2", - "/tmp/2.log", - 4, - "Finished", - "0KB/S", - "100MB", - "100%", - "[2020-07-30 11:00:00] Download success\n"); - fileTaskLogList.clear(); - fileTaskLogList.add(fileTaskLog1); - taskHostLog = buildFileTaskHostLog(1L, 0, "2020_07_29", null, "0:127.0.0.2", - fileTaskLogList); - logService.saveLog(taskHostLog); - - fileLogDocs = logService.listFileLogs(searchRequest); - assertThat(fileLogDocs).hasSize(1); - resultFileTaskLog1 = fileLogDocs.get(0); - assertThat(resultFileTaskLog1.getMode()).isEqualTo(FileTaskModeEnum.DOWNLOAD.getValue()); - assertThat(resultFileTaskLog1.getSrcIp()).isEqualTo("0:127.0.0.1"); - assertThat(resultFileTaskLog1.getSrcFile()).isEqualTo("/tmp/1.log"); - assertThat(resultFileTaskLog1.getIp()).isEqualTo("0:127.0.0.2"); - assertThat(resultFileTaskLog1.getDestFile()).isEqualTo("/tmp/2.log"); - assertThat(resultFileTaskLog1.getStatus()).isEqualTo(4); - assertThat(resultFileTaskLog1.getStatusDesc()).isEqualTo("Finished"); - assertThat(resultFileTaskLog1.getSize()).isEqualTo("100MB"); - assertThat(resultFileTaskLog1.getSpeed()).isEqualTo("0KB/S"); - assertThat(resultFileTaskLog1.getProcess()).isEqualTo("100%"); - assertThat(resultFileTaskLog1.getContent()).isEqualTo("[2020-07-30 11:00:00] Downloading...\n[2020-07-30 11:00:00] Download success\n"); - } @Test - @DisplayName("测试保存文件分发日志 - 使用hostId") - void testSaveFileLog1V2() { + @DisplayName("测试保存文件分发日志") + void testSaveFileLog() { FileTaskLogDoc fileTaskLog1 = buildFileTaskDetailLog( FileTaskModeEnum.DOWNLOAD.getValue(), + 102L, 101L, - null, + "0:127.0.0.1", + "0:::1", + 1, "/tmp/1.log", "/tmp/1.log", 102L, - null, + "0:127.0.0.2", + "0:::2", "/tmp/2.log", 3, "Downloading", @@ -304,9 +320,9 @@ void testSaveFileLog1V2() { List fileTaskLogList = new ArrayList<>(); fileTaskLogList.add(fileTaskLog1); - TaskHostLog taskHostLog = buildFileTaskHostLog(1L, 0, "2020_07_29", 102L, - null, fileTaskLogList); - logService.saveLog(taskHostLog); + TaskExecuteObjectLog taskExecuteObjectLog = buildFileTaskHostLog(1L, 0, "2020_07_29", 102L, + "0:127.0.0.2", "0:::2", fileTaskLogList); + logService.saveLog(taskExecuteObjectLog); FileLogQuery searchRequest = FileLogQuery.builder() .stepInstanceId(1L) @@ -318,9 +334,15 @@ void testSaveFileLog1V2() { assertThat(fileLogDocs).hasSize(1); FileTaskLogDoc resultFileTaskLog1 = fileLogDocs.get(0); assertThat(resultFileTaskLog1.getMode()).isEqualTo(FileTaskModeEnum.DOWNLOAD.getValue()); + assertThat(resultFileTaskLog1.getHostId()).isEqualTo(102L); assertThat(resultFileTaskLog1.getSrcHostId()).isEqualTo(101L); + assertThat(resultFileTaskLog1.getSrcIp()).isEqualTo("0:127.0.0.1"); + assertThat(resultFileTaskLog1.getSrcIpv6()).isEqualTo("0:::1"); + assertThat(resultFileTaskLog1.getSrcFileType()).isEqualTo(1); assertThat(resultFileTaskLog1.getSrcFile()).isEqualTo("/tmp/1.log"); - assertThat(resultFileTaskLog1.getHostId()).isEqualTo(102L); + assertThat(resultFileTaskLog1.getDestHostId()).isEqualTo(102L); + assertThat(resultFileTaskLog1.getDestIp()).isEqualTo("0:127.0.0.2"); + assertThat(resultFileTaskLog1.getDestIpv6()).isEqualTo("0:::2"); assertThat(resultFileTaskLog1.getDestFile()).isEqualTo("/tmp/2.log"); assertThat(resultFileTaskLog1.getStatus()).isEqualTo(3); assertThat(resultFileTaskLog1.getStatusDesc()).isEqualTo("Downloading"); @@ -332,12 +354,16 @@ void testSaveFileLog1V2() { // 再次插入日志，验证日志更新场景 fileTaskLog1 = buildFileTaskDetailLog( FileTaskModeEnum.DOWNLOAD.getValue(), + 102L, 101L, - null, + "0:127.0.0.1", + "0:::1", + 1, "/tmp/1.log", "/tmp/1.log", 102L, - null, + "0:127.0.0.2", + "0:::2", "/tmp/2.log", 4, "Finished", @@ -347,49 +373,65 @@ void testSaveFileLog1V2() { "[2020-07-30 11:00:00] Download success\n"); fileTaskLogList.clear(); fileTaskLogList.add(fileTaskLog1); - taskHostLog = buildFileTaskHostLog(1L, 0, "2020_07_29", 102L, null, + taskExecuteObjectLog = buildFileTaskHostLog(1L, 0, "2020_07_29", 102L, "0:127.0.0.2", "0:::2", fileTaskLogList); - logService.saveLog(taskHostLog); + logService.saveLog(taskExecuteObjectLog); fileLogDocs = logService.listFileLogs(searchRequest); assertThat(fileLogDocs).hasSize(1); resultFileTaskLog1 = fileLogDocs.get(0); assertThat(resultFileTaskLog1.getMode()).isEqualTo(FileTaskModeEnum.DOWNLOAD.getValue()); + assertThat(resultFileTaskLog1.getHostId()).isEqualTo(102L); assertThat(resultFileTaskLog1.getSrcHostId()).isEqualTo(101L); + assertThat(resultFileTaskLog1.getSrcIp()).isEqualTo("0:127.0.0.1"); + assertThat(resultFileTaskLog1.getSrcIpv6()).isEqualTo("0:::1"); + assertThat(resultFileTaskLog1.getSrcFileType()).isEqualTo(1); assertThat(resultFileTaskLog1.getSrcFile()).isEqualTo("/tmp/1.log"); - assertThat(resultFileTaskLog1.getHostId()).isEqualTo(102L); + assertThat(resultFileTaskLog1.getDestHostId()).isEqualTo(102L); + assertThat(resultFileTaskLog1.getDestIp()).isEqualTo("0:127.0.0.2"); + assertThat(resultFileTaskLog1.getDestIpv6()).isEqualTo("0:::2"); assertThat(resultFileTaskLog1.getDestFile()).isEqualTo("/tmp/2.log"); assertThat(resultFileTaskLog1.getStatus()).isEqualTo(4); assertThat(resultFileTaskLog1.getStatusDesc()).isEqualTo("Finished"); assertThat(resultFileTaskLog1.getSize()).isEqualTo("100MB"); assertThat(resultFileTaskLog1.getSpeed()).isEqualTo("0KB/S"); assertThat(resultFileTaskLog1.getProcess()).isEqualTo("100%"); - assertThat(resultFileTaskLog1.getContent()).isEqualTo("[2020-07-30 11:00:00] Downloading...\n[2020-07-30 11:00:00] Download success\n"); + assertThat(resultFileTaskLog1.getContent()).isEqualTo("[2020-07-30 11:00:00] Downloading...\n[2020-07-30 " + + "11:00:00] Download success\n"); } } - - private TaskHostLog buildFileTaskHostLog(long stepInstanceId, int executeCount, String jobCreateDate, Long hostId, - String ip, List fileTaskLogs) { - TaskHostLog taskHostLog = new TaskHostLog(); - taskHostLog.setStepInstanceId(stepInstanceId); - taskHostLog.setExecuteCount(executeCount); - taskHostLog.setJobCreateDate(jobCreateDate); - taskHostLog.setHostId(hostId); - taskHostLog.setIp(ip); - taskHostLog.setFileTaskLogs(fileTaskLogs); - taskHostLog.setLogType(LogTypeEnum.FILE.getValue()); - return taskHostLog; + private TaskExecuteObjectLog buildFileTaskHostLog(long stepInstanceId, + int executeCount, + String jobCreateDate, + Long hostId, + String ip, + String ipv6, + List fileTaskLogs) { + TaskExecuteObjectLog taskExecuteObjectLog = new TaskExecuteObjectLog(); + taskExecuteObjectLog.setStepInstanceId(stepInstanceId); + taskExecuteObjectLog.setExecuteCount(executeCount); + taskExecuteObjectLog.setJobCreateDate(jobCreateDate); + taskExecuteObjectLog.setHostId(hostId); + taskExecuteObjectLog.setIp(ip); + taskExecuteObjectLog.setIpv6(ipv6); + taskExecuteObjectLog.setFileTaskLogs(fileTaskLogs); + taskExecuteObjectLog.setLogType(LogTypeEnum.FILE.getValue()); + return taskExecuteObjectLog; } FileTaskLogDoc buildFileTaskDetailLog(Integer mode, + Long hostId, Long srcHostId, String srcIp, + String srcIpv6, + Integer srcFileType, String srcFileName, - String srcFileDisplayName, + String displaySrcFile, Long destHostId, String destIp, + String destIpv6, String destFileName, Integer status, String statusDesc, @@ -399,13 +441,20 @@ FileTaskLogDoc buildFileTaskDetailLog(Integer mode, String content) { FileTaskLogDoc fileTaskLogDoc = new FileTaskLogDoc(); fileTaskLogDoc.setMode(mode); + fileTaskLogDoc.setHostId(hostId); + fileTaskLogDoc.setSrcHostId(srcHostId); fileTaskLogDoc.setSrcIp(srcIp); + fileTaskLogDoc.setSrcIpv6(srcIpv6); + fileTaskLogDoc.setSrcFileType(srcFileType); fileTaskLogDoc.setSrcFile(srcFileName); - fileTaskLogDoc.setDisplaySrcFile(srcFileDisplayName); - fileTaskLogDoc.setHostId(destHostId); - fileTaskLogDoc.setIp(destIp); + fileTaskLogDoc.setDisplaySrcFile(displaySrcFile); + fileTaskLogDoc.setDestFile(destFileName); + fileTaskLogDoc.setDestHostId(destHostId); + fileTaskLogDoc.setDestIp(destIp); + fileTaskLogDoc.setDestIpv6(destIpv6); + fileTaskLogDoc.setStatus(status); fileTaskLogDoc.setStatusDesc(statusDesc); fileTaskLogDoc.setSpeed(speed); diff --git a/src/backend/job-logsvr/boot-job-logsvr/src/test/java/com/tencent/bk/job/logsvr/util/CollectionNameUtilTest.java b/src/backend/job-logsvr/boot-job-logsvr/src/test/java/com/tencent/bk/job/logsvr/util/CollectionNameUtilTest.java new file mode 100644 index 0000000000..afd48ddd4b --- /dev/null +++ b/src/backend/job-logsvr/boot-job-logsvr/src/test/java/com/tencent/bk/job/logsvr/util/CollectionNameUtilTest.java @@ -0,0 +1,77 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.logsvr.util; + +import com.tencent.bk.job.logsvr.consts.LogTypeEnum; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * 集合名称工具类测试用例 + */ +public class CollectionNameUtilTest { + + @Test + @DisplayName("集合名称工具类-测试构建日志集合名") + public void testBuildLogCollectionName() { + assertThat(CollectionNameUtil.buildLogCollectionName("2025_01_01", LogTypeEnum.SCRIPT)).isEqualTo( + "job_log_script_2025_01_01"); + assertThat(CollectionNameUtil.buildLogCollectionName("2025_05_01", LogTypeEnum.SCRIPT)).isEqualTo( + "job_log_script_2025_05_01"); + assertThat(CollectionNameUtil.buildLogCollectionName("2025_01_01", LogTypeEnum.FILE)).isEqualTo( + "job_log_file_2025_01_01"); + assertThat(CollectionNameUtil.buildLogCollectionName("2025_05_01", LogTypeEnum.FILE)).isEqualTo( + "job_log_file_2025_05_01"); + } + + @Test + @DisplayName("集合名称工具类-测试获取类型名称") + public void testGetLogTypeName() { + assertThat(CollectionNameUtil.getLogTypeName(LogTypeEnum.SCRIPT)).isEqualTo("script"); + assertThat(CollectionNameUtil.getLogTypeName(LogTypeEnum.FILE)).isEqualTo("file"); + } + + @Test + @DisplayName("集合名称工具类-构建日志集合名称中不带日期的部分") + public void testGetCollectionNamePrefix() { + assertThat(CollectionNameUtil.buildCollectionNamePrefix(LogTypeEnum.SCRIPT)) + .isEqualTo("job_log_script_"); + assertThat(CollectionNameUtil.buildCollectionNamePrefix(LogTypeEnum.FILE)) + .isEqualTo("job_log_file_"); + } + + @Test + @DisplayName("集合名称工具类-测试从日志集合名中提取日期部分") + public void testCollectionNameToDateStr() { + assertThat(CollectionNameUtil.collectionNameToDateStr("job_log_script_2025_01_01")) + .isEqualTo("2025_01_01"); + assertThat(CollectionNameUtil.collectionNameToDateStr("job_log_file_2024_12_31")) + .isEqualTo("2024_12_31"); + assertThat(CollectionNameUtil.collectionNameToDateStr("job_log_api_2025_01_01")) + .isNull(); + } +} diff --git a/src/backend/job-logsvr/boot-job-logsvr/src/test/java/com/tencent/bk/job/logsvr/util/LogFieldUtilTest.java b/src/backend/job-logsvr/boot-job-logsvr/src/test/java/com/tencent/bk/job/logsvr/util/LogFieldUtilTest.java new file mode 100644 index 0000000000..4b81990274 --- /dev/null +++ b/src/backend/job-logsvr/boot-job-logsvr/src/test/java/com/tencent/bk/job/logsvr/util/LogFieldUtilTest.java @@ -0,0 +1,60 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.logsvr.util; + +import org.junit.jupiter.api.Test; + +import java.time.LocalDateTime; +import java.time.ZoneOffset; + +import static java.time.format.DateTimeFormatter.ISO_OFFSET_DATE_TIME; +import static org.assertj.core.api.Assertions.assertThat; + +/** + * 日志字段工具类测试用例 + */ +public class LogFieldUtilTest { + + @Test + public void testBuildJobCreateDate() { + String timeStr = "2024-07-26T00:00:00+00:00"; + LocalDateTime time = LocalDateTime.parse(timeStr, ISO_OFFSET_DATE_TIME); + long timeMills = time.toInstant(ZoneOffset.UTC).getEpochSecond() * 1000L; + System.out.println(timeStr + ": " + timeMills); + assertThat(LogFieldUtil.buildJobCreateDate(timeMills)).isEqualTo("2024_07_26"); + + timeStr = "2024-07-26T12:00:00+00:00"; + time = LocalDateTime.parse(timeStr, ISO_OFFSET_DATE_TIME); + timeMills = time.toInstant(ZoneOffset.UTC).getEpochSecond() * 1000L; + System.out.println(timeStr + ": " + timeMills); + assertThat(LogFieldUtil.buildJobCreateDate(timeMills)).isEqualTo("2024_07_26"); + + timeStr = "2024-07-26T23:59:59+00:00"; + time = LocalDateTime.parse(timeStr, ISO_OFFSET_DATE_TIME); + timeMills = time.toInstant(ZoneOffset.UTC).getEpochSecond() * 1000L; + System.out.println(timeStr + ": " + timeMills); + assertThat(LogFieldUtil.buildJobCreateDate(timeMills)).isEqualTo("2024_07_26"); + } +} diff --git a/src/backend/job-logsvr/boot-job-logsvr/src/test/resources/application-test.yml b/src/backend/job-logsvr/boot-job-logsvr/src/test/resources/application-test.yml index 8237d24386..38883b5f35 100644 --- a/src/backend/job-logsvr/boot-job-logsvr/src/test/resources/application-test.yml +++ b/src/backend/job-logsvr/boot-job-logsvr/src/test/resources/application-test.yml @@ -2,6 +2,25 @@ spring: data: mongodb: uri: ${job.logsvr.mongodb.uri} + mvc: + pathmatch: + matching-strategy: ant_path_matcher cloud: config: enabled: false + kubernetes: + discovery: + enabled: false +de: + flapdoodle: + mongodb: + embedded: + version: 4.0.0 +bk-api-gateway: + gse: + url: gse.apigw.com + cmdb: + url: cmdb.apigw.com +esb: + service: + url: esb.service diff --git a/src/backend/job-logsvr/boot-job-logsvr/src/test/resources/bootstrap.yml b/src/backend/job-logsvr/boot-job-logsvr/src/test/resources/bootstrap.yml index d8e0e80944..7b31fbcfbc 100644 --- a/src/backend/job-logsvr/boot-job-logsvr/src/test/resources/bootstrap.yml +++ b/src/backend/job-logsvr/boot-job-logsvr/src/test/resources/bootstrap.yml @@ -1,5 +1,9 @@ spring: application: name: job-logsvr + cloud: + kubernetes: + enabled: false + server: - port: 19806 \ No newline at end of file + port: 19806 diff --git a/src/backend/job-logsvr/boot-job-logsvr/src/test/resources/logback-spring.xml b/src/backend/job-logsvr/boot-job-logsvr/src/test/resources/logback-spring.xml index c8862a0af8..e85bc6514b 100644 --- a/src/backend/job-logsvr/boot-job-logsvr/src/test/resources/logback-spring.xml +++ b/src/backend/job-logsvr/boot-job-logsvr/src/test/resources/logback-spring.xml @@ -1,78 +1,20 @@ - - - - - logback + value="[%date{yyyy-MM-dd HH:mm:ss.SSS}] %-5level %logger{36}:%method:%line - %msg%n"/> - - ${LOGSVR_LOG_FILE} - - ${LOGSVR_LOG_FILE}-%d{yyyyMMdd_HH}.%i.log - 1GB - 15 - 10GB - - - ${LOG_PATTERN} - UTF-8 - - - - ${ERROR_LOG_FILE} - - ${ERROR_LOG_FILE}-%d{yyyy-MM-dd}.%i.log - 1GB - 15 - 10GB - - - ${LOG_PATTERN} - UTF-8 - - + logback - + ${LOG_PATTERN} utf8 - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/backend/job-logsvr/boot-job-logsvr/src/test/resources/test.properties b/src/backend/job-logsvr/boot-job-logsvr/src/test/resources/test.properties index 8f09aea9ec..55a0c790d5 100644 --- a/src/backend/job-logsvr/boot-job-logsvr/src/test/resources/test.properties +++ b/src/backend/job-logsvr/boot-job-logsvr/src/test/resources/test.properties @@ -1,11 +1,11 @@ # -# Tencent is pleased to support the open source community by making BK-JOB����������ҵƽ̨ available. +# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. # -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +# Copyright (C) 2021 Tencent. All rights reserved. # -# BK-JOB����������ҵƽ̨ is licensed under the MIT License. +# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. # -# License for BK-JOB����������ҵƽ̨: +# License for BK-JOB蓝鲸智云作业平台: # -------------------------------------------------------------------- # Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated # documentation files (the "Software"), to deal in the Software without restriction, including without limitation diff --git a/src/backend/job-logsvr/build.gradle b/src/backend/job-logsvr/build.gradle index 7a7081b142..37fc2c6734 100644 --- a/src/backend/job-logsvr/build.gradle +++ b/src/backend/job-logsvr/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-logsvr/service-job-logsvr/build.gradle b/src/backend/job-logsvr/service-job-logsvr/build.gradle index 6f4a519348..968cf38d34 100644 --- a/src/backend/job-logsvr/service-job-logsvr/build.gradle +++ b/src/backend/job-logsvr/service-job-logsvr/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,14 +27,12 @@ dependencies { api project(":job-logsvr:api-job-logsvr") api project(':commons:common-security') api project(':commons:common-web') + api project(':commons:common-mongodb') api("org.springframework.boot:spring-boot-starter-web") api("org.springframework.boot:spring-boot-starter-data-mongodb") api("org.springframework.cloud:spring-cloud-starter-sleuth") implementation 'org.apache.commons:commons-collections4' testImplementation 'org.springframework.boot:spring-boot-starter-test' - testImplementation 'de.flapdoodle.embed:de.flapdoodle.embed.mongo' - - implementation "ch.qos.logback:logback-core" implementation "ch.qos.logback:logback-classic" implementation "org.slf4j:slf4j-api" diff --git a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/api/service/impl/ServiceLogResourceImpl.java b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/api/service/impl/ServiceLogResourceImpl.java index 622b5e3842..d0ca08b906 100644 --- a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/api/service/impl/ServiceLogResourceImpl.java +++ b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/api/service/impl/ServiceLogResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -33,14 +33,13 @@ import com.tencent.bk.job.logsvr.model.FileTaskLogDoc; import com.tencent.bk.job.logsvr.model.ScriptLogQuery; import com.tencent.bk.job.logsvr.model.ScriptTaskLogDoc; -import com.tencent.bk.job.logsvr.model.TaskHostLog; +import com.tencent.bk.job.logsvr.model.TaskExecuteObjectLog; import com.tencent.bk.job.logsvr.model.service.ServiceBatchSaveLogRequest; +import com.tencent.bk.job.logsvr.model.service.ServiceExecuteObjectLogDTO; +import com.tencent.bk.job.logsvr.model.service.ServiceExecuteObjectLogsDTO; +import com.tencent.bk.job.logsvr.model.service.ServiceExecuteObjectScriptLogDTO; import com.tencent.bk.job.logsvr.model.service.ServiceFileLogQueryRequest; import com.tencent.bk.job.logsvr.model.service.ServiceFileTaskLogDTO; -import com.tencent.bk.job.logsvr.model.service.ServiceHostLogDTO; -import com.tencent.bk.job.logsvr.model.service.ServiceHostLogsDTO; -import com.tencent.bk.job.logsvr.model.service.ServiceSaveLogRequest; -import com.tencent.bk.job.logsvr.model.service.ServiceScriptLogDTO; import com.tencent.bk.job.logsvr.model.service.ServiceScriptLogQueryRequest; import com.tencent.bk.job.logsvr.service.LogService; import lombok.extern.slf4j.Slf4j; @@ -66,87 +65,105 @@ public ServiceLogResourceImpl(LogService logService) { this.logService = logService; } - @Override - public InternalResponse saveLog(ServiceSaveLogRequest request) { - TaskHostLog taskHostLog = convertToTaskLog(request.getLogType(), request.getJobCreateDate(), - request.getStepInstanceId(), request.getExecuteCount(), request.getBatch(), request.getHostId(), - request.getIp(), request.getScriptLog(), request.getFileTaskLogs()); - logService.saveLog(taskHostLog); - return InternalResponse.buildSuccessResp(null); - } - - private TaskHostLog convertToTaskLog(Integer logType, - String jobCreateDate, - long stepInstanceId, - int executeCount, - Integer batch, - Long hostId, - String ip, - ServiceScriptLogDTO scriptLog, - List serviceFileTaskLogs) { - TaskHostLog taskHostLog = new TaskHostLog(); - taskHostLog.setLogType(logType); - taskHostLog.setStepInstanceId(stepInstanceId); - taskHostLog.setExecuteCount(executeCount); - taskHostLog.setBatch(batch); - taskHostLog.setHostId(hostId); - taskHostLog.setIp(ip); - taskHostLog.setJobCreateDate(jobCreateDate); - if (scriptLog != null) { - taskHostLog.setScriptTaskLog(new ScriptTaskLogDoc(stepInstanceId, executeCount, batch, hostId, ip, - scriptLog.getContent(), scriptLog.getOffset())); - } - if (CollectionUtils.isNotEmpty(serviceFileTaskLogs)) { - List fileTaskLogs = serviceFileTaskLogs.stream() - .map(FileTaskLogDoc::convert).collect(Collectors.toList()); - taskHostLog.setFileTaskLogs(fileTaskLogs); - } - return taskHostLog; - } - @Override public InternalResponse saveLogs(ServiceBatchSaveLogRequest request) { if (CollectionUtils.isEmpty(request.getLogs())) { return InternalResponse.buildSuccessResp(null); } - List taskHostLogs = + List taskExecuteObjectLogs = request.getLogs().stream() - .map(log -> convertToTaskLog(request.getLogType(), request.getJobCreateDate(), log.getStepInstanceId(), - log.getExecuteCount(), log.getBatch(), log.getHostId(), log.getIp(), log.getScriptLog(), - log.getFileTaskLogs())) + .map(log -> convertToTaskExecuteObjectLog( + request.getLogType(), + request.getJobCreateDate(), + log)) .collect(Collectors.toList()); LogTypeEnum logType = LogTypeEnum.getLogType(request.getLogType()); - logService.saveLogs(logType, taskHostLogs); + logService.saveLogs(logType, taskExecuteObjectLogs); return InternalResponse.buildSuccessResp(null); } - @Override - public InternalResponse getScriptHostLogByIp(String jobCreateDate, - Long stepInstanceId, - Integer executeCount, - String ip, - Integer batch) { - return InternalResponse.buildSuccessResp( - getScriptHostLog(jobCreateDate, stepInstanceId, executeCount, null, batch, ip)); + private TaskExecuteObjectLog convertToTaskExecuteObjectLog(Integer logType, + String jobCreateDate, + ServiceExecuteObjectLogDTO log) { + TaskExecuteObjectLog taskExecuteObjectLog = new TaskExecuteObjectLog(); + taskExecuteObjectLog.setLogType(logType); + taskExecuteObjectLog.setStepInstanceId(log.getStepInstanceId()); + taskExecuteObjectLog.setExecuteCount(log.getExecuteCount()); + taskExecuteObjectLog.setBatch(log.getBatch()); + if (log.getExecuteObjectId() != null) { + taskExecuteObjectLog.setExecuteObjectId(log.getExecuteObjectId()); + ServiceExecuteObjectScriptLogDTO scriptLog = log.getScriptLog(); + if (scriptLog != null) { + taskExecuteObjectLog.setScriptTaskLog(new ScriptTaskLogDoc(log.getStepInstanceId(), + log.getExecuteCount(), log.getBatch(), log.getExecuteObjectId(), scriptLog.getContent(), + scriptLog.getOffset())); + } + List fileTaskLogs = log.getFileTaskLogs(); + if (CollectionUtils.isNotEmpty(fileTaskLogs)) { + List fileTaskLogDocs = fileTaskLogs.stream() + .map(FileTaskLogDoc::convert).collect(Collectors.toList()); + taskExecuteObjectLog.setFileTaskLogs(fileTaskLogDocs); + } + } else { + // 兼容历史版本使用 hostId的方式 + taskExecuteObjectLog.setHostId(log.getHostId()); + taskExecuteObjectLog.setIp(log.getCloudIp()); + taskExecuteObjectLog.setIpv6(log.getCloudIpv6()); + ServiceExecuteObjectScriptLogDTO scriptLog = log.getScriptLog(); + if (scriptLog != null) { + taskExecuteObjectLog.setScriptTaskLog(new ScriptTaskLogDoc(log.getStepInstanceId(), + log.getExecuteCount(), log.getBatch(), log.getHostId(), log.getCloudIp(), log.getCloudIpv6(), + scriptLog.getContent(), scriptLog.getOffset())); + } + List fileTaskLogs = log.getFileTaskLogs(); + if (CollectionUtils.isNotEmpty(fileTaskLogs)) { + List fileTaskLogDocs = fileTaskLogs.stream() + .map(FileTaskLogDoc::convert).collect(Collectors.toList()); + taskExecuteObjectLog.setFileTaskLogs(fileTaskLogDocs); + } + } + taskExecuteObjectLog.setJobCreateDate(jobCreateDate); + + return taskExecuteObjectLog; } - private ServiceHostLogDTO toServiceLogDTO(TaskHostLog taskHostLog) { - if (taskHostLog == null) { + private ServiceExecuteObjectLogDTO toServiceLogDTO(TaskExecuteObjectLog taskExecuteObjectLog) { + if (taskExecuteObjectLog == null) { return null; } - ServiceHostLogDTO result = new ServiceHostLogDTO(); - result.setStepInstanceId(taskHostLog.getStepInstanceId()); - result.setExecuteCount(taskHostLog.getExecuteCount()); - result.setBatch(taskHostLog.getBatch()); - result.setHostId(taskHostLog.getHostId()); - result.setIp(taskHostLog.getIp()); - if (StringUtils.isNotEmpty(taskHostLog.getScriptContent())) { - result.setScriptLog(new ServiceScriptLogDTO(taskHostLog.getHostId(), taskHostLog.getIp(), - taskHostLog.getScriptContent())); + ServiceExecuteObjectLogDTO result = new ServiceExecuteObjectLogDTO(); + result.setStepInstanceId(taskExecuteObjectLog.getStepInstanceId()); + result.setExecuteCount(taskExecuteObjectLog.getExecuteCount()); + result.setBatch(taskExecuteObjectLog.getBatch()); + if (taskExecuteObjectLog.getExecuteObjectId() != null) { + result.setExecuteObjectId(taskExecuteObjectLog.getExecuteObjectId()); + } else { + result.setHostId(taskExecuteObjectLog.getHostId()); + result.setCloudIp(taskExecuteObjectLog.getIp()); + result.setCloudIpv6(taskExecuteObjectLog.getIpv6()); + } + if (StringUtils.isNotEmpty(taskExecuteObjectLog.getScriptContent())) { + if (taskExecuteObjectLog.getExecuteObjectId() != null) { + result.setScriptLog(new ServiceExecuteObjectScriptLogDTO( + taskExecuteObjectLog.getExecuteObjectId(), + taskExecuteObjectLog.getScriptContent(), + 0, + 0) + ); + } else { + result.setScriptLog(new ServiceExecuteObjectScriptLogDTO( + taskExecuteObjectLog.getHostId(), + taskExecuteObjectLog.getIp(), + taskExecuteObjectLog.getIpv6(), + taskExecuteObjectLog.getScriptContent(), + 0, + 0) + ); + } } - if (CollectionUtils.isNotEmpty(taskHostLog.getFileTaskLogs())) { - result.setFileTaskLogs(taskHostLog.getFileTaskLogs().stream() + if (CollectionUtils.isNotEmpty(taskExecuteObjectLog.getFileTaskLogs())) { + result.setFileTaskLogs(taskExecuteObjectLog.getFileTaskLogs().stream() .map(FileTaskLogDoc::toServiceFileTaskLogDTO) .collect(Collectors.toList())); } @@ -154,93 +171,80 @@ private ServiceHostLogDTO toServiceLogDTO(TaskHostLog taskHostLog) { } @Override - public InternalResponse getScriptHostLogByHostId(String jobCreateDate, - Long stepInstanceId, - Integer executeCount, - Long hostId, - Integer batch) { + public InternalResponse getScriptHostLogByHostId(String jobCreateDate, + Long stepInstanceId, + Integer executeCount, + Long hostId, + Integer batch) { return InternalResponse.buildSuccessResp( - getScriptHostLog(jobCreateDate, stepInstanceId, executeCount, hostId, batch, null)); + getScriptExecuteObjectLog(jobCreateDate, stepInstanceId, executeCount, batch, null, hostId)); } - private ServiceHostLogDTO getScriptHostLog(String jobCreateDate, - Long stepInstanceId, - Integer executeCount, - Long hostId, - Integer batch, - String ip) { + @Override + public InternalResponse getScriptLogByExecuteObjectId(String jobCreateDate, + Long stepInstanceId, + Integer executeCount, + String executeObjectId, + Integer batch) { + return InternalResponse.buildSuccessResp( + getScriptExecuteObjectLog(jobCreateDate, stepInstanceId, executeCount, batch, executeObjectId, null)); + } + + private ServiceExecuteObjectLogDTO getScriptExecuteObjectLog(String jobCreateDate, + Long stepInstanceId, + Integer executeCount, + Integer batch, + String executeObjectId, + Long hostId) { ScriptLogQuery query = new ScriptLogQuery(jobCreateDate, stepInstanceId, executeCount, batch, - hostId == null ? null : Collections.singletonList(hostId), - StringUtils.isEmpty(ip) ? null : Collections.singletonList(ip)); - List taskHostLogs = logService.listScriptLogs(query); - if (CollectionUtils.isEmpty(taskHostLogs)) { + executeObjectId == null ? null : Collections.singletonList(executeObjectId), + hostId == null ? null : Collections.singletonList(hostId)); + List taskExecuteObjectLogs = logService.listScriptLogs(query); + if (CollectionUtils.isEmpty(taskExecuteObjectLogs)) { return null; } - return toServiceLogDTO(taskHostLogs.get(0)); + return toServiceLogDTO(taskExecuteObjectLogs.get(0)); } @Override - public InternalResponse> listScriptLogs(String jobCreateDate, - Long stepInstanceId, - Integer executeCount, - ServiceScriptLogQueryRequest query) { + public InternalResponse> listScriptExecuteObjectLogs( + String jobCreateDate, + Long stepInstanceId, + Integer executeCount, + ServiceScriptLogQueryRequest query) { + ScriptLogQuery scriptLogQuery = new ScriptLogQuery(jobCreateDate, stepInstanceId, executeCount, - query.getBatch(), query.getHostIds(), query.getIps()); - List taskHostLogs = logService.listScriptLogs(scriptLogQuery); - List scriptLogs = - taskHostLogs.stream().map(this::toServiceLogDTO).collect(Collectors.toList()); + query.getBatch(), query.getExecuteObjectIds(), query.getHostIds()); + List taskExecuteObjectLogs = logService.listScriptLogs(scriptLogQuery); + List scriptLogs = + taskExecuteObjectLogs.stream().map(this::toServiceLogDTO).collect(Collectors.toList()); return InternalResponse.buildSuccessResp(scriptLogs); } - @Override - public InternalResponse getFileHostLogByIp(String jobCreateDate, - Long stepInstanceId, - Integer executeCount, - String ip, - Integer mode, - Integer batch) { - FileLogQuery query = FileLogQuery.builder() - .jobCreateDate(jobCreateDate) - .stepInstanceId(stepInstanceId) - .executeCount(executeCount) - .mode(mode) - .batch(batch) - .ips(Collections.singletonList(ip)) - .build(); - List fileTaskLogs = logService.listFileLogs(query); - ServiceHostLogDTO result = new ServiceHostLogDTO(); - result.setStepInstanceId(stepInstanceId); - result.setExecuteCount(executeCount); - result.setBatch(batch); - result.setIp(ip); - if (CollectionUtils.isNotEmpty(fileTaskLogs)) { - result.setFileTaskLogs(fileTaskLogs.stream() - .map(FileTaskLogDoc::toServiceFileTaskLogDTO) - .collect(Collectors.toList())); - } - return InternalResponse.buildSuccessResp(result); - } @Override - public InternalResponse getFileHostLogByHostId(String jobCreateDate, Long stepInstanceId, - Integer executeCount, Long hostId, Integer mode, - Integer batch) { + public InternalResponse getFileLogByExecuteObjectId(String jobCreateDate, + Long stepInstanceId, + Integer executeCount, + String executeObjectId, + Integer mode, + Integer batch) { FileLogQuery query = FileLogQuery.builder() .jobCreateDate(jobCreateDate) .stepInstanceId(stepInstanceId) .executeCount(executeCount) .mode(mode) .batch(batch) - .hostIds(Collections.singletonList(hostId)) + .executeObjectIds(Collections.singletonList(executeObjectId)) .build(); List fileTaskLogs = logService.listFileLogs(query); - ServiceHostLogDTO result = new ServiceHostLogDTO(); + ServiceExecuteObjectLogDTO result = new ServiceExecuteObjectLogDTO(); result.setStepInstanceId(stepInstanceId); result.setExecuteCount(executeCount); result.setBatch(batch); - result.setHostId(hostId); + result.setExecuteObjectId(executeObjectId); if (CollectionUtils.isNotEmpty(fileTaskLogs)) { result.setFileTaskLogs(fileTaskLogs.stream() .map(FileTaskLogDoc::toServiceFileTaskLogDTO) @@ -262,7 +266,6 @@ public InternalResponse> listFileHostLogs(String job .executeCount(executeCount) .batch(batch) .mode(mode) - .ips(StringUtils.isEmpty(ip) ? null : Collections.singletonList(ip)) .hostIds(hostId == null ? null : Collections.singletonList(hostId)) .build(); @@ -276,12 +279,12 @@ public InternalResponse> listFileHostLogs(String job } @Override - public InternalResponse listFileHostLogsByTaskIds(String jobCreateDate, - Long stepInstanceId, - Integer executeCount, - Integer batch, - List taskIds) { - ServiceHostLogDTO result = new ServiceHostLogDTO(); + public InternalResponse listFileLogsByTaskIds(String jobCreateDate, + Long stepInstanceId, + Integer executeCount, + Integer batch, + List taskIds) { + ServiceExecuteObjectLogDTO result = new ServiceExecuteObjectLogDTO(); result.setStepInstanceId(stepInstanceId); result.setExecuteCount(executeCount); if (CollectionUtils.isEmpty(taskIds)) { @@ -297,24 +300,43 @@ public InternalResponse listFileHostLogsByTaskIds(String jobC } @Override - public InternalResponse listFileHostLogs(ServiceFileLogQueryRequest request) { + public InternalResponse> listTaskFileLogsByTaskIds(String jobCreateDate, + Long stepInstanceId, + Integer executeCount, + Integer batch, + List taskIds) { + List fileTaskLogs = new ArrayList<>(); + + if (CollectionUtils.isEmpty(taskIds)) { + return InternalResponse.buildSuccessResp(fileTaskLogs); + } + List fileTaskLogDocs = logService.getFileLogsByTaskIds(jobCreateDate, stepInstanceId, + executeCount, batch, taskIds); + if (CollectionUtils.isEmpty(fileTaskLogDocs)) { + return InternalResponse.buildSuccessResp(fileTaskLogs); + } + return InternalResponse.buildSuccessResp( + fileTaskLogDocs.stream().map(FileTaskLogDoc::toServiceFileTaskLogDTO).collect(Collectors.toList())); + } + + @Override + public InternalResponse listFileHostLogs(ServiceFileLogQueryRequest request) { FileLogQuery query = FileLogQuery.builder() .stepInstanceId(request.getStepInstanceId()) .executeCount(request.getExecuteCount()) .jobCreateDate(request.getJobCreateDate()) .batch(request.getBatch()) .mode(request.getMode()) - .ips(request.getIps()) .hostIds(request.getHostIds()) .build(); - ServiceHostLogsDTO ipLogsResult = new ServiceHostLogsDTO(); - ipLogsResult.setStepInstanceId(request.getStepInstanceId()); - ipLogsResult.setExecuteCount(request.getExecuteCount()); + ServiceExecuteObjectLogsDTO executeObjectLogs = new ServiceExecuteObjectLogsDTO(); + executeObjectLogs.setStepInstanceId(request.getStepInstanceId()); + executeObjectLogs.setExecuteCount(request.getExecuteCount()); List fileTaskLogs = logService.listFileLogs(query); if (CollectionUtils.isEmpty(fileTaskLogs)) { - return InternalResponse.buildSuccessResp(ipLogsResult); + return InternalResponse.buildSuccessResp(executeObjectLogs); } List fileLogs = fileTaskLogs.stream().map(FileTaskLogDoc::toServiceFileTaskLogDTO) @@ -333,26 +355,136 @@ public InternalResponse listFileHostLogs(ServiceFileLogQuery }); }); - List ipLogs = new ArrayList<>(); - ipLogsResult.setIpLogs(ipLogs); + List ipLogs = new ArrayList<>(); + executeObjectLogs.setIpLogs(ipLogs); cloudIpAndLogs.forEach((cloudIp, logs) -> { - ServiceHostLogDTO ipLog = new ServiceHostLogDTO(); - ipLog.setIp(cloudIp); + ServiceExecuteObjectLogDTO ipLog = new ServiceExecuteObjectLogDTO(); + ipLog.setCloudIp(cloudIp); ipLog.setFileTaskLogs(logs); ipLogs.add(ipLog); }); - return InternalResponse.buildSuccessResp(ipLogsResult); + return InternalResponse.buildSuccessResp(executeObjectLogs); } @Override - public InternalResponse> getIpsByKeyword(String jobCreateDate, - Long stepInstanceId, - Integer executeCount, - Integer batch, - String keyword) { - List ips = logService.getIpsByKeyword(jobCreateDate, stepInstanceId, executeCount, batch, keyword); - return InternalResponse.buildSuccessResp(ips); + public InternalResponse> listFileExecuteObjectLogs( + String jobCreateDate, + Long stepInstanceId, + Integer executeCount, + ServiceFileLogQueryRequest request) { + + List executeObjectLogs = new ArrayList<>(); + + FileLogQuery query = FileLogQuery.builder() + .jobCreateDate(jobCreateDate) + .stepInstanceId(stepInstanceId) + .executeCount(executeCount) + .batch(request.getBatch()) + .mode(request.getMode()) + .hostIds(request.getHostIds()) + .executeObjectIds(request.getExecuteObjectIds()) + .build(); + + List fileTaskLogs = logService.listFileLogs(query); + if (CollectionUtils.isEmpty(fileTaskLogs)) { + return InternalResponse.buildSuccessResp(executeObjectLogs); + } + + boolean isExistExecuteObjectId = StringUtils.isNotEmpty(fileTaskLogs.get(0).getExecuteObjectId()); + + if (isExistExecuteObjectId) { + Map> logGroups = new HashMap<>(); + fileTaskLogs.forEach(fileTaskLogDoc -> { + List logGroup = logGroups.computeIfAbsent(fileTaskLogDoc.getExecuteObjectId(), + k -> new ArrayList<>()); + logGroup.add(fileTaskLogDoc); + }); + logGroups.forEach( + (executeObjectId, logGroup) -> { + ServiceExecuteObjectLogDTO executeObjectLog = new ServiceExecuteObjectLogDTO(); + executeObjectLog.setStepInstanceId(stepInstanceId); + executeObjectLog.setExecuteCount(executeCount); + executeObjectLog.setBatch(request.getBatch()); + executeObjectLog.setExecuteObjectId(executeObjectId); + executeObjectLog.setFileTaskLogs(logGroup.stream().map(FileTaskLogDoc::toServiceFileTaskLogDTO) + .collect(Collectors.toList())); + executeObjectLogs.add(executeObjectLog); + } + ); + } else { + Map> logGroups = new HashMap<>(); + fileTaskLogs.forEach(fileTaskLogDoc -> { + List logGroup = logGroups.computeIfAbsent(fileTaskLogDoc.getHostId(), + k -> new ArrayList<>()); + logGroup.add(fileTaskLogDoc); + }); + logGroups.forEach( + (hostId, logGroup) -> { + ServiceExecuteObjectLogDTO executeObjectLog = new ServiceExecuteObjectLogDTO(); + executeObjectLog.setStepInstanceId(stepInstanceId); + executeObjectLog.setExecuteCount(executeCount); + executeObjectLog.setBatch(request.getBatch()); + executeObjectLog.setHostId(hostId); + executeObjectLog.setFileTaskLogs(logGroup.stream().map(FileTaskLogDoc::toServiceFileTaskLogDTO) + .collect(Collectors.toList())); + executeObjectLogs.add(executeObjectLog); + } + ); + } + + return InternalResponse.buildSuccessResp(executeObjectLogs); + } + + @Override + public InternalResponse> queryHostsByLogKeyword(String jobCreateDate, + Long stepInstanceId, + Integer executeCount, + Integer batch, + String keyword) { + List hosts = logService.getHostsByKeyword(jobCreateDate, stepInstanceId, executeCount, batch, keyword); + return InternalResponse.buildSuccessResp(hosts); + } + + + @Override + public InternalResponse getFileHostLogByHostId(String jobCreateDate, + Long stepInstanceId, + Integer executeCount, + Long hostId, + Integer mode, + Integer batch) { + FileLogQuery query = FileLogQuery.builder() + .jobCreateDate(jobCreateDate) + .stepInstanceId(stepInstanceId) + .executeCount(executeCount) + .mode(mode) + .batch(batch) + .hostIds(Collections.singletonList(hostId)) + .build(); + List fileTaskLogs = logService.listFileLogs(query); + ServiceExecuteObjectLogDTO result = new ServiceExecuteObjectLogDTO(); + result.setStepInstanceId(stepInstanceId); + result.setExecuteCount(executeCount); + result.setBatch(batch); + result.setHostId(hostId); + if (CollectionUtils.isNotEmpty(fileTaskLogs)) { + result.setFileTaskLogs(fileTaskLogs.stream() + .map(FileTaskLogDoc::toServiceFileTaskLogDTO) + .collect(Collectors.toList())); + } + return InternalResponse.buildSuccessResp(result); } + + @Override + public InternalResponse> queryExecuteObjectsByLogKeyword(String jobCreateDate, + Long stepInstanceId, + Integer executeCount, + Integer batch, + String keyword) { + List executeObjectIds = logService.getExecuteObjectIdsByKeyword(jobCreateDate, stepInstanceId, + executeCount, batch, keyword); + return InternalResponse.buildSuccessResp(executeObjectIds); + } } diff --git a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/FileLogQuery.java b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/FileLogQuery.java index 323f9781c6..61fb491b90 100644 --- a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/FileLogQuery.java +++ b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/FileLogQuery.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,16 +25,18 @@ package com.tencent.bk.job.logsvr.model; import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.CompatibleType; import lombok.Builder; import lombok.Getter; import lombok.Setter; +import lombok.ToString; import java.util.List; -import java.util.StringJoiner; @Getter @Setter @Builder +@ToString public class FileLogQuery { /** * 作业实例创建时间,格式yyyy_MM_dd @@ -44,15 +46,17 @@ public class FileLogQuery { * 作业步骤实例ID */ private Long stepInstanceId; - /** - * 执行任务的主机ip列表 - */ - @CompatibleImplementation(name = "rolling_execute", explain = "兼容字段，后续用hostIds替换", version = "3.7.x") - private List ips; + /** * 执行任务的主机ID列表 */ + @Deprecated + @CompatibleImplementation(name = "execute_object", type = CompatibleType.HISTORY_DATA, deprecatedVersion = "3.9.x") private List hostIds; + /** + * 执行对象实例 ID + */ + private List executeObjectIds; /** * 执行次数 */ @@ -67,15 +71,4 @@ public class FileLogQuery { * @see com.tencent.bk.job.logsvr.consts.FileTaskModeEnum */ private Integer mode; - - @Override - public String toString() { - return new StringJoiner(", ", FileLogQuery.class.getSimpleName() + "[", "]") - .add("jobCreateDate='" + jobCreateDate + "'") - .add("stepInstanceId=" + stepInstanceId) - .add("ips='" + ips + "'") - .add("executeCount=" + executeCount) - .add("mode=" + mode) - .toString(); - } } diff --git a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/FileTaskLogDoc.java b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/FileTaskLogDoc.java index 1f952727d2..f65883a285 100644 --- a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/FileTaskLogDoc.java +++ b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/FileTaskLogDoc.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,9 +24,8 @@ package com.tencent.bk.job.logsvr.model; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.CompatibleType; import com.tencent.bk.job.logsvr.consts.FileTaskModeEnum; import com.tencent.bk.job.logsvr.model.service.ServiceFileTaskLogDTO; import lombok.Data; @@ -41,125 +40,178 @@ * 文件任务执行日志 - MongoDB Doc */ @Data -@JsonInclude(JsonInclude.Include.NON_EMPTY) @Document public class FileTaskLogDoc { + /** + * 步骤实例ID + */ + @Field(FileTaskLogDocField.STEP_ID) + private Long stepId; + /** + * 执行次数 + */ + @Field(FileTaskLogDocField.EXECUTE_COUNT) + private Integer executeCount; + /** + * 滚动执行批次 + */ + @Field(FileTaskLogDocField.BATCH) + private Integer batch; /** * 任务ID */ - @JsonProperty("taskId") - @Field("taskId") + @Field(FileTaskLogDocField.TASK_ID) private String taskId; /** * 文件任务模式，mode: 0-upload;1-download */ - @JsonProperty("mode") - @Field("mode") + @Field(FileTaskLogDocField.MODE) private Integer mode; /** - * ip - 真实IP。当mode=0时,ip=上传源IP;mode=1时,ip=下载目标IP + * 执行对象 ID。当mode=0时,executeObjectId=上传源的执行对象ID;mode=1时,executeObjectId=下载目标的执行对象ID */ - @CompatibleImplementation(name = "rolling_execute", explain = "兼容字段，后续用hostId替换", version = "3.7.x") - @JsonProperty("ip") - @Field("ip") - private String ip; + @Field(FileTaskLogDocField.EXECUTE_OBJECT_ID) + private String executeObjectId; /** * hostId。当mode=0时,hostId=上传源hostId;mode=1时,hostId=下载目标hostId */ - @JsonProperty("hostId") - @Field("hostId") + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA, + explain = "兼容历史数据使用, 新版本将不再使用该字段") + @Deprecated + @Field(FileTaskLogDocField.HOST_ID) private Long hostId; /** - * 文件源IP + * 文件源-执行对象 ID + */ + @Field(FileTaskLogDocField.SRC_EXECUTE_OBJECT_ID) + private String srcExecuteObjectId; + + /** + * 文件源主机(云区域:IPv4) */ - @CompatibleImplementation(name = "rolling_execute", explain = "兼容字段，后续用srcHostId替换", version = "3.7.x") - @JsonProperty("srcIp") - @Field("srcIp") + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA, + explain = "兼容历史数据使用, 新版本将不再使用该字段") + @Deprecated + @Field(FileTaskLogDocField.SRC_IP) private String srcIp; + /** + * 文件源主机(云区域:IPv6) + */ + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA, + explain = "兼容历史数据使用, 新版本将不再使用该字段") + @Deprecated + @Field(FileTaskLogDocField.SRC_IPV6) + private String srcIpv6; + /** * 文件源主机ID */ - @JsonProperty("srcHostId") - @Field("srcHostId") + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA, + explain = "兼容历史数据使用, 新版本将不再使用该字段") + @Deprecated + @Field(FileTaskLogDocField.SRC_HOST_ID) private Long srcHostId; /** - * 文件源IP - 显示 + * 目标-执行对象 ID + */ + @Field(FileTaskLogDocField.DEST_EXECUTE_OBJECT_ID) + private String destExecuteObjectId; + + /** + * 目标主机(云区域:IPv4) + */ + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA, + explain = "兼容历史数据使用, 新版本将不再使用该字段") + @Deprecated + @Field(FileTaskLogDocField.DEST_IP) + private String destIp; + + /** + * 目标主机(云区域:IPv6) + */ + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA, + explain = "兼容历史数据使用, 新版本将不再使用该字段") + @Deprecated + @Field(FileTaskLogDocField.DEST_IPV6) + private String destIpv6; + + /** + * 目标主机主机ID */ - @JsonProperty("displaySrcIp") - @Field("displaySrcIp") - private String displaySrcIp; + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA, + explain = "兼容历史数据使用, 新版本将不再使用该字段") + @Deprecated + @Field(FileTaskLogDocField.DEST_HOST_ID) + private Long destHostId; /** * 目标文件路径 */ - @JsonProperty("destFile") - @Field("destFile") + @Field(FileTaskLogDocField.DEST_FILE) private String destFile; /** * 源文件路径 - 用于显示 */ - @JsonProperty("displaySrcFile") - @Field("displaySrcFile") + @Field(FileTaskLogDocField.DISPLAY_SRC_FILE) private String displaySrcFile; /** * 源文件路径 - 真实路径 */ - @JsonProperty("srcFile") - @Field("srcFile") + @Field(FileTaskLogDocField.SRC_FILE) private String srcFile; + /** + * 源文件类型 + */ + @Field(FileTaskLogDocField.SRC_FILE_TYPE) + private Integer srcFileType; + /** * 文件大小 */ - @JsonProperty("size") - @Field("size") + @Field(FileTaskLogDocField.SIZE) private String size; /** * 文件任务状态 */ - @JsonProperty("status") - @Field("status") + @Field(FileTaskLogDocField.STATUS) private Integer status; /** * 文件任务状态描述 */ - @JsonProperty("statusDesc") - @Field("statusDesc") + @Field(FileTaskLogDocField.STATUS_DESC) private String statusDesc; /** * 速度 */ - @JsonProperty("speed") - @Field("speed") + @Field(FileTaskLogDocField.SPEED) private String speed; /** * 进度 */ - @JsonProperty("process") - @Field("process") + @Field(FileTaskLogDocField.PROCESS) private String process; /** - * 日志内容在mongodb中按照list的方式存储，与mongodb中的字段contentList对应 + * 日志内容在mongodb中按照list的方式存储 */ - @JsonProperty("contentList") - @Field("contentList") + @Field(FileTaskLogDocField.CONTENT_LIST) private List contentList; /** - * 日志内容 + * 拼接处理后的日志的内容。该字段不会写入到db */ private String content; @@ -168,22 +220,27 @@ public static FileTaskLogDoc convert(ServiceFileTaskLogDTO serviceFileLog) { fileLog.setContent(serviceFileLog.getContent()); fileLog.setMode(serviceFileLog.getMode()); if (FileTaskModeEnum.UPLOAD.getValue().equals(serviceFileLog.getMode())) { - fileLog.setSrcIp(serviceFileLog.getSrcIp()); - fileLog.setIp(serviceFileLog.getSrcIp()); fileLog.setHostId(serviceFileLog.getSrcHostId()); - fileLog.setSrcFile(serviceFileLog.getSrcFile()); - fileLog.setDisplaySrcFile(serviceFileLog.getDisplaySrcFile()); - fileLog.setDisplaySrcIp(serviceFileLog.getDisplaySrcIp()); + fileLog.setExecuteObjectId(serviceFileLog.getSrcExecuteObjectId()); } else if (FileTaskModeEnum.DOWNLOAD.getValue().equals(serviceFileLog.getMode())) { fileLog.setHostId(serviceFileLog.getDestHostId()); - fileLog.setIp(serviceFileLog.getDestIp()); - fileLog.setSrcIp(serviceFileLog.getSrcIp()); - fileLog.setSrcHostId(serviceFileLog.getSrcHostId()); - fileLog.setDisplaySrcIp(serviceFileLog.getDisplaySrcIp()); - fileLog.setSrcFile(serviceFileLog.getSrcFile()); - fileLog.setDisplaySrcFile(serviceFileLog.getDisplaySrcFile()); + fileLog.setExecuteObjectId(serviceFileLog.getDestExecuteObjectId()); + // dest + fileLog.setDestExecuteObjectId(serviceFileLog.getDestExecuteObjectId()); + fileLog.setDestHostId(serviceFileLog.getDestHostId()); + fileLog.setDestIp(serviceFileLog.getDestIp()); + fileLog.setDestIpv6(serviceFileLog.getDestIpv6()); fileLog.setDestFile(serviceFileLog.getDestFile()); } + // source + fileLog.setSrcExecuteObjectId(serviceFileLog.getSrcExecuteObjectId()); + fileLog.setSrcHostId(serviceFileLog.getSrcHostId()); + fileLog.setSrcIp(serviceFileLog.getSrcIp()); + fileLog.setSrcIpv6(serviceFileLog.getSrcIpv6()); + fileLog.setSrcFile(serviceFileLog.getSrcFile()); + fileLog.setDisplaySrcFile(serviceFileLog.getDisplaySrcFile()); + fileLog.setSrcFileType(serviceFileLog.getSrcFileType()); + fileLog.setSize(serviceFileLog.getSize()); fileLog.setProcess(serviceFileLog.getProcess()); fileLog.setSpeed(serviceFileLog.getSpeed()); @@ -203,14 +260,20 @@ public ServiceFileTaskLogDTO toServiceFileTaskLogDTO() { fileLog.setContent(content); } fileLog.setMode(mode); + fileLog.setSrcExecuteObjectId(srcExecuteObjectId); fileLog.setSrcHostId(srcHostId); fileLog.setSrcIp(srcIp); - fileLog.setDisplaySrcIp(displaySrcIp); + fileLog.setSrcIpv6(srcIpv6); + fileLog.setSrcFileType(srcFileType); fileLog.setDisplaySrcFile(displaySrcFile); fileLog.setSrcFile(srcFile); - fileLog.setDestHostId(hostId); - fileLog.setDestIp(ip); - fileLog.setDestFile(destFile); + if (FileTaskModeEnum.DOWNLOAD.getValue().equals(mode)) { + fileLog.setDestExecuteObjectId(destExecuteObjectId); + fileLog.setDestHostId(destHostId != null ? destHostId : hostId); + fileLog.setDestIp(destIp); + fileLog.setDestIpv6(destIpv6); + fileLog.setDestFile(destFile); + } fileLog.setProcess(process); fileLog.setSize(size); fileLog.setSpeed(speed); @@ -219,36 +282,29 @@ public ServiceFileTaskLogDTO toServiceFileTaskLogDTO() { return fileLog; } - @CompatibleImplementation(name = "rolling_execute", explain = "兼容方法，后续不再使用ip相关参数，发布完成后删除", - version = "3.6.x") public String buildTaskId() { StringBuilder sb = new StringBuilder(); sb.append(mode).append("_"); if (FileTaskModeEnum.UPLOAD.getValue().equals(mode)) { - if (hostId != null) { - sb.append(hostId).append("_"); - } else { - sb.append(displaySrcIp).append("_"); - } - sb.append(displaySrcFile); + sb.append(buildCompatibleExecuteObjectId(executeObjectId, hostId)) + .append("_") + .append(displaySrcFile); } else { - if (srcHostId != null) { - sb.append(srcHostId).append("_"); - } else { - sb.append(displaySrcIp).append("_"); - } - // 暂时不加入源文件，GSE暂不支持 -// sb.append(displaySrcFile).append("_"); - if (hostId != null) { - sb.append(hostId).append("_"); - } else { - sb.append(ip).append("_"); - } - sb.append(destFile); + sb.append(buildCompatibleExecuteObjectId(srcExecuteObjectId, srcHostId)) + .append("_") + .append(displaySrcFile) + .append("_") + .append(buildCompatibleExecuteObjectId(executeObjectId, hostId)) + .append("_") + .append(destFile); } return sb.toString(); } + private String buildCompatibleExecuteObjectId(String executeObjectId, Long hostId) { + return StringUtils.isNotEmpty(executeObjectId) ? executeObjectId : String.valueOf(hostId); + } + public String getTaskId() { return this.taskId == null ? buildTaskId() : this.taskId; } diff --git a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/FileTaskLogDocField.java b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/FileTaskLogDocField.java new file mode 100644 index 0000000000..d9034892f3 --- /dev/null +++ b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/FileTaskLogDocField.java @@ -0,0 +1,33 @@ +package com.tencent.bk.job.logsvr.model; + +/** + * 存储文件任务执行日志的 MongoDB Doc 字段名 + */ +public interface FileTaskLogDocField { + String STEP_ID = "stepId"; + String EXECUTE_COUNT = "executeCount"; + String BATCH = "batch"; + String TASK_ID = "taskId"; + String MODE = "mode"; + String IP = "ip"; + String HOST_ID = "hostId"; + String SRC_IP = "srcIp"; + String SRC_IPV6 = "srcIpv6"; + String SRC_HOST_ID = "srcHostId"; + String SRC_FILE_TYPE = "srcFileType"; + String SRC_FILE = "srcFile"; + String DISPLAY_SRC_FILE = "displaySrcFile"; + String DEST_IP = "destIp"; + String DEST_IPV6 = "destIpv6"; + String DEST_HOST_ID = "destHostId"; + String DEST_FILE = "destFile"; + String SIZE = "size"; + String STATUS = "status"; + String STATUS_DESC = "statusDesc"; + String SPEED = "speed"; + String PROCESS = "process"; + String CONTENT_LIST = "contentList"; + String EXECUTE_OBJECT_ID = "executeObjectId"; + String SRC_EXECUTE_OBJECT_ID = "srcExecuteObjectId"; + String DEST_EXECUTE_OBJECT_ID = "destExecuteObjectId"; +} diff --git a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/ScriptLogQuery.java b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/ScriptLogQuery.java index 9ba53e9614..2ed68d5cd4 100644 --- a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/ScriptLogQuery.java +++ b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/ScriptLogQuery.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,6 +25,7 @@ package com.tencent.bk.job.logsvr.model; import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.CompatibleType; import lombok.Data; import lombok.NoArgsConstructor; @@ -41,15 +42,16 @@ public class ScriptLogQuery { * 作业步骤实例ID */ private Long stepInstanceId; - /** - * 执行任务的主机ip列表 - */ - @CompatibleImplementation(name = "rolling_execute", explain = "兼容字段，后续用hostIds替换", version = "3.7.x") - private List ips; /** * 执行任务的主机ID列表 */ + @Deprecated + @CompatibleImplementation(name = "execute_object", type = CompatibleType.HISTORY_DATA, deprecatedVersion = "3.9.x") private List hostIds; + /** + * 执行对象实例 ID + */ + private List executeObjectIds; /** * 执行次数 */ @@ -63,13 +65,13 @@ public ScriptLogQuery(String jobCreateDate, Long stepInstanceId, Integer executeCount, Integer batch, - List hostIds, - List ips) { + List executeObjectIds, + List hostIds) { this.jobCreateDate = jobCreateDate; this.stepInstanceId = stepInstanceId; this.executeCount = executeCount; this.batch = batch; + this.executeObjectIds = executeObjectIds; this.hostIds = hostIds; - this.ips = ips; } } diff --git a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/ScriptTaskLogDoc.java b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/ScriptTaskLogDoc.java index 99d6bfd926..4817c4ee44 100644 --- a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/ScriptTaskLogDoc.java +++ b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/ScriptTaskLogDoc.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,8 +25,8 @@ package com.tencent.bk.job.logsvr.model; import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.annotation.JsonProperty; import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.CompatibleType; import lombok.Data; import lombok.NoArgsConstructor; import org.springframework.data.mongodb.core.mapping.Document; @@ -46,52 +46,81 @@ public class ScriptTaskLogDoc { /** * 作业步骤实例ID */ - @JsonProperty("stepId") - @Field("stepId") + @Field(ScriptTaskLogDocField.STEP_ID) private Long stepInstanceId; /** * 执行次数 */ - @JsonProperty("executeCount") - @Field("executeCount") + @Field(ScriptTaskLogDocField.EXECUTE_COUNT) private Integer executeCount; /** * 滚动执行批次 */ - @JsonProperty("batch") - @Field("batch") + @Field(ScriptTaskLogDocField.BATCH) private Integer batch; /** - * 执行任务的主机ip + * 执行对象 ID */ - @CompatibleImplementation(name = "rolling_execute", explain = "兼容字段，后续用hostId替换", version = "3.7.x") - @JsonProperty("ip") - @Field("ip") + @Field(ScriptTaskLogDocField.EXECUTE_OBJECT_ID) + private String executeObjectId; + /** + * 执行任务的主机ipv4 + */ + @Deprecated + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA, + explain = "兼容历史数据使用, 新版本将不再使用该字段") + @Field(ScriptTaskLogDocField.IP) private String ip; + /** + * 执行任务的主机ipv6 + */ + @Deprecated + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA, + explain = "兼容历史数据使用, 新版本将不再使用该字段") + @Field(ScriptTaskLogDocField.IPV6) + private String ipv6; /** * 执行任务的主机hostId */ - @JsonProperty("hostId") - @Field("hostId") + @Deprecated + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA, + explain = "兼容历史数据使用, 新版本将不再使用该字段") + @Field(ScriptTaskLogDocField.HOST_ID) private Long hostId; /** * 日志内容 */ - @JsonProperty("content") - @Field("content") + @Field(ScriptTaskLogDocField.CONTENT) private String content; /** - * 日志偏移 - 字节 + * 日志偏移 - 单位(byte) */ - @JsonProperty("offset") - @Field("offset") + @Field(ScriptTaskLogDocField.OFFSET) private Integer offset; + public ScriptTaskLogDoc(Long stepInstanceId, + Integer executeCount, + Integer batch, + String executeObjectId, + String content, + Integer offset) { + this.stepInstanceId = stepInstanceId; + this.executeCount = executeCount; + this.batch = batch; + this.executeObjectId = executeObjectId; + this.content = content; + this.offset = offset; + } + + @Deprecated + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA, + explain = "兼容历史数据使用") public ScriptTaskLogDoc(Long stepInstanceId, Integer executeCount, Integer batch, Long hostId, String ip, + String ipv6, String content, Integer offset) { this.stepInstanceId = stepInstanceId; @@ -99,6 +128,7 @@ public ScriptTaskLogDoc(Long stepInstanceId, this.batch = batch; this.hostId = hostId; this.ip = ip; + this.ipv6 = ipv6; this.content = content; this.offset = offset; } diff --git a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/ScriptTaskLogDocField.java b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/ScriptTaskLogDocField.java new file mode 100644 index 0000000000..15b059434b --- /dev/null +++ b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/ScriptTaskLogDocField.java @@ -0,0 +1,16 @@ +package com.tencent.bk.job.logsvr.model; + +/** + * 存储脚本任务执行日志的 MongoDB Doc 字段名 + */ +public interface ScriptTaskLogDocField { + String STEP_ID = "stepId"; + String EXECUTE_COUNT = "executeCount"; + String BATCH = "batch"; + String EXECUTE_OBJECT_ID = "executeObjectId"; + String HOST_ID = "hostId"; + String IP = "ip"; + String IPV6 = "ipv6"; + String CONTENT = "content"; + String OFFSET = "offset"; +} diff --git a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/TaskExecuteObjectLog.java b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/TaskExecuteObjectLog.java new file mode 100644 index 0000000000..e78b06d1c0 --- /dev/null +++ b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/TaskExecuteObjectLog.java @@ -0,0 +1,99 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.logsvr.model; + +import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.CompatibleType; +import lombok.Getter; +import lombok.Setter; + +import java.util.List; + +/** + * 执行对象任务执行日志 + */ +@Getter +@Setter +public class TaskExecuteObjectLog { + /** + * 作业实例创建时间,格式yyyy_MM_dd + */ + private String jobCreateDate; + /** + * 作业步骤实例ID + */ + private Long stepInstanceId; + /** + * 执行对象 ID + */ + private String executeObjectId; + /** + * 主机ipv4,格式: 云区域ID:IPv4 + */ + @Deprecated + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA, + explain = "兼容历史数据使用, 新版本将不再使用该字段") + private String ip; + /** + * 主机ipv6,格式: 云区域ID:IPv6 + */ + @Deprecated + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA, + explain = "兼容历史数据使用, 新版本将不再使用该字段") + private String ipv6; + /** + * 主机ID + */ + @Deprecated + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.HISTORY_DATA, + explain = "兼容历史数据使用, 新版本将不再使用该字段") + private Long hostId; + /** + * 执行次数 + */ + private Integer executeCount; + /** + * 滚动批次 + */ + private Integer batch; + /** + * 脚本任务执行日志 + */ + private ScriptTaskLogDoc scriptTaskLog; + /** + * 脚本任务执行日志内容 + */ + private String scriptContent; + /** + * 文件任务执行日志 + */ + List fileTaskLogs; + /** + * 日志类型 + * + * @see com.tencent.bk.job.logsvr.consts.LogTypeEnum + */ + private Integer logType; +} diff --git a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/TaskHostLog.java b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/TaskHostLog.java deleted file mode 100644 index f030b4af36..0000000000 --- a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/model/TaskHostLog.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.logsvr.model; - -import com.tencent.bk.job.common.annotation.CompatibleImplementation; -import lombok.Getter; -import lombok.Setter; -import org.apache.commons.lang3.StringUtils; - -import java.util.List; -import java.util.StringJoiner; - -/** - * 任务执行日志 - */ -@Getter -@Setter -public class TaskHostLog { - /** - * 文件任务执行日志 - */ - List fileTaskLogs; - /** - * 作业实例创建时间,格式yyyy_MM_dd - */ - private String jobCreateDate; - /** - * 作业步骤实例ID - */ - private Long stepInstanceId; - /** - * 执行任务的主机ip - */ - @CompatibleImplementation(explain = "兼容参数,由于IP不再唯一，后续使用hostId参数替换", version = "3.7.x") - private String ip; - /** - * 主机ID - */ - private Long hostId; - /** - * 执行次数 - */ - private Integer executeCount; - /** - * 滚动批次 - */ - private Integer batch; - /** - * 脚本任务执行日志 - */ - private ScriptTaskLogDoc scriptTaskLog; - /** - * 脚本任务执行日志内容 - */ - private String scriptContent; - /** - * 日志类型 - * - * @see com.tencent.bk.job.logsvr.consts.LogTypeEnum - */ - private Integer logType; - - @Override - public String toString() { - return new StringJoiner(", ", TaskHostLog.class.getSimpleName() + "[", "]") - .add("jobCreateDate='" + jobCreateDate + "'") - .add("stepInstanceId=" + stepInstanceId) - .add("hostId='" + hostId + "'") - .add("ip='" + ip + "'") - .add("executeCount=" + executeCount) - .add("batch=" + batch) - .add("scriptContentLength=" + (StringUtils.isEmpty(scriptContent) ? 0 : scriptContent.length())) - .add("logType=" + logType) - .toString(); - } -} diff --git a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/mongo/CollectionLoader.java b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/mongo/CollectionLoader.java index 7167d019d9..c18f3416d4 100644 --- a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/mongo/CollectionLoader.java +++ b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/mongo/CollectionLoader.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/mongo/CollectionLoaderBase.java b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/mongo/CollectionLoaderBase.java index 3ce525b0e8..57239c621b 100644 --- a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/mongo/CollectionLoaderBase.java +++ b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/mongo/CollectionLoaderBase.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -47,7 +47,7 @@ public class CollectionLoaderBase implements CollectionLoader { @Value("${job.logsvr.mongodb.shard.enabled:false}") protected boolean enableSharding; - private Map collectionShardStatusMap = new ConcurrentHashMap<>(); + private final Map collectionShardStatusMap = new ConcurrentHashMap<>(); @Override public MongoCollection load(MongoTemplate mongoTemplate, String collectionName) { diff --git a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/mongo/FileLogsCollectionLoader.java b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/mongo/FileLogsCollectionLoader.java index c41ee042b3..ed2b7d2d67 100644 --- a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/mongo/FileLogsCollectionLoader.java +++ b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/mongo/FileLogsCollectionLoader.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,6 +26,8 @@ import com.mongodb.client.MongoCollection; import com.mongodb.client.model.IndexOptions; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.CompatibleType; import lombok.extern.slf4j.Slf4j; import org.bson.Document; import org.springframework.data.mongodb.core.MongoTemplate; @@ -40,7 +42,13 @@ @Slf4j public class FileLogsCollectionLoader extends CollectionLoaderBase { + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.DEPLOY, + explain = "兼容历史数据使用。新版本切换完成之后将使用 IDX_STEP_EXECUTE_COUNT_MODE_EXECUTE_OBJECT_ID 索引") + @Deprecated private static final String IDX_STEP_EXECUTE_COUNT_MODE_HOST_ID = "stepId_executeCount_mode_hostId"; + + private static final String IDX_STEP_EXECUTE_COUNT_MODE_EXECUTE_OBJECT_ID + = "stepId_executeCount_mode_executeObjectId"; private static final String IDX_STEP_EXECUTE_COUNT_TASK_ID = "stepId_executeCount_taskId"; private static final String IDX_STEP_ID_HASHED = "stepId_hashed"; @@ -75,7 +83,18 @@ private void createIndexIfUnavailable(MongoCollection collection, List indexOptions); log.info("Create index {} for collection: {} successfully!", IDX_STEP_EXECUTE_COUNT_MODE_HOST_ID, collectionName); + } + if (!indexes.contains(IDX_STEP_EXECUTE_COUNT_MODE_EXECUTE_OBJECT_ID)) { + log.info("Create index {} for collection: {} start...", IDX_STEP_EXECUTE_COUNT_MODE_EXECUTE_OBJECT_ID, + collectionName); + IndexOptions indexOptions = new IndexOptions(); + indexOptions.background(false); + indexOptions.name(IDX_STEP_EXECUTE_COUNT_MODE_EXECUTE_OBJECT_ID); + collection.createIndex(Document.parse("{\"stepId\":1,\"executeCount\":1,\"mode\":1,\"executeObjectId\":1}"), + indexOptions); + log.info("Create index {} for collection: {} successfully!", IDX_STEP_EXECUTE_COUNT_MODE_EXECUTE_OBJECT_ID, + collectionName); } if (!indexes.contains(IDX_STEP_EXECUTE_COUNT_TASK_ID)) { diff --git a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/mongo/JobLogWriteConcernResolver.java b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/mongo/JobLogWriteConcernResolver.java index 08227e9c0b..1c8820abb6 100644 --- a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/mongo/JobLogWriteConcernResolver.java +++ b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/mongo/JobLogWriteConcernResolver.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/mongo/LogCollectionFactory.java b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/mongo/LogCollectionFactory.java index f48f3b85bd..08d91f52a8 100644 --- a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/mongo/LogCollectionFactory.java +++ b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/mongo/LogCollectionFactory.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -46,8 +46,8 @@ @Component @Slf4j public class LogCollectionFactory { - private MongoTemplate mongoTemplate; - private LogCollectionLoaderFactory loaderFactory; + private final MongoTemplate mongoTemplate; + private final LogCollectionLoaderFactory loaderFactory; private final LoadingCache> collectionCache = diff --git a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/mongo/LogCollectionLoaderFactory.java b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/mongo/LogCollectionLoaderFactory.java index 46fe893a75..c51e478d6e 100644 --- a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/mongo/LogCollectionLoaderFactory.java +++ b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/mongo/LogCollectionLoaderFactory.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -29,8 +29,8 @@ @Component public class LogCollectionLoaderFactory { - private ScriptLogsCollectionLoader scriptLogsCollectionLoader; - private FileLogsCollectionLoader fileLogsCollectionLoader; + private final ScriptLogsCollectionLoader scriptLogsCollectionLoader; + private final FileLogsCollectionLoader fileLogsCollectionLoader; @Autowired public LogCollectionLoaderFactory(ScriptLogsCollectionLoader scriptLogsCollectionLoader, diff --git a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/mongo/ScriptLogsCollectionLoader.java b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/mongo/ScriptLogsCollectionLoader.java index 2f4c7d3347..515217b4fe 100644 --- a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/mongo/ScriptLogsCollectionLoader.java +++ b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/mongo/ScriptLogsCollectionLoader.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,6 +26,8 @@ import com.mongodb.client.MongoCollection; import com.mongodb.client.model.IndexOptions; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; +import com.tencent.bk.job.common.constant.CompatibleType; import lombok.extern.slf4j.Slf4j; import org.bson.Document; import org.springframework.data.mongodb.core.MongoTemplate; @@ -40,7 +42,12 @@ @Slf4j public class ScriptLogsCollectionLoader extends CollectionLoaderBase { + @CompatibleImplementation(name = "execute_object", deprecatedVersion = "3.9.x", type = CompatibleType.DEPLOY, + explain = "兼容历史数据使用。新版本切换完成之后将使用 IDX_STEP_EXECUTE_COUNT_MODE_EXECUTE_OBJECT_ID 索引") + @Deprecated private static final String IDX_STEP_ID_EXECUTE_COUNT_HOST_ID = "stepId_executeCount_hostId"; + + private static final String IDX_STEP_ID_EXECUTE_COUNT_EXECUTE_OBJECT_ID = "stepId_executeCount_executeObjectId"; private static final String IDX_STEP_ID_HASHED = "stepId_hashed"; @Override @@ -74,6 +81,19 @@ private void createIndexIfUnavailable(MongoCollection collection, List log.info("Create index {} for collection: {} successfully!", IDX_STEP_ID_EXECUTE_COUNT_HOST_ID, collectionName); } + + if (!indexes.contains(IDX_STEP_ID_EXECUTE_COUNT_EXECUTE_OBJECT_ID)) { + log.info("Create index {} for collection: {} start...", + IDX_STEP_ID_EXECUTE_COUNT_EXECUTE_OBJECT_ID, collectionName); + IndexOptions indexOptions = new IndexOptions(); + indexOptions.background(false); + indexOptions.name(IDX_STEP_ID_EXECUTE_COUNT_EXECUTE_OBJECT_ID); + collection.createIndex(Document.parse("{\"stepId\":1,\"executeCount\":1,\"executeObjectId\":1}"), + indexOptions); + log.info("Create index {} for collection: {} successfully!", IDX_STEP_ID_EXECUTE_COUNT_EXECUTE_OBJECT_ID, + collectionName); + } + log.info("Create index for collection : {} successfully!", collectionName); } } diff --git a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/service/LogService.java b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/service/LogService.java index e134d3ddf1..8eefcee755 100644 --- a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/service/LogService.java +++ b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/service/LogService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -30,7 +30,7 @@ import com.tencent.bk.job.logsvr.model.FileLogQuery; import com.tencent.bk.job.logsvr.model.FileTaskLogDoc; import com.tencent.bk.job.logsvr.model.ScriptLogQuery; -import com.tencent.bk.job.logsvr.model.TaskHostLog; +import com.tencent.bk.job.logsvr.model.TaskExecuteObjectLog; import java.util.List; @@ -41,17 +41,17 @@ public interface LogService { /** * 保存执行日志 * - * @param taskHostLog 执行日志 + * @param taskExecuteObjectLog 执行日志 */ - void saveLog(TaskHostLog taskHostLog); + void saveLog(TaskExecuteObjectLog taskExecuteObjectLog); /** * 保存执行日志 * - * @param logType 日志类型 - * @param taskHostLogs 执行日志 + * @param logType 日志类型 + * @param taskExecuteObjectLogs 执行日志 */ - void saveLogs(LogTypeEnum logType, List taskHostLogs); + void saveLogs(LogTypeEnum logType, List taskExecuteObjectLogs); /** * 批量获取脚本执行日志 @@ -60,7 +60,7 @@ public interface LogService { * @return 日志内容 * @throws ServiceException 异常 */ - List listScriptLogs(ScriptLogQuery scriptLogQuery); + List listScriptLogs(ScriptLogQuery scriptLogQuery); /** * 查询文件任务执行日志 @@ -96,10 +96,26 @@ List getFileLogsByTaskIds(String jobCreateDate, * @param keyword 查询关键字 * @return ip */ - List getIpsByKeyword(String jobCreateDate, - long stepInstanceId, - int executeCount, - Integer batch, - String keyword); + List getHostsByKeyword(String jobCreateDate, + long stepInstanceId, + int executeCount, + Integer batch, + String keyword); + + /** + * 返回日志内容包含关键字的任务对应的执行对象 ID 集合 + * + * @param jobCreateDate 创建时间 + * @param stepInstanceId 步骤ID + * @param executeCount 执行次数 + * @param batch 滚动执行批次 + * @param keyword 查询关键字 + * @return 执行对象 ID 集合 + */ + List getExecuteObjectIdsByKeyword(String jobCreateDate, + long stepInstanceId, + int executeCount, + Integer batch, + String keyword); } diff --git a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/service/impl/LogServiceImpl.java b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/service/impl/LogServiceImpl.java index 900c16c75c..d13ad61351 100644 --- a/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/service/impl/LogServiceImpl.java +++ b/src/backend/job-logsvr/service-job-logsvr/src/main/java/com/tencent/bk/job/logsvr/service/impl/LogServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -32,15 +32,19 @@ import com.mongodb.client.model.WriteModel; import com.tencent.bk.job.common.exception.ServiceException; import com.tencent.bk.job.common.model.dto.HostDTO; -import com.tencent.bk.job.common.util.BatchUtil; +import com.tencent.bk.job.common.util.CollectionUtil; +import com.tencent.bk.job.common.util.StringUtil; import com.tencent.bk.job.logsvr.consts.LogTypeEnum; import com.tencent.bk.job.logsvr.model.FileLogQuery; import com.tencent.bk.job.logsvr.model.FileTaskLogDoc; +import com.tencent.bk.job.logsvr.model.FileTaskLogDocField; import com.tencent.bk.job.logsvr.model.ScriptLogQuery; import com.tencent.bk.job.logsvr.model.ScriptTaskLogDoc; -import com.tencent.bk.job.logsvr.model.TaskHostLog; +import com.tencent.bk.job.logsvr.model.ScriptTaskLogDocField; +import com.tencent.bk.job.logsvr.model.TaskExecuteObjectLog; import com.tencent.bk.job.logsvr.mongo.LogCollectionFactory; import com.tencent.bk.job.logsvr.service.LogService; +import com.tencent.bk.job.logsvr.util.CollectionNameUtil; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -63,6 +67,8 @@ @Slf4j public class LogServiceImpl implements LogService { private static final int BATCH_SIZE = 100; + private static final char[] SPECIAL_CHAR = {'*', '(', ')', '+', '?', '\\', '$', '^', '>', '.'}; + private static final String[] ESCAPE_CHAR = {"\\*", "\\(", "\\)", "\\+", "\\?", "\\\\", "\\$", "\\^", "\\>", "\\."}; private final MongoTemplate mongoTemplate; private final LogCollectionFactory logCollectionFactory; @@ -73,32 +79,32 @@ public LogServiceImpl(MongoTemplate mongoTemplate, LogCollectionFactory logColle } @Override - public void saveLog(TaskHostLog taskHostLog) { - if (taskHostLog.getLogType().equals(LogTypeEnum.SCRIPT.getValue())) { - writeScriptLog(taskHostLog); - } else if (taskHostLog.getLogType().equals(LogTypeEnum.FILE.getValue())) { - writeFileLog(taskHostLog); + public void saveLog(TaskExecuteObjectLog taskExecuteObjectLog) { + if (taskExecuteObjectLog.getLogType().equals(LogTypeEnum.SCRIPT.getValue())) { + writeScriptLog(taskExecuteObjectLog); + } else if (taskExecuteObjectLog.getLogType().equals(LogTypeEnum.FILE.getValue())) { + writeFileLog(taskExecuteObjectLog); } } @Override - public void saveLogs(LogTypeEnum logType, List taskHostLogs) { + public void saveLogs(LogTypeEnum logType, List taskExecuteObjectLogs) { if (logType == LogTypeEnum.SCRIPT) { - batchWriteScriptLogs(taskHostLogs); + batchWriteScriptLogs(taskExecuteObjectLogs); } else if (logType == LogTypeEnum.FILE) { - batchWriteFileLogs(taskHostLogs); + batchWriteFileLogs(taskExecuteObjectLogs); } } - private void batchWriteScriptLogs(List taskHostLogs) { - String jobCreateDate = taskHostLogs.get(0).getJobCreateDate(); - String collectionName = buildLogCollectionName(jobCreateDate, LogTypeEnum.SCRIPT); - List scriptLogDocList = taskHostLogs.stream() - .map(taskIpLog -> buildScriptLogDoc(taskIpLog.getScriptTaskLog())).collect(Collectors.toList()); - List> batchDocList = BatchUtil.buildBatchList(scriptLogDocList, BATCH_SIZE); + private void batchWriteScriptLogs(List taskExecuteObjectLogs) { + String jobCreateDate = taskExecuteObjectLogs.get(0).getJobCreateDate(); + String collectionName = CollectionNameUtil.buildLogCollectionName(jobCreateDate, LogTypeEnum.SCRIPT); + List scriptLogDocList = taskExecuteObjectLogs.stream() + .map(taskHostLog -> buildScriptLogDoc(taskHostLog.getScriptTaskLog())).collect(Collectors.toList()); + List> batchDocList = CollectionUtil.partitionList(scriptLogDocList, BATCH_SIZE); long start = System.currentTimeMillis(); - batchDocList.parallelStream().forEach(docs -> + batchDocList.forEach(docs -> logCollectionFactory.getCollection(collectionName) .insertMany(docs, new InsertManyOptions().ordered(false))); long end = System.currentTimeMillis(); @@ -106,36 +112,35 @@ private void batchWriteScriptLogs(List taskHostLogs) { } - private void batchWriteFileLogs(List taskHostLogs) { - String jobCreateDate = taskHostLogs.get(0).getJobCreateDate(); - String collectionName = buildLogCollectionName(jobCreateDate, LogTypeEnum.FILE); + private void batchWriteFileLogs(List taskExecuteObjectLogs) { + String jobCreateDate = taskExecuteObjectLogs.get(0).getJobCreateDate(); + String collectionName = CollectionNameUtil.buildLogCollectionName(jobCreateDate, LogTypeEnum.FILE); - List> updateOps = buildUpdateOpsFileTask(taskHostLogs); - List>> batchList = BatchUtil.buildBatchList(updateOps, BATCH_SIZE); + List> updateOps = buildUpdateOpsFileTask(taskExecuteObjectLogs); + List>> batchList = CollectionUtil.partitionList(updateOps, BATCH_SIZE); long start = System.currentTimeMillis(); - batchList.parallelStream().forEach(batchOps -> logCollectionFactory.getCollection(collectionName) + batchList.forEach(batchOps -> logCollectionFactory.getCollection(collectionName) .bulkWrite(batchOps, new BulkWriteOptions().ordered(false))); long end = System.currentTimeMillis(); - log.warn("Batch write file logs, stepInstanceId: {}, opSize: {}, cost: {} ms", - taskHostLogs.get(0).getStepInstanceId(), updateOps.size(), end - start); + log.info("Batch write file logs, stepInstanceId: {}, opSize: {}, cost: {} ms", + taskExecuteObjectLogs.get(0).getStepInstanceId(), updateOps.size(), end - start); } - private List> buildUpdateOpsFileTask(List taskHostLogs) { + private List> buildUpdateOpsFileTask(List taskExecuteObjectLogs) { List> updateOps = new ArrayList<>(); - taskHostLogs.forEach(taskHostLog -> { - long stepInstanceId = taskHostLog.getStepInstanceId(); - String ip = taskHostLog.getIp(); - int executeCount = taskHostLog.getExecuteCount(); - Integer batch = taskHostLog.getBatch(); - List fileTaskLogs = taskHostLog.getFileTaskLogs(); - - if (CollectionUtils.isNotEmpty(taskHostLog.getFileTaskLogs())) { + taskExecuteObjectLogs.forEach(taskExecuteObjectLog -> { + long stepInstanceId = taskExecuteObjectLog.getStepInstanceId(); + int executeCount = taskExecuteObjectLog.getExecuteCount(); + Integer batch = taskExecuteObjectLog.getBatch(); + List fileTaskLogs = taskExecuteObjectLog.getFileTaskLogs(); + + if (CollectionUtils.isNotEmpty(taskExecuteObjectLog.getFileTaskLogs())) { fileTaskLogs.forEach(fileTaskLog -> { BasicDBObject filter = buildQueryDocForFileTaskLog(stepInstanceId, executeCount, batch, fileTaskLog); BasicDBObject update = buildUpdateDocForFileTaskLog(stepInstanceId, executeCount, batch, - taskHostLog.getHostId(), ip, fileTaskLog); + taskExecuteObjectLog.getExecuteObjectId(), taskExecuteObjectLog.getHostId(), fileTaskLog); UpdateOneModel updateOp = new UpdateOneModel<>(filter, update, new UpdateOptions().upsert(true)); updateOps.add(updateOp); @@ -145,18 +150,19 @@ private List> buildUpdateOpsFileTask(List task return updateOps; } - private void writeScriptLog(TaskHostLog taskHostLog) { - if (taskHostLog == null || taskHostLog.getScriptTaskLog() == null) { + private void writeScriptLog(TaskExecuteObjectLog taskExecuteObjectLog) { + if (taskExecuteObjectLog == null || taskExecuteObjectLog.getScriptTaskLog() == null) { return; } - LogTypeEnum logType = LogTypeEnum.getLogType(taskHostLog.getLogType()); + LogTypeEnum logType = LogTypeEnum.getLogType(taskExecuteObjectLog.getLogType()); long start = System.currentTimeMillis(); - long stepInstanceId = taskHostLog.getStepInstanceId(); - String collectionName = buildLogCollectionName(taskHostLog.getJobCreateDate(), logType); + long stepInstanceId = taskExecuteObjectLog.getStepInstanceId(); + String collectionName = CollectionNameUtil.buildLogCollectionName(taskExecuteObjectLog.getJobCreateDate(), + logType); try { - Document scriptLogDoc = buildScriptLogDoc(taskHostLog.getScriptTaskLog()); + Document scriptLogDoc = buildScriptLogDoc(taskExecuteObjectLog.getScriptTaskLog()); logCollectionFactory.getCollection(collectionName).insertOne(scriptLogDoc); } finally { long cost = (System.currentTimeMillis() - start); @@ -166,32 +172,43 @@ private void writeScriptLog(TaskHostLog taskHostLog) { } } - private void writeFileLog(TaskHostLog taskHostLog) { - if (taskHostLog.getFileTaskLogs().size() == 1) { - taskHostLog.getFileTaskLogs().forEach( - fileTaskLog -> writeFileLog(taskHostLog.getJobCreateDate(), taskHostLog.getStepInstanceId(), - taskHostLog.getExecuteCount(), taskHostLog.getBatch(), taskHostLog.getHostId(), taskHostLog.getIp(), + private void writeFileLog(TaskExecuteObjectLog taskExecuteObjectLog) { + if (taskExecuteObjectLog.getFileTaskLogs().size() == 1) { + taskExecuteObjectLog.getFileTaskLogs().forEach( + fileTaskLog -> writeFileLog( + taskExecuteObjectLog.getJobCreateDate(), + taskExecuteObjectLog.getStepInstanceId(), + taskExecuteObjectLog.getExecuteCount(), + taskExecuteObjectLog.getBatch(), + taskExecuteObjectLog.getExecuteObjectId(), + taskExecuteObjectLog.getHostId(), fileTaskLog)); } else { - batchWriteFileLogs(Collections.singletonList(taskHostLog)); + batchWriteFileLogs(Collections.singletonList(taskExecuteObjectLog)); } } private Document buildScriptLogDoc(ScriptTaskLogDoc scriptTaskLog) { Document doc = new Document(); - doc.put("stepId", scriptTaskLog.getStepInstanceId()); - doc.put("executeCount", scriptTaskLog.getExecuteCount()); + doc.put(ScriptTaskLogDocField.STEP_ID, scriptTaskLog.getStepInstanceId()); + doc.put(ScriptTaskLogDocField.EXECUTE_COUNT, scriptTaskLog.getExecuteCount()); if (scriptTaskLog.getBatch() != null && scriptTaskLog.getBatch() > 0) { - doc.put("batch", scriptTaskLog.getBatch()); + doc.put(ScriptTaskLogDocField.BATCH, scriptTaskLog.getBatch()); + } + if (StringUtils.isNotEmpty(scriptTaskLog.getExecuteObjectId())) { + doc.put(ScriptTaskLogDocField.EXECUTE_OBJECT_ID, scriptTaskLog.getExecuteObjectId()); } if (scriptTaskLog.getHostId() != null) { - doc.put("hostId", scriptTaskLog.getHostId()); + doc.put(ScriptTaskLogDocField.HOST_ID, scriptTaskLog.getHostId()); } if (StringUtils.isNotEmpty(scriptTaskLog.getIp())) { - doc.put("ip", scriptTaskLog.getIp()); + doc.put(ScriptTaskLogDocField.IP, scriptTaskLog.getIp()); + } + if (StringUtils.isNotEmpty(scriptTaskLog.getIpv6())) { + doc.put(ScriptTaskLogDocField.IPV6, scriptTaskLog.getIpv6()); } - doc.put("content", scriptTaskLog.getContent()); - doc.put("offset", scriptTaskLog.getOffset()); + doc.put(ScriptTaskLogDocField.CONTENT, scriptTaskLog.getContent()); + doc.put(ScriptTaskLogDocField.OFFSET, scriptTaskLog.getOffset()); return doc; } @@ -199,15 +216,15 @@ private void writeFileLog(String jobCreateDate, long stepInstanceId, int executeCount, Integer batch, + String executeObjectId, Long hostId, - String ip, FileTaskLogDoc fileTaskLog) { long start = System.currentTimeMillis(); - String collectionName = buildLogCollectionName(jobCreateDate, LogTypeEnum.FILE); + String collectionName = CollectionNameUtil.buildLogCollectionName(jobCreateDate, LogTypeEnum.FILE); try { BasicDBObject filter = buildQueryDocForFileTaskLog(stepInstanceId, executeCount, batch, fileTaskLog); - BasicDBObject update = buildUpdateDocForFileTaskLog(stepInstanceId, executeCount, batch, hostId, ip, - fileTaskLog); + BasicDBObject update = buildUpdateDocForFileTaskLog(stepInstanceId, executeCount, batch, executeObjectId, + hostId, fileTaskLog); logCollectionFactory.getCollection(collectionName) .updateOne(filter, update, new UpdateOptions().upsert(true)); } finally { @@ -218,75 +235,95 @@ private void writeFileLog(String jobCreateDate, } } - private BasicDBObject buildQueryDocForFileTaskLog(long stepInstanceId, int executeCount, Integer batch, + private BasicDBObject buildQueryDocForFileTaskLog(long stepInstanceId, + int executeCount, + Integer batch, FileTaskLogDoc fileTaskLog) { BasicDBObject filter = new BasicDBObject(); - filter.append("stepId", stepInstanceId); - filter.append("executeCount", executeCount); + filter.append(FileTaskLogDocField.STEP_ID, stepInstanceId); + filter.append(FileTaskLogDocField.EXECUTE_COUNT, executeCount); if (batch != null && batch > 0) { - filter.append("batch", batch); + filter.append(FileTaskLogDocField.BATCH, batch); } - filter.append("taskId", fileTaskLog.getTaskId()); + filter.append(FileTaskLogDocField.TASK_ID, fileTaskLog.getTaskId()); return filter; } private BasicDBObject buildUpdateDocForFileTaskLog(long stepInstanceId, int executeCount, Integer batch, + String executeObjectId, Long hostId, - String ip, FileTaskLogDoc fileTaskLog) { BasicDBObject update = new BasicDBObject(); BasicDBObject setDBObject = new BasicDBObject(); BasicDBObject pushDBObject = new BasicDBObject(); - setDBObject.append("stepId", stepInstanceId) - .append("executeCount", executeCount) - .append("mode", fileTaskLog.getMode()) - .append("taskId", fileTaskLog.getTaskId()); - if (hostId != null) { - setDBObject.append("hostId", hostId); + setDBObject.append(FileTaskLogDocField.STEP_ID, stepInstanceId) + .append(FileTaskLogDocField.EXECUTE_COUNT, executeCount) + .append(FileTaskLogDocField.MODE, fileTaskLog.getMode()) + .append(FileTaskLogDocField.TASK_ID, fileTaskLog.getTaskId()); + if (StringUtils.isNotEmpty(executeObjectId)) { + setDBObject.append(FileTaskLogDocField.EXECUTE_OBJECT_ID, executeObjectId); } - if (StringUtils.isNotEmpty(ip)) { - setDBObject.append("ip", ip); + if (hostId != null) { + setDBObject.append(FileTaskLogDocField.HOST_ID, hostId); } if (batch != null && batch > 0) { - setDBObject.append("batch", batch); + setDBObject.append(FileTaskLogDocField.BATCH, batch); + } + if (StringUtils.isNotEmpty(fileTaskLog.getSrcExecuteObjectId())) { + setDBObject.append(FileTaskLogDocField.SRC_EXECUTE_OBJECT_ID, fileTaskLog.getSrcExecuteObjectId()); } if (fileTaskLog.getSrcHostId() != null) { - setDBObject.append("srcHostId", fileTaskLog.getSrcHostId()); + setDBObject.append(FileTaskLogDocField.SRC_HOST_ID, fileTaskLog.getSrcHostId()); } if (StringUtils.isNotEmpty(fileTaskLog.getSrcIp())) { - setDBObject.append("srcIp", fileTaskLog.getSrcIp()); + setDBObject.append(FileTaskLogDocField.SRC_IP, fileTaskLog.getSrcIp()); } - if (StringUtils.isNotEmpty(fileTaskLog.getDisplaySrcIp())) { - setDBObject.append("displaySrcIp", fileTaskLog.getDisplaySrcIp()); + if (StringUtils.isNotEmpty(fileTaskLog.getSrcIpv6())) { + setDBObject.append(FileTaskLogDocField.SRC_IPV6, fileTaskLog.getSrcIpv6()); + } + if (fileTaskLog.getSrcFileType() != null) { + setDBObject.append(FileTaskLogDocField.SRC_FILE_TYPE, fileTaskLog.getSrcFileType()); } if (StringUtils.isNotEmpty(fileTaskLog.getSrcFile())) { - setDBObject.append("srcFile", fileTaskLog.getSrcFile()); + setDBObject.append(FileTaskLogDocField.SRC_FILE, fileTaskLog.getSrcFile()); } if (StringUtils.isNotEmpty(fileTaskLog.getDisplaySrcFile())) { - setDBObject.append("displaySrcFile", fileTaskLog.getDisplaySrcFile()); + setDBObject.append(FileTaskLogDocField.DISPLAY_SRC_FILE, fileTaskLog.getDisplaySrcFile()); + } + if (StringUtils.isNotEmpty(fileTaskLog.getDestExecuteObjectId())) { + setDBObject.append(FileTaskLogDocField.DEST_EXECUTE_OBJECT_ID, fileTaskLog.getDestExecuteObjectId()); + } + if (fileTaskLog.getDestHostId() != null) { + setDBObject.append(FileTaskLogDocField.DEST_HOST_ID, fileTaskLog.getDestHostId()); + } + if (StringUtils.isNotEmpty(fileTaskLog.getSrcIp())) { + setDBObject.append(FileTaskLogDocField.DEST_IP, fileTaskLog.getDestIp()); + } + if (StringUtils.isNotEmpty(fileTaskLog.getSrcIpv6())) { + setDBObject.append(FileTaskLogDocField.DEST_IPV6, fileTaskLog.getDestIpv6()); } if (StringUtils.isNotEmpty(fileTaskLog.getDestFile())) { - setDBObject.append("destFile", fileTaskLog.getDestFile()); + setDBObject.append(FileTaskLogDocField.DEST_FILE, fileTaskLog.getDestFile()); } if (StringUtils.isNotEmpty(fileTaskLog.getSize())) { - setDBObject.append("size", fileTaskLog.getSize()); + setDBObject.append(FileTaskLogDocField.SIZE, fileTaskLog.getSize()); } if (fileTaskLog.getStatus() != null) { - setDBObject.append("status", fileTaskLog.getStatus()); + setDBObject.append(FileTaskLogDocField.STATUS, fileTaskLog.getStatus()); } if (StringUtils.isNotEmpty(fileTaskLog.getStatusDesc())) { - setDBObject.append("statusDesc", fileTaskLog.getStatusDesc()); + setDBObject.append(FileTaskLogDocField.STATUS_DESC, fileTaskLog.getStatusDesc()); } if (StringUtils.isNotEmpty(fileTaskLog.getSpeed())) { - setDBObject.append("speed", fileTaskLog.getSpeed()); + setDBObject.append(FileTaskLogDocField.SPEED, fileTaskLog.getSpeed()); } if (StringUtils.isNotEmpty(fileTaskLog.getProcess())) { - setDBObject.append("process", fileTaskLog.getProcess()); + setDBObject.append(FileTaskLogDocField.PROCESS, fileTaskLog.getProcess()); } if (StringUtils.isNotEmpty(fileTaskLog.getContent())) { - pushDBObject.append("contentList", fileTaskLog.getContent()); + pushDBObject.append(FileTaskLogDocField.CONTENT_LIST, fileTaskLog.getContent()); } update.put("$set", setDBObject); @@ -296,24 +333,11 @@ private BasicDBObject buildUpdateDocForFileTaskLog(long stepInstanceId, return update; } - private String buildLogCollectionName(String jobCreateDate, LogTypeEnum logType) { - return "job_log_" + getLogTypeName(logType) + "_" + jobCreateDate; - } - - private String getLogTypeName(LogTypeEnum logType) { - if (logType == LogTypeEnum.SCRIPT) { - return "script"; - } else if (logType == LogTypeEnum.FILE) { - return "file"; - } else { - throw new IllegalArgumentException("Invalid logType"); - } - } - @Override - public List listScriptLogs(ScriptLogQuery scriptLogQuery) throws ServiceException { + public List listScriptLogs(ScriptLogQuery scriptLogQuery) throws ServiceException { long start = System.currentTimeMillis(); - String collectionName = buildLogCollectionName(scriptLogQuery.getJobCreateDate(), LogTypeEnum.SCRIPT); + String collectionName = CollectionNameUtil.buildLogCollectionName(scriptLogQuery.getJobCreateDate(), + LogTypeEnum.SCRIPT); try { Query query = buildScriptLogMongoQuery(scriptLogQuery); @@ -324,7 +348,8 @@ public List listScriptLogs(ScriptLogQuery scriptLogQuery) throws Se return Collections.emptyList(); } - return groupScriptTaskLogsByHost(scriptLogQuery.getStepInstanceId(), scriptLogQuery.getExecuteCount(), + return groupScriptTaskLogsByExecuteObject(scriptLogQuery.getStepInstanceId(), + scriptLogQuery.getExecuteCount(), scriptLogQuery.getBatch(), scriptLogs); } finally { long cost = (System.currentTimeMillis() - start); @@ -338,26 +363,27 @@ private Query buildScriptLogMongoQuery(ScriptLogQuery scriptLogQuery) { long stepInstanceId = scriptLogQuery.getStepInstanceId(); int executeCount = scriptLogQuery.getExecuteCount(); Integer batch = scriptLogQuery.getBatch(); - List ips = scriptLogQuery.getIps(); List hostIds = scriptLogQuery.getHostIds(); + List executeObjectIds = scriptLogQuery.getExecuteObjectIds(); Query query = new Query(); - query.addCriteria(Criteria.where("stepId").is(stepInstanceId)); - query.addCriteria(Criteria.where("executeCount").is(executeCount)); + query.addCriteria(Criteria.where(ScriptTaskLogDocField.STEP_ID).is(stepInstanceId)); + query.addCriteria(Criteria.where(ScriptTaskLogDocField.EXECUTE_COUNT).is(executeCount)); if (batch != null && batch > 0) { - query.addCriteria(Criteria.where("batch").is(batch)); + query.addCriteria(Criteria.where(ScriptTaskLogDocField.BATCH).is(batch)); } - if (CollectionUtils.isNotEmpty(hostIds)) { - if (hostIds.size() == 1) { - query.addCriteria(Criteria.where("hostId").is(hostIds.get(0))); + // executeObjectIds/hostIds 两个参数二选一，优先使用 executeObjectIds + if (CollectionUtils.isNotEmpty(scriptLogQuery.getExecuteObjectIds())) { + if (executeObjectIds.size() == 1) { + query.addCriteria(Criteria.where(ScriptTaskLogDocField.EXECUTE_OBJECT_ID).is(executeObjectIds.get(0))); } else { - query.addCriteria(Criteria.where("hostId").in(hostIds)); + query.addCriteria(Criteria.where(ScriptTaskLogDocField.EXECUTE_OBJECT_ID).in(executeObjectIds)); } - } else if (CollectionUtils.isNotEmpty(ips)) { - if (ips.size() == 1) { - query.addCriteria(Criteria.where("ip").is(ips.get(0))); + } else if (CollectionUtils.isNotEmpty(hostIds)) { + if (hostIds.size() == 1) { + query.addCriteria(Criteria.where(ScriptTaskLogDocField.HOST_ID).is(hostIds.get(0))); } else { - query.addCriteria(Criteria.where("ip").in(ips)); + query.addCriteria(Criteria.where(ScriptTaskLogDocField.HOST_ID).in(hostIds)); } } @@ -366,7 +392,8 @@ private Query buildScriptLogMongoQuery(ScriptLogQuery scriptLogQuery) { @Override public List listFileLogs(FileLogQuery getLogRequest) { - String collectionName = buildLogCollectionName(getLogRequest.getJobCreateDate(), LogTypeEnum.FILE); + String collectionName = CollectionNameUtil.buildLogCollectionName(getLogRequest.getJobCreateDate(), + LogTypeEnum.FILE); long start = System.currentTimeMillis(); try { @@ -388,91 +415,100 @@ public List listFileLogs(FileLogQuery getLogRequest) { private Query buildFileLogMongoQuery(FileLogQuery getLogRequest) { Query query = new Query(); - query.addCriteria(Criteria.where("stepId").is(getLogRequest.getStepInstanceId())); - query.addCriteria(Criteria.where("executeCount").is(getLogRequest.getExecuteCount())); + query.addCriteria(Criteria.where(FileTaskLogDocField.STEP_ID).is(getLogRequest.getStepInstanceId())); + query.addCriteria(Criteria.where(FileTaskLogDocField.EXECUTE_COUNT).is(getLogRequest.getExecuteCount())); if (getLogRequest.getMode() != null) { - query.addCriteria(Criteria.where("mode").is(getLogRequest.getMode())); + query.addCriteria(Criteria.where(FileTaskLogDocField.MODE).is(getLogRequest.getMode())); } - if (CollectionUtils.isNotEmpty(getLogRequest.getHostIds())) { - if (getLogRequest.getHostIds().size() > 1) { - query.addCriteria(Criteria.where("hostId").in(getLogRequest.getHostIds())); + // executeObjectIds/hostIds 两个参数二选一，优先使用 executeObjectIds + if (CollectionUtils.isNotEmpty(getLogRequest.getExecuteObjectIds())) { + if (getLogRequest.getExecuteObjectIds().size() > 1) { + query.addCriteria(Criteria.where(FileTaskLogDocField.EXECUTE_OBJECT_ID) + .in(getLogRequest.getExecuteObjectIds())); } else { - query.addCriteria(Criteria.where("hostId").is(getLogRequest.getHostIds().get(0))); + query.addCriteria(Criteria.where(FileTaskLogDocField.EXECUTE_OBJECT_ID) + .is(getLogRequest.getExecuteObjectIds().get(0))); } - } else if (CollectionUtils.isNotEmpty(getLogRequest.getIps())) { - if (getLogRequest.getIps().size() > 1) { - query.addCriteria(Criteria.where("ip").in(getLogRequest.getIps())); + } else if (CollectionUtils.isNotEmpty(getLogRequest.getHostIds())) { + if (getLogRequest.getHostIds().size() > 1) { + query.addCriteria(Criteria.where(FileTaskLogDocField.HOST_ID).in(getLogRequest.getHostIds())); } else { - query.addCriteria(Criteria.where("ip").is(getLogRequest.getIps().get(0))); + query.addCriteria(Criteria.where(FileTaskLogDocField.HOST_ID).is(getLogRequest.getHostIds().get(0))); } } if (getLogRequest.getBatch() != null && getLogRequest.getBatch() > 0) { - query.addCriteria(Criteria.where("batch").is(getLogRequest.getBatch())); + query.addCriteria(Criteria.where(FileTaskLogDocField.BATCH).is(getLogRequest.getBatch())); } return query; } - private List groupScriptTaskLogsByHost(long stepInstanceId, - int executeCount, - Integer batch, - List scriptTaskLogs) { - List taskHostLogs = new ArrayList<>(); - boolean existHostIdField = scriptTaskLogs.get(0).getHostId() != null; - if (existHostIdField) { - Map> scriptLogsGroups = new HashMap<>(); + private List groupScriptTaskLogsByExecuteObject(long stepInstanceId, + int executeCount, + Integer batch, + List scriptTaskLogs) { + List taskExecuteObjectLogs = new ArrayList<>(); + boolean existExecuteObjectIdField = scriptTaskLogs.get(0).getExecuteObjectId() != null; + if (existExecuteObjectIdField) { + Map> scriptLogsGroups = new HashMap<>(); scriptTaskLogs.forEach(scriptTaskLog -> { - List scriptLogGroup = scriptLogsGroups.computeIfAbsent(scriptTaskLog.getHostId(), - k -> new ArrayList<>()); + List scriptLogGroup = scriptLogsGroups.computeIfAbsent( + scriptTaskLog.getExecuteObjectId(), k -> new ArrayList<>()); scriptLogGroup.add(scriptTaskLog); }); scriptLogsGroups.forEach( - (hostId, scriptLogGroup) -> - taskHostLogs.add( - buildTaskHostLog(stepInstanceId, executeCount, batch, scriptLogGroup))); + (executeObjectId, scriptLogGroup) -> + taskExecuteObjectLogs.add( + buildTaskExecuteObjectLog(stepInstanceId, executeCount, batch, scriptLogGroup))); } else { - Map> scriptLogsGroups = new HashMap<>(); + // 兼容 hostId + Map> scriptLogsGroups = new HashMap<>(); scriptTaskLogs.forEach(scriptTaskLog -> { - List scriptLogGroup = scriptLogsGroups.computeIfAbsent(scriptTaskLog.getIp(), + List scriptLogGroup = scriptLogsGroups.computeIfAbsent(scriptTaskLog.getHostId(), k -> new ArrayList<>()); scriptLogGroup.add(scriptTaskLog); }); scriptLogsGroups.forEach( - (ip, scriptLogGroup) -> - taskHostLogs.add( - buildTaskHostLog(stepInstanceId, executeCount, batch,scriptLogGroup))); + (hostId, scriptLogGroup) -> + taskExecuteObjectLogs.add( + buildTaskExecuteObjectLog(stepInstanceId, executeCount, batch, scriptLogGroup))); } - return taskHostLogs; + return taskExecuteObjectLogs; } - private TaskHostLog buildTaskHostLog(long stepInstanceId, int executeCount, Integer batch, - List scriptLogs) { - TaskHostLog taskHostLog = new TaskHostLog(); - taskHostLog.setStepInstanceId(stepInstanceId); - taskHostLog.setExecuteCount(executeCount); - taskHostLog.setBatch(batch); - taskHostLog.setHostId(scriptLogs.get(0).getHostId()); - taskHostLog.setIp(scriptLogs.get(0).getIp()); + private TaskExecuteObjectLog buildTaskExecuteObjectLog(long stepInstanceId, + int executeCount, + Integer batch, + List scriptLogs) { + TaskExecuteObjectLog taskExecuteObjectLog = new TaskExecuteObjectLog(); + taskExecuteObjectLog.setStepInstanceId(stepInstanceId); + taskExecuteObjectLog.setExecuteCount(executeCount); + taskExecuteObjectLog.setBatch(batch); + taskExecuteObjectLog.setExecuteObjectId(scriptLogs.get(0).getExecuteObjectId()); + taskExecuteObjectLog.setHostId(scriptLogs.get(0).getHostId()); + taskExecuteObjectLog.setIp(scriptLogs.get(0).getIp()); + taskExecuteObjectLog.setIpv6(scriptLogs.get(0).getIpv6()); scriptLogs.sort(ScriptTaskLogDoc.LOG_OFFSET_COMPARATOR); - taskHostLog.setScriptContent(scriptLogs.stream().map(ScriptTaskLogDoc::getContent).collect(Collectors.joining(""))); + taskExecuteObjectLog.setScriptContent(scriptLogs.stream() + .map(ScriptTaskLogDoc::getContent).collect(Collectors.joining(""))); - return taskHostLog; + return taskExecuteObjectLog; } @Override public List getFileLogsByTaskIds(String jobCreateDate, long stepInstanceId, int executeCount, Integer batch, List taskIds) { long start = System.currentTimeMillis(); - String collectionName = buildLogCollectionName(jobCreateDate, LogTypeEnum.FILE); + String collectionName = CollectionNameUtil.buildLogCollectionName(jobCreateDate, LogTypeEnum.FILE); try { Query query = new Query(); - query.addCriteria(Criteria.where("stepId").is(stepInstanceId)); - query.addCriteria(Criteria.where("executeCount").is(executeCount)); - query.addCriteria(Criteria.where("taskId").in(taskIds)); + query.addCriteria(Criteria.where(FileTaskLogDocField.STEP_ID).is(stepInstanceId)); + query.addCriteria(Criteria.where(FileTaskLogDocField.EXECUTE_COUNT).is(executeCount)); + query.addCriteria(Criteria.where(FileTaskLogDocField.TASK_ID).in(taskIds)); if (batch != null && batch > 0) { - query.addCriteria(Criteria.where("batch").is(batch)); + query.addCriteria(Criteria.where(FileTaskLogDocField.BATCH).is(batch)); } List fileTaskLogs = mongoTemplate.find(query, FileTaskLogDoc.class, collectionName); if (CollectionUtils.isNotEmpty(fileTaskLogs)) { @@ -490,11 +526,11 @@ public List getFileLogsByTaskIds(String jobCreateDate, long step } @Override - public List getIpsByKeyword(String jobCreateDate, long stepInstanceId, int executeCount, - Integer batch, String keyword) { - String collectionName = buildLogCollectionName(jobCreateDate, LogTypeEnum.SCRIPT); + public List getHostsByKeyword(String jobCreateDate, long stepInstanceId, int executeCount, + Integer batch, String keyword) { + String collectionName = CollectionNameUtil.buildLogCollectionName(jobCreateDate, LogTypeEnum.SCRIPT); Query query = buildQueryForKeywordSearch(stepInstanceId, executeCount, batch, keyword); - query.fields().include("ip", "hostId"); + query.fields().include(ScriptTaskLogDocField.IP, ScriptTaskLogDocField.HOST_ID); List logs = mongoTemplate.find(query, ScriptTaskLogDoc.class, collectionName); if (logs.isEmpty()) { return Collections.emptyList(); @@ -518,18 +554,37 @@ private HostDTO extractHostInfo(ScriptTaskLogDoc log) { private Query buildQueryForKeywordSearch(long stepInstanceId, int executeCount, Integer batch, String keyword) { Query query = new Query(); - query.addCriteria(Criteria.where("stepId").is(stepInstanceId)); + query.addCriteria(Criteria.where(ScriptTaskLogDocField.STEP_ID).is(stepInstanceId)); if (executeCount == 0) { - query.addCriteria(Criteria.where("executeCount").is(executeCount)); + query.addCriteria(Criteria.where(ScriptTaskLogDocField.EXECUTE_COUNT).is(executeCount)); } else { - query.addCriteria(Criteria.where("executeCount").lte(executeCount)); + query.addCriteria(Criteria.where(ScriptTaskLogDocField.EXECUTE_COUNT).lte(executeCount)); } if (batch != null && batch > 0) { - query.addCriteria(Criteria.where("batch").is(batch)); + query.addCriteria(Criteria.where(ScriptTaskLogDocField.BATCH).is(batch)); } - Pattern pattern = Pattern.compile(keyword.replaceAll("['$&|`;#]", ""), - Pattern.LITERAL | Pattern.CASE_INSENSITIVE); + keyword = StringUtil.escape(keyword, SPECIAL_CHAR, ESCAPE_CHAR); + Pattern pattern = Pattern.compile(keyword, Pattern.LITERAL | Pattern.CASE_INSENSITIVE); query.addCriteria(Criteria.where("content").regex(pattern)); return query; } + + @Override + public List getExecuteObjectIdsByKeyword(String jobCreateDate, + long stepInstanceId, + int executeCount, + Integer batch, + String keyword) { + String collectionName = CollectionNameUtil.buildLogCollectionName(jobCreateDate, LogTypeEnum.SCRIPT); + Query query = buildQueryForKeywordSearch(stepInstanceId, executeCount, batch, keyword); + query.fields().include(ScriptTaskLogDocField.EXECUTE_OBJECT_ID); + List logs = mongoTemplate.find(query, ScriptTaskLogDoc.class, collectionName); + if (logs.isEmpty()) { + return Collections.emptyList(); + } + return logs.stream() + .map(ScriptTaskLogDoc::getExecuteObjectId) + .distinct() + .collect(Collectors.toList()); + } } diff --git a/src/backend/job-logsvr/service-job-logsvr/src/main/resources/i18n/message.properties b/src/backend/job-logsvr/service-job-logsvr/src/main/resources/i18n/message.properties new file mode 100644 index 0000000000..5414b99b69 --- /dev/null +++ b/src/backend/job-logsvr/service-job-logsvr/src/main/resources/i18n/message.properties @@ -0,0 +1,23 @@ +# +# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. +# +# Copyright (C) 2021 Tencent. All rights reserved. +# +# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. +# +# License for BK-JOB蓝鲸智云作业平台: +# -------------------------------------------------------------------- +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF +# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# diff --git a/src/backend/job-logsvr/service-job-logsvr/src/main/resources/i18n/message_en.properties b/src/backend/job-logsvr/service-job-logsvr/src/main/resources/i18n/message_en.properties new file mode 100644 index 0000000000..5414b99b69 --- /dev/null +++ b/src/backend/job-logsvr/service-job-logsvr/src/main/resources/i18n/message_en.properties @@ -0,0 +1,23 @@ +# +# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. +# +# Copyright (C) 2021 Tencent. All rights reserved. +# +# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. +# +# License for BK-JOB蓝鲸智云作业平台: +# -------------------------------------------------------------------- +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF +# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# diff --git a/src/backend/job-logsvr/service-job-logsvr/src/main/resources/i18n/message_en_US.properties b/src/backend/job-logsvr/service-job-logsvr/src/main/resources/i18n/message_en_US.properties new file mode 100644 index 0000000000..5414b99b69 --- /dev/null +++ b/src/backend/job-logsvr/service-job-logsvr/src/main/resources/i18n/message_en_US.properties @@ -0,0 +1,23 @@ +# +# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. +# +# Copyright (C) 2021 Tencent. All rights reserved. +# +# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. +# +# License for BK-JOB蓝鲸智云作业平台: +# -------------------------------------------------------------------- +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF +# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# diff --git a/src/backend/job-logsvr/service-job-logsvr/src/main/resources/i18n/message_zh.properties b/src/backend/job-logsvr/service-job-logsvr/src/main/resources/i18n/message_zh.properties new file mode 100644 index 0000000000..5414b99b69 --- /dev/null +++ b/src/backend/job-logsvr/service-job-logsvr/src/main/resources/i18n/message_zh.properties @@ -0,0 +1,23 @@ +# +# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. +# +# Copyright (C) 2021 Tencent. All rights reserved. +# +# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. +# +# License for BK-JOB蓝鲸智云作业平台: +# -------------------------------------------------------------------- +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF +# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# diff --git a/src/backend/job-logsvr/service-job-logsvr/src/main/resources/i18n/message_zh_CN.properties b/src/backend/job-logsvr/service-job-logsvr/src/main/resources/i18n/message_zh_CN.properties new file mode 100644 index 0000000000..5414b99b69 --- /dev/null +++ b/src/backend/job-logsvr/service-job-logsvr/src/main/resources/i18n/message_zh_CN.properties @@ -0,0 +1,23 @@ +# +# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. +# +# Copyright (C) 2021 Tencent. All rights reserved. +# +# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. +# +# License for BK-JOB蓝鲸智云作业平台: +# -------------------------------------------------------------------- +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF +# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# diff --git a/src/backend/job-manage/api-common-job-manage/build.gradle b/src/backend/job-manage/api-common-job-manage/build.gradle new file mode 100644 index 0000000000..8313411560 --- /dev/null +++ b/src/backend/job-manage/api-common-job-manage/build.gradle @@ -0,0 +1,29 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +dependencies { + implementation 'com.fasterxml.jackson.core:jackson-core' + implementation 'com.fasterxml.jackson.core:jackson-databind' + implementation 'com.fasterxml.jackson.core:jackson-annotations' +} diff --git a/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/CredentialTypeEnum.java b/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/CredentialTypeEnum.java new file mode 100644 index 0000000000..5d10456371 --- /dev/null +++ b/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/CredentialTypeEnum.java @@ -0,0 +1,52 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.api.common.constants; + +public enum CredentialTypeEnum { + APP_ID_SECRET_KEY(1, "AccessKey+SecretKey"), + PASSWORD(2, "单一密码"), + USERNAME_PASSWORD(3, "用户名+密码"), + SECRET_KEY(4, "单一SecretKey"); + + private final Integer value; + private final String description; + + public static String getAllNameStr() { + StringBuilder sb = new StringBuilder(); + CredentialTypeEnum[] values = CredentialTypeEnum.values(); + for (int i = 0; i < values.length; i++) { + sb.append(values[i].name()); + if (i < values.length - 1) { + sb.append(","); + } + } + return sb.toString(); + } + + CredentialTypeEnum(Integer type, String description) { + this.value = type; + this.description = description; + } +} diff --git a/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/EnableStatusEnum.java b/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/EnableStatusEnum.java new file mode 100644 index 0000000000..f27d452c9c --- /dev/null +++ b/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/EnableStatusEnum.java @@ -0,0 +1,49 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.api.common.constants; + +/** + * 启用状态 + */ +public enum EnableStatusEnum { + /** + * 启用 + */ + ENABLED(1), + /** + * 停用 + */ + DISABLED(0); + + private final int value; + + EnableStatusEnum(int val) { + this.value = val; + } + + public int getValue() { + return value; + } +} diff --git a/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/JobResourceStatusEnum.java b/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/JobResourceStatusEnum.java new file mode 100644 index 0000000000..1dc7847fac --- /dev/null +++ b/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/JobResourceStatusEnum.java @@ -0,0 +1,89 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.api.common.constants; + +/** + * JOB资源状态 + */ + +public enum JobResourceStatusEnum { + /** + * 未上线 + */ + DRAFT(0), + /** + * 已上线 + */ + ONLINE(1), + /** + * 已下线 + */ + OFFLINE(2), + /** + * 禁用 + */ + DISABLED(3); + + private final int status; + + JobResourceStatusEnum(int status) { + this.status = status; + } + + public static JobResourceStatusEnum getJobResourceStatus(Integer status) { + if (status == null) { + return null; + } + if (status == ONLINE.getValue()) { + return ONLINE; + } else if (status == DRAFT.getValue()) { + return DRAFT; + } else if (status == DISABLED.getValue()) { + return DISABLED; + } else if (status == OFFLINE.getValue()) { + return OFFLINE; + } else { + throw new IllegalArgumentException("No JobResourceStatusEnum constant: " + status); + } + } + + public int getValue() { + return status; + } + + public String getStatusI18nKey() { + if (this == DRAFT) { + return "job.resource.status.draft"; + } else if (this == ONLINE) { + return "job.resource.status.online"; + } else if (this == DISABLED) { + return "job.resource.status.disabled"; + } else if (this == OFFLINE) { + return "job.resource.status.offline"; + } else { + return ""; + } + } +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/globalsetting/OSTypeEnum.java b/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/OSTypeEnum.java similarity index 93% rename from src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/globalsetting/OSTypeEnum.java rename to src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/OSTypeEnum.java index 099c67a9a1..1a4b254461 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/globalsetting/OSTypeEnum.java +++ b/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/OSTypeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.manage.common.consts.globalsetting; +package com.tencent.bk.job.manage.api.common.constants; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonValue; diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/RuleMatchHandleActionEnum.java b/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/RuleMatchHandleActionEnum.java similarity index 94% rename from src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/RuleMatchHandleActionEnum.java rename to src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/RuleMatchHandleActionEnum.java index cc5c7f142d..269f6ec338 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/RuleMatchHandleActionEnum.java +++ b/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/RuleMatchHandleActionEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.manage.common.consts; +package com.tencent.bk.job.manage.api.common.constants; /** * 匹配高危语句规则处理动作 diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/TemplateTypeEnum.java b/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/TemplateTypeEnum.java similarity index 93% rename from src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/TemplateTypeEnum.java rename to src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/TemplateTypeEnum.java index 2ff1ac2ee3..c06f4cd7b0 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/TemplateTypeEnum.java +++ b/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/TemplateTypeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.manage.common.consts; +package com.tencent.bk.job.manage.api.common.constants; import com.fasterxml.jackson.annotation.JsonValue; import lombok.AllArgsConstructor; diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/account/AccountTypeEnum.java b/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/account/AccountTypeEnum.java similarity index 89% rename from src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/account/AccountTypeEnum.java rename to src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/account/AccountTypeEnum.java index 6c068f4c9e..8718e5ec29 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/account/AccountTypeEnum.java +++ b/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/account/AccountTypeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,14 +22,13 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.manage.common.consts.account; +package com.tencent.bk.job.manage.api.common.constants.account; -import com.tencent.bk.job.manage.common.consts.globalsetting.OSTypeEnum; + +import com.tencent.bk.job.manage.api.common.constants.OSTypeEnum; /** * 账号类型 - * - * @date 2019/09/19 */ public enum AccountTypeEnum { LINUX(1, "Linux"), @@ -56,6 +55,13 @@ public static AccountTypeEnum valueOf(Integer type) { return null; } + public static boolean isValid(Integer type) { + if (type == null) { + return false; + } + return valueOf(type) != null; + } + public static boolean isValidDbType(Integer type) { if (type == null) { return false; diff --git a/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/rule/HighRiskGrammarActionEnum.java b/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/rule/HighRiskGrammarActionEnum.java new file mode 100644 index 0000000000..75bb8b4b3c --- /dev/null +++ b/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/rule/HighRiskGrammarActionEnum.java @@ -0,0 +1,74 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.api.common.constants.rule; + +import lombok.Getter; + +/** + * 处理动作枚举,1:扫描,2:拦截 + */ +@Getter +public enum HighRiskGrammarActionEnum { + SCAN(1, "scan"), INTERCEPT(2, "intercept"); + + private final Integer code; + private final String name; + + HighRiskGrammarActionEnum(Integer code, String name) { + this.code = code; + this.name = name; + } + + public static String getName(Integer type) { + for (HighRiskGrammarActionEnum highRiskGrammarActionEnum : values()) { + if (highRiskGrammarActionEnum.code.equals(type)) { + return highRiskGrammarActionEnum.getName(); + } + } + return ""; + } + + public static HighRiskGrammarActionEnum valueOf(Integer type) { + for (HighRiskGrammarActionEnum highRiskGrammarActionEnum : values()) { + if (highRiskGrammarActionEnum.code.equals(type)) { + return highRiskGrammarActionEnum; + } + } + return null; + } + + /** + * 判断参数合法性 + */ + public static boolean isValid(Integer code) { + for (HighRiskGrammarActionEnum highRiskGrammarActionEnum : HighRiskGrammarActionEnum.values()) { + if (highRiskGrammarActionEnum.getCode().equals(code)) { + return true; + } + } + return false; + } + +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/rule/HighRiskGrammarRuleStatusEnum.java b/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/rule/HighRiskGrammarRuleStatusEnum.java similarity index 95% rename from src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/rule/HighRiskGrammarRuleStatusEnum.java rename to src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/rule/HighRiskGrammarRuleStatusEnum.java index e6f605bf25..40f5c1db4a 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/rule/HighRiskGrammarRuleStatusEnum.java +++ b/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/rule/HighRiskGrammarRuleStatusEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.manage.common.consts.rule; +package com.tencent.bk.job.manage.api.common.constants.rule; import lombok.Getter; diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/script/ScriptCategoryEnum.java b/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/script/ScriptCategoryEnum.java similarity index 93% rename from src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/script/ScriptCategoryEnum.java rename to src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/script/ScriptCategoryEnum.java index 08e168448c..a51907ad03 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/script/ScriptCategoryEnum.java +++ b/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/script/ScriptCategoryEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.manage.common.consts.script; +package com.tencent.bk.job.manage.api.common.constants.script; /** * @date 2019/09/19 diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/script/ScriptCheckErrorLevelEnum.java b/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/script/ScriptCheckErrorLevelEnum.java similarity index 91% rename from src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/script/ScriptCheckErrorLevelEnum.java rename to src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/script/ScriptCheckErrorLevelEnum.java index e5828d52c6..b9f0895e8c 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/script/ScriptCheckErrorLevelEnum.java +++ b/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/script/ScriptCheckErrorLevelEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.manage.common.consts.script; +package com.tencent.bk.job.manage.api.common.constants.script; /** * 脚本检查错误级别 @@ -41,7 +41,7 @@ public enum ScriptCheckErrorLevelEnum { */ FATAL(3); - private int level; + private final int level; ScriptCheckErrorLevelEnum(int level) { this.level = level; diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/script/ScriptScopeEnum.java b/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/script/ScriptScopeEnum.java similarity index 91% rename from src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/script/ScriptScopeEnum.java rename to src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/script/ScriptScopeEnum.java index 1d5630c696..46bd09c285 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/script/ScriptScopeEnum.java +++ b/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/script/ScriptScopeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.manage.common.consts.script; +package com.tencent.bk.job.manage.api.common.constants.script; /** * 脚本可见范围 @@ -34,7 +34,7 @@ public enum ScriptScopeEnum { APP(0), PUBLIC(1); - private int scope; + private final int scope; ScriptScopeEnum(int scope) { this.scope = scope; diff --git a/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/script/ScriptTypeEnum.java b/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/script/ScriptTypeEnum.java new file mode 100644 index 0000000000..08c8c6bfa0 --- /dev/null +++ b/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/script/ScriptTypeEnum.java @@ -0,0 +1,105 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.api.common.constants.script; + +import lombok.Getter; + +@Getter +public enum ScriptTypeEnum { + /** + * 脚本类型枚举类 + */ + SHELL(1, "shell", ".sh"), + BAT(2, "bat", ".bat"), + PERL(3, "perl", ".pl"), + PYTHON(4, "python", ".py"), + POWERSHELL(5, "powershell", ".ps1"), + SQL(6, "sql", ".sql"); + + private final Integer value; + private final String name; + private final String ext; + + ScriptTypeEnum(Integer type, String name, String ext) { + this.value = type; + this.name = name; + this.ext = ext; + } + + public static String getName(Integer type) { + for (ScriptTypeEnum scriptTypeEnum : values()) { + if (scriptTypeEnum.value.equals(type)) { + return scriptTypeEnum.getName(); + } + } + return ""; + } + + public static ScriptTypeEnum valOf(Integer type) { + for (ScriptTypeEnum scriptTypeEnum : values()) { + if (scriptTypeEnum.value.equals(type)) { + return scriptTypeEnum; + } + } + return null; + } + + public static boolean isValid(Integer type) { + if (type == null) { + return false; + } + return valOf(type) != null; + } + + public static ScriptTypeEnum getTypeByExt(String ext) { + if (SHELL.ext.equals(ext)) { + return SHELL; + } else if (BAT.ext.equals(ext)) { + return BAT; + } else if (PERL.ext.equals(ext)) { + return PERL; + } else if (PYTHON.ext.equals(ext)) { + return PYTHON; + } else if (POWERSHELL.ext.equals(ext)) { + return POWERSHELL; + } else if (SQL.ext.equals(ext)) { + return SQL; + } else { + return null; + } + } + + /** + * 获取脚本类型的字典名称的排序 + * + * @return 排序 + */ + public static ScriptTypeEnum[] getScriptTypeNameAscSort() { + // 按照脚本语言名称字典顺序排序.Bat(2)->Perl(3)->Powershell(5)->Python(4)->Shell(1)->SQL(6) + return new ScriptTypeEnum[]{ + BAT, PERL, POWERSHELL, PYTHON, SHELL, SQL + }; + } +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/task/TaskApprovalTypeEnum.java b/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/task/TaskApprovalTypeEnum.java similarity index 93% rename from src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/task/TaskApprovalTypeEnum.java rename to src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/task/TaskApprovalTypeEnum.java index 8e167abe9e..dcdc96803a 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/task/TaskApprovalTypeEnum.java +++ b/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/task/TaskApprovalTypeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.manage.common.consts.task; +package com.tencent.bk.job.manage.api.common.constants.task; import lombok.AllArgsConstructor; import lombok.Getter; diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/task/TaskFileTypeEnum.java b/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/task/TaskFileTypeEnum.java similarity index 92% rename from src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/task/TaskFileTypeEnum.java rename to src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/task/TaskFileTypeEnum.java index 74120ea65a..0cd4819b9e 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/task/TaskFileTypeEnum.java +++ b/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/task/TaskFileTypeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,14 +22,11 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.manage.common.consts.task; +package com.tencent.bk.job.manage.api.common.constants.task; import lombok.AllArgsConstructor; import lombok.Getter; -/** - * @since 3/10/2019 17:08 - */ @Getter @AllArgsConstructor public enum TaskFileTypeEnum { @@ -39,7 +36,7 @@ public enum TaskFileTypeEnum { SERVER(1), /** - * 本地文件 + * Job本地文件 */ LOCAL(2), diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/task/TaskPlanTypeEnum.java b/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/task/TaskPlanTypeEnum.java similarity index 80% rename from src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/task/TaskPlanTypeEnum.java rename to src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/task/TaskPlanTypeEnum.java index 299ca8b992..9ae5030db4 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/task/TaskPlanTypeEnum.java +++ b/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/task/TaskPlanTypeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,15 +22,13 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.manage.common.consts.task; +package com.tencent.bk.job.manage.api.common.constants.task; -import com.tencent.bk.job.common.constant.ErrorCode; -import com.tencent.bk.job.common.exception.InvalidParamException; import lombok.AllArgsConstructor; import lombok.Getter; /** - * @since 27/11/2019 15:07 + * 执行方案类型 */ @Getter @AllArgsConstructor @@ -44,15 +42,15 @@ public enum TaskPlanTypeEnum { */ DEBUG(1); - private int type; + private final int type; - public TaskPlanTypeEnum valueOf(int type) throws InvalidParamException { + public TaskPlanTypeEnum valueOf(int type) { for (TaskPlanTypeEnum planType : values()) { if (planType.type == type) { return planType; } } - throw new InvalidParamException(ErrorCode.INTERNAL_ERROR); + throw new IllegalArgumentException("No TaskPlanTypeEnum constant: " + type); } public int getValue() { diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/task/TaskScriptSourceEnum.java b/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/task/TaskScriptSourceEnum.java similarity index 93% rename from src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/task/TaskScriptSourceEnum.java rename to src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/task/TaskScriptSourceEnum.java index 8619fde179..be91f98f0b 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/task/TaskScriptSourceEnum.java +++ b/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/task/TaskScriptSourceEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.manage.common.consts.task; +package com.tencent.bk.job.manage.api.common.constants.task; import lombok.AllArgsConstructor; import lombok.Getter; diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/task/TaskStepTypeEnum.java b/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/task/TaskStepTypeEnum.java similarity index 93% rename from src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/task/TaskStepTypeEnum.java rename to src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/task/TaskStepTypeEnum.java index 59e06a7b67..71ab04ff21 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/task/TaskStepTypeEnum.java +++ b/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/task/TaskStepTypeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.manage.common.consts.task; +package com.tencent.bk.job.manage.api.common.constants.task; /** * 作业步骤类型 diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/task/TaskTemplateStatusEnum.java b/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/task/TaskTemplateStatusEnum.java similarity index 93% rename from src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/task/TaskTemplateStatusEnum.java rename to src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/task/TaskTemplateStatusEnum.java index a339e007bf..ae8ea2fb87 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/task/TaskTemplateStatusEnum.java +++ b/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/task/TaskTemplateStatusEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.manage.common.consts.task; +package com.tencent.bk.job.manage.api.common.constants.task; import lombok.AllArgsConstructor; import lombok.Getter; diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/task/TaskTypeEnum.java b/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/task/TaskTypeEnum.java similarity index 92% rename from src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/task/TaskTypeEnum.java rename to src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/task/TaskTypeEnum.java index 3110afb8a5..1c40df57b9 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/task/TaskTypeEnum.java +++ b/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/task/TaskTypeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.manage.common.consts.task; +package com.tencent.bk.job.manage.api.common.constants.task; import lombok.AllArgsConstructor; import lombok.Getter; diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/whiteip/ActionScopeEnum.java b/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/whiteip/ActionScopeEnum.java similarity index 93% rename from src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/whiteip/ActionScopeEnum.java rename to src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/whiteip/ActionScopeEnum.java index b42aee5d2f..04d4464254 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/whiteip/ActionScopeEnum.java +++ b/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/whiteip/ActionScopeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.manage.common.consts.whiteip; +package com.tencent.bk.job.manage.api.common.constants.whiteip; import lombok.AllArgsConstructor; import lombok.Getter; diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/whiteip/Keys.java b/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/whiteip/Keys.java similarity index 91% rename from src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/whiteip/Keys.java rename to src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/whiteip/Keys.java index 5cb3eb270f..f8b594ef8a 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/whiteip/Keys.java +++ b/src/backend/job-manage/api-common-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/whiteip/Keys.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.manage.common.consts.whiteip; +package com.tencent.bk.job.manage.api.common.constants.whiteip; /** * @Description @@ -40,7 +40,9 @@ public class Keys { public static final String KEY_LAST_MODIFY_USER = "lastModifyUser"; public static final String KEY_LAST_MODIFY_TIME = "lastModifyTime"; public static final String KEY_CLOUD_AREA_ID = "cloudAreaId"; + public static final String KEY_HOST_ID = "hostId"; public static final String KEY_IP = "ip"; + public static final String KEY_IPV6 = "ipv6"; public static final String KEY_IP_LIST = "ipList"; public static final String KEY_ACTION_SCOPE = "actionScope"; public static final String KEY_ACTION_SCOPE_LIST = "actionScopeList"; diff --git a/src/backend/job-manage/api-job-manage/build.gradle b/src/backend/job-manage/api-job-manage/build.gradle index 191505f4fd..3a1dccd67b 100644 --- a/src/backend/job-manage/api-job-manage/build.gradle +++ b/src/backend/job-manage/api-job-manage/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,8 +26,11 @@ dependencies { api project(':commons:common') api project(':commons:common-i18n') api project(':commons:common-iam') + api project(':commons:cmdb-sdk') + api project(':commons:notice-sdk') api project(':job-execute:api-job-execute') api(project(":commons:common-api")) + api(project(":job-manage:api-common-job-manage")) implementation "org.springframework:spring-web" implementation "javax.ws.rs:javax.ws.rs-api" implementation("org.apache.commons:commons-collections4") diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/AbstractLocalCacheAppScopeMappingService.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/AbstractLocalCacheAppScopeMappingService.java index 233ea4b7d9..8dbd47b04f 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/AbstractLocalCacheAppScopeMappingService.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/AbstractLocalCacheAppScopeMappingService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -135,6 +135,12 @@ public AppResourceScope getAppResourceScope(Long appId, String scopeType, String } } + @Override + public AppResourceScope getAppResourceScope(Long appId) { + ResourceScope resourceScope = getScopeByAppId(appId); + return new AppResourceScope(appId, resourceScope); + } + @Override public AppResourceScope getAppResourceScope(String scopeType, String scopeId) { return new AppResourceScope(scopeType, scopeId, getAppIdByScope(scopeType, scopeId)); diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/AppScopeMappingServiceImpl.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/AppScopeMappingServiceImpl.java index 1f68290ae1..6440a8a6d0 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/AppScopeMappingServiceImpl.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/AppScopeMappingServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -43,6 +43,7 @@ public class AppScopeMappingServiceImpl extends AbstractLocalCacheAppScopeMappin public AppScopeMappingServiceImpl(ServiceApplicationResource applicationResource) { this.applicationResource = applicationResource; + GlobalAppScopeMappingService.register(this); } @Override @@ -55,7 +56,7 @@ public Long queryAppByScope(ResourceScope resourceScope) throws NotFoundExceptio } if (app.getId() == null) { // 如果查询到的业务缺少ID参数，抛出异常避免缓存非法数据 - log.error("Empty appId for application, reject cache!"); + log.error("Empty appId for application, reject cache! query scope: {}", resourceScope); throw new InternalException("Empty appId for application", ErrorCode.INTERNAL_ERROR); } return app.getId(); @@ -70,7 +71,7 @@ public ResourceScope queryScopeByAppId(Long appId) throws NotFoundException { } if (StringUtils.isEmpty(app.getScopeType()) || StringUtils.isEmpty(app.getScopeId())) { // 如果查询到的业务缺少scopeType|scopeId参数，抛出异常避免缓存非法数据 - log.error("Empty scopeType|scopeId for application, reject cache!"); + log.error("Empty scopeType|scopeId for application, reject cache! query appId: {}", appId); throw new InternalException("Empty scopeType|scopeId for application", ErrorCode.INTERNAL_ERROR); } return new ResourceScope(app.getScopeType(), app.getScopeId()); diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/GlobalAppScopeMappingService.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/GlobalAppScopeMappingService.java new file mode 100644 index 0000000000..c5a79cdb1a --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/GlobalAppScopeMappingService.java @@ -0,0 +1,64 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage; + +import com.tencent.bk.job.common.service.AppScopeMappingService; +import com.tencent.bk.job.common.util.ApplicationContextRegister; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +public class GlobalAppScopeMappingService { + private static volatile AppScopeMappingService globalAppScopeMappingService; + + private static final Object lock = new Object(); + + private GlobalAppScopeMappingService() { + + } + + public static AppScopeMappingService get() { + AppScopeMappingService service = globalAppScopeMappingService; + if (service == null) { + synchronized (lock) { + service = globalAppScopeMappingService; + if (service == null) { + service = + ApplicationContextRegister.getBean(AppScopeMappingService.class); + } + } + } + return service; + } + + public static void register(AppScopeMappingService appScopeMappingService) { + synchronized (lock) { + if (globalAppScopeMappingService != null) { + log.info("AppScopeMappingService is already register"); + return; + } + globalAppScopeMappingService = appScopeMappingService; + } + } +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/globalsetting/GlobalSettingKeys.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/globalsetting/GlobalSettingKeys.java similarity index 85% rename from src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/globalsetting/GlobalSettingKeys.java rename to src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/globalsetting/GlobalSettingKeys.java index 1d48d1a8c3..6cee918fb7 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/globalsetting/GlobalSettingKeys.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/globalsetting/GlobalSettingKeys.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.manage.common.consts.globalsetting; +package com.tencent.bk.job.manage.api.common.constants.globalsetting; /** * @Description @@ -52,4 +52,10 @@ public class GlobalSettingKeys { public static final String KEY_ENABLE_UPLOAD_TO_ARTIFACTORY = "ENABLE_UPLOAD_TO_ARTIFACTORY"; //Job自有业务集是否已完全迁移至CMDB public static final String KEY_IS_BIZSET_MIGRATED_TO_CMDB = "IS_BIZSET_MIGRATED_TO_CMDB"; + // 助手信息 + public static final String KEY_BK_HELPER = "BK_HELPER"; + // 是否对接消息通知中心 + public static final String KEY_ENABLE_BK_NOTICE = "ENABLE_BK_NOTICE"; + // 消息通知中心是否注册成功 + public static final String KEY_BK_NOTICE_REGISTERED_SUCCESS = "BK_NOTICE_REGISTERED_SUCCESS"; } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/globalsetting/RelatedUrlKeys.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/globalsetting/RelatedUrlKeys.java similarity index 86% rename from src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/globalsetting/RelatedUrlKeys.java rename to src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/globalsetting/RelatedUrlKeys.java index d0411a8cdc..7079c7de71 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/globalsetting/RelatedUrlKeys.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/globalsetting/RelatedUrlKeys.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.manage.common.consts.globalsetting; +package com.tencent.bk.job.manage.api.common.constants.globalsetting; public class RelatedUrlKeys { // CMDB @@ -37,4 +37,6 @@ public class RelatedUrlKeys { public static final String KEY_BK_DOC_JOB_ROOT_URL = "BK_DOC_JOB_ROOT_URL"; // 问题反馈 public static final String KEY_BK_FEED_BACK_ROOT_URL = "BK_FEED_BACK_ROOT_URL"; + // 全局配置（title/footer/name/logo/产品商标）相关的前端资源文件base.js基础地址 + public static final String KEY_BK_SHARED_RES_BASE_JS_URL = "BK_SHARED_RES_BASE_JS_URL"; } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/globalsetting/RestrictModeEnum.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/globalsetting/RestrictModeEnum.java similarity index 94% rename from src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/globalsetting/RestrictModeEnum.java rename to src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/globalsetting/RestrictModeEnum.java index 36d793a279..43169aa9e3 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/globalsetting/RestrictModeEnum.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/globalsetting/RestrictModeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.manage.common.consts.globalsetting; +package com.tencent.bk.job.manage.api.common.constants.globalsetting; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonValue; diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/globalsetting/StorageUnitEnum.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/globalsetting/StorageUnitEnum.java similarity index 91% rename from src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/globalsetting/StorageUnitEnum.java rename to src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/globalsetting/StorageUnitEnum.java index cf7a485ef3..162d016965 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/globalsetting/StorageUnitEnum.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/globalsetting/StorageUnitEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.manage.common.consts.globalsetting; +package com.tencent.bk.job.manage.api.common.constants.globalsetting; import lombok.AllArgsConstructor; import lombok.Getter; diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/notify/ExecuteStatusEnum.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/notify/ExecuteStatusEnum.java new file mode 100644 index 0000000000..fe70aa8fdd --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/notify/ExecuteStatusEnum.java @@ -0,0 +1,85 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.api.common.constants.notify; + +import com.tencent.bk.job.common.util.I18nUtil; +import com.tencent.bk.job.manage.model.web.vo.notify.ExecuteStatusVO; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.val; + +import java.util.ArrayList; +import java.util.List; + +@Getter +@AllArgsConstructor +public enum ExecuteStatusEnum { + /** + * 执行成功 + */ + SUCCESS(1, "job.manage.execute.status.success"), + + /** + * 执行失败 + */ + FAIL(2, "job.manage.execute.status.fail"), + + /** + * 等待执行 + */ + READY(3, "job.manage.execute.status.ready"); + + private final int status; + private final String i18nCode; + + public static ExecuteStatusEnum get(int status) { + val values = ExecuteStatusEnum.values(); + for (int i = 0; i < values.length; i++) { + if (values[i].status == status) { + return values[i]; + } + } + return null; + } + + public static String getName(int status) { + val values = ExecuteStatusEnum.values(); + for (int i = 0; i < values.length; i++) { + if (values[i].status == status) { + return values[i].name(); + } + } + return null; + } + + public static List getVOList() { + List resultList = new ArrayList(); + val values = ExecuteStatusEnum.values(); + for (int i = 0; i < values.length; i++) { + resultList.add(new ExecuteStatusVO(values[i].name(), I18nUtil.getI18nMessage(values[i].getI18nCode()))); + } + return resultList; + } +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/notify/JobRoleEnum.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/notify/JobRoleEnum.java similarity index 88% rename from src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/notify/JobRoleEnum.java rename to src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/notify/JobRoleEnum.java index 90468f3ff6..5d299549b2 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/notify/JobRoleEnum.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/notify/JobRoleEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.manage.common.consts.notify; +package com.tencent.bk.job.manage.api.common.constants.notify; import com.tencent.bk.job.common.util.I18nUtil; import com.tencent.bk.job.manage.model.web.vo.notify.RoleVO; @@ -70,4 +70,13 @@ public static RoleVO getVO(String role) { JobRoleEnum roleEnum = JobRoleEnum.valueOf(role); return new RoleVO(roleEnum.name(), I18nUtil.getI18nMessage(roleEnum.getDefaultName())); } + + public static boolean isJobRole(String role) { + for (JobRoleEnum jobRole : values()) { + if (jobRole.name().equals(role)) { + return true; + } + } + return false; + } } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/notify/NotifyConsts.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/notify/NotifyConsts.java similarity index 96% rename from src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/notify/NotifyConsts.java rename to src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/notify/NotifyConsts.java index e39940c34f..14d504605b 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/notify/NotifyConsts.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/notify/NotifyConsts.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.manage.common.consts.notify; +package com.tencent.bk.job.manage.api.common.constants.notify; /** * @Description diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/notify/ResourceTypeEnum.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/notify/ResourceTypeEnum.java similarity index 86% rename from src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/notify/ResourceTypeEnum.java rename to src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/notify/ResourceTypeEnum.java index 2b4a51d017..c1ee1ea739 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/notify/ResourceTypeEnum.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/notify/ResourceTypeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.manage.common.consts.notify; +package com.tencent.bk.job.manage.api.common.constants.notify; import com.tencent.bk.job.common.util.I18nUtil; import com.tencent.bk.job.manage.model.web.vo.notify.ResourceTypeVO; @@ -39,19 +39,25 @@ public enum ResourceTypeEnum { /** * 快速执行脚本 */ - SCRIPT(1, "job.manage.resource.type.script"), + SCRIPT(1, "job.manage.resource.type.script", true), /** * 快速分发文件 */ - FILE(3, "job.manage.resource.type.file"), + FILE(3, "job.manage.resource.type.file", true), /** * 执行方案 */ - JOB(2, "job.manage.resource.type.job"); + JOB(2, "job.manage.resource.type.job", true), + + /** + * 定时任务 + */ + CRON(4, "job.manage.resource.type.cron", false); private final int type; private final String defaultName; + private final boolean showInPageTemplate; public static ResourceTypeEnum get(int type) { val values = ResourceTypeEnum.values(); @@ -77,6 +83,9 @@ public static List getVOList() { List resultList = new ArrayList(); val values = ResourceTypeEnum.values(); for (int i = 0; i < values.length; i++) { + if (!values[i].showInPageTemplate) { + continue; + } resultList.add(new ResourceTypeVO(values[i].name(), I18nUtil.getI18nMessage(values[i].getDefaultName()))); } return resultList; diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/notify/TriggerTypeEnum.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/notify/TriggerTypeEnum.java similarity index 95% rename from src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/notify/TriggerTypeEnum.java rename to src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/notify/TriggerTypeEnum.java index 2682aff940..0f7d693908 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/notify/TriggerTypeEnum.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/constants/notify/TriggerTypeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.manage.common.consts.notify; +package com.tencent.bk.job.manage.api.common.constants.notify; import com.tencent.bk.job.common.util.I18nUtil; import com.tencent.bk.job.manage.model.web.vo.notify.TriggerTypeVO; diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/EsbGetDBAccountListResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/EsbGetDBAccountListResource.java index 51a3e9b426..00a88bd337 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/EsbGetDBAccountListResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/EsbGetDBAccountListResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,12 +25,14 @@ package com.tencent.bk.job.manage.api.esb; import com.tencent.bk.job.common.annotation.EsbAPI; +import com.tencent.bk.job.common.constant.JobCommonHeaders; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.manage.model.esb.EsbDBAccountDTO; import com.tencent.bk.job.manage.model.esb.request.EsbGetDBAccountListRequest; import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @@ -46,6 +48,8 @@ public interface EsbGetDBAccountListResource { @PostMapping("/get_own_db_account_list") EsbResp> getUserOwnDbAccountList( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbGetDBAccountListRequest request diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/EsbGetJobDetailResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/EsbGetJobDetailResource.java index ef6762dae9..afef64c536 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/EsbGetJobDetailResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/EsbGetJobDetailResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,12 +25,14 @@ package com.tencent.bk.job.manage.api.esb; import com.tencent.bk.job.common.annotation.EsbAPI; +import com.tencent.bk.job.common.constant.JobCommonHeaders; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.manage.model.esb.EsbJobDetailDTO; import com.tencent.bk.job.manage.model.esb.request.EsbGetJobDetailRequest; import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @@ -44,6 +46,8 @@ public interface EsbGetJobDetailResource { @PostMapping("/get_job_detail") EsbResp getJobDetail( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbGetJobDetailRequest request diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/EsbGetJobListResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/EsbGetJobListResource.java index 892e3e3ddf..8b16bc2037 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/EsbGetJobListResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/EsbGetJobListResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,12 +25,14 @@ package com.tencent.bk.job.manage.api.esb; import com.tencent.bk.job.common.annotation.EsbAPI; +import com.tencent.bk.job.common.constant.JobCommonHeaders; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.manage.model.esb.EsbJobBasicInfoDTO; import com.tencent.bk.job.manage.model.esb.request.EsbGetJobListRequest; import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @@ -46,6 +48,8 @@ public interface EsbGetJobListResource { @PostMapping("/get_job_list") EsbResp> getJobList( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbGetJobListRequest request diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/EsbGetOSAccountResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/EsbGetOSAccountResource.java index ecd872e513..6e57d1d5f7 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/EsbGetOSAccountResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/EsbGetOSAccountResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,12 +25,14 @@ package com.tencent.bk.job.manage.api.esb; import com.tencent.bk.job.common.annotation.EsbAPI; +import com.tencent.bk.job.common.constant.JobCommonHeaders; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.manage.model.esb.EsbAccountDTO; import com.tencent.bk.job.manage.model.esb.request.EsbGetOSAccountListRequest; import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @@ -46,6 +48,8 @@ public interface EsbGetOSAccountResource { @PostMapping("/get_os_account") EsbResp> getAppOsAccountList( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbGetOSAccountListRequest request diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/EsbGetPublicScriptListResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/EsbGetPublicScriptListResource.java index 6dd8810e41..49c409a5a0 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/EsbGetPublicScriptListResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/EsbGetPublicScriptListResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,6 +25,7 @@ package com.tencent.bk.job.manage.api.esb; import com.tencent.bk.job.common.annotation.EsbAPI; +import com.tencent.bk.job.common.constant.JobCommonHeaders; import com.tencent.bk.job.common.esb.model.EsbPageData; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.manage.model.esb.EsbScriptDTO; @@ -32,6 +33,7 @@ import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @@ -45,6 +47,8 @@ public interface EsbGetPublicScriptListResource { @PostMapping("/get_public_script_list") EsbResp> getPublicScriptList( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbGetPublicScriptListRequest request diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/EsbGetScriptDetailResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/EsbGetScriptDetailResource.java index 8d3d0257f7..dc63736f04 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/EsbGetScriptDetailResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/EsbGetScriptDetailResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,12 +25,14 @@ package com.tencent.bk.job.manage.api.esb; import com.tencent.bk.job.common.annotation.EsbAPI; +import com.tencent.bk.job.common.constant.JobCommonHeaders; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.manage.model.esb.EsbScriptDTO; import com.tencent.bk.job.manage.model.esb.request.EsbGetScriptDetailRequest; import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @@ -44,6 +46,8 @@ public interface EsbGetScriptDetailResource { @PostMapping("/get_script_detail") EsbResp getScriptDetail( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbGetScriptDetailRequest request diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/EsbGetScriptListResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/EsbGetScriptListResource.java index 7795736335..7fc9fa0c16 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/EsbGetScriptListResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/EsbGetScriptListResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,6 +25,7 @@ package com.tencent.bk.job.manage.api.esb; import com.tencent.bk.job.common.annotation.EsbAPI; +import com.tencent.bk.job.common.constant.JobCommonHeaders; import com.tencent.bk.job.common.esb.model.EsbPageData; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.manage.model.esb.EsbScriptDTO; @@ -32,6 +33,7 @@ import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @@ -45,6 +47,8 @@ public interface EsbGetScriptListResource { @PostMapping("/get_script_list") EsbResp> getScriptList( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbGetScriptListRequest request diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/v3/EsbAccountV3Resource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/v3/EsbAccountV3Resource.java index f2dc18d1c8..9d1aac6e85 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/v3/EsbAccountV3Resource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/v3/EsbAccountV3Resource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,9 +25,13 @@ package com.tencent.bk.job.manage.api.esb.v3; import com.tencent.bk.job.common.annotation.EsbAPI; +import com.tencent.bk.job.common.constant.AccountCategoryEnum; import com.tencent.bk.job.common.constant.JobCommonHeaders; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.common.esb.model.job.v3.EsbPageDataV3; +import com.tencent.bk.job.common.validation.CheckEnum; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbCreateAccountV3Req; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbDeleteAccountV3Req; import com.tencent.bk.job.manage.model.esb.v3.request.EsbGetAccountListV3Req; import com.tencent.bk.job.manage.model.esb.v3.response.EsbAccountV3DTO; import org.springframework.validation.annotation.Validated; @@ -44,11 +48,14 @@ */ @RequestMapping("/esb/api/v3") @RestController +@Validated @EsbAPI public interface EsbAccountV3Resource { @PostMapping("/get_account_list") EsbResp> getAccountListUsingPost( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbGetAccountListV3Req request @@ -61,8 +68,31 @@ EsbResp> getAccountList( @RequestParam(value = "bk_biz_id", required = false) Long bizId, @RequestParam(value = "bk_scope_type", required = false) String scopeType, @RequestParam(value = "bk_scope_id", required = false) String scopeId, - @RequestParam(value = "category", required = false) Integer category, + @RequestParam(value = "category", required = false) + @CheckEnum(enumClass = AccountCategoryEnum.class, enumMethod = "isValid", + message = "{validation.constraints.AccountCategory_illegal.message}") + Integer category, + @RequestParam(value = "account", required = false) String account, + @RequestParam(value = "alias", required = false) String alias, @RequestParam(value = "start", required = false) Integer start, @RequestParam(value = "length", required = false) Integer length); + @PostMapping("/create_account") + EsbResp createAccount( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, + @RequestBody + @Validated + EsbCreateAccountV3Req req + ); + + @PostMapping("/delete_account") + EsbResp deleteAccountUsingPost( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, + @RequestBody + @Validated + EsbDeleteAccountV3Req req + ); + } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/v3/EsbAgentInfoV3Resource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/v3/EsbAgentInfoV3Resource.java new file mode 100644 index 0000000000..12c519fe76 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/v3/EsbAgentInfoV3Resource.java @@ -0,0 +1,57 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.api.esb.v3; + +import com.tencent.bk.job.common.annotation.EsbAPI; +import com.tencent.bk.job.common.constant.JobCommonHeaders; +import com.tencent.bk.job.common.esb.model.EsbResp; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbQueryAgentInfoV3Req; +import com.tencent.bk.job.manage.model.esb.v3.response.EsbQueryAgentInfoV3Resp; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +/** + * 查询Agent信息API-V3 + */ +@RequestMapping("/esb/api/v3") +@RestController +@Validated +@EsbAPI +public interface EsbAgentInfoV3Resource { + + @PostMapping("/query_agent_info") + EsbResp queryAgentInfo( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, + @RequestBody + @Validated + EsbQueryAgentInfoV3Req req + ); + +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/v3/EsbCredentialV3Resource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/v3/EsbCredentialV3Resource.java index 64cea7e623..70147a3dac 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/v3/EsbCredentialV3Resource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/v3/EsbCredentialV3Resource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,17 +25,23 @@ package com.tencent.bk.job.manage.api.esb.v3; import com.tencent.bk.job.common.annotation.EsbAPI; +import com.tencent.bk.job.common.constant.JobCommonHeaders; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.manage.model.esb.v3.request.EsbCreateOrUpdateCredentialV3Req; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbGetCredentialDetailV3Req; import com.tencent.bk.job.manage.model.esb.v3.response.EsbCredentialSimpleInfoV3DTO; +import com.tencent.bk.job.manage.model.esb.v3.response.EsbCredentialV3DTO; import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; /** - * 凭据API-V3 + * 凭证API-V3 */ @RequestMapping("/esb/api/v3") @RestController @@ -44,6 +50,8 @@ public interface EsbCredentialV3Resource { @PostMapping("/create_credential") EsbResp createCredential( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbCreateOrUpdateCredentialV3Req req @@ -51,9 +59,34 @@ EsbResp createCredential( @PostMapping("/update_credential") EsbResp updateCredential( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbCreateOrUpdateCredentialV3Req req ); + @GetMapping("/get_credential_detail") + EsbResp getCredentialDetail( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, + @RequestParam(value = "bk_biz_id", required = false) Long bizId, + @RequestParam(value = "bk_scope_type", required = false) String scopeType, + @RequestParam(value = "bk_scope_id", required = false) String scopeId, + @RequestParam(value = "id") String id); + + /** + * 获取凭证详情 + * + * @param req 查询请求 + * @return 凭证详情 + */ + @PostMapping("/get_credential_detail") + EsbResp getCredentialDetailUsingPost( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, + @RequestBody + @Validated + EsbGetCredentialDetailV3Req req); + } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/v3/EsbDangerousRuleV3Resource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/v3/EsbDangerousRuleV3Resource.java new file mode 100644 index 0000000000..90c1256eec --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/v3/EsbDangerousRuleV3Resource.java @@ -0,0 +1,105 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.api.esb.v3; + +import com.tencent.bk.job.common.annotation.EsbAPI; +import com.tencent.bk.job.common.constant.JobCommonHeaders; +import com.tencent.bk.job.common.esb.model.EsbResp; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbCreateDangerousRuleV3Req; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbGetDangerousRuleV3Req; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbManageDangerousRuleV3Req; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbUpdateDangerousRuleV3Req; +import com.tencent.bk.job.manage.model.esb.v3.response.EsbDangerousRuleV3DTO; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; + +/** + * 高危语句规则管理API-V3 + */ +@RequestMapping("/esb/api/v3") +@RestController +@EsbAPI +public interface EsbDangerousRuleV3Resource { + + @PostMapping("/create_dangerous_rule") + EsbResp createDangerousRule( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, + @RequestBody + @Validated + EsbCreateDangerousRuleV3Req request + ); + + @PostMapping("/update_dangerous_rule") + EsbResp updateDangerousRule( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, + @RequestBody + @Validated + EsbUpdateDangerousRuleV3Req request + ); + + @PostMapping("/delete_dangerous_rule") + EsbResp deleteDangerousRule( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, + @RequestBody + @Validated + EsbManageDangerousRuleV3Req request + ); + + @PostMapping("/get_dangerous_rule_list") + EsbResp> getDangerousRuleListUsingPost( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, + @RequestBody + @Validated + EsbGetDangerousRuleV3Req request + ); + + @PostMapping("/enable_dangerous_rule") + EsbResp enableDangerousRule( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, + @RequestBody + @Validated + EsbManageDangerousRuleV3Req request + ); + + @PostMapping("/disable_dangerous_rule") + EsbResp disableDangerousRule( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, + @RequestBody + @Validated + EsbManageDangerousRuleV3Req request + ); +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/v3/EsbLocalFileV3Resource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/v3/EsbLocalFileV3Resource.java index 9af2188b89..9da72a335f 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/v3/EsbLocalFileV3Resource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/v3/EsbLocalFileV3Resource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,12 +25,14 @@ package com.tencent.bk.job.manage.api.esb.v3; import com.tencent.bk.job.common.annotation.EsbAPI; +import com.tencent.bk.job.common.constant.JobCommonHeaders; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.manage.model.esb.v3.request.EsbGenLocalFileUploadUrlV3Req; import com.tencent.bk.job.manage.model.esb.v3.response.EsbUploadUrlV3DTO; import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @@ -44,6 +46,8 @@ public interface EsbLocalFileV3Resource { @PostMapping("/generate_local_file_upload_url") EsbResp generateLocalFileUploadUrl( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbGenLocalFileUploadUrlV3Req req diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/v3/EsbPlanV3Resource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/v3/EsbPlanV3Resource.java index 0de3b7c593..e5cf9d5b25 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/v3/EsbPlanV3Resource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/v3/EsbPlanV3Resource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -78,6 +78,8 @@ EsbResp getPlanDetail( @PostMapping("/get_job_plan_list") EsbResp> getPlanListUsingPost( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbGetPlanListV3Request request @@ -85,6 +87,8 @@ EsbResp> getPlanListUsingPost( @PostMapping("/get_job_plan_detail") EsbResp getPlanDetailUsingPost( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbGetPlanDetailV3Request request diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/v3/EsbPublicScriptV3Resource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/v3/EsbPublicScriptV3Resource.java index 926ff1a501..06bad5ab97 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/v3/EsbPublicScriptV3Resource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/v3/EsbPublicScriptV3Resource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,9 +28,19 @@ import com.tencent.bk.job.common.constant.JobCommonHeaders; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.common.esb.model.job.v3.EsbPageDataV3; +import com.tencent.bk.job.common.validation.Create; +import com.tencent.bk.job.common.validation.Delete; +import com.tencent.bk.job.common.validation.Update; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbCreatePublicScriptV3Req; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbCreatePublicScriptVersionV3Req; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbDeletePublicScriptV3Req; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbDeletePublicScriptVersionV3Req; import com.tencent.bk.job.manage.model.esb.v3.request.EsbGetPublicScriptListV3Request; import com.tencent.bk.job.manage.model.esb.v3.request.EsbGetPublicScriptVersionDetailV3Request; import com.tencent.bk.job.manage.model.esb.v3.request.EsbGetPublicScriptVersionListV3Request; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbManagePublicScriptVersionV3Req; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbUpdatePublicScriptBasicV3Req; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbUpdatePublicScriptVersionV3Req; import com.tencent.bk.job.manage.model.esb.v3.response.EsbScriptV3DTO; import com.tencent.bk.job.manage.model.esb.v3.response.EsbScriptVersionDetailV3DTO; import org.springframework.validation.annotation.Validated; @@ -79,6 +89,8 @@ EsbResp getPublicScriptVersionDetail( @PostMapping("/get_public_script_list") EsbResp> getPublicScriptListUsingPost( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbGetPublicScriptListV3Request request @@ -86,6 +98,8 @@ EsbResp> getPublicScriptListUsingPost( @PostMapping("/get_public_script_version_list") EsbResp> getPublicScriptVersionListUsingPost( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbGetPublicScriptVersionListV3Request request @@ -93,9 +107,82 @@ EsbResp> getPublicScriptVersionListUs @PostMapping("/get_public_script_version_detail") EsbResp getPublicScriptVersionDetailUsingPost( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbGetPublicScriptVersionDetailV3Request request ); + @PostMapping("/create_public_script") + EsbResp createPublicScript( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, + @RequestBody + @Validated(Create.class) + EsbCreatePublicScriptV3Req request + ); + + @PostMapping("/create_public_script_version") + EsbResp createPublicScriptVersion( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, + @RequestBody + @Validated(Create.class) + EsbCreatePublicScriptVersionV3Req request + ); + + @PostMapping("/delete_public_script") + EsbResp deletePublicScript( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, + @RequestBody + @Validated(Delete.class) + EsbDeletePublicScriptV3Req request + ); + + @PostMapping("/delete_public_script_version") + EsbResp deletePublicScriptVersion( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, + @RequestBody + @Validated(Delete.class) + EsbDeletePublicScriptVersionV3Req request + ); + + @PostMapping("/disable_public_script_version") + EsbResp disablePublicScriptVersion( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, + @RequestBody + @Validated(Update.class) + EsbManagePublicScriptVersionV3Req request + ); + + @PostMapping("/publish_public_script_version") + EsbResp publishPublicScriptVersion( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, + @RequestBody + @Validated(Update.class) + EsbManagePublicScriptVersionV3Req request + ); + + @PostMapping("/update_public_script_basic") + EsbResp updatePublicScriptBasic( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, + @RequestBody + @Validated(Update.class) + EsbUpdatePublicScriptBasicV3Req request + ); + + @PostMapping("/update_public_script_version") + EsbResp updatePublicScriptVersion( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, + @RequestBody + @Validated(Update.class) + EsbUpdatePublicScriptVersionV3Req request + ); } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/v3/EsbScriptV3Resource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/v3/EsbScriptV3Resource.java index 84e8816287..eb707eecbd 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/v3/EsbScriptV3Resource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/v3/EsbScriptV3Resource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,9 +28,21 @@ import com.tencent.bk.job.common.constant.JobCommonHeaders; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.common.esb.model.job.v3.EsbPageDataV3; +import com.tencent.bk.job.common.validation.Create; +import com.tencent.bk.job.common.validation.Delete; +import com.tencent.bk.job.common.validation.Update; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbCheckScriptV3Req; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbCreateScriptV3Req; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbCreateScriptVersionV3Req; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbDeleteScriptV3Req; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbDeleteScriptVersionV3Req; import com.tencent.bk.job.manage.model.esb.v3.request.EsbGetScriptListV3Req; import com.tencent.bk.job.manage.model.esb.v3.request.EsbGetScriptVersionDetailV3Req; import com.tencent.bk.job.manage.model.esb.v3.request.EsbGetScriptVersionListV3Req; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbManageScriptVersionV3Req; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbUpdateScriptBasicV3Req; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbUpdateScriptVersionV3Req; +import com.tencent.bk.job.manage.model.esb.v3.response.EsbCheckScriptV3DTO; import com.tencent.bk.job.manage.model.esb.v3.response.EsbScriptV3DTO; import com.tencent.bk.job.manage.model.esb.v3.response.EsbScriptVersionDetailV3DTO; import org.springframework.validation.annotation.Validated; @@ -42,6 +54,8 @@ import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import java.util.List; + /** * 脚本相关API-V3 */ @@ -88,6 +102,8 @@ EsbResp getScriptVersionDetail( @PostMapping("/get_script_list") EsbResp> getScriptListUsingPost( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbGetScriptListV3Req request @@ -95,6 +111,8 @@ EsbResp> getScriptListUsingPost( @PostMapping("/get_script_version_list") EsbResp> getScriptVersionListUsingPost( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbGetScriptVersionListV3Req request @@ -102,9 +120,89 @@ EsbResp> getScriptVersionListUsingPos @PostMapping("/get_script_version_detail") EsbResp getScriptVersionDetailUsingPost( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbGetScriptVersionDetailV3Req request ); + @PostMapping("/create_script") + EsbResp createScript( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, + @RequestBody + @Validated(Create.class) + EsbCreateScriptV3Req request + ); + + @PostMapping("/create_script_version") + EsbResp createScriptVersion( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, + @RequestBody + @Validated(Create.class) + EsbCreateScriptVersionV3Req request + ); + + @PostMapping("/delete_script") + EsbResp deleteScript( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, + @RequestBody + @Validated(Delete.class) + EsbDeleteScriptV3Req request + ); + + @PostMapping("/delete_script_version") + EsbResp deleteScriptVersion( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, + @RequestBody + @Validated(Delete.class) + EsbDeleteScriptVersionV3Req request + ); + + @PostMapping("/disable_script_version") + EsbResp disableScriptVersion( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, + @RequestBody + @Validated(Update.class) + EsbManageScriptVersionV3Req request + ); + + @PostMapping("/publish_script_version") + EsbResp publishScriptVersion( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, + @RequestBody + @Validated(Update.class) + EsbManageScriptVersionV3Req request + ); + + @PostMapping("/update_script_basic") + EsbResp updateScriptBasic( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, + @RequestBody + @Validated(Update.class) + EsbUpdateScriptBasicV3Req request + ); + + @PostMapping("/update_script_version") + EsbResp updateScriptVersion( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, + @RequestBody + @Validated(Update.class) + EsbUpdateScriptVersionV3Req request + ); + + @PostMapping("/check_script") + EsbResp> checkScript( + @RequestBody + @Validated + EsbCheckScriptV3Req request + ); } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/v3/EsbServiceInfoV3Resource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/v3/EsbServiceInfoV3Resource.java new file mode 100644 index 0000000000..a4f360e806 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/v3/EsbServiceInfoV3Resource.java @@ -0,0 +1,56 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.api.esb.v3; + +import com.tencent.bk.job.common.annotation.EsbAPI; +import com.tencent.bk.job.common.constant.JobCommonHeaders; +import com.tencent.bk.job.common.esb.model.EsbResp; +import com.tencent.bk.job.manage.model.esb.v3.response.EsbServiceVersionV3DTO; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +/** + * 服务信息API-V3 + */ +@RequestMapping("/esb/api/v3") +@RestController +@EsbAPI +public interface EsbServiceInfoV3Resource { + + @GetMapping("/get_latest_service_version") + EsbResp getLatestServiceVersion( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode + ); + + @PostMapping("/get_latest_service_version") + EsbResp getLatestServiceVersionUsingPost( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode + ); +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/v3/EsbTemplateV3Resource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/v3/EsbTemplateV3Resource.java index 2db7754162..292fdd54ae 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/v3/EsbTemplateV3Resource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/v3/EsbTemplateV3Resource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -66,6 +66,8 @@ EsbResp> getTemplateList( @PostMapping("/get_job_template_list") EsbResp> getTemplateListUsingPost( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, @RequestBody @Validated EsbGetTemplateListV3Request request diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/IamAccountCallbackResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/IamAccountCallbackResource.java index 8c04ecdad7..e01f9963ff 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/IamAccountCallbackResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/IamAccountCallbackResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/IamPublicScriptCallbackResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/IamPublicScriptCallbackResource.java index aa647ddb39..80eb048906 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/IamPublicScriptCallbackResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/IamPublicScriptCallbackResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/IamScriptCallbackResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/IamScriptCallbackResource.java index 4079143f56..a7734c3c4a 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/IamScriptCallbackResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/IamScriptCallbackResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/IamTagCallbackResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/IamTagCallbackResource.java index cd29e4eaac..b42914e882 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/IamTagCallbackResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/IamTagCallbackResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/IamTaskPlanCallbackResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/IamTaskPlanCallbackResource.java index 97698e893f..0346abe82b 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/IamTaskPlanCallbackResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/IamTaskPlanCallbackResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/IamTaskTemplateCallbackResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/IamTaskTemplateCallbackResource.java index 770318aeaf..7e4f1217d6 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/IamTaskTemplateCallbackResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/IamTaskTemplateCallbackResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/IamTicketCallbackResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/IamTicketCallbackResource.java index db6834ce65..8af6fe3e3c 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/IamTicketCallbackResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/IamTicketCallbackResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceAccountResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceAccountResource.java index f6d3cb11fa..44a1158725 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceAccountResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceAccountResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -29,6 +29,7 @@ import com.tencent.bk.job.common.model.Response; import com.tencent.bk.job.manage.model.inner.ServiceAccountDTO; import com.tencent.bk.job.manage.model.web.request.AccountCreateUpdateReq; +import com.tentent.bk.job.common.api.feign.annotation.SmartFeignClient; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; @@ -37,18 +38,15 @@ import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestHeader; -import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; import java.util.List; /** * 账号服务 */ -@RequestMapping("/service/account") @Api(tags = {"job-manage:service:Account_Management"}) -@RestController +@SmartFeignClient(value = "job-manage", contextId = "accountResource") @InternalAPI public interface ServiceAccountResource { /** @@ -57,7 +55,7 @@ public interface ServiceAccountResource { * @param accountId * @return */ - @GetMapping("/{accountId}") + @GetMapping("/service/account/{accountId}") @ApiOperation(value = "根据账号id获取账号信息", produces = "application/json") InternalResponse getAccountByAccountId(@ApiParam(value = "账号ID", required = true) @PathVariable("accountId") Long accountId); @@ -69,7 +67,7 @@ public interface ServiceAccountResource { * @param account * @return */ - @GetMapping("/app/{appId}/accounts/{account}") + @GetMapping("/service/account/app/{appId}/accounts/{account}") @ApiOperation(value = "根据账号名获取账号信息", produces = "application/json") InternalResponse getAccountByAccountName( @@ -85,14 +83,14 @@ public interface ServiceAccountResource { * @param alias * @return */ - @GetMapping("/app/{appId}/category/{category}/alias/{alias}") + @GetMapping("/service/account/app/{appId}/category/{category}/alias/{alias}") @ApiOperation(value = "根据账号别名获取业务下的账号信息", produces = "application/json") InternalResponse getAccountByCategoryAndAliasInApp( @ApiParam(value = "业务ID", required = true) @PathVariable("appId") Long appId, @ApiParam(value = "账号用途，1-系统账号，2-DB账号", required = true) @PathVariable("category") Integer category, @ApiParam(value = "账号名称", required = true) @PathVariable("alias") String alias); - @PostMapping("/app/{appId}/saveOrGetAccount") + @PostMapping("/service/account/app/{appId}/saveOrGetAccount") InternalResponse saveOrGetAccount( @ApiParam("用户名，网关自动传入") @RequestHeader("username") String username, @ApiParam("创建时间") @RequestHeader(value = "X-Create-Time", required = false) Long createTime, @@ -101,7 +99,7 @@ InternalResponse saveOrGetAccount( @PathVariable("appId") Long appId, @RequestBody AccountCreateUpdateReq accountCreateUpdateReq); @ApiOperation(value = "新增账号", produces = "application/json") - @PostMapping(value = "/app/{appId}/account") + @PostMapping(value = "/service/account/app/{appId}/account") Response saveAccount( @ApiParam(value = "用户名，网关自动传入", required = true) @RequestHeader("username") @@ -114,7 +112,7 @@ Response saveAccount( ); @ApiOperation(value = "获取业务下的账号列表，返回简单的账号信息", produces = "application/json") - @GetMapping("/account/app/{appId}/accounts") + @GetMapping("/service/account/account/app/{appId}/accounts") Response> listAccounts( @ApiParam(value = "业务ID", required = true) @PathVariable("appId") diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceApplicationResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceApplicationResource.java index a44508eaf1..4a713dae5d 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceApplicationResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceApplicationResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,6 +28,7 @@ import com.tencent.bk.job.common.model.InternalResponse; import com.tencent.bk.job.manage.model.inner.ServiceAppBaseInfoDTO; import com.tencent.bk.job.manage.model.inner.resp.ServiceApplicationDTO; +import com.tentent.bk.job.common.api.feign.annotation.SmartFeignClient; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; @@ -35,13 +36,11 @@ import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; import java.util.List; -@RequestMapping("/service") @Api(tags = {"job-manage:service:App_Management"}) -@RestController +@SmartFeignClient(value = "job-manage", contextId = "applicationResource") @InternalAPI public interface ServiceApplicationResource { /** @@ -49,7 +48,7 @@ public interface ServiceApplicationResource { * * @return 业务列表 */ - @RequestMapping("/app/list/normal") + @RequestMapping("/service/app/list/normal") InternalResponse> listNormalApps(); /** @@ -57,7 +56,7 @@ public interface ServiceApplicationResource { * * @return 业务集、全业务列表 */ - @RequestMapping("/app/list/bizSet") + @RequestMapping("/service/app/list/bizSet") InternalResponse> listBizSetApps(); /** @@ -68,7 +67,7 @@ public interface ServiceApplicationResource { * @return 业务 */ @ApiOperation("根据Job业务id查询业务") - @RequestMapping("/app/{appId}") + @RequestMapping("/service/app/{appId}") ServiceApplicationDTO queryAppById(@ApiParam(value = "Job业务ID", required = true) @PathVariable("appId") Long appId); @@ -79,7 +78,7 @@ ServiceApplicationDTO queryAppById(@ApiParam(value = "Job业务ID", required = t * @return 业务列表 */ @ApiOperation("根据Job业务id批量查询业务") - @RequestMapping("/apps") + @RequestMapping("/service/apps") List listAppsByAppIds(@ApiParam(value = "业务ID列表，英文逗号分隔", required = true) @RequestParam("appIds") String appIds); @@ -91,15 +90,24 @@ List listAppsByAppIds(@ApiParam(value = "业务ID列表 * @return 业务 */ @ApiOperation("根据资源范围查询业务") - @RequestMapping("/app/scope/{scopeType}/{scopeId}") + @RequestMapping("/service/app/scope/{scopeType}/{scopeId}") ServiceApplicationDTO queryAppByScope(@ApiParam(value = "资源范围类型", allowableValues = "1-业务,2-业务集", required = true) @PathVariable("scopeType") String scopeType, @ApiParam(value = "资源范围ID", required = true) @PathVariable("scopeId") String scopeId); @ApiOperation(value = "获取业务列表", produces = "application/json") - @GetMapping("/app/list") + @GetMapping("/service/app/list") InternalResponse> listApps( @ApiParam(value = "资源范围类型", allowableValues = "1-业务,2-业务集") @RequestParam(value = "scopeType", required = false) String scopeType); + + @ApiOperation(value = "获取所有已归档的业务(集)id", produces = "application/json") + @GetMapping("/service/app/listArchived") + InternalResponse> listAllAppIdOfArchivedScope(); + + @ApiOperation("根据Job业务id查询业务是否存在") + @RequestMapping("/service/app/exists/{appId}") + InternalResponse existsAppById(@ApiParam(value = "Job业务ID", required = true) + @PathVariable("appId") Long appId); } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceBackupTmpResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceBackupTmpResource.java index 9032eb6108..f80cdfdd54 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceBackupTmpResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceBackupTmpResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,14 +28,13 @@ import com.tencent.bk.job.common.model.Response; import com.tencent.bk.job.manage.model.web.vo.task.TaskPlanVO; import com.tencent.bk.job.manage.model.web.vo.task.TaskTemplateVO; +import com.tentent.bk.job.common.api.feign.annotation.SmartFeignClient; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestHeader; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; import java.util.List; @@ -43,13 +42,12 @@ * TODO 后续需要优化 * 临时提供给job-backup的API */ -@RequestMapping("/service/tmp") @Api(tags = {"job-manage:service:tmp_for_backup"}) -@RestController +@SmartFeignClient(value = "job-manage", contextId = "backupResource") @InternalAPI public interface ServiceBackupTmpResource { @ApiOperation(value = "根据模版 ID 获取模版信息", produces = "application/json") - @GetMapping("/app/{appId}/template/{templateId}") + @GetMapping("/service/tmp/app/{appId}/template/{templateId}") Response getTemplateById( @RequestHeader("username") String username, @@ -60,7 +58,7 @@ Response getTemplateById( ); @ApiOperation(value = "根据执行方案 ID 获取执行方案信息", produces = "application/json") - @GetMapping("/app/{appId}/task/plan/{templateId}/{planId}") + @GetMapping("/service/tmp/app/{appId}/task/plan/{templateId}/{planId}") Response getPlanById( @ApiParam(value = "用户名，网关自动传入") @RequestHeader("username") @@ -76,7 +74,7 @@ Response getPlanById( ); @ApiOperation(value = "获取执行方案基本信息列表", produces = "application/json") - @GetMapping("/app/{appId}/task/plan/{templateId}") + @GetMapping("/service/tmp/app/{appId}/task/plan/{templateId}") Response> listPlans( @ApiParam(value = "用户名，网关自动传入") @RequestHeader("username") diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceCheckScriptResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceCheckScriptResource.java index f376e7b3fe..6ead1da0ad 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceCheckScriptResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceCheckScriptResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,24 +28,22 @@ import com.tencent.bk.job.common.model.InternalResponse; import com.tencent.bk.job.manage.model.inner.ServiceScriptCheckResultItemDTO; import com.tencent.bk.job.manage.model.inner.request.ServiceCheckScriptRequest; +import com.tentent.bk.job.common.api.feign.annotation.SmartFeignClient; import io.swagger.annotations.Api; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; import java.util.List; /** * 脚本检查服务 */ -@RequestMapping("/service/script/check") @Api(tags = {"job-manage:service:Script_Check"}) -@RestController +@SmartFeignClient(value = "job-manage", contextId = "checkScriptResource") @InternalAPI public interface ServiceCheckScriptResource { - @PostMapping + @PostMapping("/service/script/check") InternalResponse> check( @RequestBody ServiceCheckScriptRequest checkScriptRequest); } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceCredentialResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceCredentialResource.java index 7d03ccaef7..686f6065cb 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceCredentialResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceCredentialResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,60 +26,26 @@ import com.tencent.bk.job.common.annotation.InternalAPI; import com.tencent.bk.job.common.model.InternalResponse; -import com.tencent.bk.job.manage.model.inner.resp.ServiceBasicCredentialDTO; import com.tencent.bk.job.manage.model.inner.resp.ServiceCredentialDTO; -import com.tencent.bk.job.manage.model.web.request.CredentialCreateUpdateReq; +import com.tentent.bk.job.common.api.feign.annotation.SmartFeignClient; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.PutMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestHeader; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; @Api(tags = {"job-manage:service:Credential"}) -@RequestMapping("/service/credentials/app/{appId}") -@RestController +@SmartFeignClient(value = "job-manage", contextId = "credentialResource") @InternalAPI public interface ServiceCredentialResource { - @ApiOperation(value = "获取凭据详情", produces = "application/json") - @GetMapping("/ids/{id}") + @ApiOperation(value = "获取凭证详情", produces = "application/json") + @GetMapping("/service/credentials/app/{appId}/ids/{id}") InternalResponse getCredentialById( @ApiParam(value = "业务ID", required = true) @PathVariable("appId") Long appId, - @ApiParam("凭据ID") + @ApiParam("凭证ID") @PathVariable("id") String id ); - - @ApiOperation(value = "新建凭据", produces = "application/json") - @PostMapping("/create") - InternalResponse createCredential( - @ApiParam(value = "用户名，网关自动传入", required = true) - @RequestHeader("username") - String username, - @ApiParam(value = "业务ID", required = true) - @PathVariable("appId") Long appId, - @ApiParam(value = "创建或更新请求体", required = true) - @RequestBody - CredentialCreateUpdateReq createUpdateReq - ); - - @ApiOperation(value = "更新凭据", produces = "application/json") - @PutMapping("/update") - InternalResponse updateCredential( - @ApiParam(value = "用户名，网关自动传入", required = true) - @RequestHeader("username") - String username, - @ApiParam(value = "业务ID", required = true) - @PathVariable("appId") Long appId, - @ApiParam(value = "创建或更新请求体", required = true) - @RequestBody - CredentialCreateUpdateReq createUpdateReq - ); } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceCrontabCallbackResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceCrontabCallbackResource.java index eb47f01f89..ae62c4bea3 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceCrontabCallbackResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceCrontabCallbackResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -29,19 +29,17 @@ import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; @Api(tags = {"job-manage:service:CrontabCallback"}) -@RequestMapping("/service/callback/crontab") @RestController @InternalAPI public interface ServiceCrontabCallbackResource { @ApiOperation(value = "同步业务", produces = "application/json") - @GetMapping("/syncApp") + @GetMapping("/service/callback/crontab/syncApp") InternalResponse syncApp(); @ApiOperation(value = "同步主机", produces = "application/json") - @GetMapping("/syncHost") + @GetMapping("/service/callback/crontab/syncHost") InternalResponse syncHost(); } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceGlobalSettingsResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceGlobalSettingsResource.java index 94fb128bf1..4a1546e40c 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceGlobalSettingsResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceGlobalSettingsResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,23 +27,32 @@ import com.tencent.bk.job.common.annotation.InternalAPI; import com.tencent.bk.job.common.model.InternalResponse; import com.tencent.bk.job.manage.model.inner.ServiceFileUploadSettingDTO; +import com.tentent.bk.job.common.api.feign.annotation.SmartFeignClient; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; @Api(tags = {"job-manage:service:GlobalSettings"}) -@RequestMapping("/service/globalSettings") -@RestController +@SmartFeignClient(value = "job-manage", contextId = "globalSettingsResource") @InternalAPI public interface ServiceGlobalSettingsResource { + /** + * 获取文档中心基础Url + * Deprecated：3.9.x中不推荐使用，3.10.x中下线，请使用getDocJobRootUrl + * + * @return 文档中心基础Url + */ + @Deprecated @ApiOperation(value = "获取文档中心基础Url", produces = "application/json") - @GetMapping("/docCenterBaseUrl") + @GetMapping("/service/globalSettings/docCenterBaseUrl") InternalResponse getDocCenterBaseUrl(); + @ApiOperation(value = "获取文档中心Job文档基础Url", produces = "application/json") + @GetMapping("/service/globalSettings/docJobRootUrl") + InternalResponse getDocJobRootUrl(); + @ApiOperation(value = "获取文件上传设置", produces = "application/json") - @GetMapping("/fileUploadSettings") + @GetMapping("/service/globalSettings/fileUploadSettings") InternalResponse getFileUploadSettings(); } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceHostResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceHostResource.java index eb527ba3e2..873f1cfa4a 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceHostResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceHostResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -29,59 +29,60 @@ import com.tencent.bk.job.manage.model.inner.ServiceHostDTO; import com.tencent.bk.job.manage.model.inner.ServiceHostStatusDTO; import com.tencent.bk.job.manage.model.inner.ServiceListAppHostResultDTO; +import com.tencent.bk.job.manage.model.inner.request.ServiceBatchGetAppHostsReq; +import com.tencent.bk.job.manage.model.inner.request.ServiceBatchGetHostToposReq; import com.tencent.bk.job.manage.model.inner.request.ServiceBatchGetHostsReq; -import com.tencent.bk.job.manage.model.inner.request.ServiceCheckAppHostsReq; import com.tencent.bk.job.manage.model.inner.request.ServiceGetHostStatusByDynamicGroupReq; -import com.tencent.bk.job.manage.model.inner.request.ServiceGetHostStatusByIpReq; +import com.tencent.bk.job.manage.model.inner.request.ServiceGetHostStatusByHostReq; import com.tencent.bk.job.manage.model.inner.request.ServiceGetHostStatusByNodeReq; +import com.tencent.bk.job.manage.model.inner.request.ServiceGetHostsByCloudIpv6Req; +import com.tencent.bk.job.manage.model.inner.resp.ServiceHostTopoDTO; +import com.tentent.bk.job.common.api.feign.annotation.SmartFeignClient; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; import java.util.List; -@RequestMapping("/service") @Api(tags = {"job-manage:service:Host_Management"}) -@RestController +@SmartFeignClient(value = "job-manage", contextId = "hostResource") @InternalAPI public interface ServiceHostResource { @ApiOperation(value = "查询节点下的主机状态", produces = "application/json") - @PostMapping("/app/{appId}/host/status/nodes") + @PostMapping("/service/app/{appId}/host/status/nodes") InternalResponse> getHostStatusByNode( @PathVariable("appId") Long appId, @RequestBody ServiceGetHostStatusByNodeReq req ); @ApiOperation(value = "查询动态分组下的主机状态", produces = "application/json") - @PostMapping("/app/{appId}/host/status/dynamicGroups") + @PostMapping("/service/app/{appId}/host/status/dynamicGroups") InternalResponse> getHostStatusByDynamicGroup( @PathVariable("appId") Long appId, @RequestBody ServiceGetHostStatusByDynamicGroupReq req ); - @ApiOperation(value = "查询IP对应的主机状态", produces = "application/json") - @PostMapping("/app/{appId}/host/status/ips") - InternalResponse> getHostStatusByIp( + @ApiOperation(value = "查询主机对应的主机状态", produces = "application/json") + @PostMapping("/service/app/{appId}/host/status/hosts") + InternalResponse> getHostStatusByHost( @PathVariable("appId") Long appId, - @RequestBody ServiceGetHostStatusByIpReq req + @RequestBody ServiceGetHostStatusByHostReq req ); /** - * 获取业务下的主机并返回主机详情 + * 查询业务下的主机 * * @param appId Job业务ID * @param req 请求 */ - @ApiOperation(value = "检查主机是否在业务下", produces = "application/json") - @PostMapping("/app/{appId}/host/batchGet") + @ApiOperation(value = "查询业务下的主机", produces = "application/json") + @PostMapping("/service/app/{appId}/host/batchGet") InternalResponse batchGetAppHosts( @PathVariable("appId") Long appId, - @RequestBody ServiceCheckAppHostsReq req + @RequestBody ServiceBatchGetAppHostsReq req ); /** @@ -91,8 +92,32 @@ InternalResponse batchGetAppHosts( * @return 主机信息 */ @ApiOperation(value = "检查主机是否在业务下", produces = "application/json") - @PostMapping("/hosts/batchGet") + @PostMapping("/service/hosts/batchGet") InternalResponse> batchGetHosts( @RequestBody - ServiceBatchGetHostsReq req); + ServiceBatchGetHostsReq req); + + /** + * 通过云区域ID与Ipv6地址获取主机信息 + * + * @param req 请求 + * @return 主机信息 + */ + @ApiOperation(value = "通过云区域ID与Ipv6地址查询主机信息", produces = "application/json") + @PostMapping("/service/hosts/getByCloudIpv6") + InternalResponse> getHostsByCloudIpv6( + @RequestBody + ServiceGetHostsByCloudIpv6Req req); + + /** + * 批量获取主机拓扑信息 + * + * @param req 请求 + * @return 主机拓扑信息 + */ + @ApiOperation(value = "批量获取主机拓扑信息", produces = "application/json") + @PostMapping("/service/hostTopos/batchGet") + InternalResponse> batchGetHostTopos( + @RequestBody + ServiceBatchGetHostToposReq req); } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceMetricsResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceMetricsResource.java index 6e21f6dfff..7d5b878458 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceMetricsResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceMetricsResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,29 +26,29 @@ import com.tencent.bk.job.common.annotation.InternalAPI; import com.tencent.bk.job.common.model.InternalResponse; -import com.tencent.bk.job.manage.common.consts.JobResourceStatusEnum; -import com.tencent.bk.job.manage.common.consts.account.AccountTypeEnum; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskFileTypeEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskScriptSourceEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskStepTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.JobResourceStatusEnum; +import com.tencent.bk.job.manage.api.common.constants.account.AccountTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskFileTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskScriptSourceEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskStepTypeEnum; +import com.tentent.bk.job.common.api.feign.annotation.SmartFeignClient; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RequestHeader; -import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; + +import java.util.Map; @Api(tags = {"job-manage:service:Metrics"}) -@RequestMapping("/service/metrics") -@RestController +@SmartFeignClient(value = "job-manage", contextId = "manageMetricsResource") @InternalAPI public interface ServiceMetricsResource { @ApiOperation(value = "接入业务总量", produces = "application/json") - @GetMapping("/apps/count") + @GetMapping("/service/metrics/apps/count") InternalResponse countApps( @ApiParam("用户名，网关自动传入") @RequestHeader("username") @@ -56,7 +56,7 @@ InternalResponse countApps( ); @ApiOperation(value = "作业模板量", produces = "application/json") - @GetMapping("/templates/count") + @GetMapping("/service/metrics/templates/count") InternalResponse countTemplates( @ApiParam(value = "业务Id") @RequestParam(value = "appId", required = false) @@ -64,7 +64,7 @@ InternalResponse countTemplates( ); @ApiOperation(value = "执行方案量", produces = "application/json") - @GetMapping("/taskPlans/count") + @GetMapping("/service/metrics/taskPlans/count") InternalResponse countTaskPlans( @ApiParam(value = "业务Id") @RequestParam(value = "appId", required = false) @@ -72,7 +72,7 @@ InternalResponse countTaskPlans( ); @ApiOperation(value = "作业模板的步骤量", produces = "application/json") - @GetMapping("/templates/step/count") + @GetMapping("/service/metrics/templates/step/count") InternalResponse countTemplateSteps( @ApiParam(value = "业务Id") @RequestParam(value = "appId", required = false) @@ -89,7 +89,7 @@ InternalResponse countTemplateSteps( ); @ApiOperation(value = "脚本总量", produces = "application/json") - @GetMapping("/script/count") + @GetMapping("/service/metrics/script/count") InternalResponse countScripts( @ApiParam(value = "业务Id") @RequestParam(value = "appId", required = false) @@ -103,7 +103,7 @@ InternalResponse countScripts( ); @ApiOperation(value = "被引用的脚本总量", produces = "application/json") - @GetMapping("/script/cited/count") + @GetMapping("/service/metrics/script/cited/count") InternalResponse countCiteScripts( @ApiParam(value = "业务Id") @RequestParam(value = "appId", required = false) @@ -111,7 +111,7 @@ InternalResponse countCiteScripts( ); @ApiOperation(value = "引用脚本的步骤总量", produces = "application/json") - @GetMapping("/step/citeScript/count") + @GetMapping("/service/metrics/step/citeScript/count") InternalResponse countCiteScriptSteps( @ApiParam(value = "业务Id") @RequestParam(value = "appId", required = false) @@ -119,7 +119,7 @@ InternalResponse countCiteScriptSteps( ); @ApiOperation(value = "脚本版本总量", produces = "application/json") - @GetMapping("/scriptVersions/count") + @GetMapping("/service/metrics/scriptVersions/count") InternalResponse countScriptVersions( @ApiParam(value = "业务Id") @RequestParam(value = "appId", required = false) @@ -133,7 +133,7 @@ InternalResponse countScriptVersions( ); @ApiOperation(value = "账号总量", produces = "application/json") - @GetMapping("/accounts/count") + @GetMapping("/service/metrics/accounts/count") InternalResponse countAccounts( @ApiParam("账号类型") @RequestParam(value = "accountType", required = false) @@ -141,15 +141,19 @@ InternalResponse countAccounts( ); @ApiOperation(value = "主机总量", produces = "application/json") - @GetMapping("/hosts/count") + @GetMapping("/service/metrics/hosts/count") InternalResponse countHostsByOsType( @ApiParam("系统类型") @RequestParam(value = "osType", required = false) String osType ); + @ApiOperation(value = "主机的操作系统类型分布数据", produces = "application/json") + @GetMapping("/service/metrics/hosts/groupByOsType") + InternalResponse> groupHostByOsType(); + @ApiOperation(value = "某个标签在某业务下的被引数量", produces = "application/json") - @GetMapping("/tags/citedCount") + @GetMapping("/service/metrics/tags/citedCount") InternalResponse tagCitedCount( @ApiParam(value = "业务Id") @RequestParam(value = "appId", required = false) diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceNotificationResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceNotificationResource.java index a19216e6c1..ad9036549d 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceNotificationResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceNotificationResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,68 +26,117 @@ import com.tencent.bk.job.common.annotation.InternalAPI; import com.tencent.bk.job.common.model.InternalResponse; +import com.tencent.bk.job.common.model.dto.notify.CustomNotifyDTO; +import com.tencent.bk.job.manage.model.inner.ServiceSpecificResourceNotifyPolicyDTO; import com.tencent.bk.job.manage.model.inner.ServiceAppRoleDTO; import com.tencent.bk.job.manage.model.inner.ServiceNotificationMessage; import com.tencent.bk.job.manage.model.inner.ServiceNotifyChannelDTO; import com.tencent.bk.job.manage.model.inner.ServiceTemplateNotificationDTO; import com.tencent.bk.job.manage.model.inner.ServiceTriggerTemplateNotificationDTO; import com.tencent.bk.job.manage.model.inner.ServiceUserNotificationDTO; +import com.tentent.bk.job.common.api.feign.annotation.SmartFeignClient; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; +import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestHeader; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; import java.util.List; @Api(tags = {"job-manage:service:Notification"}) -@RequestMapping("/service/notification") -@RestController +@SmartFeignClient(value = "job-manage", contextId = "notificationResource") @InternalAPI public interface ServiceNotificationResource { @ApiOperation(value = "发送通知给用户（渠道在配置文件中配置，默认所有渠道）", produces = "application/json") - @PostMapping("/sendNotificationsToUsers") + @PostMapping("/service/notification/sendNotificationsToUsers") InternalResponse sendNotificationsToUsers( @ApiParam("通知接受者与消息内容") @RequestBody ServiceUserNotificationDTO serviceUserNotificationDTO ); @ApiOperation(value = "发送通知给管理员（渠道在配置文件中配置，默认所有渠道）", produces = "application/json") - @PostMapping("/sendNotificationsToAdministrators") + @PostMapping("/service/notification/sendNotificationsToAdministrators") InternalResponse sendNotificationsToAdministrators( @ApiParam("消息内容") @RequestBody ServiceNotificationMessage serviceNotificationMessage ); @ApiOperation(value = "触发模板消息通知", produces = "application/json") - @PostMapping("/triggerTemplateNotification") + @PostMapping("/service/notification/triggerTemplateNotification") InternalResponse triggerTemplateNotification( @RequestBody ServiceTriggerTemplateNotificationDTO triggerTemplateNotification ); @ApiOperation(value = "根据模板发送消息通知", produces = "application/json") - @PostMapping("/sendTemplateNotification") + @PostMapping("/service/notification/sendTemplateNotification") InternalResponse sendTemplateNotification( @ApiParam("根据模板发送消息通知") @RequestBody ServiceTemplateNotificationDTO templateNotificationDTO ); @ApiOperation(value = "获取通知角色列表", produces = "application/json") - @GetMapping("/getNotifyRoles") + @GetMapping("/service/notification/getNotifyRoles") InternalResponse> getNotifyRoles( @ApiParam("语言") @RequestHeader("lang") String lang ); @ApiOperation(value = "获取通知渠道", produces = "application/json") - @GetMapping("/getNotifyChannels") + @GetMapping("/service/notification/getNotifyChannels") InternalResponse> getNotifyChannels( @ApiParam("语言") @RequestHeader("lang") String lang ); + + @ApiOperation(value = "创建或更新特定资源消息通知策略", produces = "application/json") + @PostMapping("/service/notification/notifyPolicy") + InternalResponse createOrUpdateSpecificResourceNotifyPolicy( + @ApiParam("操作人") + @RequestHeader("username") + String username, + @ApiParam("业务id") + @RequestHeader("appId") + Long appId, + @ApiParam("消息通知策略") + @RequestBody + ServiceSpecificResourceNotifyPolicyDTO serviceNotifyPolicyDTO + ); + + @ApiOperation(value = "删除特定资源通知策略", produces = "application/json") + @DeleteMapping("/service/notification/resourceNotifyPolicy/resourceType/{resourceType}/resourceId/{resourceId}") + InternalResponse deleteSpecificResourceNotifyPolicy( + @ApiParam("业务id") + @RequestHeader("appId") + Long appId, + @ApiParam("资源类型") + @PathVariable("resourceType") + Integer resourceType, + @ApiParam("资源id") + @PathVariable("resourceId") + String resourceId + ); + + @ApiOperation(value = "获取特定资源通知策略", produces = "application/json") + @GetMapping("/service/notification/resourceNotifyPolicy/resourceType/{resourceType}/resourceId/{resourceId}" + + "/triggerType/{triggerType}") + InternalResponse getSpecificResourceNotifyPolicy( + @ApiParam("业务id") + @RequestHeader("appId") + Long appId, + @ApiParam("资源类型") + @PathVariable("resourceType") + Integer resourceType, + @ApiParam("资源id") + @PathVariable("resourceId") + String resourceId, + @ApiParam("触发类型") + @PathVariable("triggerType") + Integer triggerType + ); + } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceScriptResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceScriptResource.java index 1827b32c65..8bbdd9fa42 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceScriptResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceScriptResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,7 +27,8 @@ import com.tencent.bk.job.common.annotation.InternalAPI; import com.tencent.bk.job.common.model.InternalResponse; import com.tencent.bk.job.manage.model.inner.ServiceScriptDTO; -import com.tencent.bk.job.manage.model.web.request.ScriptCreateUpdateReq; +import com.tencent.bk.job.manage.model.web.request.ScriptCreateReq; +import com.tentent.bk.job.common.api.feign.annotation.SmartFeignClient; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; @@ -37,16 +38,13 @@ import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestHeader; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; @Api(tags = {"job-manage:service:Script_Management"}) -@RequestMapping("/service/script") -@RestController +@SmartFeignClient(value = "job-manage", contextId = "scriptResource") @InternalAPI public interface ServiceScriptResource { @ApiOperation(value = "根据业务ID、脚本版本ID获取脚本", produces = "application/json") - @GetMapping("/app/{appId}/scriptVersion/{scriptVersionId}") + @GetMapping("/service/script/app/{appId}/scriptVersion/{scriptVersionId}") InternalResponse getScriptByAppIdAndScriptVersionId(@RequestHeader("username") String username, @PathVariable("appId") @@ -55,11 +53,11 @@ InternalResponse getScriptByAppIdAndScriptVersionId(@RequestHe Long scriptVersionId); @ApiOperation(value = "根据脚本版本ID获取脚本", produces = "application/json") - @GetMapping("/scriptVersion/{scriptVersionId}") + @GetMapping("/service/script/scriptVersion/{scriptVersionId}") InternalResponse getScriptByScriptVersionId(@PathVariable("scriptVersionId") Long scriptVersionId); @ApiOperation(value = "指定Id创建脚本版本", produces = "application/json") - @PostMapping("/app/{appId}/createScriptWithVersionId") + @PostMapping("/service/script/app/{appId}/createScriptWithVersionId") InternalResponse> createScriptWithVersionId( @ApiParam("用户名，网关自动传入") @RequestHeader("username") String username, @ApiParam("创建时间") @RequestHeader(value = "X-Create-Time", required = false) Long createTime, @@ -68,13 +66,13 @@ InternalResponse> createScriptWithVersionId( @ApiParam("脚本状态") @RequestHeader(value = "X-Script-Status", required = false) Integer scriptStatus, @ApiParam(value = "业务ID", required = true, example = "2") @PathVariable("appId") Long appId, @ApiParam(value = "新增/更新的脚本对象", name = "scriptCreateUpdateReq", - required = true) @RequestBody ScriptCreateUpdateReq scriptCreateUpdateReq); + required = true) @RequestBody ScriptCreateReq scriptCreateReq); @ApiOperation(value = "获取脚本基本信息", produces = "application/json") - @GetMapping("/{scriptId}") + @GetMapping("/service/script/{scriptId}") InternalResponse getBasicScriptInfo(@PathVariable("scriptId") String scriptId); @ApiOperation(value = "获取已上线版本", produces = "application/json") - @GetMapping("/scriptVersion/online/{scriptId}") + @GetMapping("/service/script/scriptVersion/online/{scriptId}") InternalResponse getOnlineScriptVersion(@PathVariable("scriptId") String scriptId); } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceScriptTemplateResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceScriptTemplateResource.java new file mode 100644 index 0000000000..0748c0db7a --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceScriptTemplateResource.java @@ -0,0 +1,43 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.api.inner; + +import com.tencent.bk.job.common.annotation.InternalAPI; +import com.tencent.bk.job.common.model.InternalResponse; +import com.tentent.bk.job.common.api.feign.annotation.SmartFeignClient; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestParam; + +@Api(tags = {"job-manage:service:Script_Template_Management"}) +@SmartFeignClient(value = "job-manage", contextId = "scriptTemplateResource") +@InternalAPI +public interface ServiceScriptTemplateResource { + + @ApiOperation(value = "获取脚本模板", produces = "application/json") + @GetMapping("/service/scriptTemplate") + InternalResponse getScriptTemplate(@RequestParam("type") Integer type); +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceSyncResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceSyncResource.java index b0feb5f292..f7fdedc8cc 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceSyncResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceSyncResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,6 +28,7 @@ import com.tencent.bk.job.common.model.InternalResponse; import com.tencent.bk.job.manage.model.inner.ServiceHostInfoDTO; import com.tencent.bk.job.manage.model.inner.resp.ServiceApplicationDTO; +import com.tentent.bk.job.common.api.feign.annotation.SmartFeignClient; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; @@ -35,16 +36,14 @@ import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PutMapping; import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; import java.util.List; /** * 同步业务、主机资源服务 */ -@RequestMapping("/service/sync") @Api(tags = {"job-manage:service:App_Management"}) -@RestController +@SmartFeignClient(value = "job-manage", contextId = "syncResource") @InternalAPI public interface ServiceSyncResource { /** @@ -52,7 +51,7 @@ public interface ServiceSyncResource { * * @return */ - @RequestMapping("/app/list") + @RequestMapping("/service/sync/app/list") List listAllApps(); /** @@ -61,7 +60,7 @@ public interface ServiceSyncResource { * @param appId 业务ID * @return */ - @GetMapping("/host/app/{appId}") + @GetMapping("/service/sync/host/app/{appId}") @ApiOperation(value = "根据业务ID获取主机", produces = "application/json") InternalResponse> getHostByAppId( @ApiParam(value = "业务ID", required = true) @@ -73,7 +72,7 @@ InternalResponse> getHostByAppId( * @param bizId 业务ID * @return */ - @PutMapping("/syncHost/biz/{bizId}") + @PutMapping("/service/sync/syncHost/biz/{bizId}") @ApiOperation(value = "根据业务ID同步主机", produces = "application/json") InternalResponse syncHostByBizId( @ApiParam(value = "业务ID", required = true) @@ -84,7 +83,7 @@ InternalResponse syncHostByBizId( * * @return */ - @PutMapping("/bizWatch/enable") + @PutMapping("/service/sync/bizWatch/enable") @ApiOperation(value = "开启业务事件监听", produces = "application/json") InternalResponse enableBizWatch(); @@ -93,7 +92,7 @@ InternalResponse syncHostByBizId( * * @return */ - @PutMapping("/bizWatch/disable") + @PutMapping("/service/sync/bizWatch/disable") @ApiOperation(value = "关闭业务事件监听", produces = "application/json") InternalResponse disableBizWatch(); @@ -102,7 +101,7 @@ InternalResponse syncHostByBizId( * * @return */ - @PutMapping("/hostWatch/enable") + @PutMapping("/service/sync/hostWatch/enable") @ApiOperation(value = "开启主机事件监听", produces = "application/json") InternalResponse enableHostWatch(); @@ -111,7 +110,7 @@ InternalResponse syncHostByBizId( * * @return */ - @PutMapping("/hostWatch/disable") + @PutMapping("/service/sync/hostWatch/disable") @ApiOperation(value = "关闭主机事件监听", produces = "application/json") InternalResponse disableHostWatch(); @@ -120,7 +119,7 @@ InternalResponse syncHostByBizId( * * @return */ - @PutMapping("/syncApp/enable") + @PutMapping("/service/sync/syncApp/enable") @ApiOperation(value = "开启业务同步", produces = "application/json") InternalResponse enableSyncApp(); @@ -129,7 +128,7 @@ InternalResponse syncHostByBizId( * * @return */ - @PutMapping("/syncApp/disable") + @PutMapping("/service/sync/syncApp/disable") @ApiOperation(value = "关闭业务同步", produces = "application/json") InternalResponse disableSyncApp(); @@ -138,7 +137,7 @@ InternalResponse syncHostByBizId( * * @return */ - @PutMapping("/syncHost/enable") + @PutMapping("/service/sync/syncHost/enable") @ApiOperation(value = "开启主机同步", produces = "application/json") InternalResponse enableSyncHost(); @@ -147,7 +146,7 @@ InternalResponse syncHostByBizId( * * @return */ - @PutMapping("/syncHost/disable") + @PutMapping("/service/sync/syncHost/disable") @ApiOperation(value = "关闭主机同步", produces = "application/json") InternalResponse disableSyncHost(); @@ -156,7 +155,7 @@ InternalResponse syncHostByBizId( * * @return */ - @PutMapping("/syncAgentStatus/enable") + @PutMapping("/service/sync/syncAgentStatus/enable") @ApiOperation(value = "开启主机状态同步", produces = "application/json") InternalResponse enableSyncAgentStatus(); @@ -165,7 +164,7 @@ InternalResponse syncHostByBizId( * * @return */ - @PutMapping("/syncAgentStatus/disable") + @PutMapping("/service/sync/syncAgentStatus/disable") @ApiOperation(value = "关闭主机状态同步", produces = "application/json") InternalResponse disableSyncAgentStatus(); } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceTagResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceTagResource.java index e846d3a36a..4e8073d0e1 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceTagResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceTagResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,24 +27,22 @@ import com.tencent.bk.job.common.annotation.InternalAPI; import com.tencent.bk.job.common.model.InternalResponse; import com.tencent.bk.job.manage.model.inner.ServiceTagDTO; +import com.tentent.bk.job.common.api.feign.annotation.SmartFeignClient; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; import java.util.List; @Api(tags = {"job-manage:Service:Tag"}) -@RequestMapping("/service/tag") -@RestController +@SmartFeignClient(value = "job-manage", contextId = "tagResource") @InternalAPI public interface ServiceTagResource { @ApiOperation(value = "获取业务下的所有标签", produces = "application/json") - @GetMapping("/list") + @GetMapping("/service/tag/list") InternalResponse> listTags( @ApiParam(value = "业务Id") @RequestParam(value = "appId", required = false) @@ -52,7 +50,7 @@ InternalResponse> listTags( ); @ApiOperation(value = "获取所有公共标签", produces = "application/json") - @GetMapping("/list/public") + @GetMapping("/service/tag/list/public") InternalResponse> listPublicTags(); } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceTaskPlanResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceTaskPlanResource.java index 7b07662cda..a4600c9b49 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceTaskPlanResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceTaskPlanResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -30,6 +30,7 @@ import com.tencent.bk.job.manage.model.inner.ServiceTaskPlanDTO; import com.tencent.bk.job.manage.model.inner.ServiceTaskVariableDTO; import com.tencent.bk.job.manage.model.web.vo.task.TaskPlanVO; +import com.tentent.bk.job.common.api.feign.annotation.SmartFeignClient; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; @@ -38,64 +39,61 @@ import org.springframework.web.bind.annotation.PutMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestHeader; -import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; import java.util.List; @Api(tags = {"job-manage:service:Task_Plan_Management"}) -@RequestMapping("/service") -@RestController +@SmartFeignClient(value = "job-manage", contextId = "taskPlanResource") @InternalAPI public interface ServiceTaskPlanResource { @ApiOperation(value = "根据执行方案ID获取执行方案信息", produces = "application/json") - @GetMapping("/app/{appId}/plan/planId/{planId}/basic") + @GetMapping("/service/app/{appId}/plan/planId/{planId}/basic") InternalResponse getPlanBasicInfoById( @ApiParam(value = "业务ID", required = true) @PathVariable("appId") Long appId, @ApiParam(value = "执行方案ID", required = true) @PathVariable("planId") Long planId); @ApiOperation(value = "根据执行方案ID获取执行方案信息", produces = "application/json") - @GetMapping("/plan/{planId}/planName") + @GetMapping("/service/plan/{planId}/planName") InternalResponse getPlanName( @ApiParam(value = "执行方案ID", required = true) @PathVariable("planId") Long planId); @ApiOperation(value = "根据执行方案全局变量name获取id", produces = "application/json") - @GetMapping("/plan/{planId}/globalVar/getIdByName/{globalVarName}") + @GetMapping("/service/plan/{planId}/globalVar/getIdByName/{globalVarName}") InternalResponse getGlobalVarIdByName( @ApiParam(value = "执行方案ID", required = true) @PathVariable("planId") Long planId, @ApiParam(value = "全局变量名称", required = true) @PathVariable("globalVarName") String globalVarName ); @ApiOperation(value = "根据执行方案全局变量name获取实例", produces = "application/json") - @GetMapping("/plan/{planId}/globalVar/name/{globalVarName}") + @GetMapping("/service/plan/{planId}/globalVar/name/{globalVarName}") InternalResponse getGlobalVarByName( @ApiParam(value = "执行方案ID", required = true) @PathVariable("planId") Long planId, @ApiParam(value = "全局变量名称", required = true) @PathVariable("globalVarName") String globalVarName ); @ApiOperation(value = "根据执行方案全局变量id获取name", produces = "application/json") - @GetMapping("/plan/{planId}/globalVar/getNameById/{globalVarId}") + @GetMapping("/service/plan/{planId}/globalVar/getNameById/{globalVarId}") InternalResponse getGlobalVarNameById( @ApiParam(value = "执行方案ID", required = true) @PathVariable("planId") Long planId, @ApiParam(value = "全局变量ID", required = true) @PathVariable("globalVarId") Long globalVarId ); @ApiOperation(value = "根据执行方案全局变量id获取实例", produces = "application/json") - @GetMapping("/plan/{planId}/globalVar/id/{globalVarId}") + @GetMapping("/service/plan/{planId}/globalVar/id/{globalVarId}") InternalResponse getGlobalVarById( @ApiParam(value = "执行方案ID", required = true) @PathVariable("planId") Long planId, @ApiParam(value = "全局变量ID", required = true) @PathVariable("globalVarId") Long globalVarId ); @ApiOperation(value = "根据执行方案ID获取执行方案业务Id", produces = "application/json") - @GetMapping("/plan/{planId}/planAppId") + @GetMapping("/service/plan/{planId}/planAppId") InternalResponse getPlanAppId( @ApiParam(value = "执行方案ID", required = true) @PathVariable("planId") Long planId); @ApiOperation(value = "根据执行方案ID获取执行方案信息", produces = "application/json") - @GetMapping("/app/{appId}/plan/planId/{planId}") + @GetMapping("/service/app/{appId}/plan/planId/{planId}") InternalResponse getPlanById( @ApiParam(value = "业务ID", required = true) @PathVariable("appId") Long appId, @@ -105,7 +103,7 @@ InternalResponse getPlanById( @RequestParam(value = "includeDisabledSteps", required = false, defaultValue = "false") Boolean includeDisabledSteps); - @PutMapping("/app/{appId}/plan/{templateId}/{planId}/createPlanWithId") + @PutMapping("/service/app/{appId}/plan/{templateId}/{planId}/createPlanWithId") InternalResponse createPlanWithIdForMigration( @ApiParam(value = "用户名，网关自动传入") @RequestHeader("username") String username, @ApiParam(value = "业务 ID", required = true, example = "2") @PathVariable("appId") Long appId, @@ -115,7 +113,7 @@ InternalResponse createPlanWithIdForMigration( @ApiParam(value = "修改时间") @RequestHeader(value = "X-Update-Time", required = false) Long lastModifyTime, @ApiParam(value = "最后修改人") @RequestHeader(value = "X-Update-User", required = false) String lastModifyUser); - @GetMapping("/app/{appId}/plan/check") + @GetMapping("/service/app/{appId}/plan/check") InternalResponse checkIdAndName( @ApiParam(value = "业务 ID", required = true, example = "2") @PathVariable("appId") Long appId, @ApiParam(value = "模版 ID") @RequestParam("templateId") Long templateId, @@ -123,7 +121,7 @@ InternalResponse checkIdAndName( @ApiParam(value = "执行方案名称", required = true) @RequestParam("planName") String name); @ApiOperation(value = "导入执行方案", produces = "application/json") - @PutMapping("/app/{appId}/plan/{templateId}/savePlanForImport") + @PutMapping("/service/app/{appId}/plan/{templateId}/savePlanForImport") InternalResponse savePlanForImport( @ApiParam(value = "用户名，网关自动传入") @RequestHeader("username") String username, @ApiParam(value = "业务 ID", required = true, example = "2") @PathVariable("appId") Long appId, @@ -131,7 +129,7 @@ InternalResponse savePlanForImport( @ApiParam(value = "创建时间") @RequestHeader(value = "X-Create-Time", required = false) Long createTime, @ApiParam(value = "执行方案信息", required = true) @RequestBody TaskPlanVO planInfo); - @GetMapping("/app/{appId}/plan/{templateId}/{planId}/variable") + @GetMapping("/service/app/{appId}/plan/{templateId}/{planId}/variable") InternalResponse> getPlanVariable( @ApiParam(value = "用户名，网关自动传入") @RequestHeader("username") String username, @ApiParam(value = "业务 ID", required = true, example = "2") @PathVariable("appId") Long appId, @@ -139,14 +137,14 @@ InternalResponse> getPlanVariable( @ApiParam(value = "执行方案 ID") @PathVariable("planId") Long planId); @ApiOperation(value = "获取执行方案基本信息列表", produces = "application/json") - @GetMapping("/{templateId}") + @GetMapping("/service/{templateId}") InternalResponse> listPlans( @ApiParam(value = "用户名，网关自动传入") @RequestHeader("username") String username, @ApiParam(value = "业务 ID", required = true, example = "2") @PathVariable("appId") Long appId, @ApiParam(value = "模版 ID", required = true) @PathVariable(value = "templateId") Long templateId); @ApiOperation(value = "获取模板对应的执行方案Id", produces = "application/json") - @GetMapping("/plan/planIds/template/{templateId}") + @GetMapping("/service/plan/planIds/template/{templateId}") InternalResponse> listPlanIds( @ApiParam(value = "模版 ID", required = true) @PathVariable(value = "templateId") Long templateId); } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceTaskTemplateResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceTaskTemplateResource.java index 1d681ce990..2319271284 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceTaskTemplateResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceTaskTemplateResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -31,6 +31,7 @@ import com.tencent.bk.job.manage.model.inner.ServiceTaskTemplateDTO; import com.tencent.bk.job.manage.model.inner.ServiceTaskVariableDTO; import com.tencent.bk.job.manage.model.web.request.TaskTemplateCreateUpdateReq; +import com.tentent.bk.job.common.api.feign.annotation.SmartFeignClient; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; @@ -40,20 +41,17 @@ import org.springframework.web.bind.annotation.PutMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestHeader; -import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; import java.util.List; @Api(tags = {"job-manage:service:Task_Template_Management"}) -@RequestMapping("/service") -@RestController +@SmartFeignClient(value = "job-manage", contextId = "taskTemplateResource") @InternalAPI public interface ServiceTaskTemplateResource { @ApiOperation(value = "同步脚本更新消息", produces = "application/json") - @PostMapping("/app/{appId}/template/script/{scriptId}/update_message") + @PostMapping("/service/app/{appId}/template/script/{scriptId}/update_message") InternalResponse sendScriptUpdateMessage( @ApiParam(value = "业务 ID", required = true) @PathVariable("appId") Long appId, @ApiParam(value = "脚本 ID", required = true) @PathVariable("scriptId") String scriptId, @@ -61,24 +59,24 @@ InternalResponse sendScriptUpdateMessage( @ApiParam(value = "脚本状态 1 - 上线 2 - 下线 3 - 禁用", required = true) @RequestParam("status") Integer status); @ApiOperation(value = "根据模版 ID 获取模版信息", produces = "application/json") - @GetMapping("/app/{appId}/template/{templateId}") + @GetMapping("/service/app/{appId}/template/{templateId}") InternalResponse getTemplateById( @ApiParam(value = "用户名，网关自动传入") @RequestHeader("username") String username, @ApiParam(value = "业务 ID", required = true, example = "2") @PathVariable("appId") Long appId, @ApiParam(value = "模版 ID") @PathVariable("templateId") Long templateId); @ApiOperation(value = "根据模版 ID 获取模版信息", produces = "application/json") - @GetMapping("/template/{templateId}") + @GetMapping("/service/template/{templateId}") InternalResponse getTemplateById( @ApiParam(value = "模版 ID") @PathVariable("templateId") Long templateId); @ApiOperation(value = "根据模版 ID 获取模版名称", produces = "application/json") - @GetMapping("/template/{templateId}/templateName") + @GetMapping("/service/template/{templateId}/templateName") InternalResponse getTemplateNameById( @ApiParam(value = "模版 ID") @PathVariable("templateId") Long templateId); @ApiOperation(value = "更新模版", produces = "application/json") - @PutMapping("/app/{appId}/template/{templateId}/saveTemplateWithVariableId") + @PutMapping("/service/app/{appId}/template/{templateId}/saveTemplateWithVariableId") InternalResponse saveTemplateForMigration( @ApiParam(value = "用户名，网关自动传入") @RequestHeader("username") String username, @ApiParam(value = "业务 ID", required = true, example = "2") @PathVariable("appId") Long appId, @@ -91,21 +89,21 @@ InternalResponse saveTemplateForMigration( required = true) @RequestBody TaskTemplateCreateUpdateReq taskTemplateCreateUpdateReq); @ApiOperation(value = "校验模版 ID 和名称", produces = "application/json") - @GetMapping("/app/{appId}/template/check") + @GetMapping("/service/app/{appId}/template/check") InternalResponse checkIdAndName( @ApiParam(value = "业务 ID", required = true, example = "2") @PathVariable("appId") Long appId, @ApiParam(value = "模版 ID") @RequestParam("templateId") Long templateId, @ApiParam(value = "模版名称", required = true) @RequestParam("templateName") String name); @ApiOperation(value = "根据模版 ID 获取模版变量信息", produces = "application/json") - @GetMapping("/app/{appId}/template/{templateId}/variable") + @GetMapping("/service/app/{appId}/template/{templateId}/variable") InternalResponse> getTemplateVariable( @ApiParam(value = "用户名，网关自动传入") @RequestHeader("username") String username, @ApiParam(value = "业务 ID", required = true, example = "2") @PathVariable("appId") Long appId, @ApiParam(value = "模版 ID") @PathVariable("templateId") Long templateId); @ApiOperation(value = "获取模版信息列表", produces = "application/json") - @GetMapping("/app/{appId}/template/list") + @GetMapping("/service/app/{appId}/template/list") InternalResponse> listPageTaskTemplates( @ApiParam(value = "业务 ID", required = true, example = "2") @PathVariable("appId") diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceUserResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceUserResource.java index b50032dd49..4c303fc622 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceUserResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceUserResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,24 +26,22 @@ import com.tencent.bk.job.common.annotation.InternalAPI; import com.tencent.bk.job.common.model.InternalResponse; +import com.tentent.bk.job.common.api.feign.annotation.SmartFeignClient; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; import java.util.Set; @Api(tags = {"job-manage:service:User"}) -@RequestMapping("/service/user") -@RestController +@SmartFeignClient(value = "job-manage", contextId = "userResource") @InternalAPI public interface ServiceUserResource { @ApiOperation(value = "根据业务Id与角色获取用户信息", produces = "application/json") - @GetMapping("/getUsersByRoles") + @GetMapping("/service/user/getUsersByRoles") InternalResponse> getUsersByRoles( @ApiParam("业务Id") @RequestParam(value = "appId", required = true) diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceWhiteIPResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceWhiteIPResource.java index 8ee0fec63b..7682b9fda3 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceWhiteIPResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/ServiceWhiteIPResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,6 +28,7 @@ import com.tencent.bk.job.common.model.InternalResponse; import com.tencent.bk.job.manage.model.inner.ServiceWhiteIPInfo; import com.tencent.bk.job.manage.model.web.request.whiteip.WhiteIPRecordCreateUpdateReq; +import com.tentent.bk.job.common.api.feign.annotation.SmartFeignClient; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; @@ -35,20 +36,17 @@ import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestHeader; -import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestParam; -import org.springframework.web.bind.annotation.RestController; import java.util.List; @Api(tags = {"job-manage:service:WhiteIP"}) -@RequestMapping("/service/whiteip") -@RestController +@SmartFeignClient(value = "job-manage", contextId = "whiteIpResource") @InternalAPI public interface ServiceWhiteIPResource { @ApiOperation(value = "获取指定IP在白名单中的生效范围（脚本执行：SCRIPT_EXECUTE/文件分发：FILE_DISTRIBUTION）", produces = "application/json") - @GetMapping("/getWhiteIPActionScopes") + @GetMapping("/service/whiteip/getWhiteIPActionScopes") InternalResponse> getWhiteIPActionScopes( @ApiParam("业务Id") @RequestParam(value = "appId", required = false) @@ -58,15 +56,18 @@ InternalResponse> getWhiteIPActionScopes( String ip, @ApiParam("云区域Id") @RequestParam(value = "cloudAreaId", required = false) - Long cloudAreaId + Long cloudAreaId, + @ApiParam("主机hostId") + @RequestParam(value = "hostId", required = false) + Long hostId ); @ApiOperation(value = "获取白名单内IP详情信息", produces = "application/json") - @GetMapping("/listWhiteIPInfos") + @GetMapping("/service/whiteip/listWhiteIPInfos") InternalResponse> listWhiteIPInfos(); @ApiOperation(value = "新增/更新IP白名单", produces = "application/json") - @PostMapping("") + @PostMapping("/service/whiteip") InternalResponse saveWhiteIP( @ApiParam(value = "用户名，网关自动传入", required = true) @RequestHeader("username") diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/op/EventReplayOpResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/op/EventReplayOpResource.java new file mode 100644 index 0000000000..1fa669872e --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/op/EventReplayOpResource.java @@ -0,0 +1,53 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.api.op; + +import com.tencent.bk.job.common.cc.model.result.HostEventDetail; +import com.tencent.bk.job.common.cc.model.result.ResourceEvent; +import com.tencent.bk.job.common.model.Response; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@Api(tags = {"job-manage:api:EventReplay-OP"}) +@RequestMapping("/op/eventReplay") +@RestController +public interface EventReplayOpResource { + + @ApiOperation(value = "重放主机事件", produces = "application/json") + @PostMapping("/host") + Response replayHostEvent( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") String username, + @ApiParam(value = "主机事件", required = true) + @RequestBody ResourceEvent event + ); + +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/op/FlowControlOpResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/op/FlowControlOpResource.java index 7f097c5e81..8fba4d94b0 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/op/FlowControlOpResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/op/FlowControlOpResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/tmp/TmpAppAccountResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/tmp/TmpAppAccountResource.java deleted file mode 100644 index 9b7384dd6a..0000000000 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/tmp/TmpAppAccountResource.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.api.tmp; - -import com.tencent.bk.job.common.model.Response; -import com.tencent.bk.job.manage.model.tmp.TmpAccountCreateUpdateReq; -import io.swagger.annotations.Api; -import io.swagger.annotations.ApiOperation; -import io.swagger.annotations.ApiParam; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.PostMapping; -import org.springframework.web.bind.annotation.RequestBody; -import org.springframework.web.bind.annotation.RequestHeader; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - -@RestController -@Api(tags = {"job-manage:tmp:App_Account"}) -@RequestMapping("/tmp/account/app/{appId}") -public interface TmpAppAccountResource { - - @ApiOperation(value = "新增账号（不校验alias）", produces = "application/json") - @PostMapping("/account") - Response saveAccount( - @ApiParam(value = "用户名，网关自动传入", required = true) @RequestHeader("username") String username, - @ApiParam(value = "业务 ID", required = true) @PathVariable("appId") Long appId, - @ApiParam("创建时间") @RequestHeader(value = "X-Create-Time", required = false) Long createTime, - @ApiParam("修改时间") @RequestHeader(value = "X-Update-Time", required = false) Long lastModifyTime, - @ApiParam("最后修改人") @RequestHeader(value = "X-Update-User", required = false) String lastModifyUser, - @ApiParam("是否使用当前时间") @RequestHeader(value = "X-Use-Current-Time", required = false) Boolean useCurrentTime, - @ApiParam(value = "创建账号请求") @RequestBody TmpAccountCreateUpdateReq accountCreateUpdateReq); - -} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/tmp/TmpMigrationResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/tmp/TmpMigrationResource.java deleted file mode 100644 index 0517b89821..0000000000 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/tmp/TmpMigrationResource.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.api.tmp; - -import com.tencent.bk.job.common.model.Response; -import com.tencent.bk.job.manage.model.tmp.MigrationPlanBasic; -import io.swagger.annotations.Api; -import io.swagger.annotations.ApiOperation; -import io.swagger.annotations.ApiParam; -import org.springframework.web.bind.annotation.GetMapping; -import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RequestHeader; -import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RestController; - -import java.util.List; - -@RestController -@Api(tags = {"job-manage:tmp:Migration"}) -@RequestMapping("/tmp/migration") -public interface TmpMigrationResource { - - @ApiOperation(value = "获取业务下的执行方案的基础信息", produces = "application/json") - @GetMapping("/app/{appId}/plan/basic/list") - Response> listAppPlanBasicInfo( - @ApiParam(value = "用户名，网关自动传入", required = true) @RequestHeader("username") String username, - @ApiParam(value = "业务 ID", required = true) @PathVariable("appId") Long appId); - -} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebAppAccountResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebAppAccountResource.java index b30b2ebd99..b4fa7630c2 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebAppAccountResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebAppAccountResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -58,7 +58,7 @@ public interface WebAppAccountResource { @ApiOperation(value = "新增账号", produces = "application/json") @PostMapping(value = "/scope/{scopeType}/{scopeId}/account") - Response saveAccount( + Response saveAccount( @ApiParam(value = "用户名，网关自动传入", required = true) @RequestHeader("username") String username, @@ -77,8 +77,8 @@ Response saveAccount( ); @ApiOperation(value = "更新账号", produces = "application/json") - @PutMapping(value = "/scope/{scopeType}/{scopeId}/account") - Response updateAccount( + @PutMapping(value = "/scope/{scopeType}/{scopeId}/account/{accountId}") + Response updateAccount( @ApiParam(value = "用户名，网关自动传入", required = true) @RequestHeader("username") String username, @@ -91,6 +91,9 @@ Response updateAccount( @ApiParam(value = "资源范围ID", required = true) @PathVariable(value = "scopeId") String scopeId, + @ApiParam(value = "账号ID", required = true) + @PathVariable(value = "accountId") + Long accountId, @ApiParam(value = "更新账号请求") @RequestBody @Validated AccountCreateUpdateReq accountCreateUpdateReq @@ -132,6 +135,9 @@ Response> listAppAccounts( @ApiParam("更新人：模糊搜索") @RequestParam(value = "lastModifyUser", required = false) String lastModifyUser, + @ApiParam("描述：模糊搜索") + @RequestParam(value = "remark", required = false) + String remark, @ApiParam("分页-开始") @RequestParam(value = "start", required = false) Integer start, @@ -204,7 +210,7 @@ Response> listAccounts( @ApiParam(value = "资源范围ID", required = true) @PathVariable(value = "scopeId") String scopeId, - @ApiParam(value = "账号用途,1-系统账号，2-DB账号,不传表示所有用途", required = false) + @ApiParam(value = "账号用途,1-系统账号，2-DB账号,不传表示所有用途") @RequestParam(value = "category", required = false) Integer category ); diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebAppResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebAppResource.java index 9c239570eb..19b02bb99f 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebAppResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebAppResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,22 +25,11 @@ package com.tencent.bk.job.manage.api.web; import com.tencent.bk.job.common.annotation.WebAPI; -import com.tencent.bk.job.common.model.PageData; import com.tencent.bk.job.common.model.Response; import com.tencent.bk.job.common.model.dto.AppResourceScope; -import com.tencent.bk.job.common.model.vo.HostInfoVO; -import com.tencent.bk.job.common.model.vo.TargetNodeVO; -import com.tencent.bk.job.manage.model.web.request.AgentStatisticsReq; -import com.tencent.bk.job.manage.model.web.request.IpCheckReq; import com.tencent.bk.job.manage.model.web.request.app.FavorAppReq; -import com.tencent.bk.job.manage.model.web.request.ipchooser.AppTopologyTreeNode; -import com.tencent.bk.job.manage.model.web.request.ipchooser.ListHostByBizTopologyNodesReq; import com.tencent.bk.job.manage.model.web.vo.AppVO; -import com.tencent.bk.job.manage.model.web.vo.CcTopologyNodeVO; -import com.tencent.bk.job.manage.model.web.vo.DynamicGroupInfoVO; -import com.tencent.bk.job.manage.model.web.vo.NodeInfoVO; import com.tencent.bk.job.manage.model.web.vo.PageDataWithAvailableIdList; -import com.tencent.bk.job.manage.model.web.vo.index.AgentStatistics; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; @@ -55,8 +44,6 @@ import org.springframework.web.bind.annotation.RestController; import springfox.documentation.annotations.ApiIgnore; -import java.util.List; - /** * 业务管理 WEB API */ @@ -116,282 +103,4 @@ Response cancelFavorApp( @RequestBody FavorAppReq favorAppReq ); - @ApiOperation(value = "获取业务下的机器列表", produces = "application/json") - @GetMapping(value = {"/scope/{scopeType}/{scopeId}/host"}) - Response> listAppHost( - @ApiParam("用户名，网关自动传入") - @RequestHeader("username") - String username, - @ApiIgnore - @RequestAttribute(value = "appResourceScope") - AppResourceScope appResourceScope, - @ApiParam(value = "资源范围类型", required = true) - @PathVariable(value = "scopeType") - String scopeType, - @ApiParam(value = "资源范围ID", required = true) - @PathVariable(value = "scopeId") - String scopeId, - @ApiParam(value = "分页-开始") - @RequestParam(value = "start", required = false) - Integer start, - @ApiParam(value = "分页-每页大小") - @RequestParam(value = "pageSize", required = false) - Integer pageSize, - @ApiParam(value = "模块类型 0-所有模块 1-普通模块，2-DB模块") - @RequestParam(value = "moduleType", - required = false) Long moduleType, - @ApiParam(value = "ip，搜索条件，模糊匹配") - @RequestParam(value = "ipCondition", required = false) - String ipCondition - ); - - @ApiOperation(value = "获取业务拓扑列表", produces = "application/json") - @GetMapping(value = {"/scope/{scopeType}/{scopeId}/topology"}) - Response listAppTopologyTree( - @ApiParam("用户名，网关自动传入") - @RequestHeader("username") - String username, - @ApiIgnore - @RequestAttribute(value = "appResourceScope") - AppResourceScope appResourceScope, - @ApiParam(value = "资源范围类型", required = true) - @PathVariable(value = "scopeType") - String scopeType, - @ApiParam(value = "资源范围ID", required = true) - @PathVariable(value = "scopeId") - String scopeId - ); - - @ApiOperation(value = "获取业务拓扑主机列表", produces = "application/json") - @GetMapping(value = {"/scope/{scopeType}/{scopeId}/topology/host"}) - Response listAppTopologyHostTree( - @ApiParam("用户名，网关自动传入") - @RequestHeader("username") - String username, - @ApiIgnore - @RequestAttribute(value = "appResourceScope") - AppResourceScope appResourceScope, - @ApiParam(value = "资源范围类型", required = true) - @PathVariable(value = "scopeType") - String scopeType, - @ApiParam(value = "资源范围ID", required = true) - @PathVariable(value = "scopeId") - String scopeId - ); - - @ApiOperation(value = "获取业务拓扑树（含各节点主机数）", produces = "application/json") - @GetMapping(value = {"/scope/{scopeType}/{scopeId}/topology/hostCount"}) - Response listAppTopologyHostCountTree( - @ApiParam("用户名，网关自动传入") - @RequestHeader("username") - String username, - @ApiIgnore - @RequestAttribute(value = "appResourceScope") - AppResourceScope appResourceScope, - @ApiParam(value = "资源范围类型", required = true) - @PathVariable(value = "scopeType") - String scopeType, - @ApiParam(value = "资源范围ID", required = true) - @PathVariable(value = "scopeId") - String scopeId - ); - - @ApiOperation(value = "IP选择器根据拓扑节点集合获取机器列表", produces = "application/json") - @PostMapping(value = {"/scope/{scopeType}/{scopeId}/topology/hosts/nodes"}) - Response> listHostByBizTopologyNodes( - @ApiParam("用户名，网关自动传入") - @RequestHeader("username") - String username, - @ApiIgnore - @RequestAttribute(value = "appResourceScope") - AppResourceScope appResourceScope, - @ApiParam(value = "资源范围类型", required = true) - @PathVariable(value = "scopeType") - String scopeType, - @ApiParam(value = "资源范围ID", required = true) - @PathVariable(value = "scopeId") - String scopeId, - @ApiParam(value = "拓扑节点集合与分页信息", required = true) - @RequestBody - ListHostByBizTopologyNodesReq req - ); - - @ApiOperation(value = "IP选择器根据拓扑节点集合获取机器列表（纯IP），返回IP格式为[cloudId:IP]" - , produces = "application/json") - @PostMapping(value = {"/scope/{scopeType}/{scopeId}/topology/IPs/nodes"}) - Response> listIpByBizTopologyNodes( - @ApiParam("用户名，网关自动传入") - @RequestHeader("username") - String username, - @ApiIgnore - @RequestAttribute(value = "appResourceScope") - AppResourceScope appResourceScope, - @ApiParam(value = "资源范围类型", required = true) - @PathVariable(value = "scopeType") - String scopeType, - @ApiParam(value = "资源范围ID", required = true) - @PathVariable(value = "scopeId") - String scopeId, - @ApiParam(value = "拓扑节点集合与分页信息", required = true) - @RequestBody - ListHostByBizTopologyNodesReq req - ); - - @ApiOperation(value = "获取节点详情", produces = "application/json") - @PostMapping(value = {"/scope/{scopeType}/{scopeId}/node/detail"}) - Response> getNodeDetail( - @ApiParam("用户名，网关自动传入") - @RequestHeader("username") - String username, - @ApiIgnore - @RequestAttribute(value = "appResourceScope") - AppResourceScope appResourceScope, - @ApiParam(value = "资源范围类型", required = true) - @PathVariable(value = "scopeType") - String scopeType, - @ApiParam(value = "资源范围ID", required = true) - @PathVariable(value = "scopeId") - String scopeId, - @ApiParam(value = "选中的拓扑节点列表(将拓扑树节点中的objectId与instanceId传入)", required = true) - @RequestBody - List targetNodeVOList - ); - - @ApiOperation(value = "获取节点拓扑路径", produces = "application/json") - @PostMapping(value = {"/scope/{scopeType}/{scopeId}/node/queryPath"}) - Response>> queryNodePaths( - @ApiParam("用户名，网关自动传入") - @RequestHeader("username") - String username, - @ApiIgnore - @RequestAttribute(value = "appResourceScope") - AppResourceScope appResourceScope, - @ApiParam(value = "资源范围类型", required = true) - @PathVariable(value = "scopeType") - String scopeType, - @ApiParam(value = "资源范围ID", required = true) - @PathVariable(value = "scopeId") - String scopeId, - @ApiParam(value = "需要查询拓扑路径的节点列表(将拓扑树节点中的objectId与instanceId传入)", required = true) - @RequestBody - List targetNodeVOList - ); - - @ApiOperation(value = "根据模块获取机器列表", produces = "application/json") - @PostMapping(value = {"/scope/{scopeType}/{scopeId}/host/node"}) - Response> listHostByNode( - @ApiParam("用户名，网关自动传入") - @RequestHeader("username") - String username, - @ApiIgnore - @RequestAttribute(value = "appResourceScope") - AppResourceScope appResourceScope, - @ApiParam(value = "资源范围类型", required = true) - @PathVariable(value = "scopeType") - String scopeType, - @ApiParam(value = "资源范围ID", required = true) - @PathVariable(value = "scopeId") - String scopeId, - @ApiParam(value = "选中的拓扑节点列表(将拓扑树节点中的objectId与instanceId传入)", required = true) - @RequestBody - List targetNodeVOList - ); - - @ApiOperation(value = "获取业务动态分组列表", produces = "application/json") - @GetMapping(value = {"/scope/{scopeType}/{scopeId}/dynamicGroup"}) - Response> listAppDynamicGroup( - @ApiParam("用户名，网关自动传入") - @RequestHeader("username") - String username, - @ApiIgnore - @RequestAttribute(value = "appResourceScope") - AppResourceScope appResourceScope, - @ApiParam(value = "资源范围类型", required = true) - @PathVariable(value = "scopeType") - String scopeType, - @ApiParam(value = "资源范围ID", required = true) - @PathVariable(value = "scopeId") - String scopeId - ); - - @ApiOperation(value = "获取业务动态分组主机列表") - @GetMapping(value = {"/scope/{scopeType}/{scopeId}/dynamicGroup/{dynamicGroupId}"}) - Response> listAppDynamicGroupHost( - @ApiParam("用户名，网关自动传入") - @RequestHeader("username") - String username, - @ApiIgnore - @RequestAttribute(value = "appResourceScope") - AppResourceScope appResourceScope, - @ApiParam(value = "资源范围类型", required = true) - @PathVariable(value = "scopeType") - String scopeType, - @ApiParam(value = "资源范围ID", required = true) - @PathVariable(value = "scopeId") - String scopeId, - @ApiParam(value = "动态分组 ID，逗号分割", required = true) - @PathVariable("dynamicGroupId") - List dynamicGroupIds - ); - - @ApiOperation(value = "获取业务动态分组信息(不含主机)") - @GetMapping(value = {"/scope/{scopeType}/{scopeId}/dynamicGroup/{dynamicGroupId}/detailWithoutHosts"}) - Response> listAppDynamicGroupWithoutHosts( - @ApiParam("用户名，网关自动传入") - @RequestHeader("username") - String username, - @ApiIgnore - @RequestAttribute(value = "appResourceScope") - AppResourceScope appResourceScope, - @ApiParam(value = "资源范围类型", required = true) - @PathVariable(value = "scopeType") - String scopeType, - @ApiParam(value = "资源范围ID", required = true) - @PathVariable(value = "scopeId") - String scopeId, - @ApiParam(value = "动态分组 ID，逗号分割", required = true) - @PathVariable("dynamicGroupId") - List dynamicGroupIds - ); - - @ApiOperation(value = "根据输入 IP 获取机器信息") - @PostMapping(value = {"/scope/{scopeType}/{scopeId}/ip/check"}) - Response> listHostByIp( - @ApiParam("用户名，网关自动传入") - @RequestHeader("username") - String username, - @ApiIgnore - @RequestAttribute(value = "appResourceScope") - AppResourceScope appResourceScope, - @ApiParam(value = "资源范围类型", required = true) - @PathVariable(value = "scopeType") - String scopeType, - @ApiParam(value = "资源范围ID", required = true) - @PathVariable(value = "scopeId") - String scopeId, - @ApiParam(value = "用户输入的 IP 列表", required = true) - @RequestBody - IpCheckReq req - ); - - @ApiOperation(value = "查询主机统计信息") - @PostMapping(value = {"/scope/{scopeType}/{scopeId}/host/statistics"}) - Response agentStatistics( - @ApiParam("用户名，网关自动传入") - @RequestHeader("username") - String username, - @ApiIgnore - @RequestAttribute(value = "appResourceScope") - AppResourceScope appResourceScope, - @ApiParam(value = "资源范围类型", required = true) - @PathVariable(value = "scopeType") - String scopeType, - @ApiParam(value = "资源范围ID", required = true) - @PathVariable(value = "scopeId") - String scopeId, - @ApiParam(value = "请求体", required = true) - @RequestBody - AgentStatisticsReq agentStatisticsReq - ); - } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebContainerResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebContainerResource.java new file mode 100644 index 0000000000..6bf46d2bb1 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebContainerResource.java @@ -0,0 +1,168 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.api.web; + +import com.tencent.bk.job.common.annotation.WebAPI; +import com.tencent.bk.job.common.model.PageData; +import com.tencent.bk.job.common.model.Response; +import com.tencent.bk.job.common.model.dto.AppResourceScope; +import com.tencent.bk.job.common.model.vo.ContainerVO; +import com.tencent.bk.job.manage.model.web.request.chooser.ListTopologyTreesReq; +import com.tencent.bk.job.manage.model.web.request.chooser.container.ContainerCheckReq; +import com.tencent.bk.job.manage.model.web.request.chooser.container.ContainerDetailReq; +import com.tencent.bk.job.manage.model.web.request.chooser.container.ContainerIdWithMeta; +import com.tencent.bk.job.manage.model.web.request.chooser.container.ListContainerByTopologyNodesReq; +import com.tencent.bk.job.manage.model.web.vo.chooser.container.ContainerTopologyNodeVO; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestAttribute; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import springfox.documentation.annotations.ApiIgnore; + +import java.util.List; + +/** + * 容器管理 WEB API + */ +@Api(tags = "job-manage:web:Container_Management") +@RequestMapping("/web/scope/{scopeType}/{scopeId}/") +@RestController +@WebAPI +public interface WebContainerResource { + + // 容器选择器标准接口-1 + @ApiOperation(value = "获取容器拓扑树（含各节点容器数）", produces = "application/json") + @PostMapping(value = {"/topology/container"}) + Response> listTopologyTrees( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId, + @ApiParam(value = "资源范围信息") + @RequestBody(required = false) + ListTopologyTreesReq req + ); + + // 容器选择器标准接口-2 + @ApiOperation(value = "容器选择器根据拓扑节点集合获取容器列表", produces = "application/json") + @PostMapping(value = {"/topology/containers/nodes"}) + Response> listContainerByTopologyNodes( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId, + @ApiParam(value = "拓扑节点集合与分页信息", required = true) + @RequestBody + ListContainerByTopologyNodesReq req + ); + + // 容器选择器标准接口-3 + @ApiOperation(value = "容器选择器根据拓扑节点集合获取容器资源ID列表，用于跨页全选容器功能" + , produces = "application/json") + @PostMapping(value = {"/topology/containerIds/nodes"}) + Response> listContainerIdByTopologyNodes( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId, + @ApiParam(value = "拓扑节点集合与分页信息", required = true) + @RequestBody + ListContainerByTopologyNodesReq req + ); + + + // 容器选择器标准接口-4 + @ApiOperation(value = "根据用户选择/输入的容器信息获取容器") + @PostMapping(value = {"/container/check"}) + Response> checkContainers( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId, + @ApiParam(value = "用户选择/输入的容器信息", required = true) + @RequestBody + ContainerCheckReq req + ); + + // 容器选择器标准接口-5 + @ApiOperation(value = "根据容器资源 ID批量查询容器详情信息") + @PostMapping(value = {"/containers/details"}) + Response> getContainerDetails( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId, + @ApiParam(value = "容器ID及元数据信息", required = true) + @RequestBody + ContainerDetailReq req + ); + +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebCredentialResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebCredentialResource.java index 1d154e3bca..51ba27217a 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebCredentialResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebCredentialResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -29,6 +29,7 @@ import com.tencent.bk.job.common.model.Response; import com.tencent.bk.job.common.model.dto.AppResourceScope; import com.tencent.bk.job.manage.model.web.request.CredentialCreateUpdateReq; +import com.tencent.bk.job.manage.model.web.vo.CredentialBasicVO; import com.tencent.bk.job.manage.model.web.vo.CredentialVO; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; @@ -37,6 +38,7 @@ import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; import org.springframework.web.bind.annotation.RequestAttribute; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestHeader; @@ -51,7 +53,7 @@ @WebAPI public interface WebCredentialResource { - @ApiOperation(value = "获取凭据列表", produces = "application/json") + @ApiOperation(value = "获取凭证列表", produces = "application/json") @GetMapping("/list") Response> listCredentials( @ApiParam("用户名，网关自动传入") @@ -89,10 +91,33 @@ Response> listCredentials( Integer pageSize ); + @ApiOperation(value = "分页获取凭证基础信息", produces = "application/json") + @GetMapping("/basicInfo/list") + Response> listCredentialBasicInfo( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId, + @ApiParam("分页-开始，不传默认为0") + @RequestParam(value = "start", required = false) + Integer start, + @ApiParam("分页-每页大小，不传默认拉取全量数据") + @RequestParam(value = "pageSize", required = false) + Integer pageSize + ); + - @ApiOperation(value = "新增/更新凭据", produces = "application/json") - @PostMapping("") - Response saveCredential( + @ApiOperation(value = "新增凭证", produces = "application/json") + @PostMapping + Response createCredential( @ApiParam(value = "用户名，网关自动传入", required = true) @RequestHeader("username") String username, @@ -105,13 +130,36 @@ Response saveCredential( @ApiParam(value = "资源范围ID", required = true) @PathVariable(value = "scopeId") String scopeId, - @ApiParam(value = "创建或更新请求体", required = true) + @ApiParam(value = "创建请求体", required = true) + @RequestBody + CredentialCreateUpdateReq createUpdateReq + ); + + @ApiOperation(value = "更新凭证", produces = "application/json") + @PutMapping("/{credentialId}") + Response updateCredential( + @ApiParam(value = "用户名，网关自动传入", required = true) + @RequestHeader("username") + String username, + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "credentialId") + String credentialId, + @ApiParam(value = "更新请求体", required = true) @RequestBody CredentialCreateUpdateReq createUpdateReq ); - @ApiOperation(value = "删除凭据", produces = "application/json") + @ApiOperation(value = "删除凭证", produces = "application/json") @DeleteMapping("/ids/{id}") Response deleteCredentialById( @ApiParam("用户名，网关自动传入") @@ -126,7 +174,7 @@ Response deleteCredentialById( @ApiParam(value = "资源范围ID", required = true) @PathVariable(value = "scopeId") String scopeId, - @ApiParam("凭据ID") + @ApiParam("凭证ID") @PathVariable("id") String id ); diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebCustomSettingsResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebCustomSettingsResource.java index 87daf1dbcf..479858d4d4 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebCustomSettingsResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebCustomSettingsResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,84 +26,90 @@ import com.tencent.bk.job.common.annotation.WebAPI; import com.tencent.bk.job.common.model.Response; -import com.tencent.bk.job.manage.model.web.request.customsetting.ScriptTemplateCreateUpdateReq; -import com.tencent.bk.job.manage.model.web.request.customsetting.ScriptTemplateRenderReq; -import com.tencent.bk.job.manage.model.web.vo.customsetting.ScriptTemplateVO; -import com.tencent.bk.job.manage.model.web.vo.customsetting.ScriptTemplateVariableVO; +import com.tencent.bk.job.common.model.dto.AppResourceScope; +import com.tencent.bk.job.manage.model.web.request.customsetting.BatchGetCustomSettingsReq; +import com.tencent.bk.job.manage.model.web.request.customsetting.DeleteCustomSettingsReq; +import com.tencent.bk.job.manage.model.web.request.customsetting.SaveCustomSettingsReq; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; -import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.DeleteMapping; +import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestAttribute; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RequestMapping; -import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; +import springfox.documentation.annotations.ApiIgnore; -import java.util.List; +import java.util.Map; -@Api(tags = {"job-manage:web:CustomSettings_ScriptTemplate"}) -@RequestMapping("/web/customSettings/scriptTemplate") +@Api(tags = {"job-manage:web:CustomSettings"}) +@RequestMapping("/web/customSettings/scope/{scopeType}/{scopeId}") @RestController @WebAPI public interface WebCustomSettingsResource { - @ApiOperation(value = "获取用户自定义的脚本模板", produces = "application/json") - @GetMapping - Response> listUserCustomScriptTemplate( - @ApiParam("用户名，网关自动传入") - @RequestHeader("username") - String username, - @ApiParam("脚本类型，1:shell,2:bat,3:perl,4:python,5:PowerShell,6:sql;支持传入多个,用英文逗号分隔;如果不传入任何值，默认返回全部脚本类型的模板") - @RequestParam(value = "scriptLanguages", required = false) - String scriptLanguages); - - @ApiOperation(value = "获取渲染后的用户自定义的脚本模板", produces = "application/json") - @GetMapping("/rendered") - Response> listRenderedUserCustomScriptTemplate( - @ApiParam("用户名，网关自动传入") - @RequestHeader("username") - String username, - @ApiParam("脚本类型，1:shell,2:bat,3:perl,4:python,5:PowerShell,6:sql;支持传入多个,用英文逗号分隔;如果不传入任何值，默认返回全部脚本类型的模板") - @RequestParam(value = "scriptLanguages", required = false) - String scriptLanguages, - @ApiParam(value = "资源范围类型") - @RequestParam(value = "scopeType", required = false) - String scopeType, - @ApiParam(value = "资源范围ID") - @RequestParam(value = "scopeId", required = false) - String scopeId - ); - - @ApiOperation(value = "保存用户自定义的脚本模板", produces = "application/json") - @PostMapping - Response saveScriptTemplate( + // 标准接口19 + @ApiOperation(value = "保存多个配置项，返回成功保存的配置项内容", produces = "application/json") + @PostMapping("") + Response>> saveCustomSettings( @ApiParam(value = "用户名，网关自动传入", required = true) @RequestHeader("username") String username, - @ApiParam(value = "创建或更新请求体", required = true) + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId, + @ApiParam(value = "保存配置项请求体", required = true) @RequestBody - ScriptTemplateCreateUpdateReq req + SaveCustomSettingsReq req ); - @ApiOperation(value = "渲染自定义的脚本模板", produces = "application/json") - @PostMapping("/render") - Response renderScriptTemplate( - @ApiParam(value = "用户名，网关自动传入", required = true) + // 标准接口20 + @ApiOperation(value = "批量获取多个配置项内容，返回配置项内容Map，Key为配置模块", produces = "application/json") + @PostMapping("/batchGet") + Response>> batchGetCustomSettings( + @ApiParam("用户名，网关自动传入") @RequestHeader("username") String username, - @ApiParam(value = "脚本模板渲染请求", required = true) - @RequestBody - ScriptTemplateRenderReq req - ); + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId, + @ApiParam(value = "批量获取配置项请求体", required = true) + @RequestBody(required = false) + BatchGetCustomSettingsReq req); - @ApiOperation(value = "获取用户自定义的脚本模板变量", produces = "application/json") - @GetMapping("/variables") - Response> listScriptTemplateVariables( + // 标准接口21 + @ApiOperation(value = "删除多个模块配置项内容，返回删除成功的模块配置数量", produces = "application/json") + @DeleteMapping("") + Response deleteCustomSettings( @ApiParam("用户名，网关自动传入") @RequestHeader("username") - String username - ); + String username, + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId, + @ApiParam(value = "删除配置项请求体", required = true) + @RequestBody + DeleteCustomSettingsReq req); } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebCustomSettingsScriptTemplateResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebCustomSettingsScriptTemplateResource.java new file mode 100644 index 0000000000..9304a62a29 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebCustomSettingsScriptTemplateResource.java @@ -0,0 +1,109 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.api.web; + +import com.tencent.bk.job.common.annotation.WebAPI; +import com.tencent.bk.job.common.model.Response; +import com.tencent.bk.job.manage.model.web.request.customsetting.ScriptTemplateCreateUpdateReq; +import com.tencent.bk.job.manage.model.web.request.customsetting.ScriptTemplateRenderReq; +import com.tencent.bk.job.manage.model.web.vo.customsetting.ScriptTemplateVO; +import com.tencent.bk.job.manage.model.web.vo.customsetting.ScriptTemplateVariableVO; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; + +@Api(tags = {"job-manage:web:CustomSettings_ScriptTemplate"}) +@RequestMapping("/web/customSettings/scriptTemplate") +@RestController +@WebAPI +public interface WebCustomSettingsScriptTemplateResource { + + @ApiOperation(value = "获取用户自定义的脚本模板", produces = "application/json") + @GetMapping + Response> listUserCustomScriptTemplate( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiParam("脚本类型，1:shell,2:bat,3:perl,4:python,5:PowerShell,6:sql;支持传入多个,用英文逗号分隔;如果不传入任何值，默认返回全部脚本类型的模板") + @RequestParam(value = "scriptLanguages", required = false) + String scriptLanguages); + + @ApiOperation(value = "获取渲染后的用户自定义的脚本模板", produces = "application/json") + @GetMapping("/rendered") + Response> listRenderedUserCustomScriptTemplate( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiParam("脚本类型，1:shell,2:bat,3:perl,4:python,5:PowerShell,6:sql;支持传入多个,用英文逗号分隔;如果不传入任何值，默认返回全部脚本类型的模板") + @RequestParam(value = "scriptLanguages", required = false) + String scriptLanguages, + @ApiParam(value = "资源范围类型") + @RequestParam(value = "scopeType", required = false) + String scopeType, + @ApiParam(value = "资源范围ID") + @RequestParam(value = "scopeId", required = false) + String scopeId + ); + + @ApiOperation(value = "保存用户自定义的脚本模板", produces = "application/json") + @PostMapping + Response saveScriptTemplate( + @ApiParam(value = "用户名，网关自动传入", required = true) + @RequestHeader("username") + String username, + @ApiParam(value = "创建或更新请求体", required = true) + @RequestBody + ScriptTemplateCreateUpdateReq req + ); + + @ApiOperation(value = "渲染自定义的脚本模板", produces = "application/json") + @PostMapping("/render") + Response renderScriptTemplate( + @ApiParam(value = "用户名，网关自动传入", required = true) + @RequestHeader("username") + String username, + @ApiParam(value = "脚本模板渲染请求", required = true) + @RequestBody + ScriptTemplateRenderReq req + ); + + @ApiOperation(value = "获取用户自定义的脚本模板变量", produces = "application/json") + @GetMapping("/variables") + Response> listScriptTemplateVariables( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username + ); + +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebDangerousRuleResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebDangerousRuleResource.java index 88c2617926..736e142190 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebDangerousRuleResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebDangerousRuleResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -41,6 +41,7 @@ import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; import org.springframework.web.bind.annotation.RestController; import java.util.List; @@ -56,19 +57,45 @@ public interface WebDangerousRuleResource { Response> listDangerousRules( @ApiParam("用户名，网关自动传入") @RequestHeader("username") - String username + String username, + @ApiParam(value = "表达式：模糊搜索") + @RequestParam(value = "expression", required = false) + String expression, + @ApiParam("规则说明：模糊搜索") + @RequestParam(value = "description", required = false) + String description, + @ApiParam("脚本类型：可多选") + @RequestParam(value = "scriptTypeList", required = false) + List scriptTypeList, + @ApiParam("动作：可多选") + @RequestParam(value = "action", required = false) + List action ); - @ApiOperation(value = "添加/修改高危语句规则", produces = "application/json") + @ApiOperation(value = "创建高危语句规则", produces = "application/json") @PostMapping - Response addOrUpdateDangerousRule( + Response createDangerousRule( + @ApiParam(value = "用户名，网关自动传入", required = true) + @RequestHeader("username") + String username, + @ApiParam(value = "创建或更新请求体", required = true) + @RequestBody + @Validated AddOrUpdateDangerousRuleReq req + ); + + @ApiOperation(value = "修改高危语句规则", produces = "application/json") + @PutMapping("/{id}") + Response updateDangerousRule( @ApiParam(value = "用户名，网关自动传入", required = true) @RequestHeader("username") String username, + @ApiParam("高危语句规则ID") + @PathVariable("id") + Long id, @ApiParam(value = "创建或更新请求体", required = true) @RequestBody - @Validated AddOrUpdateDangerousRuleReq req + @Validated AddOrUpdateDangerousRuleReq req ); diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebEnvResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebEnvResource.java new file mode 100644 index 0000000000..fa05ebf5cb --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebEnvResource.java @@ -0,0 +1,62 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.api.web; + +import com.tencent.bk.job.common.annotation.WebAPI; +import com.tencent.bk.job.common.model.Response; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import springfox.documentation.annotations.ApiIgnore; + +import java.util.Map; + +/** + * Job 环境相关 Web API + */ +@Api(tags = {"job-manage:web:env"}) +@RequestMapping("/web/env") +@WebAPI +public interface WebEnvResource { + + + /** + * 获取Job环境的属性 + * + * @param username 用户名 + * @return Map, key: 属性名, value: 属性值 + */ + @ApiOperation(value = "获取Job环境的属性", produces = "application/json") + @GetMapping("/properties") + Response> getJobEnvProperties( + @ApiIgnore + @ApiParam(value = "用户名，网关自动传入", required = true) + @RequestHeader("username") + String username + ); +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebFeatureToggleResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebFeatureToggleResource.java new file mode 100644 index 0000000000..53cdfd3b50 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebFeatureToggleResource.java @@ -0,0 +1,59 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.api.web; + +import com.tencent.bk.job.common.annotation.WebAPI; +import com.tencent.bk.job.common.model.Response; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import springfox.documentation.annotations.ApiIgnore; + +import java.util.Map; + +/** + * Job 特性开关 Web API + */ +@Api(tags = {"job-manage:web:feature"}) +@RequestMapping("/web/feature/toggle") +@WebAPI +public interface WebFeatureToggleResource { + + + /** + * 获取特性开关配置 + */ + @ApiOperation(value = "获取Job功能特性开关配置", produces = "application/json") + @GetMapping("/list") + Response> listFeatureToggle( + @ApiIgnore + @ApiParam(value = "用户名，网关自动传入", required = true) + @RequestHeader("username") + String username + ); +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebFileUploadResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebFileUploadResource.java index 8cbaa9114f..47b26684a9 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebFileUploadResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebFileUploadResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebGlobalSettingsQueryResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebGlobalSettingsQueryResource.java index 54d182dd0e..0b7811cce4 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebGlobalSettingsQueryResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebGlobalSettingsQueryResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,11 +24,13 @@ package com.tencent.bk.job.manage.api.web; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; import com.tencent.bk.job.common.annotation.WebAPI; +import com.tencent.bk.job.common.constant.CompatibleType; import com.tencent.bk.job.common.model.Response; import com.tencent.bk.job.manage.model.web.vo.globalsetting.AccountNameRulesWithDefaultVO; import com.tencent.bk.job.manage.model.web.vo.globalsetting.NotifyChannelWithIconVO; -import com.tencent.bk.job.manage.model.web.vo.globalsetting.TitleFooterVO; +import com.tencent.bk.job.manage.model.web.vo.globalsetting.PlatformInfoVO; import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; @@ -89,11 +91,11 @@ Response getApplyBusinessUrl( @ApiParam("用户名，网关自动传入") @RequestHeader("username") String username, - @ApiParam(value = "资源范围类型", required = true) - @RequestParam(value = "scopeType") + @ApiParam(value = "资源范围类型") + @RequestParam(value = "scopeType", required = false) String scopeType, - @ApiParam(value = "资源范围ID", required = true) - @RequestParam(value = "scopeId") + @ApiParam(value = "资源范围ID") + @RequestParam(value = "scopeId", required = false) String scopeId ); @@ -113,9 +115,11 @@ Response getCMDBAppIndexUrl( ); - @ApiOperation(value = "获取Title与Footer", produces = "application/json") - @GetMapping("/titleFooter") - Response getTitleFooter(); + @CompatibleImplementation(name = "platform_info", deprecatedVersion = "3.11.x", type = CompatibleType.DEPLOY, + explain = "发布完成后可以删除") + @ApiOperation(value = "获取渲染后的平台设置", produces = "application/json") + @GetMapping("/platformInfo") + Response getRenderedPlatformInfo(); @ApiOperation(value = "获取文档中心根路径", produces = "application/json") diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebGlobalSettingsResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebGlobalSettingsResource.java index 19105e77e2..c593f0f334 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebGlobalSettingsResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebGlobalSettingsResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,12 +24,13 @@ package com.tencent.bk.job.manage.api.web; +import com.tencent.bk.job.common.annotation.CompatibleImplementation; import com.tencent.bk.job.common.annotation.WebAPI; +import com.tencent.bk.job.common.constant.CompatibleType; import com.tencent.bk.job.common.model.Response; import com.tencent.bk.job.manage.model.web.request.globalsetting.AccountNameRulesReq; import com.tencent.bk.job.manage.model.web.request.globalsetting.FileUploadSettingReq; import com.tencent.bk.job.manage.model.web.request.globalsetting.HistoryExpireReq; -import com.tencent.bk.job.manage.model.web.request.globalsetting.SetTitleFooterReq; import com.tencent.bk.job.manage.model.web.request.notify.ChannelTemplatePreviewReq; import com.tencent.bk.job.manage.model.web.request.notify.ChannelTemplateReq; import com.tencent.bk.job.manage.model.web.request.notify.NotifyBlackUsersReq; @@ -37,7 +38,7 @@ import com.tencent.bk.job.manage.model.web.vo.globalsetting.AccountNameRulesWithDefaultVO; import com.tencent.bk.job.manage.model.web.vo.globalsetting.FileUploadSettingVO; import com.tencent.bk.job.manage.model.web.vo.globalsetting.NotifyChannelWithIconVO; -import com.tencent.bk.job.manage.model.web.vo.globalsetting.TitleFooterWithDefaultVO; +import com.tencent.bk.job.manage.model.web.vo.globalsetting.PlatformInfoWithDefaultVO; import com.tencent.bk.job.manage.model.web.vo.notify.ChannelTemplateDetailWithDefaultVO; import com.tencent.bk.job.manage.model.web.vo.notify.ChannelTemplateStatusVO; import com.tencent.bk.job.manage.model.web.vo.notify.NotifyBlackUserInfoVO; @@ -232,20 +233,11 @@ Response getFileUploadSettings( String username ); - @ApiOperation(value = "设置Title与Footer", produces = "application/json") - @PostMapping("/titleFooter") - Response setTitleFooter( - @ApiParam(value = "用户名，网关自动传入", required = true) - @RequestHeader("username") - String username, - @ApiParam(value = "设置Title与Footer请求体", required = true) - @RequestBody - SetTitleFooterReq req - ); - - @ApiOperation(value = "获取Title与Footer", produces = "application/json") - @GetMapping("/titleFooterWithDefault") - Response getTitleFooterWithDefault( + @CompatibleImplementation(name = "platform_info", deprecatedVersion = "3.11.x", type = CompatibleType.DEPLOY, + explain = "发布完成后可以删除") + @ApiOperation(value = "获取平台信息-包含默认配置", produces = "application/json") + @GetMapping("/platformInfoWithDefault") + Response getPlatformInfoWithDefault( @ApiParam(value = "用户名，网关自动传入", required = true) @RequestHeader("username") String username diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebHostResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebHostResource.java new file mode 100644 index 0000000000..17928ebead --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebHostResource.java @@ -0,0 +1,323 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.api.web; + +import com.tencent.bk.job.common.annotation.WebAPI; +import com.tencent.bk.job.common.model.PageData; +import com.tencent.bk.job.common.model.Response; +import com.tencent.bk.job.common.model.dto.AppResourceScope; +import com.tencent.bk.job.common.model.vo.HostInfoVO; +import com.tencent.bk.job.manage.model.web.request.AgentStatisticsReq; +import com.tencent.bk.job.manage.model.web.request.HostCheckReq; +import com.tencent.bk.job.manage.model.web.request.chooser.ListTopologyTreesReq; +import com.tencent.bk.job.manage.model.web.request.chooser.host.GetHostAgentStatisticsByDynamicGroupsReq; +import com.tencent.bk.job.manage.model.web.request.chooser.host.GetHostAgentStatisticsByNodesReq; +import com.tencent.bk.job.manage.model.web.request.chooser.host.HostDetailReq; +import com.tencent.bk.job.manage.model.web.request.chooser.host.HostIdWithMeta; +import com.tencent.bk.job.manage.model.web.request.chooser.host.ListDynamicGroupsReq; +import com.tencent.bk.job.manage.model.web.request.chooser.host.ListHostByBizTopologyNodesReq; +import com.tencent.bk.job.manage.model.web.request.chooser.host.PageListHostsByDynamicGroupReq; +import com.tencent.bk.job.manage.model.web.request.chooser.host.QueryNodesPathReq; +import com.tencent.bk.job.manage.model.web.vo.CcTopologyNodeVO; +import com.tencent.bk.job.manage.model.web.vo.DynamicGroupBasicVO; +import com.tencent.bk.job.manage.model.web.vo.common.AgentStatistics; +import com.tencent.bk.job.manage.model.web.vo.ipchooser.DynamicGroupHostStatisticsVO; +import com.tencent.bk.job.manage.model.web.vo.ipchooser.NodeHostStatisticsVO; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestAttribute; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import springfox.documentation.annotations.ApiIgnore; + +import java.util.List; + +/** + * 主机管理 WEB API + */ +@Api(tags = {"job-manage:web:Host_Management"}) +@RequestMapping("/web") +@RestController +@WebAPI +public interface WebHostResource { + + @ApiOperation(value = "查询主机统计信息") + @PostMapping(value = {"/scope/{scopeType}/{scopeId}/host/statistics"}) + Response agentStatistics( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId, + @ApiParam(value = "请求体", required = true) + @RequestBody + AgentStatisticsReq agentStatisticsReq + ); + + + @ApiOperation(value = "批量获取动态分组信息(不含主机)") + @GetMapping(value = {"/scope/{scopeType}/{scopeId}/dynamicGroup"}) + Response> listAllDynamicGroups( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId + ); + + // 标准接口1 + @ApiOperation(value = "获取拓扑树（含各节点主机数）", produces = "application/json") + @PostMapping(value = {"/scope/{scopeType}/{scopeId}/topology/hostCount"}) + Response> listTopologyHostCountTrees( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId, + @ApiParam(value = "资源范围信息") + @RequestBody(required = false) + ListTopologyTreesReq req + ); + + // 标准接口2 + @ApiOperation(value = "获取多个节点的拓扑路径", produces = "application/json") + @PostMapping(value = {"/scope/{scopeType}/{scopeId}/nodes/queryPath"}) + Response>> queryNodePaths( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId, + @ApiParam(value = "获取多个节点的拓扑路径请求体", required = true) + @RequestBody + QueryNodesPathReq req + ); + + // 标准接口3 + @ApiOperation(value = "IP选择器根据拓扑节点集合获取机器列表", produces = "application/json") + @PostMapping(value = {"/scope/{scopeType}/{scopeId}/topology/hosts/nodes"}) + Response> listHostByBizTopologyNodes( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId, + @ApiParam(value = "拓扑节点集合与分页信息", required = true) + @RequestBody + ListHostByBizTopologyNodesReq req + ); + + // 标准接口4 + @ApiOperation(value = "IP选择器根据拓扑节点集合获取机器hostIds" + , produces = "application/json") + @PostMapping(value = {"/scope/{scopeType}/{scopeId}/topology/hostIds/nodes"}) + Response> listHostIdByBizTopologyNodes( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId, + @ApiParam(value = "拓扑节点集合与分页信息", required = true) + @RequestBody + ListHostByBizTopologyNodesReq req + ); + + // 标准接口5 + @ApiOperation(value = "获取多个节点下的主机统计信息", produces = "application/json") + @PostMapping(value = {"/scope/{scopeType}/{scopeId}/host/agentStatistics/nodes"}) + Response> getHostAgentStatisticsByNodes( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId, + @ApiParam(value = "要查询主机Agent状态统计信息的拓扑节点列表(将拓扑树节点中的objectId与instanceId传入)", required = true) + @RequestBody + GetHostAgentStatisticsByNodesReq req + ); + + // 标准接口6 + @ApiOperation(value = "批量获取动态分组信息(不含主机)") + @PostMapping(value = {"/scope/{scopeType}/{scopeId}/dynamicGroups"}) + Response> listDynamicGroups( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId, + @ApiParam(value = "批量获取动态分组信息请求体", required = true) + @RequestBody(required = false) + ListDynamicGroupsReq req + ); + + // 标准接口7 + @ApiOperation(value = "获取多个动态分组下的主机Agent状态统计信息", produces = "application/json") + @PostMapping(value = {"/scope/{scopeType}/{scopeId}/host/agentStatistics/dynamicGroups"}) + Response> getHostAgentStatisticsByDynamicGroups( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId, + @ApiParam(value = "请求体", required = true) + @RequestBody + GetHostAgentStatisticsByDynamicGroupsReq req + ); + + // 标准接口8 + @ApiOperation(value = "分页查询某个动态分组下的主机列表") + @PostMapping(value = {"/scope/{scopeType}/{scopeId}/hosts/dynamicGroup"}) + Response> pageListHostsByDynamicGroup( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId, + @ApiParam(value = "分页查询某些动态分组下的主机列表请求体", required = true) + @RequestBody + @Validated + PageListHostsByDynamicGroupReq req + ); + + // 标准接口9 + @ApiOperation(value = "根据用户选择/输入的主机信息获取真实存在的机器信息") + @PostMapping(value = {"/scope/{scopeType}/{scopeId}/host/check"}) + Response> checkHosts( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId, + @ApiParam(value = "用户选择/输入的主机信息", required = true) + @RequestBody + HostCheckReq req + ); + + // 标准接口10 + @ApiOperation(value = "根据hostId批量查询主机详情信息") + @PostMapping(value = {"/scope/{scopeType}/{scopeId}/hosts/details"}) + Response> getHostDetails( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId, + @ApiParam(value = "主机ID及元数据信息", required = true) + @RequestBody + HostDetailReq req + ); + +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebIndexResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebIndexResource.java index be4cdd65c1..153159d5d8 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebIndexResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebIndexResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -29,7 +29,7 @@ import com.tencent.bk.job.common.model.Response; import com.tencent.bk.job.common.model.dto.AppResourceScope; import com.tencent.bk.job.common.model.vo.HostInfoVO; -import com.tencent.bk.job.manage.model.web.vo.index.AgentStatistics; +import com.tencent.bk.job.manage.model.web.vo.common.AgentStatistics; import com.tencent.bk.job.manage.model.web.vo.index.GreetingVO; import com.tencent.bk.job.manage.model.web.vo.index.JobAndScriptStatistics; import com.tencent.bk.job.manage.model.web.vo.task.TaskTemplateVO; diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebNoticeResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebNoticeResource.java new file mode 100644 index 0000000000..f59761617b --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebNoticeResource.java @@ -0,0 +1,67 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.api.web; + +import com.tencent.bk.job.common.annotation.WebAPI; +import com.tencent.bk.job.common.model.Response; +import com.tencent.bk.job.manage.model.web.vo.notice.AnnouncementVO; +import io.swagger.annotations.Api; +import io.swagger.annotations.ApiOperation; +import io.swagger.annotations.ApiParam; +import org.springframework.validation.annotation.Validated; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +import javax.validation.constraints.Min; +import java.util.List; + +/** + * 对接蓝鲸消息中心相关的 WEB API + */ +@Validated +@Api(tags = {"job-manage:web:Notice"}) +@RequestMapping("/web/notice") +@RestController +@WebAPI +public interface WebNoticeResource { + + @ApiOperation(value = "获获取公告列表（返回码1217001表示消息通知中心API不存在，该环境未对接消息通知中心，需要兼容处理）", produces = "application/json") + @GetMapping(value = {"/announcement/currentAnnouncements"}) + Response> getCurrentAnnouncements( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @Min(message = "{validation.constraints.InvalidAnnouncementsOffset.message}", value = 0L) + @RequestParam(value = "offset", required = false) + Integer offset, + @Min(message = "{validation.constraints.InvalidAnnouncementsLimit.message}", value = 1L) + @RequestParam(value = "limit", required = false) + Integer limit + ); + +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebNotifyResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebNotifyResource.java index bd71662c73..1861a5ac92 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebNotifyResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebNotifyResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebPermissionResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebPermissionResource.java index 7cfbf382dc..cf0a58122c 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebPermissionResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebPermissionResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -40,7 +40,7 @@ import org.springframework.web.bind.annotation.RestController; @Api(tags = {"job-manage:web:Permission"}) -@RequestMapping("/web/permission/") +@RequestMapping("/web/permission/manage") @RestController @WebAPI public interface WebPermissionResource { @@ -90,11 +90,11 @@ Response checkOperationPermission( Response checkOperationPermission( @ApiParam("用户名，网关自动传入") @RequestHeader("username") String username, - @ApiParam(value = "资源范围类型", required = true) - @RequestParam(value = "scopeType") + @ApiParam(value = "资源范围类型") + @RequestParam(value = "scopeType", required = false) String scopeType, - @ApiParam(value = "资源范围ID", required = true) - @RequestParam(value = "scopeId") + @ApiParam(value = "资源范围ID") + @RequestParam(value = "scopeId", required = false) String scopeId, @ApiParam("操作ID,取值为: [script/create,script/view," + "script/edit,script/delete,script/execute," + diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebPublicScriptResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebPublicScriptResource.java index e13a5d89d7..5ad3361dfc 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebPublicScriptResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebPublicScriptResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,10 +27,13 @@ import com.tencent.bk.job.common.annotation.WebAPI; import com.tencent.bk.job.common.model.PageData; import com.tencent.bk.job.common.model.Response; -import com.tencent.bk.job.manage.model.web.request.ScriptCreateUpdateReq; +import com.tencent.bk.job.common.validation.Create; +import com.tencent.bk.job.common.validation.Update; +import com.tencent.bk.job.manage.model.web.request.ScriptCreateReq; import com.tencent.bk.job.manage.model.web.request.ScriptInfoUpdateReq; import com.tencent.bk.job.manage.model.web.request.ScriptSyncReq; import com.tencent.bk.job.manage.model.web.request.ScriptTagBatchPatchReq; +import com.tencent.bk.job.manage.model.web.request.ScriptVersionCreateUpdateReq; import com.tencent.bk.job.manage.model.web.vo.BasicScriptVO; import com.tencent.bk.job.manage.model.web.vo.ScriptVO; import com.tencent.bk.job.manage.model.web.vo.TagCountVO; @@ -41,6 +44,7 @@ import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; +import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; @@ -95,7 +99,7 @@ Response getOnlineScriptVersionByScriptId( @ApiParam(value = "脚本ID", required = true, example = "1") @PathVariable("scriptId") String scriptId); - @ApiOperation(value = "获取脚本列表", produces = "application/json") + @ApiOperation(value = "获取公共脚本列表", produces = "application/json") @GetMapping("/script/list") Response> listPageScript( @ApiParam("用户名，网关自动传入") @@ -159,7 +163,7 @@ Response> listScriptVersion( @ApiOperation(value = "根据条件查询业务下的脚本名称列表", produces = "application/json") @GetMapping("/scriptNames") - Response listAppScriptNames( + Response> listPublicScriptNames( @ApiParam("用户名，网关自动传入") @RequestHeader("username") String username, @ApiParam("脚本名称") @@ -175,7 +179,7 @@ Response> listScriptOnline( @ApiOperation(value = "更新脚本元数据，比如脚本描述、名称、标签", produces = "application/json") @PutMapping("/script/{scriptId}/info") - Response updateScriptInfo( + Response updateScriptInfo( @ApiParam("用户名，网关自动传入") @RequestHeader("username") String username, @ApiParam(value = "脚本ID", required = true, example = "uuid") @@ -185,13 +189,47 @@ Response updateScriptInfo( @RequestBody ScriptInfoUpdateReq scriptInfoUpdateReq); - @ApiOperation(value = "更新脚本", produces = "application/json") + @ApiOperation(value = "创建脚本", produces = "application/json") @PostMapping("/script") Response saveScript( @ApiParam("用户名，网关自动传入") - @RequestHeader("username") String username, - @ApiParam(value = "新增/更新的脚本对象", name = "scriptCreateUpdateReq", required = true) - @RequestBody ScriptCreateUpdateReq scriptCreateUpdateReq); + @RequestHeader("username") + String username, + @ApiParam(value = "新增/更新的脚本对象", name = "request", required = true) + @RequestBody + @Validated + ScriptCreateReq request + ); + + @ApiOperation(value = "新增脚本版本", produces = "application/json") + @PostMapping("/script/{scriptId}/scriptVersion") + Response saveScriptVersion( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @PathVariable(value = "scriptId") + String scriptId, + @ApiParam(value = "新增脚本版本请求", name = "request", required = true) + @RequestBody + @Validated(Create.class) + ScriptVersionCreateUpdateReq request + ); + + @ApiOperation(value = "更新脚本版本", produces = "application/json") + @PutMapping("/script/{scriptId}/scriptVersion/{scriptVersionId}") + Response updateScriptVersion( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @PathVariable(value = "scriptId") + String scriptId, + @PathVariable(value = "scriptVersionId") + Long scriptVersionId, + @ApiParam(value = "更新脚本版本请求", name = "request", required = true) + @RequestBody + @Validated(Update.class) + ScriptVersionCreateUpdateReq request + ); @ApiOperation(value = "上线脚本", produces = "application/json") @PutMapping("/script/{scriptId}/scriptVersion/{scriptVersionId}/publish") @@ -263,9 +301,9 @@ Response> syncScripts( Response getPublicScriptCiteCount( @ApiParam("用户名，网关自动传入") @RequestHeader("username") String username, - @ApiParam(value = "脚本ID", required = false, example = "1") + @ApiParam(value = "脚本ID", example = "1") @RequestParam(value = "scriptId") String scriptId, - @ApiParam(value = "脚本版本ID", required = false, example = "1") + @ApiParam(value = "脚本版本ID", example = "1") @RequestParam(value = "scriptVersionId", required = false) Long scriptVersionId ); @@ -274,9 +312,9 @@ Response getPublicScriptCiteCount( Response getPublicScriptCiteInfo( @ApiParam("用户名，网关自动传入") @RequestHeader("username") String username, - @ApiParam(value = "脚本ID", required = false, example = "1") + @ApiParam(value = "脚本ID", example = "1") @RequestParam("scriptId") String scriptId, - @ApiParam(value = "脚本版本ID", required = false, example = "1") + @ApiParam(value = "脚本版本ID", example = "1") @RequestParam(value = "scriptVersionId", required = false) Long scriptVersionId ); diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebPublicTagResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebPublicTagResource.java index c0a318d9f5..8e9c5b948f 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebPublicTagResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebPublicTagResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebScriptResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebScriptResource.java index 0aeabc8842..42c26d3719 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebScriptResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebScriptResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,11 +28,14 @@ import com.tencent.bk.job.common.model.PageData; import com.tencent.bk.job.common.model.Response; import com.tencent.bk.job.common.model.dto.AppResourceScope; +import com.tencent.bk.job.common.validation.Create; +import com.tencent.bk.job.common.validation.Update; import com.tencent.bk.job.manage.model.web.request.ScriptCheckReq; -import com.tencent.bk.job.manage.model.web.request.ScriptCreateUpdateReq; +import com.tencent.bk.job.manage.model.web.request.ScriptCreateReq; import com.tencent.bk.job.manage.model.web.request.ScriptInfoUpdateReq; import com.tencent.bk.job.manage.model.web.request.ScriptSyncReq; import com.tencent.bk.job.manage.model.web.request.ScriptTagBatchPatchReq; +import com.tencent.bk.job.manage.model.web.request.ScriptVersionCreateUpdateReq; import com.tencent.bk.job.manage.model.web.vo.BasicScriptVO; import com.tencent.bk.job.manage.model.web.vo.ScriptCheckResultItemVO; import com.tencent.bk.job.manage.model.web.vo.ScriptVO; @@ -44,6 +47,7 @@ import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; +import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; @@ -112,7 +116,7 @@ Response getScript( @ApiOperation(value = "根据脚本ID获取脚本基本信息", produces = "application/json") @GetMapping(value = {"/scope/{scopeType}/{scopeId}/script/basic/{scriptId}"}) - Response getScriptBasicInfo( + Response getScriptBasicInfo( @ApiParam("用户名，网关自动传入") @RequestHeader("username") String username, @@ -146,10 +150,7 @@ Response getOnlineScriptVersionByScriptId( @PathVariable(value = "scopeId") String scopeId, @ApiParam(value = "脚本ID", required = true, example = "1") - @PathVariable("scriptId") String scriptId, - @ApiParam(value = "是否公共脚本") - @RequestParam(value = "publicScript", required = false, defaultValue = "false") - Boolean publicScript + @PathVariable("scriptId") String scriptId ); @ApiOperation(value = "获取脚本列表", produces = "application/json") @@ -166,9 +167,6 @@ Response> listPageScript( @ApiParam(value = "资源范围ID", required = true) @PathVariable(value = "scopeId") String scopeId, - @ApiParam(value = "是否公共脚本") - @RequestParam(value = "publicScript", required = false, defaultValue = "false") - Boolean publicScript, @ApiParam(value = "脚本名称") @RequestParam(value = "name", required = false) String name, @@ -252,7 +250,7 @@ Response> listScriptVersion( @ApiOperation(value = "更新脚本元数据，比如脚本描述、名称、标签", produces = "application/json") @PutMapping("/scope/{scopeType}/{scopeId}/script/{scriptId}/info") - Response updateScriptInfo( + Response updateScriptInfo( @ApiParam("用户名，网关自动传入") @RequestHeader("username") String username, @@ -268,13 +266,13 @@ Response updateScriptInfo( @ApiParam(value = "脚本ID", required = true, example = "uuid") @PathVariable("scriptId") String scriptId, - @ApiParam(value = "脚本元数据更新请求报文", name = "scriptInfoUpdateReq", required = true) + @ApiParam(value = "脚本元数据更新请求报文", name = "request", required = true) @RequestBody - ScriptInfoUpdateReq scriptInfoUpdateReq + ScriptInfoUpdateReq request ); - @ApiOperation(value = "更新脚本", produces = "application/json") + @ApiOperation(value = "创建脚本", produces = "application/json") @PostMapping("/scope/{scopeType}/{scopeId}/script") Response saveScript( @ApiParam("用户名，网关自动传入") @@ -289,9 +287,57 @@ Response saveScript( @ApiParam(value = "资源范围ID", required = true) @PathVariable(value = "scopeId") String scopeId, - @ApiParam(value = "新增/更新的脚本对象", name = "scriptCreateUpdateReq", required = true) + @ApiParam(value = "新增/更新的脚本对象", name = "request", required = true) @RequestBody - ScriptCreateUpdateReq scriptCreateUpdateReq + ScriptCreateReq request + ); + + @ApiOperation(value = "新增脚本版本", produces = "application/json") + @PostMapping("/scope/{scopeType}/{scopeId}/script/{scriptId}/scriptVersion") + Response saveScriptVersion( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId, + @PathVariable(value = "scriptId") + String scriptId, + @ApiParam(value = "新增脚本版本请求", name = "request", required = true) + @RequestBody + @Validated(Create.class) + ScriptVersionCreateUpdateReq request + ); + + @ApiOperation(value = "更新脚本版本", produces = "application/json") + @PutMapping("/scope/{scopeType}/{scopeId}/script/{scriptId}/scriptVersion/{scriptVersionId}") + Response updateScriptVersion( + @ApiParam("用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId, + @PathVariable(value = "scriptId") + String scriptId, + @PathVariable(value = "scriptVersionId") + Long scriptVersionId, + @ApiParam(value = "更新脚本版本请求", name = "request", required = true) + @RequestBody + @Validated(Update.class) + ScriptVersionCreateUpdateReq request ); @ApiOperation(value = "上线脚本", produces = "application/json") @@ -382,7 +428,7 @@ Response deleteScriptByScriptVersionId( @ApiOperation(value = "根据条件查询业务下的脚本名称列表", produces = "application/json") @GetMapping(value = {"/scope/{scopeType}/{scopeId}/scriptNames"}) - Response listAppScriptNames( + Response> listAppScriptNames( @ApiParam("用户名，网关自动传入") @RequestHeader("username") String username, diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebServiceInfoResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebServiceInfoResource.java index 6ccd37b784..864808fe5c 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebServiceInfoResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebServiceInfoResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebTagResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebTagResource.java index dc21cfe929..7326584ecf 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebTagResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebTagResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebTaskPlanResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebTaskPlanResource.java index d07b7c2db3..8c97d644a6 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebTaskPlanResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebTaskPlanResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -30,6 +30,7 @@ import com.tencent.bk.job.common.model.dto.AppResourceScope; import com.tencent.bk.job.manage.model.web.request.TaskPlanCreateUpdateReq; import com.tencent.bk.job.manage.model.web.request.TaskVariableValueUpdateReq; +import com.tencent.bk.job.manage.model.web.vo.task.TaskPlanBasicInfoVO; import com.tencent.bk.job.manage.model.web.vo.task.TaskPlanSyncInfoVO; import com.tencent.bk.job.manage.model.web.vo.task.TaskPlanVO; import io.swagger.annotations.Api; @@ -183,7 +184,7 @@ Response getDebugPlan( @ApiOperation(value = "更新执行方案", produces = "application/json") @PutMapping("/scope/{scopeType}/{scopeId}/task/plan/{templateId}/{planId}") - Response savePlan( + Response updatePlan( @ApiParam(value = "用户名，网关自动传入") @RequestHeader("username") String username, @@ -199,7 +200,7 @@ Response savePlan( @ApiParam(value = "模版 ID", required = true) @PathVariable("templateId") Long templateId, - @ApiParam(value = "执行方案 ID，新建时填 0", required = true) + @ApiParam(value = "执行方案 ID", required = true) @PathVariable("planId") Long planId, @ApiParam(value = "更新的执行方案对象", name = "planCreateUpdateReq") @@ -207,6 +208,29 @@ Response savePlan( TaskPlanCreateUpdateReq taskPlanCreateUpdateReq ); + @ApiOperation(value = "新增执行方案", produces = "application/json") + @PostMapping("/scope/{scopeType}/{scopeId}/task/plan/{templateId}") + Response createPlan( + @ApiParam(value = "用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId, + @ApiParam(value = "模版 ID", required = true) + @PathVariable("templateId") + Long templateId, + @ApiParam(value = "更新的执行方案对象", name = "planCreateUpdateReq") + @RequestBody + TaskPlanCreateUpdateReq taskPlanCreateUpdateReq + ); + @ApiOperation(value = "删除执行方案", produces = "application/json") @DeleteMapping("/scope/{scopeType}/{scopeId}/task/plan/{templateId}/{planId}") Response deletePlan( @@ -250,6 +274,26 @@ Response> listPlanBasicInfoByIds( String planIds ); + @ApiOperation(value = "根据执行方案 ID 批量拉执行方案基础信息", produces = "application/json") + @GetMapping(value = {"/scope/{scopeType}/{scopeId}/task/plan/basicInfo"}) + Response> listTaskPlanBasicInfoByIds( + @ApiParam(value = "用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId, + @ApiParam(value = "执行方案 ID 列表，逗号分隔", required = true, example = "1,2,3") + @QueryParam("ids") + String planIds + ); + @ApiOperation(value = "检查执行方案名称是否已占用", produces = "application/json") @GetMapping("/scope/{scopeType}/{scopeId}/task/plan/{templateId}/{planId}/check_name") Response checkPlanName( diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebTaskTemplateResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebTaskTemplateResource.java index b8b41c2fb2..2c2216e569 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebTaskTemplateResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebTaskTemplateResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -40,6 +40,7 @@ import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.PutMapping; import org.springframework.web.bind.annotation.RequestAttribute; import org.springframework.web.bind.annotation.RequestBody; @@ -133,9 +134,31 @@ Response getTemplateById( Long templateId ); + + @ApiOperation(value = "新建模版", produces = "application/json") + @PostMapping + Response createTemplate( + @ApiParam(value = "用户名，网关自动传入") + @RequestHeader("username") + String username, + @ApiIgnore + @RequestAttribute(value = "appResourceScope") + AppResourceScope appResourceScope, + @ApiParam(value = "资源范围类型", required = true) + @PathVariable(value = "scopeType") + String scopeType, + @ApiParam(value = "资源范围ID", required = true) + @PathVariable(value = "scopeId") + String scopeId, + @ApiParam(value = "新增模版对象", name = "request", required = true) + @RequestBody + @Validated + TaskTemplateCreateUpdateReq request + ); + @ApiOperation(value = "更新模版", produces = "application/json") @PutMapping("/{templateId}") - Response saveTemplate( + Response updateTemplate( @ApiParam(value = "用户名，网关自动传入") @RequestHeader("username") String username, @@ -148,13 +171,13 @@ Response saveTemplate( @ApiParam(value = "资源范围ID", required = true) @PathVariable(value = "scopeId") String scopeId, - @ApiParam(value = "模版 ID 新建填 0", required = true) + @ApiParam(value = "模版 ID", required = true) @PathVariable("templateId") Long templateId, - @ApiParam(value = "新增/更新的模版对象", name = "templateCreateUpdateReq", required = true) + @ApiParam(value = "新增/更新的模版对象", name = "request", required = true) @RequestBody @Validated - TaskTemplateCreateUpdateReq taskTemplateCreateUpdateReq + TaskTemplateCreateUpdateReq request ); @ApiOperation(value = "删除模版", produces = "application/json") @@ -212,9 +235,9 @@ Response updateTemplateBasicInfo( @ApiParam(value = "模版ID", required = true) @PathVariable("templateId") Long templateId, - @ApiParam(value = "模版元数据更新请求报文", name = "templateBasicInfoUpdateReq", required = true) + @ApiParam(value = "模版元数据更新请求报文", name = "request", required = true) @RequestBody - TemplateBasicInfoUpdateReq templateBasicInfoUpdateReq + TemplateBasicInfoUpdateReq request ); @ApiOperation(value = "新增收藏", produces = "application/json") diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebWhiteIPResource.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebWhiteIPResource.java index 4939119ec1..24a7faf3de 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebWhiteIPResource.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/WebWhiteIPResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -34,10 +34,12 @@ import io.swagger.annotations.Api; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiParam; +import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.PutMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RequestMapping; @@ -91,13 +93,29 @@ Response> listWhiteIP( ); - @ApiOperation(value = "新增/更新IP白名单", produces = "application/json") - @PostMapping("/") - Response saveWhiteIP( + @ApiOperation(value = "新增IP白名单", produces = "application/json") + @PostMapping + Response createWhiteIP( @ApiParam(value = "用户名，网关自动传入", required = true) @RequestHeader("username") String username, @ApiParam(value = "创建或更新请求体", required = true) + @Validated + @RequestBody + WhiteIPRecordCreateUpdateReq createUpdateReq + ); + + @ApiOperation(value = "更新IP白名单", produces = "application/json") + @PutMapping("/{id}") + Response updateWhiteIP( + @ApiParam(value = "用户名，网关自动传入", required = true) + @RequestHeader("username") + String username, + @ApiParam("IP白名单记录ID") + @PathVariable("id") + Long id, + @ApiParam(value = "创建或更新请求体", required = true) + @Validated @RequestBody WhiteIPRecordCreateUpdateReq createUpdateReq ); diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/CredentialTypeEnum.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/CredentialTypeEnum.java deleted file mode 100644 index 99d59d70b6..0000000000 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/CredentialTypeEnum.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.common.consts; - -public enum CredentialTypeEnum { - APP_ID_SECRET_KEY(1, "AccessKey+SecretKey"), - PASSWORD(2, "单一密码"), - USERNAME_PASSWORD(3, "用户名+密码"), - SECRET_KEY(4, "单一SecretKey"); - - private final Integer value; - private final String description; - - public static String getAllNameStr() { - StringBuilder sb = new StringBuilder(); - CredentialTypeEnum[] values = CredentialTypeEnum.values(); - for (int i = 0; i < values.length; i++) { - sb.append(values[i].name()); - if (i < values.length - 1) { - sb.append(","); - } - } - return sb.toString(); - } - - CredentialTypeEnum(Integer type, String description) { - this.value = type; - this.description = description; - } -} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/EnableStatusEnum.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/EnableStatusEnum.java deleted file mode 100644 index d8b3b60779..0000000000 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/EnableStatusEnum.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.common.consts; - -/** - * 启用状态 - */ -public enum EnableStatusEnum { - /** - * 启用 - */ - ENABLED(1), - /** - * 停用 - */ - DISABLED(0); - - private final int value; - - EnableStatusEnum(int val) { - this.value = val; - } - - public int getValue() { - return value; - } -} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/JobResourceStatusEnum.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/JobResourceStatusEnum.java deleted file mode 100644 index 3cceb51a7e..0000000000 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/JobResourceStatusEnum.java +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.common.consts; - -/** - * JOB资源状态 - */ - -public enum JobResourceStatusEnum { - /** - * 未上线 - */ - DRAFT(0), - /** - * 已上线 - */ - ONLINE(1), - /** - * 已下线 - */ - OFFLINE(2), - /** - * 禁用 - */ - DISABLED(3); - - private final int status; - - JobResourceStatusEnum(int status) { - this.status = status; - } - - public static JobResourceStatusEnum getJobResourceStatus(Integer status) { - if (status == null) { - return null; - } - if (status == ONLINE.getValue()) { - return ONLINE; - } else if (status == DRAFT.getValue()) { - return DRAFT; - } else if (status == DISABLED.getValue()) { - return DISABLED; - } else if (status == OFFLINE.getValue()) { - return OFFLINE; - } else { - throw new IllegalArgumentException("No JobResourceStatusEnum constant: " + status); - } - } - - public int getValue() { - return status; - } - - public String getStatusI18nKey() { - if (this == DRAFT) { - return "job.resource.status.draft"; - } else if (this == ONLINE) { - return "job.resource.status.online"; - } else if (this == DISABLED) { - return "job.resource.status.disabled"; - } else if (this == OFFLINE) { - return "job.resource.status.offline"; - } else { - return ""; - } - } -} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/notify/ExecuteStatusEnum.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/notify/ExecuteStatusEnum.java deleted file mode 100644 index 043bf5292a..0000000000 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/notify/ExecuteStatusEnum.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.common.consts.notify; - -import com.tencent.bk.job.common.util.I18nUtil; -import com.tencent.bk.job.manage.model.web.vo.notify.ExecuteStatusVO; -import lombok.AllArgsConstructor; -import lombok.Getter; -import lombok.val; - -import java.util.ArrayList; -import java.util.List; - -@Getter -@AllArgsConstructor -public enum ExecuteStatusEnum { - /** - * 执行成功 - */ - SUCCESS(1, "job.manage.execute.status.success"), - - /** - * 执行失败 - */ - FAIL(2, "job.manage.execute.status.fail"), - - /** - * 等待执行 - */ - READY(3, "job.manage.execute.status.ready"); - - private final int status; - private final String i18nCode; - - public static ExecuteStatusEnum get(int status) { - val values = ExecuteStatusEnum.values(); - for (int i = 0; i < values.length; i++) { - if (values[i].status == status) { - return values[i]; - } - } - return null; - } - - public static String getName(int status) { - val values = ExecuteStatusEnum.values(); - for (int i = 0; i < values.length; i++) { - if (values[i].status == status) { - return values[i].name(); - } - } - return null; - } - - public static List getVOList() { - List resultList = new ArrayList(); - val values = ExecuteStatusEnum.values(); - for (int i = 0; i < values.length; i++) { - resultList.add(new ExecuteStatusVO(values[i].name(), I18nUtil.getI18nMessage(values[i].getI18nCode()))); - } - return resultList; - } -} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/rule/HighRiskGrammarActionEnum.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/rule/HighRiskGrammarActionEnum.java deleted file mode 100644 index 164850e8a6..0000000000 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/rule/HighRiskGrammarActionEnum.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.common.consts.rule; - -import lombok.Getter; - -/** - * 处理动作枚举,1:扫描,2:拦截 - */ -@Getter -public enum HighRiskGrammarActionEnum { - SCAN(1, "scan"), INTERCEPT(2, "intercept"); - - private final Integer code; - private final String name; - - HighRiskGrammarActionEnum(Integer code, String name) { - this.code = code; - this.name = name; - } - - public static String getName(Integer type) { - for (HighRiskGrammarActionEnum highRiskGrammarActionEnum : values()) { - if (highRiskGrammarActionEnum.code.equals(type)) { - return highRiskGrammarActionEnum.getName(); - } - } - return ""; - } - - public static HighRiskGrammarActionEnum valueOf(Integer type) { - for (HighRiskGrammarActionEnum highRiskGrammarActionEnum : values()) { - if (highRiskGrammarActionEnum.code.equals(type)) { - return highRiskGrammarActionEnum; - } - } - return null; - } - - /** - * 判断参数合法性 - */ - public static boolean isValid(Integer code) { - for (HighRiskGrammarActionEnum highRiskGrammarActionEnum : HighRiskGrammarActionEnum.values()) { - if (highRiskGrammarActionEnum.getCode().equals(code)) { - return true; - } - } - return false; - } - -} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/script/ScriptTypeEnum.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/script/ScriptTypeEnum.java deleted file mode 100644 index bc0f87064d..0000000000 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/common/consts/script/ScriptTypeEnum.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.common.consts.script; - -import lombok.Getter; - -/** - * @date 2019/09/19 - */ -@Getter -public enum ScriptTypeEnum { - /** - * 脚本类型枚举类 - */ - SHELL(1, "shell", ".sh"), - BAT(2, "bat", ".bat"), - PERL(3, "perl", ".pl"), - PYTHON(4, "python", ".py"), - POWERSHELL(5, "powershell", ".ps1"), - SQL(6, "sql", ".sql"); - - private final Integer value; - private final String name; - private final String ext; - - ScriptTypeEnum(Integer type, String name, String ext) { - this.value = type; - this.name = name; - this.ext = ext; - } - - public static String getName(Integer type) { - for (ScriptTypeEnum scriptTypeEnum : values()) { - if (scriptTypeEnum.value.equals(type)) { - return scriptTypeEnum.getName(); - } - } - return ""; - } - - public static ScriptTypeEnum valueOf(Integer type) { - for (ScriptTypeEnum scriptTypeEnum : values()) { - if (scriptTypeEnum.value.equals(type)) { - return scriptTypeEnum; - } - } - return null; - } - - public static boolean isValid(Integer type) { - if (type == null) { - return false; - } - return valueOf(type) != null; - } - - public static ScriptTypeEnum getTypeByExt(String ext) { - if (SHELL.ext.equals(ext)) { - return SHELL; - } else if (BAT.ext.equals(ext)) { - return BAT; - } else if (PERL.ext.equals(ext)) { - return PERL; - } else if (PYTHON.ext.equals(ext)) { - return PYTHON; - } else if (POWERSHELL.ext.equals(ext)) { - return POWERSHELL; - } else if (SQL.ext.equals(ext)) { - return SQL; - } else { - return null; - } - } - - public static String getExtByValue(int value) { - ScriptTypeEnum scriptType = valueOf(value); - if (scriptType != null) { - return scriptType.getExt(); - } else { - return ""; - } - } - - /** - * 获取脚本类型的字典名称的排序 - * - * @return 排序 - */ - public static ScriptTypeEnum[] getScriptTypeNameAscSort() { - // 按照脚本语言名称字典顺序排序.Bat(2)->Perl(3)->Powershell(5)->Python(4)->Shell(1)->SQL(6) - return new ScriptTypeEnum[]{ - BAT, PERL, POWERSHELL, PYTHON, SHELL, SQL - }; - } -} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/credential/AccessKeySecretKey.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/credential/AccessKeySecretKey.java index 0db698a943..60def99a2e 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/credential/AccessKeySecretKey.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/credential/AccessKeySecretKey.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/credential/Password.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/credential/Password.java index d287203825..13bf12cb79 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/credential/Password.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/credential/Password.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/credential/SecretKey.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/credential/SecretKey.java index 0d71f33d89..87129cfd58 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/credential/SecretKey.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/credential/SecretKey.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/credential/UsernamePassword.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/credential/UsernamePassword.java index 0f0f9bfc76..bee01a4fac 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/credential/UsernamePassword.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/credential/UsernamePassword.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/EsbAccountDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/EsbAccountDTO.java index a9eb14908a..7eca3c0221 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/EsbAccountDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/EsbAccountDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,6 +25,7 @@ package com.tencent.bk.job.manage.model.esb; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; import com.tencent.bk.job.common.esb.model.EsbAppScopeDTO; import lombok.Getter; import lombok.Setter; @@ -35,16 +36,22 @@ @ToString public class EsbAccountDTO extends EsbAppScopeDTO { + @JsonPropertyDescription("Account ID") private Long id; + @JsonPropertyDescription("Account") private String account; + @JsonPropertyDescription("Account creator") private String creator; + @JsonPropertyDescription("Account OS") private String os; + @JsonPropertyDescription("Account alias") private String alias; + @JsonPropertyDescription("Create time") @JsonProperty("create_time") private String createTime; diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/EsbDBAccountDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/EsbDBAccountDTO.java index 0b9484e243..c30846baa3 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/EsbDBAccountDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/EsbDBAccountDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,8 +27,10 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.tencent.bk.job.common.esb.model.EsbAppScopeDTO; import lombok.Data; +import lombok.EqualsAndHashCode; @Data +@EqualsAndHashCode(callSuper = true) public class EsbDBAccountDTO extends EsbAppScopeDTO { @JsonProperty("db_account_id") diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/EsbJobBasicInfoDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/EsbJobBasicInfoDTO.java index 2846e5e6f7..67a9e534a2 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/EsbJobBasicInfoDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/EsbJobBasicInfoDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/EsbJobDetailDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/EsbJobDetailDTO.java index f526789d72..e704d8cd25 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/EsbJobDetailDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/EsbJobDetailDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,17 +27,15 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.tencent.bk.job.common.esb.model.EsbAppScopeDTO; import lombok.Data; -import lombok.Getter; -import lombok.Setter; +import lombok.EqualsAndHashCode; import java.util.List; /** * 作业执行方案详情 */ +@EqualsAndHashCode(callSuper = true) @Data -@Getter -@Setter public class EsbJobDetailDTO extends EsbAppScopeDTO { /** * id diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/EsbScriptDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/EsbScriptDTO.java index aa5fae4f5b..4ccac5dc92 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/EsbScriptDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/EsbScriptDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/EsbStepDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/EsbStepDTO.java index aa0c4cbf70..9f75c8c8dd 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/EsbStepDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/EsbStepDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/EsbTaskVariableDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/EsbTaskVariableDTO.java index 2bc1ac0ae6..47bbe95e5b 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/EsbTaskVariableDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/EsbTaskVariableDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/request/EsbBaseListRequest.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/request/EsbBaseListRequest.java index eb735ee1aa..c9583949f5 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/request/EsbBaseListRequest.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/request/EsbBaseListRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/request/EsbGetDBAccountListRequest.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/request/EsbGetDBAccountListRequest.java index 7bd59864a4..a7c5a4c73e 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/request/EsbGetDBAccountListRequest.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/request/EsbGetDBAccountListRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/request/EsbGetJobDetailRequest.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/request/EsbGetJobDetailRequest.java index 38fbb6b64b..b5ef4695ba 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/request/EsbGetJobDetailRequest.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/request/EsbGetJobDetailRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/request/EsbGetJobListRequest.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/request/EsbGetJobListRequest.java index 2948ecae47..cd59da318a 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/request/EsbGetJobListRequest.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/request/EsbGetJobListRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/request/EsbGetOSAccountListRequest.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/request/EsbGetOSAccountListRequest.java index 6e9851b385..bd275b60d7 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/request/EsbGetOSAccountListRequest.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/request/EsbGetOSAccountListRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/request/EsbGetPublicScriptListRequest.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/request/EsbGetPublicScriptListRequest.java index ef444c85cd..e8b91e8073 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/request/EsbGetPublicScriptListRequest.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/request/EsbGetPublicScriptListRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -47,7 +47,7 @@ public class EsbGetPublicScriptListRequest extends EsbReq { /** * 脚本类型 * - * @see com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum + * @see com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum */ @JsonProperty("script_type") private Integer scriptType; diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/request/EsbGetScriptDetailRequest.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/request/EsbGetScriptDetailRequest.java index 5ee38fb76c..4937b1c097 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/request/EsbGetScriptDetailRequest.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/request/EsbGetScriptDetailRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/request/EsbGetScriptListRequest.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/request/EsbGetScriptListRequest.java index 71f6302b95..1fd387a99f 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/request/EsbGetScriptListRequest.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/request/EsbGetScriptListRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -51,7 +51,7 @@ public class EsbGetScriptListRequest extends EsbAppScopeReq { /** * 脚本类型 * - * @see com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum + * @see com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum */ @JsonProperty("script_type") private Integer scriptType; diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbCheckScriptV3Req.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbCheckScriptV3Req.java new file mode 100644 index 0000000000..09bfd9e94b --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbCheckScriptV3Req.java @@ -0,0 +1,60 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.esb.v3.request; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.esb.model.EsbJobReq; +import com.tencent.bk.job.common.validation.CheckEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; +import io.swagger.annotations.ApiModel; +import lombok.Data; +import lombok.EqualsAndHashCode; + +import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.NotNull; + +/** + * 高危脚本检测请求 + */ +@Data +@ApiModel("高危脚本检测请求报文") +@EqualsAndHashCode(callSuper = true) +public class EsbCheckScriptV3Req extends EsbJobReq { + /** + * 脚本类型 + * @see com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum + */ + @JsonProperty("script_language") + @NotNull(message = "{validation.constraints.ScriptType_empty.message}") + @CheckEnum(enumClass = ScriptTypeEnum.class, enumMethod = "isValid", + message = "{validation.constraints.ScriptType_illegal.message}") + private Integer type; + + /** + * 脚本内容，需Base64编码 + */ + @NotEmpty(message = "{validation.constraints.ScriptContent_empty.message}") + private String content; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbCreateAccountV3Req.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbCreateAccountV3Req.java new file mode 100644 index 0000000000..676735bac3 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbCreateAccountV3Req.java @@ -0,0 +1,84 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.esb.v3.request; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.constant.AccountCategoryEnum; +import com.tencent.bk.job.common.esb.model.EsbAppScopeReq; +import com.tencent.bk.job.common.util.json.SkipLogFields; +import com.tencent.bk.job.common.validation.CheckEnum; +import com.tencent.bk.job.manage.api.common.constants.account.AccountTypeEnum; +import io.swagger.annotations.ApiModel; +import lombok.Data; +import lombok.ToString; +import org.hibernate.validator.constraints.Length; + +import javax.validation.constraints.NotEmpty; + +@Data +@ApiModel("账号创建请求") +@ToString(exclude = {"password"}) +public class EsbCreateAccountV3Req extends EsbAppScopeReq { + + /** + * 帐号名称 + */ + @NotEmpty(message = "{validation.constraints.AccountName_empty.message}") + private String account; + + /** + * 账号类型：1-Linux，2-Windows，9-Mysql，10-Oracle，11-DB2 + */ + @CheckEnum(enumClass = AccountTypeEnum.class, enumMethod = "isValid", + message = "{validation.constraints.AccountType_illegal.message}") + private Integer type; + + /** + * 账号用途：1-系统账号，2-数据库账号 + */ + @CheckEnum(enumClass = AccountCategoryEnum.class, enumMethod = "isValid", + message = "{validation.constraints.AccountCategory_illegal.message}") + private Integer category; + + /** + * 系统账号的密码(Windows) + */ + @SkipLogFields + @Length(max = 255, message = "{validation.constraints.AccountPassword_tooLong.message}") + private String password; + + /** + * 别名 + */ + @Length(max = 255, message = "{validation.constraints.AccountAlias_tooLong.message}") + private String alias; + + /** + * 描述 + */ + @JsonProperty("description") + @Length(max = 1024, message = "{validation.constraints.AccountDescription_tooLong.message}") + private String remark; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbCreateDangerousRuleV3Req.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbCreateDangerousRuleV3Req.java new file mode 100644 index 0000000000..796b7edcf0 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbCreateDangerousRuleV3Req.java @@ -0,0 +1,76 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.esb.v3.request; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.esb.model.EsbJobReq; +import com.tencent.bk.job.common.validation.CheckEnum; +import com.tencent.bk.job.common.validation.ValidRegexPattern; +import com.tencent.bk.job.manage.api.common.constants.rule.HighRiskGrammarActionEnum; +import io.swagger.annotations.ApiModel; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.hibernate.validator.constraints.Length; + +import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.NotNull; +import java.util.List; + +/** + * 创建高危语句规则请求 + */ +@Data +@ApiModel("创建高危语句规则请求报文") +@EqualsAndHashCode(callSuper = true) +public class EsbCreateDangerousRuleV3Req extends EsbJobReq { + /** + * 表达式 + */ + @NotEmpty(message = "{validation.constraints.InvalidJobHighRiskGrammarRegex_empty.message}") + @Length(max = 250, message = "{validation.constraints.InvalidJobHighRiskGrammarRegex_outOfLength.message}") + @ValidRegexPattern(message = "{validation.constraints.InvalidJobHighRiskGrammarRegex_wrongExpr.message}") + private String expression; + + /** + * 脚本类型 + */ + @JsonProperty("script_language_list") + @NotEmpty(message = "{validation.constraints.ScriptTypeList_empty.message}") + private List scriptTypeList; + + /** + * 规则描述 + */ + @Length(max = 1000, message = "{validation.constraints.InvalidHighRiskRegularDescription_outOfLength.message}") + private String description; + + /** + * 处理动作 + */ + @NotNull(message = "{validation.constraints.InvalidHighRiskGrammarHandleAction.message}") + @CheckEnum(enumClass = HighRiskGrammarActionEnum.class, enumMethod = "isValid", + message = "{validation.constraints.InvalidHighRiskGrammarHandleAction.message}") + private Integer action; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbCreateOrUpdateCredentialV3Req.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbCreateOrUpdateCredentialV3Req.java index 478324649d..f3bd62d115 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbCreateOrUpdateCredentialV3Req.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbCreateOrUpdateCredentialV3Req.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -30,24 +30,24 @@ import lombok.EqualsAndHashCode; /** - * 新建凭据请求 + * 新建凭证请求 */ @EqualsAndHashCode(callSuper = true) @Data public class EsbCreateOrUpdateCredentialV3Req extends EsbAppScopeReq { /** - * 凭据ID + * 凭证ID */ private String id; /** - * 凭据名称 + * 凭证名称 */ private String name; /** - * 凭据类型 + * 凭证类型 */ private String type; diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbCreatePublicScriptV3Req.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbCreatePublicScriptV3Req.java new file mode 100644 index 0000000000..d0de0f60df --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbCreatePublicScriptV3Req.java @@ -0,0 +1,68 @@ +package com.tencent.bk.job.manage.model.esb.v3.request; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.esb.model.EsbJobReq; +import com.tencent.bk.job.common.validation.CheckEnum; +import com.tencent.bk.job.common.validation.Create; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; +import io.swagger.annotations.ApiModel; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.hibernate.validator.constraints.Length; + +import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.NotNull; +import javax.validation.constraints.Pattern; + +/** + * 创建公共脚本请求 + */ +@Data +@ApiModel("创建公共脚本请求报文") +@EqualsAndHashCode(callSuper = true) +public class EsbCreatePublicScriptV3Req extends EsbJobReq { + /** + * 脚本名称 + */ + @NotEmpty(message = "{validation.constraints.ScriptName_empty.message}", groups = Create.class) + @Length(max = 60, message = "{validation.constraints.ScriptName_outOfLength.message}", groups = Create.class) + @Pattern(regexp = "^[^\\\\|/:*<>\"?]+$", message = "{validation.constraints.ScriptName_illegal.message}", + groups = Create.class) + private String name; + + /** + * 脚本描述 + */ + private String description; + + /** + * 脚本类型 + * @see com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum + */ + @JsonProperty("script_language") + @NotNull(message = "{validation.constraints.ScriptType_empty.message}",groups = Create.class) + @CheckEnum(enumClass = ScriptTypeEnum.class, enumMethod = "isValid", + message = "{validation.constraints.ScriptType_illegal.message}", groups = Create.class) + private Integer type; + + /** + * 脚本内容，需Base64编码 + */ + @NotEmpty(message = "{validation.constraints.ScriptContent_empty.message}", groups = Create.class) + private String content; + + /** + * 脚本版本 + */ + @NotEmpty(message = "{validation.constraints.ScriptVersion_empty.message}", groups = Create.class) + @Length(max = 60, message = "{validation.constraints.ScriptVersion_outOfLength.message}", groups = Create.class) + @Pattern(regexp = "^[A-Za-z0-9_\\-#@.]+$", message = "{validation.constraints.ScriptVersion_illegal.message}", + groups = Create.class) + private String version; + + /** + * 版本描述 + */ + @JsonProperty("version_desc") + private String versionDesc; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbCreatePublicScriptVersionV3Req.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbCreatePublicScriptVersionV3Req.java new file mode 100644 index 0000000000..c39fdd0dba --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbCreatePublicScriptVersionV3Req.java @@ -0,0 +1,48 @@ +package com.tencent.bk.job.manage.model.esb.v3.request; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.esb.model.EsbJobReq; +import com.tencent.bk.job.common.validation.Create; +import io.swagger.annotations.ApiModel; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.hibernate.validator.constraints.Length; + +import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.Pattern; + +/** + * 创建公共脚本版本请求 + */ +@Data +@ApiModel("创建公共脚本版本请求报文") +@EqualsAndHashCode(callSuper = true) +public class EsbCreatePublicScriptVersionV3Req extends EsbJobReq { + /** + * 脚本ID + */ + @NotEmpty(message = "{validation.constraints.ScriptId_empty.message}", groups = Create.class) + @JsonProperty("script_id") + private String scriptId; + + /** + * 脚本内容，需Base64编码 + */ + @NotEmpty(message = "{validation.constraints.ScriptContent_empty.message}", groups = Create.class) + private String content; + + /** + * 脚本版本 + */ + @NotEmpty(message = "{validation.constraints.ScriptVersion_empty.message}", groups = Create.class) + @Length(max = 60, message = "{validation.constraints.ScriptVersion_outOfLength.message}", groups = Create.class) + @Pattern(regexp = "^[A-Za-z0-9_\\-#@.]+$", message = "{validation.constraints.ScriptVersion_illegal.message}", + groups = Create.class) + private String version; + + /** + * 版本描述 + */ + @JsonProperty("version_desc") + private String versionDesc; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbCreateScriptV3Req.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbCreateScriptV3Req.java new file mode 100644 index 0000000000..9e0d8a51ae --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbCreateScriptV3Req.java @@ -0,0 +1,92 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.esb.v3.request; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.esb.model.EsbAppScopeReq; +import com.tencent.bk.job.common.validation.CheckEnum; +import com.tencent.bk.job.common.validation.Create; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; +import io.swagger.annotations.ApiModel; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.hibernate.validator.constraints.Length; + +import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.NotNull; +import javax.validation.constraints.Pattern; + +/** + * 创建脚本请求 + */ +@EqualsAndHashCode(callSuper = true) +@Data +@ApiModel("创建业务脚本请求报文") +public class EsbCreateScriptV3Req extends EsbAppScopeReq { + + /** + * 脚本名称 + */ + @NotEmpty(message = "{validation.constraints.ScriptName_empty.message}", groups = Create.class) + @Length(max = 60, message = "{validation.constraints.ScriptName_outOfLength.message}", groups = Create.class) + @Pattern(regexp = "^[^\\\\|/:*<>\"?]+$", message = "{validation.constraints.ScriptName_illegal.message}", + groups = Create.class) + private String name; + /** + * 脚本描述 + */ + private String description; + + /** + * 脚本类型 + * @see com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum + */ + @JsonProperty("script_language") + @NotNull(message = "{validation.constraints.ScriptType_empty.message}",groups = Create.class) + @CheckEnum(enumClass = ScriptTypeEnum.class, enumMethod = "isValid", + message = "{validation.constraints.ScriptType_illegal.message}", groups = Create.class) + private Integer type; + + /** + * 脚本内容，需Base64编码 + */ + @NotEmpty(message = "{validation.constraints.ScriptContent_empty.message}", groups = Create.class) + private String content; + + /** + * 脚本版本 + */ + @NotEmpty(message = "{validation.constraints.ScriptVersion_empty.message}", groups = Create.class) + @Length(max = 60, message = "{validation.constraints.ScriptVersion_outOfLength.message}", groups = Create.class) + @Pattern(regexp = "^[A-Za-z0-9_\\-#@.]+$", message = "{validation.constraints.ScriptVersion_illegal.message}", + groups = Create.class) + private String version; + + /** + * 版本描述 + */ + @JsonProperty("version_desc") + private String versionDesc; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbCreateScriptVersionV3Req.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbCreateScriptVersionV3Req.java new file mode 100644 index 0000000000..e1937c6ab8 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbCreateScriptVersionV3Req.java @@ -0,0 +1,73 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.esb.v3.request; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.esb.model.EsbAppScopeReq; +import com.tencent.bk.job.common.validation.Create; +import io.swagger.annotations.ApiModel; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.hibernate.validator.constraints.Length; + +import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.Pattern; + +/** + * 创建脚本版本请求 + */ +@EqualsAndHashCode(callSuper = true) +@Data +@ApiModel("创建业务脚本版本请求报文") +public class EsbCreateScriptVersionV3Req extends EsbAppScopeReq { + + /** + * 脚本ID + */ + @NotEmpty(message = "{validation.constraints.ScriptId_empty.message}", groups = Create.class) + @JsonProperty("script_id") + private String scriptId; + + /** + * 脚本内容，需Base64编码 + */ + @NotEmpty(message = "{validation.constraints.ScriptContent_empty.message}", groups = Create.class) + private String content; + + /** + * 脚本版本 + */ + @NotEmpty(message = "{validation.constraints.ScriptVersion_empty.message}", groups = Create.class) + @Length(max = 60, message = "{validation.constraints.ScriptVersion_outOfLength.message}", groups = Create.class) + @Pattern(regexp = "^[A-Za-z0-9_\\-#@.]+$", message = "{validation.constraints.ScriptVersion_illegal.message}", + groups = Create.class) + private String version; + + /** + * 版本描述 + */ + @JsonProperty("version_desc") + private String versionDesc; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbDeleteAccountV3Req.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbDeleteAccountV3Req.java new file mode 100644 index 0000000000..e80f2d9f8b --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbDeleteAccountV3Req.java @@ -0,0 +1,43 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.esb.v3.request; + +import com.tencent.bk.job.common.esb.model.EsbAppScopeReq; +import io.swagger.annotations.ApiModel; +import lombok.Data; + +import javax.validation.constraints.NotNull; + +@Data +@ApiModel("账号删除请求") +public class EsbDeleteAccountV3Req extends EsbAppScopeReq { + + /** + * 帐号ID + */ + @NotNull(message = "{validation.constraints.AccountId_empty.message}") + private Long id; + +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbDeletePublicScriptV3Req.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbDeletePublicScriptV3Req.java new file mode 100644 index 0000000000..1e1beb9a0b --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbDeletePublicScriptV3Req.java @@ -0,0 +1,50 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.esb.v3.request; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.esb.model.EsbJobReq; +import com.tencent.bk.job.common.validation.Delete; +import io.swagger.annotations.ApiModel; +import lombok.Data; +import lombok.EqualsAndHashCode; + +import javax.validation.constraints.NotEmpty; + +/** + * 删除公共脚本请求 + */ +@EqualsAndHashCode(callSuper = true) +@Data +@ApiModel("删除公共脚本请求报文") +public class EsbDeletePublicScriptV3Req extends EsbJobReq { + + /** + * 脚本ID + */ + @NotEmpty(message = "{validation.constraints.ScriptId_empty.message}", groups = Delete.class) + @JsonProperty("script_id") + private String scriptId; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbDeletePublicScriptVersionV3Req.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbDeletePublicScriptVersionV3Req.java new file mode 100644 index 0000000000..9a99c84d43 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbDeletePublicScriptVersionV3Req.java @@ -0,0 +1,58 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.esb.v3.request; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.esb.model.EsbJobReq; +import com.tencent.bk.job.common.validation.Delete; +import io.swagger.annotations.ApiModel; +import lombok.Data; +import lombok.EqualsAndHashCode; + +import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.NotNull; + +/** + * 删除公共脚本版本请求 + */ +@EqualsAndHashCode(callSuper = true) +@Data +@ApiModel("删除公共脚本版本请求报文") +public class EsbDeletePublicScriptVersionV3Req extends EsbJobReq { + + /** + * 脚本ID + */ + @NotEmpty(message = "{validation.constraints.ScriptId_empty.message}", groups = Delete.class) + @JsonProperty("script_id") + private String scriptId; + + /** + * 脚本版本ID + */ + @NotNull(message = "{validation.constraints.ScriptVersionId_empty.message}", groups = Delete.class) + @JsonProperty("script_version_id") + private Long scriptVersionId; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbDeleteScriptV3Req.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbDeleteScriptV3Req.java new file mode 100644 index 0000000000..7feae0f98d --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbDeleteScriptV3Req.java @@ -0,0 +1,50 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.esb.v3.request; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.esb.model.EsbAppScopeReq; +import com.tencent.bk.job.common.validation.Delete; +import io.swagger.annotations.ApiModel; +import lombok.Data; +import lombok.EqualsAndHashCode; + +import javax.validation.constraints.NotEmpty; + +/** + * 删除脚本请求 + */ +@EqualsAndHashCode(callSuper = true) +@Data +@ApiModel("删除业务脚本请求报文") +public class EsbDeleteScriptV3Req extends EsbAppScopeReq { + + /** + * 脚本ID + */ + @NotEmpty(message = "{validation.constraints.ScriptId_empty.message}", groups = Delete.class) + @JsonProperty("script_id") + private String scriptId; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbDeleteScriptVersionV3Req.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbDeleteScriptVersionV3Req.java new file mode 100644 index 0000000000..bb5a20390f --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbDeleteScriptVersionV3Req.java @@ -0,0 +1,58 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.esb.v3.request; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.esb.model.EsbAppScopeReq; +import com.tencent.bk.job.common.validation.Delete; +import io.swagger.annotations.ApiModel; +import lombok.Data; +import lombok.EqualsAndHashCode; + +import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.NotNull; + +/** + * 删除脚本版本请求 + */ +@EqualsAndHashCode(callSuper = true) +@Data +@ApiModel("删除业务脚本版本请求报文") +public class EsbDeleteScriptVersionV3Req extends EsbAppScopeReq { + + /** + * 脚本ID + */ + @NotEmpty(message = "{validation.constraints.ScriptId_empty.message}", groups = Delete.class) + @JsonProperty("script_id") + private String scriptId; + + /** + * 脚本版本ID + */ + @NotNull(message = "{validation.constraints.ScriptVersionId_empty.message}", groups = Delete.class) + @JsonProperty("script_version_id") + private Long scriptVersionId; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGenLocalFileUploadUrlV3Req.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGenLocalFileUploadUrlV3Req.java index 8eb19e1815..bff2f08192 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGenLocalFileUploadUrlV3Req.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGenLocalFileUploadUrlV3Req.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetAccountListV3Req.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetAccountListV3Req.java index e419430c40..71fa3be501 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetAccountListV3Req.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetAccountListV3Req.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,9 @@ package com.tencent.bk.job.manage.model.esb.v3.request; +import com.tencent.bk.job.common.constant.AccountCategoryEnum; import com.tencent.bk.job.common.esb.model.EsbAppScopeReq; +import com.tencent.bk.job.common.validation.CheckEnum; import lombok.Data; import lombok.EqualsAndHashCode; @@ -37,8 +39,20 @@ public class EsbGetAccountListV3Req extends EsbAppScopeReq { /** * 账号用途（1：系统账号，2：DB账号），不传则不区分 */ + @CheckEnum(enumClass = AccountCategoryEnum.class, enumMethod = "isValid", + message = "{validation.constraints.AccountCategory_illegal.message}") private Integer category; + /** + * 账号名称 + */ + private String account; + + /** + * 账号别名 + */ + private String alias; + /** * 起始位置 */ diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetCredentialDetailV3Req.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetCredentialDetailV3Req.java new file mode 100644 index 0000000000..b8069035a4 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetCredentialDetailV3Req.java @@ -0,0 +1,19 @@ +package com.tencent.bk.job.manage.model.esb.v3.request; + +import com.tencent.bk.job.common.esb.model.EsbAppScopeReq; +import com.tencent.bk.job.common.validation.NotBlankField; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.EqualsAndHashCode; + +@EqualsAndHashCode(callSuper = true) +@Data +public class EsbGetCredentialDetailV3Req extends EsbAppScopeReq { + + /** + * 凭证id + */ + @ApiModelProperty(value = "凭证id") + @NotBlankField(fieldName = "id") + private String id; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetDangerousRuleV3Req.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetDangerousRuleV3Req.java new file mode 100644 index 0000000000..8f329fa6a4 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetDangerousRuleV3Req.java @@ -0,0 +1,62 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.esb.v3.request; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.esb.model.EsbJobReq; +import io.swagger.annotations.ApiModel; +import lombok.Data; +import lombok.EqualsAndHashCode; + +import java.util.List; + +/** + * 查询高危语句规则请求 + */ +@Data +@ApiModel("查询高危语句规则请求") +@EqualsAndHashCode(callSuper = true) +public class EsbGetDangerousRuleV3Req extends EsbJobReq { + /** + * 表达式 + */ + private String expression; + + /** + * 脚本类型 + */ + @JsonProperty("script_language_list") + private List scriptTypeList; + + /** + * 规则描述 + */ + private String description; + + /** + * 处理动作 + */ + private Integer action; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetPlanDetailV3Request.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetPlanDetailV3Request.java index 84d7452e3e..726eed616b 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetPlanDetailV3Request.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetPlanDetailV3Request.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,7 +27,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.esb.model.EsbAppScopeReq; -import com.tencent.bk.job.common.model.ValidateResult; +import com.tencent.bk.job.common.exception.InvalidParamException; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.extern.slf4j.Slf4j; @@ -48,11 +48,10 @@ public class EsbGetPlanDetailV3Request extends EsbAppScopeReq { @JsonProperty("job_plan_id") private Long planId; - public ValidateResult validate() { + public void validate() { if (this.getPlanId() == null || this.getPlanId() <= 0) { log.warn("Plan Id is empty or illegal!|{}", this.getPlanId()); - return ValidateResult.fail(ErrorCode.MISSING_OR_ILLEGAL_PARAM_WITH_PARAM_NAME, "job_plan_id"); + throw new InvalidParamException(ErrorCode.MISSING_OR_ILLEGAL_PARAM_WITH_PARAM_NAME, "job_plan_id"); } - return ValidateResult.pass(); } } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetPlanListV3Request.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetPlanListV3Request.java index 05e3ce7650..dcd5740cab 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetPlanListV3Request.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetPlanListV3Request.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetPublicScriptListV3Request.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetPublicScriptListV3Request.java index 5228c8c293..f6d0c728ce 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetPublicScriptListV3Request.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetPublicScriptListV3Request.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetPublicScriptVersionDetailV3Request.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetPublicScriptVersionDetailV3Request.java index e0e6a0e6da..fe053ad256 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetPublicScriptVersionDetailV3Request.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetPublicScriptVersionDetailV3Request.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetPublicScriptVersionListV3Request.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetPublicScriptVersionListV3Request.java index bec1ec8f00..b1c5c0bd1d 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetPublicScriptVersionListV3Request.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetPublicScriptVersionListV3Request.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetScriptListV3Req.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetScriptListV3Req.java index 6dc38b6bb9..a508a31372 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetScriptListV3Req.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetScriptListV3Req.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetScriptVersionDetailV3Req.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetScriptVersionDetailV3Req.java index d1de7b29b9..adab4b8267 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetScriptVersionDetailV3Req.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetScriptVersionDetailV3Req.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetScriptVersionListV3Req.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetScriptVersionListV3Req.java index 86b0741a7b..b4592e4838 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetScriptVersionListV3Req.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetScriptVersionListV3Req.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetTemplateListV3Request.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetTemplateListV3Request.java index c2060dfd09..dadb5ecfc9 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetTemplateListV3Request.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbGetTemplateListV3Request.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbManageDangerousRuleV3Req.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbManageDangerousRuleV3Req.java new file mode 100644 index 0000000000..0c985548c0 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbManageDangerousRuleV3Req.java @@ -0,0 +1,46 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.esb.v3.request; + +import com.tencent.bk.job.common.esb.model.EsbJobReq; +import io.swagger.annotations.ApiModel; +import lombok.Data; +import lombok.EqualsAndHashCode; + +import javax.validation.constraints.NotNull; + +/** + * 启停、删除高危语句规则请求， + */ +@Data +@ApiModel("启停、删除高危语句规则请求报文") +@EqualsAndHashCode(callSuper = true) +public class EsbManageDangerousRuleV3Req extends EsbJobReq { + /** + * 高危语句规则ID + */ + @NotNull(message = "{validation.constraints.InvalidJobHighRiskGrammarId_empty.message}") + private Long id; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbManagePublicScriptVersionV3Req.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbManagePublicScriptVersionV3Req.java new file mode 100644 index 0000000000..631676b990 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbManagePublicScriptVersionV3Req.java @@ -0,0 +1,58 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.esb.v3.request; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.esb.model.EsbJobReq; +import com.tencent.bk.job.common.validation.Update; +import io.swagger.annotations.ApiModel; +import lombok.Data; +import lombok.EqualsAndHashCode; + +import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.NotNull; + +/** + * 管理公共脚本版本请求 + */ +@EqualsAndHashCode(callSuper = true) +@Data +@ApiModel("管理公共脚本版本请求报文") +public class EsbManagePublicScriptVersionV3Req extends EsbJobReq { + + /** + * 脚本ID + */ + @NotEmpty(message = "{validation.constraints.ScriptId_empty.message}", groups = Update.class) + @JsonProperty("script_id") + private String scriptId; + + /** + * 脚本版本ID + */ + @NotNull(message = "{validation.constraints.ScriptVersionId_empty.message}", groups = Update.class) + @JsonProperty("script_version_id") + private Long scriptVersionId; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbManageScriptVersionV3Req.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbManageScriptVersionV3Req.java new file mode 100644 index 0000000000..bde2244a6e --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbManageScriptVersionV3Req.java @@ -0,0 +1,58 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.esb.v3.request; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.esb.model.EsbAppScopeReq; +import com.tencent.bk.job.common.validation.Update; +import io.swagger.annotations.ApiModel; +import lombok.Data; +import lombok.EqualsAndHashCode; + +import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.NotNull; + +/** + * 管理脚本版本请求 + */ +@EqualsAndHashCode(callSuper = true) +@Data +@ApiModel("管理业务脚本请求报文") +public class EsbManageScriptVersionV3Req extends EsbAppScopeReq { + + /** + * 脚本ID + */ + @NotEmpty(message = "{validation.constraints.ScriptId_empty.message}", groups = Update.class) + @JsonProperty("script_id") + private String scriptId; + + /** + * 脚本版本ID + */ + @NotNull(message = "{validation.constraints.ScriptVersionId_empty.message}", groups = Update.class) + @JsonProperty("script_version_id") + private Long scriptVersionId; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbQueryAgentInfoV3Req.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbQueryAgentInfoV3Req.java new file mode 100644 index 0000000000..3f0c979ccb --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbQueryAgentInfoV3Req.java @@ -0,0 +1,49 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.esb.v3.request; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.esb.model.EsbAppScopeReq; +import lombok.Data; +import lombok.EqualsAndHashCode; + +import javax.validation.constraints.Size; +import java.util.List; + +/** + * 查询Agent信息请求 + */ +@EqualsAndHashCode(callSuper = true) +@Data +public class EsbQueryAgentInfoV3Req extends EsbAppScopeReq { + + /** + * 主机ID列表 + */ + @Size(max = 5000, message = "{validation.constraints.queryAgentInfoHostIds_tooMany.message}") + @JsonProperty(value = "host_id_list") + private List hostIdList; + +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbUpdateDangerousRuleV3Req.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbUpdateDangerousRuleV3Req.java new file mode 100644 index 0000000000..039d0e9a9a --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbUpdateDangerousRuleV3Req.java @@ -0,0 +1,82 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.esb.v3.request; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.esb.model.EsbJobReq; +import com.tencent.bk.job.common.validation.CheckEnum; +import com.tencent.bk.job.common.validation.ValidRegexPattern; +import com.tencent.bk.job.manage.api.common.constants.rule.HighRiskGrammarActionEnum; +import io.swagger.annotations.ApiModel; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.hibernate.validator.constraints.Length; + +import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.NotNull; +import java.util.List; + +/** + * 修改高危语句规则请求 + */ +@Data +@ApiModel("修改高危语句规则请求报文") +@EqualsAndHashCode(callSuper = true) +public class EsbUpdateDangerousRuleV3Req extends EsbJobReq { + /** + * 高危语句规则ID + */ + @NotNull(message = "{validation.constraints.InvalidJobHighRiskGrammarId_empty.message}") + private Long id; + + /** + * 表达式 + */ + @NotEmpty(message = "{validation.constraints.InvalidJobHighRiskGrammarRegex_empty.message}") + @Length(max = 250, message = "{validation.constraints.InvalidJobHighRiskGrammarRegex_outOfLength.message}") + @ValidRegexPattern(message = "{validation.constraints.InvalidJobHighRiskGrammarRegex_wrongExpr.message}") + private String expression; + + /** + * 脚本类型 + */ + @JsonProperty("script_language_list") + @NotEmpty(message = "{validation.constraints.ScriptTypeList_empty.message}") + private List scriptTypeList; + + /** + * 规则描述 + */ + @Length(max = 1000, message = "{validation.constraints.InvalidHighRiskRegularDescription_outOfLength.message}") + private String description; + + /** + * 处理动作 + */ + @NotNull(message = "{validation.constraints.InvalidHighRiskGrammarHandleAction.message}") + @CheckEnum(enumClass = HighRiskGrammarActionEnum.class, enumMethod = "isValid", + message = "{validation.constraints.InvalidHighRiskGrammarHandleAction.message}") + private Integer action; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbUpdatePublicScriptBasicV3Req.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbUpdatePublicScriptBasicV3Req.java new file mode 100644 index 0000000000..20bb02ae76 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbUpdatePublicScriptBasicV3Req.java @@ -0,0 +1,68 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.esb.v3.request; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.esb.model.EsbJobReq; +import com.tencent.bk.job.common.validation.Create; +import com.tencent.bk.job.common.validation.Update; +import io.swagger.annotations.ApiModel; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.hibernate.validator.constraints.Length; + +import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.Pattern; + +/** + * 更新公共脚本基础信息请求 + */ +@EqualsAndHashCode(callSuper = true) +@Data +@ApiModel("更新公共脚本基础信息请求报文") +public class EsbUpdatePublicScriptBasicV3Req extends EsbJobReq { + + /** + * 脚本ID + */ + @NotEmpty(message = "{validation.constraints.ScriptId_empty.message}", groups = Update.class) + @JsonProperty("script_id") + private String scriptId; + + /** + * 脚本名称 + */ + @NotEmpty(message = "{validation.constraints.ScriptName_empty.message}", groups = Update.class) + @Length(max = 60, message = "{validation.constraints.ScriptName_outOfLength.message}", groups = Update.class) + @Pattern(regexp = "^[^\\\\|/:*<>\"?]+$", message = "{validation.constraints.ScriptName_illegal.message}", + groups = Create.class) + private String name; + + /** + * 脚本描述 + */ + private String description; + +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbUpdatePublicScriptVersionV3Req.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbUpdatePublicScriptVersionV3Req.java new file mode 100644 index 0000000000..24bb3cb56d --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbUpdatePublicScriptVersionV3Req.java @@ -0,0 +1,71 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.esb.v3.request; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.esb.model.EsbJobReq; +import com.tencent.bk.job.common.validation.Update; +import io.swagger.annotations.ApiModel; +import lombok.Data; +import lombok.EqualsAndHashCode; + +import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.NotNull; + +/** + * 更新公共脚本版本请求 + */ +@EqualsAndHashCode(callSuper = true) +@Data +@ApiModel("更新公共脚本版本请求报文") +public class EsbUpdatePublicScriptVersionV3Req extends EsbJobReq { + + /** + * 脚本ID + */ + @NotEmpty(message = "{validation.constraints.ScriptId_empty.message}", groups = Update.class) + @JsonProperty("script_id") + private String scriptId; + + /** + * 脚本版本ID + */ + @NotNull(message = "{validation.constraints.ScriptVersion_empty.message}", groups = Update.class) + @JsonProperty("script_version_id") + private Long scriptVersionId; + + /** + * 脚本内容，需Base64编码 + */ + @NotEmpty(message = "{validation.constraints.ScriptContent_empty.message}", groups = Update.class) + private String content; + + /** + * 版本描述 + */ + @JsonProperty("version_desc") + private String versionDesc; + +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbUpdateScriptBasicV3Req.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbUpdateScriptBasicV3Req.java new file mode 100644 index 0000000000..e4cae64d64 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbUpdateScriptBasicV3Req.java @@ -0,0 +1,68 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.esb.v3.request; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.esb.model.EsbAppScopeReq; +import com.tencent.bk.job.common.validation.Create; +import com.tencent.bk.job.common.validation.Update; +import io.swagger.annotations.ApiModel; +import lombok.Data; +import lombok.EqualsAndHashCode; +import org.hibernate.validator.constraints.Length; + +import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.Pattern; + +/** + * 更新脚本基础信息请求 + */ +@EqualsAndHashCode(callSuper = true) +@Data +@ApiModel("更新业务脚本基础信息请求报文") +public class EsbUpdateScriptBasicV3Req extends EsbAppScopeReq { + + /** + * 脚本ID + */ + @NotEmpty(message = "{validation.constraints.ScriptId_empty.message}", groups = Update.class) + @JsonProperty("script_id") + private String scriptId; + + /** + * 脚本名称 + */ + @NotEmpty(message = "{validation.constraints.ScriptName_empty.message}", groups = Update.class) + @Length(max = 60, message = "{validation.constraints.ScriptName_outOfLength.message}", groups = Update.class) + @Pattern(regexp = "^[^\\\\|/:*<>\"?]+$", message = "{validation.constraints.ScriptName_illegal.message}", + groups = Create.class) + private String name; + + /** + * 脚本描述 + */ + private String description; + +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbUpdateScriptVersionV3Req.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbUpdateScriptVersionV3Req.java new file mode 100644 index 0000000000..fe0379275f --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/request/EsbUpdateScriptVersionV3Req.java @@ -0,0 +1,71 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.esb.v3.request; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.esb.model.EsbAppScopeReq; +import com.tencent.bk.job.common.validation.Update; +import io.swagger.annotations.ApiModel; +import lombok.Data; +import lombok.EqualsAndHashCode; + +import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.NotNull; + +/** + * 更新脚本版本请求 + */ +@EqualsAndHashCode(callSuper = true) +@Data +@ApiModel("更新业务脚本版本请求报文") +public class EsbUpdateScriptVersionV3Req extends EsbAppScopeReq { + + /** + * 脚本ID + */ + @NotEmpty(message = "{validation.constraints.ScriptId_empty.message}", groups = Update.class) + @JsonProperty("script_id") + private String scriptId; + + /** + * 脚本版本ID + */ + @NotNull(message = "{validation.constraints.ScriptVersionId_empty.message}", groups = Update.class) + @JsonProperty("script_version_id") + private Long scriptVersionId; + + /** + * 脚本内容，需Base64编码 + */ + @NotEmpty(message = "{validation.constraints.ScriptContent_empty.message}", groups = Update.class) + private String content; + + /** + * 版本描述 + */ + @JsonProperty("version_desc") + private String versionDesc; + +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbAccountV3DTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbAccountV3DTO.java index 3c1dc3cc6b..1f882eed5b 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbAccountV3DTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbAccountV3DTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,36 +25,57 @@ package com.tencent.bk.job.manage.model.esb.v3.response; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; import com.tencent.bk.job.common.esb.model.EsbAppScopeDTO; import lombok.Data; +import lombok.EqualsAndHashCode; @Data +@EqualsAndHashCode(callSuper = true) public class EsbAccountV3DTO extends EsbAppScopeDTO { + @JsonPropertyDescription("Account ID") private Long id; + @JsonPropertyDescription("Account name") private String account; + @JsonPropertyDescription("Account alias") private String alias; // 账号用途（1：系统账号，2：DB账号） + @JsonPropertyDescription("Account usage. 1- OS Account, 2 - DB Account") private int category; // 账号类型（1：Linux，2：Windows，9：MySQL，10：Oracle，11：DB2） + @JsonPropertyDescription("Account type. 1 - Linux, 2 - Windows, 9 - MySQL, 10 - Oracle, 11 - DB2") private int type; @JsonProperty("db_system_account_id") + @JsonPropertyDescription("System account id for db account") private Long dbSystemAccountId; + @JsonPropertyDescription("Account OS") private String os; + @JsonPropertyDescription("Creator") private String creator; @JsonProperty("create_time") + @JsonPropertyDescription("Create time") private Long createTime; @JsonProperty("last_modify_user") + @JsonPropertyDescription("Last modify user") private String lastModifyUser; + @JsonPropertyDescription("Last modify time") @JsonProperty("last_modify_time") private Long lastModifyTime; + + /** + * 账号描述 + */ + @JsonPropertyDescription("Account description") + @JsonProperty("description") + private String description; } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbAgentInfoV3DTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbAgentInfoV3DTO.java new file mode 100644 index 0000000000..11c9ab902a --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbAgentInfoV3DTO.java @@ -0,0 +1,48 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.esb.v3.response; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@EqualsAndHashCode +public class EsbAgentInfoV3DTO { + + @JsonPropertyDescription("Host ID") + @JsonProperty(value = "bk_host_id") + private Long hostId; + + @JsonPropertyDescription("Agent status: 0 - abnormal, 1 - normal") + private int status; + + /** + * Agent版本 + */ + @JsonPropertyDescription("Agent Version") + private String version; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbCheckScriptV3DTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbCheckScriptV3DTO.java new file mode 100644 index 0000000000..188e6477d8 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbCheckScriptV3DTO.java @@ -0,0 +1,59 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.esb.v3.response; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +@Data +public class EsbCheckScriptV3DTO { + /** + * 错误所在行数 + */ + private Integer line; + + /** + * 脚本所在行的内容 + */ + @JsonProperty("line_content") + private String lineContent; + + /** + * 匹配的内容 + */ + @JsonProperty("match_content") + private String matchContent; + + /** + * 错误级别 + * @see com.tencent.bk.job.manage.api.common.constants.script.ScriptCheckErrorLevelEnum + */ + private Integer level; + + /** + * 检查项描述 + */ + private String description; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbCredentialSimpleInfoV3DTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbCredentialSimpleInfoV3DTO.java index f119357a6d..41f442fdb2 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbCredentialSimpleInfoV3DTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbCredentialSimpleInfoV3DTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,15 +24,21 @@ package com.tencent.bk.job.manage.model.esb.v3.response; -import lombok.AllArgsConstructor; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; import lombok.Data; import lombok.NoArgsConstructor; @Data @NoArgsConstructor -@AllArgsConstructor public class EsbCredentialSimpleInfoV3DTO { - + @JsonPropertyDescription("Credential id") private String id; + @JsonPropertyDescription("Credential name") + private String name; + + public EsbCredentialSimpleInfoV3DTO(String id, String name) { + this.id = id; + this.name = name; + } } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbCredentialV3DTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbCredentialV3DTO.java new file mode 100644 index 0000000000..2405ce946c --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbCredentialV3DTO.java @@ -0,0 +1,87 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.esb.v3.response; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; +import com.tencent.bk.job.common.esb.model.EsbAppScopeDTO; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + +@EqualsAndHashCode(callSuper = true) +@Data +@NoArgsConstructor +public class EsbCredentialV3DTO extends EsbAppScopeDTO { + + /** + * id + */ + @JsonPropertyDescription("Credential id") + private String id; + + /** + * 凭证名称 + */ + @JsonPropertyDescription("Credential name") + private String name; + + /** + * 凭证类型 + */ + @JsonPropertyDescription("Credential type") + private String type; + + /** + * 描述 + */ + @JsonPropertyDescription("Credential name") + private String description; + + /** + * 创建人 + */ + @JsonPropertyDescription("Creator") + private String creator; + /** + * 创建时间 + */ + @JsonProperty("create_time") + @JsonPropertyDescription("Create time") + private Long createTime; + /** + * 更新人 + */ + @JsonProperty("last_modify_user") + @JsonPropertyDescription("Last modify user") + private String lastModifyUser; + + /** + * 更新时间 + */ + @JsonProperty("last_modify_time") + @JsonPropertyDescription("Last modify time") + private Long lastModifyTime; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbDangerousRuleV3DTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbDangerousRuleV3DTO.java new file mode 100644 index 0000000000..a182c1a6ed --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbDangerousRuleV3DTO.java @@ -0,0 +1,88 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.esb.v3.response; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +import java.util.List; + +@Data +public class EsbDangerousRuleV3DTO { + /** + * 高危语句规则ID + */ + private Long id; + + /** + * 表达式 + */ + private String expression; + + /** + * 脚本语言列表 + */ + @JsonProperty("script_language_list") + private List scriptTypeList; + + /** + * 规则描述 + */ + private String description; + + /** + * 处理动作: 1 - 扫描, 2 - 拦截 + */ + private Integer action; + + /** + * 启用状态: 0 - 停用, 1 - 启用 + */ + private Integer status; + + /** + * 创建人 + */ + private String creator; + + /** + * 创建时间Unix时间戳（ms） + */ + @JsonProperty("create_time") + private Long createTime; + + /** + * 最近一次修改人 + */ + @JsonProperty("last_modify_user") + private String lastModifyUser; + + /** + * 最近一次修改时间Unix时间戳（ms） + */ + @JsonProperty("last_modify_time") + private Long lastModifyTime; + +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbFileStepV3DTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbFileStepV3DTO.java deleted file mode 100644 index 13b953ff81..0000000000 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbFileStepV3DTO.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.model.esb.v3.response; - -import com.fasterxml.jackson.annotation.JsonProperty; -import com.tencent.bk.job.common.esb.model.job.v3.EsbFileDestinationV3DTO; -import com.tencent.bk.job.common.esb.model.job.v3.EsbFileSourceV3DTO; -import lombok.Data; - -import java.util.List; - -/** - * 文件分发步骤 - * - * @since 17/11/2020 20:37 - */ -@Data -public class EsbFileStepV3DTO { - /** - * 源文件列表 - */ - @JsonProperty("file_source_list") - private List fileSourceList; - - /** - * 分发目标信息 - */ - @JsonProperty("file_destination") - private EsbFileDestinationV3DTO fileDestination; - - /** - * 超时 - */ - private Long timeout; - - /** - * 源机器上传限速 - */ - @JsonProperty("source_speed_limit") - private Long sourceSpeedLimit; - - /** - * 目标机器下载限速 - */ - @JsonProperty("destination_speed_limit") - private Long destinationSpeedLimit; - - /** - * 传输模式 - */ - @JsonProperty("transfer_mode") - private Integer transferMode; -} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbPlanBasicInfoV3DTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbPlanBasicInfoV3DTO.java index f138aa4291..d6343c1e01 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbPlanBasicInfoV3DTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbPlanBasicInfoV3DTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,52 +25,63 @@ package com.tencent.bk.job.manage.model.esb.v3.response; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; import com.tencent.bk.job.common.esb.model.EsbAppScopeDTO; import lombok.Data; +import lombok.EqualsAndHashCode; /** * 执行方案基础信息 * * @since 15/10/2020 17:54 */ +@EqualsAndHashCode(callSuper = true) @Data public class EsbPlanBasicInfoV3DTO extends EsbAppScopeDTO { /** * 执行方案 ID */ + @JsonPropertyDescription("Job plan id") private Long id; /** * 作业模版 ID */ @JsonProperty("job_template_id") + @JsonPropertyDescription("Job template id") private Long templateId; /** * 执行方案名称 */ + @JsonPropertyDescription("Job plan name") private String name; /** * 创建人 */ + @JsonPropertyDescription("Creator") private String creator; /** * 创建时间 */ @JsonProperty("create_time") + @JsonPropertyDescription("Create time") private Long createTime; /** * 最后更新人 */ @JsonProperty("last_modify_user") + @JsonPropertyDescription("Last modify user") private String lastModifyUser; /** * 最后更新时间 */ @JsonProperty("last_modify_time") + @JsonPropertyDescription("Last modify time") private Long lastModifyTime; + } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbPlanInfoV3DTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbPlanInfoV3DTO.java index e2fc67ad8b..f65b0621c0 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbPlanInfoV3DTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbPlanInfoV3DTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,7 +25,9 @@ package com.tencent.bk.job.manage.model.esb.v3.response; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; import com.tencent.bk.job.common.esb.model.job.v3.EsbGlobalVarV3DTO; +import com.tencent.bk.job.common.esb.model.job.v3.resp.EsbStepV3DTO; import lombok.Data; import lombok.EqualsAndHashCode; @@ -42,11 +44,13 @@ public class EsbPlanInfoV3DTO extends EsbPlanBasicInfoV3DTO { * 步骤信息 */ @JsonProperty("step_list") + @JsonPropertyDescription("Job plan steps") private List stepList; /** * 全局变量信息 */ @JsonProperty("global_var_list") + @JsonPropertyDescription("Job plan global variables") private List globalVarList; } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbQueryAgentInfoV3Resp.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbQueryAgentInfoV3Resp.java new file mode 100644 index 0000000000..6de841a648 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbQueryAgentInfoV3Resp.java @@ -0,0 +1,41 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.esb.v3.response; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; +import lombok.Data; +import lombok.EqualsAndHashCode; + +import java.util.List; + +@Data +@EqualsAndHashCode +public class EsbQueryAgentInfoV3Resp { + + @JsonPropertyDescription("Agent info list") + @JsonProperty(value = "agent_info_list") + private List agentInfoList; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbScriptStepV3DTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbScriptStepV3DTO.java deleted file mode 100644 index 5e8f194d20..0000000000 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbScriptStepV3DTO.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.model.esb.v3.response; - -import com.fasterxml.jackson.annotation.JsonProperty; -import com.tencent.bk.job.common.esb.model.job.v3.EsbAccountV3BasicDTO; -import com.tencent.bk.job.common.esb.model.job.v3.EsbServerV3DTO; -import lombok.Data; - -/** - * @since 17/11/2020 20:37 - */ -@Data -public class EsbScriptStepV3DTO { - @JsonProperty("script_type") - private Integer type; - - @JsonProperty("script_id") - private String scriptId; - - @JsonProperty("script_version_id") - private Long scriptVersionId; - - @JsonProperty("script_content") - private String content; - - @JsonProperty("script_language") - private Integer language; - - @JsonProperty("script_param") - private String scriptParam; - - @JsonProperty("script_timeout") - private Long scriptTimeout; - - private EsbAccountV3BasicDTO account; - - private EsbServerV3DTO server; - - @JsonProperty("is_param_sensitive") - private Integer secureParam; -} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbScriptV3DTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbScriptV3DTO.java index a64e33c15f..e0070ebec4 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbScriptV3DTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbScriptV3DTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,6 +25,7 @@ package com.tencent.bk.job.manage.model.esb.v3.response; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; import com.tencent.bk.job.common.esb.model.EsbAppScopeDTO; import lombok.Getter; import lombok.Setter; @@ -34,25 +35,37 @@ @Setter @ToString public class EsbScriptV3DTO extends EsbAppScopeDTO { + + @JsonPropertyDescription("Script id") private String id; + @JsonPropertyDescription("Script name") private String name; @JsonProperty("script_language") + @JsonPropertyDescription("Script language") private int type; + @JsonPropertyDescription("Creator") private String creator; + @JsonPropertyDescription("Create time") @JsonProperty("create_time") private Long createTime; @JsonProperty("last_modify_user") + @JsonPropertyDescription("Last modify user") private String lastModifyUser; @JsonProperty("last_modify_time") + @JsonPropertyDescription("Last modify time") private Long lastModifyTime; @JsonProperty("online_script_version_id") + @JsonPropertyDescription("Online script version id") private Long onlineScriptVersionId; + @JsonProperty("description") + @JsonPropertyDescription("Script description") + private String description; } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbScriptVersionDetailV3DTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbScriptVersionDetailV3DTO.java index 3979b2517b..ac7d14e6ff 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbScriptVersionDetailV3DTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbScriptVersionDetailV3DTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,6 +25,7 @@ package com.tencent.bk.job.manage.model.esb.v3.response; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; import com.tencent.bk.job.common.esb.model.EsbAppScopeDTO; import lombok.Getter; import lombok.Setter; @@ -37,34 +38,58 @@ public class EsbScriptVersionDetailV3DTO extends EsbAppScopeDTO { /** * 脚本版本ID */ + @JsonPropertyDescription("Script version ID") private Long id; @JsonProperty("script_id") + @JsonPropertyDescription("Script ID") private String scriptId; /** * 脚本名称 */ + @JsonPropertyDescription("Script name") private String name; + @JsonPropertyDescription("Script version") private String version; + @JsonPropertyDescription("Script content") private String content; // 脚本版本状态（0：未上线，1：已上线，2：已下线，3：已禁用） - private int status; + @JsonPropertyDescription("Script status") + private Integer status; + @JsonPropertyDescription("Script version description") @JsonProperty("version_desc") private String versionDesc; + @JsonPropertyDescription("Creator") private String creator; @JsonProperty("create_time") + @JsonPropertyDescription("Create time") private Long createTime; @JsonProperty("last_modify_user") + @JsonPropertyDescription("Last modify user") private String lastModifyUser; + @JsonPropertyDescription("Last modify time") @JsonProperty("last_modify_time") private Long lastModifyTime; + + /** + * 脚本语言:1 - shell, 2 - bat, 3 - perl, 4 - python, 5 - powershell + */ + @JsonProperty("script_language") + @JsonPropertyDescription("Script language") + private Integer type; + + /** + * 脚本描述 + */ + @JsonPropertyDescription("Script description") + private String description; } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbServiceVersionV3DTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbServiceVersionV3DTO.java new file mode 100644 index 0000000000..d4f1760433 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbServiceVersionV3DTO.java @@ -0,0 +1,39 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.esb.v3.response; + +import com.fasterxml.jackson.annotation.JsonPropertyDescription; +import lombok.Data; +import lombok.EqualsAndHashCode; + +@Data +@EqualsAndHashCode +public class EsbServiceVersionV3DTO { + /** + * 服务版本号 + */ + @JsonPropertyDescription("Service Version") + private String version; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbStepV3DTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbStepV3DTO.java deleted file mode 100644 index d0312d589f..0000000000 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbStepV3DTO.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.model.esb.v3.response; - -import com.fasterxml.jackson.annotation.JsonProperty; -import lombok.Data; - -/** - * @since 17/11/2020 16:41 - */ -@Data -public class EsbStepV3DTO { - private Long id; - - private String name; - - private Integer type; - - @JsonProperty("script_info") - private EsbScriptStepV3DTO scriptInfo; - - @JsonProperty("file_info") - private EsbFileStepV3DTO fileInfo; - - @JsonProperty("approval_info") - private EsbApprovalStepV3DTO approvalInfo; -} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbTagV3DTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbTagV3DTO.java new file mode 100644 index 0000000000..428755380f --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbTagV3DTO.java @@ -0,0 +1,54 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.esb.v3.response; + +import com.fasterxml.jackson.annotation.JsonPropertyDescription; +import com.tencent.bk.job.common.esb.model.EsbAppScopeDTO; +import lombok.Data; + +/** + * 标签 + */ +@Data +public class EsbTagV3DTO extends EsbAppScopeDTO { + /** + * 标签 ID + */ + @JsonPropertyDescription("Tag id") + private Long id; + + /** + * 标签名 + */ + @JsonPropertyDescription("Tag name") + private String name; + + /** + * 描述 + */ + @JsonPropertyDescription("Tag description") + private String description; + +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbTemplateBasicInfoV3DTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbTemplateBasicInfoV3DTO.java index 3444b777a2..30fbc7fb13 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbTemplateBasicInfoV3DTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbTemplateBasicInfoV3DTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,6 +25,7 @@ package com.tencent.bk.job.manage.model.esb.v3.response; import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; import com.tencent.bk.job.common.esb.model.EsbAppScopeDTO; import lombok.Getter; import lombok.Setter; @@ -41,33 +42,47 @@ public class EsbTemplateBasicInfoV3DTO extends EsbAppScopeDTO { /** * 作业模版 ID */ + @JsonPropertyDescription("Job template id") private Long id; /** * 作业模版名称 */ + @JsonPropertyDescription("Job template name") private String name; /** * 创建人 */ + @JsonPropertyDescription("Creator") private String creator; /** * 创建时间 */ @JsonProperty("create_time") + @JsonPropertyDescription("Create time") private Long createTime; /** * 最后更新人 */ @JsonProperty("last_modify_user") + @JsonPropertyDescription("Last modify user") private String lastModifyUser; /** * 最后更新时间 */ @JsonProperty("last_modify_time") + @JsonPropertyDescription("Last modify time") private Long lastModifyTime; + + /** + * 模板描述 + */ + @JsonProperty("description") + @JsonPropertyDescription("Description") + private String description; + } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbTemplateInfoV3DTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbTemplateInfoV3DTO.java new file mode 100644 index 0000000000..843f174086 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbTemplateInfoV3DTO.java @@ -0,0 +1,57 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.esb.v3.response; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyDescription; +import com.tencent.bk.job.common.esb.model.job.v3.EsbGlobalVarV3DTO; +import com.tencent.bk.job.common.esb.model.job.v3.resp.EsbStepV3DTO; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +import java.util.List; + +/** + * 作业模版信息 + */ +@Getter +@Setter +@ToString +public class EsbTemplateInfoV3DTO extends EsbTemplateBasicInfoV3DTO { + /** + * 步骤信息 + */ + @JsonProperty("step_list") + @JsonPropertyDescription("Job template steps") + private List stepList; + + /** + * 全局变量信息 + */ + @JsonProperty("global_var_list") + @JsonPropertyDescription("Job template global variables") + private List globalVarList; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbUploadUrlV3DTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbUploadUrlV3DTO.java index b8ea8bbf99..00cba74848 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbUploadUrlV3DTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/esb/v3/response/EsbUploadUrlV3DTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceAccountDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceAccountDTO.java index 6fa3e9e517..f6a4a9cda4 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceAccountDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceAccountDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceAppBaseInfoDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceAppBaseInfoDTO.java index fcd352a75e..10849d71e2 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceAppBaseInfoDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceAppBaseInfoDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceAppRoleDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceAppRoleDTO.java index 67938f6237..f32ecb891a 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceAppRoleDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceAppRoleDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceApplicationAttrsDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceApplicationAttrsDTO.java index 9f00e8ecd3..2a5180c296 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceApplicationAttrsDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceApplicationAttrsDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceFileUploadSettingDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceFileUploadSettingDTO.java index a221d20cde..2d0c10a6b6 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceFileUploadSettingDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceFileUploadSettingDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceHostDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceHostDTO.java index 55d077befa..2b8070ef12 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceHostDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceHostDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,7 +25,9 @@ package com.tencent.bk.job.manage.model.inner; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; import lombok.AllArgsConstructor; import lombok.Builder; import lombok.Getter; @@ -48,16 +50,31 @@ public class ServiceHostDTO { */ private Long hostId; + /** + * AgentID + */ + private String agentId; + /** * 云区域ID */ private Long cloudAreaId; /** - * ip + * 云区域名称 + */ + private String cloudAreaName; + + /** + * ipv4 */ private String ip; + /** + * ipv6 + */ + private String ipv6; + /** * 业务ID */ @@ -68,18 +85,90 @@ public class ServiceHostDTO { */ private Long bizId; + /** + * agent存活状态，0-异常，1-正常 + */ + @JsonInclude(JsonInclude.Include.NON_NULL) + private Integer alive; + + /** + * 操作系统名称 + */ + private String osName; + + /** + * 操作系统类型 + */ + private String osType; + + /** + * 操作系统类型名称 + */ + private String osTypeName; + /** + * 主机名称 + */ + private String hostname; + + /** + * 所属云厂商ID + */ + private String cloudVendorId; + + /** + * 所属云厂商名称 + */ + private String cloudVendorName; + @JsonIgnore public String getCloudIp() { return cloudAreaId + ":" + ip; } + @JsonIgnore + public String getHostIdOrCloudIp() { + if (hostId != null && hostId > 0) { + return String.valueOf(hostId); + } + return cloudAreaId + ":" + ip; + } + public static ServiceHostDTO fromApplicationHostDTO(ApplicationHostDTO host) { return ServiceHostDTO.builder() .bizId(host.getBizId()) .appId(host.getAppId()) .hostId(host.getHostId()) .cloudAreaId(host.getCloudAreaId()) + .cloudAreaName(host.getCloudAreaName()) .ip(host.getIp()) + .ipv6(host.getIpv6()) + .agentId(host.getAgentId()) + .osName(host.getOsName()) + .osType(host.getOsType()) + .osTypeName(host.getOsTypeName()) + .hostname(host.getHostName()) + .cloudVendorId(host.getCloudVendorId()) + .cloudVendorName(host.getCloudVendorName()) .build(); } + + public static HostDTO toHostDTO(ServiceHostDTO serviceHostDTO) { + if (serviceHostDTO == null) { + return null; + } + HostDTO hostDTO = new HostDTO(); + hostDTO.setHostId(serviceHostDTO.getHostId()); + hostDTO.setAgentId(serviceHostDTO.getAgentId()); + hostDTO.setIp(serviceHostDTO.getIp()); + hostDTO.setIpv6(serviceHostDTO.getIpv6()); + hostDTO.setBkCloudId(serviceHostDTO.getCloudAreaId()); + hostDTO.setBkCloudName(serviceHostDTO.getCloudAreaName()); + hostDTO.setHostname(serviceHostDTO.getHostname()); + hostDTO.setCloudVendorId(serviceHostDTO.getCloudVendorId()); + hostDTO.setCloudVendorName(serviceHostDTO.getCloudVendorName()); + hostDTO.setOsType(serviceHostDTO.getOsType()); + hostDTO.setOsTypeName(serviceHostDTO.getOsTypeName()); + hostDTO.setOsName(serviceHostDTO.getOsName()); + return hostDTO; + } } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceHostInfoDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceHostInfoDTO.java index 43026297e8..c64ac0a586 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceHostInfoDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceHostInfoDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -37,11 +37,11 @@ public class ServiceHostInfoDTO { @ApiModelProperty("服务器ID") private Long hostId; - @ApiModelProperty("主机IP") + @ApiModelProperty("主机IPv4") private String ip; - @ApiModelProperty("展示用的IP，主要针对多内网IP问题") - private String displayIp; + @ApiModelProperty("主机ipv6") + private String ipv6; @ApiModelProperty("云区域ID") private Long cloudAreaId; diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceHostStatusDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceHostStatusDTO.java index 04e54dbf5c..7dd9972055 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceHostStatusDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceHostStatusDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,6 +25,7 @@ package com.tencent.bk.job.manage.model.inner; import com.fasterxml.jackson.annotation.JsonInclude; +import com.tencent.bk.job.common.constant.JobConstants; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; @@ -39,9 +40,6 @@ public class ServiceHostStatusDTO { @ApiModelProperty("服务器ID") private Long hostId; - @ApiModelProperty("主机IP") - private String ip; - @ApiModelProperty("GSE Agent状态") private int alive; @@ -57,4 +55,8 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(hostId); } + + public boolean isAgentAlive() { + return alive == JobConstants.GSE_AGENT_STATUS_VALUE_ALIVE; + } } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceIdNameCheckDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceIdNameCheckDTO.java index a9cdb8b03c..36bfc27b0c 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceIdNameCheckDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceIdNameCheckDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceListAppHostResultDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceListAppHostResultDTO.java index 51ec865b96..0557e76f10 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceListAppHostResultDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceListAppHostResultDTO.java @@ -10,6 +10,11 @@ */ @Data public class ServiceListAppHostResultDTO { + /** + * 空的实例 + */ + public static final ServiceListAppHostResultDTO EMPTY = new ServiceListAppHostResultDTO(); + /** * 合法的主机（在当前业务下) */ @@ -22,9 +27,4 @@ public class ServiceListAppHostResultDTO { * 在其他业务下的主机 */ private List notInAppHosts; -// /** -// * 主机白名单 -// */ -// private Map> whiteHosts; - } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceNotificationDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceNotificationDTO.java index 975ca49481..dca3b6de0c 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceNotificationDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceNotificationDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceNotificationMessage.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceNotificationMessage.java index c5cfd2c79e..cfddcfb09d 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceNotificationMessage.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceNotificationMessage.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceNotificationTriggerDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceNotificationTriggerDTO.java index c34904ee00..2f2c2c52f5 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceNotificationTriggerDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceNotificationTriggerDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -46,21 +46,26 @@ public class ServiceNotificationTriggerDTO { /** * 触发方式 * - * @see com.tencent.bk.job.manage.common.consts.notify.TriggerTypeEnum + * @see com.tencent.bk.job.manage.api.common.constants.notify.TriggerTypeEnum */ private Integer triggerType; /** * 触发通知的资源类型 * - * @see com.tencent.bk.job.manage.common.consts.notify.ResourceTypeEnum + * @see com.tencent.bk.job.manage.api.common.constants.notify.ResourceTypeEnum */ private Integer resourceType; + /** + * 定时任务ID + */ + private Long cronTaskId; + /** * 资源操作结果 * - * @see com.tencent.bk.job.manage.common.consts.notify.ExecuteStatusEnum + * @see com.tencent.bk.job.manage.api.common.constants.notify.ExecuteStatusEnum */ private Integer resourceExecuteStatus; } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceNotifyChannelDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceNotifyChannelDTO.java index 6779bef05f..ea94d2bb12 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceNotifyChannelDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceNotifyChannelDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceRolesUsersNotificationDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceRolesUsersNotificationDTO.java index 07b5c15808..17eafeaa2f 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceRolesUsersNotificationDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceRolesUsersNotificationDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -38,7 +38,7 @@ public class ServiceRolesUsersNotificationDTO { /** * 触发通知的资源类型 * - * @see com.tencent.bk.job.manage.common.consts.notify.ResourceTypeEnum + * @see com.tencent.bk.job.manage.api.common.constants.notify.ResourceTypeEnum */ private Integer resourceType; /** diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceScriptCheckResultItemDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceScriptCheckResultItemDTO.java index 31d77e948b..4b06292157 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceScriptCheckResultItemDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceScriptCheckResultItemDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,8 +24,8 @@ package com.tencent.bk.job.manage.model.inner; -import com.tencent.bk.job.manage.common.consts.RuleMatchHandleActionEnum; -import com.tencent.bk.job.manage.common.consts.script.ScriptCheckErrorLevelEnum; +import com.tencent.bk.job.manage.api.common.constants.RuleMatchHandleActionEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptCheckErrorLevelEnum; import lombok.Data; import lombok.NoArgsConstructor; diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceScriptDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceScriptDTO.java index 8fb8517a46..4f2063c2f0 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceScriptDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceScriptDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,7 @@ package com.tencent.bk.job.manage.model.inner; -import com.tencent.bk.job.manage.common.consts.JobResourceStatusEnum; +import com.tencent.bk.job.manage.api.common.constants.JobResourceStatusEnum; import lombok.Getter; import lombok.Setter; import lombok.ToString; diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceSpecificResourceNotifyPolicyDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceSpecificResourceNotifyPolicyDTO.java new file mode 100644 index 0000000000..bb8e066c88 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceSpecificResourceNotifyPolicyDTO.java @@ -0,0 +1,56 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.inner; + +import lombok.Data; + +import java.util.List; +import java.util.Map; + +@Data +public class ServiceSpecificResourceNotifyPolicyDTO { + + private Long appId; + + /** + * 触发类型 + * @see com.tencent.bk.job.manage.api.common.constants.notify.TriggerTypeEnum; + */ + private Integer triggerType; + + /** + * 资源类型 + * @see com.tencent.bk.job.manage.api.common.constants.notify.ResourceTypeEnum + */ + private Integer resourceType; + + private Long resourceId; + + private List roleList; + + private List extraObserverList; + + private Map> resourceStatusChannelMap; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTagDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTagDTO.java index 5e35cc51d1..2d79fb7e5c 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTagDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTagDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskApprovalStepDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskApprovalStepDTO.java index 532254c551..1aaf9d656e 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskApprovalStepDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskApprovalStepDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskApprovalUserDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskApprovalUserDTO.java index 3c59b0743c..040ce04769 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskApprovalUserDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskApprovalUserDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskFileInfoDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskFileInfoDTO.java index 6ea9f6aeaf..5bfebd8c30 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskFileInfoDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskFileInfoDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskFileStepDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskFileStepDTO.java index 4dcdab325a..ac473946da 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskFileStepDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskFileStepDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskHostNodeDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskHostNodeDTO.java index c2ab87a4a1..2384d05fd0 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskHostNodeDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskHostNodeDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskNodeInfoDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskNodeInfoDTO.java index ef684d8c6a..de551cde7a 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskNodeInfoDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskNodeInfoDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskPlanDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskPlanDTO.java index 0734beb728..d5559007b1 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskPlanDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskPlanDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -51,7 +51,7 @@ public class ServiceTaskPlanDTO { * 执行方案类型 */ @ApiModelProperty(value = "是否调试执行方案") - private Boolean debugTask; + private boolean debugTask; /** * 执行方案 ID diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskScriptStepDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskScriptStepDTO.java index a8dde2bfd4..01368b0657 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskScriptStepDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskScriptStepDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,8 @@ package com.tencent.bk.job.manage.model.inner; -import com.tencent.bk.job.manage.common.consts.task.TaskScriptSourceEnum; +import com.tencent.bk.job.manage.api.common.constants.JobResourceStatusEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskScriptSourceEnum; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; @@ -39,7 +40,7 @@ public class ServiceTaskScriptStepDTO { private Long scriptVersionId; /** - * @see com.tencent.bk.job.manage.common.consts.JobResourceStatusEnum + * @see JobResourceStatusEnum */ @ApiModelProperty("脚本状态") private Integer scriptStatus; @@ -55,7 +56,7 @@ public class ServiceTaskScriptStepDTO { /** * 脚本类型 * - * @see com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum + * @see com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum */ @ApiModelProperty("脚本语言类型") private Integer scriptType; @@ -63,7 +64,7 @@ public class ServiceTaskScriptStepDTO { /** * 脚本类型 * - * @see com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum + * @see com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum */ @ApiModelProperty("脚本语言类型") private Integer type; @@ -74,6 +75,9 @@ public class ServiceTaskScriptStepDTO { @ApiModelProperty("脚本参数") private String scriptParam; + @ApiModelProperty("Windows解释器") + private String windowsInterpreter; + @ApiModelProperty("脚本超时时间") private Long scriptTimeout; diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskStepDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskStepDTO.java index 63c735121e..675dd9635c 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskStepDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskStepDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskTargetDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskTargetDTO.java index 5d397f6021..a1b223c1b9 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskTargetDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskTargetDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskTemplateDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskTemplateDTO.java index 2ad1d4fcee..b60cd70921 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskTemplateDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskTemplateDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskVariableDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskVariableDTO.java index 5544623e8a..bf4599e66e 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskVariableDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTaskVariableDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTemplateNotificationDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTemplateNotificationDTO.java index 155aac70dc..3c4690d40c 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTemplateNotificationDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTemplateNotificationDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -57,7 +57,7 @@ public class ServiceTemplateNotificationDTO { * 触发通知的资源类型 * 用于获取【资源所属者】角色对应用户 * - * @see com.tencent.bk.job.manage.common.consts.notify.ResourceTypeEnum + * @see com.tencent.bk.job.manage.api.common.constants.notify.ResourceTypeEnum */ private Integer resourceType; diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTriggerTemplateNotificationDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTriggerTemplateNotificationDTO.java index ee91df7960..1d2c4c8710 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTriggerTemplateNotificationDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceTriggerTemplateNotificationDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceUserNotificationDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceUserNotificationDTO.java index 3aa55164d5..01b374e9cc 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceUserNotificationDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceUserNotificationDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceWhiteIPInfo.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceWhiteIPInfo.java index b1fa6b615a..3ff232736f 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceWhiteIPInfo.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/ServiceWhiteIPInfo.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -35,8 +35,7 @@ @AllArgsConstructor @NoArgsConstructor public class ServiceWhiteIPInfo { - private Long cloudId; - private String ip; + private Long hostId; private boolean isForAllApp; //（脚本执行：SCRIPT_EXECUTE/文件分发：FILE_DISTRIBUTION） private List allAppActionScopeList; @@ -45,8 +44,7 @@ public class ServiceWhiteIPInfo { @Override public String toString() { return "ServiceWhiteIPInfo{" + - "cloudId=" + cloudId + - ", ip='" + ip + '\'' + + "hostId=" + hostId + ", isForAllApp=" + isForAllApp + ", allAppActionScopeList=" + allAppActionScopeList + ", appIdActionScopeMap=" + appIdActionScopeMap + diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceBatchGetAppHostsReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceBatchGetAppHostsReq.java new file mode 100644 index 0000000000..0178b17cbd --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceBatchGetAppHostsReq.java @@ -0,0 +1,86 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.inner.request; + +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.InvalidParamException; +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.common.util.ip.IpUtils; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; + +import java.util.List; + +/** + * 查询业务主机 + */ +@Getter +@Setter +@ToString +@NoArgsConstructor +@Slf4j +public class ServiceBatchGetAppHostsReq { + /** + * 查询的主机列表 + */ + private List hosts; + /** + * 是否需要刷新主机的bk_agent_id + */ + private boolean refreshAgentId; + + + public ServiceBatchGetAppHostsReq(List hosts, boolean refreshAgentId) { + this.hosts = hosts; + this.refreshAgentId = refreshAgentId; + } + + public void validate() throws InvalidParamException { + if (CollectionUtils.isEmpty(hosts)) { + log.error("Empty param: hosts"); + throw new InvalidParamException( + ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME_AND_REASON, + new Object[]{ + "hosts", + "hosts must not be empty" + }); + } + hosts.forEach(host -> { + if (host.getHostId() == null && !IpUtils.checkCloudIp(host.toCloudIp())) { + log.error("Invalid host: {}", host); + throw new InvalidParamException( + ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME_AND_REASON, + new Object[]{ + "hosts", + "Invalid host" + }); + } + }); + } +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceBatchGetHostToposReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceBatchGetHostToposReq.java new file mode 100644 index 0000000000..3d97f09b30 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceBatchGetHostToposReq.java @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.inner.request; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; + +import java.util.List; + +/** + * 根据主机ID获取主机拓扑路径请求 + */ +@Getter +@Setter +@ToString +@NoArgsConstructor +@AllArgsConstructor +public class ServiceBatchGetHostToposReq { + List hostIdList; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceBatchGetHostsReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceBatchGetHostsReq.java index d58e8a643a..ef61bdb1b2 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceBatchGetHostsReq.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceBatchGetHostsReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -40,7 +40,7 @@ @ToString @NoArgsConstructor public class ServiceBatchGetHostsReq { - List hosts; + private List hosts; public ServiceBatchGetHostsReq(List hosts) { this.hosts = hosts; diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceCheckAppHostsReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceCheckAppHostsReq.java deleted file mode 100644 index 6a77b216fa..0000000000 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceCheckAppHostsReq.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.model.inner.request; - -import com.tencent.bk.job.common.model.dto.HostDTO; -import lombok.Getter; -import lombok.NoArgsConstructor; -import lombok.Setter; -import lombok.ToString; - -import java.util.List; - -/** - * 检查主机是否在业务下请求 - */ -@Getter -@Setter -@ToString -@NoArgsConstructor -public class ServiceCheckAppHostsReq { - List hosts; - - public ServiceCheckAppHostsReq(List hosts) { - this.hosts = hosts; - } -} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceCheckScriptRequest.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceCheckScriptRequest.java index b2bc62d0ee..1af014072b 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceCheckScriptRequest.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceCheckScriptRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,7 +25,7 @@ package com.tencent.bk.job.manage.model.inner.request; import com.tencent.bk.job.common.validation.CheckEnum; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; @@ -49,7 +49,7 @@ public class ServiceCheckScriptRequest { /** * 脚本类型 * - * @see com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum + * @see com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum */ @ApiModelProperty("脚本类型") @CheckEnum(enumClass = ScriptTypeEnum.class, enumMethod = "isValid") diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceGetHostStatusByDynamicGroupReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceGetHostStatusByDynamicGroupReq.java index 0c85c2510f..44e7912d26 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceGetHostStatusByDynamicGroupReq.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceGetHostStatusByDynamicGroupReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceGetHostStatusByHostReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceGetHostStatusByHostReq.java new file mode 100644 index 0000000000..54b9e2299e --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceGetHostStatusByHostReq.java @@ -0,0 +1,39 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.inner.request; + +import com.tencent.bk.job.common.model.dto.HostDTO; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +@AllArgsConstructor +@NoArgsConstructor +@Data +public class ServiceGetHostStatusByHostReq { + List hostList; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceGetHostStatusByIpReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceGetHostStatusByIpReq.java deleted file mode 100644 index 129432b3ea..0000000000 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceGetHostStatusByIpReq.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.model.inner.request; - -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.util.List; - -@AllArgsConstructor -@NoArgsConstructor -@Data -public class ServiceGetHostStatusByIpReq { - List ipList; -} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceGetHostStatusByNodeReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceGetHostStatusByNodeReq.java index 066c424159..4ad4eba850 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceGetHostStatusByNodeReq.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceGetHostStatusByNodeReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,7 @@ package com.tencent.bk.job.manage.model.inner.request; -import com.tencent.bk.job.manage.model.web.request.ipchooser.AppTopologyTreeNode; +import com.tencent.bk.job.manage.model.web.request.chooser.host.BizTopoNode; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; @@ -35,5 +35,5 @@ @NoArgsConstructor @Data public class ServiceGetHostStatusByNodeReq { - List treeNodeList; + List treeNodeList; } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceGetHostsByCloudIpv6Req.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceGetHostsByCloudIpv6Req.java new file mode 100644 index 0000000000..012c07da84 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceGetHostsByCloudIpv6Req.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.inner.request; + +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; + +/** + * 根据云区域ID与Ipv6地址获取主机请求 + */ +@Getter +@Setter +@ToString +@NoArgsConstructor +@AllArgsConstructor +public class ServiceGetHostsByCloudIpv6Req { + Long cloudAreaId; + String ipv6; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceListPageTaskTemplatesRequest.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceListPageTaskTemplatesRequest.java index 63b8298203..ea58b08790 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceListPageTaskTemplatesRequest.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/request/ServiceListPageTaskTemplatesRequest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/resp/ServiceApplicationDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/resp/ServiceApplicationDTO.java index b51aad50cb..a8a3ff9d18 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/resp/ServiceApplicationDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/resp/ServiceApplicationDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/resp/ServiceBasicCredentialDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/resp/ServiceBasicCredentialDTO.java index 14e794c26a..e4bdbd36d1 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/resp/ServiceBasicCredentialDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/resp/ServiceBasicCredentialDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -30,7 +30,7 @@ import lombok.Data; import lombok.NoArgsConstructor; -@ApiModel("凭据基本信息") +@ApiModel("凭证基本信息") @Data @AllArgsConstructor @NoArgsConstructor diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/resp/ServiceCredentialDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/resp/ServiceCredentialDTO.java index b5e782d06f..f425ca2748 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/resp/ServiceCredentialDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/resp/ServiceCredentialDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -29,7 +29,7 @@ import io.swagger.annotations.ApiModelProperty; import lombok.Data; -@ApiModel("凭据") +@ApiModel("凭证") @Data public class ServiceCredentialDTO { /** @@ -53,8 +53,8 @@ public class ServiceCredentialDTO { @ApiModelProperty("类型") private String type; /** - * 凭据 + * 凭证 */ - @ApiModelProperty("凭据") + @ApiModelProperty("凭证") private CommonCredential credential; } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/resp/ServiceCredentialDisplayDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/resp/ServiceCredentialDisplayDTO.java index e0d14539e0..8b9e717d82 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/resp/ServiceCredentialDisplayDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/resp/ServiceCredentialDisplayDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -29,7 +29,7 @@ import lombok.AllArgsConstructor; import lombok.Data; -@ApiModel("凭据") +@ApiModel("凭证") @AllArgsConstructor @Data public class ServiceCredentialDisplayDTO { diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/resp/ServiceHostTopoDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/resp/ServiceHostTopoDTO.java new file mode 100644 index 0000000000..a24aba1aa5 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/resp/ServiceHostTopoDTO.java @@ -0,0 +1,60 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.inner.resp; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + +/** + * 主机拓扑 + */ +@Data +@EqualsAndHashCode +@NoArgsConstructor +@AllArgsConstructor +public class ServiceHostTopoDTO { + /** + * 主机Id + */ + private Long hostId; + /** + * 业务ID + */ + private Long bizId; + /** + * 集群ID + */ + private Long setId; + /** + * 模块ID + */ + private Long moduleId; + /** + * CMDB中的数据最后修改时间 + */ + private Long lastTime; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/task/ServiceTaskVariableDTO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/task/ServiceTaskVariableDTO.java index 1fe1e1d80a..0f4c154652 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/task/ServiceTaskVariableDTO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/inner/task/ServiceTaskVariableDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/migration/AddHostIdMigrationReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/migration/AddHostIdMigrationReq.java new file mode 100644 index 0000000000..59c83d8df1 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/migration/AddHostIdMigrationReq.java @@ -0,0 +1,41 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.migration; + +import com.tencent.bk.job.common.model.dto.ResourceScope; +import lombok.Data; + +import java.util.List; + +/** + * 作业模板、执行方案等包含的主机数据，在原来的云区域+ip的基础上，填充hostID属性 - 请求 + */ +@Data +public class AddHostIdMigrationReq { + // 迁移数据限定的资源范围，不传该参数则表示迁移所有资源范围下的数据 + private List scopeList; + + private boolean dryRun; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/migration/AddHostIdResult.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/migration/AddHostIdResult.java new file mode 100644 index 0000000000..496e48be9b --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/migration/AddHostIdResult.java @@ -0,0 +1,34 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.migration; + +import lombok.NoArgsConstructor; + +@NoArgsConstructor +public class AddHostIdResult extends MigrationRecordsResult { + public AddHostIdResult(String task) { + super(task); + } +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/migration/BkPlatformInfo.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/migration/BkPlatformInfo.java new file mode 100644 index 0000000000..0e69d54f82 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/migration/BkPlatformInfo.java @@ -0,0 +1,81 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.migration; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +public class BkPlatformInfo { + + @JsonProperty("bkAppCode") + private String bkAppCode = "bk_job"; + + @JsonProperty("name") + private String name = "蓝鲸作业平台"; + + @JsonProperty("nameEn") + private String nameEn = "BlueKing JOB"; + + @JsonProperty("appLogo") + private String appLogo = "/static/images/logo.png"; + + @JsonProperty("favicon") + private String favicon = "/static/images/favicon.icon"; + + @JsonProperty("helperText") + private String helperText = "联系BK助手"; + + @JsonProperty("helperTextEn") + private String helperTextEn = "Contact BK Assistant"; + + @JsonProperty("helperLink") + private String helperLink = "wxwork://message/?username=BK%E5%8A%A9%E6%89%8B"; + + @JsonProperty("brandImg") + private String brandImg = "brand_img.png"; + + @JsonProperty("brandImgEn") + private String brandImgEn = "brand_img.png"; + + @JsonProperty("brandName") + private String brandName = "腾讯蓝鲸智云"; + + @JsonProperty("brandNameEn") + private String brandNameEn = "BlueKing"; + + @JsonProperty("footerInfo") + private String footerInfo = "[技术支持](https://wpa1.qq.com/KziXGWJs?_type=wpa&qidian=true) | [社区论坛](https://bk" + + ".tencent.com/s-mart/community/) | [产品官网](https://bk.tencent.com/index/)"; + + @JsonProperty("footerInfoEn") + private String footerInfoEn = "[Support](https://wpa1.qq.com/KziXGWJs?_type=wpa&qidian=true) | [Forum](https://bk" + + ".tencent.com/s-mart/community/) | [Official](https://bk.tencent.com/index/)"; + + @JsonProperty("footerCopyright") + private String footerCopyright = "Copyright © 2012 Tencent BlueKing. All Rights Reserved. {{version}}"; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/migration/EncryptDbAccountPasswordReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/migration/EncryptDbAccountPasswordReq.java index 033b2dc486..91e69570c0 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/migration/EncryptDbAccountPasswordReq.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/migration/EncryptDbAccountPasswordReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/migration/MigrationRecordsResult.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/migration/MigrationRecordsResult.java new file mode 100644 index 0000000000..7d60d4e1e6 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/migration/MigrationRecordsResult.java @@ -0,0 +1,41 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.migration; + +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +public class MigrationRecordsResult { + private String task; + private int totalRecords; + private int successRecords; + private boolean success; + + public MigrationRecordsResult(String task) { + this.task = task; + } +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/migration/SetBizSetMigrationStatusReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/migration/SetBizSetMigrationStatusReq.java index 3ff1baddb3..483a0de21c 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/migration/SetBizSetMigrationStatusReq.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/migration/SetBizSetMigrationStatusReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/migration/UpdateAppIdForWhiteIpMigrationReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/migration/UpdateAppIdForWhiteIpMigrationReq.java new file mode 100644 index 0000000000..fa81751b5a --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/migration/UpdateAppIdForWhiteIpMigrationReq.java @@ -0,0 +1,35 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.migration; + +import lombok.Data; + +/** + * IP白名单更新AppId，全业务ID->代表所有业务的ID - 请求 + */ +@Data +public class UpdateAppIdForWhiteIpMigrationReq { + private boolean dryRun; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/migration/UpdateAppIdResult.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/migration/UpdateAppIdResult.java new file mode 100644 index 0000000000..75dd808bfd --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/migration/UpdateAppIdResult.java @@ -0,0 +1,35 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.migration; + +import lombok.NoArgsConstructor; + +@NoArgsConstructor +public class UpdateAppIdResult extends MigrationRecordsResult { + + public UpdateAppIdResult(String task) { + super(task); + } +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/op/req/ConfigFlowControlReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/op/req/ConfigFlowControlReq.java index e1aa1a7498..026a52b655 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/op/req/ConfigFlowControlReq.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/op/req/ConfigFlowControlReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/tmp/MigrationPlanBasic.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/tmp/MigrationPlanBasic.java index aa07ec99f1..0e85f368d3 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/tmp/MigrationPlanBasic.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/tmp/MigrationPlanBasic.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/tmp/TmpAccountCreateUpdateReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/tmp/TmpAccountCreateUpdateReq.java index b31f34629e..a53f4e42c6 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/tmp/TmpAccountCreateUpdateReq.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/tmp/TmpAccountCreateUpdateReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/AccountCreateUpdateReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/AccountCreateUpdateReq.java index abdd224333..58b8a0f9b3 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/AccountCreateUpdateReq.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/AccountCreateUpdateReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -37,8 +37,10 @@ @ApiModel("账号创建、更新请求") @ToString(exclude = {"password", "dbPassword"}) public class AccountCreateUpdateReq { - - @ApiModelProperty(value = "ID,更新账号的时候需要传入，新建账号不需要") + /** + * 账号ID + */ + @ApiModelProperty(value = "账号ID ID", hidden = true) private Long id; /** * 帐号名称 @@ -81,7 +83,7 @@ public class AccountCreateUpdateReq { private String password; @ApiModelProperty(value = "DB端口,创建/更新DB账号的时候必传") - @Range(min = 0, max = 65535, message = "{validation.constraints.InvalidPort.message}") + @Range(min = 1, max = 65535, message = "{validation.constraints.InvalidPort.message}") private Integer dbPort; @ApiModelProperty(value = "DB账号关联的系统账号,创建/更新DB账号的时候必传") diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/AgentStatisticsReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/AgentStatisticsReq.java index 8c107d2155..fc84d1c714 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/AgentStatisticsReq.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/AgentStatisticsReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,38 +24,36 @@ package com.tencent.bk.job.manage.model.web.request; -import com.tencent.bk.job.manage.model.web.request.ipchooser.AppTopologyTreeNode; +import com.tencent.bk.job.common.model.vo.DynamicGroupIdWithMeta; +import com.tencent.bk.job.manage.model.web.request.chooser.host.BizTopoNode; +import com.tencent.bk.job.manage.model.web.request.chooser.host.HostIdWithMeta; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; +import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; /** * 主机状态批量查询请求 */ - @Data @ApiModel("主机状态批量查询请求报文") public class AgentStatisticsReq { - /** - * IP列表 - */ - @ApiModelProperty(value = "IP列表", required = false) - List ipList; - - /** - * 业务拓扑节点列表 - */ - @ApiModelProperty(value = "业务拓扑节点列表(传 objectId 与 instanceId ，其余字段置空即可)", required = false) - List appTopoNodeList; - - /** - * 动态分组Id列表 - */ - @ApiModelProperty(value = "动态分组Id列表", required = false) - List dynamicGroupIds; -} + @ApiModelProperty(value = "主机列表") + List hostList = new ArrayList<>(); + @ApiModelProperty(value = "业务拓扑节点列表(传 objectId 与 instanceId ，其余字段置空即可)") + List nodeList; + @ApiModelProperty(value = "动态分组列表") + List dynamicGroupList; + + public List getDynamicGroupIds() { + return dynamicGroupList.stream() + .map(DynamicGroupIdWithMeta::getId) + .collect(Collectors.toList()); + } +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/BatchGetPlanReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/BatchGetPlanReq.java index ecd07eff09..68fa1e12b4 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/BatchGetPlanReq.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/BatchGetPlanReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/BatchPatchResourceTagReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/BatchPatchResourceTagReq.java index 93d038cf2f..b0fabf41dd 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/BatchPatchResourceTagReq.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/BatchPatchResourceTagReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/CredentialCreateUpdateReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/CredentialCreateUpdateReq.java index 722fce73c1..bfefd9ac86 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/CredentialCreateUpdateReq.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/CredentialCreateUpdateReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,22 +24,25 @@ package com.tencent.bk.job.manage.model.web.request; -import com.tencent.bk.job.manage.common.consts.CredentialTypeEnum; import com.tencent.bk.job.common.model.dto.CommonCredential; +import com.tencent.bk.job.manage.api.common.constants.CredentialTypeEnum; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; -import static com.tencent.bk.job.manage.common.consts.CredentialTypeEnum.APP_ID_SECRET_KEY; -import static com.tencent.bk.job.manage.common.consts.CredentialTypeEnum.PASSWORD; -import static com.tencent.bk.job.manage.common.consts.CredentialTypeEnum.SECRET_KEY; -import static com.tencent.bk.job.manage.common.consts.CredentialTypeEnum.USERNAME_PASSWORD; +import static com.tencent.bk.job.manage.api.common.constants.CredentialTypeEnum.APP_ID_SECRET_KEY; +import static com.tencent.bk.job.manage.api.common.constants.CredentialTypeEnum.PASSWORD; +import static com.tencent.bk.job.manage.api.common.constants.CredentialTypeEnum.SECRET_KEY; +import static com.tencent.bk.job.manage.api.common.constants.CredentialTypeEnum.USERNAME_PASSWORD; @Data -@ApiModel("凭据创建/更新请求") +@ApiModel("凭证创建/更新请求") public class CredentialCreateUpdateReq { - @ApiModelProperty(value = "ID,更新凭据的时候需要传入，新建时不需要", required = false) + /** + * 凭证ID + */ + @ApiModelProperty(value = "凭证 ID", hidden = true) private String id; /** * 名称 diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/HostCheckReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/HostCheckReq.java new file mode 100644 index 0000000000..f8f281ca11 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/HostCheckReq.java @@ -0,0 +1,66 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.web.request; + +import com.tencent.bk.job.manage.api.common.constants.whiteip.ActionScopeEnum; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import org.apache.commons.lang3.StringUtils; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +@Data +@ApiModel("主机检查请求报文") +public class HostCheckReq { + + @ApiModelProperty(value = "应用场景：脚本执行/文件分发，Key为actionScope，Value分别为SCRIPT_EXECUTE/FILE_DISTRIBUTION") + Map meta = new HashMap<>(); + + @ApiModelProperty(value = "hostId列表", required = true) + List hostIdList = new ArrayList<>(); + + @ApiModelProperty(value = "IP列表，单个IP格式：cloudAreaId:[ip]或cloudAreaId:ip或ip") + List ipList = new ArrayList<>(); + + @ApiModelProperty(value = "IPv6列表，单个IPv6格式：cloudAreaId:[ipv6]或ipv6") + List ipv6List = new ArrayList<>(); + + @ApiModelProperty(value = "关键字列表，可匹配主机名称") + List keyList = new ArrayList<>(); + + public ActionScopeEnum getActionScope() { + String actionScopeName = (String) meta.get("actionScope"); + if (StringUtils.isBlank(actionScopeName)) { + return null; + } + return ActionScopeEnum.valueOf(actionScopeName); + } +} + + diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/IpCheckReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/IpCheckReq.java deleted file mode 100644 index 089a297dcd..0000000000 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/IpCheckReq.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.model.web.request; - -import com.tencent.bk.job.manage.common.consts.whiteip.ActionScopeEnum; -import io.swagger.annotations.ApiModel; -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; - -import java.util.List; - -@Data -@ApiModel("IP检查请求报文") -public class IpCheckReq { - - @ApiModelProperty(value = "应用场景：脚本执行/文件分发", required = false) - ActionScopeEnum actionScope; - - @ApiModelProperty(value = "IP列表，单个IP格式：cloudAreaId:ip", required = true) - List ipList; - -} - - diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/OperationPermissionReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/OperationPermissionReq.java index e04dc962e1..ed1f516d6d 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/OperationPermissionReq.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/OperationPermissionReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/ScriptCheckReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/ScriptCheckReq.java index 47a536d877..e8b10e3584 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/ScriptCheckReq.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/ScriptCheckReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/ScriptCreateReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/ScriptCreateReq.java new file mode 100644 index 0000000000..74790a1d2e --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/ScriptCreateReq.java @@ -0,0 +1,95 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.web.request; + +import com.tencent.bk.job.common.validation.CheckEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; +import com.tencent.bk.job.manage.model.web.vo.TagVO; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import org.hibernate.validator.constraints.Length; + +import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.NotNull; +import javax.validation.constraints.Pattern; +import java.util.List; + +/** + * 脚本新增请求 + */ +@Data +@ApiModel("脚本新增请求") +public class ScriptCreateReq { + + /** + * 脚本名称 + */ + @ApiModelProperty(value = "脚本名称", required = true, example = "scriptName") + @NotEmpty(message = "{validation.constraints.ScriptName_empty.message}") + @Length(max = 60, message = "{validation.constraints.ScriptName_outOfLength.message}") + @Pattern(regexp = "^[^\\\\|/:*<>\"?]+$", message = "{validation.constraints.ScriptName_illegal.message}") + private String name; + + /** + * 脚本类型 + */ + @ApiModelProperty(value = "脚本类型,创建脚本时需要传入") + @NotNull(message = "{validation.constraints.ScriptType_empty.message}") + @CheckEnum(enumClass = ScriptTypeEnum.class, enumMethod = "isValid", + message = "{validation.constraints.ScriptType_illegal.message}") + private Integer type; + + /** + * 脚本内容 + */ + @ApiModelProperty(value = "脚本内容,创建脚本时需要传入，BASE64编码") + @NotEmpty(message = "{validation.constraints.ScriptContent_empty.message}") + private String content; + + /** + * 脚本的版本号 + */ + @ApiModelProperty(value = "版本号，新增脚本时需要传入") + @NotEmpty(message = "{validation.constraints.ScriptVersion_empty.message}") + @Length(max = 60, message = "{validation.constraints.ScriptVersion_outOfLength.message}") + @Pattern(regexp = "^[A-Za-z0-9_\\-#@.]+$", message = "{validation.constraints.ScriptVersion_illegal.message}") + private String version; + + /** + * 脚本标签 + */ + @ApiModelProperty(value = "脚本标签，新增/更新脚本需要传入") + private List tags; + + /** + * 脚本描述 + */ + @ApiModelProperty(value = "脚本描述") + private String description; + +} + + diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/ScriptCreateUpdateReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/ScriptCreateUpdateReq.java deleted file mode 100644 index bd7c5904ac..0000000000 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/ScriptCreateUpdateReq.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.model.web.request; - -import com.tencent.bk.job.manage.model.web.vo.TagVO; -import io.swagger.annotations.ApiModel; -import io.swagger.annotations.ApiModelProperty; -import lombok.Data; - -import java.util.List; - -/** - * 脚本新增、更新请求 - */ - -@Data -@ApiModel("脚本新增、更新请求报文") -public class ScriptCreateUpdateReq { - /** - * 脚本版本ID，对应某个版本的脚本的ID - */ - @ApiModelProperty(value = "脚本版本ID,更新脚本版本时需要传入", required = false) - private Long scriptVersionId; - /** - * 脚本ID，一个脚本包含多个版本的脚本 - */ - @ApiModelProperty(value = "脚本ID,新增/更新脚本版本时需要传入", required = false) - private String id; - /** - * 脚本名称 - */ - @ApiModelProperty(value = "脚本名称", required = true, example = "scriptName") - private String name; - /** - * 脚本类型 - */ - @ApiModelProperty(value = "脚本类型,新增脚本/脚本版本时需要传入", required = false) - private Integer type; - /** - * 脚本内容 - */ - @ApiModelProperty(value = "脚本内容,新增脚本/脚本版本时需要传入，BASE64编码", required = false) - private String content; - - /** - * 创建者 - */ - @ApiModelProperty(value = "创建者，新增脚本/脚本版本时需要传入", required = false) - private String creator; - - /** - * 脚本的版本号 - */ - @ApiModelProperty(value = "版本号，新增脚本/脚本版本时需要传入", required = false) - private String version; - /** - * 脚本标签 - */ - @ApiModelProperty(value = "脚本标签，新增/更新脚本需要传入", required = false) - private List tags; - - /** - * 脚本描述 - */ - @ApiModelProperty(value = "脚本描述", required = false) - private String description; - - /** - * 脚本版本描述 - */ - @ApiModelProperty(value = "脚本版本描述", required = false) - private String versionDesc; - - -} - - diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/ScriptInfoUpdateReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/ScriptInfoUpdateReq.java index 82fe2455f7..9bf88222d5 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/ScriptInfoUpdateReq.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/ScriptInfoUpdateReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,15 +24,25 @@ package com.tencent.bk.job.manage.model.web.request; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.InvalidParamException; +import com.tencent.bk.job.common.util.check.IlegalCharChecker; +import com.tencent.bk.job.common.util.check.MaxLengthChecker; +import com.tencent.bk.job.common.util.check.NotEmptyChecker; +import com.tencent.bk.job.common.util.check.StringCheckHelper; +import com.tencent.bk.job.common.util.check.TrimChecker; +import com.tencent.bk.job.common.util.check.exception.StringCheckException; import com.tencent.bk.job.manage.model.web.vo.TagVO; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; +import lombok.extern.slf4j.Slf4j; import java.util.List; @Data @ApiModel("脚本元数据更新请求报文") +@Slf4j public class ScriptInfoUpdateReq { /** * 脚本更新的字段，目前支持：scriptName/scriptDesc/scriptTags @@ -48,4 +58,16 @@ public class ScriptInfoUpdateReq { @ApiModelProperty(value = "脚本标签", required = false) private List scriptTags; + + + public void validateScriptName() { + try { + StringCheckHelper stringCheckHelper = new StringCheckHelper(new TrimChecker(), + new NotEmptyChecker(), new IlegalCharChecker(), new MaxLengthChecker(60)); + this.scriptName = stringCheckHelper.checkAndGetResult(scriptName); + } catch (StringCheckException e) { + log.warn("scriptName is invalid:", e); + throw new InvalidParamException(ErrorCode.SCRIPT_NAME_INVALID); + } + } } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/ScriptSyncReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/ScriptSyncReq.java index 46fbb524e8..5d031ddd33 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/ScriptSyncReq.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/ScriptSyncReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/ScriptTagBatchPatchReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/ScriptTagBatchPatchReq.java index d39a8659af..f5109f1720 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/ScriptTagBatchPatchReq.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/ScriptTagBatchPatchReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,14 +25,19 @@ package com.tencent.bk.job.manage.model.web.request; import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.InvalidParamException; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; import java.util.List; @Data @ApiModel("作业模板标签批量更新请求") +@Slf4j public class ScriptTagBatchPatchReq { @ApiModelProperty(value = "脚本 ID 列表", required = true) @@ -46,4 +51,11 @@ public class ScriptTagBatchPatchReq { @ApiModelProperty(value = "新增的标签ID列表") @JsonProperty("deleteTagIdList") private List deleteTagIdList; + + public void validate() { + if (CollectionUtils.isEmpty(this.idList)) { + log.warn("ScriptTagBatchUpdateReq->idList is empty"); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME, "idList"); + } + } } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/ScriptVersionCreateUpdateReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/ScriptVersionCreateUpdateReq.java new file mode 100644 index 0000000000..672017b13a --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/ScriptVersionCreateUpdateReq.java @@ -0,0 +1,69 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.web.request; + +import com.tencent.bk.job.common.validation.Create; +import com.tencent.bk.job.common.validation.Update; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import org.hibernate.validator.constraints.Length; + +import javax.validation.constraints.NotEmpty; +import javax.validation.constraints.Pattern; + +/** + * 脚本版本新增、更新请求 + */ +@Data +@ApiModel("脚本版本新增、更新请求报文") +public class ScriptVersionCreateUpdateReq { + + /** + * 脚本内容 + */ + @ApiModelProperty(value = "脚本内容,新增脚本/脚本版本时需要传入，BASE64编码") + @NotEmpty(message = "{validation.constraints.ScriptContent_empty.message}", groups = {Create.class, Update.class}) + private String content; + + /** + * 脚本的版本号 + */ + @ApiModelProperty(value = "版本号，新增脚本版本时需要传入") + @NotEmpty(message = "{validation.constraints.ScriptVersion_empty.message}", groups = Create.class) + @Length(max = 60, message = "{validation.constraints.ScriptVersion_outOfLength.message}", groups = Create.class) + @Pattern(regexp = "^[A-Za-z0-9_\\-#@.]+$", message = "{validation.constraints.ScriptVersion_illegal.message}", + groups = Create.class) + private String version; + + /** + * 脚本版本描述 + */ + @ApiModelProperty(value = "脚本版本描述") + private String versionDesc; + +} + + diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/TagCreateUpdateReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/TagCreateUpdateReq.java index e7b582d5ac..77d6e9b862 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/TagCreateUpdateReq.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/TagCreateUpdateReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/TaskPlanCreateUpdateReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/TaskPlanCreateUpdateReq.java index 5e006aa964..0e3ebdad15 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/TaskPlanCreateUpdateReq.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/TaskPlanCreateUpdateReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -40,13 +40,15 @@ public class TaskPlanCreateUpdateReq { /** * 执行方案 ID */ - @ApiModelProperty(value = "执行方案 ID，更新和删除时需要传入", required = true) + @Deprecated + @ApiModelProperty(value = "执行方案 ID", hidden = true) private Long id; /** * 模版 ID */ - @ApiModelProperty(value = "模版 ID", required = true) + @Deprecated + @ApiModelProperty(value = "模版 ID", hidden = true) private Long templateId; /** diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/TaskTemplateCreateUpdateReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/TaskTemplateCreateUpdateReq.java index 34906323d8..b65bd51882 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/TaskTemplateCreateUpdateReq.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/TaskTemplateCreateUpdateReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,8 @@ package com.tencent.bk.job.manage.model.web.request; -import com.tencent.bk.job.common.util.JobContextUtil; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.util.check.IlegalCharChecker; import com.tencent.bk.job.common.util.check.MaxLengthChecker; import com.tencent.bk.job.common.util.check.NotEmptyChecker; @@ -45,9 +46,6 @@ import java.util.List; import java.util.Set; -/** - * @since 15/10/2019 20:27 - */ @Slf4j @Data @EqualsAndHashCode(callSuper = true) @@ -66,15 +64,15 @@ public class TaskTemplateCreateUpdateReq extends TemplateBasicInfoUpdateReq { @ApiModelProperty(value = "模版变量, 新增、修改、删除时需要传入") private List variables; - public boolean validate() { + public void validate() throws InvalidParamException { // 模板名称检查 try { StringCheckHelper stringCheckHelper = new StringCheckHelper(new TrimChecker(), new NotEmptyChecker(), new IlegalCharChecker(), new MaxLengthChecker(60)); this.setName(stringCheckHelper.checkAndGetResult(this.getName())); } catch (StringCheckException e) { - log.warn("Template name is invalid:", e); - return false; + log.warn("Template name is invalid, name: {}", getName()); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } if (this.getDescription() == null) { this.setDescription(""); @@ -85,42 +83,27 @@ public boolean validate() { } if (isCreate) { if (CollectionUtils.isEmpty(steps)) { - JobContextUtil.addDebugMessage("Empty template step!"); - return false; + log.warn("Empty template step!"); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } } if (CollectionUtils.isNotEmpty(steps)) { for (TaskStepVO step : steps) { - if (!step.validate(isCreate)) { - JobContextUtil.addDebugMessage("Validate step failed!"); - return false; - } - try { - StringCheckHelper stepCheckHelper = new StringCheckHelper(new TrimChecker(), new NotEmptyChecker(), - new IlegalCharChecker(), new MaxLengthChecker(60)); - step.setName(stepCheckHelper.checkAndGetResult(step.getName())); - } catch (StringCheckException e) { - log.warn("Step name is invalid:", e); - return false; - } + step.validate(isCreate); } } if (CollectionUtils.isNotEmpty(variables)) { Set variableNameList = new HashSet<>(); for (TaskVariableVO variable : variables) { - if (!variable.validate(isCreate)) { - JobContextUtil.addDebugMessage("Validate variable failed!"); - return false; - } + variable.validate(isCreate); if (variableNameList.contains(variable.getName())) { - JobContextUtil.addDebugMessage("Variable name duplicated!"); - return false; + log.warn("Variable name duplicated!"); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } if (variable.getDelete() != 1) { variableNameList.add(variable.getName()); } } } - return true; } } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/TaskVariableValueUpdateReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/TaskVariableValueUpdateReq.java index 2e85b3d8ed..c1cf15d7ed 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/TaskVariableValueUpdateReq.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/TaskVariableValueUpdateReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/TemplateBasicInfoUpdateReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/TemplateBasicInfoUpdateReq.java index 63a48e23fd..5fc2e453e4 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/TemplateBasicInfoUpdateReq.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/TemplateBasicInfoUpdateReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -40,7 +40,8 @@ public class TemplateBasicInfoUpdateReq { /** * 模版 ID */ - @ApiModelProperty(value = "模版 ID, 更新和删除时需要传入") + @Deprecated + @ApiModelProperty(value = "作业模板 ID", hidden = true) private Long id; /** diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/TemplateTagBatchPatchReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/TemplateTagBatchPatchReq.java index 2f00f8f19d..84b2f74332 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/TemplateTagBatchPatchReq.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/TemplateTagBatchPatchReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/app/FavorAppReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/app/FavorAppReq.java index cad87332c4..8364f41350 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/app/FavorAppReq.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/app/FavorAppReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/ListTopologyTreesReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/ListTopologyTreesReq.java new file mode 100644 index 0000000000..ddefd57f50 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/ListTopologyTreesReq.java @@ -0,0 +1,43 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.web.request.chooser; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.util.List; + +@Data +@ApiModel("获取拓扑树请求") +public class ListTopologyTreesReq { + + @ApiModelProperty(value = "是否获取所有资源范围的拓扑结构，默认为false", required = true) + Boolean allScope = false; + + @ApiModelProperty(value = "要获取拓扑树的资源范围列表") + List scopeList; + +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/Scope.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/Scope.java new file mode 100644 index 0000000000..02bd8f86c2 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/Scope.java @@ -0,0 +1,39 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.web.request.chooser; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +@Data +@ApiModel("资源范围") +public class Scope { + @ApiModelProperty(value = "资源范围类型", required = true) + String scopeType; + + @ApiModelProperty(value = "资源范围Id", required = true) + String scopeId; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/container/ContainerCheckReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/container/ContainerCheckReq.java new file mode 100644 index 0000000000..e8f08b594e --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/container/ContainerCheckReq.java @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.web.request.chooser.container; + +import com.fasterxml.jackson.annotation.JsonInclude; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +@Data +@ApiModel("容器检查请求报文") +@JsonInclude(JsonInclude.Include.NON_NULL) +public class ContainerCheckReq { + + @ApiModelProperty(value = "容器Uid列表", required = true) + private List uidList = new ArrayList<>(); + +} + + diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/container/ContainerDetailReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/container/ContainerDetailReq.java new file mode 100644 index 0000000000..d1f8aa7fa3 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/container/ContainerDetailReq.java @@ -0,0 +1,43 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.web.request.chooser.container; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +@Data +@ApiModel("容器详情请求报文") +public class ContainerDetailReq { + + @ApiModelProperty(value = "containerId及元数据列表", required = true) + List containerList = new ArrayList<>(); + +} + + diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/container/ContainerIdWithMeta.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/container/ContainerIdWithMeta.java new file mode 100644 index 0000000000..c2b76a406c --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/container/ContainerIdWithMeta.java @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.web.request.chooser.container; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.Map; + +@Data +@ApiModel("容器ID及元数据") +@NoArgsConstructor +public class ContainerIdWithMeta { + @ApiModelProperty(value = "容器资源 ID", required = true) + private Long id; + + @ApiModelProperty(value = "容器元数据；Job不支持") + Map meta; + + public ContainerIdWithMeta(Long id) { + this.id = id; + } +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/container/ListContainerByTopologyNodesReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/container/ListContainerByTopologyNodesReq.java new file mode 100644 index 0000000000..9063300cdb --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/container/ListContainerByTopologyNodesReq.java @@ -0,0 +1,76 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.web.request.chooser.container; + +import com.tencent.bk.job.manage.model.web.vo.chooser.container.ContainerTopologyNodeVO; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; +import java.util.Map; + + +@NoArgsConstructor +@AllArgsConstructor +@Data +@ApiModel("通过拓扑节点集合获取容器集合") +public class ListContainerByTopologyNodesReq { + + @ApiModelProperty(value = "拓扑节点列表", required = true) + private List nodeList; + + @ApiModelProperty(value = "筛选条件：容器ID列表") + private List containerUidList; + + @ApiModelProperty(value = "筛选条件：容器名称列表, 支持模糊检索") + private List containerNameKeywordList; + + @ApiModelProperty(value = "筛选条件：Pod名称列表, 支持模糊检索") + private List podNameKeywordList; + + @ApiModelProperty(value = "筛选条件：Pod label") + private Map podLabels; + + @ApiModelProperty(value = "数据起始位置") + private Integer start; + + @ApiModelProperty(value = "拉取数量") + private Integer pageSize; + + /** + * 检查分页参数是否有效 + * + * @return 分页参数是否有效 + */ + public boolean hasValidPageParams() { + return start != null + && start >= 0 + && pageSize != null + && pageSize > 0; + } +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/host/BizTopoNode.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/host/BizTopoNode.java new file mode 100644 index 0000000000..a2d54c5566 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/host/BizTopoNode.java @@ -0,0 +1,85 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.web.request.chooser.host; + +import com.tencent.bk.job.common.model.vo.TargetNodeVO; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; +import java.util.Objects; + +@NoArgsConstructor +@AllArgsConstructor +@Data +@ApiModel("业务拓扑节点信息") +public class BizTopoNode { + @ApiModelProperty(value = "节点类型Id", required = true) + private String objectId; + @ApiModelProperty(value = "节点类型名称") + private String objectName; + @ApiModelProperty(value = "节点实例Id", required = true) + private Long instanceId; + @ApiModelProperty(value = "节点实例名称") + private String instanceName; + @ApiModelProperty(value = "子节点列表") + private List childs; + + public static BizTopoNode fromTargetNodeVO(TargetNodeVO targetNodeVO) { + BizTopoNode node = new BizTopoNode(); + node.setObjectId(targetNodeVO.getObjectId()); + node.setInstanceId(targetNodeVO.getInstanceId()); + return node; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (!(o instanceof BizTopoNode)) return false; + BizTopoNode that = (BizTopoNode) o; + return Objects.equals(objectId, that.objectId) && + Objects.equals(instanceId, that.instanceId); + } + + @Override + public int hashCode() { + return Objects.hash(objectId, instanceId); + } + + public String getSimpleDesc() { + return "(" + objectId + "," + instanceId + ")"; + } + + @Override + public String toString() { + return "BizTopoNode{" + + "objectId='" + objectId + '\'' + + ", instanceId=" + instanceId + + '}'; + } +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/host/GetHostAgentStatisticsByDynamicGroupsReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/host/GetHostAgentStatisticsByDynamicGroupsReq.java new file mode 100644 index 0000000000..e2654ce7e0 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/host/GetHostAgentStatisticsByDynamicGroupsReq.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.web.request.chooser.host; + +import com.tencent.bk.job.common.model.vo.DynamicGroupIdWithMeta; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +@NoArgsConstructor +@AllArgsConstructor +@Data +@ApiModel("通过动态分组列表获取主机Agent状态统计信息") +public class GetHostAgentStatisticsByDynamicGroupsReq { + + @ApiModelProperty(value = "动态分组ID及元数据列表", required = true) + List dynamicGroupList; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/host/GetHostAgentStatisticsByNodesReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/host/GetHostAgentStatisticsByNodesReq.java new file mode 100644 index 0000000000..00a9b24e9b --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/host/GetHostAgentStatisticsByNodesReq.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.web.request.chooser.host; + +import com.tencent.bk.job.common.model.vo.TargetNodeVO; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +@NoArgsConstructor +@AllArgsConstructor +@Data +@ApiModel("通过拓扑节点集合获取主机Agent状态统计信息") +public class GetHostAgentStatisticsByNodesReq { + + @ApiModelProperty(value = "拓扑节点列表", required = true) + List nodeList; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/host/HostDetailReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/host/HostDetailReq.java new file mode 100644 index 0000000000..0f5b120ddc --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/host/HostDetailReq.java @@ -0,0 +1,43 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.web.request.chooser.host; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.util.ArrayList; +import java.util.List; + +@Data +@ApiModel("主机详情请求报文") +public class HostDetailReq { + + @ApiModelProperty(value = "hostId及元数据列表", required = true) + List hostList = new ArrayList<>(); + +} + + diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/host/HostIdWithMeta.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/host/HostIdWithMeta.java new file mode 100644 index 0000000000..c436c2099f --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/host/HostIdWithMeta.java @@ -0,0 +1,43 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.web.request.chooser.host; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Data; + +import java.util.Map; + +@AllArgsConstructor +@Data +@ApiModel("主机ID及元数据") +public class HostIdWithMeta { + @ApiModelProperty(value = "主机HostId", required = true) + Long hostId; + + @ApiModelProperty(value = "主机元数据") + Map meta; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/host/ListDynamicGroupsReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/host/ListDynamicGroupsReq.java new file mode 100644 index 0000000000..b4898e0163 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/host/ListDynamicGroupsReq.java @@ -0,0 +1,41 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.web.request.chooser.host; + +import com.tencent.bk.job.common.model.vo.DynamicGroupIdWithMeta; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.util.List; + +@Data +@ApiModel("批量获取动态分组请求体") +public class ListDynamicGroupsReq { + + @ApiModelProperty(value = "要获取的动态分组ID及元数据列表，不传表示获取所有") + List dynamicGroupList; + +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/host/ListHostByBizTopologyNodesReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/host/ListHostByBizTopologyNodesReq.java new file mode 100644 index 0000000000..e984abdf54 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/host/ListHostByBizTopologyNodesReq.java @@ -0,0 +1,72 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.web.request.chooser.host; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +/** + * @Description + * @Date 2020/3/23 + * @Version 1.0 + */ +@NoArgsConstructor +@AllArgsConstructor +@Data +@ApiModel("通过拓扑节点集合获取主机集合") +public class ListHostByBizTopologyNodesReq { + + @ApiModelProperty(value = "拓扑节点列表", required = true) + List nodeList; + + @ApiModelProperty(value = "搜索内容（同时对主机IP/IPv6/主机名称/操作系统名称/云区域名称进行模糊搜索）") + String searchContent; + + @ApiModelProperty(value = "筛选条件：ip关键字列表，ip与列表中任意一个关键字相似即命中") + List ipKeyList; + + @ApiModelProperty(value = "筛选条件：ipv6关键字列表，ipv6与列表中任意一个关键字相似即命中") + List ipv6KeyList; + + @ApiModelProperty(value = "筛选条件：主机名称关键字列表，主机名称与列表中任意一个关键字相似即命中") + List hostNameKeyList; + + @ApiModelProperty(value = "筛选条件：操作系统名称关键字列表，操作系统名称与列表中任意一个关键字相似即命中") + List osNameKeyList; + + @ApiModelProperty(value = "筛选条件：alive：0为Agent异常，1为Agent正常，不传则不筛选") + Integer alive; + + @ApiModelProperty(value = "数据起始位置") + Long start; + + @ApiModelProperty(value = "拉取数量") + Long pageSize; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/ipchooser/ListNodeChildrenReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/host/ListNodeChildrenReq.java similarity index 91% rename from src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/ipchooser/ListNodeChildrenReq.java rename to src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/host/ListNodeChildrenReq.java index e58055e025..ff2d1c8ac5 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/ipchooser/ListNodeChildrenReq.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/host/ListNodeChildrenReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.manage.model.web.request.ipchooser; +package com.tencent.bk.job.manage.model.web.request.chooser.host; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; @@ -38,7 +38,7 @@ public class ListNodeChildrenReq { @ApiModelProperty(value = "业务拓扑节点", required = true) - AppTopologyTreeNode appTopoNode; + BizTopoNode appTopoNode; @ApiModelProperty(value = "数据起始位置", required = false) Long offset; diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/host/PageListHostsByDynamicGroupReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/host/PageListHostsByDynamicGroupReq.java new file mode 100644 index 0000000000..a56862164c --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/host/PageListHostsByDynamicGroupReq.java @@ -0,0 +1,52 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.web.request.chooser.host; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import io.swagger.annotations.ApiParam; +import lombok.Data; + +import javax.validation.constraints.NotNull; +import java.util.Map; + +@Data +@ApiModel("根据动态分组分页查询主机请求体") +public class PageListHostsByDynamicGroupReq { + + @ApiModelProperty(value = "动态分组ID，若传多个请使用英文逗号分隔", required = true) + @NotNull(message = "{validation.constraints.emptyDynamicGroupId.message}") + String id; + + @ApiModelProperty(value = "动态分组元数据") + Map meta; + + @ApiParam(value = "数据起始位置，不传默认为0") + Long start = 0L; + + @ApiParam(value = "拉取的数据量，不传默认拉取全量") + Long pageSize = -1L; + +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/host/QueryNodesPathReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/host/QueryNodesPathReq.java new file mode 100644 index 0000000000..c4c3c9fa76 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/chooser/host/QueryNodesPathReq.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.web.request.chooser.host; + +import com.tencent.bk.job.common.model.vo.TargetNodeVO; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +@NoArgsConstructor +@AllArgsConstructor +@Data +@ApiModel("查询多个拓扑节点的拓扑路径") +public class QueryNodesPathReq { + + @ApiModelProperty(value = "需要查询拓扑路径的节点列表(将拓扑树节点中的objectId与instanceId传入)", required = true) + List nodeList; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/customsetting/BatchGetCustomSettingsReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/customsetting/BatchGetCustomSettingsReq.java new file mode 100644 index 0000000000..18e403c2cb --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/customsetting/BatchGetCustomSettingsReq.java @@ -0,0 +1,43 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.web.request.customsetting; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.util.List; + +/** + * 批量获取用户自定义配置项请求 + */ +@Data +@ApiModel("批量获取用户自定义配置项请求") +public class BatchGetCustomSettingsReq { + + @ApiModelProperty(value = "要批量获取配置项的模块列表，不传表示获取所有模块配置项") + private List moduleList; + +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/customsetting/DeleteCustomSettingsReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/customsetting/DeleteCustomSettingsReq.java new file mode 100644 index 0000000000..093dc1428b --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/customsetting/DeleteCustomSettingsReq.java @@ -0,0 +1,43 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.web.request.customsetting; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.util.List; + +/** + * 删除用户自定义配置项请求 + */ +@Data +@ApiModel("删除用户自定义配置项请求") +public class DeleteCustomSettingsReq { + + @ApiModelProperty(value = "要删除配置项的模块列表，不传表示删除所有模块配置项") + private List moduleList; + +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/customsetting/SaveCustomSettingsReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/customsetting/SaveCustomSettingsReq.java new file mode 100644 index 0000000000..fc9789dfc4 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/customsetting/SaveCustomSettingsReq.java @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.web.request.customsetting; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +import java.util.Map; + +/** + * 用户自定义配置项新增、更新请求 + */ +@Data +@ApiModel("用户自定义配置项新增、更新请求") +public class SaveCustomSettingsReq { + + @ApiModelProperty( + value = "要保存的配置项Map：key：（字符串，表示配置模块），value：（Map，包含多个配置项内容）", + required = true) + private Map> settingsMap; + +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/customsetting/ScriptTemplateCreateUpdateReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/customsetting/ScriptTemplateCreateUpdateReq.java index e0c34554db..1a105b2381 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/customsetting/ScriptTemplateCreateUpdateReq.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/customsetting/ScriptTemplateCreateUpdateReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/customsetting/ScriptTemplateRenderReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/customsetting/ScriptTemplateRenderReq.java index b5c06b0569..9ffbd74b68 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/customsetting/ScriptTemplateRenderReq.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/customsetting/ScriptTemplateRenderReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/globalsetting/AccountNameRule.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/globalsetting/AccountNameRule.java index 34064d3c27..88c8810ac6 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/globalsetting/AccountNameRule.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/globalsetting/AccountNameRule.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,7 @@ package com.tencent.bk.job.manage.model.web.request.globalsetting; -import com.tencent.bk.job.manage.common.consts.globalsetting.OSTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.OSTypeEnum; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.AllArgsConstructor; diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/globalsetting/AccountNameRulesReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/globalsetting/AccountNameRulesReq.java index dd6e3fd4d6..3892e87705 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/globalsetting/AccountNameRulesReq.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/globalsetting/AccountNameRulesReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/globalsetting/AddOrUpdateDangerousRuleReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/globalsetting/AddOrUpdateDangerousRuleReq.java index 0664772b96..6375c6dcb9 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/globalsetting/AddOrUpdateDangerousRuleReq.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/globalsetting/AddOrUpdateDangerousRuleReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,8 +27,8 @@ import com.tencent.bk.job.common.validation.CheckEnum; import com.tencent.bk.job.common.validation.Update; import com.tencent.bk.job.common.validation.ValidRegexPattern; -import com.tencent.bk.job.manage.common.consts.rule.HighRiskGrammarActionEnum; -import com.tencent.bk.job.manage.common.consts.rule.HighRiskGrammarRuleStatusEnum; +import com.tencent.bk.job.manage.api.common.constants.rule.HighRiskGrammarActionEnum; +import com.tencent.bk.job.manage.api.common.constants.rule.HighRiskGrammarRuleStatusEnum; import com.tencent.bk.job.manage.validation.provider.DangerousRuleGroupSequenceProvider; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; @@ -48,7 +48,9 @@ @Data @ApiModel("新增、更新高危语句规则") public class AddOrUpdateDangerousRuleReq { - @ApiModelProperty("高危语句规则Id，新增传-1") + /** + * 内部字段 + */ private Long id; @ApiModelProperty("表达式") @NotEmpty(message = "{validation.constraints.InvalidJobHighRiskGrammarRegex_empty.message}") diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/globalsetting/FileUploadSettingReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/globalsetting/FileUploadSettingReq.java index 85252c67c2..7f96ce7ed1 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/globalsetting/FileUploadSettingReq.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/globalsetting/FileUploadSettingReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,8 +25,8 @@ package com.tencent.bk.job.manage.model.web.request.globalsetting; import com.tencent.bk.job.common.validation.CheckEnum; -import com.tencent.bk.job.manage.common.consts.globalsetting.RestrictModeEnum; -import com.tencent.bk.job.manage.common.consts.globalsetting.StorageUnitEnum; +import com.tencent.bk.job.manage.api.common.constants.globalsetting.RestrictModeEnum; +import com.tencent.bk.job.manage.api.common.constants.globalsetting.StorageUnitEnum; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.AllArgsConstructor; @@ -62,7 +62,7 @@ public class FileUploadSettingReq { private Integer restrictMode; @ApiModelProperty("后缀列表") - private List<@Pattern(regexp = "^\\.[A-Za-z0-9_-]{1,24}$", + private List<@Pattern(regexp = "^(\\.[A-Za-z0-9_-]{1,24})+$", message = "{validation.constraints.InvalidUploadFileSuffix.message}") String> suffixList; } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/globalsetting/HistoryExpireReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/globalsetting/HistoryExpireReq.java index 2fb74d89c4..d572e47f59 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/globalsetting/HistoryExpireReq.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/globalsetting/HistoryExpireReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/globalsetting/MoveDangerousRuleReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/globalsetting/MoveDangerousRuleReq.java index 8cfd1892af..e3c643b63e 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/globalsetting/MoveDangerousRuleReq.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/globalsetting/MoveDangerousRuleReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/globalsetting/SetTitleFooterReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/globalsetting/SetTitleFooterReq.java index 5ac8308b36..c42fded699 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/globalsetting/SetTitleFooterReq.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/globalsetting/SetTitleFooterReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -30,11 +30,6 @@ import lombok.Data; import lombok.NoArgsConstructor; -/** - * @Description - * @Date 2020/2/27 - * @Version 1.0 - */ @AllArgsConstructor @NoArgsConstructor @Data diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/ipchooser/AppTopologyTreeNode.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/ipchooser/AppTopologyTreeNode.java deleted file mode 100644 index 6e69adfd9b..0000000000 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/ipchooser/AppTopologyTreeNode.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.model.web.request.ipchooser; - -import io.swagger.annotations.ApiModel; -import io.swagger.annotations.ApiModelProperty; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.util.List; -import java.util.Objects; - -/** - * @Description - * @Date 2020/3/19 - * @Version 1.0 - */ -@NoArgsConstructor -@AllArgsConstructor -@Data -@ApiModel("业务拓扑节点信息") -public class AppTopologyTreeNode { - @ApiModelProperty(value = "节点类型Id", required = true) - private String objectId; - @ApiModelProperty(value = "节点类型名称") - private String objectName; - @ApiModelProperty(value = "节点实例Id", required = true) - private Long instanceId; - @ApiModelProperty(value = "节点实例名称") - private String instanceName; - @ApiModelProperty(value = "子节点列表") - private List childs; - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (!(o instanceof AppTopologyTreeNode)) return false; - AppTopologyTreeNode that = (AppTopologyTreeNode) o; - return Objects.equals(objectId, that.objectId) && - Objects.equals(instanceId, that.instanceId); - } - - @Override - public int hashCode() { - return Objects.hash(objectId, instanceId); - } - - public String getSimpleDesc() { - return "(" + objectId + "," + instanceId + ")"; - } - - @Override - public String toString() { - return "AppTopologyTreeNode{" + - "objectId='" + objectId + '\'' + - ", instanceId=" + instanceId + - '}'; - } -} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/ipchooser/ListHostByBizTopologyNodesReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/ipchooser/ListHostByBizTopologyNodesReq.java deleted file mode 100644 index 474461832e..0000000000 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/ipchooser/ListHostByBizTopologyNodesReq.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.model.web.request.ipchooser; - -import io.swagger.annotations.ApiModel; -import io.swagger.annotations.ApiModelProperty; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -import java.util.List; - -/** - * @Description - * @Date 2020/3/23 - * @Version 1.0 - */ -@NoArgsConstructor -@AllArgsConstructor -@Data -@ApiModel("通过拓扑节点集合获取主机集合") -public class ListHostByBizTopologyNodesReq { - - @ApiModelProperty(value = "业务拓扑节点列表", required = true) - List appTopoNodeList; - - @ApiModelProperty(value = "搜索内容（同时对主机IP/主机名/操作系统/云区域名称进行模糊搜索）", required = false) - String searchContent; - - @ApiModelProperty(value = "筛选条件：agentStatus：0为异常，1为正常，不传则不筛选）", required = false) - Integer agentStatus; - - @ApiModelProperty(value = "数据起始位置", required = false) - Long start; - - @ApiModelProperty(value = "拉取数量", required = false) - Long pageSize; -} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/notify/ChannelTemplatePreviewReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/notify/ChannelTemplatePreviewReq.java index 7340e5b78d..0c287b2986 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/notify/ChannelTemplatePreviewReq.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/notify/ChannelTemplatePreviewReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/notify/ChannelTemplateReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/notify/ChannelTemplateReq.java index 7ec804992c..0bd412f393 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/notify/ChannelTemplateReq.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/notify/ChannelTemplateReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/notify/NotifyBlackUsersReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/notify/NotifyBlackUsersReq.java index ad8a5d1a9b..1494812bbc 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/notify/NotifyBlackUsersReq.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/notify/NotifyBlackUsersReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/notify/NotifyPoliciesCreateUpdateReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/notify/NotifyPoliciesCreateUpdateReq.java index e07318166d..dcf4143b07 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/notify/NotifyPoliciesCreateUpdateReq.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/notify/NotifyPoliciesCreateUpdateReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/notify/ResourceStatusChannel.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/notify/ResourceStatusChannel.java index 4d44d15c2c..f19d5447ea 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/notify/ResourceStatusChannel.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/notify/ResourceStatusChannel.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,14 +24,16 @@ package com.tencent.bk.job.manage.model.web.request.notify; -import com.tencent.bk.job.manage.common.consts.notify.ExecuteStatusEnum; +import com.tencent.bk.job.manage.api.common.constants.notify.ExecuteStatusEnum; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; +import java.util.ArrayList; import java.util.List; +import java.util.Map; @NoArgsConstructor @AllArgsConstructor @@ -44,4 +46,21 @@ public class ResourceStatusChannel { @ApiModelProperty(value = "通知渠道Code列表", required = true) private List channelList; + + public static ResourceStatusChannel fromKV(Integer key, List value) { + ExecuteStatusEnum executeStatus = ExecuteStatusEnum.get(key); + if (executeStatus == null) { + throw new IllegalArgumentException("Parse ResourceStatusChannel with illegal ExecuteStatusEnum[" + + key + "]"); + } + return new ResourceStatusChannel(executeStatus, value); + } + + public static List fromMap(Map> map) { + List statusChannelList = new ArrayList<>(); + for (Map.Entry> entry : map.entrySet()) { + statusChannelList.add(fromKV(entry.getKey(), entry.getValue())); + } + return statusChannelList; + } } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/notify/SetAvailableNotifyChannelReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/notify/SetAvailableNotifyChannelReq.java index 3e6099d017..94f63dbc06 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/notify/SetAvailableNotifyChannelReq.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/notify/SetAvailableNotifyChannelReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/notify/TriggerPolicy.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/notify/TriggerPolicy.java index c2796055ff..4236f3f23a 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/notify/TriggerPolicy.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/notify/TriggerPolicy.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,8 +24,8 @@ package com.tencent.bk.job.manage.model.web.request.notify; -import com.tencent.bk.job.manage.common.consts.notify.ResourceTypeEnum; -import com.tencent.bk.job.manage.common.consts.notify.TriggerTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.notify.ResourceTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.notify.TriggerTypeEnum; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/whiteip/WhiteIPRecordCreateUpdateReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/whiteip/WhiteIPRecordCreateUpdateReq.java index 59fdee1b04..63bcb46f45 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/whiteip/WhiteIPRecordCreateUpdateReq.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/whiteip/WhiteIPRecordCreateUpdateReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,27 +25,35 @@ package com.tencent.bk.job.manage.model.web.request.whiteip; import com.tencent.bk.job.common.model.dto.ResourceScope; +import com.tencent.bk.job.manage.model.web.request.chooser.host.HostIdWithMeta; +import com.tencent.bk.job.manage.model.web.request.whiteip.validation.CheckWhiteIpScope; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; +import java.util.ArrayList; import java.util.List; @Data @ApiModel("IP白名单记录创建请求") +@CheckWhiteIpScope public class WhiteIPRecordCreateUpdateReq { - @ApiModelProperty(value = "ID,更新IP白名单记录的时候需要传入，新建时不需要", required = false) + /** + * 内部字段 + */ + @Deprecated + @ApiModelProperty(value = "白名单记录 ID", hidden = true) private Long id; - @ApiModelProperty(value = "多个资源范围列表", required = true) - private List scopeList; + @ApiModelProperty(value = "是否对所有资源范围生效，默认为false") + private boolean allScope = false; - @ApiModelProperty(value = "云区域ID", required = true) - private Long cloudAreaId; + @ApiModelProperty(value = "多个资源范围列表") + private List scopeList; - @ApiModelProperty(value = "IPs(换行分隔)", required = true) - private String ipStr; + @ApiModelProperty(value = "主机列表", required = true) + private List hostList = new ArrayList<>(); @ApiModelProperty(value = "备注", required = true) private String remark; diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/whiteip/WhiteIPRecordDeleteReq.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/whiteip/WhiteIPRecordDeleteReq.java index 7a49f30ded..8826dc23e1 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/whiteip/WhiteIPRecordDeleteReq.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/whiteip/WhiteIPRecordDeleteReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/whiteip/validation/CheckWhiteIpScope.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/whiteip/validation/CheckWhiteIpScope.java new file mode 100644 index 0000000000..97cacfc9cc --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/whiteip/validation/CheckWhiteIpScope.java @@ -0,0 +1,50 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.web.request.whiteip.validation; + +import javax.validation.Constraint; +import javax.validation.Payload; +import java.lang.annotation.Documented; +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +import static java.lang.annotation.ElementType.ANNOTATION_TYPE; +import static java.lang.annotation.ElementType.FIELD; +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.ElementType.PARAMETER; +import static java.lang.annotation.ElementType.TYPE_USE; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +@Target({FIELD, METHOD, PARAMETER, ANNOTATION_TYPE, TYPE_USE}) +@Retention(RUNTIME) +@Constraint(validatedBy = WhiteIpScopeValidator.class) +@Documented +public @interface CheckWhiteIpScope { + String message() default "{validation.constraints.InvalidWhiteIpScope.message}"; + + Class[] groups() default {}; + + Class[] payload() default {}; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/whiteip/validation/WhiteIpScopeValidator.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/whiteip/validation/WhiteIpScopeValidator.java new file mode 100644 index 0000000000..16b4de1eb3 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/request/whiteip/validation/WhiteIpScopeValidator.java @@ -0,0 +1,41 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.web.request.whiteip.validation; + +import com.tencent.bk.job.manage.model.web.request.whiteip.WhiteIPRecordCreateUpdateReq; +import org.apache.commons.collections.CollectionUtils; + +import javax.validation.ConstraintValidator; +import javax.validation.ConstraintValidatorContext; + +public class WhiteIpScopeValidator implements ConstraintValidator { + + @Override + public boolean isValid(WhiteIPRecordCreateUpdateReq whiteIPRecordCreateUpdateReq, + ConstraintValidatorContext constraintValidatorContext) { + return whiteIPRecordCreateUpdateReq.isAllScope() + || CollectionUtils.isNotEmpty(whiteIPRecordCreateUpdateReq.getScopeList()); + } +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/AccountVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/AccountVO.java index 4808e7f427..299a314235 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/AccountVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/AccountVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/AppVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/AppVO.java index 466835c0ab..bf5399dae3 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/AppVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/AppVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/BasicScriptVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/BasicScriptVO.java index 9943060e00..86f660c97a 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/BasicScriptVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/BasicScriptVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,12 +24,15 @@ package com.tencent.bk.job.manage.model.web.vo; -import com.tencent.bk.job.manage.common.consts.JobResourceStatusEnum; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.tencent.bk.job.common.util.json.LongTimestampSerializer; +import com.tencent.bk.job.manage.api.common.constants.JobResourceStatusEnum; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Getter; import lombok.Setter; +import java.util.List; import java.util.StringJoiner; /** @@ -45,49 +48,55 @@ public class BasicScriptVO { * 脚本版本ID */ @ApiModelProperty(value = "脚本版本ID") - private Long scriptVersionId; + protected Long scriptVersionId; /** * 脚本ID，一个脚本下面包含多个版本的脚本 */ @ApiModelProperty(value = "脚本ID") - private String id; + protected String id; /** * 脚本名称 */ @ApiModelProperty(value = "脚本名称") - private String name; + protected String name; /** * 脚本类型 */ @ApiModelProperty(value = "脚本类型") - private Integer type; + protected Integer type; + + /** + * 脚本类型名称 + */ + protected String typeName; + /** * 是否公共脚本 */ @ApiModelProperty(value = "是否公共脚本") - private Boolean publicScript; + protected Boolean publicScript; /** * 资源范围类型 */ @ApiModelProperty(value = "资源范围类型", allowableValues = "biz-业务,biz_set-业务集") - private String scopeType; + protected String scopeType; /** * 资源范围ID */ @ApiModelProperty("资源范围ID") - private String scopeId; + protected String scopeId; /** * 脚本大种类: 0 系统的执行脚本(如shell,bat,python等), 1 SQL执行脚本 */ @ApiModelProperty(value = "脚本分类: 0 系统的执行脚本(如shell,bat,python等), 1 SQL执行脚本") - private Integer category; + protected Integer category; /** * 脚本的版本号 */ @ApiModelProperty(value = "版本号") - private String version; + protected String version; /** * 脚本状态 @@ -95,14 +104,61 @@ public class BasicScriptVO { * @see JobResourceStatusEnum */ @ApiModelProperty(value = "脚本状态,0:未发布，1:已上线") - private Integer status; + protected Integer status; + @ApiModelProperty(value = "脚本状态描述") + protected String statusDesc; @ApiModelProperty("是否可以查看") private Boolean canView; - @ApiModelProperty("是否可以管理") - private Boolean canManage; + protected Boolean canManage; + @ApiModelProperty("是否可以克隆") + protected Boolean canClone; + + /** + * 创建者 + */ + @ApiModelProperty(value = "创建者") + protected String creator; + + /** + * 创建时间 + */ + @ApiModelProperty(value = "创建时间") + @JsonSerialize(using = LongTimestampSerializer.class) + protected Long createTime; + + /** + * 最后修改人 + */ + @ApiModelProperty(value = "最后更新者") + protected String lastModifyUser; + + /** + * 最后修改时间 + */ + @ApiModelProperty(value = "最后更新时间") + @JsonSerialize(using = LongTimestampSerializer.class) + protected Long lastModifyTime; + + /** + * 脚本标签 + */ + @ApiModelProperty(value = "脚本标签") + protected List tags; + + /** + * 脚本版本描述 + */ + @ApiModelProperty(value = "脚本版本描述") + protected String versionDesc; + + /** + * 脚本描述 + */ + @ApiModelProperty(value = "脚本描述") + protected String description; @Override public String toString() { @@ -111,11 +167,24 @@ public String toString() { .add("id='" + id + "'") .add("name='" + name + "'") .add("type=" + type) + .add("typeName='" + typeName + "'") .add("publicScript=" + publicScript) + .add("scopeType='" + scopeType + "'") + .add("scopeId='" + scopeId + "'") .add("category=" + category) .add("version='" + version + "'") .add("status=" + status) - .add("content='*******'") + .add("statusDesc='" + statusDesc + "'") + .add("canView=" + canView) + .add("canManage=" + canManage) + .add("canClone=" + canClone) + .add("creator='" + creator + "'") + .add("createTime=" + createTime) + .add("lastModifyUser='" + lastModifyUser + "'") + .add("lastModifyTime=" + lastModifyTime) + .add("tags=" + tags) + .add("versionDesc='" + versionDesc + "'") + .add("description='" + description + "'") .toString(); } } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/CcTopologyNodeVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/CcTopologyNodeVO.java index c9ec2f89f2..800633cdac 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/CcTopologyNodeVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/CcTopologyNodeVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,6 +25,7 @@ package com.tencent.bk.job.manage.model.web.vo; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; import com.tencent.bk.job.common.model.vo.HostInfoVO; import io.swagger.annotations.ApiModelProperty; import lombok.Getter; @@ -34,18 +35,17 @@ import java.util.List; import java.util.Set; -/** - * @date 2019/1/31 - */ @Getter @Setter @ToString +@JsonInclude(JsonInclude.Include.NON_NULL) public class CcTopologyNodeVO { private Long instanceId; private String instanceName; private String objectId; private String objectName; - private Boolean expanded = true; + // 当前层级节点数据是否使用懒加载，作业平台目前不涉及，取值一直为false即可 + private Boolean lazy = false; private List child; @ApiModelProperty("节点包含的 IP 列表") diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/CredentialBasicVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/CredentialBasicVO.java new file mode 100644 index 0000000000..17ef3b91fd --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/CredentialBasicVO.java @@ -0,0 +1,47 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.web.vo; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +@ApiModel("凭证") +@Data +public class CredentialBasicVO { + /** + * 主键Id + */ + @ApiModelProperty("主键Id") + private String id; + /** + * 名称 + */ + @ApiModelProperty("名称") + private String name; + + @ApiModelProperty("是否可以使用") + private Boolean canUse; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/CredentialVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/CredentialVO.java index de79504482..845a859cf5 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/CredentialVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/CredentialVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -30,7 +30,7 @@ import io.swagger.annotations.ApiModelProperty; import lombok.Data; -@ApiModel("凭据") +@ApiModel("凭证") @Data public class CredentialVO { /** diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/DynamicGroupBasicVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/DynamicGroupBasicVO.java new file mode 100644 index 0000000000..73a87a5632 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/DynamicGroupBasicVO.java @@ -0,0 +1,51 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.web.vo; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Data; + +import java.util.Map; + +@ApiModel("动态分组信息") +@Data +@AllArgsConstructor +public class DynamicGroupBasicVO { + + @ApiModelProperty("动态分组 ID") + private String id; + + @ApiModelProperty("动态分组名称") + private String name; + + @ApiModelProperty("最后修改时间") + private String lastTime; + + @ApiModelProperty(value = "动态分组元数据") + Map meta; + +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/DynamicGroupInfoVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/DynamicGroupInfoVO.java index f9c3036ea1..af84d51edd 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/DynamicGroupInfoVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/DynamicGroupInfoVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/FeatureVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/FeatureVO.java new file mode 100644 index 0000000000..d015153128 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/FeatureVO.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.web.vo; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; + +@ApiModel("功能特性开关") +@Data +public class FeatureVO { + /** + * 特性ID + */ + @ApiModelProperty("特性ID") + private String id; + /** + * 是否启用特性 + */ + @ApiModelProperty("是否启用特性") + private boolean enabled; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/HostInfoWithModulesVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/HostInfoWithModulesVO.java index c7294bbcd9..a80dd1019c 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/HostInfoWithModulesVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/HostInfoWithModulesVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,7 +26,7 @@ import com.tencent.bk.job.common.model.vo.CloudAreaInfoVO; import com.tencent.bk.job.common.util.JobContextUtil; -import com.tencent.bk.job.manage.model.web.request.ipchooser.AppTopologyTreeNode; +import com.tencent.bk.job.manage.model.web.request.chooser.host.BizTopoNode; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.AllArgsConstructor; @@ -70,7 +70,7 @@ public class HostInfoWithModulesVO { private String os; @ApiModelProperty("所有模块节点信息") - private List moduleNodes; + private List moduleNodes; public boolean validate(boolean isCreate) { if (cloudAreaInfo == null) { diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/NodeInfoVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/NodeInfoVO.java index d5b269199f..3c5f1325d5 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/NodeInfoVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/NodeInfoVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/PageDataWithAvailableIdList.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/PageDataWithAvailableIdList.java index ba36349a69..eefbda54fb 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/PageDataWithAvailableIdList.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/PageDataWithAvailableIdList.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/ScriptCheckResultItemVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/ScriptCheckResultItemVO.java index 42962f3e51..d6eb9381d5 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/ScriptCheckResultItemVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/ScriptCheckResultItemVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/ScriptCitedTaskPlanVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/ScriptCitedTaskPlanVO.java index 72ab211756..991802c1f1 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/ScriptCitedTaskPlanVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/ScriptCitedTaskPlanVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/ScriptCitedTemplateVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/ScriptCitedTemplateVO.java index beb57b3331..ed587b809d 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/ScriptCitedTemplateVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/ScriptCitedTemplateVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/ScriptVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/ScriptVO.java index 64d3bd0fac..a05527a0a7 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/ScriptVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/ScriptVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,9 +25,6 @@ package com.tencent.bk.job.manage.model.web.vo; import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.databind.annotation.JsonSerialize; -import com.tencent.bk.job.common.util.json.LongTimestampSerializer; -import com.tencent.bk.job.manage.common.consts.JobResourceStatusEnum; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Getter; @@ -43,108 +40,14 @@ @Setter @ApiModel("脚本信息") @JsonInclude(JsonInclude.Include.NON_EMPTY) -public class ScriptVO { - /** - * 脚本版本ID - */ - @ApiModelProperty(value = "脚本版本ID") - private Long scriptVersionId; - /** - * 脚本ID，一个脚本下面包含多个版本的脚本 - */ - @ApiModelProperty(value = "脚本ID") - private String id; - /** - * 脚本名称 - */ - @ApiModelProperty(value = "脚本名称") - private String name; - /** - * 脚本类型 - */ - @ApiModelProperty(value = "脚本类型") - private Integer type; - /** - * 脚本类型名称 - */ - private String typeName; - /** - * 是否公共脚本 - */ - @ApiModelProperty(value = "是否公共脚本") - private Boolean publicScript; +public class ScriptVO extends BasicScriptVO { - /** - * 资源范围类型 - */ - @ApiModelProperty(value = "资源范围类型", allowableValues = "biz-业务,biz_set-业务集") - private String scopeType; - - /** - * 资源范围ID - */ - @ApiModelProperty("资源范围ID") - private String scopeId; /** * 脚本内容 */ @ApiModelProperty(value = "脚本内容") private String content; - /** - * 创建者 - */ - @ApiModelProperty(value = "创建者") - private String creator; - - /** - * 创建时间 - */ - @ApiModelProperty(value = "创建时间") - @JsonSerialize(using = LongTimestampSerializer.class) - private Long createTime; - - /** - * 最后修改人 - */ - @ApiModelProperty(value = "最后更新者") - private String lastModifyUser; - - /** - * 最后修改时间 - */ - @ApiModelProperty(value = "最后更新时间") - @JsonSerialize(using = LongTimestampSerializer.class) - private Long lastModifyTime; - - /** - * 脚本大种类: 0 系统的执行脚本(如shell,bat,python等), 1 SQL执行脚本 - */ - @ApiModelProperty(value = "脚本分类: 0 系统的执行脚本(如shell,bat,python等), 1 SQL执行脚本") - private Integer category; - /** - * 脚本的版本号 - */ - @ApiModelProperty(value = "版本号") - private String version; - - /** - * 脚本标签 - */ - @ApiModelProperty(value = "脚本标签") - private List tags; - - /** - * 脚本状态 - * - * @see JobResourceStatusEnum - */ - @ApiModelProperty(value = "脚本状态,0:未发布，1:已上线") - private Integer status; - - @ApiModelProperty(value = "脚本状态描述") - private String statusDesc; - @ApiModelProperty(value = "脚本版本列表") private List scriptVersions; @@ -154,47 +57,41 @@ public class ScriptVO { @ApiModelProperty(value = "关联的执行方案数量") private Integer relatedTaskPlanNum; - /** - * 脚本版本描述 - */ - @ApiModelProperty(value = "脚本版本描述") - private String versionDesc; - - /** - * 脚本描述 - */ - @ApiModelProperty(value = "脚本描述") - private String description; - - @ApiModelProperty("是否可以查看") - private Boolean canView; - @ApiModelProperty("是否可以管理") - private Boolean canManage; - @ApiModelProperty("是否可以克隆") - private Boolean canClone; /** * 是否支持同步 */ @ApiModelProperty(value = "是否支持同步") private Boolean syncEnabled = false; - @Override public String toString() { return new StringJoiner(", ", ScriptVO.class.getSimpleName() + "[", "]") + .add("content=******") + .add("scriptVersions=" + scriptVersions) + .add("relatedTaskTemplateNum=" + relatedTaskTemplateNum) + .add("relatedTaskPlanNum=" + relatedTaskPlanNum) + .add("syncEnabled=" + syncEnabled) .add("scriptVersionId=" + scriptVersionId) .add("id='" + id + "'") .add("name='" + name + "'") .add("type=" + type) + .add("typeName='" + typeName + "'") .add("publicScript=" + publicScript) - .add("scopeType=" + scopeType) - .add("scopeId=" + scopeId) + .add("scopeType='" + scopeType + "'") + .add("scopeId='" + scopeId + "'") .add("category=" + category) .add("version='" + version + "'") - .add("tags='" + tags + "'") .add("status=" + status) - .add("desc='" + versionDesc + "'") - .add("content='*******'") + .add("statusDesc='" + statusDesc + "'") + .add("canManage=" + canManage) + .add("canClone=" + canClone) + .add("creator='" + creator + "'") + .add("createTime=" + createTime) + .add("lastModifyUser='" + lastModifyUser + "'") + .add("lastModifyTime=" + lastModifyTime) + .add("tags=" + tags) + .add("versionDesc='" + versionDesc + "'") + .add("description='" + description + "'") .toString(); } } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/TagCountVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/TagCountVO.java index 3ccfb26529..f4854efcf8 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/TagCountVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/TagCountVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/TagVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/TagVO.java index 68a5347f15..9aae1dc784 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/TagVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/TagVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/TemplateStepIDVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/TemplateStepIDVO.java index 43932ae760..d32058b6dd 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/TemplateStepIDVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/TemplateStepIDVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/UploadLocalFileResultVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/UploadLocalFileResultVO.java index 4e6334cfae..1bfb9a0090 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/UploadLocalFileResultVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/UploadLocalFileResultVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/chooser/container/ContainerTopologyNodeVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/chooser/container/ContainerTopologyNodeVO.java new file mode 100644 index 0000000000..617a74edd4 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/chooser/container/ContainerTopologyNodeVO.java @@ -0,0 +1,66 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.web.vo.chooser.container; + +import com.fasterxml.jackson.annotation.JsonInclude; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +import java.util.List; + +/** + * 容器拓扑节点 + */ +@Getter +@Setter +@ToString +@JsonInclude(JsonInclude.Include.NON_NULL) +@ApiModel("容器拓扑节点") +public class ContainerTopologyNodeVO { + @ApiModelProperty("节点实例 ID") + private Long instanceId; + + @ApiModelProperty("节点实例名称") + private String instanceName; + + @ApiModelProperty("节点类型 ID") + private String objectId; + + @ApiModelProperty("节点类型名称") + private String objectName; + + // 当前层级节点数据是否使用懒加载，作业平台目前不涉及，取值一直为false即可 + @ApiModelProperty("当前层级节点数据是否使用懒加载") + private Boolean lazy = false; + + @ApiModelProperty("子节点") + private List child; + + @ApiModelProperty("节点包含的容器数量") + private int count; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/index/AgentStatistics.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/common/AgentStatistics.java similarity index 80% rename from src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/index/AgentStatistics.java rename to src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/common/AgentStatistics.java index d2c5fe3e63..55ff88ec8c 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/index/AgentStatistics.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/common/AgentStatistics.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,7 +22,7 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.manage.model.web.vo.index; +package com.tencent.bk.job.manage.model.web.vo.common; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; @@ -38,8 +38,19 @@ @ApiModel("Agent统计数据") @Data public class AgentStatistics { + @ApiModelProperty("正常数") - private int normalNum; + private int aliveCount; + @ApiModelProperty("异常数") - private int abnormalNum; + private int notAliveCount; + + @ApiModelProperty("总数") + private int totalCount; + + public AgentStatistics(int aliveCount, int notAliveCount) { + this.aliveCount = aliveCount; + this.notAliveCount = notAliveCount; + this.totalCount = aliveCount + notAliveCount; + } } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/customsetting/ScriptTemplateVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/customsetting/ScriptTemplateVO.java index ef5e8e1f6e..b49ae1e3c2 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/customsetting/ScriptTemplateVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/customsetting/ScriptTemplateVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/customsetting/ScriptTemplateVariableVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/customsetting/ScriptTemplateVariableVO.java index 8462d81e17..593c99ca98 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/customsetting/ScriptTemplateVariableVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/customsetting/ScriptTemplateVariableVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/globalsetting/AccountNameRuleVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/globalsetting/AccountNameRuleVO.java index 0fc2b26a95..263ad5cdb9 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/globalsetting/AccountNameRuleVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/globalsetting/AccountNameRuleVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,7 @@ package com.tencent.bk.job.manage.model.web.vo.globalsetting; -import com.tencent.bk.job.manage.common.consts.globalsetting.OSTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.OSTypeEnum; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.AllArgsConstructor; diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/globalsetting/AccountNameRulesWithDefaultVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/globalsetting/AccountNameRulesWithDefaultVO.java index f6f444a165..459969e430 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/globalsetting/AccountNameRulesWithDefaultVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/globalsetting/AccountNameRulesWithDefaultVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/globalsetting/DangerousRuleVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/globalsetting/DangerousRuleVO.java index 50af25bf0d..6c9134ef49 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/globalsetting/DangerousRuleVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/globalsetting/DangerousRuleVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,11 @@ package com.tencent.bk.job.manage.model.web.vo.globalsetting; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.tencent.bk.job.common.util.json.LongTimestampDeserializer; +import com.tencent.bk.job.common.util.json.LongTimestampSerializer; +import com.tencent.bk.job.manage.model.esb.v3.response.EsbDangerousRuleV3DTO; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.AllArgsConstructor; @@ -51,4 +56,31 @@ public class DangerousRuleVO { private Integer action; @ApiModelProperty("启用状态,1:启用,2:停止") private Integer status; + @ApiModelProperty("创建人") + private String creator; + @JsonSerialize(using = LongTimestampSerializer.class) + @JsonDeserialize(using = LongTimestampDeserializer.class) + @ApiModelProperty("创建时间") + private Long createTime; + @ApiModelProperty("最近更新人") + private String lastModifier; + @JsonSerialize(using = LongTimestampSerializer.class) + @JsonDeserialize(using = LongTimestampDeserializer.class) + @ApiModelProperty("最近更新时间") + private Long lastModifyTime; + + public EsbDangerousRuleV3DTO toEsbDangerousRuleV3DTO() { + EsbDangerousRuleV3DTO esbDangerousRuleV3DTO = new EsbDangerousRuleV3DTO(); + esbDangerousRuleV3DTO.setId(id); + esbDangerousRuleV3DTO.setExpression(expression); + esbDangerousRuleV3DTO.setScriptTypeList(scriptTypeList); + esbDangerousRuleV3DTO.setDescription(description); + esbDangerousRuleV3DTO.setAction(action); + esbDangerousRuleV3DTO.setStatus(status); + esbDangerousRuleV3DTO.setCreator(creator); + esbDangerousRuleV3DTO.setCreateTime(createTime); + esbDangerousRuleV3DTO.setLastModifyUser(lastModifier); + esbDangerousRuleV3DTO.setLastModifyTime(lastModifyTime); + return esbDangerousRuleV3DTO; + } } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/globalsetting/FileUploadSettingVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/globalsetting/FileUploadSettingVO.java index 040ed58e4d..b336f7b9e5 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/globalsetting/FileUploadSettingVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/globalsetting/FileUploadSettingVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/globalsetting/NotifyChannelWithIconVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/globalsetting/NotifyChannelWithIconVO.java index d892a1cb68..ef8842ef8d 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/globalsetting/NotifyChannelWithIconVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/globalsetting/NotifyChannelWithIconVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/globalsetting/PlatformInfoVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/globalsetting/PlatformInfoVO.java new file mode 100644 index 0000000000..e7d37a82a2 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/globalsetting/PlatformInfoVO.java @@ -0,0 +1,51 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.web.vo.globalsetting; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; + +@EqualsAndHashCode(callSuper = true) +@NoArgsConstructor +@Data +@ApiModel("全局设置-平台设置") +public class PlatformInfoVO extends TitleFooterVO { + @ApiModelProperty("助手链接") + private String helperContactLink; + + public PlatformInfoVO(TitleFooterVO titleFooterVO) { + this(titleFooterVO, null); + } + + public PlatformInfoVO(TitleFooterVO titleFooterVO, + String helperContactLink) { + super(titleFooterVO.getTitleHead(), titleFooterVO.getTitleSeparator(), + titleFooterVO.getFooterLink(), titleFooterVO.getFooterCopyRight()); + this.helperContactLink = helperContactLink; + } +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/globalsetting/PlatformInfoWithDefaultVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/globalsetting/PlatformInfoWithDefaultVO.java new file mode 100644 index 0000000000..650ccdffcc --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/globalsetting/PlatformInfoWithDefaultVO.java @@ -0,0 +1,45 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.web.vo.globalsetting; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * 平台信息VO(含默认配置) + */ +@NoArgsConstructor +@AllArgsConstructor +@ApiModel("平台信息VO(含默认配置)") +@Data +public class PlatformInfoWithDefaultVO { + @ApiModelProperty("当前已设置的平台信息") + private PlatformInfoVO currentPlatformInfo; + @ApiModelProperty("默认的平台信息") + private PlatformInfoVO defaultPlatformInfo; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/globalsetting/TitleFooterVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/globalsetting/TitleFooterVO.java index 3a2a570edf..5f411589e5 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/globalsetting/TitleFooterVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/globalsetting/TitleFooterVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,19 +26,12 @@ import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; -import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; -/** - * @Description - * @Date 2020/2/27 - * @Version 1.0 - */ -@AllArgsConstructor @NoArgsConstructor @Data -@ApiModel("全站Title与Footer") +@ApiModel("全局设置-平台设置-title/footer") public class TitleFooterVO { @ApiModelProperty("Title固定头文字") private String titleHead; @@ -48,4 +41,11 @@ public class TitleFooterVO { private String footerLink; @ApiModelProperty("Footer第二行copyright内容，Markdown格式") private String footerCopyRight; + + public TitleFooterVO(String titleHead, String titleSeparator, String footerLink, String footerCopyRight) { + this.titleHead = titleHead; + this.titleSeparator = titleSeparator; + this.footerLink = footerLink; + this.footerCopyRight = footerCopyRight; + } } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/globalsetting/TitleFooterWithDefaultVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/globalsetting/TitleFooterWithDefaultVO.java deleted file mode 100644 index ca0c09506a..0000000000 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/globalsetting/TitleFooterWithDefaultVO.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.model.web.vo.globalsetting; - -import io.swagger.annotations.ApiModel; -import io.swagger.annotations.ApiModelProperty; -import lombok.AllArgsConstructor; -import lombok.Data; -import lombok.NoArgsConstructor; - -/** - * 平台TitleFooter配置VO - */ -@NoArgsConstructor -@AllArgsConstructor -@ApiModel("平台TitleFooter配置（含默认配置）") -@Data -public class TitleFooterWithDefaultVO { - @ApiModelProperty("当前已设置的TitleFooter信息") - private TitleFooterVO currentTitleFooter; - @ApiModelProperty("默认TitleFooter信息") - private TitleFooterVO defaultTitleFooter; -} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/index/ExecuteRecord.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/index/ExecuteRecord.java index bdd55126ed..1eecaac68b 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/index/ExecuteRecord.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/index/ExecuteRecord.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/index/GreetingVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/index/GreetingVO.java index d986793ac0..e104b909a4 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/index/GreetingVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/index/GreetingVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/index/JobAndScriptStatistics.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/index/JobAndScriptStatistics.java index b5b277603a..e551dbfc69 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/index/JobAndScriptStatistics.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/index/JobAndScriptStatistics.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/index/MyFavorJobVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/index/MyFavorJobVO.java index cdf52c78ba..826f80f1b5 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/index/MyFavorJobVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/index/MyFavorJobVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/ipchooser/DynamicGroupHostStatisticsVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/ipchooser/DynamicGroupHostStatisticsVO.java new file mode 100644 index 0000000000..78d24e4688 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/ipchooser/DynamicGroupHostStatisticsVO.java @@ -0,0 +1,54 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.web.vo.ipchooser; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.tencent.bk.job.manage.model.web.vo.DynamicGroupBasicVO; +import com.tencent.bk.job.manage.model.web.vo.common.AgentStatistics; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * @since 13/12/2019 17:31 + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +@ApiModel("动态分组主机Agent状态统计信息") +@JsonInclude(JsonInclude.Include.NON_NULL) +public class DynamicGroupHostStatisticsVO { + /** + * 云区域 ID + */ + @ApiModelProperty(value = "动态分组信息", required = true) + private DynamicGroupBasicVO dynamicGroup; + + @ApiModelProperty(value = "主机Agent状态统计信息", required = true) + private AgentStatistics agentStatistics; + +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/ipchooser/NodeHostStatisticsVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/ipchooser/NodeHostStatisticsVO.java new file mode 100644 index 0000000000..c876be1801 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/ipchooser/NodeHostStatisticsVO.java @@ -0,0 +1,54 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.web.vo.ipchooser; + +import com.fasterxml.jackson.annotation.JsonInclude; +import com.tencent.bk.job.common.model.vo.TargetNodeVO; +import com.tencent.bk.job.manage.model.web.vo.common.AgentStatistics; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * @since 13/12/2019 17:31 + */ +@Data +@AllArgsConstructor +@NoArgsConstructor +@ApiModel("节点主机Agent状态统计信息") +@JsonInclude(JsonInclude.Include.NON_NULL) +public class NodeHostStatisticsVO { + /** + * 云区域 ID + */ + @ApiModelProperty(value = "节点信息", required = true) + private TargetNodeVO node; + + @ApiModelProperty(value = "主机Agent状态统计信息", required = true) + private AgentStatistics agentStatistics; + +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notice/AnnouncementVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notice/AnnouncementVO.java new file mode 100644 index 0000000000..33ad26fe07 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notice/AnnouncementVO.java @@ -0,0 +1,74 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.web.vo.notice; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.notice.model.AnnouncementDTO; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@NoArgsConstructor +@AllArgsConstructor +@ApiModel("公告内容") +@Data +public class AnnouncementVO { + + @ApiModelProperty("ID") + private Long id; + + @ApiModelProperty("标题") + private String title; + + @ApiModelProperty("内容") + private String content; + + @ApiModelProperty("公告类型: event(活动通知)/announce(平台公告)") + @JsonProperty("announce_type") + private String announceType; + + @ApiModelProperty("开始时间") + @JsonProperty("start_time") + private String startTime; + + @ApiModelProperty("结束时间") + @JsonProperty("end_time") + private String endTime; + + public static AnnouncementVO fromDTO(AnnouncementDTO announcementDTO) { + AnnouncementVO announcementVO = new AnnouncementVO(); + announcementVO.setId(announcementDTO.getId()); + announcementVO.setTitle(announcementDTO.getTitle()); + announcementVO.setContent(announcementDTO.getContent()); + announcementVO.setAnnounceType(announcementDTO.getAnnounceType()); + announcementVO.setStartTime(announcementDTO.getStartTime()); + announcementVO.setEndTime(announcementDTO.getEndTime()); + return announcementVO; + } +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/ChannelTemplateDetailVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/ChannelTemplateDetailVO.java index fb61c3dbfc..eae6661ee7 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/ChannelTemplateDetailVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/ChannelTemplateDetailVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/ChannelTemplateDetailWithDefaultVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/ChannelTemplateDetailWithDefaultVO.java index 08520f42cb..ebaff1b74d 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/ChannelTemplateDetailWithDefaultVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/ChannelTemplateDetailWithDefaultVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/ChannelTemplateStatusVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/ChannelTemplateStatusVO.java index 9f3f4f27d7..a1dd21acf5 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/ChannelTemplateStatusVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/ChannelTemplateStatusVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/ExecuteStatusVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/ExecuteStatusVO.java index 33ef771a63..0454ebbb94 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/ExecuteStatusVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/ExecuteStatusVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/NotifyBlackUserInfoVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/NotifyBlackUserInfoVO.java index d54eb77229..45f448aaae 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/NotifyBlackUserInfoVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/NotifyBlackUserInfoVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/PageTemplateVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/PageTemplateVO.java index 69acf046ec..5c9047ae0f 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/PageTemplateVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/PageTemplateVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/ResourceStatusChannelVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/ResourceStatusChannelVO.java index 1aad8cc822..283bc76025 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/ResourceStatusChannelVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/ResourceStatusChannelVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/ResourceTypeVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/ResourceTypeVO.java index 49cf2f8856..3dbb715e9e 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/ResourceTypeVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/ResourceTypeVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/RoleVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/RoleVO.java index d5d810064b..bcec1a4511 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/RoleVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/RoleVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/TemplateBasicInfo.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/TemplateBasicInfo.java index 73319cee1d..43491e22c6 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/TemplateBasicInfo.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/TemplateBasicInfo.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/TriggerPolicyVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/TriggerPolicyVO.java index 9fe742df39..45f53d379b 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/TriggerPolicyVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/TriggerPolicyVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/TriggerTypeVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/TriggerTypeVO.java index 0a6ccd8bfe..6665bbcb68 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/TriggerTypeVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/TriggerTypeVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/UserVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/UserVO.java index 7f70bb6d5d..90fa3d36ba 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/UserVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/notify/UserVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/script/ScriptCiteCountVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/script/ScriptCiteCountVO.java index f33ab9b612..809cea5493 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/script/ScriptCiteCountVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/script/ScriptCiteCountVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/script/ScriptCiteInfoVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/script/ScriptCiteInfoVO.java index f2f235fd76..8524f5546f 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/script/ScriptCiteInfoVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/script/ScriptCiteInfoVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/script/ScriptRelatedTemplateStepVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/script/ScriptRelatedTemplateStepVO.java index 1d5ceee02e..b82fae3c07 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/script/ScriptRelatedTemplateStepVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/script/ScriptRelatedTemplateStepVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/script/ScriptSyncResultVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/script/ScriptSyncResultVO.java index 05427e04fa..55608c8470 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/script/ScriptSyncResultVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/script/ScriptSyncResultVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/serviceinfo/ServiceInfoVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/serviceinfo/ServiceInfoVO.java index 2862f88536..0ac5eb17e9 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/serviceinfo/ServiceInfoVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/serviceinfo/ServiceInfoVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/serviceinfo/ServiceInstanceInfoVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/serviceinfo/ServiceInstanceInfoVO.java index 5590d78260..b62011f01a 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/serviceinfo/ServiceInstanceInfoVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/serviceinfo/ServiceInstanceInfoVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/CloudAreaInfoVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/CloudAreaInfoVO.java index 941079c630..15122bd07b 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/CloudAreaInfoVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/CloudAreaInfoVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,12 +25,14 @@ package com.tencent.bk.job.manage.model.web.vo.task; import com.fasterxml.jackson.annotation.JsonInclude; -import com.tencent.bk.job.common.util.JobContextUtil; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.InvalidParamException; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.AllArgsConstructor; import lombok.Data; import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; /** * @since 13/12/2019 17:31 @@ -40,9 +42,10 @@ @NoArgsConstructor @ApiModel("云区域信息") @JsonInclude(JsonInclude.Include.NON_NULL) +@Slf4j public class CloudAreaInfoVO { /** - * 云区域 ID：1是腾讯云 2是投后公司 3是私有云 + * 云区域 ID */ @ApiModelProperty(value = "云区域 ID", required = true) private Long id; @@ -50,11 +53,10 @@ public class CloudAreaInfoVO { @ApiModelProperty("云区域名称") private String name; - public boolean validate(boolean isCreate) { - if (id != null && id >= 0) { - return true; + public void validate(boolean isCreate) throws InvalidParamException { + if (id == null || id < 0) { + log.warn("Invalid cloud area info! id: {}", id); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } - JobContextUtil.addDebugMessage("Invalid cloud area info!"); - return false; } } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TargetNodeVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TargetNodeVO.java index 8f696e02af..ba45360027 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TargetNodeVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TargetNodeVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,10 +25,12 @@ package com.tencent.bk.job.manage.model.web.vo.task; import com.fasterxml.jackson.annotation.JsonInclude; -import com.tencent.bk.job.common.util.JobContextUtil; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.InvalidParamException; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; /** @@ -37,6 +39,7 @@ @Data @ApiModel("目标节点信息") @JsonInclude(JsonInclude.Include.NON_NULL) +@Slf4j public class TargetNodeVO { @ApiModelProperty(value = "节点 ID，对应拓扑树节点中的instanceId", required = true) private Long id; @@ -44,11 +47,14 @@ public class TargetNodeVO { @ApiModelProperty(value = "节点类型 biz-业务 set-集群 module-模块 xxx-用户自定义节点类型，对应拓扑树节点中的objectId", required = true) private String type; - public boolean validate(boolean isCreate) { - if (id != null && id > 0 && StringUtils.isNotBlank(type)) { - return true; + public void validate() throws InvalidParamException { + if (id == null || id <= 0) { + log.warn("Invalid target node id, id: {}", id); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); + } + if (StringUtils.isBlank(type)) { + log.warn("Blank target node type"); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } - JobContextUtil.addDebugMessage("Target node info does not have id or type"); - return false; } } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskApprovalStepVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskApprovalStepVO.java index 77bfbb61b0..8df38216b4 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskApprovalStepVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskApprovalStepVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,11 +24,13 @@ package com.tencent.bk.job.manage.model.web.vo.task; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.model.vo.UserRoleInfoVO; -import com.tencent.bk.job.common.util.JobContextUtil; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; +import lombok.extern.slf4j.Slf4j; import java.util.Collections; import java.util.List; @@ -38,6 +40,7 @@ */ @Data @ApiModel("任务审批步骤信息") +@Slf4j public class TaskApprovalStepVO { @ApiModelProperty(value = "审批类型 暂未启用 1-任意人审批 2-所有人审批") @@ -52,7 +55,7 @@ public class TaskApprovalStepVO { @ApiModelProperty("通知渠道") private List notifyChannel; - public boolean validate(boolean isCreate) { + public void validate(boolean isCreate) throws InvalidParamException { if (notifyChannel == null) { notifyChannel = Collections.emptyList(); } @@ -60,9 +63,9 @@ public boolean validate(boolean isCreate) { approvalMessage = ""; } if (approvalUser == null) { - JobContextUtil.addDebugMessage("Approval step must have user!"); - return false; + log.warn("Approval step must have user!"); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } - return approvalUser.validate(); + approvalUser.validate(); } } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskFileDestinationInfoVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskFileDestinationInfoVO.java index 4d4663a30f..48b1043a5f 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskFileDestinationInfoVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskFileDestinationInfoVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskFileSourceInfoVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskFileSourceInfoVO.java index 8e3fbf0e45..1b31fb9374 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskFileSourceInfoVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskFileSourceInfoVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,12 +24,14 @@ package com.tencent.bk.job.manage.model.web.vo.task; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.model.vo.TaskTargetVO; -import com.tencent.bk.job.common.util.JobContextUtil; +import com.tencent.bk.job.common.util.FilePathValidateUtil; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; -import org.apache.commons.lang3.StringUtils; +import lombok.extern.slf4j.Slf4j; import org.springframework.util.CollectionUtils; import java.util.List; @@ -39,6 +41,7 @@ */ @Data @ApiModel("步骤源文件信息") +@Slf4j public class TaskFileSourceInfoVO { @ApiModelProperty(value = "文件 ID 新建填 0") @@ -68,50 +71,46 @@ public class TaskFileSourceInfoVO { @ApiModelProperty(value = "文件源ID，来自文件源的文件需要传入") private Integer fileSourceId; - public boolean validate(boolean isCreate) { + public void validate(boolean isCreate) throws InvalidParamException { if (fileType == null || fileType <= 0 || fileType > 3) { - JobContextUtil.addDebugMessage("Invalid file type!"); - return false; + log.warn("Invalid file type!"); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } switch (fileType) { case 1: if (CollectionUtils.isEmpty(fileLocation)) { - JobContextUtil.addDebugMessage("Empty file location!"); - return false; + log.warn("Empty file location!"); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } for (String file : fileLocation) { - if (StringUtils.isBlank(file)) { - JobContextUtil.addDebugMessage("Invalid file location!"); - return false; + if (!FilePathValidateUtil.validateFileSystemAbsolutePath(file)) { + log.warn("fileLocation is illegal!"); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } } - if (!host.validate(isCreate)) { - JobContextUtil.addDebugMessage("Invalid host!"); - return false; - } + host.validate(); if (account == null || account <= 0) { - JobContextUtil.addDebugMessage("Invalid host account!"); - return false; + log.warn("Invalid host account!"); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } break; case 2: try { Long.valueOf(fileSize); } catch (NumberFormatException e) { - JobContextUtil.addDebugMessage("Invalid file size!"); - return false; + log.warn("Invalid file size!"); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } break; case 3: if (fileSourceId == null || fileSourceId <= 0) { - JobContextUtil.addDebugMessage("Invalid fileSourceId!"); - return false; + log.warn("Invalid fileSourceId!"); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } break; default: - JobContextUtil.addDebugMessage("Invalid file type!"); - return false; + log.warn("Invalid file type!"); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } - return true; } } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskFileStepVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskFileStepVO.java index 5e854c8f9d..af37d4feec 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskFileStepVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskFileStepVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,22 +27,21 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.tencent.bk.job.common.constant.DuplicateHandlerEnum; import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.constant.JobConstants; import com.tencent.bk.job.common.constant.NotExistPathHandlerEnum; import com.tencent.bk.job.common.exception.InvalidParamException; -import com.tencent.bk.job.common.util.JobContextUtil; +import com.tencent.bk.job.common.util.FilePathValidateUtil; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; -import org.apache.commons.lang3.StringUtils; +import lombok.extern.slf4j.Slf4j; import org.springframework.util.CollectionUtils; import java.util.List; -/** - * @since 16/10/2019 14:46 - */ @Data @ApiModel("步骤文件信息") +@Slf4j public class TaskFileStepVO { @ApiModelProperty("源文件列表") @@ -100,39 +99,33 @@ public static Integer getTransferMode(DuplicateHandlerEnum duplicateHandlerEnum, } } - public boolean validate(boolean isCreate) { + public void validate(boolean isCreate) throws InvalidParamException { if (CollectionUtils.isEmpty(fileSourceList)) { - JobContextUtil.addDebugMessage("Empty origin file list!"); - return false; + log.warn("Empty origin file list!"); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } for (TaskFileSourceInfoVO taskFileSourceInfoVO : fileSourceList) { - if (!taskFileSourceInfoVO.validate(isCreate)) { - JobContextUtil.addDebugMessage("Invalid file info!"); - return false; - } + taskFileSourceInfoVO.validate(isCreate); } if (fileDestination == null) { - JobContextUtil.addDebugMessage("Empty destination info!"); - return false; + log.warn("Empty destination info!"); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } - if (StringUtils.isBlank(fileDestination.getPath())) { - JobContextUtil.addDebugMessage("Empty destination location!"); - return false; + if (!FilePathValidateUtil.validateFileSystemAbsolutePath(fileDestination.getPath())) { + log.warn("fileDestinationPath is illegal! fileDestinationPath: {}", fileDestination.getPath()); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } if (fileDestination.getAccount() == null || fileDestination.getAccount() <= 0) { - JobContextUtil.addDebugMessage("Empty account!"); - return false; - } - if (!fileDestination.getServer().validate(isCreate)) { - JobContextUtil.addDebugMessage("Invalid destination!"); - return false; + log.warn("Empty account!"); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } + fileDestination.getServer().validate(); if (transferMode == null || transferMode < 1 || transferMode > 4) { - JobContextUtil.addDebugMessage("Invalid transferMode setting!"); - return false; + log.warn("Invalid transferMode : {}", transferMode); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } if (timeout == null || timeout < 0) { - timeout = 0L; + timeout = (long) JobConstants.DEFAULT_JOB_TIMEOUT_SECONDS; } if (originSpeedLimit == null || originSpeedLimit <= 0) { originSpeedLimit = null; @@ -143,28 +136,5 @@ public boolean validate(boolean isCreate) { if (ignoreError == null || ignoreError < 0) { ignoreError = 0; } - return true; - } - - public Integer getNotExistPathHandler() { - if (transferMode == 1) { - return NotExistPathHandlerEnum.STEP_FAIL.getValue(); - } else { - return NotExistPathHandlerEnum.CREATE_DIR.getValue(); - } - } - - public Integer getDuplicateHandler() { - switch (transferMode) { - case 1: - case 2: - return DuplicateHandlerEnum.OVERWRITE.getId(); - case 3: - return DuplicateHandlerEnum.GROUP_BY_IP.getId(); - case 4: - return DuplicateHandlerEnum.GROUP_BY_DATE_AND_IP.getId(); - default: - throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); - } } } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskHostNodeVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskHostNodeVO.java index d34c146c31..857f5f1163 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskHostNodeVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskHostNodeVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,11 +25,13 @@ package com.tencent.bk.job.manage.model.web.vo.task; import com.fasterxml.jackson.annotation.JsonInclude; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.model.vo.HostInfoVO; -import com.tencent.bk.job.common.util.JobContextUtil; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.util.CollectionUtils; @@ -41,6 +43,7 @@ @Data @ApiModel("主机节点信息") @JsonInclude(JsonInclude.Include.NON_NULL) +@Slf4j public class TaskHostNodeVO { @ApiModelProperty("机器 IP 列表") @@ -52,35 +55,32 @@ public class TaskHostNodeVO { @ApiModelProperty("动态分组 ID") private List dynamicGroupList; - public boolean validate(boolean isCreate) { + public void validate(boolean isCreate) throws InvalidParamException { boolean allEmpty = true; if (!CollectionUtils.isEmpty(topoNodeList)) { allEmpty = false; for (TargetNodeVO targetNodeVO : topoNodeList) { - if (!targetNodeVO.validate(isCreate)) { - JobContextUtil.addDebugMessage("Host node info validate failed!"); - return false; - } + targetNodeVO.validate(); } } if (!CollectionUtils.isEmpty(dynamicGroupList)) { allEmpty = false; for (String dynamicGroup : dynamicGroupList) { if (!StringUtils.isNoneBlank(dynamicGroup)) { - JobContextUtil.addDebugMessage("Host dynamic group id is empty!"); - return false; + log.warn("Host dynamic group id is empty!"); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } } } if (!CollectionUtils.isEmpty(ipList)) { allEmpty = false; for (HostInfoVO hostInfoVO : ipList) { - if (!hostInfoVO.validate(isCreate)) { - JobContextUtil.addDebugMessage("Host info validate failed!"); - return false; - } + hostInfoVO.validate(); } } - return !allEmpty; + if (allEmpty) { + log.warn("TaskHostNode is empty!"); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); + } } } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskPlanBasicInfoVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskPlanBasicInfoVO.java new file mode 100644 index 0000000000..0251864334 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskPlanBasicInfoVO.java @@ -0,0 +1,91 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.web.vo.task; + +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.tencent.bk.job.common.util.json.LongTimestampDeserializer; +import com.tencent.bk.job.common.util.json.LongTimestampSerializer; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; +import lombok.extern.slf4j.Slf4j; + +@Slf4j +@Getter +@Setter +@ToString +@ApiModel("执行方案信息") +public class TaskPlanBasicInfoVO { + + /** + * 执行方案 ID + */ + @ApiModelProperty(value = "执行方案 ID") + private Long id; + + /** + * 执行方案名称 + */ + @ApiModelProperty(value = "执行方案名称") + private String name; + + /** + * 模版 ID + */ + @ApiModelProperty(value = "模版 ID") + private Long templateId; + + + /** + * 创建者 + */ + @ApiModelProperty(value = "创建者") + private String creator; + + /** + * 创建时间 + */ + @ApiModelProperty(value = "创建时间") + @JsonSerialize(using = LongTimestampSerializer.class) + @JsonDeserialize(using = LongTimestampDeserializer.class) + private Long createTime; + + /** + * 最后修改人 + */ + @ApiModelProperty(value = "最后更新者") + private String lastModifyUser; + + /** + * 最后修改时间 + */ + @ApiModelProperty(value = "最后更新时间") + @JsonSerialize(using = LongTimestampSerializer.class) + @JsonDeserialize(using = LongTimestampDeserializer.class) + private Long lastModifyTime; +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskPlanSyncInfoVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskPlanSyncInfoVO.java index 9216b68365..0efad5d717 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskPlanSyncInfoVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskPlanSyncInfoVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskPlanVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskPlanVO.java index 443967d93c..4c3a7782da 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskPlanVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskPlanVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,6 +26,8 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.util.JobContextUtil; import com.tencent.bk.job.common.util.check.IlegalCharChecker; import com.tencent.bk.job.common.util.check.MaxLengthChecker; @@ -186,7 +188,8 @@ public class TaskPlanVO { @ApiModelProperty(value = "是否有删除权限") private Boolean canDelete; - public boolean validateForImport() { + public void validateForImport() throws InvalidParamException { + JobContextUtil.setAllowMigration(true); // 模板名称检查 try { StringCheckHelper stringCheckHelper = new StringCheckHelper(new TrimChecker(), new NotEmptyChecker(), @@ -194,41 +197,26 @@ public boolean validateForImport() { this.setName(stringCheckHelper.checkAndGetResult(this.getName())); } catch (StringCheckException e) { log.warn("Template name is invalid:", e); - return false; + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } boolean isCreate = false; if (CollectionUtils.isNotEmpty(stepList)) { for (TaskStepVO step : stepList) { - if (!step.validate(isCreate)) { - JobContextUtil.addDebugMessage("Validate step failed!"); - return false; - } - try { - StringCheckHelper stepCheckHelper = new StringCheckHelper(new TrimChecker(), new NotEmptyChecker(), - new IlegalCharChecker(), new MaxLengthChecker(60)); - step.setName(stepCheckHelper.checkAndGetResult(step.getName())); - } catch (StringCheckException e) { - log.warn("Step name is invalid:", e); - return false; - } + step.validate(isCreate); } } if (CollectionUtils.isNotEmpty(variableList)) { Set variableNameList = new HashSet<>(); for (TaskVariableVO variable : variableList) { - if (!variable.validate(isCreate)) { - JobContextUtil.addDebugMessage("Validate variable failed!"); - return false; - } + variable.validate(isCreate); if (variableNameList.contains(variable.getName())) { - JobContextUtil.addDebugMessage("Variable name duplicated!"); - return false; + log.warn("Variable name duplicated! name: {}", variable.getName()); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } if (variable.getDelete() != null && variable.getDelete() != 1) { variableNameList.add(variable.getName()); } } } - return true; } } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskScriptStepVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskScriptStepVO.java index a2aa22c166..0f06c5570c 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskScriptStepVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskScriptStepVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,20 +24,25 @@ package com.tencent.bk.job.manage.model.web.vo.task; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.constant.JobConstants; +import com.tencent.bk.job.common.constant.MySQLTextDataType; +import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.model.vo.TaskTargetVO; -import com.tencent.bk.job.common.util.JobContextUtil; +import com.tencent.bk.job.common.validation.EndWith; +import com.tencent.bk.job.common.validation.MaxLength; +import com.tencent.bk.job.common.validation.NotExceedMySQLTextFieldLength; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.hibernate.validator.constraints.Range; -/** - * @since 16/10/2019 14:46 - */ @Data @ApiModel("任务脚本步骤信息") +@Slf4j public class TaskScriptStepVO { @ApiModelProperty(value = "脚本类型 1-本地脚本 2-引用业务脚本 3-引用公共脚本") @@ -50,16 +55,33 @@ public class TaskScriptStepVO { private Long scriptVersionId; @ApiModelProperty("脚本内容") + @NotExceedMySQLTextFieldLength( + fieldName = "content", + fieldType = MySQLTextDataType.MEDIUMTEXT, + base64 = true + ) private String content; @ApiModelProperty("脚本语言") private Integer scriptLanguage; @ApiModelProperty("脚本参数") + @NotExceedMySQLTextFieldLength( + fieldName = "scriptParam", + fieldType = MySQLTextDataType.TEXT, + base64 = false + ) private String scriptParam; + @ApiModelProperty("自定义Windows解释器路径") + @EndWith(fieldName = "windowsInterpreter", value = ".exe", + message = "{validation.constraints.WinInterpreterInvalidSuffix.message}") + @MaxLength(value = 260, + message = "{validation.constraints.WindowsInterpreterExceedMaxLength.message}") + private String windowsInterpreter; + @ApiModelProperty("脚本超时时间") - @Range(min = JobConstants.MIN_JOB_TIMEOUT_SECONDS, max= JobConstants.MAX_JOB_TIMEOUT_SECONDS, + @Range(min = JobConstants.MIN_JOB_TIMEOUT_SECONDS, max = JobConstants.MAX_JOB_TIMEOUT_SECONDS, message = "{validation.constraints.InvalidJobTimeout_outOfRange.message}") private Long timeout; @@ -81,10 +103,10 @@ public class TaskScriptStepVO { @ApiModelProperty("忽略错误 0 - 不忽略 1 - 忽略") private Integer ignoreError; - public boolean validate(boolean isCreate) { + public void validate(boolean isCreate) throws InvalidParamException { if (scriptSource == null || scriptSource <= 0) { - JobContextUtil.addDebugMessage("Missing script step type!"); - return false; + log.warn("Invalid script source : {}", scriptSource); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } if (status == null || status > 0b11 || status < 0) { status = 0; @@ -92,8 +114,8 @@ public boolean validate(boolean isCreate) { switch (scriptSource) { case 1: if (StringUtils.isBlank(content)) { - JobContextUtil.addDebugMessage("Local step missing content!"); - return false; + log.warn("Local step missing content!"); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } scriptId = null; scriptVersionId = null; @@ -101,28 +123,35 @@ public boolean validate(boolean isCreate) { case 2: case 3: if (StringUtils.isBlank(scriptId) || scriptVersionId == null || scriptVersionId <= 0) { - JobContextUtil.addDebugMessage("Link script missing script id or script version id!"); - return false; + log.warn("Link script missing script id or script version id!"); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } content = null; break; default: - JobContextUtil.addDebugMessage("Unknown script type!"); - return false; + log.warn("Unknown script type!"); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } if (account == null || account <= 0) { - JobContextUtil.addDebugMessage("Missing execute account!"); - return false; + log.warn("Missing execute account!"); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } if (StringUtils.isNotBlank(scriptParam) && scriptParam.length() > 5000) { scriptParam = scriptParam.substring(0, 5000); } - if (timeout == null) { - timeout = 3600L; - } if (ignoreError == null || ignoreError < 0) { ignoreError = 0; } - return executeTarget.validate(isCreate); + executeTarget.validate(); + } + + /** + * 获取去除首尾空格后的windowsInterpreter + * + * @return Trim后的windowsInterpreter + */ + @JsonIgnore + public String getTrimmedWindowsInterpreter() { + return windowsInterpreter != null ? windowsInterpreter.trim() : null; } } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskStepVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskStepVO.java index 5952d64a5e..4b13424918 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskStepVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskStepVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,20 +24,27 @@ package com.tencent.bk.job.manage.model.web.vo.task; -import com.tencent.bk.job.common.util.JobContextUtil; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.InvalidParamException; +import com.tencent.bk.job.common.util.check.IlegalCharChecker; +import com.tencent.bk.job.common.util.check.MaxLengthChecker; +import com.tencent.bk.job.common.util.check.NotEmptyChecker; +import com.tencent.bk.job.common.util.check.StringCheckHelper; +import com.tencent.bk.job.common.util.check.TrimChecker; +import com.tencent.bk.job.common.util.check.exception.StringCheckException; +import com.tencent.bk.job.manage.api.common.constants.task.TaskStepTypeEnum; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import javax.validation.Valid; import java.util.List; -/** - * @since 16/10/2019 10:37 - */ @Data @ApiModel("任务步骤信息") +@Slf4j public class TaskStepVO { @ApiModelProperty("步骤 ID 仅在更新、删除时填写") @@ -73,42 +80,54 @@ public class TaskStepVO { @ApiModelProperty(value = "引用的全局变量") private List refVariables; - public boolean validate(boolean isCreate) { + public void validate(boolean isCreate) throws InvalidParamException { if (isCreate) { if (id != null && id > 0) { - JobContextUtil.addDebugMessage("Create request has step id!"); - return false; + log.warn("Create request has step id!"); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } if (StringUtils.isBlank(name)) { - JobContextUtil.addDebugMessage("Create request missing step name!"); - return false; + log.warn("Create request missing step name!"); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } } if (delete != null && delete == 1) { - return true; + return; + } + try { + StringCheckHelper stepCheckHelper = new StringCheckHelper(new TrimChecker(), new NotEmptyChecker(), + new IlegalCharChecker(), new MaxLengthChecker(60)); + this.name = stepCheckHelper.checkAndGetResult(name); + } catch (StringCheckException e) { + log.warn("Step name is invalid, stepName: {}", name); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } - switch (type) { - case 1: - if (scriptStepInfo == null || !scriptStepInfo.validate(isCreate)) { - JobContextUtil.addDebugMessage("Script step info validate failed!"); - return false; + TaskStepTypeEnum stepType = TaskStepTypeEnum.valueOf(type); + switch (stepType) { + case SCRIPT: + if (scriptStepInfo == null) { + log.warn("Empty script step"); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } + scriptStepInfo.validate(isCreate); break; - case 2: - if (fileStepInfo == null || !fileStepInfo.validate(isCreate)) { - JobContextUtil.addDebugMessage("File step info validate failed!"); - return false; + case FILE: + if (fileStepInfo == null) { + log.warn("Empty file step"); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } + fileStepInfo.validate(isCreate); break; - case 3: - if (approvalStepInfo == null || !approvalStepInfo.validate(isCreate)) { - JobContextUtil.addDebugMessage("Approval step info validate failed!"); - return false; + case APPROVAL: + if (approvalStepInfo == null) { + log.warn("Empty approval step"); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } + approvalStepInfo.validate(isCreate); break; default: - return false; + log.warn("Invalid step type: {}", type); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } - return true; } } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskTemplateVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskTemplateVO.java index 8e35fb3542..ed4c098905 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskTemplateVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskTemplateVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -37,9 +37,6 @@ import java.util.List; -/** - * @since 23/10/2019 16:07 - */ @Getter @Setter @ToString @@ -99,6 +96,7 @@ public class TaskTemplateVO { /** * 模版内的脚本状态 + * */ @ApiModelProperty(value = "模版内脚本状态 0 - 正常 1 - 需要更新 2 - 被禁用 3 - 需要更新和被禁用的都存在") private Integer scriptStatus; diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskVariableVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskVariableVO.java index 7a58da97ba..adaa900df7 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskVariableVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/task/TaskVariableVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,11 +24,14 @@ package com.tencent.bk.job.manage.model.web.vo.task; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.model.vo.TaskTargetVO; import com.tencent.bk.job.common.util.JobContextUtil; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.Data; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; /** @@ -36,6 +39,7 @@ */ @Data @ApiModel("全局变量信息") +@Slf4j public class TaskVariableVO { @ApiModelProperty(value = "变量 ID 新增时无需填写，删除时仅需填写 id 和 delete", required = true) @@ -71,28 +75,27 @@ public class TaskVariableVO { @ApiModelProperty(value = "删除 0-不删除 1-删除，仅在删除时填写") private Integer delete; - public boolean validate(boolean isCreate) { + public void validate(boolean isCreate) throws InvalidParamException { if (isCreate && !JobContextUtil.isAllowMigration()) { if (id != null && id > 0) { - JobContextUtil.addDebugMessage("Create request has variable id!"); - return false; + log.warn("Create request has variable id!"); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } } if (StringUtils.isBlank(name)) { - JobContextUtil.addDebugMessage("Empty variable name!"); - return false; + log.warn("Empty variable name!"); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } if (description == null) { description = ""; } if (type <= 0 || type > 6) { - JobContextUtil.addDebugMessage("Invalid variable type!"); - return false; + log.warn("Invalid variable type: {}", type); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } if (required == null || required > 1) { - JobContextUtil.addDebugMessage("Invalid require option!"); - return false; + log.warn("Invalid require option : {}", required); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } - return true; } } diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/whiteip/ActionScopeVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/whiteip/ActionScopeVO.java index 09ef2d8846..58fbeff54e 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/whiteip/ActionScopeVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/whiteip/ActionScopeVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/whiteip/ScopeVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/whiteip/ScopeVO.java new file mode 100644 index 0000000000..fe254347f1 --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/whiteip/ScopeVO.java @@ -0,0 +1,51 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.web.vo.whiteip; + +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * 资源范围VO + */ +@NoArgsConstructor +@ApiModel("资源范围") +@Data +public class ScopeVO { + @ApiModelProperty("资源范围类型") + private String scopeType; + @ApiModelProperty("资源范围ID") + private String scopeId; + @ApiModelProperty("业务名称") + private String name; + + public ScopeVO(String scopeType, String scopeId, String name) { + this.scopeType = scopeType; + this.scopeId = scopeId; + this.name = name; + } +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/whiteip/WhiteIPHostVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/whiteip/WhiteIPHostVO.java new file mode 100644 index 0000000000..3ef13a46fc --- /dev/null +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/whiteip/WhiteIPHostVO.java @@ -0,0 +1,53 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.web.vo.whiteip; + +import com.fasterxml.jackson.annotation.JsonInclude; +import io.swagger.annotations.ApiModel; +import io.swagger.annotations.ApiModelProperty; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@NoArgsConstructor +@AllArgsConstructor +@ApiModel("IP白名单中的主机信息") +@Data +@JsonInclude(JsonInclude.Include.NON_NULL) +public class WhiteIPHostVO { + + @ApiModelProperty(value = "服务器 ID", required = true) + private Long hostId; + + @ApiModelProperty("云区域ID") + private Long cloudAreaId; + + @ApiModelProperty("主机 IP") + private String ip; + + @ApiModelProperty("主机 IPv6") + private String ipv6; + +} diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/whiteip/WhiteIPRecordVO.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/whiteip/WhiteIPRecordVO.java index 91ee230927..fe1470f202 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/whiteip/WhiteIPRecordVO.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/model/web/vo/whiteip/WhiteIPRecordVO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,7 +26,6 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.tencent.bk.job.common.util.json.LongTimestampSerializer; -import com.tencent.bk.job.manage.model.web.vo.AppVO; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; import lombok.AllArgsConstructor; @@ -47,12 +46,18 @@ public class WhiteIPRecordVO { private Long id; @ApiModelProperty("云区域ID") private Long cloudAreaId; - @ApiModelProperty("IP列表") - private List ipList; + + @ApiModelProperty("主机列表") + private List hostList; @ApiModelProperty("生效范围") private List actionScopeList; - @ApiModelProperty("业务") - private List appList; + + @ApiModelProperty(value = "是否对所有资源范围生效，默认为false") + private boolean allScope = false; + + @ApiModelProperty("生效的资源范围列表") + private List scopeList; + @ApiModelProperty("备注") private String remark; @ApiModelProperty("创建人") diff --git a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/validation/provider/DangerousRuleGroupSequenceProvider.java b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/validation/provider/DangerousRuleGroupSequenceProvider.java index 73fe91ebb8..1730f2b5c1 100644 --- a/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/validation/provider/DangerousRuleGroupSequenceProvider.java +++ b/src/backend/job-manage/api-job-manage/src/main/java/com/tencent/bk/job/manage/validation/provider/DangerousRuleGroupSequenceProvider.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/boot-job-manage/build.gradle b/src/backend/job-manage/boot-job-manage/build.gradle index 52601ccd35..a31ab6d7dd 100644 --- a/src/backend/job-manage/boot-job-manage/build.gradle +++ b/src/backend/job-manage/boot-job-manage/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -30,10 +30,9 @@ dependencies { api project(":commons:common-i18n") implementation 'org.springframework.boot:spring-boot-starter-jdbc' implementation 'org.springframework.cloud:spring-cloud-starter-bootstrap' - implementation 'org.springframework.cloud:spring-cloud-starter-bus-amqp' implementation 'org.springframework:spring-webmvc' implementation(group: 'org.springframework.boot', name: 'spring-boot-starter-data-redis') - runtimeOnly('mysql:mysql-connector-java') + runtimeOnly('com.mysql:mysql-connector-j') testImplementation("com.h2database:h2") } @@ -47,8 +46,7 @@ task renameArtifacts(type: Copy) { destinationDir file('build/libs/') rename "boot-job-manage-${version}.jar", "job-manage-${version}.jar" } -renameArtifacts.dependsOn jar -assemble.dependsOn renameArtifacts +renameArtifacts.dependsOn assemble task copyToLatestJar(type: Copy) { group = "local" @@ -59,3 +57,4 @@ task copyToLatestJar(type: Copy) { } copyToLatestJar.dependsOn assemble apply from: "$rootDir/task_job_package.gradle" +copyToRelease.dependsOn renameArtifacts diff --git a/src/backend/job-manage/boot-job-manage/src/main/java/com/tencent/bk/job/manage/JobManageBootApplication.java b/src/backend/job-manage/boot-job-manage/src/main/java/com/tencent/bk/job/manage/JobManageBootApplication.java index 08f4a2e0ce..521a3316b3 100644 --- a/src/backend/job-manage/boot-job-manage/src/main/java/com/tencent/bk/job/manage/JobManageBootApplication.java +++ b/src/backend/job-manage/boot-job-manage/src/main/java/com/tencent/bk/job/manage/JobManageBootApplication.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,19 +24,20 @@ package com.tencent.bk.job.manage; -import com.tencent.bk.job.common.config.FeatureToggleConfig; +import com.tencent.bk.job.common.service.boot.JobBootApplication; import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.boot.autoconfigure.data.redis.RedisAutoConfiguration; -import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.boot.autoconfigure.availability.ApplicationAvailabilityAutoConfiguration; +import org.springframework.boot.autoconfigure.jooq.JooqAutoConfiguration; import org.springframework.cache.annotation.EnableCaching; import org.springframework.cloud.openfeign.EnableFeignClients; import org.springframework.scheduling.annotation.EnableScheduling; -@SpringBootApplication(scanBasePackages = "com.tencent.bk.job", exclude = {RedisAutoConfiguration.class}) -@EnableConfigurationProperties({FeatureToggleConfig.class}) +@JobBootApplication( + scanBasePackages = "com.tencent.bk.job.manage", + exclude = {JooqAutoConfiguration.class, ApplicationAvailabilityAutoConfiguration.class}, + excludeName = {"org.springframework.cloud.kubernetes.client.discovery.KubernetesDiscoveryClientAutoConfiguration"}) @EnableCaching -@EnableFeignClients +@EnableFeignClients(basePackages = "com.tencent.bk.job") @EnableScheduling public class JobManageBootApplication { diff --git a/src/backend/job-manage/boot-job-manage/src/main/java/com/tencent/bk/job/manage/config/DbConfig.java b/src/backend/job-manage/boot-job-manage/src/main/java/com/tencent/bk/job/manage/config/DbConfig.java deleted file mode 100644 index 5e9127bdbb..0000000000 --- a/src/backend/job-manage/boot-job-manage/src/main/java/com/tencent/bk/job/manage/config/DbConfig.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.config; - -import org.jooq.ConnectionProvider; -import org.jooq.DSLContext; -import org.jooq.SQLDialect; -import org.jooq.impl.DataSourceConnectionProvider; -import org.jooq.impl.DefaultConfiguration; -import org.jooq.impl.DefaultDSLContext; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.boot.context.properties.ConfigurationProperties; -import org.springframework.boot.jdbc.DataSourceBuilder; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.DependsOn; -import org.springframework.context.annotation.Primary; -import org.springframework.jdbc.core.JdbcTemplate; -import org.springframework.jdbc.datasource.DataSourceTransactionManager; -import org.springframework.transaction.annotation.EnableTransactionManagement; - -import javax.sql.DataSource; - -/** - * @date 2019/09/19 - */ -@Configuration -@EnableTransactionManagement -public class DbConfig { - @Qualifier("job-manage-data-source") - @Primary - @Bean(name = "job-manage-data-source") - @ConfigurationProperties(prefix = "spring.datasource.job-manage") - public DataSource dataSource() { - return DataSourceBuilder.create().build(); - } - - @Qualifier("transactionManager") - @Bean(name = "transactionManager") - @DependsOn("job-manage-data-source") - @Primary - public DataSourceTransactionManager transactionManager(@Qualifier("job-manage-data-source") DataSource dataSource) { - return new DataSourceTransactionManager(dataSource); - } - - @Qualifier("job-manage-jdbc-template") - @Bean(name = "job-manage-jdbc-template") - public JdbcTemplate jdbcTemplate(@Qualifier("job-manage-data-source") DataSource dataSource) { - return new JdbcTemplate(dataSource); - } - - @Qualifier("job-manage-dsl-context") - @Bean(name = "job-manage-dsl-context") - public DSLContext dslContext(@Qualifier("job-manage-jooq-conf") org.jooq.Configuration configuration) { - return new DefaultDSLContext(configuration); - } - - @Qualifier("job-manage-jooq-conf") - @Bean(name = "job-manage-jooq-conf") - public org.jooq.Configuration - jooqConf(@Qualifier("job-manage-conn-provider") ConnectionProvider connectionProvider) { - return new DefaultConfiguration().derive(connectionProvider).derive(SQLDialect.MYSQL); - } - - @Qualifier("job-manage-conn-provider") - @Bean(name = "job-manage-conn-provider") - public ConnectionProvider connectionProvider(@Qualifier("job-manage-data-source") DataSource dataSource) { - return new DataSourceConnectionProvider(dataSource); - } - -} diff --git a/src/backend/job-manage/boot-job-manage/src/main/java/com/tencent/bk/job/manage/config/FilterConfig.java b/src/backend/job-manage/boot-job-manage/src/main/java/com/tencent/bk/job/manage/config/FilterConfig.java deleted file mode 100644 index de0003b896..0000000000 --- a/src/backend/job-manage/boot-job-manage/src/main/java/com/tencent/bk/job/manage/config/FilterConfig.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.config; - -import com.tencent.bk.job.common.web.filter.RepeatableReadWriteServletRequestResponseFilter; -import org.springframework.boot.web.servlet.FilterRegistrationBean; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Configuration -public class FilterConfig { - @Bean - public FilterRegistrationBean repeatableRSRRFilterRegister() { - FilterRegistrationBean registration = - new FilterRegistrationBean<>(); - registration.setFilter(repeatableRRRFilter()); - registration.addUrlPatterns("/esb/api/*"); - registration.setName("repeatableReadRequestResponseFilter"); - registration.setOrder(0); - return registration; - } - - @Bean(name = "repeatableReadRequestResponseFilter") - public RepeatableReadWriteServletRequestResponseFilter repeatableRRRFilter() { - return new RepeatableReadWriteServletRequestResponseFilter(); - } -} diff --git a/src/backend/job-manage/boot-job-manage/src/main/java/com/tencent/bk/job/manage/config/InterceptorConfiguration.java b/src/backend/job-manage/boot-job-manage/src/main/java/com/tencent/bk/job/manage/config/InterceptorConfiguration.java deleted file mode 100644 index 4415e81e21..0000000000 --- a/src/backend/job-manage/boot-job-manage/src/main/java/com/tencent/bk/job/manage/config/InterceptorConfiguration.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.config; - -import com.tencent.bk.job.common.iam.interceptor.AuthAppInterceptor; -import com.tencent.bk.job.common.iam.interceptor.JobIamInterceptor; -import com.tencent.bk.job.common.web.interceptor.EsbApiLogInterceptor; -import com.tencent.bk.job.common.web.interceptor.EsbReqRewriteInterceptor; -import com.tencent.bk.job.common.web.interceptor.JobCommonInterceptor; -import com.tencent.bk.job.common.web.interceptor.ServiceSecurityInterceptor; -import com.tencent.bk.job.manage.common.interceptor.UriPermissionInterceptor; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Configuration; -import org.springframework.web.servlet.config.annotation.InterceptorRegistry; -import org.springframework.web.servlet.config.annotation.WebMvcConfigurer; - -/** - * @since 6/11/2019 10:38 - */ -@Configuration -public class InterceptorConfiguration implements WebMvcConfigurer { - - private final JobCommonInterceptor jobCommonInterceptor; - private final AuthAppInterceptor authAppInterceptor; - private final UriPermissionInterceptor uriPermissionInterceptor; - private final EsbApiLogInterceptor esbApiLogInterceptor; - private final EsbReqRewriteInterceptor esbReqRewriteInterceptor; - private final ServiceSecurityInterceptor serviceSecurityInterceptor; - private final JobIamInterceptor iamInterceptor; - - @Autowired - public InterceptorConfiguration( - JobCommonInterceptor jobCommonInterceptor, - AuthAppInterceptor authAppInterceptor, - UriPermissionInterceptor uriPermissionInterceptor, - EsbApiLogInterceptor esbApiLogInterceptor, - EsbReqRewriteInterceptor esbReqRewriteInterceptor, - ServiceSecurityInterceptor serviceSecurityInterceptor, - JobIamInterceptor iamInterceptor - ) { - this.jobCommonInterceptor = jobCommonInterceptor; - this.authAppInterceptor = authAppInterceptor; - this.uriPermissionInterceptor = uriPermissionInterceptor; - this.esbApiLogInterceptor = esbApiLogInterceptor; - this.esbReqRewriteInterceptor = esbReqRewriteInterceptor; - this.serviceSecurityInterceptor = serviceSecurityInterceptor; - this.iamInterceptor = iamInterceptor; - } - - @Override - public void addInterceptors(InterceptorRegistry registry) { - // 注册拦截器 - registry.addInterceptor(jobCommonInterceptor).addPathPatterns("/**").order(0); - registry.addInterceptor(serviceSecurityInterceptor).addPathPatterns("/**").order(20); - registry.addInterceptor(uriPermissionInterceptor) - .addPathPatterns( - uriPermissionInterceptor.getControlUriPatterns() - ).order(30); - registry.addInterceptor(iamInterceptor).addPathPatterns("/iam/api/v1/resources/**").order(40); - registry.addInterceptor(esbApiLogInterceptor).addPathPatterns("/esb/api/**").order(50); - registry.addInterceptor(esbReqRewriteInterceptor).addPathPatterns("/esb/api/**").order(60); - registry.addInterceptor(authAppInterceptor).addPathPatterns("/web/**", "/esb/api/**").order(70); - } -} diff --git a/src/backend/job-manage/boot-job-manage/src/main/java/com/tencent/bk/job/manage/config/JobManageConfiguration.java b/src/backend/job-manage/boot-job-manage/src/main/java/com/tencent/bk/job/manage/config/JobManageConfiguration.java new file mode 100644 index 0000000000..f319a45cf5 --- /dev/null +++ b/src/backend/job-manage/boot-job-manage/src/main/java/com/tencent/bk/job/manage/config/JobManageConfiguration.java @@ -0,0 +1,39 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.config; + +import com.tencent.bk.job.common.service.AppScopeMappingService; +import com.tencent.bk.job.common.web.interceptor.AppResourceScopeInterceptor; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +@Configuration +public class JobManageConfiguration { + + @Bean + public AppResourceScopeInterceptor appResourceScopeInterceptor(AppScopeMappingService appScopeMappingService) { + return new AppResourceScopeInterceptor(appScopeMappingService); + } +} diff --git a/src/backend/job-manage/boot-job-manage/src/main/java/com/tencent/bk/job/manage/config/RedisCacheConfig.java b/src/backend/job-manage/boot-job-manage/src/main/java/com/tencent/bk/job/manage/config/RedisCacheConfig.java deleted file mode 100644 index 236cb90a5a..0000000000 --- a/src/backend/job-manage/boot-job-manage/src/main/java/com/tencent/bk/job/manage/config/RedisCacheConfig.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.config; - -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.PropertyAccessor; -import com.fasterxml.jackson.databind.ObjectMapper; -import org.springframework.cache.CacheManager; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.context.annotation.Primary; -import org.springframework.data.redis.cache.RedisCacheConfiguration; -import org.springframework.data.redis.cache.RedisCacheManager; -import org.springframework.data.redis.cache.RedisCacheWriter; -import org.springframework.data.redis.connection.RedisConnectionFactory; -import org.springframework.data.redis.serializer.GenericJackson2JsonRedisSerializer; -import org.springframework.data.redis.serializer.RedisSerializationContext; -import org.springframework.data.redis.serializer.RedisSerializer; - -import java.time.Duration; -import java.util.HashMap; -import java.util.Map; - -/** - * @Description - * @Date 2020/2/26 - * @Version 1.0 - */ -@Configuration -public class RedisCacheConfig { - - RedisCacheConfig() { - } - - /** - * 使用GenericJackson2JsonRedisSerializer序列化器 - */ - private RedisSerializer valueSerializer() { - ObjectMapper objectMapper = new ObjectMapper(); - objectMapper.setVisibility(PropertyAccessor.ALL, JsonAutoDetect.Visibility.ANY); - objectMapper.enableDefaultTyping(ObjectMapper.DefaultTyping.NON_FINAL); - return new GenericJackson2JsonRedisSerializer(objectMapper); - } - - private RedisCacheConfiguration getRedisCacheConfigurationWithTtl(Integer seconds) { - RedisCacheConfiguration redisCacheConfiguration = RedisCacheConfiguration.defaultCacheConfig(); - redisCacheConfiguration = redisCacheConfiguration.serializeValuesWith( - RedisSerializationContext - .SerializationPair - .fromSerializer(valueSerializer()) - ).entryTtl(Duration.ofSeconds(seconds)); - return redisCacheConfiguration; - } - - /** - * 缓存过期时间自定义配置 - */ - @Bean - @Primary - public CacheManager cacheManager(RedisConnectionFactory redisConnectionFactory) { - // 初始化一个RedisCacheWriter - RedisCacheWriter redisCacheWriter = RedisCacheWriter.nonLockingRedisCacheWriter(redisConnectionFactory); - //默认5秒过期 - RedisCacheConfiguration defaultCacheConfig = - RedisCacheConfiguration.defaultCacheConfig() - .serializeValuesWith(RedisSerializationContext.SerializationPair.fromSerializer(valueSerializer())) - .entryTtl(Duration.ofSeconds(5)); - // 每一类信息进行缓存配置 - Map redisCacheConfigurationMap = new HashMap<>(); - //单个业务信息3秒过期 - redisCacheConfigurationMap.put("appInfoCache", this.getRedisCacheConfigurationWithTtl(3)); - //拓扑信息10秒过期 - redisCacheConfigurationMap.put("appTopologyCache", this.getRedisCacheConfigurationWithTtl(10)); - // 初始化RedisCacheManager - RedisCacheManager cacheManager = new RedisCacheManager(redisCacheWriter, defaultCacheConfig, - redisCacheConfigurationMap); - return cacheManager; - } -} diff --git a/src/backend/job-manage/boot-job-manage/src/main/java/com/tencent/bk/job/manage/config/ScheduleConfig.java b/src/backend/job-manage/boot-job-manage/src/main/java/com/tencent/bk/job/manage/config/ScheduleConfig.java deleted file mode 100644 index ce49eb7a95..0000000000 --- a/src/backend/job-manage/boot-job-manage/src/main/java/com/tencent/bk/job/manage/config/ScheduleConfig.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.config; - -import lombok.extern.slf4j.Slf4j; -import org.springframework.context.annotation.Configuration; -import org.springframework.scheduling.annotation.SchedulingConfigurer; -import org.springframework.scheduling.config.ScheduledTaskRegistrar; - -import java.util.concurrent.ScheduledThreadPoolExecutor; - -/** - * @Description - * @Date 2020/2/24 - * @Version 1.0 - */ - -//设定一个长度5的定时任务线程池 -@Slf4j -@Configuration -public class ScheduleConfig implements SchedulingConfigurer { - @Override - public void configureTasks(ScheduledTaskRegistrar taskRegistrar) { - taskRegistrar.setScheduler(new ScheduledThreadPoolExecutor(5, (r, executor) -> log.error( - "ScheduledThreadPoolExecutor rejected a runnable"))); - } -} diff --git a/src/backend/job-manage/boot-job-manage/src/main/java/com/tencent/bk/job/manage/config/SwaggerConfig.java b/src/backend/job-manage/boot-job-manage/src/main/java/com/tencent/bk/job/manage/config/SwaggerConfig.java index 4006b8245c..2a5a93c708 100644 --- a/src/backend/job-manage/boot-job-manage/src/main/java/com/tencent/bk/job/manage/config/SwaggerConfig.java +++ b/src/backend/job-manage/boot-job-manage/src/main/java/com/tencent/bk/job/manage/config/SwaggerConfig.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,38 +24,40 @@ package com.tencent.bk.job.manage.config; +import com.tencent.bk.job.common.service.config.JobCommonConfig; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Profile; import springfox.documentation.builders.PathSelectors; import springfox.documentation.builders.RequestHandlerSelectors; +import springfox.documentation.oas.annotations.EnableOpenApi; import springfox.documentation.spi.DocumentationType; import springfox.documentation.spring.web.plugins.Docket; -import springfox.documentation.swagger2.annotations.EnableSwagger2; import java.util.Arrays; import java.util.HashSet; /** - * @date 2019/09/19 + * Swagger 配置 */ @Configuration -@EnableSwagger2 -@Profile({"dev", "test", "local"}) +@EnableOpenApi +@Profile({"dev", "local"}) public class SwaggerConfig { - private final JobManageConfig jobManageConfig; + private final JobCommonConfig jobCommonConfig; @Autowired - public SwaggerConfig(JobManageConfig jobManageConfig) { - this.jobManageConfig = jobManageConfig; + public SwaggerConfig(JobCommonConfig jobCommonConfig) { + this.jobCommonConfig = jobCommonConfig; } @Bean public Docket api() { - return new Docket(DocumentationType.SWAGGER_2) - .host(jobManageConfig.getSwaggerUrl() + "/job-manage") + return new Docket(DocumentationType.OAS_30) + .host(jobCommonConfig.getSwaggerUrl()) + .pathMapping("job-manage") .protocols(new HashSet<>(Arrays.asList("http", "https"))) .select() .apis(RequestHandlerSelectors.basePackage("com.tencent.bk.job.manage.api")) diff --git a/src/backend/job-manage/boot-job-manage/src/main/resources/application.yml b/src/backend/job-manage/boot-job-manage/src/main/resources/application.yml index 171be41030..bd9e53f60e 100644 --- a/src/backend/job-manage/boot-job-manage/src/main/resources/application.yml +++ b/src/backend/job-manage/boot-job-manage/src/main/resources/application.yml @@ -1,20 +1,24 @@ spring: + main: + allow-bean-definition-overriding: true profiles: active: prod - -ribbon: - ServerListRefreshInterval: 3000 - + mvc: + pathmatch: + matching-strategy: ant_path_matcher management: health: livenessstate: enabled: true readinessstate: enabled: true + # 使用spring cloud stream rabbitmq, 禁用掉springboot的rabbitmq health检查 + rabbit: + enabled: false endpoints: web: exposure: - include: health,configprops,env,beans,conditions,loggers,metrics,mappings,prometheus,scheduledtasks,info,refresh + include: health,configprops,env,beans,conditions,loggers,metrics,mappings,prometheus,scheduledtasks,info,refresh,busrefresh,bindings base-path: /actuator enabled-by-default: false endpoint: @@ -45,6 +49,8 @@ management: enabled: true refresh: enabled: true + busrefresh: + enabled: true server: port: 19803 shutdown: graceful diff --git a/src/backend/job-manage/boot-job-manage/src/main/resources/bootstrap.yml b/src/backend/job-manage/boot-job-manage/src/main/resources/bootstrap.yml index 665d6a09be..420dd22f1c 100644 --- a/src/backend/job-manage/boot-job-manage/src/main/resources/bootstrap.yml +++ b/src/backend/job-manage/boot-job-manage/src/main/resources/bootstrap.yml @@ -11,3 +11,5 @@ spring: name: ${JOB_APPLICATION_CONFIGMAP_NAME} - namespace: ${KUBERNETES_NAMESPACE} name: ${JOB_COMMON_CONFIGMAP_NAME} + - namespace: ${KUBERNETES_NAMESPACE} + name: ${JOB_COMMON_REFRESHABLE_CONFIGMAP_NAME} diff --git a/src/backend/job-manage/boot-job-manage/src/main/resources/i18n/message.properties b/src/backend/job-manage/boot-job-manage/src/main/resources/i18n/message.properties deleted file mode 100644 index ddcc596a9f..0000000000 --- a/src/backend/job-manage/boot-job-manage/src/main/resources/i18n/message.properties +++ /dev/null @@ -1,98 +0,0 @@ -# -# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. -# -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. -# -# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. -# -# License for BK-JOB蓝鲸智云作业平台: -# -------------------------------------------------------------------- -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF -# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - -job.resource.status.online=已上线 -job.resource.status.draft=未上线 -job.resource.status.offline=已下线 -job.resource.status.disabled=禁用 -job.account.category.system=系统账号 -job.account.category.db=数据库账号 - - -#script check code -103701001=脚本首行没有定义合法的脚本类型，例如： #!/bin/bash -103701002=请用'cd ... || exit' 或者 'cd ... || return' 处理目录不存在的逻辑 -103702001=危险：出现递归逻辑fork炸弹, 禁止在函数中调用自己 -103702002=危险：删除高危目录，可能会影响到系统的稳定! -103702101=禁止不加条件扫描根目录! -103702102=警告:有条件扫描根目录，建议指定具体目录，不建议直接扫描根目录 -103702103=警告:建议加上扫描条件! -103702111=危险:不要重定向数据到块设备! -103702112=危险:不要对块设备进行格式化文件系统! -103702113=危险:不要对块设备进行格式化! -103702114=危险:不要用dd命令进行块设备操作! -job.task.plan.debug=调试 - - -#消息通知部分 -#资源类型 -job.manage.resource.type.script=快速执行脚本 -job.manage.resource.type.file=快速分发文件 -job.manage.resource.type.job=执行方案 -#触发方式 -job.manage.trigger.type.page.execute=页面执行 -job.manage.trigger.type.api.invoke=API调用 -job.manage.trigger.type.timer.task=定时任务 -#执行状态 -job.manage.execute.status.success=执行成功 -job.manage.execute.status.fail=执行失败 -job.manage.execute.status.ready=等待执行 -#角色 -job.manage.role.resource.owner=资源所属者 -job.manage.role.resource.trigger.user=任务执行人 -job.manage.role.extra.observer=额外通知人 -#通知渠道 -job.manage.notify.channel.sms=短信 -job.manage.notify.channel.email=邮件 -job.manage.notify.channel.wechat=微信 -#通知模板名称 -job.manage.notify.template.name.confirmation=人工确认 -job.manage.notify.template.name.executeSuccess=执行成功 -job.manage.notify.template.name.executeFailure=执行失败 -job.manage.notify.template.name.beforeCronJobExecute=定时执行前 -job.manage.notify.template.name.beforeCronJobEnd=定时结束前 -job.manage.notify.template.name.cronJobFailed=定时启动失败 - -# IP白名单部分 -job.manage.whiteip.actionScope.scriptExecute=脚本执行 -job.manage.whiteip.actionScope.fileDistribution=文件分发 - -# 全局设置部分 -job.manage.globalsettings.defaultNameRules.description.linux=由小写字母或下划线开头，和 2~32个字母 / 数字 / _ 或 - 组成的字符 -job.manage.globalsettings.defaultNameRules.description.windows=由1~32个大小写字母或横杠组成的字符 -job.manage.globalsettings.defaultNameRules.description.database=由1~16个大小写 / 数字 / . / - / _ 组成的字符 -job.manage.globalsettings.defaultTitleHead=作业平台 | 腾讯蓝鲸智云 -job.manage.globalsettings.contactBKHelper=联系BK助手 -job.manage.globalsettings.BKDesktop=蓝鲸桌面 -job.manage.globalsettings.ee.footerLink=[技术支持](https://wpa1.qq.com/KziXGWJs?_type=wpa&qidian=true) | [社区论坛](https://bk.tencent.com/s-mart/community) | [产品官网](https://bk.tencent.com/index/) -job.manage.globalsettings.ee.footerCopyRight=Copyright © 2012-{{current_year}} Tencent BlueKing. All Rights Reserved. {{current_ver}} - -job.custom.settings.script.template.variable.datetime.desc=当前日期时间 -job.custom.settings.script.template.variable.username.desc=用户名 -job.custom.settings.script.template.variable.display_name.desc=全名 -job.custom.settings.script.template.variable.biz_id.desc=业务(集)ID -job.custom.settings.script.template.variable.biz_name.desc=业务(集)名称 - -# CMDB相关资源 -cmdb.object.name.biz=业务 diff --git a/src/backend/job-manage/boot-job-manage/src/main/resources/i18n/message_en.properties b/src/backend/job-manage/boot-job-manage/src/main/resources/i18n/message_en.properties deleted file mode 100644 index 02edd8fbf0..0000000000 --- a/src/backend/job-manage/boot-job-manage/src/main/resources/i18n/message_en.properties +++ /dev/null @@ -1,98 +0,0 @@ -# -# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. -# -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. -# -# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. -# -# License for BK-JOB蓝鲸智云作业平台: -# -------------------------------------------------------------------- -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF -# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - -job.resource.status.online=Online -job.resource.status.draft=Stand-by -job.resource.status.offline=Offline -job.resource.status.disabled=Banned -job.account.category.system=System Account -job.account.category.db=Database Account - - -#script check code -103701001=The first line of the script does not define a legal script type(e.g.: #!/bin/bash). -103701002=Plz use 'cd ... || exit' or 'cd ... || return' to determine if directory does not exist. -103702001=Danger: Recursive logic fork bomb appears, prohibit calling yourself in the function. -103702002=Danger: Deleting high-risk directories may affect the stability of the system! -103702101=Prohibit unconditional scanning of the root directory! -103702102=Warning: It is not recommended to scan the root directory directly. -103702103=Warning: It is recommended to add some conditions! -103702111=Danger: Do not redirect data to block devices! -103702112=Danger: Do not format the file system for block devices! -103702113=Danger: Do not format block devices! -103702114=Danger: Do not use 'dd' for block devices! -job.task.plan.debug=Debug - - -#消息通知部分 -#资源类型 -job.manage.resource.type.script=Script -job.manage.resource.type.file=File -job.manage.resource.type.job=Job -#触发方式 -job.manage.trigger.type.page.execute=Web UI -job.manage.trigger.type.api.invoke=API -job.manage.trigger.type.timer.task=Cron -#执行状态 -job.manage.execute.status.success=Success -job.manage.execute.status.fail=Failed -job.manage.execute.status.ready=Waitting -#角色 -job.manage.role.resource.owner=Owner -job.manage.role.resource.trigger.user=Operater -job.manage.role.extra.observer=Additional -#通知渠道 -job.manage.notify.channel.sms=SMS -job.manage.notify.channel.email=Mail -job.manage.notify.channel.wechat=WeChat -#通知模板名称 -job.manage.notify.template.name.confirmation=Confirm -job.manage.notify.template.name.executeSuccess=Success -job.manage.notify.template.name.executeFailure=Fail -job.manage.notify.template.name.beforeCronJobExecute=Before Cron Start -job.manage.notify.template.name.beforeCronJobEnd=Before Cron End -job.manage.notify.template.name.cronJobFailed=Cron Unlaunched - -# IP白名单部分 -job.manage.whiteip.actionScope.scriptExecute=Script Execution -job.manage.whiteip.actionScope.fileDistribution=File Transfer - -# 全局设置部分 -job.manage.globalsettings.defaultNameRules.description.linux=Start with lowercase or underscore, followed by lowercase / numbers / _ or -, up to 32 characters -job.manage.globalsettings.defaultNameRules.description.windows=Composed of 1 to 32 uppercase and lowercase or hyphen symbol -job.manage.globalsettings.defaultNameRules.description.database=Composed of 1 to 16 uppercase / lowercase / numbers / . / - / _ -job.manage.globalsettings.defaultTitleHead=Job | Tencent BlueKing -job.manage.globalsettings.contactBKHelper=Contact BK-Helper -job.manage.globalsettings.BKDesktop=BlueKing Desktop -job.manage.globalsettings.ee.footerLink=[Support](https://wpa1.qq.com/KziXGWJs?_type=wpa&qidian=true) | [Forum](https://bk.tencent.com/s-mart/community) | [Official](https://bk.tencent.com/index/) -job.manage.globalsettings.ee.footerCopyRight=Copyright © 2012-{{current_year}} Tencent BlueKing. All Rights Reserved. {{current_ver}} - -job.custom.settings.script.template.variable.datetime.desc=Current date and time -job.custom.settings.script.template.variable.username.desc=Username -job.custom.settings.script.template.variable.display_name.desc=User full name -job.custom.settings.script.template.variable.biz_id.desc=Business(set) -job.custom.settings.script.template.variable.biz_name.desc=Business(set) Name - -# CMDB相关资源 -cmdb.object.name.biz=Business diff --git a/src/backend/job-manage/boot-job-manage/src/main/resources/i18n/message_en_US.properties b/src/backend/job-manage/boot-job-manage/src/main/resources/i18n/message_en_US.properties deleted file mode 100644 index 02edd8fbf0..0000000000 --- a/src/backend/job-manage/boot-job-manage/src/main/resources/i18n/message_en_US.properties +++ /dev/null @@ -1,98 +0,0 @@ -# -# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. -# -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. -# -# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. -# -# License for BK-JOB蓝鲸智云作业平台: -# -------------------------------------------------------------------- -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF -# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - -job.resource.status.online=Online -job.resource.status.draft=Stand-by -job.resource.status.offline=Offline -job.resource.status.disabled=Banned -job.account.category.system=System Account -job.account.category.db=Database Account - - -#script check code -103701001=The first line of the script does not define a legal script type(e.g.: #!/bin/bash). -103701002=Plz use 'cd ... || exit' or 'cd ... || return' to determine if directory does not exist. -103702001=Danger: Recursive logic fork bomb appears, prohibit calling yourself in the function. -103702002=Danger: Deleting high-risk directories may affect the stability of the system! -103702101=Prohibit unconditional scanning of the root directory! -103702102=Warning: It is not recommended to scan the root directory directly. -103702103=Warning: It is recommended to add some conditions! -103702111=Danger: Do not redirect data to block devices! -103702112=Danger: Do not format the file system for block devices! -103702113=Danger: Do not format block devices! -103702114=Danger: Do not use 'dd' for block devices! -job.task.plan.debug=Debug - - -#消息通知部分 -#资源类型 -job.manage.resource.type.script=Script -job.manage.resource.type.file=File -job.manage.resource.type.job=Job -#触发方式 -job.manage.trigger.type.page.execute=Web UI -job.manage.trigger.type.api.invoke=API -job.manage.trigger.type.timer.task=Cron -#执行状态 -job.manage.execute.status.success=Success -job.manage.execute.status.fail=Failed -job.manage.execute.status.ready=Waitting -#角色 -job.manage.role.resource.owner=Owner -job.manage.role.resource.trigger.user=Operater -job.manage.role.extra.observer=Additional -#通知渠道 -job.manage.notify.channel.sms=SMS -job.manage.notify.channel.email=Mail -job.manage.notify.channel.wechat=WeChat -#通知模板名称 -job.manage.notify.template.name.confirmation=Confirm -job.manage.notify.template.name.executeSuccess=Success -job.manage.notify.template.name.executeFailure=Fail -job.manage.notify.template.name.beforeCronJobExecute=Before Cron Start -job.manage.notify.template.name.beforeCronJobEnd=Before Cron End -job.manage.notify.template.name.cronJobFailed=Cron Unlaunched - -# IP白名单部分 -job.manage.whiteip.actionScope.scriptExecute=Script Execution -job.manage.whiteip.actionScope.fileDistribution=File Transfer - -# 全局设置部分 -job.manage.globalsettings.defaultNameRules.description.linux=Start with lowercase or underscore, followed by lowercase / numbers / _ or -, up to 32 characters -job.manage.globalsettings.defaultNameRules.description.windows=Composed of 1 to 32 uppercase and lowercase or hyphen symbol -job.manage.globalsettings.defaultNameRules.description.database=Composed of 1 to 16 uppercase / lowercase / numbers / . / - / _ -job.manage.globalsettings.defaultTitleHead=Job | Tencent BlueKing -job.manage.globalsettings.contactBKHelper=Contact BK-Helper -job.manage.globalsettings.BKDesktop=BlueKing Desktop -job.manage.globalsettings.ee.footerLink=[Support](https://wpa1.qq.com/KziXGWJs?_type=wpa&qidian=true) | [Forum](https://bk.tencent.com/s-mart/community) | [Official](https://bk.tencent.com/index/) -job.manage.globalsettings.ee.footerCopyRight=Copyright © 2012-{{current_year}} Tencent BlueKing. All Rights Reserved. {{current_ver}} - -job.custom.settings.script.template.variable.datetime.desc=Current date and time -job.custom.settings.script.template.variable.username.desc=Username -job.custom.settings.script.template.variable.display_name.desc=User full name -job.custom.settings.script.template.variable.biz_id.desc=Business(set) -job.custom.settings.script.template.variable.biz_name.desc=Business(set) Name - -# CMDB相关资源 -cmdb.object.name.biz=Business diff --git a/src/backend/job-manage/boot-job-manage/src/main/resources/i18n/message_zh.properties b/src/backend/job-manage/boot-job-manage/src/main/resources/i18n/message_zh.properties deleted file mode 100644 index ddcc596a9f..0000000000 --- a/src/backend/job-manage/boot-job-manage/src/main/resources/i18n/message_zh.properties +++ /dev/null @@ -1,98 +0,0 @@ -# -# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. -# -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. -# -# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. -# -# License for BK-JOB蓝鲸智云作业平台: -# -------------------------------------------------------------------- -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF -# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - -job.resource.status.online=已上线 -job.resource.status.draft=未上线 -job.resource.status.offline=已下线 -job.resource.status.disabled=禁用 -job.account.category.system=系统账号 -job.account.category.db=数据库账号 - - -#script check code -103701001=脚本首行没有定义合法的脚本类型，例如： #!/bin/bash -103701002=请用'cd ... || exit' 或者 'cd ... || return' 处理目录不存在的逻辑 -103702001=危险：出现递归逻辑fork炸弹, 禁止在函数中调用自己 -103702002=危险：删除高危目录，可能会影响到系统的稳定! -103702101=禁止不加条件扫描根目录! -103702102=警告:有条件扫描根目录，建议指定具体目录，不建议直接扫描根目录 -103702103=警告:建议加上扫描条件! -103702111=危险:不要重定向数据到块设备! -103702112=危险:不要对块设备进行格式化文件系统! -103702113=危险:不要对块设备进行格式化! -103702114=危险:不要用dd命令进行块设备操作! -job.task.plan.debug=调试 - - -#消息通知部分 -#资源类型 -job.manage.resource.type.script=快速执行脚本 -job.manage.resource.type.file=快速分发文件 -job.manage.resource.type.job=执行方案 -#触发方式 -job.manage.trigger.type.page.execute=页面执行 -job.manage.trigger.type.api.invoke=API调用 -job.manage.trigger.type.timer.task=定时任务 -#执行状态 -job.manage.execute.status.success=执行成功 -job.manage.execute.status.fail=执行失败 -job.manage.execute.status.ready=等待执行 -#角色 -job.manage.role.resource.owner=资源所属者 -job.manage.role.resource.trigger.user=任务执行人 -job.manage.role.extra.observer=额外通知人 -#通知渠道 -job.manage.notify.channel.sms=短信 -job.manage.notify.channel.email=邮件 -job.manage.notify.channel.wechat=微信 -#通知模板名称 -job.manage.notify.template.name.confirmation=人工确认 -job.manage.notify.template.name.executeSuccess=执行成功 -job.manage.notify.template.name.executeFailure=执行失败 -job.manage.notify.template.name.beforeCronJobExecute=定时执行前 -job.manage.notify.template.name.beforeCronJobEnd=定时结束前 -job.manage.notify.template.name.cronJobFailed=定时启动失败 - -# IP白名单部分 -job.manage.whiteip.actionScope.scriptExecute=脚本执行 -job.manage.whiteip.actionScope.fileDistribution=文件分发 - -# 全局设置部分 -job.manage.globalsettings.defaultNameRules.description.linux=由小写字母或下划线开头，和 2~32个字母 / 数字 / _ 或 - 组成的字符 -job.manage.globalsettings.defaultNameRules.description.windows=由1~32个大小写字母或横杠组成的字符 -job.manage.globalsettings.defaultNameRules.description.database=由1~16个大小写 / 数字 / . / - / _ 组成的字符 -job.manage.globalsettings.defaultTitleHead=作业平台 | 腾讯蓝鲸智云 -job.manage.globalsettings.contactBKHelper=联系BK助手 -job.manage.globalsettings.BKDesktop=蓝鲸桌面 -job.manage.globalsettings.ee.footerLink=[技术支持](https://wpa1.qq.com/KziXGWJs?_type=wpa&qidian=true) | [社区论坛](https://bk.tencent.com/s-mart/community) | [产品官网](https://bk.tencent.com/index/) -job.manage.globalsettings.ee.footerCopyRight=Copyright © 2012-{{current_year}} Tencent BlueKing. All Rights Reserved. {{current_ver}} - -job.custom.settings.script.template.variable.datetime.desc=当前日期时间 -job.custom.settings.script.template.variable.username.desc=用户名 -job.custom.settings.script.template.variable.display_name.desc=全名 -job.custom.settings.script.template.variable.biz_id.desc=业务(集)ID -job.custom.settings.script.template.variable.biz_name.desc=业务(集)名称 - -# CMDB相关资源 -cmdb.object.name.biz=业务 diff --git a/src/backend/job-manage/boot-job-manage/src/main/resources/i18n/message_zh_CN.properties b/src/backend/job-manage/boot-job-manage/src/main/resources/i18n/message_zh_CN.properties deleted file mode 100644 index ddcc596a9f..0000000000 --- a/src/backend/job-manage/boot-job-manage/src/main/resources/i18n/message_zh_CN.properties +++ /dev/null @@ -1,98 +0,0 @@ -# -# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. -# -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. -# -# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. -# -# License for BK-JOB蓝鲸智云作业平台: -# -------------------------------------------------------------------- -# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated -# documentation files (the "Software"), to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and -# to permit persons to whom the Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included in all copies or substantial portions of -# the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO -# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF -# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -# IN THE SOFTWARE. -# - -job.resource.status.online=已上线 -job.resource.status.draft=未上线 -job.resource.status.offline=已下线 -job.resource.status.disabled=禁用 -job.account.category.system=系统账号 -job.account.category.db=数据库账号 - - -#script check code -103701001=脚本首行没有定义合法的脚本类型，例如： #!/bin/bash -103701002=请用'cd ... || exit' 或者 'cd ... || return' 处理目录不存在的逻辑 -103702001=危险：出现递归逻辑fork炸弹, 禁止在函数中调用自己 -103702002=危险：删除高危目录，可能会影响到系统的稳定! -103702101=禁止不加条件扫描根目录! -103702102=警告:有条件扫描根目录，建议指定具体目录，不建议直接扫描根目录 -103702103=警告:建议加上扫描条件! -103702111=危险:不要重定向数据到块设备! -103702112=危险:不要对块设备进行格式化文件系统! -103702113=危险:不要对块设备进行格式化! -103702114=危险:不要用dd命令进行块设备操作! -job.task.plan.debug=调试 - - -#消息通知部分 -#资源类型 -job.manage.resource.type.script=快速执行脚本 -job.manage.resource.type.file=快速分发文件 -job.manage.resource.type.job=执行方案 -#触发方式 -job.manage.trigger.type.page.execute=页面执行 -job.manage.trigger.type.api.invoke=API调用 -job.manage.trigger.type.timer.task=定时任务 -#执行状态 -job.manage.execute.status.success=执行成功 -job.manage.execute.status.fail=执行失败 -job.manage.execute.status.ready=等待执行 -#角色 -job.manage.role.resource.owner=资源所属者 -job.manage.role.resource.trigger.user=任务执行人 -job.manage.role.extra.observer=额外通知人 -#通知渠道 -job.manage.notify.channel.sms=短信 -job.manage.notify.channel.email=邮件 -job.manage.notify.channel.wechat=微信 -#通知模板名称 -job.manage.notify.template.name.confirmation=人工确认 -job.manage.notify.template.name.executeSuccess=执行成功 -job.manage.notify.template.name.executeFailure=执行失败 -job.manage.notify.template.name.beforeCronJobExecute=定时执行前 -job.manage.notify.template.name.beforeCronJobEnd=定时结束前 -job.manage.notify.template.name.cronJobFailed=定时启动失败 - -# IP白名单部分 -job.manage.whiteip.actionScope.scriptExecute=脚本执行 -job.manage.whiteip.actionScope.fileDistribution=文件分发 - -# 全局设置部分 -job.manage.globalsettings.defaultNameRules.description.linux=由小写字母或下划线开头，和 2~32个字母 / 数字 / _ 或 - 组成的字符 -job.manage.globalsettings.defaultNameRules.description.windows=由1~32个大小写字母或横杠组成的字符 -job.manage.globalsettings.defaultNameRules.description.database=由1~16个大小写 / 数字 / . / - / _ 组成的字符 -job.manage.globalsettings.defaultTitleHead=作业平台 | 腾讯蓝鲸智云 -job.manage.globalsettings.contactBKHelper=联系BK助手 -job.manage.globalsettings.BKDesktop=蓝鲸桌面 -job.manage.globalsettings.ee.footerLink=[技术支持](https://wpa1.qq.com/KziXGWJs?_type=wpa&qidian=true) | [社区论坛](https://bk.tencent.com/s-mart/community) | [产品官网](https://bk.tencent.com/index/) -job.manage.globalsettings.ee.footerCopyRight=Copyright © 2012-{{current_year}} Tencent BlueKing. All Rights Reserved. {{current_ver}} - -job.custom.settings.script.template.variable.datetime.desc=当前日期时间 -job.custom.settings.script.template.variable.username.desc=用户名 -job.custom.settings.script.template.variable.display_name.desc=全名 -job.custom.settings.script.template.variable.biz_id.desc=业务(集)ID -job.custom.settings.script.template.variable.biz_name.desc=业务(集)名称 - -# CMDB相关资源 -cmdb.object.name.biz=业务 diff --git a/src/backend/job-manage/boot-job-manage/src/main/resources/logback-spring.xml b/src/backend/job-manage/boot-job-manage/src/main/resources/logback-spring.xml new file mode 100644 index 0000000000..9f69e27f46 --- /dev/null +++ b/src/backend/job-manage/boot-job-manage/src/main/resources/logback-spring.xml @@ -0,0 +1,89 @@ + + + logback + + + + + + + + + + + + + ${SYNC_APP_HOST_LOG_FILE} + + ${SYNC_APP_HOST_LOG_FILE}-%d{yyyyMMdd_HH}.log.%i + 1GB + ${LOG_KEEP_HOURS} + 20GB + ${CLEAN_HISTORY_ON_START} + + + ${FILE_LOG_PATTERN} + UTF-8 + + + + + + ${NOTICE_LOG_FILE} + + ${NOTICE_LOG_FILE}-%d{yyyyMMdd}.log.%i + 1GB + ${LOG_KEEP_DAYS} + 2GB + ${CLEAN_HISTORY_ON_START} + + + ${FILE_LOG_PATTERN} + UTF-8 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/backend/job-manage/boot-job-manage/src/main/resources/logback/logback-app-props.xml b/src/backend/job-manage/boot-job-manage/src/main/resources/logback/logback-app-props.xml deleted file mode 100644 index fbd1f1b92e..0000000000 --- a/src/backend/job-manage/boot-job-manage/src/main/resources/logback/logback-app-props.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - diff --git a/src/backend/job-manage/boot-job-manage/src/main/resources/logback/logback-app.xml b/src/backend/job-manage/boot-job-manage/src/main/resources/logback/logback-app.xml deleted file mode 100644 index 419cc1aa9a..0000000000 --- a/src/backend/job-manage/boot-job-manage/src/main/resources/logback/logback-app.xml +++ /dev/null @@ -1,40 +0,0 @@ - - - - - ${SYNC_APP_HOST_LOG_FILE} - - ${SYNC_APP_HOST_LOG_FILE}-%d{yyyyMMdd_HH}.log.%i - 1GB - 120 - 20GB - - - ${FILE_LOG_PATTERN} - UTF-8 - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/AccountDAOImplIntegrationTest.java b/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/AccountDAOImplIntegrationTest.java index bdcac5f80e..e82325a836 100644 --- a/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/AccountDAOImplIntegrationTest.java +++ b/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/AccountDAOImplIntegrationTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,10 +24,10 @@ package com.tencent.bk.job.manage.dao; +import com.tencent.bk.job.common.constant.AccountCategoryEnum; import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; -import com.tencent.bk.job.manage.common.consts.account.AccountCategoryEnum; -import com.tencent.bk.job.manage.common.consts.account.AccountTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.account.AccountTypeEnum; import com.tencent.bk.job.manage.model.dto.AccountDTO; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -297,7 +297,13 @@ public void testGetAccountByAlias() { @Test @DisplayName("测试获取业务下指定分类的账号") public void testListAllAppAccount() { - List accounts = accountDAO.listAllAppAccount(2L, AccountCategoryEnum.SYSTEM, null); + List accounts = accountDAO.listAppAccount( + 2L, + AccountCategoryEnum.SYSTEM, + null, + null, + null + ); assertThat(accounts).hasSize(3).extracting("id").containsOnly(1L, 2L, 4L); } @@ -319,6 +325,7 @@ void testBatchUpdateDbAccountPassword() { AccountDTO account1 = new AccountDTO(); account1.setId(3L); account1.setDbPassword("ax798sdfs"); + account1.setCategory(AccountCategoryEnum.DB); updateAccounts.add(account1); accountDAO.batchUpdateDbAccountPassword(updateAccounts); diff --git a/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/ApplicationDAOImplIntegrationTest.java b/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/ApplicationDAOImplIntegrationTest.java index a8d33fcbfd..6040b2d35e 100644 --- a/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/ApplicationDAOImplIntegrationTest.java +++ b/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/ApplicationDAOImplIntegrationTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/ResourceTagDAOIntegrationTest.java b/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/ResourceTagDAOIntegrationTest.java index eb37df5f84..4c3a2a3003 100644 --- a/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/ResourceTagDAOIntegrationTest.java +++ b/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/ResourceTagDAOIntegrationTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -61,7 +61,8 @@ public void listResourceTagsByResourceIds() { List scriptResourceTags = resourceTagDAO.listResourceTags(JobResourceTypeEnum.APP_SCRIPT.getValue(), scriptResourceIdList); assertThat(scriptResourceTags).hasSize(2); - assertThat(scriptResourceTags).extracting("resourceType").containsOnly(JobResourceTypeEnum.APP_SCRIPT.getValue()); + assertThat(scriptResourceTags).extracting("resourceType") + .containsOnly(JobResourceTypeEnum.APP_SCRIPT.getValue()); assertThat(scriptResourceTags).extracting("resourceId").containsOnly("1"); assertThat(scriptResourceTags).extracting("tagId").containsOnly(1L, 2L); @@ -71,7 +72,8 @@ public void listResourceTagsByResourceIds() { List templateResourceTags = resourceTagDAO.listResourceTags(JobResourceTypeEnum.TEMPLATE.getValue(), templateResourceIdList); assertThat(templateResourceTags).hasSize(3); - assertThat(templateResourceTags).extracting("resourceType").containsOnly(JobResourceTypeEnum.TEMPLATE.getValue()); + assertThat(templateResourceTags).extracting("resourceType") + .containsOnly(JobResourceTypeEnum.TEMPLATE.getValue()); assertThat(templateResourceTags).extracting("resourceId").containsOnly("1", "2"); assertThat(templateResourceTags).extracting("tagId").containsOnly(1L, 2L); } @@ -81,7 +83,8 @@ public void listResourceTagsByResourceId() { List scriptResourceTags = resourceTagDAO.listResourceTags(JobResourceTypeEnum.APP_SCRIPT.getValue(), String.valueOf(1)); assertThat(scriptResourceTags).hasSize(2); - assertThat(scriptResourceTags).extracting("resourceType").containsOnly(JobResourceTypeEnum.APP_SCRIPT.getValue()); + assertThat(scriptResourceTags).extracting("resourceType") + .containsOnly(JobResourceTypeEnum.APP_SCRIPT.getValue()); assertThat(scriptResourceTags).extracting("resourceId").containsOnly("1"); assertThat(scriptResourceTags).extracting("tagId").containsOnly(1L, 2L); } @@ -90,16 +93,20 @@ public void listResourceTagsByResourceId() { public void listResourceTagsByTagId() { List resourceTags = resourceTagDAO.listResourceTags(1L); assertThat(resourceTags).hasSize(4); - assertThat(resourceTags).extracting("resourceType").containsOnly(JobResourceTypeEnum.APP_SCRIPT.getValue(), JobResourceTypeEnum.TEMPLATE.getValue()); + assertThat(resourceTags).extracting("resourceType") + .containsOnly(JobResourceTypeEnum.APP_SCRIPT.getValue(), + JobResourceTypeEnum.TEMPLATE.getValue()); assertThat(resourceTags).extracting("resourceId").containsOnly("1", "2"); assertThat(resourceTags).extracting("tagId").containsOnly(1L); } @Test public void listResourceTagsByTagIdAndResourceType() { - List resourceTags = resourceTagDAO.listResourceTags(1L, JobResourceTypeEnum.APP_SCRIPT.getValue()); + List resourceTags = resourceTagDAO.listResourceTags(1L, + JobResourceTypeEnum.APP_SCRIPT.getValue()); assertThat(resourceTags).hasSize(2); - assertThat(resourceTags).extracting("resourceType").containsOnly(JobResourceTypeEnum.APP_SCRIPT.getValue()); + assertThat(resourceTags).extracting("resourceType") + .containsOnly(JobResourceTypeEnum.APP_SCRIPT.getValue()); assertThat(resourceTags).extracting("resourceId").containsOnly("1", "2"); assertThat(resourceTags).extracting("tagId").containsOnly(1L); } @@ -108,7 +115,9 @@ public void listResourceTagsByTagIdAndResourceType() { public void listResourceTagsByTagIdS() { List resourceTags = resourceTagDAO.listResourceTags(Arrays.asList(1L, 2L)); assertThat(resourceTags).hasSize(6); - assertThat(resourceTags).extracting("resourceType").containsOnly(JobResourceTypeEnum.APP_SCRIPT.getValue(), JobResourceTypeEnum.TEMPLATE.getValue()); + assertThat(resourceTags).extracting("resourceType") + .containsOnly(JobResourceTypeEnum.APP_SCRIPT.getValue(), + JobResourceTypeEnum.TEMPLATE.getValue()); assertThat(resourceTags).extracting("resourceId").containsOnly("1", "2"); assertThat(resourceTags).extracting("tagId").containsOnly(1L, 2L); } @@ -116,12 +125,18 @@ public void listResourceTagsByTagIdS() { @Test public void batchSaveResourceTags() { List resourceTags = new ArrayList<>(); - ResourceTagDTO resourceTag1 = new ResourceTagDTO(JobResourceTypeEnum.APP_SCRIPT.getValue(), "1000", 1L); - ResourceTagDTO resourceTag2 = new ResourceTagDTO(JobResourceTypeEnum.APP_SCRIPT.getValue(), "1000", 2L); - ResourceTagDTO resourceTag3 = new ResourceTagDTO(JobResourceTypeEnum.APP_SCRIPT.getValue(), "1001", 1L); - ResourceTagDTO resourceTag4 = new ResourceTagDTO(JobResourceTypeEnum.TEMPLATE.getValue(), "1000", 1L); - ResourceTagDTO resourceTag5 = new ResourceTagDTO(JobResourceTypeEnum.TEMPLATE.getValue(), "1000", 2L); - ResourceTagDTO resourceTag6 = new ResourceTagDTO(JobResourceTypeEnum.TEMPLATE.getValue(), "1001", 1L); + ResourceTagDTO resourceTag1 = new ResourceTagDTO( + JobResourceTypeEnum.APP_SCRIPT.getValue(), "1000", 1L); + ResourceTagDTO resourceTag2 = new ResourceTagDTO( + JobResourceTypeEnum.APP_SCRIPT.getValue(), "1000", 2L); + ResourceTagDTO resourceTag3 = new ResourceTagDTO( + JobResourceTypeEnum.APP_SCRIPT.getValue(), "1001", 1L); + ResourceTagDTO resourceTag4 = new ResourceTagDTO( + JobResourceTypeEnum.TEMPLATE.getValue(), "1000", 1L); + ResourceTagDTO resourceTag5 = new ResourceTagDTO( + JobResourceTypeEnum.TEMPLATE.getValue(), "1000", 2L); + ResourceTagDTO resourceTag6 = new ResourceTagDTO( + JobResourceTypeEnum.TEMPLATE.getValue(), "1001", 1L); resourceTags.add(resourceTag1); resourceTags.add(resourceTag2); resourceTags.add(resourceTag3); @@ -137,7 +152,8 @@ public void batchSaveResourceTags() { List scriptResourceTags = resourceTagDAO.listResourceTags(JobResourceTypeEnum.APP_SCRIPT.getValue(), scriptResourceIdList); assertThat(scriptResourceTags).hasSize(3); - assertThat(scriptResourceTags).extracting("resourceType").containsOnly(JobResourceTypeEnum.APP_SCRIPT.getValue()); + assertThat(scriptResourceTags).extracting("resourceType") + .containsOnly(JobResourceTypeEnum.APP_SCRIPT.getValue()); assertThat(scriptResourceTags).extracting("resourceId").containsOnly("1000", "1001"); assertThat(scriptResourceTags).extracting("tagId").containsOnly(1L, 2L); @@ -147,7 +163,8 @@ public void batchSaveResourceTags() { List templateResourceTags = resourceTagDAO.listResourceTags(JobResourceTypeEnum.TEMPLATE.getValue(), templateResourceIdList); assertThat(templateResourceTags).hasSize(3); - assertThat(templateResourceTags).extracting("resourceType").containsOnly(JobResourceTypeEnum.TEMPLATE.getValue()); + assertThat(templateResourceTags).extracting("resourceType") + .containsOnly(JobResourceTypeEnum.TEMPLATE.getValue()); assertThat(templateResourceTags).extracting("resourceId").containsOnly("1000", "1001"); assertThat(templateResourceTags).extracting("tagId").containsOnly(1L, 2L); @@ -156,8 +173,10 @@ public void batchSaveResourceTags() { @Test public void givenDuplicateKeyWhenBatchSaveResourceTagsThenSuccess() { List resourceTags = new ArrayList<>(); - ResourceTagDTO resourceTag1 = new ResourceTagDTO(JobResourceTypeEnum.APP_SCRIPT.getValue(), "1000", 1L); - ResourceTagDTO resourceTag2 = new ResourceTagDTO(JobResourceTypeEnum.APP_SCRIPT.getValue(), "1000", 1L); + ResourceTagDTO resourceTag1 = new ResourceTagDTO( + JobResourceTypeEnum.APP_SCRIPT.getValue(), "1000", 1L); + ResourceTagDTO resourceTag2 = new ResourceTagDTO( + JobResourceTypeEnum.APP_SCRIPT.getValue(), "1000", 1L); resourceTags.add(resourceTag1); resourceTags.add(resourceTag2); @@ -168,7 +187,8 @@ public void givenDuplicateKeyWhenBatchSaveResourceTagsThenSuccess() { List scriptResourceTags = resourceTagDAO.listResourceTags(JobResourceTypeEnum.APP_SCRIPT.getValue(), scriptResourceIdList); assertThat(scriptResourceTags).hasSize(1); - assertThat(scriptResourceTags).extracting("resourceType").containsOnly(JobResourceTypeEnum.APP_SCRIPT.getValue()); + assertThat(scriptResourceTags).extracting("resourceType") + .containsOnly(JobResourceTypeEnum.APP_SCRIPT.getValue()); assertThat(scriptResourceTags).extracting("resourceId").containsOnly("1000"); assertThat(scriptResourceTags).extracting("tagId").containsOnly(1L); } @@ -178,22 +198,26 @@ public void deleteResourceTagsByResourceId() { boolean result = resourceTagDAO.deleteResourceTags(JobResourceTypeEnum.APP_SCRIPT.getValue(), "1"); assertThat(result).isEqualTo(true); - List tags = resourceTagDAO.listResourceTags(JobResourceTypeEnum.APP_SCRIPT.getValue(), Collections.singletonList("1")); + List tags = resourceTagDAO.listResourceTags(JobResourceTypeEnum.APP_SCRIPT.getValue(), + Collections.singletonList("1")); assertThat(tags).hasSize(0); } @Test public void deleteResourceTagByResourceAndTagId() { - boolean result = resourceTagDAO.deleteResourceTag(JobResourceTypeEnum.APP_SCRIPT.getValue(), "1", 1L); + boolean result = resourceTagDAO.deleteResourceTag( + JobResourceTypeEnum.APP_SCRIPT.getValue(), "1", 1L); assertThat(result).isEqualTo(true); } @Test public void deleteResourceTagsByResourceAndTagIds() { - boolean result = resourceTagDAO.deleteResourceTags(JobResourceTypeEnum.APP_SCRIPT.getValue(), "1", Collections.singletonList(1L)); + boolean result = resourceTagDAO.deleteResourceTags(JobResourceTypeEnum.APP_SCRIPT.getValue(), "1", + Collections.singletonList(1L)); assertThat(result).isEqualTo(true); - List tags = resourceTagDAO.listResourceTags(JobResourceTypeEnum.APP_SCRIPT.getValue(), Collections.singletonList("1")); + List tags = resourceTagDAO.listResourceTags(JobResourceTypeEnum.APP_SCRIPT.getValue(), + Collections.singletonList("1")); assertThat(tags).hasSize(1); assertThat(tags.get(0).getTagId()).isEqualTo(2L); } diff --git a/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/ScriptDAOImplIntegrationTest.java b/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/ScriptDAOImplIntegrationTest.java index bb3dfa53da..7a56a417ad 100644 --- a/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/ScriptDAOImplIntegrationTest.java +++ b/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/ScriptDAOImplIntegrationTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,11 +27,11 @@ import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; import com.tencent.bk.job.common.util.JobUUID; -import com.tencent.bk.job.manage.common.consts.JobResourceStatusEnum; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.JobResourceStatusEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; import com.tencent.bk.job.manage.model.dto.ScriptDTO; import com.tencent.bk.job.manage.model.query.ScriptQuery; -import org.jooq.generated.tables.Script; +import com.tencent.bk.job.manage.model.tables.Script; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; @@ -242,18 +242,17 @@ public void whenSaveScriptThenStore() { } @Test - public void whenUpdateScriptThenStore() { - ScriptDTO script = new ScriptDTO(); - script.setLastModifyUser("user2"); - script.setId("dc65a20cd91811e993a2309c2357fc12"); - script.setName("new_name"); + public void updateScriptLastModify() { + String lastModifyUser = "user2"; + String scriptId = "dc65a20cd91811e993a2309c2357fc12"; + long time = System.currentTimeMillis(); - scriptDAO.updateScript(script); + scriptDAO.updateScriptLastModify(scriptId, lastModifyUser, time); - ScriptDTO updatedScript = getScriptById("dc65a20cd91811e993a2309c2357fc12"); + ScriptDTO updatedScript = getScriptById(scriptId); assertThat(updatedScript).isNotNull(); - assertThat(updatedScript.getLastModifyUser()).isEqualTo(script.getLastModifyUser()); - assertThat(updatedScript.getName()).isEqualTo("new_name"); + assertThat(updatedScript.getLastModifyUser()).isEqualTo(lastModifyUser); + assertThat(updatedScript.getLastModifyTime()).isEqualTo(time); } private ScriptDTO getScriptById(String scriptId) { @@ -452,8 +451,9 @@ void whenListPageScriptVersionThenReturnOrderedPageResult() { BaseSearchCondition baseSearchCondition = new BaseSearchCondition(); baseSearchCondition.setStart(0); baseSearchCondition.setLength(Integer.MAX_VALUE); + scriptCondition.setBaseSearchCondition(baseSearchCondition); - PageData pageData = scriptDAO.listPageScriptVersion(scriptCondition, baseSearchCondition); + PageData pageData = scriptDAO.listPageScriptVersion(scriptCondition); assertThat(pageData).isNotNull(); assertThat(pageData.getPageSize()).isEqualTo(Integer.MAX_VALUE); assertThat(pageData.getStart()).isEqualTo(0); diff --git a/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/ScriptRelateTaskPlanDAOImplIntegrationTest.java b/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/ScriptRelateTaskPlanDAOImplIntegrationTest.java index 1a2c447761..47e4c964d3 100644 --- a/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/ScriptRelateTaskPlanDAOImplIntegrationTest.java +++ b/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/ScriptRelateTaskPlanDAOImplIntegrationTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/ScriptRelateTemplateDAOImplIntegrationTest.java b/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/ScriptRelateTemplateDAOImplIntegrationTest.java index 49a20da32c..8637a2c781 100644 --- a/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/ScriptRelateTemplateDAOImplIntegrationTest.java +++ b/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/ScriptRelateTemplateDAOImplIntegrationTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/TagDAOIntegrationTest.java b/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/TagDAOIntegrationTest.java index 76cdcfeb7c..f29c68768b 100644 --- a/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/TagDAOIntegrationTest.java +++ b/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/TagDAOIntegrationTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/globalsetting/impl/DangerousRuleDAOImplIntegrationTest.java b/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/globalsetting/impl/DangerousRuleDAOImplIntegrationTest.java index cc313ad359..287d9e9a33 100644 --- a/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/globalsetting/impl/DangerousRuleDAOImplIntegrationTest.java +++ b/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/globalsetting/impl/DangerousRuleDAOImplIntegrationTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,7 +26,8 @@ import com.tencent.bk.job.manage.dao.globalsetting.DangerousRuleDAO; import com.tencent.bk.job.manage.model.dto.globalsetting.DangerousRuleDTO; -import org.jooq.DSLContext; +import com.tencent.bk.job.manage.model.query.DangerousRuleQuery; +import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; import org.springframework.beans.factory.annotation.Autowired; @@ -37,6 +38,9 @@ import org.springframework.test.context.jdbc.SqlConfig; import org.springframework.test.context.junit.jupiter.SpringExtension; +import java.util.Arrays; +import java.util.List; + import static org.assertj.core.api.Assertions.assertThat; /** @@ -53,27 +57,98 @@ class DangerousRuleDAOImplIntegrationTest { @Autowired private DangerousRuleDAO dangerousRuleDAO; - @Autowired - private DSLContext dslContext; - @Test void listDangerousRulesByScriptType() { DangerousRuleDTO dangerousRule = new DangerousRuleDTO(); dangerousRule.setScriptType(1); - assertThat(dangerousRuleDAO.listDangerousRules(dslContext, dangerousRule)).hasSize(5); + assertThat(dangerousRuleDAO.listDangerousRules(dangerousRule)).hasSize(8); dangerousRule.setScriptType(2); - assertThat(dangerousRuleDAO.listDangerousRules(dslContext, dangerousRule)).hasSize(4); + assertThat(dangerousRuleDAO.listDangerousRules(dangerousRule)).hasSize(5); dangerousRule.setScriptType(3); - assertThat(dangerousRuleDAO.listDangerousRules(dslContext, dangerousRule)).hasSize(3); + assertThat(dangerousRuleDAO.listDangerousRules(dangerousRule)).hasSize(4); dangerousRule.setScriptType(4); - assertThat(dangerousRuleDAO.listDangerousRules(dslContext, dangerousRule)).hasSize(2); + assertThat(dangerousRuleDAO.listDangerousRules(dangerousRule)).hasSize(5); dangerousRule.setScriptType(5); - assertThat(dangerousRuleDAO.listDangerousRules(dslContext, dangerousRule)).hasSize(1); + assertThat(dangerousRuleDAO.listDangerousRules(dangerousRule)).hasSize(2); dangerousRule.setScriptType(6); - assertThat(dangerousRuleDAO.listDangerousRules(dslContext, dangerousRule)).hasSize(0); + assertThat(dangerousRuleDAO.listDangerousRules(dangerousRule)).hasSize(3); dangerousRule.setScriptType(7); - assertThat(dangerousRuleDAO.listDangerousRules(dslContext, dangerousRule)).hasSize(0); + assertThat(dangerousRuleDAO.listDangerousRules(dangerousRule)).hasSize(0); dangerousRule.setScriptType(8); - assertThat(dangerousRuleDAO.listDangerousRules(dslContext, dangerousRule)).hasSize(0); + assertThat(dangerousRuleDAO.listDangerousRules(dangerousRule)).hasSize(0); + } + + @Test + @DisplayName("测试高危语句规则条件查询") + void listDangerousRulesByQuery() { + // 按sriptType模糊查询，略，因H2数据库不支持msyql原生的位运算 + + // 语法检测表达式模糊查询 + DangerousRuleQuery query = DangerousRuleQuery + .builder() + .expression("sql") + .build(); + List dangerousRuleDTOS = dangerousRuleDAO.listDangerousRules(query); + assertThat(dangerousRuleDTOS).hasSize(3); + query.setExpression("shell"); + dangerousRuleDTOS = dangerousRuleDAO.listDangerousRules(query); + assertThat(dangerousRuleDTOS).hasSize(8); + query.setExpression("per"); + dangerousRuleDTOS = dangerousRuleDAO.listDangerousRules(query); + assertThat(dangerousRuleDTOS).hasSize(4); + query.setExpression("l"); + dangerousRuleDTOS = dangerousRuleDAO.listDangerousRules(query); + assertThat(dangerousRuleDTOS).hasSize(9); + query.setExpression("python"); + dangerousRuleDTOS = dangerousRuleDAO.listDangerousRules(query); + assertThat(dangerousRuleDTOS).hasSize(5); + query.setExpression("java"); + dangerousRuleDTOS = dangerousRuleDAO.listDangerousRules(query); + assertThat(dangerousRuleDTOS).hasSize(0); + + // 规则说明模式查询 + query.setExpression(null); + query.setDescription("Bat"); + dangerousRuleDTOS = dangerousRuleDAO.listDangerousRules(query); + assertThat(dangerousRuleDTOS).hasSize(5); + query.setDescription("test"); + dangerousRuleDTOS = dangerousRuleDAO.listDangerousRules(query); + assertThat(dangerousRuleDTOS).hasSize(0); + query.setDescription("pow"); + dangerousRuleDTOS = dangerousRuleDAO.listDangerousRules(query); + assertThat(dangerousRuleDTOS).hasSize(0); + query.setDescription(",SQL-6"); + dangerousRuleDTOS = dangerousRuleDAO.listDangerousRules(query); + assertThat(dangerousRuleDTOS).hasSize(2); + query.setDescription("Shell-1,Python-4"); + dangerousRuleDTOS = dangerousRuleDAO.listDangerousRules(query); + assertThat(dangerousRuleDTOS).hasSize(1); + + // 拦截动作查询 + query.setDescription(null); + query.setAction(Arrays.asList(new Byte[]{1})); + dangerousRuleDTOS = dangerousRuleDAO.listDangerousRules(query); + assertThat(dangerousRuleDTOS).hasSize(7); + query.setAction(Arrays.asList(new Byte[]{0})); + dangerousRuleDTOS = dangerousRuleDAO.listDangerousRules(query); + assertThat(dangerousRuleDTOS).hasSize(3); + query.setAction(Arrays.asList(new Byte[]{0, 1})); + dangerousRuleDTOS = dangerousRuleDAO.listDangerousRules(query); + assertThat(dangerousRuleDTOS).hasSize(10); + + // 多添加查询 + query.setAction(Arrays.asList(new Byte[]{0})); + query.setExpression("shell"); + dangerousRuleDTOS = dangerousRuleDAO.listDangerousRules(query); + assertThat(dangerousRuleDTOS).hasSize(2); + query.setAction(Arrays.asList(new Byte[]{0})); + query.setExpression("sql"); + dangerousRuleDTOS = dangerousRuleDAO.listDangerousRules(query); + assertThat(dangerousRuleDTOS).hasSize(0); + query.setAction(Arrays.asList(new Byte[]{1})); + query.setExpression("sql"); + query.setDescription("SQL"); + dangerousRuleDTOS = dangerousRuleDAO.listDangerousRules(query); + assertThat(dangerousRuleDTOS).hasSize(3); } } diff --git a/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateApprovalStepDAOImplIntegrationTest.java b/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateApprovalStepDAOImplIntegrationTest.java index e878f38037..aa1b485f31 100644 --- a/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateApprovalStepDAOImplIntegrationTest.java +++ b/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateApprovalStepDAOImplIntegrationTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,7 +25,7 @@ package com.tencent.bk.job.manage.dao.template.impl; import com.tencent.bk.job.common.model.dto.UserRoleInfoDTO; -import com.tencent.bk.job.manage.common.consts.task.TaskApprovalTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskApprovalTypeEnum; import com.tencent.bk.job.manage.dao.TaskApprovalStepDAO; import com.tencent.bk.job.manage.model.dto.task.TaskApprovalStepDTO; import org.junit.jupiter.api.BeforeEach; diff --git a/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateDAOImplIntegrationTest.java b/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateDAOImplIntegrationTest.java index fd0b28687c..b5748eb99a 100644 --- a/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateDAOImplIntegrationTest.java +++ b/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateDAOImplIntegrationTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,7 +25,7 @@ package com.tencent.bk.job.manage.dao.template.impl; import com.tencent.bk.job.common.util.date.DateUtils; -import com.tencent.bk.job.manage.common.consts.task.TaskTemplateStatusEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskTemplateStatusEnum; import com.tencent.bk.job.manage.dao.template.TaskTemplateDAO; import com.tencent.bk.job.manage.model.dto.TagDTO; import com.tencent.bk.job.manage.model.dto.task.TaskTemplateInfoDTO; @@ -81,10 +81,12 @@ void initTest() { TEMPLATE_INFO_1.setDescription("这是一个测试模版。这段描述是随便写的，没什么意义。1"); TEMPLATE_INFO_1.setCreator("userC"); TEMPLATE_INFO_1.setStatus(TaskTemplateStatusEnum.NEW); - TEMPLATE_INFO_1.setCreateTime(LocalDateTime.of(2019, 10, 1, 0, 0, 0, 0).toEpochSecond(ZoneOffset.ofHours(8))); + TEMPLATE_INFO_1.setCreateTime(LocalDateTime.of(2019, 10, 1, 0, 0, 0, 0) + .toEpochSecond(ZoneOffset.ofHours(8))); TEMPLATE_INFO_1.setLastModifyUser("userC"); TEMPLATE_INFO_1 - .setLastModifyTime(LocalDateTime.of(2019, 10, 1, 0, 0, 0, 0).toEpochSecond(ZoneOffset.ofHours(8))); + .setLastModifyTime(LocalDateTime.of(2019, 10, 1, 0, 0, 0, 0) + .toEpochSecond(ZoneOffset.ofHours(8))); TEMPLATE_INFO_1.setTags(Stream.of(1L, 2L, 3L).map(tagId -> { TagDTO tagInfo = new TagDTO(); tagInfo.setId(tagId); @@ -101,10 +103,12 @@ void initTest() { TEMPLATE_INFO_2.setDescription("这是一个测试模版。这段描述是随便写的，没什么意义。2"); TEMPLATE_INFO_2.setCreator("userC"); TEMPLATE_INFO_2.setStatus(TaskTemplateStatusEnum.IN_REVIEW); - TEMPLATE_INFO_2.setCreateTime(LocalDateTime.of(2019, 10, 1, 0, 0, 0, 0).toEpochSecond(ZoneOffset.ofHours(8))); + TEMPLATE_INFO_2.setCreateTime(LocalDateTime.of(2019, 10, 1, 0, 0, 0, 0) + .toEpochSecond(ZoneOffset.ofHours(8))); TEMPLATE_INFO_2.setLastModifyUser("userT"); TEMPLATE_INFO_2 - .setLastModifyTime(LocalDateTime.of(2019, 10, 2, 0, 0, 0, 0).toEpochSecond(ZoneOffset.ofHours(8))); + .setLastModifyTime(LocalDateTime.of(2019, 10, 2, 0, 0, 0, 0) + .toEpochSecond(ZoneOffset.ofHours(8))); TEMPLATE_INFO_2.setTags(Stream.of(2L, 3L, 4L).map(tagId -> { TagDTO tagInfo = new TagDTO(); tagInfo.setId(tagId); @@ -121,10 +125,12 @@ void initTest() { TEMPLATE_INFO_3.setDescription("这是一个测试模版。这段描述是随便写的，没什么意义。3"); TEMPLATE_INFO_3.setCreator("userC"); TEMPLATE_INFO_3.setStatus(TaskTemplateStatusEnum.PUBLISHED); - TEMPLATE_INFO_3.setCreateTime(LocalDateTime.of(2019, 10, 1, 0, 0, 0, 0).toEpochSecond(ZoneOffset.ofHours(8))); + TEMPLATE_INFO_3.setCreateTime(LocalDateTime.of(2019, 10, 1, 0, 0, 0, 0) + .toEpochSecond(ZoneOffset.ofHours(8))); TEMPLATE_INFO_3.setLastModifyUser("userC"); TEMPLATE_INFO_3 - .setLastModifyTime(LocalDateTime.of(2019, 10, 8, 0, 0, 0, 0).toEpochSecond(ZoneOffset.ofHours(8))); + .setLastModifyTime(LocalDateTime.of(2019, 10, 8, 0, 0, 0, 0) + .toEpochSecond(ZoneOffset.ofHours(8))); TEMPLATE_INFO_3.setTags(Stream.of(3L, 4L, 5L).map(tagId -> { TagDTO tagInfo = new TagDTO(); tagInfo.setId(tagId); @@ -141,10 +147,12 @@ void initTest() { TEMPLATE_INFO_4.setDescription("这是一个测试模版。这段描述是随便写的，没什么意义。4"); TEMPLATE_INFO_4.setCreator("userT"); TEMPLATE_INFO_4.setStatus(TaskTemplateStatusEnum.REJECTED); - TEMPLATE_INFO_4.setCreateTime(LocalDateTime.of(2019, 10, 1, 0, 0, 0, 0).toEpochSecond(ZoneOffset.ofHours(8))); + TEMPLATE_INFO_4.setCreateTime(LocalDateTime.of(2019, 10, 1, 0, 0, 0, 0) + .toEpochSecond(ZoneOffset.ofHours(8))); TEMPLATE_INFO_4.setLastModifyUser("userT"); TEMPLATE_INFO_4 - .setLastModifyTime(LocalDateTime.of(2019, 10, 4, 0, 0, 0, 0).toEpochSecond(ZoneOffset.ofHours(8))); + .setLastModifyTime(LocalDateTime.of(2019, 10, 4, 0, 0, 0, 0) + .toEpochSecond(ZoneOffset.ofHours(8))); TEMPLATE_INFO_4.setTags(Stream.of(1L, 3L, 5L).map(tagId -> { TagDTO tagInfo = new TagDTO(); tagInfo.setId(tagId); @@ -161,10 +169,12 @@ void initTest() { TEMPLATE_INFO_5.setDescription("这是一个测试模版。这段描述是随便写的，没什么意义。5"); TEMPLATE_INFO_5.setCreator("userT"); TEMPLATE_INFO_5.setStatus(TaskTemplateStatusEnum.PUBLISHED); - TEMPLATE_INFO_5.setCreateTime(LocalDateTime.of(2019, 10, 1, 0, 0, 0, 0).toEpochSecond(ZoneOffset.ofHours(8))); + TEMPLATE_INFO_5.setCreateTime(LocalDateTime.of(2019, 10, 1, 0, 0, 0, 0) + .toEpochSecond(ZoneOffset.ofHours(8))); TEMPLATE_INFO_5.setLastModifyUser("userC"); TEMPLATE_INFO_5 - .setLastModifyTime(LocalDateTime.of(2019, 10, 5, 0, 0, 0, 0).toEpochSecond(ZoneOffset.ofHours(8))); + .setLastModifyTime(LocalDateTime.of(2019, 10, 5, 0, 0, 0, 0) + .toEpochSecond(ZoneOffset.ofHours(8))); TEMPLATE_INFO_5.setTags(Stream.of(2L, 4L, 6L).map(tagId -> { TagDTO tagInfo = new TagDTO(); tagInfo.setId(tagId); @@ -181,10 +191,12 @@ void initTest() { TEMPLATE_INFO_6.setDescription("这是一个测试模版。这段描述是随便写的，没什么意义。6"); TEMPLATE_INFO_6.setCreator("userT"); TEMPLATE_INFO_6.setStatus(TaskTemplateStatusEnum.IN_REVIEW); - TEMPLATE_INFO_6.setCreateTime(LocalDateTime.of(2019, 10, 1, 0, 0, 0, 0).toEpochSecond(ZoneOffset.ofHours(8))); + TEMPLATE_INFO_6.setCreateTime(LocalDateTime.of(2019, 10, 1, 0, 0, 0, 0) + .toEpochSecond(ZoneOffset.ofHours(8))); TEMPLATE_INFO_6.setLastModifyUser("userT"); TEMPLATE_INFO_6 - .setLastModifyTime(LocalDateTime.of(2019, 10, 3, 1, 0, 0, 0).toEpochSecond(ZoneOffset.ofHours(8))); + .setLastModifyTime(LocalDateTime.of(2019, 10, 3, 1, 0, 0, 0) + .toEpochSecond(ZoneOffset.ofHours(8))); TEMPLATE_INFO_6.setTags(Stream.of(1L, 4L, 7L).map(tagId -> { TagDTO tagInfo = new TagDTO(); tagInfo.setId(tagId); @@ -201,10 +213,12 @@ void initTest() { TEMPLATE_INFO_7.setDescription("这是一个测试模版。这段描述是随便写的，没什么意义。7"); TEMPLATE_INFO_7.setCreator("userT"); TEMPLATE_INFO_7.setStatus(TaskTemplateStatusEnum.NEW); - TEMPLATE_INFO_7.setCreateTime(LocalDateTime.of(2019, 10, 1, 0, 0, 0, 0).toEpochSecond(ZoneOffset.ofHours(8))); + TEMPLATE_INFO_7.setCreateTime(LocalDateTime.of(2019, 10, 1, 0, 0, 0, 0) + .toEpochSecond(ZoneOffset.ofHours(8))); TEMPLATE_INFO_7.setLastModifyUser("userC"); TEMPLATE_INFO_7 - .setLastModifyTime(LocalDateTime.of(2019, 10, 2, 1, 0, 0, 0).toEpochSecond(ZoneOffset.ofHours(8))); + .setLastModifyTime(LocalDateTime.of(2019, 10, 2, 1, 0, 0, 0) + .toEpochSecond(ZoneOffset.ofHours(8))); TEMPLATE_INFO_7.setTags(Stream.of(2L, 5L, 8L).map(tagId -> { TagDTO tagInfo = new TagDTO(); tagInfo.setId(tagId); @@ -221,10 +235,12 @@ void initTest() { TEMPLATE_INFO_8.setDescription("这是一个测试模版。这段描述是随便写的，没什么意义。8"); TEMPLATE_INFO_8.setCreator("userC"); TEMPLATE_INFO_8.setStatus(TaskTemplateStatusEnum.PUBLISHED); - TEMPLATE_INFO_8.setCreateTime(LocalDateTime.of(2019, 10, 1, 0, 0, 0, 0).toEpochSecond(ZoneOffset.ofHours(8))); + TEMPLATE_INFO_8.setCreateTime(LocalDateTime.of(2019, 10, 1, 0, 0, 0, 0) + .toEpochSecond(ZoneOffset.ofHours(8))); TEMPLATE_INFO_8.setLastModifyUser("userT"); TEMPLATE_INFO_8 - .setLastModifyTime(LocalDateTime.of(2019, 10, 6, 0, 0, 0, 0).toEpochSecond(ZoneOffset.ofHours(8))); + .setLastModifyTime(LocalDateTime.of(2019, 10, 6, 0, 0, 0, 0) + .toEpochSecond(ZoneOffset.ofHours(8))); TEMPLATE_INFO_8.setTags(Stream.of(3L, 6L, 9L).map(tagId -> { TagDTO tagInfo = new TagDTO(); tagInfo.setId(tagId); @@ -238,7 +254,8 @@ void initTest() { @Test void giveNormalTemplateIdReturnTemplateInfo() { - TaskTemplateInfoDTO template = taskTemplateDAO.getTaskTemplateById(TEMPLATE_INFO_1.getAppId(), TEMPLATE_INFO_1.getId()); + TaskTemplateInfoDTO template = taskTemplateDAO.getTaskTemplateById( + TEMPLATE_INFO_1.getAppId(), TEMPLATE_INFO_1.getId()); assertThat(template).isNotNull(); assertThat(template.getId()).isEqualTo(TEMPLATE_INFO_1.getId()); assertThat(template.getAppId()).isEqualTo(TEMPLATE_INFO_1.getAppId()); diff --git a/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateFavoriteDAOImplIntegrationTest.java b/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateFavoriteDAOImplIntegrationTest.java index 37e4154b06..9c4c34aa45 100644 --- a/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateFavoriteDAOImplIntegrationTest.java +++ b/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateFavoriteDAOImplIntegrationTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateFileInfoDAOImplIntegrationTest.java b/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateFileInfoDAOImplIntegrationTest.java index bac3704c9d..9a58025919 100644 --- a/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateFileInfoDAOImplIntegrationTest.java +++ b/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateFileInfoDAOImplIntegrationTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,7 +25,7 @@ package com.tencent.bk.job.manage.dao.template.impl; import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; -import com.tencent.bk.job.manage.common.consts.task.TaskFileTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskFileTypeEnum; import com.tencent.bk.job.manage.dao.TaskFileInfoDAO; import com.tencent.bk.job.manage.model.dto.task.TaskFileInfoDTO; import com.tencent.bk.job.manage.model.dto.task.TaskHostNodeDTO; diff --git a/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateFileStepDAOImplIntegrationTest.java b/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateFileStepDAOImplIntegrationTest.java index 9f7ad0f48b..782f98679a 100644 --- a/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateFileStepDAOImplIntegrationTest.java +++ b/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateFileStepDAOImplIntegrationTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateScriptStepDAOImplIntegrationTest.java b/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateScriptStepDAOImplIntegrationTest.java index 9e36ba20d2..eaae27f346 100644 --- a/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateScriptStepDAOImplIntegrationTest.java +++ b/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateScriptStepDAOImplIntegrationTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,8 +24,8 @@ package com.tencent.bk.job.manage.dao.template.impl; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskScriptSourceEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskScriptSourceEnum; import com.tencent.bk.job.manage.dao.TaskScriptStepDAO; import com.tencent.bk.job.manage.model.dto.task.TaskScriptStepDTO; import org.junit.jupiter.api.BeforeEach; diff --git a/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateStepDAOImplIntegrationTest.java b/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateStepDAOImplIntegrationTest.java index ed011bd0d3..d6664c46d4 100644 --- a/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateStepDAOImplIntegrationTest.java +++ b/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateStepDAOImplIntegrationTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,7 +25,7 @@ package com.tencent.bk.job.manage.dao.template.impl; import com.tencent.bk.job.common.util.RandomUtil; -import com.tencent.bk.job.manage.common.consts.task.TaskStepTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskStepTypeEnum; import com.tencent.bk.job.manage.dao.TaskStepDAO; import com.tencent.bk.job.manage.model.dto.task.TaskStepDTO; import org.junit.jupiter.api.BeforeEach; diff --git a/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateVariableDAOImplIntegrationTest.java b/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateVariableDAOImplIntegrationTest.java index 36bd804d7f..71e3a5eb47 100644 --- a/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateVariableDAOImplIntegrationTest.java +++ b/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateVariableDAOImplIntegrationTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -39,7 +39,11 @@ import org.springframework.test.context.jdbc.SqlConfig; import org.springframework.test.context.junit.jupiter.SpringExtension; -import java.util.*; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Random; +import java.util.UUID; import static org.assertj.core.api.Assertions.assertThat; @@ -217,18 +221,15 @@ void givenWrongVariableIdOrTemplateIdReturnDeleteFailed() { @Test void givenNewVariableInfoReturnUpdateSuccess() { - assertThat(taskVariableDAO.updateVariableById(VARIABLE_1)).isTrue(); + assertThat(taskVariableDAO.updateVarByParentResourceIdAndTplVarId(VARIABLE_1)).isTrue(); VARIABLE_1.setName(UUID.randomUUID().toString()); - VARIABLE_1.setType(TaskVariableTypeEnum.CIPHER); + VARIABLE_1.setType(TaskVariableTypeEnum.STRING); VARIABLE_1.setDescription(UUID.randomUUID().toString()); VARIABLE_1.setDefaultValue(UUID.randomUUID().toString()); VARIABLE_1.setChangeable(!VARIABLE_1.getChangeable()); VARIABLE_1.setRequired(!VARIABLE_1.getRequired()); - assertThat(taskVariableDAO.updateVariableById(VARIABLE_1)).isTrue(); + assertThat(taskVariableDAO.updateVarByParentResourceIdAndTplVarId(VARIABLE_1)).isTrue(); - assertThat(taskVariableDAO.getVariableById(VARIABLE_1.getTemplateId(), VARIABLE_1.getId())) - .isNotEqualTo(VARIABLE_1); - VARIABLE_1.setType(TaskVariableTypeEnum.STRING); assertThat(taskVariableDAO.getVariableById(VARIABLE_1.getTemplateId(), VARIABLE_1.getId())) .isEqualTo(VARIABLE_1); } @@ -237,13 +238,13 @@ void givenNewVariableInfoReturnUpdateSuccess() { void givenWrongVariableInfoReturnUpdateFailed() { VARIABLE_1.setId(getRandomPositiveLong()); VARIABLE_1.setTemplateId(getRandomPositiveLong()); - assertThat(taskVariableDAO.updateVariableById(VARIABLE_1)).isFalse(); + assertThat(taskVariableDAO.updateVarByParentResourceIdAndTplVarId(VARIABLE_1)).isFalse(); VARIABLE_2.setTemplateId(getRandomPositiveLong()); - assertThat(taskVariableDAO.updateVariableById(VARIABLE_2)).isFalse(); + assertThat(taskVariableDAO.updateVarByParentResourceIdAndTplVarId(VARIABLE_2)).isFalse(); VARIABLE_3.setId(getRandomPositiveLong()); - assertThat(taskVariableDAO.updateVariableById(VARIABLE_3)).isFalse(); + assertThat(taskVariableDAO.updateVarByParentResourceIdAndTplVarId(VARIABLE_3)).isFalse(); } } diff --git a/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/package-info.java b/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/package-info.java index 256fe7ec70..12fef3e95d 100644 --- a/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/package-info.java +++ b/src/backend/job-manage/boot-job-manage/src/test/java/com/tencent/bk/job/manage/package-info.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/boot-job-manage/src/test/resources/application-test.yml b/src/backend/job-manage/boot-job-manage/src/test/resources/application-test.yml index bee4e55ec3..0400e83847 100644 --- a/src/backend/job-manage/boot-job-manage/src/test/resources/application-test.yml +++ b/src/backend/job-manage/boot-job-manage/src/test/resources/application-test.yml @@ -1,4 +1,7 @@ spring: + mvc: + pathmatch: + matching-strategy: ant_path_matcher datasource: job-manage: driver-class-name: org.h2.Driver @@ -12,8 +15,24 @@ spring: idle-timeout: 600000 poolName: "job-manage" validationTimeout: 5000 + cloud: + config: + enabled: false + kubernetes: + discovery: + enabled: false job: security: service: public-key-base64: LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FROEFNSUlCQ2dLQ0FRRUFyTXN3emh3QkVUTktQOThTcElna0VjZThPNVlQWm5aUDVHVW1ZUVhhSHdHWmg5UHJwNDFseS9xbWk5enEwTlRCODRyaXhSZHZyU1c0cCtCUjk4TVRzRmpYaUlRYVBxRTdRUitpdDdoT1VQNVJMQW9KcWlvL3NlZlRIdlZxVW1GcitYU3NST05rek5selRVaVcvMFRNY0cwNVdIS3hORno5YTB2aXo1T01PeE5lUzhqS1ZiRStIMHlQS0JhR1JHMEdZNTZTS1BLbXh1RHk5a2tyVUtGSXZycldSZkh2bzVsK0xsT3IyQmVGT3FHcEhmeW1FaXlkbFRxWHNlbGlsYTRxMUoxbnBZRDYwY3JIY1ZRV3d2Ri9CZUkrZytxVkF5YzNaWmhYeEhoNWNMcmZsK3pqMktsUEFEZVIvZktubHFqKzFBQm4yaFRoVWJ2WjRCdXhTdWxOa1FJREFRQUIKLS0tLS1FTkQgUFVCTElDIEtFWS0tLS0t private-key-base64: LS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tCk1JSUV1d0lCQURBTkJna3Foa2lHOXcwQkFRRUZBQVNDQktVd2dnU2hBZ0VBQW9JQkFRQ3N5ekRPSEFFUk0wby8zeEtraUNRUng3dzdsZzltZGsva1pTWmhCZG9mQVptSDArdW5qV1hMK3FhTDNPclExTUh6aXVMRkYyK3RKYmluNEZIM3d4T3dXTmVJaEJvK29UdEJINkszdUU1US9sRXNDZ21xS2oreDU5TWU5V3BTWVd2NWRLeEU0MlRNMlhOTlNKYi9STXh3YlRsWWNyRTBYUDFyUytMUGs0dzdFMTVMeU1wVnNUNGZUSThvRm9aRWJRWmpucElvOHFiRzRQTDJTU3RRb1VpK3V0WkY4ZStqbVg0dVU2dllGNFU2b2FrZC9LWVNMSjJWT3BleDZXS1ZyaXJVbldlbGdQclJ5c2R4VkJiQzhYOEY0ajZENnBVREp6ZGxtRmZFZUhsd3V0K1g3T1BZcVU4QU41SDk4cWVXcVA3VUFHZmFGT0ZSdTluZ0c3Rks2VTJSQWdNQkFBRUNnZ0VBZlkvZGREaUhVZjlxV2dWNG41VEtXUjFDSEM2TUhGR205bm5HTE1TNjNzTWoxTDdiUkZNTWdQQXY2L3hwcWJ5Zy81K1I3OVI2ODJBdzBkVEEyNENuSXdNOEE5aXkxWVlGNytuVUxSckIxanNuc3cvTVBCb2RWT0wzMkV4UHFYTmZxZEx0aDlXUm53WDRzbFVvWEhDYStxdHorMkRFZ2g5OGkyYUdkMzVwYlZjT2p0czlWc3FtZE5uQTdLNFVQWkd4aVFheG1rLzBOVEVNY2RqMUl4bWNlZjNqL1RUVkFiR05IdWVtYTJKeUdZdk52N09BZUN3VGJSdW5TNW1PSmptRHlFSmp4dndaVWZwanZoM0w0SkZzMGRJUm5wZ2VtTkY3NytqUmFTTy9WUVlqc1FLelB0SnEwcHFneHUwd0RoQUVBeXpjYWI2NW8wSm1hZzNJUjZNQzdRS0JnUUR1MThHNFRVaTlSVk8yZS9YdWtDSUNiOVZISHlMeVFIUDh4NDhBVFZmVENtSVBEMUduTWp5TlFmSnhsUlNma3UyUjJIM1IyRC81YUNpNUYwTi9USG51OWVqYlMvS1NoYjFxMFl5TWF1TDkxQlZjMkEyeDZlRzRPNFFpT2daT0gzaE0raUVSREJxOExNQ1dILzVtYVJDeDhVRmRQTnpHMUtXVVNwTEJXM3l2ZXdLQmdRQzVOTS9yN09Wc3F6OGlDUjBXa2JramN2eWU4MkJNcGFsZVkyRHNWQVMwRHNMNlMvZlAzY2VUdEJQVmdJL244Sk16b1ZNZE5OK3htbXM4Wi9rMGM5OGlLTzRUeE0zN2daYkIwUE8rWGRyeUZkRFhTRDZXTm1heWpYUG5MUndBZmRwbFlWdFNqVGxWMWJzWWIySnhXNXY2RUVSaVlLYnNnRGNpWTA5ZnYva0RZd0ovUnVmR212QldwOW50QUQvTXd3WUhFcklnbnZ5ZlkwdS9JMHdiSi92T0Z0aitRM3BJdzFvbW44ajBNTVFSVzA1RE9Ra01VVC9odmlrdDdjVDZSTkJ2WW9HZW4zdnNoNU1zcUltTk1DS0xRTSsxaDlxY05qVTR6WGpkd1V0NGs3akQvaFpEdXN2ZEpBQWxMR3hUR2hRVzRMeStxdTltbTZDRmg0Q2NFTXNZT1FLQmdCa0pEWTRyd0x1V3pucndWbkZGazR3NkwvSGdua1RlSSszeGF4bW5KZGVkSXpnb2FzQTdqNDNreXNXU2ttRmRBTmo2MXJ4YytGd09ycTVFczhnbVhhUkZlaUZMK2pna1JWZS93aU1hbHMwZ3E5RmVINis0Zk50b0NhSFhWREN3MG9QZnpETjFpYW1Uc2EzVTZLN0FIbTZJSW1aV0FGcDUzbm9VaUpjOTRSQlhBb0dCQUxMcmQrbnljNzVEbjZ0ZWdVT0JQRy9HR3FYWkZGdmd6emJRRnR0SmR0NTlMZ0NBanUxMjJMRExsRjZreTB6Q0JsK0h6Q0kyUXJXVEp4cXUxSTZvbndaZDlOMllKVlNWQlFOV285NTJUamRkOHcvSkExYlNqK3JpTU1WQ0txT1BNRHNiOENEaGlEbVM3RlBEM0trTVlmcUxJOVhNRTVvNVUvNFJzcEZUZWozRQotLS0tLUVORCBQUklWQVRFIEtFWS0tLS0t +bk-api-gateway: + gse: + url: gse.apigw.com + bkNotice: + url: bk-notice.apigw.com + cmdb: + url: cmdb.apigw.com +esb: + service: + url: esb.service diff --git a/src/backend/job-manage/boot-job-manage/src/test/resources/bootstrap.yml b/src/backend/job-manage/boot-job-manage/src/test/resources/bootstrap.yml index 1030a885b9..055d1ab90e 100644 --- a/src/backend/job-manage/boot-job-manage/src/test/resources/bootstrap.yml +++ b/src/backend/job-manage/boot-job-manage/src/test/resources/bootstrap.yml @@ -2,4 +2,7 @@ spring: application: name: job-manage profiles: - active: test \ No newline at end of file + active: test + cloud: + kubernetes: + enabled: false diff --git a/src/backend/job-manage/boot-job-manage/src/test/resources/init_account_data.sql b/src/backend/job-manage/boot-job-manage/src/test/resources/init_account_data.sql index 4168a73ade..5384f7a495 100644 --- a/src/backend/job-manage/boot-job-manage/src/test/resources/init_account_data.sql +++ b/src/backend/job-manage/boot-job-manage/src/test/resources/init_account_data.sql @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,14 +25,14 @@ truncate table account; insert into job_manage.account (id,account,alias,category,type,app_id,grantee,remark,os,password,db_password,db_port,db_system_account_id, -creator,create_time,last_modify_user,last_modify_time) + creator,create_time,last_modify_user,last_modify_time) values (1,'root','root',1,1,2,'user1,user2','root-linux','Linux',NULL,NULL,NULL,NULL,'admin', 1569550210000, 'admin', 1569550210000); insert into job_manage.account (id,account,alias,category,type,app_id,grantee,remark,os,password,db_password,db_port,db_system_account_id, -creator,create_time,last_modify_user,last_modify_time) + creator,create_time,last_modify_user,last_modify_time) values (2,'system','system',1,2,2,'user1,user2','system-window','Windows','mypassword',NULL,NULL,NULL,'admin', 1569550210000, 'admin', 1569550210000); insert into job_manage.account (id,account,alias,category,type,app_id,grantee,remark,os,password,db_password,db_port,db_system_account_id, -creator,create_time,last_modify_user,last_modify_time) -values (3,'job','job',2,9,2,'user1,user2','db-mysql-job',NULL,NULL,'dbpassword',3600,1,'admin', 1569550210000, 'admin', 1569550210000); + creator,create_time,last_modify_user,last_modify_time) +values (3,'job','job',2,9,2,'user1,user2','db-mysql-job',NULL,NULL,'YXZ/qDVUEdeMAs/8aTuhNAL+RaY95fl8zGlQXko7nMg=',3600,1,'admin', 1569550210000, 'admin', 1569550210000); insert into job_manage.account (id,account,alias,category,type,app_id,grantee,remark,os,password,db_password,db_port,db_system_account_id, -creator,create_time,last_modify_user,last_modify_time) + creator,create_time,last_modify_user,last_modify_time) values (4,'root','root-v2',1,1,2,'user1,user2','root-linux','Linux',NULL,NULL,NULL,NULL,'admin', 1569550210000, 'admin', 1569636611000); diff --git a/src/backend/job-manage/boot-job-manage/src/test/resources/init_analysis_task_data.sql b/src/backend/job-manage/boot-job-manage/src/test/resources/init_analysis_task_data.sql index bf58578175..277822f9dd 100644 --- a/src/backend/job-manage/boot-job-manage/src/test/resources/init_analysis_task_data.sql +++ b/src/backend/job-manage/boot-job-manage/src/test/resources/init_analysis_task_data.sql @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/boot-job-manage/src/test/resources/init_analysis_task_instance_data.sql b/src/backend/job-manage/boot-job-manage/src/test/resources/init_analysis_task_instance_data.sql index bbe47420dd..6f2b0df1db 100644 --- a/src/backend/job-manage/boot-job-manage/src/test/resources/init_analysis_task_instance_data.sql +++ b/src/backend/job-manage/boot-job-manage/src/test/resources/init_analysis_task_instance_data.sql @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/boot-job-manage/src/test/resources/init_application_data.sql b/src/backend/job-manage/boot-job-manage/src/test/resources/init_application_data.sql index 8c29ed39b9..b4e68846ad 100644 --- a/src/backend/job-manage/boot-job-manage/src/test/resources/init_application_data.sql +++ b/src/backend/job-manage/boot-job-manage/src/test/resources/init_application_data.sql @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/boot-job-manage/src/test/resources/init_dangerous_rule_data.sql b/src/backend/job-manage/boot-job-manage/src/test/resources/init_dangerous_rule_data.sql index 6c24ca8065..88158cd9aa 100644 --- a/src/backend/job-manage/boot-job-manage/src/test/resources/init_dangerous_rule_data.sql +++ b/src/backend/job-manage/boot-job-manage/src/test/resources/init_dangerous_rule_data.sql @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,9 +25,14 @@ USE job_manage; TRUNCATE TABLE `dangerous_rule`; INSERT INTO `dangerous_rule` (expression, script_type, description, priority, creator, last_modify_user, create_time, - last_modify_time) -VALUES ('a', 1, 'a', 5, 'userC', 'userC', 1600000000, 1600000000), - ('b', 3, 'b', 3, 'userC', 'userC', 1600000000, 1600000000), - ('c', 7, 'c', 1, 'userC', 'userC', 1600000000, 1600000000), - ('d', 15, 'd', 4, 'userC', 'userC', 1600000000, 1600000000), - ('e', 31, 'e', 2, 'userC', 'userC', 1600000000, 1600000000); + last_modify_time, action, status) +VALUES ('shell',1,'Shell-1',1,'test1','test1','1607439270294','1607439270294',0,1), + ('shell,bat,perl',7,'Shell-1,Bat-2,Perl-3',1,'test1','test1','1607439270294','1607439270294',1,1), + ('shell,bat',3,'Shell-1,Bat-2',1,'test1','test1','1607439270294','1607439270294',1,1), + ('shell,bat,perl,python',15,'Shell-1,Bat-2,Perl-3,Python-4',1,'test1','test1','1607439270294','1607439270294',1,1), + ('shell,bat,perl,python,powershell',31,'Shell-1,Bat-2,Perl-3,Python-4,Powershell-5',1,'test1','test1','1607439270294','1607439270294',0,1), + ('shell,bat,perl,python,powershell,sql',63,'Shell-1,Bat-2,Perl-3,Python-4,Powershell-5,SQL-6',1,'test1','test1','1607439270294','1607439270294',1,1), + ('sql',32,'SQL-6',1,'test1','test1','1607439270294','1607439270294',1,1), + ('shell,sql',33,'Shell-1,SQL-6',1,'test1','test1','1607439270294','1607439270294',1,1), + ('shell,python',9,'Shell-1,Python-4',1,'test1','test1','1607439270294','1607439270294',1,1), + ('python',8,'Python-4',1,'test1','test1','1607439270294','1607439270294',0,1); diff --git a/src/backend/job-manage/boot-job-manage/src/test/resources/init_resource_tag_data.sql b/src/backend/job-manage/boot-job-manage/src/test/resources/init_resource_tag_data.sql index 215fb1fc59..27ee1d6fc1 100644 --- a/src/backend/job-manage/boot-job-manage/src/test/resources/init_resource_tag_data.sql +++ b/src/backend/job-manage/boot-job-manage/src/test/resources/init_resource_tag_data.sql @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/boot-job-manage/src/test/resources/init_schema.sql b/src/backend/job-manage/boot-job-manage/src/test/resources/init_schema.sql index e6922c900c..ca1c08a417 100644 --- a/src/backend/job-manage/boot-job-manage/src/test/resources/init_schema.sql +++ b/src/backend/job-manage/boot-job-manage/src/test/resources/init_schema.sql @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -282,6 +282,7 @@ CREATE TABLE `task_plan_step_script` `is_secure_param` TINYINT(1) UNSIGNED NOT NULL DEFAULT '0', `is_latest_version` TINYINT(1) UNSIGNED NOT NULL DEFAULT '1', `ignore_error` TINYINT(1) UNSIGNED NOT NULL, + `windows_interpreter` VARCHAR(260) NULL DEFAULT NULL, PRIMARY KEY (`id`), KEY (`step_id`) USING BTREE, KEY (`script_id`) USING BTREE, @@ -445,6 +446,7 @@ CREATE TABLE `task_template_step_script` `is_secure_param` TINYINT(1) UNSIGNED NOT NULL DEFAULT '0', `is_latest_version` TINYINT(1) UNSIGNED NOT NULL DEFAULT '1', `ignore_error` TINYINT(1) UNSIGNED NOT NULL, + `windows_interpreter` VARCHAR(260) NULL DEFAULT NULL, PRIMARY KEY (`id`), UNIQUE KEY (`step_id`) USING BTREE, KEY (`script_id`) USING BTREE, diff --git a/src/backend/job-manage/boot-job-manage/src/test/resources/init_script_data.sql b/src/backend/job-manage/boot-job-manage/src/test/resources/init_script_data.sql index ef7b3aec27..6b68ad1377 100644 --- a/src/backend/job-manage/boot-job-manage/src/test/resources/init_script_data.sql +++ b/src/backend/job-manage/boot-job-manage/src/test/resources/init_script_data.sql @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/boot-job-manage/src/test/resources/init_script_relate_task_plan_data.sql b/src/backend/job-manage/boot-job-manage/src/test/resources/init_script_relate_task_plan_data.sql index 6b4954f7ef..0e1d7ba8d4 100644 --- a/src/backend/job-manage/boot-job-manage/src/test/resources/init_script_relate_task_plan_data.sql +++ b/src/backend/job-manage/boot-job-manage/src/test/resources/init_script_relate_task_plan_data.sql @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/boot-job-manage/src/test/resources/init_script_relate_task_template_data.sql b/src/backend/job-manage/boot-job-manage/src/test/resources/init_script_relate_task_template_data.sql index 542c15d572..6c7a6534c4 100644 --- a/src/backend/job-manage/boot-job-manage/src/test/resources/init_script_relate_task_template_data.sql +++ b/src/backend/job-manage/boot-job-manage/src/test/resources/init_script_relate_task_template_data.sql @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/boot-job-manage/src/test/resources/init_tag_data.sql b/src/backend/job-manage/boot-job-manage/src/test/resources/init_tag_data.sql index 1ffee7b0b8..a1a8a12176 100644 --- a/src/backend/job-manage/boot-job-manage/src/test/resources/init_tag_data.sql +++ b/src/backend/job-manage/boot-job-manage/src/test/resources/init_tag_data.sql @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/boot-job-manage/src/test/resources/template/init_template_approval_step_data.sql b/src/backend/job-manage/boot-job-manage/src/test/resources/template/init_template_approval_step_data.sql index 5826709e7c..f65d3ab931 100644 --- a/src/backend/job-manage/boot-job-manage/src/test/resources/template/init_template_approval_step_data.sql +++ b/src/backend/job-manage/boot-job-manage/src/test/resources/template/init_template_approval_step_data.sql @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/boot-job-manage/src/test/resources/template/init_template_data.sql b/src/backend/job-manage/boot-job-manage/src/test/resources/template/init_template_data.sql index e3c5a8235e..349b4995ff 100644 --- a/src/backend/job-manage/boot-job-manage/src/test/resources/template/init_template_data.sql +++ b/src/backend/job-manage/boot-job-manage/src/test/resources/template/init_template_data.sql @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/boot-job-manage/src/test/resources/template/init_template_favorite_data.sql b/src/backend/job-manage/boot-job-manage/src/test/resources/template/init_template_favorite_data.sql index 07486533c5..f205bd4e5f 100644 --- a/src/backend/job-manage/boot-job-manage/src/test/resources/template/init_template_favorite_data.sql +++ b/src/backend/job-manage/boot-job-manage/src/test/resources/template/init_template_favorite_data.sql @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/boot-job-manage/src/test/resources/template/init_template_file_info_data.sql b/src/backend/job-manage/boot-job-manage/src/test/resources/template/init_template_file_info_data.sql index 42aff33a7c..ed6c0c6276 100644 --- a/src/backend/job-manage/boot-job-manage/src/test/resources/template/init_template_file_info_data.sql +++ b/src/backend/job-manage/boot-job-manage/src/test/resources/template/init_template_file_info_data.sql @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/boot-job-manage/src/test/resources/template/init_template_file_step_data.sql b/src/backend/job-manage/boot-job-manage/src/test/resources/template/init_template_file_step_data.sql index 134f241685..2d54d96e7f 100644 --- a/src/backend/job-manage/boot-job-manage/src/test/resources/template/init_template_file_step_data.sql +++ b/src/backend/job-manage/boot-job-manage/src/test/resources/template/init_template_file_step_data.sql @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,6 +27,6 @@ TRUNCATE TABLE `task_template_step_file`; INSERT INTO `task_template_step_file` (id, step_id, destination_file_location, execute_account, destination_host_list, timeout, origin_speed_limit, target_speed_limit, ignore_error, duplicate_handler) -VALUES (1, 1000, '/root/tmp', 1, '1.1.1.1,2.2.2.2', 6000, 100, 200, 0, 1), - (2, 2000, '/home/user1/tmp', 2, '3.3.3.3,4.4.4.4,6.6.6.6', 6000, null, null, 1, 2), - (3, 3000, '/home/userC/tmp', 3, '10.10.10.10', 6000, 100, 200, 0, 1); +VALUES (1, 1000, '/root/tmp', 1, null, 6000, 100, 200, 0, 1), + (2, 2000, '/home/user1/tmp', 2, null, 6000, null, null, 1, 2), + (3, 3000, '/home/userC/tmp', 3, null, 6000, 100, 200, 0, 1); diff --git a/src/backend/job-manage/boot-job-manage/src/test/resources/template/init_template_script_step_data.sql b/src/backend/job-manage/boot-job-manage/src/test/resources/template/init_template_script_step_data.sql index 1550a888bb..ca922e8f8e 100644 --- a/src/backend/job-manage/boot-job-manage/src/test/resources/template/init_template_script_step_data.sql +++ b/src/backend/job-manage/boot-job-manage/src/test/resources/template/init_template_script_step_data.sql @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/boot-job-manage/src/test/resources/template/init_template_step_data.sql b/src/backend/job-manage/boot-job-manage/src/test/resources/template/init_template_step_data.sql index 04621cb373..92f1dcb3f5 100644 --- a/src/backend/job-manage/boot-job-manage/src/test/resources/template/init_template_step_data.sql +++ b/src/backend/job-manage/boot-job-manage/src/test/resources/template/init_template_step_data.sql @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/boot-job-manage/src/test/resources/template/init_template_variable_data.sql b/src/backend/job-manage/boot-job-manage/src/test/resources/template/init_template_variable_data.sql index b9b00f3a25..0109f1f0e8 100644 --- a/src/backend/job-manage/boot-job-manage/src/test/resources/template/init_template_variable_data.sql +++ b/src/backend/job-manage/boot-job-manage/src/test/resources/template/init_template_variable_data.sql @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,18 +25,18 @@ USE `job_manage`; TRUNCATE TABLE `task_template_variable`; INSERT INTO `task_template_variable` (id, template_id, name, type, default_value, description, is_changeable, is_required) - VALUES (1, 10000, '测试1', 1, 'test1', '这是一个测试变量1', true, true); +VALUES (1, 10000, '测试1', 1, 'test1', '这是一个测试变量1', true, true); INSERT INTO `task_template_variable` (id, template_id, name, type, default_value, description, is_changeable, is_required) - VALUES (2, 10000, '测试2', 2, 'test2', '这是一个测试变量2', false, false); +VALUES (2, 10000, '测试2', 2, 'test2', '这是一个测试变量2', false, false); INSERT INTO `task_template_variable` (id, template_id, name, type, default_value, description, is_changeable, is_required) - VALUES (3, 10000, '测试3', 3, 'test3', '这是一个测试变量3', true, false); +VALUES (3, 10000, '测试3', 3, 'test3', '这是一个测试变量3', true, false); INSERT INTO `task_template_variable` (id, template_id, name, type, default_value, description, is_changeable, is_required) - VALUES (4, 10000, '测试4', 4, 'test4', '这是一个测试变量4', false, true); +VALUES (4, 10000, '测试4', 4, 'test4', '这是一个测试变量4', false, true); INSERT INTO `task_template_variable` (id, template_id, name, type, default_value, description, is_changeable, is_required) - VALUES (5, 20000, '测试5', 1, 'test5', '这是一个测试变量5', true, true); +VALUES (5, 20000, '测试5', 1, 'test5', '这是一个测试变量5', true, true); INSERT INTO `task_template_variable` (id, template_id, name, type, default_value, description, is_changeable, is_required) - VALUES (6, 20000, '测试6', 2, 'test6', '这是一个测试变量6', false, false); +VALUES (6, 20000, '测试6', 2, 'test6', '这是一个测试变量6', false, false); INSERT INTO `task_template_variable` (id, template_id, name, type, default_value, description, is_changeable, is_required) - VALUES (7, 20000, '测试7', 3, 'test7', '这是一个测试变量7', true, false); +VALUES (7, 20000, '测试7', 3, 'test7', '这是一个测试变量7', true, false); INSERT INTO `task_template_variable` (id, template_id, name, type, default_value, description, is_changeable, is_required) - VALUES (8, 20000, '测试8', 4, 'test8', '这是一个测试变量8', false, true); +VALUES (8, 20000, '测试8', 4, 'test8', '这是一个测试变量8', false, true); diff --git a/src/backend/job-manage/boot-job-manage/src/test/resources/test.properties b/src/backend/job-manage/boot-job-manage/src/test/resources/test.properties index 90b9e3b79c..8653b0d6f4 100644 --- a/src/backend/job-manage/boot-job-manage/src/test/resources/test.properties +++ b/src/backend/job-manage/boot-job-manage/src/test/resources/test.properties @@ -1,7 +1,7 @@ # # Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. # -# Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. +# Copyright (C) 2021 Tencent. All rights reserved. # # BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. # diff --git a/src/backend/job-manage/build.gradle b/src/backend/job-manage/build.gradle index a758222971..7be64bd2a1 100644 --- a/src/backend/job-manage/build.gradle +++ b/src/backend/job-manage/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/model-job-manage/build.gradle b/src/backend/job-manage/model-job-manage/build.gradle index 53cd347e49..8006176375 100644 --- a/src/backend/job-manage/model-job-manage/build.gradle +++ b/src/backend/job-manage/model-job-manage/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/build.gradle b/src/backend/job-manage/service-job-manage/build.gradle index 6b7e21f646..d3f073c279 100644 --- a/src/backend/job-manage/service-job-manage/build.gradle +++ b/src/backend/job-manage/service-job-manage/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -29,14 +29,17 @@ dependencies { api project(":job-manage:model-job-manage") api project(":job-crontab:api-job-crontab") api project(":job-analysis:api-job-analysis") + api project(":job-file-gateway:api-job-file-gateway") api project(":commons:common") + api project(":commons:common-crypto") api project(":commons:common-security") api project(":commons:common-redis") - api project(":commons:cmdb-sdk-ext") api project(":commons:artifactory-sdk") api project(":commons:paas-sdk") api project(":commons:esb-sdk") api project(":commons:gse-sdk") + api project(":commons:cmdb-sdk") + api project(":commons:common-mysql") implementation "org.springframework.boot:spring-boot-starter-web" implementation "org.springframework.boot:spring-boot-starter-jdbc" implementation "org.springframework.boot:spring-boot-starter-jooq" @@ -54,4 +57,5 @@ dependencies { implementation 'joda-time:joda-time' testImplementation 'org.springframework.boot:spring-boot-starter-test' testImplementation 'org.apache.commons:commons-lang3' + testImplementation("com.h2database:h2") } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/ScriptDTOBuilder.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/ScriptDTOBuilder.java index a5fbbd6fb3..287f793a57 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/ScriptDTOBuilder.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/common/ScriptDTOBuilder.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,11 +25,18 @@ package com.tencent.bk.job.manage.api.common; import com.tencent.bk.job.common.util.Base64Util; -import com.tencent.bk.job.manage.common.consts.script.ScriptCategoryEnum; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptCategoryEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; import com.tencent.bk.job.manage.model.dto.ScriptDTO; import com.tencent.bk.job.manage.model.dto.TagDTO; -import com.tencent.bk.job.manage.model.web.request.ScriptCreateUpdateReq; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbCreatePublicScriptV3Req; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbCreatePublicScriptVersionV3Req; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbCreateScriptV3Req; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbCreateScriptVersionV3Req; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbUpdatePublicScriptVersionV3Req; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbUpdateScriptVersionV3Req; +import com.tencent.bk.job.manage.model.web.request.ScriptCreateReq; +import com.tencent.bk.job.manage.model.web.request.ScriptVersionCreateUpdateReq; import com.tencent.bk.job.manage.model.web.vo.TagVO; import org.springframework.stereotype.Service; @@ -39,10 +46,14 @@ @Service public class ScriptDTOBuilder { - public ScriptDTO buildFromCreateUpdateReq(ScriptCreateUpdateReq req) { + /** + * ScriptCreateReq -> ScriptDTO 转换 + * + * @param req 请求 + * @return 转换后的脚本 + */ + public ScriptDTO buildFromScriptCreateReq(ScriptCreateReq req) { ScriptDTO scriptDTO = new ScriptDTO(); - scriptDTO.setId(req.getId()); - scriptDTO.setScriptVersionId(req.getScriptVersionId()); scriptDTO.setName(req.getName()); scriptDTO.setType(req.getType()); if (req.getType() != null) { @@ -65,7 +76,93 @@ public ScriptDTO buildFromCreateUpdateReq(ScriptCreateUpdateReq req) { } scriptDTO.setVersion(req.getVersion()); scriptDTO.setDescription(req.getDescription()); + return scriptDTO; + } + + /** + * ScriptVersionCreateUpdateReq -> ScriptDTO 转换 + * + * @param req 请求 + * @return 转换后的脚本 + */ + public ScriptDTO buildFromScriptVersionCreateUpdateReq(ScriptVersionCreateUpdateReq req) { + ScriptDTO scriptDTO = new ScriptDTO(); + scriptDTO.setContent(Base64Util.decodeContentToStr(req.getContent())); + scriptDTO.setVersion(req.getVersion()); + scriptDTO.setVersionDesc(req.getVersionDesc()); + return scriptDTO; + } + + public ScriptDTO buildFromCreateUpdateReq(EsbUpdateScriptVersionV3Req req) { + ScriptDTO scriptDTO = new ScriptDTO(); + scriptDTO.setId(req.getScriptId()); + scriptDTO.setContent(Base64Util.decodeContentToStr(req.getContent())); + scriptDTO.setVersionDesc(req.getVersionDesc()); + scriptDTO.setScriptVersionId(req.getScriptVersionId()); + return scriptDTO; + } + + public ScriptDTO buildFromCreateUpdateReq(EsbUpdatePublicScriptVersionV3Req req) { + ScriptDTO scriptDTO = new ScriptDTO(); + scriptDTO.setId(req.getScriptId()); + scriptDTO.setContent(Base64Util.decodeContentToStr(req.getContent())); + scriptDTO.setVersionDesc(req.getVersionDesc()); + scriptDTO.setScriptVersionId(req.getScriptVersionId()); + return scriptDTO; + } + + public ScriptDTO buildFromEsbCreateReq(EsbCreateScriptV3Req req) { + ScriptDTO scriptDTO = new ScriptDTO(); + scriptDTO.setName(req.getName()); + scriptDTO.setType(req.getType()); + if (req.getType() != null) { + if (!req.getType().equals(ScriptTypeEnum.SQL.getValue())) { + scriptDTO.setCategory(ScriptCategoryEnum.SYSSCRIPT.getValue()); + } else { + scriptDTO.setCategory(ScriptCategoryEnum.SQLSCRIPT.getValue()); + } + } + scriptDTO.setContent(Base64Util.decodeContentToStr(req.getContent())); + scriptDTO.setVersion(req.getVersion()); + scriptDTO.setDescription(req.getDescription()); scriptDTO.setVersionDesc(req.getVersionDesc()); return scriptDTO; } + + public ScriptDTO buildFromEsbCreateReq(EsbCreatePublicScriptV3Req req) { + ScriptDTO scriptDTO = new ScriptDTO(); + scriptDTO.setName(req.getName()); + scriptDTO.setType(req.getType()); + if (req.getType() != null) { + if (!req.getType().equals(ScriptTypeEnum.SQL.getValue())) { + scriptDTO.setCategory(ScriptCategoryEnum.SYSSCRIPT.getValue()); + } else { + scriptDTO.setCategory(ScriptCategoryEnum.SQLSCRIPT.getValue()); + } + } + scriptDTO.setContent(Base64Util.decodeContentToStr(req.getContent())); + scriptDTO.setVersion(req.getVersion()); + scriptDTO.setDescription(req.getDescription()); + scriptDTO.setVersionDesc(req.getVersionDesc()); + return scriptDTO; + } + + public ScriptDTO buildFromEsbCreateReq(EsbCreateScriptVersionV3Req req) { + ScriptDTO scriptDTO = new ScriptDTO(); + scriptDTO.setId(req.getScriptId()); + scriptDTO.setContent(Base64Util.decodeContentToStr(req.getContent())); + scriptDTO.setVersion(req.getVersion()); + scriptDTO.setVersionDesc(req.getVersionDesc()); + return scriptDTO; + } + + public ScriptDTO buildFromEsbCreateReq(EsbCreatePublicScriptVersionV3Req req) { + ScriptDTO scriptDTO = new ScriptDTO(); + scriptDTO.setId(req.getScriptId()); + scriptDTO.setContent(Base64Util.decodeContentToStr(req.getContent())); + scriptDTO.setVersion(req.getVersion()); + scriptDTO.setVersionDesc(req.getVersionDesc()); + return scriptDTO; + } + } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/EsbGetDBAccountListResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/EsbGetDBAccountListResourceImpl.java index cdb96c89d3..69a73e0bdb 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/EsbGetDBAccountListResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/EsbGetDBAccountListResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,17 +24,15 @@ package com.tencent.bk.job.manage.api.esb.impl; +import com.tencent.bk.job.common.constant.AccountCategoryEnum; import com.tencent.bk.job.common.esb.metrics.EsbApiTimed; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.common.esb.util.EsbDTOAppScopeMappingHelper; import com.tencent.bk.job.common.exception.InvalidParamException; -import com.tencent.bk.job.common.i18n.service.MessageI18nService; import com.tencent.bk.job.common.metrics.CommonMetricNames; import com.tencent.bk.job.common.model.ValidateResult; -import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.manage.api.esb.EsbGetDBAccountListResource; -import com.tencent.bk.job.manage.common.consts.account.AccountCategoryEnum; import com.tencent.bk.job.manage.model.dto.AccountDTO; import com.tencent.bk.job.manage.model.esb.EsbDBAccountDTO; import com.tencent.bk.job.manage.model.esb.request.EsbGetDBAccountListRequest; @@ -54,36 +52,31 @@ @Slf4j public class EsbGetDBAccountListResourceImpl implements EsbGetDBAccountListResource { private final AccountService accountService; - private final MessageI18nService i18nService; - private final AppScopeMappingService appScopeMappingService; @Autowired - public EsbGetDBAccountListResourceImpl(AccountService accountService, - MessageI18nService i18nService, - AppScopeMappingService appScopeMappingService) { + public EsbGetDBAccountListResourceImpl(AccountService accountService) { this.accountService = accountService; - this.i18nService = i18nService; - this.appScopeMappingService = appScopeMappingService; } @Override @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v2_get_own_db_account_list"}) - public EsbResp> getUserOwnDbAccountList(EsbGetDBAccountListRequest request) { - request.fillAppResourceScope(appScopeMappingService); + public EsbResp> getUserOwnDbAccountList(String username, + String appCode, + EsbGetDBAccountListRequest request) { ValidateResult checkResult = checkRequest(request); if (!checkResult.isPass()) { log.warn("Get db account list, request is illegal!"); throw new InvalidParamException(checkResult); } long appId = request.getAppId(); - List dbAccounts = accountService.listAllAppAccount(appId, AccountCategoryEnum.DB); + List dbAccounts = accountService.listAppAccount(appId, AccountCategoryEnum.DB); List grantedDbAccounts = new ArrayList<>(); dbAccounts.forEach(dbAccount -> { if (StringUtils.isBlank(dbAccount.getGrantees())) { grantedDbAccounts.add(dbAccount); } else { List grantees = Arrays.asList(dbAccount.getGrantees().split(",")); - if (grantees.contains(request.getUserName())) { + if (grantees.contains(username)) { grantedDbAccounts.add(dbAccount); } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/EsbGetJobDetailResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/EsbGetJobDetailResourceImpl.java index 264654c99e..a4f97b2cba 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/EsbGetJobDetailResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/EsbGetJobDetailResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,8 @@ package com.tencent.bk.job.manage.api.esb.impl; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.audit.annotations.AuditRequestBody; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.constant.TaskVariableTypeEnum; import com.tencent.bk.job.common.esb.metrics.EsbApiTimed; @@ -32,18 +34,15 @@ import com.tencent.bk.job.common.esb.model.job.EsbIpDTO; import com.tencent.bk.job.common.esb.util.EsbDTOAppScopeMappingHelper; import com.tencent.bk.job.common.exception.InvalidParamException; -import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; -import com.tencent.bk.job.common.iam.model.AuthResult; -import com.tencent.bk.job.common.iam.service.BusinessAuthService; +import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.metrics.CommonMetricNames; import com.tencent.bk.job.common.model.ValidateResult; import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; -import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.common.util.date.DateUtils; +import com.tencent.bk.job.manage.api.common.constants.task.TaskFileTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskScriptSourceEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskStepTypeEnum; import com.tencent.bk.job.manage.api.esb.EsbGetJobDetailResource; -import com.tencent.bk.job.manage.auth.PlanAuthService; -import com.tencent.bk.job.manage.common.consts.task.TaskFileTypeEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskStepTypeEnum; import com.tencent.bk.job.manage.model.dto.AccountDTO; import com.tencent.bk.job.manage.model.dto.ScriptDTO; import com.tencent.bk.job.manage.model.dto.task.TaskApprovalStepDTO; @@ -61,6 +60,7 @@ import com.tencent.bk.job.manage.model.esb.request.EsbGetJobDetailRequest; import com.tencent.bk.job.manage.model.inner.ServiceAccountDTO; import com.tencent.bk.job.manage.service.AccountService; +import com.tencent.bk.job.manage.service.PublicScriptService; import com.tencent.bk.job.manage.service.ScriptService; import com.tencent.bk.job.manage.service.plan.TaskPlanService; import lombok.extern.slf4j.Slf4j; @@ -79,30 +79,25 @@ public class EsbGetJobDetailResourceImpl implements EsbGetJobDetailResource { private final TaskPlanService taskPlanService; private final ScriptService scriptService; + private final PublicScriptService publicScriptService; private final AccountService accountService; - private final BusinessAuthService businessAuthService; - private final PlanAuthService planAuthService; - private final AppScopeMappingService appScopeMappingService; public EsbGetJobDetailResourceImpl(TaskPlanService taskPlanService, ScriptService scriptService, - AccountService accountService, - BusinessAuthService businessAuthService, - PlanAuthService planAuthService, - AppScopeMappingService appScopeMappingService) { + PublicScriptService publicScriptService, + AccountService accountService) { this.taskPlanService = taskPlanService; this.scriptService = scriptService; + this.publicScriptService = publicScriptService; this.accountService = accountService; - this.businessAuthService = businessAuthService; - this.planAuthService = planAuthService; - this.appScopeMappingService = appScopeMappingService; } @Override @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v2_get_job_detail"}) - public EsbResp getJobDetail(EsbGetJobDetailRequest request) { - request.fillAppResourceScope(appScopeMappingService); - + @AuditEntry(actionId = ActionId.VIEW_JOB_PLAN) + public EsbResp getJobDetail(String username, + String appCode, + @AuditRequestBody EsbGetJobDetailRequest request) { ValidateResult checkResult = checkRequest(request); if (!checkResult.isPass()) { log.warn("Get job detail, request is illegal!"); @@ -111,24 +106,7 @@ public EsbResp getJobDetail(EsbGetJobDetailRequest request) { Long appId = request.getAppId(); Long jobId = request.getPlanId(); - TaskPlanInfoDTO taskPlan = taskPlanService.getTaskPlanById(appId, jobId); - if (taskPlan == null) { - AuthResult authResult = - businessAuthService.authAccessBusiness(request.getUserName(), request.getAppResourceScope()); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } else { - log.info("Get job detail, job is not exist! appId={}, jobId={}", appId, jobId); - return EsbResp.buildSuccessResp(null); - } - } - - AuthResult authResult = - planAuthService.authViewJobPlan(request.getUserName(), request.getAppResourceScope(), - taskPlan.getTemplateId(), request.getPlanId(), taskPlan.getName()); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } + TaskPlanInfoDTO taskPlan = taskPlanService.getTaskPlan(username, appId, jobId); return EsbResp.buildSuccessResp(buildJobDetail(taskPlan)); } @@ -208,7 +186,7 @@ private void buildScriptStep(EsbStepDTO esbStep, TaskScriptStepDTO scriptStep, esbStep.setType(scriptStep.getScriptSource().getType()); if (scriptStep.getScriptVersionId() != null && scriptStep.getScriptVersionId() > 0) { - ScriptDTO script = scriptService.getScriptVersion(scriptStep.getScriptVersionId()); + ScriptDTO script = getScriptVersion(scriptStep.getScriptSource(), scriptStep.getScriptVersionId()); if (script == null) { log.warn("Plan related script is not exist, planId={}, scriptVersionId={}", scriptStep.getPlanId(), scriptStep.getScriptVersionId()); @@ -235,6 +213,20 @@ private void buildScriptStep(EsbStepDTO esbStep, TaskScriptStepDTO scriptStep, fillStepTargetServerInfo(esbStep, scriptStep.getExecuteTarget()); } + private ScriptDTO getScriptVersion(TaskScriptSourceEnum scriptSource, long scriptVersionId) { + ScriptDTO scriptVersion = null; + switch (scriptSource) { + case CITING: + scriptVersion = scriptService.getScriptVersion(scriptVersionId); + break; + case PUBLIC: + scriptVersion = publicScriptService.getScriptVersion(scriptVersionId); + break; + } + return scriptVersion; + + } + private void fillStepTargetServerInfo(EsbStepDTO esbStep, TaskTargetDTO targetServer) { if (targetServer != null && StringUtils.isEmpty(targetServer.getVariable()) && targetServer.getHostNodeList() != null) { @@ -253,7 +245,7 @@ private List convertToEsbIpDTOList(List hostList) List ipList = new ArrayList<>(); if (hostList != null && !hostList.isEmpty()) { hostList.forEach(host -> { - EsbIpDTO ipDTO = new EsbIpDTO(host.getCloudAreaId(), host.getIp()); + EsbIpDTO ipDTO = new EsbIpDTO(host.getHostId(), host.getCloudAreaId(), host.getIp()); ipList.add(ipDTO); }); } @@ -331,7 +323,7 @@ private EsbTaskVariableDTO buildVariable(TaskVariableDTO variable) { variableDTO.setCategory(variable.getType().getType()); if (variable.getType() == TaskVariableTypeEnum.HOST_LIST && StringUtils.isNotBlank(variable.getDefaultValue())) { - TaskTargetDTO target = TaskTargetDTO.fromString(variable.getDefaultValue()); + TaskTargetDTO target = TaskTargetDTO.fromJsonString(variable.getDefaultValue()); if (target == null) { log.error("Variable target is empty! variableId:{}", variable.getId()); return variableDTO; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/EsbGetJobListResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/EsbGetJobListResourceImpl.java index f4d817be4c..f73b2e3ac4 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/EsbGetJobListResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/EsbGetJobListResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,14 +28,11 @@ import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.common.esb.util.EsbDTOAppScopeMappingHelper; import com.tencent.bk.job.common.exception.InvalidParamException; -import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; -import com.tencent.bk.job.common.iam.model.AuthResult; import com.tencent.bk.job.common.iam.service.BusinessAuthService; import com.tencent.bk.job.common.metrics.CommonMetricNames; import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; import com.tencent.bk.job.common.model.ValidateResult; -import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.manage.api.esb.EsbGetJobListResource; import com.tencent.bk.job.manage.model.dto.TaskPlanQueryDTO; @@ -58,21 +55,19 @@ public class EsbGetJobListResourceImpl implements EsbGetJobListResource { private final TaskPlanService taskPlanService; private final BusinessAuthService businessAuthService; - private final AppScopeMappingService appScopeMappingService; public EsbGetJobListResourceImpl(TaskPlanService taskPlanService, - BusinessAuthService businessAuthService, - AppScopeMappingService appScopeMappingService) { + BusinessAuthService businessAuthService) { this.taskPlanService = taskPlanService; this.businessAuthService = businessAuthService; - this.appScopeMappingService = appScopeMappingService; } @Override @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v2_get_job_list"}) - public EsbResp> getJobList(EsbGetJobListRequest request) { - request.fillAppResourceScope(appScopeMappingService); + public EsbResp> getJobList(String username, + String appCode, + EsbGetJobListRequest request) { ValidateResult checkResult = checkRequest(request); if (!checkResult.isPass()) { log.warn("Get job list, request is illegal!"); @@ -80,12 +75,6 @@ public EsbResp> getJobList(EsbGetJobListRequest request } long appId = request.getAppId(); - AuthResult authResult = - businessAuthService.authAccessBusiness(request.getUserName(), request.getAppResourceScope()); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } - TaskPlanQueryDTO taskPlanQueryDTO = new TaskPlanQueryDTO(); taskPlanQueryDTO.setAppId(appId); taskPlanQueryDTO.setName(request.getName()); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/EsbGetOSAccountResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/EsbGetOSAccountResourceImpl.java index 8641093fd7..4869aa832b 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/EsbGetOSAccountResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/EsbGetOSAccountResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,14 +24,13 @@ package com.tencent.bk.job.manage.api.esb.impl; +import com.tencent.bk.job.common.constant.AccountCategoryEnum; import com.tencent.bk.job.common.esb.metrics.EsbApiTimed; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.common.esb.util.EsbDTOAppScopeMappingHelper; import com.tencent.bk.job.common.metrics.CommonMetricNames; -import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.manage.api.esb.EsbGetOSAccountResource; -import com.tencent.bk.job.manage.common.consts.account.AccountCategoryEnum; import com.tencent.bk.job.manage.model.dto.AccountDTO; import com.tencent.bk.job.manage.model.esb.EsbAccountDTO; import com.tencent.bk.job.manage.model.esb.request.EsbGetOSAccountListRequest; @@ -49,21 +48,19 @@ @Slf4j public class EsbGetOSAccountResourceImpl implements EsbGetOSAccountResource { private final AccountService accountService; - private final AppScopeMappingService appScopeMappingService; @Autowired - public EsbGetOSAccountResourceImpl(AccountService accountService, - AppScopeMappingService appScopeMappingService) { + public EsbGetOSAccountResourceImpl(AccountService accountService) { this.accountService = accountService; - this.appScopeMappingService = appScopeMappingService; } @Override @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v2_get_os_account"}) - public EsbResp> getAppOsAccountList(EsbGetOSAccountListRequest request) { - request.fillAppResourceScope(appScopeMappingService); + public EsbResp> getAppOsAccountList(String username, + String appCode, + EsbGetOSAccountListRequest request) { long appId = request.getAppId(); - List systemAccounts = accountService.listAllAppAccount(appId, AccountCategoryEnum.SYSTEM); + List systemAccounts = accountService.listAppAccount(appId, AccountCategoryEnum.SYSTEM); return EsbResp.buildSuccessResp(convertToEsbAccountDTOList(systemAccounts)); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/EsbGetPublicScriptListResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/EsbGetPublicScriptListResourceImpl.java index 8e3479b044..e5f89c7adf 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/EsbGetPublicScriptListResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/EsbGetPublicScriptListResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -35,14 +35,14 @@ import com.tencent.bk.job.common.model.PageData; import com.tencent.bk.job.common.model.ValidateResult; import com.tencent.bk.job.common.util.date.DateUtils; +import com.tencent.bk.job.manage.api.common.constants.JobResourceStatusEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; import com.tencent.bk.job.manage.api.esb.EsbGetPublicScriptListResource; -import com.tencent.bk.job.manage.common.consts.JobResourceStatusEnum; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; import com.tencent.bk.job.manage.model.dto.ScriptDTO; import com.tencent.bk.job.manage.model.esb.EsbScriptDTO; import com.tencent.bk.job.manage.model.esb.request.EsbGetPublicScriptListRequest; import com.tencent.bk.job.manage.model.query.ScriptQuery; -import com.tencent.bk.job.manage.service.ScriptService; +import com.tencent.bk.job.manage.service.PublicScriptService; import lombok.extern.slf4j.Slf4j; import org.springframework.web.bind.annotation.RestController; @@ -56,15 +56,17 @@ @Slf4j public class EsbGetPublicScriptListResourceImpl implements EsbGetPublicScriptListResource { - private final ScriptService scriptService; + private final PublicScriptService publicScriptService; - public EsbGetPublicScriptListResourceImpl(ScriptService scriptService) { - this.scriptService = scriptService; + public EsbGetPublicScriptListResourceImpl(PublicScriptService publicScriptService) { + this.publicScriptService = publicScriptService; } @Override @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v2_get_public_script_list"}) - public EsbResp> getPublicScriptList(EsbGetPublicScriptListRequest request) { + public EsbResp> getPublicScriptList(String username, + String appCode, + EsbGetPublicScriptListRequest request) { ValidateResult checkResult = checkRequest(request); if (!checkResult.isPass()) { log.warn("Get public script list, request is illegal!"); @@ -85,8 +87,9 @@ public EsbResp> getPublicScriptList(EsbGetPublicScript BaseSearchCondition baseSearchCondition = new BaseSearchCondition(); baseSearchCondition.setStart(request.getStart()); baseSearchCondition.setLength(request.getLength()); + scriptQuery.setBaseSearchCondition(baseSearchCondition); - PageData pageScripts = scriptService.listPageScriptVersion(scriptQuery, baseSearchCondition); + PageData pageScripts = publicScriptService.listPageScriptVersion(scriptQuery); EsbPageData result = convertToPageEsbScriptDTO(pageScripts, returnScriptContent); return EsbResp.buildSuccessResp(result); } @@ -130,7 +133,7 @@ private EsbPageData convertToPageEsbScriptDTO(PageData } private ValidateResult checkRequest(EsbGetPublicScriptListRequest request) { - if (request.getScriptType() != null && ScriptTypeEnum.valueOf(request.getScriptType()) == null) { + if (request.getScriptType() != null && ScriptTypeEnum.valOf(request.getScriptType()) == null) { log.warn("ScriptType:{} is illegal!", request.getScriptType()); return ValidateResult.fail(ErrorCode.MISSING_OR_ILLEGAL_PARAM_WITH_PARAM_NAME, "script_type"); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/EsbGetScriptDetailResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/EsbGetScriptDetailResourceImpl.java index 34be270237..9f5c5475b4 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/EsbGetScriptDetailResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/EsbGetScriptDetailResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,24 +24,30 @@ package com.tencent.bk.job.manage.api.esb.impl; +import com.tencent.bk.audit.annotations.ActionAuditRecord; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.audit.annotations.AuditInstanceRecord; +import com.tencent.bk.audit.context.ActionAuditContext; +import com.tencent.bk.job.common.audit.constants.EventContentConstants; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.esb.metrics.EsbApiTimed; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.common.esb.util.EsbDTOAppScopeMappingHelper; import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.exception.NotFoundException; +import com.tencent.bk.job.common.iam.constant.ActionId; +import com.tencent.bk.job.common.iam.constant.ResourceTypeId; import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; import com.tencent.bk.job.common.iam.model.AuthResult; import com.tencent.bk.job.common.metrics.CommonMetricNames; import com.tencent.bk.job.common.model.ValidateResult; -import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.manage.api.esb.EsbGetScriptDetailResource; import com.tencent.bk.job.manage.auth.ScriptAuthService; import com.tencent.bk.job.manage.model.dto.ScriptDTO; import com.tencent.bk.job.manage.model.esb.EsbScriptDTO; import com.tencent.bk.job.manage.model.esb.request.EsbGetScriptDetailRequest; -import com.tencent.bk.job.manage.service.ScriptService; +import com.tencent.bk.job.manage.service.ScriptManager; import lombok.extern.slf4j.Slf4j; import org.springframework.web.bind.annotation.RestController; @@ -51,22 +57,28 @@ @RestController @Slf4j public class EsbGetScriptDetailResourceImpl implements EsbGetScriptDetailResource { - private final ScriptService scriptService; + private final ScriptManager scriptManager; private final ScriptAuthService scriptAuthService; - private final AppScopeMappingService appScopeMappingService; - public EsbGetScriptDetailResourceImpl(ScriptService scriptService, - ScriptAuthService scriptAuthService, - AppScopeMappingService appScopeMappingService) { - this.scriptService = scriptService; + public EsbGetScriptDetailResourceImpl(ScriptManager scriptManager, + ScriptAuthService scriptAuthService) { + this.scriptManager = scriptManager; this.scriptAuthService = scriptAuthService; - this.appScopeMappingService = appScopeMappingService; } @Override @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v2_get_script_detail"}) - public EsbResp getScriptDetail(EsbGetScriptDetailRequest request) { - request.fillAppResourceScope(appScopeMappingService); + @AuditEntry(actionId = ActionId.VIEW_SCRIPT) + @ActionAuditRecord( + actionId = ActionId.VIEW_SCRIPT, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.SCRIPT + ), + content = EventContentConstants.VIEW_SCRIPT + ) + public EsbResp getScriptDetail(String username, + String appCode, + EsbGetScriptDetailRequest request) { ValidateResult checkResult = checkRequest(request); if (!checkResult.isPass()) { log.warn("Get script detail, request is illegal!"); @@ -75,16 +87,21 @@ public EsbResp getScriptDetail(EsbGetScriptDetailRequest request) Long appId = request.getAppId(); - ScriptDTO scriptVersion = scriptService.getScriptVersion(request.getScriptVersionId()); + ScriptDTO scriptVersion = scriptManager.getScriptVersion(request.getAppId(), + request.getScriptVersionId()); if (scriptVersion == null) { log.warn("Cannot find scriptVersion by id {}", request.getScriptVersionId()); throw new NotFoundException(ErrorCode.SCRIPT_VERSION_NOT_EXIST); } String scriptId = scriptVersion.getId(); + + // 审计 + ActionAuditContext.current().setInstanceId(scriptId).setInstanceName(scriptVersion.getName()); + // 非公共脚本鉴权 if (!scriptVersion.isPublicScript()) { AuthResult authResult = - scriptAuthService.authViewScript(request.getUserName(), request.getAppResourceScope(), scriptId, null); + scriptAuthService.authViewScript(username, request.getAppResourceScope(), scriptId, null); if (!authResult.isPass()) { throw new PermissionDeniedException(authResult); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/EsbGetScriptListResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/EsbGetScriptListResourceImpl.java index 902c913d1a..eb9d344e95 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/EsbGetScriptListResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/EsbGetScriptListResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -37,12 +37,11 @@ import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; import com.tencent.bk.job.common.model.ValidateResult; -import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.common.util.date.DateUtils; +import com.tencent.bk.job.manage.api.common.constants.JobResourceStatusEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; import com.tencent.bk.job.manage.api.esb.EsbGetScriptListResource; import com.tencent.bk.job.manage.auth.EsbAuthService; -import com.tencent.bk.job.manage.common.consts.JobResourceStatusEnum; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; import com.tencent.bk.job.manage.model.dto.ScriptDTO; import com.tencent.bk.job.manage.model.esb.EsbScriptDTO; import com.tencent.bk.job.manage.model.esb.request.EsbGetScriptListRequest; @@ -65,21 +64,19 @@ public class EsbGetScriptListResourceImpl implements EsbGetScriptListResource { private final ScriptService scriptService; private final EsbAuthService authService; - private final AppScopeMappingService appScopeMappingService; public EsbGetScriptListResourceImpl(ScriptService scriptService, - EsbAuthService authService, - AppScopeMappingService appScopeMappingService) { + EsbAuthService authService) { this.scriptService = scriptService; this.authService = authService; - this.appScopeMappingService = appScopeMappingService; } @Override @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v2_get_script_list"}) - public EsbResp> getScriptList(EsbGetScriptListRequest request) { - request.fillAppResourceScope(appScopeMappingService); + public EsbResp> getScriptList(String username, + String appCode, + EsbGetScriptListRequest request) { ValidateResult checkResult = checkRequest(request); if (!checkResult.isPass()) { log.warn("Get script list, request is illegal!"); @@ -107,8 +104,9 @@ public EsbResp> getScriptList(EsbGetScriptListRequest BaseSearchCondition baseSearchCondition = new BaseSearchCondition(); baseSearchCondition.setStart(request.getStart()); baseSearchCondition.setLength(request.getLength()); + scriptQuery.setBaseSearchCondition(baseSearchCondition); - PageData pageScripts = scriptService.listPageScriptVersion(scriptQuery, baseSearchCondition); + PageData pageScripts = scriptService.listPageScriptVersion(scriptQuery); if (returnScriptContent) { List scriptDTOList = pageScripts.getData(); if (scriptDTOList == null) { @@ -117,12 +115,12 @@ public EsbResp> getScriptList(EsbGetScriptListRequest // 鉴权 Map idNameMap = new HashMap<>(); // 过滤掉公共脚本 - List resourceIds = scriptDTOList.parallelStream().filter(it -> !it.isPublicScript()).map(it -> { + List resourceIds = scriptDTOList.stream().filter(it -> !it.isPublicScript()).map(it -> { idNameMap.put(it.getId(), it.getName()); return it.getId(); }).collect(Collectors.toList()); if (!resourceIds.isEmpty()) { - EsbResp authFailResp = authService.batchAuthJobResources(request.getUserName(), ActionId.VIEW_SCRIPT, + EsbResp authFailResp = authService.batchAuthJobResources(username, ActionId.VIEW_SCRIPT, request.getAppResourceScope(), ResourceTypeEnum.SCRIPT, resourceIds, idNameMap); if (authFailResp != null) { return authFailResp; @@ -186,7 +184,7 @@ private ValidateResult checkRequest(EsbGetScriptListRequest request) { } // 如果script_type=0,表示查询所有类型 if (request.getScriptType() != null && request.getScriptType() > 0 - && ScriptTypeEnum.valueOf(request.getScriptType()) == null) { + && ScriptTypeEnum.valOf(request.getScriptType()) == null) { log.warn("ScriptType:{} is illegal!", request.getScriptType()); return ValidateResult.fail(ErrorCode.MISSING_OR_ILLEGAL_PARAM_WITH_PARAM_NAME, "script_type"); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/v3/EsbAccountResourceV3Impl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/v3/EsbAccountResourceV3Impl.java index 78ec970916..6063c561ae 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/v3/EsbAccountResourceV3Impl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/v3/EsbAccountResourceV3Impl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,18 +24,27 @@ package com.tencent.bk.job.manage.api.esb.impl.v3; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.audit.annotations.AuditRequestBody; +import com.tencent.bk.job.common.constant.AccountCategoryEnum; import com.tencent.bk.job.common.esb.metrics.EsbApiTimed; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.common.esb.model.job.v3.EsbPageDataV3; +import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.metrics.CommonMetricNames; import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.service.AppScopeMappingService; +import com.tencent.bk.job.common.util.date.DateUtils; +import com.tencent.bk.job.manage.api.common.constants.account.AccountTypeEnum; import com.tencent.bk.job.manage.api.esb.v3.EsbAccountV3Resource; import com.tencent.bk.job.manage.model.dto.AccountDTO; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbCreateAccountV3Req; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbDeleteAccountV3Req; import com.tencent.bk.job.manage.model.esb.v3.request.EsbGetAccountListV3Req; import com.tencent.bk.job.manage.model.esb.v3.response.EsbAccountV3DTO; import com.tencent.bk.job.manage.service.AccountService; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RestController; @@ -57,29 +66,51 @@ public EsbAccountResourceV3Impl(AccountService accountService, @Override @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_get_account_list"}) - public EsbResp> getAccountListUsingPost(EsbGetAccountListV3Req request) { - request.fillAppResourceScope(appScopeMappingService); + public EsbResp> getAccountListUsingPost(String username, + String appCode, + EsbGetAccountListV3Req request) { long appId = request.getAppId(); + BaseSearchCondition baseSearchCondition = buildBaseSearchCondition(request.getStart(), request.getLength()); + AccountCategoryEnum category = AccountCategoryEnum.valOf(request.getCategory()); + List accountList = accountService.listAppAccount( + appId, + category, + request.getAccount(), + request.getAlias(), + baseSearchCondition + ); + List accountV3DTOList = convertToEsbAccountV3DTOList(accountList); + Integer accountCount = accountService.countAppAccount( + appId, + category, + request.getAccount(), + request.getAlias() + ); + EsbPageDataV3 esbPageData = new EsbPageDataV3<>(); + esbPageData.setTotal(accountCount.longValue()); + esbPageData.setStart(baseSearchCondition.getStart()); + esbPageData.setLength(baseSearchCondition.getLength()); + esbPageData.setData(accountV3DTOList); + return EsbResp.buildSuccessResp(esbPageData); + } + + private BaseSearchCondition buildBaseSearchCondition(Integer rawStart, Integer rawLength) { BaseSearchCondition baseSearchCondition = new BaseSearchCondition(); int start = 0; - if (request.getStart() != null && request.getStart() > 0) { - start = request.getStart(); + if (rawStart != null && rawStart > 0) { + start = rawStart; } baseSearchCondition.setStart(start); int length = 20; - if (request.getLength() != null && request.getLength() > 0) { - length = request.getLength(); + if (rawLength != null && rawLength > 0) { + length = rawLength; + } + int maxLength = 1000; + if (length > maxLength) { + length = maxLength; } baseSearchCondition.setLength(length); - List accountList = accountService.listAllAppAccount(appId, null, baseSearchCondition); - List accountV3DTOList = convertToEsbAccountV3DTOList(accountList); - Integer accountCount = accountService.countAllAppAccount(appId, null); - EsbPageDataV3 esbPageData = new EsbPageDataV3<>(); - esbPageData.setTotal(accountCount.longValue()); - esbPageData.setStart(start); - esbPageData.setLength(length); - esbPageData.setData(accountV3DTOList); - return EsbResp.buildSuccessResp(esbPageData); + return baseSearchCondition; } private List convertToEsbAccountV3DTOList(List accounts) { @@ -101,17 +132,72 @@ public EsbResp> getAccountList(String username, String scopeType, String scopeId, Integer category, + String account, + String alias, Integer start, Integer length) { EsbGetAccountListV3Req request = new EsbGetAccountListV3Req(); - request.setUserName(username); - request.setAppCode(appCode); request.setBizId(bizId); request.setScopeType(scopeType); request.setScopeId(scopeId); request.setCategory(category); + request.setAccount(account); + request.setAlias(alias); request.setStart(start); request.setLength(length); - return getAccountListUsingPost(request); + request.fillAppResourceScope(appScopeMappingService); + return getAccountListUsingPost(username, appCode, request); + } + + @Override + @AuditEntry(actionId = ActionId.CREATE_ACCOUNT) + public EsbResp createAccount(String username, + String appCode, + @AuditRequestBody EsbCreateAccountV3Req req) { + AccountDTO accountDTO = buildCreateAccountDTO(username, req.getAppId(), req); + AccountDTO createdAccountDTO = accountService.createAccount(username, accountDTO); + return EsbResp.buildSuccessResp(createdAccountDTO.toEsbAccountV3DTO()); + } + + @SuppressWarnings("DuplicatedCode") + public AccountDTO buildCreateAccountDTO(String operator, long appId, EsbCreateAccountV3Req req) { + AccountDTO accountDTO = new AccountDTO(); + accountDTO.setAppId(appId); + accountDTO.setAccount(req.getAccount()); + accountDTO.setCategory(AccountCategoryEnum.valOf(req.getCategory())); + accountDTO.setType(AccountTypeEnum.valueOf(req.getType())); + if (StringUtils.isBlank(req.getAlias())) { + accountDTO.setAlias(req.getAccount()); + } else { + accountDTO.setAlias(req.getAlias()); + } + accountDTO.setRemark(req.getRemark()); + + if (AccountCategoryEnum.SYSTEM.getValue().equals(req.getCategory())) { + if (AccountTypeEnum.WINDOW.getType().equals(req.getType())) { + accountDTO.setOs("Windows"); + } else { + accountDTO.setOs("Linux"); + } + accountDTO.setPassword(req.getPassword()); + } + + accountDTO.setCreator(operator); + accountDTO.setCreateTime(DateUtils.currentTimeMillis()); + accountDTO.setLastModifyUser(operator); + accountDTO.setLastModifyTime(DateUtils.currentTimeMillis()); + + return accountDTO; + } + + @Override + @AuditEntry(actionId = ActionId.MANAGE_ACCOUNT) + public EsbResp deleteAccountUsingPost(String username, + String appCode, + @AuditRequestBody EsbDeleteAccountV3Req req) { + Long appId = appScopeMappingService.getAppIdByScope(req.getScopeType(), req.getScopeId()); + AccountDTO accountDTO = accountService.getAccount(appId, req.getId()); + accountService.deleteAccount(username, appId, req.getId()); + return EsbResp.buildSuccessResp(accountDTO.toEsbAccountV3DTO()); } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/v3/EsbAgentInfoResourceV3Impl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/v3/EsbAgentInfoResourceV3Impl.java new file mode 100644 index 0000000000..a2dbd96c3e --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/v3/EsbAgentInfoResourceV3Impl.java @@ -0,0 +1,182 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.api.esb.impl.v3; + +import com.tencent.bk.job.common.esb.model.EsbResp; +import com.tencent.bk.job.common.gse.constants.AgentStateStatusEnum; +import com.tencent.bk.job.common.gse.v2.model.req.ListAgentStateReq; +import com.tencent.bk.job.common.gse.v2.model.resp.AgentState; +import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; +import com.tencent.bk.job.common.model.dto.ResourceScope; +import com.tencent.bk.job.manage.api.esb.v3.EsbAgentInfoV3Resource; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbQueryAgentInfoV3Req; +import com.tencent.bk.job.manage.model.esb.v3.response.EsbAgentInfoV3DTO; +import com.tencent.bk.job.manage.model.esb.v3.response.EsbQueryAgentInfoV3Resp; +import com.tencent.bk.job.manage.service.agent.status.ScopeAgentStatusService; +import com.tencent.bk.job.manage.service.host.HostService; +import com.tencent.bk.job.manage.service.host.ScopeHostService; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections.CollectionUtils; +import org.jooq.tools.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RestController; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; + +@RestController +@Slf4j +public class EsbAgentInfoResourceV3Impl implements EsbAgentInfoV3Resource { + + private final HostService hostService; + private final ScopeAgentStatusService scopeAgentStatusService; + private final ScopeHostService scopeHostService; + + @Autowired + public EsbAgentInfoResourceV3Impl(HostService hostService, + ScopeAgentStatusService scopeAgentStatusService, + ScopeHostService scopeHostService) { + this.hostService = hostService; + this.scopeAgentStatusService = scopeAgentStatusService; + this.scopeHostService = scopeHostService; + } + + @Override + public EsbResp queryAgentInfo(String username, + String appCode, + EsbQueryAgentInfoV3Req req) { + ResourceScope resourceScope = new ResourceScope(req.getScopeType(), req.getScopeId()); + boolean needToUseGseV2 = scopeAgentStatusService.needToUseGseV2(resourceScope); + + List hostIdList = req.getHostIdList(); + List validHostIdList = scopeHostService.filterScopeHostIds( + req.getAppResourceScope(), + new HashSet<>(hostIdList) + ); + hostIdList.removeAll(validHostIdList); + if (CollectionUtils.isNotEmpty(hostIdList)) { + log.warn("Ignore hostIds not in {}:{}", resourceScope, hostIdList); + } + Map hostIdAgentIdMap = generateHostIdAgentIdMap(needToUseGseV2, validHostIdList); + + ListAgentStateReq listAgentStateReq = new ListAgentStateReq(); + listAgentStateReq.setAgentIdList(new ArrayList<>(new HashSet<>(hostIdAgentIdMap.values()))); + List agentStateList = scopeAgentStatusService.listAgentState(resourceScope, listAgentStateReq); + + List agentInfoList = buildEsbAgentInfoList( + agentStateList, + validHostIdList, + hostIdAgentIdMap, + needToUseGseV2 + ); + + EsbQueryAgentInfoV3Resp resp = new EsbQueryAgentInfoV3Resp(); + resp.setAgentInfoList(agentInfoList); + return EsbResp.buildSuccessResp(resp); + } + + private Map generateHostIdAgentIdMap(boolean needToUseGseV2, List hostIdList) { + Map hostInfoMap = hostService.listHostsByHostIds(hostIdList); + Map hostIdAgentIdMap = new HashMap<>(); + if (needToUseGseV2) { + // 使用AgentId + for (Long hostId : hostIdList) { + ApplicationHostDTO hostDTO = hostInfoMap.get(hostId); + if (hostDTO != null) { + String agentId = hostDTO.getAgentId(); + if (!StringUtils.isBlank(agentId)) { + hostIdAgentIdMap.put(hostId, agentId); + } else { + log.info("AgentId of host({}) is blank, ignore", hostDTO); + } + } else { + log.info("Cannot find host by hostId:{}", hostId); + } + } + return hostIdAgentIdMap; + } + // 使用CloudIp + for (Long hostId : hostIdList) { + ApplicationHostDTO hostDTO = hostInfoMap.get(hostId); + if (hostDTO != null) { + String cloudIp = hostDTO.getCloudIp(); + if (!StringUtils.isBlank(cloudIp)) { + hostIdAgentIdMap.put(hostId, cloudIp); + } else { + log.info("cloudIp of host({}) is blank, ignore", hostDTO); + } + } else { + log.info("Cannot find host by hostId:{}", hostId); + } + } + return hostIdAgentIdMap; + } + + private List buildEsbAgentInfoList(List agentStateList, + List hostIdList, + Map hostIdAgentIdMap, + boolean needToUseGseV2) { + Map agentStateMap = new HashMap<>(); + for (AgentState agentState : agentStateList) { + agentStateMap.put(agentState.getAgentId(), agentState); + } + List agentInfoList = new ArrayList<>(); + for (Long hostId : hostIdList) { + String agentId = hostIdAgentIdMap.get(hostId); + if (StringUtils.isBlank(agentId)) { + log.info("Cannot find agentId by hostId={}", hostId); + continue; + } + AgentState agentState = agentStateMap.get(agentId); + if (agentState == null) { + log.warn("Cannot find agentState from gse by hostId={}, agentId={}", hostId, agentId); + continue; + } + EsbAgentInfoV3DTO agentInfoV3DTO = new EsbAgentInfoV3DTO(); + agentInfoV3DTO.setHostId(hostId); + agentInfoV3DTO.setStatus(getEsbAgentStatus(agentState)); + String agentVersion = agentState.getVersion(); + if (StringUtils.isBlank(agentVersion)) { + if (!needToUseGseV2) { + agentInfoV3DTO.setVersion("V1"); + } + } else { + agentInfoV3DTO.setVersion(agentVersion); + } + agentInfoList.add(agentInfoV3DTO); + } + return agentInfoList; + } + + private Integer getEsbAgentStatus(AgentState agentState) { + if (agentState.getStatusCode() == AgentStateStatusEnum.RUNNING.getValue()) { + return 1; + } + return 0; + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/v3/EsbCredentialResourceV3Impl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/v3/EsbCredentialResourceV3Impl.java index 4b83291fb3..76d1bd63ad 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/v3/EsbCredentialResourceV3Impl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/v3/EsbCredentialResourceV3Impl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,21 +24,24 @@ package com.tencent.bk.job.manage.api.esb.impl.v3; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.audit.annotations.AuditRequestBody; import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.esb.metrics.EsbApiTimed; import com.tencent.bk.job.common.esb.model.EsbResp; -import com.tencent.bk.job.common.exception.InternalException; import com.tencent.bk.job.common.exception.InvalidParamException; -import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; -import com.tencent.bk.job.common.iam.model.AuthResult; -import com.tencent.bk.job.common.model.InternalResponse; +import com.tencent.bk.job.common.iam.constant.ActionId; +import com.tencent.bk.job.common.metrics.CommonMetricNames; import com.tencent.bk.job.common.service.AppScopeMappingService; +import com.tencent.bk.job.manage.api.common.constants.CredentialTypeEnum; import com.tencent.bk.job.manage.api.esb.v3.EsbCredentialV3Resource; -import com.tencent.bk.job.manage.api.inner.ServiceCredentialResource; -import com.tencent.bk.job.manage.common.consts.CredentialTypeEnum; +import com.tencent.bk.job.manage.model.dto.CredentialDTO; import com.tencent.bk.job.manage.model.esb.v3.request.EsbCreateOrUpdateCredentialV3Req; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbGetCredentialDetailV3Req; import com.tencent.bk.job.manage.model.esb.v3.response.EsbCredentialSimpleInfoV3DTO; -import com.tencent.bk.job.manage.model.inner.resp.ServiceBasicCredentialDTO; +import com.tencent.bk.job.manage.model.esb.v3.response.EsbCredentialV3DTO; import com.tencent.bk.job.manage.model.web.request.CredentialCreateUpdateReq; +import com.tencent.bk.job.manage.service.CredentialService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -48,28 +51,31 @@ @RestController @Slf4j public class EsbCredentialResourceV3Impl implements EsbCredentialV3Resource { - private final ServiceCredentialResource credentialService; + private final CredentialService credentialService; private final AppScopeMappingService appScopeMappingService; @Autowired - public EsbCredentialResourceV3Impl(ServiceCredentialResource credentialService, + public EsbCredentialResourceV3Impl(CredentialService credentialService, AppScopeMappingService appScopeMappingService) { this.credentialService = credentialService; this.appScopeMappingService = appScopeMappingService; } @Override - public EsbResp createCredential(EsbCreateOrUpdateCredentialV3Req req) { - req.fillAppResourceScope(appScopeMappingService); + @AuditEntry( + actionId = ActionId.CREATE_TICKET + ) + public EsbResp createCredential( + String username, + String appCode, + @AuditRequestBody EsbCreateOrUpdateCredentialV3Req req) { checkCreateParam(req); - return saveCredential(req); - } - @Override - public EsbResp updateCredential(EsbCreateOrUpdateCredentialV3Req req) { - req.fillAppResourceScope(appScopeMappingService); - checkUpdateParam(req); - return saveCredential(req); + CredentialCreateUpdateReq createUpdateReq = convertToCreateUpdateReq(req); + CredentialDTO createCredential = credentialService.createCredential(username, req.getAppId(), + createUpdateReq); + + return EsbResp.buildSuccessResp(createCredential.toEsbCredentialSimpleInfoV3DTO()); } private void checkCreateParam(EsbCreateOrUpdateCredentialV3Req req) { @@ -85,6 +91,51 @@ private void checkCreateParam(EsbCreateOrUpdateCredentialV3Req req) { } } + @Override + @AuditEntry( + actionId = ActionId.MANAGE_TICKET + ) + public EsbResp updateCredential( + String username, + String appCode, + @AuditRequestBody EsbCreateOrUpdateCredentialV3Req req) { + checkUpdateParam(req); + + CredentialCreateUpdateReq createUpdateReq = convertToCreateUpdateReq(req); + CredentialDTO updateCredential = credentialService.updateCredential(username, req.getAppId(), + createUpdateReq); + + return EsbResp.buildSuccessResp(updateCredential.toEsbCredentialSimpleInfoV3DTO()); + } + + @Override + @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "get_credential_detail"}) + public EsbResp getCredentialDetail( + String username, + String appCode, + Long bizId, + String scopeType, + String scopeId, + String id) { + EsbGetCredentialDetailV3Req req = new EsbGetCredentialDetailV3Req(); + req.setBizId(bizId); + req.setScopeType(scopeType); + req.setScopeId(scopeId); + req.setId(id); + req.fillAppResourceScope(appScopeMappingService); + return getCredentialDetailUsingPost(username, appCode, req); + } + + @Override + @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "get_credential_detail"}) + public EsbResp getCredentialDetailUsingPost( + String username, + String appCode, + @AuditRequestBody EsbGetCredentialDetailV3Req req) { + CredentialDTO credentialDTO = credentialService.getCredentialById(req.getId()); + return EsbResp.buildSuccessResp(credentialDTO.toEsbCredentialV3DTO()); + } + private void checkUpdateParam(EsbCreateOrUpdateCredentialV3Req req) { if (StringUtils.isBlank(req.getId())) { throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME_AND_REASON, @@ -92,31 +143,6 @@ private void checkUpdateParam(EsbCreateOrUpdateCredentialV3Req req) { } } - private EsbResp saveCredential(EsbCreateOrUpdateCredentialV3Req req) { - CredentialCreateUpdateReq createUpdateReq = convertToCreateUpdateReq(req); - InternalResponse resp; - if (req.getId() == null) { - resp = credentialService.createCredential( - req.getUserName(), - req.getAppId(), - createUpdateReq - ); - } else { - resp = credentialService.updateCredential( - req.getUserName(), - req.getAppId(), - createUpdateReq - ); - } - if (resp.getAuthResult() != null) { - throw new PermissionDeniedException(AuthResult.fromAuthResultDTO(resp.getAuthResult())); - } else if (!resp.isSuccess()) { - throw new InternalException(resp.getCode()); - } - ServiceBasicCredentialDTO data = resp.getData(); - return EsbResp.buildSuccessResp(new EsbCredentialSimpleInfoV3DTO(data.getId())); - } - private CredentialCreateUpdateReq convertToCreateUpdateReq( EsbCreateOrUpdateCredentialV3Req req ) { diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/v3/EsbDangerousRuleV3ResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/v3/EsbDangerousRuleV3ResourceImpl.java new file mode 100644 index 0000000000..bcd289f32a --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/v3/EsbDangerousRuleV3ResourceImpl.java @@ -0,0 +1,149 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.api.esb.impl.v3; + +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.audit.annotations.AuditRequestBody; +import com.tencent.bk.job.common.esb.metrics.EsbApiTimed; +import com.tencent.bk.job.common.esb.model.EsbResp; +import com.tencent.bk.job.common.iam.constant.ActionId; +import com.tencent.bk.job.common.metrics.CommonMetricNames; +import com.tencent.bk.job.manage.api.common.constants.EnableStatusEnum; +import com.tencent.bk.job.manage.api.esb.v3.EsbDangerousRuleV3Resource; +import com.tencent.bk.job.manage.model.dto.globalsetting.DangerousRuleDTO; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbCreateDangerousRuleV3Req; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbGetDangerousRuleV3Req; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbManageDangerousRuleV3Req; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbUpdateDangerousRuleV3Req; +import com.tencent.bk.job.manage.model.esb.v3.response.EsbDangerousRuleV3DTO; +import com.tencent.bk.job.manage.model.query.DangerousRuleQuery; +import com.tencent.bk.job.manage.model.web.request.globalsetting.AddOrUpdateDangerousRuleReq; +import com.tencent.bk.job.manage.model.web.vo.globalsetting.DangerousRuleVO; +import com.tencent.bk.job.manage.service.DangerousRuleService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RestController; + +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +@RestController +@Slf4j +public class EsbDangerousRuleV3ResourceImpl implements EsbDangerousRuleV3Resource { + private final DangerousRuleService dangerousRuleService; + + @Autowired + public EsbDangerousRuleV3ResourceImpl(DangerousRuleService dangerousRuleService) { + this.dangerousRuleService = dangerousRuleService; + } + + @Override + @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_create_dangerous_rule"}) + @AuditEntry(actionId = ActionId.HIGH_RISK_DETECT_RULE) + public EsbResp createDangerousRule(String username, + String appCode, + @AuditRequestBody EsbCreateDangerousRuleV3Req request) { + AddOrUpdateDangerousRuleReq req = new AddOrUpdateDangerousRuleReq(); + req.setExpression(request.getExpression()); + req.setScriptTypeList(request.getScriptTypeList()); + req.setDescription(request.getDescription()); + req.setAction(request.getAction()); + DangerousRuleDTO dangerousRule = dangerousRuleService.createDangerousRule(username, req); + return EsbResp.buildSuccessResp(dangerousRule.toEsbDangerousRuleV3DTO()); + } + + @Override + @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_update_dangerous_rule"}) + @AuditEntry(actionId = ActionId.HIGH_RISK_DETECT_RULE) + public EsbResp updateDangerousRule(String username, + String appCode, + @AuditRequestBody EsbUpdateDangerousRuleV3Req request) { + AddOrUpdateDangerousRuleReq req = new AddOrUpdateDangerousRuleReq(); + DangerousRuleDTO dangerousRuleDTO = dangerousRuleService.getDangerousRuleById(request.getId()); + req.setId(request.getId()); + req.setExpression(request.getExpression()); + req.setScriptTypeList(request.getScriptTypeList()); + req.setDescription(request.getDescription()); + req.setAction(request.getAction()); + req.setStatus(dangerousRuleDTO.getStatus()); + DangerousRuleDTO updateDangerousRuleDTO = dangerousRuleService.updateDangerousRule(username, req); + return EsbResp.buildSuccessResp(updateDangerousRuleDTO.toEsbDangerousRuleV3DTO()); + } + + @Override + @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_delete_dangerous_rule"}) + @AuditEntry(actionId = ActionId.HIGH_RISK_DETECT_RULE) + public EsbResp deleteDangerousRule(String username, + String appCode, + @AuditRequestBody EsbManageDangerousRuleV3Req request) { + dangerousRuleService.deleteDangerousRuleById(username, request.getId()); + return EsbResp.buildSuccessResp(null); + } + + @Override + @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_get_dangerous_rule_list"}) + @AuditEntry(actionId = ActionId.HIGH_RISK_DETECT_RULE) + public EsbResp> getDangerousRuleListUsingPost( + String username, + String appCode, + @AuditRequestBody EsbGetDangerousRuleV3Req request) { + DangerousRuleQuery query = DangerousRuleQuery.builder() + .expression(request.getExpression()) + .description(request.getDescription()) + .scriptTypeList(request.getScriptTypeList()) + .action(request.getAction() != null ? Collections.singletonList(request.getAction().byteValue()) : null) + .build(); + List dangerousRuleVOS = dangerousRuleService.listDangerousRules(query); + List esbDangerousRuleV3DTOList = dangerousRuleVOS.stream() + .map(DangerousRuleVO::toEsbDangerousRuleV3DTO) + .collect(Collectors.toList()); + return EsbResp.buildSuccessResp(esbDangerousRuleV3DTOList); + } + + @Override + @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_enable_dangerous_rule"}) + @AuditEntry(actionId = ActionId.HIGH_RISK_DETECT_RULE) + public EsbResp enableDangerousRule(String username, + String appCode, + @AuditRequestBody EsbManageDangerousRuleV3Req request) { + DangerousRuleDTO dangerousRuleDTO = dangerousRuleService.updateDangerousRuleStatus(username, + request.getId(), + EnableStatusEnum.ENABLED); + return EsbResp.buildSuccessResp(dangerousRuleDTO.toEsbManageDangerousRuleV3DTO()); + } + + @Override + @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_disable_dangerous_rule"}) + @AuditEntry(actionId = ActionId.HIGH_RISK_DETECT_RULE) + public EsbResp disableDangerousRule(String username, + String appCode, + @AuditRequestBody EsbManageDangerousRuleV3Req request) { + DangerousRuleDTO dangerousRuleDTO = dangerousRuleService.updateDangerousRuleStatus(username, + request.getId(), + EnableStatusEnum.DISABLED); + return EsbResp.buildSuccessResp(dangerousRuleDTO.toEsbManageDangerousRuleV3DTO()); + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/v3/EsbLocalFileResourceV3Impl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/v3/EsbLocalFileResourceV3Impl.java index 5cd6dc4b2e..62f979674d 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/v3/EsbLocalFileResourceV3Impl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/v3/EsbLocalFileResourceV3Impl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,23 +24,23 @@ package com.tencent.bk.job.manage.api.esb.impl.v3; +import com.tencent.bk.job.common.artifactory.config.ArtifactoryConfig; import com.tencent.bk.job.common.artifactory.model.dto.TempUrlInfo; import com.tencent.bk.job.common.artifactory.sdk.ArtifactoryClient; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.esb.constants.EsbConsts; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.common.exception.InvalidParamException; -import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.common.util.StringUtil; import com.tencent.bk.job.common.util.Utils; import com.tencent.bk.job.common.util.check.ParamCheckUtil; import com.tencent.bk.job.manage.api.esb.v3.EsbLocalFileV3Resource; -import com.tencent.bk.job.manage.config.ArtifactoryConfig; import com.tencent.bk.job.manage.config.LocalFileConfigForManage; import com.tencent.bk.job.manage.model.esb.v3.request.EsbGenLocalFileUploadUrlV3Req; import com.tencent.bk.job.manage.model.esb.v3.response.EsbUploadUrlV3DTO; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.web.bind.annotation.RestController; import java.io.File; @@ -57,22 +57,20 @@ public class EsbLocalFileResourceV3Impl implements EsbLocalFileV3Resource { private final ArtifactoryConfig artifactoryConfig; private final LocalFileConfigForManage localFileConfigForManage; private final ArtifactoryClient artifactoryClient; - private final AppScopeMappingService appScopeMappingService; @Autowired public EsbLocalFileResourceV3Impl(ArtifactoryConfig artifactoryConfig, LocalFileConfigForManage localFileConfigForManage, - ArtifactoryClient artifactoryClient, - AppScopeMappingService appScopeMappingService) { + @Qualifier("jobArtifactoryClient") ArtifactoryClient artifactoryClient) { this.artifactoryConfig = artifactoryConfig; this.localFileConfigForManage = localFileConfigForManage; this.artifactoryClient = artifactoryClient; - this.appScopeMappingService = appScopeMappingService; } @Override - public EsbResp generateLocalFileUploadUrl(EsbGenLocalFileUploadUrlV3Req req) { - req.fillAppResourceScope(appScopeMappingService); + public EsbResp generateLocalFileUploadUrl(String username, + String appCode, + EsbGenLocalFileUploadUrlV3Req req) { // 参数检查 // fileNameList List fileNameList = req.getFileNameList(); @@ -92,7 +90,7 @@ public EsbResp generateLocalFileUploadUrl(EsbGenLocalFileUplo sb.append(File.separatorChar); sb.append(Utils.getUUID()); sb.append(File.separator); - sb.append(req.getUserName()); + sb.append(username); sb.append(File.separatorChar); sb.append(fileName); String filePath = sb.toString(); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/v3/EsbPlanV3ResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/v3/EsbPlanV3ResourceImpl.java index 5b56389487..938af56e0c 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/v3/EsbPlanV3ResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/v3/EsbPlanV3ResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,23 +24,23 @@ package com.tencent.bk.job.manage.api.esb.impl.v3; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.audit.annotations.AuditRequestBody; import com.tencent.bk.job.common.esb.metrics.EsbApiTimed; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.common.esb.model.job.v3.EsbPageDataV3; import com.tencent.bk.job.common.esb.util.EsbDTOAppScopeMappingHelper; -import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; -import com.tencent.bk.job.common.iam.model.AuthResult; -import com.tencent.bk.job.common.iam.service.BusinessAuthService; +import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.metrics.CommonMetricNames; import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; import com.tencent.bk.job.common.model.ValidateResult; import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.manage.api.esb.v3.EsbPlanV3Resource; -import com.tencent.bk.job.manage.auth.PlanAuthService; import com.tencent.bk.job.manage.manager.variable.StepRefVariableParser; import com.tencent.bk.job.manage.model.dto.TaskPlanQueryDTO; import com.tencent.bk.job.manage.model.dto.task.TaskPlanInfoDTO; +import com.tencent.bk.job.manage.model.dto.task.TaskStepDTO; import com.tencent.bk.job.manage.model.esb.v3.request.EsbGetPlanDetailV3Request; import com.tencent.bk.job.manage.model.esb.v3.request.EsbGetPlanListV3Request; import com.tencent.bk.job.manage.model.esb.v3.response.EsbPlanBasicInfoV3DTO; @@ -50,6 +50,9 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RestController; +import java.util.List; +import java.util.stream.Collectors; + /** * @since 15/10/2020 18:08 */ @@ -58,22 +61,17 @@ public class EsbPlanV3ResourceImpl implements EsbPlanV3Resource { private final TaskPlanService taskPlanService; - private final BusinessAuthService businessAuthService; - private final PlanAuthService planAuthService; private final AppScopeMappingService appScopeMappingService; @Autowired public EsbPlanV3ResourceImpl(TaskPlanService taskPlanService, - BusinessAuthService businessAuthService, - PlanAuthService planAuthService, AppScopeMappingService appScopeMappingService) { this.taskPlanService = taskPlanService; - this.businessAuthService = businessAuthService; - this.planAuthService = planAuthService; this.appScopeMappingService = appScopeMappingService; } @Override + @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_get_job_plan_list"}) public EsbResp> getPlanList(String username, String appCode, Long bizId, @@ -90,8 +88,6 @@ public EsbResp> getPlanList(String username Integer start, Integer length) { EsbGetPlanListV3Request request = new EsbGetPlanListV3Request(); - request.setUserName(username); - request.setAppCode(appCode); request.setBizId(bizId); request.setScopeType(scopeType); request.setScopeId(scopeId); @@ -105,26 +101,32 @@ public EsbResp> getPlanList(String username request.setLastModifyTimeEnd(lastModifyTimeEnd); request.setStart(start); request.setLength(length); - return getPlanListUsingPost(request); + request.fillAppResourceScope(appScopeMappingService); + return getPlanListUsingPost(username, appCode, request); } @Override - public EsbResp getPlanDetail(String username, String appCode, Long bizId, - String scopeType, String scopeId, Long planId) { + @AuditEntry(actionId = ActionId.VIEW_JOB_PLAN) + public EsbResp getPlanDetail(String username, + String appCode, + Long bizId, + String scopeType, + String scopeId, + Long planId) { EsbGetPlanDetailV3Request request = new EsbGetPlanDetailV3Request(); - request.setUserName(username); - request.setAppCode(appCode); request.setBizId(bizId); request.setScopeType(scopeType); request.setScopeId(scopeId); request.setPlanId(planId); - return getPlanDetailUsingPost(request); + request.fillAppResourceScope(appScopeMappingService); + return getPlanDetailUsingPost(username, appCode, request); } @Override @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_get_job_plan_list"}) - public EsbResp> getPlanListUsingPost(EsbGetPlanListV3Request request) { - request.fillAppResourceScope(appScopeMappingService); + public EsbResp> getPlanListUsingPost(String username, + String appCode, + EsbGetPlanListV3Request request) { ValidateResult checkResult = checkRequest(request); if (!checkResult.isPass()) { log.warn("Get plan list, request is illegal!"); @@ -133,13 +135,6 @@ public EsbResp> getPlanListUsingPost(EsbGet long appId = request.getAppId(); - AuthResult authResult = - businessAuthService.authAccessBusiness( - request.getUserName(), request.getAppResourceScope()); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } - TaskPlanQueryDTO taskPlanQueryDTO = new TaskPlanQueryDTO(); taskPlanQueryDTO.setAppId(appId); taskPlanQueryDTO.setName(request.getName()); @@ -173,29 +168,23 @@ public EsbResp> getPlanListUsingPost(EsbGet @Override @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_get_job_plan_detail"}) - public EsbResp getPlanDetailUsingPost(EsbGetPlanDetailV3Request request) { - request.fillAppResourceScope(appScopeMappingService); - ValidateResult validateResult = request.validate(); - if (validateResult.isPass()) { - TaskPlanInfoDTO taskPlanInfo = taskPlanService.getTaskPlanById(request.getAppId(), request.getPlanId()); - if (taskPlanInfo != null) { - AuthResult authResult = - planAuthService.authViewJobPlan(request.getUserName(), - request.getAppResourceScope(), - taskPlanInfo.getTemplateId(), request.getPlanId(), taskPlanInfo.getName()); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } - - // 解析步骤引用全局变量的信息 - StepRefVariableParser.parseStepRefVars(taskPlanInfo.getStepList(), taskPlanInfo.getVariableList()); - return EsbResp.buildSuccessResp(TaskPlanInfoDTO.toEsbPlanInfoV3(taskPlanInfo)); - } - return EsbResp.buildSuccessResp(null); - } else { - log.warn("Get plan detail request is illegal!"); - return EsbResp.buildCommonFailResp(validateResult); - } + @AuditEntry(actionId = ActionId.VIEW_JOB_PLAN) + public EsbResp getPlanDetailUsingPost(String username, + String appCode, + @AuditRequestBody EsbGetPlanDetailV3Request request) { + request.validate(); + + TaskPlanInfoDTO taskPlanInfo = taskPlanService.getTaskPlan(username, + request.getAppId(), request.getPlanId()); + + List enabledTaskStepList = taskPlanInfo.getStepList() + .stream() + .filter(taskStep -> taskStep.getEnable() != 0) + .collect(Collectors.toList()); + + // 解析启用的步骤引用全局变量的信息 + StepRefVariableParser.parseStepRefVars(enabledTaskStepList, taskPlanInfo.getVariableList()); + return EsbResp.buildSuccessResp(TaskPlanInfoDTO.toEsbPlanInfoV3(taskPlanInfo)); } private ValidateResult checkRequest(EsbGetPlanListV3Request request) { diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/v3/EsbPublicScriptResourceV3Impl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/v3/EsbPublicScriptResourceV3Impl.java index 9b4b55c787..ff597adbd1 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/v3/EsbPublicScriptResourceV3Impl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/v3/EsbPublicScriptResourceV3Impl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,29 +24,42 @@ package com.tencent.bk.job.manage.api.esb.impl.v3; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.audit.annotations.AuditRequestBody; import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.constant.JobCommonHeaders; import com.tencent.bk.job.common.esb.metrics.EsbApiTimed; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.common.esb.model.job.v3.EsbPageDataV3; import com.tencent.bk.job.common.exception.InvalidParamException; +import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.metrics.CommonMetricNames; import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; +import com.tencent.bk.job.manage.api.common.ScriptDTOBuilder; +import com.tencent.bk.job.manage.api.common.constants.JobResourceStatusEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; import com.tencent.bk.job.manage.api.esb.v3.EsbPublicScriptV3Resource; -import com.tencent.bk.job.manage.common.consts.JobResourceStatusEnum; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; import com.tencent.bk.job.manage.model.dto.ScriptDTO; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbCreatePublicScriptV3Req; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbCreatePublicScriptVersionV3Req; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbDeletePublicScriptV3Req; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbDeletePublicScriptVersionV3Req; import com.tencent.bk.job.manage.model.esb.v3.request.EsbGetPublicScriptListV3Request; import com.tencent.bk.job.manage.model.esb.v3.request.EsbGetPublicScriptVersionDetailV3Request; import com.tencent.bk.job.manage.model.esb.v3.request.EsbGetPublicScriptVersionListV3Request; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbManagePublicScriptVersionV3Req; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbUpdatePublicScriptBasicV3Req; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbUpdatePublicScriptVersionV3Req; import com.tencent.bk.job.manage.model.esb.v3.response.EsbScriptV3DTO; import com.tencent.bk.job.manage.model.esb.v3.response.EsbScriptVersionDetailV3DTO; import com.tencent.bk.job.manage.model.query.ScriptQuery; -import com.tencent.bk.job.manage.service.ScriptService; +import com.tencent.bk.job.manage.service.PublicScriptService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RequestHeader; import org.springframework.web.bind.annotation.RestController; import java.util.ArrayList; @@ -59,14 +72,18 @@ @Slf4j public class EsbPublicScriptResourceV3Impl implements EsbPublicScriptV3Resource { - private final ScriptService scriptService; + private final ScriptDTOBuilder scriptDTOBuilder; + private final PublicScriptService publicScriptService; @Autowired - public EsbPublicScriptResourceV3Impl(ScriptService scriptService) { - this.scriptService = scriptService; + public EsbPublicScriptResourceV3Impl(PublicScriptService publicScriptService, + ScriptDTOBuilder scriptDTOBuilder) { + this.publicScriptService = publicScriptService; + this.scriptDTOBuilder = scriptDTOBuilder; } @Override + @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_get_public_script_list"}) public EsbResp> getPublicScriptList(String username, String appCode, String name, @@ -74,16 +91,15 @@ public EsbResp> getPublicScriptList(String usernam Integer start, Integer length) { EsbGetPublicScriptListV3Request request = new EsbGetPublicScriptListV3Request(); - request.setUserName(username); - request.setAppCode(appCode); request.setName(name); request.setScriptLanguage(scriptLanguage); request.setStart(start); request.setLength(length); - return getPublicScriptListUsingPost(request); + return getPublicScriptListUsingPost(username, appCode, request); } @Override + @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_get_public_script_version_list"}) public EsbResp> getPublicScriptVersionList(String username, String appCode, String scriptId, @@ -91,37 +107,37 @@ public EsbResp> getPublicScriptVersio Integer start, Integer length) { EsbGetPublicScriptVersionListV3Request request = new EsbGetPublicScriptVersionListV3Request(); - request.setUserName(username); - request.setAppCode(appCode); request.setScriptId(scriptId); request.setReturnScriptContent(returnScriptContent); request.setStart(start); request.setLength(length); - return getPublicScriptVersionListUsingPost(request); + return getPublicScriptVersionListUsingPost(username, appCode, request); } @Override + @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_get_script_version_detail"}) public EsbResp getPublicScriptVersionDetail(String username, String appCode, Long scriptVersionId, String scriptId, String version) { EsbGetPublicScriptVersionDetailV3Request request = new EsbGetPublicScriptVersionDetailV3Request(); - request.setUserName(username); - request.setAppCode(appCode); request.setId(scriptVersionId); request.setScriptId(scriptId); request.setVersion(version); - return getPublicScriptVersionDetailUsingPost(request); + return getPublicScriptVersionDetailUsingPost(username, appCode, request); } @Override @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_get_public_script_list"}) public EsbResp> getPublicScriptListUsingPost( + String username, + String appCode, EsbGetPublicScriptListV3Request request) { checkEsbGetPublicScriptListV3Req(request); ScriptQuery scriptQuery = new ScriptQuery(); + scriptQuery.setAppId(PUBLIC_APP_ID); scriptQuery.setPublicScript(true); scriptQuery.setName(request.getName()); // 如果script_type=0,表示查询所有类型,不需要传查询条件 @@ -133,8 +149,9 @@ public EsbResp> getPublicScriptListUsingPost( BaseSearchCondition baseSearchCondition = new BaseSearchCondition(); baseSearchCondition.setStart(request.getStart()); baseSearchCondition.setLength(request.getLength()); + scriptQuery.setBaseSearchCondition(baseSearchCondition); - PageData pageScripts = scriptService.listPageScript(scriptQuery, baseSearchCondition); + PageData pageScripts = publicScriptService.listPageScript(scriptQuery); setOnlineScriptVersionInfo(pageScripts.getData()); EsbPageDataV3 result = EsbPageDataV3.from(pageScripts, ScriptDTO::toEsbScriptV3DTO); @@ -144,17 +161,21 @@ public EsbResp> getPublicScriptListUsingPost( @Override @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_get_public_script_version_list"}) public EsbResp> getPublicScriptVersionListUsingPost( + String username, + String appCode, EsbGetPublicScriptVersionListV3Request request) { checkEsbGetPublicScriptVersionListV3Req(request); ScriptQuery scriptQuery = new ScriptQuery(); + scriptQuery.setAppId(PUBLIC_APP_ID); scriptQuery.setPublicScript(true); scriptQuery.setId(request.getScriptId()); BaseSearchCondition baseSearchCondition = BaseSearchCondition.pageCondition(request.getStart(), request.getLength()); + scriptQuery.setBaseSearchCondition(baseSearchCondition); - PageData pageScriptVersions = scriptService.listPageScriptVersion(scriptQuery, baseSearchCondition); + PageData pageScriptVersions = publicScriptService.listPageScriptVersion(scriptQuery); EsbPageDataV3 result = EsbPageDataV3.from(pageScriptVersions, ScriptDTO::toEsbScriptVersionDetailV3DTO); @@ -169,6 +190,8 @@ public EsbResp> getPublicScriptVersio @Override @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_get_public_script_version_detail"}) public EsbResp getPublicScriptVersionDetailUsingPost( + String username, + String appCode, EsbGetPublicScriptVersionDetailV3Request request) { checkEsbGetPublicScriptVersionDetailV3Req(request); @@ -177,9 +200,9 @@ public EsbResp getPublicScriptVersionDetailUsingPos Long id = request.getId(); ScriptDTO scriptVersion; if (id != null && id > 0) { - scriptVersion = scriptService.getScriptVersion(null, PUBLIC_APP_ID, id); + scriptVersion = publicScriptService.getScriptVersion(id); } else { - scriptVersion = scriptService.getByScriptIdAndVersion(null, PUBLIC_APP_ID, scriptId, version); + scriptVersion = publicScriptService.getByScriptIdAndVersion(scriptId, version); } EsbScriptVersionDetailV3DTO result = null; @@ -190,12 +213,137 @@ public EsbResp getPublicScriptVersionDetailUsingPos return EsbResp.buildSuccessResp(result); } + @Override + @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_create_public_script"}) + @AuditEntry(actionId = ActionId.CREATE_PUBLIC_SCRIPT) + public EsbResp createPublicScript( + String username, + String appCode, + @AuditRequestBody EsbCreatePublicScriptV3Req request) { + String userName = username; + + ScriptDTO script = scriptDTOBuilder.buildFromEsbCreateReq(request); + script.setAppId(PUBLIC_APP_ID); + script.setPublicScript(true); + script.setCreator(userName); + script.setLastModifyUser(userName); + ScriptDTO savedScript = publicScriptService.saveScript(userName, script); + + EsbScriptVersionDetailV3DTO result = null; + if (savedScript != null) { + result = savedScript.toEsbCreateScriptV3DTO(); + } + return EsbResp.buildSuccessResp(result); + } + + + @Override + @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_create_public_script_version"}) + @AuditEntry(actionId = ActionId.MANAGE_PUBLIC_SCRIPT_INSTANCE) + public EsbResp createPublicScriptVersion( + String username, + String appCode, + @AuditRequestBody EsbCreatePublicScriptVersionV3Req request) { + String userName = username; + + ScriptDTO script = scriptDTOBuilder.buildFromEsbCreateReq(request); + script.setAppId(PUBLIC_APP_ID); + script.setPublicScript(true); + script.setCreator(userName); + script.setLastModifyUser(userName); + ScriptDTO savedScript = publicScriptService.saveScriptVersion(userName, script); + EsbScriptVersionDetailV3DTO result = null; + if (savedScript != null) { + result = savedScript.toEsbCreateScriptV3DTO(); + } + return EsbResp.buildSuccessResp(result); + } + + @Override + @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_delete_public_script"}) + @AuditEntry(actionId = ActionId.MANAGE_PUBLIC_SCRIPT_INSTANCE) + public EsbResp deletePublicScript(String username, + String appCode, + @AuditRequestBody EsbDeletePublicScriptV3Req request) { + publicScriptService.deleteScript(username, request.getScriptId()); + return EsbResp.buildSuccessResp(null); + } + + @Override + @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_delete_public_script_version"}) + @AuditEntry(actionId = ActionId.MANAGE_PUBLIC_SCRIPT_INSTANCE) + public EsbResp deletePublicScriptVersion(String username, + String appCode, + @AuditRequestBody EsbDeletePublicScriptVersionV3Req request) { + publicScriptService.deleteScriptVersion(username, request.getScriptVersionId()); + return EsbResp.buildSuccessResp(null); + } + + @Override + @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_disable_public_script_version"}) + @AuditEntry(actionId = ActionId.MANAGE_PUBLIC_SCRIPT_INSTANCE) + public EsbResp disablePublicScriptVersion( + String username, + String appCode, + @AuditRequestBody EsbManagePublicScriptVersionV3Req request) { + publicScriptService.disableScript(username, request.getScriptId(), request.getScriptVersionId()); + ScriptDTO scriptVersion = publicScriptService.getScriptVersion(request.getScriptVersionId()); + return EsbResp.buildSuccessResp(scriptVersion.toEsbManageScriptV3DTO()); + } + + @Override + @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_publish_public_script_version"}) + @AuditEntry(actionId = ActionId.MANAGE_PUBLIC_SCRIPT_INSTANCE) + public EsbResp publishPublicScriptVersion( + String username, + String appCode, + @AuditRequestBody EsbManagePublicScriptVersionV3Req request) { + publicScriptService.publishScript(username, request.getScriptId(), request.getScriptVersionId()); + ScriptDTO scriptVersion = publicScriptService.getScriptVersion(request.getScriptVersionId()); + return EsbResp.buildSuccessResp(scriptVersion.toEsbManageScriptV3DTO()); + } + + @Override + @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_update_public_script_basic"}) + @AuditEntry(actionId = ActionId.MANAGE_PUBLIC_SCRIPT_INSTANCE) + public EsbResp updatePublicScriptBasic( + String username, + String appCode, + @AuditRequestBody EsbUpdatePublicScriptBasicV3Req request) { + String scriptId = request.getScriptId(); + publicScriptService.updateScriptName(username, scriptId, request.getName()); + if (StringUtils.isNotEmpty(request.getDescription())) { + publicScriptService.updateScriptDesc(username, scriptId, request.getDescription()); + } + + ScriptDTO scriptDTO = publicScriptService.getScript(scriptId); + EsbScriptV3DTO updatePublicScriptV3DTO = scriptDTO.toEsbScriptV3DTO(); + return EsbResp.buildSuccessResp(updatePublicScriptV3DTO); + } + + @Override + @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_update_public_script_version"}) + @AuditEntry(actionId = ActionId.MANAGE_PUBLIC_SCRIPT_INSTANCE) + public EsbResp updatePublicScriptVersion( + @RequestHeader(value = JobCommonHeaders.USERNAME) String username, + @RequestHeader(value = JobCommonHeaders.APP_CODE) String appCode, + @AuditRequestBody EsbUpdatePublicScriptVersionV3Req request) { + ScriptDTO scriptVersionDTO = scriptDTOBuilder.buildFromCreateUpdateReq(request); + scriptVersionDTO.setAppId(PUBLIC_APP_ID); + scriptVersionDTO.setPublicScript(true); + scriptVersionDTO.setCreator(username); + scriptVersionDTO.setLastModifyUser(username); + publicScriptService.updateScriptVersion(username, scriptVersionDTO); + ScriptDTO scriptDTO = publicScriptService.getScriptVersion(request.getScriptVersionId()); + return EsbResp.buildSuccessResp(scriptDTO.toEsbCreateScriptV3DTO()); + } + private void checkEsbGetPublicScriptListV3Req(EsbGetPublicScriptListV3Request request) { request.adjustPageParam(); // 如果script_type=0,表示查询所有类型 if (request.getScriptLanguage() != null && request.getScriptLanguage() > 0 - && ScriptTypeEnum.valueOf(request.getScriptLanguage()) == null) { + && ScriptTypeEnum.valOf(request.getScriptLanguage()) == null) { log.warn("Param [type]:[{}] is illegal!", request.getScriptLanguage()); throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME, "type"); } @@ -207,7 +355,8 @@ private void setOnlineScriptVersionInfo(List scripts) { for (ScriptDTO script : scripts) { scriptIdList.add(script.getId()); } - Map onlineScriptMap = scriptService.batchGetOnlineScriptVersionByScriptIds(scriptIdList); + Map onlineScriptMap = + publicScriptService.batchGetOnlineScriptVersionByScriptIds(scriptIdList); for (ScriptDTO script : scripts) { ScriptDTO onlineScriptVersion = onlineScriptMap.get(script.getId()); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/v3/EsbScriptResourceV3Impl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/v3/EsbScriptResourceV3Impl.java index 1da572e7e4..e618cc791f 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/v3/EsbScriptResourceV3Impl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/v3/EsbScriptResourceV3Impl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,57 +24,87 @@ package com.tencent.bk.job.manage.api.esb.impl.v3; +import com.tencent.bk.audit.annotations.ActionAuditRecord; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.audit.annotations.AuditInstanceRecord; +import com.tencent.bk.audit.annotations.AuditRequestBody; +import com.tencent.bk.job.common.audit.constants.EventContentConstants; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.esb.metrics.EsbApiTimed; import com.tencent.bk.job.common.esb.model.EsbResp; import com.tencent.bk.job.common.esb.model.job.v3.EsbPageDataV3; import com.tencent.bk.job.common.exception.InvalidParamException; -import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; -import com.tencent.bk.job.common.iam.model.AuthResult; +import com.tencent.bk.job.common.i18n.service.MessageI18nService; +import com.tencent.bk.job.common.iam.constant.ActionId; +import com.tencent.bk.job.common.iam.constant.ResourceTypeId; import com.tencent.bk.job.common.metrics.CommonMetricNames; import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; import com.tencent.bk.job.common.model.dto.AppResourceScope; import com.tencent.bk.job.common.service.AppScopeMappingService; +import com.tencent.bk.job.common.validation.ScopeValidator; +import com.tencent.bk.job.manage.api.common.ScriptDTOBuilder; +import com.tencent.bk.job.manage.api.common.constants.JobResourceStatusEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; import com.tencent.bk.job.manage.api.esb.v3.EsbScriptV3Resource; import com.tencent.bk.job.manage.auth.ScriptAuthService; -import com.tencent.bk.job.manage.common.consts.JobResourceStatusEnum; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; +import com.tencent.bk.job.manage.model.dto.ScriptCheckResultItemDTO; import com.tencent.bk.job.manage.model.dto.ScriptDTO; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbCheckScriptV3Req; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbCreateScriptV3Req; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbCreateScriptVersionV3Req; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbDeleteScriptV3Req; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbDeleteScriptVersionV3Req; import com.tencent.bk.job.manage.model.esb.v3.request.EsbGetScriptListV3Req; import com.tencent.bk.job.manage.model.esb.v3.request.EsbGetScriptVersionDetailV3Req; import com.tencent.bk.job.manage.model.esb.v3.request.EsbGetScriptVersionListV3Req; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbManageScriptVersionV3Req; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbUpdateScriptBasicV3Req; +import com.tencent.bk.job.manage.model.esb.v3.request.EsbUpdateScriptVersionV3Req; +import com.tencent.bk.job.manage.model.esb.v3.response.EsbCheckScriptV3DTO; import com.tencent.bk.job.manage.model.esb.v3.response.EsbScriptV3DTO; import com.tencent.bk.job.manage.model.esb.v3.response.EsbScriptVersionDetailV3DTO; import com.tencent.bk.job.manage.model.query.ScriptQuery; +import com.tencent.bk.job.manage.service.ScriptCheckService; import com.tencent.bk.job.manage.service.ScriptService; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.codec.binary.Base64; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.web.bind.annotation.RestController; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.stream.Collectors; @RestController @Slf4j public class EsbScriptResourceV3Impl implements EsbScriptV3Resource { private final ScriptService scriptService; private final ScriptAuthService scriptAuthService; + private final ScriptDTOBuilder scriptDTOBuilder; + private final ScriptCheckService scriptCheckService; + protected final MessageI18nService i18nService; private final AppScopeMappingService appScopeMappingService; public EsbScriptResourceV3Impl(ScriptService scriptService, ScriptAuthService scriptAuthService, + ScriptDTOBuilder scriptDTOBuilder, + ScriptCheckService scriptCheckService, + MessageI18nService i18nService, AppScopeMappingService appScopeMappingService) { this.scriptService = scriptService; this.scriptAuthService = scriptAuthService; + this.scriptDTOBuilder = scriptDTOBuilder; + this.scriptCheckService = scriptCheckService; + this.i18nService = i18nService; this.appScopeMappingService = appScopeMappingService; } @Override + @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_get_script_list"}) public EsbResp> getScriptList(String username, String appCode, Long bizId, @@ -84,9 +114,8 @@ public EsbResp> getScriptList(String username, Integer scriptLanguage, Integer start, Integer length) { + ScopeValidator.validate(bizId, scopeType, scopeId); EsbGetScriptListV3Req request = new EsbGetScriptListV3Req(); - request.setUserName(username); - request.setAppCode(appCode); request.setBizId(bizId); request.setScopeType(scopeType); request.setScopeId(scopeId); @@ -94,10 +123,13 @@ public EsbResp> getScriptList(String username, request.setScriptLanguage(scriptLanguage); request.setStart(start); request.setLength(length); - return getScriptListUsingPost(request); + request.fillAppResourceScope(appScopeMappingService); + return getScriptListUsingPost(username, appCode, request); } @Override + @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_get_script_version_list"}) + @AuditEntry(actionId = ActionId.VIEW_SCRIPT) public EsbResp> getScriptVersionList(String username, String appCode, Long bizId, @@ -107,9 +139,8 @@ public EsbResp> getScriptVersionList( boolean returnScriptContent, Integer start, Integer length) { + ScopeValidator.validate(bizId, scopeType, scopeId); EsbGetScriptVersionListV3Req request = new EsbGetScriptVersionListV3Req(); - request.setUserName(username); - request.setAppCode(appCode); request.setBizId(bizId); request.setScopeType(scopeType); request.setScopeId(scopeId); @@ -117,10 +148,13 @@ public EsbResp> getScriptVersionList( request.setReturnScriptContent(returnScriptContent); request.setStart(start); request.setLength(length); - return getScriptVersionListUsingPost(request); + request.fillAppResourceScope(appScopeMappingService); + return getScriptVersionListUsingPost(username, appCode, request); } @Override + @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_get_script_version_detail"}) + @AuditEntry(actionId = ActionId.VIEW_SCRIPT) public EsbResp getScriptVersionDetail(String username, String appCode, Long bizId, @@ -129,24 +163,35 @@ public EsbResp getScriptVersionDetail(String userna Long scriptVersionId, String scriptId, String version) { + ScopeValidator.validate(bizId, scopeType, scopeId); EsbGetScriptVersionDetailV3Req request = new EsbGetScriptVersionDetailV3Req(); - request.setUserName(username); - request.setAppCode(appCode); request.setBizId(bizId); request.setScopeType(scopeType); request.setScopeId(scopeId); request.setId(scriptVersionId); request.setScriptId(scriptId); request.setVersion(version); - return getScriptVersionDetailUsingPost(request); + request.fillAppResourceScope(appScopeMappingService); + return getScriptVersionDetailUsingPost(username, appCode, request); } @Override @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_get_script_list"}) - public EsbResp> getScriptListUsingPost(EsbGetScriptListV3Req request) { - request.fillAppResourceScope(appScopeMappingService); + public EsbResp> getScriptListUsingPost(String username, + String appCode, + EsbGetScriptListV3Req request) { checkEsbGetScriptListV3Req(request); + ScriptQuery scriptQuery = buildListPageScriptQuery(request); + + PageData pageScripts = scriptService.listPageScript(scriptQuery); + setOnlineScriptVersionInfo(pageScripts.getData()); + + EsbPageDataV3 result = EsbPageDataV3.from(pageScripts, ScriptDTO::toEsbScriptV3DTO); + return EsbResp.buildSuccessResp(result); + } + + private ScriptQuery buildListPageScriptQuery(EsbGetScriptListV3Req request) { long appId = request.getAppId(); ScriptQuery scriptQuery = new ScriptQuery(); scriptQuery.setAppId(appId); @@ -161,12 +206,9 @@ public EsbResp> getScriptListUsingPost(EsbGetScrip BaseSearchCondition baseSearchCondition = new BaseSearchCondition(); baseSearchCondition.setStart(request.getStart()); baseSearchCondition.setLength(request.getLength()); + scriptQuery.setBaseSearchCondition(baseSearchCondition); - PageData pageScripts = scriptService.listPageScript(scriptQuery, baseSearchCondition); - setOnlineScriptVersionInfo(pageScripts.getData()); - - EsbPageDataV3 result = EsbPageDataV3.from(pageScripts, ScriptDTO::toEsbScriptV3DTO); - return EsbResp.buildSuccessResp(result); + return scriptQuery; } private void checkEsbGetScriptListV3Req(EsbGetScriptListV3Req request) { @@ -174,7 +216,7 @@ private void checkEsbGetScriptListV3Req(EsbGetScriptListV3Req request) { // 如果script_type=0,表示查询所有类型 if (request.getScriptLanguage() != null && request.getScriptLanguage() > 0 - && ScriptTypeEnum.valueOf(request.getScriptLanguage()) == null) { + && ScriptTypeEnum.valOf(request.getScriptLanguage()) == null) { log.warn("Param [type]:[{}] is illegal!", request.getScriptLanguage()); throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME, "type"); } @@ -200,23 +242,26 @@ private void setOnlineScriptVersionInfo(List scripts) { @Override @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_get_script_version_list"}) + @AuditEntry(actionId = ActionId.VIEW_SCRIPT) + @ActionAuditRecord( + actionId = ActionId.VIEW_SCRIPT, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.SCRIPT + ), + content = EventContentConstants.VIEW_SCRIPT + ) public EsbResp> getScriptVersionListUsingPost( - EsbGetScriptVersionListV3Req request) { - request.fillAppResourceScope(appScopeMappingService); + String username, + String appCode, + @AuditRequestBody EsbGetScriptVersionListV3Req request) { checkEsbGetScriptVersionListV3Req(request); - long appId = request.getAppId(); - ScriptQuery scriptQuery = new ScriptQuery(); - scriptQuery.setAppId(appId); - scriptQuery.setPublicScript(false); - scriptQuery.setId(request.getScriptId()); - - BaseSearchCondition baseSearchCondition = BaseSearchCondition.pageCondition(request.getStart(), - request.getLength()); + scriptAuthService.authViewScript(username, request.getAppResourceScope(), request.getScriptId(), + null).denyIfNoPermission(); - PageData pageScriptVersions = scriptService.listPageScriptVersion(scriptQuery, baseSearchCondition); + ScriptQuery scriptQuery = buildListScriptVersionQuery(request); - batchAuthViewScript(request.getUserName(), request.getAppResourceScope(), pageScriptVersions.getData()); + PageData pageScriptVersions = scriptService.listPageScriptVersion(scriptQuery); EsbPageDataV3 result = EsbPageDataV3.from(pageScriptVersions, ScriptDTO::toEsbScriptVersionDetailV3DTO); @@ -228,6 +273,20 @@ public EsbResp> getScriptVersionListU return EsbResp.buildSuccessResp(result); } + private ScriptQuery buildListScriptVersionQuery(EsbGetScriptVersionListV3Req request) { + long appId = request.getAppId(); + ScriptQuery scriptQuery = new ScriptQuery(); + scriptQuery.setAppId(appId); + scriptQuery.setPublicScript(false); + scriptQuery.setId(request.getScriptId()); + + BaseSearchCondition baseSearchCondition = BaseSearchCondition.pageCondition(request.getStart(), + request.getLength()); + scriptQuery.setBaseSearchCondition(baseSearchCondition); + + return scriptQuery; + } + private void checkEsbGetScriptVersionListV3Req(EsbGetScriptVersionListV3Req request) { if (StringUtils.isBlank(request.getScriptId())) { log.warn("Param [script_id] is empty!"); @@ -235,26 +294,13 @@ private void checkEsbGetScriptVersionListV3Req(EsbGetScriptVersionListV3Req requ } } - private void batchAuthViewScript(String username, AppResourceScope appResourceScope, List scripts) { - if (CollectionUtils.isNotEmpty(scripts)) { - // 鉴权 - List resourceIds = - scripts.stream().map(ScriptDTO::getId).distinct().collect(Collectors.toList()); - if (!resourceIds.isEmpty()) { - AuthResult authResult = scriptAuthService.batchAuthResultViewScript(username, - appResourceScope, resourceIds); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } - } - } - } - @Override @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_get_script_version_detail"}) + @AuditEntry(actionId = ActionId.VIEW_SCRIPT) public EsbResp getScriptVersionDetailUsingPost( - EsbGetScriptVersionDetailV3Req request) { - request.fillAppResourceScope(appScopeMappingService); + String username, + String appCode, + @AuditRequestBody EsbGetScriptVersionDetailV3Req request) { checkEsbGetScriptVersionDetailV3Req(request); long appId = request.getAppId(); @@ -263,17 +309,9 @@ public EsbResp getScriptVersionDetailUsingPost( Long id = request.getId(); ScriptDTO scriptVersion; if (id != null && id > 0) { - scriptVersion = scriptService.getScriptVersion(null, appId, id); + scriptVersion = scriptService.getScriptVersion(username, appId, id); } else { - scriptVersion = scriptService.getByScriptIdAndVersion(null, appId, scriptId, version); - } - - if (scriptVersion != null) { - AuthResult authResult = scriptAuthService.authViewScript(request.getUserName(), request.getAppResourceScope(), - scriptVersion.getId(), null); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } + scriptVersion = scriptService.getByScriptIdAndVersion(username, appId, scriptId, version); } EsbScriptVersionDetailV3DTO result = null; @@ -284,6 +322,160 @@ public EsbResp getScriptVersionDetailUsingPost( return EsbResp.buildSuccessResp(result); } + @Override + @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_create_script"}) + @AuditEntry(actionId = ActionId.CREATE_SCRIPT) + public EsbResp createScript( + String username, + String appCode, + @AuditRequestBody EsbCreateScriptV3Req request) { + AppResourceScope appResourceScope = request.getAppResourceScope(); + + ScriptDTO script = scriptDTOBuilder.buildFromEsbCreateReq(request); + script.setAppId(appResourceScope.getAppId()); + script.setPublicScript(false); + script.setCreator(username); + script.setLastModifyUser(username); + ScriptDTO savedScript = scriptService.createScript(username, script); + + EsbScriptVersionDetailV3DTO result = savedScript.toEsbCreateScriptV3DTO(); + return EsbResp.buildSuccessResp(result); + } + + @Override + @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_create_script_version"}) + @AuditEntry(actionId = ActionId.CREATE_SCRIPT) + public EsbResp createScriptVersion( + String username, + String appCode, + @AuditRequestBody EsbCreateScriptVersionV3Req request) { + AppResourceScope appResourceScope = request.getAppResourceScope(); + + ScriptDTO script = scriptDTOBuilder.buildFromEsbCreateReq(request); + script.setAppId(appResourceScope.getAppId()); + script.setPublicScript(false); + script.setCreator(username); + script.setLastModifyUser(username); + ScriptDTO savedScript = scriptService.createScriptVersion(username, script); + + EsbScriptVersionDetailV3DTO result = null; + if (savedScript != null) { + result = savedScript.toEsbCreateScriptV3DTO(); + } + return EsbResp.buildSuccessResp(result); + } + + @Override + @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_delete_script"}) + @AuditEntry(actionId = ActionId.MANAGE_SCRIPT) + public EsbResp deleteScript(String username, + String appCode, + @AuditRequestBody EsbDeleteScriptV3Req request) { + scriptService.deleteScript(username, request.getAppId(), request.getScriptId()); + return EsbResp.buildSuccessResp(null); + } + + @Override + @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_delete_script_version"}) + @AuditEntry(actionId = ActionId.MANAGE_SCRIPT) + public EsbResp deleteScriptVersion(String username, + String appCode, + @AuditRequestBody EsbDeleteScriptVersionV3Req request) { + scriptService.deleteScriptVersion(username, request.getAppResourceScope().getAppId(), + request.getScriptVersionId()); + return EsbResp.buildSuccessResp(null); + } + + @Override + @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_disable_script_version"}) + @AuditEntry(actionId = ActionId.MANAGE_SCRIPT) + public EsbResp disableScriptVersion( + String username, + String appCode, + @AuditRequestBody EsbManageScriptVersionV3Req request) { + scriptService.disableScript(request.getAppResourceScope().getAppId(), username, + request.getScriptId(), request.getScriptVersionId()); + ScriptDTO scriptVersion = scriptService.getScriptVersion(request.getScriptVersionId()); + return EsbResp.buildSuccessResp(scriptVersion.toEsbManageScriptV3DTO()); + } + + @Override + @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_publish_script_version"}) + @AuditEntry(actionId = ActionId.MANAGE_SCRIPT) + public EsbResp publishScriptVersion( + String username, + String appCode, + @AuditRequestBody EsbManageScriptVersionV3Req request) { + scriptService.publishScript(request.getAppResourceScope().getAppId(), username, + request.getScriptId(), request.getScriptVersionId()); + ScriptDTO scriptVersion = scriptService.getScriptVersion(request.getScriptVersionId()); + return EsbResp.buildSuccessResp(scriptVersion.toEsbManageScriptV3DTO()); + } + + @Override + @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_update_script_basic"}) + @AuditEntry(actionId = ActionId.MANAGE_SCRIPT) + public EsbResp updateScriptBasic( + String username, + String appCode, + @AuditRequestBody EsbUpdateScriptBasicV3Req request) { + String scriptId = request.getScriptId(); + AppResourceScope appResourceScope = request.getAppResourceScope(); + scriptService.updateScriptName(appResourceScope.getAppId(), username, scriptId, request.getName()); + if (StringUtils.isNotEmpty(request.getDescription())) { + scriptService.updateScriptDesc(appResourceScope.getAppId(), username, scriptId, request.getDescription()); + } + + ScriptDTO scriptDTO = scriptService.getScript(appResourceScope.getAppId(), scriptId); + EsbScriptV3DTO updateScriptV3DTO = scriptDTO.toEsbScriptV3DTO(); + return EsbResp.buildSuccessResp(updateScriptV3DTO); + } + + @Override + @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_update_script_version"}) + @AuditEntry(actionId = ActionId.MANAGE_SCRIPT) + public EsbResp updateScriptVersion( + String username, + String appCode, + @AuditRequestBody EsbUpdateScriptVersionV3Req request) { + ScriptDTO scriptVersionDTO = scriptDTOBuilder.buildFromCreateUpdateReq(request); + scriptVersionDTO.setAppId(request.getAppResourceScope().getAppId()); + scriptVersionDTO.setPublicScript(false); + scriptVersionDTO.setCreator(username); + scriptVersionDTO.setLastModifyUser(username); + scriptService.updateScriptVersion(username, scriptVersionDTO); + ScriptDTO scriptDTO = scriptService.getScriptVersion(request.getScriptVersionId()); + return EsbResp.buildSuccessResp(scriptDTO.toEsbCreateScriptV3DTO()); + } + + @Override + @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_check_script"}) + public EsbResp> checkScript(EsbCheckScriptV3Req request) { + String content = new String(Base64.decodeBase64(request.getContent()), StandardCharsets.UTF_8); + List checkResultItems = + scriptCheckService.check(ScriptTypeEnum.valOf(request.getType()), content); + List checkScriptDTOS = new ArrayList<>(); + if (checkResultItems != null) { + for (ScriptCheckResultItemDTO checkResultItem : checkResultItems) { + EsbCheckScriptV3DTO checkScriptDTO = new EsbCheckScriptV3DTO(); + checkScriptDTO.setLine(checkResultItem.getLine()); + checkScriptDTO.setLineContent(checkResultItem.getLineContent()); + checkScriptDTO.setMatchContent(checkResultItem.getMatchContent()); + checkScriptDTO.setLevel(checkResultItem.getLevel().getValue()); + if (StringUtils.isNotBlank(checkResultItem.getCheckItemCode())) { + String desc = i18nService.getI18n(checkResultItem.getCheckItemCode()); + if (StringUtils.isNotBlank(desc) && !checkResultItem.getCheckItemCode().equals(desc)) { + checkScriptDTO.setDescription(desc); + } + } else { + checkScriptDTO.setDescription(checkResultItem.getDescription()); + } + checkScriptDTOS.add(checkScriptDTO); + } + } + return EsbResp.buildSuccessResp(checkScriptDTOS); + } + private void checkEsbGetScriptVersionDetailV3Req(EsbGetScriptVersionDetailV3Req request) { if (request.getId() != null && request.getId() > 0) { // 如果ID合法，那么忽略其他参数 diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/v3/EsbServiceInfoV3ResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/v3/EsbServiceInfoV3ResourceImpl.java new file mode 100644 index 0000000000..357c658e0e --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/v3/EsbServiceInfoV3ResourceImpl.java @@ -0,0 +1,90 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.api.esb.impl.v3; + +import com.tencent.bk.job.common.constant.ProfileEnum; +import com.tencent.bk.job.common.discovery.ServiceInfoProvider; +import com.tencent.bk.job.common.discovery.model.ServiceInstanceInfoDTO; +import com.tencent.bk.job.common.esb.model.EsbResp; +import com.tencent.bk.job.common.util.CompareUtil; +import com.tencent.bk.job.manage.api.esb.v3.EsbServiceInfoV3Resource; +import com.tencent.bk.job.manage.model.esb.v3.response.EsbServiceVersionV3DTO; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Profile; +import org.springframework.web.bind.annotation.RestController; + +import java.util.List; + +@RestController +@Slf4j +@Profile("!" + ProfileEnum.Constants.TEST) +public class EsbServiceInfoV3ResourceImpl implements EsbServiceInfoV3Resource { + + private final ServiceInfoProvider serviceInfoProvider; + + @Autowired + public EsbServiceInfoV3ResourceImpl(ServiceInfoProvider serviceInfoProvider) { + this.serviceInfoProvider = serviceInfoProvider;; + } + + + @Override + public EsbResp getLatestServiceVersion(String username, + String appCode) { + return getLatestServiceVersionUsingPost(username, appCode); + } + + @Override + public EsbResp getLatestServiceVersionUsingPost(String username, + String appCode) { + List instanceInfoDTOList = serviceInfoProvider.listServiceInfo(); + ServiceInstanceInfoDTO latestVersionInstance = findLatestVersionInstance(instanceInfoDTOList); + EsbServiceVersionV3DTO esbServiceVersionV3DTO = new EsbServiceVersionV3DTO(); + if (latestVersionInstance != null) { + esbServiceVersionV3DTO.setVersion(latestVersionInstance.getVersion()); + } + return EsbResp.buildSuccessResp(esbServiceVersionV3DTO); + } + + private static ServiceInstanceInfoDTO findLatestVersionInstance(List instanceInfoDTOList) { + if (instanceInfoDTOList == null || instanceInfoDTOList.isEmpty()) { + return null; + } + + ServiceInstanceInfoDTO latestInstance = instanceInfoDTOList.get(0); + String latestVersion = latestInstance.getVersion(); + + for (int i = 1; i < instanceInfoDTOList.size(); i++) { + ServiceInstanceInfoDTO currentInstance = instanceInfoDTOList.get(i); + String currentVersion = currentInstance.getVersion(); + if (CompareUtil.compareVersion(currentVersion, latestVersion) > 0) { + latestInstance = currentInstance; + latestVersion = currentVersion; + } + } + return latestInstance; + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/v3/EsbTemplateV3ResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/v3/EsbTemplateV3ResourceImpl.java index c4d34b5fac..96d31730c2 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/v3/EsbTemplateV3ResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/esb/impl/v3/EsbTemplateV3ResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -29,14 +29,10 @@ import com.tencent.bk.job.common.esb.model.job.v3.EsbPageDataV3; import com.tencent.bk.job.common.esb.util.EsbDTOAppScopeMappingHelper; import com.tencent.bk.job.common.exception.InvalidParamException; -import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; -import com.tencent.bk.job.common.iam.model.AuthResult; -import com.tencent.bk.job.common.iam.service.BusinessAuthService; import com.tencent.bk.job.common.metrics.CommonMetricNames; import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; import com.tencent.bk.job.common.model.ValidateResult; -import com.tencent.bk.job.common.model.dto.AppResourceScope; import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.manage.api.esb.v3.EsbTemplateV3Resource; import com.tencent.bk.job.manage.model.dto.task.TaskTemplateInfoDTO; @@ -55,15 +51,12 @@ @RestController public class EsbTemplateV3ResourceImpl implements EsbTemplateV3Resource { private final TaskTemplateService taskTemplateService; - private final BusinessAuthService businessAuthService; private final AppScopeMappingService appScopeMappingService; @Autowired public EsbTemplateV3ResourceImpl(TaskTemplateService taskTemplateService, - BusinessAuthService businessAuthService, AppScopeMappingService appScopeMappingService) { this.taskTemplateService = taskTemplateService; - this.businessAuthService = businessAuthService; this.appScopeMappingService = appScopeMappingService; } @@ -83,8 +76,6 @@ public EsbResp> getTemplateList(String Integer start, Integer length) { EsbGetTemplateListV3Request request = new EsbGetTemplateListV3Request(); - request.setUserName(username); - request.setAppCode(appCode); request.setBizId(bizId); request.setScopeType(scopeType); request.setScopeId(scopeId); @@ -97,14 +88,16 @@ public EsbResp> getTemplateList(String request.setLastModifyTimeStart(lastModifyTimeStart); request.setStart(start); request.setLength(length); - return getTemplateListUsingPost(request); + request.fillAppResourceScope(appScopeMappingService); + return getTemplateListUsingPost(username, appCode, request); } @Override @EsbApiTimed(value = CommonMetricNames.ESB_API, extraTags = {"api_name", "v3_get_job_template_list"}) public EsbResp> getTemplateListUsingPost( + String username, + String appCode, EsbGetTemplateListV3Request request) { - request.fillAppResourceScope(appScopeMappingService); ValidateResult checkResult = checkRequest(request); if (!checkResult.isPass()) { log.warn("Get template list, request is illegal!"); @@ -112,13 +105,6 @@ public EsbResp> getTemplateListUsingPos } long appId = request.getAppId(); - AuthResult authResult = - businessAuthService.authAccessBusiness( - request.getUserName(), new AppResourceScope(request.getAppId())); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } - BaseSearchCondition baseSearchCondition = new BaseSearchCondition(); if (request.getStart() != null) { baseSearchCondition.setStart(request.getStart()); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/impl/AbstractScriptCallbackHelper.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/impl/AbstractScriptCallbackHelper.java new file mode 100644 index 0000000000..ac0ea8aa1d --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/impl/AbstractScriptCallbackHelper.java @@ -0,0 +1,140 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.api.iam.impl; + +import com.tencent.bk.job.common.iam.constant.ResourceTypeId; +import com.tencent.bk.job.common.iam.service.BaseIamCallbackService; +import com.tencent.bk.job.common.model.BaseSearchCondition; +import com.tencent.bk.job.common.model.dto.ResourceScope; +import com.tencent.bk.job.manage.model.dto.ScriptBasicDTO; +import com.tencent.bk.job.manage.model.dto.ScriptDTO; +import com.tencent.bk.job.manage.model.query.ScriptQuery; +import com.tencent.bk.job.manage.service.ApplicationService; +import com.tencent.bk.sdk.iam.dto.PathInfoDTO; +import com.tencent.bk.sdk.iam.dto.callback.request.CallbackRequestDTO; +import com.tencent.bk.sdk.iam.dto.callback.request.IamSearchCondition; +import com.tencent.bk.sdk.iam.dto.callback.response.CallbackBaseResponseDTO; +import com.tencent.bk.sdk.iam.dto.callback.response.FetchInstanceInfoResponseDTO; +import com.tencent.bk.sdk.iam.dto.callback.response.InstanceInfoDTO; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +public abstract class AbstractScriptCallbackHelper extends BaseIamCallbackService { + + private final ApplicationService applicationService; + + protected AbstractScriptCallbackHelper(ApplicationService applicationService) { + this.applicationService = applicationService; + } + + protected InstanceInfoDTO convert(ScriptDTO script) { + InstanceInfoDTO instanceInfo = new InstanceInfoDTO(); + instanceInfo.setId(String.valueOf(script.getId())); + instanceInfo.setDisplayName(script.getName()); + return instanceInfo; + } + + protected ScriptQuery buildBasicScriptQuery(CallbackRequestDTO callbackRequest) { + IamSearchCondition searchCondition = IamSearchCondition.fromReq(callbackRequest); + BaseSearchCondition baseSearchCondition = new BaseSearchCondition(); + baseSearchCondition.setStart(searchCondition.getStart().intValue()); + baseSearchCondition.setLength(searchCondition.getLength().intValue()); + + ScriptQuery scriptQuery = new ScriptQuery(); + boolean isPublicScript = isPublicScript(); + scriptQuery.setPublicScript(isPublicScript); + if (!isPublicScript) { + Long appId = applicationService.getAppIdByScope(extractResourceScopeCondition(searchCondition)); + scriptQuery.setAppId(appId); + } + scriptQuery.setBaseSearchCondition(baseSearchCondition); + return scriptQuery; + } + + protected CallbackBaseResponseDTO buildFetchInstanceResp(List queryScriptIdList, + List scriptBasicDTOList) { + List instanceAttributeInfoList = new ArrayList<>(); + Map scriptBasicDTOMap = new HashMap<>(scriptBasicDTOList.size()); + Set appIdSet = new HashSet<>(); + for (ScriptBasicDTO scriptBasicDTO : scriptBasicDTOList) { + scriptBasicDTOMap.put(scriptBasicDTO.getId(), scriptBasicDTO); + appIdSet.add(scriptBasicDTO.getAppId()); + } + // Job app --> CMDB biz/businessSet转换 + Map appIdScopeMap = applicationService.getScopeByAppIds(appIdSet); + for (String id : queryScriptIdList) { + ScriptBasicDTO scriptBasicDTO = scriptBasicDTOMap.get(id); + if (scriptBasicDTO == null) { + logNotExistId(id); + continue; + } + try { + InstanceInfoDTO instanceInfo = buildInstance(scriptBasicDTO, appIdScopeMap); + instanceAttributeInfoList.add(instanceInfo); + } catch (Exception e) { + logBuildInstanceFailure(scriptBasicDTO, e); + } + } + + FetchInstanceInfoResponseDTO fetchInstanceInfoResponse = new FetchInstanceInfoResponseDTO(); + fetchInstanceInfoResponse.setCode(0L); + fetchInstanceInfoResponse.setData(instanceAttributeInfoList); + return fetchInstanceInfoResponse; + } + + protected InstanceInfoDTO buildInstance(ScriptBasicDTO scriptBasicDTO, + Map appIdScopeMap) { + // 拓扑路径构建 + List path = new ArrayList<>(); + PathInfoDTO rootNode = new PathInfoDTO(); + if (isPublicScript()) { + // 公共脚本 + rootNode.setType(ResourceTypeId.PUBLIC_SCRIPT); + rootNode.setId(scriptBasicDTO.getId()); + } else { + // 业务脚本 + Long appId = scriptBasicDTO.getAppId(); + rootNode = getPathNodeByAppId(appId, appIdScopeMap); + PathInfoDTO scriptNode = new PathInfoDTO(); + scriptNode.setType(ResourceTypeId.SCRIPT); + scriptNode.setId(scriptBasicDTO.getId()); + rootNode.setChild(scriptNode); + } + path.add(rootNode); + // 实例组装 + InstanceInfoDTO instanceInfo = new InstanceInfoDTO(); + instanceInfo.setId(scriptBasicDTO.getId()); + instanceInfo.setDisplayName(scriptBasicDTO.getName()); + instanceInfo.setPath(path); + return instanceInfo; + } + + abstract boolean isPublicScript(); +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/impl/IamAccountCallbackResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/impl/IamAccountCallbackResourceImpl.java index 82d2205aed..ad26e9b7d5 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/impl/IamAccountCallbackResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/impl/IamAccountCallbackResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,7 @@ package com.tencent.bk.job.manage.api.iam.impl; +import com.tencent.bk.audit.utils.json.JsonSchemaUtils; import com.tencent.bk.job.common.iam.constant.ResourceTypeId; import com.tencent.bk.job.common.iam.service.BaseIamCallbackService; import com.tencent.bk.job.common.iam.util.IamRespUtil; @@ -33,6 +34,7 @@ import com.tencent.bk.job.manage.api.iam.IamAccountCallbackResource; import com.tencent.bk.job.manage.model.dto.AccountDTO; import com.tencent.bk.job.manage.model.dto.AccountDisplayDTO; +import com.tencent.bk.job.manage.model.esb.EsbAccountDTO; import com.tencent.bk.job.manage.service.AccountService; import com.tencent.bk.job.manage.service.ApplicationService; import com.tencent.bk.sdk.iam.dto.PathInfoDTO; @@ -40,11 +42,13 @@ import com.tencent.bk.sdk.iam.dto.callback.request.IamSearchCondition; import com.tencent.bk.sdk.iam.dto.callback.response.CallbackBaseResponseDTO; import com.tencent.bk.sdk.iam.dto.callback.response.FetchInstanceInfoResponseDTO; +import com.tencent.bk.sdk.iam.dto.callback.response.FetchResourceTypeSchemaResponseDTO; import com.tencent.bk.sdk.iam.dto.callback.response.InstanceInfoDTO; import com.tencent.bk.sdk.iam.dto.callback.response.ListInstanceResponseDTO; import com.tencent.bk.sdk.iam.dto.callback.response.SearchInstanceResponseDTO; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Pair; +import org.slf4j.helpers.MessageFormatter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RestController; @@ -144,7 +148,11 @@ protected CallbackBaseResponseDTO fetchInstanceResp( long id = Long.parseLong(instanceId); accountIdList.add(id); } catch (NumberFormatException e) { - log.error("Parse account id failed!|{}", instanceId, e); + String msg = MessageFormatter.format( + "Parse account id failed!|{}", + instanceId + ).getMessage(); + log.error(msg, e); } } Map accountInfoMap = accountService.getAccountDisplayInfoMapByIds(accountIdList); @@ -173,6 +181,14 @@ protected CallbackBaseResponseDTO fetchInstanceResp( return fetchInstanceInfoResponse; } + @Override + protected FetchResourceTypeSchemaResponseDTO fetchResourceTypeSchemaResp( + CallbackRequestDTO callbackRequest) { + FetchResourceTypeSchemaResponseDTO resp = new FetchResourceTypeSchemaResponseDTO(); + resp.setData(JsonSchemaUtils.generateJsonSchema(EsbAccountDTO.class)); + return resp; + } + @Override public CallbackBaseResponseDTO callback(CallbackRequestDTO callbackRequest) { return baseCallback(callbackRequest); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/impl/IamPublicScriptCallbackResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/impl/IamPublicScriptCallbackResourceImpl.java index 6a32aa84a5..0efa5d1740 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/impl/IamPublicScriptCallbackResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/impl/IamPublicScriptCallbackResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,16 +24,10 @@ package com.tencent.bk.job.manage.api.iam.impl; -import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.manage.api.iam.IamPublicScriptCallbackResource; -import com.tencent.bk.job.manage.model.query.ScriptQuery; -import com.tencent.bk.job.manage.service.ApplicationService; -import com.tencent.bk.job.manage.service.ScriptService; import com.tencent.bk.sdk.iam.dto.callback.request.CallbackRequestDTO; -import com.tencent.bk.sdk.iam.dto.callback.request.IamSearchCondition; import com.tencent.bk.sdk.iam.dto.callback.response.CallbackBaseResponseDTO; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.tuple.Pair; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RestController; @@ -41,37 +35,16 @@ @Slf4j public class IamPublicScriptCallbackResourceImpl implements IamPublicScriptCallbackResource { - private final ScriptCallbackHelper scriptCallbackHelper; + private final PublicScriptCallbackHelper publicScriptCallbackHelper; @Autowired - public IamPublicScriptCallbackResourceImpl(ScriptService scriptService, - ApplicationService applicationService) { - this.scriptCallbackHelper = new ScriptCallbackHelper(scriptService, new ScriptCallbackHelper.IGetBasicInfo() { - @Override - public Pair getBasicQueryCondition(CallbackRequestDTO callbackRequest) { - return getBasicQueryConditionImpl(callbackRequest); - } - - @Override - public boolean isPublicScript() { - return true; - } - }, applicationService); - } - - private Pair getBasicQueryConditionImpl(CallbackRequestDTO callbackRequest) { - IamSearchCondition searchCondition = IamSearchCondition.fromReq(callbackRequest); - BaseSearchCondition baseSearchCondition = new BaseSearchCondition(); - baseSearchCondition.setStart(searchCondition.getStart().intValue()); - baseSearchCondition.setLength(searchCondition.getLength().intValue()); - - ScriptQuery scriptQuery = new ScriptQuery(); - scriptQuery.setPublicScript(true); - return Pair.of(scriptQuery, baseSearchCondition); + public IamPublicScriptCallbackResourceImpl( + PublicScriptCallbackHelper publicScriptCallbackHelper) { + this.publicScriptCallbackHelper = publicScriptCallbackHelper; } @Override public CallbackBaseResponseDTO callback(CallbackRequestDTO callbackRequest) { - return scriptCallbackHelper.doCallback(callbackRequest); + return publicScriptCallbackHelper.doCallback(callbackRequest); } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/impl/IamScriptCallbackResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/impl/IamScriptCallbackResourceImpl.java index 28182416a0..8f8a86b2ce 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/impl/IamScriptCallbackResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/impl/IamScriptCallbackResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,54 +24,21 @@ package com.tencent.bk.job.manage.api.iam.impl; -import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.manage.api.iam.IamScriptCallbackResource; -import com.tencent.bk.job.manage.model.query.ScriptQuery; -import com.tencent.bk.job.manage.service.ApplicationService; -import com.tencent.bk.job.manage.service.ScriptService; import com.tencent.bk.sdk.iam.dto.callback.request.CallbackRequestDTO; -import com.tencent.bk.sdk.iam.dto.callback.request.IamSearchCondition; import com.tencent.bk.sdk.iam.dto.callback.response.CallbackBaseResponseDTO; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.tuple.Pair; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RestController; @RestController @Slf4j public class IamScriptCallbackResourceImpl implements IamScriptCallbackResource { - private final ScriptCallbackHelper scriptCallbackHelper; - private final ApplicationService applicationService; @Autowired - public IamScriptCallbackResourceImpl(ScriptService scriptService, - ApplicationService applicationService) { - this.scriptCallbackHelper = new ScriptCallbackHelper(scriptService, new ScriptCallbackHelper.IGetBasicInfo() { - @Override - public Pair getBasicQueryCondition(CallbackRequestDTO callbackRequest) { - return getBasicQueryConditionImpl(callbackRequest); - } - - @Override - public boolean isPublicScript() { - return false; - } - }, applicationService); - this.applicationService = applicationService; - } - - private Pair getBasicQueryConditionImpl(CallbackRequestDTO callbackRequest) { - IamSearchCondition searchCondition = IamSearchCondition.fromReq(callbackRequest); - BaseSearchCondition baseSearchCondition = new BaseSearchCondition(); - baseSearchCondition.setStart(searchCondition.getStart().intValue()); - baseSearchCondition.setLength(searchCondition.getLength().intValue()); - - ScriptQuery scriptQuery = new ScriptQuery(); - Long appId = - applicationService.getAppIdByScope(scriptCallbackHelper.extractResourceScopeCondition(searchCondition)); - scriptQuery.setAppId(appId); - return Pair.of(scriptQuery, baseSearchCondition); + public IamScriptCallbackResourceImpl(ScriptCallbackHelper scriptCallbackHelper) { + this.scriptCallbackHelper = scriptCallbackHelper; } @Override diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/impl/IamTagCallbackResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/impl/IamTagCallbackResourceImpl.java index 1dd3b0793d..545787e8e0 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/impl/IamTagCallbackResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/impl/IamTagCallbackResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,7 @@ package com.tencent.bk.job.manage.api.iam.impl; +import com.tencent.bk.audit.utils.json.JsonSchemaUtils; import com.tencent.bk.job.common.iam.constant.ResourceTypeId; import com.tencent.bk.job.common.iam.service.BaseIamCallbackService; import com.tencent.bk.job.common.iam.util.IamRespUtil; @@ -32,6 +33,7 @@ import com.tencent.bk.job.common.model.dto.ResourceScope; import com.tencent.bk.job.manage.api.iam.IamTagCallbackResource; import com.tencent.bk.job.manage.model.dto.TagDTO; +import com.tencent.bk.job.manage.model.esb.v3.response.EsbTagV3DTO; import com.tencent.bk.job.manage.service.ApplicationService; import com.tencent.bk.job.manage.service.TagService; import com.tencent.bk.sdk.iam.dto.PathInfoDTO; @@ -39,11 +41,13 @@ import com.tencent.bk.sdk.iam.dto.callback.request.IamSearchCondition; import com.tencent.bk.sdk.iam.dto.callback.response.CallbackBaseResponseDTO; import com.tencent.bk.sdk.iam.dto.callback.response.FetchInstanceInfoResponseDTO; +import com.tencent.bk.sdk.iam.dto.callback.response.FetchResourceTypeSchemaResponseDTO; import com.tencent.bk.sdk.iam.dto.callback.response.InstanceInfoDTO; import com.tencent.bk.sdk.iam.dto.callback.response.ListInstanceResponseDTO; import com.tencent.bk.sdk.iam.dto.callback.response.SearchInstanceResponseDTO; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Pair; +import org.slf4j.helpers.MessageFormatter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RestController; @@ -139,7 +143,11 @@ protected CallbackBaseResponseDTO fetchInstanceResp(CallbackRequestDTO callbackR Long tagId = Long.parseLong(instanceId); tagIdList.add(tagId); } catch (NumberFormatException e) { - log.error("Parse tag id failed!|{}", instanceId, e); + String msg = MessageFormatter.format( + "Parse tag id failed!|{}", + instanceId + ).getMessage(); + log.error(msg, e); } } List tagDTOList = tagService.listTagInfoByIds(tagIdList); @@ -176,4 +184,12 @@ protected CallbackBaseResponseDTO fetchInstanceResp(CallbackRequestDTO callbackR public CallbackBaseResponseDTO callback(CallbackRequestDTO callbackRequest) { return baseCallback(callbackRequest); } + + @Override + protected FetchResourceTypeSchemaResponseDTO fetchResourceTypeSchemaResp( + CallbackRequestDTO callbackRequest) { + FetchResourceTypeSchemaResponseDTO resp = new FetchResourceTypeSchemaResponseDTO(); + resp.setData(JsonSchemaUtils.generateJsonSchema(EsbTagV3DTO.class)); + return resp; + } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/impl/IamTaskPlanCallbackResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/impl/IamTaskPlanCallbackResourceImpl.java index bc77aa98b2..a952b8418b 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/impl/IamTaskPlanCallbackResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/impl/IamTaskPlanCallbackResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,7 @@ package com.tencent.bk.job.manage.api.iam.impl; +import com.tencent.bk.audit.utils.json.JsonSchemaUtils; import com.tencent.bk.job.common.iam.constant.ResourceTypeId; import com.tencent.bk.job.common.iam.service.BaseIamCallbackService; import com.tencent.bk.job.common.iam.util.IamRespUtil; @@ -34,6 +35,7 @@ import com.tencent.bk.job.manage.model.dto.TaskPlanQueryDTO; import com.tencent.bk.job.manage.model.dto.task.TaskPlanBasicInfoDTO; import com.tencent.bk.job.manage.model.dto.task.TaskPlanInfoDTO; +import com.tencent.bk.job.manage.model.esb.v3.response.EsbPlanInfoV3DTO; import com.tencent.bk.job.manage.service.ApplicationService; import com.tencent.bk.job.manage.service.plan.TaskPlanService; import com.tencent.bk.sdk.iam.dto.PathInfoDTO; @@ -41,11 +43,13 @@ import com.tencent.bk.sdk.iam.dto.callback.request.IamSearchCondition; import com.tencent.bk.sdk.iam.dto.callback.response.CallbackBaseResponseDTO; import com.tencent.bk.sdk.iam.dto.callback.response.FetchInstanceInfoResponseDTO; +import com.tencent.bk.sdk.iam.dto.callback.response.FetchResourceTypeSchemaResponseDTO; import com.tencent.bk.sdk.iam.dto.callback.response.InstanceInfoDTO; import com.tencent.bk.sdk.iam.dto.callback.response.ListInstanceResponseDTO; import com.tencent.bk.sdk.iam.dto.callback.response.SearchInstanceResponseDTO; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.tuple.Pair; +import org.slf4j.helpers.MessageFormatter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RestController; @@ -152,7 +156,11 @@ protected CallbackBaseResponseDTO fetchInstanceResp( long id = Long.parseLong(instanceId); planIdList.add(id); } catch (NumberFormatException e) { - log.error("Parse plan id failed!|{}", instanceId, e); + String msg = MessageFormatter.format( + "Parse plan id failed!|{}", + instanceId + ).getMessage(); + log.error(msg, e); } } List planBasicInfoDTOList = planService.listTaskPlanByIds(planIdList); @@ -189,4 +197,12 @@ protected CallbackBaseResponseDTO fetchInstanceResp( public CallbackBaseResponseDTO callback(CallbackRequestDTO callbackRequest) { return baseCallback(callbackRequest); } + + @Override + protected FetchResourceTypeSchemaResponseDTO fetchResourceTypeSchemaResp( + CallbackRequestDTO callbackRequest) { + FetchResourceTypeSchemaResponseDTO resp = new FetchResourceTypeSchemaResponseDTO(); + resp.setData(JsonSchemaUtils.generateJsonSchema(EsbPlanInfoV3DTO.class)); + return resp; + } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/impl/IamTaskTemplateCallbackResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/impl/IamTaskTemplateCallbackResourceImpl.java index 0a0ce6adfe..1af29ec524 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/impl/IamTaskTemplateCallbackResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/impl/IamTaskTemplateCallbackResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,7 @@ package com.tencent.bk.job.manage.api.iam.impl; +import com.tencent.bk.audit.utils.json.JsonSchemaUtils; import com.tencent.bk.job.common.iam.constant.ResourceTypeId; import com.tencent.bk.job.common.iam.service.BaseIamCallbackService; import com.tencent.bk.job.common.iam.util.IamRespUtil; @@ -32,6 +33,7 @@ import com.tencent.bk.job.common.model.dto.ResourceScope; import com.tencent.bk.job.manage.api.iam.IamTaskTemplateCallbackResource; import com.tencent.bk.job.manage.model.dto.task.TaskTemplateInfoDTO; +import com.tencent.bk.job.manage.model.esb.v3.response.EsbTemplateInfoV3DTO; import com.tencent.bk.job.manage.model.query.TaskTemplateQuery; import com.tencent.bk.job.manage.service.ApplicationService; import com.tencent.bk.job.manage.service.template.TaskTemplateService; @@ -40,10 +42,12 @@ import com.tencent.bk.sdk.iam.dto.callback.request.IamSearchCondition; import com.tencent.bk.sdk.iam.dto.callback.response.CallbackBaseResponseDTO; import com.tencent.bk.sdk.iam.dto.callback.response.FetchInstanceInfoResponseDTO; +import com.tencent.bk.sdk.iam.dto.callback.response.FetchResourceTypeSchemaResponseDTO; import com.tencent.bk.sdk.iam.dto.callback.response.InstanceInfoDTO; import com.tencent.bk.sdk.iam.dto.callback.response.ListInstanceResponseDTO; import com.tencent.bk.sdk.iam.dto.callback.response.SearchInstanceResponseDTO; import lombok.extern.slf4j.Slf4j; +import org.slf4j.helpers.MessageFormatter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RestController; @@ -139,7 +143,11 @@ protected CallbackBaseResponseDTO fetchInstanceResp( long id = Long.parseLong(instanceId); templateIdList.add(id); } catch (NumberFormatException e) { - log.error("Parse template id failed!|{}", instanceId, e); + String msg = MessageFormatter.format( + "Parse template id failed!|{}", + instanceId + ).getMessage(); + log.error(msg, e); } } List taskTemplateInfoDTOList = @@ -177,4 +185,12 @@ protected CallbackBaseResponseDTO fetchInstanceResp( public CallbackBaseResponseDTO callback(CallbackRequestDTO callbackRequest) { return baseCallback(callbackRequest); } + + @Override + protected FetchResourceTypeSchemaResponseDTO fetchResourceTypeSchemaResp( + CallbackRequestDTO callbackRequest) { + FetchResourceTypeSchemaResponseDTO resp = new FetchResourceTypeSchemaResponseDTO(); + resp.setData(JsonSchemaUtils.generateJsonSchema(EsbTemplateInfoV3DTO.class)); + return resp; + } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/impl/IamTicketCallbackResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/impl/IamTicketCallbackResourceImpl.java index c413155c0c..66393c6d67 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/impl/IamTicketCallbackResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/impl/IamTicketCallbackResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,7 @@ package com.tencent.bk.job.manage.api.iam.impl; +import com.tencent.bk.audit.utils.json.JsonSchemaUtils; import com.tencent.bk.job.common.iam.constant.ResourceTypeId; import com.tencent.bk.job.common.iam.service.BaseIamCallbackService; import com.tencent.bk.job.common.iam.util.IamRespUtil; @@ -32,6 +33,7 @@ import com.tencent.bk.job.common.model.dto.ResourceScope; import com.tencent.bk.job.manage.api.iam.IamTicketCallbackResource; import com.tencent.bk.job.manage.model.dto.CredentialDTO; +import com.tencent.bk.job.manage.model.esb.v3.response.EsbCredentialSimpleInfoV3DTO; import com.tencent.bk.job.manage.model.inner.resp.ServiceCredentialDisplayDTO; import com.tencent.bk.job.manage.service.ApplicationService; import com.tencent.bk.job.manage.service.CredentialService; @@ -40,6 +42,7 @@ import com.tencent.bk.sdk.iam.dto.callback.request.IamSearchCondition; import com.tencent.bk.sdk.iam.dto.callback.response.CallbackBaseResponseDTO; import com.tencent.bk.sdk.iam.dto.callback.response.FetchInstanceInfoResponseDTO; +import com.tencent.bk.sdk.iam.dto.callback.response.FetchResourceTypeSchemaResponseDTO; import com.tencent.bk.sdk.iam.dto.callback.response.InstanceInfoDTO; import com.tencent.bk.sdk.iam.dto.callback.response.ListInstanceResponseDTO; import com.tencent.bk.sdk.iam.dto.callback.response.SearchInstanceResponseDTO; @@ -184,4 +187,12 @@ protected CallbackBaseResponseDTO fetchInstanceResp( public CallbackBaseResponseDTO callback(CallbackRequestDTO callbackRequest) { return baseCallback(callbackRequest); } + + @Override + protected FetchResourceTypeSchemaResponseDTO fetchResourceTypeSchemaResp( + CallbackRequestDTO callbackRequest) { + FetchResourceTypeSchemaResponseDTO resp = new FetchResourceTypeSchemaResponseDTO(); + resp.setData(JsonSchemaUtils.generateJsonSchema(EsbCredentialSimpleInfoV3DTO.class)); + return resp; + } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/impl/PublicScriptCallbackHelper.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/impl/PublicScriptCallbackHelper.java new file mode 100644 index 0000000000..1dc5cb0504 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/impl/PublicScriptCallbackHelper.java @@ -0,0 +1,103 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.api.iam.impl; + +import com.tencent.bk.audit.utils.json.JsonSchemaUtils; +import com.tencent.bk.job.common.iam.util.IamRespUtil; +import com.tencent.bk.job.common.model.PageData; +import com.tencent.bk.job.manage.model.dto.ScriptBasicDTO; +import com.tencent.bk.job.manage.model.dto.ScriptDTO; +import com.tencent.bk.job.manage.model.esb.v3.response.EsbScriptV3DTO; +import com.tencent.bk.job.manage.model.query.ScriptQuery; +import com.tencent.bk.job.manage.service.ApplicationService; +import com.tencent.bk.job.manage.service.PublicScriptService; +import com.tencent.bk.sdk.iam.dto.callback.request.CallbackRequestDTO; +import com.tencent.bk.sdk.iam.dto.callback.request.IamSearchCondition; +import com.tencent.bk.sdk.iam.dto.callback.response.CallbackBaseResponseDTO; +import com.tencent.bk.sdk.iam.dto.callback.response.FetchResourceTypeSchemaResponseDTO; +import com.tencent.bk.sdk.iam.dto.callback.response.ListInstanceResponseDTO; +import com.tencent.bk.sdk.iam.dto.callback.response.SearchInstanceResponseDTO; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.List; + +@Slf4j +@Service +public class PublicScriptCallbackHelper extends AbstractScriptCallbackHelper { + private final PublicScriptService publicScriptService; + + @Autowired + public PublicScriptCallbackHelper(PublicScriptService publicScriptService, + ApplicationService applicationService) { + super(applicationService); + this.publicScriptService = publicScriptService; + } + + @Override + protected ListInstanceResponseDTO listInstanceResp(CallbackRequestDTO callbackRequest) { + ScriptQuery scriptQuery = buildBasicScriptQuery(callbackRequest); + PageData scriptDTOPageData = publicScriptService.listPageScript(scriptQuery); + + return IamRespUtil.getListInstanceRespFromPageData(scriptDTOPageData, this::convert); + } + + + @Override + protected SearchInstanceResponseDTO searchInstanceResp(CallbackRequestDTO callbackRequest) { + + ScriptQuery scriptQuery = buildBasicScriptQuery(callbackRequest); + scriptQuery.setName(callbackRequest.getFilter().getKeyword()); + PageData accountDTOPageData = publicScriptService.listPageScript(scriptQuery); + + return IamRespUtil.getSearchInstanceRespFromPageData(accountDTOPageData, this::convert); + } + + @Override + protected CallbackBaseResponseDTO fetchInstanceResp(CallbackRequestDTO callbackRequest) { + IamSearchCondition searchCondition = IamSearchCondition.fromReq(callbackRequest); + List scriptIdList = searchCondition.getIdList(); + List scriptBasicDTOList = publicScriptService.listScriptBasicInfoByScriptIds(scriptIdList); + return buildFetchInstanceResp(scriptIdList, scriptBasicDTOList); + } + + @Override + boolean isPublicScript() { + return true; + } + + public CallbackBaseResponseDTO doCallback(CallbackRequestDTO callbackRequest) { + return baseCallback(callbackRequest); + } + + @Override + protected FetchResourceTypeSchemaResponseDTO fetchResourceTypeSchemaResp( + CallbackRequestDTO callbackRequest) { + FetchResourceTypeSchemaResponseDTO resp = new FetchResourceTypeSchemaResponseDTO(); + resp.setData(JsonSchemaUtils.generateJsonSchema(EsbScriptV3DTO.class)); + return resp; + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/impl/ScriptCallbackHelper.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/impl/ScriptCallbackHelper.java index fc971f2819..c6cedc8bae 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/impl/ScriptCallbackHelper.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/iam/impl/ScriptCallbackHelper.java @@ -1,155 +1,103 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + package com.tencent.bk.job.manage.api.iam.impl; -import com.tencent.bk.job.common.iam.constant.ResourceTypeId; -import com.tencent.bk.job.common.iam.service.BaseIamCallbackService; +import com.tencent.bk.audit.utils.json.JsonSchemaUtils; import com.tencent.bk.job.common.iam.util.IamRespUtil; -import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; -import com.tencent.bk.job.common.model.dto.ResourceScope; import com.tencent.bk.job.manage.model.dto.ScriptBasicDTO; import com.tencent.bk.job.manage.model.dto.ScriptDTO; +import com.tencent.bk.job.manage.model.esb.v3.response.EsbScriptV3DTO; import com.tencent.bk.job.manage.model.query.ScriptQuery; import com.tencent.bk.job.manage.service.ApplicationService; import com.tencent.bk.job.manage.service.ScriptService; -import com.tencent.bk.sdk.iam.dto.PathInfoDTO; import com.tencent.bk.sdk.iam.dto.callback.request.CallbackRequestDTO; import com.tencent.bk.sdk.iam.dto.callback.request.IamSearchCondition; import com.tencent.bk.sdk.iam.dto.callback.response.CallbackBaseResponseDTO; -import com.tencent.bk.sdk.iam.dto.callback.response.FetchInstanceInfoResponseDTO; -import com.tencent.bk.sdk.iam.dto.callback.response.InstanceInfoDTO; +import com.tencent.bk.sdk.iam.dto.callback.response.FetchResourceTypeSchemaResponseDTO; import com.tencent.bk.sdk.iam.dto.callback.response.ListInstanceResponseDTO; import com.tencent.bk.sdk.iam.dto.callback.response.SearchInstanceResponseDTO; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.tuple.Pair; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; import java.util.List; -import java.util.Map; -import java.util.Set; @Slf4j -public class ScriptCallbackHelper extends BaseIamCallbackService { +@Service +public class ScriptCallbackHelper extends AbstractScriptCallbackHelper { private final ScriptService scriptService; - private final IGetBasicInfo basicInfoInterface; - private final ApplicationService applicationService; - public ScriptCallbackHelper( - ScriptService scriptService, - IGetBasicInfo basicInfoInterface, - ApplicationService applicationService) { + @Autowired + public ScriptCallbackHelper(ScriptService scriptService, + ApplicationService applicationService) { + super(applicationService); this.scriptService = scriptService; - this.basicInfoInterface = basicInfoInterface; - this.applicationService = applicationService; - } - - public interface IGetBasicInfo { - Pair getBasicQueryCondition(CallbackRequestDTO callbackRequest); - - boolean isPublicScript(); - } - - - private InstanceInfoDTO convert(ScriptDTO script) { - InstanceInfoDTO instanceInfo = new InstanceInfoDTO(); - instanceInfo.setId(String.valueOf(script.getId())); - instanceInfo.setDisplayName(script.getName()); - return instanceInfo; } @Override protected ListInstanceResponseDTO listInstanceResp(CallbackRequestDTO callbackRequest) { - Pair basicQueryCond = - basicInfoInterface.getBasicQueryCondition(callbackRequest); - - ScriptQuery scriptQuery = basicQueryCond.getLeft(); - BaseSearchCondition baseSearchCondition = basicQueryCond.getRight(); - PageData scriptDTOPageData = scriptService.listPageScript(scriptQuery, - baseSearchCondition); + ScriptQuery scriptQuery = buildBasicScriptQuery(callbackRequest); + PageData scriptDTOPageData = scriptService.listPageScript(scriptQuery); return IamRespUtil.getListInstanceRespFromPageData(scriptDTOPageData, this::convert); } + @Override protected SearchInstanceResponseDTO searchInstanceResp(CallbackRequestDTO callbackRequest) { - Pair basicQueryCond = - basicInfoInterface.getBasicQueryCondition(callbackRequest); - ScriptQuery scriptQuery = basicQueryCond.getLeft(); - BaseSearchCondition baseSearchCondition = basicQueryCond.getRight(); - + ScriptQuery scriptQuery = buildBasicScriptQuery(callbackRequest); scriptQuery.setName(callbackRequest.getFilter().getKeyword()); - PageData accountDTOPageData = scriptService.listPageScript(scriptQuery, - baseSearchCondition); + PageData accountDTOPageData = scriptService.listPageScript(scriptQuery); return IamRespUtil.getSearchInstanceRespFromPageData(accountDTOPageData, this::convert); } - private InstanceInfoDTO buildInstance(ScriptBasicDTO scriptBasicDTO, - Map appIdScopeMap) { - // 拓扑路径构建 - List path = new ArrayList<>(); - PathInfoDTO rootNode = new PathInfoDTO(); - if (basicInfoInterface.isPublicScript()) { - // 公共脚本 - rootNode.setType(ResourceTypeId.PUBLIC_SCRIPT); - rootNode.setId(scriptBasicDTO.getId()); - } else { - // 业务脚本 - Long appId = scriptBasicDTO.getAppId(); - rootNode = getPathNodeByAppId(appId, appIdScopeMap); - PathInfoDTO scriptNode = new PathInfoDTO(); - scriptNode.setType(ResourceTypeId.SCRIPT); - scriptNode.setId(scriptBasicDTO.getId()); - rootNode.setChild(scriptNode); - } - path.add(rootNode); - // 实例组装 - InstanceInfoDTO instanceInfo = new InstanceInfoDTO(); - instanceInfo.setId(scriptBasicDTO.getId()); - instanceInfo.setDisplayName(scriptBasicDTO.getName()); - instanceInfo.setPath(path); - return instanceInfo; - } - @Override - protected CallbackBaseResponseDTO fetchInstanceResp( - CallbackRequestDTO callbackRequest - ) { + protected CallbackBaseResponseDTO fetchInstanceResp(CallbackRequestDTO callbackRequest) { IamSearchCondition searchCondition = IamSearchCondition.fromReq(callbackRequest); - List instanceAttributeInfoList = new ArrayList<>(); List scriptIdList = searchCondition.getIdList(); List scriptBasicDTOList = scriptService.listScriptBasicInfoByScriptIds(scriptIdList); - Map scriptBasicDTOMap = new HashMap<>(scriptBasicDTOList.size()); - Set appIdSet = new HashSet<>(); - for (ScriptBasicDTO scriptBasicDTO : scriptBasicDTOList) { - scriptBasicDTOMap.put(scriptBasicDTO.getId(), scriptBasicDTO); - appIdSet.add(scriptBasicDTO.getAppId()); - } - // Job app --> CMDB biz/businessSet转换 - Map appIdScopeMap = applicationService.getScopeByAppIds(appIdSet); - for (String id : searchCondition.getIdList()) { - ScriptBasicDTO scriptBasicDTO = scriptBasicDTOMap.get(id); - if (scriptBasicDTO == null || scriptBasicDTO.isPublicScript() != basicInfoInterface.isPublicScript()) { - logNotExistId(id); - continue; - } - try { - InstanceInfoDTO instanceInfo = buildInstance(scriptBasicDTO, appIdScopeMap); - instanceAttributeInfoList.add(instanceInfo); - } catch (Exception e) { - logBuildInstanceFailure(scriptBasicDTO, e); - } - } + return buildFetchInstanceResp(scriptIdList, scriptBasicDTOList); + } - FetchInstanceInfoResponseDTO fetchInstanceInfoResponse = new FetchInstanceInfoResponseDTO(); - fetchInstanceInfoResponse.setCode(0L); - fetchInstanceInfoResponse.setData(instanceAttributeInfoList); - return fetchInstanceInfoResponse; + @Override + boolean isPublicScript() { + return false; } public CallbackBaseResponseDTO doCallback(CallbackRequestDTO callbackRequest) { return baseCallback(callbackRequest); } + + @Override + protected FetchResourceTypeSchemaResponseDTO fetchResourceTypeSchemaResp( + CallbackRequestDTO callbackRequest) { + FetchResourceTypeSchemaResponseDTO resp = new FetchResourceTypeSchemaResponseDTO(); + resp.setData(JsonSchemaUtils.generateJsonSchema(EsbScriptV3DTO.class)); + return resp; + } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceAccountResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceAccountResourceImpl.java index 87f4f57706..6a6e9ce8db 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceAccountResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceAccountResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,7 @@ package com.tencent.bk.job.manage.api.inner.impl; +import com.tencent.bk.job.common.constant.AccountCategoryEnum; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.exception.NotFoundException; @@ -32,12 +33,12 @@ import com.tencent.bk.job.common.model.InternalResponse; import com.tencent.bk.job.common.model.Response; import com.tencent.bk.job.common.model.dto.AppResourceScope; +import com.tencent.bk.job.common.mysql.JobTransactional; import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.common.util.ArrayUtil; import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.manage.api.inner.ServiceAccountResource; import com.tencent.bk.job.manage.auth.AccountAuthService; -import com.tencent.bk.job.manage.common.consts.account.AccountCategoryEnum; import com.tencent.bk.job.manage.model.dto.AccountDTO; import com.tencent.bk.job.manage.model.inner.ServiceAccountDTO; import com.tencent.bk.job.manage.model.web.request.AccountCreateUpdateReq; @@ -134,6 +135,7 @@ private ServiceAccountDTO toServiceAccountDTO(AccountDTO accountDTO) { } @Override + @JobTransactional(transactionManager = "jobManageTransactionManager") public InternalResponse saveOrGetAccount(String username, Long createTime, Long lastModifyTime, String lastModifyUser, Long appId, AccountCreateUpdateReq accountCreateUpdateReq) { @@ -163,9 +165,9 @@ public InternalResponse saveOrGetAccount(String username, Lon newAccount.setLastModifyUser(username); } - long accountId = accountService.saveAccount(newAccount); - newAccount.setId(accountId); - return InternalResponse.buildSuccessResp(newAccount.toServiceAccountDTO()); + AccountDTO savedAccount = accountService.createAccount(newAccount); + newAccount.setId(savedAccount.getId()); + return InternalResponse.buildSuccessResp(savedAccount.toServiceAccountDTO()); } @Override @@ -181,15 +183,15 @@ public Response saveAccount(String username, AccountDTO newAccount = accountService.buildCreateAccountDTO(username, appResourceScope.getAppId(), accountCreateUpdateReq); - long accountId = accountService.saveAccount(newAccount); - accountAuthService.registerAccount(username, accountId, newAccount.getAlias()); - return Response.buildSuccessResp(accountId); + AccountDTO savedAccount = accountService.createAccount(newAccount); + accountAuthService.registerAccount(username, savedAccount.getId(), newAccount.getAlias()); + return Response.buildSuccessResp(savedAccount.getId()); } @Override public Response> listAccounts(Long appId, Integer category) { List accountDTOS = - accountService.listAllAppAccount(appId, AccountCategoryEnum.valOf(category)); + accountService.listAppAccount(appId, AccountCategoryEnum.valOf(category)); List accounts = new ArrayList<>(); if (accountDTOS != null && !accountDTOS.isEmpty()) { for (AccountDTO accountDTO : accountDTOS) { diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceApplicationResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceApplicationResourceImpl.java index 3320570261..1cae0b9e20 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceApplicationResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceApplicationResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,9 @@ package com.tencent.bk.job.manage.api.inner.impl; +import com.tencent.bk.job.common.cc.model.bizset.BizSetInfo; +import com.tencent.bk.job.common.cc.sdk.IBizCmdbClient; +import com.tencent.bk.job.common.cc.sdk.IBizSetCmdbClient; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.constant.ResourceScopeTypeEnum; import com.tencent.bk.job.common.exception.NotFoundException; @@ -40,6 +43,7 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RestController; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Set; @@ -49,17 +53,23 @@ @RestController public class ServiceApplicationResourceImpl implements ServiceApplicationResource { private final ApplicationService applicationService; + private final IBizCmdbClient bizCmdbClient; + private final IBizSetCmdbClient bizSetCmdbClient; @Autowired - public ServiceApplicationResourceImpl(ApplicationService applicationService) { + public ServiceApplicationResourceImpl(ApplicationService applicationService, + IBizCmdbClient bizCmdbClient, + IBizSetCmdbClient bizSetCmdbClient) { this.applicationService = applicationService; + this.bizCmdbClient = bizCmdbClient; + this.bizSetCmdbClient = bizSetCmdbClient; } @Override public InternalResponse> listNormalApps() { List appList = applicationService.listAllApps(); List resultList = - appList.parallelStream().filter(ApplicationDTO::isBiz) + appList.stream().filter(ApplicationDTO::isBiz) .map(this::convertToServiceAppBaseInfo).collect(Collectors.toList()); return InternalResponse.buildSuccessResp(resultList); } @@ -69,7 +79,7 @@ public InternalResponse> listBizSetApps() { List applicationInfoDTOList = applicationService.listAppsByScopeType(ResourceScopeTypeEnum.BIZ_SET); List resultList = - applicationInfoDTOList.parallelStream().map(this::convertToServiceApp).collect(Collectors.toList()); + applicationInfoDTOList.stream().map(this::convertToServiceApp).collect(Collectors.toList()); return InternalResponse.buildSuccessResp(resultList); } @@ -149,4 +159,60 @@ public InternalResponse> listApps(String scopeType) appList.stream().map(this::convertToServiceApp).collect(Collectors.toList()); return InternalResponse.buildSuccessResp(resultList); } + + @Override + public InternalResponse> listAllAppIdOfArchivedScope() { + List loaclAllDeletedApps = applicationService.listAllDeletedApps(); + log.debug("find archived app from local, size={}", loaclAllDeletedApps.size()); + + List bizApps = loaclAllDeletedApps.stream() + .filter(app -> ResourceScopeTypeEnum.BIZ.getValue().equals(app.getScope().getType().getValue())) + .collect(Collectors.toList()); + List bizIds = bizApps.stream() + .map(bizApp -> Long.valueOf(bizApp.getScope().getId())) + .collect(Collectors.toList()); + + List bizSetApps = loaclAllDeletedApps.stream() + .filter(app -> ResourceScopeTypeEnum.BIZ_SET.getValue().equals(app.getScope().getType().getValue())) + .collect(Collectors.toList()); + List bizSetIds = bizSetApps.stream() + .map(bizSetApp -> Long.valueOf(bizSetApp.getScope().getId())) + .collect(Collectors.toList()); + + List archivedIds = new ArrayList<>(); + + if (CollectionUtils.isNotEmpty(bizApps)) { + List ccAllBizApps = bizCmdbClient.ListBizAppByIds(bizIds); + Set ccBizAppScopeIds = ccAllBizApps.stream() + .map(ccBizApp -> ccBizApp.getScope().getId()) + .collect(Collectors.toSet()); + archivedIds.addAll(bizApps.stream().filter(bizAppInfoDTO -> + !ccBizAppScopeIds.contains(bizAppInfoDTO.getScope().getId())) + .map(ApplicationDTO::getId) + .collect(Collectors.toList())); + } + if (CollectionUtils.isNotEmpty(bizSetApps)) { + List bizSetInfos = bizSetCmdbClient.ListBizSetByIds(bizSetIds); + Set ccBizSetAppScopeIds = bizSetInfos.stream() + .map(ccBizSetApp -> String.valueOf(ccBizSetApp.getId())) + .collect(Collectors.toSet()); + archivedIds.addAll(bizSetApps.stream().filter(bizAppInfoDTO -> + !ccBizSetAppScopeIds.contains(bizAppInfoDTO.getScope().getId())) + .map(ApplicationDTO::getId) + .collect(Collectors.toList())); + } + log.debug("finally find archived appIds={}", archivedIds); + return InternalResponse.buildSuccessResp(archivedIds); + } + + @Override + public InternalResponse existsAppById(Long appId) { + try { + ApplicationDTO appInfo = applicationService.getAppByAppId(appId); + return InternalResponse.buildSuccessResp(appInfo != null); + } catch (NotFoundException e){ + log.info("biz/bizSet not exist, appId={}", appId); + return InternalResponse.buildSuccessResp(false); + } + } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceBackupTmpResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceBackupTmpResourceImpl.java index e2261fc082..e6b4f61a66 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceBackupTmpResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceBackupTmpResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceCheckScriptResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceCheckScriptResourceImpl.java index 8eb7f27741..f8d86b02a3 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceCheckScriptResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceCheckScriptResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,8 +25,8 @@ package com.tencent.bk.job.manage.api.inner.impl; import com.tencent.bk.job.common.model.InternalResponse; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; import com.tencent.bk.job.manage.api.inner.ServiceCheckScriptResource; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; import com.tencent.bk.job.manage.model.dto.ScriptCheckResultItemDTO; import com.tencent.bk.job.manage.model.inner.ServiceScriptCheckResultItemDTO; import com.tencent.bk.job.manage.model.inner.request.ServiceCheckScriptRequest; @@ -51,7 +51,7 @@ public ServiceCheckScriptResourceImpl(ScriptCheckService scriptCheckService) { @Override public InternalResponse> check(ServiceCheckScriptRequest checkScriptRequest) { List checkResultItems = scriptCheckService.checkScriptWithDangerousRule( - ScriptTypeEnum.valueOf(checkScriptRequest.getScriptType()), checkScriptRequest.getScriptContent()); + ScriptTypeEnum.valOf(checkScriptRequest.getScriptType()), checkScriptRequest.getScriptContent()); return InternalResponse.buildSuccessResp(checkResultItems.stream() .map(ScriptCheckResultItemDTO::toServiceScriptCheckResultDTO) .collect(Collectors.toList())); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceCredentialResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceCredentialResourceImpl.java index 6a23b652e3..d3f9e69c77 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceCredentialResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceCredentialResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,17 +24,12 @@ package com.tencent.bk.job.manage.api.inner.impl; -import com.tencent.bk.job.common.iam.model.AuthResult; import com.tencent.bk.job.common.model.InternalResponse; -import com.tencent.bk.job.common.model.dto.AppResourceScope; import com.tencent.bk.job.manage.api.inner.ServiceCredentialResource; -import com.tencent.bk.job.manage.auth.TicketAuthService; -import com.tencent.bk.job.manage.model.inner.resp.ServiceBasicCredentialDTO; +import com.tencent.bk.job.manage.model.dto.CredentialDTO; import com.tencent.bk.job.manage.model.inner.resp.ServiceCredentialDTO; -import com.tencent.bk.job.manage.model.web.request.CredentialCreateUpdateReq; import com.tencent.bk.job.manage.service.CredentialService; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RestController; @@ -43,64 +38,16 @@ @Slf4j public class ServiceCredentialResourceImpl implements ServiceCredentialResource { - private final TicketAuthService ticketAuthService; private final CredentialService credentialService; @Autowired - public ServiceCredentialResourceImpl(TicketAuthService ticketAuthService, CredentialService credentialService) { - this.ticketAuthService = ticketAuthService; + public ServiceCredentialResourceImpl(CredentialService credentialService) { this.credentialService = credentialService; } @Override public InternalResponse getCredentialById(Long appId, String id) { - ServiceCredentialDTO serviceCredentialDTO = credentialService.getServiceCredentialById(appId, id); - return InternalResponse.buildSuccessResp(serviceCredentialDTO); - } - - @Override - public InternalResponse createCredential( - String username, - Long appId, - CredentialCreateUpdateReq createUpdateReq - ) { - return saveCredential(username, appId, createUpdateReq); - } - - @Override - public InternalResponse updateCredential( - String username, - Long appId, - CredentialCreateUpdateReq createUpdateReq - ) { - return saveCredential(username, appId, createUpdateReq); - } - - private InternalResponse saveCredential( - String username, - Long appId, - CredentialCreateUpdateReq createUpdateReq - ) { - AuthResult authResult; - if (StringUtils.isBlank(createUpdateReq.getId())) { - authResult = checkCreateTicketPermission(username, appId); - } else { - authResult = checkManageTicketPermission(username, appId, createUpdateReq.getId()); - } - if (!authResult.isPass()) { - return InternalResponse.buildAuthFailResp(AuthResult.toAuthResultDTO(authResult)); - } - String credentialId = credentialService.saveCredential(username, appId, createUpdateReq); - return InternalResponse.buildSuccessResp(new ServiceBasicCredentialDTO(credentialId)); - } - - public AuthResult checkCreateTicketPermission(String username, Long appId) { - // 需要拥有在业务下创建凭证的权限 - return ticketAuthService.authCreateTicket(username, new AppResourceScope(appId)); - } - - public AuthResult checkManageTicketPermission(String username, Long appId, String credentialId) { - // 需要拥有在业务下管理某个具体凭证的权限 - return ticketAuthService.authManageTicket(username, new AppResourceScope(appId), credentialId, null); + CredentialDTO credentialDTO = credentialService.getCredentialById(appId, id); + return InternalResponse.buildSuccessResp(credentialDTO.toServiceCredentialDTO()); } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceCrontabCallbackResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceCrontabCallbackResourceImpl.java index 960928611b..32aabd4299 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceCrontabCallbackResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceCrontabCallbackResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceGlobalSettingsResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceGlobalSettingsResourceImpl.java index 6554359547..57d2c947e4 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceGlobalSettingsResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceGlobalSettingsResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,7 +28,7 @@ import com.tencent.bk.job.manage.api.inner.ServiceGlobalSettingsResource; import com.tencent.bk.job.manage.model.inner.ServiceFileUploadSettingDTO; import com.tencent.bk.job.manage.model.web.vo.globalsetting.FileUploadSettingVO; -import com.tencent.bk.job.manage.service.GlobalSettingsService; +import com.tencent.bk.job.manage.service.globalsetting.GlobalSettingsService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RestController; @@ -49,14 +49,20 @@ public InternalResponse getDocCenterBaseUrl() { return InternalResponse.buildSuccessResp(globalSettingsService.getDocCenterBaseUrl()); } + @Override + public InternalResponse getDocJobRootUrl() { + return InternalResponse.buildSuccessResp(globalSettingsService.getDocJobRootUrl()); + } + @Override public InternalResponse getFileUploadSettings() { FileUploadSettingVO fileUploadSettingVO = globalSettingsService.getFileUploadSettings(); return InternalResponse.buildSuccessResp(convertToServiceFileUploadSettingDTO(fileUploadSettingVO)); } + private ServiceFileUploadSettingDTO convertToServiceFileUploadSettingDTO(FileUploadSettingVO fileUploadSettingVO) { ServiceFileUploadSettingDTO serviceFileUploadSettingDTO = new ServiceFileUploadSettingDTO(); - if(fileUploadSettingVO != null){ + if (fileUploadSettingVO != null) { serviceFileUploadSettingDTO.setSuffixList(fileUploadSettingVO.getSuffixList()); serviceFileUploadSettingDTO.setUnit(fileUploadSettingVO.getUnit()); serviceFileUploadSettingDTO.setAmount(fileUploadSettingVO.getAmount()); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceHostResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceHostResourceImpl.java index 2ae4086f5f..0c2f57dab3 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceHostResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceHostResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,47 +25,71 @@ package com.tencent.bk.job.manage.api.inner.impl; import com.tencent.bk.job.common.constant.ErrorCode; -import com.tencent.bk.job.common.constant.JobConstants; import com.tencent.bk.job.common.exception.NotImplementedException; import com.tencent.bk.job.common.model.InternalResponse; +import com.tencent.bk.job.common.model.dto.AppResourceScope; import com.tencent.bk.job.common.model.dto.ApplicationDTO; import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; -import com.tencent.bk.job.common.model.dto.DynamicGroupWithHost; import com.tencent.bk.job.common.model.dto.HostDTO; -import com.tencent.bk.job.common.model.vo.HostInfoVO; +import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.manage.api.inner.ServiceHostResource; +import com.tencent.bk.job.manage.dao.HostTopoDAO; +import com.tencent.bk.job.manage.model.dto.HostTopoDTO; import com.tencent.bk.job.manage.model.inner.ServiceHostDTO; import com.tencent.bk.job.manage.model.inner.ServiceHostStatusDTO; import com.tencent.bk.job.manage.model.inner.ServiceListAppHostResultDTO; +import com.tencent.bk.job.manage.model.inner.request.ServiceBatchGetAppHostsReq; +import com.tencent.bk.job.manage.model.inner.request.ServiceBatchGetHostToposReq; import com.tencent.bk.job.manage.model.inner.request.ServiceBatchGetHostsReq; -import com.tencent.bk.job.manage.model.inner.request.ServiceCheckAppHostsReq; import com.tencent.bk.job.manage.model.inner.request.ServiceGetHostStatusByDynamicGroupReq; -import com.tencent.bk.job.manage.model.inner.request.ServiceGetHostStatusByIpReq; +import com.tencent.bk.job.manage.model.inner.request.ServiceGetHostStatusByHostReq; import com.tencent.bk.job.manage.model.inner.request.ServiceGetHostStatusByNodeReq; -import com.tencent.bk.job.manage.model.web.request.ipchooser.AppTopologyTreeNode; -import com.tencent.bk.job.manage.model.web.vo.NodeInfoVO; +import com.tencent.bk.job.manage.model.inner.request.ServiceGetHostsByCloudIpv6Req; +import com.tencent.bk.job.manage.model.inner.resp.ServiceHostTopoDTO; +import com.tencent.bk.job.manage.model.web.request.chooser.host.BizTopoNode; import com.tencent.bk.job.manage.service.ApplicationService; -import com.tencent.bk.job.manage.service.HostService; +import com.tencent.bk.job.manage.service.host.BizTopoHostService; +import com.tencent.bk.job.manage.service.host.HostDetailService; +import com.tencent.bk.job.manage.service.host.HostService; +import com.tencent.bk.job.manage.service.host.impl.BizDynamicGroupHostService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RestController; import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; @Slf4j @RestController public class ServiceHostResourceImpl implements ServiceHostResource { + private final AppScopeMappingService appScopeMappingService; private final ApplicationService applicationService; private final HostService hostService; + private final BizTopoHostService bizTopoHostService; + private final BizDynamicGroupHostService bizDynamicGroupHostService; + private final HostDetailService hostDetailService; + private final HostTopoDAO hostTopoDAO; @Autowired - public ServiceHostResourceImpl(ApplicationService applicationService, - HostService hostService) { + public ServiceHostResourceImpl(AppScopeMappingService appScopeMappingService, + ApplicationService applicationService, + HostService hostService, + BizTopoHostService bizTopoHostService, + BizDynamicGroupHostService bizDynamicGroupHostService, + HostDetailService hostDetailService, + HostTopoDAO hostTopoDAO) { + this.appScopeMappingService = appScopeMappingService; this.applicationService = applicationService; this.hostService = hostService; + this.bizTopoHostService = bizTopoHostService; + this.bizDynamicGroupHostService = bizDynamicGroupHostService; + this.hostDetailService = hostDetailService; + this.hostTopoDAO = hostTopoDAO; } @Override @@ -76,23 +100,16 @@ public InternalResponse> getHostStatusByNode(Long app String msg = "topo node of bizset not supported yet"; throw new NotImplementedException(msg, ErrorCode.NOT_SUPPORT_FEATURE); } - List treeNodeList = req.getTreeNodeList(); - List nodeInfoVOList = hostService.getBizHostsByNode( - JobConstants.DEFAULT_SYSTEM_USER_ADMIN, - appDTO.getBizIdIfBizApp(), - treeNodeList - ); - List hostStatusDTOList = new ArrayList<>(); - nodeInfoVOList.parallelStream().forEach(nodeInfoVO -> nodeInfoVO.getIpListStatus().forEach(hostInfoVO -> { + List treeNodeList = req.getTreeNodeList(); + List hostList = bizTopoHostService.listHostByNodes(appDTO.getBizIdIfBizApp(), treeNodeList); + Set hostStatusDTOSet = new HashSet<>(); + hostList.forEach(host -> { ServiceHostStatusDTO serviceHostStatusDTO = new ServiceHostStatusDTO(); - serviceHostStatusDTO.setHostId(hostInfoVO.getHostId()); - serviceHostStatusDTO.setIp(hostInfoVO.getIp()); - serviceHostStatusDTO.setAlive(hostInfoVO.getAlive()); - if (!hostStatusDTOList.contains(serviceHostStatusDTO)) { - hostStatusDTOList.add(serviceHostStatusDTO); - } - })); - return InternalResponse.buildSuccessResp(hostStatusDTOList); + serviceHostStatusDTO.setHostId(host.getHostId()); + serviceHostStatusDTO.setAlive(host.getAgentStatusValue()); + hostStatusDTOSet.add(serviceHostStatusDTO); + }); + return InternalResponse.buildSuccessResp(new ArrayList<>(hostStatusDTOSet)); } @Override @@ -106,43 +123,33 @@ public InternalResponse> getHostStatusByDynamicGroup( throw new NotImplementedException(msg, ErrorCode.NOT_SUPPORT_FEATURE); } List dynamicGroupIdList = req.getDynamicGroupIdList(); - List dynamicGroupWithHostList = hostService.getBizDynamicGroupHostList( - JobConstants.DEFAULT_SYSTEM_USER_ADMIN, - appDTO.getBizIdIfBizApp(), + AppResourceScope appResourceScope = new AppResourceScope(appId); + appScopeMappingService.fillAppResourceScope(appResourceScope); + List hostList = bizDynamicGroupHostService.listHostByDynamicGroups( + appResourceScope, dynamicGroupIdList ); - List hostStatusDTOList = new ArrayList<>(); - dynamicGroupWithHostList.parallelStream().forEach(dynamicGroupInfoDTO -> - dynamicGroupInfoDTO.getIpListStatus().forEach(hostInfoVO -> { - ServiceHostStatusDTO serviceHostStatusDTO = new ServiceHostStatusDTO(); - serviceHostStatusDTO.setHostId(hostInfoVO.getHostId()); - serviceHostStatusDTO.setIp(hostInfoVO.getIp()); - serviceHostStatusDTO.setAlive(hostInfoVO.getGseAgentAlive() ? 1 : 0); - if (!hostStatusDTOList.contains(serviceHostStatusDTO)) { - hostStatusDTOList.add(serviceHostStatusDTO); - } - })); - return InternalResponse.buildSuccessResp(hostStatusDTOList); + Set hostStatusDTOSet = new HashSet<>(); + hostList.forEach(applicationHostDTO -> { + ServiceHostStatusDTO serviceHostStatusDTO = new ServiceHostStatusDTO(); + serviceHostStatusDTO.setHostId(applicationHostDTO.getHostId()); + serviceHostStatusDTO.setAlive(applicationHostDTO.getGseAgentAlive() ? 1 : 0); + hostStatusDTOSet.add(serviceHostStatusDTO); + }); + return InternalResponse.buildSuccessResp(new ArrayList<>(hostStatusDTOSet)); } @Override - public InternalResponse> getHostStatusByIp(Long appId, - ServiceGetHostStatusByIpReq req) { - List ipList = req.getIpList(); - List hostInfoVOList = hostService.getHostsByIp( - JobConstants.DEFAULT_SYSTEM_USER_ADMIN, - appId, - null, - ipList - ); + public InternalResponse> getHostStatusByHost(Long appId, + ServiceGetHostStatusByHostReq req) { + List hostDTOList = hostService.listHosts(req.getHostList()); List hostStatusDTOList = new ArrayList<>(); - hostInfoVOList.forEach(hostInfoVO -> { - ServiceHostStatusDTO serviceHostStatusDTO = new ServiceHostStatusDTO(); - serviceHostStatusDTO.setHostId(hostInfoVO.getHostId()); - serviceHostStatusDTO.setIp(hostInfoVO.getCloudAreaInfo().getId() + ":" + hostInfoVO.getIp()); - serviceHostStatusDTO.setAlive(hostInfoVO.getAlive()); - if (!hostStatusDTOList.contains(serviceHostStatusDTO)) { - hostStatusDTOList.add(serviceHostStatusDTO); + hostDTOList.forEach(host -> { + ServiceHostStatusDTO hostStatusDTO = new ServiceHostStatusDTO(); + hostStatusDTO.setHostId(host.getHostId()); + hostStatusDTO.setAlive(host.getAgentStatusValue()); + if (!hostStatusDTOList.contains(hostStatusDTO)) { + hostStatusDTOList.add(hostStatusDTO); } }); return InternalResponse.buildSuccessResp(hostStatusDTOList); @@ -150,21 +157,51 @@ public InternalResponse> getHostStatusByIp(Long appId @Override public InternalResponse batchGetAppHosts(Long appId, - ServiceCheckAppHostsReq req) { - return InternalResponse.buildSuccessResp(hostService.listAppHosts(appId, req.getHosts())); + ServiceBatchGetAppHostsReq req) { + req.validate(); + ServiceListAppHostResultDTO result = + hostService.listAppHostsPreferCache(appId, req.getHosts(), req.isRefreshAgentId()); + if (CollectionUtils.isNotEmpty(result.getValidHosts())) { + hostDetailService.fillDetailForHosts(result.getValidHosts()); + } + return InternalResponse.buildSuccessResp(result); } @Override public InternalResponse> batchGetHosts(ServiceBatchGetHostsReq req) { - List hostIps = req.getHosts(); - List hosts = hostService.listHosts(hostIps); + List queryHosts = req.getHosts(); + List hosts = hostService.listHosts(queryHosts); if (CollectionUtils.isEmpty(hosts)) { - return InternalResponse.buildSuccessResp(null); + return InternalResponse.buildSuccessResp(Collections.emptyList()); } + hostDetailService.fillDetailForApplicationHosts(hosts); return InternalResponse.buildSuccessResp( hosts.stream() .map(ServiceHostDTO::fromApplicationHostDTO) .collect(Collectors.toList())); } + + @Override + public InternalResponse> getHostsByCloudIpv6(ServiceGetHostsByCloudIpv6Req req) { + List hosts = hostService.listHostsByCloudIpv6(req.getCloudAreaId(), req.getIpv6()); + if (CollectionUtils.isEmpty(hosts)) { + return InternalResponse.buildSuccessResp(Collections.emptyList()); + } + hostDetailService.fillDetailForApplicationHosts(hosts); + return InternalResponse.buildSuccessResp( + hosts.stream() + .map(ServiceHostDTO::fromApplicationHostDTO) + .collect(Collectors.toList())); + } + + @Override + public InternalResponse> batchGetHostTopos(ServiceBatchGetHostToposReq req) { + List hostTopoDTOList = hostTopoDAO.listHostTopoByHostIds(req.getHostIdList()); + List serviceHostTopoDTOList = hostTopoDTOList.stream() + .map(HostTopoDTO::toServiceHostTopoDTO) + .collect(Collectors.toList()); + return InternalResponse.buildSuccessResp(serviceHostTopoDTOList); + } + } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceMetricsResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceMetricsResourceImpl.java index 1bb94b51df..4c842ae2eb 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceMetricsResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceMetricsResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,19 +25,19 @@ package com.tencent.bk.job.manage.api.inner.impl; import com.tencent.bk.job.common.model.InternalResponse; +import com.tencent.bk.job.manage.api.common.constants.JobResourceStatusEnum; +import com.tencent.bk.job.manage.api.common.constants.account.AccountTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskFileTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskScriptSourceEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskStepTypeEnum; import com.tencent.bk.job.manage.api.inner.ServiceMetricsResource; -import com.tencent.bk.job.manage.common.consts.JobResourceStatusEnum; -import com.tencent.bk.job.manage.common.consts.account.AccountTypeEnum; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskFileTypeEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskScriptSourceEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskStepTypeEnum; import com.tencent.bk.job.manage.model.dto.ResourceTagDTO; import com.tencent.bk.job.manage.service.AccountService; import com.tencent.bk.job.manage.service.ApplicationService; -import com.tencent.bk.job.manage.service.HostService; -import com.tencent.bk.job.manage.service.ScriptService; +import com.tencent.bk.job.manage.service.ScriptManager; import com.tencent.bk.job.manage.service.TagService; +import com.tencent.bk.job.manage.service.host.HostService; import com.tencent.bk.job.manage.service.plan.TaskPlanService; import com.tencent.bk.job.manage.service.template.TaskTemplateService; import lombok.extern.slf4j.Slf4j; @@ -45,27 +45,31 @@ import org.springframework.web.bind.annotation.RestController; import java.util.List; +import java.util.Map; -@RestController +@RestController("manageMetricsResource") @Slf4j public class ServiceMetricsResourceImpl implements ServiceMetricsResource { private final ApplicationService applicationService; private final AccountService accountService; - private final ScriptService scriptService; + private final ScriptManager scriptManager; private final TaskTemplateService taskTemplateService; private final TaskPlanService taskPlanService; private final HostService hostService; private final TagService tagService; @Autowired - public ServiceMetricsResourceImpl(ApplicationService applicationService, AccountService accountService, - ScriptService scriptService, TaskTemplateService taskTemplateService, - TaskPlanService taskPlanService, HostService hostService, + public ServiceMetricsResourceImpl(ApplicationService applicationService, + AccountService accountService, + ScriptManager scriptManager, + TaskTemplateService taskTemplateService, + TaskPlanService taskPlanService, + HostService hostService, TagService tagService) { this.applicationService = applicationService; this.accountService = accountService; - this.scriptService = scriptService; + this.scriptManager = scriptManager; this.taskTemplateService = taskTemplateService; this.taskPlanService = taskPlanService; this.hostService = hostService; @@ -89,33 +93,33 @@ public InternalResponse countTaskPlans(Long appId) { @Override public InternalResponse countTemplateSteps(Long appId, TaskStepTypeEnum taskStepType, - TaskScriptSourceEnum scriptSource, TaskFileTypeEnum fileType) { + TaskScriptSourceEnum scriptSource, TaskFileTypeEnum fileType) { return InternalResponse.buildSuccessResp(taskTemplateService.countTemplateSteps(appId, taskStepType, scriptSource, fileType)); } @Override public InternalResponse countScripts(Long appId, ScriptTypeEnum scriptTypeEnum, - JobResourceStatusEnum jobResourceStatusEnum) { - return InternalResponse.buildSuccessResp(scriptService.countScripts(appId, scriptTypeEnum, + JobResourceStatusEnum jobResourceStatusEnum) { + return InternalResponse.buildSuccessResp(scriptManager.countScripts(appId, scriptTypeEnum, jobResourceStatusEnum)); } @Override public InternalResponse countCiteScripts(Long appId) { - return InternalResponse.buildSuccessResp(scriptService.countCiteScripts(appId)); + return InternalResponse.buildSuccessResp(scriptManager.countCiteScripts(appId)); } @Override public InternalResponse countCiteScriptSteps(Long appId) { - List scriptIdList = scriptService.listScriptIds(appId); + List scriptIdList = scriptManager.listScriptIds(appId); return InternalResponse.buildSuccessResp(taskTemplateService.countCiteScriptSteps(appId, scriptIdList)); } @Override public InternalResponse countScriptVersions(Long appId, ScriptTypeEnum scriptTypeEnum, - JobResourceStatusEnum jobResourceStatusEnum) { - return InternalResponse.buildSuccessResp(scriptService.countScriptVersions(appId, scriptTypeEnum, + JobResourceStatusEnum jobResourceStatusEnum) { + return InternalResponse.buildSuccessResp(scriptManager.countScriptVersions(appId, scriptTypeEnum, jobResourceStatusEnum)); } @@ -129,9 +133,14 @@ public InternalResponse countHostsByOsType(String osType) { return InternalResponse.buildSuccessResp(hostService.countHostsByOsType(osType)); } + @Override + public InternalResponse> groupHostByOsType() { + return InternalResponse.buildSuccessResp(hostService.groupHostByOsType()); + } + @Override public InternalResponse tagCitedCount(Long appId, Long tagId) { List scriptResourceTags = tagService.listResourceTagsByTagId(appId, tagId); - return InternalResponse.buildSuccessResp(Long.valueOf(scriptResourceTags.size())); + return InternalResponse.buildSuccessResp((long) scriptResourceTags.size()); } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceNotificationResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceNotificationResourceImpl.java index c026f1597e..372d3bda1f 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceNotificationResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceNotificationResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,9 +26,11 @@ import com.tencent.bk.job.common.cc.model.AppRoleDTO; import com.tencent.bk.job.common.model.InternalResponse; +import com.tencent.bk.job.common.model.dto.notify.CustomNotifyDTO; import com.tencent.bk.job.common.util.PrefConsts; import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.manage.api.inner.ServiceNotificationResource; +import com.tencent.bk.job.manage.model.inner.ServiceSpecificResourceNotifyPolicyDTO; import com.tencent.bk.job.manage.model.dto.notify.NotifyEsbChannelDTO; import com.tencent.bk.job.manage.model.inner.ServiceAppRoleDTO; import com.tencent.bk.job.manage.model.inner.ServiceNotificationMessage; @@ -57,15 +59,20 @@ public ServiceNotificationResourceImpl(NotifyService notifyService) { @Override public InternalResponse sendNotificationsToUsers(ServiceUserNotificationDTO serviceUserNotificationDTO) { - log.info(String.format("Input:%s", JsonUtils.toJson(serviceUserNotificationDTO))); - return InternalResponse.buildSuccessResp(notifyService.asyncSendNotificationsToUsers(serviceUserNotificationDTO)); + if (log.isDebugEnabled()) { + log.debug("Input: {}", JsonUtils.toJson(serviceUserNotificationDTO)); + } + return InternalResponse.buildSuccessResp( + notifyService.asyncSendNotificationsToUsers(serviceUserNotificationDTO)); } @Override public InternalResponse sendNotificationsToAdministrators( ServiceNotificationMessage serviceNotificationMessage ) { - log.info(String.format("Input:%s", JsonUtils.toJson(serviceNotificationMessage))); + if (log.isDebugEnabled()) { + log.debug("Input: {}", JsonUtils.toJson(serviceNotificationMessage)); + } return InternalResponse.buildSuccessResp( notifyService.asyncSendNotificationsToAdministrators(serviceNotificationMessage) ); @@ -75,7 +82,9 @@ public InternalResponse sendNotificationsToAdministrators( public InternalResponse triggerTemplateNotification( ServiceTriggerTemplateNotificationDTO triggerTemplateNotification ) { - log.info(String.format("Input:%s", JsonUtils.toJson(triggerTemplateNotification))); + if (log.isDebugEnabled()) { + log.debug("Input: {}", JsonUtils.toJson(triggerTemplateNotification)); + } StopWatch watch = new StopWatch(); watch.start("triggerTemplateNotification"); Integer result = notifyService.triggerTemplateNotification(triggerTemplateNotification); @@ -90,13 +99,14 @@ public InternalResponse triggerTemplateNotification( @Override public InternalResponse sendTemplateNotification(ServiceTemplateNotificationDTO templateNotificationDTO) { - log.info(String.format("Input:%s", JsonUtils.toJson(templateNotificationDTO))); + if (log.isDebugEnabled()) { + log.debug("Input: {}", JsonUtils.toJson(templateNotificationDTO)); + } return InternalResponse.buildSuccessResp(notifyService.sendTemplateNotification(templateNotificationDTO)); } @Override public InternalResponse> getNotifyRoles(String lang) { - log.info(String.format("Input:%s", lang)); List roles = notifyService.listRoles(); List result = roles.stream().map(role -> new ServiceAppRoleDTO(role.getId(), role.getName())).collect(Collectors.toList()); @@ -105,11 +115,40 @@ public InternalResponse> getNotifyRoles(String lang) { @Override public InternalResponse> getNotifyChannels(String lang) { - log.info(String.format("Input:%s", lang)); List channels = notifyService.listAllNotifyChannel(); List result = channels.stream().map(channel -> new ServiceNotifyChannelDTO(channel.getType(), channel.getLabel())).collect(Collectors.toList()); return InternalResponse.buildSuccessResp(result); } + + @Override + public InternalResponse createOrUpdateSpecificResourceNotifyPolicy( + String username, + Long appId, + ServiceSpecificResourceNotifyPolicyDTO serviceNotifyPolicyDTO) { + + return InternalResponse.buildSuccessResp(notifyService.saveSpecificResourceNotifyPolicies( + appId, + username, + serviceNotifyPolicyDTO + )); + } + + @Override + public InternalResponse deleteSpecificResourceNotifyPolicy(Long appId, + Integer resourceType, String resourceId) { + return InternalResponse.buildSuccessResp(notifyService.deleteAppResourceNotifyPolicies( + appId, + resourceType, + resourceId + )); + } + + @Override + public InternalResponse getSpecificResourceNotifyPolicy(Long appId, Integer resourceType, + String resourceId, Integer triggerType) { + return InternalResponse.buildSuccessResp(notifyService.getSpecificResourceNotifyPolicy( + appId, resourceType, resourceId, triggerType)); + } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceScriptResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceScriptResourceImpl.java index 6bc02a6700..baad757e66 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceScriptResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceScriptResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -29,14 +29,15 @@ import com.tencent.bk.job.common.exception.NotFoundException; import com.tencent.bk.job.common.i18n.service.MessageI18nService; import com.tencent.bk.job.common.model.InternalResponse; +import com.tencent.bk.job.common.mysql.JobTransactional; import com.tencent.bk.job.common.util.ArrayUtil; import com.tencent.bk.job.manage.api.common.ScriptDTOBuilder; import com.tencent.bk.job.manage.api.inner.ServiceScriptResource; import com.tencent.bk.job.manage.model.dto.ScriptDTO; import com.tencent.bk.job.manage.model.dto.converter.ScriptConverter; import com.tencent.bk.job.manage.model.inner.ServiceScriptDTO; -import com.tencent.bk.job.manage.model.web.request.ScriptCreateUpdateReq; -import com.tencent.bk.job.manage.service.ScriptService; +import com.tencent.bk.job.manage.model.web.request.ScriptCreateReq; +import com.tencent.bk.job.manage.service.ScriptManager; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; @@ -46,16 +47,16 @@ @RestController @Slf4j public class ServiceScriptResourceImpl implements ServiceScriptResource { - private final ScriptService scriptService; + private final ScriptManager scriptManager; private final MessageI18nService i18nService; private final ScriptDTOBuilder scriptDTOBuilder; @Autowired - public ServiceScriptResourceImpl(MessageI18nService i18nService, ScriptService scriptService, + public ServiceScriptResourceImpl(MessageI18nService i18nService, ScriptManager scriptManager, ScriptDTOBuilder scriptDTOBuilder) { this.i18nService = i18nService; - this.scriptService = scriptService; + this.scriptManager = scriptManager; this.scriptDTOBuilder = scriptDTOBuilder; } @@ -71,7 +72,7 @@ public InternalResponse getScriptByAppIdAndScriptVersionId(Str log.warn("Get script version by id, param scriptVersionId is empty"); throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } - ScriptDTO script = scriptService.getScriptVersion(username, appId, scriptVersionId); + ScriptDTO script = scriptManager.getScriptVersion(appId, scriptVersionId); if (script == null) { log.warn("Get script version by id:{}, the script is not exist", scriptVersionId); throw new NotFoundException(ErrorCode.SCRIPT_NOT_EXIST, ArrayUtil.toArray(scriptVersionId)); @@ -87,7 +88,7 @@ public InternalResponse getScriptByScriptVersionId(Long script throw new InvalidParamException(ErrorCode.MISSING_PARAM); } - ScriptDTO script = scriptService.getScriptVersion(scriptVersionId); + ScriptDTO script = scriptManager.getScriptVersion(scriptVersionId); if (script == null) { log.warn("Get script version by id:{}, the script is not exist", scriptVersionId); throw new NotFoundException(ErrorCode.SCRIPT_NOT_EXIST, ArrayUtil.toArray(scriptVersionId)); @@ -98,15 +99,16 @@ public InternalResponse getScriptByScriptVersionId(Long script } @Override + @JobTransactional(transactionManager = "jobManageTransactionManager") public InternalResponse> createScriptWithVersionId(String username, Long createTime, - Long lastModifyTime, String lastModifyUser, - Integer scriptStatus, Long appId, - ScriptCreateUpdateReq scriptCreateUpdateReq) { + Long lastModifyTime, String lastModifyUser, + Integer scriptStatus, Long appId, + ScriptCreateReq scriptCreateReq) { if (log.isDebugEnabled()) { log.debug("createScriptWithVersionId,operator={},appId={},script={},status={}", username, appId, - scriptCreateUpdateReq, scriptStatus); + scriptCreateReq, scriptStatus); } - ScriptDTO script = scriptDTOBuilder.buildFromCreateUpdateReq(scriptCreateUpdateReq); + ScriptDTO script = scriptDTOBuilder.buildFromScriptCreateReq(scriptCreateReq); script.setAppId(appId); script.setCreator(username); if (StringUtils.isNotBlank(lastModifyUser)) { @@ -118,7 +120,7 @@ public InternalResponse> createScriptWithVersionId(String use script.setStatus(scriptStatus); } return InternalResponse.buildSuccessResp( - scriptService.createScriptWithVersionId(username, appId, script, createTime, lastModifyTime)); + scriptManager.createScriptWithVersionId(appId, script, createTime, lastModifyTime)); } @Override @@ -128,7 +130,7 @@ public InternalResponse getBasicScriptInfo(String scriptId) { throw new InvalidParamException(ErrorCode.MISSING_PARAM); } - ScriptDTO script = scriptService.getScriptWithoutTagByScriptId(scriptId); + ScriptDTO script = scriptManager.getScriptWithoutTagByScriptId(scriptId); if (script == null) { log.warn("Get script by id:{}, the script is not exist", scriptId); throw new NotFoundException(ErrorCode.SCRIPT_NOT_EXIST, ArrayUtil.toArray(scriptId)); @@ -144,7 +146,7 @@ public InternalResponse getOnlineScriptVersion(String scriptId throw new InvalidParamException(ErrorCode.MISSING_PARAM); } - ScriptDTO script = scriptService.getOnlineScriptVersionByScriptId(scriptId); + ScriptDTO script = scriptManager.getOnlineScriptVersionByScriptId(scriptId); if (script == null) { return InternalResponse.buildSuccessResp(null); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceScriptTemplateResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceScriptTemplateResourceImpl.java new file mode 100644 index 0000000000..99fa9756fc --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceScriptTemplateResourceImpl.java @@ -0,0 +1,70 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.api.inner.impl; + +import com.tencent.bk.job.common.model.InternalResponse; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; +import com.tencent.bk.job.manage.api.inner.ServiceScriptTemplateResource; +import lombok.extern.slf4j.Slf4j; +import org.slf4j.helpers.MessageFormatter; +import org.springframework.util.StreamUtils; +import org.springframework.web.bind.annotation.RestController; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; + +@Slf4j +@RestController +public class ServiceScriptTemplateResourceImpl implements ServiceScriptTemplateResource { + @Override + public InternalResponse getScriptTemplate(Integer type) { + ScriptTypeEnum typeEnum = ScriptTypeEnum.valOf(type); + if (typeEnum == null) { + log.warn("Unknown script type: {}", type); + return InternalResponse.buildSuccessResp(null); + } + String scriptTemplateFileName = typeEnum.getName(); + String scriptTemplateFilePath = "script_template/" + scriptTemplateFileName; + InputStream inputStream = getClass().getClassLoader().getResourceAsStream(scriptTemplateFilePath); + try { + String scriptTemplate = StreamUtils.copyToString(inputStream, StandardCharsets.UTF_8); + return InternalResponse.buildSuccessResp(scriptTemplate); + } catch (IOException e) { + String message = MessageFormatter.format( + "Fail to read content of {}", + scriptTemplateFilePath + ).getMessage(); + log.warn(message, e); + } finally { + try { + inputStream.close(); + } catch (IOException e) { + log.warn("Fail to close inputStream", e); + } + } + return InternalResponse.buildSuccessResp(null); + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceSyncResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceSyncResourceImpl.java index 256b959e69..426e0f181d 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceSyncResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceSyncResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -33,8 +33,8 @@ import com.tencent.bk.job.manage.model.inner.ServiceHostInfoDTO; import com.tencent.bk.job.manage.model.inner.resp.ServiceApplicationDTO; import com.tencent.bk.job.manage.service.ApplicationService; -import com.tencent.bk.job.manage.service.HostService; import com.tencent.bk.job.manage.service.SyncService; +import com.tencent.bk.job.manage.service.host.HostService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RestController; @@ -155,7 +155,7 @@ private ServiceHostInfoDTO convertToServiceHostInfo(long appId, ApplicationHostD serviceHostInfo.setAppId(appId); serviceHostInfo.setCloudAreaId(hostInfo.getCloudAreaId()); serviceHostInfo.setIp(hostInfo.getIp()); - serviceHostInfo.setDisplayIp(hostInfo.getDisplayIp()); + serviceHostInfo.setIpv6(hostInfo.getIpv6()); serviceHostInfo.setHostId(hostInfo.getHostId()); return serviceHostInfo; } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceTagResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceTagResourceImpl.java index 890b102222..78cf7b97ba 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceTagResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceTagResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -57,12 +57,12 @@ private ServiceTagDTO convert(TagDTO tagDTO) { @Override public InternalResponse> listTags(Long appId) { List tags = tagService.listTags(appId, null); - return InternalResponse.buildSuccessResp(tags.parallelStream().map(this::convert).collect(Collectors.toList())); + return InternalResponse.buildSuccessResp(tags.stream().map(this::convert).collect(Collectors.toList())); } @Override public InternalResponse> listPublicTags() { List tags = tagService.listTags(JobConstants.PUBLIC_APP_ID, null); - return InternalResponse.buildSuccessResp(tags.parallelStream().map(this::convert).collect(Collectors.toList())); + return InternalResponse.buildSuccessResp(tags.stream().map(this::convert).collect(Collectors.toList())); } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceTaskPlanResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceTaskPlanResourceImpl.java index 424d743b9e..237fa6c865 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceTaskPlanResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceTaskPlanResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,17 +24,18 @@ package com.tencent.bk.job.manage.api.inner.impl; +import com.tencent.bk.job.common.constant.AccountCategoryEnum; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.constant.TaskVariableTypeEnum; import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.exception.NotFoundException; -import com.tencent.bk.job.common.i18n.service.MessageI18nService; import com.tencent.bk.job.common.model.InternalResponse; +import com.tencent.bk.job.common.mysql.JobTransactional; import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.manage.api.common.constants.task.TaskFileTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskScriptSourceEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskStepTypeEnum; import com.tencent.bk.job.manage.api.inner.ServiceTaskPlanResource; -import com.tencent.bk.job.manage.common.consts.account.AccountCategoryEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskFileTypeEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskStepTypeEnum; import com.tencent.bk.job.manage.model.dto.AccountDTO; import com.tencent.bk.job.manage.model.dto.ScriptDTO; import com.tencent.bk.job.manage.model.dto.converter.TaskStepConverter; @@ -60,6 +61,7 @@ import com.tencent.bk.job.manage.model.web.vo.task.TaskPlanVO; import com.tencent.bk.job.manage.service.AbstractTaskVariableService; import com.tencent.bk.job.manage.service.AccountService; +import com.tencent.bk.job.manage.service.PublicScriptService; import com.tencent.bk.job.manage.service.ScriptService; import com.tencent.bk.job.manage.service.plan.TaskPlanService; import lombok.extern.slf4j.Slf4j; @@ -84,23 +86,21 @@ public class ServiceTaskPlanResourceImpl implements ServiceTaskPlanResource { private final AbstractTaskVariableService taskVariableService; private final ScriptService scriptService; + private final PublicScriptService publicScriptService; private final AccountService accountService; - private final MessageI18nService i18nService; - @Autowired public ServiceTaskPlanResourceImpl( TaskPlanService taskPlanService, @Qualifier("TaskPlanVariableServiceImpl") AbstractTaskVariableService taskVariableService, - ScriptService scriptService, - AccountService accountService, - MessageI18nService i18nService) { + ScriptService scriptService, PublicScriptService publicScriptService, + AccountService accountService) { this.taskPlanService = taskPlanService; this.taskVariableService = taskVariableService; this.scriptService = scriptService; + this.publicScriptService = publicScriptService; this.accountService = accountService; - this.i18nService = i18nService; } @Override @@ -109,7 +109,7 @@ public InternalResponse getPlanBasicInfoById(Long appId, Lon throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } List planList = - taskPlanService.listPlanBasicInfoByIds(appId, Collections.singletonList(planId)); + taskPlanService.listPlanBasicInfoWithVariablesByIds(appId, Collections.singletonList(planId)); if (CollectionUtils.isNotEmpty(planList)) { TaskPlanInfoDTO plan = planList.get(0); ServiceTaskPlanDTO planDTO = new ServiceTaskPlanDTO(); @@ -206,6 +206,7 @@ public InternalResponse getPlanById(Long appId, Long planId, } @Override + @JobTransactional(transactionManager = "jobManageTransactionManager") public InternalResponse createPlanWithIdForMigration( String username, Long appId, @@ -221,7 +222,7 @@ public InternalResponse createPlanWithIdForMigration( @Override public InternalResponse checkIdAndName(Long appId, Long templateId, Long planId, - String name) { + String name) { boolean idResult = taskPlanService.checkPlanId(planId); boolean nameResult = taskPlanService.checkPlanName(appId, templateId, 0L, name); @@ -233,26 +234,23 @@ public InternalResponse checkIdAndName(Long appId, Long t @Override public InternalResponse savePlanForImport(String username, Long appId, Long templateId, - Long createTime, TaskPlanVO planInfo) { - if (planInfo.validateForImport()) { - TaskPlanInfoDTO taskPlanInfo = TaskPlanInfoDTO.fromVO(username, appId, planInfo); - if (createTime != null && createTime > 0) { - taskPlanInfo.setCreateTime(createTime); - } - Long finalTemplateId = taskPlanService.saveTaskPlanForBackup(taskPlanInfo); - return InternalResponse.buildSuccessResp(finalTemplateId); - } else { - throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); + Long createTime, TaskPlanVO planInfo) { + planInfo.validateForImport(); + TaskPlanInfoDTO taskPlanInfo = TaskPlanInfoDTO.fromVO(username, appId, planInfo); + if (createTime != null && createTime > 0) { + taskPlanInfo.setCreateTime(createTime); } + Long finalTemplateId = taskPlanService.saveTaskPlanForBackup(taskPlanInfo); + return InternalResponse.buildSuccessResp(finalTemplateId); } @Override public InternalResponse> getPlanVariable(String username, Long appId, Long templateId, - Long planId) { + Long planId) { List taskVariableList = taskVariableService.listVariablesByParentId(planId); if (CollectionUtils.isNotEmpty(taskVariableList)) { List variableList = - taskVariableList.parallelStream().map(TaskVariableDTO::toServiceDTO).collect(Collectors.toList()); + taskVariableList.stream().map(TaskVariableDTO::toServiceDTO).collect(Collectors.toList()); return InternalResponse.buildSuccessResp(variableList); } return InternalResponse.buildSuccessResp(Collections.emptyList()); @@ -261,7 +259,7 @@ public InternalResponse> getPlanVariable(String use @Override public InternalResponse> listPlans(String username, Long appId, Long templateId) { List taskPlanInfoDTOList = taskPlanService.listTaskPlansBasicInfo(appId, templateId); - List resultList = taskPlanInfoDTOList.parallelStream().map(it -> { + List resultList = taskPlanInfoDTOList.stream().map(it -> { ServiceTaskPlanDTO serviceTaskPlanDTO = new ServiceTaskPlanDTO(); serviceTaskPlanDTO.setId(it.getId()); serviceTaskPlanDTO.setName(it.getName()); @@ -273,7 +271,7 @@ public InternalResponse> listPlans(String username, Lon List serviceTaskStepDTOList = new ArrayList<>(); if (taskStepDTOList != null && !taskStepDTOList.isEmpty()) { serviceTaskStepDTOList = - taskStepDTOList.parallelStream() + taskStepDTOList.stream() .map(TaskStepConverter::convertToServiceTaskStepDTO).collect(Collectors.toList()); } serviceTaskPlanDTO.setStepList(serviceTaskStepDTOList); @@ -282,7 +280,7 @@ public InternalResponse> listPlans(String username, Lon List serviceTaskVariableDTOList = new ArrayList<>(); if (variableList != null && !variableList.isEmpty()) { serviceTaskVariableDTOList = - variableList.parallelStream() + variableList.stream() .map(TaskVariableConverter::convertToServiceTaskVariableDTO) .collect(Collectors.toList()); } @@ -349,7 +347,7 @@ private ServiceTaskScriptStepDTO buildScriptStep(TaskScriptStepDTO scriptStep, scriptStepDTO.setType(scriptStep.getLanguage().getValue()); if (scriptStep.getScriptVersionId() != null && scriptStep.getScriptVersionId() > 0) { - ScriptDTO script = scriptService.getScriptVersion(scriptStep.getScriptVersionId()); + ScriptDTO script = getScriptVersion(scriptStep.getScriptSource(), scriptStep.getScriptVersionId()); if (script == null) { log.warn("Plan related script is not exist, planId={}, scriptVersionId={}", scriptStep.getPlanId(), scriptStep.getScriptVersionId()); @@ -363,6 +361,7 @@ private ServiceTaskScriptStepDTO buildScriptStep(TaskScriptStepDTO scriptStep, } scriptStepDTO.setScriptParam(scriptStep.getScriptParam()); + scriptStepDTO.setWindowsInterpreter(scriptStep.getWindowsInterpreter()); scriptStepDTO.setScriptTimeout(scriptStep.getTimeout()); scriptStepDTO.setSecureParam(scriptStep.getSecureParam()); @@ -374,6 +373,19 @@ private ServiceTaskScriptStepDTO buildScriptStep(TaskScriptStepDTO scriptStep, return scriptStepDTO; } + private ScriptDTO getScriptVersion(TaskScriptSourceEnum scriptSource, long scriptVersionId) { + ScriptDTO scriptVersion = null; + switch (scriptSource) { + case CITING: + scriptVersion = scriptService.getScriptVersion(scriptVersionId); + break; + case PUBLIC: + scriptVersion = publicScriptService.getScriptVersion(scriptVersionId); + break; + } + return scriptVersion; + } + private ServiceAccountDTO buildAccount(Long accountId, Map cacheAccountMap) { ServiceAccountDTO cacheAccount = cacheAccountMap.get(accountId); if (cacheAccount != null) { @@ -466,9 +478,7 @@ private List buildVariables(TaskPlanInfoDTO plan) { return null; } List variableDTOS = new ArrayList<>(); - plan.getVariableList().forEach(variableDTO -> { - variableDTOS.add(buildVariable(variableDTO)); - }); + plan.getVariableList().forEach(variableDTO -> variableDTOS.add(buildVariable(variableDTO))); return variableDTOS; } @@ -483,7 +493,7 @@ private ServiceTaskVariableDTO buildVariable(TaskVariableDTO variable) { if (variable.getType() == TaskVariableTypeEnum.HOST_LIST && StringUtils.isNotBlank(variable.getDefaultValue())) { variableDTO - .setDefaultTargetValue(TaskTargetDTO.fromString(variable.getDefaultValue()).toServiceTaskTargetDTO()); + .setDefaultTargetValue(TaskTargetDTO.fromJsonString(variable.getDefaultValue()).toServiceTaskTargetDTO()); } return variableDTO; } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceTaskTemplateResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceTaskTemplateResourceImpl.java index 7537f99b5b..fa07729f4d 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceTaskTemplateResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceTaskTemplateResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,20 +26,25 @@ import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.constant.JobConstants; -import com.tencent.bk.job.common.exception.InvalidParamException; +import com.tencent.bk.job.common.constant.TaskVariableTypeEnum; import com.tencent.bk.job.common.exception.NotFoundException; import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; import com.tencent.bk.job.common.iam.model.AuthResult; -import com.tencent.bk.job.common.iam.service.WebAuthService; import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.InternalResponse; import com.tencent.bk.job.common.model.PageData; import com.tencent.bk.job.common.model.dto.AppResourceScope; +import com.tencent.bk.job.common.mysql.JobTransactional; import com.tencent.bk.job.common.util.JobContextUtil; +import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.manage.api.common.constants.task.TaskStepTypeEnum; import com.tencent.bk.job.manage.api.inner.ServiceTaskTemplateResource; import com.tencent.bk.job.manage.auth.TemplateAuthService; -import com.tencent.bk.job.manage.common.consts.JobResourceStatusEnum; +import com.tencent.bk.job.manage.migration.AddHostIdForTemplateAndPlanMigrationTask; import com.tencent.bk.job.manage.model.dto.TagDTO; +import com.tencent.bk.job.manage.model.dto.task.TaskFileInfoDTO; +import com.tencent.bk.job.manage.model.dto.task.TaskStepDTO; +import com.tencent.bk.job.manage.model.dto.task.TaskTargetDTO; import com.tencent.bk.job.manage.model.dto.task.TaskTemplateInfoDTO; import com.tencent.bk.job.manage.model.dto.task.TaskVariableDTO; import com.tencent.bk.job.manage.model.inner.ServiceIdNameCheckDTO; @@ -50,14 +55,18 @@ import com.tencent.bk.job.manage.service.AbstractTaskVariableService; import com.tencent.bk.job.manage.service.TagService; import com.tencent.bk.job.manage.service.template.TaskTemplateService; +import com.tencent.bk.job.manage.service.template.impl.TemplateScriptStatusUpdateService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.web.bind.annotation.RestController; +import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -68,22 +77,26 @@ public class ServiceTaskTemplateResourceImpl implements ServiceTaskTemplateResou private final TaskTemplateService templateService; private final AbstractTaskVariableService taskVariableService; - private final WebAuthService authService; private final TemplateAuthService templateAuthService; private final TagService tagService; + private final AddHostIdForTemplateAndPlanMigrationTask addHostIdService; + + private final TemplateScriptStatusUpdateService templateScriptStatusUpdateService; @Autowired public ServiceTaskTemplateResourceImpl( TaskTemplateService templateService, @Qualifier("TaskTemplateVariableServiceImpl") AbstractTaskVariableService taskVariableService, - WebAuthService authService, TemplateAuthService templateAuthService, - TagService tagService) { + TagService tagService, + AddHostIdForTemplateAndPlanMigrationTask addHostIdService, + TemplateScriptStatusUpdateService templateScriptStatusUpdateService) { this.templateService = templateService; this.taskVariableService = taskVariableService; - this.authService = authService; this.templateAuthService = templateAuthService; this.tagService = tagService; + this.addHostIdService = addHostIdService; + this.templateScriptStatusUpdateService = templateScriptStatusUpdateService; } @Override @@ -93,8 +106,8 @@ public InternalResponse sendScriptUpdateMessage( Long scriptVersionId, Integer status ) { - return InternalResponse.buildSuccessResp(templateService.updateScriptStatus(appId, scriptId, scriptVersionId, - JobResourceStatusEnum.getJobResourceStatus(status))); + templateScriptStatusUpdateService.refreshTemplateScriptStatusByScript(scriptId, scriptVersionId); + return InternalResponse.buildSuccessResp(null); } @Override @@ -123,6 +136,7 @@ public InternalResponse getTemplateNameById(Long templateId) { } @Override + @JobTransactional(transactionManager = "jobManageTransactionManager") public InternalResponse saveTemplateForMigration( String username, Long appId, @@ -156,19 +170,98 @@ public InternalResponse saveTemplateForMigration( log.warn("Skip create perm check for migration!"); } } - if (taskTemplateCreateUpdateReq.validate()) { - TaskTemplateInfoDTO templateInfo = TaskTemplateInfoDTO.fromReq(username, appId, - taskTemplateCreateUpdateReq); - bindExistTagsToImportedTemplate(appId, templateInfo); - templateInfo.setCreator(username); - Long finalTemplateId = templateService.saveTaskTemplateForMigration(templateInfo, createTime, - lastModifyTime, lastModifyUser); - templateAuthService.registerTemplate( - finalTemplateId, taskTemplateCreateUpdateReq.getName(), username); - return InternalResponse.buildSuccessResp(finalTemplateId); - } else { - throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); + taskTemplateCreateUpdateReq.validate(); + + TaskTemplateInfoDTO templateInfo = TaskTemplateInfoDTO.fromReq(username, appId, + taskTemplateCreateUpdateReq); + int count = addHostIdForHostsInStepAndVariables(templateInfo); + log.info("{} hostIds added for template {}:{}", count, templateInfo.getAppId(), templateInfo.getName()); + bindExistTagsToImportedTemplate(appId, templateInfo); + templateInfo.setCreator(username); + Long finalTemplateId = templateService.saveTaskTemplateForMigration(templateInfo, createTime, + lastModifyTime, lastModifyUser); + templateAuthService.registerTemplate( + finalTemplateId, taskTemplateCreateUpdateReq.getName(), username); + return InternalResponse.buildSuccessResp(finalTemplateId); + } + + private int addHostIdForHostsInStepAndVariables(TaskTemplateInfoDTO templateInfo) { + List targetList = new ArrayList<>(); + // 步骤：收集主机信息 + List stepList = templateInfo.getStepList(); + for (TaskStepDTO step : stepList) { + TaskStepTypeEnum type = step.getType(); + if (type == TaskStepTypeEnum.SCRIPT) { + TaskTargetDTO executeTarget = step.getScriptStepInfo().getExecuteTarget(); + targetList.add(executeTarget); + } else if (type == TaskStepTypeEnum.FILE) { + TaskTargetDTO destinationHostList = step.getFileStepInfo().getDestinationHostList(); + targetList.add(destinationHostList); + List originFileList = step.getFileStepInfo().getOriginFileList(); + for (TaskFileInfoDTO originFile : originFileList) { + TaskTargetDTO host = originFile.getHost(); + targetList.add(host); + } + } + } + // 变量：收集主机信息 + List variableList = templateInfo.getVariableList(); + if (variableList == null) { + variableList = new ArrayList<>(); + } + Map varTargetMap = new HashMap<>(); + for (TaskVariableDTO taskVariableDTO : variableList) { + if (taskVariableDTO.getType() == TaskVariableTypeEnum.HOST_LIST) { + TaskTargetDTO target = TaskTargetDTO.fromJsonString(taskVariableDTO.getDefaultValue()); + if (target != null) { + targetList.add(target); + varTargetMap.put(taskVariableDTO.getId(), target); + } + } + } + return addHostIdForTargets(stepList, variableList, targetList, varTargetMap); + } + + private int addHostIdForTargets(List stepList, + List variableList, + List targetList, + Map varTargetMap) { + int count = 0; + // 批量查询更新 + addHostIdService.addIpAndHostIdMappings(targetList); + for (TaskStepDTO step : stepList) { + TaskStepTypeEnum type = step.getType(); + if (type == TaskStepTypeEnum.SCRIPT) { + TaskTargetDTO executeTarget = step.getScriptStepInfo().getExecuteTarget(); + if (addHostIdService.fillHostId(executeTarget)) { + count += 1; + } + } else if (type == TaskStepTypeEnum.FILE) { + TaskTargetDTO destinationHostList = step.getFileStepInfo().getDestinationHostList(); + if (addHostIdService.fillHostId(destinationHostList)) { + count += 1; + } + List originFileList = step.getFileStepInfo().getOriginFileList(); + for (TaskFileInfoDTO originFile : originFileList) { + TaskTargetDTO host = originFile.getHost(); + if (addHostIdService.fillHostId(host)) { + count += 1; + } + } + } + } + for (TaskVariableDTO taskVariableDTO : variableList) { + if (taskVariableDTO.getType() == TaskVariableTypeEnum.HOST_LIST) { + TaskTargetDTO target = varTargetMap.get(taskVariableDTO.getId()); + if (target != null) { + if (addHostIdService.fillHostId(target)) { + taskVariableDTO.setDefaultValue(JsonUtils.toJson(target)); + count += 1; + } + } + } } + return count; } /* @@ -204,7 +297,7 @@ public InternalResponse> getTemplateVariable(String List taskVariableList = taskVariableService.listVariablesByParentId(templateId); if (CollectionUtils.isNotEmpty(taskVariableList)) { List variableList = - taskVariableList.parallelStream().map(TaskVariableDTO::toServiceDTO).collect(Collectors.toList()); + taskVariableList.stream().map(TaskVariableDTO::toServiceDTO).collect(Collectors.toList()); return InternalResponse.buildSuccessResp(variableList); } return InternalResponse.buildSuccessResp(Collections.emptyList()); @@ -228,7 +321,7 @@ public InternalResponse> listPageTaskTemplates( resultData.setTotal(templateListPage.getTotal()); resultData.setStart(templateListPage.getStart()); resultData.setPageSize(templateListPage.getPageSize()); - resultData.setData(templateListPage.getData().parallelStream().map(TaskTemplateInfoDTO::toServiceDTO) + resultData.setData(templateListPage.getData().stream().map(TaskTemplateInfoDTO::toServiceDTO) .collect(Collectors.toList())); return InternalResponse.buildSuccessResp(resultData); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceUserResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceUserResourceImpl.java index 525af02740..e2a05ad2e5 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceUserResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceUserResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceWhiteIPResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceWhiteIPResourceImpl.java index 9d1447a858..27794fba44 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceWhiteIPResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/inner/impl/ServiceWhiteIPResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,6 +26,7 @@ import com.tencent.bk.job.common.model.InternalResponse; import com.tencent.bk.job.manage.api.inner.ServiceWhiteIPResource; +import com.tencent.bk.job.manage.model.dto.whiteip.WhiteIPRecordDTO; import com.tencent.bk.job.manage.model.inner.ServiceWhiteIPInfo; import com.tencent.bk.job.manage.model.web.request.whiteip.WhiteIPRecordCreateUpdateReq; import com.tencent.bk.job.manage.service.WhiteIPService; @@ -46,8 +47,8 @@ public ServiceWhiteIPResourceImpl(WhiteIPService whiteIPService) { @Override - public InternalResponse> getWhiteIPActionScopes(Long appId, String ip, Long cloudAreaId) { - return InternalResponse.buildSuccessResp(whiteIPService.getWhiteIPActionScopes(appId, ip, cloudAreaId)); + public InternalResponse> getWhiteIPActionScopes(Long appId, String ip, Long cloudAreaId, Long hostId) { + return InternalResponse.buildSuccessResp(whiteIPService.getWhiteIPActionScopes(appId, ip, cloudAreaId, hostId)); } @Override @@ -57,6 +58,12 @@ public InternalResponse> listWhiteIPInfos() { @Override public InternalResponse saveWhiteIP(String username, WhiteIPRecordCreateUpdateReq createUpdateReq) { - return InternalResponse.buildSuccessResp(whiteIPService.saveWhiteIP(username, createUpdateReq)); + WhiteIPRecordDTO record; + if (createUpdateReq.getId() != null && createUpdateReq.getId() > 0) { + record = whiteIPService.updateWhiteIP(username, createUpdateReq); + } else { + record = whiteIPService.createWhiteIP(username, createUpdateReq); + } + return InternalResponse.buildSuccessResp(record.getId()); } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/migration/impl/MigrationResource.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/migration/impl/MigrationResource.java index be82dfae09..a0fc3d0aa6 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/migration/impl/MigrationResource.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/migration/impl/MigrationResource.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,40 +24,70 @@ package com.tencent.bk.job.manage.api.migration.impl; +import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.model.Response; +import com.tencent.bk.job.common.model.dto.ResourceScope; +import com.tencent.bk.job.common.service.AppScopeMappingService; +import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.manage.migration.AddHostIdForTemplateAndPlanMigrationTask; +import com.tencent.bk.job.manage.migration.AddHostIdForWhiteIpMigrationTask; import com.tencent.bk.job.manage.migration.EncryptDbAccountPasswordMigrationTask; import com.tencent.bk.job.manage.migration.ResourceTagsMigrationTask; +import com.tencent.bk.job.manage.migration.UpdateAppIdForWhiteIpMigrationTask; import com.tencent.bk.job.manage.model.dto.ResourceTagDTO; +import com.tencent.bk.job.manage.model.migration.AddHostIdMigrationReq; +import com.tencent.bk.job.manage.model.migration.BkPlatformInfo; +import com.tencent.bk.job.manage.model.migration.MigrationRecordsResult; import com.tencent.bk.job.manage.model.migration.SetBizSetMigrationStatusReq; +import com.tencent.bk.job.manage.model.migration.UpdateAppIdForWhiteIpMigrationReq; +import com.tencent.bk.job.manage.service.globalsetting.BkPlatformInfoService; import com.tencent.bk.job.manage.service.impl.BizSetService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; +import java.util.ArrayList; import java.util.List; +import java.util.Map; /** * 微服务升级 */ -@RequestMapping("/migration") +@RequestMapping("/manage/migration") @Slf4j -@RestController +@RestController("jobManageMigrationResource") public class MigrationResource { private final EncryptDbAccountPasswordMigrationTask encryptDbAccountPasswordMigrationTask; private final ResourceTagsMigrationTask resourceTagsMigrationTask; private final BizSetService bizSetService; + private final AddHostIdForTemplateAndPlanMigrationTask addHostIdForTemplateAndPlanMigrationTask; + private final UpdateAppIdForWhiteIpMigrationTask updateAppIdForWhiteIpMigrationTask; + private final AddHostIdForWhiteIpMigrationTask addHostIdForWhiteIpMigrationTask; + private final AppScopeMappingService appScopeMappingService; + private final BkPlatformInfoService bkPlatformInfoService; @Autowired public MigrationResource( EncryptDbAccountPasswordMigrationTask encryptDbAccountPasswordMigrationTask, ResourceTagsMigrationTask resourceTagsMigrationTask, - BizSetService bizSetService) { + BizSetService bizSetService, + AddHostIdForTemplateAndPlanMigrationTask addHostIdForTemplateAndPlanMigrationTask, + UpdateAppIdForWhiteIpMigrationTask updateAppIdForWhiteIpMigrationTask, + AddHostIdForWhiteIpMigrationTask addHostIdForWhiteIpMigrationTask, + AppScopeMappingService appScopeMappingService, + BkPlatformInfoService bkPlatformInfoService) { this.encryptDbAccountPasswordMigrationTask = encryptDbAccountPasswordMigrationTask; this.resourceTagsMigrationTask = resourceTagsMigrationTask; this.bizSetService = bizSetService; + this.addHostIdForTemplateAndPlanMigrationTask = addHostIdForTemplateAndPlanMigrationTask; + this.updateAppIdForWhiteIpMigrationTask = updateAppIdForWhiteIpMigrationTask; + this.addHostIdForWhiteIpMigrationTask = addHostIdForWhiteIpMigrationTask; + this.appScopeMappingService = appScopeMappingService; + this.bkPlatformInfoService = bkPlatformInfoService; } /** @@ -83,4 +113,56 @@ public Response> upgradeResourceTags() { public Response setBizSetMigrationStatus(@RequestBody SetBizSetMigrationStatusReq req) { return Response.buildSuccessResp(bizSetService.setBizSetMigratedToCMDB(req.getMigrated())); } + + /** + * IP白名单AppId更新，全业务ID->代表所有业务的ID + */ + @PostMapping("/action/updateAppIdForWhiteIpMigrationTask") + public Response updateAppIdForWhiteIpMigrationTask(@RequestBody UpdateAppIdForWhiteIpMigrationReq req) { + List results = new ArrayList<>(); + results.add(updateAppIdForWhiteIpMigrationTask.execute(req.isDryRun())); + boolean success = results.stream().allMatch(MigrationRecordsResult::isSuccess); + return success ? Response.buildSuccessResp(JsonUtils.toJson(results)) : + Response.buildCommonFailResp(ErrorCode.MIGRATION_FAIL, new String[]{"UpdateAppIdForWhiteIpMigrationTask", + JsonUtils.toJson(results)}); + } + + /** + * IP白名单、作业模板、执行方案等包含的主机数据，在原来的云区域+ip的基础上，填充hostID属性 + */ + @PostMapping("/action/addHostIdMigrationTask") + public Response addHostIdMigrationTask(@RequestBody AddHostIdMigrationReq req) { + List results = new ArrayList<>(); + List scopeList = req.getScopeList(); + List appIdList = null; + if (scopeList != null) { + Map scopeAppIdMap = appScopeMappingService.getAppIdByScopeList(scopeList); + if (scopeAppIdMap.size() == scopeList.size()) { + appIdList = new ArrayList<>(scopeAppIdMap.values()); + } else { + scopeList.removeIf(scopeAppIdMap::containsKey); + return Response.buildCommonFailResp(ErrorCode.MIGRATION_FAIL, new String[]{ + "AddHostIdMigrationTask", + "Cannot find appId by scope:" + scopeList + } + ); + } + } + if (appIdList == null) { + results.add(addHostIdForWhiteIpMigrationTask.execute(req.isDryRun())); + } + results.addAll(addHostIdForTemplateAndPlanMigrationTask.execute(appIdList, req.isDryRun())); + boolean success = results.stream().allMatch(MigrationRecordsResult::isSuccess); + return success ? Response.buildSuccessResp(JsonUtils.toJson(results)) : + Response.buildCommonFailResp(ErrorCode.MIGRATION_FAIL, new String[]{"AddHostIdMigrationTask", + JsonUtils.toJson(results)}); + } + + /** + * 获取蓝鲸统一规范的平台配置信息 + */ + @GetMapping("/action/getBkPlatformInfo") + public Response getBkPlatformInfo() { + return Response.buildSuccessResp(bkPlatformInfoService.getCurrentBkPlatformInfo()); + } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/op/impl/EventReplayOpResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/op/impl/EventReplayOpResourceImpl.java new file mode 100644 index 0000000000..5718723a40 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/op/impl/EventReplayOpResourceImpl.java @@ -0,0 +1,53 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.api.op.impl; + +import com.tencent.bk.job.common.cc.model.result.HostEventDetail; +import com.tencent.bk.job.common.cc.model.result.ResourceEvent; +import com.tencent.bk.job.common.model.Response; +import com.tencent.bk.job.manage.api.op.EventReplayOpResource; +import com.tencent.bk.job.manage.service.impl.sync.HostEventWatcher; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RestController; + +@Slf4j +@RestController +public class EventReplayOpResourceImpl implements EventReplayOpResource { + + private final HostEventWatcher hostEventWatcher; + + @Autowired + public EventReplayOpResourceImpl(HostEventWatcher hostEventWatcher) { + this.hostEventWatcher = hostEventWatcher; + } + + @Override + public Response replayHostEvent(String username, ResourceEvent event) { + hostEventWatcher.handleEvent(event); + return Response.buildSuccessResp(null); + } + +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/op/impl/FlowControlOpResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/op/impl/FlowControlOpResourceImpl.java index 383100f862..2036e41de2 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/op/impl/FlowControlOpResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/op/impl/FlowControlOpResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -29,6 +29,7 @@ import com.tencent.bk.job.manage.api.op.FlowControlOpResource; import com.tencent.bk.job.manage.model.op.req.ConfigFlowControlReq; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.ObjectProvider; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RestController; @@ -41,8 +42,8 @@ public class FlowControlOpResourceImpl implements FlowControlOpResource { private final FlowController globalFlowController; @Autowired - public FlowControlOpResourceImpl(FlowController flowController) { - globalFlowController = flowController; + public FlowControlOpResourceImpl(ObjectProvider flowControllerObjectProvider) { + globalFlowController = flowControllerObjectProvider.getIfAvailable(); } protected void logInput(Object... args) { diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/tmp/impl/TmpAppAccountResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/tmp/impl/TmpAppAccountResourceImpl.java deleted file mode 100644 index b77574483b..0000000000 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/tmp/impl/TmpAppAccountResourceImpl.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.api.tmp.impl; - -import com.tencent.bk.job.common.constant.ErrorCode; -import com.tencent.bk.job.common.exception.AlreadyExistsException; -import com.tencent.bk.job.common.exception.InvalidParamException; -import com.tencent.bk.job.common.i18n.service.MessageI18nService; -import com.tencent.bk.job.common.model.Response; -import com.tencent.bk.job.common.util.Utils; -import com.tencent.bk.job.common.util.json.JsonUtils; -import com.tencent.bk.job.manage.api.tmp.TmpAppAccountResource; -import com.tencent.bk.job.manage.common.consts.account.AccountCategoryEnum; -import com.tencent.bk.job.manage.common.consts.account.AccountTypeEnum; -import com.tencent.bk.job.manage.model.dto.AccountDTO; -import com.tencent.bk.job.manage.model.tmp.TmpAccountCreateUpdateReq; -import com.tencent.bk.job.manage.service.AccountService; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.joda.time.DateTimeUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.RestController; - -@Slf4j -@RestController -public class TmpAppAccountResourceImpl implements TmpAppAccountResource { - private AccountService accountService; - private MessageI18nService i18nService; - - @Autowired - public TmpAppAccountResourceImpl(AccountService accountService, MessageI18nService i18nService) { - this.accountService = accountService; - this.i18nService = i18nService; - } - - - public AccountDTO buildCreateAccountDTO(String operator, long appId, Long createTime, Long lastModifyTime, - String lastModifyUser, TmpAccountCreateUpdateReq req) { - AccountDTO accountDTO = new AccountDTO(); - accountDTO.setId(req.getId()); - accountDTO.setAppId(appId); - accountDTO.setAccount(req.getAccount()); - accountDTO.setCategory(AccountCategoryEnum.valOf(req.getCategory())); - accountDTO.setType(AccountTypeEnum.valueOf(req.getType())); - if (StringUtils.isBlank(req.getAlias())) { - accountDTO.setAlias(req.getAccount()); - } else { - accountDTO.setAlias(req.getAlias()); - } - accountDTO.setRemark(req.getRemark()); - accountDTO.setGrantees(Utils.concatStringWithSeperator(req.getGrantees(), ",")); - - if (AccountCategoryEnum.SYSTEM.getValue().equals(req.getCategory())) { - if (AccountTypeEnum.WINDOW.getType().equals(req.getType())) { - accountDTO.setOs("Windows"); - } else { - accountDTO.setOs("Linux"); - } - accountDTO.setPassword(req.getPassword()); - } - - if (AccountCategoryEnum.DB.getValue().equals(req.getCategory())) { - accountDTO.setDbPort(req.getDbPort()); - accountDTO.setDbPassword(req.getDbPassword()); - accountDTO.setDbSystemAccountId(req.getDbSystemAccountId()); - } - - accountDTO.setCreator(operator); - accountDTO.setCreateTime(createTime); - accountDTO.setLastModifyUser(lastModifyUser); - accountDTO.setLastModifyTime(lastModifyTime); - - return accountDTO; - } - - @Override - public Response saveAccount(String username, Long appId, Long createTime, Long lastModifyTime, - String lastModifyUser, Boolean useCurrentTime, - TmpAccountCreateUpdateReq accountCreateUpdateReq) { - if (useCurrentTime != null && useCurrentTime) { - if (createTime == null) { - createTime = DateTimeUtils.currentTimeMillis(); - } - if (lastModifyTime == null) { - lastModifyTime = DateTimeUtils.currentTimeMillis(); - } - } - if (!accountService.checkCreateParam(accountCreateUpdateReq, false, false)) { - throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); - } - Long accountId = accountCreateUpdateReq.getId(); - if (accountId != null) { - AccountDTO accountDTO = accountService.getAccountById(accountId); - if (accountDTO != null) { - if (accountDTO.getAppId().longValue() == appId - && accountDTO.getAccount().equals(accountCreateUpdateReq.getAccount()) - && accountDTO.getAlias().equals(accountCreateUpdateReq.getAlias()) - ) { - log.warn("same account exist, account={}, skip", JsonUtils.toJson(accountDTO)); - return Response.buildSuccessResp(accountId); - } else { - // 报错，相同ID账号已存在 - throw new AlreadyExistsException(ErrorCode.ACCOUNT_ALIAS_EXIST); - } - } - } - - if ((accountId == null || accountId == 0) - && StringUtils.isNotBlank(accountCreateUpdateReq.getAlias())) { - AccountDTO account; - switch (accountCreateUpdateReq.getCategory()) { - case 1: - account = accountService.getAccount(appId, AccountCategoryEnum.SYSTEM, - accountCreateUpdateReq.getAlias()); - break; - case 2: - account = accountService.getAccount(appId, AccountCategoryEnum.DB, - accountCreateUpdateReq.getAlias()); - break; - default: - throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); - } - if (account != null) { - if (account.getAccount().equals(accountCreateUpdateReq.getAccount())) { - return Response.buildSuccessResp(account.getId()); - } - } - } - - AccountDTO newAccount = buildCreateAccountDTO(username, appId, createTime, lastModifyTime, lastModifyUser, - accountCreateUpdateReq); - accountId = accountService.saveAccount(newAccount); - return Response.buildSuccessResp(accountId); - } -} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/tmp/impl/TmpMigrationResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/tmp/impl/TmpMigrationResourceImpl.java deleted file mode 100644 index 8a61fa7515..0000000000 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/tmp/impl/TmpMigrationResourceImpl.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.api.tmp.impl; - -import com.tencent.bk.job.common.model.BaseSearchCondition; -import com.tencent.bk.job.common.model.PageData; -import com.tencent.bk.job.common.model.Response; -import com.tencent.bk.job.manage.api.tmp.TmpMigrationResource; -import com.tencent.bk.job.manage.dao.plan.TaskPlanDAO; -import com.tencent.bk.job.manage.model.dto.TaskPlanQueryDTO; -import com.tencent.bk.job.manage.model.dto.task.TaskPlanInfoDTO; -import com.tencent.bk.job.manage.model.dto.task.TaskTemplateInfoDTO; -import com.tencent.bk.job.manage.model.query.TaskTemplateQuery; -import com.tencent.bk.job.manage.model.tmp.MigrationPlanBasic; -import com.tencent.bk.job.manage.service.template.TaskTemplateService; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.web.bind.annotation.RestController; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -@Slf4j -@RestController -public class TmpMigrationResourceImpl implements TmpMigrationResource { - private final TaskTemplateService taskTemplateService; - private final TaskPlanDAO taskPlanDAO; - - @Autowired - public TmpMigrationResourceImpl(TaskTemplateService taskTemplateService, - TaskPlanDAO taskPlanDAO) { - this.taskTemplateService = taskTemplateService; - this.taskPlanDAO = taskPlanDAO; - } - - @Override - public Response> listAppPlanBasicInfo(String username, Long appId) { - BaseSearchCondition baseSearchCondition = new BaseSearchCondition(); - // Get all - baseSearchCondition.setLength(Integer.MAX_VALUE); - - TaskTemplateQuery query = TaskTemplateQuery.builder().appId(appId).baseSearchCondition(baseSearchCondition) - .build(); - PageData templatePageData = taskTemplateService.listPageTaskTemplates(query); - if (templatePageData == null || CollectionUtils.isEmpty(templatePageData.getData())) { - log.info("Template is empty, appId: {}", appId); - return Response.buildSuccessResp(Collections.emptyList()); - } - Map templateIdNameMap = new HashMap<>(); - templatePageData.getData().forEach(template -> templateIdNameMap.put(template.getId(), template.getName())); - - TaskPlanQueryDTO taskPlanQuery = new TaskPlanQueryDTO(); - taskPlanQuery.setAppId(appId); - PageData pagePlanData = taskPlanDAO.listPageTaskPlans(taskPlanQuery, - baseSearchCondition, Collections.emptyList()); - if (pagePlanData == null || CollectionUtils.isEmpty(pagePlanData.getData())) { - log.info("Plan is empty, appId: {}", appId); - return Response.buildSuccessResp(Collections.emptyList()); - } - List planList = new ArrayList<>(pagePlanData.getData().size()); - pagePlanData.getData().forEach(plan -> { - MigrationPlanBasic planBasic = new MigrationPlanBasic(); - planBasic.setPlanId(plan.getId()); - planBasic.setPlanName(plan.getName()); - planBasic.setTemplateId(plan.getTemplateId()); - planBasic.setTemplateName(templateIdNameMap.get(plan.getTemplateId())); - planList.add(planBasic); - }); - - return Response.buildSuccessResp(planList); - } -} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/BaseWebScriptResource.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/BaseWebScriptResource.java new file mode 100644 index 0000000000..778b68e2e1 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/BaseWebScriptResource.java @@ -0,0 +1,359 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.api.web.impl; + +import com.tencent.bk.job.common.constant.JobResourceTypeEnum; +import com.tencent.bk.job.common.i18n.service.MessageI18nService; +import com.tencent.bk.job.common.model.BaseSearchCondition; +import com.tencent.bk.job.common.model.PageData; +import com.tencent.bk.job.common.model.dto.AppResourceScope; +import com.tencent.bk.job.manage.api.common.ScriptDTOBuilder; +import com.tencent.bk.job.manage.api.common.constants.JobResourceStatusEnum; +import com.tencent.bk.job.manage.model.dto.ResourceTagDTO; +import com.tencent.bk.job.manage.model.dto.ScriptDTO; +import com.tencent.bk.job.manage.model.dto.ScriptSyncTemplateStepDTO; +import com.tencent.bk.job.manage.model.dto.SyncScriptResultDTO; +import com.tencent.bk.job.manage.model.dto.TagDTO; +import com.tencent.bk.job.manage.model.dto.converter.ScriptConverter; +import com.tencent.bk.job.manage.model.dto.converter.ScriptRelatedTemplateStepConverter; +import com.tencent.bk.job.manage.model.dto.script.ScriptCitedTaskPlanDTO; +import com.tencent.bk.job.manage.model.dto.script.ScriptCitedTaskTemplateDTO; +import com.tencent.bk.job.manage.model.query.ScriptQuery; +import com.tencent.bk.job.manage.model.web.request.ScriptInfoUpdateReq; +import com.tencent.bk.job.manage.model.web.vo.BasicScriptVO; +import com.tencent.bk.job.manage.model.web.vo.ScriptCitedTaskPlanVO; +import com.tencent.bk.job.manage.model.web.vo.ScriptCitedTemplateVO; +import com.tencent.bk.job.manage.model.web.vo.ScriptVO; +import com.tencent.bk.job.manage.model.web.vo.TagVO; +import com.tencent.bk.job.manage.model.web.vo.script.ScriptCiteCountVO; +import com.tencent.bk.job.manage.model.web.vo.script.ScriptCiteInfoVO; +import com.tencent.bk.job.manage.model.web.vo.script.ScriptRelatedTemplateStepVO; +import com.tencent.bk.job.manage.model.web.vo.script.ScriptSyncResultVO; +import com.tencent.bk.job.manage.service.ScriptManager; +import com.tencent.bk.job.manage.service.TagService; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.Collectors; + +import static com.tencent.bk.job.common.constant.JobConstants.PUBLIC_APP_ID; + +/** + * 脚本Resource通用实现 + */ +public class BaseWebScriptResource { + + protected final MessageI18nService i18nService; + + protected final ScriptDTOBuilder scriptDTOBuilder; + + protected final TagService tagService; + + private final ScriptManager scriptManager; + + public BaseWebScriptResource(MessageI18nService i18nService, + ScriptDTOBuilder scriptDTOBuilder, + TagService tagService, + ScriptManager scriptManager) { + this.i18nService = i18nService; + this.scriptDTOBuilder = scriptDTOBuilder; + this.tagService = tagService; + this.scriptManager = scriptManager; + } + + public ScriptQuery buildListPageScriptQuery(AppResourceScope appResourceScope, + String name, + Integer type, + String tags, + Long panelTag, + Integer panelType, + String creator, + String lastModifyUser, + String scriptId, + String content, + Integer start, + Integer pageSize, + String orderField, + Integer order) { + ScriptQuery scriptQuery = new ScriptQuery(); + boolean queryPublic = appResourceScope == null || appResourceScope.getAppId() == null + || appResourceScope.getAppId().equals(PUBLIC_APP_ID); + scriptQuery.setPublicScript(queryPublic); + scriptQuery.setAppId(queryPublic ? PUBLIC_APP_ID : appResourceScope.getAppId()); + scriptQuery.setId(scriptId); + scriptQuery.setName(name); + scriptQuery.setType(type); + if (panelType != null && panelType == 2) { + scriptQuery.setUntaggedScript(true); + } else { + addTagCondition(scriptQuery, tags, panelTag); + } + scriptQuery.setContentKeyword(content); + + BaseSearchCondition baseSearchCondition = new BaseSearchCondition(); + baseSearchCondition.setLength(pageSize); + baseSearchCondition.setStart(start); + baseSearchCondition.setOrderField(orderField); + baseSearchCondition.setOrder(order); + baseSearchCondition.setCreator(creator); + baseSearchCondition.setLastModifyUser(lastModifyUser); + + scriptQuery.setBaseSearchCondition(baseSearchCondition); + + return scriptQuery; + } + + private void addTagCondition(ScriptQuery query, String tags, Long panelTagId) { + if (StringUtils.isNotBlank(tags)) { + query.setTagIds(Arrays.stream(tags.split(",")).map(tagIdStr -> { + try { + return Long.parseLong(tagIdStr); + } catch (NumberFormatException e) { + return null; + } + }).filter(Objects::nonNull).collect(Collectors.toList())); + } + if (panelTagId != null && panelTagId > 0) { + // Frontend need additional param to tell where the tag from + TagDTO tagInfo = new TagDTO(); + tagInfo.setId(panelTagId); + if (CollectionUtils.isEmpty(query.getTagIds())) { + query.setTagIds(Collections.singletonList(panelTagId)); + } else { + query.getTagIds().add(panelTagId); + } + } + } + + protected PageData pageVOs(PageData pageData, + Integer start, + Integer pageSize) { + List resultScripts = new ArrayList<>(); + if (pageData == null) { + PageData resultPageData = new PageData<>(); + resultPageData.setStart(start); + resultPageData.setPageSize(pageSize); + resultPageData.setTotal(0L); + resultPageData.setData(resultScripts); + return resultPageData; + } + + for (ScriptDTO scriptDTO : pageData.getData()) { + ScriptVO scriptVO = ScriptConverter.convertToScriptVO(scriptDTO); + resultScripts.add(scriptVO); + } + + PageData resultPageData = new PageData<>(); + resultPageData.setStart(pageData.getStart()); + resultPageData.setPageSize(pageData.getPageSize()); + resultPageData.setTotal(pageData.getTotal()); + resultPageData.setData(resultScripts); + + return resultPageData; + } + + protected void setScriptCiteCount(List scriptVOS) { + for (ScriptVO scriptVO : scriptVOS) { + String scriptId = scriptVO.getId(); + Long scriptVersionId = scriptVO.getScriptVersionId(); + Integer taskTemplateCiteCount = scriptManager.getScriptTemplateCiteCount( + scriptId, scriptVersionId); + scriptVO.setRelatedTaskTemplateNum(taskTemplateCiteCount); + Integer taskPlanCiteCount = scriptManager.getScriptTaskPlanCiteCount( + scriptId, scriptVersionId); + scriptVO.setRelatedTaskPlanNum(taskPlanCiteCount); + } + } + + protected void setOnlineScriptVersionInfo(List scripts) { + if (scripts != null && !scripts.isEmpty()) { + List scriptIdList = new ArrayList<>(); + for (ScriptVO script : scripts) { + scriptIdList.add(script.getId()); + } + Map onlineScriptMap = scriptManager.batchGetOnlineScriptVersionByScriptIds(scriptIdList); + + for (ScriptVO scriptVO : scripts) { + ScriptDTO onlineScriptVersion = onlineScriptMap.get(scriptVO.getId()); + if (onlineScriptVersion != null) { + scriptVO.setScriptVersionId(onlineScriptVersion.getScriptVersionId()); + scriptVO.setVersion(onlineScriptVersion.getVersion()); + } + } + } + } + + protected List extractTags(ScriptInfoUpdateReq scriptInfoUpdateReq) { + List tags = new ArrayList<>(); + if (scriptInfoUpdateReq.getScriptTags() != null && !scriptInfoUpdateReq.getScriptTags().isEmpty()) { + for (TagVO tagVO : scriptInfoUpdateReq.getScriptTags()) { + TagDTO tagDTO = new TagDTO(); + tagDTO.setId(tagVO.getId()); + tagDTO.setName(tagVO.getName()); + tags.add(tagDTO); + } + } + return tags; + } + + protected ScriptCiteCountVO getScriptCiteCountOfAllScript(String scriptId, Long scriptVersionId) { + Integer templateCiteCount = scriptManager.getScriptTemplateCiteCount(scriptId, scriptVersionId); + Integer taskPlanCiteCount = scriptManager.getScriptTaskPlanCiteCount(scriptId, scriptVersionId); + return new ScriptCiteCountVO(templateCiteCount, taskPlanCiteCount); + } + + protected ScriptCiteInfoVO getScriptCiteInfoOfAllScript(String scriptId, Long scriptVersionId) { + List citedTemplateList = scriptManager.getScriptCitedTemplates( + scriptId, scriptVersionId); + if (citedTemplateList == null) { + citedTemplateList = Collections.emptyList(); + } + List citedTemplateVOList = + citedTemplateList.parallelStream().map(ScriptCitedTaskTemplateDTO::toVO).collect(Collectors.toList()); + List citedTaskPlanList = scriptManager.getScriptCitedTaskPlans( + scriptId, scriptVersionId); + if (citedTaskPlanList == null) { + citedTaskPlanList = Collections.emptyList(); + } + List citedTaskPlanVOList = + citedTaskPlanList.parallelStream().map(ScriptCitedTaskPlanDTO::toVO).collect(Collectors.toList()); + return new ScriptCiteInfoVO(citedTemplateVOList, citedTaskPlanVOList); + } + + protected List convertToScriptVOList(List scriptList) { + if (CollectionUtils.isEmpty(scriptList)) { + return Collections.emptyList(); + } + + return scriptList.stream().map(ScriptConverter::convertToScriptVO) + .collect(Collectors.toList()); + } + + protected List convertToBasicScriptVOList(List scriptList) { + if (CollectionUtils.isEmpty(scriptList)) { + return Collections.emptyList(); + } + + return scriptList.stream().map(ScriptConverter::convertToBasicScriptVO) + .collect(Collectors.toList()); + } + + protected void batchPatchResourceTags(JobResourceTypeEnum resourceType, + List scriptIdList, + List addTagIdList, + List deleteTagIdList) { + List addResourceTags = null; + List deleteResourceTags = null; + if (CollectionUtils.isNotEmpty(addTagIdList)) { + addResourceTags = tagService.buildResourceTags(resourceType.getValue(), + scriptIdList.stream().map(String::valueOf).collect(Collectors.toList()), + addTagIdList); + } + if (CollectionUtils.isNotEmpty(deleteTagIdList)) { + deleteResourceTags = tagService.buildResourceTags(resourceType.getValue(), + scriptIdList.stream().map(String::valueOf).collect(Collectors.toList()), + deleteTagIdList); + } + tagService.batchPatchResourceTags(addResourceTags, deleteResourceTags); + } + + protected List listScriptSyncTemplateSteps(Long appId, + String scriptId, + Long scriptVersionId) { + List steps = getSyncTemplateSteps(appId, scriptId, scriptVersionId); + if (CollectionUtils.isEmpty(steps)) { + return Collections.emptyList(); + } + + List stepVOS = + steps.stream() + .map(ScriptRelatedTemplateStepConverter::convertToScriptRelatedTemplateStepVO) + .collect(Collectors.toList()); + stepVOS.forEach(stepVO -> { + JobResourceStatusEnum scriptStatus = JobResourceStatusEnum.getJobResourceStatus(stepVO.getScriptStatus()); + if (scriptStatus != null) { + stepVO.setScriptStatusDesc(i18nService.getI18n(scriptStatus.getStatusI18nKey())); + } + stepVO.setCanEdit(true); + }); + return stepVOS; + } + + protected List getSyncTemplateSteps(Long appId, + String scriptId, + Long scriptVersionId) { + List steps = scriptManager.listScriptSyncTemplateSteps(appId, scriptId); + if (CollectionUtils.isEmpty(steps)) { + return Collections.emptyList(); + } + // 过滤掉已经是最新的模板步骤 + steps = + steps.stream().filter(step -> + !scriptVersionId.equals(step.getScriptVersionId())) + .collect(Collectors.toList()); + return steps; + } + + protected List convertToSyncResultVOs(List syncResults, + AppResourceScope appResourceScope) { + List syncResultVOS = new ArrayList<>(syncResults.size()); + for (SyncScriptResultDTO syncResult : syncResults) { + ScriptSyncResultVO syncVO = new ScriptSyncResultVO(); + ScriptSyncTemplateStepDTO syncStep = syncResult.getTemplateStep(); + if (syncStep.getAppId() != null && !syncStep.getAppId().equals(PUBLIC_APP_ID)) { + syncVO.setScopeType(appResourceScope.getType().getValue()); + syncVO.setScopeId(appResourceScope.getId()); + } + syncVO.setScriptId(syncStep.getScriptId()); + syncVO.setScriptVersionId(syncStep.getScriptVersionId()); + syncVO.setScriptVersion(syncStep.getScriptVersion()); + syncVO.setScriptName(syncStep.getScriptName()); + syncVO.setScriptStatus(syncStep.getScriptStatus()); + JobResourceStatusEnum scriptStatus = JobResourceStatusEnum.getJobResourceStatus(syncStep.getScriptStatus()); + if (scriptStatus != null) { + syncVO.setScriptStatusDesc(i18nService.getI18n(scriptStatus.getStatusI18nKey())); + } + syncVO.setStepId(syncStep.getStepId()); + syncVO.setTemplateId(syncStep.getTemplateId()); + syncVO.setStepName(syncStep.getStepName()); + syncVO.setTemplateName(syncStep.getTemplateName()); + if (syncResult.isSuccess()) { + syncVO.setSyncStatus(ScriptSyncResultVO.SYNC_SUCCESS); + } else { + syncVO.setSyncStatus(ScriptSyncResultVO.SYNC_FAIL); + syncVO.setFailMsg(i18nService.getI18n(String.valueOf(syncResult.getErrorCode()))); + } + syncResultVOS.add(syncVO); + } + + return syncResultVOS; + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebAppAccountResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebAppAccountResourceImpl.java index df33ccff85..0ba133b5b6 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebAppAccountResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebAppAccountResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,28 +24,23 @@ package com.tencent.bk.job.manage.api.web.impl; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.audit.annotations.AuditRequestBody; +import com.tencent.bk.job.common.constant.AccountCategoryEnum; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.constant.FeatureToggleModeEnum; -import com.tencent.bk.job.common.exception.FailedPreconditionException; import com.tencent.bk.job.common.exception.InvalidParamException; -import com.tencent.bk.job.common.exception.NotFoundException; -import com.tencent.bk.job.common.i18n.service.MessageI18nService; -import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; +import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.iam.model.AuthResult; import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; import com.tencent.bk.job.common.model.Response; import com.tencent.bk.job.common.model.dto.AppResourceScope; -import com.tencent.bk.job.common.model.dto.ResourceScope; -import com.tencent.bk.job.common.service.AppScopeMappingService; -import com.tencent.bk.job.common.util.ApplicationContextRegister; -import com.tencent.bk.job.common.util.ArrayUtil; import com.tencent.bk.job.common.util.Utils; import com.tencent.bk.job.common.util.date.DateUtils; +import com.tencent.bk.job.manage.api.common.constants.account.AccountTypeEnum; import com.tencent.bk.job.manage.api.web.WebAppAccountResource; import com.tencent.bk.job.manage.auth.AccountAuthService; -import com.tencent.bk.job.manage.common.consts.account.AccountCategoryEnum; -import com.tencent.bk.job.manage.common.consts.account.AccountTypeEnum; import com.tencent.bk.job.manage.config.JobManageConfig; import com.tencent.bk.job.manage.model.dto.AccountDTO; import com.tencent.bk.job.manage.model.web.request.AccountCreateUpdateReq; @@ -68,69 +63,55 @@ @RestController public class WebAppAccountResourceImpl implements WebAppAccountResource { private final AccountService accountService; - private final MessageI18nService i18nService; private final AccountAuthService accountAuthService; private final JobManageConfig jobManageConfig; @Autowired public WebAppAccountResourceImpl(AccountService accountService, - MessageI18nService i18nService, AccountAuthService accountAuthService, JobManageConfig jobManageConfig) { this.accountService = accountService; - this.i18nService = i18nService; this.accountAuthService = accountAuthService; this.jobManageConfig = jobManageConfig; } @Override - public Response saveAccount(String username, - AppResourceScope appResourceScope, - String scopeType, - String scopeId, - AccountCreateUpdateReq accountCreateUpdateReq) { - AuthResult authResult = checkCreateAccountPermission(username, appResourceScope); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } + @AuditEntry(actionId = ActionId.CREATE_ACCOUNT) + public Response saveAccount(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + @AuditRequestBody AccountCreateUpdateReq accountCreateUpdateReq) { accountService.checkCreateParam(accountCreateUpdateReq, true, true); AccountDTO newAccount = accountService.buildCreateAccountDTO(username, appResourceScope.getAppId(), accountCreateUpdateReq); - long accountId = accountService.saveAccount(newAccount); - accountAuthService.registerAccount( - username, - accountId, - newAccount.getAlias() - ); - return Response.buildSuccessResp(accountId); + AccountDTO savedAccount = accountService.createAccount(username, newAccount); + return Response.buildSuccessResp(savedAccount.toAccountVO()); } @Override - public Response updateAccount(String username, - AppResourceScope appResourceScope, - String scopeType, - String scopeId, - AccountCreateUpdateReq accountCreateUpdateReq) { - Long accountId = accountCreateUpdateReq.getId(); - AccountDTO account = accountService.getAccountById(accountId); - if (account == null) { - log.info("Account is not exist, accountId={}", accountId); - throw new NotFoundException(ErrorCode.ACCOUNT_NOT_EXIST, ArrayUtil.toArray(accountId)); - } - checkManageAccountPermission(username, appResourceScope, accountId); + @AuditEntry(actionId = ActionId.MANAGE_ACCOUNT) + public Response updateAccount(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + Long accountId, + @AuditRequestBody AccountCreateUpdateReq accountCreateUpdateReq) { + accountCreateUpdateReq.setId(accountId); if (!checkUpdateAccountParam(accountCreateUpdateReq)) { throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } - AccountDTO updateAccount = buildUpdateAccountDTO(account, username, accountCreateUpdateReq); - accountService.updateAccount(updateAccount); - return Response.buildSuccessResp(null); + + AccountDTO updateAccount = buildUpdateAccountDTO(username, appResourceScope.getAppId(), accountCreateUpdateReq); + AccountDTO savedAccount = accountService.updateAccount(username, updateAccount); + return Response.buildSuccessResp(savedAccount.toAccountVO()); } private boolean checkUpdateAccountParam(AccountCreateUpdateReq req) { // 账号名称是不能更新的，所以这里不用校验 if (req.getId() == null) { - log.warn("Id is invalid, id={}", req.getId()); + log.warn("Id is invalid, id=null"); return false; } if (req.getCategory() != null && req.getCategory().equals(AccountCategoryEnum.DB.getValue()) @@ -142,11 +123,10 @@ private boolean checkUpdateAccountParam(AccountCreateUpdateReq req) { return true; } - private AccountDTO buildUpdateAccountDTO(AccountDTO existAccount, String operator, AccountCreateUpdateReq req) { + private AccountDTO buildUpdateAccountDTO(String operator, long appId, AccountCreateUpdateReq req) { AccountDTO accountDTO = new AccountDTO(); - accountDTO.setAppId(existAccount.getAppId()); - accountDTO.setType(existAccount.getType()); - accountDTO.setCategory(existAccount.getCategory()); + accountDTO.setId(req.getId()); + accountDTO.setAppId(appId); accountDTO.setAlias(req.getAlias()); accountDTO.setRemark(req.getRemark()); accountDTO.setGrantees(Utils.concatStringWithSeperator(req.getGrantees(), ",")); @@ -162,7 +142,6 @@ private AccountDTO buildUpdateAccountDTO(AccountDTO existAccount, String operato accountDTO.setDbSystemAccountId(req.getDbSystemAccountId()); accountDTO.setLastModifyUser(operator); accountDTO.setLastModifyTime(DateUtils.currentTimeMillis()); - accountDTO.setId(req.getId()); return accountDTO; } @@ -179,6 +158,7 @@ public Response> listAppAccounts(String username, Integer type, String creator, String lastModifyUser, + String remark, Integer start, Integer pageSize, String orderField, @@ -203,6 +183,7 @@ public Response> listAppAccounts(String username, accountQuery.setType(AccountTypeEnum.valueOf(type)); accountQuery.setCreator(creator); accountQuery.setLastModifyUser(lastModifyUser); + accountQuery.setRemark(remark); } pageData = accountService.listPageAccount(accountQuery, baseSearchCondition); } else { @@ -217,95 +198,39 @@ public Response> listAppAccounts(String username, List accountVOS = new ArrayList<>(); if (pageData.getData() != null) { for (AccountDTO accountDTO : pageData.getData()) { - AccountVO accountVO = convertToAccountVO(accountDTO); - accountVOS.add(accountVO); + accountVOS.add(accountDTO.toAccountVO()); } } // 添加权限数据 List canManageIdList = accountAuthService.batchAuthManageAccount(username, appResourceScope, - accountVOS.parallelStream().map(AccountVO::getId).collect(Collectors.toList())); + accountVOS.stream().map(AccountVO::getId).collect(Collectors.toList())); accountVOS.forEach(it -> it.setCanManage(canManageIdList.contains(it.getId()))); result.setData(accountVOS); result.setCanCreate(checkCreateAccountPermission(username, appResourceScope).isPass()); return Response.buildSuccessResp(result); } - private AccountVO convertToAccountVO(AccountDTO accountDTO) { - AccountVO accountVO = new AccountVO(); - accountVO.setId(accountDTO.getId()); - - AppScopeMappingService appScopeMappingService = - ApplicationContextRegister.getBean(AppScopeMappingService.class); - ResourceScope resourceScope = appScopeMappingService.getScopeByAppId(accountDTO.getAppId()); - accountVO.setScopeType(resourceScope.getType().getValue()); - accountVO.setScopeId(resourceScope.getId()); - accountVO.setAccount(accountDTO.getAccount()); - accountVO.setAlias(accountDTO.getAlias()); - accountVO.setCategory(accountDTO.getCategory().getValue()); - accountVO.setCategoryName(i18nService.getI18n(accountDTO.getCategory().getI18nKey())); - accountVO.setType(accountDTO.getType().getType()); - accountVO.setTypeName(accountDTO.getType().getName()); - accountVO.setOwnerUsers(Utils.getNotBlankSplitList(accountDTO.getGrantees(), ",")); - accountVO.setRemark(accountDTO.getRemark()); - accountVO.setOs(accountDTO.getOs()); - accountVO.setPassword("******"); - accountVO.setDbPort(accountDTO.getDbPort()); - accountVO.setDbPassword("******"); - accountVO.setDbSystemAccountId(accountDTO.getDbSystemAccountId()); - accountVO.setLastModifyUser(accountDTO.getLastModifyUser()); - accountVO.setCreator(accountDTO.getCreator()); - if (accountDTO.getCreateTime() != null) { - accountVO.setCreateTime(accountDTO.getCreateTime()); - } - if (accountDTO.getLastModifyTime() != null) { - accountVO.setLastModifyTime(accountDTO.getLastModifyTime()); - } - return accountVO; - } - @Override + @AuditEntry(actionId = ActionId.MANAGE_ACCOUNT) public Response deleteAccount(String username, AppResourceScope appResourceScope, String scopeType, String scopeId, Long accountId) { - - log.info("Delete account, operator={}, scope={}, accountId={}", username, appResourceScope, accountId); - AccountDTO account = accountService.getAccountById(accountId); - if (account == null) { - log.info("Account is not exist, accountId={}", accountId); - throw new NotFoundException(ErrorCode.ACCOUNT_NOT_EXIST, ArrayUtil.toArray(accountId)); - } - checkManageAccountPermission(username, appResourceScope, accountId); - if (accountService.isAccountRefByAnyStep(accountId)) { - log.info("Account:{} is ref by step, should not delete!", accountId); - throw new FailedPreconditionException(ErrorCode.DELETE_REF_ACCOUNT_FORBIDDEN); - } - if (account.getCategory() == AccountCategoryEnum.SYSTEM - && accountService.isSystemAccountRefByDbAccount(accountId)) { - log.info("Account:{} is ref by db account, should not delete!", accountId); - throw new FailedPreconditionException(ErrorCode.DELETE_REF_ACCOUNT_FORBIDDEN); - } - accountService.deleteAccount(accountId); + accountService.deleteAccount(username, appResourceScope.getAppId(), accountId); return Response.buildSuccessResp(null); } @Override + @AuditEntry(actionId = ActionId.USE_ACCOUNT) public Response getAccountById(String username, AppResourceScope appResourceScope, String scopeType, String scopeId, Long accountId) { - AccountDTO account = accountService.getAccountById(accountId); - if (account == null) { - log.info("Account is not exist, accountId={}", accountId); - throw new NotFoundException(ErrorCode.ACCOUNT_NOT_EXIST, ArrayUtil.toArray(accountId)); - } - - checkManageAccountPermission(username, appResourceScope, accountId); - - return Response.buildSuccessResp(convertToAccountVO(account)); + AccountDTO account = accountService.getAccount(username, appResourceScope.getAppId(), accountId); + return Response.buildSuccessResp(account.toAccountVO()); } @Override @@ -318,14 +243,12 @@ public Response> listAccounts(String username, throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } List accountDTOS = - accountService.listAllAppAccount(appResourceScope.getAppId(), AccountCategoryEnum.valOf(category)); + accountService.listAppAccount(appResourceScope.getAppId(), AccountCategoryEnum.valOf(category)); List accountVOS = new ArrayList<>(); if (accountDTOS != null && !accountDTOS.isEmpty()) { List accountIdList = new ArrayList<>(); for (AccountDTO accountDTO : accountDTOS) { - AccountVO accountVO = convertToAccountVO(accountDTO); - accountVO.setPassword("******"); - accountVO.setDbPassword("******"); + AccountVO accountVO = accountDTO.toAccountVO(); accountVO.setCanManage(true); accountVO.setCanUse(true); accountIdList.add(accountDTO.getId()); @@ -348,15 +271,6 @@ private AuthResult checkCreateAccountPermission(String username, AppResourceScop return accountAuthService.authCreateAccount(username, appResourceScope); } - private void checkManageAccountPermission(String username, - AppResourceScope appResourceScope, - Long accountId) { - AuthResult authResult = accountAuthService.authManageAccount(username, appResourceScope, accountId, null); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } - } - private void setUseAccountPermission(String username, AppResourceScope appResourceScope, List accountVOS) { diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebAppResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebAppResourceImpl.java index 134ee0e61f..6062cdeb92 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebAppResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebAppResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,48 +24,28 @@ package com.tencent.bk.job.manage.api.web.impl; -import com.tencent.bk.job.common.cc.model.InstanceTopologyDTO; -import com.tencent.bk.job.common.constant.ErrorCode; -import com.tencent.bk.job.common.constant.ResourceScopeTypeEnum; import com.tencent.bk.job.common.exception.NotFoundException; -import com.tencent.bk.job.common.exception.NotImplementedException; import com.tencent.bk.job.common.iam.dto.AppResourceScopeResult; import com.tencent.bk.job.common.iam.service.AppAuthService; -import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; import com.tencent.bk.job.common.model.Response; import com.tencent.bk.job.common.model.dto.AppResourceScope; import com.tencent.bk.job.common.model.dto.ApplicationDTO; -import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; -import com.tencent.bk.job.common.model.dto.DynamicGroupWithHost; -import com.tencent.bk.job.common.model.vo.HostInfoVO; -import com.tencent.bk.job.common.model.vo.TargetNodeVO; import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.common.util.CompareUtil; import com.tencent.bk.job.common.util.PageUtil; import com.tencent.bk.job.manage.api.web.WebAppResource; -import com.tencent.bk.job.manage.common.TopologyHelper; import com.tencent.bk.job.manage.model.dto.ApplicationFavorDTO; -import com.tencent.bk.job.manage.model.web.request.AgentStatisticsReq; -import com.tencent.bk.job.manage.model.web.request.IpCheckReq; import com.tencent.bk.job.manage.model.web.request.app.FavorAppReq; -import com.tencent.bk.job.manage.model.web.request.ipchooser.AppTopologyTreeNode; -import com.tencent.bk.job.manage.model.web.request.ipchooser.ListHostByBizTopologyNodesReq; import com.tencent.bk.job.manage.model.web.vo.AppVO; -import com.tencent.bk.job.manage.model.web.vo.CcTopologyNodeVO; -import com.tencent.bk.job.manage.model.web.vo.DynamicGroupInfoVO; -import com.tencent.bk.job.manage.model.web.vo.NodeInfoVO; import com.tencent.bk.job.manage.model.web.vo.PageDataWithAvailableIdList; -import com.tencent.bk.job.manage.model.web.vo.index.AgentStatistics; import com.tencent.bk.job.manage.service.ApplicationService; -import com.tencent.bk.job.manage.service.HostService; import com.tencent.bk.job.manage.service.impl.ApplicationFavorService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RestController; import java.util.ArrayList; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -79,18 +59,15 @@ public class WebAppResourceImpl implements WebAppResource { private final ApplicationService applicationService; private final ApplicationFavorService applicationFavorService; private final AppAuthService appAuthService; - private final HostService hostService; private final AppScopeMappingService appScopeMappingService; @Autowired public WebAppResourceImpl(ApplicationService applicationService, ApplicationFavorService applicationFavorService, AppAuthService appAuthService, - HostService hostService, AppScopeMappingService appScopeMappingService) { this.applicationService = applicationService; this.applicationFavorService = applicationFavorService; - this.hostService = hostService; this.appAuthService = appAuthService; this.appScopeMappingService = appScopeMappingService; } @@ -228,254 +205,4 @@ public Response cancelFavorApp(String username, applicationFavorService.cancelFavorApp(username, applicationDTO.getId()) ); } - - @Override - public Response> listAppHost(String username, - AppResourceScope appResourceScope, - String scopeType, - String scopeId, - Integer start, - Integer pageSize, - Long moduleType, - String ipCondition) { - ApplicationHostDTO applicationHostInfoCondition = new ApplicationHostDTO(); - applicationHostInfoCondition.setBizId(appResourceScope.getAppId()); - applicationHostInfoCondition.setIp(ipCondition); - if (moduleType != null) { - applicationHostInfoCondition.getModuleType().add(moduleType); - } - - BaseSearchCondition baseSearchCondition = new BaseSearchCondition(); - if (start == null || start < 0) { - start = 0; - } - if (pageSize == null || pageSize <= 0) { - pageSize = 10; - } - baseSearchCondition.setStart(start); - baseSearchCondition.setLength(pageSize); - - PageData appHostInfoPageData = - hostService.listAppHost(applicationHostInfoCondition, baseSearchCondition); - PageData finalHostInfoPageData = new PageData<>(); - finalHostInfoPageData.setTotal(appHostInfoPageData.getTotal()); - finalHostInfoPageData.setStart(appHostInfoPageData.getStart()); - finalHostInfoPageData.setPageSize(appHostInfoPageData.getPageSize()); - finalHostInfoPageData - .setData(appHostInfoPageData.getData().stream() - .map(TopologyHelper::convertToHostInfoVO).collect(Collectors.toList())); - return Response.buildSuccessResp(finalHostInfoPageData); - } - - @Override - public Response listAppTopologyTree(String username, - AppResourceScope appResourceScope, - String scopeType, - String scopeId) { - return Response.buildSuccessResp( - hostService.listAppTopologyTree(username, appResourceScope) - ); - } - - @Override - public Response listAppTopologyHostTree(String username, - AppResourceScope appResourceScope, - String scopeType, - String scopeId) { - return Response.buildSuccessResp(hostService.listAppTopologyHostTree(username, appResourceScope)); - } - - @Override - public Response listAppTopologyHostCountTree(String username, - AppResourceScope appResourceScope, - String scopeType, - String scopeId) { - return Response.buildSuccessResp(hostService.listAppTopologyHostCountTree(username, - appResourceScope)); - } - - @Override - public Response> listHostByBizTopologyNodes(String username, - AppResourceScope appResourceScope, - String scopeType, - String scopeId, - ListHostByBizTopologyNodesReq req) { - return Response.buildSuccessResp( - hostService.listHostByAppTopologyNodes( - username, appResourceScope, req - ) - ); - } - - @Override - public Response> listIpByBizTopologyNodes(String username, - AppResourceScope appResourceScope, - String scopeType, - String scopeId, - ListHostByBizTopologyNodesReq req) { - return Response.buildSuccessResp( - hostService.listIPByBizTopologyNodes( - username, appResourceScope, req - ) - ); - } - - @Override - public Response> getNodeDetail(String username, - AppResourceScope appResourceScope, - String scopeType, - String scopeId, - List targetNodeVOList) { - List treeNodeList = hostService.getAppTopologyTreeNodeDetail(username, - appResourceScope, - targetNodeVOList.stream().map(it -> new AppTopologyTreeNode( - it.getType(), - "", - it.getId(), - "", - null - )).collect(Collectors.toList())); - return Response.buildSuccessResp(treeNodeList); - } - - @Override - public Response>> queryNodePaths(String username, - AppResourceScope appResourceScope, - String scopeType, - String scopeId, - List targetNodeVOList) { - ApplicationDTO appDTO = applicationService.getAppByScope(appResourceScope); - if (appDTO.isBizSet()) { - return Response.buildSuccessResp(Collections.emptyList()); - } - List> pathList = hostService.queryBizNodePaths( - username, - appDTO.getBizIdIfBizApp(), - targetNodeVOList.stream().map(it -> { - InstanceTopologyDTO instanceTopologyDTO = new InstanceTopologyDTO(); - instanceTopologyDTO.setObjectId(it.getType()); - instanceTopologyDTO.setInstanceId(it.getId()); - return instanceTopologyDTO; - }).collect(Collectors.toList())); - List> resultList = new ArrayList<>(); - for (List instanceTopologyDTOS : pathList) { - if (instanceTopologyDTOS == null) { - resultList.add(null); - } else { - resultList.add(instanceTopologyDTOS.stream().map(it -> { - CcTopologyNodeVO ccTopologyNodeVO = new CcTopologyNodeVO(); - ccTopologyNodeVO.setObjectId(it.getObjectId()); - ccTopologyNodeVO.setObjectName(it.getObjectName()); - ccTopologyNodeVO.setInstanceId(it.getInstanceId()); - ccTopologyNodeVO.setInstanceName(it.getInstanceName()); - return ccTopologyNodeVO; - }).collect(Collectors.toList())); - } - } - return Response.buildSuccessResp(resultList); - } - - @Override - public Response> listHostByNode(String username, - AppResourceScope appResourceScope, - String scopeType, - String scopeId, - List targetNodeVOList) { - ApplicationDTO appDTO = applicationService.getAppByScope(appResourceScope); - if (appDTO.isBizSet()) { - String msg = "topo node of bizset not supported yet"; - throw new NotImplementedException(msg, ErrorCode.NOT_SUPPORT_FEATURE); - } - List moduleHostInfoList = hostService.getBizHostsByNode( - username, - appDTO.getBizIdIfBizApp(), - targetNodeVOList.stream().map(it -> new AppTopologyTreeNode( - it.getType(), - "", - it.getId(), - "", - null - )).collect(Collectors.toList()) - ); - return Response.buildSuccessResp(moduleHostInfoList); - } - - @Override - public Response> listAppDynamicGroup(String username, - AppResourceScope appResourceScope, - String scopeType, - String scopeId) { - ApplicationDTO applicationDTO = applicationService.getAppByAppId(appResourceScope.getAppId()); - // 业务集动态分组暂不支持 - if (!applicationDTO.isBiz()) { - return Response.buildSuccessResp(new ArrayList<>()); - } - List dynamicGroupList = hostService.getAppDynamicGroupList( - username, appResourceScope - ); - List dynamicGroupInfoList = dynamicGroupList.parallelStream() - .map(TopologyHelper::convertToDynamicGroupInfoVO) - .collect(Collectors.toList()); - return Response.buildSuccessResp(dynamicGroupInfoList); - } - - @Override - public Response> listAppDynamicGroupHost(String username, - AppResourceScope appResourceScope, - String scopeType, - String scopeId, - List dynamicGroupIds) { - // 目前只有业务支持动态分组 - if (appResourceScope.getType() == ResourceScopeTypeEnum.BIZ) { - List dynamicGroupList = - hostService.getBizDynamicGroupHostList( - username, Long.parseLong(appResourceScope.getId()), dynamicGroupIds - ); - List dynamicGroupInfoList = dynamicGroupList.parallelStream() - .map(TopologyHelper::convertToDynamicGroupInfoVO) - .collect(Collectors.toList()); - return Response.buildSuccessResp(dynamicGroupInfoList); - } - return Response.buildSuccessResp(Collections.emptyList()); - } - - @Override - public Response> listAppDynamicGroupWithoutHosts(String username, - AppResourceScope appResourceScope, - String scopeType, - String scopeId, - List dynamicGroupIds) { - List dynamicGroupList = hostService.getAppDynamicGroupList( - username, appResourceScope - ); - List dynamicGroupInfoList = dynamicGroupList.parallelStream() - .filter(dynamicGroupInfoDTO -> dynamicGroupIds.contains(dynamicGroupInfoDTO.getId())) - .map(TopologyHelper::convertToDynamicGroupInfoVO) - .collect(Collectors.toList()); - return Response.buildSuccessResp(dynamicGroupInfoList); - } - - @Override - public Response> listHostByIp(String username, - AppResourceScope appResourceScope, - String scopeType, - String scopeId, - IpCheckReq req) { - return Response.buildSuccessResp(hostService.getHostsByIp( - username, - appResourceScope.getAppId(), - req.getActionScope(), - req.getIpList()) - ); - } - - @Override - public Response agentStatistics(String username, - AppResourceScope appResourceScope, - String scopeType, - String scopeId, - AgentStatisticsReq agentStatisticsReq) { - return Response.buildSuccessResp(hostService.getAgentStatistics(username, appResourceScope.getAppId(), - agentStatisticsReq)); - } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebContainerResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebContainerResourceImpl.java new file mode 100644 index 0000000000..aaa83ac780 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebContainerResourceImpl.java @@ -0,0 +1,274 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.api.web.impl; + +import com.tencent.bk.job.common.cc.model.container.ContainerDetailDTO; +import com.tencent.bk.job.common.cc.model.container.KubeNodeDTO; +import com.tencent.bk.job.common.cc.model.container.KubeTopologyDTO; +import com.tencent.bk.job.common.i18n.service.MessageI18nService; +import com.tencent.bk.job.common.model.PageData; +import com.tencent.bk.job.common.model.Response; +import com.tencent.bk.job.common.model.dto.AppResourceScope; +import com.tencent.bk.job.common.model.dto.ApplicationDTO; +import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; +import com.tencent.bk.job.common.model.vo.ContainerVO; +import com.tencent.bk.job.common.util.toggle.ToggleEvaluateContext; +import com.tencent.bk.job.common.util.toggle.ToggleStrategyContextParams; +import com.tencent.bk.job.common.util.toggle.feature.FeatureIdConstants; +import com.tencent.bk.job.common.util.toggle.feature.FeatureToggle; +import com.tencent.bk.job.manage.api.web.WebContainerResource; +import com.tencent.bk.job.manage.model.mapper.ContainerMapper; +import com.tencent.bk.job.manage.model.query.ContainerQuery; +import com.tencent.bk.job.manage.model.web.request.chooser.ListTopologyTreesReq; +import com.tencent.bk.job.manage.model.web.request.chooser.container.ContainerCheckReq; +import com.tencent.bk.job.manage.model.web.request.chooser.container.ContainerDetailReq; +import com.tencent.bk.job.manage.model.web.request.chooser.container.ContainerIdWithMeta; +import com.tencent.bk.job.manage.model.web.request.chooser.container.ListContainerByTopologyNodesReq; +import com.tencent.bk.job.manage.model.web.vo.chooser.container.ContainerTopologyNodeVO; +import com.tencent.bk.job.manage.service.ApplicationService; +import com.tencent.bk.job.manage.service.ContainerService; +import com.tencent.bk.job.manage.service.host.HostService; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RestController; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@RestController +@Slf4j +public class WebContainerResourceImpl implements WebContainerResource { + + private final ContainerService containerService; + private final ApplicationService applicationService; + private final MessageI18nService i18nService; + private final HostService hostService; + + @Autowired + public WebContainerResourceImpl(ContainerService containerService, + ApplicationService applicationService, + MessageI18nService i18nService, + HostService hostService) { + this.containerService = containerService; + this.applicationService = applicationService; + this.i18nService = i18nService; + this.hostService = hostService; + } + + @Override + public Response> listTopologyTrees(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + ListTopologyTreesReq req) { + if (appResourceScope.isBizSet()) { + // 业务集暂时不支持容器拓扑 + return Response.buildSuccessResp(buildEmptyTopoTree(scopeType, scopeId)); + } + if (!isContainerExecuteSupport(appResourceScope)) { + // 未开启"容器执行"特性的灰度，返回空的容器拓扑 + return Response.buildSuccessResp(buildEmptyTopoTree(scopeType, scopeId)); + } + + KubeTopologyDTO topo = containerService.getBizKubeCacheTopo(Long.parseLong(scopeId)); + + ContainerTopologyNodeVO topoVO = new ContainerTopologyNodeVO(); + topoVO.setInstanceId(topo.getBiz().getId()); + topoVO.setInstanceName(topo.getBiz().getName()); + topoVO.setObjectId("biz"); + topoVO.setObjectName(i18nService.getI18n("cmdb.object.name.biz")); + topoVO.setCount(topo.getBiz().getCount()); + + if (CollectionUtils.isNotEmpty(topo.getNodes())) { + List clusters = new ArrayList<>(); + for (KubeNodeDTO node : topo.getNodes()) { + clusters.add(convertToContainerTopologyNodeVO(node)); + } + topoVO.setChild(clusters); + } + + return Response.buildSuccessResp( + Collections.singletonList( + topoVO + ) + ); + } + + private boolean isContainerExecuteSupport(AppResourceScope appResourceScope) { + if (appResourceScope.isBizSet()) { + return false; + } + return FeatureToggle.checkFeature( + FeatureIdConstants.FEATURE_CONTAINER_EXECUTE, + ToggleEvaluateContext.builder().addContextParam( + ToggleStrategyContextParams.CTX_PARAM_RESOURCE_SCOPE, appResourceScope)); + } + + private ContainerTopologyNodeVO convertToContainerTopologyNodeVO(KubeNodeDTO node) { + ContainerTopologyNodeVO nodeVO = new ContainerTopologyNodeVO(); + nodeVO.setObjectId(node.getKind()); + nodeVO.setObjectName(node.getKind()); + nodeVO.setInstanceId(node.getId()); + nodeVO.setInstanceName(node.getName()); + nodeVO.setCount(node.getCount()); + List children = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(node.getNodes())) { + for (KubeNodeDTO childNode : node.getNodes()) { + children.add(convertToContainerTopologyNodeVO(childNode)); + } + } + nodeVO.setChild(children); + return nodeVO; + } + + private List buildEmptyTopoTree(String scopeType, String scopeId) { + ContainerTopologyNodeVO topo = new ContainerTopologyNodeVO(); + ApplicationDTO bizSetApp = applicationService.getAppByScope(scopeType, scopeId); + topo.setInstanceId(bizSetApp.getId()); + topo.setInstanceName(bizSetApp.getName()); + if (bizSetApp.isBiz()) { + topo.setObjectId("biz"); + topo.setObjectName(i18nService.getI18n("cmdb.object.name.biz")); + } else if (bizSetApp.isBizSet()) { + topo.setObjectId("biz_set"); + topo.setObjectName(i18nService.getI18n("cmdb.object.name.biz_set")); + } + topo.setCount(0); + return Collections.singletonList(topo); + } + + @Override + public Response> listContainerByTopologyNodes(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + ListContainerByTopologyNodesReq req) { + if (!isContainerExecuteSupport(appResourceScope)) { + // 未开启"容器执行"特性的灰度，返回空的容器拓扑 + return Response.buildSuccessResp(PageData.emptyPageData(req.getStart(), req.getPageSize())); + } + ContainerQuery containerQuery = + ContainerQuery.fromListContainerByTopologyNodesReq(Long.parseLong(scopeId), req); + + PageData pageData = containerService.listPageKubeContainerByTopo(containerQuery); + + PageData containerVOPageData = + PageData.from(pageData, ContainerMapper::toContainerVO); + fillNodesHostInfo(containerVOPageData.getData()); + + return Response.buildSuccessResp(containerVOPageData); + } + + private void fillNodesHostInfo(Collection containerVOs) { + List hostIds = containerVOs.stream() + .map(ContainerVO::getNodeHostId).distinct().collect(Collectors.toList()); + Map hostMap = hostService.listHostsByHostIds(hostIds); + + containerVOs.forEach(containerVO -> { + Long nodeHostId = containerVO.getNodeHostId(); + ApplicationHostDTO nodeHost = hostMap.get(nodeHostId); + if (nodeHost != null) { + containerVO.setNodeIp(nodeHost.getPrimaryIp()); + } + }); + } + + @Override + public Response> listContainerIdByTopologyNodes( + String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + ListContainerByTopologyNodesReq req + ) { + if (!isContainerExecuteSupport(appResourceScope)) { + // 未开启"容器执行"特性的灰度，返回空的容器拓扑 + return Response.buildSuccessResp(PageData.emptyPageData(req.getStart(), req.getPageSize())); + } + ContainerQuery containerQuery = + ContainerQuery.fromListContainerByTopologyNodesReq(Long.parseLong(scopeId), req); + + PageData pageData; + if (containerQuery.getPageCondition() != null) { + pageData = containerService.listPageKubeContainerByTopo(containerQuery); + } else { + List containerList = containerService.listKubeContainerByTopo(containerQuery); + pageData = new PageData<>(0, Integer.MAX_VALUE, + containerList == null ? 0L : containerList.size(), containerList); + } + + PageData containerIdPageData = + PageData.from(pageData, container -> new ContainerIdWithMeta(container.getContainer().getId())); + + return Response.buildSuccessResp(containerIdPageData); + } + + @Override + public Response> checkContainers(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + ContainerCheckReq req) { + if (!isContainerExecuteSupport(appResourceScope)) { + // 未开启"容器执行"特性的灰度，返回空的容器拓扑 + return Response.buildSuccessResp(Collections.emptyList()); + } + List containers = + containerService.listKubeContainerByUIds(Long.parseLong(scopeId), req.getUidList()); + + return Response.buildSuccessResp(toContainerVOS(containers)); + } + + private List toContainerVOS(List containers) { + if (CollectionUtils.isEmpty(containers)) { + return Collections.emptyList(); + } + + List containerVOS = containers.stream() + .map(ContainerMapper::toContainerVO).collect(Collectors.toList()); + fillNodesHostInfo(containerVOS); + + return containerVOS; + } + + @Override + public Response> getContainerDetails(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + ContainerDetailReq req) { + List containers = containerService.listKubeContainerByIds( + Long.parseLong(scopeId), + req.getContainerList().stream() + .map(ContainerIdWithMeta::getId).collect(Collectors.toList())); + + return Response.buildSuccessResp(toContainerVOS(containers)); + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebCredentialResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebCredentialResourceImpl.java index 3ac9564ef4..c8bee95673 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebCredentialResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebCredentialResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,9 @@ package com.tencent.bk.job.manage.api.web.impl; -import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.audit.annotations.AuditRequestBody; +import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.iam.model.AuthResult; import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; @@ -34,15 +36,14 @@ import com.tencent.bk.job.manage.auth.TicketAuthService; import com.tencent.bk.job.manage.model.dto.CredentialDTO; import com.tencent.bk.job.manage.model.web.request.CredentialCreateUpdateReq; +import com.tencent.bk.job.manage.model.web.vo.CredentialBasicVO; import com.tencent.bk.job.manage.model.web.vo.CredentialVO; import com.tencent.bk.job.manage.service.CredentialService; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RestController; import java.util.List; -import java.util.Objects; import java.util.stream.Collectors; @@ -84,7 +85,7 @@ public Response> listCredentials(String username, baseSearchCondition.setLength(pageSize); PageData pageData = credentialService.listCredentials(credentialQuery, baseSearchCondition); List credentialVOList = - pageData.getData().parallelStream().map(CredentialDTO::toVO).collect(Collectors.toList()); + pageData.getData().stream().map(CredentialDTO::toVO).collect(Collectors.toList()); PageData finalPageData = new PageData<>(); finalPageData.setStart(pageData.getStart()); finalPageData.setPageSize(pageData.getPageSize()); @@ -95,45 +96,73 @@ public Response> listCredentials(String username, } @Override - public Response saveCredential(String username, - AppResourceScope appResourceScope, - String scopeType, - String scopeId, - CredentialCreateUpdateReq createUpdateReq) { - AuthResult authResult; - if (StringUtils.isBlank(createUpdateReq.getId())) { - authResult = checkCreateTicketPermission(username, appResourceScope); - } else { - authResult = checkManageTicketPermission(username, appResourceScope, createUpdateReq.getId()); - } - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } - return Response.buildSuccessResp(credentialService.saveCredential(username, appResourceScope.getAppId(), - createUpdateReq)); + public Response> listCredentialBasicInfo(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + Integer start, + Integer pageSize) { + BaseSearchCondition baseSearchCondition = new BaseSearchCondition(); + baseSearchCondition.setStart(start); + baseSearchCondition.setLength(pageSize); + PageData pageData = credentialService.listCredentialBasicInfo( + appResourceScope.getAppId(), + baseSearchCondition + ); + List credentialBasicVOList = + pageData.getData().stream().map(CredentialDTO::toBasicVO).collect(Collectors.toList()); + PageData finalPageData = new PageData<>(); + finalPageData.setStart(pageData.getStart()); + finalPageData.setPageSize(pageData.getPageSize()); + finalPageData.setTotal(pageData.getTotal()); + finalPageData.setData(credentialBasicVOList); + addPermissionDataForBasicVO(username, appResourceScope, finalPageData); + return Response.buildSuccessResp(finalPageData); + } + + @Override + @AuditEntry(actionId = ActionId.CREATE_TICKET) + public Response createCredential(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + @AuditRequestBody CredentialCreateUpdateReq createUpdateReq) { + CredentialDTO credential = + credentialService.createCredential(username, appResourceScope.getAppId(), createUpdateReq); + return Response.buildSuccessResp(credential.toVO()); } @Override + @AuditEntry(actionId = ActionId.MANAGE_TICKET) + public Response updateCredential(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + String credentialId, + @AuditRequestBody CredentialCreateUpdateReq createUpdateReq) { + createUpdateReq.setId(credentialId); + CredentialDTO credential = credentialService.updateCredential(username, appResourceScope.getAppId(), + createUpdateReq); + return Response.buildSuccessResp(credential.toVO()); + } + + @Override + @AuditEntry(actionId = ActionId.MANAGE_TICKET) public Response deleteCredentialById(String username, AppResourceScope appResourceScope, String scopeType, String scopeId, String id) { - AuthResult authResult = checkManageTicketPermission(username, appResourceScope, id); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } - return Response.buildSuccessResp(credentialService.deleteCredentialById(username, appResourceScope.getAppId(), - id)); + return Response.buildSuccessResp( + credentialService.deleteCredentialById(username, appResourceScope.getAppId(), id)); } private void addPermissionData(String username, AppResourceScope appResourceScope, PageData credentialVOPageData) { List credentialVOList = credentialVOPageData.getData(); // 添加权限数据 - List credentialIdList = credentialVOList.parallelStream() + List credentialIdList = credentialVOList.stream() .map(CredentialVO::getId) - .map(Objects::toString) .collect(Collectors.toList()); List canManageIdList = ticketAuthService.batchAuthManageTicket(username, appResourceScope, credentialIdList); @@ -146,14 +175,23 @@ private void addPermissionData(String username, AppResourceScope appResourceScop credentialVOPageData.setCanCreate(checkCreateTicketPermission(username, appResourceScope).isPass()); } + private void addPermissionDataForBasicVO(String username, AppResourceScope appResourceScope, + PageData credentialBasicVOPageData) { + List credentialBasicVOList = credentialBasicVOPageData.getData(); + // 添加权限数据 + List credentialIdList = credentialBasicVOList.stream() + .map(CredentialBasicVO::getId) + .collect(Collectors.toList()); + List canUseIdList = + ticketAuthService.batchAuthUseTicket(username, appResourceScope, credentialIdList); + credentialBasicVOList.forEach(it -> { + it.setCanUse(canUseIdList.contains(it.getId())); + }); + credentialBasicVOPageData.setCanCreate(checkCreateTicketPermission(username, appResourceScope).isPass()); + } + public AuthResult checkCreateTicketPermission(String username, AppResourceScope appResourceScope) { // 需要拥有在业务下创建凭证的权限 return ticketAuthService.authCreateTicket(username, appResourceScope); } - - private AuthResult checkManageTicketPermission(String username, AppResourceScope appResourceScope, - String credentialId) { - // 需要拥有在业务下管理某个具体凭证的权限 - return ticketAuthService.authManageTicket(username, appResourceScope, credentialId, null); - } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebCustomSettingsResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebCustomSettingsResourceImpl.java index c22e5d23b6..b88fe901e6 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebCustomSettingsResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebCustomSettingsResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,161 +24,55 @@ package com.tencent.bk.job.manage.api.web.impl; -import com.tencent.bk.job.common.constant.ErrorCode; -import com.tencent.bk.job.common.i18n.service.MessageI18nService; import com.tencent.bk.job.common.model.Response; -import com.tencent.bk.job.common.model.ValidateResult; import com.tencent.bk.job.common.model.dto.AppResourceScope; -import com.tencent.bk.job.common.service.AppScopeMappingService; -import com.tencent.bk.job.common.util.Base64Util; import com.tencent.bk.job.manage.api.web.WebCustomSettingsResource; -import com.tencent.bk.job.manage.common.constants.ScriptTemplateVariableEnum; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; -import com.tencent.bk.job.manage.model.dto.customsetting.ScriptTemplateDTO; -import com.tencent.bk.job.manage.model.dto.customsetting.ScriptTemplateVariableRenderDTO; -import com.tencent.bk.job.manage.model.web.request.customsetting.ScriptTemplateCreateUpdateReq; -import com.tencent.bk.job.manage.model.web.request.customsetting.ScriptTemplateRenderReq; -import com.tencent.bk.job.manage.model.web.vo.customsetting.ScriptTemplateVO; -import com.tencent.bk.job.manage.model.web.vo.customsetting.ScriptTemplateVariableVO; -import com.tencent.bk.job.manage.service.CustomScriptTemplateService; +import com.tencent.bk.job.manage.model.web.request.customsetting.BatchGetCustomSettingsReq; +import com.tencent.bk.job.manage.model.web.request.customsetting.DeleteCustomSettingsReq; +import com.tencent.bk.job.manage.model.web.request.customsetting.SaveCustomSettingsReq; +import com.tencent.bk.job.manage.service.impl.CustomSettingsService; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RestController; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; +import java.util.Map; @RestController @Slf4j public class WebCustomSettingsResourceImpl implements WebCustomSettingsResource { - private final CustomScriptTemplateService customScriptTemplateService; - private final MessageI18nService i18nService; - private final AppScopeMappingService appScopeMappingService; - @Autowired - public WebCustomSettingsResourceImpl(CustomScriptTemplateService customScriptTemplateService, - MessageI18nService i18nService, - AppScopeMappingService appScopeMappingService) { - this.customScriptTemplateService = customScriptTemplateService; - this.i18nService = i18nService; - this.appScopeMappingService = appScopeMappingService; - } - - @Override - public Response> listUserCustomScriptTemplate(String username, - String scriptLanguages) { - List scriptTemplates = getUserCustomScriptTemplate(username, scriptLanguages); - - List scriptTemplateVOS = scriptTemplates.stream() - .map(this::toScriptTemplateVO).collect(Collectors.toList()); - return Response.buildSuccessResp(scriptTemplateVOS); - } - - private List getUserCustomScriptTemplate(String username, - String scriptLanguages) { - List scriptTemplates = customScriptTemplateService.listCustomScriptTemplate(username); - if (CollectionUtils.isEmpty(scriptTemplates)) { - return Collections.emptyList(); - } - - if (StringUtils.isNotEmpty(scriptLanguages)) { - Set languages = new HashSet<>(); - for (String language : scriptLanguages.split(",")) { - languages.add(Integer.parseInt(language)); - } - scriptTemplates = scriptTemplates.stream() - .filter(scriptTemplate -> languages.contains(scriptTemplate.getScriptLanguage())) - .collect(Collectors.toList()); - } - return scriptTemplates; - } + private final CustomSettingsService customSettingsService; - private ScriptTemplateVO toScriptTemplateVO(ScriptTemplateDTO scriptTemplate) { - String scriptContent = scriptTemplate.getScriptContent(); - if (StringUtils.isNotEmpty(scriptContent)) { - scriptContent = Base64Util.encodeContentToStr(scriptContent); - } - return new ScriptTemplateVO(scriptTemplate.getScriptLanguage(), scriptContent); - } - - @Override - public Response> listRenderedUserCustomScriptTemplate(String username, - String scriptLanguages, - String scopeType, - String scopeId) { - AppResourceScope appResourceScope = appScopeMappingService.getAppResourceScope(null, scopeType, scopeId); - List scriptTemplates = getUserCustomScriptTemplate(username, scriptLanguages); - scriptTemplates.forEach( - scriptTemplate -> customScriptTemplateService.renderScriptTemplate( - new ScriptTemplateVariableRenderDTO(appResourceScope.getType().getValue(), - appResourceScope.getId(), username), scriptTemplate)); - List scriptTemplateVOS = scriptTemplates.stream().map(this::toScriptTemplateVO) - .collect(Collectors.toList()); - return Response.buildSuccessResp(scriptTemplateVOS); + @Autowired + public WebCustomSettingsResourceImpl( + CustomSettingsService customSettingsService) { + this.customSettingsService = customSettingsService; } @Override - public Response saveScriptTemplate(String username, ScriptTemplateCreateUpdateReq req) { - ValidateResult validateResult = checkScriptTemplateCreateUpdateReq(req); - if (!validateResult.isPass()) { - return Response.buildCommonFailResp(validateResult.getErrorCode(), validateResult.getErrorParams()); - } - String scriptContent = req.getScriptContent(); - if (StringUtils.isNotEmpty(scriptContent)) { - scriptContent = Base64Util.decodeContentToStr(scriptContent); - } - ScriptTemplateDTO scriptTemplate = new ScriptTemplateDTO(req.getScriptLanguage(), scriptContent); - customScriptTemplateService.saveScriptTemplate(username, scriptTemplate); - return Response.buildSuccessResp(null); - } - - private ValidateResult checkScriptTemplateCreateUpdateReq(ScriptTemplateCreateUpdateReq req) { - if (req.getScriptLanguage() == null || ScriptTypeEnum.valueOf(req.getScriptLanguage()) == null) { - return ValidateResult.fail(ErrorCode.ILLEGAL_PARAM); - } - if (req.getScriptContent() == null) { - req.setScriptContent(""); - } - return ValidateResult.pass(); + public Response>> saveCustomSettings(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + SaveCustomSettingsReq req) { + return Response.buildSuccessResp(customSettingsService.saveCustomSettings(username, appResourceScope, req)); } @Override - public Response renderScriptTemplate(String username, ScriptTemplateRenderReq req) { - String scriptContent = req.getScriptContent(); - if (StringUtils.isNotEmpty(scriptContent)) { - scriptContent = Base64Util.decodeContentToStr(scriptContent); - } - - ScriptTemplateDTO scriptTemplate = new ScriptTemplateDTO(req.getScriptLanguage(), scriptContent); - - ScriptTemplateVariableRenderDTO variableRender = new ScriptTemplateVariableRenderDTO(req.getScopeType(), - req.getScopeId(), username); - if (req.getScopeType() == null && req.getScopeId() == null) { - variableRender.addDefaultValue(ScriptTemplateVariableEnum.BIZ_ID.getName(), - ScriptTemplateVariableEnum.BIZ_ID.getDemo()); - variableRender.addDefaultValue(ScriptTemplateVariableEnum.BIZ_NAME.getName(), - ScriptTemplateVariableEnum.BIZ_NAME.getDemo()); - } - customScriptTemplateService.renderScriptTemplate(variableRender, scriptTemplate); - return Response.buildSuccessResp(toScriptTemplateVO(scriptTemplate)); + public Response>> batchGetCustomSettings(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + BatchGetCustomSettingsReq req) { + return Response.buildSuccessResp(customSettingsService.batchGetCustomSettings(username, appResourceScope, req)); } @Override - public Response> listScriptTemplateVariables(String username) { - List variableVOS = new ArrayList<>(); - for (ScriptTemplateVariableEnum variable : ScriptTemplateVariableEnum.values()) { - ScriptTemplateVariableVO variableVO = new ScriptTemplateVariableVO(); - variableVO.setName(variable.getName()); - variableVO.setDescription(i18nService.getI18n(variable.getDescI18nKey())); - variableVO.setDemo(variable.getDemo()); - variableVOS.add(variableVO); - } - return Response.buildSuccessResp(variableVOS); + public Response deleteCustomSettings(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + DeleteCustomSettingsReq req) { + return Response.buildSuccessResp(customSettingsService.deleteCustomSettings(username, appResourceScope, req)); } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebCustomSettingsScriptTemplateResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebCustomSettingsScriptTemplateResourceImpl.java new file mode 100644 index 0000000000..16548dc2d2 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebCustomSettingsScriptTemplateResourceImpl.java @@ -0,0 +1,184 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.api.web.impl; + +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.i18n.service.MessageI18nService; +import com.tencent.bk.job.common.model.Response; +import com.tencent.bk.job.common.model.ValidateResult; +import com.tencent.bk.job.common.model.dto.AppResourceScope; +import com.tencent.bk.job.common.service.AppScopeMappingService; +import com.tencent.bk.job.common.util.Base64Util; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; +import com.tencent.bk.job.manage.api.web.WebCustomSettingsScriptTemplateResource; +import com.tencent.bk.job.manage.common.constants.ScriptTemplateVariableEnum; +import com.tencent.bk.job.manage.model.dto.customsetting.ScriptTemplateDTO; +import com.tencent.bk.job.manage.model.dto.customsetting.ScriptTemplateVariableRenderDTO; +import com.tencent.bk.job.manage.model.web.request.customsetting.ScriptTemplateCreateUpdateReq; +import com.tencent.bk.job.manage.model.web.request.customsetting.ScriptTemplateRenderReq; +import com.tencent.bk.job.manage.model.web.vo.customsetting.ScriptTemplateVO; +import com.tencent.bk.job.manage.model.web.vo.customsetting.ScriptTemplateVariableVO; +import com.tencent.bk.job.manage.service.CustomScriptTemplateService; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RestController; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +@RestController +@Slf4j +public class WebCustomSettingsScriptTemplateResourceImpl implements WebCustomSettingsScriptTemplateResource { + private final CustomScriptTemplateService customScriptTemplateService; + private final MessageI18nService i18nService; + private final AppScopeMappingService appScopeMappingService; + + @Autowired + public WebCustomSettingsScriptTemplateResourceImpl(CustomScriptTemplateService customScriptTemplateService, + MessageI18nService i18nService, + AppScopeMappingService appScopeMappingService) { + this.customScriptTemplateService = customScriptTemplateService; + this.i18nService = i18nService; + this.appScopeMappingService = appScopeMappingService; + } + + @Override + public Response> listUserCustomScriptTemplate(String username, + String scriptLanguages) { + List scriptTemplates = getUserCustomScriptTemplate(username, scriptLanguages); + + List scriptTemplateVOS = scriptTemplates.stream() + .map(this::toScriptTemplateVO).collect(Collectors.toList()); + return Response.buildSuccessResp(scriptTemplateVOS); + } + + private List getUserCustomScriptTemplate(String username, + String scriptLanguages) { + List scriptTemplates = customScriptTemplateService.listCustomScriptTemplate(username); + if (CollectionUtils.isEmpty(scriptTemplates)) { + return Collections.emptyList(); + } + + if (StringUtils.isNotEmpty(scriptLanguages)) { + Set languages = new HashSet<>(); + for (String language : scriptLanguages.split(",")) { + languages.add(Integer.parseInt(language)); + } + scriptTemplates = scriptTemplates.stream() + .filter(scriptTemplate -> languages.contains(scriptTemplate.getScriptLanguage())) + .collect(Collectors.toList()); + } + return scriptTemplates; + } + + private ScriptTemplateVO toScriptTemplateVO(ScriptTemplateDTO scriptTemplate) { + String scriptContent = scriptTemplate.getScriptContent(); + if (StringUtils.isNotEmpty(scriptContent)) { + scriptContent = Base64Util.encodeContentToStr(scriptContent); + } + return new ScriptTemplateVO(scriptTemplate.getScriptLanguage(), scriptContent); + } + + @Override + public Response> listRenderedUserCustomScriptTemplate(String username, + String scriptLanguages, + String scopeType, + String scopeId) { + AppResourceScope appResourceScope = appScopeMappingService.getAppResourceScope(null, scopeType, scopeId); + List scriptTemplates = getUserCustomScriptTemplate(username, scriptLanguages); + scriptTemplates.forEach( + scriptTemplate -> customScriptTemplateService.renderScriptTemplate( + new ScriptTemplateVariableRenderDTO(appResourceScope.getType().getValue(), + appResourceScope.getId(), username), scriptTemplate)); + List scriptTemplateVOS = scriptTemplates.stream().map(this::toScriptTemplateVO) + .collect(Collectors.toList()); + return Response.buildSuccessResp(scriptTemplateVOS); + } + + @Override + public Response saveScriptTemplate(String username, ScriptTemplateCreateUpdateReq req) { + ValidateResult validateResult = checkScriptTemplateCreateUpdateReq(req); + if (!validateResult.isPass()) { + return Response.buildCommonFailResp(validateResult.getErrorCode(), validateResult.getErrorParams()); + } + String scriptContent = req.getScriptContent(); + if (StringUtils.isNotEmpty(scriptContent)) { + scriptContent = Base64Util.decodeContentToStr(scriptContent); + } + ScriptTemplateDTO scriptTemplate = new ScriptTemplateDTO(req.getScriptLanguage(), scriptContent); + customScriptTemplateService.saveScriptTemplate(username, scriptTemplate); + return Response.buildSuccessResp(null); + } + + private ValidateResult checkScriptTemplateCreateUpdateReq(ScriptTemplateCreateUpdateReq req) { + if (req.getScriptLanguage() == null || ScriptTypeEnum.valOf(req.getScriptLanguage()) == null) { + return ValidateResult.fail(ErrorCode.ILLEGAL_PARAM); + } + if (req.getScriptContent() == null) { + req.setScriptContent(""); + } + return ValidateResult.pass(); + } + + @Override + public Response renderScriptTemplate(String username, ScriptTemplateRenderReq req) { + String scriptContent = req.getScriptContent(); + if (StringUtils.isNotEmpty(scriptContent)) { + scriptContent = Base64Util.decodeContentToStr(scriptContent); + } + + ScriptTemplateDTO scriptTemplate = new ScriptTemplateDTO(req.getScriptLanguage(), scriptContent); + + ScriptTemplateVariableRenderDTO variableRender = new ScriptTemplateVariableRenderDTO(req.getScopeType(), + req.getScopeId(), username); + if (req.getScopeType() == null && req.getScopeId() == null) { + variableRender.addDefaultValue(ScriptTemplateVariableEnum.BIZ_ID.getName(), + ScriptTemplateVariableEnum.BIZ_ID.getDemo()); + variableRender.addDefaultValue(ScriptTemplateVariableEnum.BIZ_NAME.getName(), + ScriptTemplateVariableEnum.BIZ_NAME.getDemo()); + } + customScriptTemplateService.renderScriptTemplate(variableRender, scriptTemplate); + return Response.buildSuccessResp(toScriptTemplateVO(scriptTemplate)); + } + + @Override + public Response> listScriptTemplateVariables(String username) { + List variableVOS = new ArrayList<>(); + for (ScriptTemplateVariableEnum variable : ScriptTemplateVariableEnum.values()) { + ScriptTemplateVariableVO variableVO = new ScriptTemplateVariableVO(); + variableVO.setName(variable.getName()); + variableVO.setDescription(i18nService.getI18n(variable.getDescI18nKey())); + variableVO.setDemo(variable.getDemo()); + variableVOS.add(variableVO); + } + return Response.buildSuccessResp(variableVOS); + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebDangerousRuleResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebDangerousRuleResourceImpl.java index 81c8de4270..134a2100a7 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebDangerousRuleResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebDangerousRuleResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,8 +24,14 @@ package com.tencent.bk.job.manage.api.web.impl; +import com.tencent.bk.audit.annotations.ActionAuditRecord; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.job.common.audit.constants.EventContentConstants; +import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.model.Response; import com.tencent.bk.job.manage.api.web.WebDangerousRuleResource; +import com.tencent.bk.job.manage.model.dto.globalsetting.DangerousRuleDTO; +import com.tencent.bk.job.manage.model.query.DangerousRuleQuery; import com.tencent.bk.job.manage.model.web.request.globalsetting.AddOrUpdateDangerousRuleReq; import com.tencent.bk.job.manage.model.web.request.globalsetting.MoveDangerousRuleReq; import com.tencent.bk.job.manage.model.web.vo.globalsetting.DangerousRuleVO; @@ -47,21 +53,51 @@ public WebDangerousRuleResourceImpl(DangerousRuleService dangerousRuleService) { } @Override - public Response> listDangerousRules(String username) { - return Response.buildSuccessResp(dangerousRuleService.listDangerousRules(username)); + @AuditEntry(actionId = ActionId.HIGH_RISK_DETECT_RULE) + @ActionAuditRecord( + actionId = ActionId.HIGH_RISK_DETECT_RULE, + content = EventContentConstants.VIEW_HIGH_RISK_DETECT_RULE + ) + public Response> listDangerousRules(String username, + String expression, + String description, + List scriptTypeList, + List action) { + DangerousRuleQuery query = DangerousRuleQuery.builder() + .expression(expression) + .description(description) + .scriptTypeList(scriptTypeList) + .action(action) + .build(); + return Response.buildSuccessResp(dangerousRuleService.listDangerousRules(query)); + } + + + @Override + @AuditEntry(actionId = ActionId.HIGH_RISK_DETECT_RULE) + public Response createDangerousRule(String username, AddOrUpdateDangerousRuleReq req) { + DangerousRuleDTO dangerousRule = dangerousRuleService.createDangerousRule(username, req); + return Response.buildSuccessResp(dangerousRule.toVO()); } @Override - public Response addOrUpdateDangerousRule(String username, AddOrUpdateDangerousRuleReq req) { - return Response.buildSuccessResp(dangerousRuleService.addOrUpdateDangerousRule(username, req)); + @AuditEntry(actionId = ActionId.HIGH_RISK_DETECT_RULE) + public Response updateDangerousRule(String username, + Long id, + AddOrUpdateDangerousRuleReq req) { + req.setId(id); + DangerousRuleDTO dangerousRule = dangerousRuleService.updateDangerousRule(username, req); + return Response.buildSuccessResp(dangerousRule.toVO()); } @Override + @AuditEntry(actionId = ActionId.HIGH_RISK_DETECT_RULE) public Response moveDangerousRule(String username, MoveDangerousRuleReq req) { return Response.buildSuccessResp(dangerousRuleService.moveDangerousRule(username, req)); } @Override + @AuditEntry(actionId = ActionId.HIGH_RISK_DETECT_RULE) public Response deleteDangerousRuleById(String username, Long id) { return Response.buildSuccessResp(dangerousRuleService.deleteDangerousRuleById(username, id)); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebEnvResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebEnvResourceImpl.java new file mode 100644 index 0000000000..a5c91fd739 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebEnvResourceImpl.java @@ -0,0 +1,72 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.api.web.impl; + +import com.tencent.bk.job.common.config.BkConfig; +import com.tencent.bk.job.common.esb.config.EsbProperties; +import com.tencent.bk.job.common.model.Response; +import com.tencent.bk.job.manage.api.web.WebEnvResource; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RestController; + +import java.util.HashMap; +import java.util.Map; + +@RestController +@Slf4j +public class WebEnvResourceImpl implements WebEnvResource { + + private final EsbProperties esbProperties; + private final BkConfig bkConfig; + + @Autowired + public WebEnvResourceImpl(EsbProperties esbProperties, BkConfig bkConfig) { + this.esbProperties = esbProperties; + this.bkConfig = bkConfig; + } + + @Override + public Response> getJobEnvProperties(String username) { + Map properties = new HashMap<>(); + properties.put("esb.url", standardUrl(esbProperties.getService().getPublicUrl())); + properties.put("bkDomain", bkConfig.getBkDomain()); + return Response.buildSuccessResp(properties); + } + + private String standardUrl(String url) { + if (StringUtils.isBlank(url)) { + return url; + } + // 移除最后的/ + if (url.endsWith("/")) { + return url.substring(0, url.length() - 1); + } else { + return url; + } + } + +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebFeatureToggleResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebFeatureToggleResourceImpl.java new file mode 100644 index 0000000000..be67024bd8 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebFeatureToggleResourceImpl.java @@ -0,0 +1,75 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.api.web.impl; + +import com.tencent.bk.job.common.model.Response; +import com.tencent.bk.job.common.util.toggle.feature.Feature; +import com.tencent.bk.job.common.util.toggle.feature.FeatureIdConstants; +import com.tencent.bk.job.common.util.toggle.feature.FeatureManager; +import com.tencent.bk.job.manage.api.web.WebFeatureToggleResource; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RestController; + +import java.util.HashMap; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +@RestController +@Slf4j +public class WebFeatureToggleResourceImpl implements WebFeatureToggleResource { + + private final FeatureManager featureManager; + + private static final Set frontendFeatures = new HashSet<>(); + + static { + // 增强功能-容器执行 + frontendFeatures.add(FeatureIdConstants.FEATURE_CONTAINER_EXECUTE); + } + + @Autowired + public WebFeatureToggleResourceImpl(FeatureManager featureManager) { + this.featureManager = featureManager; + } + + @Override + public Response> listFeatureToggle(String username) { + Map features = featureManager.listFeatures().stream() + .collect(Collectors.toMap(Feature::getId, feature -> feature, (oldValue, newValue) -> newValue)); + + Map featureToggles = new HashMap<>(); + + frontendFeatures.forEach(frontendFeatureId -> { + Feature feature = features.get(frontendFeatureId); + // 产品需求-无需考虑灰度策略情况 + featureToggles.put(frontendFeatureId, feature != null && feature.isEnabled()); + }); + + return Response.buildSuccessResp(featureToggles); + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebFileUploadResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebFileUploadResourceImpl.java index 5e80892a21..0dbd4117e3 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebFileUploadResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebFileUploadResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,6 +25,7 @@ package com.tencent.bk.job.manage.api.web.impl; import com.google.common.collect.Lists; +import com.tencent.bk.job.common.artifactory.config.ArtifactoryConfig; import com.tencent.bk.job.common.artifactory.model.dto.NodeDTO; import com.tencent.bk.job.common.artifactory.model.dto.TempUrlInfo; import com.tencent.bk.job.common.artifactory.sdk.ArtifactoryClient; @@ -33,24 +34,27 @@ import com.tencent.bk.job.common.exception.InternalException; import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.model.Response; +import com.tencent.bk.job.common.util.FilePathUtils; import com.tencent.bk.job.common.util.Utils; import com.tencent.bk.job.common.util.file.PathUtil; +import com.tencent.bk.job.manage.api.common.constants.globalsetting.RestrictModeEnum; import com.tencent.bk.job.manage.api.web.WebFileUploadResource; -import com.tencent.bk.job.manage.common.consts.globalsetting.RestrictModeEnum; -import com.tencent.bk.job.manage.config.ArtifactoryConfig; import com.tencent.bk.job.manage.config.LocalFileConfigForManage; import com.tencent.bk.job.manage.config.StorageSystemConfig; import com.tencent.bk.job.manage.model.web.request.GenUploadTargetReq; import com.tencent.bk.job.manage.model.web.vo.UploadLocalFileResultVO; import com.tencent.bk.job.manage.model.web.vo.UploadTargetVO; import com.tencent.bk.job.manage.model.web.vo.globalsetting.FileUploadSettingVO; -import com.tencent.bk.job.manage.service.GlobalSettingsService; +import com.tencent.bk.job.manage.service.globalsetting.GlobalSettingsService; import io.netty.util.internal.StringUtil; import lombok.extern.slf4j.Slf4j; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; +import org.slf4j.helpers.FormattingTuple; +import org.slf4j.helpers.MessageFormatter; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartFile; @@ -72,8 +76,9 @@ public class WebFileUploadResourceImpl implements WebFileUploadResource { @Autowired public WebFileUploadResourceImpl( StorageSystemConfig storageSystemConfig, - ArtifactoryConfig artifactoryConfig, LocalFileConfigForManage localFileConfigForManage, - ArtifactoryClient artifactoryClient, + ArtifactoryConfig artifactoryConfig, + LocalFileConfigForManage localFileConfigForManage, + @Qualifier("jobArtifactoryClient") ArtifactoryClient artifactoryClient, GlobalSettingsService globalSettingsService) { this.storageSystemConfig = storageSystemConfig; this.artifactoryConfig = artifactoryConfig; @@ -85,9 +90,9 @@ public WebFileUploadResourceImpl( /** * 将上传的文件保存至本机挂载的NFS * - * @param username - * @param uploadFiles - * @return + * @param username 用户名 + * @param uploadFiles 上传的文件 + * @return 文件上传结果 */ private List saveFileToLocal(String username, MultipartFile[] uploadFiles) { String uploadPath = storageSystemConfig.getJobStorageRootPath() + "/localupload/"; @@ -96,9 +101,13 @@ private List saveFileToLocal(String username, Multipart for (MultipartFile file : uploadFiles) { UploadLocalFileResultVO result = new UploadLocalFileResultVO(); try { + String originalFileName = file.getOriginalFilename(); + if (StringUtils.isBlank(originalFileName)) { + continue; + } String fileName = Utils.getUUID() + File.separatorChar + username + File.separatorChar + - file.getOriginalFilename(); + FilePathUtils.parseDirAndFileName(originalFileName).getRight(); String fullFileName = uploadPath.concat(fileName); File theFile = new File(fullFileName); @@ -155,9 +164,9 @@ private List saveFileToLocal(String username, Multipart /** * 将上传的文件保存至蓝鲸制品库 * - * @param username - * @param uploadFiles - * @return + * @param username 用户名 + * @param uploadFiles 上传的文件 + * @return 文件上传结果 */ private List saveFileToArtifactory(String username, MultipartFile[] uploadFiles) { List fileUploadResults = Lists.newArrayListWithCapacity(uploadFiles.length); @@ -184,15 +193,13 @@ private List saveFileToArtifactory(String username, Mul fileResultVO.setMd5(nodeDTO.getMd5()); fileResultVO.setStatus(0); } catch (IOException e) { - String errMsg = String.format( - "Fail to upload file %s to artifactory project {} repo {}", - filePath, - project, - repo + FormattingTuple errMsg = MessageFormatter.arrayFormat( + "Fail to upload file {} to artifactory project {} repo {}", + new String[]{filePath, project, repo} ); fileResultVO.setStatus(-1); - log.error(errMsg, e); - throw new InternalException(errMsg, ErrorCode.ARTIFACTORY_API_DATA_ERROR); + log.error(errMsg.getMessage(), e); + throw new InternalException(errMsg.getMessage(), ErrorCode.ARTIFACTORY_API_DATA_ERROR); } finally { fileUploadResults.add(fileResultVO); } @@ -297,7 +304,7 @@ public Response genUploadTarget(String username, GenUploadTarget return Response.buildSuccessResp( new UploadTargetVO( urlInfoList - .parallelStream() + .stream() .map(TempUrlInfo::getUrl) .collect(Collectors.toList()) ) diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebGlobalSettingsQueryResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebGlobalSettingsQueryResourceImpl.java index 7d6d6c4cb1..058900e6d2 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebGlobalSettingsQueryResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebGlobalSettingsQueryResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,8 +24,8 @@ package com.tencent.bk.job.manage.api.web.impl; +import com.tencent.bk.audit.annotations.AuditEntry; import com.tencent.bk.job.analysis.consts.AnalysisConsts; -import com.tencent.bk.job.common.constant.JobConstants; import com.tencent.bk.job.common.constant.ResourceScopeTypeEnum; import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.iam.model.AuthResult; @@ -38,11 +38,12 @@ import com.tencent.bk.job.manage.config.JobManageConfig; import com.tencent.bk.job.manage.model.web.vo.globalsetting.AccountNameRulesWithDefaultVO; import com.tencent.bk.job.manage.model.web.vo.globalsetting.NotifyChannelWithIconVO; -import com.tencent.bk.job.manage.model.web.vo.globalsetting.TitleFooterVO; +import com.tencent.bk.job.manage.model.web.vo.globalsetting.PlatformInfoVO; import com.tencent.bk.job.manage.service.ApplicationService; -import com.tencent.bk.job.manage.service.GlobalSettingsService; -import com.tencent.bk.job.manage.service.ScriptService; +import com.tencent.bk.job.manage.service.PublicScriptService; +import com.tencent.bk.job.manage.service.globalsetting.GlobalSettingsService; import lombok.extern.slf4j.Slf4j; +import org.jooq.tools.StringUtils; import org.springframework.beans.factory.DisposableBean; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; @@ -64,7 +65,7 @@ public class WebGlobalSettingsQueryResourceImpl implements WebGlobalSettingsQuer private final JobManageConfig jobManageConfig; private final NoResourceScopeAuthService noResourceScopeAuthService; private final AppAuthService appAuthService; - private final ScriptService scriptService; + private final PublicScriptService publicScriptService; private final ThreadPoolExecutor adminAuthExecutor; @Autowired @@ -73,23 +74,25 @@ public WebGlobalSettingsQueryResourceImpl(GlobalSettingsService globalSettingsSe JobManageConfig jobManageConfig, NoResourceScopeAuthService noResourceScopeAuthService, AppAuthService appAuthService, - ScriptService scriptService, + PublicScriptService publicScriptService, @Qualifier("adminAuthExecutor") ThreadPoolExecutor adminAuthExecutor) { this.globalSettingsService = globalSettingsService; this.applicationService = applicationService; this.jobManageConfig = jobManageConfig; this.noResourceScopeAuthService = noResourceScopeAuthService; this.appAuthService = appAuthService; - this.scriptService = scriptService; + this.publicScriptService = publicScriptService; this.adminAuthExecutor = adminAuthExecutor; } @Override + @AuditEntry(actionId = ActionId.GLOBAL_SETTINGS) public Response> listNotifyChannel(String username) { return Response.buildSuccessResp(globalSettingsService.listNotifyChannel(username)); } @Override + @AuditEntry(actionId = ActionId.GLOBAL_SETTINGS) public Response getAccountNameRules(String username) { return Response.buildSuccessResp(globalSettingsService.getAccountNameRules()); } @@ -133,7 +136,7 @@ public Response isAdmin(String username) { // 是否能管理某些公共脚本 List canManagePublicScriptIds = noResourceScopeAuthService.batchAuthManagePublicScript(username, - scriptService.listScriptIds(JobConstants.PUBLIC_APP_ID)); + publicScriptService.listScriptIds()); // 是否能够创建公共脚本 AuthResult authResult = noResourceScopeAuthService.authCreatePublicScript(username); flag.set(flag.get() || !canManagePublicScriptIds.isEmpty() || authResult.isPass()); @@ -209,6 +212,9 @@ public Response getCMDBServerUrl(String username) { @Override public Response getApplyBusinessUrl(String username, String scopeType, String scopeId) { + if (StringUtils.isBlank(scopeType) || StringUtils.isBlank(scopeId)) { + return Response.buildSuccessResp(appAuthService.getBusinessApplyUrl(null)); + } AppResourceScope appResourceScope = new AppResourceScope(scopeType, scopeId, null); ApplicationDTO applicationDTO = applicationService.getAppByScope(appResourceScope); if (applicationDTO != null) { @@ -230,8 +236,8 @@ public Response getCMDBAppIndexUrl(String username, String scopeType, St } @Override - public Response getTitleFooter() { - return Response.buildSuccessResp(globalSettingsService.getTitleFooter()); + public Response getRenderedPlatformInfo() { + return Response.buildSuccessResp(globalSettingsService.getRenderedPlatformInfo()); } @Override diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebGlobalSettingsResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebGlobalSettingsResourceImpl.java index 488f9e1546..e839c85dc5 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebGlobalSettingsResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebGlobalSettingsResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,12 +24,16 @@ package com.tencent.bk.job.manage.api.web.impl; +import com.tencent.bk.audit.annotations.ActionAuditRecord; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.audit.annotations.AuditRequestBody; +import com.tencent.bk.job.common.audit.constants.EventContentConstants; +import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.model.Response; import com.tencent.bk.job.manage.api.web.WebGlobalSettingsResource; import com.tencent.bk.job.manage.model.web.request.globalsetting.AccountNameRulesReq; import com.tencent.bk.job.manage.model.web.request.globalsetting.FileUploadSettingReq; import com.tencent.bk.job.manage.model.web.request.globalsetting.HistoryExpireReq; -import com.tencent.bk.job.manage.model.web.request.globalsetting.SetTitleFooterReq; import com.tencent.bk.job.manage.model.web.request.notify.ChannelTemplatePreviewReq; import com.tencent.bk.job.manage.model.web.request.notify.ChannelTemplateReq; import com.tencent.bk.job.manage.model.web.request.notify.NotifyBlackUsersReq; @@ -37,12 +41,12 @@ import com.tencent.bk.job.manage.model.web.vo.globalsetting.AccountNameRulesWithDefaultVO; import com.tencent.bk.job.manage.model.web.vo.globalsetting.FileUploadSettingVO; import com.tencent.bk.job.manage.model.web.vo.globalsetting.NotifyChannelWithIconVO; -import com.tencent.bk.job.manage.model.web.vo.globalsetting.TitleFooterWithDefaultVO; +import com.tencent.bk.job.manage.model.web.vo.globalsetting.PlatformInfoWithDefaultVO; import com.tencent.bk.job.manage.model.web.vo.notify.ChannelTemplateDetailWithDefaultVO; import com.tencent.bk.job.manage.model.web.vo.notify.ChannelTemplateStatusVO; import com.tencent.bk.job.manage.model.web.vo.notify.NotifyBlackUserInfoVO; import com.tencent.bk.job.manage.model.web.vo.notify.UserVO; -import com.tencent.bk.job.manage.service.GlobalSettingsService; +import com.tencent.bk.job.manage.service.globalsetting.GlobalSettingsService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.web.bind.annotation.RestController; @@ -61,26 +65,41 @@ public WebGlobalSettingsResourceImpl(GlobalSettingsService globalSettingsService } @Override + @AuditEntry(actionId = ActionId.GLOBAL_SETTINGS) + @ActionAuditRecord( + actionId = ActionId.GLOBAL_SETTINGS, + content = EventContentConstants.VIEW_GLOBAL_SETTINGS + ) public Response> listNotifyChannel(String username) { return Response.buildSuccessResp(globalSettingsService.listNotifyChannel(username)); } @Override - public Response setAvailableNotifyChannel(String username, SetAvailableNotifyChannelReq req) { + @AuditEntry(actionId = ActionId.GLOBAL_SETTINGS) + public Response setAvailableNotifyChannel(String username, + @AuditRequestBody SetAvailableNotifyChannelReq req) { return Response.buildSuccessResp(globalSettingsService.setAvailableNotifyChannel(username, req)); } @Override - public Response saveChannelTemplate(String username, ChannelTemplateReq req) { + @AuditEntry(actionId = ActionId.GLOBAL_SETTINGS) + public Response saveChannelTemplate(String username, + @AuditRequestBody ChannelTemplateReq req) { return Response.buildSuccessResp(globalSettingsService.saveChannelTemplate(username, req)); } @Override - public Response sendChannelTemplate(String username, ChannelTemplatePreviewReq req) { + public Response sendChannelTemplate(String username, + @AuditRequestBody ChannelTemplatePreviewReq req) { return Response.buildSuccessResp(globalSettingsService.sendChannelTemplate(username, req)); } @Override + @AuditEntry(actionId = ActionId.GLOBAL_SETTINGS) + @ActionAuditRecord( + actionId = ActionId.GLOBAL_SETTINGS, + content = EventContentConstants.VIEW_GLOBAL_SETTINGS + ) public Response getChannelTemplateDetail(String username, String channelCode, String messageTypeCode) { @@ -89,63 +108,101 @@ public Response getChannelTemplateDetail(Str } @Override + @AuditEntry(actionId = ActionId.GLOBAL_SETTINGS) + @ActionAuditRecord( + actionId = ActionId.GLOBAL_SETTINGS, + content = EventContentConstants.VIEW_GLOBAL_SETTINGS + ) public Response> listChannelTemplateStatus(String username) { return Response.buildSuccessResp(globalSettingsService.listChannelTemplateStatus(username)); } @Override + @AuditEntry(actionId = ActionId.GLOBAL_SETTINGS) + @ActionAuditRecord( + actionId = ActionId.GLOBAL_SETTINGS, + content = EventContentConstants.VIEW_GLOBAL_SETTINGS + ) public Response> listUsers(String username, String prefixStr, Long offset, Long limit) { return Response.buildSuccessResp(globalSettingsService.listUsers(username, prefixStr, offset, limit)); } @Override + @AuditEntry(actionId = ActionId.GLOBAL_SETTINGS) + @ActionAuditRecord( + actionId = ActionId.GLOBAL_SETTINGS, + content = EventContentConstants.VIEW_GLOBAL_SETTINGS + ) public Response> listNotifyBlackUsers(String username, Integer start, Integer pageSize) { return Response.buildSuccessResp(globalSettingsService.listNotifyBlackUsers(username, start, pageSize)); } @Override - public Response> saveNotifyBlackUsers(String username, NotifyBlackUsersReq req) { + @AuditEntry(actionId = ActionId.GLOBAL_SETTINGS) + public Response> saveNotifyBlackUsers(String username, + @AuditRequestBody NotifyBlackUsersReq req) { return Response.buildSuccessResp(globalSettingsService.saveNotifyBlackUsers(username, req)); } @Override + @AuditEntry(actionId = ActionId.GLOBAL_SETTINGS) + @ActionAuditRecord( + actionId = ActionId.GLOBAL_SETTINGS, + content = EventContentConstants.VIEW_GLOBAL_SETTINGS + ) public Response getHistoryExpireTime(String username) { return Response.buildSuccessResp(globalSettingsService.getHistoryExpireTime(username)); } @Override - public Response setHistoryExpireTime(String username, HistoryExpireReq req) { + @AuditEntry(actionId = ActionId.GLOBAL_SETTINGS) + public Response setHistoryExpireTime(String username, + @AuditRequestBody HistoryExpireReq req) { return Response.buildSuccessResp(globalSettingsService.setHistoryExpireTime(username, req)); } @Override + @AuditEntry(actionId = ActionId.GLOBAL_SETTINGS) + @ActionAuditRecord( + actionId = ActionId.GLOBAL_SETTINGS, + content = EventContentConstants.VIEW_GLOBAL_SETTINGS + ) public Response getAccountNameRules(String username) { return Response.buildSuccessResp(globalSettingsService.getAccountNameRules()); } @Override - public Response setAccountNameRules(String username, AccountNameRulesReq req) { + @AuditEntry(actionId = ActionId.GLOBAL_SETTINGS) + public Response setAccountNameRules(String username, + @AuditRequestBody AccountNameRulesReq req) { return Response.buildSuccessResp(globalSettingsService.setAccountNameRules(username, req)); } @Override - public Response saveFileUploadSettings(String username, FileUploadSettingReq req) { + @AuditEntry(actionId = ActionId.GLOBAL_SETTINGS) + public Response saveFileUploadSettings(String username, + @AuditRequestBody FileUploadSettingReq req) { return Response.buildSuccessResp(globalSettingsService.saveFileUploadSettings(username, req)); } @Override + @AuditEntry(actionId = ActionId.GLOBAL_SETTINGS) + @ActionAuditRecord( + actionId = ActionId.GLOBAL_SETTINGS, + content = EventContentConstants.VIEW_GLOBAL_SETTINGS + ) public Response getFileUploadSettings(String username) { return Response.buildSuccessResp(globalSettingsService.getFileUploadSettings()); } @Override - public Response setTitleFooter(String username, SetTitleFooterReq req) { - return Response.buildSuccessResp(globalSettingsService.setTitleFooter(username, req)); - } - - @Override - public Response getTitleFooterWithDefault(String username) { - return Response.buildSuccessResp(globalSettingsService.getTitleFooterWithDefault(username)); + @AuditEntry(actionId = ActionId.GLOBAL_SETTINGS) + @ActionAuditRecord( + actionId = ActionId.GLOBAL_SETTINGS, + content = EventContentConstants.VIEW_GLOBAL_SETTINGS + ) + public Response getPlatformInfoWithDefault(String username) { + return Response.buildSuccessResp(globalSettingsService.getPlatformInfoWithDefault(username)); } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebHostResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebHostResourceImpl.java new file mode 100644 index 0000000000..8361bf4bef --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebHostResourceImpl.java @@ -0,0 +1,555 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.api.web.impl; + +import com.tencent.bk.job.common.cc.model.InstanceTopologyDTO; +import com.tencent.bk.job.common.cc.sdk.BkNetClient; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.constant.ResourceScopeTypeEnum; +import com.tencent.bk.job.common.model.PageData; +import com.tencent.bk.job.common.model.Response; +import com.tencent.bk.job.common.model.dto.AppResourceScope; +import com.tencent.bk.job.common.model.dto.ApplicationDTO; +import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; +import com.tencent.bk.job.common.model.vo.CloudAreaInfoVO; +import com.tencent.bk.job.common.model.vo.DynamicGroupIdWithMeta; +import com.tencent.bk.job.common.model.vo.HostInfoVO; +import com.tencent.bk.job.common.model.vo.TargetNodeVO; +import com.tencent.bk.job.common.util.PageUtil; +import com.tencent.bk.job.common.util.ip.IpUtils; +import com.tencent.bk.job.manage.api.common.constants.whiteip.ActionScopeEnum; +import com.tencent.bk.job.manage.api.web.WebHostResource; +import com.tencent.bk.job.manage.model.dto.DynamicGroupDTO; +import com.tencent.bk.job.manage.model.web.request.AgentStatisticsReq; +import com.tencent.bk.job.manage.model.web.request.HostCheckReq; +import com.tencent.bk.job.manage.model.web.request.chooser.ListTopologyTreesReq; +import com.tencent.bk.job.manage.model.web.request.chooser.host.BizTopoNode; +import com.tencent.bk.job.manage.model.web.request.chooser.host.GetHostAgentStatisticsByDynamicGroupsReq; +import com.tencent.bk.job.manage.model.web.request.chooser.host.GetHostAgentStatisticsByNodesReq; +import com.tencent.bk.job.manage.model.web.request.chooser.host.HostDetailReq; +import com.tencent.bk.job.manage.model.web.request.chooser.host.HostIdWithMeta; +import com.tencent.bk.job.manage.model.web.request.chooser.host.ListDynamicGroupsReq; +import com.tencent.bk.job.manage.model.web.request.chooser.host.ListHostByBizTopologyNodesReq; +import com.tencent.bk.job.manage.model.web.request.chooser.host.PageListHostsByDynamicGroupReq; +import com.tencent.bk.job.manage.model.web.request.chooser.host.QueryNodesPathReq; +import com.tencent.bk.job.manage.model.web.vo.CcTopologyNodeVO; +import com.tencent.bk.job.manage.model.web.vo.DynamicGroupBasicVO; +import com.tencent.bk.job.manage.model.web.vo.common.AgentStatistics; +import com.tencent.bk.job.manage.model.web.vo.ipchooser.DynamicGroupHostStatisticsVO; +import com.tencent.bk.job.manage.model.web.vo.ipchooser.NodeHostStatisticsVO; +import com.tencent.bk.job.manage.service.ApplicationService; +import com.tencent.bk.job.manage.service.host.BizTopoHostService; +import com.tencent.bk.job.manage.service.host.HostDetailService; +import com.tencent.bk.job.manage.service.host.HostService; +import com.tencent.bk.job.manage.service.host.ScopeHostService; +import com.tencent.bk.job.manage.service.host.ScopeTopoHostService; +import com.tencent.bk.job.manage.service.host.WhiteIpAwareScopeHostService; +import com.tencent.bk.job.manage.service.host.impl.BizDynamicGroupHostService; +import com.tencent.bk.job.manage.service.impl.BizDynamicGroupService; +import com.tencent.bk.job.manage.service.impl.ScopeDynamicGroupService; +import com.tencent.bk.job.manage.service.impl.agent.AgentStatusService; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RestController; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +@RestController +@Slf4j +public class WebHostResourceImpl implements WebHostResource { + + private final ApplicationService applicationService; + private final HostService hostService; + private final ScopeTopoHostService scopeTopoHostService; + private final ScopeHostService scopeHostService; + private final WhiteIpAwareScopeHostService whiteIpAwareScopeHostService; + private final AgentStatusService agentStatusService; + private final HostDetailService hostDetailService; + private final BizTopoHostService bizTopoHostService; + private final BizDynamicGroupService bizDynamicGroupService; + private final BizDynamicGroupHostService bizDynamicGroupHostService; + private final ScopeDynamicGroupService scopeDynamicGroupHostService; + + @Autowired + public WebHostResourceImpl(ApplicationService applicationService, + HostService hostService, + ScopeTopoHostService scopeTopoHostService, + ScopeHostService scopeHostService, + WhiteIpAwareScopeHostService whiteIpAwareScopeHostService, + AgentStatusService agentStatusService, + HostDetailService hostDetailService, + BizTopoHostService bizTopoHostService, + BizDynamicGroupService bizDynamicGroupService, + BizDynamicGroupHostService bizDynamicGroupHostService, + ScopeDynamicGroupService scopeDynamicGroupHostService) { + this.applicationService = applicationService; + this.hostService = hostService; + this.scopeTopoHostService = scopeTopoHostService; + this.scopeHostService = scopeHostService; + this.whiteIpAwareScopeHostService = whiteIpAwareScopeHostService; + this.agentStatusService = agentStatusService; + this.hostDetailService = hostDetailService; + this.bizTopoHostService = bizTopoHostService; + this.bizDynamicGroupService = bizDynamicGroupService; + this.bizDynamicGroupHostService = bizDynamicGroupHostService; + this.scopeDynamicGroupHostService = scopeDynamicGroupHostService; + } + + // 标准接口1 + @Override + public Response> listTopologyHostCountTrees(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + ListTopologyTreesReq req) { + return Response.buildSuccessResp( + Collections.singletonList( + scopeTopoHostService.listAppTopologyHostCountTree(username, appResourceScope) + ) + ); + } + + // 标准接口3 + @Override + public Response> listHostByBizTopologyNodes(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + ListHostByBizTopologyNodesReq req) { + Pair pagePair = PageUtil.normalizePageParam(req.getStart(), req.getPageSize()); + PageData pageHostList = scopeHostService.searchHost( + appResourceScope, + req.getNodeList(), + req.getAlive(), + req.getSearchContent(), + req.getIpKeyList(), + req.getIpv6KeyList(), + req.getHostNameKeyList(), + req.getOsNameKeyList(), + pagePair.getLeft(), + pagePair.getRight() + ); + hostDetailService.fillDetailForApplicationHosts(pageHostList.getData()); + return Response.buildSuccessResp(PageUtil.transferPageData( + pageHostList, + ApplicationHostDTO::toVO + )); + } + + // 标准接口4 + @Override + public Response> listHostIdByBizTopologyNodes(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + ListHostByBizTopologyNodesReq req) { + // 参数标准化 + Pair pagePair = PageUtil.normalizePageParam(req.getStart(), req.getPageSize()); + PageData pageData = scopeHostService.listHostIdByBizTopologyNodes( + appResourceScope, + req.getNodeList(), + req.getSearchContent(), + req.getAlive(), + req.getIpKeyList(), + req.getIpv6KeyList(), + req.getHostNameKeyList(), + req.getOsNameKeyList(), + pagePair.getLeft(), + pagePair.getRight() + ); + PageData finalPageData = PageUtil.transferPageData( + pageData, + hostId -> new HostIdWithMeta(hostId, null) + ); + return Response.buildSuccessResp(finalPageData); + } + + // 标准接口2 + @Override + public Response>> queryNodePaths(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + QueryNodesPathReq req) { + return queryNodePaths(username, appResourceScope, req.getNodeList()); + } + + public Response>> queryNodePaths(String username, + AppResourceScope appResourceScope, + List targetNodeVOList) { + ApplicationDTO appDTO = applicationService.getAppByScope(appResourceScope); + if (appDTO.isBizSet()) { + return Response.buildSuccessResp(Collections.emptyList()); + } + List> pathList = hostService.queryBizNodePaths( + username, + appDTO.getBizIdIfBizApp(), + targetNodeVOList.stream().map(it -> { + InstanceTopologyDTO instanceTopologyDTO = new InstanceTopologyDTO(); + instanceTopologyDTO.setObjectId(it.getObjectId()); + instanceTopologyDTO.setInstanceId(it.getInstanceId()); + return instanceTopologyDTO; + }).collect(Collectors.toList())); + List> resultList = new ArrayList<>(); + for (List instanceTopologyDTOS : pathList) { + if (instanceTopologyDTOS == null) { + continue; + } + resultList.add(instanceTopologyDTOS.stream().map(it -> { + CcTopologyNodeVO ccTopologyNodeVO = new CcTopologyNodeVO(); + ccTopologyNodeVO.setObjectId(it.getObjectId()); + ccTopologyNodeVO.setObjectName(it.getObjectName()); + ccTopologyNodeVO.setInstanceId(it.getInstanceId()); + ccTopologyNodeVO.setInstanceName(it.getInstanceName()); + return ccTopologyNodeVO; + }).collect(Collectors.toList())); + } + return Response.buildSuccessResp(resultList); + } + + // 标准接口5 + @Override + public Response> getHostAgentStatisticsByNodes(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + GetHostAgentStatisticsByNodesReq req) { + if (ResourceScopeTypeEnum.BIZ_SET.getValue().equals(scopeType)) { + return Response.buildCommonFailResp(ErrorCode.NOT_SUPPORT_FEATURE_FOR_BIZ_SET); + } + List nodeList = req.getNodeList(); + if (CollectionUtils.isEmpty(nodeList)) { + return Response.buildSuccessResp(Collections.emptyList()); + } + List resultList = new ArrayList<>(); + // TODO:性能优化 + for (TargetNodeVO node : nodeList) { + List hostList = bizTopoHostService.listHostByNode( + Long.parseLong(scopeId), + BizTopoNode.fromTargetNodeVO(node) + ); + AgentStatistics agentStatistics = agentStatusService.calcAgentStatistics(hostList); + resultList.add(new NodeHostStatisticsVO(node, agentStatistics)); + } + return Response.buildSuccessResp(resultList); + } + + // 标准接口6 + @Override + public Response> listDynamicGroups(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + ListDynamicGroupsReq req) { + List idWithMetaList = req == null ? null : req.getDynamicGroupList(); + Set ids = idWithMetaList == null ? null : + idWithMetaList.stream().map(DynamicGroupIdWithMeta::getId).collect(Collectors.toSet()); + List dynamicGroupList = scopeDynamicGroupHostService.listOrderedDynamicGroup( + appResourceScope, + ids + ); + List dynamicGroupInfoList = dynamicGroupList.stream() + .map(DynamicGroupDTO::toBasicVO) + .collect(Collectors.toList()); + return Response.buildSuccessResp(dynamicGroupInfoList); + } + + // 标准接口7 + @Override + public Response> getHostAgentStatisticsByDynamicGroups( + String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + GetHostAgentStatisticsByDynamicGroupsReq req + ) { + if (ResourceScopeTypeEnum.BIZ_SET.getValue().equals(scopeType)) { + return Response.buildCommonFailResp(ErrorCode.NOT_SUPPORT_FEATURE_FOR_BIZ_SET); + } + List idWithMetaList = req.getDynamicGroupList(); + if (CollectionUtils.isEmpty(idWithMetaList)) { + return Response.buildSuccessResp(Collections.emptyList()); + } + List idList = idWithMetaList.stream().map(DynamicGroupIdWithMeta::getId).collect(Collectors.toList()); + List dynamicGroupList = + bizDynamicGroupService.listDynamicGroup(Long.parseLong(scopeId), idList); + List resultList = new ArrayList<>(); + for (DynamicGroupDTO dynamicGroupDTO : dynamicGroupList) { + DynamicGroupHostStatisticsVO statisticsVO = new DynamicGroupHostStatisticsVO(); + statisticsVO.setDynamicGroup(dynamicGroupDTO.toBasicVO()); + List hostList = bizDynamicGroupHostService.listHostByDynamicGroup( + appResourceScope, + dynamicGroupDTO.getId() + ); + AgentStatistics agentStatistics = agentStatusService.calcAgentStatistics(hostList); + statisticsVO.setAgentStatistics(agentStatistics); + resultList.add(statisticsVO); + } + return Response.buildSuccessResp(resultList); + } + + // 标准接口8 + @Override + public Response> pageListHostsByDynamicGroup(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + PageListHostsByDynamicGroupReq req) { + PageData pageData = bizDynamicGroupHostService.pageHostByDynamicGroups( + appResourceScope, + req.getId(), + req.getStart().intValue(), + req.getPageSize().intValue() + ); + // 填充Agent状态数据 + if (CollectionUtils.isNotEmpty(pageData.getData())) { + agentStatusService.fillRealTimeAgentStatus(pageData.getData()); + } + return Response.buildSuccessResp(PageUtil.transferPageData(pageData, ApplicationHostDTO::toVO)); + } + + // 标准接口9 + @Override + public Response> checkHosts(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + HostCheckReq req) { + // 查出所有主机 + List hostDTOList = findHosts(appResourceScope, req); + // 填充实时agent状态 + agentStatusService.fillRealTimeAgentStatus(hostDTOList); + List hostList = hostDTOList.stream() + .map(ApplicationHostDTO::toVO) + .collect(Collectors.toList()); + // 填充云区域名称 + hostList.forEach(hostInfoVO -> { + CloudAreaInfoVO cloudAreaInfo = hostInfoVO.getCloudArea(); + if (cloudAreaInfo != null + && cloudAreaInfo.getId() != null + && StringUtils.isBlank(cloudAreaInfo.getName())) { + cloudAreaInfo.setName(BkNetClient.getCloudAreaNameFromCache(cloudAreaInfo.getId())); + } + }); + return Response.buildSuccessResp(hostList); + } + + private List findHosts(AppResourceScope appResourceScope, HostCheckReq req) { + List hostDTOList = new ArrayList<>(); + // 根据主机ID解析主机 + findHostsByHostIds(appResourceScope, req.getActionScope(), req.getHostIdList(), hostDTOList); + // 根据Ipv4解析主机 + findHostsByIpv4s(appResourceScope, req.getActionScope(), req.getIpList(), hostDTOList); + // 根据Ipv6解析主机 + findHostsByIpv6s(appResourceScope, req.getActionScope(), req.getIpv6List(), hostDTOList); + // 根据关键字（主机名称）解析主机 + findHostsByKeys(appResourceScope, req.getActionScope(), req.getKeyList(), hostDTOList); + // 去重 + Set hostIdSet = new HashSet<>(); + Iterator iterator = hostDTOList.iterator(); + while (iterator.hasNext()) { + ApplicationHostDTO hostDTO = iterator.next(); + if (hostIdSet.contains(hostDTO.getHostId())) { + iterator.remove(); + } else { + hostIdSet.add(hostDTO.getHostId()); + } + } + return hostDTOList; + } + + private void findHostsByHostIds(AppResourceScope appResourceScope, + ActionScopeEnum actionScope, + List hostIdList, + List hostDTOList) { + if (CollectionUtils.isNotEmpty(hostIdList)) { + // 根据hostId查资源范围及白名单内的主机详情 + hostDTOList.addAll(whiteIpAwareScopeHostService.getScopeHostsIncludingWhiteIPByHostId( + appResourceScope, + actionScope, + hostIdList + )); + } + } + + private void findHostsByIpv4s(AppResourceScope appResourceScope, + ActionScopeEnum actionScope, + List ipOrCloudIpList, + List hostDTOList) { + if (CollectionUtils.isEmpty(ipOrCloudIpList)) { + return; + } + Pair, Set> pair = IpUtils.parseCleanIpv4AndCloudIpv4s(ipOrCloudIpList); + Set ipSet = pair.getLeft(); + Set cloudIpSet = pair.getRight(); + // 根据ip地址查资源范围及白名单内的主机详情 + if (CollectionUtils.isNotEmpty(ipSet)) { + hostDTOList.addAll(whiteIpAwareScopeHostService.getScopeHostsIncludingWhiteIPByIp( + appResourceScope, + actionScope, + ipSet + )); + } + if (CollectionUtils.isNotEmpty(cloudIpSet)) { + hostDTOList.addAll(whiteIpAwareScopeHostService.getScopeHostsIncludingWhiteIPByCloudIp( + appResourceScope, + actionScope, + cloudIpSet + )); + } + } + + private void findHostsByIpv6s(AppResourceScope appResourceScope, + ActionScopeEnum actionScope, + List ipv6OrCloudIpv6List, + List hostDTOList) { + if (CollectionUtils.isEmpty(ipv6OrCloudIpv6List)) { + return; + } + Pair, Set>> pair = IpUtils.parseFullIpv6AndCloudIpv6s(ipv6OrCloudIpv6List); + Set ipv6Set = pair.getLeft(); + Set> cloudIpv6Set = pair.getRight(); + Set allIpv6Set = new HashSet<>(ipv6Set); + Map ipv6CloudIdMap = new HashMap<>(); + for (Pair cloudIpv6 : cloudIpv6Set) { + allIpv6Set.add(cloudIpv6.getRight()); + ipv6CloudIdMap.put(cloudIpv6.getRight(), cloudIpv6.getLeft()); + } + List hostList = whiteIpAwareScopeHostService.getScopeHostsIncludingWhiteIPByIpv6( + appResourceScope, + actionScope, + allIpv6Set + ); + for (ApplicationHostDTO host : hostList) { + String ipv6 = host.getIpv6(); + Long cloudId = host.getCloudAreaId(); + // 未指定云区域ID的数据匹配所有ipv6符合的数据 + if (ipv6Set.contains(ipv6) || + // 指定了云区域ID的数据需要精确匹配 + (ipv6CloudIdMap.containsKey(ipv6) && cloudId.equals(ipv6CloudIdMap.get(ipv6)))) { + // 根据ipv6地址查资源范围及白名单内的主机详情 + hostDTOList.add(host); + } + } + } + + private void findHostsByKeys(AppResourceScope appResourceScope, + ActionScopeEnum actionScope, + List keyList, + List hostDTOList) { + if (CollectionUtils.isEmpty(keyList)) { + return; + } + // 根据关键字（主机名称）查资源范围及白名单内的主机详情 + hostDTOList.addAll(whiteIpAwareScopeHostService.getScopeHostsIncludingWhiteIPByKey( + appResourceScope, + actionScope, + keyList + )); + } + + // 标准接口10 + @Override + public Response> getHostDetails(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + HostDetailReq req) { + Collection hostIds = req.getHostList().stream() + .filter(hostIdWithMeta -> hostIdWithMeta.getHostId() != null) + .map(HostIdWithMeta::getHostId) + .collect(Collectors.toList()); + List hostList = hostDetailService.listHostDetails(appResourceScope, hostIds); + // 排序：Agent异常机器在前，Agent正常机器在后 + List hostInfoVOList = hostList.stream() + .filter(hostDTO -> !hostDTO.getGseAgentAlive()) + .map(ApplicationHostDTO::toVO) + .collect(Collectors.toList()); + hostInfoVOList.addAll(hostList.stream() + .filter(ApplicationHostDTO::getGseAgentAlive) + .map(ApplicationHostDTO::toVO) + .collect(Collectors.toList())); + return Response.buildSuccessResp(hostInfoVOList); + } + + @Override + public Response agentStatistics(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + AgentStatisticsReq agentStatisticsReq) { + List allHostList = new ArrayList<>(); + List hostList = agentStatisticsReq.getHostList(); + if (CollectionUtils.isNotEmpty(hostList)) { + List hostsById = scopeHostService.getScopeHostsByIds( + appResourceScope, + hostList.stream().map(HostIdWithMeta::getHostId).collect(Collectors.toList()) + ); + log.debug("hostsById={}", hostsById); + allHostList.addAll(hostsById); + } + List nodeList = agentStatisticsReq.getNodeList(); + if (CollectionUtils.isNotEmpty(nodeList)) { + long bizId = Long.parseLong(scopeId); + List hostsByNode = new ArrayList<>(); + for (BizTopoNode node : nodeList) { + hostsByNode.addAll(bizTopoHostService.listHostByNode(bizId, node)); + } + log.debug("hostsByNode={}", hostsByNode); + allHostList.addAll(hostsByNode); + } + List dynamicGroupIdList = agentStatisticsReq.getDynamicGroupIds(); + if (CollectionUtils.isNotEmpty(dynamicGroupIdList)) { + List hostsByDynamicGroup = new ArrayList<>(); + for (String id : dynamicGroupIdList) { + hostsByDynamicGroup.addAll(bizDynamicGroupHostService.listHostByDynamicGroup(appResourceScope, id)); + } + log.debug("hostsByDynamicGroup={}", hostsByDynamicGroup); + allHostList.addAll(hostsByDynamicGroup); + } + AgentStatistics agentStatistics = agentStatusService.calcAgentStatistics(allHostList); + return Response.buildSuccessResp(agentStatistics); + } + + @Override + public Response> listAllDynamicGroups(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId) { + return listDynamicGroups(username, appResourceScope, scopeType, scopeId, null); + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebIndexResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebIndexResourceImpl.java index 866b7cb251..395d5aeeb2 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebIndexResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebIndexResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -29,7 +29,7 @@ import com.tencent.bk.job.common.model.dto.AppResourceScope; import com.tencent.bk.job.common.model.vo.HostInfoVO; import com.tencent.bk.job.manage.api.web.WebIndexResource; -import com.tencent.bk.job.manage.model.web.vo.index.AgentStatistics; +import com.tencent.bk.job.manage.model.web.vo.common.AgentStatistics; import com.tencent.bk.job.manage.model.web.vo.index.GreetingVO; import com.tencent.bk.job.manage.model.web.vo.index.JobAndScriptStatistics; import com.tencent.bk.job.manage.model.web.vo.task.TaskTemplateVO; @@ -41,7 +41,7 @@ import java.util.List; -@RestController +@RestController("jobManageWebIndexResource") @Slf4j public class WebIndexResourceImpl implements WebIndexResource { diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebNoticeResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebNoticeResourceImpl.java new file mode 100644 index 0000000000..ba4ea66390 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebNoticeResourceImpl.java @@ -0,0 +1,73 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.api.web.impl; + +import com.tencent.bk.job.common.i18n.locale.BkConsts; +import com.tencent.bk.job.common.i18n.locale.LocaleUtils; +import com.tencent.bk.job.common.model.Response; +import com.tencent.bk.job.common.notice.IBkNoticeClient; +import com.tencent.bk.job.common.notice.config.BkNoticeProperties; +import com.tencent.bk.job.common.util.JobContextUtil; +import com.tencent.bk.job.manage.api.web.WebNoticeResource; +import com.tencent.bk.job.manage.model.web.vo.notice.AnnouncementVO; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.web.bind.annotation.RestController; + +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +@RestController +@Slf4j +public class WebNoticeResourceImpl implements WebNoticeResource { + + private final IBkNoticeClient bkNoticeClient; + private final BkNoticeProperties bkNoticeProperties; + + @Autowired + public WebNoticeResourceImpl(@Autowired(required = false) IBkNoticeClient bkNoticeClient, + BkNoticeProperties bkNoticeProperties) { + this.bkNoticeClient = bkNoticeClient; + this.bkNoticeProperties = bkNoticeProperties; + } + + @Override + public Response> getCurrentAnnouncements(String username, Integer offset, Integer limit) { + if (!bkNoticeProperties.isEnabled()) { + log.info("bkNotice not enabled, please check config value: bkNotice.enabled"); + return Response.buildSuccessResp(Collections.emptyList()); + } + String userLang = JobContextUtil.getUserLang(); + String bkLang = LocaleUtils.getBkLang(userLang); + if (bkLang == null) { + bkLang = BkConsts.HEADER_VALUE_LANG_EN; + } + List resultList = bkNoticeClient.getCurrentAnnouncements(bkLang, offset, limit).stream() + .map(AnnouncementVO::fromDTO) + .collect(Collectors.toList()); + return Response.buildSuccessResp(resultList); + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebNotifyResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebNotifyResourceImpl.java index 5ff638bb24..ad1ebd68f8 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebNotifyResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebNotifyResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,8 @@ package com.tencent.bk.job.manage.api.web.impl; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; import com.tencent.bk.job.common.iam.model.AuthResult; import com.tencent.bk.job.common.model.Response; @@ -77,6 +79,7 @@ public Response> listAppDefaultNotifyPolicies(String usern } @Override + @AuditEntry(actionId = ActionId.NOTIFICATION_SETTING) public Response saveAppDefaultNotifyPolicies(String username, AppResourceScope appResourceScope, String scopeType, diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebPermissionResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebPermissionResourceImpl.java index 3cca884769..2b36f0d47b 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebPermissionResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebPermissionResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -49,7 +49,7 @@ import org.apache.commons.lang3.StringUtils; import org.springframework.web.bind.annotation.RestController; -@RestController +@RestController("jobManageWebPermissionResourceImpl") @Slf4j public class WebPermissionResourceImpl implements WebPermissionResource { private final WebAuthService webAuthService; @@ -234,10 +234,20 @@ private Response checkJobTemplateOperationPermission( ); case "edit": templateId = Long.parseLong(resourceId); + AuthResult authEditResult = templateAuthService.authEditJobTemplate( + username, + appResourceScope, + templateId + ); + AuthResult authViewResult = templateAuthService.authViewJobTemplate( + username, + appResourceScope, + templateId + ); return Response.buildSuccessResp( webAuthService.toAuthResultVO( isReturnApplyUrl, - templateAuthService.authEditJobTemplate(username, appResourceScope, templateId) + authEditResult.mergeAuthResult(authViewResult) ) ); case "delete": @@ -507,7 +517,6 @@ public Response checkOperationPermission( } String resourceType = resourceAndAction[0]; String action = resourceAndAction[1]; - AppResourceScope appResourceScope = new AppResourceScope(scopeType, scopeId, null); boolean isReturnApplyUrl = returnPermissionDetail != null && returnPermissionDetail; switch (resourceType) { @@ -516,31 +525,38 @@ public Response checkOperationPermission( return Response.buildSuccessResp( webAuthService.toAuthResultVO( isReturnApplyUrl, - businessAuthService.authAccessBusiness(username, appResourceScope) + businessAuthService.authAccessBusiness(username, buildResourceScope(scopeType, scopeId)) ) ); } break; case "script": - return checkScriptOperationPermission(username, appResourceScope, action, resourceId, isReturnApplyUrl); + return checkScriptOperationPermission(username, buildResourceScope(scopeType, scopeId), + action, resourceId, isReturnApplyUrl); case "public_script": return checkPublicScriptOperationPermission(username, action, resourceId, isReturnApplyUrl); case "job_template": - return checkJobTemplateOperationPermission(username, appResourceScope, action, resourceId, - isReturnApplyUrl); + return checkJobTemplateOperationPermission(username, buildResourceScope(scopeType, scopeId), + action, resourceId, isReturnApplyUrl); case "job_plan": - return checkJobPlanOperationPermission(username, appResourceScope, action, resourceId, - isReturnApplyUrl); + return checkJobPlanOperationPermission(username, buildResourceScope(scopeType, scopeId), + action, resourceId, isReturnApplyUrl); case "account": - return checkAccountOperationPermission(username, appResourceScope, action, resourceId, - isReturnApplyUrl); + return checkAccountOperationPermission(username, buildResourceScope(scopeType, scopeId), + action, resourceId, isReturnApplyUrl); case "whitelist": return checkWhiteIPOperationPermission(username, action, isReturnApplyUrl); case "tag": - return checkTagOperationPermission(username, appResourceScope, action, resourceId, isReturnApplyUrl); + return checkTagOperationPermission(username, buildResourceScope(scopeType, scopeId), + action, resourceId, isReturnApplyUrl); case "ticket": - return checkTicketOperationPermission(username, appResourceScope, action, resourceId, isReturnApplyUrl); + return checkTicketOperationPermission(username, buildResourceScope(scopeType, scopeId), + action, resourceId, isReturnApplyUrl); } return Response.buildSuccessResp(AuthResultVO.fail()); } + + private AppResourceScope buildResourceScope(String scopeType, String scopeId) { + return new AppResourceScope(scopeType, scopeId, null); + } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebPublicScriptResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebPublicScriptResourceImpl.java index 311a227e7d..835b4384ce 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebPublicScriptResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebPublicScriptResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,14 +24,33 @@ package com.tencent.bk.job.manage.api.web.impl; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.audit.annotations.AuditRequestBody; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.constant.JobResourceTypeEnum; +import com.tencent.bk.job.common.exception.InvalidParamException; +import com.tencent.bk.job.common.exception.NotFoundException; +import com.tencent.bk.job.common.i18n.service.MessageI18nService; +import com.tencent.bk.job.common.iam.constant.ActionId; +import com.tencent.bk.job.common.iam.model.AuthResult; import com.tencent.bk.job.common.model.PageData; import com.tencent.bk.job.common.model.Response; +import com.tencent.bk.job.manage.api.common.ScriptDTOBuilder; +import com.tencent.bk.job.manage.api.common.constants.JobResourceStatusEnum; import com.tencent.bk.job.manage.api.web.WebPublicScriptResource; -import com.tencent.bk.job.manage.api.web.WebScriptResource; -import com.tencent.bk.job.manage.model.web.request.ScriptCreateUpdateReq; +import com.tencent.bk.job.manage.auth.NoResourceScopeAuthService; +import com.tencent.bk.job.manage.model.dto.ScriptDTO; +import com.tencent.bk.job.manage.model.dto.ScriptSyncTemplateStepDTO; +import com.tencent.bk.job.manage.model.dto.SyncScriptResultDTO; +import com.tencent.bk.job.manage.model.dto.TagDTO; +import com.tencent.bk.job.manage.model.dto.TemplateStepIDDTO; +import com.tencent.bk.job.manage.model.dto.converter.ScriptConverter; +import com.tencent.bk.job.manage.model.query.ScriptQuery; +import com.tencent.bk.job.manage.model.web.request.ScriptCreateReq; import com.tencent.bk.job.manage.model.web.request.ScriptInfoUpdateReq; import com.tencent.bk.job.manage.model.web.request.ScriptSyncReq; import com.tencent.bk.job.manage.model.web.request.ScriptTagBatchPatchReq; +import com.tencent.bk.job.manage.model.web.request.ScriptVersionCreateUpdateReq; import com.tencent.bk.job.manage.model.web.vo.BasicScriptVO; import com.tencent.bk.job.manage.model.web.vo.ScriptVO; import com.tencent.bk.job.manage.model.web.vo.TagCountVO; @@ -39,46 +58,89 @@ import com.tencent.bk.job.manage.model.web.vo.script.ScriptCiteInfoVO; import com.tencent.bk.job.manage.model.web.vo.script.ScriptRelatedTemplateStepVO; import com.tencent.bk.job.manage.model.web.vo.script.ScriptSyncResultVO; +import com.tencent.bk.job.manage.service.PublicScriptService; +import com.tencent.bk.job.manage.service.ScriptManager; +import com.tencent.bk.job.manage.service.TagService; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.web.bind.annotation.RestController; +import java.util.ArrayList; import java.util.List; +import java.util.stream.Collectors; +import static com.tencent.bk.job.common.constant.JobConstants.PUBLIC_APP_ID; + +/** + * 公共脚本Resource + */ @RestController @Slf4j -public class WebPublicScriptResourceImpl implements WebPublicScriptResource { - /** - * 脚本处理的通用服务，与普通脚本不同的是，公共脚本的业务ID=0 - * 鉴权在通用脚本逻辑中处理 - */ - private final WebScriptResource scriptResourceProxy; +public class WebPublicScriptResourceImpl extends BaseWebScriptResource implements WebPublicScriptResource { + private final PublicScriptService publicScriptService; + + private final NoResourceScopeAuthService noResourceScopeAuthService; @Autowired - public WebPublicScriptResourceImpl(@Qualifier("webScriptResourceImpl") WebScriptResource scriptResourceProxy) { - this.scriptResourceProxy = scriptResourceProxy; + public WebPublicScriptResourceImpl(PublicScriptService publicScriptService, + MessageI18nService i18nService, + ScriptDTOBuilder scriptDTOBuilder, + NoResourceScopeAuthService noResourceScopeAuthService, + TagService tagService, + ScriptManager scriptManager) { + super(i18nService, scriptDTOBuilder, tagService, scriptManager); + this.publicScriptService = publicScriptService; + this.noResourceScopeAuthService = noResourceScopeAuthService; } @Override - public Response getScriptVersionDetail(String username, Long scriptVersionId) { - return scriptResourceProxy.getScriptVersionDetail(username, null, null, null, scriptVersionId); + public Response getScriptVersionDetail(String username, + Long scriptVersionId) { + ScriptDTO script = publicScriptService.getScriptVersion(scriptVersionId); + + ScriptVO scriptVO = ScriptConverter.convertToScriptVO(script); + return Response.buildSuccessResp(scriptVO); } + @Override - public Response getScript(String username, String scriptId) { - return scriptResourceProxy.getScript(username, null, null, null, scriptId); + public Response getScript(String username, + String scriptId) { + ScriptDTO script = publicScriptService.getScript(scriptId); + + List scriptVersions = publicScriptService.listScriptVersion(scriptId); + if (CollectionUtils.isEmpty(scriptVersions)) { + throw new NotFoundException(ErrorCode.SCRIPT_NOT_EXIST); + } + + List scriptVersionVOS = convertToScriptVOList(scriptVersions); + ScriptVO scriptVO = ScriptConverter.convertToScriptVO(script); + scriptVO.setScriptVersions(scriptVersionVOS); + + return Response.buildSuccessResp(scriptVO); } @Override - public Response getScriptBasicInfo(String username, String scriptId) { - return scriptResourceProxy.getScriptBasicInfo(username, null, null, null, scriptId); + public Response getScriptBasicInfo(String username, + String scriptId) { + ScriptDTO script = publicScriptService.getScript(scriptId); + ScriptVO scriptVO = ScriptConverter.convertToScriptVO(script); + return Response.buildSuccessResp(scriptVO); } @Override - public Response getOnlineScriptVersionByScriptId(String username, String scriptId) { - return scriptResourceProxy.getOnlineScriptVersionByScriptId(username, null, null, - null, scriptId, true); + public Response getOnlineScriptVersionByScriptId(String username, + String scriptId) { + ScriptDTO onlineScriptVersion = + publicScriptService.getOnlineScriptVersionByScriptId(scriptId); + if (onlineScriptVersion == null) { + return Response.buildSuccessResp(null); + } + + ScriptVO onlineScriptVO = ScriptConverter.convertToScriptVO(onlineScriptVersion); + return Response.buildSuccessResp(onlineScriptVO); } @Override @@ -96,99 +158,313 @@ public Response> listPageScript(String username, Integer pageSize, String orderField, Integer order) { - return scriptResourceProxy.listPageScript(username, null, null, null, true, name, - type, tags, panelTag, panelType, creator, lastModifyUser, scriptId, content, - start, pageSize, orderField, order); + ScriptQuery scriptQuery = buildListPageScriptQuery(null, name, type, tags, panelTag, + panelType, creator, lastModifyUser, scriptId, content, start, pageSize, orderField, order); + + PageData pageData = publicScriptService.listPageScript(scriptQuery); + PageData resultPageData = pageVOs(pageData, start, pageSize); + + //设置脚本引用信息 + setScriptCiteCount(resultPageData.getData()); + + // 设置脚本的最新版本 + setOnlineScriptVersionInfo(resultPageData.getData()); + + // 设置权限 + processPermissionForList(username, resultPageData); + resultPageData.setExistAny(publicScriptService.isExistAnyPublicScript()); + + return Response.buildSuccessResp(resultPageData); } + private void processPermissionForList(String username, + PageData resultPageData) { + resultPageData.setCanCreate( + noResourceScopeAuthService.authCreatePublicScript(username).isPass() + ); + + resultPageData.getData() + .forEach(script -> script.setCanManage( + noResourceScopeAuthService.authManagePublicScript(username, script.getId()).isPass())); + resultPageData.getData().forEach(script -> script.setCanView(true)); + } + + @Override - public Response> listScriptBasicInfo(String username, List scriptIds) { - return scriptResourceProxy.listScriptBasicInfo(username, null, null, null, scriptIds); + @AuditEntry(actionId = ActionId.MANAGE_PUBLIC_SCRIPT_INSTANCE) + public Response updateScriptInfo(String username, + String scriptId, + @AuditRequestBody ScriptInfoUpdateReq request) { + String updateField = request.getUpdateField(); + boolean isUpdateDesc = "scriptDesc".equals(updateField); + boolean isUpdateName = "scriptName".equals(updateField); + boolean isUpdateTags = "scriptTags".equals(updateField); + + if (StringUtils.isBlank(updateField) || !(isUpdateDesc || isUpdateName || isUpdateTags)) { + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); + } + + ScriptDTO updateScript; + if (isUpdateDesc) { + updateScript = publicScriptService.updateScriptDesc(username, scriptId, request.getScriptDesc()); + } else if (isUpdateName) { + updateScript = updateScriptName(username, scriptId, request); + } else { + updateScript = updateScriptTags(username, scriptId, request); + } + return Response.buildSuccessResp(ScriptConverter.convertToScriptVO(updateScript)); + } + + private ScriptDTO updateScriptName(String operator, String scriptId, ScriptInfoUpdateReq scriptInfoUpdateReq) { + scriptInfoUpdateReq.validateScriptName(); + return publicScriptService.updateScriptName(operator, scriptId, scriptInfoUpdateReq.getScriptName()); + } + + private ScriptDTO updateScriptTags(String operator, String scriptId, ScriptInfoUpdateReq scriptInfoUpdateReq) { + List tags = extractTags(scriptInfoUpdateReq); + return publicScriptService.updateScriptTags(operator, scriptId, tags); } @Override - public Response> listScriptVersion(String username, String scriptId) { - return scriptResourceProxy.listScriptVersion(username, null, null, null, scriptId); + public Response> listScriptBasicInfo(String username, + List scriptIds) { + ScriptQuery scriptQuery = new ScriptQuery(); + scriptQuery.setAppId(PUBLIC_APP_ID); + scriptQuery.setIds(scriptIds); + scriptQuery.setPublicScript(true); + List scripts = publicScriptService.listScripts(scriptQuery); + if (CollectionUtils.isNotEmpty(scripts)) { + scripts = scripts.stream().filter(script -> script.getAppId().equals(PUBLIC_APP_ID)) + .collect(Collectors.toList()); + } + + List scriptVOS = scripts.stream().map(ScriptConverter::convertToScriptVO) + .collect(Collectors.toList()); + return Response.buildSuccessResp(scriptVOS); } @Override - public Response listAppScriptNames(String username, String scriptName) { - return scriptResourceProxy.listAppScriptNames(username, null, null, null, scriptName); + public Response> listScriptVersion(String username, + String scriptId) { + // 鉴权 + AuthResult manageAuthResult = noResourceScopeAuthService.authManagePublicScript(username, scriptId); + + List scripts = publicScriptService.listScriptVersion(scriptId); + List resultVOS = new ArrayList<>(); + if (scripts != null && !scripts.isEmpty()) { + for (ScriptDTO scriptDTO : scripts) { + ScriptVO scriptVO = ScriptConverter.convertToScriptVO(scriptDTO); + scriptVO.setCanView(true); + scriptVO.setCanManage(manageAuthResult.isPass()); + // 克隆需要管理权限 + scriptVO.setCanClone(true); + + // 是否支持同步操作 + if (scriptDTO.getStatus().equals(JobResourceStatusEnum.ONLINE.getValue())) { + List syncSteps = getSyncTemplateSteps(PUBLIC_APP_ID, scriptId, + scriptDTO.getScriptVersionId()); + scriptVO.setSyncEnabled(!syncSteps.isEmpty()); + } else { + scriptVO.setSyncEnabled(false); + } + resultVOS.add(scriptVO); + } + } + + // 统计被引用次数 + setScriptCiteCount(resultVOS); + + return Response.buildSuccessResp(resultVOS); } @Override - public Response> listScriptOnline(String username) { - return scriptResourceProxy.listScriptOnline(username, null, null, null); + @AuditEntry(actionId = ActionId.CREATE_PUBLIC_SCRIPT) + public Response saveScript(String username, + @AuditRequestBody ScriptCreateReq request) { + + ScriptDTO script = buildCreateScriptDTO(request, username); + ScriptDTO savedScript = publicScriptService.saveScript(username, script); + ScriptVO scriptVO = ScriptConverter.convertToScriptVO(savedScript); + return Response.buildSuccessResp(scriptVO); + } + + private ScriptDTO buildCreateScriptDTO(ScriptCreateReq request, + String username) { + ScriptDTO script = scriptDTOBuilder.buildFromScriptCreateReq(request); + script.setAppId(PUBLIC_APP_ID); + script.setPublicScript(true); + script.setCreator(username); + script.setLastModifyUser(username); + return script; + } + + @Override + @AuditEntry(actionId = ActionId.MANAGE_PUBLIC_SCRIPT_INSTANCE) + public Response saveScriptVersion(String username, + String scriptId, + @AuditRequestBody ScriptVersionCreateUpdateReq request) { + ScriptDTO script = buildCreateOrUpdateScriptVersion(true, request, scriptId, + null, username); + ScriptDTO savedScript = publicScriptService.saveScriptVersion(username, script); + + ScriptVO scriptVO = ScriptConverter.convertToScriptVO(savedScript); + return Response.buildSuccessResp(scriptVO); + } + + private ScriptDTO buildCreateOrUpdateScriptVersion(boolean isCreate, + ScriptVersionCreateUpdateReq request, + String scriptId, + Long scriptVersionId, + String username) { + ScriptDTO script = scriptDTOBuilder.buildFromScriptVersionCreateUpdateReq(request); + script.setId(scriptId); + script.setAppId(PUBLIC_APP_ID); + script.setPublicScript(true); + script.setLastModifyUser(username); + if (isCreate) { + script.setCreator(username); + } else { + script.setScriptVersionId(scriptVersionId); + } + return script; + } + + @Override + @AuditEntry(actionId = ActionId.MANAGE_PUBLIC_SCRIPT_INSTANCE) + public Response updateScriptVersion(String username, + String scriptId, + Long scriptVersionId, + @AuditRequestBody ScriptVersionCreateUpdateReq request) { + ScriptDTO script = buildCreateOrUpdateScriptVersion(false, request, scriptId, + scriptVersionId, username); + ScriptDTO savedScriptVersion = publicScriptService.updateScriptVersion(username, script); + + ScriptVO scriptVO = ScriptConverter.convertToScriptVO(savedScriptVersion); + return Response.buildSuccessResp(scriptVO); } @Override - public Response updateScriptInfo(String username, String scriptId, ScriptInfoUpdateReq scriptInfoUpdateReq) { - return scriptResourceProxy.updateScriptInfo(username, null, null, null, scriptId, - scriptInfoUpdateReq); + @AuditEntry(actionId = ActionId.MANAGE_PUBLIC_SCRIPT_INSTANCE) + public Response publishScriptVersion(String username, + String scriptId, + Long scriptVersionId) { + publicScriptService.publishScript(username, scriptId, scriptVersionId); + return Response.buildSuccessResp(null); } @Override - public Response saveScript(String username, ScriptCreateUpdateReq scriptCreateUpdateReq) { - return scriptResourceProxy.saveScript(username, null, null, null, scriptCreateUpdateReq); + @AuditEntry(actionId = ActionId.MANAGE_PUBLIC_SCRIPT_INSTANCE) + public Response disableScriptVersion(String username, + String scriptId, + Long scriptVersionId) { + publicScriptService.disableScript(username, scriptId, scriptVersionId); + return Response.buildSuccessResp(null); } @Override - public Response publishScriptVersion(String username, String scriptId, Long scriptVersionId) { - return scriptResourceProxy.publishScriptVersion(username, null, null, null, scriptId, - scriptVersionId); + @AuditEntry(actionId = ActionId.MANAGE_PUBLIC_SCRIPT_INSTANCE) + public Response deleteScriptByScriptId(String username, + String scriptId) { + publicScriptService.deleteScript(username, scriptId); + return Response.buildSuccessResp(null); } @Override - public Response disableScriptVersion(String username, String scriptId, Long scriptVersionId) { - return scriptResourceProxy.disableScriptVersion(username, null, null, null, scriptId, - scriptVersionId); + @AuditEntry(actionId = ActionId.MANAGE_PUBLIC_SCRIPT_INSTANCE) + public Response deleteScriptByScriptVersionId(String username, + Long scriptVersionId) { + publicScriptService.deleteScriptVersion(username, scriptVersionId); + return Response.buildSuccessResp(null); } @Override - public Response deleteScriptByScriptId(String username, String scriptId) { - return scriptResourceProxy.deleteScriptByScriptId(username, null, null, null, scriptId); + public Response> listPublicScriptNames(String username, + String scriptName) { + List scriptNames = publicScriptService.listScriptNames(scriptName); + return Response.buildSuccessResp(scriptNames); } @Override - public Response deleteScriptByScriptVersionId(String username, Long scriptVersionId) { - return scriptResourceProxy.deleteScriptByScriptVersionId(username, null, null, null, scriptVersionId); + public Response> listScriptOnline(String username) { + List scriptList = publicScriptService.listOnlineScript(); + List scriptVOList = convertToBasicScriptVOList(scriptList); + processScriptPermission(username, scriptVOList); + return Response.buildSuccessResp(scriptVOList); + } + + private void processScriptPermission(String username, + List scriptList) { + scriptList.forEach(script -> { + AuthResult managePermAuthResult = noResourceScopeAuthService.authManagePublicScript(username, + script.getId()); + script.setCanManage(managePermAuthResult.isPass()); + script.setCanView(true); + }); } @Override public Response> listScriptSyncTemplateSteps(String username, String scriptId, Long scriptVersionId) { - return scriptResourceProxy.listScriptSyncTemplateSteps(username, null, null, null, scriptId, - scriptVersionId); + List stepVOS = listScriptSyncTemplateSteps(PUBLIC_APP_ID, + scriptId, scriptVersionId); + return Response.buildSuccessResp(stepVOS); } @Override - public Response> syncScripts(String username, String scriptId, - Long scriptVersionId, ScriptSyncReq scriptSyncReq) { - return scriptResourceProxy.syncScripts(username, null, null, null, scriptId, scriptVersionId, - scriptSyncReq); + @AuditEntry(actionId = ActionId.EDIT_JOB_TEMPLATE) + public Response> syncScripts(String username, + String scriptId, + Long scriptVersionId, + @AuditRequestBody ScriptSyncReq scriptSyncReq) { + List templateStepIDs = new ArrayList<>(scriptSyncReq.getSteps().size()); + scriptSyncReq.getSteps().forEach(step -> + templateStepIDs.add(new TemplateStepIDDTO(step.getTemplateId(), step.getStepId()))); + + List syncResults = publicScriptService.syncScriptToTaskTemplate(username, scriptId, + scriptVersionId, templateStepIDs); + List syncResultVOS = convertToSyncResultVOs(syncResults, null); + return Response.buildSuccessResp(syncResultVOS); } @Override - public Response getPublicScriptCiteCount(String username, String scriptId, - Long scriptVersionId) { - return scriptResourceProxy.getScriptCiteCount(username, null, null, null, scriptId, scriptVersionId); + public Response getPublicScriptCiteInfo(String username, + String scriptId, + Long scriptVersionId) { + ScriptCiteInfoVO scriptCiteInfoVO = getScriptCiteInfoOfAllScript(scriptId, scriptVersionId); + return Response.buildSuccessResp(scriptCiteInfoVO); } @Override - public Response getPublicScriptCiteInfo(String username, String scriptId, - Long scriptVersionId) { - return scriptResourceProxy.getScriptCiteInfo(username, null, null, null, scriptId, scriptVersionId); + public Response getPublicScriptCiteCount(String username, + String scriptId, + Long scriptVersionId) { + ScriptCiteCountVO scriptCiteCountVO = getScriptCiteCountOfAllScript( + scriptId, scriptVersionId); + return Response.buildSuccessResp(scriptCiteCountVO); } @Override + @AuditEntry(actionId = ActionId.MANAGE_PUBLIC_SCRIPT_INSTANCE) public Response batchUpdatePublicScriptTags(String username, - ScriptTagBatchPatchReq tagBatchUpdateReq) { - return scriptResourceProxy.batchUpdateScriptTags(username, null, null, null, tagBatchUpdateReq); + @AuditRequestBody ScriptTagBatchPatchReq req) { + // 校验 + req.validate(); + + // 鉴权 + List scriptIdList = req.getIdList(); + noResourceScopeAuthService.batchAuthResultManagePublicScript(username, scriptIdList).denyIfNoPermission(); + + batchPatchResourceTags(JobResourceTypeEnum.PUBLIC_SCRIPT, scriptIdList, req.getAddTagIdList(), + req.getDeleteTagIdList()); + + return Response.buildSuccessResp(true); } + @Override public Response getTagPublicScriptCount(String username) { - return scriptResourceProxy.getTagScriptCount(username, null, null, null); + return Response.buildSuccessResp(publicScriptService.getTagScriptCount()); } + } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebPublicTagResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebPublicTagResourceImpl.java index e4b24d389b..59d8333c4c 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebPublicTagResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebPublicTagResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebScriptResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebScriptResourceImpl.java index 030e57922f..5202c3d2fc 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebScriptResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebScriptResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,8 @@ package com.tencent.bk.job.manage.api.web.impl; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.audit.annotations.AuditRequestBody; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.constant.JobResourceTypeEnum; import com.tencent.bk.job.common.exception.FailedPreconditionException; @@ -31,31 +33,19 @@ import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.exception.NotFoundException; import com.tencent.bk.job.common.i18n.service.MessageI18nService; -import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; +import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.iam.model.AuthResult; -import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; import com.tencent.bk.job.common.model.Response; -import com.tencent.bk.job.common.model.ValidateResult; import com.tencent.bk.job.common.model.dto.AppResourceScope; -import com.tencent.bk.job.common.util.ArrayUtil; import com.tencent.bk.job.common.util.Base64Util; -import com.tencent.bk.job.common.util.check.IlegalCharChecker; -import com.tencent.bk.job.common.util.check.MaxLengthChecker; -import com.tencent.bk.job.common.util.check.NotEmptyChecker; -import com.tencent.bk.job.common.util.check.StringCheckHelper; -import com.tencent.bk.job.common.util.check.TrimChecker; -import com.tencent.bk.job.common.util.check.WhiteCharChecker; -import com.tencent.bk.job.common.util.check.exception.StringCheckException; import com.tencent.bk.job.common.util.file.CharsetDetectHelper; import com.tencent.bk.job.common.util.file.EncodingUtils; import com.tencent.bk.job.manage.api.common.ScriptDTOBuilder; +import com.tencent.bk.job.manage.api.common.constants.JobResourceStatusEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; import com.tencent.bk.job.manage.api.web.WebScriptResource; -import com.tencent.bk.job.manage.auth.NoResourceScopeAuthService; import com.tencent.bk.job.manage.auth.ScriptAuthService; -import com.tencent.bk.job.manage.common.consts.JobResourceStatusEnum; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; -import com.tencent.bk.job.manage.model.dto.ResourceTagDTO; import com.tencent.bk.job.manage.model.dto.ScriptCheckResultItemDTO; import com.tencent.bk.job.manage.model.dto.ScriptDTO; import com.tencent.bk.job.manage.model.dto.ScriptSyncTemplateStepDTO; @@ -63,27 +53,23 @@ import com.tencent.bk.job.manage.model.dto.TagDTO; import com.tencent.bk.job.manage.model.dto.TemplateStepIDDTO; import com.tencent.bk.job.manage.model.dto.converter.ScriptConverter; -import com.tencent.bk.job.manage.model.dto.converter.ScriptRelatedTemplateStepConverter; -import com.tencent.bk.job.manage.model.dto.script.ScriptCitedTaskPlanDTO; -import com.tencent.bk.job.manage.model.dto.script.ScriptCitedTaskTemplateDTO; import com.tencent.bk.job.manage.model.query.ScriptQuery; import com.tencent.bk.job.manage.model.web.request.ScriptCheckReq; -import com.tencent.bk.job.manage.model.web.request.ScriptCreateUpdateReq; +import com.tencent.bk.job.manage.model.web.request.ScriptCreateReq; import com.tencent.bk.job.manage.model.web.request.ScriptInfoUpdateReq; import com.tencent.bk.job.manage.model.web.request.ScriptSyncReq; import com.tencent.bk.job.manage.model.web.request.ScriptTagBatchPatchReq; +import com.tencent.bk.job.manage.model.web.request.ScriptVersionCreateUpdateReq; import com.tencent.bk.job.manage.model.web.vo.BasicScriptVO; import com.tencent.bk.job.manage.model.web.vo.ScriptCheckResultItemVO; -import com.tencent.bk.job.manage.model.web.vo.ScriptCitedTaskPlanVO; -import com.tencent.bk.job.manage.model.web.vo.ScriptCitedTemplateVO; import com.tencent.bk.job.manage.model.web.vo.ScriptVO; import com.tencent.bk.job.manage.model.web.vo.TagCountVO; -import com.tencent.bk.job.manage.model.web.vo.TagVO; import com.tencent.bk.job.manage.model.web.vo.script.ScriptCiteCountVO; import com.tencent.bk.job.manage.model.web.vo.script.ScriptCiteInfoVO; import com.tencent.bk.job.manage.model.web.vo.script.ScriptRelatedTemplateStepVO; import com.tencent.bk.job.manage.model.web.vo.script.ScriptSyncResultVO; import com.tencent.bk.job.manage.service.ScriptCheckService; +import com.tencent.bk.job.manage.service.ScriptManager; import com.tencent.bk.job.manage.service.ScriptService; import com.tencent.bk.job.manage.service.TagService; import lombok.extern.slf4j.Slf4j; @@ -91,195 +77,100 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.validation.annotation.Validated; import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartFile; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; import java.util.List; -import java.util.Map; -import java.util.Objects; import java.util.stream.Collectors; -import static com.tencent.bk.job.common.constant.JobConstants.PUBLIC_APP_ID; - +/** + * 业务脚本Resource + */ @RestController @Slf4j -public class WebScriptResourceImpl implements WebScriptResource { +public class WebScriptResourceImpl extends BaseWebScriptResource implements WebScriptResource { private final ScriptService scriptService; - private final MessageI18nService i18nService; - private final ScriptCheckService scriptCheckService; - private final ScriptDTOBuilder scriptDTOBuilder; - private final ScriptAuthService scriptAuthService; - private final NoResourceScopeAuthService noResourceScopeAuthService; - - private final TagService tagService; - @Autowired public WebScriptResourceImpl(ScriptService scriptService, MessageI18nService i18nService, ScriptCheckService scriptCheckService, ScriptDTOBuilder scriptDTOBuilder, ScriptAuthService scriptAuthService, - NoResourceScopeAuthService noResourceScopeAuthService, - TagService tagService) { + TagService tagService, + ScriptManager scriptManager) { + super(i18nService, scriptDTOBuilder, tagService, scriptManager); this.scriptService = scriptService; - this.i18nService = i18nService; this.scriptCheckService = scriptCheckService; - this.scriptDTOBuilder = scriptDTOBuilder; this.scriptAuthService = scriptAuthService; - this.noResourceScopeAuthService = noResourceScopeAuthService; - this.tagService = tagService; } @Override + @AuditEntry(actionId = ActionId.VIEW_SCRIPT) public Response getScriptVersionDetail(String username, AppResourceScope appResourceScope, String scopeType, String scopeId, Long scriptVersionId) { - Long appId = appResourceScope == null ? PUBLIC_APP_ID : appResourceScope.getAppId(); - if (scriptVersionId == null || scriptVersionId <= 0) { - log.warn("Get script version by id, param scriptVersionId is empty"); - throw new InvalidParamException(ErrorCode.MISSING_PARAM); - } - ScriptDTO script = scriptService.getScriptVersion(username, appId, scriptVersionId); - if (script == null) { - log.warn("Get script version by id, appId={},id={}, the script is not exist", appId, - scriptVersionId); - throw new NotFoundException(ErrorCode.SCRIPT_NOT_EXIST, ArrayUtil.toArray(scriptVersionId)); - } - - // 鉴权 - AuthResult authResult = checkScriptViewPermission(username, appResourceScope, script); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } - + ScriptDTO script = scriptService.getScriptVersion(username, appResourceScope.getAppId(), scriptVersionId); ScriptVO scriptVO = ScriptConverter.convertToScriptVO(script); - - if (!checkPublicScriptVersionViewPermission(username, scriptVO)) { - return Response.buildSuccessResp(null); - } - - scriptVO.setTypeName(ScriptTypeEnum.getName(scriptVO.getType())); - // 给前端的脚本内容需要base64编码 - scriptVO.setContent(Base64Util.encodeContentToStr(script.getContent())); return Response.buildSuccessResp(scriptVO); } - private boolean checkPublicScriptVersionViewPermission(String username, ScriptVO script) { - if (script.getPublicScript() != null && !script.getPublicScript()) { - return true; - } - - AuthResult authResult = checkScriptManagePermission(username, null, script.getId()); - if (authResult.isPass()) { - return true; - } else { - // if user does not have public script management permission, only return online public script version list - return script.getStatus() == JobResourceStatusEnum.ONLINE.getValue(); - } - } - - private List excludeNotOnlinePublicScriptVersion(String username, - AppResourceScope appResourceScope, - String scriptId, - List scriptVersions) { - AuthResult authResult = checkScriptManagePermission(username, appResourceScope, scriptId); - if (authResult.isPass()) { - return scriptVersions; - } else { - // if user does not have public script management permission, only return online public script version list - return scriptVersions.stream().filter(scriptVersion -> - scriptVersion.getStatus() == JobResourceStatusEnum.ONLINE.getValue()) - .collect(Collectors.toList()); - } - } - @Override + @AuditEntry(actionId = ActionId.VIEW_SCRIPT) public Response getScript(String username, AppResourceScope appResourceScope, String scopeType, String scopeId, String scriptId) { - Long appId = appResourceScope == null ? PUBLIC_APP_ID : appResourceScope.getAppId(); - ScriptDTO script = scriptService.getScript(username, appId, scriptId); - if (script == null) { - throw new NotFoundException(ErrorCode.SCRIPT_NOT_EXIST); - } - - // 鉴权 - AuthResult authResult = checkScriptViewPermission(username, appResourceScope, script); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } - - List scriptVersions = scriptService.listScriptVersion(username, appId, scriptId); - if (scriptVersions == null || scriptVersions.isEmpty()) { - throw new NotFoundException(ErrorCode.SCRIPT_NOT_EXIST); - } + ScriptDTO script = scriptService.getScript(username, appResourceScope.getAppId(), scriptId); + ScriptVO scriptVO = ScriptConverter.convertToScriptVO(script); - List scriptVersionVOS = new ArrayList<>(); - for (ScriptDTO scriptVersion : scriptVersions) { - ScriptVO scriptVO = ScriptConverter.convertToScriptVO(scriptVersion); - JobResourceStatusEnum status = JobResourceStatusEnum.getJobResourceStatus(scriptVO.getStatus()); - scriptVO.setStatusDesc(i18nService.getI18n(status != null ? status.getStatusI18nKey() : null)); - scriptVO.setTypeName(ScriptTypeEnum.getName(scriptVO.getType())); - // 给前端的脚本内容需要base64编码 - scriptVO.setContent(Base64Util.encodeContentToStr(scriptVersion.getContent())); - scriptVersionVOS.add(scriptVO); + List scriptVersions = scriptService.listScriptVersion(appResourceScope.getAppId(), scriptId); + if (CollectionUtils.isNotEmpty(scriptVersions)) { + List scriptVersionVOS = scriptVersions.stream().map(ScriptConverter::convertToScriptVO) + .collect(Collectors.toList()); + scriptVO.setScriptVersions(scriptVersionVOS); } - ScriptVO scriptVO = ScriptConverter.convertToScriptVO(script); - scriptVO.setScriptVersions(scriptVersionVOS); return Response.buildSuccessResp(scriptVO); } @Override - public Response getScriptBasicInfo(String username, - AppResourceScope appResourceScope, - String scopeType, - String scopeId, - String scriptId) { - Long appId = appResourceScope == null ? PUBLIC_APP_ID : appResourceScope.getAppId(); - ScriptDTO script = scriptService.getScript(username, appId, scriptId); - if (script == null) { - throw new NotFoundException(ErrorCode.SCRIPT_NOT_EXIST); - } - ScriptVO scriptVO = ScriptConverter.convertToScriptVO(script); + @AuditEntry(actionId = ActionId.VIEW_SCRIPT) + public Response getScriptBasicInfo(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + String scriptId) { + ScriptDTO script = scriptService.getScript(username, appResourceScope.getAppId(), scriptId); + BasicScriptVO scriptVO = ScriptConverter.convertToBasicScriptVO(script); return Response.buildSuccessResp(scriptVO); } @Override + @AuditEntry(actionId = ActionId.VIEW_SCRIPT) public Response getOnlineScriptVersionByScriptId(String username, AppResourceScope appResourceScope, String scopeType, String scopeId, - String scriptId, - Boolean publicScript) { - long appId = appResourceScope == null ? PUBLIC_APP_ID : appResourceScope.getAppId(); + String scriptId) { ScriptDTO onlineScriptVersion = scriptService.getOnlineScriptVersionByScriptId(username, - appId, scriptId); + appResourceScope.getAppId(), scriptId); if (onlineScriptVersion == null) { return Response.buildSuccessResp(null); } - // 鉴权 - AuthResult authResult = checkScriptViewPermission(username, appResourceScope, onlineScriptVersion); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } - ScriptVO onlineScriptVO = ScriptConverter.convertToScriptVO(onlineScriptVersion); return Response.buildSuccessResp(onlineScriptVO); } @@ -289,7 +180,6 @@ public Response> listPageScript(String username, AppResourceScope appResourceScope, String scopeType, String scopeId, - Boolean publicScript, String name, Integer type, String tags, @@ -303,201 +193,58 @@ public Response> listPageScript(String username, Integer pageSize, String orderField, Integer order) { - ScriptQuery scriptQuery = new ScriptQuery(); - if (publicScript != null && publicScript) { - scriptQuery.setPublicScript(true); - scriptQuery.setAppId(PUBLIC_APP_ID); - } else { - scriptQuery.setPublicScript(false); - scriptQuery.setAppId(appResourceScope.getAppId()); - } - scriptQuery.setId(scriptId); - scriptQuery.setName(name); - scriptQuery.setType(type); - if (panelType != null && panelType == 2) { - scriptQuery.setUntaggedScript(true); - } else { - addTagCondition(scriptQuery, tags, panelTag); - } - scriptQuery.setContentKeyword(content); - - BaseSearchCondition baseSearchCondition = new BaseSearchCondition(); - baseSearchCondition.setLength(pageSize); - baseSearchCondition.setStart(start); - baseSearchCondition.setOrderField(orderField); - baseSearchCondition.setOrder(order); - baseSearchCondition.setCreator(creator); - baseSearchCondition.setLastModifyUser(lastModifyUser); - - PageData pageData = scriptService.listPageScript(scriptQuery, baseSearchCondition); - List resultScripts = new ArrayList<>(); - if (pageData == null) { - PageData resultPageData = new PageData<>(); - resultPageData.setStart(start); - resultPageData.setPageSize(pageSize); - resultPageData.setTotal(0L); - resultPageData.setData(resultScripts); - return Response.buildSuccessResp(resultPageData); - } + ScriptQuery scriptQuery = buildListPageScriptQuery(appResourceScope, name, type, tags, panelTag, panelType, + creator, lastModifyUser, scriptId, content, start, pageSize, orderField, order); - for (ScriptDTO scriptDTO : pageData.getData()) { - ScriptVO scriptVO = ScriptConverter.convertToScriptVO(scriptDTO); - resultScripts.add(scriptVO); - } - for (ScriptVO scriptVO : resultScripts) { - String resultScriptId = scriptVO.getId(); - Integer taskTemplateCiteCount = scriptService.getScriptTemplateCiteCount(username, scriptQuery.getAppId() - , resultScriptId, null); - scriptVO.setRelatedTaskTemplateNum(taskTemplateCiteCount); - Integer taskPlanCiteCount = scriptService.getScriptTaskPlanCiteCount(username, scriptQuery.getAppId(), - resultScriptId, null); - scriptVO.setRelatedTaskPlanNum(taskPlanCiteCount); - } + PageData pageData = scriptService.listPageScript(scriptQuery); + PageData resultPageData = pageVOs(pageData, start, pageSize); - // 脚本类型处理 - for (ScriptVO scriptVO : resultScripts) { - scriptVO.setTypeName(ScriptTypeEnum.getName(scriptVO.getType())); - } + //设置脚本引用信息 + setScriptCiteCount(resultPageData.getData()); // 设置脚本的最新版本 - setOnlineScriptVersionInfo(resultScripts); - - PageData resultPageData = new PageData<>(); - resultPageData.setStart(pageData.getStart()); - resultPageData.setPageSize(pageData.getPageSize()); - resultPageData.setTotal(pageData.getTotal()); - resultPageData.setData(resultScripts); + setOnlineScriptVersionInfo(resultPageData.getData()); + // 设置权限 processPermissionForList(username, appResourceScope, resultPageData); - processAnyScriptExistFlag(appResourceScope, publicScript, resultPageData); + resultPageData.setExistAny(scriptService.isExistAnyAppScript(appResourceScope.getAppId())); return Response.buildSuccessResp(resultPageData); } - private void addTagCondition(ScriptQuery query, String tags, Long panelTagId) { - if (StringUtils.isNotBlank(tags)) { - query.setTagIds(Arrays.stream(tags.split(",")).map(tagIdStr -> { - try { - return Long.parseLong(tagIdStr); - } catch (NumberFormatException e) { - return null; - } - }).filter(Objects::nonNull).collect(Collectors.toList())); - } - if (panelTagId != null && panelTagId > 0) { - // Frontend need additional param to tell where the tag from - TagDTO tagInfo = new TagDTO(); - tagInfo.setId(panelTagId); - if (CollectionUtils.isEmpty(query.getTagIds())) { - query.setTagIds(Collections.singletonList(panelTagId)); - } else { - query.getTagIds().add(panelTagId); - } - } - } - - private void processAnyScriptExistFlag(AppResourceScope appResourceScope, Boolean publicScript, - PageData resultPageData) { - if (publicScript != null && publicScript) { - resultPageData.setExistAny(scriptService.isExistAnyPublicScript()); - } else { - resultPageData.setExistAny(scriptService.isExistAnyAppScript(appResourceScope.getAppId())); - } - } - private void processPermissionForList(String username, AppResourceScope appResourceScope, PageData resultPageData) { - boolean isQueryPublicScript = (appResourceScope == null); - if (isQueryPublicScript) { - resultPageData.setCanCreate( - noResourceScopeAuthService.authCreatePublicScript(username).isPass() - ); - } else { - resultPageData.setCanCreate( - scriptAuthService.authCreateScript(username, appResourceScope).isPass()); - } + resultPageData.setCanCreate( + scriptAuthService.authCreateScript(username, appResourceScope).isPass()); List scriptIdList = new ArrayList<>(); + resultPageData.getData().forEach(script -> scriptIdList.add(script.getId())); + + List allowedManageScriptIdList = + scriptAuthService.batchAuthManageScript(username, appResourceScope, scriptIdList); + List allowedViewScriptIdList = + scriptAuthService.batchAuthViewScript(username, appResourceScope, scriptIdList); + resultPageData.getData() + .forEach(script -> script.setCanManage(allowedManageScriptIdList.contains(script.getId()))); resultPageData.getData().forEach(script -> { - scriptIdList.add(script.getId()); - }); - if (isQueryPublicScript) { - resultPageData.getData() - .forEach(script -> script.setCanManage( - noResourceScopeAuthService.authManagePublicScript(username, script.getId()).isPass())); - resultPageData.getData().forEach(script -> script.setCanView(true)); - } else { - List allowedManageScriptIdList = - scriptAuthService.batchAuthManageScript(username, appResourceScope, scriptIdList); - List allowedViewScriptIdList = - scriptAuthService.batchAuthViewScript(username, appResourceScope, scriptIdList); - resultPageData.getData() - .forEach(script -> script.setCanManage(allowedManageScriptIdList.contains(script.getId()))); - resultPageData.getData().forEach(script -> { - boolean canView = allowedViewScriptIdList.contains(script.getId()); - script.setCanView(canView); - if (!canView) { - script.setContent("******"); - } - }); - } - } - - private void setOnlineScriptVersionInfo(List scripts) { - if (scripts != null && !scripts.isEmpty()) { - List scriptIdList = new ArrayList<>(); - for (ScriptVO script : scripts) { - scriptIdList.add(script.getId()); + boolean canView = allowedViewScriptIdList.contains(script.getId()); + script.setCanView(canView); + if (!canView) { + script.setContent("******"); } - Map onlineScriptMap = scriptService.batchGetOnlineScriptVersionByScriptIds(scriptIdList); - - for (ScriptVO scriptVO : scripts) { - ScriptDTO onlineScriptVersion = onlineScriptMap.get(scriptVO.getId()); - if (onlineScriptVersion != null) { - scriptVO.setScriptVersionId(onlineScriptVersion.getScriptVersionId()); - scriptVO.setVersion(onlineScriptVersion.getVersion()); - } - } - } - } - - private void updateScriptName(String username, Long appId, String scriptId, - ScriptInfoUpdateReq scriptInfoUpdateReq) { - try { - StringCheckHelper stringCheckHelper = new StringCheckHelper(new TrimChecker(), - new NotEmptyChecker(), new IlegalCharChecker(), new MaxLengthChecker(60)); - scriptInfoUpdateReq - .setScriptName(stringCheckHelper.checkAndGetResult(scriptInfoUpdateReq.getScriptName())); - } catch (StringCheckException e) { - log.warn("scriptName is invalid:", e); - throw new InvalidParamException(ErrorCode.SCRIPT_NAME_INVALID); - } - scriptService.updateScriptName(appId, username, scriptId, scriptInfoUpdateReq.getScriptName()); - } - - private void updateScriptTags(String username, Long appId, String scriptId, - ScriptInfoUpdateReq scriptInfoUpdateReq) { - List tags = new ArrayList<>(); - if (scriptInfoUpdateReq.getScriptTags() != null && !scriptInfoUpdateReq.getScriptTags().isEmpty()) { - for (TagVO tagVO : scriptInfoUpdateReq.getScriptTags()) { - TagDTO tagDTO = new TagDTO(); - tagDTO.setId(tagVO.getId()); - tagDTO.setName(tagVO.getName()); - tags.add(tagDTO); - } - } - scriptService.updateScriptTags(appId, username, scriptId, tags); + }); } @Override - public Response updateScriptInfo(String username, - AppResourceScope appResourceScope, - String scopeType, - String scopeId, - String scriptId, - ScriptInfoUpdateReq scriptInfoUpdateReq) { - Long appId = appResourceScope == null ? PUBLIC_APP_ID : appResourceScope.getAppId(); - String updateField = scriptInfoUpdateReq.getUpdateField(); + @AuditEntry(actionId = ActionId.MANAGE_SCRIPT) + public Response updateScriptInfo(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + String scriptId, + @AuditRequestBody ScriptInfoUpdateReq request) { + long appId = appResourceScope.getAppId(); + String updateField = request.getUpdateField(); boolean isUpdateDesc = "scriptDesc".equals(updateField); boolean isUpdateName = "scriptName".equals(updateField); boolean isUpdateTags = "scriptTags".equals(updateField); @@ -506,21 +253,28 @@ public Response updateScriptInfo(String username, throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } - // 鉴权 - AuthResult authResult = checkScriptManagePermission(username, appResourceScope, scriptId); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } - + ScriptDTO updateScript; if (isUpdateDesc) { - scriptService.updateScriptDesc(appId, username, scriptId, - scriptInfoUpdateReq.getScriptDesc()); + updateScript = scriptService.updateScriptDesc(appId, username, scriptId, + request.getScriptDesc()); } else if (isUpdateName) { - updateScriptName(username, appId, scriptId, scriptInfoUpdateReq); + updateScript = updateScriptName(username, appId, scriptId, request); } else { - updateScriptTags(username, appId, scriptId, scriptInfoUpdateReq); + updateScript = updateScriptTags(username, appId, scriptId, request); } - return Response.buildSuccessResp(null); + return Response.buildSuccessResp(ScriptConverter.convertToScriptVO(updateScript)); + } + + private ScriptDTO updateScriptName(String username, Long appId, String scriptId, + ScriptInfoUpdateReq scriptInfoUpdateReq) { + scriptInfoUpdateReq.validateScriptName(); + return scriptService.updateScriptName(appId, username, scriptId, scriptInfoUpdateReq.getScriptName()); + } + + private ScriptDTO updateScriptTags(String username, Long appId, String scriptId, + ScriptInfoUpdateReq scriptInfoUpdateReq) { + List tags = extractTags(scriptInfoUpdateReq); + return scriptService.updateScriptTags(appId, username, scriptId, tags); } @Override @@ -529,11 +283,11 @@ public Response> listScriptBasicInfo(String username, String scopeType, String scopeId, List scriptIds) { - Long appId = appResourceScope == null ? PUBLIC_APP_ID : appResourceScope.getAppId(); + Long appId = appResourceScope.getAppId(); ScriptQuery scriptQuery = new ScriptQuery(); scriptQuery.setAppId(appId); scriptQuery.setIds(scriptIds); - scriptQuery.setPublicScript(appResourceScope == null); + scriptQuery.setPublicScript(false); List scripts = scriptService.listScripts(scriptQuery); if (CollectionUtils.isNotEmpty(scripts)) { scripts = scripts.stream().filter(script -> script.getAppId().equals(appId)) @@ -551,43 +305,30 @@ public Response> listScriptVersion(String username, String scopeType, String scopeId, String scriptId) { - Long appId = appResourceScope == null ? PUBLIC_APP_ID : appResourceScope.getAppId(); - // 鉴权 - AuthResult viewAuthResult = checkScriptViewPermission(username, appResourceScope, scriptId); - if (!viewAuthResult.isPass()) { - throw new PermissionDeniedException(viewAuthResult); + long appId = appResourceScope.getAppId(); + ScriptDTO script = scriptService.getScriptByScriptId(scriptId); + if (script == null) { + throw new NotFoundException(ErrorCode.SCRIPT_NOT_EXIST); } - AuthResult manageAuthResult = checkScriptManagePermission(username, appResourceScope, scriptId); - List scripts = scriptService.listScriptVersion(username, appId, scriptId); + // 鉴权 + AuthResult viewAuthResult = scriptAuthService.authViewScript(username, appResourceScope, scriptId, null); + AuthResult manageAuthResult = scriptAuthService.authManageScript(username, appResourceScope, scriptId, null); + + List scripts = scriptService.listScriptVersion(appId, scriptId); List resultVOS = new ArrayList<>(); if (scripts != null && !scripts.isEmpty()) { - for (ScriptDTO scriptDTO : scripts) { ScriptVO scriptVO = ScriptConverter.convertToScriptVO(scriptDTO); - JobResourceStatusEnum status = JobResourceStatusEnum.getJobResourceStatus(scriptVO.getStatus()); - scriptVO.setStatusDesc(i18nService.getI18n(status != null ? status.getStatusI18nKey() : null)); - // 给前端的脚本内容需要base64编码 - scriptVO.setContent(Base64Util.encodeContentToStr(scriptDTO.getContent())); - scriptVO.setTypeName(ScriptTypeEnum.getName(scriptVO.getType())); - scriptVO.setCanView(viewAuthResult.isPass()); scriptVO.setCanManage(manageAuthResult.isPass()); // 克隆需要管理权限 scriptVO.setCanClone(manageAuthResult.isPass()); - // 统计被引用次数 - Integer taskTemplateCiteCount = scriptService.getScriptTemplateCiteCount(username, - appId, scriptDTO.getId(), scriptDTO.getScriptVersionId()); - scriptVO.setRelatedTaskTemplateNum(taskTemplateCiteCount); - Integer taskPlanCiteCount = scriptService.getScriptTaskPlanCiteCount(username, - appId, scriptDTO.getId(), scriptDTO.getScriptVersionId()); - scriptVO.setRelatedTaskPlanNum(taskPlanCiteCount); - // 是否支持同步操作 if (scriptDTO.getStatus().equals(JobResourceStatusEnum.ONLINE.getValue())) { - List syncSteps = getSyncTemplateSteps(username, - appId, scriptId, scriptDTO.getScriptVersionId()); + List syncSteps = getSyncTemplateSteps(appId, scriptId, + scriptDTO.getScriptVersionId()); scriptVO.setSyncEnabled(!syncSteps.isEmpty()); } else { scriptVO.setSyncEnabled(false); @@ -597,163 +338,149 @@ public Response> listScriptVersion(String username, } } - resultVOS = excludeNotOnlinePublicScriptVersion(username, appResourceScope, scriptId, resultVOS); + // 统计被引用次数 + setScriptCiteCount(resultVOS); + return Response.buildSuccessResp(resultVOS); } @Override + @AuditEntry(actionId = ActionId.CREATE_SCRIPT) public Response saveScript(String username, AppResourceScope appResourceScope, String scopeType, String scopeId, - ScriptCreateUpdateReq scriptCreateUpdateReq) { - Long appId = appResourceScope == null ? PUBLIC_APP_ID : appResourceScope.getAppId(); - log.info("Save script,operator={},appId={},script={}", username, appId, scriptCreateUpdateReq.toString()); + @AuditRequestBody @Validated ScriptCreateReq request) { - AuthResult authResult = checkSaveScript(username, appResourceScope, scriptCreateUpdateReq); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } + ScriptDTO script = buildCreateScriptDTO(request, appResourceScope, username); + ScriptDTO savedScript = scriptService.createScript(username, script); - try { - StringCheckHelper scriptNameCheckHelper = new StringCheckHelper(new TrimChecker(), new NotEmptyChecker(), - new IlegalCharChecker(), new MaxLengthChecker(60)); - scriptCreateUpdateReq.setName(scriptNameCheckHelper.checkAndGetResult(scriptCreateUpdateReq.getName())); - } catch (StringCheckException e) { - log.warn("Script name [{}] is invalid", scriptCreateUpdateReq.getName()); - throw new InvalidParamException(ErrorCode.SCRIPT_NAME_INVALID); - } - try { - StringCheckHelper scriptVersionCheckHelper = new StringCheckHelper(new TrimChecker(), new NotEmptyChecker(), - new WhiteCharChecker("A-Za-z0-9_\\-#@\\."), new MaxLengthChecker(60)); - scriptCreateUpdateReq - .setVersion(scriptVersionCheckHelper.checkAndGetResult(scriptCreateUpdateReq.getVersion())); - } catch (StringCheckException e) { - log.warn("Script version [{}] is invalid", scriptCreateUpdateReq.getVersion()); - throw new InvalidParamException(ErrorCode.SCRIPT_VERSION_ILLEGAL); - } - ScriptDTO script = scriptDTOBuilder.buildFromCreateUpdateReq(scriptCreateUpdateReq); - script.setAppId(appId); - script.setPublicScript(appResourceScope == null); + ScriptVO scriptVO = ScriptConverter.convertToScriptVO(savedScript); + return Response.buildSuccessResp(scriptVO); + } + + + private ScriptDTO buildCreateScriptDTO(ScriptCreateReq request, + AppResourceScope appResourceScope, + String username) { + ScriptDTO script = scriptDTOBuilder.buildFromScriptCreateReq(request); + script.setAppId(appResourceScope.getAppId()); + script.setPublicScript(false); script.setCreator(username); script.setLastModifyUser(username); - ScriptDTO savedScript = scriptService.saveScript(username, appId, script); - // 只在新建脚本时新建关联权限，编辑/复制并新建版本时不动 - if (StringUtils.isBlank(scriptCreateUpdateReq.getId())) { - if (script.isPublicScript()) { - // 公共脚本 - noResourceScopeAuthService.registerPublicScript(savedScript.getId(), script.getName(), username); - } else { - // 业务脚本 - scriptAuthService.registerScript(savedScript.getId(), script.getName(), username); - } - } - ScriptVO scriptVO = new ScriptVO(); - scriptVO.setScriptVersionId(savedScript.getScriptVersionId()); - scriptVO.setId(savedScript.getId()); + return script; + } + + @Override + @AuditEntry(actionId = ActionId.MANAGE_SCRIPT) + public Response saveScriptVersion(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + String scriptId, + @AuditRequestBody ScriptVersionCreateUpdateReq request) { + + ScriptDTO script = buildCreateOrUpdateScriptVersion(true, request, scriptId, null, + appResourceScope, username); + ScriptDTO savedScript = scriptService.createScriptVersion(username, script); + + ScriptVO scriptVO = ScriptConverter.convertToScriptVO(savedScript); return Response.buildSuccessResp(scriptVO); } - private AuthResult checkSaveScript(String username, AppResourceScope appResourceScope, - ScriptCreateUpdateReq scriptCreateUpdateReq) { - Long scriptVersionId = scriptCreateUpdateReq.getScriptVersionId(); - boolean isCreateNew = scriptVersionId == null || scriptVersionId < 0; - // 创建脚本版本鉴管理权限 - if (!StringUtils.isBlank(scriptCreateUpdateReq.getId())) { - isCreateNew = false; + private ScriptDTO buildCreateOrUpdateScriptVersion(boolean isCreate, + ScriptVersionCreateUpdateReq request, + String scriptId, + Long scriptVersionId, + AppResourceScope appResourceScope, + String username) { + ScriptDTO script = scriptDTOBuilder.buildFromScriptVersionCreateUpdateReq(request); + script.setId(scriptId); + script.setAppId(appResourceScope.getAppId()); + script.setPublicScript(false); + script.setLastModifyUser(username); + if (isCreate) { + script.setCreator(username); + } else { + script.setScriptVersionId(scriptVersionId); } - return isCreateNew ? checkScriptCreatePermission(username, appResourceScope) - : checkScriptManagePermission(username, appResourceScope, scriptCreateUpdateReq.getId()); + return script; } @Override + @AuditEntry(actionId = ActionId.MANAGE_SCRIPT) + public Response updateScriptVersion(String username, + AppResourceScope appResourceScope, + String scopeType, String scopeId, + String scriptId, + Long scriptVersionId, + @AuditRequestBody ScriptVersionCreateUpdateReq request) { + + ScriptDTO script = buildCreateOrUpdateScriptVersion(false, request, scriptId, scriptVersionId, + appResourceScope, username); + ScriptDTO savedScriptVersion = scriptService.updateScriptVersion(username, script); + + ScriptVO scriptVO = ScriptConverter.convertToScriptVO(savedScriptVersion); + return Response.buildSuccessResp(scriptVO); + } + + @Override + @AuditEntry(actionId = ActionId.MANAGE_SCRIPT) public Response publishScriptVersion(String username, AppResourceScope appResourceScope, String scopeType, String scopeId, String scriptId, Long scriptVersionId) { - Long appId = appResourceScope == null ? PUBLIC_APP_ID : appResourceScope.getAppId(); - log.info("Publish script version, scope={}, scriptId={}, scriptVersionId={}, username={}", appResourceScope, - scriptId, scriptVersionId, username); - - AuthResult authResult = checkScriptManagePermission(username, appResourceScope, scriptId); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } - - scriptService.publishScript(appId, username, scriptId, scriptVersionId); + scriptService.publishScript(appResourceScope.getAppId(), username, scriptId, scriptVersionId); return Response.buildSuccessResp(null); } @Override + @AuditEntry(actionId = ActionId.MANAGE_SCRIPT) public Response disableScriptVersion(String username, AppResourceScope appResourceScope, String scopeType, String scopeId, String scriptId, Long scriptVersionId) { - log.info("Disable script version, scope={}, scriptId={}, scriptVersionId={}, username={}", appResourceScope, - scriptId, scriptVersionId, username); - Long appId = appResourceScope == null ? PUBLIC_APP_ID : appResourceScope.getAppId(); - - AuthResult authResult = checkScriptManagePermission(username, appResourceScope, scriptId); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } - - scriptService.disableScript(appId, username, scriptId, scriptVersionId); + scriptService.disableScript(appResourceScope.getAppId(), username, scriptId, scriptVersionId); return Response.buildSuccessResp(null); } @Override + @AuditEntry(actionId = ActionId.MANAGE_SCRIPT) public Response deleteScriptByScriptId(String username, AppResourceScope appResourceScope, String scopeType, String scopeId, String scriptId) { - log.info("Delete script[{}], operator={}, scope={}", scriptId, username, appResourceScope); - Long appId = appResourceScope == null ? PUBLIC_APP_ID : appResourceScope.getAppId(); - - AuthResult authResult = checkScriptManagePermission(username, appResourceScope, scriptId); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } - - scriptService.deleteScript(username, appId, scriptId); + scriptService.deleteScript(username, appResourceScope.getAppId(), scriptId); return Response.buildSuccessResp(null); } @Override + @AuditEntry(actionId = ActionId.MANAGE_SCRIPT) public Response deleteScriptByScriptVersionId(String username, AppResourceScope appResourceScope, String scopeType, String scopeId, Long scriptVersionId) { log.info("Delete scriptVersion[{}], operator={}, scope={}", scriptVersionId, username, appResourceScope); - Long appId = appResourceScope == null ? PUBLIC_APP_ID : appResourceScope.getAppId(); - ScriptDTO script = scriptService.getScriptVersion(username, appId, scriptVersionId); - if (script == null) { - return Response.buildCommonFailResp(ErrorCode.SCRIPT_NOT_EXIST); - } - - AuthResult authResult = checkScriptManagePermission(username, appResourceScope, script.getId()); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } + long appId = appResourceScope.getAppId(); + ScriptDTO script = scriptService.getScriptVersion(appId, scriptVersionId); scriptService.deleteScriptVersion(username, appId, scriptVersionId); return Response.buildSuccessResp(null); } @Override - public Response listAppScriptNames(String username, - AppResourceScope appResourceScope, - String scopeType, - String scopeId, - String scriptName) { - Long appId = appResourceScope == null ? PUBLIC_APP_ID : appResourceScope.getAppId(); - List scriptNames = scriptService.listScriptNames(appId, scriptName); + public Response> listAppScriptNames(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + String scriptName) { + List scriptNames = scriptService.listScriptNames(appResourceScope.getAppId(), scriptName); return Response.buildSuccessResp(scriptNames); } @@ -762,18 +489,8 @@ public Response> listScriptOnline(String username, AppResourceScope appResourceScope, String scopeType, String scopeId) { - long queryAppId = appResourceScope == null ? PUBLIC_APP_ID : appResourceScope.getAppId(); - - List scriptList = scriptService.listOnlineScriptForApp(username, queryAppId); - if (scriptList == null || scriptList.isEmpty()) { - return Response.buildSuccessResp(Collections.emptyList()); - } - - List scriptVOList = new ArrayList<>(); - for (ScriptDTO script : scriptList) { - BasicScriptVO basicScriptVO = ScriptConverter.convertToBasicScriptVO(script); - scriptVOList.add(basicScriptVO); - } + List scriptList = scriptService.listOnlineScript(username, appResourceScope.getAppId()); + List scriptVOList = convertToBasicScriptVOList(scriptList); processScriptPermission(username, appResourceScope, scriptVOList); return Response.buildSuccessResp(scriptVOList); } @@ -782,27 +499,15 @@ private void processScriptPermission(String username, AppResourceScope appResour List scriptList) { List scriptIdList = new ArrayList<>(); scriptList.forEach(script -> scriptIdList.add(script.getId())); - boolean isPublicScript = appResourceScope == null; - if (isPublicScript) { - scriptList.forEach(script -> { - // TODO:batchAuth - AuthResult managePermAuthResult = noResourceScopeAuthService.authManagePublicScript(username, - script.getId()); - script.setCanManage(managePermAuthResult.isPass()); - script.setCanView(true); + List allowedManageScriptIdList = + scriptAuthService.batchAuthManageScript(username, appResourceScope, scriptIdList); + List allowedViewScriptIdList = + scriptAuthService.batchAuthViewScript(username, appResourceScope, scriptIdList); + scriptList + .forEach(script -> { + script.setCanManage(allowedManageScriptIdList.contains(script.getId())); + script.setCanView(allowedViewScriptIdList.contains(script.getId())); }); - } else { - - List allowedManageScriptIdList = - scriptAuthService.batchAuthManageScript(username, appResourceScope, scriptIdList); - List allowedViewScriptIdList = - scriptAuthService.batchAuthViewScript(username, appResourceScope, scriptIdList); - scriptList - .forEach(script -> { - script.setCanManage(allowedManageScriptIdList.contains(script.getId())); - script.setCanView(allowedViewScriptIdList.contains(script.getId())); - }); - } } @Override @@ -814,7 +519,7 @@ public Response> checkScript(String username, Scri String content = new String(Base64.decodeBase64(scriptCheckReq.getContent()), StandardCharsets.UTF_8); List checkResultItems = - scriptCheckService.check(ScriptTypeEnum.valueOf(scriptCheckReq.getScriptType()), content); + scriptCheckService.check(ScriptTypeEnum.valOf(scriptCheckReq.getScriptType()), content); List checkResultItemVOS = new ArrayList<>(); if (checkResultItems != null) { @@ -822,7 +527,8 @@ public Response> checkScript(String username, Scri ScriptCheckResultItemVO checkResultVO = new ScriptCheckResultItemVO(); checkResultVO.setCode(checkResultItem.getCheckItemCode()); checkResultVO.setLevel(checkResultItem.getLevel().getValue()); - checkResultVO.setAction(checkResultItem.getAction() == null ? null : checkResultItem.getAction().getValue()); + checkResultVO.setAction(checkResultItem.getAction() == null ? null : + checkResultItem.getAction().getValue()); checkResultVO.setLine(checkResultItem.getLine()); checkResultVO.setLineContent(checkResultItem.getLineContent()); checkResultVO.setMatchContent(checkResultItem.getMatchContent()); @@ -894,43 +600,6 @@ private String fileToString(MultipartFile is) { return fileContent; } - private AuthResult checkScriptViewPermission(String username, AppResourceScope appResourceScope, String scriptId) { - boolean isPublicScript = (appResourceScope == null); - if (isPublicScript) { - // 公共脚本默认公开，无需查看权限 - return AuthResult.pass(); - } - return scriptAuthService.authViewScript(username, appResourceScope, scriptId, null); - } - - private AuthResult checkScriptViewPermission(String username, AppResourceScope appResourceScope, ScriptDTO script) { - if (script.isPublicScript()) { - // 公共脚本默认公开，无需查看权限 - return AuthResult.pass(); - } - return scriptAuthService.authViewScript(username, appResourceScope, script.getId(), null); - } - - - private AuthResult checkScriptManagePermission(String username, AppResourceScope appResourceScope, - String scriptId) { - boolean isPublicScript = appResourceScope == null; - if (isPublicScript) { - return noResourceScopeAuthService.authManagePublicScript(username, scriptId); - } else { - return scriptAuthService.authManageScript(username, appResourceScope, scriptId, null); - } - } - - private AuthResult checkScriptCreatePermission(String username, AppResourceScope appResourceScope) { - boolean isPublicScript = appResourceScope == null; - if (isPublicScript) { - return noResourceScopeAuthService.authCreatePublicScript(username); - } else { - return scriptAuthService.authCreateScript(username, appResourceScope); - } - } - @Override public Response> listScriptSyncTemplateSteps(String username, AppResourceScope appResourceScope, @@ -938,44 +607,13 @@ public Response> listScriptSyncTemplateSteps(S String scopeId, String scriptId, Long scriptVersionId) { - Long appId = appResourceScope == null ? PUBLIC_APP_ID : appResourceScope.getAppId(); - List steps = getSyncTemplateSteps(username, appId, scriptId, scriptVersionId); - if (CollectionUtils.isEmpty(steps)) { - return Response.buildSuccessResp(Collections.emptyList()); - } - - List stepVOS = - steps.stream() - .map(ScriptRelatedTemplateStepConverter::convertToScriptRelatedTemplateStepVO) - .collect(Collectors.toList()); - stepVOS.forEach(stepVO -> { - JobResourceStatusEnum scriptStatus = JobResourceStatusEnum.getJobResourceStatus(stepVO.getScriptStatus()); - if (scriptStatus != null) { - stepVO.setScriptStatusDesc(i18nService.getI18n(scriptStatus.getStatusI18nKey())); - } - stepVO.setCanEdit(true); - }); + List stepVOS = listScriptSyncTemplateSteps(appResourceScope.getAppId(), + scriptId, scriptVersionId); return Response.buildSuccessResp(stepVOS); } - private List getSyncTemplateSteps(String username, - Long appId, - String scriptId, - Long scriptVersionId) { - List steps = scriptService.listScriptSyncTemplateSteps(username, appId, scriptId); - if (CollectionUtils.isEmpty(steps)) { - return Collections.emptyList(); - } - // 过滤掉已经是最新的模板步骤 - steps = - steps.stream().filter(step -> - !scriptVersionId.equals(step.getScriptVersionId())) - .collect(Collectors.toList()); - return steps; - } - - @Override + @AuditEntry(actionId = ActionId.EDIT_JOB_TEMPLATE) public Response> syncScripts(String username, AppResourceScope appResourceScope, String scopeType, @@ -983,77 +621,17 @@ public Response> syncScripts(String username, String scriptId, Long scriptVersionId, ScriptSyncReq scriptSyncReq) { - Long appId = appResourceScope == null ? PUBLIC_APP_ID : appResourceScope.getAppId(); + long appId = appResourceScope.getAppId(); List templateStepIDs = new ArrayList<>(scriptSyncReq.getSteps().size()); - scriptSyncReq.getSteps().forEach(step -> { - templateStepIDs.add(new TemplateStepIDDTO(step.getTemplateId(), step.getStepId())); - }); + scriptSyncReq.getSteps().forEach(step -> + templateStepIDs.add(new TemplateStepIDDTO(step.getTemplateId(), step.getStepId()))); List syncResults = scriptService.syncScriptToTaskTemplate(username, appId, scriptId, scriptVersionId, templateStepIDs); - List syncResultVOS = new ArrayList<>(syncResults.size()); - for (SyncScriptResultDTO syncResult : syncResults) { - ScriptSyncResultVO syncVO = new ScriptSyncResultVO(); - ScriptSyncTemplateStepDTO syncStep = syncResult.getTemplateStep(); - if (syncStep.getAppId() != null && !syncStep.getAppId().equals(PUBLIC_APP_ID)) { - syncVO.setScopeType(scopeType); - syncVO.setScopeId(scopeId); - } - syncVO.setScriptId(syncStep.getScriptId()); - syncVO.setScriptVersionId(syncStep.getScriptVersionId()); - syncVO.setScriptVersion(syncStep.getScriptVersion()); - syncVO.setScriptName(syncStep.getScriptName()); - syncVO.setScriptStatus(syncStep.getScriptStatus()); - JobResourceStatusEnum scriptStatus = JobResourceStatusEnum.getJobResourceStatus(syncStep.getScriptStatus()); - if (scriptStatus != null) { - syncVO.setScriptStatusDesc(i18nService.getI18n(scriptStatus.getStatusI18nKey())); - } - syncVO.setStepId(syncStep.getStepId()); - syncVO.setTemplateId(syncStep.getTemplateId()); - syncVO.setStepName(syncStep.getStepName()); - syncVO.setTemplateName(syncStep.getTemplateName()); - if (syncResult.isSuccess()) { - syncVO.setSyncStatus(ScriptSyncResultVO.SYNC_SUCCESS); - } else { - syncVO.setSyncStatus(ScriptSyncResultVO.SYNC_FAIL); - syncVO.setFailMsg(i18nService.getI18n(String.valueOf(syncResult.getErrorCode()))); - } - syncResultVOS.add(syncVO); - } + List syncResultVOS = convertToSyncResultVOs(syncResults, appResourceScope); return Response.buildSuccessResp(syncResultVOS); } - private ScriptCiteCountVO getScriptCiteCountOfAllScript(String username, - Long appId, - String scriptId, - Long scriptVersionId) { - Integer templateCiteCount = scriptService.getScriptTemplateCiteCount(username, appId, scriptId, - scriptVersionId); - Integer taskPlanCiteCount = scriptService.getScriptTaskPlanCiteCount(username, appId, scriptId, - scriptVersionId); - return new ScriptCiteCountVO(templateCiteCount, taskPlanCiteCount); - } - - private ScriptCiteInfoVO getScriptCiteInfoOfAllScript(String username, - Long appId, - String scriptId, - Long scriptVersionId) { - List citedTemplateList = scriptService.getScriptCitedTemplates(username, appId, - scriptId, scriptVersionId); - if (citedTemplateList == null) { - citedTemplateList = Collections.emptyList(); - } - List citedTemplateVOList = - citedTemplateList.parallelStream().map(ScriptCitedTaskTemplateDTO::toVO).collect(Collectors.toList()); - List citedTaskPlanList = scriptService.getScriptCitedTaskPlans(username, appId, - scriptId, scriptVersionId); - if (citedTaskPlanList == null) { - citedTaskPlanList = Collections.emptyList(); - } - List citedTaskPlanVOList = - citedTaskPlanList.parallelStream().map(ScriptCitedTaskPlanDTO::toVO).collect(Collectors.toList()); - return new ScriptCiteInfoVO(citedTemplateVOList, citedTaskPlanVOList); - } @Override public Response getScriptCiteInfo(String username, @@ -1062,9 +640,7 @@ public Response getScriptCiteInfo(String username, String scopeId, String scriptId, Long scriptVersionId) { - Long appId = appResourceScope == null ? PUBLIC_APP_ID : appResourceScope.getAppId(); - ScriptCiteInfoVO scriptCiteInfoVO = getScriptCiteInfoOfAllScript(username, appId, - scriptId, scriptVersionId); + ScriptCiteInfoVO scriptCiteInfoVO = getScriptCiteInfoOfAllScript(scriptId, scriptVersionId); return Response.buildSuccessResp(scriptCiteInfoVO); } @@ -1075,77 +651,38 @@ public Response getScriptCiteCount(String username, String scopeId, String scriptId, Long scriptVersionId) { - Long appId = appResourceScope == null ? PUBLIC_APP_ID : appResourceScope.getAppId(); - ScriptCiteCountVO scriptCiteCountVO = getScriptCiteCountOfAllScript(username, appId, - scriptId, scriptVersionId); + ScriptCiteCountVO scriptCiteCountVO = getScriptCiteCountOfAllScript(scriptId, scriptVersionId); return Response.buildSuccessResp(scriptCiteCountVO); } @Override + @AuditEntry(actionId = ActionId.MANAGE_SCRIPT) public Response batchUpdateScriptTags(String username, AppResourceScope appResourceScope, String scopeType, String scopeId, ScriptTagBatchPatchReq req) { - ValidateResult validateResult = checkScriptTagBatchPatchReq(req); - if (!validateResult.isPass()) { - return Response.buildValidateFailResp(validateResult); + req.validate(); + if (CollectionUtils.isEmpty(req.getAddTagIdList()) && CollectionUtils.isEmpty(req.getDeleteTagIdList())) { + return Response.buildSuccessResp(true); } - boolean isPublicScript = appResourceScope == null; + // 鉴权 List scriptIdList = req.getIdList(); - AuthResult authResult; - if (isPublicScript) { - authResult = noResourceScopeAuthService.batchAuthResultManagePublicScript(username, scriptIdList); - } else { - authResult = scriptAuthService.batchAuthResultManageScript(username, appResourceScope, scriptIdList); - } + scriptAuthService.batchAuthResultManageScript(username, appResourceScope, scriptIdList).denyIfNoPermission(); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } - - List addResourceTags = null; - List deleteResourceTags = null; - Integer resourceType = appResourceScope == null ? - JobResourceTypeEnum.PUBLIC_SCRIPT.getValue() : - JobResourceTypeEnum.APP_SCRIPT.getValue(); - if (CollectionUtils.isNotEmpty(req.getAddTagIdList())) { - addResourceTags = tagService.buildResourceTags(resourceType, - scriptIdList.stream().map(String::valueOf).collect(Collectors.toList()), - req.getAddTagIdList()); - } - if (CollectionUtils.isNotEmpty(req.getDeleteTagIdList())) { - deleteResourceTags = tagService.buildResourceTags(resourceType, - scriptIdList.stream().map(String::valueOf).collect(Collectors.toList()), - req.getDeleteTagIdList()); - } - tagService.batchPatchResourceTags(addResourceTags, deleteResourceTags); + batchPatchResourceTags(JobResourceTypeEnum.APP_SCRIPT, req.getIdList(), req.getAddTagIdList(), + req.getDeleteTagIdList()); return Response.buildSuccessResp(true); } - private ValidateResult checkScriptTagBatchPatchReq(ScriptTagBatchPatchReq req) { - if (CollectionUtils.isEmpty(req.getIdList())) { - log.warn("ScriptTagBatchUpdateReq->idList is empty"); - return ValidateResult.fail(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME, "idList"); - } - - if (CollectionUtils.isEmpty(req.getAddTagIdList()) && CollectionUtils.isEmpty(req.getDeleteTagIdList())) { - log.warn("ScriptTagBatchUpdateReq->No script tags changed!"); - return ValidateResult.fail(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME, - "addTagIdList|deleteTagIdList"); - } - return ValidateResult.pass(); - } - @Override public Response getTagScriptCount(String username, AppResourceScope appResourceScope, String scopeType, String scopeId) { - Long appId = appResourceScope != null ? appResourceScope.getAppId() : PUBLIC_APP_ID; - return Response.buildSuccessResp(scriptService.getTagScriptCount(appId)); + return Response.buildSuccessResp(scriptService.getTagScriptCount(appResourceScope.getAppId())); } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebServiceInfoResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebServiceInfoResourceImpl.java index 7471e076d2..3138b57a90 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebServiceInfoResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebServiceInfoResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,18 +24,25 @@ package com.tencent.bk.job.manage.api.web.impl; +import com.tencent.bk.audit.annotations.ActionAuditRecord; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.job.common.audit.constants.EventContentConstants; +import com.tencent.bk.job.common.constant.ProfileEnum; +import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.model.Response; import com.tencent.bk.job.manage.api.web.WebServiceInfoResource; import com.tencent.bk.job.manage.model.web.vo.serviceinfo.ServiceInfoVO; import com.tencent.bk.job.manage.service.impl.ServiceInfoService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Profile; import org.springframework.web.bind.annotation.RestController; import java.util.List; @RestController @Slf4j +@Profile("!" + ProfileEnum.Constants.TEST) public class WebServiceInfoResourceImpl implements WebServiceInfoResource { private final ServiceInfoService serviceInfoService; @@ -46,6 +53,11 @@ public WebServiceInfoResourceImpl(ServiceInfoService serviceInfoService) { } @Override + @AuditEntry(actionId = ActionId.SERVICE_STATE_ACCESS) + @ActionAuditRecord( + actionId = ActionId.SERVICE_STATE_ACCESS, + content = EventContentConstants.VIEW_PLATFORM_SERVICE_STAT + ) public Response> listServiceInfo(String username) { return Response.buildSuccessResp(serviceInfoService.listServiceInfo()); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebTagResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebTagResourceImpl.java index 12740e3762..c4892c7c8c 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebTagResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebTagResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,8 +24,11 @@ package com.tencent.bk.job.manage.api.web.impl; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.audit.annotations.AuditRequestBody; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.constant.JobResourceTypeEnum; +import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; import com.tencent.bk.job.common.iam.model.AuthResult; import com.tencent.bk.job.common.model.BaseSearchCondition; @@ -33,7 +36,6 @@ import com.tencent.bk.job.common.model.Response; import com.tencent.bk.job.common.model.ValidateResult; import com.tencent.bk.job.common.model.dto.AppResourceScope; -import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.manage.api.web.WebTagResource; import com.tencent.bk.job.manage.auth.NoResourceScopeAuthService; import com.tencent.bk.job.manage.auth.ScriptAuthService; @@ -72,8 +74,7 @@ public WebTagResourceImpl(TagService tagService, TagAuthService tagAuthService, ScriptAuthService scriptAuthService, TemplateAuthService templateAuthService, - NoResourceScopeAuthService noResourceScopeAuthService, - AppScopeMappingService appScopeMappingService) { + NoResourceScopeAuthService noResourceScopeAuthService) { this.tagService = tagService; this.tagAuthService = tagAuthService; this.scriptAuthService = scriptAuthService; @@ -179,16 +180,13 @@ public Response> listTagsBasic(String username, } @Override + @AuditEntry(actionId = ActionId.MANAGE_TAG) public Response updateTagInfo(String username, AppResourceScope appResourceScope, String scopeType, String scopeId, Long tagId, TagCreateUpdateReq tagCreateUpdateReq) { - AuthResult authResult = checkManageTagPermission(username, appResourceScope, tagId); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } TagDTO tag = new TagDTO(); tag.setId(tagId); tag.setAppId(appResourceScope.getAppId()); @@ -198,46 +196,28 @@ public Response updateTagInfo(String username, } @Override + @AuditEntry(actionId = ActionId.CREATE_TAG) public Response saveTagInfo(String username, AppResourceScope appResourceScope, String scopeType, String scopeId, - TagCreateUpdateReq tagCreateUpdateReq) { - AuthResult authResult = checkCreateTagPermission(username, appResourceScope); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } + @AuditRequestBody TagCreateUpdateReq tagCreateUpdateReq) { TagDTO tag = new TagDTO(); tag.setAppId(appResourceScope.getAppId()); tag.setName(tagCreateUpdateReq.getName()); tag.setDescription(tagCreateUpdateReq.getDescription()); - Long tagId = tagService.insertNewTag(username, tag); - tagAuthService.registerTag(tagId, tagCreateUpdateReq.getName(), username); - - TagDTO savedTag = tagService.getTagInfoById(appResourceScope.getAppId(), tagId); - return Response.buildSuccessResp(TagDTO.toVO(savedTag)); - } - - private AuthResult checkManageTagPermission(String username, AppResourceScope appResourceScope, Long tagId) { - return tagAuthService.authManageTag(username, appResourceScope, tagId, null); - } - - private AuthResult checkCreateTagPermission(String username, AppResourceScope appResourceScope) { - return tagAuthService.authCreateTag(username, appResourceScope); + TagDTO newTag = tagService.createTag(username, tag); + return Response.buildSuccessResp(TagDTO.toVO(newTag)); } @Override + @AuditEntry(actionId = ActionId.MANAGE_TAG) public Response deleteTag(String username, AppResourceScope appResourceScope, String scopeType, String scopeId, Long tagId) { - AuthResult authResult = checkManageTagPermission(username, appResourceScope, tagId); - - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } - tagService.deleteTag(tagId); + tagService.deleteTag(username, appResourceScope.getAppId(), tagId); return Response.buildSuccessResp(true); } @@ -310,7 +290,7 @@ private AuthResult checkTagRelatedResourcesUpdatePermission(String username, App authResult = authResult.mergeAuthResult( templateAuthService.batchAuthResultEditJobTemplate( username, appResourceScope, - resources.parallelStream().map(Long::valueOf).collect(Collectors.toList()) + resources.stream().map(Long::valueOf).collect(Collectors.toList()) ) ); break; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebTaskPlanResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebTaskPlanResourceImpl.java index 482e5534d4..6273cd1d40 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebTaskPlanResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebTaskPlanResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,18 +24,19 @@ package com.tencent.bk.job.manage.api.web.impl; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.audit.annotations.AuditRequestBody; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.exception.InternalException; import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.exception.NotFoundException; +import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; import com.tencent.bk.job.common.iam.model.AuthResult; -import com.tencent.bk.job.common.iam.service.BusinessAuthService; import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; import com.tencent.bk.job.common.model.Response; import com.tencent.bk.job.common.model.dto.AppResourceScope; -import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.common.util.check.IlegalCharChecker; import com.tencent.bk.job.common.util.check.MaxLengthChecker; import com.tencent.bk.job.common.util.check.NotEmptyChecker; @@ -55,10 +56,12 @@ import com.tencent.bk.job.manage.model.query.TaskTemplateQuery; import com.tencent.bk.job.manage.model.web.request.TaskPlanCreateUpdateReq; import com.tencent.bk.job.manage.model.web.request.TaskVariableValueUpdateReq; +import com.tencent.bk.job.manage.model.web.vo.task.TaskPlanBasicInfoVO; import com.tencent.bk.job.manage.model.web.vo.task.TaskPlanSyncInfoVO; import com.tencent.bk.job.manage.model.web.vo.task.TaskPlanVO; import com.tencent.bk.job.manage.service.CronJobService; import com.tencent.bk.job.manage.service.TaskFavoriteService; +import com.tencent.bk.job.manage.service.host.HostService; import com.tencent.bk.job.manage.service.plan.TaskPlanService; import com.tencent.bk.job.manage.service.template.TaskTemplateService; import lombok.extern.slf4j.Slf4j; @@ -80,7 +83,7 @@ import java.util.stream.Collectors; /** - * @since 19/11/2019 16:30 + * 执行方案 Web Resource */ @Slf4j @RestController @@ -90,26 +93,25 @@ public class WebTaskPlanResourceImpl implements WebTaskPlanResource { private final TaskTemplateService templateService; private final TaskFavoriteService taskFavoriteService; private final CronJobService cronJobService; - private final BusinessAuthService businessAuthService; private final TemplateAuthService templateAuthService; private final PlanAuthService planAuthService; + private final HostService hostService; @Autowired public WebTaskPlanResourceImpl(TaskPlanService planService, TaskTemplateService templateService, @Qualifier("TaskPlanFavoriteServiceImpl") TaskFavoriteService taskFavoriteService, CronJobService cronJobService, - BusinessAuthService businessAuthService, TemplateAuthService templateAuthService, PlanAuthService planAuthService, - AppScopeMappingService appScopeMappingService) { + HostService hostService) { this.planService = planService; this.templateService = templateService; this.taskFavoriteService = taskFavoriteService; this.cronJobService = cronJobService; - this.businessAuthService = businessAuthService; this.templateAuthService = templateAuthService; this.planAuthService = planAuthService; + this.hostService = hostService; } @Override @@ -125,10 +127,6 @@ public Response> listAllPlans(String username, String lastModifyUser, Integer start, Integer pageSize) { - AuthResult authResult = businessAuthService.authAccessBusiness(username, appResourceScope); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } List favoriteList = taskFavoriteService.listFavorites(appResourceScope.getAppId(), username); TaskPlanQueryDTO taskPlanQueryDTO = new TaskPlanQueryDTO(); @@ -215,10 +213,6 @@ public Response> listPlans(String username, String scopeType, String scopeId, Long templateId) { - AuthResult authResult = businessAuthService.authAccessBusiness(username, appResourceScope); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } List taskPlanList = listPlansByTemplateId(username, appResourceScope, templateId); return Response.buildSuccessResp(taskPlanList); @@ -247,9 +241,9 @@ private List listPlansByTemplateId(String username, AppResourceScope private void processPlanPermission(String username, AppResourceScope appResourceScope, List taskPlanList) { List jobTemplateIdList = - taskPlanList.parallelStream().map(TaskPlanVO::getTemplateId).collect(Collectors.toList()); + taskPlanList.stream().map(TaskPlanVO::getTemplateId).collect(Collectors.toList()); List jobPlanIdList = - taskPlanList.parallelStream().map(TaskPlanVO::getId).collect(Collectors.toList()); + taskPlanList.stream().map(TaskPlanVO::getId).collect(Collectors.toList()); List allowedViewPlan = planAuthService.batchAuthViewJobPlan(username, appResourceScope, jobTemplateIdList, jobPlanIdList); @@ -291,11 +285,6 @@ public Response> batchGetPlans(String username, return Response.buildCommonFailResp(ErrorCode.ILLEGAL_PARAM); } - AuthResult authResult = businessAuthService.authAccessBusiness(username, appResourceScope); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } - List planList = new ArrayList<>(); for (Long templateId : templateIdList) { List templatePlanList = listPlansByTemplateId(username, appResourceScope, @@ -309,6 +298,9 @@ public Response> batchGetPlans(String username, } @Override + @AuditEntry( + actionId = ActionId.VIEW_JOB_PLAN + ) public Response getPlanById(String username, AppResourceScope appResourceScope, String scopeType, @@ -320,15 +312,8 @@ public Response getPlanById(String username, if (taskTemplateBasicInfo == null) { throw new NotFoundException(ErrorCode.TASK_PLAN_NOT_EXIST); } - TaskPlanInfoDTO taskPlan = planService.getTaskPlanById(appResourceScope.getAppId(), templateId, planId); - if (taskPlan == null) { - throw new NotFoundException(ErrorCode.TASK_PLAN_NOT_EXIST); - } - AuthResult authResult = planAuthService.authViewJobPlan(username, appResourceScope, templateId, - planId, taskPlan.getName()); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } + + TaskPlanInfoDTO taskPlan = planService.getTaskPlan(username, appResourceScope.getAppId(), templateId, planId); StepRefVariableParser.parseStepRefVars(taskPlan.getStepList(), taskPlan.getVariableList()); @@ -346,6 +331,7 @@ public Response getPlanById(String username, taskPlanVO.setCanDelete(planAuthService.authDeleteJobPlan(username, appResourceScope, templateId, planId, taskPlan.getName()) .isPass()); + return Response.buildSuccessResp(taskPlanVO); } @@ -377,30 +363,46 @@ public Response getDebugPlan(String username, } @Override - public Response savePlan(String username, - AppResourceScope appResourceScope, - String scopeType, - String scopeId, - Long templateId, - Long planId, - TaskPlanCreateUpdateReq taskPlanCreateUpdateReq) { + @AuditEntry( + actionId = ActionId.EDIT_JOB_PLAN + ) + public Response updatePlan(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + Long templateId, + Long planId, + @AuditRequestBody TaskPlanCreateUpdateReq taskPlanCreateUpdateReq) { taskPlanCreateUpdateReq.setTemplateId(templateId); - AuthResult authResult; - if (planId > 0) { - if (planService.isDebugPlan(appResourceScope.getAppId(), templateId, planId)) { - authResult = planAuthService.authEditJobPlan(username, appResourceScope, templateId, - planId, null); - } else { - authResult = planAuthService.authEditJobPlan(username, appResourceScope, templateId, - planId, null); - } - taskPlanCreateUpdateReq.setId(planId); + taskPlanCreateUpdateReq.setId(planId); + + // 检查执行方案名称 + checkPlanName(taskPlanCreateUpdateReq); + + TaskPlanInfoDTO savedPlan; + if (planService.isDebugPlan(appResourceScope.getAppId(), templateId, planId)) { + savedPlan = updateDebugTaskPlan(username, appResourceScope, taskPlanCreateUpdateReq); } else { - authResult = planAuthService.authCreateJobPlan(username, appResourceScope, templateId, null); - } - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); + savedPlan = updateTaskPlan(username, appResourceScope, taskPlanCreateUpdateReq); } + return Response.buildSuccessResp(TaskPlanInfoDTO.toVO(savedPlan)); + } + + private TaskPlanInfoDTO updateTaskPlan(String username, + AppResourceScope appResourceScope, + TaskPlanCreateUpdateReq taskPlanCreateUpdateReq) { + return planService.updateTaskPlan(username, TaskPlanInfoDTO.fromReq(username, + appResourceScope.getAppId(), taskPlanCreateUpdateReq)); + } + + private TaskPlanInfoDTO updateDebugTaskPlan(String username, + AppResourceScope appResourceScope, + TaskPlanCreateUpdateReq taskPlanCreateUpdateReq) { + return planService.updateDebugTaskPlan(username, TaskPlanInfoDTO.fromReq(username, + appResourceScope.getAppId(), taskPlanCreateUpdateReq)); + } + + private void checkPlanName(TaskPlanCreateUpdateReq taskPlanCreateUpdateReq) { // 检查执行方案名称 try { StringCheckHelper stringCheckHelper = new StringCheckHelper(new TrimChecker(), new NotEmptyChecker(), @@ -410,27 +412,42 @@ public Response savePlan(String username, log.warn("TaskPlan name is invalid:", e); throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); } - Long savedPlanId = planService.saveTaskPlan(TaskPlanInfoDTO.fromReq(username, appResourceScope.getAppId(), - taskPlanCreateUpdateReq)); - if (planId == 0) { - planAuthService.registerPlan(savedPlanId, taskPlanCreateUpdateReq.getName(), username); - } - return Response.buildSuccessResp(savedPlanId); } @Override + @AuditEntry( + actionId = ActionId.CREATE_JOB_PLAN + ) + public Response createPlan(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + Long templateId, + @AuditRequestBody TaskPlanCreateUpdateReq taskPlanCreateUpdateReq) { + taskPlanCreateUpdateReq.setTemplateId(templateId); + // 检查执行方案名称 + checkPlanName(taskPlanCreateUpdateReq); + + TaskPlanInfoDTO savedPlan = planService.createTaskPlan(username, TaskPlanInfoDTO.fromReq(username, + appResourceScope.getAppId(), taskPlanCreateUpdateReq)); + + return Response.buildSuccessResp(TaskPlanInfoDTO.toVO(savedPlan)); + } + + + @Override + @AuditEntry( + actionId = ActionId.DELETE_JOB_PLAN + ) public Response deletePlan(String username, AppResourceScope appResourceScope, String scopeType, String scopeId, Long templateId, Long planId) { - AuthResult authResult = planAuthService.authDeleteJobPlan(username, appResourceScope, templateId, - planId, null); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } - return Response.buildSuccessResp(planService.deleteTaskPlan(appResourceScope.getAppId(), templateId, planId)); + planService.deleteTaskPlan(username, appResourceScope.getAppId(), templateId, planId); + + return Response.buildSuccessResp(true); } @Override @@ -439,15 +456,10 @@ public Response> listPlanBasicInfoByIds(String username, String scopeType, String scopeId, String planIds) { - - AuthResult authResult = businessAuthService.authAccessBusiness(username, appResourceScope); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } if (StringUtils.isNotEmpty(planIds)) { List planIdList = Arrays.stream(planIds.split(",")).filter(Objects::nonNull).map(Long::valueOf) .filter(id -> id > 0).collect(Collectors.toList()); - List taskPlanInfoList = planService.listPlanBasicInfoByIds(appResourceScope.getAppId(), + List taskPlanInfoList = planService.listPlanBasicInfoWithVariablesByIds(appResourceScope.getAppId(), planIdList); fillCronInfo(appResourceScope.getAppId(), taskPlanInfoList); List taskPlanList = @@ -459,6 +471,31 @@ public Response> listPlanBasicInfoByIds(String username, } } + @Override + public Response> listTaskPlanBasicInfoByIds(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + String planIds) { + if (StringUtils.isEmpty(planIds)) { + return Response.buildSuccessResp(new ArrayList<>()); + } + List planIdList = Arrays.stream(planIds.split(",")) + .filter(Objects::nonNull) + .map(Long::valueOf) + .filter(id -> id > 0) + .collect(Collectors.toList()); + List taskPlanInfoList = planService.listPlanBasicInfoByIds( + appResourceScope.getAppId(), + planIdList + ); + List taskPlanList = + taskPlanInfoList.stream() + .map(TaskPlanInfoDTO::toBasicInfoVO) + .collect(Collectors.toList()); + return Response.buildSuccessResp(taskPlanList); + } + @Override public Response checkPlanName(String username, AppResourceScope appResourceScope, @@ -506,6 +543,9 @@ public Response syncInfo(String username, } @Override + @AuditEntry( + actionId = ActionId.SYNC_JOB_PLAN + ) public Response syncConfirm(String username, AppResourceScope appResourceScope, String scopeType, @@ -529,10 +569,6 @@ public Response addFavorite(String username, String scopeId, Long templateId, Long planId) { - AuthResult authResult = businessAuthService.authAccessBusiness(username, appResourceScope); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } return Response.buildSuccessResp(taskFavoriteService.addFavorite(appResourceScope.getAppId(), username, planId)); } @@ -544,10 +580,6 @@ public Response removeFavorite(String username, String scopeId, Long templateId, Long planId) { - AuthResult authResult = businessAuthService.authAccessBusiness(username, appResourceScope); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } return Response.buildSuccessResp(taskFavoriteService.deleteFavorite(appResourceScope.getAppId(), username, planId)); } @@ -559,11 +591,6 @@ public Response getPlanBasicInfoById(String username, Long planId) { } TaskPlanInfoDTO taskPlanInfo = planService.getTaskPlanById(planId); if (taskPlanInfo != null) { - AuthResult authResult = businessAuthService.authAccessBusiness( - username, new AppResourceScope(taskPlanInfo.getAppId())); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } taskPlanInfo.setStepList(null); taskPlanInfo.setVariableList(null); return Response.buildSuccessResp(TaskPlanInfoDTO.toVO(taskPlanInfo)); @@ -572,6 +599,9 @@ public Response getPlanBasicInfoById(String username, Long planId) { } @Override + @AuditEntry( + actionId = ActionId.EDIT_JOB_PLAN + ) public Response batchUpdatePlanVariableValueByName(String username, AppResourceScope appResourceScope, String scopeType, @@ -601,7 +631,7 @@ public Response batchUpdatePlanVariableValueByName(String username, Long modifyTime = DateUtils.currentTimeSeconds(); - List planInfoList = planVariableInfoList.parallelStream().map(planVariableInfo -> { + List planInfoList = planVariableInfoList.stream().map(planVariableInfo -> { TaskPlanInfoDTO planInfo = new TaskPlanInfoDTO(); planInfo.setAppId(appResourceScope.getAppId()); planInfo.setLastModifyUser(username); @@ -609,7 +639,7 @@ public Response batchUpdatePlanVariableValueByName(String username, planInfo.setId(planVariableInfo.getPlanId()); planInfo.setTemplateId(planVariableInfo.getTemplateId()); if (CollectionUtils.isNotEmpty(planVariableInfo.getVariableInfoList())) { - planInfo.setVariableList(planVariableInfo.getVariableInfoList().parallelStream() + planInfo.setVariableList(planVariableInfo.getVariableInfoList().stream() .map(variableVO -> { variableVO.setRequired(0); variableVO.setChangeable(0); @@ -628,7 +658,7 @@ public Response batchUpdatePlanVariableValueByName(String username, private void fillCronInfo(Long appId, List planInfoList) { try { - List planIds = planInfoList.parallelStream() + List planIds = planInfoList.stream() .map(TaskPlanInfoDTO::getId).collect(Collectors.toList()); Map> cronJobByPlanIds = cronJobService.batchListCronJobByPlanIds(appId, planIds); if (MapUtils.isNotEmpty(cronJobByPlanIds)) { diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebTaskTemplateResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebTaskTemplateResourceImpl.java index a6fe67b815..9938d93b88 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebTaskTemplateResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebTaskTemplateResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,10 +25,13 @@ package com.tencent.bk.job.manage.api.web.impl; import com.google.common.base.CaseFormat; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.audit.annotations.AuditRequestBody; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.constant.JobResourceTypeEnum; import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.exception.NotFoundException; +import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; import com.tencent.bk.job.common.iam.model.AuthResult; import com.tencent.bk.job.common.model.BaseSearchCondition; @@ -36,10 +39,11 @@ import com.tencent.bk.job.common.model.Response; import com.tencent.bk.job.common.model.ValidateResult; import com.tencent.bk.job.common.model.dto.AppResourceScope; +import com.tencent.bk.job.common.mysql.JobTransactional; +import com.tencent.bk.job.manage.api.common.constants.TemplateTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskTemplateStatusEnum; import com.tencent.bk.job.manage.api.web.WebTaskTemplateResource; import com.tencent.bk.job.manage.auth.TemplateAuthService; -import com.tencent.bk.job.manage.common.consts.TemplateTypeEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskTemplateStatusEnum; import com.tencent.bk.job.manage.manager.variable.StepRefVariableParser; import com.tencent.bk.job.manage.model.dto.ResourceTagDTO; import com.tencent.bk.job.manage.model.dto.TagDTO; @@ -59,7 +63,6 @@ import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.transaction.annotation.Transactional; import org.springframework.web.bind.annotation.RestController; import java.util.ArrayList; @@ -70,7 +73,7 @@ import java.util.stream.Collectors; /** - * @since 16/10/2019 16:16 + * 作业模板Resource */ @Slf4j @RestController @@ -113,7 +116,7 @@ public Response> listPageTemplates(String username, Integer pageSize, String orderField, Integer order) { - + Long appId = appResourceScope.getAppId(); List favoriteList = taskFavoriteService.listFavorites(appId, username); @@ -211,21 +214,15 @@ private void addTagCondition(TaskTemplateQuery query, String tags, Long panelTag } @Override + @AuditEntry(actionId = ActionId.VIEW_JOB_TEMPLATE) public Response getTemplateById(String username, AppResourceScope appResourceScope, String scopeType, String scopeId, Long templateId) { - - AuthResult authResult = templateAuthService.authViewJobTemplate(username, appResourceScope, - templateId); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } - TaskTemplateInfoDTO templateInfo = templateService.getTaskTemplateById(appResourceScope.getAppId(), templateId); - if (templateInfo == null) { - throw new NotFoundException(ErrorCode.TEMPLATE_NOT_EXIST); - } + TaskTemplateInfoDTO templateInfo = templateService.getTaskTemplate(username, + appResourceScope.getAppId(), templateId); + StepRefVariableParser.parseStepRefVars(templateInfo.getStepList(), templateInfo.getVariableList()); TaskTemplateVO taskTemplateVO = TaskTemplateInfoDTO.toVO(templateInfo); @@ -242,55 +239,58 @@ public Response getTemplateById(String username, } @Override - public Response saveTemplate(String username, - AppResourceScope appResourceScope, - String scopeType, - String scopeId, - Long templateId, - TaskTemplateCreateUpdateReq taskTemplateCreateUpdateReq) { - - AuthResult authResult; - if (templateId > 0) { - taskTemplateCreateUpdateReq.setId(templateId); - authResult = templateAuthService.authEditJobTemplate(username, appResourceScope, templateId); - } else { - authResult = templateAuthService.authCreateJobTemplate(username, appResourceScope); - } - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } - - if (taskTemplateCreateUpdateReq.validate()) { - Long finalTemplateId = templateService - .saveTaskTemplate(TaskTemplateInfoDTO.fromReq(username, appResourceScope.getAppId(), - taskTemplateCreateUpdateReq)); - templateAuthService.registerTemplate(finalTemplateId, taskTemplateCreateUpdateReq.getName(), username); - return Response.buildSuccessResp(finalTemplateId); - } else { - throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); - } + @AuditEntry( + actionId = ActionId.CREATE_JOB_TEMPLATE + ) + public Response createTemplate(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + @AuditRequestBody TaskTemplateCreateUpdateReq request) { + + request.validate(); + + TaskTemplateInfoDTO createdTemplate = templateService.saveTaskTemplate(username, + TaskTemplateInfoDTO.fromReq(username, appResourceScope.getAppId(), request)); + return Response.buildSuccessResp(TaskTemplateInfoDTO.toVO(createdTemplate)); } @Override - @Transactional(rollbackFor = {Exception.class, Error.class}) + @JobTransactional(transactionManager = "jobManageTransactionManager") + @AuditEntry( + actionId = ActionId.DELETE_JOB_TEMPLATE, + subActionIds = {ActionId.DELETE_JOB_PLAN} + ) public Response deleteTemplate(String username, AppResourceScope appResourceScope, String scopeType, String scopeId, Long templateId) { - - AuthResult authResult = templateAuthService.authDeleteJobTemplate(username, appResourceScope, - templateId); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } Long appId = appResourceScope.getAppId(); - if (templateService.deleteTaskTemplate(appId, templateId)) { - taskFavoriteService.deleteFavorite(appId, username, templateId); - return Response.buildSuccessResp(true); - } - return Response.buildSuccessResp(false); + templateService.deleteTaskTemplate(username, appId, templateId); + taskFavoriteService.deleteFavorite(appId, username, templateId); + + return Response.buildSuccessResp(true); + } + + @Override + @AuditEntry( + actionId = ActionId.EDIT_JOB_TEMPLATE + ) + public Response updateTemplate(String username, + AppResourceScope appResourceScope, + String scopeType, + String scopeId, + Long templateId, + @AuditRequestBody TaskTemplateCreateUpdateReq request) { + request.setId(templateId); + request.validate(); + + TaskTemplateInfoDTO updatedTemplate = templateService.updateTaskTemplate( + username, TaskTemplateInfoDTO.fromReq(username, appResourceScope.getAppId(), request)); + + return Response.buildSuccessResp(TaskTemplateInfoDTO.toVO(updatedTemplate)); } @Override @@ -298,32 +298,23 @@ public Response getTagTemplateCount(String username, AppResourceScope appResourceScope, String scopeType, String scopeId) { - + return Response.buildSuccessResp(templateService.getTagTemplateCount(appResourceScope.getAppId())); } @Override + @AuditEntry( + actionId = ActionId.EDIT_JOB_TEMPLATE + ) public Response updateTemplateBasicInfo(String username, AppResourceScope appResourceScope, String scopeType, String scopeId, Long templateId, - TemplateBasicInfoUpdateReq templateBasicInfoUpdateReq) { - - if (templateId > 0) { - templateBasicInfoUpdateReq.setId(templateId); - } else { - throw new NotFoundException(ErrorCode.TEMPLATE_NOT_EXIST); - } - AuthResult authResult = templateAuthService.authEditJobTemplate(username, appResourceScope, - templateId); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } - - return Response.buildSuccessResp( - templateService.saveTaskTemplateBasicInfo( - TaskTemplateInfoDTO.fromBasicReq(username, appResourceScope.getAppId(), templateBasicInfoUpdateReq))); + @AuditRequestBody TemplateBasicInfoUpdateReq request) { + templateService.saveTaskTemplateBasicInfo(username, + TaskTemplateInfoDTO.fromBasicReq(username, appResourceScope.getAppId(), request)); + return Response.buildSuccessResp(true); } @Override @@ -332,7 +323,7 @@ public Response addFavorite(String username, String scopeType, String scopeId, Long templateId) { - + return Response.buildSuccessResp(taskFavoriteService.addFavorite(appResourceScope.getAppId(), username, templateId)); } @@ -343,7 +334,7 @@ public Response removeFavorite(String username, String scopeType, String scopeId, Long templateId) { - + return Response.buildSuccessResp(taskFavoriteService.deleteFavorite(appResourceScope.getAppId(), username, templateId)); } @@ -355,7 +346,7 @@ public Response checkTemplateName(String username, String scopeId, Long templateId, String name) { - + return Response.buildSuccessResp(templateService.checkTemplateName(appResourceScope.getAppId(), templateId, name)); } @@ -366,7 +357,7 @@ public Response> listTemplateBasicInfoByIds(String username String scopeType, String scopeId, List templateIds) { - + List taskTemplateBasicInfo = templateService.listTaskTemplateBasicInfoByIds(appResourceScope.getAppId(), templateIds); List templateBasicInfoVOList = taskTemplateBasicInfo.stream() @@ -380,7 +371,7 @@ public Response batchPatchTemplateTags(String username, String scopeType, String scopeId, TemplateTagBatchPatchReq req) { - + ValidateResult validateResult = checkTemplateTagBatchPatchReq(req); if (!validateResult.isPass()) { throw new InvalidParamException(validateResult); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebVersionResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebVersionResourceImpl.java deleted file mode 100644 index c7222f08c9..0000000000 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebVersionResourceImpl.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.api.web.impl; - -import com.tencent.bk.job.common.web.controller.WebVersionResource; -import lombok.extern.slf4j.Slf4j; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.actuate.autoconfigure.endpoint.condition.ConditionalOnAvailableEndpoint; -import org.springframework.boot.actuate.info.InfoEndpoint; -import org.springframework.web.bind.annotation.RestController; - -import java.util.Map; - -/** - * @since 17/7/2020 13:47 - */ -@Slf4j -@RestController -@ConditionalOnAvailableEndpoint(endpoint = InfoEndpoint.class) -public class WebVersionResourceImpl implements WebVersionResource { - - private final InfoEndpoint infoEndpoint; - - @Autowired - public WebVersionResourceImpl(InfoEndpoint infoEndpoint) { - this.infoEndpoint = infoEndpoint; - } - - @Override - public Map getVersion() { - return infoEndpoint.info(); - } -} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebWhiteIPResourceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebWhiteIPResourceImpl.java index 786c47196d..bb57fcaf58 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebWhiteIPResourceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/api/web/impl/WebWhiteIPResourceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,11 @@ package com.tencent.bk.job.manage.api.web.impl; +import com.tencent.bk.audit.annotations.ActionAuditRecord; +import com.tencent.bk.audit.annotations.AuditEntry; +import com.tencent.bk.audit.annotations.AuditRequestBody; +import com.tencent.bk.job.common.audit.constants.EventContentConstants; +import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; import com.tencent.bk.job.common.iam.model.AuthResult; import com.tencent.bk.job.common.model.PageData; @@ -32,6 +37,8 @@ import com.tencent.bk.job.common.model.vo.CloudAreaInfoVO; import com.tencent.bk.job.manage.api.web.WebWhiteIPResource; import com.tencent.bk.job.manage.auth.NoResourceScopeAuthService; +import com.tencent.bk.job.manage.dao.whiteip.ActionScopeDAO; +import com.tencent.bk.job.manage.model.dto.whiteip.WhiteIPRecordDTO; import com.tencent.bk.job.manage.model.web.request.whiteip.WhiteIPRecordCreateUpdateReq; import com.tencent.bk.job.manage.model.web.vo.whiteip.ActionScopeVO; import com.tencent.bk.job.manage.model.web.vo.whiteip.WhiteIPRecordVO; @@ -41,6 +48,7 @@ import org.springframework.web.bind.annotation.RestController; import java.util.List; +import java.util.stream.Collectors; @RestController @Slf4j @@ -48,15 +56,23 @@ public class WebWhiteIPResourceImpl implements WebWhiteIPResource { private final WhiteIPService whiteIPService; private final NoResourceScopeAuthService noResourceScopeAuthService; + private final ActionScopeDAO actionScopeDAO; @Autowired public WebWhiteIPResourceImpl(WhiteIPService whiteIPService, - NoResourceScopeAuthService noResourceScopeAuthService) { + NoResourceScopeAuthService noResourceScopeAuthService, + ActionScopeDAO actionScopeDAO) { this.whiteIPService = whiteIPService; this.noResourceScopeAuthService = noResourceScopeAuthService; + this.actionScopeDAO = actionScopeDAO; } @Override + @AuditEntry(actionId = ActionId.MANAGE_WHITELIST) + @ActionAuditRecord( + actionId = ActionId.MANAGE_WHITELIST, + content = EventContentConstants.VIEW_WHITE_LIST + ) public Response> listWhiteIP( String username, String ipStr, @@ -86,20 +102,41 @@ public Response> listWhiteIP( } @Override - public Response saveWhiteIP(String username, WhiteIPRecordCreateUpdateReq createUpdateReq) { - Long id = createUpdateReq.getId(); - if (id != null && id > 0) { - AuthResult authResult = noResourceScopeAuthService.authManageWhiteList(username); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } - } else { - AuthResult authResult = noResourceScopeAuthService.authCreateWhiteList(username); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } + @AuditEntry(actionId = ActionId.CREATE_WHITELIST) + public Response createWhiteIP(String username, + @AuditRequestBody WhiteIPRecordCreateUpdateReq createUpdateReq) { + AuthResult authResult = noResourceScopeAuthService.authCreateWhiteList(username); + if (!authResult.isPass()) { + throw new PermissionDeniedException(authResult); } - return Response.buildSuccessResp(whiteIPService.saveWhiteIP(username, createUpdateReq)); + + WhiteIPRecordDTO record = whiteIPService.createWhiteIP(username, createUpdateReq); + return Response.buildSuccessResp(toWhiteIPRecordVO(record)); + } + + private WhiteIPRecordVO toWhiteIPRecordVO(WhiteIPRecordDTO record) { + WhiteIPRecordVO vo = record.toVO(); + vo.setActionScopeList(record.getActionScopeList().stream().map(actionScope -> { + long actionScopeId = actionScope.getActionScopeId(); + return actionScopeDAO.getActionScopeVOById(actionScopeId); + }).collect(Collectors.toList())); + + return vo; + } + + @Override + @AuditEntry(actionId = ActionId.MANAGE_WHITELIST) + public Response updateWhiteIP(String username, + Long id, + @AuditRequestBody WhiteIPRecordCreateUpdateReq createUpdateReq) { + createUpdateReq.setId(id); + AuthResult authResult = noResourceScopeAuthService.authManageWhiteList(username); + if (!authResult.isPass()) { + throw new PermissionDeniedException(authResult); + } + + WhiteIPRecordDTO record = whiteIPService.updateWhiteIP(username, createUpdateReq); + return Response.buildSuccessResp(toWhiteIPRecordVO(record)); } @Override @@ -108,7 +145,9 @@ public Response getWhiteIPDetailById(String username, Long id) if (!authResult.isPass()) { throw new PermissionDeniedException(authResult); } - return Response.buildSuccessResp(whiteIPService.getWhiteIPDetailById(username, id)); + + WhiteIPRecordDTO record = whiteIPService.getWhiteIPDetailById(username, id); + return Response.buildSuccessResp(toWhiteIPRecordVO(record)); } @Override @@ -122,6 +161,7 @@ public Response> listActionScope(String username) { } @Override + @AuditEntry(actionId = ActionId.MANAGE_WHITELIST) public Response deleteWhiteIPById(String username, Long id) { AuthResult authResult = noResourceScopeAuthService.authManageWhiteList(username); if (!authResult.isPass()) { diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/AccountAuthService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/AccountAuthService.java index dcb40af28e..6b6909089b 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/AccountAuthService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/AccountAuthService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/EsbAuthService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/EsbAuthService.java index 5856014964..779a9b3f2f 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/EsbAuthService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/EsbAuthService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/NoResourceScopeAuthService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/NoResourceScopeAuthService.java index 14bd551aba..902780f87b 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/NoResourceScopeAuthService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/NoResourceScopeAuthService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/NotificationAuthService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/NotificationAuthService.java index bc0ad06504..3be8d27f28 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/NotificationAuthService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/NotificationAuthService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/PlanAuthService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/PlanAuthService.java index bd05c7f9b4..751e459aae 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/PlanAuthService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/PlanAuthService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/ScriptAuthService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/ScriptAuthService.java index be353cfa88..c41a2b0346 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/ScriptAuthService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/ScriptAuthService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/TagAuthService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/TagAuthService.java index 798f58fe36..e16baac834 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/TagAuthService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/TagAuthService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/TemplateAuthService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/TemplateAuthService.java index b1f103b22e..c424a5e6fc 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/TemplateAuthService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/TemplateAuthService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -80,18 +80,6 @@ AuthResult authDeleteJobTemplate(String username, AppResourceScope appResourceScope, Long jobTemplateId); - /** - * 资源范围下调试作业模板鉴权 - * - * @param username 用户名 - * @param appResourceScope 资源范围 - * @param jobTemplateId 作业模板ID - * @return 鉴权结果 - */ - AuthResult authDebugJobTemplate(String username, - AppResourceScope appResourceScope, - Long jobTemplateId); - /** * 资源范围下查看作业模板批量鉴权 * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/TicketAuthService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/TicketAuthService.java index e2b90ac16e..f394a40f18 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/TicketAuthService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/TicketAuthService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -93,4 +93,14 @@ List batchAuthManageTicket(String username, List batchAuthUseTicket(String username, AppResourceScope appResourceScope, List ticketIdList); + + /** + * 注册凭证实例 + * + * @param creator 资源实例创建者 + * @param id 资源实例 ID + * @param name 资源实例名称 + * @return 是否注册成功 + */ + boolean registerTicket(String creator, String id, String name); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/impl/AccountAuthServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/impl/AccountAuthServiceImpl.java index ffc07dbc29..08eb0e2641 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/impl/AccountAuthServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/impl/AccountAuthServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -103,8 +103,8 @@ public List batchAuthManageAccount(String username, List accountIdList) { List allowIdList = appAuthService.batchAuth(username, ActionId.MANAGE_ACCOUNT, appResourceScope, ResourceTypeEnum.ACCOUNT, - accountIdList.parallelStream().map(Object::toString).collect(Collectors.toList())); - return allowIdList.parallelStream().map(Long::valueOf).collect(Collectors.toList()); + accountIdList.stream().map(Object::toString).collect(Collectors.toList())); + return allowIdList.stream().map(Long::valueOf).collect(Collectors.toList()); } @Override @@ -113,8 +113,8 @@ public List batchAuthUseAccount(String username, List accountIdList) { List allowIdList = appAuthService.batchAuth(username, ActionId.USE_ACCOUNT, appResourceScope, ResourceTypeEnum.ACCOUNT, - accountIdList.parallelStream().map(Object::toString).collect(Collectors.toList())); - return allowIdList.parallelStream().map(Long::valueOf).collect(Collectors.toList()); + accountIdList.stream().map(Object::toString).collect(Collectors.toList())); + return allowIdList.stream().map(Long::valueOf).collect(Collectors.toList()); } @Override diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/impl/EsbAuthServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/impl/EsbAuthServiceImpl.java index 9d1d93cb7a..b922ccf376 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/impl/EsbAuthServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/impl/EsbAuthServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/impl/NoResourceScopeAuthServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/impl/NoResourceScopeAuthServiceImpl.java index 219fea805d..37e78eed21 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/impl/NoResourceScopeAuthServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/impl/NoResourceScopeAuthServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/impl/NotificationAuthServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/impl/NotificationAuthServiceImpl.java index eb89f2d6f9..0d2659993b 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/impl/NotificationAuthServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/impl/NotificationAuthServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/impl/PlanAuthServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/impl/PlanAuthServiceImpl.java index 5c2bc907c5..4f0738eb3c 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/impl/PlanAuthServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/impl/PlanAuthServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -167,9 +167,10 @@ public List batchAuthViewJobPlan(String username, List jobPlanIdList) { List planPermissionResourceList = buildPlanPermissionResource( appResourceScope, jobTemplateIdList, jobPlanIdList); - List allowedPlanIdList = appAuthService.batchAuth(username, ActionId.VIEW_JOB_PLAN, appResourceScope, - planPermissionResourceList).parallelStream().collect(Collectors.toList()); - return allowedPlanIdList.parallelStream().map(Long::valueOf).collect(Collectors.toList()); + List allowedPlanIdList = new ArrayList<>(appAuthService.batchAuth(username, ActionId.VIEW_JOB_PLAN, + appResourceScope, + planPermissionResourceList)); + return allowedPlanIdList.stream().map(Long::valueOf).collect(Collectors.toList()); } @Override @@ -179,9 +180,10 @@ public List batchAuthEditJobPlan(String username, List jobPlanIdList) { List planPermissionResourceList = buildPlanPermissionResource( appResourceScope, jobTemplateIdList, jobPlanIdList); - List allowedPlanIdList = appAuthService.batchAuth(username, ActionId.EDIT_JOB_PLAN, appResourceScope, - planPermissionResourceList).parallelStream().collect(Collectors.toList()); - return allowedPlanIdList.parallelStream().map(Long::valueOf).collect(Collectors.toList()); + List allowedPlanIdList = new ArrayList<>(appAuthService.batchAuth(username, ActionId.EDIT_JOB_PLAN, + appResourceScope, + planPermissionResourceList)); + return allowedPlanIdList.stream().map(Long::valueOf).collect(Collectors.toList()); } @Override @@ -191,9 +193,10 @@ public List batchAuthDeleteJobPlan(String username, List jobPlanIdList) { List planPermissionResourceList = buildPlanPermissionResource( appResourceScope, jobTemplateIdList, jobPlanIdList); - List allowedPlanIdList = appAuthService.batchAuth(username, ActionId.DELETE_JOB_PLAN, appResourceScope, - planPermissionResourceList).parallelStream().collect(Collectors.toList()); - return allowedPlanIdList.parallelStream().map(Long::valueOf).collect(Collectors.toList()); + List allowedPlanIdList = new ArrayList<>(appAuthService.batchAuth(username, ActionId.DELETE_JOB_PLAN, + appResourceScope, + planPermissionResourceList)); + return allowedPlanIdList.stream().map(Long::valueOf).collect(Collectors.toList()); } @Override diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/impl/ScriptAuthServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/impl/ScriptAuthServiceImpl.java index 4cd05f222c..880237a32e 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/impl/ScriptAuthServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/impl/ScriptAuthServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/impl/TagAuthServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/impl/TagAuthServiceImpl.java index a29166eec0..b954ae30bf 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/impl/TagAuthServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/impl/TagAuthServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -87,8 +87,8 @@ public List batchAuthManageTag(String username, List tagIdList) { List allowedIdList = appAuthService.batchAuth(username, ActionId.MANAGE_TAG, appResourceScope, ResourceTypeEnum.TAG, - tagIdList.parallelStream().map(Object::toString).collect(Collectors.toList())); - return allowedIdList.parallelStream().map(Long::valueOf).collect(Collectors.toList()); + tagIdList.stream().map(Object::toString).collect(Collectors.toList())); + return allowedIdList.stream().map(Long::valueOf).collect(Collectors.toList()); } @Override diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/impl/TemplateAuthServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/impl/TemplateAuthServiceImpl.java index b19da279eb..1189ae4134 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/impl/TemplateAuthServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/impl/TemplateAuthServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -107,27 +107,14 @@ public AuthResult authDeleteJobTemplate(String username, ); } - @Override - public AuthResult authDebugJobTemplate(String username, - AppResourceScope appResourceScope, - Long jobTemplateId) { - return authService.auth( - username, - ActionId.DEBUG_JOB_TEMPLATE, - ResourceTypeEnum.TEMPLATE, - jobTemplateId.toString(), - buildAppScopePath(appResourceScope) - ); - } - @Override public List batchAuthViewJobTemplate(String username, AppResourceScope appResourceScope, List jobTemplateIdList) { List allowedIdList = appAuthService.batchAuth(username, ActionId.VIEW_JOB_TEMPLATE, appResourceScope, ResourceTypeEnum.TEMPLATE, - jobTemplateIdList.parallelStream().map(Object::toString).collect(Collectors.toList())); - return allowedIdList.parallelStream().map(Long::valueOf).collect(Collectors.toList()); + jobTemplateIdList.stream().map(Object::toString).collect(Collectors.toList())); + return allowedIdList.stream().map(Long::valueOf).collect(Collectors.toList()); } @Override @@ -136,8 +123,8 @@ public List batchAuthEditJobTemplate(String username, List jobTemplateIdList) { List allowedIdList = appAuthService.batchAuth(username, ActionId.EDIT_JOB_TEMPLATE, appResourceScope, ResourceTypeEnum.TEMPLATE, - jobTemplateIdList.parallelStream().map(Object::toString).collect(Collectors.toList())); - return allowedIdList.parallelStream().map(Long::valueOf).collect(Collectors.toList()); + jobTemplateIdList.stream().map(Object::toString).collect(Collectors.toList())); + return allowedIdList.stream().map(Long::valueOf).collect(Collectors.toList()); } @Override @@ -159,8 +146,8 @@ public List batchAuthDeleteJobTemplate(String username, List jobTemplateIdList) { List allowedIdList = appAuthService.batchAuth(username, ActionId.DELETE_JOB_TEMPLATE, appResourceScope, ResourceTypeEnum.TEMPLATE, - jobTemplateIdList.parallelStream().map(Object::toString).collect(Collectors.toList())); - return allowedIdList.parallelStream().map(Long::valueOf).collect(Collectors.toList()); + jobTemplateIdList.stream().map(Object::toString).collect(Collectors.toList())); + return allowedIdList.stream().map(Long::valueOf).collect(Collectors.toList()); } @Override diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/impl/TicketAuthServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/impl/TicketAuthServiceImpl.java index a7ce3282fa..69b9674d2d 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/impl/TicketAuthServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/auth/impl/TicketAuthServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,6 +26,7 @@ import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.iam.constant.ResourceTypeEnum; +import com.tencent.bk.job.common.iam.constant.ResourceTypeId; import com.tencent.bk.job.common.iam.model.AuthResult; import com.tencent.bk.job.common.iam.service.AppAuthService; import com.tencent.bk.job.common.iam.service.AuthService; @@ -118,4 +119,9 @@ public List batchAuthUseTicket(String username, ticketIdList ); } + + @Override + public boolean registerTicket(String creator, String id, String name) { + return authService.registerResource(id, name, ResourceTypeId.TICKET, creator, null); + } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/client/ServiceCronJobResourceClient.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/client/ServiceCronJobResourceClient.java deleted file mode 100644 index 9a79916ccd..0000000000 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/client/ServiceCronJobResourceClient.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.client; - -import com.tencent.bk.job.crontab.api.inner.ServiceCronJobResource; -import org.springframework.cloud.openfeign.FeignClient; - -/** - * 定时任务内部接口远程调用客户端 - * - * @since 21/2/2020 11:19 - */ -@FeignClient(value = "job-crontab", contextId = "crontab") -public interface ServiceCronJobResourceClient extends ServiceCronJobResource { -} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/client/ServiceInnerCronJobResourceClient.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/client/ServiceInnerCronJobResourceClient.java deleted file mode 100644 index b14ad56082..0000000000 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/client/ServiceInnerCronJobResourceClient.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.client; - -import com.tencent.bk.job.crontab.api.inner.ServiceInnerCronJobResource; -import org.springframework.cloud.openfeign.FeignClient; - -/** - * 内置定时任务远程调用客户端 - */ -@FeignClient(value = "job-crontab", contextId = "innerCronJob") -public interface ServiceInnerCronJobResourceClient extends ServiceInnerCronJobResource { -} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/client/ServiceTaskExecuteResultResourceClient.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/client/ServiceTaskExecuteResultResourceClient.java deleted file mode 100644 index 2214e0c62e..0000000000 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/client/ServiceTaskExecuteResultResourceClient.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.client; - -import com.tencent.bk.job.execute.api.inner.ServiceTaskExecuteResultResource; -import org.springframework.cloud.openfeign.FeignClient; - -/** - * @Description - * @Date 2020/3/9 - * @Version 1.0 - */ - -@FeignClient(value = "job-execute", contextId = "execute") -public interface ServiceTaskExecuteResultResourceClient extends ServiceTaskExecuteResultResource { -} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/common/TopologyHelper.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/common/TopologyHelper.java index 1378009434..1a5c320413 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/common/TopologyHelper.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/common/TopologyHelper.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,24 +25,14 @@ package com.tencent.bk.job.manage.common; import com.tencent.bk.job.common.cc.model.InstanceTopologyDTO; -import com.tencent.bk.job.common.cc.model.TopologyNodeInfoDTO; import com.tencent.bk.job.common.cc.sdk.CmdbClientFactory; -import com.tencent.bk.job.common.cc.service.CloudAreaService; import com.tencent.bk.job.common.constant.CcNodeTypeEnum; import com.tencent.bk.job.common.constant.ErrorCode; -import com.tencent.bk.job.common.constant.ResourceScopeTypeEnum; import com.tencent.bk.job.common.exception.InternalException; -import com.tencent.bk.job.common.gse.service.QueryAgentStatusClient; import com.tencent.bk.job.common.model.dto.ApplicationDTO; -import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; -import com.tencent.bk.job.common.model.dto.DynamicGroupWithHost; -import com.tencent.bk.job.common.model.vo.CloudAreaInfoVO; -import com.tencent.bk.job.common.model.vo.HostInfoVO; import com.tencent.bk.job.common.util.JobContextUtil; import com.tencent.bk.job.manage.dao.ApplicationDAO; import com.tencent.bk.job.manage.model.web.vo.CcTopologyNodeVO; -import com.tencent.bk.job.manage.model.web.vo.DynamicGroupInfoVO; -import com.tencent.bk.job.manage.model.web.vo.NodeInfoVO; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -62,7 +52,6 @@ import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; -import java.util.stream.Collectors; /** * 拓扑信息缓存、更新工具类 @@ -77,24 +66,19 @@ public class TopologyHelper { private static final Map>> BIZ_NODE_TYPE_NAME_MAP = new ConcurrentHashMap<>(); - private final QueryAgentStatusClient queryAgentStatusClient; private final ApplicationDAO applicationDAO; - private final CloudAreaService cloudAreaService; @Autowired - public TopologyHelper(ApplicationDAO applicationDAO, QueryAgentStatusClient queryAgentStatusClient, - CloudAreaService cloudAreaService) { + public TopologyHelper(ApplicationDAO applicationDAO) { this.applicationDAO = applicationDAO; - this.queryAgentStatusClient = queryAgentStatusClient; - this.cloudAreaService = cloudAreaService; } /** * 递归遍历查找节点路径 * - * @param treeNode - * @param targetNode - * @return + * @param treeNode 节点树 + * @param targetNode 目标节点 + * @return 节点路径 */ public static List findTopoPath( InstanceTopologyDTO treeNode, @@ -119,9 +103,9 @@ public static List findTopoPath( /** * 递归遍历查找多个节点路径 * - * @param treeNode - * @param targetNodes - * @return + * @param treeNode 节点树 + * @param targetNodes 要查找的目标节点 + * @return 节点路径列表 */ public static List> findTopoPaths(InstanceTopologyDTO treeNode, List targetNodes) { @@ -181,10 +165,6 @@ public static CcTopologyNodeVO convertToCcTopologyTree(InstanceTopologyDTO bizIn node.setObjectName(bizInstTopo.getObjectName()); node.setInstanceId(bizInstTopo.getInstanceId()); node.setInstanceName(bizInstTopo.getInstanceName()); - // 模块不需要展开,其他的默认展开 - if (CcNodeTypeEnum.MODULE.getType().equals(bizInstTopo.getObjectId())) { - node.setExpanded(false); - } List children = new ArrayList<>(); if (bizInstTopo.getChild() != null && !bizInstTopo.getChild().isEmpty()) { for (InstanceTopologyDTO childBizInstTopo : bizInstTopo.getChild()) { @@ -195,80 +175,6 @@ public static CcTopologyNodeVO convertToCcTopologyTree(InstanceTopologyDTO bizIn return node; } - /** - * 将作业平台内部主机信息转换为展示用主机信息 - * - * @param hostInfo 作业平台内主机信息 - * @return 展示用主机信息 - */ - public static HostInfoVO convertToHostInfoVO(ApplicationHostDTO hostInfo) { - if (hostInfo == null) { - return null; - } - HostInfoVO hostInfoVO = new HostInfoVO(); - hostInfoVO.setHostId(hostInfo.getHostId()); - hostInfoVO.setIp(hostInfo.getIp()); - hostInfoVO.setDisplayIp(hostInfo.getDisplayIp()); - hostInfoVO.setIpDesc(hostInfo.getIpDesc()); - if (hostInfo.getGseAgentAlive() != null) { - hostInfoVO.setAlive(hostInfo.getGseAgentAlive() ? 1 : 0); - } else { - hostInfoVO.setAlive(0); - } - hostInfoVO.setCloudAreaInfo(new CloudAreaInfoVO(hostInfo.getCloudAreaId(), - CloudAreaService.getCloudAreaNameFromCache(hostInfo.getCloudAreaId()))); - hostInfoVO.setOs(hostInfo.getOs()); - return hostInfoVO; - } - - /** - * 将作业平台内部拓扑信息转换为展示用拓扑信息 - * - * @param nodeInfo 作业平台内拓扑结构 - * @return 展示用拓扑结构 - */ - public static NodeInfoVO convertToNodeInfoVO(TopologyNodeInfoDTO nodeInfo) { - NodeInfoVO nodeInfoVO = new NodeInfoVO(); - nodeInfoVO.setId(nodeInfo.getId()); - nodeInfoVO.setName(nodeInfo.getName()); - nodeInfoVO.setNodeType(nodeInfo.getType().getType()); - nodeInfoVO.setIpList(nodeInfo.getIpList()); - if (CollectionUtils.isNotEmpty(nodeInfo.getIpListStatus())) { - nodeInfoVO.setIpListStatus( - nodeInfo.getIpListStatus().parallelStream() - .map(TopologyHelper::convertToHostInfoVO).collect(Collectors.toList())); - } - return nodeInfoVO; - } - - /** - * 将作业平台内部动态分组信息转换为展示用动态分组信息 - * - * @param dynamicGroupWithHost 作业平台内部动态分组信息 - * @return 展示用动态分组信息 - */ - public static DynamicGroupInfoVO convertToDynamicGroupInfoVO(DynamicGroupWithHost dynamicGroupWithHost) { - if (dynamicGroupWithHost == null) { - return null; - } - DynamicGroupInfoVO dynamicGroupInfoVO = new DynamicGroupInfoVO(); - dynamicGroupInfoVO.setScopeType(ResourceScopeTypeEnum.BIZ.getValue()); - dynamicGroupInfoVO.setScopeId(dynamicGroupWithHost.getBizId().toString()); - dynamicGroupInfoVO.setScopeName(dynamicGroupWithHost.getBizName()); - dynamicGroupInfoVO.setId(dynamicGroupWithHost.getId()); - dynamicGroupInfoVO.setOwner(dynamicGroupWithHost.getOwner()); - dynamicGroupInfoVO.setOwnerName(dynamicGroupWithHost.getOwnerName()); - dynamicGroupInfoVO.setName(dynamicGroupWithHost.getName()); - dynamicGroupInfoVO.setType(dynamicGroupWithHost.getType()); - if (dynamicGroupWithHost.getIpListStatus() != null) { - dynamicGroupInfoVO.setIpListStatus(dynamicGroupWithHost.getIpListStatus().parallelStream() - .map(TopologyHelper::convertToHostInfoVO).collect(Collectors.toList())); - } else { - dynamicGroupInfoVO.setIpListStatus(null); - } - return dynamicGroupInfoVO; - } - @PostConstruct private void initCache() { TopologyNameCacheThread topologyNameCacheThread = new TopologyNameCacheThread(); @@ -292,40 +198,6 @@ public InstanceTopologyDTO getTopologyTreeByApplication(ApplicationDTO applicati return instanceTopology; } - /** - * 根据拓扑节点 ID 和类型获取节点名称 - * - * @param appId 业务 ID - * @param nodeId 节点 ID - * @param nodeType 节点类型 - * @return 节点名称 - */ - public String getTopologyNodeName(Long appId, Long nodeId, String nodeType) { - Map> nodeTypeNameMap = BIZ_NODE_TYPE_NAME_MAP.get(appId); - ApplicationDTO appInfo = applicationDAO.getAppById(appId); - if (appInfo == null) { - return String.valueOf(nodeId); - } - if (nodeTypeNameMap == null || nodeTypeNameMap.get(nodeType) == null) { - InstanceTopologyDTO topology = getTopologyTreeByApplication(appInfo); - processTopologyNodeName(topology, null); - } - if (CcNodeTypeEnum.BIZ.getType().equals(nodeType)) { - return appInfo.getName(); - } - nodeTypeNameMap = BIZ_NODE_TYPE_NAME_MAP.get(appId); - if (nodeTypeNameMap != null) { - if (nodeTypeNameMap.get(nodeType) != null) { - String name = nodeTypeNameMap.get(nodeType).get(nodeId); - if (StringUtils.isBlank(name)) { - name = String.valueOf(nodeId); - } - return name; - } - } - return String.valueOf(nodeId); - } - public List getBizSetSubBizIds(ApplicationDTO appInfo) { if (appInfo.isAllBizSet()) { return applicationDAO.listAllBizAppBizIds(); @@ -341,52 +213,6 @@ public List getBizSetSubBizIds(ApplicationDTO appInfo) { } } - /** - * 向主机信息中填充云区域名称 - *

- * 不可删除！ - * - * @param hostInfoList 主机信息列表 - */ - public void fillCloudAreaName(List hostInfoList) { - if (CollectionUtils.isEmpty(hostInfoList)) { - return; - } - for (HostInfoVO hostInfoVO : hostInfoList) { - if (hostInfoVO != null) { - if (hostInfoVO.getCloudAreaInfo() != null) { - hostInfoVO.getCloudAreaInfo() - .setName(cloudAreaService.getCloudAreaName(hostInfoVO.getCloudAreaInfo().getId())); - } - } - } - } - - /** - * 根据 IP 地址列表批量获取机器 Agent 状态 - * - * @param appId 业务 ID - * @param ipList IP 地址列表 - * @return 机器 Agent 状态信息列表 - */ - public List getIpStatusListByIps(long appId, List ipList) { - List ipInfoList = new ArrayList<>(); - if (CollectionUtils.isEmpty(ipList)) { - return ipInfoList; - } - Map agentStatusMap = - queryAgentStatusClient.batchGetAgentStatus(ipList); - for (String ip : ipList) { - ApplicationHostDTO ipInfo = new ApplicationHostDTO(); - ipInfo.setCloudAreaId(Long.valueOf(ip.split(":")[0])); - ipInfo.setBizId(appId); - ipInfo.setIp(ip.split(":")[1]); - ipInfo.setGseAgentAlive(agentStatusMap.get(ip) != null && (agentStatusMap.get(ip).status == 1)); - ipInfoList.add(ipInfo); - } - return ipInfoList; - } - /** * 更新拓扑节点名称信息缓存 * @@ -402,7 +228,7 @@ private void processTopologyNodeName( if (BIZ_NODE_TYPE_NAME_MAP.get(appId) == null) { BIZ_NODE_TYPE_NAME_MAP.put(appId, new ConcurrentHashMap<>(3)); } - // Root node, app, find all child + // root node, app, find all child if (CollectionUtils.isNotEmpty(topology.getChild())) { topology.getChild().forEach(child -> processTopologyNodeName(child, BIZ_NODE_TYPE_NAME_MAP.get(appId))); } @@ -427,6 +253,7 @@ private void processTopologyNodeName( } class TopologyNameCacheThread extends Thread { + @SuppressWarnings("InfiniteLoopStatement") @Override public void run() { this.setName("Topology-Name-Update-Thread"); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/common/client/PAASClientFactory.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/common/client/PAASClientFactory.java deleted file mode 100644 index 227a08797d..0000000000 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/common/client/PAASClientFactory.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.common.client; - -import com.google.common.collect.Maps; -import com.tencent.bk.job.common.esb.config.EsbConfig; -import com.tencent.bk.job.common.paas.user.EEPaasClient; -import com.tencent.bk.job.common.paas.user.IPaasClient; -import com.tencent.bk.job.common.util.ApplicationContextRegister; -import io.micrometer.core.instrument.MeterRegistry; -import lombok.extern.slf4j.Slf4j; - -import java.util.Map; - -@Slf4j -public class PAASClientFactory { - - private static final String CHINESE = "zh_CN"; - private static final String ENGLISH = "en_US"; - private static final String BK_LANG_CN = "zh-cn"; - private static final String BK_LANG_EN = "en"; - private static final String BK_LANG_ALL = "all"; - - private static final Map paasClients = Maps.newHashMap(); - - static { - EsbConfig esbConfig; - MeterRegistry meterRegistry; - try { - esbConfig = ApplicationContextRegister.getBean(EsbConfig.class); - meterRegistry = ApplicationContextRegister.getBean(MeterRegistry.class); - } catch (Exception e) { - log.error("Error while initialize bk config!", e); - throw e; - } - - paasClients.put(CHINESE, new EEPaasClient(esbConfig.getEsbUrl(), esbConfig.getAppCode(), - esbConfig.getAppSecret(), BK_LANG_CN, esbConfig.isUseEsbTestEnv(), meterRegistry)); - paasClients.put(ENGLISH, new EEPaasClient(esbConfig.getEsbUrl(), esbConfig.getAppCode(), - esbConfig.getAppSecret(), BK_LANG_EN, esbConfig.isUseEsbTestEnv(), meterRegistry)); - paasClients.put(BK_LANG_CN, new EEPaasClient(esbConfig.getEsbUrl(), esbConfig.getAppCode(), - esbConfig.getAppSecret(), BK_LANG_CN, esbConfig.isUseEsbTestEnv(), meterRegistry)); - paasClients.put(BK_LANG_EN, new EEPaasClient(esbConfig.getEsbUrl(), esbConfig.getAppCode(), - esbConfig.getAppSecret(), BK_LANG_EN, esbConfig.isUseEsbTestEnv(), meterRegistry)); - paasClients.put(BK_LANG_ALL, new EEPaasClient(esbConfig.getEsbUrl(), esbConfig.getAppCode(), - esbConfig.getAppSecret(), BK_LANG_ALL, esbConfig.isUseEsbTestEnv(), meterRegistry)); - } - - public static IPaasClient getClient(String language) { - if (language == null) { - language = BK_LANG_EN; - } - IPaasClient paasClient = paasClients.get(language); - if (paasClient == null) { - paasClient = paasClients.get(BK_LANG_EN); - } - return paasClient; - } - - - public static IPaasClient getClient() { - return getClient(BK_LANG_EN); - } - - public static void resetTodayStatistics() { - paasClients.entrySet().forEach(entry -> { - IPaasClient paasClient = entry.getValue(); - paasClient.resetTodayStatistics(); - }); - } -} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/common/constants/ScriptTemplateVariableEnum.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/common/constants/ScriptTemplateVariableEnum.java index ae08efda10..471f44e2a2 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/common/constants/ScriptTemplateVariableEnum.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/common/constants/ScriptTemplateVariableEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/common/interceptor/JobManageUriPermissionInterceptor.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/common/interceptor/JobManageUriPermissionInterceptor.java new file mode 100644 index 0000000000..405465beb1 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/common/interceptor/JobManageUriPermissionInterceptor.java @@ -0,0 +1,111 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.common.interceptor; + +import com.tencent.bk.job.common.annotation.JobInterceptor; +import com.tencent.bk.job.common.constant.InterceptorOrder; +import com.tencent.bk.job.common.iam.constant.ActionId; +import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; +import com.tencent.bk.job.common.iam.model.AuthResult; +import com.tencent.bk.job.common.iam.service.AuthService; +import com.tencent.bk.job.common.util.JobContextUtil; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.util.AntPathMatcher; +import org.springframework.util.PathMatcher; +import org.springframework.web.servlet.handler.HandlerInterceptorAdapter; + +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; + +/** + * Uri权限控制拦截 + */ +@Slf4j +@Component +@JobInterceptor(pathPatterns = { + "/web/whiteIP/**", + "/web/notify/users/blacklist", + "/web/globalSettings/**", + "/web/public_script/**", + "/web/public_tag/**", + "/web/serviceInfo/**", + "/web/dangerousRule/**"}, + order = InterceptorOrder.AUTH.AUTH_COMMON) +public class JobManageUriPermissionInterceptor extends HandlerInterceptorAdapter { + private final String URI_PATTERN_WHITE_IP = "/web/whiteIP/**"; + private final String URI_PATTERN_NOTIFY_BLACKLIST = "/web/notify/users/blacklist"; + private final String URI_PATTERN_GLOBAL_SETTINGS = "/web/globalSettings/**"; + private final String URI_PATTERN_PUBLIC_SCRIPT = "/web/public_script/**"; + private final String URI_PATTERN_PUBLIC_TAG = "/web/public_tag/**"; + private final String URI_PATTERN_SERVICE_INFO = "/web/serviceInfo/**"; + private final String URI_PATTERN_DANGEROUS_RULE = "/web/dangerousRule/**"; + private final AuthService authService; + private final PathMatcher pathMatcher; + + @Autowired + public JobManageUriPermissionInterceptor(AuthService authService) { + this.authService = authService; + this.pathMatcher = new AntPathMatcher(); + } + + @Override + public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) { + String username = JobContextUtil.getUsername(); + String uri = request.getRequestURI(); + + //仅超级管理员可使用管理相关接口 + if (pathMatcher.match(URI_PATTERN_NOTIFY_BLACKLIST, uri)) { + AuthResult authResult = authService.auth(username, ActionId.GLOBAL_SETTINGS); + if (!authResult.isPass()) { + throw new PermissionDeniedException(authResult); + } + } else if (pathMatcher.match(URI_PATTERN_GLOBAL_SETTINGS, uri)) { + AuthResult authResult = authService.auth(username, ActionId.GLOBAL_SETTINGS); + if (!authResult.isPass()) { + throw new PermissionDeniedException(authResult); + } + } else if (pathMatcher.match(URI_PATTERN_SERVICE_INFO, uri)) { + AuthResult authResult = authService.auth(username, ActionId.SERVICE_STATE_ACCESS); + if (!authResult.isPass()) { + throw new PermissionDeniedException(authResult); + } + } else if (pathMatcher.match(URI_PATTERN_DANGEROUS_RULE, uri)) { + AuthResult authResult = authService.auth(username, ActionId.HIGH_RISK_DETECT_RULE); + if (!authResult.isPass()) { + throw new PermissionDeniedException(authResult); + } + } + return true; + } + + @Override + public void afterCompletion(HttpServletRequest request, + HttpServletResponse response, + Object handler, Exception ex) { + + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/common/interceptor/UriPermissionInterceptor.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/common/interceptor/UriPermissionInterceptor.java deleted file mode 100644 index 328e970551..0000000000 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/common/interceptor/UriPermissionInterceptor.java +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.common.interceptor; - -import com.tencent.bk.job.common.RequestIdLogger; -import com.tencent.bk.job.common.iam.constant.ActionId; -import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; -import com.tencent.bk.job.common.iam.model.AuthResult; -import com.tencent.bk.job.common.iam.service.AuthService; -import com.tencent.bk.job.common.util.JobContextUtil; -import com.tencent.bk.job.common.util.SimpleRequestIdLogger; -import lombok.extern.slf4j.Slf4j; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Component; -import org.springframework.util.AntPathMatcher; -import org.springframework.util.PathMatcher; -import org.springframework.web.servlet.handler.HandlerInterceptorAdapter; - -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import java.util.Arrays; -import java.util.List; - -/** - * Uri权限控制拦截 - */ -@Slf4j -@Component -public class UriPermissionInterceptor extends HandlerInterceptorAdapter { - private static final RequestIdLogger logger = - new SimpleRequestIdLogger(LoggerFactory.getLogger(UriPermissionInterceptor.class)); - private final String URI_PATTERN_WHITE_IP = "/web/whiteIP/**"; - private final String URI_PATTERN_NOTIFY_BLACKLIST = "/web/notify/users/blacklist"; - private final String URI_PATTERN_GLOBAL_SETTINGS = "/web/globalSettings/**"; - private final String URI_PATTERN_PUBLIC_SCRIPT = "/web/public_script/**"; - private final String URI_PATTERN_PUBLIC_TAG = "/web/public_tag/**"; - private final String URI_PATTERN_SERVICE_INFO = "/web/serviceInfo/**"; - private final String URI_PATTERN_DANGEROUS_RULE = "/web/dangerousRule/**"; - private final AuthService authService; - private final PathMatcher pathMatcher; - - @Autowired - public UriPermissionInterceptor(AuthService authService) { - this.authService = authService; - this.pathMatcher = new AntPathMatcher(); - } - - public String[] getControlUriPatterns() { - Object[] rawArr = getControlUriPatternsList().toArray(); - String[] arr = new String[rawArr.length]; - for (int i = 0; i < rawArr.length; i++) { - arr[i] = (String) rawArr[i]; - } - return arr; - } - - private List getControlUriPatternsList() { - return Arrays.asList( - //IP白名单 - URI_PATTERN_WHITE_IP, - //通知黑名单 - URI_PATTERN_NOTIFY_BLACKLIST, - //全局设置 - URI_PATTERN_GLOBAL_SETTINGS, - // 公共脚本 - URI_PATTERN_PUBLIC_SCRIPT, - // 公共标签 - URI_PATTERN_PUBLIC_TAG, - // 服务状态 - URI_PATTERN_SERVICE_INFO, - // 高危语句规则 - URI_PATTERN_DANGEROUS_RULE - ); - } - - @Override - public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) - throws Exception { - String username = JobContextUtil.getUsername(); - String uri = request.getRequestURI(); - logger.infoWithRequestId("PermissionControlInterceptor.preHandle:username=" + username + ", uri=" + uri + ", " + - "controlUriPatterns=" + getControlUriPatternsList()); - //仅超级管理员可使用管理相关接口 - if (pathMatcher.match(URI_PATTERN_NOTIFY_BLACKLIST, uri)) { - AuthResult authResult = authService.auth(username, ActionId.GLOBAL_SETTINGS); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } - } else if (pathMatcher.match(URI_PATTERN_GLOBAL_SETTINGS, uri)) { - AuthResult authResult = authService.auth(username, ActionId.GLOBAL_SETTINGS); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } - } else if (pathMatcher.match(URI_PATTERN_SERVICE_INFO, uri)) { - AuthResult authResult = authService.auth(username, ActionId.SERVICE_STATE_ACCESS); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } - } else if (pathMatcher.match(URI_PATTERN_DANGEROUS_RULE, uri)) { - AuthResult authResult = authService.auth(username, ActionId.HIGH_RISK_DETECT_RULE); - if (!authResult.isPass()) { - throw new PermissionDeniedException(authResult); - } - } - return true; - } - - @Override - public void afterCompletion(HttpServletRequest request, HttpServletResponse response, Object handler, Exception ex) - throws Exception { - - } -} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/common/util/DbRecordMapper.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/common/util/DbRecordMapper.java index 125155823d..c7e903b57f 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/common/util/DbRecordMapper.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/common/util/DbRecordMapper.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,14 +28,16 @@ import com.tencent.bk.job.common.constant.DuplicateHandlerEnum; import com.tencent.bk.job.common.constant.NotExistPathHandlerEnum; import com.tencent.bk.job.common.model.dto.UserRoleInfoDTO; +import com.tencent.bk.job.common.mysql.util.JooqDataTypeUtil; import com.tencent.bk.job.common.util.json.JsonUtils; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskApprovalTypeEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskFileTypeEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskScriptSourceEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskStepTypeEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskTemplateStatusEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskApprovalTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskFileTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskPlanTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskScriptSourceEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskStepTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskTemplateStatusEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskTypeEnum; import com.tencent.bk.job.manage.model.dto.task.TaskApprovalStepDTO; import com.tencent.bk.job.manage.model.dto.task.TaskFileInfoDTO; import com.tencent.bk.job.manage.model.dto.task.TaskFileStepDTO; @@ -45,15 +47,16 @@ import com.tencent.bk.job.manage.model.dto.task.TaskStepDTO; import com.tencent.bk.job.manage.model.dto.task.TaskTargetDTO; import com.tencent.bk.job.manage.model.dto.task.TaskTemplateInfoDTO; +import com.tencent.bk.job.manage.model.tables.TaskPlan; +import com.tencent.bk.job.manage.model.tables.TaskTemplate; +import com.tencent.bk.job.manage.model.tables.TaskTemplateStepFileList; import lombok.extern.slf4j.Slf4j; +import org.jooq.Record; import org.jooq.Record11; import org.jooq.Record13; import org.jooq.Record15; import org.jooq.Record6; import org.jooq.Record9; -import org.jooq.generated.tables.TaskPlan; -import org.jooq.generated.tables.TaskTemplate; -import org.jooq.generated.tables.TaskTemplateStepFileList; import org.jooq.types.UByte; import org.jooq.types.ULong; @@ -159,11 +162,11 @@ public static TaskScriptStepDTO convertRecordToTaskScriptStep(Record15()); taskFileStep.setDestinationFileLocation((String) record.get(2)); taskFileStep.setExecuteAccount(((ULong) record.get(3)).longValue()); - taskFileStep.setDestinationHostList(TaskTargetDTO.fromString((String) record.get(4))); + taskFileStep.setDestinationHostList(TaskTargetDTO.fromJsonString((String) record.get(4))); taskFileStep.setTimeout(((ULong) record.get(5)).longValue()); taskFileStep.setOriginSpeedLimit(JooqDataTypeUtil.getLongFromULong((ULong) record.get(6))); taskFileStep.setTargetSpeedLimit(JooqDataTypeUtil.getLongFromULong((ULong) record.get(7))); @@ -219,7 +222,7 @@ public static TaskScriptStepDTO convertRecordToTaskScriptStep(Record15> 1) & 0x1) == 1; + } + + public static int computeScriptStatusFlags(boolean isOffline, boolean isDisabled) { + int scriptStatusBinaryFlags = 0; + + if (isOffline) { + scriptStatusBinaryFlags |= 0b1; + } + if (isDisabled) { + scriptStatusBinaryFlags |= 0b10; + } + + return scriptStatusBinaryFlags; + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/ArtifactoryConfig.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/ArtifactoryConfig.java deleted file mode 100644 index 82047aa29a..0000000000 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/ArtifactoryConfig.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.config; - -import lombok.Data; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Configuration; - -@Data -@Configuration -public class ArtifactoryConfig { - - @Value("${artifactory.base-url:}") - private String artifactoryBaseUrl; - - @Value("${artifactory.admin.username:admin}") - private String artifactoryAdminUsername; - - @Value("${artifactory.admin.password:blueking}") - private String artifactoryAdminPassword; - - @Value("${artifactory.job.username:bkjob}") - private String artifactoryJobUsername; - - @Value("${artifactory.job.password:bkjob}") - private String artifactoryJobPassword; - - @Value("${artifactory.job.project:bkjob}") - private String artifactoryJobProject; - -} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/BizSetCmdbClientAutoConfig.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/BizSetCmdbClientAutoConfig.java deleted file mode 100644 index 31683c65fc..0000000000 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/BizSetCmdbClientAutoConfig.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.config; - -import com.tencent.bk.job.common.cc.config.CmdbConfig; -import com.tencent.bk.job.common.cc.sdk.BizSetCmdbClient; -import com.tencent.bk.job.common.esb.config.EsbConfig; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Configuration -public class BizSetCmdbClientAutoConfig { - - @Bean - @Autowired - public BizSetCmdbClient bizSetCmdbClient(EsbConfig esbConfig, CmdbConfig cmdbConfig) { - return new BizSetCmdbClient(esbConfig, cmdbConfig); - } -} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/DbConfig.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/DbConfig.java new file mode 100644 index 0000000000..66b1035c50 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/DbConfig.java @@ -0,0 +1,100 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.config; + +import com.tencent.bk.job.common.mysql.util.JooqConfigurationUtil; +import org.jooq.ConnectionProvider; +import org.jooq.DSLContext; +import org.jooq.SQLDialect; +import org.jooq.impl.DataSourceConnectionProvider; +import org.jooq.impl.DefaultConfiguration; +import org.jooq.impl.DefaultDSLContext; +import org.jooq.impl.DefaultExecuteListenerProvider; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.jdbc.DataSourceBuilder; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.DependsOn; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.datasource.DataSourceTransactionManager; +import org.springframework.jdbc.datasource.TransactionAwareDataSourceProxy; +import org.springframework.transaction.annotation.EnableTransactionManagement; + +import javax.sql.DataSource; + +@Configuration(value = "jobManageDbConfig") +@EnableTransactionManagement +public class DbConfig { + @Qualifier("job-manage-data-source") + @Bean(name = "job-manage-data-source") + @ConfigurationProperties(prefix = "spring.datasource.job-manage") + public DataSource dataSource() { + return DataSourceBuilder.create().build(); + } + + @Qualifier("jobManageTransactionManager") + @Bean(name = "jobManageTransactionManager") + @DependsOn("job-manage-data-source") + public DataSourceTransactionManager transactionManager(@Qualifier("job-manage-data-source") DataSource dataSource) { + return new DataSourceTransactionManager(dataSource); + } + + @Qualifier("job-manage-jdbc-template") + @Bean(name = "job-manage-jdbc-template") + public JdbcTemplate jdbcTemplate(@Qualifier("job-manage-data-source") DataSource dataSource) { + return new JdbcTemplate(dataSource); + } + + @Qualifier("job-manage-dsl-context") + @Bean(name = "job-manage-dsl-context") + public DSLContext dslContext(@Qualifier("job-manage-jooq-conf") org.jooq.Configuration configuration) { + return new DefaultDSLContext(configuration); + } + + @Qualifier("job-manage-jooq-conf") + @Bean(name = "job-manage-jooq-conf") + public org.jooq.Configuration + jooqConf(@Qualifier("job-manage-conn-provider") ConnectionProvider connectionProvider, + DefaultExecuteListenerProvider jooqExecuteListenerProvider + ) { + return JooqConfigurationUtil.getConfiguration(connectionProvider, jooqExecuteListenerProvider); + } + + @Qualifier("job-manage-conn-provider") + @Bean(name = "job-manage-conn-provider") + public ConnectionProvider connectionProvider( + @Qualifier("jobManageTransactionAwareDataSource") DataSource dataSource) { + return new DataSourceConnectionProvider(dataSource); + } + + @Qualifier("jobManageTransactionAwareDataSource") + @Bean(name = "jobManageTransactionAwareDataSource") + public TransactionAwareDataSourceProxy + transactionAwareDataSourceProxy(@Qualifier("job-manage-data-source") DataSource dataSource) { + return new TransactionAwareDataSourceProxy(dataSource); + } + +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/ExecutorConfiguration.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/ExecutorConfiguration.java index b39426705d..cd67e7b063 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/ExecutorConfiguration.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/ExecutorConfiguration.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,8 @@ package com.tencent.bk.job.manage.config; +import com.tencent.bk.job.common.WatchableThreadPoolExecutor; +import io.micrometer.core.instrument.MeterRegistry; import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; @@ -37,31 +39,14 @@ import java.util.concurrent.atomic.AtomicInteger; @Slf4j -@Configuration +@Configuration(value = "jobManageExecutorConfig") public class ExecutorConfiguration { - @Bean("syncAgentStatusExecutor") - public ThreadPoolExecutor syncAgentStatusExecutor() { - ThreadPoolExecutor syncAgentStatusExecutor = new ThreadPoolExecutor( - 5, - 5, - 1L, - TimeUnit.SECONDS, - new ArrayBlockingQueue<>(5000), (r, executor) -> - log.error( - "syncAgentStatusExecutor Runnable rejected! executor.poolSize={}, executor.queueSize={}", - executor.getPoolSize(), - executor.getQueue().size() - ) - ); - syncAgentStatusExecutor.setThreadFactory(getThreadFactoryByNameAndSeq("syncAgentStatusExecutor-", - new AtomicInteger(1))); - return syncAgentStatusExecutor; - } - @Bean("syncHostExecutor") - public ThreadPoolExecutor syncHostExecutor() { - ThreadPoolExecutor syncHostExecutor = new ThreadPoolExecutor( + public ThreadPoolExecutor syncHostExecutor(MeterRegistry meterRegistry) { + ThreadPoolExecutor syncHostExecutor = new WatchableThreadPoolExecutor( + meterRegistry, + "syncHostExecutor", 5, 5, 1L, @@ -79,8 +64,10 @@ public ThreadPoolExecutor syncHostExecutor() { } @Bean("syncAppExecutor") - public ThreadPoolExecutor syncAppExecutor() { - ThreadPoolExecutor syncAppExecutor = new ThreadPoolExecutor( + public ThreadPoolExecutor syncAppExecutor(MeterRegistry meterRegistry) { + ThreadPoolExecutor syncAppExecutor = new WatchableThreadPoolExecutor( + meterRegistry, + "syncAppExecutor", 5, 5, 1L, @@ -99,8 +86,10 @@ public ThreadPoolExecutor syncAppExecutor() { } @Bean("notifySendExecutor") - public ThreadPoolExecutor notifySendExecutor() { - return new ThreadPoolExecutor( + public ThreadPoolExecutor notifySendExecutor(MeterRegistry meterRegistry) { + return new WatchableThreadPoolExecutor( + meterRegistry, + "notifySendExecutor", 5, 30, 60L, @@ -110,19 +99,28 @@ public ThreadPoolExecutor notifySendExecutor() { } @Bean("scriptCheckExecutor") - public ExecutorService scriptCheckExecutor() { - return new ThreadPoolExecutor( + public ExecutorService scriptCheckExecutor(MeterRegistry meterRegistry) { + return new WatchableThreadPoolExecutor( + meterRegistry, + "scriptCheckExecutor", 10, 50, 60L, TimeUnit.SECONDS, - new LinkedBlockingQueue<>() + new LinkedBlockingQueue<>(1000), + (r, executor) -> + log.error( + "scriptCheckExecutor Runnable rejected! executor.poolSize={}, executor.queueSize={}", + executor.getPoolSize(), executor.getQueue().size() + ) ); } @Bean("adminAuthExecutor") - public ThreadPoolExecutor adminAuthExecutor() { - return new ThreadPoolExecutor( + public ThreadPoolExecutor adminAuthExecutor(MeterRegistry meterRegistry) { + return new WatchableThreadPoolExecutor( + meterRegistry, + "adminAuthExecutor", 5, 5, 30, @@ -131,6 +129,19 @@ public ThreadPoolExecutor adminAuthExecutor() { ); } + @Bean("initRunnerExecutor") + public ThreadPoolExecutor initRunnerExecutor(MeterRegistry meterRegistry) { + return new WatchableThreadPoolExecutor( + meterRegistry, + "initRunnerExecutor", + 0, + 5, + 1, + TimeUnit.SECONDS, + new LinkedBlockingQueue<>() + ); + } + private ThreadFactory getThreadFactoryByNameAndSeq(String namePrefix, AtomicInteger seq) { return r -> { Thread t = new Thread(Thread.currentThread().getThreadGroup(), r, diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/GseClientAutoConfig.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/GseClientAutoConfig.java deleted file mode 100644 index 9f2f8d0641..0000000000 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/GseClientAutoConfig.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.config; - -import com.tencent.bk.job.common.gse.config.GseConfig; -import com.tencent.bk.job.common.gse.service.QueryAgentStatusClient; -import com.tencent.bk.job.common.gse.service.QueryAgentStatusClientImpl; -import io.micrometer.core.instrument.MeterRegistry; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.boot.autoconfigure.AutoConfigureAfter; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -@Configuration -@AutoConfigureAfter(GseConfigForManage.class) -public class GseClientAutoConfig { - - @Bean - @Autowired - public QueryAgentStatusClient gseCacheClient(GseConfigForManage gseConfigForManage, - MeterRegistry meterRegistry) { - GseConfig gseConfig = new GseConfig(); - gseConfig.setEnableSsl(gseConfigForManage.isEnableSsl()); - gseConfig.setGseCacheApiServerHost(gseConfigForManage.getGseCacheApiServerHosts()); - gseConfig.setGseCacheApiServerPort(gseConfigForManage.getGseCacheApiServerPort()); - gseConfig.setKeyStore(gseConfigForManage.getKeyStore()); - gseConfig.setKeyStorePass(gseConfigForManage.getKeyStorePass()); - gseConfig.setTrustStore(gseConfigForManage.getTrustStore()); - gseConfig.setTrustStorePass(gseConfigForManage.getTrustStorePass()); - gseConfig.setTrustStoreType(gseConfigForManage.getTrustStoreType()); - gseConfig.setTrustManagerType(gseConfigForManage.getTrustManagerType()); - gseConfig.setQueryBatchSize(gseConfigForManage.getGseQueryBatchSize()); - gseConfig.setQueryThreadsNum(gseConfigForManage.getGseQueryThreadsNum()); - return new QueryAgentStatusClientImpl(gseConfig, meterRegistry); - } -} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/GseConfig.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/GseConfig.java new file mode 100644 index 0000000000..7c6fb2cfe2 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/GseConfig.java @@ -0,0 +1,103 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.config; + +import com.tencent.bk.job.common.gse.config.AgentStateQueryConfig; +import com.tencent.bk.job.common.gse.constants.DefaultBeanNames; +import com.tencent.bk.job.common.gse.service.AgentStateClient; +import com.tencent.bk.job.common.gse.service.AutoChoosingAgentStateClientImpl; +import com.tencent.bk.job.common.gse.service.BizHostInfoQueryService; +import com.tencent.bk.job.common.gse.service.GseV1AgentStateClientImpl; +import com.tencent.bk.job.common.gse.service.GseV2AgentStateClientImpl; +import com.tencent.bk.job.common.gse.service.UseV2ByFeatureAgentStateClientImpl; +import com.tencent.bk.job.common.gse.v1.GseV1ApiClient; +import com.tencent.bk.job.common.gse.v2.GseV2ApiClient; +import org.springframework.beans.factory.ObjectProvider; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Primary; + +import java.util.concurrent.ThreadPoolExecutor; + +@Configuration(value = "jobManageGseConfig") +public class GseConfig { + + public static final String MANAGE_BEAN_PREFIX = "jobManage"; + public static final String MANAGE_BEAN_AGENT_STATE_CLIENT = MANAGE_BEAN_PREFIX + "AgentStateClient"; + public static final String MANAGE_BEAN_GSE_V1_AGENT_STATE_CLIENT = MANAGE_BEAN_PREFIX + "GseV1AgentStateClient"; + public static final String MANAGE_BEAN_GSE_V2_AGENT_STATE_CLIENT = MANAGE_BEAN_PREFIX + "GseV2AgentStateClient"; + public static final String MANAGE_BEAN_USE_V2_BY_FEATURE_AGENT_STATE_CLIENT = + MANAGE_BEAN_PREFIX + DefaultBeanNames.USE_V2_BY_FEATURE_AGENT_STATE_CLIENT; + + @Bean(MANAGE_BEAN_GSE_V1_AGENT_STATE_CLIENT) + public GseV1AgentStateClientImpl gseV1AgentStateClient(AgentStateQueryConfig agentStateQueryConfig, + ObjectProvider gseV1ApiClient, + @Qualifier(DefaultBeanNames.AGENT_STATUS_QUERY_THREAD_POOL_EXECUTOR) + ThreadPoolExecutor threadPoolExecutor) { + return new GseV1AgentStateClientImpl( + agentStateQueryConfig, + gseV1ApiClient.getIfAvailable(), + threadPoolExecutor + ); + } + + @Bean(MANAGE_BEAN_GSE_V2_AGENT_STATE_CLIENT) + public GseV2AgentStateClientImpl gseV2AgentStateClient(AgentStateQueryConfig agentStateQueryConfig, + ObjectProvider gseV2ApiClient, + @Qualifier(DefaultBeanNames.AGENT_STATUS_QUERY_THREAD_POOL_EXECUTOR) + ThreadPoolExecutor threadPoolExecutor) { + return new GseV2AgentStateClientImpl( + agentStateQueryConfig, + gseV2ApiClient.getIfAvailable(), + threadPoolExecutor + ); + } + + @Bean(MANAGE_BEAN_USE_V2_BY_FEATURE_AGENT_STATE_CLIENT) + public AgentStateClient useV2ByFeatureAgentStateClient(@Qualifier(MANAGE_BEAN_GSE_V1_AGENT_STATE_CLIENT) + GseV1AgentStateClientImpl gseV1AgentStateClient, + @Qualifier(MANAGE_BEAN_GSE_V2_AGENT_STATE_CLIENT) + GseV2AgentStateClientImpl gseV2AgentStateClient, + @Qualifier("jobManageBizHostInfoQueryService") + BizHostInfoQueryService bizHostInfoQueryService) { + return new UseV2ByFeatureAgentStateClientImpl( + gseV1AgentStateClient, + gseV2AgentStateClient, + bizHostInfoQueryService + ); + } + + @Primary + @Bean(MANAGE_BEAN_AGENT_STATE_CLIENT) + public AgentStateClient AutoChoosingAgentStateClientImpl( + @Qualifier(DefaultBeanNames.PREFER_V2_AGENT_STATE_CLIENT) + AgentStateClient preferV2AgentStateClient, + @Qualifier(MANAGE_BEAN_USE_V2_BY_FEATURE_AGENT_STATE_CLIENT) + AgentStateClient useV2ByFeatureAgentStateClient + ) { + return new AutoChoosingAgentStateClientImpl(preferV2AgentStateClient, useV2ByFeatureAgentStateClient); + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/GseConfigForManage.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/GseConfigForManage.java deleted file mode 100644 index 256968d27a..0000000000 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/GseConfigForManage.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.config; - -import lombok.Getter; -import lombok.Setter; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.context.annotation.Configuration; - -@Getter -@Setter -@Configuration -public class GseConfigForManage { - @Value("${gse.cache.apiserver.host}") - private String[] gseCacheApiServerHosts; - @Value("${gse.cache.apiserver.port:59313}") - private int gseCacheApiServerPort; - - @Value("${gse.ssl.enabled:true}") - private boolean enableSsl; - - @Value("${gse.ssl.keystore.path:/data/bkee/cert/gse_job_api_client.keystore}") - private String keyStore; - @Value("${gse.ssl.keystore.password:}") - private String keyStorePass; - - @Value("${gse.ssl.truststore.path:/data/bkee/cert/gse_job_api_client.truststore}") - private String trustStore; - @Value("${gse.ssl.truststore.password:}") - private String trustStorePass; - - @Value("${gse.ssl.truststore.manager-type:SunX509}") - private String trustManagerType; - @Value("${gse.ssl.truststore.store-type:JKS}") - private String trustStoreType; - - /** - * GSE 公钥证书BASE64 - */ - private String gsePublicKeyPermBase64 = - "LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUlHZk1BMEdDU3FHU0liM0RRRUJBUVVBQTRHTkFEQ0JpUUtCZ1FETEVXZk9" + - "YU2VvMXpNQ1JpRVNFTWs3OXo0cwpHYkw4VmIvZXg5K1RaR2VyN255bEh5Y0Vtb2o5aWE4K2daTmVQOFRRVmRyTExhSz" + - "IzektiT3lja2FiVE5QS0VZCmhQY0NlellEQVdleTZBS2ZHSCtYZGV0MnJDOWtzRWhrM1BqcDVuZDk4QW1KZ0VJeSt6S" + - "0FhaVZEazFvdG5Jc0EKRWxucUdXL24zaWVuN0hmSXN3SURBUUFCCi0tLS0tRU5EIFBVQkxJQyBLRVktLS0tLQo="; - - @Value("${gse.query.threads.num:5}") - private int gseQueryThreadsNum; - - @Value("${gse.query.batchSize:5000}") - private int gseQueryBatchSize; -} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/JobManageAutoConfiguration.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/JobManageAutoConfiguration.java deleted file mode 100644 index 790ab63dd7..0000000000 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/JobManageAutoConfiguration.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.config; - -import com.tencent.bk.job.common.artifactory.sdk.ArtifactoryClient; -import com.tencent.bk.job.common.encrypt.Encryptor; -import com.tencent.bk.job.common.encrypt.RSAEncryptor; -import com.tencent.bk.job.common.esb.metrics.EsbApiTimedAspect; -import io.micrometer.core.instrument.MeterRegistry; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; - -import java.io.IOException; -import java.security.GeneralSecurityException; - -@Configuration -public class JobManageAutoConfiguration { - @Bean - public ArtifactoryClient artifactoryClient(@Autowired ArtifactoryConfig artifactoryConfig, - @Autowired MeterRegistry meterRegistry) { - return new ArtifactoryClient( - artifactoryConfig.getArtifactoryBaseUrl(), - artifactoryConfig.getArtifactoryJobUsername(), - artifactoryConfig.getArtifactoryJobPassword(), - meterRegistry); - } - - static class CmdbConfigSetter { - CmdbConfigSetter() { - } - } - - @Bean - public EsbApiTimedAspect esbApiTimedAspect(@Autowired MeterRegistry meterRegistry) { - return new EsbApiTimedAspect(meterRegistry); - } - - @Bean("gseRsaEncryptor") - public Encryptor rsaEncryptor(@Autowired GseConfigForManage config) throws IOException, GeneralSecurityException { - return new RSAEncryptor(config.getGsePublicKeyPermBase64()); - } -} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/JobManageConfig.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/JobManageConfig.java index 41819ca7fd..eaf5644afb 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/JobManageConfig.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/JobManageConfig.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -50,21 +50,30 @@ public class JobManageConfig { @Value("${job.edition:ee}") private String jobEdition; - @Value("${job.web.url:}") - private String jobWebUrl; - @Value("${bk.doc.root:}") private String bkDocRoot; @Value("${bk.feedback.root:}") private String bkFeedBackRoot; + @Value("${bk.sharedResUrl:}") + private String bkSharedResUrl; + + @Value("${bk.sharedBaseJsPath:/bk_job/base.js}") + private String bkSharedBaseJsPath; + @Value("${bk.ce.root:https://bk.tencent.com}") private String bkCERoot; @Value("${job.manage.sync.app.enabled:true}") private boolean enableSyncApp; + @Value("${job.manage.sync.app.deleteProtect.enabled:false}") + private boolean enableAppDeleteProtect; + + @Value("${job.manage.sync.app.deleteProtect.maxRatio:0.5}") + private float maxAppDeleteRatio; + @Value("${job.manage.sync.host.enabled:true}") private boolean enableSyncHost; @@ -77,9 +86,6 @@ public class JobManageConfig { @Value("${job.manage.sync.hostEvent.handlerNum:3}") private int hostEventHandlerNum; - @Value("${swagger.url:swagger.job.com}") - private String swaggerUrl; - //---------------------------- Cmdb Config --------------------------------- @Value("${cmdb.default.supplier.account:0}") private String defaultSupplierAccount; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/JobTicketConfig.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/JobTicketConfig.java index 1d3fcf0adb..2d3461a63b 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/JobTicketConfig.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/JobTicketConfig.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/LocalFileConfigForManage.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/LocalFileConfigForManage.java index 67d6671a67..63df4a23e8 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/LocalFileConfigForManage.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/LocalFileConfigForManage.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -38,6 +38,9 @@ public class LocalFileConfigForManage { @Value("${local-file.artifactory.repo:localupload}") private String localUploadRepo; + @Value("${local-file.expire-delete:true}") + private boolean expireDelete = true; + @Value("${local-file.expire-days:7}") private Integer expireDays; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/ScheduleConfig.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/ScheduleConfig.java new file mode 100644 index 0000000000..f851d82b84 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/ScheduleConfig.java @@ -0,0 +1,49 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.config; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.annotation.Configuration; +import org.springframework.scheduling.annotation.SchedulingConfigurer; +import org.springframework.scheduling.config.ScheduledTaskRegistrar; + +import java.util.concurrent.ScheduledThreadPoolExecutor; + +/** + * @Description + * @Date 2020/2/24 + * @Version 1.0 + */ + +//设定一个长度5的定时任务线程池 +@Slf4j +@Configuration +public class ScheduleConfig implements SchedulingConfigurer { + @Override + public void configureTasks(ScheduledTaskRegistrar taskRegistrar) { + taskRegistrar.setScheduler(new ScheduledThreadPoolExecutor(5, (r, executor) -> log.error( + "ScheduledThreadPoolExecutor rejected a runnable"))); + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/StorageSystemConfig.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/StorageSystemConfig.java index d9a23a44db..f1f5a9c6bf 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/StorageSystemConfig.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/StorageSystemConfig.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -32,7 +32,7 @@ * 存储配置 */ @Getter -@Configuration +@Configuration("jobManageStorageSystemConfig") public class StorageSystemConfig { @Value("${job.storage.root-path:/data/bkee/job/data}") private String jobStorageRootPath; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/listener/ApplicationReadyEventListener.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/listener/ApplicationReadyEventListener.java deleted file mode 100644 index 0cf04a4074..0000000000 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/listener/ApplicationReadyEventListener.java +++ /dev/null @@ -1,183 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.config.listener; - -import com.tencent.bk.job.common.paas.model.EsbNotifyChannelDTO; -import com.tencent.bk.job.common.util.StringUtil; -import com.tencent.bk.job.manage.common.consts.notify.ExecuteStatusEnum; -import com.tencent.bk.job.manage.common.consts.notify.JobRoleEnum; -import com.tencent.bk.job.manage.common.consts.notify.NotifyConsts; -import com.tencent.bk.job.manage.common.consts.notify.ResourceTypeEnum; -import com.tencent.bk.job.manage.common.consts.notify.TriggerTypeEnum; -import com.tencent.bk.job.manage.dao.notify.AvailableEsbChannelDAO; -import com.tencent.bk.job.manage.dao.notify.NotifyTriggerPolicyDAO; -import com.tencent.bk.job.manage.model.dto.notify.AvailableEsbChannelDTO; -import com.tencent.bk.job.manage.model.web.request.notify.NotifyPoliciesCreateUpdateReq; -import com.tencent.bk.job.manage.model.web.request.notify.ResourceStatusChannel; -import com.tencent.bk.job.manage.model.web.request.notify.TriggerPolicy; -import com.tencent.bk.job.manage.service.GlobalSettingsService; -import com.tencent.bk.job.manage.service.NotifyService; -import com.tencent.bk.job.manage.service.PaaSService; -import lombok.extern.slf4j.Slf4j; -import org.jooq.DSLContext; -import org.jooq.impl.DSL; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.context.event.ApplicationReadyEvent; -import org.springframework.context.ApplicationListener; -import org.springframework.lang.NonNull; -import org.springframework.stereotype.Component; - -import java.io.IOException; -import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -@Slf4j -@Component -public class ApplicationReadyEventListener implements ApplicationListener { - - final PaaSService paaSService; - final GlobalSettingsService globalSettingsService; - final NotifyService notifyService; - final AvailableEsbChannelDAO availableEsbChannelDAO; - final NotifyTriggerPolicyDAO notifyTriggerPolicyDAO; - final DSLContext dslContext; - - @Value("${notify.default.channels.available:mail,weixin,rtx}") - private final String defaultAvailableNotifyChannelsStr = "mail,weixin,rtx"; - - @Autowired - public ApplicationReadyEventListener(PaaSService paaSService, GlobalSettingsService globalSettingsService, - NotifyService notifyService, AvailableEsbChannelDAO availableEsbChannelDAO, - NotifyTriggerPolicyDAO notifyTriggerPolicyDAO, DSLContext dslContext) { - this.paaSService = paaSService; - this.globalSettingsService = globalSettingsService; - this.notifyService = notifyService; - this.availableEsbChannelDAO = availableEsbChannelDAO; - this.notifyTriggerPolicyDAO = notifyTriggerPolicyDAO; - this.dslContext = dslContext; - } - - @Override - public void onApplicationEvent(@NonNull ApplicationReadyEvent event) { - // 应用启动完成后的一些初始化操作 - // 1.消息通知默认配置 - initDefaultNotifyChannels(); - // 2.用户侧默认配置 - try { - if (notifyTriggerPolicyDAO.countDefaultPolicies() == 0) { - // - NotifyPoliciesCreateUpdateReq req = new NotifyPoliciesCreateUpdateReq(); - List triggerPolicyList = new ArrayList<>(); - //1.页面执行策略 - TriggerPolicy pageExecuteTriggerPolicy = new TriggerPolicy(); - pageExecuteTriggerPolicy.setTriggerType(TriggerTypeEnum.PAGE_EXECUTE); - setDefaultTargetAndChannels(pageExecuteTriggerPolicy); - //2.API调用策略 - TriggerPolicy apiInvokeTriggerPolicy = new TriggerPolicy(); - apiInvokeTriggerPolicy.setTriggerType(TriggerTypeEnum.API_INVOKE); - setDefaultTargetAndChannels(apiInvokeTriggerPolicy); - //3.定时任务策略 - TriggerPolicy timerTaskTriggerPolicy = new TriggerPolicy(); - timerTaskTriggerPolicy.setTriggerType(TriggerTypeEnum.TIMER_TASK); - setDefaultTargetAndChannels(timerTaskTriggerPolicy); - //策略集成 - triggerPolicyList.add(pageExecuteTriggerPolicy); - triggerPolicyList.add(apiInvokeTriggerPolicy); - triggerPolicyList.add(timerTaskTriggerPolicy); - req.setTriggerPoliciesList(triggerPolicyList); - notifyService.saveAppDefaultNotifyPoliciesToLocal( - NotifyConsts.JOB_ADMINISTRATOR_NAME, - NotifyConsts.DEFAULT_APP_ID, - NotifyConsts.JOB_ADMINISTRATOR_NAME, - req - ); - } else { - log.info("default notify policies already configured"); - } - } catch (Exception e) { - log.error("Fail to config default notify policies", e); - } - } - - private void initDefaultNotifyChannels() { - try { - log.info("init default notify channels"); - List esbNotifyChannelDTOList = paaSService.getAllChannelList("", "100"); - if (esbNotifyChannelDTOList == null) { - log.error("Fail to get notify channels from esb, null"); - return; - } - dslContext.transaction(configuration -> { - DSLContext context = DSL.using(configuration); - if (!globalSettingsService.isNotifyChannelConfiged(context)) { - globalSettingsService.setNotifyChannelConfiged(context); - availableEsbChannelDAO.deleteAll(context); - for (EsbNotifyChannelDTO esbNotifyChannelDTO : esbNotifyChannelDTOList) { - if (!esbNotifyChannelDTO.isActive()) { - continue; - } - availableEsbChannelDAO.insertAvailableEsbChannel( - context, - new AvailableEsbChannelDTO( - esbNotifyChannelDTO.getType(), - true, - "admin", - LocalDateTime.now() - ) - ); - } - } - }); - } catch (IOException e) { - log.error("Fail to get notify channels from esb", e); - } catch (Exception e) { - log.error("Fail to init default notify channels", e); - } - } - - private void setDefaultTargetAndChannels(TriggerPolicy policy) { - //默认通知任务执行人 - policy.setRoleList( - Collections.singletonList(JobRoleEnum.JOB_RESOURCE_TRIGGER_USER.name()) - ); - policy.setExtraObserverList(Collections.emptyList()); - //三种资源类型默认全部选中 - List pageExecuteResourceTypeList = new ArrayList<>(); - pageExecuteResourceTypeList.add(ResourceTypeEnum.SCRIPT); - pageExecuteResourceTypeList.add(ResourceTypeEnum.FILE); - pageExecuteResourceTypeList.add(ResourceTypeEnum.JOB); - policy.setResourceTypeList(pageExecuteResourceTypeList); - //成功、失败均默认邮件通知 - List resourceStatusChannelList = new ArrayList<>(); - List defaultAvailableChannelList = StringUtil.strToList(defaultAvailableNotifyChannelsStr, - String.class, "[,;]"); - resourceStatusChannelList.add(new ResourceStatusChannel(ExecuteStatusEnum.FAIL, - defaultAvailableChannelList)); - policy.setResourceStatusChannelList(resourceStatusChannelList); - } -} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/listener/NotifyInitListener.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/listener/NotifyInitListener.java new file mode 100644 index 0000000000..888486bbed --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/config/listener/NotifyInitListener.java @@ -0,0 +1,178 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.config.listener; + +import com.tencent.bk.job.common.mysql.JobTransactional; +import com.tencent.bk.job.common.paas.cmsi.CmsiApiClient; +import com.tencent.bk.job.common.paas.model.EsbNotifyChannelDTO; +import com.tencent.bk.job.common.util.StringUtil; +import com.tencent.bk.job.manage.api.common.constants.notify.ExecuteStatusEnum; +import com.tencent.bk.job.manage.api.common.constants.notify.JobRoleEnum; +import com.tencent.bk.job.manage.api.common.constants.notify.NotifyConsts; +import com.tencent.bk.job.manage.api.common.constants.notify.ResourceTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.notify.TriggerTypeEnum; +import com.tencent.bk.job.manage.dao.notify.AvailableEsbChannelDAO; +import com.tencent.bk.job.manage.dao.notify.NotifyTriggerPolicyDAO; +import com.tencent.bk.job.manage.model.dto.notify.AvailableEsbChannelDTO; +import com.tencent.bk.job.manage.model.web.request.notify.NotifyPoliciesCreateUpdateReq; +import com.tencent.bk.job.manage.model.web.request.notify.ResourceStatusChannel; +import com.tencent.bk.job.manage.model.web.request.notify.TriggerPolicy; +import com.tencent.bk.job.manage.service.NotifyService; +import com.tencent.bk.job.manage.service.globalsetting.GlobalSettingsService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.context.event.ApplicationReadyEvent; +import org.springframework.context.ApplicationListener; +import org.springframework.context.annotation.Profile; +import org.springframework.lang.NonNull; +import org.springframework.stereotype.Component; + +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +@Slf4j +@Component +@Profile("!test") +public class NotifyInitListener implements ApplicationListener { + + private final CmsiApiClient cmsiApiClient; + private final GlobalSettingsService globalSettingsService; + private final NotifyService notifyService; + private final AvailableEsbChannelDAO availableEsbChannelDAO; + private final NotifyTriggerPolicyDAO notifyTriggerPolicyDAO; + + @Value("${job.manage.notify.default.channels.available:mail,weixin,rtx}") + private final String defaultAvailableNotifyChannelsStr = "mail,weixin,rtx"; + + @Autowired + public NotifyInitListener(CmsiApiClient cmsiApiClient, + GlobalSettingsService globalSettingsService, + NotifyService notifyService, + AvailableEsbChannelDAO availableEsbChannelDAO, + NotifyTriggerPolicyDAO notifyTriggerPolicyDAO) { + this.cmsiApiClient = cmsiApiClient; + this.globalSettingsService = globalSettingsService; + this.notifyService = notifyService; + this.availableEsbChannelDAO = availableEsbChannelDAO; + this.notifyTriggerPolicyDAO = notifyTriggerPolicyDAO; + } + + @Override + public void onApplicationEvent(@NonNull ApplicationReadyEvent event) { + // 应用启动完成后的一些初始化操作 + // 1.消息通知默认配置 + initDefaultNotifyChannels(); + // 2.用户侧默认配置 + try { + if (notifyTriggerPolicyDAO.countDefaultPolicies() == 0) { + // + NotifyPoliciesCreateUpdateReq req = new NotifyPoliciesCreateUpdateReq(); + List triggerPolicyList = new ArrayList<>(); + //1.页面执行策略 + TriggerPolicy pageExecuteTriggerPolicy = new TriggerPolicy(); + pageExecuteTriggerPolicy.setTriggerType(TriggerTypeEnum.PAGE_EXECUTE); + setDefaultTargetAndChannels(pageExecuteTriggerPolicy); + //2.API调用策略 + TriggerPolicy apiInvokeTriggerPolicy = new TriggerPolicy(); + apiInvokeTriggerPolicy.setTriggerType(TriggerTypeEnum.API_INVOKE); + setDefaultTargetAndChannels(apiInvokeTriggerPolicy); + //3.定时任务策略 + TriggerPolicy timerTaskTriggerPolicy = new TriggerPolicy(); + timerTaskTriggerPolicy.setTriggerType(TriggerTypeEnum.TIMER_TASK); + setDefaultTargetAndChannels(timerTaskTriggerPolicy); + //策略集成 + triggerPolicyList.add(pageExecuteTriggerPolicy); + triggerPolicyList.add(apiInvokeTriggerPolicy); + triggerPolicyList.add(timerTaskTriggerPolicy); + req.setTriggerPoliciesList(triggerPolicyList); + notifyService.saveAppDefaultNotifyPoliciesToLocal( + NotifyConsts.JOB_ADMINISTRATOR_NAME, + NotifyConsts.DEFAULT_APP_ID, + NotifyConsts.JOB_ADMINISTRATOR_NAME, + req + ); + } else { + log.info("default notify policies already configured"); + } + } catch (Exception e) { + log.error("Fail to config default notify policies", e); + } + } + + private void initDefaultNotifyChannels() { + log.info("init default notify channels"); + List esbNotifyChannelDTOList = cmsiApiClient.getNotifyChannelList(); + if (esbNotifyChannelDTOList == null) { + log.error("Fail to get notify channels from esb, null"); + return; + } + saveDefaultNotifyChannelsToDb(esbNotifyChannelDTOList); + } + + @JobTransactional(transactionManager = "jobManageTransactionManager") + public void saveDefaultNotifyChannelsToDb(List esbNotifyChannelDTOList) { + if (!globalSettingsService.isNotifyChannelConfiged()) { + globalSettingsService.setNotifyChannelConfiged(); + availableEsbChannelDAO.deleteAll(); + for (EsbNotifyChannelDTO esbNotifyChannelDTO : esbNotifyChannelDTOList) { + if (!esbNotifyChannelDTO.isActive()) { + continue; + } + availableEsbChannelDAO.insertAvailableEsbChannel( + new AvailableEsbChannelDTO( + esbNotifyChannelDTO.getType(), + true, + "admin", + LocalDateTime.now() + ) + ); + } + } + } + + private void setDefaultTargetAndChannels(TriggerPolicy policy) { + //默认通知任务执行人 + policy.setRoleList( + Collections.singletonList(JobRoleEnum.JOB_RESOURCE_TRIGGER_USER.name()) + ); + policy.setExtraObserverList(Collections.emptyList()); + //三种资源类型默认全部选中 + List pageExecuteResourceTypeList = new ArrayList<>(); + pageExecuteResourceTypeList.add(ResourceTypeEnum.SCRIPT); + pageExecuteResourceTypeList.add(ResourceTypeEnum.FILE); + pageExecuteResourceTypeList.add(ResourceTypeEnum.JOB); + policy.setResourceTypeList(pageExecuteResourceTypeList); + //成功、失败均默认邮件通知 + List resourceStatusChannelList = new ArrayList<>(); + List defaultAvailableChannelList = StringUtil.strToList(defaultAvailableNotifyChannelsStr, + String.class, "[,;]"); + resourceStatusChannelList.add(new ResourceStatusChannel(ExecuteStatusEnum.FAIL, + defaultAvailableChannelList)); + policy.setResourceStatusChannelList(resourceStatusChannelList); + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/crypto/CredentialCryptoService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/crypto/CredentialCryptoService.java new file mode 100644 index 0000000000..172ef60d6c --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/crypto/CredentialCryptoService.java @@ -0,0 +1,74 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.crypto; + +import com.tencent.bk.job.common.crypto.CryptoScenarioEnum; +import com.tencent.bk.job.common.crypto.JobCryptorNames; +import com.tencent.bk.job.common.crypto.SymmetricCryptoService; +import com.tencent.bk.sdk.crypto.cryptor.consts.CryptorNames; +import com.tencent.bk.sdk.crypto.util.CryptorMetaUtil; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +/** + * DB账号密码相关加解密服务 + */ +@Slf4j +@Service +public class CredentialCryptoService { + + private final SymmetricCryptoService symmetricCryptoService; + + @Autowired + public CredentialCryptoService(SymmetricCryptoService symmetricCryptoService) { + this.symmetricCryptoService = symmetricCryptoService; + } + + public String getCredentialEncryptAlgorithmByCipher(String cipher) { + if (StringUtils.isEmpty(cipher)) { + return CryptorNames.NONE; + } + String algorithm = CryptorMetaUtil.getCryptorNameFromCipher(cipher); + if (algorithm != null) { + return algorithm; + } + return JobCryptorNames.AES_CBC; + } + + public String encryptCredential(String credentialValue) { + return symmetricCryptoService.encryptToBase64Str(credentialValue, CryptoScenarioEnum.CREDENTIAL); + } + + public String decryptCredential(String encryptedCredential) { + String algorithm = getCredentialEncryptAlgorithmByCipher(encryptedCredential); + if (StringUtils.isBlank(algorithm)) { + return encryptedCredential; + } + return symmetricCryptoService.decrypt(encryptedCredential, algorithm); + } + +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/AbstractTaskStepDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/AbstractTaskStepDAO.java index 10d413ae0a..e339a36aa7 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/AbstractTaskStepDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/AbstractTaskStepDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/AccountDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/AccountDAO.java index b8b8e69e2e..5b9946685a 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/AccountDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/AccountDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,10 +24,10 @@ package com.tencent.bk.job.manage.dao; +import com.tencent.bk.job.common.constant.AccountCategoryEnum; import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; -import com.tencent.bk.job.manage.common.consts.account.AccountCategoryEnum; -import com.tencent.bk.job.manage.common.consts.account.AccountTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.account.AccountTypeEnum; import com.tencent.bk.job.manage.model.dto.AccountDTO; import com.tencent.bk.job.manage.model.dto.AccountDisplayDTO; @@ -41,7 +41,7 @@ public interface AccountDAO { /** * 新增账号 * - * @param account + * @param account 账号信息 * @return 账号ID */ long saveAccountWithId(AccountDTO account); @@ -49,7 +49,7 @@ public interface AccountDAO { /** * 新增账号 * - * @param account + * @param account 账号信息 * @return 账号ID */ long saveAccount(AccountDTO account); @@ -57,57 +57,57 @@ public interface AccountDAO { /** * 根据ID获取账号信息 * - * @param accountId - * @return + * @param accountId 账号ID + * @return 账号信息 */ AccountDTO getAccountById(Long accountId); /** * 根据ID批量获取账号信息 * - * @param accountIds - * @return + * @param accountIds 账号ID集合 + * @return 账号展示信息列表 */ List listAccountDisplayInfoByIds(Collection accountIds); /** * 根据appId与account获取账号信息 * - * @param appId - * @param account - * @return + * @param appId Job业务ID + * @param account 账号名 + * @return 账号信息 */ AccountDTO getAccountByAccount(Long appId, String account); /** * 更新账号 * - * @param account + * @param account 账号信息 */ void updateAccount(AccountDTO account); /** * 删除账号 * - * @param accountId + * @param accountId 账号ID */ void deleteAccount(Long accountId); /** * 分页查询账号列表 * - * @param accountQuery - * @param baseSearchCondition - * @return + * @param accountQuery 账号查询条件 + * @param baseSearchCondition 基础搜索条件 + * @return 账号分页数据 */ PageData listPageAccount(AccountDTO accountQuery, BaseSearchCondition baseSearchCondition); /** * 分页搜索账号列表 * - * @param keyword - * @param baseSearchCondition - * @return + * @param keyword 关键字 + * @param baseSearchCondition 基础搜索条件 + * @return 账号分页数据 */ PageData searchPageAccount(Long appId, String keyword, BaseSearchCondition baseSearchCondition); @@ -116,12 +116,17 @@ public interface AccountDAO { * * @param appId 业务ID * @param category 账号类型，如果传入null，表示所有类型 - * @return + * @param account 账号名称 + * @param alias 账号别名 + * @return 账号列表 */ - List listAllAppAccount(Long appId, AccountCategoryEnum category, - BaseSearchCondition baseSearchCondition); + List listAppAccount(Long appId, + AccountCategoryEnum category, + String account, + String alias, + BaseSearchCondition baseSearchCondition); - Integer countAllAppAccount(Long appId, AccountCategoryEnum category); + Integer countAppAccount(Long appId, AccountCategoryEnum category, String account, String alias); /** * 根据别名获取账号信息 @@ -163,7 +168,7 @@ List listAllAppAccount(Long appId, AccountCategoryEnum category, * 判断系统账号是否被DB账号引用 * * @param accountId 系统账号ID - * @return + * @return 是否被引用 */ boolean isAccountRefByDbAccount(Long accountId); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/ApplicationDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/ApplicationDAO.java index ff36c50c7c..dca5617c18 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/ApplicationDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/ApplicationDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,7 +27,6 @@ import com.tencent.bk.job.common.constant.ResourceScopeTypeEnum; import com.tencent.bk.job.common.model.dto.ApplicationDTO; import com.tencent.bk.job.common.model.dto.ResourceScope; -import org.jooq.DSLContext; import java.util.Collection; import java.util.List; @@ -57,27 +56,25 @@ public interface ApplicationDAO { List listAppsByScopeType(ResourceScopeTypeEnum scopeType); - Long insertApp(DSLContext dslContext, ApplicationDTO applicationDTO); + Long insertApp(ApplicationDTO applicationDTO); - int updateApp(DSLContext dslContext, ApplicationDTO applicationDTO); + int updateApp(ApplicationDTO applicationDTO); /** * 恢复已删除的Job业务 * - * @param dslContext DB操作删上下文 - * @param appId Job业务ID + * @param appId Job业务ID * @return 受影响数据行数 */ - int restoreDeletedApp(DSLContext dslContext, long appId); + int restoreDeletedApp(long appId); /** * 对Job业务进行软删除 * - * @param dslContext DB操作删上下文 - * @param appId Job业务ID + * @param appId Job业务ID * @return 受影响数据行数 */ - int deleteAppByIdSoftly(DSLContext dslContext, long appId); + int deleteAppByIdSoftly(long appId); Integer countApps(); @@ -92,4 +89,11 @@ public interface ApplicationDAO { * @return 业务 */ ApplicationDTO getAppByScopeIncludingDeleted(ResourceScope scope); + + /** + * 获取作业平台所有已删除的业务 + * + * @return 业务列表 + */ + List listAllDeletedApps(); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/ApplicationFavorDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/ApplicationFavorDAO.java index b10cd95465..ea4ae49438 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/ApplicationFavorDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/ApplicationFavorDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/ApplicationHostDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/ApplicationHostDAO.java index 91ff7dba24..2e71c91820 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/ApplicationHostDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/ApplicationHostDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,13 +24,16 @@ package com.tencent.bk.job.manage.dao; -import com.tencent.bk.job.common.gse.constants.AgentStatusEnum; import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; +import com.tencent.bk.job.common.model.dto.BasicHostDTO; +import com.tencent.bk.job.common.model.dto.HostSimpleDTO; +import com.tencent.bk.job.common.model.dto.HostStatusNumStatisticsDTO; import java.util.Collection; import java.util.List; +import java.util.Map; /** * @since 4/11/2019 15:01 @@ -38,16 +41,13 @@ public interface ApplicationHostDAO { // 查询类操作 - - boolean existsHost(long bizId, String ip); - - boolean existAppHostInfoByHostId(ApplicationHostDTO applicationHostDTO); - boolean existAppHostInfoByHostId(Long hostId); ApplicationHostDTO getHostById(Long hostId); - ApplicationHostDTO getLatestHost(long bizId, long cloudAreaId, String ip); + List listHostInfoByIps(Collection ips); + + List listHostInfoByCloudIps(Collection cloudIps); /** * 查询近期未更新的主机ID @@ -59,14 +59,24 @@ public interface ApplicationHostDAO { */ List listHostId(long bizId, long minUpdateTimeMills, long maxUpdateTimeMills); + List listHostInfoByIps(Long bizId, Collection ips); + List listHostInfoByBizId(long bizId); + List listBasicHostInfo(Collection hostIds); + List listAllHostInfo(Long start, Long limit); List listHostInfoByBizIds(Collection bizIds, Long start, Long limit); List listHostInfoByHostIds(Collection hostIds); + List listHostInfoByIpv6s(Collection ipv6s); + + List listHostInfoByCloudIpv6(Long cloudAreaId, String ipv6); + + List listHostInfoByHostNames(Collection hostNames); + List listHostInfoBySearchContents(Collection bizIds, Collection moduleIds, Collection cloudAreaIds, @@ -82,21 +92,50 @@ List listHostInfoBySearchContents(Collection bizIds, PageData listHostInfoByPage(ApplicationHostDTO applicationHostInfoCondition, BaseSearchCondition baseSearchCondition); - // count类操作 + List getHostIdListBySearchContents(Collection appIds, + Collection moduleIds, + Collection cloudAreaIds, + List searchContents, + Integer agentAlive, + Long start, + Long limit); + Long countHostInfoBySearchContents(Collection bizIds, Collection moduleIds, Collection cloudAreaIds, List searchContents, Integer agentStatus); - /** - * 根据ID与Agent状态查询主机数量 - * - * @param hostIds 主机Id集合 - * @param agentStatus Agent状态 - * @return 主机数量 - */ - Long countHostByIdAndStatus(Collection hostIds, AgentStatusEnum agentStatus); + List listHostInfoByMultiKeys(Collection bizIds, + Collection moduleIds, + Collection cloudAreaIds, + Collection ipKeys, + Collection ipv6Keys, + Collection hostNameKeys, + Collection osNameKeys, + Integer agentAlive, + Long start, + Long limit); + + List getHostIdListByMultiKeys(Collection bizIds, + Collection moduleIds, + Collection cloudAreaIds, + Collection ipKeys, + Collection ipv6Keys, + Collection hostNameKeys, + Collection osNameKeys, + Integer agentAlive, + Long start, + Long limit); + + Long countHostInfoByMultiKeys(Collection bizIds, + Collection moduleIds, + Collection cloudAreaIds, + Collection ipKeys, + Collection ipv6Keys, + Collection hostNameKeys, + Collection osNameKeys, + Integer agentAlive); long countHostsByBizIds(Collection bizIds); @@ -104,34 +143,55 @@ Long countHostInfoBySearchContents(Collection bizIds, long countHostsByOsType(String osType); + Map groupHostByOsType(); + + List listHostInfoByBizAndIps(Collection bizIds, Collection ips); + + List listHostInfoByBizAndCloudIps(Collection bizIds, Collection cloudIps); + + List listHostInfoByBizAndIpv6s(Collection bizIds, Collection ipv6s); + + List listHostInfoByBizAndHostNames(Collection bizIds, Collection hostNames); + /** - * 根据ip查询主机 + * 查询全部主机，主机对象只有主要属性 * - * @param cloudIps 主机ip(云区域+ip)列表 + * @return 主机列表 */ - List listHostsByIps(Collection cloudIps); - - // 新增、更新类操作 + List listAllHostSimpleInfo(); - int insertHostWithoutTopo(ApplicationHostDTO applicationHostDTO); + /** + * 查询全部主机，主机对象只有基础属性 + * + * @return 主机列表 + */ + List listAllBasicHost(); - void insertOrUpdateHost(ApplicationHostDTO hostDTO); + /** + * 批量更新主机状态 + * + * @param status 主机状态 + * @param hostIdList 主机id列表 + * @return 成功更新的条数 + */ + int batchUpdateHostStatusByHostIds(int status, List hostIdList); - int batchInsertAppHostInfo(List applicationHostDTOList); + // 新增、更新类操作 + int insertHostWithoutTopo(ApplicationHostDTO applicationHostDTO); - void updateHostAttrsById(ApplicationHostDTO applicationHostDTO); + int batchInsertHost(List applicationHostDTOList); - void updateBizHostInfoByHostId(Long bizId, ApplicationHostDTO applicationHostDTO); + int updateHostAttrsByHostId(ApplicationHostDTO applicationHostDTO); - int updateBizHostInfoByHostId(Long bizId, ApplicationHostDTO applicationHostDTO, boolean updateTopo); + int updateHostAttrsBeforeLastTime(ApplicationHostDTO applicationHostDTO); - int batchUpdateBizHostInfoByHostId(List applicationHostDTOList); + int batchUpdateHostsBeforeLastTime(List applicationHostDTOList); int syncHostTopo(Long hostId); - // 删除类操作 - int deleteBizHostInfoById(Long bizId, Long hostId); + // 删除类操作 + int deleteHostBeforeOrEqualLastTime(Long bizId, Long hostId, Long lastTime); /** * 根据传入的主机ID批量删除主机 @@ -150,6 +210,15 @@ Long countHostInfoBySearchContents(Collection bizIds, */ int batchDeleteBizHostInfoById(Long bizId, List hostIdList); + /** + * 根据ip查询主机 + * 删除某个业务下的全部主机，用于业务被删除后清理主机 + * 根据cloudIp查询主机 + * + * @param cloudIps 主机ip(云区域+ip)集合 + */ + List listHostsByCloudIps(Collection cloudIps); + /** * 删除某个业务下的全部主机，用于业务被删除后清理主机 * @@ -157,4 +226,20 @@ Long countHostInfoBySearchContents(Collection bizIds, * @return 删除的主机数量 */ int deleteBizHostInfoByBizId(long bizId); + + /** + * 根据主机基础信息进行批量删除 + * + * @param basicHostList 主机基础信息列表 + * @return 成功删除的主机数量 + */ + int deleteByBasicHost(List basicHostList); + + /** + * 根据业务id统计主机状态数量 + * + * @param bizIds 业务id + * @return 状态数量 + */ + List countHostStatusNumByBizIds(List bizIds); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/CredentialDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/CredentialDAO.java index 81b78c6fbd..40a6c79d9d 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/CredentialDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/CredentialDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,21 +28,22 @@ import com.tencent.bk.job.common.model.PageData; import com.tencent.bk.job.manage.model.dto.CredentialDTO; import com.tencent.bk.job.manage.model.inner.resp.ServiceCredentialDisplayDTO; -import org.jooq.DSLContext; import java.util.Collection; import java.util.List; public interface CredentialDAO { - String insertCredential(DSLContext dslContext, CredentialDTO credentialDTO); + String insertCredential(CredentialDTO credentialDTO); - String updateCredentialById(DSLContext dslContext, CredentialDTO credentialDTO); + String updateCredentialById(CredentialDTO credentialDTO); - int deleteCredentialById(DSLContext dslContext, String id); + int deleteCredentialById(String id); - CredentialDTO getCredentialById(DSLContext dslContext, String id); + CredentialDTO getCredentialById(String id); - List listCredentialDisplayInfoByIds(DSLContext dslContext, Collection ids); + List listCredentialDisplayInfoByIds(Collection ids); PageData listCredentials(CredentialDTO credentialQuery, BaseSearchCondition baseSearchCondition); + + PageData listCredentialBasicInfo(Long appId, BaseSearchCondition baseSearchCondition); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/HostTopoDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/HostTopoDAO.java index eba048fe01..86d6bcba39 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/HostTopoDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/HostTopoDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,34 +25,68 @@ package com.tencent.bk.job.manage.dao; import com.tencent.bk.job.manage.model.dto.HostTopoDTO; +import org.apache.commons.lang3.tuple.Pair; import java.util.Collection; import java.util.List; public interface HostTopoDAO { - void insertHostTopo(HostTopoDTO hostTopoDTO); + int insertHostTopo(HostTopoDTO hostTopoDTO); int batchInsertHostTopo(List hostTopoDTOList); void deleteHostTopoByHostId(Long appId, Long hostId); - void deleteHostTopo(Long hostId, Long appId, Long setId, Long moduleId); + int deleteHostTopoBeforeOrEqualLastTime(Long hostId, Long bizId, Long setId, Long moduleId, Long lastTime); int batchDeleteHostTopo(List hostIdList); + int batchDeleteWithLastTime(List hostTopoList); + int batchDeleteHostTopo(Long bizId, List hostIdList); - int countHostTopo(Long bizId, Long hostId); + int batchUpdateBeforeLastTime(List hostTopoList); + + int updateBeforeLastTime(HostTopoDTO hostTopo); List listHostTopoByHostId(Long hostId); + List listHostTopoByHostIds(Collection hostId); + + List listHostTopoByModuleIds(Collection moduleIds); + List listHostTopoByModuleIds(Collection moduleIds, Long start, Long limit); /** - * 根据CMDB业务IDs查询下属主机ID列表 + * 根据要排除的hostId查询其他拓扑 + * + * @param excludeHostIds 要排除的hostId集合 + * @return 拓扑列表 + */ + List listHostTopoByExcludeHostIds(Collection excludeHostIds); + + /** + * 根据CMDB业务ID与主机ID集合查询下属主机ID列表 * - * @param bizIds 业务ID集合 + * @param bizIds 业务ID集合 + * @param hostIds 主机ID集合 * @return 主机ID列表 */ - List listHostIdByBizIds(Collection bizIds); + List listHostIdByBizAndHostIds(Collection bizIds, Collection hostIds); + + /** + * 根据hostId查询所属模块Id + * + * @param hostId 主机ID + * @return 模块ID列表 + */ + List listModuleIdByHostId(Long hostId); + + /** + * 根据业务ID批量查询主机Id与模块Id信息 + * + * @param bizId 业务ID + * @return <主机ID,模块ID>列表 + */ + List> listHostIdAndModuleIdByBizId(Long bizId); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/ResourceTagDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/ResourceTagDAO.java index e1002de4b7..6b7b4f0f6a 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/ResourceTagDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/ResourceTagDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/ScriptCitedTaskTemplateDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/ScriptCitedTaskTemplateDAO.java index 653bd68b16..87c39f8257 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/ScriptCitedTaskTemplateDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/ScriptCitedTaskTemplateDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/ScriptDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/ScriptDAO.java index 577c1eeeb6..e352702feb 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/ScriptDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/ScriptDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,12 +26,11 @@ import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; -import com.tencent.bk.job.manage.common.consts.JobResourceStatusEnum; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.JobResourceStatusEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; import com.tencent.bk.job.manage.model.dto.ScriptBasicDTO; import com.tencent.bk.job.manage.model.dto.ScriptDTO; import com.tencent.bk.job.manage.model.query.ScriptQuery; -import org.jooq.DSLContext; import java.util.Collection; import java.util.List; @@ -103,75 +102,33 @@ public interface ScriptDAO { String saveScript(ScriptDTO script); /** - * 新增脚本 + * 更新脚本最新更新人、更新时间 * - * @param script - * @ - */ - String saveScript(ScriptDTO script, long createTime, long lastModifyTime); - - /** - * 新增脚本 - * - * @param script - * @ - */ - String saveScript(DSLContext dslContext, ScriptDTO script, long createTime, long lastModifyTime); - - /** - * 更新脚本 - * - * @param script - */ - void updateScript(ScriptDTO script); - - /** - * 更新脚本 - * - * @param script - */ - void updateScript(ScriptDTO script, long lastModifyTime); - - /** - * 更新脚本 - * - * @param script + * @param scriptId 脚本 ID + * @param lastModifyUser 更新人 + * @param lastModifyTime 最后更新时间 */ - void updateScript(DSLContext dslContext, ScriptDTO script, long lastModifyTime); + void updateScriptLastModify(String scriptId, String lastModifyUser, Long lastModifyTime); /** * 删除脚本 * - * @param scriptId + * @param scriptId 脚本 ID */ void deleteScript(String scriptId); /** * 新增脚本版本 * - * @param scriptVersion + * @param scriptVersion 脚本版本 */ Long saveScriptVersion(ScriptDTO scriptVersion); - /** - * 新增脚本版本 - * - * @param scriptVersion - */ - Long saveScriptVersion(ScriptDTO scriptVersion, long createTime, long lastModifyTime); - - /** - * 新增脚本版本 - * - * @param scriptVersion - */ - Long saveScriptVersion(DSLContext dslContext, ScriptDTO scriptVersion, long createTime, long lastModifyTime); - /** * 根据脚本ID查询所有的版本 * - * @param scriptId - * @return + * @param scriptId 脚本 ID + * @return 脚本版本列表 */ List listScriptVersionsByScriptId(String scriptId); @@ -288,11 +245,10 @@ PageData listPageOnlineScript(ScriptQuery scriptCondition, /** * 分页查询脚本版本 * - * @param scriptQuery - * @param baseSearchCondition - * @return + * @param scriptQuery 查询条件 + * @return 脚本版本分页 */ - PageData listPageScriptVersion(ScriptQuery scriptQuery, BaseSearchCondition baseSearchCondition); + PageData listPageScriptVersion(ScriptQuery scriptQuery); /** * 获取脚本已上线版本信息 @@ -389,4 +345,6 @@ PageData listPageOnlineScript(ScriptQuery scriptCondition, * @return Map> */ Map> listAllScriptTagsCompatible(); + + Map batchGetScriptVersionStatus(Collection scriptVersionIds); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/ScriptRelateJobTemplateDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/ScriptRelateJobTemplateDAO.java index 47c1711cd4..fe2da071b8 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/ScriptRelateJobTemplateDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/ScriptRelateJobTemplateDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/ScriptRelateTaskPlanDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/ScriptRelateTaskPlanDAO.java index 285214b9ce..7bae6eb445 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/ScriptRelateTaskPlanDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/ScriptRelateTaskPlanDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/TagDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/TagDAO.java index 7954087212..f3fe9c0e36 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/TagDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/TagDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/TaskApprovalStepDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/TaskApprovalStepDAO.java index 75d4190ea9..9637f68344 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/TaskApprovalStepDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/TaskApprovalStepDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/TaskFavoriteDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/TaskFavoriteDAO.java index c6d20a91f7..7c5f206b0f 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/TaskFavoriteDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/TaskFavoriteDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/TaskFileInfoDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/TaskFileInfoDAO.java index 0e96794055..d5df256724 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/TaskFileInfoDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/TaskFileInfoDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,6 +25,7 @@ package com.tencent.bk.job.manage.dao; import com.tencent.bk.job.manage.model.dto.task.TaskFileInfoDTO; +import com.tencent.bk.job.manage.model.dto.task.TaskTargetDTO; import java.util.List; import java.util.Map; @@ -101,4 +102,18 @@ public interface TaskFileInfoDAO { boolean deleteFileInfosByStepId(long stepId); List listLocalFileByStepId(List stepIdList); + + /** + * 查询所有的步骤的源文件的主机 + */ + Map listStepFileHosts(); + + /** + * 更新步骤文件源的主机的值 + * + * @param recordId 记录id + * @param value 值 + * @return 更新结果 + */ + boolean updateStepFileHosts(Long recordId, String value); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/TaskFileStepDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/TaskFileStepDAO.java index dda4e52f1e..54a4ac7c0d 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/TaskFileStepDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/TaskFileStepDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,8 +25,9 @@ package com.tencent.bk.job.manage.dao; import com.tencent.bk.job.common.constant.DuplicateHandlerEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskFileTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskFileTypeEnum; import com.tencent.bk.job.manage.model.dto.task.TaskFileStepDTO; +import com.tencent.bk.job.manage.model.dto.task.TaskTargetDTO; import java.util.List; import java.util.Map; @@ -103,4 +104,18 @@ default void fixTargerLocation(TaskFileStepDTO fileStep) { } } } + + /** + * 查询所有的步骤的目标主机 + */ + Map listStepTargets(); + + /** + * 更新步骤的目标主机的值 + * + * @param recordId 记录id + * @param value 值 + * @return 更新结果 + */ + boolean updateStepTargets(Long recordId, String value); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/TaskScriptStepDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/TaskScriptStepDAO.java index 5a424eba84..a5ad6dd5e6 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/TaskScriptStepDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/TaskScriptStepDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,7 @@ package com.tencent.bk.job.manage.dao; -import com.tencent.bk.job.manage.common.consts.task.TaskScriptSourceEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskScriptSourceEnum; import com.tencent.bk.job.manage.model.dto.task.TaskScriptStepDTO; import java.util.List; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/TaskStepDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/TaskStepDAO.java index 90cd135f7e..f2f1560dcf 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/TaskStepDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/TaskStepDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/TaskVariableDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/TaskVariableDAO.java index 8fc79aa8c9..68d4079d22 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/TaskVariableDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/TaskVariableDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -54,7 +54,7 @@ public interface TaskVariableDAO { * 根据执行方案 ID 和 变量名称查询变量信息 * * @param parentId 父资源ID - * @param varName 变量名称 + * @param varName 变量名称 * @return 变量信息 */ TaskVariableDTO getVariableByName(long parentId, String varName); @@ -76,12 +76,12 @@ public interface TaskVariableDAO { List batchInsertVariables(List variableList); /** - * 根据变量 ID 更新变量信息 + * 根据 父资源（作业模板/执行方案）ID+变量ID 更新变量信息 * * @param variable 变量信息 * @return 是否更新成功 */ - boolean updateVariableById(TaskVariableDTO variable); + boolean updateVarByParentResourceIdAndTplVarId(TaskVariableDTO variable); /** * 根据变量 ID 和父 ID 删除变量信息 diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/customsetting/CustomScriptTemplateDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/customsetting/CustomScriptTemplateDAO.java index 605df2c7ea..f8ad11c0d0 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/customsetting/CustomScriptTemplateDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/customsetting/CustomScriptTemplateDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/customsetting/UserCustomSettingDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/customsetting/UserCustomSettingDAO.java new file mode 100644 index 0000000000..bc7d8a18fd --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/customsetting/UserCustomSettingDAO.java @@ -0,0 +1,192 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.dao.customsetting; + +import com.tencent.bk.job.common.mysql.JobTransactional; +import com.tencent.bk.job.common.util.ListUtil; +import com.tencent.bk.job.manage.model.dto.customsetting.UserCustomSettingDTO; +import com.tencent.bk.job.manage.model.tables.UserCustomSetting; +import com.tencent.bk.job.manage.model.tables.records.UserCustomSettingRecord; +import lombok.val; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.jooq.Batch; +import org.jooq.BatchBindStep; +import org.jooq.DSLContext; +import org.jooq.Record; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Repository; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +@Repository +public class UserCustomSettingDAO { + + private final UserCustomSetting defaultTable = UserCustomSetting.USER_CUSTOM_SETTING; + private final DSLContext dslContext; + + @Autowired + public UserCustomSettingDAO(@Qualifier("job-manage-dsl-context") DSLContext dslContext) { + this.dslContext = dslContext; + } + + @JobTransactional(transactionManager = "jobManageTransactionManager") + public int batchSave(List customSettingList) { + List keyList = extractKeyList(customSettingList); + Set existKeys = new HashSet<>(listExistKeys(keyList)); + Pair, List> pair = + ListUtil.separate(customSettingList, it -> existKeys.contains(it.getKey())); + List updateList = pair.getLeft(); + List insertList = pair.getRight(); + int updateAffectedNum = batchUpdate(updateList); + int insertAffectedNum = batchInsert(insertList); + return updateAffectedNum + insertAffectedNum; + } + + public int batchDelete(Collection keyList) { + if (CollectionUtils.isEmpty(keyList)) { + return 0; + } + return dslContext.deleteFrom(defaultTable) + .where(defaultTable.KEY.in(keyList)) + .execute(); + } + + public List listAll() { + return dslContext.select( + defaultTable.USERNAME, defaultTable.APP_ID, defaultTable.MODULE, defaultTable.VALUE, + defaultTable.LAST_MODIFY_USER, defaultTable.LAST_MODIFY_TIME) + .from(defaultTable) + .fetch() + .map(this::convert); + } + + public List batchGet(Collection keyList) { + if (CollectionUtils.isEmpty(keyList)) { + return Collections.emptyList(); + } + return dslContext.select( + defaultTable.USERNAME, defaultTable.APP_ID, defaultTable.MODULE, defaultTable.VALUE, + defaultTable.LAST_MODIFY_USER, defaultTable.LAST_MODIFY_TIME) + .from(defaultTable) + .where(defaultTable.KEY.in(keyList)) + .fetch() + .map(this::convert); + } + + private List extractKeyList(List customSettingList) { + if (CollectionUtils.isEmpty(customSettingList)) { + return Collections.emptyList(); + } + return customSettingList.stream().map(UserCustomSettingDTO::getKey).collect(Collectors.toList()); + } + + private List listExistKeys(List keyList) { + if (CollectionUtils.isEmpty(keyList)) { + return Collections.emptyList(); + } + return Arrays.asList(dslContext.select(defaultTable.KEY).from(defaultTable) + .where(defaultTable.KEY.in(keyList)) + .fetch() + .intoArray(defaultTable.KEY)); + } + + private int batchInsert(List customSettingList) { + if (CollectionUtils.isEmpty(customSettingList)) { + return 0; + } + int affectedNum = 0; + val insertQuery = dslContext.insertInto(defaultTable, + defaultTable.USERNAME, + defaultTable.APP_ID, + defaultTable.MODULE, + defaultTable.KEY, + defaultTable.VALUE, + defaultTable.LAST_MODIFY_USER, + defaultTable.LAST_MODIFY_TIME + ).values((String) null, null, null, null, null, null, null); + BatchBindStep batchQuery = dslContext.batch(insertQuery); + for (UserCustomSettingDTO userCustomSettingDTO : customSettingList) { + batchQuery = batchQuery.bind( + userCustomSettingDTO.getUsername(), + userCustomSettingDTO.getAppId(), + userCustomSettingDTO.getModule(), + userCustomSettingDTO.getKey(), + userCustomSettingDTO.getValue(), + userCustomSettingDTO.getLastModifyUser(), + System.currentTimeMillis() + ); + } + int[] results = batchQuery.execute(); + for (int result : results) { + affectedNum += result; + } + return affectedNum; + } + + private int batchUpdate(List customSettingList) { + if (CollectionUtils.isEmpty(customSettingList)) { + return 0; + } + int affectedNum = 0; + List records = new ArrayList<>(); + for (UserCustomSettingDTO userCustomSettingDTO : customSettingList) { + UserCustomSettingRecord record = new UserCustomSettingRecord(); + record.set(defaultTable.USERNAME, userCustomSettingDTO.getUsername()); + record.set(defaultTable.APP_ID, userCustomSettingDTO.getAppId()); + record.set(defaultTable.MODULE, userCustomSettingDTO.getModule()); + record.set(defaultTable.KEY, userCustomSettingDTO.getKey()); + record.set(defaultTable.VALUE, userCustomSettingDTO.getValue()); + record.set(defaultTable.LAST_MODIFY_USER, userCustomSettingDTO.getLastModifyUser()); + record.set(defaultTable.LAST_MODIFY_TIME, System.currentTimeMillis()); + records.add(record); + } + Batch batchQuery = dslContext.batchUpdate(records); + int[] results = batchQuery.execute(); + for (int result : results) { + affectedNum += result; + } + return affectedNum; + } + + private UserCustomSettingDTO convert(Record record) { + UserCustomSettingDTO userCustomSettingDTO = new UserCustomSettingDTO(); + userCustomSettingDTO.setUsername(record.get(defaultTable.USERNAME)); + userCustomSettingDTO.setAppId(record.get(defaultTable.APP_ID)); + userCustomSettingDTO.setModule(record.get(defaultTable.MODULE)); + userCustomSettingDTO.setValue(record.get(defaultTable.VALUE)); + userCustomSettingDTO.setLastModifyUser(record.get(defaultTable.LAST_MODIFY_USER)); + userCustomSettingDTO.setLastModifyTime(record.get(defaultTable.LAST_MODIFY_TIME)); + return userCustomSettingDTO; + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/customsetting/impl/CustomScriptTemplateDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/customsetting/impl/CustomScriptTemplateDAOImpl.java index a1ff1bd22b..31505a6552 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/customsetting/impl/CustomScriptTemplateDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/customsetting/impl/CustomScriptTemplateDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,14 +24,14 @@ package com.tencent.bk.job.manage.dao.customsetting.impl; -import com.tencent.bk.job.manage.common.util.JooqDataTypeUtil; +import com.tencent.bk.job.common.mysql.util.JooqDataTypeUtil; import com.tencent.bk.job.manage.dao.customsetting.CustomScriptTemplateDAO; import com.tencent.bk.job.manage.model.dto.customsetting.ScriptTemplateDTO; +import com.tencent.bk.job.manage.model.tables.UserCustomScriptTemplate; import lombok.extern.slf4j.Slf4j; import org.jooq.DSLContext; import org.jooq.Record; import org.jooq.Result; -import org.jooq.generated.tables.UserCustomScriptTemplate; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; @@ -44,7 +44,7 @@ public class CustomScriptTemplateDAOImpl implements CustomScriptTemplateDAO { private final UserCustomScriptTemplate TB = UserCustomScriptTemplate.USER_CUSTOM_SCRIPT_TEMPLATE; - private DSLContext ctx; + private final DSLContext ctx; @Autowired public CustomScriptTemplateDAOImpl(@Qualifier("job-manage-dsl-context") DSLContext ctx) { diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/globalsetting/DangerousRuleDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/globalsetting/DangerousRuleDAO.java index b131c9f232..e8bcce0ed9 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/globalsetting/DangerousRuleDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/globalsetting/DangerousRuleDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,27 +24,32 @@ package com.tencent.bk.job.manage.dao.globalsetting; +import com.tencent.bk.job.manage.api.common.constants.EnableStatusEnum; import com.tencent.bk.job.manage.model.dto.globalsetting.DangerousRuleDTO; -import org.jooq.DSLContext; +import com.tencent.bk.job.manage.model.query.DangerousRuleQuery; import java.util.List; public interface DangerousRuleDAO { - Long insertDangerousRule(DSLContext dslContext, DangerousRuleDTO dangerousRuleDTO); + Long insertDangerousRule(DangerousRuleDTO dangerousRuleDTO); - int updateDangerousRule(DSLContext dslContext, DangerousRuleDTO dangerousRuleDTO); + int updateDangerousRule(DangerousRuleDTO dangerousRuleDTO); - int deleteDangerousRuleById(DSLContext dslContext, Long id); + int deleteDangerousRuleById(Long id); - DangerousRuleDTO getDangerousRuleById(DSLContext dslContext, Long id); + DangerousRuleDTO getDangerousRuleById(Long id); - DangerousRuleDTO getDangerousRuleByPriority(DSLContext dslContext, int priority); + DangerousRuleDTO getDangerousRuleByPriority(int priority); - List listDangerousRules(DSLContext dslContext); + List listDangerousRules(); - List listDangerousRules(DSLContext dslContext, DangerousRuleDTO dangerousRuleQuery); + List listDangerousRules(DangerousRuleDTO dangerousRuleQuery); - int getMaxPriority(DSLContext dslContext); + List listDangerousRules(DangerousRuleQuery query); - int getMinPriority(DSLContext dslContext); + int getMaxPriority(); + + int getMinPriority(); + + int updateDangerousRuleStatus(String userName, Long id, EnableStatusEnum status); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/globalsetting/GlobalSettingDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/globalsetting/GlobalSettingDAO.java index 3c10b78d7b..98938039fd 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/globalsetting/GlobalSettingDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/globalsetting/GlobalSettingDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,19 +25,16 @@ package com.tencent.bk.job.manage.dao.globalsetting; import com.tencent.bk.job.manage.model.dto.GlobalSettingDTO; -import org.jooq.DSLContext; public interface GlobalSettingDAO { int upsertGlobalSetting(GlobalSettingDTO globalSettingDTO); - int insertGlobalSetting(DSLContext dslContext, GlobalSettingDTO globalSettingDTO); + int insertGlobalSetting(GlobalSettingDTO globalSettingDTO); - int updateGlobalSetting(DSLContext dslContext, GlobalSettingDTO globalSettingDTO); + int updateGlobalSetting(GlobalSettingDTO globalSettingDTO); - int deleteGlobalSetting(DSLContext dslContext, String key); - - GlobalSettingDTO getGlobalSetting(DSLContext dslContext, String key); + int deleteGlobalSetting(String key); GlobalSettingDTO getGlobalSetting(String key); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/globalsetting/impl/DangerousRuleDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/globalsetting/impl/DangerousRuleDAOImpl.java index 99a3b41b77..655509eade 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/globalsetting/impl/DangerousRuleDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/globalsetting/impl/DangerousRuleDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,19 +24,26 @@ package com.tencent.bk.job.manage.dao.globalsetting.impl; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; -import com.tencent.bk.job.manage.common.util.JooqDataTypeUtil; +import com.tencent.bk.job.common.mysql.util.JooqDataTypeUtil; +import com.tencent.bk.job.manage.api.common.constants.EnableStatusEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; import com.tencent.bk.job.manage.dao.globalsetting.DangerousRuleDAO; import com.tencent.bk.job.manage.model.dto.globalsetting.DangerousRuleDTO; +import com.tencent.bk.job.manage.model.query.DangerousRuleQuery; +import com.tencent.bk.job.manage.model.tables.DangerousRule; +import com.tencent.bk.job.manage.model.tables.records.DangerousRuleRecord; import lombok.extern.slf4j.Slf4j; import lombok.val; +import org.apache.commons.lang3.StringUtils; import org.jooq.Condition; import org.jooq.DSLContext; import org.jooq.Record; +import org.jooq.Result; import org.jooq.conf.ParamType; -import org.jooq.generated.tables.DangerousRule; import org.jooq.impl.DSL; import org.jooq.types.ULong; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; import java.util.ArrayList; @@ -49,6 +56,12 @@ public class DangerousRuleDAOImpl implements DangerousRuleDAO { private static final DangerousRule T = DangerousRule.DANGEROUS_RULE; + private final DSLContext dslContext; + + @Autowired + public DangerousRuleDAOImpl(@Qualifier("job-manage-dsl-context") DSLContext dslContext) { + this.dslContext = dslContext; + } private void setDefaultValue(DangerousRuleDTO dangerousRuleDTO) { if (dangerousRuleDTO.getScriptType() == null) { @@ -57,7 +70,7 @@ private void setDefaultValue(DangerousRuleDTO dangerousRuleDTO) { } @Override - public Long insertDangerousRule(DSLContext dslContext, DangerousRuleDTO dangerousRuleDTO) { + public Long insertDangerousRule(DangerousRuleDTO dangerousRuleDTO) { setDefaultValue(dangerousRuleDTO); val query = dslContext.insertInto(T, T.EXPRESSION, @@ -92,7 +105,7 @@ public Long insertDangerousRule(DSLContext dslContext, DangerousRuleDTO dangerou } @Override - public int updateDangerousRule(DSLContext dslContext, DangerousRuleDTO dangerousRuleDTO) { + public int updateDangerousRule(DangerousRuleDTO dangerousRuleDTO) { val query = dslContext.update(T) .set(T.EXPRESSION, dangerousRuleDTO.getExpression()) .set(T.DESCRIPTION, dangerousRuleDTO.getDescription()) @@ -114,14 +127,14 @@ public int updateDangerousRule(DSLContext dslContext, DangerousRuleDTO dangerous @Override - public int deleteDangerousRuleById(DSLContext dslContext, Long id) { + public int deleteDangerousRuleById(Long id) { return dslContext.deleteFrom(T).where( T.ID.eq(id) ).execute(); } @Override - public DangerousRuleDTO getDangerousRuleById(DSLContext dslContext, Long id) { + public DangerousRuleDTO getDangerousRuleById(Long id) { val record = dslContext.selectFrom(T).where( T.ID.eq(id) ).fetchOne(); @@ -133,7 +146,7 @@ val record = dslContext.selectFrom(T).where( } @Override - public DangerousRuleDTO getDangerousRuleByPriority(DSLContext dslContext, int priority) { + public DangerousRuleDTO getDangerousRuleByPriority(int priority) { val record = dslContext.selectFrom(T).where( T.PRIORITY.eq(priority) ).fetchOne(); @@ -145,7 +158,7 @@ val record = dslContext.selectFrom(T).where( } @Override - public List listDangerousRules(DSLContext dslContext) { + public List listDangerousRules() { val records = dslContext.selectFrom(T).orderBy(T.PRIORITY).fetch(); if (records.isEmpty()) { return Collections.emptyList(); @@ -155,7 +168,7 @@ public List listDangerousRules(DSLContext dslContext) { } @Override - public List listDangerousRules(DSLContext dslContext, DangerousRuleDTO dangerousRuleQuery) { + public List listDangerousRules(DangerousRuleDTO dangerousRuleQuery) { Integer scriptType = dangerousRuleQuery.getScriptType(); List conditions = new ArrayList<>(); if (dangerousRuleQuery.getStatus() != null) { @@ -171,12 +184,48 @@ public List listDangerousRules(DSLContext dslContext, Dangerou return dangerousRuleList; } int typeFlag = 1 << scriptType - 1; - return dangerousRuleList.stream().filter(rule -> (rule.getScriptType() & (typeFlag)) == typeFlag).collect(Collectors.toList()); + return dangerousRuleList.stream() + .filter(rule -> (rule.getScriptType() & (typeFlag)) == typeFlag) + .collect(Collectors.toList()); + } + } + + @Override + public List listDangerousRules(DangerousRuleQuery query) { + List conditions = buildConditionList(query); + Result records = dslContext.selectFrom(T) + .where(conditions) + .orderBy(T.PRIORITY) + .fetch(); + return records.map(this::convertRecordToDto); + } + + private List buildConditionList(DangerousRuleQuery query) { + List conditions = new ArrayList<>(); + if (StringUtils.isNotBlank(query.getExpression())) { + conditions.add(T.EXPRESSION.like("%" + query.getExpression() + "%")); + } + if (StringUtils.isNotBlank(query.getDescription())) { + conditions.add(T.DESCRIPTION.like("%" + query.getDescription() + "%")); + } + if (query.getScriptTypeList() != null) { + List typeList = query.getScriptTypeList(); + int scriptType = 0; + for (Byte type : typeList) { + if (type > 0 && type <= 8) { + scriptType |= 1 << (type - 1); + } + } + conditions.add(DSL.bitAnd(T.SCRIPT_TYPE, scriptType).greaterThan(0)); + } + if (query.getAction() != null) { + conditions.add(T.ACTION.in(query.getAction())); } + return conditions; } @Override - public int getMaxPriority(DSLContext dslContext) { + public int getMaxPriority() { val record = dslContext.select(DSL.max(T.PRIORITY)).from(T).fetchOne(); if (record == null || record.value1() == null) { return 0; @@ -186,7 +235,7 @@ val record = dslContext.select(DSL.max(T.PRIORITY)).from(T).fetchOne(); } @Override - public int getMinPriority(DSLContext dslContext) { + public int getMinPriority() { val record = dslContext.select(DSL.min(T.PRIORITY)).from(T).fetchOne(); if (record == null || record.value1() == null) { return 0; @@ -195,6 +244,22 @@ val record = dslContext.select(DSL.min(T.PRIORITY)).from(T).fetchOne(); } } + @Override + public int updateDangerousRuleStatus(String userName, Long id, EnableStatusEnum status) { + val query = dslContext.update(T) + .set(T.LAST_MODIFY_USER, userName) + .set(T.LAST_MODIFY_TIME, ULong.valueOf(System.currentTimeMillis())) + .set(T.STATUS, (byte) status.getValue()) + .where(T.ID.eq(id)); + try { + return query.execute(); + } catch (Exception e) { + val sql = query.getSQL(ParamType.INLINED); + log.error(sql); + throw e; + } + } + private DangerousRuleDTO convertRecordToDto(Record record) { return new DangerousRuleDTO( record.get(T.ID), @@ -209,4 +274,5 @@ private DangerousRuleDTO convertRecordToDto(Record record) { record.get(T.ACTION).intValue(), record.get(T.STATUS).intValue()); } + } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/globalsetting/impl/GlobalSettingDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/globalsetting/impl/GlobalSettingDAOImpl.java index 30b0b09a1f..5a91efe30e 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/globalsetting/impl/GlobalSettingDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/globalsetting/impl/GlobalSettingDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,50 +24,44 @@ package com.tencent.bk.job.manage.dao.globalsetting.impl; +import com.tencent.bk.job.common.mysql.JobTransactional; import com.tencent.bk.job.manage.dao.globalsetting.GlobalSettingDAO; import com.tencent.bk.job.manage.model.dto.GlobalSettingDTO; +import com.tencent.bk.job.manage.model.tables.GlobalSetting; import lombok.extern.slf4j.Slf4j; import lombok.val; import org.jooq.DSLContext; import org.jooq.conf.ParamType; -import org.jooq.generated.tables.GlobalSetting; -import org.jooq.impl.DSL; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; import java.util.concurrent.atomic.AtomicInteger; -/** - * @Description - * @Date 2020/2/28 - * @Version 1.0 - */ @Repository @Slf4j public class GlobalSettingDAOImpl implements GlobalSettingDAO { - private final DSLContext defaultDSLContext; + private final DSLContext dslContext; private static final GlobalSetting defaultTable = GlobalSetting.GLOBAL_SETTING; @Autowired - public GlobalSettingDAOImpl(DSLContext dslContext) { - this.defaultDSLContext = dslContext; + public GlobalSettingDAOImpl(@Qualifier("job-manage-dsl-context") DSLContext dslContext) { + this.dslContext = dslContext; } @Override + @JobTransactional(transactionManager = "jobManageTransactionManager") public int upsertGlobalSetting(GlobalSettingDTO globalSettingDTO) { AtomicInteger affectedNum = new AtomicInteger(0); - defaultDSLContext.transaction(configuration -> { - DSLContext context = DSL.using(configuration); - deleteGlobalSetting(context, globalSettingDTO.getKey()); - affectedNum.set(insertGlobalSetting(context, globalSettingDTO)); - }); + deleteGlobalSetting(globalSettingDTO.getKey()); + affectedNum.set(insertGlobalSetting(globalSettingDTO)); return affectedNum.get(); } @Override - public int insertGlobalSetting(DSLContext dslContext, GlobalSettingDTO globalSettingDTO) { + public int insertGlobalSetting(GlobalSettingDTO globalSettingDTO) { val query = dslContext.insertInto(defaultTable, defaultTable.KEY, defaultTable.VALUE, @@ -87,7 +81,7 @@ public int insertGlobalSetting(DSLContext dslContext, GlobalSettingDTO globalSet } @Override - public int updateGlobalSetting(DSLContext dslContext, GlobalSettingDTO globalSettingDTO) { + public int updateGlobalSetting(GlobalSettingDTO globalSettingDTO) { val query = dslContext.update(defaultTable) .set(defaultTable.VALUE, globalSettingDTO.getValue()) .set(defaultTable.DECRIPTION, globalSettingDTO.getDescription()) @@ -102,7 +96,7 @@ public int updateGlobalSetting(DSLContext dslContext, GlobalSettingDTO globalSet } @Override - public int deleteGlobalSetting(DSLContext dslContext, String key) { + public int deleteGlobalSetting(String key) { return dslContext.deleteFrom(defaultTable).where( defaultTable.KEY.eq(key) ).execute(); @@ -110,11 +104,6 @@ public int deleteGlobalSetting(DSLContext dslContext, String key) { @Override public GlobalSettingDTO getGlobalSetting(String key) { - return getGlobalSetting(defaultDSLContext, key); - } - - @Override - public GlobalSettingDTO getGlobalSetting(DSLContext dslContext, String key) { val record = dslContext.selectFrom(defaultTable).where( defaultTable.KEY.eq(key) ).fetchOne(); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/AccountDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/AccountDAOImpl.java index 80424bf50f..b049785a59 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/AccountDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/AccountDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,15 +24,23 @@ package com.tencent.bk.job.manage.dao.impl; +import com.tencent.bk.job.common.constant.AccountCategoryEnum; +import com.tencent.bk.job.common.crypto.scenario.DbPasswordCryptoService; import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; +import com.tencent.bk.job.common.mysql.util.JooqDataTypeUtil; import com.tencent.bk.job.common.util.date.DateUtils; -import com.tencent.bk.job.manage.common.consts.account.AccountCategoryEnum; -import com.tencent.bk.job.manage.common.consts.account.AccountTypeEnum; -import com.tencent.bk.job.manage.common.util.JooqDataTypeUtil; +import com.tencent.bk.job.manage.api.common.constants.account.AccountTypeEnum; import com.tencent.bk.job.manage.dao.AccountDAO; import com.tencent.bk.job.manage.model.dto.AccountDTO; import com.tencent.bk.job.manage.model.dto.AccountDisplayDTO; +import com.tencent.bk.job.manage.model.tables.Account; +import com.tencent.bk.job.manage.model.tables.TaskTemplate; +import com.tencent.bk.job.manage.model.tables.TaskTemplateStep; +import com.tencent.bk.job.manage.model.tables.TaskTemplateStepFile; +import com.tencent.bk.job.manage.model.tables.TaskTemplateStepFileList; +import com.tencent.bk.job.manage.model.tables.TaskTemplateStepScript; +import com.tencent.bk.job.manage.model.tables.records.AccountRecord; import lombok.val; import org.apache.commons.lang3.StringUtils; import org.jooq.Condition; @@ -44,12 +52,6 @@ import org.jooq.SortField; import org.jooq.TableField; import org.jooq.UpdateSetMoreStep; -import org.jooq.generated.tables.Account; -import org.jooq.generated.tables.TaskTemplate; -import org.jooq.generated.tables.TaskTemplateStep; -import org.jooq.generated.tables.TaskTemplateStepFile; -import org.jooq.generated.tables.TaskTemplateStepFileList; -import org.jooq.generated.tables.TaskTemplateStepScript; import org.jooq.types.UByte; import org.jooq.types.ULong; import org.springframework.beans.factory.annotation.Autowired; @@ -64,56 +66,118 @@ @Repository public class AccountDAOImpl implements AccountDAO { private static final Account TB_ACCOUNT = Account.ACCOUNT; - private static final TableField[] ALL_FILED = {TB_ACCOUNT.ID, TB_ACCOUNT.ACCOUNT_, TB_ACCOUNT.ALIAS, - TB_ACCOUNT.CATEGORY, TB_ACCOUNT.TYPE, TB_ACCOUNT.APP_ID, TB_ACCOUNT.GRANTEE, TB_ACCOUNT.REMARK, - TB_ACCOUNT.OS, TB_ACCOUNT.PASSWORD, TB_ACCOUNT.DB_PASSWORD, TB_ACCOUNT.DB_PORT, TB_ACCOUNT.DB_SYSTEM_ACCOUNT_ID, - TB_ACCOUNT.CREATOR, TB_ACCOUNT.CREATE_TIME, TB_ACCOUNT.LAST_MODIFY_USER, TB_ACCOUNT.LAST_MODIFY_TIME}; - private DSLContext ctx; + private static final TableField[] ALL_FILED = { + TB_ACCOUNT.ID, + TB_ACCOUNT.ACCOUNT_, + TB_ACCOUNT.ALIAS, + TB_ACCOUNT.CATEGORY, + TB_ACCOUNT.TYPE, + TB_ACCOUNT.APP_ID, + TB_ACCOUNT.GRANTEE, + TB_ACCOUNT.REMARK, + TB_ACCOUNT.OS, + TB_ACCOUNT.PASSWORD, + TB_ACCOUNT.DB_PASSWORD, + TB_ACCOUNT.DB_PORT, + TB_ACCOUNT.DB_SYSTEM_ACCOUNT_ID, + TB_ACCOUNT.CREATOR, + TB_ACCOUNT.CREATE_TIME, + TB_ACCOUNT.LAST_MODIFY_USER, + TB_ACCOUNT.LAST_MODIFY_TIME + }; + private final DSLContext ctx; + private final DbPasswordCryptoService dbPasswordCryptoService; @Autowired - public AccountDAOImpl(@Qualifier("job-manage-dsl-context") DSLContext create) { + public AccountDAOImpl(@Qualifier("job-manage-dsl-context") DSLContext create, + DbPasswordCryptoService dbPasswordCryptoService) { this.ctx = create; + this.dbPasswordCryptoService = dbPasswordCryptoService; } @Override public long saveAccountWithId(AccountDTO account) { - Record record = ctx.insertInto(TB_ACCOUNT, TB_ACCOUNT.ID, TB_ACCOUNT.ACCOUNT_, TB_ACCOUNT.ALIAS, - TB_ACCOUNT.CATEGORY, TB_ACCOUNT.TYPE, TB_ACCOUNT.APP_ID, TB_ACCOUNT.GRANTEE, - TB_ACCOUNT.REMARK, TB_ACCOUNT.OS, TB_ACCOUNT.PASSWORD, TB_ACCOUNT.DB_PASSWORD, TB_ACCOUNT.DB_PORT, + Record record = ctx.insertInto( + TB_ACCOUNT, + TB_ACCOUNT.ID, + TB_ACCOUNT.ACCOUNT_, + TB_ACCOUNT.ALIAS, + TB_ACCOUNT.CATEGORY, + TB_ACCOUNT.TYPE, + TB_ACCOUNT.APP_ID, + TB_ACCOUNT.GRANTEE, + TB_ACCOUNT.REMARK, + TB_ACCOUNT.OS, + TB_ACCOUNT.PASSWORD, + TB_ACCOUNT.DB_PASSWORD, + TB_ACCOUNT.DB_PORT, TB_ACCOUNT.DB_SYSTEM_ACCOUNT_ID, - TB_ACCOUNT.CREATOR, TB_ACCOUNT.CREATE_TIME, TB_ACCOUNT.LAST_MODIFY_USER, TB_ACCOUNT.LAST_MODIFY_TIME) - .values(account.getId(), account.getAccount(), account.getAlias(), - JooqDataTypeUtil.getByteFromInteger(account.getCategory().getValue()), - JooqDataTypeUtil.getByteFromInteger(account.getType().getType()), account.getAppId(), - account.getGrantees(), - account.getRemark(), account.getOs(), account.getPassword(), account.getDbPassword(), - account.getDbPort(), - account.getDbSystemAccountId(), account.getCreator(), ULong.valueOf(account.getCreateTime()), - account.getLastModifyUser(), - ULong.valueOf(account.getLastModifyTime())) - .returning(TB_ACCOUNT.ID) + TB_ACCOUNT.CREATOR, + TB_ACCOUNT.CREATE_TIME, + TB_ACCOUNT.LAST_MODIFY_USER, + TB_ACCOUNT.LAST_MODIFY_TIME + ).values( + account.getId(), + account.getAccount(), + account.getAlias(), + JooqDataTypeUtil.getByteFromInteger(account.getCategory().getValue()), + JooqDataTypeUtil.getByteFromInteger(account.getType().getType()), + account.getAppId(), + account.getGrantees(), + account.getRemark(), + account.getOs(), + account.getPassword(), + dbPasswordCryptoService.encryptDbPasswordIfNeeded(account.getCategory(), account.getDbPassword()), + account.getDbPort(), + account.getDbSystemAccountId(), + account.getCreator(), + ULong.valueOf(account.getCreateTime()), + account.getLastModifyUser(), + ULong.valueOf(account.getLastModifyTime()) + ).returning(TB_ACCOUNT.ID) .fetchOne(); + assert record != null; return record.get(TB_ACCOUNT.ID); } @Override public long saveAccount(AccountDTO account) { - Record record = ctx.insertInto(TB_ACCOUNT, TB_ACCOUNT.ACCOUNT_, TB_ACCOUNT.ALIAS, TB_ACCOUNT.CATEGORY, - TB_ACCOUNT.TYPE, TB_ACCOUNT.APP_ID, TB_ACCOUNT.GRANTEE, - TB_ACCOUNT.REMARK, TB_ACCOUNT.OS, TB_ACCOUNT.PASSWORD, TB_ACCOUNT.DB_PASSWORD, TB_ACCOUNT.DB_PORT, + Record record = ctx.insertInto(TB_ACCOUNT, + TB_ACCOUNT.ACCOUNT_, + TB_ACCOUNT.ALIAS, + TB_ACCOUNT.CATEGORY, + TB_ACCOUNT.TYPE, + TB_ACCOUNT.APP_ID, + TB_ACCOUNT.GRANTEE, + TB_ACCOUNT.REMARK, + TB_ACCOUNT.OS, + TB_ACCOUNT.PASSWORD, + TB_ACCOUNT.DB_PASSWORD, + TB_ACCOUNT.DB_PORT, TB_ACCOUNT.DB_SYSTEM_ACCOUNT_ID, - TB_ACCOUNT.CREATOR, TB_ACCOUNT.CREATE_TIME, TB_ACCOUNT.LAST_MODIFY_USER, TB_ACCOUNT.LAST_MODIFY_TIME) - .values(account.getAccount(), account.getAlias(), - JooqDataTypeUtil.getByteFromInteger(account.getCategory().getValue()), - JooqDataTypeUtil.getByteFromInteger(account.getType().getType()), account.getAppId(), - account.getGrantees(), - account.getRemark(), account.getOs(), account.getPassword(), account.getDbPassword(), - account.getDbPort(), - account.getDbSystemAccountId(), account.getCreator(), ULong.valueOf(account.getCreateTime()), - account.getLastModifyUser(), - ULong.valueOf(DateUtils.currentTimeMillis())) - .returning(TB_ACCOUNT.ID) + TB_ACCOUNT.CREATOR, + TB_ACCOUNT.CREATE_TIME, + TB_ACCOUNT.LAST_MODIFY_USER, + TB_ACCOUNT.LAST_MODIFY_TIME + ).values(account.getAccount(), + account.getAlias(), + JooqDataTypeUtil.getByteFromInteger(account.getCategory().getValue()), + JooqDataTypeUtil.getByteFromInteger(account.getType().getType()), + account.getAppId(), + account.getGrantees(), + account.getRemark(), + account.getOs(), + account.getPassword(), + dbPasswordCryptoService.encryptDbPasswordIfNeeded(account.getCategory(), account.getDbPassword()), + account.getDbPort(), + account.getDbSystemAccountId(), + account.getCreator(), + ULong.valueOf(account.getCreateTime()), + account.getLastModifyUser(), + ULong.valueOf(DateUtils.currentTimeMillis()) + ).returning(TB_ACCOUNT.ID) .fetchOne(); + assert record != null; return record.get(TB_ACCOUNT.ID); } @@ -134,8 +198,8 @@ public List listAccountDisplayInfoByIds(Collection acco TB_ACCOUNT.ALIAS, TB_ACCOUNT.CATEGORY, TB_ACCOUNT.TYPE, - TB_ACCOUNT.APP_ID) - .from(TB_ACCOUNT) + TB_ACCOUNT.APP_ID + ).from(TB_ACCOUNT) .where(conditions) .and(TB_ACCOUNT.IS_DELETED.eq(UByte.valueOf(0))) .fetch(); @@ -210,7 +274,15 @@ private AccountDTO extract(Record record) { account.setRemark(record.get(TB_ACCOUNT.REMARK)); account.setOs(record.get(TB_ACCOUNT.OS)); account.setPassword(record.get(TB_ACCOUNT.PASSWORD)); - account.setDbPassword(record.get(TB_ACCOUNT.DB_PASSWORD)); + + // 解密DB账号密码 + String encryptedDbPassword = record.get(TB_ACCOUNT.DB_PASSWORD); + String dbPassword = dbPasswordCryptoService.decryptDbPasswordIfNeeded( + account.getCategory(), + encryptedDbPassword + ); + + account.setDbPassword(dbPassword); account.setDbPort(record.get(TB_ACCOUNT.DB_PORT)); account.setDbSystemAccountId(record.get(TB_ACCOUNT.DB_SYSTEM_ACCOUNT_ID)); account.setCreator(record.get(TB_ACCOUNT.CREATOR)); @@ -222,7 +294,7 @@ private AccountDTO extract(Record record) { @Override public void updateAccount(AccountDTO account) { - UpdateSetMoreStep update = ctx.update(TB_ACCOUNT) + UpdateSetMoreStep update = ctx.update(TB_ACCOUNT) .set(TB_ACCOUNT.GRANTEE, account.getGrantees()) .set(TB_ACCOUNT.REMARK, account.getRemark()) .set(TB_ACCOUNT.DB_PORT, account.getDbPort()) @@ -236,7 +308,10 @@ public void updateAccount(AccountDTO account) { update.set(TB_ACCOUNT.PASSWORD, account.getPassword()); } if (StringUtils.isNotEmpty(account.getDbPassword())) { - update.set(TB_ACCOUNT.DB_PASSWORD, account.getDbPassword()); + update.set(TB_ACCOUNT.DB_PASSWORD, dbPasswordCryptoService.encryptDbPasswordIfNeeded( + account.getCategory(), + account.getDbPassword() + )); } update.where(TB_ACCOUNT.ID.eq(account.getId())) .execute(); @@ -252,12 +327,6 @@ private void deleteAccountHardly(Long accountId) { .where(TB_ACCOUNT.ID.eq(accountId)).execute(); } - private void deleteAccountSoftly(Long accountId) { - ctx.update(TB_ACCOUNT) - .set(TB_ACCOUNT.IS_DELETED, UByte.valueOf(1)) - .where(TB_ACCOUNT.ID.eq(accountId)).execute(); - } - @Override public PageData searchPageAccount(Long appId, String keyword, BaseSearchCondition baseSearchCondition) { @@ -306,7 +375,7 @@ public PageData listPageAccountByConditions(BaseSearchCondition base int start = baseSearchCondition.getStartOrDefault(0); int length = baseSearchCondition.getLengthOrDefault(10); - Result result = + Result result = ctx.select(ALL_FILED) .from(TB_ACCOUNT) .where(conditions) @@ -334,7 +403,9 @@ public PageData listPageAccountByConditions(BaseSearchCondition base */ private long getPageAccountCount(Long appId, String keyword, BaseSearchCondition baseSearchCondition) { List conditions = buildConditionList(appId, keyword, baseSearchCondition); - return ctx.selectCount().from(TB_ACCOUNT).where(conditions).fetchOne(0, Long.class); + Long count = ctx.selectCount().from(TB_ACCOUNT).where(conditions).fetchOne(0, Long.class); + assert count != null; + return count; } @@ -343,7 +414,13 @@ private long getPageAccountCount(Long appId, String keyword, BaseSearchCondition */ private long getPageAccountCount(AccountDTO accountQuery, BaseSearchCondition baseSearchCondition) { List conditions = buildConditionList(accountQuery, baseSearchCondition); - return ctx.selectCount().from(TB_ACCOUNT).where(conditions).fetchOne(0, Long.class); + return countAccountByConditions(conditions); + } + + private long countAccountByConditions(List conditions) { + Long count = ctx.selectCount().from(TB_ACCOUNT).where(conditions).fetchOne(0, Long.class); + assert count != null; + return count; } private List buildConditionList(AccountDTO accountQuery, BaseSearchCondition baseSearchCondition) { @@ -369,6 +446,9 @@ private List buildConditionList(AccountDTO accountQuery, BaseSearchCo if (StringUtils.isNotBlank(accountQuery.getLastModifyUser())) { conditions.add(TB_ACCOUNT.LAST_MODIFY_USER.like("%" + accountQuery.getLastModifyUser() + "%")); } + if (StringUtils.isNotBlank(accountQuery.getRemark())) { + conditions.add(TB_ACCOUNT.REMARK.like("%" + accountQuery.getRemark() + "%")); + } conditions.add(TB_ACCOUNT.APP_ID.eq(accountQuery.getAppId())); conditions.add(TB_ACCOUNT.IS_DELETED.eq(UByte.valueOf(0))); @@ -377,7 +457,7 @@ private List buildConditionList(AccountDTO accountQuery, BaseSearchCo private List buildConditionList(Long appId, String keyword, BaseSearchCondition baseSearchCondition) { List conditions = new ArrayList<>(); - Condition condition = null; + Condition condition; if (keyword != null) { String likeKeyword = "%" + keyword + "%"; condition = TB_ACCOUNT.ID.like(likeKeyword); @@ -385,6 +465,7 @@ private List buildConditionList(Long appId, String keyword, BaseSearc condition.or(TB_ACCOUNT.ALIAS.like(likeKeyword)); condition.or(TB_ACCOUNT.CREATOR.like(likeKeyword)); condition.or(TB_ACCOUNT.LAST_MODIFY_USER.like(likeKeyword)); + condition.or(TB_ACCOUNT.REMARK.like(likeKeyword)); conditions.add(condition); } conditions.add(TB_ACCOUNT.APP_ID.eq(appId)); @@ -392,7 +473,7 @@ private List buildConditionList(Long appId, String keyword, BaseSearc return conditions; } - private List genBaseConditions(Long appId, AccountCategoryEnum category) { + private List genBaseConditions(Long appId, AccountCategoryEnum category, String account, String alias) { List conditions = new ArrayList<>(); conditions.add(TB_ACCOUNT.IS_DELETED.eq(UByte.valueOf(0))); if (appId != null) { @@ -401,19 +482,28 @@ private List genBaseConditions(Long appId, AccountCategoryEnum catego if (category != null) { conditions.add(TB_ACCOUNT.CATEGORY.eq(JooqDataTypeUtil.getByteFromInteger(category.getValue()))); } + if (StringUtils.isNotBlank(account)) { + conditions.add(TB_ACCOUNT.ACCOUNT_.eq(account)); + } + if (StringUtils.isNotBlank(alias)) { + conditions.add(TB_ACCOUNT.ALIAS.eq(alias)); + } return conditions; } @Override - public List listAllAppAccount(Long appId, AccountCategoryEnum category, - BaseSearchCondition baseSearchCondition) { - List conditions = genBaseConditions(appId, category); + public List listAppAccount(Long appId, + AccountCategoryEnum category, + String account, + String alias, + BaseSearchCondition baseSearchCondition) { + List conditions = genBaseConditions(appId, category, account, alias); return listAllAppAccount(conditions, baseSearchCondition); } @Override - public Integer countAllAppAccount(Long appId, AccountCategoryEnum category) { - List conditions = genBaseConditions(appId, category); + public Integer countAppAccount(Long appId, AccountCategoryEnum category, String account, String alias) { + List conditions = genBaseConditions(appId, category, account, alias); return countAllAppAccount(conditions); } @@ -429,10 +519,10 @@ public Integer countAllAppAccount(Collection conditions) { public List listAllAppAccount(Collection conditions, BaseSearchCondition baseSearchCondition) { - SelectConditionStep select = ctx.select(ALL_FILED) + SelectConditionStep select = ctx.select(ALL_FILED) .from(TB_ACCOUNT) .where(conditions); - Result result = null; + Result result; if (baseSearchCondition == null) { result = select.fetch(); } else { @@ -448,9 +538,7 @@ public List listAllAppAccount(Collection conditions, } List accountDTOS = new ArrayList<>(); if (result.size() != 0) { - result.into(record -> { - accountDTOS.add(extract(record)); - }); + result.into(record -> accountDTOS.add(extract(record))); } return accountDTOS; } @@ -470,16 +558,10 @@ public AccountDTO getAccount(Long appId, AccountCategoryEnum category, String al @Override public AccountDTO getAccount(Long appId, AccountCategoryEnum category, AccountTypeEnum type, String account, String alias) { - List conditions = genBaseConditions(appId, category); + List conditions = genBaseConditions(appId, category, account, alias); if (type != null) { conditions.add(TB_ACCOUNT.TYPE.eq(JooqDataTypeUtil.getByteFromInteger(type.getType()))); } - if (StringUtils.isNotBlank(account)) { - conditions.add(TB_ACCOUNT.ACCOUNT_.eq(account)); - } - if (StringUtils.isNotBlank(alias)) { - conditions.add(TB_ACCOUNT.ALIAS.eq(alias)); - } Record record = ctx.select(ALL_FILED) .from(TB_ACCOUNT) .where(conditions) @@ -587,8 +669,12 @@ public void batchUpdateDbAccountPassword(List updatedAccounts) { } private void updateDbPasswordAccount(DSLContext dsl, AccountDTO account) { - dsl.update(TB_ACCOUNT).set(TB_ACCOUNT.DB_PASSWORD, account.getDbPassword()) - .where(TB_ACCOUNT.ID.eq(account.getId())) + dsl.update(TB_ACCOUNT).set( + TB_ACCOUNT.DB_PASSWORD, dbPasswordCryptoService.encryptDbPasswordIfNeeded( + account.getCategory(), + account.getDbPassword() + ) + ).where(TB_ACCOUNT.ID.eq(account.getId())) .and(TB_ACCOUNT.CATEGORY.eq(JooqDataTypeUtil.getByteFromInteger(AccountCategoryEnum.DB.getValue()))) .execute(); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/ApplicationDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/ApplicationDAOImpl.java index c6c694517f..3ad85edd00 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/ApplicationDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/ApplicationDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -33,6 +33,7 @@ import com.tencent.bk.job.common.model.dto.ResourceScope; import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.manage.dao.ApplicationDAO; +import com.tencent.bk.job.manage.model.tables.Application; import lombok.extern.slf4j.Slf4j; import lombok.val; import org.apache.commons.collections4.CollectionUtils; @@ -43,9 +44,9 @@ import org.jooq.Result; import org.jooq.TableField; import org.jooq.conf.ParamType; -import org.jooq.generated.tables.Application; import org.jooq.types.UByte; import org.jooq.types.ULong; +import org.slf4j.helpers.MessageFormatter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; @@ -74,18 +75,18 @@ public class ApplicationDAOImpl implements ApplicationDAO { T_APP.ATTRS }; - private final DSLContext context; + private final DSLContext dslContext; @Autowired - public ApplicationDAOImpl(@Qualifier("job-manage-dsl-context") DSLContext context) { - this.context = context; + public ApplicationDAOImpl(@Qualifier("job-manage-dsl-context") DSLContext dslContext) { + this.dslContext = dslContext; } @Override public boolean existBiz(long bizId) { - val records = context.selectZero() + val records = dslContext.selectZero() .from(T_APP) - .where(T_APP.IS_DELETED.eq(UByte.valueOf(Bool.FALSE.getValue()))) + .where(T_APP.IS_DELETED.eq(UByte.valueOf(Bool.FALSE.byteValue()))) .and(T_APP.BK_SCOPE_TYPE.eq(ResourceScopeTypeEnum.BIZ.getValue())) .and(T_APP.BK_SCOPE_ID.eq("" + bizId)) .limit(1) @@ -95,10 +96,10 @@ public boolean existBiz(long bizId) { @Override public ApplicationDTO getAppById(long appId) { - Record record = context.select(ALL_FIELDS) + Record record = dslContext.select(ALL_FIELDS) .from(T_APP) .where(T_APP.APP_ID.eq(ULong.valueOf(appId))) - .and(T_APP.IS_DELETED.eq(UByte.valueOf(Bool.FALSE.getValue()))) + .and(T_APP.IS_DELETED.eq(UByte.valueOf(Bool.FALSE.byteValue()))) .fetchOne(); if (record != null) { return extract(record); @@ -134,7 +135,7 @@ private List listAppsByConditions(List conditions) { if (conditions == null) { conditions = new ArrayList<>(); } - Result result = context + Result result = dslContext .select(ALL_FIELDS) .from(T_APP) .where(conditions) @@ -155,7 +156,7 @@ public List listAllApps() { @Override public List listAppsByAppIds(List appIdList) { List conditions = getBasicNotDeletedConditions(); - conditions.add(T_APP.APP_ID.in(appIdList.parallelStream().map(ULong::valueOf).collect(Collectors.toList()))); + conditions.add(T_APP.APP_ID.in(appIdList.stream().map(ULong::valueOf).collect(Collectors.toList()))); return listAppsByConditions(conditions); } @@ -164,7 +165,7 @@ public List listBizAppsByBizIds(Collection bizIdList) { List conditions = getBasicNotDeletedConditions(); conditions.add(T_APP.BK_SCOPE_TYPE.eq(ResourceScopeTypeEnum.BIZ.getValue())); conditions.add( - T_APP.BK_SCOPE_ID.in(bizIdList.parallelStream().map(Object::toString) + T_APP.BK_SCOPE_ID.in(bizIdList.stream().map(Object::toString) .collect(Collectors.toList())) ); return listAppsByConditions(conditions); @@ -174,7 +175,7 @@ public List listBizAppsByBizIds(Collection bizIdList) { public List listAllBizAppBizIds() { List conditions = getBasicNotDeletedConditions(); conditions.add(T_APP.BK_SCOPE_TYPE.equal(ResourceScopeTypeEnum.BIZ.getValue())); - Result> records = context + Result> records = dslContext .select(T_APP.BK_SCOPE_ID) .from(T_APP) .where(conditions) @@ -211,7 +212,7 @@ public List listAppsByScopeType(ResourceScopeTypeEnum scopeType) } @Override - public Long insertApp(DSLContext dslContext, ApplicationDTO applicationDTO) { + public Long insertApp(ApplicationDTO applicationDTO) { ResourceScope scope = applicationDTO.getScope(); val query = dslContext.insertInto(T_APP, T_APP.APP_NAME, @@ -230,7 +231,7 @@ public Long insertApp(DSLContext dslContext, ApplicationDTO applicationDTO) { scope == null ? null : scope.getType().getValue(), scope == null ? null : scope.getId(), applicationDTO.getAttrs() == null ? null : JsonUtils.toJson(applicationDTO.getAttrs()), - UByte.valueOf(Bool.FALSE.getValue()) + UByte.valueOf(Bool.FALSE.byteValue()) ); try { val record = query.returning(T_APP.APP_ID).fetchOne(); @@ -243,13 +244,17 @@ val record = query.returning(T_APP.APP_ID).fetchOne(); ); } } catch (Exception e) { - log.error("Fail to insertAppInfo:SQL={}", query.getSQL(ParamType.INLINED), e); + String msg = MessageFormatter.arrayFormat( + "Fail to insertAppInfo:SQL={}", + new String[]{query.getSQL(ParamType.INLINED)} + ).getMessage(); + log.error(msg, e); throw e; } } @Override - public int updateApp(DSLContext dslContext, ApplicationDTO applicationDTO) { + public int updateApp(ApplicationDTO applicationDTO) { val query = dslContext.update(T_APP) .set(T_APP.APP_NAME, applicationDTO.getName()) .set(T_APP.BK_SUPPLIER_ACCOUNT, applicationDTO.getBkSupplierAccount()) @@ -261,9 +266,9 @@ public int updateApp(DSLContext dslContext, ApplicationDTO applicationDTO) { } @Override - public int restoreDeletedApp(DSLContext dslContext, long appId) { + public int restoreDeletedApp(long appId) { val query = dslContext.update(T_APP) - .set(T_APP.IS_DELETED, UByte.valueOf(Bool.FALSE.getValue())) + .set(T_APP.IS_DELETED, UByte.valueOf(Bool.FALSE.byteValue())) .where(T_APP.APP_ID.eq(ULong.valueOf(appId))); int affectedNum = query.execute(); if (log.isDebugEnabled()) { @@ -273,7 +278,7 @@ public int restoreDeletedApp(DSLContext dslContext, long appId) { } @Override - public int deleteAppByIdSoftly(DSLContext dslContext, long appId) { + public int deleteAppByIdSoftly(long appId) { val query = dslContext.update(T_APP) .set(T_APP.IS_DELETED, UByte.valueOf(1)) .where(T_APP.APP_ID.eq(ULong.valueOf(appId))); @@ -286,15 +291,15 @@ public int deleteAppByIdSoftly(DSLContext dslContext, long appId) { @Override public Integer countApps() { - return context.selectCount() + return dslContext.selectCount() .from(T_APP) - .where(T_APP.IS_DELETED.eq(UByte.valueOf(Bool.FALSE.getValue()))) + .where(T_APP.IS_DELETED.eq(UByte.valueOf(Bool.FALSE.byteValue()))) .fetchOne(0, Integer.class); } @Override public Integer countBizSetAppsWithDeleted() { - return context.selectCount() + return dslContext.selectCount() .from(T_APP) .where(T_APP.BK_SCOPE_TYPE.eq(ResourceScopeTypeEnum.BIZ_SET.getValue())) .fetchOne(0, Integer.class); @@ -302,11 +307,11 @@ public Integer countBizSetAppsWithDeleted() { @Override public ApplicationDTO getAppByScope(ResourceScope scope) { - Record record = context.select(ALL_FIELDS) + Record record = dslContext.select(ALL_FIELDS) .from(T_APP) .where(T_APP.BK_SCOPE_TYPE.eq(scope.getType().getValue())) .and(T_APP.BK_SCOPE_ID.eq(scope.getId())) - .and(T_APP.IS_DELETED.eq(UByte.valueOf(Bool.FALSE.getValue()))) + .and(T_APP.IS_DELETED.eq(UByte.valueOf(Bool.FALSE.byteValue()))) .fetchOne(); if (record != null) { return extract(record); @@ -316,7 +321,7 @@ public ApplicationDTO getAppByScope(ResourceScope scope) { @Override public ApplicationDTO getAppByScopeIncludingDeleted(ResourceScope scope) { - Record record = context.select(ALL_FIELDS) + Record record = dslContext.select(ALL_FIELDS) .from(T_APP) .where(T_APP.BK_SCOPE_TYPE.eq(scope.getType().getValue())) .and(T_APP.BK_SCOPE_ID.eq(scope.getId())) @@ -326,4 +331,16 @@ public ApplicationDTO getAppByScopeIncludingDeleted(ResourceScope scope) { } return null; } + + @Override + public List listAllDeletedApps() { + List conditions = getBasicDeletedConditions(); + return listAppsByConditions(conditions); + } + + private List getBasicDeletedConditions() { + List conditions = new ArrayList<>(); + conditions.add(T_APP.IS_DELETED.eq(UByte.valueOf(1))); + return conditions; + } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/ApplicationFavorDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/ApplicationFavorDAOImpl.java index e323704d39..68c11ace13 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/ApplicationFavorDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/ApplicationFavorDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,14 +26,14 @@ import com.tencent.bk.job.manage.dao.ApplicationFavorDAO; import com.tencent.bk.job.manage.model.dto.ApplicationFavorDTO; +import com.tencent.bk.job.manage.model.tables.ApplicationFavor; +import com.tencent.bk.job.manage.model.tables.records.ApplicationFavorRecord; import lombok.extern.slf4j.Slf4j; import lombok.val; import org.jooq.Condition; import org.jooq.DSLContext; import org.jooq.Result; import org.jooq.conf.ParamType; -import org.jooq.generated.tables.ApplicationFavor; -import org.jooq.generated.tables.records.ApplicationFavorRecord; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/ApplicationHostDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/ApplicationHostDAOImpl.java index 5ecd9272b8..8cc60a1823 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/ApplicationHostDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/ApplicationHostDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,21 +28,25 @@ import com.tencent.bk.job.common.constant.JobConstants; import com.tencent.bk.job.common.constant.ResourceScopeTypeEnum; import com.tencent.bk.job.common.exception.InternalException; -import com.tencent.bk.job.common.gse.constants.AgentStatusEnum; import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; import com.tencent.bk.job.common.model.dto.ApplicationDTO; import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; +import com.tencent.bk.job.common.model.dto.BasicHostDTO; +import com.tencent.bk.job.common.model.dto.HostSimpleDTO; +import com.tencent.bk.job.common.model.dto.HostStatusNumStatisticsDTO; import com.tencent.bk.job.common.model.dto.ResourceScope; +import com.tencent.bk.job.common.mysql.JobTransactional; +import com.tencent.bk.job.common.mysql.util.JooqDataTypeUtil; import com.tencent.bk.job.common.util.StringUtil; import com.tencent.bk.job.common.util.TagUtils; -import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.manage.common.TopologyHelper; -import com.tencent.bk.job.manage.common.util.JooqDataTypeUtil; import com.tencent.bk.job.manage.dao.ApplicationDAO; import com.tencent.bk.job.manage.dao.ApplicationHostDAO; import com.tencent.bk.job.manage.dao.HostTopoDAO; import com.tencent.bk.job.manage.model.dto.HostTopoDTO; +import com.tencent.bk.job.manage.model.tables.Host; +import com.tencent.bk.job.manage.model.tables.HostTopo; import lombok.extern.slf4j.Slf4j; import lombok.val; import lombok.var; @@ -51,15 +55,16 @@ import org.jooq.BatchBindStep; import org.jooq.Condition; import org.jooq.DSLContext; +import org.jooq.Field; import org.jooq.Query; import org.jooq.Record; import org.jooq.Record1; +import org.jooq.Record2; import org.jooq.Result; import org.jooq.SelectSeekStep2; import org.jooq.TableField; import org.jooq.conf.ParamType; -import org.jooq.generated.tables.Host; -import org.jooq.generated.tables.HostTopo; +import org.jooq.impl.DSL; import org.jooq.types.UByte; import org.jooq.types.ULong; import org.slf4j.helpers.FormattingTuple; @@ -67,16 +72,19 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; -import org.springframework.transaction.annotation.Transactional; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import static org.jooq.impl.DSL.count; + /** * 主机DAO */ @@ -88,29 +96,51 @@ public class ApplicationHostDAOImpl implements ApplicationHostDAO { TABLE.HOST_ID, TABLE.APP_ID, TABLE.IP, + TABLE.IP_V6, + TABLE.AGENT_ID, TABLE.IP_DESC, TABLE.SET_IDS, TABLE.MODULE_IDS, - TABLE.CLOUD_AREA_ID, + TABLE.CLOUD_ID, TABLE.DISPLAY_IP, TABLE.OS, TABLE.OS_TYPE, TABLE.MODULE_TYPE, TABLE.IS_AGENT_ALIVE, - TABLE.CLOUD_IP + TABLE.CLOUD_IP, + TABLE.CLOUD_VENDOR_ID + }; + + private static final TableField[] SIMPLE_FIELDS = { + TABLE.HOST_ID, + TABLE.APP_ID, + TABLE.IS_AGENT_ALIVE, + TABLE.IP, + TABLE.CLOUD_ID, + TABLE.AGENT_ID, + TABLE.APP_ID, + TABLE.IP_V6, + TABLE.IP_DESC, + TABLE.OS, + TABLE.OS_TYPE }; - private final DSLContext defaultContext; + private static final TableField[] BASIC_FIELDS = { + TABLE.HOST_ID, + TABLE.LAST_TIME + }; + + private final DSLContext context; private final ApplicationDAO applicationDAO; private final HostTopoDAO hostTopoDAO; private final TopologyHelper topologyHelper; @Autowired - public ApplicationHostDAOImpl(@Qualifier("job-manage-dsl-context") DSLContext defaultContext, + public ApplicationHostDAOImpl(@Qualifier("job-manage-dsl-context") DSLContext context, ApplicationDAO applicationDAO, HostTopoDAO hostTopoDAO, TopologyHelper topologyHelper) { - this.defaultContext = defaultContext; + this.context = context; this.applicationDAO = applicationDAO; this.topologyHelper = topologyHelper; this.hostTopoDAO = hostTopoDAO; @@ -119,8 +149,8 @@ public ApplicationHostDAOImpl(@Qualifier("job-manage-dsl-context") DSLContext de @Override public ApplicationHostDTO getHostById(Long hostId) { List conditions = new ArrayList<>(); - conditions.add(TABLE.HOST_ID.eq(ULong.valueOf(hostId))); - Record record = defaultContext.select(ALL_FIELDS).from(TABLE).where(conditions).fetchOne(); + conditions.add(TABLE.HOST_ID.eq(JooqDataTypeUtil.buildULong(hostId))); + Record record = context.select(ALL_FIELDS).from(TABLE).where(conditions).fetchOne(); return extractData(record); } @@ -133,12 +163,85 @@ public List listHostId(long bizId, long minUpdateTimeMills, long maxUpdate return listHostIdByConditions(conditions); } + @Override + public List listHostInfoByIps(Collection ips) { + return listHostInfoByIps(null, ips); + } + + @Override + public List listHostInfoByCloudIps(Collection cloudIps) { + return listHostInfoByCloudIps(null, cloudIps); + } + + @Override + public List listHostInfoByIps(Long bizId, Collection ips) { + List ipList = new ArrayList<>(ips); + List hostInfoList = new ArrayList<>(); + // 分批，防止SQL超长 + int batchSize = 30000; + int start = 0; + int end = start + batchSize; + int ipSize = ipList.size(); + end = Math.min(end, ipSize); + do { + List ipSubList = ipList.subList(start, end); + hostInfoList.addAll(listHostInfoByIpsIndeed(bizId, ipSubList)); + start += batchSize; + end = start + batchSize; + end = Math.min(end, ipSize); + } while (start < ipSize); + return hostInfoList; + } + + public List listHostInfoByCloudIps(Long bizId, Collection cloudIps) { + List cloudIpList = new ArrayList<>(cloudIps); + List hostInfoList = new ArrayList<>(); + // 分批，防止SQL超长 + int batchSize = 30000; + int start = 0; + int end = start + batchSize; + int ipSize = cloudIpList.size(); + end = Math.min(end, ipSize); + do { + List ipSubList = cloudIpList.subList(start, end); + hostInfoList.addAll(listHostInfoByCloudIpsIndeed(bizId, ipSubList)); + start += batchSize; + end = start + batchSize; + end = Math.min(end, ipSize); + } while (start < ipSize); + return hostInfoList; + } + + private List listHostInfoByIpsIndeed(Long bizId, Collection ips) { + List conditions = new ArrayList<>(); + if (bizId != null) { + conditions.addAll(buildBizIdCondition(bizId)); + } + conditions.add(TABLE.IP.in(ips)); + return queryHostsByCondition(conditions); + } + + private List listHostInfoByCloudIpsIndeed(Long bizId, Collection cloudIps) { + List conditions = new ArrayList<>(); + if (bizId != null) { + conditions.addAll(buildBizIdCondition(bizId)); + } + conditions.add(TABLE.CLOUD_IP.in(cloudIps)); + return queryHostsByCondition(conditions); + } + @Override public List listHostInfoByBizId(long bizId) { List conditions = buildBizIdCondition(bizId); return listHostInfoByConditions(conditions); } + @Override + public List listBasicHostInfo(Collection hostIds) { + List conditions = buildHostIdsCondition(hostIds); + return listBasicHostInfoByConditions(conditions); + } + @Override public List listAllHostInfo(Long start, Long limit) { return listHostInfoByConditions(Collections.emptyList(), start, limit); @@ -148,11 +251,35 @@ private List listHostInfoByConditions(Collection return listHostInfoByConditions(conditions, null, null); } + private List listBasicHostInfoByConditions(Collection conditions) { + if (conditions == null) { + conditions = Collections.emptyList(); + } + val query = context.select( + TABLE.HOST_ID, + TABLE.LAST_TIME + ).from(TABLE) + .where(conditions); + Result> records = query.fetch(); + List basicHostInfoList = new ArrayList<>(); + + if (CollectionUtils.isNotEmpty(records)) { + records.forEach(record -> { + BasicHostDTO basicHost = new BasicHostDTO( + JooqDataTypeUtil.getLongFromULong(record.get(TABLE.HOST_ID)), + record.get(TABLE.LAST_TIME) + ); + basicHostInfoList.add(basicHost); + }); + } + return basicHostInfoList; + } + private List listHostIdByConditions(Collection conditions) { if (conditions == null) { conditions = Collections.emptyList(); } - val query = defaultContext.select(TABLE.HOST_ID) + val query = context.select(TABLE.HOST_ID) .from(TABLE) .where(conditions); if (log.isDebugEnabled()) { @@ -168,7 +295,7 @@ private List listHostInfoByConditions(Collection if (conditions == null) { conditions = Collections.emptyList(); } - val query = defaultContext.select(ALL_FIELDS) + val query = context.select(ALL_FIELDS) .from(TABLE) .where(conditions) .orderBy(TABLE.IS_AGENT_ALIVE.desc(), TABLE.HOST_ID.asc()); @@ -198,9 +325,9 @@ private Result fetchRecordsWithLimit(SelectSeekStep2 listHostInfo(Collection bizIds, Collection ips) { List conditions = new ArrayList<>(); if (bizIds != null) { - conditions.add(TABLE.APP_ID.in(bizIds.parallelStream().map(ULong::valueOf).collect(Collectors.toList()))); + conditions.add(TABLE.APP_ID.in(bizIds.stream().map(ULong::valueOf).collect(Collectors.toList()))); } - conditions.add(TABLE.IP.in(ips.parallelStream().map(String::trim).collect(Collectors.toList()))); + conditions.add(TABLE.IP.in(ips.stream().map(String::trim).collect(Collectors.toList()))); return listHostInfoByConditions(conditions); } @@ -208,23 +335,88 @@ public List listHostInfo(Collection bizIds, Collection public List listHostInfoByBizAndCloudIPs(Collection bizIds, Collection cloudIPs) { List conditions = new ArrayList<>(); if (bizIds != null) { - conditions.add(TABLE.APP_ID.in(bizIds.parallelStream().map(ULong::valueOf).collect(Collectors.toList()))); + conditions.add(TABLE.APP_ID.in(bizIds.stream().map(ULong::valueOf).collect(Collectors.toList()))); + } + conditions.add(TABLE.CLOUD_IP.in(cloudIPs.stream().map(String::trim).collect(Collectors.toList()))); + return listHostInfoByConditions(conditions); + } + + @Override + public List listHostInfoByBizAndIps(Collection bizIds, Collection ips) { + List conditions = new ArrayList<>(); + if (bizIds != null) { + conditions.add(TABLE.APP_ID.in(bizIds.stream().map(ULong::valueOf).collect(Collectors.toList()))); + } + conditions.add(TABLE.IP.in(ips)); + return listHostInfoByConditions(conditions); + } + + @Override + public List listHostInfoByBizAndCloudIps(Collection bizIds, Collection cloudIps) { + List conditions = new ArrayList<>(); + if (bizIds != null) { + conditions.add(TABLE.APP_ID.in(bizIds.stream().map(ULong::valueOf).collect(Collectors.toList()))); + } + conditions.add(TABLE.CLOUD_IP.in(cloudIps)); + return listHostInfoByConditions(conditions); + } + + @Override + public List listHostInfoByBizAndIpv6s(Collection bizIds, Collection ipv6s) { + List conditions = new ArrayList<>(); + if (bizIds != null) { + conditions.add(TABLE.APP_ID.in(bizIds.stream().map(ULong::valueOf).collect(Collectors.toList()))); + } + conditions.add(TABLE.IP_V6.in(ipv6s)); + return listHostInfoByConditions(conditions); + } + + @Override + public List listHostInfoByBizAndHostNames(Collection bizIds, + Collection hostNames) { + List conditions = new ArrayList<>(); + if (bizIds != null) { + conditions.add(TABLE.APP_ID.in(bizIds.stream().map(ULong::valueOf).collect(Collectors.toList()))); } - conditions.add(TABLE.CLOUD_IP.in(cloudIPs.parallelStream().map(String::trim).collect(Collectors.toList()))); + conditions.add(TABLE.IP_DESC.in(hostNames)); return listHostInfoByConditions(conditions); } @Override public List listHostInfoByBizIds(Collection bizIds, Long start, Long limit) { List conditions = new ArrayList<>(); - conditions.add(TABLE.APP_ID.in(bizIds.parallelStream().map(ULong::valueOf).collect(Collectors.toList()))); + conditions.add(TABLE.APP_ID.in(bizIds.stream().map(ULong::valueOf).collect(Collectors.toList()))); return listHostInfoByConditions(conditions, start, limit); } @Override public List listHostInfoByHostIds(Collection hostIds) { List conditions = new ArrayList<>(); - conditions.add(TABLE.HOST_ID.in(hostIds.parallelStream().map(ULong::valueOf).collect(Collectors.toList()))); + conditions.add(TABLE.HOST_ID.in(hostIds.stream().map(ULong::valueOf).collect(Collectors.toList()))); + return listHostInfoByConditions(conditions); + } + + @Override + public List listHostInfoByIpv6s(Collection ipv6s) { + List conditions = new ArrayList<>(); + conditions.add(TABLE.IP_V6.in(ipv6s)); + return listHostInfoByConditions(conditions); + } + + @Override + public List listHostInfoByCloudIpv6(Long cloudAreaId, String ipv6) { + List conditions = new ArrayList<>(); + if (cloudAreaId != null) { + conditions.add(TABLE.CLOUD_ID.eq(cloudAreaId)); + } + conditions.add(TABLE.IP_V6.like("%" + ipv6 + "%")); + return listHostInfoByConditions(conditions); + } + + @Override + public List listHostInfoByHostNames(Collection hostNames) { + List conditions = new ArrayList<>(); + conditions.add(TABLE.IP_DESC.in(hostNames)); return listHostInfoByConditions(conditions); } @@ -234,23 +426,18 @@ public Long countHostInfoBySearchContents(Collection bizIds, Collection cloudAreaIds, List searchContents, Integer agentStatus) { - List hostIdList = getHostIdListBySearchContents(bizIds, moduleIds, cloudAreaIds, searchContents, - agentStatus, null, null); + List hostIdList = getHostIdListBySearchContents( + bizIds, + moduleIds, + cloudAreaIds, + searchContents, + agentStatus, + null, + null + ); return (long) (hostIdList.size()); } - @Override - public Long countHostByIdAndStatus(Collection hostIds, AgentStatusEnum agentStatus) { - List conditions = new ArrayList<>(); - if (hostIds != null) { - conditions.add(TABLE.HOST_ID.in(hostIds)); - } - if (agentStatus != null) { - conditions.add(TABLE.IS_AGENT_ALIVE.eq(UByte.valueOf(agentStatus.getValue()))); - } - return countHostByConditions(conditions); - } - @Override public List listHostInfoBySearchContents(Collection bizIds, Collection moduleIds, @@ -259,30 +446,114 @@ public List listHostInfoBySearchContents(Collection bi Integer agentStatus, Long start, Long limit) { - List hostIdList = getHostIdListBySearchContents(bizIds, moduleIds, cloudAreaIds, searchContents, - agentStatus, start, limit); + List hostIdList = getHostIdListBySearchContents( + bizIds, + moduleIds, + cloudAreaIds, + searchContents, + agentStatus, + start, + limit + ); return listHostInfoByHostIds(hostIdList); } - public List getHostIdListBySearchContents(Collection appIds, + @Override + public List getHostIdListBySearchContents(Collection bizIds, Collection moduleIds, Collection cloudAreaIds, List searchContents, - Integer agentStatus, + Integer agentAlive, Long start, Long limit) { + List conditions = buildSearchContentsConditions( + bizIds, + moduleIds, + cloudAreaIds, + searchContents, + agentAlive + ); + return getHostIdListByConditions(conditions, start, limit); + } + + @Override + public List listHostInfoByMultiKeys(Collection bizIds, + Collection moduleIds, + Collection cloudAreaIds, + Collection ipKeys, + Collection ipv6Keys, + Collection hostNameKeys, + Collection osNameKeys, + Integer agentAlive, + Long start, + Long limit) { + List hostIdList = getHostIdListByMultiKeys( + bizIds, + moduleIds, + cloudAreaIds, + ipKeys, + ipv6Keys, + hostNameKeys, + osNameKeys, + agentAlive, + start, + limit + ); + return listHostInfoByHostIds(hostIdList); + } + + @Override + public List getHostIdListByMultiKeys(Collection bizIds, + Collection moduleIds, + Collection cloudAreaIds, + Collection ipKeys, + Collection ipv6Keys, + Collection hostNameKeys, + Collection osNameKeys, + Integer agentAlive, + Long start, + Long limit) { + List conditions = buildMultiKeysConditions( + bizIds, + moduleIds, + cloudAreaIds, + ipKeys, + ipv6Keys, + hostNameKeys, + osNameKeys, + agentAlive + ); + return getHostIdListByConditions(conditions, start, limit); + } + + public Long countHostInfoByMultiKeys(Collection bizIds, + Collection moduleIds, + Collection cloudAreaIds, + Collection ipKeys, + Collection ipv6Keys, + Collection hostNameKeys, + Collection osNameKeys, + Integer agentAlive) { + List conditions = buildMultiKeysConditions( + bizIds, + moduleIds, + cloudAreaIds, + ipKeys, + ipv6Keys, + hostNameKeys, + osNameKeys, + agentAlive + ); + return countHostIdByConditions(conditions); + } + + private List buildSearchContentsConditions(Collection bizIds, + Collection moduleIds, + Collection cloudAreaIds, + List searchContents, + Integer agentAlive) { Host tHost = Host.HOST; - HostTopo tHostTopo = HostTopo.HOST_TOPO; - List conditions = new ArrayList<>(); - if (appIds != null) { - conditions.add(tHostTopo.APP_ID.in(appIds)); - } - if (agentStatus != null) { - conditions.add(tHost.IS_AGENT_ALIVE.eq(UByte.valueOf(agentStatus))); - } - if (moduleIds != null) { - conditions.add(tHostTopo.MODULE_ID.in(moduleIds)); - } + List conditions = buildConditions(bizIds, moduleIds, agentAlive); Condition condition = null; if (searchContents != null && !searchContents.isEmpty()) { String firstContent = searchContents.get(0); @@ -290,6 +561,10 @@ public List getHostIdListBySearchContents(Collection appIds, for (int i = 1; i < searchContents.size(); i++) { condition = condition.or(tHost.IP.like("%" + searchContents.get(i) + "%")); } + condition = condition.or(tHost.IP_V6.like("%" + firstContent + "%")); + for (int i = 1; i < searchContents.size(); i++) { + condition = condition.or(tHost.IP_V6.like("%" + searchContents.get(i) + "%")); + } condition = condition.or(tHost.IP_DESC.like("%" + firstContent + "%")); for (int i = 1; i < searchContents.size(); i++) { condition = condition.or(tHost.IP_DESC.like("%" + searchContents.get(i) + "%")); @@ -301,16 +576,86 @@ public List getHostIdListBySearchContents(Collection appIds, } if (cloudAreaIds != null) { if (condition != null) { - condition = condition.or(tHost.CLOUD_AREA_ID.in(cloudAreaIds)); + condition = condition.or(tHost.CLOUD_ID.in(cloudAreaIds)); } else { - condition = tHost.CLOUD_AREA_ID.in(cloudAreaIds); + condition = tHost.CLOUD_ID.in(cloudAreaIds); } } if (condition != null) { conditions.add(condition); } + return conditions; + } + + private void addFieldMultiLikeCondition(List conditions, Field field, Collection keys) { + if (CollectionUtils.isNotEmpty(keys)) { + List keyList = new ArrayList<>(keys); + String firstContent = keyList.get(0); + Condition condition = field.like("%" + firstContent + "%"); + for (int i = 1; i < keyList.size(); i++) { + condition = condition.or(field.like("%" + keyList.get(i) + "%")); + } + conditions.add(condition); + } + } + + private List buildMultiKeysConditions(Collection bizIds, + Collection moduleIds, + Collection cloudAreaIds, + Collection ipKeys, + Collection ipv6Keys, + Collection hostNameKeys, + Collection osNameKeys, + Integer agentAlive) { + Host tHost = Host.HOST; + List conditions = buildConditions(bizIds, moduleIds, agentAlive); + if (cloudAreaIds != null) { + conditions.add(tHost.CLOUD_ID.in(cloudAreaIds)); + } + addFieldMultiLikeCondition(conditions, tHost.IP, ipKeys); + addFieldMultiLikeCondition(conditions, tHost.IP_V6, ipv6Keys); + addFieldMultiLikeCondition(conditions, tHost.IP_DESC, hostNameKeys); + addFieldMultiLikeCondition(conditions, tHost.OS, osNameKeys); + return conditions; + } + + private List buildConditions(Collection bizIds, + Collection moduleIds, + Integer agentAlive) { + Host tHost = Host.HOST; + HostTopo tHostTopo = HostTopo.HOST_TOPO; + List conditions = new ArrayList<>(); + if (bizIds != null) { + conditions.add(tHostTopo.APP_ID.in(bizIds)); + } + if (agentAlive != null) { + conditions.add(tHost.IS_AGENT_ALIVE.eq(JooqDataTypeUtil.buildUByte(agentAlive))); + } + if (moduleIds != null) { + conditions.add(tHostTopo.MODULE_ID.in(moduleIds)); + } + return conditions; + } + + private Long countHostIdByConditions(Collection conditions) { + Host tHost = Host.HOST; + HostTopo tHostTopo = HostTopo.HOST_TOPO; + return context + .select(DSL.countDistinct(tHost.HOST_ID)) + .from(tHost) + .join(tHostTopo) + .on(tHost.HOST_ID.eq(tHostTopo.HOST_ID)) + .where(conditions) + .fetchOne(0, Long.class); + } + + private List getHostIdListByConditions(Collection conditions, + Long start, + Long limit) { + Host tHost = Host.HOST; + HostTopo tHostTopo = HostTopo.HOST_TOPO; val query = - defaultContext + context .selectDistinct(tHost.HOST_ID) .select(tHost.IS_AGENT_ALIVE) .from(tHost) @@ -321,8 +666,8 @@ public List getHostIdListBySearchContents(Collection appIds, log.debug("SQL={}", query.getSQL(ParamType.INLINED)); Result records = fetchRecordsWithLimit(query, start, limit); List hostIdList = new ArrayList<>(); - if (records.size() >= 1) { - hostIdList = records.parallelStream() + if (!records.isEmpty()) { + hostIdList = records.stream() .map(record -> record.get(0, Long.class)) .collect(Collectors.toList()); } @@ -348,7 +693,7 @@ public PageData listHostInfoByPage(ApplicationHostDTO applic int start = baseSearchCondition.getStartOrDefault(0); int length = baseSearchCondition.getLengthOrDefault(10); - Result result = defaultContext.select(ALL_FIELDS) + Result result = context.select(ALL_FIELDS) .from(TABLE) .where(conditions) .orderBy(TABLE.IS_AGENT_ALIVE.desc(), TABLE.HOST_ID.asc()) @@ -370,137 +715,96 @@ public PageData listHostInfoByPage(ApplicationHostDTO applic return hostInfoPageData; } - private void setDefaultValue(ApplicationHostDTO applicationHostDTO) { - if (applicationHostDTO.getHostId() == null) { - applicationHostDTO.setHostId(0L); - } - if (applicationHostDTO.getBizId() == null) { - applicationHostDTO.setBizId(JobConstants.PUBLIC_APP_ID); - } - if (applicationHostDTO.getCloudAreaId() == null) { - applicationHostDTO.setCloudAreaId(0L); - } - if (applicationHostDTO.getSetId() == null) { - applicationHostDTO.setSetId(new ArrayList<>()); - } - if (applicationHostDTO.getGseAgentAlive() == null) { - applicationHostDTO.setGseAgentAlive(true); - } - if (applicationHostDTO.getIp() == null) { - applicationHostDTO.setIp("0.0.0.0"); - } - if (applicationHostDTO.getDisplayIp() == null) { - applicationHostDTO.setDisplayIp("0.0.0.0"); - } - } - - private List genHostTopoDTOList(ApplicationHostDTO applicationHostDTO) { - List setIdList = applicationHostDTO.getSetId(); - List moduleIdList = applicationHostDTO.getModuleId(); - if (setIdList == null || moduleIdList == null) { - return Collections.emptyList(); - } else if (setIdList.size() != moduleIdList.size()) { - throw new RuntimeException("setIdList.size()!=moduleIdList.size(),hostInfo=" + JsonUtils.toJson(applicationHostDTO)); - } else { - List hostTopoDTOList = new ArrayList<>(); - for (int i = 0; i < setIdList.size(); i++) { - hostTopoDTOList.add(new HostTopoDTO(applicationHostDTO.getHostId(), - applicationHostDTO.getBizId(), setIdList.get(i), moduleIdList.get(i))); - } - return hostTopoDTOList; - } - } - - @Transactional @Override public int insertHostWithoutTopo(ApplicationHostDTO applicationHostDTO) { - return insertOrUpdateHost(defaultContext, applicationHostDTO, false); - } - - @Transactional - @Override - public void insertOrUpdateHost(ApplicationHostDTO hostDTO) { - insertOrUpdateHost(defaultContext, hostDTO, true); + return insertOrUpdateHost(context, applicationHostDTO); } private int insertOrUpdateHost(DSLContext defaultContext, - ApplicationHostDTO applicationHostDTO, - Boolean insertTopo) { - setDefaultValue(applicationHostDTO); + ApplicationHostDTO applicationHostDTO) { int result; String finalSetIdsStr = applicationHostDTO.getSetIdsStr(); String finalModuleIdsStr = applicationHostDTO.getModuleIdsStr(); String finalModuleTypeStr = applicationHostDTO.getModuleTypeStr(); ULong bizId = ULong.valueOf(applicationHostDTO.getBizId()); String ip = applicationHostDTO.getIp(); - String ipDesc = applicationHostDTO.getIpDesc(); - ULong cloudAreaId = ULong.valueOf(applicationHostDTO.getCloudAreaId()); + String ipv6 = applicationHostDTO.preferFullIpv6(); + String agentId = applicationHostDTO.getAgentId(); + String ipDesc = applicationHostDTO.getHostName(); String displayIp = applicationHostDTO.getDisplayIp(); - String os = applicationHostDTO.getOs(); + String os = applicationHostDTO.getOsName(); String osType = applicationHostDTO.getOsType(); UByte gseAgentAlive = UByte.valueOf(applicationHostDTO.getGseAgentAlive() ? 1 : 0); String cloudIp = applicationHostDTO.getCloudIp(); + String cloudVendor = applicationHostDTO.getCloudVendorId(); + Long lastTime = applicationHostDTO.getLastTime(); var query = defaultContext.insertInto(TABLE, TABLE.HOST_ID, TABLE.APP_ID, TABLE.IP, + TABLE.IP_V6, + TABLE.AGENT_ID, TABLE.IP_DESC, TABLE.SET_IDS, TABLE.MODULE_IDS, - TABLE.CLOUD_AREA_ID, + TABLE.CLOUD_ID, TABLE.DISPLAY_IP, TABLE.OS, TABLE.OS_TYPE, TABLE.MODULE_TYPE, TABLE.IS_AGENT_ALIVE, TABLE.CLOUD_IP, - TABLE.LAST_MODIFY_TIME + TABLE.LAST_MODIFY_TIME, + TABLE.CLOUD_VENDOR_ID, + TABLE.LAST_TIME ).values( - ULong.valueOf(applicationHostDTO.getHostId()), + JooqDataTypeUtil.buildULong(applicationHostDTO.getHostId()), bizId, ip, + ipv6, + agentId, ipDesc, finalSetIdsStr, finalModuleIdsStr, - cloudAreaId, + applicationHostDTO.getCloudAreaId(), displayIp, os, osType, finalModuleTypeStr, gseAgentAlive, cloudIp, - JooqDataTypeUtil.buildULong(System.currentTimeMillis()) + JooqDataTypeUtil.buildULong(System.currentTimeMillis()), + cloudVendor, + lastTime ); try { result = query.onDuplicateKeyUpdate() .set(TABLE.APP_ID, bizId) .set(TABLE.IP, ip) + .set(TABLE.IP_V6, ipv6) + .set(TABLE.AGENT_ID, agentId) .set(TABLE.IP_DESC, ipDesc) .set(TABLE.SET_IDS, finalSetIdsStr) .set(TABLE.MODULE_IDS, finalModuleIdsStr) - .set(TABLE.CLOUD_AREA_ID, cloudAreaId) + .set(TABLE.CLOUD_ID, applicationHostDTO.getCloudAreaId()) .set(TABLE.DISPLAY_IP, displayIp) .set(TABLE.OS, os) .set(TABLE.OS_TYPE, osType) .set(TABLE.MODULE_TYPE, finalModuleTypeStr) .set(TABLE.IS_AGENT_ALIVE, gseAgentAlive) .set(TABLE.CLOUD_IP, cloudIp) + .set(TABLE.CLOUD_VENDOR_ID, cloudVendor) + .set(TABLE.LAST_TIME, lastTime) .execute(); } catch (Throwable t) { log.info("SQL=" + query.getSQL(ParamType.INLINED)); throw t; } - if (insertTopo) { - List hostTopoDTOList = genHostTopoDTOList(applicationHostDTO); - hostTopoDAO.deleteHostTopoByHostId(applicationHostDTO.getBizId(), applicationHostDTO.getHostId()); - int affectedNum = hostTopoDAO.batchInsertHostTopo(hostTopoDTOList); - log.debug("{} hostTopo inserted", affectedNum); - } return result; } @Override - public int batchInsertAppHostInfo(List applicationHostDTOList) { + public int batchInsertHost(List applicationHostDTOList) { int batchSize = 1000; int size = applicationHostDTOList.size(); int start = 0; @@ -510,21 +814,25 @@ public int batchInsertAppHostInfo(List applicationHostDTOLis end = start + batchSize; end = Math.min(end, size); List subList = applicationHostDTOList.subList(start, end); - val insertQuery = defaultContext.insertInto(TABLE, + val insertQuery = context.insertInto(TABLE, TABLE.HOST_ID, TABLE.APP_ID, TABLE.IP, + TABLE.IP_V6, + TABLE.AGENT_ID, TABLE.IP_DESC, TABLE.SET_IDS, TABLE.MODULE_IDS, - TABLE.CLOUD_AREA_ID, + TABLE.CLOUD_ID, TABLE.DISPLAY_IP, TABLE.OS, TABLE.OS_TYPE, TABLE.MODULE_TYPE, TABLE.IS_AGENT_ALIVE, TABLE.CLOUD_IP, - TABLE.LAST_MODIFY_TIME + TABLE.LAST_MODIFY_TIME, + TABLE.CLOUD_VENDOR_ID, + TABLE.LAST_TIME ).values( (ULong) null, null, @@ -539,35 +847,39 @@ public int batchInsertAppHostInfo(List applicationHostDTOLis null, null, null, + null, + null, + null, + null, null ); - BatchBindStep batchQuery = defaultContext.batch(insertQuery); - List hostTopoDTOList = new ArrayList<>(); + BatchBindStep batchQuery = context.batch(insertQuery); for (ApplicationHostDTO applicationHostDTO : subList) { - setDefaultValue(applicationHostDTO); batchQuery = batchQuery.bind( - ULong.valueOf(applicationHostDTO.getHostId()), - ULong.valueOf(applicationHostDTO.getBizId()), + JooqDataTypeUtil.buildULong(applicationHostDTO.getHostId()), + JooqDataTypeUtil.buildULong(applicationHostDTO.getBizId()), applicationHostDTO.getIp(), - applicationHostDTO.getIpDesc(), + applicationHostDTO.preferFullIpv6(), + applicationHostDTO.getAgentId(), + applicationHostDTO.getHostName(), applicationHostDTO.getSetIdsStr(), applicationHostDTO.getModuleIdsStr(), - ULong.valueOf(applicationHostDTO.getCloudAreaId()), + applicationHostDTO.getCloudAreaId(), applicationHostDTO.getDisplayIp(), - applicationHostDTO.getOs(), + applicationHostDTO.getOsName(), applicationHostDTO.getOsType(), applicationHostDTO.getModuleTypeStr(), - UByte.valueOf(applicationHostDTO.getGseAgentAlive() ? 1 : 0), + JooqDataTypeUtil.buildUByte(applicationHostDTO.getAgentStatusValue()), applicationHostDTO.getCloudIp(), - JooqDataTypeUtil.buildULong(System.currentTimeMillis()) + JooqDataTypeUtil.buildULong(System.currentTimeMillis()), + applicationHostDTO.getCloudVendorId(), + applicationHostDTO.getLastTime() ); - hostTopoDTOList.addAll(genHostTopoDTOList(applicationHostDTO)); } int[] results = batchQuery.execute(); for (int result : results) { affectedNum += result; } - hostTopoDAO.batchInsertHostTopo(hostTopoDTOList); start += batchSize; } while (end < size); return affectedNum; @@ -575,23 +887,9 @@ public int batchInsertAppHostInfo(List applicationHostDTOLis @SuppressWarnings("all") @Override - public boolean existAppHostInfoByHostId(ApplicationHostDTO applicationHostDTO) { - setDefaultValue(applicationHostDTO); - if (applicationHostDTO.getHostId() == -1L) { - return false; - } - val query = defaultContext.selectCount().from(TABLE) - .where(TABLE.APP_ID.eq(ULong.valueOf(applicationHostDTO.getBizId()))) - .and(TABLE.HOST_ID.eq(ULong.valueOf(applicationHostDTO.getHostId()))) - .and(TABLE.IP.eq(applicationHostDTO.getIp())) - .and(TABLE.IP_DESC.eq(applicationHostDTO.getIpDesc())) - .and(TABLE.SET_IDS.eq(applicationHostDTO.getSetIdsStr())) - .and(TABLE.MODULE_IDS.eq(applicationHostDTO.getModuleIdsStr())) - .and(TABLE.CLOUD_AREA_ID.eq(ULong.valueOf(applicationHostDTO.getCloudAreaId()))) - .and(TABLE.DISPLAY_IP.eq(applicationHostDTO.getDisplayIp())) - .and(TABLE.OS.eq(applicationHostDTO.getOs())) - .and(TABLE.MODULE_TYPE.eq(applicationHostDTO.getModuleTypeStr())) - .and(TABLE.IS_AGENT_ALIVE.eq(UByte.valueOf(applicationHostDTO.getGseAgentAlive() ? 1 : 0))); + public boolean existAppHostInfoByHostId(Long hostId) { + val query = context.selectCount().from(TABLE) + .where(TABLE.HOST_ID.eq(JooqDataTypeUtil.buildULong(hostId))); try { return query.fetchOne(0, Long.class) >= 1; } catch (Throwable t) { @@ -600,21 +898,24 @@ public boolean existAppHostInfoByHostId(ApplicationHostDTO applicationHostDTO) { } } - @SuppressWarnings("all") @Override - public boolean existAppHostInfoByHostId(Long hostId) { - val query = defaultContext.selectCount().from(TABLE) - .where(TABLE.HOST_ID.eq(ULong.valueOf(hostId))); - try { - return query.fetchOne(0, Long.class) >= 1; - } catch (Throwable t) { - log.info("SQL=" + query.getSQL(ParamType.INLINED)); - throw t; - } + public int updateHostAttrsByHostId(ApplicationHostDTO applicationHostDTO) { + checkHostId(applicationHostDTO); + List conditions = new ArrayList<>(); + conditions.add(TABLE.HOST_ID.eq(ULong.valueOf(applicationHostDTO.getHostId()))); + return updateHostAttrsByConditions(applicationHostDTO, conditions); } @Override - public void updateHostAttrsById(ApplicationHostDTO applicationHostDTO) { + public int updateHostAttrsBeforeLastTime(ApplicationHostDTO applicationHostDTO) { + checkHostId(applicationHostDTO); + List conditions = new ArrayList<>(); + conditions.add(TABLE.HOST_ID.eq(ULong.valueOf(applicationHostDTO.getHostId()))); + conditions.add(TABLE.LAST_TIME.lessThan(applicationHostDTO.getLastTime())); + return updateHostAttrsByConditions(applicationHostDTO, conditions); + } + + private void checkHostId(ApplicationHostDTO applicationHostDTO) { Long hostId = applicationHostDTO.getHostId(); if (hostId == null || hostId <= 0) { FormattingTuple msg = MessageFormatter.format( @@ -624,88 +925,58 @@ public void updateHostAttrsById(ApplicationHostDTO applicationHostDTO) { log.error(msg.getMessage()); throw new InternalException(msg.getMessage(), ErrorCode.INTERNAL_ERROR); } - List conditions = new ArrayList<>(); - conditions.add(TABLE.HOST_ID.eq(ULong.valueOf(applicationHostDTO.getHostId()))); - val query = defaultContext.update(TABLE) - .set(TABLE.CLOUD_AREA_ID, ULong.valueOf(applicationHostDTO.getCloudAreaId())) + } + + public int updateHostAttrsByConditions(ApplicationHostDTO applicationHostDTO, Collection conditions) { + val query = context.update(TABLE) + .set(TABLE.CLOUD_ID, applicationHostDTO.getCloudAreaId()) .set(TABLE.IP, applicationHostDTO.getIp()) + .set(TABLE.IP_V6, applicationHostDTO.preferFullIpv6()) + .set(TABLE.AGENT_ID, applicationHostDTO.getAgentId()) .set(TABLE.DISPLAY_IP, applicationHostDTO.getDisplayIp()) .set(TABLE.CLOUD_IP, applicationHostDTO.getCloudIp()) - .set(TABLE.IP_DESC, applicationHostDTO.getIpDesc()) - .set(TABLE.OS, applicationHostDTO.getOs()) + .set(TABLE.IP_DESC, applicationHostDTO.getHostName()) + .set(TABLE.OS, applicationHostDTO.getOsName()) .set(TABLE.OS_TYPE, applicationHostDTO.getOsType()) .set(TABLE.IS_AGENT_ALIVE, UByte.valueOf(applicationHostDTO.getAgentStatusValue())) .set(TABLE.LAST_MODIFY_TIME, JooqDataTypeUtil.buildULong(System.currentTimeMillis())) + .set(TABLE.CLOUD_VENDOR_ID, applicationHostDTO.getCloudVendorId()) + .set(TABLE.LAST_TIME, applicationHostDTO.getLastTime()) .where(conditions); try { - query.execute(); - } catch (Throwable t) { - log.info("SQL=" + query.getSQL(ParamType.INLINED)); - throw t; - } - } - - @Override - public void updateBizHostInfoByHostId(Long bizId, ApplicationHostDTO applicationHostDTO) { - updateBizHostInfoByHostId(bizId, applicationHostDTO, true); - } - - @Transactional - @Override - public int updateBizHostInfoByHostId(Long bizId, - ApplicationHostDTO applicationHostDTO, - boolean updateTopo) { - setDefaultValue(applicationHostDTO); - if (applicationHostDTO.getHostId() == -1L) { - return -1; - } - int affectedNum; - List conditions = new ArrayList<>(); - if (bizId != null) { - conditions.add(TABLE.APP_ID.eq(ULong.valueOf(bizId))); - } - Long hostId = applicationHostDTO.getHostId(); - if (hostId != null) { - conditions.add(TABLE.HOST_ID.eq(ULong.valueOf(applicationHostDTO.getHostId()))); - } - val query = buildQueryWithHostAndConditions(applicationHostDTO, conditions); - try { - affectedNum = query.execute(); + return query.execute(); } catch (Throwable t) { log.info("SQL=" + query.getSQL(ParamType.INLINED)); throw t; } - if (updateTopo) { - List hostTopoDTOList = genHostTopoDTOList(applicationHostDTO); - hostTopoDAO.deleteHostTopoByHostId(bizId, hostId); - hostTopoDAO.batchInsertHostTopo(hostTopoDTOList); - } - return affectedNum; } private Query buildQueryWithHostAndConditions(ApplicationHostDTO applicationHostDTO, List conditions) { - return defaultContext.update(TABLE) - .set(TABLE.APP_ID, ULong.valueOf(applicationHostDTO.getBizId())) + return context.update(TABLE) + .set(TABLE.APP_ID, JooqDataTypeUtil.buildULong(applicationHostDTO.getBizId())) .set(TABLE.IP, applicationHostDTO.getIp()) + .set(TABLE.IP_V6, applicationHostDTO.preferFullIpv6()) + .set(TABLE.AGENT_ID, applicationHostDTO.getAgentId()) .set(TABLE.CLOUD_IP, applicationHostDTO.getCloudIp()) - .set(TABLE.IP_DESC, applicationHostDTO.getIpDesc()) + .set(TABLE.IP_DESC, applicationHostDTO.getHostName()) .set(TABLE.SET_IDS, applicationHostDTO.getSetIdsStr()) .set(TABLE.MODULE_IDS, applicationHostDTO.getModuleIdsStr()) - .set(TABLE.CLOUD_AREA_ID, ULong.valueOf(applicationHostDTO.getCloudAreaId())) + .set(TABLE.CLOUD_ID, applicationHostDTO.getCloudAreaId()) .set(TABLE.DISPLAY_IP, applicationHostDTO.getDisplayIp()) - .set(TABLE.OS, applicationHostDTO.getOs()) + .set(TABLE.OS, applicationHostDTO.getOsName()) .set(TABLE.OS_TYPE, applicationHostDTO.getOsType()) .set(TABLE.MODULE_TYPE, applicationHostDTO.getModuleTypeStr()) - .set(TABLE.IS_AGENT_ALIVE, UByte.valueOf(applicationHostDTO.getGseAgentAlive() ? 1 : 0)) + .set(TABLE.IS_AGENT_ALIVE, JooqDataTypeUtil.buildUByte(applicationHostDTO.getAgentStatusValue())) .set(TABLE.LAST_MODIFY_TIME, JooqDataTypeUtil.buildULong(System.currentTimeMillis())) + .set(TABLE.CLOUD_VENDOR_ID, applicationHostDTO.getCloudVendorId()) + .set(TABLE.LAST_TIME, applicationHostDTO.getLastTime()) .where(conditions); } - @Transactional @Override - public int batchUpdateBizHostInfoByHostId(List applicationHostDTOList) { + public int batchUpdateHostsBeforeLastTime(List hostList) { int batchSize = 1000; - int size = applicationHostDTOList.size(); + int size = hostList.size(); int start = 0; int end; List queryList = new ArrayList<>(); @@ -713,53 +984,45 @@ public int batchUpdateBizHostInfoByHostId(List applicationHo do { end = start + batchSize; end = Math.min(end, size); - List subList = applicationHostDTOList.subList(start, end); - List hostTopoDTOList = new ArrayList<>(); - for (ApplicationHostDTO applicationHostDTO : subList) { - setDefaultValue(applicationHostDTO); - if (applicationHostDTO.getHostId() == -1L) { - log.warn("Unexpected hostId==-1,hostInfo={}", applicationHostDTO); - continue; - } + List subList = hostList.subList(start, end); + for (ApplicationHostDTO host : subList) { List conditions = new ArrayList<>(); - conditions.add(TABLE.HOST_ID.eq(ULong.valueOf(applicationHostDTO.getHostId()))); - conditions.add(TABLE.APP_ID.eq(ULong.valueOf(applicationHostDTO.getBizId()))); - queryList.add(buildQueryWithHostAndConditions(applicationHostDTO, conditions)); - hostTopoDTOList.addAll(genHostTopoDTOList(applicationHostDTO)); + conditions.add(TABLE.HOST_ID.eq(ULong.valueOf(host.getHostId()))); + conditions.add(TABLE.LAST_TIME.lessThan(host.getLastTime())); + queryList.add(buildQueryWithHostAndConditions(host, conditions)); } - int[] results = defaultContext.batch(queryList).execute(); + int[] results = context.batch(queryList).execute(); queryList.clear(); for (int result : results) { affectedNum += result; } - // 更新hostTopo表数据 - hostTopoDAO.batchDeleteHostTopo( - new ArrayList<>(hostTopoDTOList.stream().map(HostTopoDTO::getHostId).collect(Collectors.toSet())) - ); - hostTopoDAO.batchInsertHostTopo(hostTopoDTOList); start += batchSize; } while (end < size); return affectedNum; } + @JobTransactional(transactionManager = "jobManageTransactionManager") @Override - public int deleteBizHostInfoById(Long bizId, Long hostId) { + public int deleteHostBeforeOrEqualLastTime(Long bizId, Long hostId, Long lastTime) { int affectedNum; List conditions = new ArrayList<>(); if (bizId != null) { - conditions.add(TABLE.APP_ID.eq(ULong.valueOf(bizId))); + conditions.add(TABLE.APP_ID.eq(JooqDataTypeUtil.buildULong(bizId))); } if (hostId != null) { - conditions.add(TABLE.HOST_ID.eq(ULong.valueOf(hostId))); + conditions.add(TABLE.HOST_ID.eq(JooqDataTypeUtil.buildULong(hostId))); + } + if (lastTime != null) { + conditions.add(TABLE.LAST_TIME.lessOrEqual(lastTime)); } - affectedNum = defaultContext.deleteFrom(TABLE) + affectedNum = context.deleteFrom(TABLE) .where(conditions) .execute(); hostTopoDAO.deleteHostTopoByHostId(bizId, hostId); return affectedNum; } - @Transactional + @JobTransactional(transactionManager = "jobManageTransactionManager") @Override public int batchDeleteHostById(List hostIdList) { if (CollectionUtils.isEmpty(hostIdList)) { @@ -767,16 +1030,16 @@ public int batchDeleteHostById(List hostIdList) { } List conditions = new ArrayList<>(); conditions.add( - TABLE.HOST_ID.in(hostIdList.parallelStream().map(ULong::valueOf).collect(Collectors.toList())) + TABLE.HOST_ID.in(hostIdList.stream().map(ULong::valueOf).collect(Collectors.toList())) ); int deletedRelationNum = hostTopoDAO.batchDeleteHostTopo(hostIdList); log.info("{} host relation deleted", deletedRelationNum); - return defaultContext.deleteFrom(TABLE) + return context.deleteFrom(TABLE) .where(conditions) .execute(); } - @Transactional + @JobTransactional(transactionManager = "jobManageTransactionManager") @Override public int batchDeleteBizHostInfoById(Long bizId, List hostIdList) { int affectedNum = 0; @@ -790,18 +1053,18 @@ public int batchDeleteBizHostInfoById(Long bizId, List hostIdList) { end = Math.min(end, size); List subList = hostIdList.subList(start, end); if (bizId != null) { - queryList.add(defaultContext.deleteFrom(TABLE) + queryList.add(context.deleteFrom(TABLE) .where(TABLE.HOST_ID.in(subList.stream().map(ULong::valueOf).collect(Collectors.toList()))) .and(TABLE.APP_ID.eq(ULong.valueOf(bizId))) ); } else { - queryList.add(defaultContext.deleteFrom(TABLE) + queryList.add(context.deleteFrom(TABLE) .where(TABLE.HOST_ID.in(subList.stream().map(ULong::valueOf).collect(Collectors.toList()))) ); } // SQL语句达到批量即执行 if (queryList.size() >= batchSize) { - int[] results = defaultContext.batch(queryList).execute(); + int[] results = context.batch(queryList).execute(); queryList.clear(); for (int result : results) { affectedNum += result; @@ -811,7 +1074,7 @@ public int batchDeleteBizHostInfoById(Long bizId, List hostIdList) { start += batchSize; } while (end < size); if (!queryList.isEmpty()) { - int[] results = defaultContext.batch(queryList).execute(); + int[] results = context.batch(queryList).execute(); for (int result : results) { affectedNum += result; } @@ -837,23 +1100,63 @@ public int deleteBizHostInfoByBizId(long bizId) { } @Override - public boolean existsHost(long bizId, String ip) { - return defaultContext.fetchExists(TABLE, TABLE.APP_ID.eq(ULong.valueOf(bizId)).and(TABLE.IP.eq(ip))); + public int deleteByBasicHost(List basicHostList) { + int affectedNum = 0; + int batchSize = 1000; + int size = basicHostList.size(); + int start = 0; + int end; + List queryList = new ArrayList<>(); + do { + end = start + batchSize; + end = Math.min(end, size); + List subList = basicHostList.subList(start, end); + for (BasicHostDTO basicHost : subList) { + queryList.add(context.deleteFrom(TABLE) + .where(TABLE.HOST_ID.eq(JooqDataTypeUtil.buildULong(basicHost.getHostId()))) + .and(TABLE.LAST_TIME.eq(basicHost.getLastTime())) + ); + } + int[] results = context.batch(queryList).execute(); + queryList.clear(); + for (int result : results) { + affectedNum += result; + } + start += batchSize; + } while (end < size); + return affectedNum; + } + + @Override + public List listAllHostSimpleInfo() { + val query = context.select(SIMPLE_FIELDS) + .from(TABLE); + Result records = query.fetch(); + List hostInfoList = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(records)) { + records.map(record -> hostInfoList.add(extractSimpleData(record))); + } + return hostInfoList; } @Override - public ApplicationHostDTO getLatestHost(long bizId, long cloudAreaId, String ip) { - List conditions = new ArrayList<>(); - conditions.add(TABLE.APP_ID.eq(ULong.valueOf(bizId))); - conditions.add(TABLE.CLOUD_AREA_ID.eq(ULong.valueOf(cloudAreaId))); - conditions.add(TABLE.IP.eq(ip)); - Record record = defaultContext - .select(ALL_FIELDS) - .from(TABLE) - .where(conditions) - .orderBy(TABLE.ROW_UPDATE_TIME.desc(), TABLE.HOST_ID.asc()) - .fetchOne(); - return extractData(record); + public List listAllBasicHost() { + val query = context.select(BASIC_FIELDS) + .from(TABLE); + Result records = query.fetch(); + List basicHostList = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(records)) { + records.map(record -> basicHostList.add(extractBasicHost(record))); + } + return basicHostList; + } + + @Override + public int batchUpdateHostStatusByHostIds(int status, List hostIdList) { + return context.update(TABLE) + .set(TABLE.IS_AGENT_ALIVE, UByte.valueOf(status)) + .where(TABLE.HOST_ID.in(hostIdList)) + .execute(); } @Override @@ -878,16 +1181,38 @@ public long countHostsByOsType(String osType) { return countHostByConditions(conditions); } + @Override + public Map groupHostByOsType() { + Map groupMap = new HashMap<>(); + context.select( + TABLE.OS_TYPE, + count() + ) + .from(TABLE) + .groupBy(TABLE.OS_TYPE) + .fetch() + .map(record -> { + String osType = record.get(0, String.class); + if (StringUtils.isNotBlank(osType)) { + groupMap.put(osType, record.get(1, Integer.class)); + } else { + groupMap.put("null", record.get(1, Integer.class)); + } + return record; + }); + return groupMap; + } + @Override public int syncHostTopo(Long hostId) { ApplicationHostDTO hostInfoDTO = getHostById(hostId); if (hostInfoDTO != null) { List hostTopoDTOList = hostTopoDAO.listHostTopoByHostId(hostId); List setIds = - hostTopoDTOList.parallelStream().map(HostTopoDTO::getSetId).collect(Collectors.toList()); + hostTopoDTOList.stream().map(HostTopoDTO::getSetId).collect(Collectors.toList()); List moduleIds = - hostTopoDTOList.parallelStream().map(HostTopoDTO::getModuleId).collect(Collectors.toList()); - List moduleTypes = moduleIds.parallelStream().map(it -> 1L).collect(Collectors.toList()); + hostTopoDTOList.stream().map(HostTopoDTO::getModuleId).collect(Collectors.toList()); + List moduleTypes = moduleIds.stream().map(it -> 1L).collect(Collectors.toList()); if (!hostTopoDTOList.isEmpty()) { hostInfoDTO.setBizId(hostTopoDTOList.get(0).getBizId()); } else { @@ -896,11 +1221,28 @@ public int syncHostTopo(Long hostId) { hostInfoDTO.setSetId(setIds); hostInfoDTO.setModuleId(moduleIds); hostInfoDTO.setModuleType(moduleTypes); - return updateBizHostInfoByHostId(null, hostInfoDTO, false); + return updateHostTopoAttrsByHostId(hostInfoDTO); } return -1; } + private int updateHostTopoAttrsByHostId(ApplicationHostDTO host) { + List conditions = new ArrayList<>(); + conditions.add(TABLE.HOST_ID.eq(ULong.valueOf(host.getHostId()))); + val query = context.update(TABLE) + .set(TABLE.APP_ID, ULong.valueOf(host.getBizId())) + .set(TABLE.SET_IDS, host.getSetIdsStr()) + .set(TABLE.MODULE_IDS, host.getModuleIdsStr()) + .set(TABLE.MODULE_TYPE, host.getModuleTypeStr()) + .where(conditions); + try { + return query.execute(); + } catch (Throwable t) { + log.info("SQL=" + query.getSQL(ParamType.INLINED)); + throw t; + } + } + /** * 查询符合条件的主机数量 */ @@ -909,7 +1251,13 @@ private long countHostByConditions(List conditions) { if (conditions == null) { conditions = Collections.emptyList(); } - return defaultContext.selectCount().from(TABLE).where(conditions).fetchOne(0, Long.class); + return context.selectCount().from(TABLE).where(conditions).fetchOne(0, Long.class); + } + + private List buildHostIdsCondition(Collection hostIds) { + List conditions = new ArrayList<>(); + conditions.add(TABLE.HOST_ID.in(hostIds)); + return conditions; } private List buildBizIdCondition(long bizId) { @@ -918,7 +1266,7 @@ private List buildBizIdCondition(long bizId) { ); List conditions = new ArrayList<>(); if (appInfo.isBiz()) { - conditions.add(TABLE.APP_ID.eq(ULong.valueOf(bizId))); + conditions.add(TABLE.APP_ID.eq(JooqDataTypeUtil.buildULong(bizId))); } else if (!appInfo.isAllBizSet() && appInfo.isBizSet()) { List subBizIds = topologyHelper.getBizSetSubBizIds(appInfo); conditions.add(TABLE.APP_ID.in(subBizIds.stream().map(ULong::valueOf).collect(Collectors.toList()))); @@ -933,7 +1281,7 @@ private List buildCondition(ApplicationHostDTO applicationHostInfoCon } if (applicationHostInfoCondition.getGseAgentAlive() != null) { - conditions.add(TABLE.IS_AGENT_ALIVE.eq(UByte.valueOf(applicationHostInfoCondition.getGseAgentAlive() ? 1 + conditions.add(TABLE.IS_AGENT_ALIVE.eq(JooqDataTypeUtil.buildUByte(applicationHostInfoCondition.getGseAgentAlive() ? 1 : 0))); } @@ -953,15 +1301,41 @@ private List buildCondition(ApplicationHostDTO applicationHostInfoCon } @Override - public List listHostsByIps(Collection cloudIps) { + public List listHostsByCloudIps(Collection cloudIps) { List conditions = new ArrayList<>(); conditions.add(TABLE.CLOUD_IP.in(cloudIps)); return queryHostsByCondition(conditions); } + @Override + public List countHostStatusNumByBizIds(List bizIds) { + List conditions = new ArrayList<>(); + if (bizIds != null) { + conditions.add(HostTopo.HOST_TOPO.APP_ID.in(bizIds)); + } + var query = context.select( + TABLE.IS_AGENT_ALIVE.as(HostStatusNumStatisticsDTO.KEY_AGENT_ALIVE), + DSL.countDistinct(TABLE.HOST_ID).as(HostStatusNumStatisticsDTO.KEY_HOST_NUM) + ).from(TABLE) + .leftJoin(HostTopo.HOST_TOPO).on(TABLE.HOST_ID.eq(HostTopo.HOST_TOPO.HOST_ID)) + .where(conditions) + .groupBy(TABLE.IS_AGENT_ALIVE); + val records = query.fetch(); + List countList = new ArrayList<>(); + if (!records.isEmpty()) { + records.forEach(record -> { + HostStatusNumStatisticsDTO statisticsDTO = new HostStatusNumStatisticsDTO(); + statisticsDTO.setHostNum(record.get(HostStatusNumStatisticsDTO.KEY_HOST_NUM, Integer.class)); + statisticsDTO.setGseAgentAlive(record.get(HostStatusNumStatisticsDTO.KEY_AGENT_ALIVE, Integer.class)); + countList.add(statisticsDTO); + }); + } + return countList; + } + private List queryHostsByCondition(List conditions) { Result result = - defaultContext.select(ALL_FIELDS) + context.select(ALL_FIELDS) .from(TABLE) .where(conditions) .fetch(); @@ -982,14 +1356,16 @@ public static ApplicationHostDTO extractData(Record record) { ApplicationHostDTO applicationHostDTO = new ApplicationHostDTO(); applicationHostDTO.setBizId(record.get(TABLE.APP_ID).longValue()); applicationHostDTO.setIp(record.get(TABLE.IP)); - applicationHostDTO.setIpDesc(record.get(TABLE.IP_DESC)); + applicationHostDTO.setIpv6(record.get(TABLE.IP_V6)); + applicationHostDTO.setAgentId(record.get(TABLE.AGENT_ID)); + applicationHostDTO.setHostName(record.get(TABLE.IP_DESC)); applicationHostDTO.setGseAgentAlive(record.get(TABLE.IS_AGENT_ALIVE).intValue() == 1); List setIdList = new ArrayList<>(); String setIdsStr = record.get(TABLE.SET_IDS); if (setIdsStr != null) { List list = new ArrayList<>(); for (String id : setIdsStr.split(",")) { - if (!id.trim().equals("")) { + if (!id.trim().isEmpty()) { Long parseLong = Long.parseLong(id); list.add(parseLong); } @@ -998,13 +1374,44 @@ public static ApplicationHostDTO extractData(Record record) { } applicationHostDTO.setSetId(setIdList); applicationHostDTO.setModuleId(StringUtil.strToList(record.get(TABLE.MODULE_IDS), Long.class, ",")); - applicationHostDTO.setCloudAreaId(record.get(TABLE.CLOUD_AREA_ID).longValue()); + applicationHostDTO.setCloudAreaId(record.get(TABLE.CLOUD_ID)); applicationHostDTO.setDisplayIp(record.get(TABLE.DISPLAY_IP)); - applicationHostDTO.setOs(record.get(TABLE.OS)); + applicationHostDTO.setOsName(record.get(TABLE.OS)); applicationHostDTO.setOsType(record.get(TABLE.OS_TYPE)); applicationHostDTO.setModuleType(StringUtil.strToList(record.get(TABLE.MODULE_TYPE), Long.class, ",")); applicationHostDTO.setHostId(record.get(TABLE.HOST_ID).longValue()); applicationHostDTO.setCloudIp(record.get(TABLE.CLOUD_IP)); + applicationHostDTO.setCloudVendorId(record.get(TABLE.CLOUD_VENDOR_ID)); return applicationHostDTO; } + + public static HostSimpleDTO extractSimpleData(Record record) { + if (record == null) { + return null; + } + HostSimpleDTO hostSimpleDTO = new HostSimpleDTO(); + hostSimpleDTO.setBizId(record.get(TABLE.APP_ID).longValue()); + hostSimpleDTO.setAgentAliveStatus(record.get(TABLE.IS_AGENT_ALIVE).intValue()); + hostSimpleDTO.setHostId(record.get(TABLE.HOST_ID).longValue()); + hostSimpleDTO.setAgentId(record.get(TABLE.AGENT_ID)); + hostSimpleDTO.setIpv6(record.get(TABLE.IP_V6)); + hostSimpleDTO.setHostName(record.get(TABLE.IP_DESC)); + hostSimpleDTO.setOsName(record.get(TABLE.OS)); + hostSimpleDTO.setOsType(record.get(TABLE.OS_TYPE)); + hostSimpleDTO.setIp(record.get(TABLE.IP)); + hostSimpleDTO.setCloudAreaId(record.get(TABLE.CLOUD_ID)); + hostSimpleDTO.setCloudIp(hostSimpleDTO.getCloudAreaId() + ":" + hostSimpleDTO.getIp()); + + return hostSimpleDTO; + } + + public static BasicHostDTO extractBasicHost(Record record) { + if (record == null) { + return null; + } + BasicHostDTO basicHost = new BasicHostDTO(); + basicHost.setHostId(record.get(TABLE.HOST_ID).longValue()); + basicHost.setLastTime(record.get(TABLE.LAST_TIME)); + return basicHost; + } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/CredentialDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/CredentialDAOImpl.java index b1c21bf504..a178ff6058 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/CredentialDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/CredentialDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,27 +27,28 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; -import com.tencent.bk.job.common.util.Base64Util; +import com.tencent.bk.job.common.model.dto.CommonCredential; import com.tencent.bk.job.common.util.JobUUID; -import com.tencent.bk.job.common.util.crypto.AESUtils; import com.tencent.bk.job.common.util.json.JsonUtils; -import com.tencent.bk.job.manage.config.JobTicketConfig; +import com.tencent.bk.job.manage.crypto.CredentialCryptoService; import com.tencent.bk.job.manage.dao.CredentialDAO; -import com.tencent.bk.job.common.model.dto.CommonCredential; import com.tencent.bk.job.manage.model.dto.CredentialDTO; import com.tencent.bk.job.manage.model.inner.resp.ServiceCredentialDisplayDTO; +import com.tencent.bk.job.manage.model.tables.Credential; +import com.tencent.bk.job.manage.model.tables.records.CredentialRecord; import io.micrometer.core.instrument.util.StringUtils; import lombok.extern.slf4j.Slf4j; import lombok.val; import org.jooq.Condition; import org.jooq.DSLContext; import org.jooq.Record; +import org.jooq.Record2; +import org.jooq.Result; import org.jooq.SortField; import org.jooq.UpdateConditionStep; import org.jooq.conf.ParamType; -import org.jooq.generated.tables.Credential; -import org.jooq.generated.tables.records.CredentialRecord; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; import java.util.ArrayList; @@ -59,17 +60,18 @@ public class CredentialDAOImpl implements CredentialDAO { private static final Credential defaultTable = Credential.CREDENTIAL; - private final JobTicketConfig jobTicketConfig; - private final DSLContext defaultDSLContext; + private final CredentialCryptoService credentialCryptoService; + private final DSLContext dslContext; @Autowired - public CredentialDAOImpl(JobTicketConfig jobTicketConfig, DSLContext dslContext) { - this.jobTicketConfig = jobTicketConfig; - this.defaultDSLContext = dslContext; + public CredentialDAOImpl(@Qualifier("job-manage-dsl-context") DSLContext dslContext, + CredentialCryptoService credentialCryptoService) { + this.credentialCryptoService = credentialCryptoService; + this.dslContext = dslContext; } @Override - public String insertCredential(DSLContext dslContext, CredentialDTO credentialDTO) { + public String insertCredential(CredentialDTO credentialDTO) { String id = JobUUID.getUUID(); String sql = null; String credentialStr = JsonUtils.toJson(credentialDTO.getCredential()); @@ -92,10 +94,7 @@ public String insertCredential(DSLContext dslContext, CredentialDTO credentialDT credentialDTO.getName(), credentialDTO.getType(), credentialDTO.getDescription(), - AESUtils.encryptToBase64EncodedCipherText( - credentialStr, - jobTicketConfig.getEncryptPassword() - ), + credentialCryptoService.encryptCredential(credentialStr), credentialDTO.getCreator(), credentialDTO.getCreateTime(), credentialDTO.getLastModifyUser(), @@ -113,7 +112,7 @@ public String insertCredential(DSLContext dslContext, CredentialDTO credentialDT } @Override - public String updateCredentialById(DSLContext dslContext, CredentialDTO credentialDTO) { + public String updateCredentialById(CredentialDTO credentialDTO) { String sql = null; String credentialStr = JsonUtils.toJson(credentialDTO.getCredential()); log.debug("Update credentialStr={}", credentialStr); @@ -123,9 +122,7 @@ public String updateCredentialById(DSLContext dslContext, CredentialDTO credenti .set(defaultTable.NAME, credentialDTO.getName()) .set(defaultTable.TYPE, credentialDTO.getType()) .set(defaultTable.DESCRIPTION, credentialDTO.getDescription()) - .set(defaultTable.VALUE, AESUtils.encryptToBase64EncodedCipherText( - credentialStr, - jobTicketConfig.getEncryptPassword())) + .set(defaultTable.VALUE, credentialCryptoService.encryptCredential(credentialStr)) .set(defaultTable.LAST_MODIFY_USER, credentialDTO.getLastModifyUser()) .set(defaultTable.LAST_MODIFY_TIME, System.currentTimeMillis()) .where(defaultTable.ID.eq(credentialDTO.getId())); @@ -141,14 +138,14 @@ public String updateCredentialById(DSLContext dslContext, CredentialDTO credenti } @Override - public int deleteCredentialById(DSLContext dslContext, String id) { + public int deleteCredentialById(String id) { return dslContext.deleteFrom(defaultTable).where( defaultTable.ID.eq(id) ).execute(); } @Override - public CredentialDTO getCredentialById(DSLContext dslContext, String id) { + public CredentialDTO getCredentialById(String id) { val record = dslContext.select( defaultTable.ID, defaultTable.APP_ID, @@ -171,8 +168,7 @@ val record = dslContext.select( } @Override - public List listCredentialDisplayInfoByIds(DSLContext dslContext, - Collection ids) { + public List listCredentialDisplayInfoByIds(Collection ids) { val records = dslContext.select( defaultTable.ID, defaultTable.APP_ID, @@ -210,11 +206,18 @@ private List buildConditionList( } /** - * 查询符合条件的凭据数量 + * 查询符合条件的凭证数量 */ private long getPageCredentialCount(CredentialDTO credentialQuery, BaseSearchCondition baseSearchCondition) { List conditions = buildConditionList(credentialQuery, baseSearchCondition); - Long count = defaultDSLContext + return getPageCredentialCount(conditions); + } + + /** + * 查询符合条件的凭证数量 + */ + private long getPageCredentialCount(Collection conditions) { + Long count = dslContext .selectCount() .from(defaultTable) .where(conditions) @@ -237,9 +240,55 @@ public PageData listCredentials( return listPageCredentialByConditions(baseSearchCondition, conditions, count); } + @Override + public PageData listCredentialBasicInfo(Long appId, BaseSearchCondition baseSearchCondition) { + Collection conditions = new ArrayList<>(); + conditions.add(defaultTable.APP_ID.eq(appId)); + long count = getPageCredentialCount(conditions); + return listPageCredentialBasicInfoByConditions(baseSearchCondition, conditions, count); + } + + public PageData listPageCredentialBasicInfoByConditions( + BaseSearchCondition baseSearchCondition, + Collection conditions, + long count + ) { + Integer start = baseSearchCondition.getStart(); + Integer length = baseSearchCondition.getLength(); + val query = + dslContext.select( + defaultTable.ID, + defaultTable.NAME + ).from(defaultTable) + .where(conditions) + .orderBy(defaultTable.LAST_MODIFY_TIME.desc()); + Result> records; + if (length != null && length > 0) { + records = query.limit(start, length).fetch(); + } else { + records = query.offset(start).fetch(); + } + List credentials = new ArrayList<>(); + if (records.size() != 0) { + records.forEach(record -> { + CredentialDTO credentialDTO = new CredentialDTO(); + credentialDTO.setId(record.get(defaultTable.ID)); + credentialDTO.setName(record.get(defaultTable.NAME)); + credentials.add(credentialDTO); + }); + } + + PageData credentialPageData = new PageData<>(); + credentialPageData.setTotal(count); + credentialPageData.setPageSize(length); + credentialPageData.setData(credentials); + credentialPageData.setStart(start); + return credentialPageData; + } + public PageData listPageCredentialByConditions( BaseSearchCondition baseSearchCondition, - List conditions, + Collection conditions, long count ) { Collection> orderFields = new ArrayList<>(); @@ -271,7 +320,7 @@ public PageData listPageCredentialByConditions( int start = baseSearchCondition.getStartOrDefault(0); int length = baseSearchCondition.getLengthOrDefault(10); val records = - defaultDSLContext.select( + dslContext.select( defaultTable.ID, defaultTable.APP_ID, defaultTable.NAME, @@ -305,9 +354,8 @@ public PageData listPageCredentialByConditions( private CredentialDTO convertRecordToDto(Record record) { try { - String credentialStr = AESUtils.decryptToPlainText( - Base64Util.decodeContentToByte(record.get(defaultTable.VALUE)), - jobTicketConfig.getEncryptPassword() + String credentialStr = credentialCryptoService.decryptCredential( + record.get(defaultTable.VALUE) ); log.debug("Get credential from DB:{}", credentialStr); return new CredentialDTO( diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/HostTopoDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/HostTopoDAOImpl.java index 40d16e03e0..cd4d9001d9 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/HostTopoDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/HostTopoDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,28 +24,34 @@ package com.tencent.bk.job.manage.dao.impl; -import com.tencent.bk.job.manage.common.util.JooqDataTypeUtil; +import com.tencent.bk.job.common.mysql.util.JooqDataTypeUtil; +import com.tencent.bk.job.common.util.CollectionUtil; import com.tencent.bk.job.manage.dao.HostTopoDAO; import com.tencent.bk.job.manage.model.dto.HostTopoDTO; +import com.tencent.bk.job.manage.model.tables.HostTopo; +import com.tencent.bk.job.manage.model.tables.records.HostTopoRecord; import lombok.extern.slf4j.Slf4j; import lombok.val; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.tuple.Pair; import org.jooq.BatchBindStep; import org.jooq.Condition; import org.jooq.DSLContext; import org.jooq.DeleteConditionStep; import org.jooq.Query; import org.jooq.Result; +import org.jooq.UpdateConditionStep; import org.jooq.conf.ParamType; -import org.jooq.generated.tables.HostTopo; -import org.jooq.generated.tables.records.HostTopoRecord; import org.jooq.types.ULong; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; @Repository @@ -56,28 +62,33 @@ public class HostTopoDAOImpl implements HostTopoDAO { private final DSLContext defaultContext; @Autowired - public HostTopoDAOImpl(DSLContext dslContext) { + public HostTopoDAOImpl(@Qualifier("job-manage-dsl-context") DSLContext dslContext) { this.defaultContext = dslContext; } @Override - public void insertHostTopo(HostTopoDTO hostTopoDTO) { + public int insertHostTopo(HostTopoDTO hostTopoDTO) { val query = defaultContext.insertInto(defaultTable, defaultTable.HOST_ID, defaultTable.APP_ID, defaultTable.SET_ID, - defaultTable.MODULE_ID + defaultTable.MODULE_ID, + defaultTable.LAST_TIME ).values( ULong.valueOf(hostTopoDTO.getHostId()), ULong.valueOf(hostTopoDTO.getBizId()), hostTopoDTO.getSetId(), - hostTopoDTO.getModuleId() + hostTopoDTO.getModuleId(), + hostTopoDTO.getLastTime() ).onDuplicateKeyIgnore(); - query.execute(); + return query.execute(); } @Override public int batchInsertHostTopo(List hostTopoDTOList) { + if (CollectionUtils.isEmpty(hostTopoDTOList)) { + return 0; + } int batchSize = 1000; int size = hostTopoDTOList.size(); int start = 0; @@ -87,19 +98,17 @@ public int batchInsertHostTopo(List hostTopoDTOList) { end = start + batchSize; end = Math.min(end, size); List subList = hostTopoDTOList.subList(start, end); - if (subList.isEmpty()) { - // 避免插入空数据 - break; - } val insertQuery = defaultContext.insertInto(defaultTable, defaultTable.HOST_ID, defaultTable.APP_ID, defaultTable.SET_ID, - defaultTable.MODULE_ID + defaultTable.MODULE_ID, + defaultTable.LAST_TIME ).values( (ULong) null, null, null, + null, null ).onDuplicateKeyIgnore(); BatchBindStep batchQuery = defaultContext.batch(insertQuery); @@ -108,7 +117,8 @@ public int batchInsertHostTopo(List hostTopoDTOList) { ULong.valueOf(hostTopoDTO.getHostId()), hostTopoDTO.getBizId(), hostTopoDTO.getSetId(), - hostTopoDTO.getModuleId() + hostTopoDTO.getModuleId(), + hostTopoDTO.getLastTime() ); } int[] results = batchQuery.execute(); @@ -121,10 +131,10 @@ public int batchInsertHostTopo(List hostTopoDTOList) { } @Override - public void deleteHostTopoByHostId(Long appId, Long hostId) { + public void deleteHostTopoByHostId(Long bizId, Long hostId) { List conditions = new ArrayList<>(); - if (appId != null) { - conditions.add(defaultTable.APP_ID.eq(ULong.valueOf(appId))); + if (bizId != null) { + conditions.add(defaultTable.APP_ID.eq(ULong.valueOf(bizId))); } if (hostId != null) { conditions.add(defaultTable.HOST_ID.eq(ULong.valueOf(hostId))); @@ -135,12 +145,13 @@ public void deleteHostTopoByHostId(Long appId, Long hostId) { } @Override - public void deleteHostTopo(Long hostId, Long appId, Long setId, Long moduleId) { - defaultContext.deleteFrom(defaultTable) + public int deleteHostTopoBeforeOrEqualLastTime(Long hostId, Long bizId, Long setId, Long moduleId, Long lastTime) { + return defaultContext.deleteFrom(defaultTable) .where(defaultTable.HOST_ID.eq(ULong.valueOf(hostId))) - .and(defaultTable.APP_ID.eq(ULong.valueOf(appId))) + .and(defaultTable.APP_ID.eq(ULong.valueOf(bizId))) .and(defaultTable.SET_ID.eq(setId)) .and(defaultTable.MODULE_ID.eq(moduleId)) + .and(defaultTable.LAST_TIME.lessOrEqual(lastTime)) .execute(); } @@ -181,11 +192,83 @@ public int batchDeleteHostTopo(Long bizId, List hostIdList) { return affectedNum; } + private List buildHostTopoMainFieldCondition(HostTopoDTO hostTopo) { + List conditions = new ArrayList<>(); + conditions.add(defaultTable.HOST_ID.eq(JooqDataTypeUtil.buildULong(hostTopo.getHostId()))); + conditions.add(defaultTable.APP_ID.eq(JooqDataTypeUtil.buildULong(hostTopo.getBizId()))); + conditions.add(defaultTable.SET_ID.eq(hostTopo.getSetId())); + conditions.add(defaultTable.MODULE_ID.eq(hostTopo.getModuleId())); + return conditions; + } + + @Override + public int batchUpdateBeforeLastTime(List hostTopoList) { + if (CollectionUtils.isEmpty(hostTopoList)) { + return 0; + } + int batchSize = 1000; + List queryList = new ArrayList<>(); + int affectedNum = 0; + List> subListList = CollectionUtil.partitionList(hostTopoList, batchSize); + for (List subList : subListList) { + for (HostTopoDTO hostTopo : subList) { + List conditions = buildHostTopoMainFieldCondition(hostTopo); + conditions.add(defaultTable.LAST_TIME.lessThan(hostTopo.getLastTime())); + UpdateConditionStep step = defaultContext.update(defaultTable) + .set(defaultTable.LAST_TIME, hostTopo.getLastTime()) + .where(conditions); + queryList.add(step); + } + int[] results = defaultContext.batch(queryList).execute(); + queryList.clear(); + for (int result : results) { + affectedNum += result; + } + } + return affectedNum; + } + + @Override + public int updateBeforeLastTime(HostTopoDTO hostTopo) { + List conditions = buildHostTopoMainFieldCondition(hostTopo); + conditions.add(defaultTable.LAST_TIME.lessThan(hostTopo.getLastTime())); + return defaultContext.update(defaultTable) + .set(defaultTable.LAST_TIME, hostTopo.getLastTime()) + .where(conditions) + .execute(); + } + @Override public int batchDeleteHostTopo(List hostIdList) { return batchDeleteHostTopo(null, hostIdList); } + @Override + public int batchDeleteWithLastTime(List hostTopoList) { + if (CollectionUtils.isEmpty(hostTopoList)) { + return 0; + } + int batchSize = 1000; + List queryList = new ArrayList<>(); + int affectedNum = 0; + List> subListList = CollectionUtil.partitionList(hostTopoList, batchSize); + for (List subList : subListList) { + for (HostTopoDTO hostTopo : subList) { + List conditions = buildHostTopoMainFieldCondition(hostTopo); + conditions.add(defaultTable.LAST_TIME.eq(hostTopo.getLastTime())); + DeleteConditionStep step = defaultContext.deleteFrom(defaultTable) + .where(conditions); + queryList.add(step); + } + int[] results = defaultContext.batch(queryList).execute(); + queryList.clear(); + for (int result : results) { + affectedNum += result; + } + } + return affectedNum; + } + private List listHostTopoByConditions(Collection conditions) { return listHostTopoByConditions(conditions, null, null); } @@ -224,24 +307,28 @@ private int countHostTopoByConditions(Collection conditions) { } @Override - public int countHostTopo(Long bizId, Long hostId) { + public List listHostTopoByHostId(Long hostId) { List conditions = new ArrayList<>(); - if (hostId != null) { - conditions.add(defaultTable.HOST_ID.eq(ULong.valueOf(hostId))); - } - if (bizId != null) { - conditions.add(defaultTable.APP_ID.eq(ULong.valueOf(bizId))); - } - return countHostTopoByConditions(conditions); + conditions.add(defaultTable.HOST_ID.eq(ULong.valueOf(hostId))); + return listHostTopoByConditions(conditions); } @Override - public List listHostTopoByHostId(Long hostId) { + public List listHostTopoByHostIds(Collection hostIds) { List conditions = new ArrayList<>(); - conditions.add(defaultTable.HOST_ID.eq(ULong.valueOf(hostId))); + conditions.add(defaultTable.HOST_ID.in( + hostIds.stream() + .map(JooqDataTypeUtil::buildULong) + .collect(Collectors.toList()) + )); return listHostTopoByConditions(conditions); } + @Override + public List listHostTopoByModuleIds(Collection moduleIds) { + return listHostTopoByModuleIds(moduleIds, null, null); + } + @Override public List listHostTopoByModuleIds(Collection moduleIds, Long start, Long limit) { @@ -251,23 +338,72 @@ public List listHostTopoByModuleIds(Collection moduleIds, Lon } @Override - public List listHostIdByBizIds(Collection bizIds) { + public List listHostTopoByExcludeHostIds(Collection excludeHostIds) { List conditions = new ArrayList<>(); - if (bizIds != null) { - conditions.add(defaultTable.APP_ID.in(bizIds.stream().map(ULong::valueOf).collect(Collectors.toList()))); - } + conditions.add(defaultTable.HOST_ID.notIn(excludeHostIds)); + return listHostTopoByConditions(conditions, null, null); + } + + private List listHostIdByConditions(Collection conditions) { val query = defaultContext.select( defaultTable.HOST_ID ).from(defaultTable).where(conditions); return query.fetch().map(record -> record.get(defaultTable.HOST_ID, Long.class)); } + @Override + public List listHostIdByBizAndHostIds(Collection bizIds, Collection hostIds) { + List conditions = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(bizIds)) { + conditions.add(defaultTable.APP_ID.in(bizIds.stream() + .filter(Objects::nonNull) + .map(ULong::valueOf) + .collect(Collectors.toList())) + ); + } + if (CollectionUtils.isNotEmpty(hostIds)) { + conditions.add(defaultTable.HOST_ID.in(hostIds.stream() + .filter(Objects::nonNull) + .map(ULong::valueOf) + .collect(Collectors.toList())) + ); + } + return listHostIdByConditions(conditions); + } + + @Override + public List listModuleIdByHostId(Long hostId) { + val query = defaultContext.select( + defaultTable.MODULE_ID + ).from(defaultTable) + .where(defaultTable.HOST_ID.eq(JooqDataTypeUtil.buildULong(hostId))); + return query.fetch().map(record -> record.get(defaultTable.MODULE_ID, Long.class)); + } + + @Override + public List> listHostIdAndModuleIdByBizId(Long bizId) { + List conditions = new ArrayList<>(); + if (bizId != null) { + conditions.add(defaultTable.APP_ID.eq(JooqDataTypeUtil.buildULong(bizId))); + } + val query = defaultContext.select( + defaultTable.HOST_ID, + defaultTable.MODULE_ID + ).from(defaultTable) + .where(conditions); + return query.fetch().map(record -> Pair.of( + record.get(defaultTable.HOST_ID, Long.class), + record.get(defaultTable.MODULE_ID, Long.class) + )); + } + private HostTopoDTO convertRecordToDto(HostTopoRecord record) { return new HostTopoDTO( record.getHostId().longValue(), record.getAppId().longValue(), record.getSetId(), - record.getModuleId() + record.getModuleId(), + record.getLastTime() ); } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/ResourceTagDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/ResourceTagDAOImpl.java index a948f918bd..4ff5214c28 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/ResourceTagDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/ResourceTagDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,12 +27,12 @@ import com.tencent.bk.job.common.exception.DAOException; import com.tencent.bk.job.manage.dao.ResourceTagDAO; import com.tencent.bk.job.manage.model.dto.ResourceTagDTO; +import com.tencent.bk.job.manage.model.tables.ResourceTag; import lombok.extern.slf4j.Slf4j; import org.jooq.DSLContext; import org.jooq.Record; import org.jooq.Result; import org.jooq.TableField; -import org.jooq.generated.tables.ResourceTag; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/ScriptCitedTaskTemplateDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/ScriptCitedTaskTemplateDAOImpl.java index 38a7839efc..429b9ce04c 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/ScriptCitedTaskTemplateDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/ScriptCitedTaskTemplateDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,10 +24,13 @@ package com.tencent.bk.job.manage.dao.impl; -import com.tencent.bk.job.manage.common.consts.JobResourceStatusEnum; -import com.tencent.bk.job.manage.common.util.JooqDataTypeUtil; +import com.tencent.bk.job.common.mysql.util.JooqDataTypeUtil; +import com.tencent.bk.job.manage.api.common.constants.JobResourceStatusEnum; import com.tencent.bk.job.manage.dao.ScriptCitedTaskTemplateDAO; import com.tencent.bk.job.manage.model.dto.script.ScriptCitedTaskTemplateDTO; +import com.tencent.bk.job.manage.model.tables.ScriptVersion; +import com.tencent.bk.job.manage.model.tables.TaskTemplate; +import com.tencent.bk.job.manage.model.tables.TaskTemplateStepScript; import lombok.extern.slf4j.Slf4j; import lombok.val; import org.jooq.DSLContext; @@ -35,9 +38,6 @@ import org.jooq.Record1; import org.jooq.Result; import org.jooq.conf.ParamType; -import org.jooq.generated.tables.ScriptVersion; -import org.jooq.generated.tables.TaskTemplate; -import org.jooq.generated.tables.TaskTemplateStepScript; import org.jooq.impl.DSL; import org.jooq.types.UByte; import org.springframework.beans.factory.annotation.Autowired; @@ -53,18 +53,19 @@ @Repository public class ScriptCitedTaskTemplateDAOImpl implements ScriptCitedTaskTemplateDAO { - private DSLContext ctx; + private final DSLContext ctx; - private TaskTemplateStepScript T_TASK_TEMPLATE_STEP_SCRIPT = TaskTemplateStepScript.TASK_TEMPLATE_STEP_SCRIPT; - private ScriptVersion T_SCRIPT_VERSION = ScriptVersion.SCRIPT_VERSION; - private TaskTemplate T_TASK_TEMPLATE = TaskTemplate.TASK_TEMPLATE; + private final TaskTemplateStepScript T_TASK_TEMPLATE_STEP_SCRIPT = TaskTemplateStepScript.TASK_TEMPLATE_STEP_SCRIPT; + private final ScriptVersion T_SCRIPT_VERSION = ScriptVersion.SCRIPT_VERSION; + private final TaskTemplate T_TASK_TEMPLATE = TaskTemplate.TASK_TEMPLATE; @Autowired public ScriptCitedTaskTemplateDAOImpl(@Qualifier("job-manage-dsl-context") DSLContext ctx) { this.ctx = ctx; } - private List getDistinctRelatedTemplates(List srcScriptCitedTaskTemplates) { + private List getDistinctRelatedTemplates( + List srcScriptCitedTaskTemplates) { List resultRelatedTemplates = new ArrayList<>(); Set keySet = new HashSet<>(); for (ScriptCitedTaskTemplateDTO citedTaskTemplateDTO : srcScriptCitedTaskTemplates) { diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/ScriptDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/ScriptDAOImpl.java index 8355482a87..b311c1a942 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/ScriptDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/ScriptDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -31,13 +31,15 @@ import com.tencent.bk.job.common.util.StringUtil; import com.tencent.bk.job.common.util.TagUtils; import com.tencent.bk.job.common.util.date.DateUtils; -import com.tencent.bk.job.manage.common.consts.JobResourceStatusEnum; -import com.tencent.bk.job.manage.common.consts.script.ScriptScopeEnum; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.JobResourceStatusEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptScopeEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; import com.tencent.bk.job.manage.dao.ScriptDAO; import com.tencent.bk.job.manage.model.dto.ScriptBasicDTO; import com.tencent.bk.job.manage.model.dto.ScriptDTO; import com.tencent.bk.job.manage.model.query.ScriptQuery; +import com.tencent.bk.job.manage.model.tables.Script; +import com.tencent.bk.job.manage.model.tables.ScriptVersion; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import org.jooq.Condition; @@ -46,8 +48,6 @@ import org.jooq.Record6; import org.jooq.Result; import org.jooq.SortField; -import org.jooq.generated.tables.Script; -import org.jooq.generated.tables.ScriptVersion; import org.jooq.impl.DSL; import org.jooq.types.UByte; import org.jooq.types.ULong; @@ -71,11 +71,11 @@ public class ScriptDAOImpl implements ScriptDAO { private static final Script TB_SCRIPT = Script.SCRIPT; private static final ScriptVersion TB_SCRIPT_VERSION = ScriptVersion.SCRIPT_VERSION; - private final DSLContext create; + private final DSLContext dslContext; @Autowired - public ScriptDAOImpl(@Qualifier("job-manage-dsl-context") DSLContext create) { - this.create = create; + public ScriptDAOImpl(@Qualifier("job-manage-dsl-context") DSLContext dslContext) { + this.dslContext = dslContext; } @Override @@ -97,9 +97,9 @@ private PageData listPageScriptWithBasicQueryCondition(ScriptQuery sc int start = baseSearchCondition.getStartOrDefault(0); int length = baseSearchCondition.getLengthOrDefault(10); Result result = - create.select(TB_SCRIPT.ID, TB_SCRIPT.APP_ID, TB_SCRIPT.NAME, TB_SCRIPT.CATEGORY, - TB_SCRIPT.TYPE, TB_SCRIPT.CREATOR, TB_SCRIPT.CREATE_TIME, TB_SCRIPT.LAST_MODIFY_USER, - TB_SCRIPT.LAST_MODIFY_TIME, TB_SCRIPT.IS_PUBLIC, TB_SCRIPT.DESCRIPTION) + dslContext.select(TB_SCRIPT.ID, TB_SCRIPT.APP_ID, TB_SCRIPT.NAME, TB_SCRIPT.CATEGORY, + TB_SCRIPT.TYPE, TB_SCRIPT.CREATOR, TB_SCRIPT.CREATE_TIME, TB_SCRIPT.LAST_MODIFY_USER, + TB_SCRIPT.LAST_MODIFY_TIME, TB_SCRIPT.IS_PUBLIC, TB_SCRIPT.DESCRIPTION) .from(TB_SCRIPT) .where(buildScriptConditionList(scriptQuery, baseSearchCondition)) .orderBy(orderFields) @@ -110,7 +110,7 @@ private PageData listPageScriptWithBasicQueryCondition(ScriptQuery sc private long getPageScriptCountWithBasicQueryCondition(ScriptQuery scriptQuery, BaseSearchCondition baseSearchCondition) { List conditions = buildScriptConditionList(scriptQuery, baseSearchCondition); - Long count = create.selectCount().from(TB_SCRIPT).where(conditions).fetchOne(0, Long.class); + Long count = dslContext.selectCount().from(TB_SCRIPT).where(conditions).fetchOne(0, Long.class); return count == null ? 0 : count; } @@ -123,9 +123,9 @@ private PageData listPageScriptWithScriptVersionQueryCondition(Script int length = baseSearchCondition.getLengthOrDefault(10); Result result = - create.selectDistinct(TB_SCRIPT.ID, TB_SCRIPT.APP_ID, TB_SCRIPT.NAME, TB_SCRIPT.CATEGORY, - TB_SCRIPT.TYPE, TB_SCRIPT.CREATOR, TB_SCRIPT.CREATE_TIME, TB_SCRIPT.LAST_MODIFY_USER, - TB_SCRIPT.LAST_MODIFY_TIME, TB_SCRIPT.IS_PUBLIC, TB_SCRIPT.DESCRIPTION) + dslContext.selectDistinct(TB_SCRIPT.ID, TB_SCRIPT.APP_ID, TB_SCRIPT.NAME, TB_SCRIPT.CATEGORY, + TB_SCRIPT.TYPE, TB_SCRIPT.CREATOR, TB_SCRIPT.CREATE_TIME, TB_SCRIPT.LAST_MODIFY_USER, + TB_SCRIPT.LAST_MODIFY_TIME, TB_SCRIPT.IS_PUBLIC, TB_SCRIPT.DESCRIPTION) .from(TB_SCRIPT) .leftJoin(TB_SCRIPT_VERSION) .on(TB_SCRIPT.ID.eq(TB_SCRIPT_VERSION.SCRIPT_ID)) @@ -137,8 +137,8 @@ private PageData listPageScriptWithScriptVersionQueryCondition(Script private int getPageScriptCountWithScriptVersionQueryCondition(ScriptQuery scriptQuery, BaseSearchCondition baseSearchCondition) { - return create.fetchCount( - create.selectDistinct(TB_SCRIPT.ID) + return dslContext.fetchCount( + dslContext.selectDistinct(TB_SCRIPT.ID) .from(TB_SCRIPT) .leftJoin(TB_SCRIPT_VERSION) .on(TB_SCRIPT.ID.eq(TB_SCRIPT_VERSION.SCRIPT_ID)) @@ -204,9 +204,9 @@ private PageData pageResult(Result result, long cou @Override public List listScripts(ScriptQuery scriptQuery) { Result result = - create.select(TB_SCRIPT.ID, TB_SCRIPT.APP_ID, TB_SCRIPT.NAME, TB_SCRIPT.CATEGORY, - TB_SCRIPT.TYPE, TB_SCRIPT.CREATOR, TB_SCRIPT.CREATE_TIME, TB_SCRIPT.LAST_MODIFY_USER, - TB_SCRIPT.LAST_MODIFY_TIME, TB_SCRIPT.IS_PUBLIC, TB_SCRIPT.DESCRIPTION) + dslContext.select(TB_SCRIPT.ID, TB_SCRIPT.APP_ID, TB_SCRIPT.NAME, TB_SCRIPT.CATEGORY, + TB_SCRIPT.TYPE, TB_SCRIPT.CREATOR, TB_SCRIPT.CREATE_TIME, TB_SCRIPT.LAST_MODIFY_USER, + TB_SCRIPT.LAST_MODIFY_TIME, TB_SCRIPT.IS_PUBLIC, TB_SCRIPT.DESCRIPTION) .from(TB_SCRIPT) .where(buildScriptConditionList(scriptQuery, null)) .orderBy(TB_SCRIPT.NAME.asc()) @@ -220,23 +220,23 @@ public List listScripts(ScriptQuery scriptQuery) { @Override public ScriptDTO getScriptByScriptId(String scriptId) { - Record record = create.select(TB_SCRIPT.ID, TB_SCRIPT.APP_ID, TB_SCRIPT.NAME, TB_SCRIPT.CATEGORY, - TB_SCRIPT.TYPE, TB_SCRIPT.IS_PUBLIC, TB_SCRIPT.CREATOR, TB_SCRIPT.CREATE_TIME, - TB_SCRIPT.LAST_MODIFY_USER, TB_SCRIPT.LAST_MODIFY_TIME, TB_SCRIPT.DESCRIPTION).from(Script.SCRIPT) + Record record = dslContext.select(TB_SCRIPT.ID, TB_SCRIPT.APP_ID, TB_SCRIPT.NAME, TB_SCRIPT.CATEGORY, + TB_SCRIPT.TYPE, TB_SCRIPT.IS_PUBLIC, TB_SCRIPT.CREATOR, TB_SCRIPT.CREATE_TIME, + TB_SCRIPT.LAST_MODIFY_USER, TB_SCRIPT.LAST_MODIFY_TIME, TB_SCRIPT.DESCRIPTION).from(Script.SCRIPT) .where(TB_SCRIPT.ID.eq(scriptId)).fetchOne(); return extractScriptData(record); } @Override public List listScriptBasicInfoByScriptIds(Collection scriptIds) { - Result> records = create.select( - TB_SCRIPT.ID, - TB_SCRIPT.APP_ID, - TB_SCRIPT.NAME, - TB_SCRIPT.CATEGORY, - TB_SCRIPT.TYPE, - TB_SCRIPT.IS_PUBLIC - ).from(Script.SCRIPT) + Result> records = dslContext.select( + TB_SCRIPT.ID, + TB_SCRIPT.APP_ID, + TB_SCRIPT.NAME, + TB_SCRIPT.CATEGORY, + TB_SCRIPT.TYPE, + TB_SCRIPT.IS_PUBLIC + ).from(Script.SCRIPT) .where(TB_SCRIPT.ID.in(scriptIds)).fetch(); return records.map(this::extractScriptBasicDTO); } @@ -246,11 +246,11 @@ public ScriptDTO getScriptVersionById(long id) { Script tbScript = Script.SCRIPT.as("t1"); ScriptVersion tbScriptVersion = ScriptVersion.SCRIPT_VERSION.as("t2"); - Record record = create.select(tbScript.ID, tbScript.APP_ID, tbScript.NAME, tbScript.CATEGORY, - tbScript.TYPE, tbScript.IS_PUBLIC, tbScript.DESCRIPTION, tbScriptVersion.ID.as("scriptVersionId"), - tbScriptVersion.VERSION, tbScriptVersion.CONTENT, tbScriptVersion.STATUS, tbScriptVersion.CREATE_TIME, - tbScriptVersion.CREATOR, tbScriptVersion.LAST_MODIFY_TIME, tbScriptVersion.LAST_MODIFY_USER, - tbScriptVersion.VERSION_DESC).from(tbScriptVersion).join(tbScript) + Record record = dslContext.select(tbScript.ID, tbScript.APP_ID, tbScript.NAME, tbScript.CATEGORY, + tbScript.TYPE, tbScript.IS_PUBLIC, tbScript.DESCRIPTION, tbScriptVersion.ID.as("scriptVersionId"), + tbScriptVersion.VERSION, tbScriptVersion.CONTENT, tbScriptVersion.STATUS, tbScriptVersion.CREATE_TIME, + tbScriptVersion.CREATOR, tbScriptVersion.LAST_MODIFY_TIME, tbScriptVersion.LAST_MODIFY_USER, + tbScriptVersion.VERSION_DESC).from(tbScriptVersion).join(tbScript) .on(tbScript.ID.eq(tbScriptVersion.SCRIPT_ID)) .where(tbScriptVersion.ID.eq(ULong.valueOf(id)) .and(tbScriptVersion.IS_DELETED.eq(UByte.valueOf(0)))) @@ -271,11 +271,11 @@ public ScriptDTO getByScriptIdAndVersion(Long appId, String scriptId, String ver conditions.add(tbScriptVersion.VERSION.eq(version)); } - Record record = create.select(tbScript.ID, tbScript.APP_ID, tbScript.NAME, tbScript.CATEGORY, - tbScript.TYPE, tbScript.IS_PUBLIC, tbScript.DESCRIPTION, tbScriptVersion.ID.as("scriptVersionId"), - tbScriptVersion.VERSION, tbScriptVersion.CONTENT, tbScriptVersion.STATUS, tbScriptVersion.CREATE_TIME, - tbScriptVersion.CREATOR, tbScriptVersion.LAST_MODIFY_TIME, tbScriptVersion.LAST_MODIFY_USER, - tbScriptVersion.VERSION_DESC).from(tbScriptVersion).join(tbScript) + Record record = dslContext.select(tbScript.ID, tbScript.APP_ID, tbScript.NAME, tbScript.CATEGORY, + tbScript.TYPE, tbScript.IS_PUBLIC, tbScript.DESCRIPTION, tbScriptVersion.ID.as("scriptVersionId"), + tbScriptVersion.VERSION, tbScriptVersion.CONTENT, tbScriptVersion.STATUS, tbScriptVersion.CREATE_TIME, + tbScriptVersion.CREATOR, tbScriptVersion.LAST_MODIFY_TIME, tbScriptVersion.LAST_MODIFY_USER, + tbScriptVersion.VERSION_DESC).from(tbScriptVersion).join(tbScript) .on(tbScript.ID.eq(tbScriptVersion.SCRIPT_ID)) .where(conditions) .fetchOne(); @@ -288,11 +288,12 @@ public List batchGetScriptVersionsByIds(Collection scriptVersio ScriptVersion tbScriptVersion = ScriptVersion.SCRIPT_VERSION.as("t2"); List scriptVersionIdList = scriptVersionIds.stream().map(ULong::valueOf).collect(Collectors.toList()); - Result result = create.select(tbScript.ID, tbScript.APP_ID, tbScript.NAME, tbScript.CATEGORY, - tbScript.TYPE, tbScript.IS_PUBLIC, tbScript.DESCRIPTION, tbScriptVersion.ID.as("scriptVersionId"), - tbScriptVersion.VERSION, tbScriptVersion.CONTENT, tbScriptVersion.STATUS, tbScriptVersion.CREATE_TIME, - tbScriptVersion.CREATOR, tbScriptVersion.LAST_MODIFY_TIME, tbScriptVersion.LAST_MODIFY_USER, - tbScriptVersion.VERSION_DESC).from(tbScriptVersion).join(tbScript) + Result result = dslContext.select(tbScript.ID, tbScript.APP_ID, tbScript.NAME, + tbScript.CATEGORY, + tbScript.TYPE, tbScript.IS_PUBLIC, tbScript.DESCRIPTION, tbScriptVersion.ID.as("scriptVersionId"), + tbScriptVersion.VERSION, tbScriptVersion.CONTENT, tbScriptVersion.STATUS, tbScriptVersion.CREATE_TIME, + tbScriptVersion.CREATOR, tbScriptVersion.LAST_MODIFY_TIME, tbScriptVersion.LAST_MODIFY_USER, + tbScriptVersion.VERSION_DESC).from(tbScriptVersion).join(tbScript) .on(tbScript.ID.eq(tbScriptVersion.SCRIPT_ID)).where(tbScriptVersion.ID.in(scriptVersionIdList).and(tbScriptVersion.IS_DELETED.eq(UByte.valueOf(0)))) .fetch(); @@ -310,7 +311,6 @@ private List buildScriptConditionList(ScriptQuery scriptQuery, if (scriptQuery != null) { if (StringUtils.isNotBlank(scriptQuery.getId())) { conditions.add(TB_SCRIPT.ID.eq(scriptQuery.getId())); - return conditions; } else if (CollectionUtils.isNotEmpty(scriptQuery.getIds())) { if (scriptQuery.getIds().size() == 1) { conditions.add(TB_SCRIPT.ID.eq(scriptQuery.getIds().get(0))); @@ -367,51 +367,47 @@ private List buildScriptVersionConditionList(ScriptQuery scriptQuery, @Override public String saveScript(ScriptDTO script) { - return saveScript(create, script, DateUtils.currentTimeMillis(), DateUtils.currentTimeMillis()); - } - - @Override - public String saveScript(ScriptDTO script, long createTime, long lastModifyTime) { - return saveScript(create, script, createTime, lastModifyTime); - } - - @Override - public String saveScript(DSLContext dslContext, ScriptDTO script, long createTime, long lastModifyTime) { if (StringUtils.isBlank(script.getId())) { script.setId(JobUUID.getUUID()); } int isPublicScript = script.isPublicScript() ? ScriptScopeEnum.PUBLIC.getValue() : ScriptScopeEnum.APP.getValue(); - dslContext.insertInto(TB_SCRIPT, TB_SCRIPT.ID, TB_SCRIPT.NAME, TB_SCRIPT.APP_ID, TB_SCRIPT.CATEGORY, - TB_SCRIPT.TYPE, TB_SCRIPT.DESCRIPTION, - TB_SCRIPT.IS_PUBLIC, TB_SCRIPT.CREATOR, TB_SCRIPT.LAST_MODIFY_USER, - TB_SCRIPT.LAST_MODIFY_TIME, TB_SCRIPT.CREATE_TIME) - .values(script.getId(), script.getName(), ULong.valueOf(script.getAppId()), + dslContext.insertInto(TB_SCRIPT, + TB_SCRIPT.ID, + TB_SCRIPT.NAME, + TB_SCRIPT.APP_ID, + TB_SCRIPT.CATEGORY, + TB_SCRIPT.TYPE, + TB_SCRIPT.DESCRIPTION, + TB_SCRIPT.IS_PUBLIC, + TB_SCRIPT.CREATOR, + TB_SCRIPT.LAST_MODIFY_USER, + TB_SCRIPT.LAST_MODIFY_TIME, + TB_SCRIPT.CREATE_TIME) + .values(script.getId(), + script.getName(), + ULong.valueOf(script.getAppId()), UByte.valueOf(script.getCategory()), - UByte.valueOf(script.getType()), script.getDescription(), UByte.valueOf(isPublicScript), + UByte.valueOf(script.getType()), + script.getDescription(), + UByte.valueOf(isPublicScript), script.getCreator(), - script.getCreator(), ULong.valueOf(createTime), ULong.valueOf(lastModifyTime)).execute(); + script.getCreator(), + ULong.valueOf(script.getCreateTime() == null ? System.currentTimeMillis() : script.getCreateTime()), + ULong.valueOf(script.getLastModifyTime() == null ? System.currentTimeMillis() : + script.getLastModifyTime())) + .execute(); return script.getId(); } @Override - public void updateScript(ScriptDTO script) { - updateScript(script, DateUtils.currentTimeMillis()); - } - - @Override - public void updateScript(ScriptDTO script, long lastModifyTime) { - updateScript(create, script, lastModifyTime); - } - - @Override - public void updateScript(DSLContext dslContext, ScriptDTO script, long lastModifyTime) { + public void updateScriptLastModify(String scriptId, String lastModifyUser, Long lastModifyTime) { dslContext.update(TB_SCRIPT) - .set(TB_SCRIPT.NAME, script.getName()) - .set(TB_SCRIPT.LAST_MODIFY_USER, script.getLastModifyUser()) - .set(TB_SCRIPT.LAST_MODIFY_TIME, ULong.valueOf(lastModifyTime)) - .where(TB_SCRIPT.ID.eq(script.getId())).execute(); + .set(TB_SCRIPT.LAST_MODIFY_USER, lastModifyUser) + .set(TB_SCRIPT.LAST_MODIFY_TIME, ULong.valueOf(lastModifyTime == null ? + System.currentTimeMillis() : lastModifyTime)) + .where(TB_SCRIPT.ID.eq(scriptId)).execute(); } @Override @@ -420,34 +416,41 @@ public void deleteScript(String scriptId) { } private void deleteScriptSoftly(String scriptId) { - create.update(TB_SCRIPT).set(TB_SCRIPT.IS_DELETED, UByte.valueOf(1)) + dslContext.update(TB_SCRIPT).set(TB_SCRIPT.IS_DELETED, UByte.valueOf(1)) .where(TB_SCRIPT.ID.eq(scriptId)).execute(); } @Override - public Long saveScriptVersion(ScriptDTO scriptVersion) { - return saveScriptVersion(create, scriptVersion, DateUtils.currentTimeMillis(), DateUtils.currentTimeMillis()); - } - - @Override - public Long saveScriptVersion(ScriptDTO scriptVersion, long createTime, long lastModifyTime) { - return saveScriptVersion(create, scriptVersion, createTime, lastModifyTime); - } - - @Override - public Long saveScriptVersion(DSLContext dslContext, ScriptDTO scriptDTO, long createTime, long lastModifyTime) { + public Long saveScriptVersion(ScriptDTO scriptDTO) { Long scriptVersionId = scriptDTO.getScriptVersionId(); if (scriptVersionId == null || scriptVersionId < 0) { scriptVersionId = null; } - Record record = dslContext.insertInto(TB_SCRIPT_VERSION, TB_SCRIPT_VERSION.ID, TB_SCRIPT_VERSION.SCRIPT_ID, - TB_SCRIPT_VERSION.CONTENT, TB_SCRIPT_VERSION.VERSION_DESC, TB_SCRIPT_VERSION.VERSION, - TB_SCRIPT_VERSION.CREATOR, TB_SCRIPT_VERSION.LAST_MODIFY_USER, TB_SCRIPT_VERSION.STATUS - , TB_SCRIPT_VERSION.CREATE_TIME, TB_SCRIPT_VERSION.LAST_MODIFY_TIME) - .values(scriptVersionId == null ? null : ULong.valueOf(scriptVersionId), scriptDTO.getId(), - scriptDTO.getContent(), scriptDTO.getVersionDesc(), scriptDTO.getVersion(), scriptDTO.getCreator(), - scriptDTO.getCreator(), UByte.valueOf(scriptDTO.getStatus()), ULong.valueOf(createTime), - ULong.valueOf(lastModifyTime)) + Record record = dslContext.insertInto( + TB_SCRIPT_VERSION, + TB_SCRIPT_VERSION.ID, + TB_SCRIPT_VERSION.SCRIPT_ID, + TB_SCRIPT_VERSION.CONTENT, + TB_SCRIPT_VERSION.VERSION_DESC, + TB_SCRIPT_VERSION.VERSION, + TB_SCRIPT_VERSION.CREATOR, + TB_SCRIPT_VERSION.LAST_MODIFY_USER, + TB_SCRIPT_VERSION.STATUS, + TB_SCRIPT_VERSION.CREATE_TIME, + TB_SCRIPT_VERSION.LAST_MODIFY_TIME) + .values( + scriptVersionId == null ? null : ULong.valueOf(scriptVersionId), + scriptDTO.getId(), + scriptDTO.getContent(), + scriptDTO.getVersionDesc(), + scriptDTO.getVersion(), + scriptDTO.getCreator(), + scriptDTO.getCreator(), + UByte.valueOf(scriptDTO.getStatus()), + ULong.valueOf(scriptDTO.getCreateTime() == null ? System.currentTimeMillis() : + scriptDTO.getCreateTime()), + ULong.valueOf(scriptDTO.getLastModifyTime() == null ? System.currentTimeMillis() : + scriptDTO.getLastModifyTime())) .returning(TB_SCRIPT_VERSION.ID).fetchOne(); return record != null ? record.getValue(TB_SCRIPT_VERSION.ID).longValue() : 0; } @@ -457,11 +460,12 @@ public List listScriptVersionsByScriptId(String scriptId) { Script tbScript = Script.SCRIPT.as("t1"); ScriptVersion tbScriptVersion = ScriptVersion.SCRIPT_VERSION.as("t2"); - Result result = create.select(tbScript.ID, tbScript.APP_ID, tbScript.NAME, tbScript.CATEGORY, - tbScript.TYPE, tbScript.IS_PUBLIC, tbScript.DESCRIPTION, tbScriptVersion.ID.as("scriptVersionId"), - tbScriptVersion.VERSION, tbScriptVersion.CONTENT, tbScriptVersion.VERSION_DESC, tbScriptVersion.STATUS, - tbScriptVersion.CREATE_TIME, - tbScriptVersion.CREATOR, tbScriptVersion.LAST_MODIFY_TIME, tbScriptVersion.LAST_MODIFY_USER).from(tbScriptVersion).join(tbScript) + Result result = dslContext.select(tbScript.ID, tbScript.APP_ID, tbScript.NAME, + tbScript.CATEGORY, + tbScript.TYPE, tbScript.IS_PUBLIC, tbScript.DESCRIPTION, tbScriptVersion.ID.as("scriptVersionId"), + tbScriptVersion.VERSION, tbScriptVersion.CONTENT, tbScriptVersion.VERSION_DESC, tbScriptVersion.STATUS, + tbScriptVersion.CREATE_TIME, + tbScriptVersion.CREATOR, tbScriptVersion.LAST_MODIFY_TIME, tbScriptVersion.LAST_MODIFY_USER).from(tbScriptVersion).join(tbScript) .on(tbScript.ID.eq(tbScriptVersion.SCRIPT_ID)).where(tbScript.ID.eq(scriptId).and(tbScriptVersion.IS_DELETED.eq(UByte.valueOf(0)))) .orderBy(tbScriptVersion.LAST_MODIFY_TIME.desc()) .fetch(); @@ -470,14 +474,14 @@ public List listScriptVersionsByScriptId(String scriptId) { @Override public boolean isExistDuplicateScriptId(Long appId, String scriptId) { - int count = create.selectCount().from(TB_SCRIPT).where(TB_SCRIPT.APP_ID.eq(ULong.valueOf(appId)) + int count = dslContext.selectCount().from(TB_SCRIPT).where(TB_SCRIPT.APP_ID.eq(ULong.valueOf(appId)) .and(TB_SCRIPT.ID.eq(scriptId))).fetchOne(0, Integer.class); return (count >= 1); } @Override public boolean isExistDuplicateName(Long appId, String scriptName) { - int count = create.selectCount().from(TB_SCRIPT).where(TB_SCRIPT.APP_ID.eq(ULong.valueOf(appId))) + int count = dslContext.selectCount().from(TB_SCRIPT).where(TB_SCRIPT.APP_ID.eq(ULong.valueOf(appId))) .and(TB_SCRIPT.NAME.eq(scriptName)) .and(TB_SCRIPT.IS_DELETED.eq(UByte.valueOf(0))) .fetchOne(0, Integer.class); @@ -486,13 +490,13 @@ public boolean isExistDuplicateName(Long appId, String scriptName) { @Override public void updateScriptVersionDesc(Long scriptVersionId, String desc) { - create.update(TB_SCRIPT_VERSION).set(TB_SCRIPT_VERSION.VERSION_DESC, desc) + dslContext.update(TB_SCRIPT_VERSION).set(TB_SCRIPT_VERSION.VERSION_DESC, desc) .where(TB_SCRIPT_VERSION.ID.eq(ULong.valueOf(scriptVersionId))).execute(); } @Override public void deleteScriptVersion(Long scriptVersionId) { - create.update(TB_SCRIPT_VERSION).set(TB_SCRIPT_VERSION.IS_DELETED, UByte.valueOf(1)) + dslContext.update(TB_SCRIPT_VERSION).set(TB_SCRIPT_VERSION.IS_DELETED, UByte.valueOf(1)) .where(TB_SCRIPT_VERSION.ID.eq(ULong.valueOf(scriptVersionId))).execute(); } @@ -502,7 +506,7 @@ public void deleteScriptVersionByScriptId(String scriptId) { } private void deleteScriptVersionByScriptIdSoftly(String scriptId) { - create.update(TB_SCRIPT_VERSION).set(TB_SCRIPT_VERSION.IS_DELETED, UByte.valueOf(1)) + dslContext.update(TB_SCRIPT_VERSION).set(TB_SCRIPT_VERSION.IS_DELETED, UByte.valueOf(1)) .where(TB_SCRIPT_VERSION.SCRIPT_ID.eq(scriptId)).execute(); } @@ -582,7 +586,7 @@ private ScriptDTO extractScriptVersionData(Record record) { @Override public void updateScriptVersionStatus(Long scriptVersionId, Integer status) { - create.update(TB_SCRIPT_VERSION).set(TB_SCRIPT_VERSION.STATUS, UByte.valueOf(status)) + dslContext.update(TB_SCRIPT_VERSION).set(TB_SCRIPT_VERSION.STATUS, UByte.valueOf(status)) .where(TB_SCRIPT_VERSION.ID.eq(ULong.valueOf(scriptVersionId))).execute(); } @@ -591,15 +595,30 @@ public Map batchGetOnlineByScriptIds(List scriptIds) Script tbScript = Script.SCRIPT.as("t1"); ScriptVersion tbScriptVersion = ScriptVersion.SCRIPT_VERSION.as("t2"); - Result result = create.select(tbScript.ID, tbScript.APP_ID, tbScript.NAME, tbScript.CATEGORY, - tbScript.TYPE, tbScript.IS_PUBLIC, tbScript.DESCRIPTION, tbScriptVersion.ID.as("scriptVersionId"), - tbScriptVersion.VERSION, tbScriptVersion.CONTENT, tbScriptVersion.VERSION_DESC, tbScriptVersion.STATUS, - tbScriptVersion.CREATE_TIME, - tbScriptVersion.CREATOR, tbScriptVersion.LAST_MODIFY_TIME, tbScriptVersion.LAST_MODIFY_USER).from(tbScriptVersion).join(tbScript) - .on(tbScript.ID.eq(tbScriptVersion.SCRIPT_ID)).where(tbScript.ID.in(scriptIds) - .and(tbScriptVersion.STATUS.eq(UByte.valueOf(JobResourceStatusEnum.ONLINE.getValue()))) - .and(tbScriptVersion.IS_DELETED.eq(UByte.valueOf(0)))) - .fetch(); + Result result = + dslContext.select( + tbScript.ID, + tbScript.APP_ID, + tbScript.NAME, + tbScript.CATEGORY, + tbScript.TYPE, + tbScript.IS_PUBLIC, + tbScript.DESCRIPTION, + tbScriptVersion.ID.as("scriptVersionId"), + tbScriptVersion.VERSION, + tbScriptVersion.CONTENT, + tbScriptVersion.VERSION_DESC, + tbScriptVersion.STATUS, + tbScriptVersion.CREATE_TIME, + tbScriptVersion.CREATOR, + tbScriptVersion.LAST_MODIFY_TIME, + tbScriptVersion.LAST_MODIFY_USER) + .from(tbScriptVersion) + .join(tbScript) + .on(tbScript.ID.eq(tbScriptVersion.SCRIPT_ID)).where(tbScript.ID.in(scriptIds) + .and(tbScriptVersion.STATUS.eq(UByte.valueOf(JobResourceStatusEnum.ONLINE.getValue()))) + .and(tbScriptVersion.IS_DELETED.eq(UByte.valueOf(0)))) + .fetch(); List scriptVersions = new ArrayList<>(); if (result.size() > 0) { scriptVersions = result.map(this::extractScriptVersionData); @@ -614,7 +633,7 @@ public Map batchGetOnlineByScriptIds(List scriptIds) @Override public void updateScriptDesc(String operator, String scriptId, String desc) { - create.update(TB_SCRIPT).set(TB_SCRIPT.DESCRIPTION, desc) + dslContext.update(TB_SCRIPT).set(TB_SCRIPT.DESCRIPTION, desc) .set(TB_SCRIPT.LAST_MODIFY_TIME, ULong.valueOf(DateUtils.currentTimeMillis())) .set(TB_SCRIPT.LAST_MODIFY_USER, operator) .where(TB_SCRIPT.ID.eq(scriptId)).execute(); @@ -622,7 +641,7 @@ public void updateScriptDesc(String operator, String scriptId, String desc) { @Override public void updateScriptName(String operator, String scriptId, String name) { - create.update(TB_SCRIPT).set(Script.SCRIPT.NAME, name) + dslContext.update(TB_SCRIPT).set(Script.SCRIPT.NAME, name) .set(TB_SCRIPT.LAST_MODIFY_TIME, ULong.valueOf(DateUtils.currentTimeMillis())) .set(TB_SCRIPT.LAST_MODIFY_USER, operator) .where(TB_SCRIPT.ID.eq(scriptId)).execute(); @@ -630,7 +649,7 @@ public void updateScriptName(String operator, String scriptId, String name) { @Override public void updateScriptVersion(String operator, Long scriptVersionId, ScriptDTO scriptVersion) { - create.update(TB_SCRIPT_VERSION).set(TB_SCRIPT_VERSION.CONTENT, scriptVersion.getContent()) + dslContext.update(TB_SCRIPT_VERSION).set(TB_SCRIPT_VERSION.CONTENT, scriptVersion.getContent()) .set(TB_SCRIPT_VERSION.VERSION_DESC, scriptVersion.getVersionDesc()) .set(TB_SCRIPT_VERSION.LAST_MODIFY_USER, operator) .set(TB_SCRIPT_VERSION.LAST_MODIFY_TIME, ULong.valueOf(DateUtils.currentTimeMillis())) @@ -641,11 +660,12 @@ public void updateScriptVersion(String operator, Long scriptVersionId, ScriptDTO @Override public List listScriptNames(Long appId, String keyword) { String likePattern = "%" + keyword + "%"; - Result result = create.select(TB_SCRIPT.NAME).from(TB_SCRIPT).where(TB_SCRIPT.APP_ID.eq(ULong.valueOf(appId))) - .and(TB_SCRIPT.IS_DELETED.eq(UByte.valueOf(0))) - .and(TB_SCRIPT.NAME.like(likePattern)) - .orderBy(TB_SCRIPT.NAME.asc()) - .fetch(); + Result result = + dslContext.select(TB_SCRIPT.NAME).from(TB_SCRIPT).where(TB_SCRIPT.APP_ID.eq(ULong.valueOf(appId))) + .and(TB_SCRIPT.IS_DELETED.eq(UByte.valueOf(0))) + .and(TB_SCRIPT.NAME.like(likePattern)) + .orderBy(TB_SCRIPT.NAME.asc()) + .fetch(); List scriptNames = new ArrayList<>(); result.into(record -> { String scriptName = record.get(TB_SCRIPT.NAME); @@ -658,11 +678,11 @@ public List listOnlineScriptForApp(long appId) { Script tbScript = Script.SCRIPT.as("t1"); ScriptVersion tbScriptVersion = ScriptVersion.SCRIPT_VERSION.as("t2"); - Result result = create.select(tbScript.ID, tbScript.APP_ID, tbScript.NAME, tbScript.CATEGORY, - tbScript.TYPE, tbScript.IS_PUBLIC, tbScript.DESCRIPTION, tbScriptVersion.ID.as("scriptVersionId"), - tbScriptVersion.VERSION, tbScriptVersion.CONTENT, tbScriptVersion.VERSION_DESC, tbScriptVersion.STATUS, - tbScriptVersion.CREATE_TIME, - tbScriptVersion.CREATOR, tbScriptVersion.LAST_MODIFY_TIME, tbScriptVersion.LAST_MODIFY_USER) + Result result = dslContext.select(tbScript.ID, tbScript.APP_ID, tbScript.NAME, tbScript.CATEGORY, + tbScript.TYPE, tbScript.IS_PUBLIC, tbScript.DESCRIPTION, tbScriptVersion.ID.as("scriptVersionId"), + tbScriptVersion.VERSION, tbScriptVersion.CONTENT, tbScriptVersion.VERSION_DESC, tbScriptVersion.STATUS, + tbScriptVersion.CREATE_TIME, + tbScriptVersion.CREATOR, tbScriptVersion.LAST_MODIFY_TIME, tbScriptVersion.LAST_MODIFY_USER) .from(tbScriptVersion).join(tbScript) .on(tbScript.ID.eq(tbScriptVersion.SCRIPT_ID)).where(tbScript.APP_ID.eq(ULong.valueOf(appId)) .and(tbScriptVersion.STATUS.eq(UByte.valueOf(JobResourceStatusEnum.ONLINE.getValue()))) @@ -687,18 +707,18 @@ public PageData listPageOnlineScript(ScriptQuery scriptCondition, List conditions = buildScriptConditionList(scriptCondition, baseSearchCondition); conditions.add(ScriptVersion.SCRIPT_VERSION.STATUS.eq(UByte.valueOf(JobResourceStatusEnum.ONLINE.getValue()))); - long total = create.selectCount().from(tbScriptVersion).join(tbScript) + long total = dslContext.selectCount().from(tbScriptVersion).join(tbScript) .on(tbScript.ID.eq(tbScriptVersion.SCRIPT_ID)). - where(conditions).fetchOne(0, Long.class); + where(conditions).fetchOne(0, Long.class); int start = baseSearchCondition.getStartOrDefault(0); int length = baseSearchCondition.getLengthOrDefault(10); - Result result = create.select(tbScript.ID, tbScript.APP_ID, tbScript.NAME, tbScript.CATEGORY, - tbScript.TYPE, tbScript.IS_PUBLIC, tbScript.DESCRIPTION, tbScriptVersion.ID.as("scriptVersionId"), - tbScriptVersion.VERSION, tbScriptVersion.CONTENT, tbScriptVersion.VERSION_DESC, tbScriptVersion.STATUS, - tbScriptVersion.CREATE_TIME, - tbScriptVersion.CREATOR, tbScriptVersion.LAST_MODIFY_TIME, tbScriptVersion.LAST_MODIFY_USER).from(tbScriptVersion).join(tbScript) + Result result = dslContext.select(tbScript.ID, tbScript.APP_ID, tbScript.NAME, tbScript.CATEGORY, + tbScript.TYPE, tbScript.IS_PUBLIC, tbScript.DESCRIPTION, tbScriptVersion.ID.as("scriptVersionId"), + tbScriptVersion.VERSION, tbScriptVersion.CONTENT, tbScriptVersion.VERSION_DESC, tbScriptVersion.STATUS, + tbScriptVersion.CREATE_TIME, + tbScriptVersion.CREATOR, tbScriptVersion.LAST_MODIFY_TIME, tbScriptVersion.LAST_MODIFY_USER).from(tbScriptVersion).join(tbScript) .on(tbScript.ID.eq(tbScriptVersion.SCRIPT_ID)) .where(conditions) .orderBy(tbScriptVersion.LAST_MODIFY_TIME.desc()) @@ -719,21 +739,21 @@ public PageData listPageOnlineScript(ScriptQuery scriptCondition, } @Override - public PageData listPageScriptVersion(ScriptQuery scriptQuery, - BaseSearchCondition baseSearchCondition) { + public PageData listPageScriptVersion(ScriptQuery scriptQuery) { Script tbScript = Script.SCRIPT; ScriptVersion tbScriptVersion = ScriptVersion.SCRIPT_VERSION; - long count = getScriptVersionPageCount(scriptQuery, baseSearchCondition); + long count = getScriptVersionPageCount(scriptQuery); + BaseSearchCondition baseSearchCondition = scriptQuery.getBaseSearchCondition(); int start = baseSearchCondition.getStartOrDefault(0); int length = baseSearchCondition.getLengthOrDefault(10); - Result result = create.select(tbScript.ID, tbScript.APP_ID, tbScript.NAME, tbScript.CATEGORY, - tbScript.TYPE, tbScript.IS_PUBLIC, tbScript.DESCRIPTION, tbScriptVersion.ID.as("scriptVersionId"), - tbScriptVersion.VERSION, tbScriptVersion.CONTENT, tbScriptVersion.VERSION_DESC, tbScriptVersion.STATUS, - tbScriptVersion.CREATE_TIME, - tbScriptVersion.CREATOR, tbScriptVersion.LAST_MODIFY_TIME, tbScriptVersion.LAST_MODIFY_USER).from(tbScriptVersion).join(tbScript) + Result result = dslContext.select(tbScript.ID, tbScript.APP_ID, tbScript.NAME, tbScript.CATEGORY, + tbScript.TYPE, tbScript.IS_PUBLIC, tbScript.DESCRIPTION, tbScriptVersion.ID.as("scriptVersionId"), + tbScriptVersion.VERSION, tbScriptVersion.CONTENT, tbScriptVersion.VERSION_DESC, tbScriptVersion.STATUS, + tbScriptVersion.CREATE_TIME, + tbScriptVersion.CREATOR, tbScriptVersion.LAST_MODIFY_TIME, tbScriptVersion.LAST_MODIFY_USER).from(tbScriptVersion).join(tbScript) .on(tbScript.ID.eq(tbScriptVersion.SCRIPT_ID)) .where(buildScriptVersionConditionList(scriptQuery, baseSearchCondition)) .orderBy(tbScriptVersion.LAST_MODIFY_TIME.desc()) @@ -754,23 +774,29 @@ public PageData listPageScriptVersion(ScriptQuery scriptQuery, return scriptVersionPageData; } - private long getScriptVersionPageCount(ScriptQuery scriptQuery, BaseSearchCondition baseSearchCondition) { + private long getScriptVersionPageCount(ScriptQuery scriptQuery) { + BaseSearchCondition baseSearchCondition = scriptQuery.getBaseSearchCondition(); Script tbScript = Script.SCRIPT; ScriptVersion tbScriptVersion = ScriptVersion.SCRIPT_VERSION; List conditions = buildScriptVersionConditionList(scriptQuery, baseSearchCondition); - return create.selectCount().from(tbScriptVersion).join(tbScript).on(tbScript.ID.eq(tbScriptVersion.SCRIPT_ID)) - .where(conditions).fetchOne(0, Long.class); + return dslContext.selectCount() + .from(tbScriptVersion) + .join(tbScript) + .on(tbScript.ID.eq(tbScriptVersion.SCRIPT_ID)) + .where(conditions) + .fetchOne(0, Long.class); } @Override public ScriptDTO getOnlineScriptVersionByScriptId(long appId, String scriptId) { Script tbScript = Script.SCRIPT.as("t1"); ScriptVersion tbScriptVersion = ScriptVersion.SCRIPT_VERSION.as("t2"); - Record record = create.select(tbScript.ID, tbScript.APP_ID, tbScript.NAME, tbScript.CATEGORY, - tbScript.TYPE, tbScript.IS_PUBLIC, tbScript.DESCRIPTION, tbScriptVersion.ID.as("scriptVersionId"), - tbScriptVersion.VERSION, tbScriptVersion.CONTENT, tbScriptVersion.VERSION_DESC, tbScriptVersion.STATUS, - tbScriptVersion.CREATE_TIME, - tbScriptVersion.CREATOR, tbScriptVersion.LAST_MODIFY_TIME, tbScriptVersion.LAST_MODIFY_USER).from(tbScriptVersion).join(tbScript) + Record record = dslContext.select(tbScript.ID, tbScript.APP_ID, tbScript.NAME, tbScript.CATEGORY, + tbScript.TYPE, tbScript.IS_PUBLIC, tbScript.DESCRIPTION, tbScriptVersion.ID.as("scriptVersionId"), + tbScriptVersion.VERSION, tbScriptVersion.CONTENT, tbScriptVersion.VERSION_DESC, tbScriptVersion.STATUS, + tbScriptVersion.CREATE_TIME, + tbScriptVersion.CREATOR, tbScriptVersion.LAST_MODIFY_TIME, tbScriptVersion.LAST_MODIFY_USER) + .from(tbScriptVersion).join(tbScript) .on(tbScript.ID.eq(tbScriptVersion.SCRIPT_ID)).where(tbScript.APP_ID.eq(ULong.valueOf(appId)) .and(tbScriptVersion.SCRIPT_ID.eq(scriptId)) .and(tbScriptVersion.STATUS.eq(UByte.valueOf(JobResourceStatusEnum.ONLINE.getValue()))) @@ -783,11 +809,12 @@ public ScriptDTO getOnlineScriptVersionByScriptId(long appId, String scriptId) { public ScriptDTO getOnlineScriptVersionByScriptId(String scriptId) { Script tbScript = Script.SCRIPT.as("t1"); ScriptVersion tbScriptVersion = ScriptVersion.SCRIPT_VERSION.as("t2"); - Record record = create.select(tbScript.ID, tbScript.APP_ID, tbScript.NAME, tbScript.CATEGORY, - tbScript.TYPE, tbScript.IS_PUBLIC, tbScript.DESCRIPTION, tbScriptVersion.ID.as("scriptVersionId"), - tbScriptVersion.VERSION, tbScriptVersion.CONTENT, tbScriptVersion.VERSION_DESC, tbScriptVersion.STATUS, - tbScriptVersion.CREATE_TIME, - tbScriptVersion.CREATOR, tbScriptVersion.LAST_MODIFY_TIME, tbScriptVersion.LAST_MODIFY_USER).from(tbScriptVersion).join(tbScript) + Record record = dslContext.select(tbScript.ID, tbScript.APP_ID, tbScript.NAME, tbScript.CATEGORY, + tbScript.TYPE, tbScript.IS_PUBLIC, tbScript.DESCRIPTION, tbScriptVersion.ID.as("scriptVersionId"), + tbScriptVersion.VERSION, tbScriptVersion.CONTENT, tbScriptVersion.VERSION_DESC, tbScriptVersion.STATUS, + tbScriptVersion.CREATE_TIME, + tbScriptVersion.CREATOR, tbScriptVersion.LAST_MODIFY_TIME, tbScriptVersion.LAST_MODIFY_USER) + .from(tbScriptVersion).join(tbScript) .on(tbScript.ID.eq(tbScriptVersion.SCRIPT_ID)) .where(tbScriptVersion.SCRIPT_ID.eq(scriptId)) .and(tbScriptVersion.STATUS.eq(UByte.valueOf(JobResourceStatusEnum.ONLINE.getValue()))) @@ -799,14 +826,14 @@ public ScriptDTO getOnlineScriptVersionByScriptId(String scriptId) { @Override public long countScriptByAppId(long appId) { Script tbScript = Script.SCRIPT.as("tbScript"); - return create.selectCount().from(tbScript) + return dslContext.selectCount().from(tbScript) .where(tbScript.APP_ID.eq(ULong.valueOf(appId)) .and(tbScript.IS_DELETED.eq(UByte.valueOf(0)))).fetchOne().value1(); } @Override public boolean isExistDuplicateVersion(String scriptId, String version) { - int count = create.selectCount().from(TB_SCRIPT_VERSION) + int count = dslContext.selectCount().from(TB_SCRIPT_VERSION) .where(TB_SCRIPT_VERSION.SCRIPT_ID.eq(scriptId)) .and(TB_SCRIPT_VERSION.VERSION.eq(version)) .and(TB_SCRIPT_VERSION.IS_DELETED.eq(UByte.valueOf(0))) @@ -816,7 +843,7 @@ public boolean isExistDuplicateVersion(String scriptId, String version) { @Override public boolean isExistDuplicateScriptId(Long scriptVersionId) { - int count = create.selectCount().from(TB_SCRIPT_VERSION) + int count = dslContext.selectCount().from(TB_SCRIPT_VERSION) .where(TB_SCRIPT_VERSION.ID.eq(ULong.valueOf(scriptVersionId))) .fetchOne(0, Integer.class); return (count >= 1); @@ -827,7 +854,7 @@ public boolean isExistAnyScript(Long appId) { List conditions = new ArrayList<>(2); conditions.add(TB_SCRIPT.APP_ID.eq(ULong.valueOf(appId))); conditions.add(TB_SCRIPT.IS_DELETED.eq(UByte.valueOf(0))); - return create.fetchExists(TB_SCRIPT, conditions); + return dslContext.fetchExists(TB_SCRIPT, conditions); } @Override @@ -835,7 +862,7 @@ public boolean isExistAnyPublicScript() { List conditions = new ArrayList<>(2); conditions.add(TB_SCRIPT.IS_PUBLIC.eq(UByte.valueOf(ScriptScopeEnum.PUBLIC.getValue()))); conditions.add(TB_SCRIPT.IS_DELETED.eq(UByte.valueOf(0))); - return create.fetchExists(TB_SCRIPT, conditions); + return dslContext.fetchExists(TB_SCRIPT, conditions); } @Override @@ -854,7 +881,7 @@ public Integer countScripts(Long appId, ScriptTypeEnum scriptTypeEnum, conditions.add(tbScriptVersion.STATUS.eq(UByte.valueOf(jobResourceStatusEnum.getValue()))); } conditions.add(tbScript.IS_DELETED.eq(UByte.valueOf(0))); - return create.select(DSL.countDistinct(tbScript.ID)).from(tbScript).join(tbScriptVersion) + return dslContext.select(DSL.countDistinct(tbScript.ID)).from(tbScript).join(tbScriptVersion) .on(tbScript.ID.eq(tbScriptVersion.SCRIPT_ID)) .where(conditions).fetchOne().value1(); } @@ -862,7 +889,7 @@ public Integer countScripts(Long appId, ScriptTypeEnum scriptTypeEnum, @Override public Integer countScripts() { Script tbScript = Script.SCRIPT.as("tbScript"); - return create.selectCount().from(tbScript) + return dslContext.selectCount().from(tbScript) .where(tbScript.IS_DELETED.eq(UByte.valueOf(0))).fetchOne().value1(); } @@ -882,7 +909,7 @@ public Integer countScriptVersions(Long appId, ScriptTypeEnum scriptTypeEnum, conditions.add(tbScriptVersion.STATUS.eq(UByte.valueOf(jobResourceStatusEnum.getValue()))); } conditions.add(tbScript.IS_DELETED.eq(UByte.valueOf(0))); - return create.select(DSL.countDistinct(tbScriptVersion.ID)).from(tbScript).join(tbScriptVersion) + return dslContext.select(DSL.countDistinct(tbScriptVersion.ID)).from(tbScript).join(tbScriptVersion) .on(tbScript.ID.eq(tbScriptVersion.SCRIPT_ID)) .where(conditions).fetchOne().value1(); } @@ -894,7 +921,7 @@ public List listAppScriptIds(Long appId) { if (appId != null) { conditions.add(TB_SCRIPT.APP_ID.eq(ULong.valueOf(appId))); } - Result result = create.select(TB_SCRIPT.ID).from(TB_SCRIPT) + Result result = dslContext.select(TB_SCRIPT.ID).from(TB_SCRIPT) .where(conditions) .fetch(); List scriptIdList = new ArrayList<>(); @@ -908,7 +935,7 @@ public List listAppScriptIds(Long appId) { @Override public Map> listAllScriptTagsCompatible() { Result result = - create.select(TB_SCRIPT.ID, TB_SCRIPT.TAGS).from(TB_SCRIPT) + dslContext.select(TB_SCRIPT.ID, TB_SCRIPT.TAGS).from(TB_SCRIPT) .where(TB_SCRIPT.IS_DELETED.eq(UByte.valueOf(0))).fetch(); Map> scriptTagsMap = new HashMap<>(); result.map(record -> { @@ -919,4 +946,27 @@ public Map> listAllScriptTagsCompatible() { }); return scriptTagsMap; } + + @Override + public Map batchGetScriptVersionStatus(Collection scriptVersionIds) { + Result result = dslContext + .select(TB_SCRIPT_VERSION.ID, TB_SCRIPT_VERSION.STATUS) + .from(TB_SCRIPT_VERSION) + .where(TB_SCRIPT_VERSION.ID.in(scriptVersionIds) + .and(TB_SCRIPT_VERSION.IS_DELETED.eq(UByte.valueOf(0)))) + .fetch(); + Map scriptVersionStatusMap = new HashMap<>(); + if (result.isEmpty()) { + return scriptVersionStatusMap; + } + + result.forEach(record -> + scriptVersionStatusMap.put( + record.get(TB_SCRIPT_VERSION.ID).longValue(), + JobResourceStatusEnum.getJobResourceStatus(record.get(TB_SCRIPT_VERSION.STATUS).intValue()) + ) + ); + + return scriptVersionStatusMap; + } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/ScriptRelateJobTemplateDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/ScriptRelateJobTemplateDAOImpl.java index 33b3840b19..dc9cc8315a 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/ScriptRelateJobTemplateDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/ScriptRelateJobTemplateDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,14 +26,14 @@ import com.tencent.bk.job.manage.dao.ScriptRelateJobTemplateDAO; import com.tencent.bk.job.manage.model.dto.ScriptSyncTemplateStepDTO; +import com.tencent.bk.job.manage.model.tables.TaskTemplate; +import com.tencent.bk.job.manage.model.tables.TaskTemplateStep; +import com.tencent.bk.job.manage.model.tables.TaskTemplateStepScript; import lombok.extern.slf4j.Slf4j; import org.jooq.DSLContext; import org.jooq.Record; import org.jooq.Result; import org.jooq.Table; -import org.jooq.generated.tables.TaskTemplate; -import org.jooq.generated.tables.TaskTemplateStep; -import org.jooq.generated.tables.TaskTemplateStepScript; import org.jooq.types.UByte; import org.jooq.types.ULong; import org.springframework.beans.factory.annotation.Autowired; @@ -46,7 +46,7 @@ @Slf4j @Repository public class ScriptRelateJobTemplateDAOImpl implements ScriptRelateJobTemplateDAO { - private DSLContext ctx; + private final DSLContext ctx; @Autowired public ScriptRelateJobTemplateDAOImpl(@Qualifier("job-manage-dsl-context") DSLContext ctx) { diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/ScriptRelateTaskPlanDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/ScriptRelateTaskPlanDAOImpl.java index b35c2c487e..962014398e 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/ScriptRelateTaskPlanDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/ScriptRelateTaskPlanDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,11 +24,14 @@ package com.tencent.bk.job.manage.dao.impl; -import com.tencent.bk.job.manage.common.consts.JobResourceStatusEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskPlanTypeEnum; -import com.tencent.bk.job.manage.common.util.JooqDataTypeUtil; +import com.tencent.bk.job.common.mysql.util.JooqDataTypeUtil; +import com.tencent.bk.job.manage.api.common.constants.JobResourceStatusEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskPlanTypeEnum; import com.tencent.bk.job.manage.dao.ScriptRelateTaskPlanDAO; import com.tencent.bk.job.manage.model.dto.ScriptRelatedTaskPlanDTO; +import com.tencent.bk.job.manage.model.tables.ScriptVersion; +import com.tencent.bk.job.manage.model.tables.TaskPlan; +import com.tencent.bk.job.manage.model.tables.TaskPlanStepScript; import lombok.extern.slf4j.Slf4j; import lombok.val; import org.jooq.DSLContext; @@ -36,9 +39,6 @@ import org.jooq.Record1; import org.jooq.Result; import org.jooq.conf.ParamType; -import org.jooq.generated.tables.ScriptVersion; -import org.jooq.generated.tables.TaskPlan; -import org.jooq.generated.tables.TaskPlanStepScript; import org.jooq.impl.DSL; import org.jooq.types.UByte; import org.jooq.types.ULong; @@ -55,11 +55,11 @@ @Repository public class ScriptRelateTaskPlanDAOImpl implements ScriptRelateTaskPlanDAO { - private DSLContext ctx; + private final DSLContext ctx; - private TaskPlanStepScript T_TASK_PLAN_STEP_SCRIPT = TaskPlanStepScript.TASK_PLAN_STEP_SCRIPT; - private ScriptVersion T_SCRIPT_VERSION = ScriptVersion.SCRIPT_VERSION; - private TaskPlan T_TASK_PLAN = TaskPlan.TASK_PLAN; + private final TaskPlanStepScript T_TASK_PLAN_STEP_SCRIPT = TaskPlanStepScript.TASK_PLAN_STEP_SCRIPT; + private final ScriptVersion T_SCRIPT_VERSION = ScriptVersion.SCRIPT_VERSION; + private final TaskPlan T_TASK_PLAN = TaskPlan.TASK_PLAN; @Autowired public ScriptRelateTaskPlanDAOImpl(@Qualifier("job-manage-dsl-context") DSLContext ctx) { @@ -77,9 +77,7 @@ public Integer countScriptRelatedTaskPlan(String scriptId) { .and(T_TASK_PLAN.TYPE.eq(UByte.valueOf(TaskPlanTypeEnum.NORMAL.getValue()))); try { Result> records = query.fetch(); - if (records != null) { - return records.get(0).value1(); - } + return records.get(0).value1(); } catch (Exception e) { log.error(String.format("error SQL=%s", query.getSQL(ParamType.INLINED)), e); } @@ -98,9 +96,7 @@ public Integer countScriptVersionRelatedTaskPlan(String scriptId, long scriptVer .and(T_TASK_PLAN.TYPE.eq(UByte.valueOf(TaskPlanTypeEnum.NORMAL.getValue()))); try { Result> records = query.fetch(); - if (records != null) { - return records.get(0).value1(); - } + return records.get(0).value1(); } catch (Exception e) { log.error(String.format("error SQL=%s", query.getSQL(ParamType.INLINED)), e); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/TagDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/TagDAOImpl.java index acb13e231e..c0e4aa2935 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/TagDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/impl/TagDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -30,6 +30,7 @@ import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.manage.dao.TagDAO; import com.tencent.bk.job.manage.model.dto.TagDTO; +import com.tencent.bk.job.manage.model.tables.Tag; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.jooq.Condition; @@ -38,7 +39,6 @@ import org.jooq.Record; import org.jooq.Result; import org.jooq.TableField; -import org.jooq.generated.tables.Tag; import org.jooq.types.ULong; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/index/IndexGreetingDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/index/IndexGreetingDAO.java index 7631e9e7dd..1bd5e6dee3 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/index/IndexGreetingDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/index/IndexGreetingDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,20 +25,10 @@ package com.tencent.bk.job.manage.dao.index; import com.tencent.bk.job.manage.model.dto.index.IndexGreetingDTO; -import org.jooq.DSLContext; import java.util.List; public interface IndexGreetingDAO { - int insertIndexGreeting(DSLContext dslContext, IndexGreetingDTO indexGreetingDTO); - int updateIndexGreetingById(DSLContext dslContext, IndexGreetingDTO indexGreetingDTO); - - int deleteIndexGreetingById(DSLContext dslContext, Long id); - - IndexGreetingDTO getIndexGreetingById(DSLContext dslContext, Long id); - - List listAllIndexGreeting(DSLContext dslContext); - - List listActiveIndexGreeting(DSLContext dslContext, int currentTimeSeconds); + List listActiveIndexGreeting(int currentTimeSeconds); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/index/impl/IndexGreetingDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/index/impl/IndexGreetingDAOImpl.java index 77da08a4e7..3535c30d07 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/index/impl/IndexGreetingDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/index/impl/IndexGreetingDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,6 +26,8 @@ import com.tencent.bk.job.manage.dao.index.IndexGreetingDAO; import com.tencent.bk.job.manage.model.dto.index.IndexGreetingDTO; +import com.tencent.bk.job.manage.model.tables.IndexGreeting; +import com.tencent.bk.job.manage.model.tables.records.IndexGreetingRecord; import lombok.extern.slf4j.Slf4j; import lombok.val; import lombok.var; @@ -33,9 +35,8 @@ import org.jooq.DSLContext; import org.jooq.Result; import org.jooq.conf.ParamType; -import org.jooq.generated.tables.IndexGreeting; -import org.jooq.generated.tables.records.IndexGreetingRecord; -import org.jooq.types.ULong; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; import java.util.ArrayList; @@ -43,106 +44,29 @@ import java.util.Collections; import java.util.List; -/** - * @Description - * @Date 2020/3/6 - * @Version 1.0 - */ @Repository @Slf4j public class IndexGreetingDAOImpl implements IndexGreetingDAO { private static final IndexGreeting defaultTable = IndexGreeting.INDEX_GREETING; - @Override - public int insertIndexGreeting(DSLContext dslContext, IndexGreetingDTO indexGreetingDTO) { - val query = dslContext.insertInto(defaultTable, - defaultTable.ID, - defaultTable.START_SECONDS, - defaultTable.END_SECONDS, - defaultTable.CONTENT, - defaultTable.PRIORITY, - defaultTable.ACTIVE, - defaultTable.CREATOR, - defaultTable.CREATE_TIME, - defaultTable.LAST_MODIFY_USER, - defaultTable.LAST_MODIFY_TIME - ).values( - null, - indexGreetingDTO.getStartSeconds(), - indexGreetingDTO.getEndSeconds(), - indexGreetingDTO.getContent(), - indexGreetingDTO.getPriority(), - indexGreetingDTO.isActive(), - indexGreetingDTO.getCreator(), - ULong.valueOf(indexGreetingDTO.getCreateTime()), - indexGreetingDTO.getLastModifier(), - ULong.valueOf(indexGreetingDTO.getLastModifyTime()) - ); - val sql = query.getSQL(ParamType.INLINED); - try { - return query.execute(); - } catch (Exception e) { - log.error(sql); - throw e; - } - } - - @Override - public int updateIndexGreetingById(DSLContext dslContext, IndexGreetingDTO indexGreetingDTO) { - val query = dslContext.update(defaultTable) - .set(defaultTable.START_SECONDS, indexGreetingDTO.getStartSeconds()) - .set(defaultTable.END_SECONDS, indexGreetingDTO.getEndSeconds()) - .set(defaultTable.CONTENT, indexGreetingDTO.getContent()) - .set(defaultTable.PRIORITY, indexGreetingDTO.getEndSeconds()) - .set(defaultTable.ACTIVE, indexGreetingDTO.isActive()) - .set(defaultTable.LAST_MODIFY_USER, indexGreetingDTO.getLastModifier()) - .set(defaultTable.LAST_MODIFY_TIME, ULong.valueOf(indexGreetingDTO.getLastModifyTime())) - .where(defaultTable.ID.eq(indexGreetingDTO.getId())); - val sql = query.getSQL(ParamType.INLINED); - try { - return query.execute(); - } catch (Exception e) { - log.error(sql); - throw e; - } - } - - @Override - public int deleteIndexGreetingById(DSLContext dslContext, Long id) { - return dslContext.deleteFrom(defaultTable).where( - defaultTable.ID.eq(id) - ).execute(); - } - - @Override - public IndexGreetingDTO getIndexGreetingById(DSLContext dslContext, Long id) { - val record = dslContext.selectFrom(defaultTable).where( - defaultTable.ID.eq(id) - ).fetchOne(); - if (record == null) { - return null; - } else { - return convert(record); - } - } + private final DSLContext dslContext; - @Override - public List listAllIndexGreeting(DSLContext dslContext) { - return listIndexGreetingWithConditions(dslContext, Collections.emptyList()); + @Autowired + public IndexGreetingDAOImpl(@Qualifier("job-manage-dsl-context") DSLContext dslContext) { + this.dslContext = dslContext; } @Override - public List listActiveIndexGreeting(DSLContext dslContext, int currentTimeSeconds) { + public List listActiveIndexGreeting(int currentTimeSeconds) { List conditions = new ArrayList<>(); conditions.add(defaultTable.ACTIVE.eq(true)); conditions.add(defaultTable.START_SECONDS.lessOrEqual(currentTimeSeconds)); conditions.add(defaultTable.END_SECONDS.greaterOrEqual(currentTimeSeconds)); - return listIndexGreetingWithConditions(dslContext, conditions); + return listIndexGreetingWithConditions(conditions); } - private List listIndexGreetingWithConditions(DSLContext dslContext, - Collection conditions) { + private List listIndexGreetingWithConditions(Collection conditions) { var query = dslContext.selectFrom(defaultTable).where( conditions //默认按照优先级排序 @@ -155,7 +79,7 @@ private List listIndexGreetingWithConditions(DSLContext dslCon log.error(sql); throw e; } - if (records == null || records.isEmpty()) { + if (records.isEmpty()) { return Collections.emptyList(); } else { return records.map(this::convert); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/AvailableEsbChannelDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/AvailableEsbChannelDAO.java index 032d23ea3f..361fd7eec9 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/AvailableEsbChannelDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/AvailableEsbChannelDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,14 +25,13 @@ package com.tencent.bk.job.manage.dao.notify; import com.tencent.bk.job.manage.model.dto.notify.AvailableEsbChannelDTO; -import org.jooq.DSLContext; import java.util.List; public interface AvailableEsbChannelDAO { - int insertAvailableEsbChannel(DSLContext dslContext, AvailableEsbChannelDTO availableEsbChannelDTO); + int insertAvailableEsbChannel(AvailableEsbChannelDTO availableEsbChannelDTO); - int deleteAll(DSLContext dslContext); + int deleteAll(); - List listAvailableEsbChannel(DSLContext dslContext); + List listAvailableEsbChannel(); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/EsbAppRoleDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/EsbAppRoleDAO.java index 4e25025124..b1dc8cfb34 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/EsbAppRoleDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/EsbAppRoleDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,12 +25,11 @@ package com.tencent.bk.job.manage.dao.notify; import com.tencent.bk.job.common.cc.model.AppRoleDTO; -import org.jooq.DSLContext; import java.util.List; public interface EsbAppRoleDAO { - List listEsbAppRole(DSLContext dslContext); + List listEsbAppRole(); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/EsbUserInfoDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/EsbUserInfoDAO.java index 2c82c44ab6..31cc514c1b 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/EsbUserInfoDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/EsbUserInfoDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,16 +25,15 @@ package com.tencent.bk.job.manage.dao.notify; import com.tencent.bk.job.manage.model.dto.notify.EsbUserInfoDTO; -import org.jooq.DSLContext; import java.util.Collection; import java.util.List; public interface EsbUserInfoDAO { - int insertEsbUserInfo(DSLContext dslContext, EsbUserInfoDTO esbUserInfoDTO); + int insertEsbUserInfo(EsbUserInfoDTO esbUserInfoDTO); - int deleteEsbUserInfoById(DSLContext dslContext, Long id); + int deleteEsbUserInfoById(Long id); List listEsbUserInfo(); @@ -42,4 +41,6 @@ public interface EsbUserInfoDAO { List listEsbUserInfo(Collection userNames); + List listExistUserName(Collection userNames); + } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/NotifyBlackUserInfoDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/NotifyBlackUserInfoDAO.java index 22c7a334c7..0dafe40a01 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/NotifyBlackUserInfoDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/NotifyBlackUserInfoDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/NotifyConfigStatusDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/NotifyConfigStatusDAO.java index 21533b4fe2..d810dc75bf 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/NotifyConfigStatusDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/NotifyConfigStatusDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,11 +24,9 @@ package com.tencent.bk.job.manage.dao.notify; -import org.jooq.DSLContext; - public interface NotifyConfigStatusDAO { - int insertNotifyConfigStatus(DSLContext dslContext, String userName, Long appId); + int insertNotifyConfigStatus(String userName, Long appId); boolean exist(String userName, Long appId); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/NotifyEsbChannelDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/NotifyEsbChannelDAO.java index f74f8ba63d..22ceaf6dfe 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/NotifyEsbChannelDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/NotifyEsbChannelDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,12 +25,11 @@ package com.tencent.bk.job.manage.dao.notify; import com.tencent.bk.job.manage.model.dto.notify.NotifyEsbChannelDTO; -import org.jooq.DSLContext; import java.util.List; public interface NotifyEsbChannelDAO { - List listNotifyEsbChannel(DSLContext dslContext); + List listNotifyEsbChannel(); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/NotifyPolicyRoleTargetDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/NotifyPolicyRoleTargetDAO.java index 0c5d8f01ab..d3a1aeb5ce 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/NotifyPolicyRoleTargetDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/NotifyPolicyRoleTargetDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,7 +25,6 @@ package com.tencent.bk.job.manage.dao.notify; import com.tencent.bk.job.manage.model.dto.notify.NotifyPolicyRoleTargetDTO; -import org.jooq.DSLContext; import java.util.List; @@ -33,8 +32,8 @@ public interface NotifyPolicyRoleTargetDAO { Long insert(NotifyPolicyRoleTargetDTO notifyPolicyRoleTargetDTO); - int deleteByPolicyId(DSLContext dslContext, Long policyId); + int deleteByPolicyId(Long policyId); - List listByPolicyId(DSLContext dslContext, Long policyId); + List listByPolicyId(Long policyId); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/NotifyRoleTargetChannelDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/NotifyRoleTargetChannelDAO.java index c1f3a6c1ae..435d7754a3 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/NotifyRoleTargetChannelDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/NotifyRoleTargetChannelDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,15 +25,13 @@ package com.tencent.bk.job.manage.dao.notify; import com.tencent.bk.job.manage.model.dto.notify.NotifyRoleTargetChannelDTO; -import org.jooq.DSLContext; import java.util.List; public interface NotifyRoleTargetChannelDAO { Long insert(NotifyRoleTargetChannelDTO notifyRoleTargetChannelDTO); - int deleteByRoleTargetId(DSLContext dslContext, Long roleTargetId); + int deleteByRoleTargetId(Long roleTargetId); - List listByRoleTargetId(DSLContext dslContext, - Long roleTargetId); + List listByRoleTargetId(Long roleTargetId); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/NotifyTemplateDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/NotifyTemplateDAO.java index 02cf9e9ae6..2ecd7f6fa5 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/NotifyTemplateDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/NotifyTemplateDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/NotifyTriggerPolicyDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/NotifyTriggerPolicyDAO.java index 3c0f3acf23..af72e7f2cd 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/NotifyTriggerPolicyDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/NotifyTriggerPolicyDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,7 @@ package com.tencent.bk.job.manage.dao.notify; +import com.tencent.bk.job.common.model.dto.notify.CustomNotifyDTO; import com.tencent.bk.job.manage.model.dto.notify.NotifyTriggerPolicyDTO; import com.tencent.bk.job.manage.model.web.vo.notify.TriggerPolicyVO; @@ -34,10 +35,21 @@ public interface NotifyTriggerPolicyDAO { int deleteAppNotifyPolicies(Long appId, String triggerUser); - List list(String triggerUser, Long appId, String resourceId); + int deleteAppResourceNotifyPolicies(Long appId, Integer resourceType, String resourceId); - List list(String triggerUser, Long appId, String resourceId, - Integer resourceType, Integer triggerType, Integer executeStatus); + CustomNotifyDTO getSpecificResourceNotifyPolicy(Long appId, + Integer resourceType, + String resourceId, + Integer triggerType); + + List listAppDefault(String triggerUser, Long appId, String resourceId); + + List list(String triggerUser, + Long appId, + String resourceId, + Integer resourceType, + Integer triggerType, + Integer executeStatus); int countDefaultPolicies(); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/AvailableEsbChannelDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/AvailableEsbChannelDAOImpl.java index 0731864553..0c17cd8006 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/AvailableEsbChannelDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/AvailableEsbChannelDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,31 +26,34 @@ import com.tencent.bk.job.manage.dao.notify.AvailableEsbChannelDAO; import com.tencent.bk.job.manage.model.dto.notify.AvailableEsbChannelDTO; +import com.tencent.bk.job.manage.model.tables.AvailableEsbChannel; import lombok.val; import org.jooq.DSLContext; import org.jooq.conf.ParamType; -import org.jooq.generated.tables.AvailableEsbChannel; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; import java.util.ArrayList; import java.util.List; -/** - * @Description - * @Date 2020/1/2 - * @Version 1.0 - */ @Repository public class AvailableEsbChannelDAOImpl implements AvailableEsbChannelDAO { private static final Logger logger = LoggerFactory.getLogger(AvailableEsbChannelDAOImpl.class); private static final AvailableEsbChannel T_AVAILABLE_ESB_CHANNEL = AvailableEsbChannel.AVAILABLE_ESB_CHANNEL; private static final AvailableEsbChannel defaultTable = T_AVAILABLE_ESB_CHANNEL; + private final DSLContext dslContext; + + @Autowired + public AvailableEsbChannelDAOImpl(@Qualifier("job-manage-dsl-context") DSLContext dslContext) { + this.dslContext = dslContext; + } @Override - public int insertAvailableEsbChannel(DSLContext dslContext, AvailableEsbChannelDTO availableEsbChannelDTO) { + public int insertAvailableEsbChannel(AvailableEsbChannelDTO availableEsbChannelDTO) { val query = dslContext.insertInto(defaultTable, defaultTable.TYPE, defaultTable.ENABLE, @@ -72,12 +75,12 @@ public int insertAvailableEsbChannel(DSLContext dslContext, AvailableEsbChannelD } @Override - public int deleteAll(DSLContext dslContext) { + public int deleteAll() { return dslContext.deleteFrom(defaultTable).execute(); } @Override - public List listAvailableEsbChannel(DSLContext dslContext) { + public List listAvailableEsbChannel() { val records = dslContext.selectFrom(defaultTable).fetch(); if (records.isEmpty()) { return new ArrayList<>(); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/EsbAppRoleDAOMemCacheImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/EsbAppRoleDAOMemCacheImpl.java index 345c139de6..5685503136 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/EsbAppRoleDAOMemCacheImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/EsbAppRoleDAOMemCacheImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -33,7 +33,6 @@ import com.tencent.bk.job.manage.service.AppRoleService; import lombok.extern.slf4j.Slf4j; import lombok.val; -import org.jooq.DSLContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -43,11 +42,6 @@ import java.util.List; import java.util.concurrent.TimeUnit; -/** - * @Description - * @Date 2020/1/2 - * @Version 1.0 - */ @Slf4j @Repository public class EsbAppRoleDAOMemCacheImpl implements EsbAppRoleDAO { @@ -74,7 +68,7 @@ public EsbAppRoleDAOMemCacheImpl(AppRoleService roleService) { } @Override - public List listEsbAppRole(DSLContext dslContext) { + public List listEsbAppRole() { try { String lang = JobContextUtil.getUserLang(); logger.info(String.format("Current Lang:%s", lang)); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/EsbUserInfoDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/EsbUserInfoDAOImpl.java index 268dd71ce9..56bc0a713a 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/EsbUserInfoDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/EsbUserInfoDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,43 +26,42 @@ import com.tencent.bk.job.manage.dao.notify.EsbUserInfoDAO; import com.tencent.bk.job.manage.model.dto.notify.EsbUserInfoDTO; +import com.tencent.bk.job.manage.model.tables.EsbUserInfo; +import com.tencent.bk.job.manage.model.tables.records.EsbUserInfoRecord; import lombok.val; import org.jooq.Condition; import org.jooq.DSLContext; +import org.jooq.Record1; import org.jooq.Result; import org.jooq.conf.ParamType; -import org.jooq.generated.tables.EsbUserInfo; -import org.jooq.generated.tables.records.EsbUserInfoRecord; import org.jooq.types.ULong; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; +import org.springframework.util.CollectionUtils; import java.util.ArrayList; import java.util.Collection; +import java.util.Collections; import java.util.List; -/** - * @Description - * @Date 2020/1/2 - * @Version 1.0 - */ @Repository public class EsbUserInfoDAOImpl implements EsbUserInfoDAO { - private final DSLContext defaultDslContext; + private final DSLContext dslContext; private static final Logger logger = LoggerFactory.getLogger(EsbUserInfoDAOImpl.class); private static final EsbUserInfo T_ESB_USER_INFO = EsbUserInfo.ESB_USER_INFO; private static final EsbUserInfo defaultTable = T_ESB_USER_INFO; @Autowired - public EsbUserInfoDAOImpl(DSLContext dslContext) { - this.defaultDslContext = dslContext; + public EsbUserInfoDAOImpl(@Qualifier("job-manage-dsl-context") DSLContext dslContext) { + this.dslContext = dslContext; } @Override - public int insertEsbUserInfo(DSLContext dslContext, EsbUserInfoDTO esbUserInfoDTO) { + public int insertEsbUserInfo(EsbUserInfoDTO esbUserInfoDTO) { val query = dslContext.insertInto(defaultTable, defaultTable.ID, defaultTable.USERNAME, @@ -86,7 +85,7 @@ public int insertEsbUserInfo(DSLContext dslContext, EsbUserInfoDTO esbUserInfoDT } @Override - public int deleteEsbUserInfoById(DSLContext dslContext, Long id) { + public int deleteEsbUserInfoById(Long id) { return dslContext.deleteFrom(defaultTable).where( defaultTable.ID.eq(id) ).execute(); @@ -94,7 +93,7 @@ public int deleteEsbUserInfoById(DSLContext dslContext, Long id) { @Override public List listEsbUserInfo() { - val records = defaultDslContext.selectFrom(defaultTable).fetch(); + val records = dslContext.selectFrom(defaultTable).fetch(); if (records.isEmpty()) { return new ArrayList<>(); } else { @@ -112,14 +111,28 @@ public List listEsbUserInfo() { public List listEsbUserInfo(String prefixStr, Long limit) { List conditions = new ArrayList<>(); conditions.add(defaultTable.USERNAME.startsWith(prefixStr)); - return listEsbUserInfoByConditions(defaultDslContext, conditions, limit); + return listEsbUserInfoByConditions(dslContext, conditions, limit); } @Override public List listEsbUserInfo(Collection userNames) { List conditions = new ArrayList<>(); conditions.add(defaultTable.USERNAME.in(userNames)); - return listEsbUserInfoByConditions(defaultDslContext, conditions, null); + return listEsbUserInfoByConditions(dslContext, conditions, null); + } + + @Override + public List listExistUserName(Collection userNames) { + if (CollectionUtils.isEmpty(userNames)) { + return Collections.emptyList(); + } + List conditions = new ArrayList<>(); + conditions.add(defaultTable.USERNAME.in(userNames)); + val baseQuery = dslContext.select(defaultTable.USERNAME) + .from(defaultTable) + .where(conditions); + Result> records = baseQuery.fetch(); + return records.map(record -> record.get(defaultTable.USERNAME)); } private List listEsbUserInfoByConditions(DSLContext dslContext, List conditions, diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/NotifyBlackUserInfoDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/NotifyBlackUserInfoDAOImpl.java index 3ed5671f3e..bd66686f9a 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/NotifyBlackUserInfoDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/NotifyBlackUserInfoDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,18 +26,19 @@ import com.tencent.bk.job.manage.dao.notify.NotifyBlackUserInfoDAO; import com.tencent.bk.job.manage.model.dto.notify.NotifyBlackUserInfoDTO; +import com.tencent.bk.job.manage.model.tables.NotifyBlackUserInfo; +import com.tencent.bk.job.manage.model.tables.records.NotifyBlackUserInfoRecord; import com.tencent.bk.job.manage.model.web.vo.notify.NotifyBlackUserInfoVO; import lombok.val; import org.jooq.DSLContext; import org.jooq.Record; import org.jooq.Result; import org.jooq.conf.ParamType; -import org.jooq.generated.tables.NotifyBlackUserInfo; -import org.jooq.generated.tables.records.NotifyBlackUserInfoRecord; import org.jooq.types.ULong; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; import java.util.ArrayList; @@ -58,7 +59,7 @@ public class NotifyBlackUserInfoDAOImpl implements NotifyBlackUserInfoDAO { private final DSLContext dslContext; @Autowired - public NotifyBlackUserInfoDAOImpl(DSLContext dslContext) { + public NotifyBlackUserInfoDAOImpl(@Qualifier("job-manage-dsl-context") DSLContext dslContext) { this.dslContext = dslContext; } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/NotifyConfigStatusDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/NotifyConfigStatusDAOImpl.java index fe324a7d78..38b76a193d 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/NotifyConfigStatusDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/NotifyConfigStatusDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,22 +25,18 @@ package com.tencent.bk.job.manage.dao.notify.impl; import com.tencent.bk.job.manage.dao.notify.NotifyConfigStatusDAO; +import com.tencent.bk.job.manage.model.tables.NotifyConfigStatus; import lombok.val; import org.joda.time.DateTimeUtils; import org.jooq.DSLContext; import org.jooq.conf.ParamType; -import org.jooq.generated.tables.NotifyConfigStatus; import org.jooq.types.ULong; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; -/** - * @Description - * @Date 2020/1/2 - * @Version 1.0 - */ @Repository public class NotifyConfigStatusDAOImpl implements NotifyConfigStatusDAO { @@ -51,12 +47,12 @@ public class NotifyConfigStatusDAOImpl implements NotifyConfigStatusDAO { private final DSLContext dslContext; @Autowired - public NotifyConfigStatusDAOImpl(DSLContext dslContext) { + public NotifyConfigStatusDAOImpl(@Qualifier("job-manage-dsl-context") DSLContext dslContext) { this.dslContext = dslContext; } @Override - public int insertNotifyConfigStatus(DSLContext dslContext, String userName, Long appId) { + public int insertNotifyConfigStatus(String userName, Long appId) { val query = dslContext.insertInto(defaultTable, defaultTable.USERNAME, defaultTable.APP_ID, diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/NotifyEsbChannelDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/NotifyEsbChannelDAOImpl.java index 8dadbf3655..e185319ef0 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/NotifyEsbChannelDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/NotifyEsbChannelDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -30,33 +30,26 @@ import com.google.common.util.concurrent.UncheckedExecutionException; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.exception.InternalException; +import com.tencent.bk.job.common.paas.cmsi.CmsiApiClient; import com.tencent.bk.job.common.util.JobContextUtil; import com.tencent.bk.job.manage.dao.notify.NotifyEsbChannelDAO; import com.tencent.bk.job.manage.model.dto.notify.NotifyEsbChannelDTO; -import com.tencent.bk.job.manage.service.PaaSService; -import org.jooq.DSLContext; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.lang.NonNull; import org.springframework.stereotype.Repository; -import java.io.IOException; import java.time.LocalDateTime; import java.util.List; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -/** - * @Description - * @Date 2020/1/2 - * @Version 1.0 - */ @Repository public class NotifyEsbChannelDAOImpl implements NotifyEsbChannelDAO { private static final Logger logger = LoggerFactory.getLogger(NotifyEsbChannelDAOImpl.class); - private final PaaSService paaSService; + private final CmsiApiClient cmsiApiClient; private final LoadingCache> esbChannelCache = CacheBuilder.newBuilder() .maximumSize(10).expireAfterWrite(10, TimeUnit.MINUTES). build(new CacheLoader>() { @@ -64,33 +57,28 @@ public class NotifyEsbChannelDAOImpl implements NotifyEsbChannelDAO { public List load(@NonNull String searchKey) { logger.info("esbChannelCache searchKey=" + searchKey); List channelDtoList; - try { - //新增渠道默认为已启用 - channelDtoList = - paaSService.getAllChannelList("", "100").stream().map(it -> new NotifyEsbChannelDTO( - it.getType(), - it.getLabel(), - it.isActive(), - true, - it.getIcon(), - LocalDateTime.now() - )).collect(Collectors.toList()); - logger.info(String.format("result.size=%d", channelDtoList.size())); - return channelDtoList; - } catch (IOException e) { - logger.error("updateNotifyEsbChannel: Fail to fetch remote notifyChannel, return", e); - return null; - } + //新增渠道默认为已启用 + channelDtoList = + cmsiApiClient.getNotifyChannelList().stream().map(it -> new NotifyEsbChannelDTO( + it.getType(), + it.getLabel(), + it.isActive(), + true, + it.getIcon(), + LocalDateTime.now() + )).collect(Collectors.toList()); + logger.info(String.format("result.size=%d", channelDtoList.size())); + return channelDtoList; } } ); - public NotifyEsbChannelDAOImpl(PaaSService paaSService) { - this.paaSService = paaSService; + public NotifyEsbChannelDAOImpl(CmsiApiClient cmsiApiClient) { + this.cmsiApiClient = cmsiApiClient; } @Override - public List listNotifyEsbChannel(DSLContext dslContext) { + public List listNotifyEsbChannel() { try { String lang = JobContextUtil.getUserLang(); return esbChannelCache.get(lang); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/NotifyPolicyRoleTargetDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/NotifyPolicyRoleTargetDAOImpl.java index e7b495fefc..69fcc69160 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/NotifyPolicyRoleTargetDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/NotifyPolicyRoleTargetDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,34 +24,27 @@ package com.tencent.bk.job.manage.dao.notify.impl; -import com.tencent.bk.job.common.RequestIdLogger; -import com.tencent.bk.job.common.util.SimpleRequestIdLogger; import com.tencent.bk.job.manage.dao.notify.NotifyPolicyRoleTargetDAO; import com.tencent.bk.job.manage.dao.notify.NotifyRoleTargetChannelDAO; import com.tencent.bk.job.manage.model.dto.notify.NotifyPolicyRoleTargetDTO; +import com.tencent.bk.job.manage.model.tables.NotifyPolicyRoleTarget; +import com.tencent.bk.job.manage.model.tables.records.NotifyPolicyRoleTargetRecord; +import lombok.extern.slf4j.Slf4j; import lombok.val; import org.jooq.DSLContext; import org.jooq.Record; import org.jooq.conf.ParamType; -import org.jooq.generated.tables.NotifyPolicyRoleTarget; -import org.jooq.generated.tables.records.NotifyPolicyRoleTargetRecord; import org.jooq.types.ULong; -import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; import java.util.ArrayList; import java.util.List; -/** - * @Description - * @Date 2020/1/2 - * @Version 1.0 - */ @Repository +@Slf4j public class NotifyPolicyRoleTargetDAOImpl implements NotifyPolicyRoleTargetDAO { - private static final RequestIdLogger logger = - new SimpleRequestIdLogger(LoggerFactory.getLogger(NotifyPolicyRoleTargetDAOImpl.class)); private static final NotifyPolicyRoleTarget T_NOTIFY_POLICY_ROLE_TARGET = NotifyPolicyRoleTarget.NOTIFY_POLICY_ROLE_TARGET; private static final NotifyPolicyRoleTarget defaultTable = T_NOTIFY_POLICY_ROLE_TARGET; @@ -60,7 +53,7 @@ public class NotifyPolicyRoleTargetDAOImpl implements NotifyPolicyRoleTargetDAO private final NotifyRoleTargetChannelDAO notifyRoleTargetChannelDAO; @Autowired - public NotifyPolicyRoleTargetDAOImpl(DSLContext dslContext, + public NotifyPolicyRoleTargetDAOImpl(@Qualifier("job-manage-dsl-context") DSLContext dslContext, NotifyRoleTargetChannelDAO notifyRoleTargetChannelDAO) { this.dslContext = dslContext; this.notifyRoleTargetChannelDAO = notifyRoleTargetChannelDAO; @@ -93,13 +86,13 @@ public Long insert(NotifyPolicyRoleTargetDTO notifyPolicyRoleTargetDTO) { assert record != null; return record.get(T_NOTIFY_POLICY_ROLE_TARGET.ID); } catch (Exception e) { - logger.errorWithRequestId(sql); + log.error(sql); throw e; } } @Override - public int deleteByPolicyId(DSLContext dslContext, Long policyId) { + public int deleteByPolicyId(Long policyId) { //1.查记录 val records = dslContext.selectFrom(defaultTable).where( defaultTable.POLICY_ID.eq(policyId) @@ -108,7 +101,7 @@ public int deleteByPolicyId(DSLContext dslContext, Long policyId) { return 0; } //2.删从表 - records.forEach(record -> notifyRoleTargetChannelDAO.deleteByRoleTargetId(dslContext, record.getId())); + records.forEach(record -> notifyRoleTargetChannelDAO.deleteByRoleTargetId(record.getId())); //3.删主表 return dslContext.deleteFrom(defaultTable).where( defaultTable.ID.in(records.map(NotifyPolicyRoleTargetRecord::getId)) @@ -116,7 +109,7 @@ public int deleteByPolicyId(DSLContext dslContext, Long policyId) { } @Override - public List listByPolicyId(DSLContext dslContext, Long policyId) { + public List listByPolicyId(Long policyId) { val records = dslContext.selectFrom(T_NOTIFY_POLICY_ROLE_TARGET).where( T_NOTIFY_POLICY_ROLE_TARGET.POLICY_ID.eq(policyId) ).fetch(); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/NotifyRoleTargetChannelDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/NotifyRoleTargetChannelDAOImpl.java index 493a9fe55d..8d545f2a3e 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/NotifyRoleTargetChannelDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/NotifyRoleTargetChannelDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,38 +24,31 @@ package com.tencent.bk.job.manage.dao.notify.impl; -import com.tencent.bk.job.common.RequestIdLogger; -import com.tencent.bk.job.common.util.SimpleRequestIdLogger; import com.tencent.bk.job.manage.dao.notify.NotifyRoleTargetChannelDAO; import com.tencent.bk.job.manage.model.dto.notify.NotifyRoleTargetChannelDTO; +import com.tencent.bk.job.manage.model.tables.NotifyRoleTargetChannel; +import lombok.extern.slf4j.Slf4j; import lombok.val; import org.jooq.DSLContext; import org.jooq.Record; import org.jooq.conf.ParamType; -import org.jooq.generated.tables.NotifyRoleTargetChannel; import org.jooq.types.ULong; -import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; import java.util.List; -/** - * @Description - * @Date 2020/1/2 - * @Version 1.0 - */ @Repository +@Slf4j public class NotifyRoleTargetChannelDAOImpl implements NotifyRoleTargetChannelDAO { - private static final RequestIdLogger logger = - new SimpleRequestIdLogger(LoggerFactory.getLogger(NotifyRoleTargetChannelDAOImpl.class)); private static final NotifyRoleTargetChannel T_NOTIFY_ROLE_TARGET_CHANNEL = NotifyRoleTargetChannel.NOTIFY_ROLE_TARGET_CHANNEL; private final DSLContext dslContext; @Autowired - public NotifyRoleTargetChannelDAOImpl(DSLContext dslContext) { + public NotifyRoleTargetChannelDAOImpl(@Qualifier("job-manage-dsl-context") DSLContext dslContext) { this.dslContext = dslContext; } @@ -82,13 +75,13 @@ public Long insert(NotifyRoleTargetChannelDTO notifyRoleTargetChannelDTO) { assert record != null; return record.get(T_NOTIFY_ROLE_TARGET_CHANNEL.ID); } catch (Exception e) { - logger.errorWithRequestId(sql); + log.error(sql); throw e; } } @Override - public int deleteByRoleTargetId(DSLContext dslContext, Long roleTargetId) { + public int deleteByRoleTargetId(Long roleTargetId) { //1.无从表 //2.直接删主表 return dslContext.deleteFrom(T_NOTIFY_ROLE_TARGET_CHANNEL).where( @@ -97,8 +90,7 @@ public int deleteByRoleTargetId(DSLContext dslContext, Long roleTargetId) { } @Override - public List listByRoleTargetId(DSLContext dslContext, - Long roleTargetId) { + public List listByRoleTargetId(Long roleTargetId) { val records = dslContext.selectFrom(T_NOTIFY_ROLE_TARGET_CHANNEL).where( T_NOTIFY_ROLE_TARGET_CHANNEL.ROLE_TARGET_ID.eq(roleTargetId) ).fetch(); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/NotifyTemplateDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/NotifyTemplateDAOImpl.java index 87fb84d79a..a536e82f58 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/NotifyTemplateDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/NotifyTemplateDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,17 +26,18 @@ import com.tencent.bk.job.manage.dao.notify.NotifyTemplateDAO; import com.tencent.bk.job.manage.model.dto.notify.NotifyTemplateDTO; +import com.tencent.bk.job.manage.model.tables.NotifyTemplate; import lombok.val; import org.jooq.Condition; import org.jooq.DSLContext; import org.jooq.Record13; import org.jooq.Result; import org.jooq.conf.ParamType; -import org.jooq.generated.tables.NotifyTemplate; import org.jooq.types.ULong; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; import java.util.ArrayList; @@ -57,7 +58,7 @@ public class NotifyTemplateDAOImpl implements NotifyTemplateDAO { private final DSLContext dslContext; @Autowired - public NotifyTemplateDAOImpl(DSLContext dslContext) { + public NotifyTemplateDAOImpl(@Qualifier("job-manage-dsl-context") DSLContext dslContext) { this.dslContext = dslContext; } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/NotifyTriggerPolicyDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/NotifyTriggerPolicyDAOImpl.java index 82a68a9578..c5900ebacc 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/NotifyTriggerPolicyDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/notify/impl/NotifyTriggerPolicyDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,13 +24,12 @@ package com.tencent.bk.job.manage.dao.notify.impl; -import com.tencent.bk.job.common.RequestIdLogger; -import com.tencent.bk.job.common.util.SimpleRequestIdLogger; -import com.tencent.bk.job.manage.common.consts.notify.ExecuteStatusEnum; -import com.tencent.bk.job.manage.common.consts.notify.JobRoleEnum; -import com.tencent.bk.job.manage.common.consts.notify.NotifyConsts; -import com.tencent.bk.job.manage.common.consts.notify.ResourceTypeEnum; -import com.tencent.bk.job.manage.common.consts.notify.TriggerTypeEnum; +import com.tencent.bk.job.common.model.dto.notify.CustomNotifyDTO; +import com.tencent.bk.job.manage.api.common.constants.notify.ExecuteStatusEnum; +import com.tencent.bk.job.manage.api.common.constants.notify.JobRoleEnum; +import com.tencent.bk.job.manage.api.common.constants.notify.NotifyConsts; +import com.tencent.bk.job.manage.api.common.constants.notify.ResourceTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.notify.TriggerTypeEnum; import com.tencent.bk.job.manage.dao.notify.NotifyConfigStatusDAO; import com.tencent.bk.job.manage.dao.notify.NotifyPolicyRoleTargetDAO; import com.tencent.bk.job.manage.dao.notify.NotifyRoleTargetChannelDAO; @@ -38,19 +37,21 @@ import com.tencent.bk.job.manage.model.dto.notify.NotifyPolicyRoleTargetDTO; import com.tencent.bk.job.manage.model.dto.notify.NotifyRoleTargetChannelDTO; import com.tencent.bk.job.manage.model.dto.notify.NotifyTriggerPolicyDTO; +import com.tencent.bk.job.manage.model.tables.NotifyTriggerPolicy; +import com.tencent.bk.job.manage.model.tables.records.NotifyTriggerPolicyRecord; import com.tencent.bk.job.manage.model.web.vo.notify.TriggerPolicyVO; +import lombok.extern.slf4j.Slf4j; import lombok.val; import lombok.var; import org.jooq.Condition; import org.jooq.DSLContext; import org.jooq.Record; +import org.jooq.Record1; import org.jooq.Result; import org.jooq.conf.ParamType; -import org.jooq.generated.tables.NotifyTriggerPolicy; -import org.jooq.generated.tables.records.NotifyTriggerPolicyRecord; import org.jooq.types.ULong; -import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; import java.util.ArrayList; @@ -60,16 +61,9 @@ import java.util.Map; import java.util.stream.Collectors; -/** - * @Description - * @Date 2020/1/2 - * @Version 1.0 - */ @Repository +@Slf4j public class NotifyTriggerPolicyDAOImpl implements NotifyTriggerPolicyDAO { - - private static final RequestIdLogger logger = - new SimpleRequestIdLogger(LoggerFactory.getLogger(NotifyTriggerPolicyDAOImpl.class)); private static final NotifyTriggerPolicy T_NOTIFY_TRIGGER_POLICY = NotifyTriggerPolicy.NOTIFY_TRIGGER_POLICY; private static final NotifyTriggerPolicy defaultTable = T_NOTIFY_TRIGGER_POLICY; @@ -79,7 +73,7 @@ public class NotifyTriggerPolicyDAOImpl implements NotifyTriggerPolicyDAO { private final NotifyConfigStatusDAO notifyConfigStatusDAO; @Autowired - public NotifyTriggerPolicyDAOImpl(DSLContext dslContext, + public NotifyTriggerPolicyDAOImpl(@Qualifier("job-manage-dsl-context") DSLContext dslContext, NotifyPolicyRoleTargetDAO notifyPolicyRoleTargetDAO, NotifyRoleTargetChannelDAO notifyRoleTargetChannelDAO, NotifyConfigStatusDAO notifyConfigStatusDAO @@ -121,27 +115,28 @@ public Long insertNotifyTriggerPolicy(NotifyTriggerPolicyDTO notifyTriggerPolicy assert record != null; return record.get(T_NOTIFY_TRIGGER_POLICY.ID); } catch (Exception e) { - logger.errorWithRequestId(sql); + log.error(sql); throw e; } } - private int cascadeDelete(Result records) { + private int cascadeDelete(Result> records) { if (null == records || records.isEmpty()) { return 0; } //1.删从表 - records.forEach(record -> notifyPolicyRoleTargetDAO.deleteByPolicyId(dslContext, record.getId())); + records.forEach(record -> notifyPolicyRoleTargetDAO.deleteByPolicyId(record.get(defaultTable.ID))); //2.删主表 return dslContext.deleteFrom(defaultTable).where( - defaultTable.ID.in(records.map(NotifyTriggerPolicyRecord::getId)) + defaultTable.ID.in(records.map(record -> record.get(defaultTable.ID))) ).execute(); } @Override public int deleteAppNotifyPolicies(Long appId, String triggerUser) { // 1.查出所有记录 - val records = dslContext.selectFrom(defaultTable) + Result> records = dslContext.select(defaultTable.ID) + .from(defaultTable) .where(defaultTable.TRIGGER_USER.eq(triggerUser)) .and(defaultTable.APP_ID.eq(appId)) .and(defaultTable.RESOURCE_ID.eq(NotifyConsts.DEFAULT_RESOURCE_ID)) @@ -151,7 +146,49 @@ public int deleteAppNotifyPolicies(Long appId, String triggerUser) { } @Override - public List list(String triggerUser, Long appId, String resourceId) { + public int deleteAppResourceNotifyPolicies(Long appId, Integer resourceType, String resourceId) { + + Result> records = dslContext.select(defaultTable.ID) + .from(defaultTable) + .where(defaultTable.APP_ID.eq(appId)) + .and(defaultTable.RESOURCE_TYPE.eq(resourceType.byteValue())) + .and(defaultTable.RESOURCE_ID.eq(resourceId)) + .fetch(); + return cascadeDelete(records); + } + + @Override + public CustomNotifyDTO getSpecificResourceNotifyPolicy(Long appId, + Integer resourceType, + String resourceId, + Integer triggerType) { + var records = dslContext.selectFrom(defaultTable) + .where(defaultTable.APP_ID.eq(appId)) + .and(defaultTable.RESOURCE_TYPE.eq(resourceType.byteValue())) + .and(defaultTable.RESOURCE_ID.eq(resourceId)) + .and(defaultTable.TRIGGER_TYPE.eq(triggerType.byteValue())) + .fetch(); + if (records.isEmpty()) { + return null; + } + return extractCustomNotifyDTOFromTriggerPolicyVO( + getTriggerPolicyVO(records, TriggerTypeEnum.get(triggerType)) + ); + } + + private CustomNotifyDTO extractCustomNotifyDTOFromTriggerPolicyVO(TriggerPolicyVO triggerPolicyVO) { + if (triggerPolicyVO == null) { + return null; + } + CustomNotifyDTO notifyDTO = new CustomNotifyDTO(); + notifyDTO.setRoleList(triggerPolicyVO.getRoleList()); + notifyDTO.setExtraObserverList(triggerPolicyVO.getExtraObserverList()); + notifyDTO.buildCustomNotifyChannel(triggerPolicyVO.getResourceStatusChannelMap()); + return notifyDTO; + } + + @Override + public List listAppDefault(String triggerUser, Long appId, String resourceId) { val resultList = new ArrayList(); var records = dslContext.selectFrom(defaultTable) .where(defaultTable.TRIGGER_USER.eq(triggerUser)) @@ -160,14 +197,14 @@ public List list(String triggerUser, Long appId, String resourc .fetch(); if (records.isEmpty()) { if (!notifyConfigStatusDAO.exist(triggerUser, appId)) { - logger.warn(triggerUser + "未在业务(id=" + appId + ")下配置消息通知策略，采用业务无关通用默认策略"); + log.warn(triggerUser + "未在业务(id=" + appId + ")下配置消息通知策略，采用业务无关通用默认策略"); records = dslContext.selectFrom(defaultTable) .where(defaultTable.TRIGGER_USER.eq(NotifyConsts.DEFAULT_TRIGGER_USER)) .and(defaultTable.APP_ID.eq(NotifyConsts.DEFAULT_APP_ID)) .and(defaultTable.RESOURCE_ID.eq(resourceId)) .fetch(); if (records.isEmpty()) { - logger.info("业务无关通用默认策略未配置"); + log.info("业务无关通用默认策略未配置"); } } else { //已配置为不发送任何通知 @@ -186,8 +223,12 @@ public List list(String triggerUser, Long appId, String resourc } @Override - public List list(String triggerUser, Long appId, String resourceId - , Integer resourceType, Integer triggerType, Integer executeStatus) { + public List list(String triggerUser, + Long appId, + String resourceId, + Integer resourceType, + Integer triggerType, + Integer executeStatus) { List resultList; List conditions = new ArrayList<>(); if (null != triggerUser) { @@ -268,7 +309,7 @@ private TriggerPolicyVO getTriggerPolicyVO(List recor } }); if (currentTriggerTypeRecords.isEmpty()) { - logger.info(triggerType.getDefaultName() + " Default Policy not configed"); + log.info(triggerType.getDefaultName() + " Default Policy not configed"); //返回空数据 return getEmptyTriggerPolicyVO(triggerType); } @@ -283,7 +324,7 @@ private TriggerPolicyVO getTriggerPolicyVO(List recor //随机取一条记录即可，所有记录对应的通知对象是一样的 val triggerPolicyRecord = currentTriggerTypeRecords.get(0); List roleTargetDTOList = - notifyPolicyRoleTargetDAO.listByPolicyId(dslContext, triggerPolicyRecord.getId()); + notifyPolicyRoleTargetDAO.listByPolicyId(triggerPolicyRecord.getId()); roleTargetDTOList.forEach(it -> { if (it.isEnable()) { val appRole = it.getRole(); @@ -302,11 +343,10 @@ private TriggerPolicyVO getTriggerPolicyVO(List recor }).collect(Collectors.toList()); executeStatusDefaultRecords.forEach(it -> { List roleTargets = - notifyPolicyRoleTargetDAO.listByPolicyId(dslContext, it.getId()); + notifyPolicyRoleTargetDAO.listByPolicyId(it.getId()); if (roleTargets.size() > 0) { List roleTargetChannelDTOList = - notifyRoleTargetChannelDAO.listByRoleTargetId(dslContext, - roleTargets.get(0).getId()); + notifyRoleTargetChannelDAO.listByRoleTargetId(roleTargets.get(0).getId()); resourceStatusChannelMap.put( ExecuteStatusEnum.getName(it.getExecuteStatus()), roleTargetChannelDTOList.stream().map(NotifyRoleTargetChannelDTO::getChannel) diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/plan/TaskPlanDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/plan/TaskPlanDAO.java index 5532a41e2b..d43eb8c900 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/plan/TaskPlanDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/plan/TaskPlanDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,7 +26,7 @@ import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; -import com.tencent.bk.job.manage.common.consts.task.TaskPlanTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskPlanTypeEnum; import com.tencent.bk.job.manage.model.dto.TaskPlanQueryDTO; import com.tencent.bk.job.manage.model.dto.task.TaskPlanBasicInfoDTO; import com.tencent.bk.job.manage.model.dto.task.TaskPlanInfoDTO; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/plan/impl/TaskPlanApprovalStepDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/plan/impl/TaskPlanApprovalStepDAOImpl.java index 6600f98997..a00b3f699f 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/plan/impl/TaskPlanApprovalStepDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/plan/impl/TaskPlanApprovalStepDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -30,15 +30,15 @@ import com.tencent.bk.job.manage.common.util.DbRecordMapper; import com.tencent.bk.job.manage.dao.TaskApprovalStepDAO; import com.tencent.bk.job.manage.model.dto.task.TaskApprovalStepDTO; +import com.tencent.bk.job.manage.model.tables.TaskPlan; +import com.tencent.bk.job.manage.model.tables.TaskPlanStep; +import com.tencent.bk.job.manage.model.tables.TaskPlanStepApproval; +import com.tencent.bk.job.manage.model.tables.records.TaskPlanStepApprovalRecord; import lombok.extern.slf4j.Slf4j; import org.jooq.Condition; import org.jooq.DSLContext; import org.jooq.Record6; import org.jooq.Result; -import org.jooq.generated.tables.TaskPlan; -import org.jooq.generated.tables.TaskPlanStep; -import org.jooq.generated.tables.TaskPlanStepApproval; -import org.jooq.generated.tables.records.TaskPlanStepApprovalRecord; import org.jooq.types.UByte; import org.jooq.types.ULong; import org.springframework.beans.factory.annotation.Autowired; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/plan/impl/TaskPlanDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/plan/impl/TaskPlanDAOImpl.java index 1593ae998a..5c24923963 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/plan/impl/TaskPlanDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/plan/impl/TaskPlanDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,12 +27,14 @@ import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; import com.tencent.bk.job.common.util.json.JsonMapper; -import com.tencent.bk.job.manage.common.consts.task.TaskPlanTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskPlanTypeEnum; import com.tencent.bk.job.manage.common.util.DbRecordMapper; import com.tencent.bk.job.manage.dao.plan.TaskPlanDAO; import com.tencent.bk.job.manage.model.dto.TaskPlanQueryDTO; import com.tencent.bk.job.manage.model.dto.task.TaskPlanBasicInfoDTO; import com.tencent.bk.job.manage.model.dto.task.TaskPlanInfoDTO; +import com.tencent.bk.job.manage.model.tables.TaskPlan; +import com.tencent.bk.job.manage.model.tables.records.TaskPlanRecord; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -46,8 +48,6 @@ import org.jooq.SelectJoinStep; import org.jooq.TableField; import org.jooq.UpdateSetMoreStep; -import org.jooq.generated.tables.TaskPlan; -import org.jooq.generated.tables.records.TaskPlanRecord; import org.jooq.types.UByte; import org.jooq.types.ULong; import org.springframework.beans.factory.annotation.Autowired; @@ -152,7 +152,7 @@ public PageData listPageTaskPlans(TaskPlanQueryDTO taskPlanQuer if (CollectionUtils.isNotEmpty(excludePlanIdList)) { conditions.add( - TABLE.ID.notIn(excludePlanIdList.parallelStream().map(ULong::valueOf).collect(Collectors.toList()))); + TABLE.ID.notIn(excludePlanIdList.stream().map(ULong::valueOf).collect(Collectors.toList()))); } long count = getPageTaskPlanCount(conditions); @@ -224,7 +224,7 @@ private List buildTaskPlanQueryCondition(TaskPlanQueryDTO taskPlanQue taskPlanQuery.getTemplateIdList().add(taskPlanQuery.getTemplateId()); } if (CollectionUtils.isNotEmpty(taskPlanQuery.getTemplateIdList())) { - conditions.add(TABLE.TEMPLATE_ID.in(taskPlanQuery.getTemplateIdList().parallelStream().map(ULong::valueOf).collect(Collectors.toList()))); + conditions.add(TABLE.TEMPLATE_ID.in(taskPlanQuery.getTemplateIdList().stream().map(ULong::valueOf).collect(Collectors.toList()))); } if (taskPlanQuery.getName() != null) { conditions.add(TABLE.NAME.like("%" + taskPlanQuery.getName() + "%")); @@ -370,12 +370,12 @@ public List listTaskPlanByIds(Long appId, List planIdList if (taskPlanQuery == null) { conditions.add(TABLE.APP_ID.eq(ULong.valueOf(appId))); } - conditions.add(TABLE.ID.in(planIdList.parallelStream().map(ULong::valueOf).collect(Collectors.toList()))); + conditions.add(TABLE.ID.in(planIdList.stream().map(ULong::valueOf).collect(Collectors.toList()))); Result> result = context.select(TABLE.ID, TABLE.APP_ID, TABLE.TEMPLATE_ID, TABLE.TYPE, TABLE.NAME, TABLE.CREATOR, - TABLE.CREATE_TIME, TABLE.LAST_MODIFY_USER, TABLE.LAST_MODIFY_TIME, TABLE.FIRST_STEP_ID, - TABLE.LAST_STEP_ID, TABLE.VERSION, TABLE.IS_LATEST_VERSION).from(TABLE) + TABLE.CREATE_TIME, TABLE.LAST_MODIFY_USER, TABLE.LAST_MODIFY_TIME, TABLE.FIRST_STEP_ID, + TABLE.LAST_STEP_ID, TABLE.VERSION, TABLE.IS_LATEST_VERSION).from(TABLE) .where(conditions).fetch(); List taskPlanInfoList = new ArrayList<>(); @@ -391,13 +391,13 @@ public List listTaskPlanBasicInfoByIds(Collection pl List conditions = buildTaskPlanIdsCondition(planIds); Result> result = context.select( - TABLE.ID, - TABLE.NAME, - TABLE.VERSION, - TABLE.APP_ID, - TABLE.TEMPLATE_ID, - TABLE.TYPE - ).from(TABLE) + TABLE.ID, + TABLE.NAME, + TABLE.VERSION, + TABLE.APP_ID, + TABLE.TEMPLATE_ID, + TABLE.TYPE + ).from(TABLE) .where(conditions).fetch(); List taskPlanBasicInfoList = new ArrayList<>(); @@ -418,8 +418,9 @@ public boolean checkPlanName(Long appId, Long templateId, Long planId, String na conditions.add(TABLE.ID.notEqual(ULong.valueOf(planId))); } conditions.add(TABLE.NAME.eq(name)); - return context.selectCount().from(TABLE).where(conditions).fetchOne(0, Integer.class) == 0; + conditions.add(TABLE.TYPE.eq(UByte.valueOf(TaskPlanTypeEnum.NORMAL.getType()))); + return !context.fetchExists(TABLE, conditions); } @Override @@ -520,7 +521,7 @@ public List listAllPlanId() { conditions.add(TABLE.IS_DELETED.equal(UByte.valueOf(0))); List uLongPlanId = context.select(TABLE.ID).from(TABLE).where(conditions).fetch(TABLE.ID); if (CollectionUtils.isNotEmpty(uLongPlanId)) { - return uLongPlanId.parallelStream().map(ULong::longValue).collect(Collectors.toList()); + return uLongPlanId.stream().map(ULong::longValue).collect(Collectors.toList()); } else { return new ArrayList<>(); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/plan/impl/TaskPlanFavoriteDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/plan/impl/TaskPlanFavoriteDAOImpl.java index bd9295d4e3..b25c3b76e6 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/plan/impl/TaskPlanFavoriteDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/plan/impl/TaskPlanFavoriteDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,12 +25,12 @@ package com.tencent.bk.job.manage.dao.plan.impl; import com.tencent.bk.job.manage.dao.TaskFavoriteDAO; +import com.tencent.bk.job.manage.model.tables.TaskFavoritePlan; import lombok.extern.slf4j.Slf4j; import org.jooq.Condition; import org.jooq.DSLContext; import org.jooq.Record1; import org.jooq.Result; -import org.jooq.generated.tables.TaskFavoritePlan; import org.jooq.types.UByte; import org.jooq.types.ULong; import org.springframework.beans.factory.annotation.Autowired; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/plan/impl/TaskPlanFileInfoDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/plan/impl/TaskPlanFileInfoDAOImpl.java index 407c071177..6b8620b3c8 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/plan/impl/TaskPlanFileInfoDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/plan/impl/TaskPlanFileInfoDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,19 +26,22 @@ import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.exception.InternalException; +import com.tencent.bk.job.common.mysql.util.JooqDataTypeUtil; import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.manage.common.util.DbRecordMapper; import com.tencent.bk.job.manage.dao.TaskFileInfoDAO; import com.tencent.bk.job.manage.model.dto.task.TaskFileInfoDTO; +import com.tencent.bk.job.manage.model.dto.task.TaskTargetDTO; +import com.tencent.bk.job.manage.model.tables.TaskPlanStepFileList; +import com.tencent.bk.job.manage.model.tables.records.TaskPlanStepFileListRecord; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.jooq.Condition; import org.jooq.DSLContext; import org.jooq.InsertValuesStep8; import org.jooq.Record9; import org.jooq.Result; -import org.jooq.generated.tables.TaskPlanStepFileList; -import org.jooq.generated.tables.records.TaskPlanStepFileListRecord; import org.jooq.types.UByte; import org.jooq.types.ULong; import org.springframework.beans.factory.annotation.Autowired; @@ -130,7 +133,7 @@ public long insertFileInfo(TaskFileInfoDTO fileInfo) { JsonUtils.toJson(fileInfo.getFileLocation()), DbRecordMapper.getJooqLongValue(fileInfo.getFileSize()), fileInfo.getFileHash(), - fileInfo.getHost() == null ? null : fileInfo.getHost().toString(), + fileInfo.getHost() == null ? null : fileInfo.getHost().toJsonString(), DbRecordMapper.getJooqLongValue(fileInfo.getHostAccount()), fileInfo.getFileSourceId() ) @@ -154,7 +157,7 @@ public List batchInsertFileInfo(List fileInfoList) { JsonUtils.toJson(fileInfo.getFileLocation()), DbRecordMapper.getJooqLongValue(fileInfo.getFileSize()), fileInfo.getFileHash(), - fileInfo.getHost() == null ? null : fileInfo.getHost().toString(), + fileInfo.getHost() == null ? null : fileInfo.getHost().toJsonString(), DbRecordMapper.getJooqLongValue(fileInfo.getHostAccount()), fileInfo.getFileSourceId() )); @@ -205,11 +208,37 @@ public List listLocalFileByStepId(List stepIdList) { return new ArrayList<>(); } - List uLongStepIdList = stepIdList.parallelStream().map(ULong::valueOf).collect(Collectors.toList()); + List uLongStepIdList = stepIdList.stream().map(ULong::valueOf).collect(Collectors.toList()); List conditions = new ArrayList<>(); conditions.add(TABLE.STEP_ID.in(uLongStepIdList)); conditions.add(TABLE.FILE_TYPE.equal(UByte.valueOf(2))); return context.select(TABLE.FILE_LOCATION).from(TABLE).where(conditions).fetch(TABLE.FILE_LOCATION); } + + @Override + public Map listStepFileHosts() { + Result result = context.select(TABLE.ID, TABLE.HOST).fetch(); + Map stepTargets = new HashMap<>(); + if (result.isNotEmpty()) { + result.forEach(record -> { + Long recordId = record.get(TABLE.ID).longValue(); + TaskTargetDTO target = TaskTargetDTO.fromJsonString(record.get(TABLE.HOST)); + stepTargets.put(recordId, target); + }); + } + return stepTargets; + } + + @Override + public boolean updateStepFileHosts(Long recordId, String value) { + if (StringUtils.isBlank(value)) { + return false; + } + int result = context.update(TABLE) + .set(TABLE.HOST, value) + .where(TABLE.ID.eq(JooqDataTypeUtil.buildULong(recordId))) + .execute(); + return result == 1; + } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/plan/impl/TaskPlanFileStepDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/plan/impl/TaskPlanFileStepDAOImpl.java index 9a0c64c9bf..c611b37aa0 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/plan/impl/TaskPlanFileStepDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/plan/impl/TaskPlanFileStepDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,20 +26,23 @@ import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.exception.InternalException; -import com.tencent.bk.job.manage.common.consts.task.TaskFileTypeEnum; +import com.tencent.bk.job.common.mysql.util.JooqDataTypeUtil; +import com.tencent.bk.job.manage.api.common.constants.task.TaskFileTypeEnum; import com.tencent.bk.job.manage.common.util.DbRecordMapper; import com.tencent.bk.job.manage.dao.TaskFileStepDAO; import com.tencent.bk.job.manage.model.dto.task.TaskFileStepDTO; +import com.tencent.bk.job.manage.model.dto.task.TaskTargetDTO; +import com.tencent.bk.job.manage.model.tables.TaskPlan; +import com.tencent.bk.job.manage.model.tables.TaskPlanStep; +import com.tencent.bk.job.manage.model.tables.TaskPlanStepFile; +import com.tencent.bk.job.manage.model.tables.TaskPlanStepFileList; +import com.tencent.bk.job.manage.model.tables.records.TaskPlanStepFileRecord; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.jooq.Condition; import org.jooq.DSLContext; import org.jooq.Record11; import org.jooq.Result; -import org.jooq.generated.tables.TaskPlan; -import org.jooq.generated.tables.TaskPlanStep; -import org.jooq.generated.tables.TaskPlanStepFile; -import org.jooq.generated.tables.TaskPlanStepFileList; -import org.jooq.generated.tables.records.TaskPlanStepFileRecord; import org.jooq.types.UByte; import org.jooq.types.ULong; import org.springframework.beans.factory.annotation.Autowired; @@ -116,7 +119,7 @@ public long insertFileStep(TaskFileStepDTO fileStep) { TABLE.DUPLICATE_HANDLER, TABLE.NOT_EXIST_PATH_HANDLER) .values(ULong.valueOf(fileStep.getStepId()), fileStep.getDestinationFileLocation(), ULong.valueOf(fileStep.getExecuteAccount()), - fileStep.getDestinationHostList() == null ? null : fileStep.getDestinationHostList().toString(), + fileStep.getDestinationHostList() == null ? null : fileStep.getDestinationHostList().toJsonString(), fileStep.getTimeout() == null ? ULong.valueOf(0) : ULong.valueOf(fileStep.getTimeout()), fileStep.getOriginSpeedLimit() == null ? null : ULong.valueOf(fileStep.getOriginSpeedLimit()), fileStep.getTargetSpeedLimit() == null ? null : ULong.valueOf(fileStep.getTargetSpeedLimit()), @@ -157,4 +160,32 @@ public int countFileSteps(Long appId, TaskFileTypeEnum fileType) { .join(tableTaskPlan).on(tableTTStep.PLAN_ID.eq(tableTaskPlan.ID)) .where(conditions).fetchOne().value1(); } + + @Override + public Map listStepTargets() { + Result result = context.select(TABLE.ID, TABLE.DESTINATION_HOST_LIST).fetch(); + Map stepTargets = new HashMap<>(); + if (result.isNotEmpty()) { + result.forEach(record -> { + Long recordId = record.get(TABLE.ID).longValue(); + TaskTargetDTO target = TaskTargetDTO.fromJsonString(record.get(TABLE.DESTINATION_HOST_LIST)); + stepTargets.put(recordId, target); + }); + } + return stepTargets; + } + + @Override + public boolean updateStepTargets(Long recordId, String value) { + if (StringUtils.isBlank(value)) { + return false; + } + int result = context.update(TABLE) + .set(TABLE.DESTINATION_HOST_LIST, value) + .where(TABLE.ID.eq(JooqDataTypeUtil.buildULong(recordId))) + .execute(); + return result == 1; + } + + } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/plan/impl/TaskPlanScriptStepDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/plan/impl/TaskPlanScriptStepDAOImpl.java index 7d16285a81..60c79b5376 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/plan/impl/TaskPlanScriptStepDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/plan/impl/TaskPlanScriptStepDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,21 +25,24 @@ package com.tencent.bk.job.manage.dao.plan.impl; import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.crypto.scenario.SensitiveParamCryptoService; import com.tencent.bk.job.common.exception.InternalException; -import com.tencent.bk.job.manage.common.consts.task.TaskScriptSourceEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskTypeEnum; -import com.tencent.bk.job.manage.common.util.DbRecordMapper; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskScriptSourceEnum; import com.tencent.bk.job.manage.dao.TaskScriptStepDAO; import com.tencent.bk.job.manage.model.dto.task.TaskScriptStepDTO; +import com.tencent.bk.job.manage.model.dto.task.TaskTargetDTO; +import com.tencent.bk.job.manage.model.tables.TaskPlan; +import com.tencent.bk.job.manage.model.tables.TaskPlanStep; +import com.tencent.bk.job.manage.model.tables.TaskPlanStepScript; +import com.tencent.bk.job.manage.model.tables.records.TaskPlanStepScriptRecord; import lombok.extern.slf4j.Slf4j; import org.jooq.Condition; import org.jooq.DSLContext; +import org.jooq.Record1; import org.jooq.Record15; +import org.jooq.Record16; import org.jooq.Result; -import org.jooq.generated.tables.TaskPlan; -import org.jooq.generated.tables.TaskPlanStep; -import org.jooq.generated.tables.TaskPlanStepScript; -import org.jooq.generated.tables.records.TaskPlanStepScriptRecord; import org.jooq.impl.DSL; import org.jooq.types.UByte; import org.jooq.types.ULong; @@ -65,11 +68,47 @@ public class TaskPlanScriptStepDAOImpl implements TaskScriptStepDAO { private static final TaskPlanStep tableTTStep = TaskPlanStep.TASK_PLAN_STEP; private static final TaskPlanStepScript tableTTStepScript = TaskPlanStepScript.TASK_PLAN_STEP_SCRIPT; - private DSLContext context; + private final DSLContext context; + private final SensitiveParamCryptoService sensitiveParamCryptoService; @Autowired - public TaskPlanScriptStepDAOImpl(@Qualifier("job-manage-dsl-context") DSLContext context) { + public TaskPlanScriptStepDAOImpl(@Qualifier("job-manage-dsl-context") DSLContext context, + SensitiveParamCryptoService sensitiveParamCryptoService) { this.context = context; + this.sensitiveParamCryptoService = sensitiveParamCryptoService; + } + + private TaskScriptStepDTO convertRecordToTaskScriptStep(Record16 record) { + if (record == null) { + return null; + } + TaskScriptStepDTO taskScriptStep = new TaskScriptStepDTO(); + taskScriptStep.setId(record.get(TABLE.ID).longValue()); + taskScriptStep.setPlanId((record.get(TABLE.PLAN_ID)).longValue()); + taskScriptStep.setStepId((record.get(TABLE.STEP_ID)).longValue()); + taskScriptStep.setScriptSource(TaskScriptSourceEnum.valueOf((record.get(TABLE.SCRIPT_TYPE)).intValue())); + taskScriptStep.setScriptId(record.get(TABLE.SCRIPT_ID)); + if (record.get(TABLE.SCRIPT_VERSION_ID) != null) { + taskScriptStep.setScriptVersionId((record.get(TABLE.SCRIPT_VERSION_ID)).longValue()); + } + taskScriptStep.setContent(record.get(TABLE.CONTENT)); + taskScriptStep.setLanguage(ScriptTypeEnum.valOf((record.get(TABLE.LANGUAGE)).intValue())); + taskScriptStep.setTimeout((record.get(TABLE.SCRIPT_TIMEOUT)).longValue()); + taskScriptStep.setAccount((record.get(TABLE.EXECUTE_ACCOUNT)).longValue()); + taskScriptStep.setExecuteTarget(TaskTargetDTO.fromJsonString(record.get(TABLE.DESTINATION_HOST_LIST))); + taskScriptStep.setSecureParam((record.get(TABLE.IS_SECURE_PARAM)).intValue() == 1); + String encryptedScriptParam = record.get(TABLE.SCRIPT_PARAM); + + // 敏感参数解密 + taskScriptStep.setScriptParam(sensitiveParamCryptoService.decryptParamIfNeeded( + taskScriptStep.getSecureParam(), encryptedScriptParam + )); + + taskScriptStep.setStatus((record.get(TABLE.STATUS)).intValue()); + taskScriptStep.setIgnoreError((record.get(TABLE.IGNORE_ERROR)).intValue() == 1); + taskScriptStep.setWindowsInterpreter(record.get(TABLE.WINDOWS_INTERPRETER)); + return taskScriptStep; } @Override @@ -77,20 +116,33 @@ public List listScriptStepByParentId(long parentId) { List conditions = new ArrayList<>(); conditions.add(TABLE.PLAN_ID.eq(ULong.valueOf(parentId))); Result< - Record15> result = + Record16> result = context - .select(TABLE.ID, TABLE.PLAN_ID, TABLE.STEP_ID, TABLE.SCRIPT_TYPE, TABLE.SCRIPT_ID, - TABLE.SCRIPT_VERSION_ID, TABLE.CONTENT, TABLE.LANGUAGE, TABLE.SCRIPT_PARAM, - TABLE.SCRIPT_TIMEOUT, TABLE.EXECUTE_ACCOUNT, TABLE.DESTINATION_HOST_LIST, - TABLE.IS_SECURE_PARAM, TABLE.STATUS, TABLE.IGNORE_ERROR) - .from(TABLE).where(conditions).fetch(); + .select( + TABLE.ID, + TABLE.PLAN_ID, + TABLE.STEP_ID, + TABLE.SCRIPT_TYPE, + TABLE.SCRIPT_ID, + TABLE.SCRIPT_VERSION_ID, + TABLE.CONTENT, + TABLE.LANGUAGE, + TABLE.SCRIPT_PARAM, + TABLE.SCRIPT_TIMEOUT, + TABLE.EXECUTE_ACCOUNT, + TABLE.DESTINATION_HOST_LIST, + TABLE.IS_SECURE_PARAM, + TABLE.STATUS, + TABLE.IGNORE_ERROR, + TABLE.WINDOWS_INTERPRETER + ).from(TABLE).where(conditions).fetch(); List taskScriptStepList = new ArrayList<>(); - if (result != null && result.size() >= 1) { + if (result.size() >= 1) { result.map(record -> taskScriptStepList - .add(DbRecordMapper.convertRecordToTaskScriptStep(record, TaskTypeEnum.PLAN))); + .add(convertRecordToTaskScriptStep(record))); } return taskScriptStepList; } @@ -100,20 +152,35 @@ public Map listScriptStepByIds(List stepIdList) { List conditions = new ArrayList<>(); conditions.add(TABLE.STEP_ID.in(stepIdList.stream().map(ULong::valueOf).collect(Collectors.toList()))); Result< - Record15> result = + Record16> result = context - .select(TABLE.ID, TABLE.PLAN_ID, TABLE.STEP_ID, TABLE.SCRIPT_TYPE, TABLE.SCRIPT_ID, - TABLE.SCRIPT_VERSION_ID, TABLE.CONTENT, TABLE.LANGUAGE, TABLE.SCRIPT_PARAM, - TABLE.SCRIPT_TIMEOUT, TABLE.EXECUTE_ACCOUNT, TABLE.DESTINATION_HOST_LIST, - TABLE.IS_SECURE_PARAM, TABLE.STATUS, TABLE.IGNORE_ERROR) - .from(TABLE).where(conditions).fetch(); + .select( + TABLE.ID, + TABLE.PLAN_ID, + TABLE.STEP_ID, + TABLE.SCRIPT_TYPE, + TABLE.SCRIPT_ID, + TABLE.SCRIPT_VERSION_ID, + TABLE.CONTENT, + TABLE.LANGUAGE, + TABLE.SCRIPT_PARAM, + TABLE.SCRIPT_TIMEOUT, + TABLE.EXECUTE_ACCOUNT, + TABLE.DESTINATION_HOST_LIST, + TABLE.IS_SECURE_PARAM, + TABLE.STATUS, + TABLE.IGNORE_ERROR, + TABLE.WINDOWS_INTERPRETER + ).from(TABLE).where(conditions).fetch(); Map taskScriptStepMap = new HashMap<>(stepIdList.size()); - if (result != null && result.size() >= 1) { - result.map(record -> taskScriptStepMap.put(record.get(TABLE.STEP_ID).longValue(), - DbRecordMapper.convertRecordToTaskScriptStep(record, TaskTypeEnum.PLAN))); + if (result.size() >= 1) { + result.map(record -> taskScriptStepMap.put( + record.get(TABLE.STEP_ID).longValue(), + convertRecordToTaskScriptStep(record)) + ); } return taskScriptStepMap; } @@ -122,13 +189,27 @@ public Map listScriptStepByIds(List stepIdList) { public TaskScriptStepDTO getScriptStepById(long stepId) { List conditions = new ArrayList<>(); conditions.add(TABLE.STEP_ID.eq(ULong.valueOf(stepId))); - Record15 record = context.select(TABLE.ID, TABLE.PLAN_ID, TABLE.STEP_ID, TABLE.SCRIPT_TYPE, TABLE.SCRIPT_ID, - TABLE.SCRIPT_VERSION_ID, TABLE.CONTENT, TABLE.LANGUAGE, TABLE.SCRIPT_PARAM, TABLE.SCRIPT_TIMEOUT, - TABLE.EXECUTE_ACCOUNT, TABLE.DESTINATION_HOST_LIST, TABLE.IS_SECURE_PARAM, TABLE.STATUS, - TABLE.IGNORE_ERROR).from(TABLE).where(conditions).fetchOne(); + Record16 record = context.select( + TABLE.ID, + TABLE.PLAN_ID, + TABLE.STEP_ID, + TABLE.SCRIPT_TYPE, + TABLE.SCRIPT_ID, + TABLE.SCRIPT_VERSION_ID, + TABLE.CONTENT, + TABLE.LANGUAGE, + TABLE.SCRIPT_PARAM, + TABLE.SCRIPT_TIMEOUT, + TABLE.EXECUTE_ACCOUNT, + TABLE.DESTINATION_HOST_LIST, + TABLE.IS_SECURE_PARAM, + TABLE.STATUS, + TABLE.IGNORE_ERROR, + TABLE.WINDOWS_INTERPRETER + ).from(TABLE).where(conditions).fetchOne(); if (record != null) { - return DbRecordMapper.convertRecordToTaskScriptStep(record, TaskTypeEnum.PLAN); + return convertRecordToTaskScriptStep(record); } else { return null; } @@ -146,17 +227,41 @@ public long insertScriptStep(TaskScriptStepDTO scriptStep) { ignoreError = UByte.valueOf(1); } TaskPlanStepScriptRecord record = context.insertInto(TABLE) - .columns(TABLE.PLAN_ID, TABLE.STEP_ID, TABLE.SCRIPT_TYPE, TABLE.SCRIPT_ID, TABLE.SCRIPT_VERSION_ID, - TABLE.CONTENT, TABLE.LANGUAGE, TABLE.SCRIPT_PARAM, TABLE.SCRIPT_TIMEOUT, TABLE.EXECUTE_ACCOUNT, - TABLE.DESTINATION_HOST_LIST, TABLE.IS_SECURE_PARAM, TABLE.STATUS, TABLE.IGNORE_ERROR) - .values(ULong.valueOf(scriptStep.getPlanId()), ULong.valueOf(scriptStep.getStepId()), - UByte.valueOf(scriptStep.getScriptSource().getType()), scriptStep.getScriptId(), + .columns( + TABLE.PLAN_ID, + TABLE.STEP_ID, + TABLE.SCRIPT_TYPE, + TABLE.SCRIPT_ID, + TABLE.SCRIPT_VERSION_ID, + TABLE.CONTENT, + TABLE.LANGUAGE, + TABLE.SCRIPT_PARAM, + TABLE.SCRIPT_TIMEOUT, + TABLE.EXECUTE_ACCOUNT, + TABLE.DESTINATION_HOST_LIST, + TABLE.IS_SECURE_PARAM, + TABLE.STATUS, + TABLE.IGNORE_ERROR, + TABLE.WINDOWS_INTERPRETER + ).values( + ULong.valueOf(scriptStep.getPlanId()), + ULong.valueOf(scriptStep.getStepId()), + UByte.valueOf(scriptStep.getScriptSource().getType()), + scriptStep.getScriptId(), scriptStep.getScriptVersionId() == null ? null : ULong.valueOf(scriptStep.getScriptVersionId()), - scriptStep.getContent(), UByte.valueOf(scriptStep.getLanguage().getValue()), - scriptStep.getScriptParam(), ULong.valueOf(scriptStep.getTimeout()), - ULong.valueOf(scriptStep.getAccount()), scriptStep.getExecuteTarget().toString(), isSecureParam, - status, ignoreError) - .returning(TABLE.ID).fetchOne(); + scriptStep.getContent(), + UByte.valueOf(scriptStep.getLanguage().getValue()), + sensitiveParamCryptoService.encryptParamIfNeeded(scriptStep.getSecureParam(), + scriptStep.getScriptParam()), + ULong.valueOf(scriptStep.getTimeout()), + ULong.valueOf(scriptStep.getAccount()), + scriptStep.getExecuteTarget().toJsonString(), + isSecureParam, + status, + ignoreError, + scriptStep.getWindowsInterpreter() + ).returning(TABLE.ID).fetchOne(); + assert record != null; return record.getId().longValue(); } @@ -186,10 +291,12 @@ public int countScriptSteps(Long appId, TaskScriptSourceEnum scriptSource) { if (scriptSource != null) { conditions.add(tableTTStepScript.SCRIPT_TYPE.eq(UByte.valueOf(scriptSource.getType()))); } - return context.selectCount().from(tableTTStepScript) + Record1 record = context.selectCount().from(tableTTStepScript) .join(tableTTStep).on(tableTTStep.ID.eq(tableTTStepScript.STEP_ID)) .join(tableTaskPlan).on(tableTTStep.PLAN_ID.eq(tableTaskPlan.ID)) - .where(conditions).fetchOne().value1(); + .where(conditions).fetchOne(); + assert record != null; + return record.value1(); } @Override @@ -201,10 +308,12 @@ public int countScriptCitedByStepsByScriptIds(Long appId, List scriptIdL if (scriptIdList != null) { conditions.add(tableTTStepScript.SCRIPT_ID.in(scriptIdList)); } - return context.select(DSL.countDistinct(tableTTStepScript.SCRIPT_ID)).from(tableTTStepScript) + Record1 record = context.select(DSL.countDistinct(tableTTStepScript.SCRIPT_ID)).from(tableTTStepScript) .join(tableTTStep).on(tableTTStep.ID.eq(tableTTStepScript.STEP_ID)) .join(tableTaskPlan).on(tableTTStep.PLAN_ID.eq(tableTaskPlan.ID)) - .where(conditions).fetchOne().value1(); + .where(conditions).fetchOne(); + assert record != null; + return record.value1(); } @Override @@ -216,9 +325,11 @@ public int countScriptStepsByScriptIds(Long appId, List scriptIdList) { if (scriptIdList != null) { conditions.add(tableTTStepScript.SCRIPT_ID.in(scriptIdList)); } - return context.selectCount().from(tableTTStepScript) + Record1 record = context.selectCount().from(tableTTStepScript) .join(tableTTStep).on(tableTTStep.ID.eq(tableTTStepScript.STEP_ID)) .join(tableTaskPlan).on(tableTTStep.PLAN_ID.eq(tableTaskPlan.ID)) - .where(conditions).fetchOne().value1(); + .where(conditions).fetchOne(); + assert record != null; + return record.value1(); } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/plan/impl/TaskPlanStepDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/plan/impl/TaskPlanStepDAOImpl.java index 9e2cd5b9c2..779f68b14b 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/plan/impl/TaskPlanStepDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/plan/impl/TaskPlanStepDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,11 +24,13 @@ package com.tencent.bk.job.manage.dao.plan.impl; -import com.tencent.bk.job.manage.common.consts.task.TaskTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskTypeEnum; import com.tencent.bk.job.manage.common.util.DbRecordMapper; import com.tencent.bk.job.manage.dao.AbstractTaskStepDAO; import com.tencent.bk.job.manage.dao.TaskStepDAO; import com.tencent.bk.job.manage.model.dto.task.TaskStepDTO; +import com.tencent.bk.job.manage.model.tables.TaskPlanStep; +import com.tencent.bk.job.manage.model.tables.records.TaskPlanStepRecord; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.jooq.Condition; @@ -36,8 +38,6 @@ import org.jooq.Record11; import org.jooq.Result; import org.jooq.UpdateSetMoreStep; -import org.jooq.generated.tables.TaskPlanStep; -import org.jooq.generated.tables.records.TaskPlanStepRecord; import org.jooq.types.UByte; import org.jooq.types.ULong; import org.springframework.beans.factory.annotation.Autowired; @@ -176,7 +176,7 @@ public List listStepIdByParentId(List parentIdList) { return new ArrayList<>(); } - List uLongPlanIdList = parentIdList.parallelStream().map(ULong::valueOf).collect(Collectors.toList()); + List uLongPlanIdList = parentIdList.stream().map(ULong::valueOf).collect(Collectors.toList()); List conditions = new ArrayList<>(); conditions.add(TABLE.PLAN_ID.in(uLongPlanIdList)); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/plan/impl/TaskPlanVariableDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/plan/impl/TaskPlanVariableDAOImpl.java index 7b8cdd6d02..ecb3cfabae 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/plan/impl/TaskPlanVariableDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/plan/impl/TaskPlanVariableDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,9 +27,13 @@ import com.tencent.bk.job.common.constant.Bool; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.constant.TaskVariableTypeEnum; +import com.tencent.bk.job.common.crypto.scenario.CipherVariableCryptoService; import com.tencent.bk.job.common.exception.InternalException; +import com.tencent.bk.job.common.mysql.JobTransactional; import com.tencent.bk.job.manage.dao.TaskVariableDAO; import com.tencent.bk.job.manage.model.dto.task.TaskVariableDTO; +import com.tencent.bk.job.manage.model.tables.TaskPlanVariable; +import com.tencent.bk.job.manage.model.tables.records.TaskPlanVariableRecord; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -40,14 +44,11 @@ import org.jooq.Result; import org.jooq.TableField; import org.jooq.UpdateSetMoreStep; -import org.jooq.generated.tables.TaskPlanVariable; -import org.jooq.generated.tables.records.TaskPlanVariableRecord; import org.jooq.types.UByte; import org.jooq.types.ULong; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; -import org.springframework.transaction.annotation.Transactional; import java.util.ArrayList; import java.util.Collections; @@ -61,14 +62,17 @@ public class TaskPlanVariableDAOImpl implements TaskVariableDAO { private static final TaskPlanVariable TABLE = TaskPlanVariable.TASK_PLAN_VARIABLE; - private static final TableField[] ALL_FIELDS = {TABLE.TEMPLATE_VARIABLE_ID, TABLE.PLAN_ID, + private static final TableField[] ALL_FIELDS = {TABLE.TEMPLATE_VARIABLE_ID, TABLE.PLAN_ID, TABLE.NAME, TABLE.TYPE, TABLE.DEFAULT_VALUE, TABLE.DESCRIPTION, TABLE.IS_CHANGEABLE, TABLE.IS_REQUIRED}; private final DSLContext context; + private final CipherVariableCryptoService cipherVariableCryptoService; @Autowired - public TaskPlanVariableDAOImpl(@Qualifier("job-manage-dsl-context") DSLContext context) { + public TaskPlanVariableDAOImpl(@Qualifier("job-manage-dsl-context") DSLContext context, + CipherVariableCryptoService cipherVariableCryptoService) { this.context = context; + this.cipherVariableCryptoService = cipherVariableCryptoService; } @Override @@ -90,7 +94,13 @@ private TaskVariableDTO extract(Record record) { taskVariable.setPlanId(record.get(TABLE.PLAN_ID).longValue()); taskVariable.setName(record.get(TABLE.NAME)); taskVariable.setType(TaskVariableTypeEnum.valOf(record.get(TABLE.TYPE).intValue())); - taskVariable.setDefaultValue(record.get(TABLE.DEFAULT_VALUE)); + String encryptedDefaultValue = record.get(TABLE.DEFAULT_VALUE); + // 密文变量解密 + String defaultValue = cipherVariableCryptoService.decryptTaskVariableIfNeeded( + taskVariable.getType(), + encryptedDefaultValue + ); + taskVariable.setDefaultValue(defaultValue); taskVariable.setDescription(record.get(TABLE.DESCRIPTION)); taskVariable.setChangeable(Bool.isTrue(record.get(TABLE.IS_CHANGEABLE).byteValue())); taskVariable.setRequired(Bool.isTrue(record.get(TABLE.IS_REQUIRED).byteValue())); @@ -119,12 +129,26 @@ public TaskVariableDTO getVariableByName(long parentId, String varName) { @Override public long insertVariable(TaskVariableDTO variable) { TaskPlanVariableRecord record = context.insertInto(TABLE) - .columns(TABLE.TEMPLATE_VARIABLE_ID, TABLE.PLAN_ID, TABLE.NAME, TABLE.TYPE, TABLE.DEFAULT_VALUE, - TABLE.DESCRIPTION, TABLE.IS_CHANGEABLE, TABLE.IS_REQUIRED) - .values(ULong.valueOf(variable.getId()), ULong.valueOf(variable.getPlanId()), variable.getName(), - UByte.valueOf(variable.getType().getType()), variable.getDefaultValue(), variable.getDescription(), - getChangeable(variable.getChangeable()), getRequired(variable.getRequired())) - .returning(TABLE.TEMPLATE_VARIABLE_ID).fetchOne(); + .columns( + TABLE.TEMPLATE_VARIABLE_ID, + TABLE.PLAN_ID, + TABLE.NAME, + TABLE.TYPE, + TABLE.DEFAULT_VALUE, + TABLE.DESCRIPTION, + TABLE.IS_CHANGEABLE, + TABLE.IS_REQUIRED + ).values( + ULong.valueOf(variable.getId()), + ULong.valueOf(variable.getPlanId()), + variable.getName(), + UByte.valueOf(variable.getType().getType()), + cipherVariableCryptoService.encryptTaskVariableIfNeeded(variable.getType(), variable.getDefaultValue()), + variable.getDescription(), + getChangeable(variable.getChangeable()), + getRequired(variable.getRequired()) + ).returning(TABLE.TEMPLATE_VARIABLE_ID).fetchOne(); + assert record != null; return record.getTemplateVariableId().longValue(); } @@ -134,13 +158,29 @@ public List batchInsertVariables(List variableList) { return Collections.emptyList(); } InsertValuesStep8 insertStep = context.insertInto(TABLE).columns(TABLE.TEMPLATE_VARIABLE_ID, TABLE.PLAN_ID, - TABLE.NAME, TABLE.TYPE, TABLE.DEFAULT_VALUE, TABLE.DESCRIPTION, TABLE.IS_CHANGEABLE, TABLE.IS_REQUIRED); + UByte> insertStep = context.insertInto(TABLE) + .columns( + TABLE.TEMPLATE_VARIABLE_ID, + TABLE.PLAN_ID, + TABLE.NAME, + TABLE.TYPE, + TABLE.DEFAULT_VALUE, + TABLE.DESCRIPTION, + TABLE.IS_CHANGEABLE, + TABLE.IS_REQUIRED + ); - variableList.forEach(variable -> insertStep.values(ULong.valueOf(variable.getId()), - ULong.valueOf(variable.getPlanId()), variable.getName(), UByte.valueOf(variable.getType().getType()), - variable.getDefaultValue(), variable.getDescription(), getChangeable(variable.getChangeable()), - getRequired(variable.getRequired()))); + variableList.forEach(variable -> + insertStep.values( + ULong.valueOf(variable.getId()), + ULong.valueOf(variable.getPlanId()), + variable.getName(), + UByte.valueOf(variable.getType().getType()), + cipherVariableCryptoService.encryptTaskVariableIfNeeded(variable.getType(), variable.getDefaultValue()), + variable.getDescription(), + getChangeable(variable.getChangeable()), + getRequired(variable.getRequired()) + )); Result result = insertStep.returning(TABLE.TEMPLATE_VARIABLE_ID).fetch(); List variableIdList = new ArrayList<>(variableList.size()); @@ -161,15 +201,17 @@ public List batchInsertVariables(List variableList) { } @Override - public boolean updateVariableById(TaskVariableDTO variable) { + public boolean updateVarByParentResourceIdAndTplVarId(TaskVariableDTO variable) { List conditions = new ArrayList<>(); conditions.add(TABLE.TEMPLATE_VARIABLE_ID.eq(ULong.valueOf(variable.getId()))); conditions.add(TABLE.PLAN_ID.eq(ULong.valueOf(variable.getPlanId()))); UpdateSetMoreStep updateStep; - if (variable.getType().isNeedMask() && variable.getDefaultValue().equals(variable.getType().getMask())) { + if (variable.cipherNotChange()) { updateStep = context.update(TABLE).set(TABLE.DEFAULT_VALUE, TABLE.DEFAULT_VALUE); } else { - updateStep = context.update(TABLE).set(TABLE.DEFAULT_VALUE, variable.getDefaultValue()); + updateStep = context.update(TABLE) + .set(TABLE.DEFAULT_VALUE, + cipherVariableCryptoService.encryptTaskVariableIfNeeded(variable.getType(), variable.getDefaultValue())); } if (StringUtils.isNotBlank(variable.getName())) { updateStep.set(TABLE.NAME, variable.getName()); @@ -207,13 +249,17 @@ public boolean batchInsertVariableWithId(List variableList) { } @Override - @Transactional(rollbackFor = {Exception.class, Error.class}) + @JobTransactional(transactionManager = "jobManageTransactionManager") public boolean updateVariableByName(TaskVariableDTO variable) { List conditions = new ArrayList<>(); conditions.add(TABLE.PLAN_ID.equal(ULong.valueOf(variable.getPlanId()))); conditions.add(TABLE.NAME.equal(variable.getName())); return 1 == - context.update(TABLE).set(TABLE.DEFAULT_VALUE, variable.getDefaultValue()).where(conditions).limit(1).execute(); + context.update(TABLE) + .set(TABLE.DEFAULT_VALUE, + cipherVariableCryptoService.encryptTaskVariableIfNeeded(variable.getType(), variable.getDefaultValue())) + .where(conditions).limit(1) + .execute(); } private UByte getChangeable(Boolean changeable) { diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/template/TaskTemplateDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/template/TaskTemplateDAO.java index 11184aa7dc..c6cb721038 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/template/TaskTemplateDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/template/TaskTemplateDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,8 +27,8 @@ import com.tencent.bk.job.common.model.PageData; import com.tencent.bk.job.manage.model.dto.task.TaskTemplateInfoDTO; import com.tencent.bk.job.manage.model.query.TaskTemplateQuery; -import org.jooq.types.ULong; +import java.util.Collection; import java.util.List; import java.util.Map; @@ -206,7 +206,21 @@ public interface TaskTemplateDAO { */ boolean isExistAnyAppTemplate(Long appId); - void updateTemplateStatus(ULong templateId, int scriptStatus); + /** + * 更新作业模版引用的脚本的状态 + * + * @param templateId 作业模版 + * @param scriptStatusFlags 脚本更新状态 + */ + void updateTemplateScriptStatusFlags(Long templateId, int scriptStatusFlags); + + /** + * 批量更新作业模版引用的脚本的状态 + * + * @param templateIds 作业模版ID 列表 + * @param scriptStatusFlags 脚本更新状态 + */ + void batchUpdateTemplateScriptStatus(Collection templateIds, int scriptStatusFlags); Integer countTemplates(Long appId); @@ -217,7 +231,7 @@ public interface TaskTemplateDAO { /** * 获取模板标签(兼容老版本) * - * @return Map> + * @return Map> */ Map> listAllTemplateTagsCompatible(); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/template/TaskTemplateScriptStepDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/template/TaskTemplateScriptStepDAO.java new file mode 100644 index 0000000000..91c10fd398 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/template/TaskTemplateScriptStepDAO.java @@ -0,0 +1,58 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.dao.template; + +import com.tencent.bk.job.manage.dao.TaskScriptStepDAO; +import com.tencent.bk.job.manage.model.dto.TemplateStepScriptStatusInfo; + +import java.util.Collection; +import java.util.List; + +public interface TaskTemplateScriptStepDAO extends TaskScriptStepDAO { + + /** + * 获取引用该脚本的作业模版所有脚本步骤的脚本状态（包含其他脚本） + * + * @param scriptId 脚本 ID + * @param scriptVersionId 脚本版本 ID + */ + List listAllRelatedTemplateStepsScriptStatusInfo(String scriptId, + Long scriptVersionId); + + /** + * 获取作业模版包含的所有脚本步骤的脚本状态 + * + * @param templateId 作业模版 ID + */ + List listStepsScriptStatusInfoByTemplateId(Long templateId); + + /** + * 批量更新步骤的脚本状态flags + * + * @param stepIds 步骤 ID 列表 + * @param scriptStatusFlags 脚本状态flags + */ + void batchUpdateScriptStatusFlags(Collection stepIds, int scriptStatusFlags); +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateApprovalStepDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateApprovalStepDAOImpl.java index 4c3527cfef..ac31d6003a 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateApprovalStepDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateApprovalStepDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,15 +28,15 @@ import com.tencent.bk.job.manage.common.util.DbRecordMapper; import com.tencent.bk.job.manage.dao.TaskApprovalStepDAO; import com.tencent.bk.job.manage.model.dto.task.TaskApprovalStepDTO; +import com.tencent.bk.job.manage.model.tables.TaskTemplate; +import com.tencent.bk.job.manage.model.tables.TaskTemplateStep; +import com.tencent.bk.job.manage.model.tables.TaskTemplateStepApproval; +import com.tencent.bk.job.manage.model.tables.records.TaskTemplateStepApprovalRecord; import lombok.extern.slf4j.Slf4j; import org.jooq.Condition; import org.jooq.DSLContext; import org.jooq.Record6; import org.jooq.Result; -import org.jooq.generated.tables.TaskTemplate; -import org.jooq.generated.tables.TaskTemplateStep; -import org.jooq.generated.tables.TaskTemplateStepApproval; -import org.jooq.generated.tables.records.TaskTemplateStepApprovalRecord; import org.jooq.types.UByte; import org.jooq.types.ULong; import org.springframework.beans.factory.annotation.Autowired; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateDAOImpl.java index 8e1ef6e178..c9a28cc79c 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,11 +26,14 @@ import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; +import com.tencent.bk.job.common.mysql.util.JooqDataTypeUtil; import com.tencent.bk.job.common.util.TagUtils; import com.tencent.bk.job.manage.common.util.DbRecordMapper; import com.tencent.bk.job.manage.dao.template.TaskTemplateDAO; import com.tencent.bk.job.manage.model.dto.task.TaskTemplateInfoDTO; import com.tencent.bk.job.manage.model.query.TaskTemplateQuery; +import com.tencent.bk.job.manage.model.tables.TaskTemplate; +import com.tencent.bk.job.manage.model.tables.records.TaskTemplateRecord; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; @@ -44,8 +47,6 @@ import org.jooq.SelectJoinStep; import org.jooq.TableField; import org.jooq.UpdateSetMoreStep; -import org.jooq.generated.tables.TaskTemplate; -import org.jooq.generated.tables.records.TaskTemplateRecord; import org.jooq.types.UByte; import org.jooq.types.ULong; import org.springframework.beans.factory.annotation.Autowired; @@ -218,9 +219,9 @@ public TaskTemplateInfoDTO getDeletedTaskTemplateById(Long templateId) { private TaskTemplateInfoDTO getOneTaskTemplate(List conditions) { Record13 record = context.select(TABLE.ID, TABLE.APP_ID, TABLE.NAME, TABLE.DESCRIPTION, TABLE.CREATOR, - TABLE.STATUS, - TABLE.CREATE_TIME, TABLE.LAST_MODIFY_USER, TABLE.LAST_MODIFY_TIME, TABLE.FIRST_STEP_ID, - TABLE.LAST_STEP_ID, TABLE.VERSION, TABLE.SCRIPT_STATUS) + TABLE.STATUS, + TABLE.CREATE_TIME, TABLE.LAST_MODIFY_USER, TABLE.LAST_MODIFY_TIME, TABLE.FIRST_STEP_ID, + TABLE.LAST_STEP_ID, TABLE.VERSION, TABLE.SCRIPT_STATUS) .from(TABLE).where(conditions).fetchOne(); if (record != null) { return DbRecordMapper.convertRecordToTemplateInfo(record); @@ -232,9 +233,9 @@ private TaskTemplateInfoDTO getOneTaskTemplate(List conditions) { private List listTaskTemplate(List conditions) { Result> records = context.select(TABLE.ID, TABLE.APP_ID, TABLE.NAME, TABLE.DESCRIPTION, TABLE.CREATOR, - TABLE.STATUS, - TABLE.CREATE_TIME, TABLE.LAST_MODIFY_USER, TABLE.LAST_MODIFY_TIME, TABLE.FIRST_STEP_ID, - TABLE.LAST_STEP_ID, TABLE.VERSION, TABLE.SCRIPT_STATUS) + TABLE.STATUS, + TABLE.CREATE_TIME, TABLE.LAST_MODIFY_USER, TABLE.LAST_MODIFY_TIME, TABLE.FIRST_STEP_ID, + TABLE.LAST_STEP_ID, TABLE.VERSION, TABLE.SCRIPT_STATUS) .from(TABLE).where(conditions).fetch(); return records.map(DbRecordMapper::convertRecordToTemplateInfo); } @@ -536,9 +537,19 @@ public boolean isExistAnyAppTemplate(Long appId) { } @Override - public void updateTemplateStatus(ULong templateId, int scriptStatus) { - context.update(TABLE).set(TABLE.SCRIPT_STATUS, UByte.valueOf(scriptStatus)) - .where(TABLE.ID.equal(templateId)).execute(); + public void updateTemplateScriptStatusFlags(Long templateId, int scriptStatusFlags) { + context.update(TABLE) + .set(TABLE.SCRIPT_STATUS, JooqDataTypeUtil.buildUByte(scriptStatusFlags)) + .where(TABLE.ID.equal(ULong.valueOf(templateId))) + .execute(); + } + + @Override + public void batchUpdateTemplateScriptStatus(Collection templateIds, int scriptStatusFlags) { + context.update(TABLE) + .set(TABLE.SCRIPT_STATUS, JooqDataTypeUtil.buildUByte(scriptStatusFlags)) + .where(TABLE.ID.in(templateIds)) + .execute(); } @Override diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateFavoriteDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateFavoriteDAOImpl.java index 8224c2d666..99f35dbd92 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateFavoriteDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateFavoriteDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,12 +25,12 @@ package com.tencent.bk.job.manage.dao.template.impl; import com.tencent.bk.job.manage.dao.TaskFavoriteDAO; +import com.tencent.bk.job.manage.model.tables.TaskFavoriteTemplate; import lombok.extern.slf4j.Slf4j; import org.jooq.Condition; import org.jooq.DSLContext; import org.jooq.Record1; import org.jooq.Result; -import org.jooq.generated.tables.TaskFavoriteTemplate; import org.jooq.types.UByte; import org.jooq.types.ULong; import org.springframework.beans.factory.annotation.Autowired; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateFileInfoDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateFileInfoDAOImpl.java index c50cc0ace1..7e8d0cb96a 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateFileInfoDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateFileInfoDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,19 +26,22 @@ import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.exception.InternalException; +import com.tencent.bk.job.common.mysql.util.JooqDataTypeUtil; import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.manage.common.util.DbRecordMapper; import com.tencent.bk.job.manage.dao.TaskFileInfoDAO; import com.tencent.bk.job.manage.model.dto.task.TaskFileInfoDTO; +import com.tencent.bk.job.manage.model.dto.task.TaskTargetDTO; +import com.tencent.bk.job.manage.model.tables.TaskTemplateStepFileList; +import com.tencent.bk.job.manage.model.tables.records.TaskTemplateStepFileListRecord; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import org.jooq.Condition; import org.jooq.DSLContext; import org.jooq.InsertValuesStep8; import org.jooq.Record9; import org.jooq.Result; -import org.jooq.generated.tables.TaskTemplateStepFileList; -import org.jooq.generated.tables.records.TaskTemplateStepFileListRecord; import org.jooq.types.UByte; import org.jooq.types.ULong; import org.springframework.beans.factory.annotation.Autowired; @@ -130,7 +133,7 @@ public long insertFileInfo(TaskFileInfoDTO fileInfo) { JsonUtils.toJson(fileInfo.getFileLocation()), DbRecordMapper.getJooqLongValue(fileInfo.getFileSize()), fileInfo.getFileHash(), - fileInfo.getHost() == null ? null : fileInfo.getHost().toString(), + fileInfo.getHost() == null ? null : fileInfo.getHost().toJsonString(), DbRecordMapper.getJooqLongValue(fileInfo.getHostAccount()), fileInfo.getFileSourceId() ) @@ -155,7 +158,7 @@ public List batchInsertFileInfo(List fileInfoList) { JsonUtils.toJson(fileInfo.getFileLocation()), DbRecordMapper.getJooqLongValue(fileInfo.getFileSize()), fileInfo.getFileHash(), - fileInfo.getHost() == null ? null : fileInfo.getHost().toString(), + fileInfo.getHost() == null ? null : fileInfo.getHost().toJsonString(), DbRecordMapper.getJooqLongValue(fileInfo.getHostAccount()), fileInfo.getFileSourceId() )); @@ -187,7 +190,7 @@ public boolean updateFileInfoById(TaskFileInfoDTO fileInfo) { .set(TABLE.FILE_LOCATION, JsonUtils.toJson(fileInfo.getFileLocation())) .set(TABLE.FILE_SIZE, DbRecordMapper.getJooqLongValue(fileInfo.getFileSize())) .set(TABLE.FILE_HASH, fileInfo.getFileHash()) - .set(TABLE.HOST, fileInfo.getHost() == null ? null : fileInfo.getHost().toString()) + .set(TABLE.HOST, fileInfo.getHost() == null ? null : fileInfo.getHost().toJsonString()) .set(TABLE.HOST_ACCOUNT, DbRecordMapper.getJooqLongValue(fileInfo.getHostAccount())) .set(TABLE.FILE_SOURCE_ID, fileInfo.getFileSourceId()) .where(conditions) @@ -219,11 +222,37 @@ public List listLocalFileByStepId(List stepIdList) { return new ArrayList<>(); } - List uLongStepIdList = stepIdList.parallelStream().map(ULong::valueOf).collect(Collectors.toList()); + List uLongStepIdList = stepIdList.stream().map(ULong::valueOf).collect(Collectors.toList()); List conditions = new ArrayList<>(); conditions.add(TABLE.STEP_ID.in(uLongStepIdList)); conditions.add(TABLE.FILE_TYPE.equal(UByte.valueOf(2))); return context.select(TABLE.FILE_LOCATION).from(TABLE).where(conditions).fetch(TABLE.FILE_LOCATION); } + + @Override + public Map listStepFileHosts() { + Result result = context.select(TABLE.ID, TABLE.HOST).fetch(); + Map stepTargets = new HashMap<>(); + if (result.isNotEmpty()) { + result.forEach(record -> { + Long recordId = record.get(TABLE.ID).longValue(); + TaskTargetDTO target = TaskTargetDTO.fromJsonString(record.get(TABLE.HOST)); + stepTargets.put(recordId, target); + }); + } + return stepTargets; + } + + @Override + public boolean updateStepFileHosts(Long recordId, String value) { + if (StringUtils.isBlank(value)) { + return false; + } + int result = context.update(TABLE) + .set(TABLE.HOST, value) + .where(TABLE.ID.eq(JooqDataTypeUtil.buildULong(recordId))) + .execute(); + return result == 1; + } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateFileStepDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateFileStepDAOImpl.java index d64a2e4597..04838ad438 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateFileStepDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateFileStepDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,20 +25,23 @@ package com.tencent.bk.job.manage.dao.template.impl; import com.tencent.bk.job.common.constant.NotExistPathHandlerEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskFileTypeEnum; +import com.tencent.bk.job.common.mysql.util.JooqDataTypeUtil; +import com.tencent.bk.job.manage.api.common.constants.task.TaskFileTypeEnum; import com.tencent.bk.job.manage.common.util.DbRecordMapper; import com.tencent.bk.job.manage.dao.TaskFileStepDAO; import com.tencent.bk.job.manage.model.dto.task.TaskFileStepDTO; +import com.tencent.bk.job.manage.model.dto.task.TaskTargetDTO; +import com.tencent.bk.job.manage.model.tables.TaskTemplate; +import com.tencent.bk.job.manage.model.tables.TaskTemplateStep; +import com.tencent.bk.job.manage.model.tables.TaskTemplateStepFile; +import com.tencent.bk.job.manage.model.tables.TaskTemplateStepFileList; +import com.tencent.bk.job.manage.model.tables.records.TaskTemplateStepFileRecord; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.jooq.Condition; import org.jooq.DSLContext; import org.jooq.Record11; import org.jooq.Result; -import org.jooq.generated.tables.TaskTemplate; -import org.jooq.generated.tables.TaskTemplateStep; -import org.jooq.generated.tables.TaskTemplateStepFile; -import org.jooq.generated.tables.TaskTemplateStepFileList; -import org.jooq.generated.tables.records.TaskTemplateStepFileRecord; import org.jooq.types.UByte; import org.jooq.types.ULong; import org.springframework.beans.factory.annotation.Autowired; @@ -117,7 +120,7 @@ public long insertFileStep(TaskFileStepDTO fileStep) { TABLE.DUPLICATE_HANDLER, TABLE.NOT_EXIST_PATH_HANDLER) .values(ULong.valueOf(fileStep.getStepId()), fileStep.getDestinationFileLocation(), ULong.valueOf(fileStep.getExecuteAccount()), - fileStep.getDestinationHostList() == null ? null : fileStep.getDestinationHostList().toString(), + fileStep.getDestinationHostList() == null ? null : fileStep.getDestinationHostList().toJsonString(), fileStep.getTimeout() == null ? ULong.valueOf(0) : ULong.valueOf(fileStep.getTimeout()), fileStep.getOriginSpeedLimit() == null ? null : ULong.valueOf(fileStep.getOriginSpeedLimit()), fileStep.getTargetSpeedLimit() == null ? null : ULong.valueOf(fileStep.getTargetSpeedLimit()), @@ -145,7 +148,7 @@ public boolean updateFileStepById(TaskFileStepDTO fileStep) { return 1 == context.update(TABLE).set(TABLE.DESTINATION_FILE_LOCATION, fileStep.getDestinationFileLocation()) .set(TABLE.EXECUTE_ACCOUNT, ULong.valueOf(fileStep.getExecuteAccount())) .set(TABLE.DESTINATION_HOST_LIST, - fileStep.getDestinationHostList() == null ? null : fileStep.getDestinationHostList().toString()) + fileStep.getDestinationHostList() == null ? null : fileStep.getDestinationHostList().toJsonString()) .set(TABLE.TIMEOUT, fileStep.getTimeout() == null ? ULong.valueOf(0) : ULong.valueOf(fileStep.getTimeout())) .set(TABLE.ORIGIN_SPEED_LIMIT, fileStep.getOriginSpeedLimit() == null ? null : ULong.valueOf(fileStep.getOriginSpeedLimit())) @@ -182,4 +185,30 @@ public int countFileSteps(Long appId, TaskFileTypeEnum fileType) { .join(tableTaskTemplate).on(tableTTStep.TEMPLATE_ID.eq(tableTaskTemplate.ID)) .where(conditions).fetchOne().value1(); } + + @Override + public Map listStepTargets() { + Result result = context.select(TABLE.ID, TABLE.DESTINATION_HOST_LIST).fetch(); + Map stepTargets = new HashMap<>(); + if (result.isNotEmpty()) { + result.forEach(record -> { + Long recordId = record.get(TABLE.ID).longValue(); + TaskTargetDTO target = TaskTargetDTO.fromJsonString(record.get(TABLE.DESTINATION_HOST_LIST)); + stepTargets.put(recordId, target); + }); + } + return stepTargets; + } + + @Override + public boolean updateStepTargets(Long recordId, String value) { + if (StringUtils.isBlank(value)) { + return false; + } + int result = context.update(TABLE) + .set(TABLE.DESTINATION_HOST_LIST, value) + .where(TABLE.ID.eq(JooqDataTypeUtil.buildULong(recordId))) + .execute(); + return result == 1; + } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateScriptStepDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateScriptStepDAOImpl.java index 7ccbf3e417..721b29b016 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateScriptStepDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateScriptStepDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,17 +24,28 @@ package com.tencent.bk.job.manage.dao.template.impl; -import com.tencent.bk.job.manage.common.consts.task.TaskScriptSourceEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskTypeEnum; -import com.tencent.bk.job.manage.common.util.DbRecordMapper; -import com.tencent.bk.job.manage.dao.TaskScriptStepDAO; +import com.tencent.bk.job.common.crypto.scenario.SensitiveParamCryptoService; +import com.tencent.bk.job.common.mysql.util.JooqDataTypeUtil; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskScriptSourceEnum; +import com.tencent.bk.job.manage.dao.template.TaskTemplateScriptStepDAO; +import com.tencent.bk.job.manage.model.dto.TemplateStepScriptStatusInfo; import com.tencent.bk.job.manage.model.dto.task.TaskScriptStepDTO; +import com.tencent.bk.job.manage.model.dto.task.TaskTargetDTO; +import com.tencent.bk.job.manage.model.tables.TaskTemplate; +import com.tencent.bk.job.manage.model.tables.TaskTemplateStep; +import com.tencent.bk.job.manage.model.tables.TaskTemplateStepScript; +import com.tencent.bk.job.manage.model.tables.records.TaskTemplateStepScriptRecord; import lombok.extern.slf4j.Slf4j; -import org.jooq.*; -import org.jooq.generated.tables.TaskTemplate; -import org.jooq.generated.tables.TaskTemplateStep; -import org.jooq.generated.tables.TaskTemplateStepScript; -import org.jooq.generated.tables.records.TaskTemplateStepScriptRecord; +import org.jooq.Condition; +import org.jooq.DSLContext; +import org.jooq.Record; +import org.jooq.Record1; +import org.jooq.Record15; +import org.jooq.Record16; +import org.jooq.Record3; +import org.jooq.Result; +import org.jooq.Select; import org.jooq.impl.DSL; import org.jooq.types.UByte; import org.jooq.types.ULong; @@ -42,26 +53,65 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.stream.Collectors; -/** - * @since 3/10/2019 21:53 - */ @Slf4j @Repository("TaskTemplateScriptStepDAOImpl") -public class TaskTemplateScriptStepDAOImpl implements TaskScriptStepDAO { +public class TaskTemplateScriptStepDAOImpl implements TaskTemplateScriptStepDAO { private static final TaskTemplateStepScript TABLE = TaskTemplateStepScript.TASK_TEMPLATE_STEP_SCRIPT; private static final TaskTemplate tableTaskTemplate = TaskTemplate.TASK_TEMPLATE; private static final TaskTemplateStep tableTTStep = TaskTemplateStep.TASK_TEMPLATE_STEP; private static final TaskTemplateStepScript tableTTStepScript = TaskTemplateStepScript.TASK_TEMPLATE_STEP_SCRIPT; - private DSLContext context; + private final DSLContext context; + private final SensitiveParamCryptoService sensitiveParamCryptoService; @Autowired - public TaskTemplateScriptStepDAOImpl(@Qualifier("job-manage-dsl-context") DSLContext context) { + public TaskTemplateScriptStepDAOImpl(@Qualifier("job-manage-dsl-context") DSLContext context, + SensitiveParamCryptoService sensitiveParamCryptoService) { this.context = context; + this.sensitiveParamCryptoService = sensitiveParamCryptoService; + } + + private TaskScriptStepDTO convertRecordToTaskScriptStep(Record16 record) { + if (record == null) { + return null; + } + TaskScriptStepDTO taskScriptStep = new TaskScriptStepDTO(); + taskScriptStep.setId(record.get(TABLE.ID).longValue()); + taskScriptStep.setTemplateId((record.get(TABLE.TEMPLATE_ID)).longValue()); + taskScriptStep.setStepId((record.get(TABLE.STEP_ID)).longValue()); + taskScriptStep.setScriptSource(TaskScriptSourceEnum.valueOf((record.get(TABLE.SCRIPT_TYPE)).intValue())); + taskScriptStep.setScriptId(record.get(TABLE.SCRIPT_ID)); + if (record.get(TABLE.SCRIPT_VERSION_ID) != null) { + taskScriptStep.setScriptVersionId((record.get(TABLE.SCRIPT_VERSION_ID)).longValue()); + } + taskScriptStep.setContent(record.get(TABLE.CONTENT)); + taskScriptStep.setLanguage(ScriptTypeEnum.valOf((record.get(TABLE.LANGUAGE)).intValue())); + taskScriptStep.setTimeout((record.get(TABLE.SCRIPT_TIMEOUT)).longValue()); + taskScriptStep.setAccount((record.get(TABLE.EXECUTE_ACCOUNT)).longValue()); + taskScriptStep.setExecuteTarget(TaskTargetDTO.fromJsonString(record.get(TABLE.DESTINATION_HOST_LIST))); + taskScriptStep.setSecureParam((record.get(TABLE.IS_SECURE_PARAM)).intValue() == 1); + String encryptedScriptParam = record.get(TABLE.SCRIPT_PARAM); + + // 敏感参数解密 + taskScriptStep.setScriptParam(sensitiveParamCryptoService.decryptParamIfNeeded( + taskScriptStep.getSecureParam(), encryptedScriptParam + )); + + taskScriptStep.setStatus((record.get(TABLE.STATUS)).intValue()); + taskScriptStep.setIgnoreError((record.get(TABLE.IGNORE_ERROR)).intValue() == 1); + taskScriptStep.setWindowsInterpreter(record.get(TABLE.WINDOWS_INTERPRETER)); + return taskScriptStep; } @Override @@ -69,20 +119,34 @@ public List listScriptStepByParentId(long parentId) { List conditions = new ArrayList<>(); conditions.add(TABLE.TEMPLATE_ID.eq(ULong.valueOf(parentId))); Result< - Record15> result = + Record16> result = context - .select(TABLE.ID, TABLE.TEMPLATE_ID, TABLE.STEP_ID, TABLE.SCRIPT_TYPE, TABLE.SCRIPT_ID, - TABLE.SCRIPT_VERSION_ID, TABLE.CONTENT, TABLE.LANGUAGE, TABLE.SCRIPT_PARAM, - TABLE.SCRIPT_TIMEOUT, TABLE.EXECUTE_ACCOUNT, TABLE.DESTINATION_HOST_LIST, - TABLE.IS_SECURE_PARAM, TABLE.STATUS, TABLE.IGNORE_ERROR) - .from(TABLE).where(conditions).fetch(); + .select( + TABLE.ID, + TABLE.TEMPLATE_ID, + TABLE.STEP_ID, + TABLE.SCRIPT_TYPE, + TABLE.SCRIPT_ID, + TABLE.SCRIPT_VERSION_ID, + TABLE.CONTENT, + TABLE.LANGUAGE, + TABLE.SCRIPT_PARAM, + TABLE.SCRIPT_TIMEOUT, + TABLE.EXECUTE_ACCOUNT, + TABLE.DESTINATION_HOST_LIST, + TABLE.IS_SECURE_PARAM, + TABLE.STATUS, + TABLE.IGNORE_ERROR, + TABLE.WINDOWS_INTERPRETER + ).from(TABLE).where(conditions).fetch(); List taskScriptStepList = new ArrayList<>(); - if (result != null && result.size() >= 1) { - result.map(record -> taskScriptStepList - .add(DbRecordMapper.convertRecordToTaskScriptStep(record, TaskTypeEnum.TEMPLATE))); + if (!result.isEmpty()) { + result.map(record -> + taskScriptStepList.add(convertRecordToTaskScriptStep(record)) + ); } return taskScriptStepList; } @@ -92,20 +156,35 @@ public Map listScriptStepByIds(List stepIdList) { List conditions = new ArrayList<>(); conditions.add(TABLE.STEP_ID.in(stepIdList.stream().map(ULong::valueOf).collect(Collectors.toList()))); Result< - Record15> result = + Record16> result = context - .select(TABLE.ID, TABLE.TEMPLATE_ID, TABLE.STEP_ID, TABLE.SCRIPT_TYPE, TABLE.SCRIPT_ID, - TABLE.SCRIPT_VERSION_ID, TABLE.CONTENT, TABLE.LANGUAGE, TABLE.SCRIPT_PARAM, - TABLE.SCRIPT_TIMEOUT, TABLE.EXECUTE_ACCOUNT, TABLE.DESTINATION_HOST_LIST, - TABLE.IS_SECURE_PARAM, TABLE.STATUS, TABLE.IGNORE_ERROR) - .from(TABLE).where(conditions).fetch(); + .select( + TABLE.ID, + TABLE.TEMPLATE_ID, + TABLE.STEP_ID, + TABLE.SCRIPT_TYPE, + TABLE.SCRIPT_ID, + TABLE.SCRIPT_VERSION_ID, + TABLE.CONTENT, + TABLE.LANGUAGE, + TABLE.SCRIPT_PARAM, + TABLE.SCRIPT_TIMEOUT, + TABLE.EXECUTE_ACCOUNT, + TABLE.DESTINATION_HOST_LIST, + TABLE.IS_SECURE_PARAM, + TABLE.STATUS, + TABLE.IGNORE_ERROR, + TABLE.WINDOWS_INTERPRETER + ).from(TABLE).where(conditions).fetch(); Map taskScriptStepMap = new HashMap<>(stepIdList.size()); - if (result != null && result.size() >= 1) { - result.map(record -> taskScriptStepMap.put(record.get(TABLE.STEP_ID).longValue(), - DbRecordMapper.convertRecordToTaskScriptStep(record, TaskTypeEnum.TEMPLATE))); + if (result.size() >= 1) { + result.map(record -> taskScriptStepMap.put( + record.get(TABLE.STEP_ID).longValue(), + convertRecordToTaskScriptStep(record) + )); } return taskScriptStepMap; } @@ -114,13 +193,27 @@ public Map listScriptStepByIds(List stepIdList) { public TaskScriptStepDTO getScriptStepById(long stepId) { List conditions = new ArrayList<>(); conditions.add(TABLE.STEP_ID.eq(ULong.valueOf(stepId))); - Record15 record = context.select(TABLE.ID, TABLE.TEMPLATE_ID, TABLE.STEP_ID, TABLE.SCRIPT_TYPE, - TABLE.SCRIPT_ID, TABLE.SCRIPT_VERSION_ID, TABLE.CONTENT, TABLE.LANGUAGE, TABLE.SCRIPT_PARAM, - TABLE.SCRIPT_TIMEOUT, TABLE.EXECUTE_ACCOUNT, TABLE.DESTINATION_HOST_LIST, TABLE.IS_SECURE_PARAM, - TABLE.STATUS, TABLE.IGNORE_ERROR).from(TABLE).where(conditions).fetchOne(); + Record16 record = context.select( + TABLE.ID, + TABLE.TEMPLATE_ID, + TABLE.STEP_ID, + TABLE.SCRIPT_TYPE, + TABLE.SCRIPT_ID, + TABLE.SCRIPT_VERSION_ID, + TABLE.CONTENT, + TABLE.LANGUAGE, + TABLE.SCRIPT_PARAM, + TABLE.SCRIPT_TIMEOUT, + TABLE.EXECUTE_ACCOUNT, + TABLE.DESTINATION_HOST_LIST, + TABLE.IS_SECURE_PARAM, + TABLE.STATUS, + TABLE.IGNORE_ERROR, + TABLE.WINDOWS_INTERPRETER + ).from(TABLE).where(conditions).fetchOne(); if (record != null) { - return DbRecordMapper.convertRecordToTaskScriptStep(record, TaskTypeEnum.TEMPLATE); + return convertRecordToTaskScriptStep(record); } else { return null; } @@ -137,18 +230,41 @@ public long insertScriptStep(TaskScriptStepDTO scriptStep) { ignoreError = UByte.valueOf(1); } TaskTemplateStepScriptRecord record = context.insertInto(TABLE) - .columns(TABLE.TEMPLATE_ID, TABLE.STEP_ID, TABLE.SCRIPT_TYPE, TABLE.SCRIPT_ID, TABLE.SCRIPT_VERSION_ID, - TABLE.CONTENT, TABLE.LANGUAGE, TABLE.SCRIPT_PARAM, TABLE.SCRIPT_TIMEOUT, TABLE.EXECUTE_ACCOUNT, - TABLE.DESTINATION_HOST_LIST, TABLE.IS_SECURE_PARAM, TABLE.STATUS, TABLE.IGNORE_ERROR) - .values(ULong.valueOf(scriptStep.getTemplateId()), ULong.valueOf(scriptStep.getStepId()), - UByte.valueOf(scriptStep.getScriptSource().getType()), scriptStep.getScriptId(), + .columns( + TABLE.TEMPLATE_ID, + TABLE.STEP_ID, + TABLE.SCRIPT_TYPE, + TABLE.SCRIPT_ID, + TABLE.SCRIPT_VERSION_ID, + TABLE.CONTENT, + TABLE.LANGUAGE, + TABLE.SCRIPT_PARAM, + TABLE.SCRIPT_TIMEOUT, + TABLE.EXECUTE_ACCOUNT, + TABLE.DESTINATION_HOST_LIST, + TABLE.IS_SECURE_PARAM, + TABLE.STATUS, + TABLE.IGNORE_ERROR, + TABLE.WINDOWS_INTERPRETER + ).values( + ULong.valueOf(scriptStep.getTemplateId()), + ULong.valueOf(scriptStep.getStepId()), + UByte.valueOf(scriptStep.getScriptSource().getType()), + scriptStep.getScriptId(), scriptStep.getScriptVersionId() == null ? null : ULong.valueOf(scriptStep.getScriptVersionId()), - scriptStep.getContent(), UByte.valueOf(scriptStep.getLanguage().getValue()), - scriptStep.getScriptParam(), ULong.valueOf(scriptStep.getTimeout()), + scriptStep.getContent(), + UByte.valueOf(scriptStep.getLanguage().getValue()), + sensitiveParamCryptoService.encryptParamIfNeeded(scriptStep.getSecureParam(), + scriptStep.getScriptParam()), + ULong.valueOf(scriptStep.getTimeout()), ULong.valueOf(scriptStep.getAccount()), - scriptStep.getExecuteTarget() == null ? null : scriptStep.getExecuteTarget().toString(), isSecureParam, - UByte.valueOf(scriptStep.getStatus()), ignoreError) - .returning(TABLE.ID).fetchOne(); + scriptStep.getExecuteTarget() == null ? null : scriptStep.getExecuteTarget().toJsonString(), + isSecureParam, + UByte.valueOf(scriptStep.getStatus()), + ignoreError, + scriptStep.getWindowsInterpreter() + ).returning(TABLE.ID).fetchOne(); + assert record != null; return record.getId().longValue(); } @@ -164,19 +280,24 @@ public boolean updateScriptStepById(TaskScriptStepDTO scriptStep) { if (scriptStep.getIgnoreError() != null && scriptStep.getIgnoreError()) { ignoreError = UByte.valueOf(1); } - return 1 == context.update(TABLE).set(TABLE.SCRIPT_TYPE, UByte.valueOf(scriptStep.getScriptSource().getType())) + return 1 == context.update(TABLE) + .set(TABLE.SCRIPT_TYPE, UByte.valueOf(scriptStep.getScriptSource().getType())) .set(TABLE.SCRIPT_ID, scriptStep.getScriptId()) .set(TABLE.SCRIPT_VERSION_ID, scriptStep.getScriptVersionId() == null ? null : ULong.valueOf(scriptStep.getScriptVersionId())) .set(TABLE.CONTENT, scriptStep.getContent()) .set(TABLE.LANGUAGE, UByte.valueOf(scriptStep.getLanguage().getValue())) - .set(TABLE.SCRIPT_PARAM, scriptStep.getScriptParam()) + .set(TABLE.SCRIPT_PARAM, sensitiveParamCryptoService.encryptParamIfNeeded( + scriptStep.getSecureParam(), scriptStep.getScriptParam())) .set(TABLE.SCRIPT_TIMEOUT, ULong.valueOf(scriptStep.getTimeout())) .set(TABLE.EXECUTE_ACCOUNT, ULong.valueOf(scriptStep.getAccount())) .set(TABLE.DESTINATION_HOST_LIST, - scriptStep.getExecuteTarget() == null ? null : scriptStep.getExecuteTarget().toString()) - .set(TABLE.IS_SECURE_PARAM, isSecureParam).set(TABLE.STATUS, UByte.valueOf(scriptStep.getStatus())) - .set(TABLE.IGNORE_ERROR, ignoreError).where(conditions).limit(1).execute(); + scriptStep.getExecuteTarget() == null ? null : scriptStep.getExecuteTarget().toJsonString()) + .set(TABLE.IS_SECURE_PARAM, isSecureParam) + .set(TABLE.STATUS, UByte.valueOf(scriptStep.getStatus())) + .set(TABLE.IGNORE_ERROR, ignoreError) + .set(TABLE.WINDOWS_INTERPRETER, scriptStep.getWindowsInterpreter()) + .where(conditions).limit(1).execute(); } @Override @@ -190,7 +311,7 @@ public boolean deleteScriptStepById(long stepId) { public List batchListScriptStepIdByParentIds(List templateIdList) { List conditions = new ArrayList<>(); conditions.add( - TABLE.TEMPLATE_ID.in(templateIdList.parallelStream().map(ULong::valueOf).collect(Collectors.toList()))); + TABLE.TEMPLATE_ID.in(templateIdList.stream().map(ULong::valueOf).collect(Collectors.toList()))); conditions.add(TABLE.SCRIPT_TYPE.in(Arrays.asList(UByte.valueOf(TaskScriptSourceEnum.CITING.getType()), UByte.valueOf(TaskScriptSourceEnum.PUBLIC.getType())))); conditions.add(TABLE.SCRIPT_ID.isNotNull()); @@ -198,7 +319,7 @@ public List batchListScriptStepIdByParentIds(List templ Result> result = context .select(TABLE.TEMPLATE_ID, TABLE.SCRIPT_ID, TABLE.SCRIPT_VERSION_ID).from(TABLE).where(conditions).fetch(); List taskScriptStepList = new ArrayList<>(); - if (result != null && result.size() > 0) { + if (result.size() > 0) { result.map(record -> { TaskScriptStepDTO taskScriptStep = new TaskScriptStepDTO(); taskScriptStep.setTemplateId(record.get(TABLE.TEMPLATE_ID).longValue()); @@ -228,10 +349,16 @@ public int countScriptSteps(Long appId, TaskScriptSourceEnum scriptSource) { if (scriptSource != null) { conditions.add(tableTTStepScript.SCRIPT_TYPE.eq(UByte.valueOf(scriptSource.getType()))); } - return context.selectCount().from(tableTTStepScript) - .join(tableTTStep).on(tableTTStep.ID.eq(tableTTStepScript.STEP_ID)) - .join(tableTaskTemplate).on(tableTTStep.TEMPLATE_ID.eq(tableTaskTemplate.ID)) - .where(conditions).fetchOne().value1(); + Record1 record = context.selectCount() + .from(tableTTStepScript) + .join(tableTTStep) + .on(tableTTStep.ID.eq(tableTTStepScript.STEP_ID)) + .join(tableTaskTemplate) + .on(tableTTStep.TEMPLATE_ID.eq(tableTaskTemplate.ID)) + .where(conditions) + .fetchOne(); + assert record != null; + return record.value1(); } @Override @@ -243,10 +370,16 @@ public int countScriptCitedByStepsByScriptIds(Long appId, List scriptIdL if (scriptIdList != null) { conditions.add(tableTTStepScript.SCRIPT_ID.in(scriptIdList)); } - return context.select(DSL.countDistinct(tableTTStepScript.SCRIPT_ID)).from(tableTTStepScript) - .join(tableTTStep).on(tableTTStep.ID.eq(tableTTStepScript.STEP_ID)) - .join(tableTaskTemplate).on(tableTTStep.TEMPLATE_ID.eq(tableTaskTemplate.ID)) - .where(conditions).fetchOne().value1(); + Record1 record = context.select(DSL.countDistinct(tableTTStepScript.SCRIPT_ID)) + .from(tableTTStepScript) + .join(tableTTStep) + .on(tableTTStep.ID.eq(tableTTStepScript.STEP_ID)) + .join(tableTaskTemplate) + .on(tableTTStep.TEMPLATE_ID.eq(tableTaskTemplate.ID)) + .where(conditions) + .fetchOne(); + assert record != null; + return record.value1(); } @Override @@ -258,10 +391,75 @@ public int countScriptStepsByScriptIds(Long appId, List scriptIdList) { if (scriptIdList != null) { conditions.add(tableTTStepScript.SCRIPT_ID.in(scriptIdList)); } - return context.selectCount().from(tableTTStepScript) - .join(tableTTStep).on(tableTTStep.ID.eq(tableTTStepScript.STEP_ID)) - .join(tableTaskTemplate).on(tableTTStep.TEMPLATE_ID.eq(tableTaskTemplate.ID)) - .where(conditions).fetchOne().value1(); + Record1 record = context.selectCount() + .from(tableTTStepScript) + .join(tableTTStep) + .on(tableTTStep.ID.eq(tableTTStepScript.STEP_ID)) + .join(tableTaskTemplate) + .on(tableTTStep.TEMPLATE_ID.eq(tableTaskTemplate.ID)) + .where(conditions) + .fetchOne(); + assert record != null; + return record.value1(); + } + + public List listAllRelatedTemplateStepsScriptStatusInfo(String scriptId, + Long scriptVersionId) { + Select> templateIdSubQuery = context + .selectDistinct(TABLE.TEMPLATE_ID) + .from(TABLE) + .where(TABLE.SCRIPT_ID.eq(scriptId)) + .and(TABLE.SCRIPT_VERSION_ID.eq(ULong.valueOf(scriptVersionId))); + + Result result = context + .select(TABLE.STEP_ID, TABLE.TEMPLATE_ID, TABLE.SCRIPT_ID, TABLE.SCRIPT_VERSION_ID, TABLE.STATUS) + .from(TABLE) + .where(TABLE.TEMPLATE_ID.in(templateIdSubQuery)) + .fetch(); + if (result.isEmpty()) { + return Collections.emptyList(); + } + + List stepScriptStatusInfos = new ArrayList<>(result.size()); + result.forEach(record -> stepScriptStatusInfos.add(convertToTemplateStepScriptStatusInfo(record))); + + return stepScriptStatusInfos; + } + + @Override + public List listStepsScriptStatusInfoByTemplateId(Long templateId) { + Result result = context + .select(TABLE.STEP_ID, TABLE.TEMPLATE_ID, TABLE.SCRIPT_ID, TABLE.SCRIPT_VERSION_ID, TABLE.STATUS) + .from(TABLE) + .where(TABLE.TEMPLATE_ID.eq(ULong.valueOf(templateId))) + .and(TABLE.SCRIPT_ID.isNotNull()) + .and(TABLE.SCRIPT_VERSION_ID.isNotNull()) + .fetch(); + if (result.isEmpty()) { + return Collections.emptyList(); + } + + List stepScriptStatusInfos = new ArrayList<>(result.size()); + result.forEach(record -> stepScriptStatusInfos.add(convertToTemplateStepScriptStatusInfo(record))); + + return stepScriptStatusInfos; + } + + private TemplateStepScriptStatusInfo convertToTemplateStepScriptStatusInfo(Record record) { + TemplateStepScriptStatusInfo stepScriptStatusInfo = new TemplateStepScriptStatusInfo(); + stepScriptStatusInfo.setTemplateId(record.get(TABLE.TEMPLATE_ID).longValue()); + stepScriptStatusInfo.setStepId(record.get(TABLE.STEP_ID).longValue()); + stepScriptStatusInfo.setScriptStatusFlags(record.get(TABLE.STATUS).intValue()); + stepScriptStatusInfo.setScriptId(record.get(TABLE.SCRIPT_ID)); + stepScriptStatusInfo.setScriptVersionId(record.get(TABLE.SCRIPT_VERSION_ID).longValue()); + return stepScriptStatusInfo; } + @Override + public void batchUpdateScriptStatusFlags(Collection stepIds, int scriptStatusFlags) { + context.update(TABLE) + .set(TABLE.STATUS, JooqDataTypeUtil.buildUByte(scriptStatusFlags)) + .where(TABLE.STEP_ID.in(stepIds)) + .execute(); + } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateStepDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateStepDAOImpl.java index 56418fdfc8..f4004dfbc0 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateStepDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateStepDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,11 +24,13 @@ package com.tencent.bk.job.manage.dao.template.impl; -import com.tencent.bk.job.manage.common.consts.task.TaskTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskTypeEnum; import com.tencent.bk.job.manage.common.util.DbRecordMapper; import com.tencent.bk.job.manage.dao.AbstractTaskStepDAO; import com.tencent.bk.job.manage.dao.TaskStepDAO; import com.tencent.bk.job.manage.model.dto.task.TaskStepDTO; +import com.tencent.bk.job.manage.model.tables.TaskTemplateStep; +import com.tencent.bk.job.manage.model.tables.records.TaskTemplateStepRecord; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.jooq.Condition; @@ -36,8 +38,6 @@ import org.jooq.Record11; import org.jooq.Result; import org.jooq.UpdateSetMoreStep; -import org.jooq.generated.tables.TaskTemplateStep; -import org.jooq.generated.tables.records.TaskTemplateStepRecord; import org.jooq.types.UByte; import org.jooq.types.ULong; import org.springframework.beans.factory.annotation.Autowired; @@ -175,7 +175,7 @@ public List listStepIdByParentId(List parentIdList) { } List uLongTemplateIdList = - parentIdList.parallelStream().map(ULong::valueOf).collect(Collectors.toList()); + parentIdList.stream().map(ULong::valueOf).collect(Collectors.toList()); List conditions = new ArrayList<>(); conditions.add(TABLE.TEMPLATE_ID.in(uLongTemplateIdList)); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateVariableDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateVariableDAOImpl.java index ad502719bf..55adcca09f 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateVariableDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/template/impl/TaskTemplateVariableDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,20 +26,22 @@ import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.constant.TaskVariableTypeEnum; +import com.tencent.bk.job.common.crypto.scenario.CipherVariableCryptoService; import com.tencent.bk.job.common.exception.InternalException; -import com.tencent.bk.job.manage.common.util.JooqDataTypeUtil; +import com.tencent.bk.job.common.mysql.util.JooqDataTypeUtil; import com.tencent.bk.job.manage.dao.TaskVariableDAO; import com.tencent.bk.job.manage.model.dto.task.TaskVariableDTO; +import com.tencent.bk.job.manage.model.tables.TaskTemplateVariable; +import com.tencent.bk.job.manage.model.tables.records.TaskTemplateVariableRecord; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.jooq.Condition; import org.jooq.DSLContext; import org.jooq.InsertValuesStep7; import org.jooq.InsertValuesStep8; +import org.jooq.Record; import org.jooq.Record8; import org.jooq.Result; -import org.jooq.generated.tables.TaskTemplateVariable; -import org.jooq.generated.tables.records.TaskTemplateVariableRecord; import org.jooq.types.UByte; import org.jooq.types.ULong; import org.springframework.beans.factory.annotation.Autowired; @@ -50,6 +52,7 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.Objects; /** * @since 3/10/2019 21:54 @@ -60,11 +63,14 @@ public class TaskTemplateVariableDAOImpl implements TaskVariableDAO { private static final TaskTemplateVariable TABLE = TaskTemplateVariable.TASK_TEMPLATE_VARIABLE; - private DSLContext context; + private final DSLContext context; + private final CipherVariableCryptoService cipherVariableCryptoService; @Autowired - public TaskTemplateVariableDAOImpl(@Qualifier("job-manage-dsl-context") DSLContext context) { + public TaskTemplateVariableDAOImpl(@Qualifier("job-manage-dsl-context") DSLContext context, + CipherVariableCryptoService cipherVariableCryptoService) { this.context = context; + this.cipherVariableCryptoService = cipherVariableCryptoService; } @Override @@ -77,15 +83,22 @@ public List listVariablesByParentId(long parentId) { Record8> records = context - .select(TABLE.ID, TABLE.TEMPLATE_ID, TABLE.NAME, TABLE.TYPE, TABLE.DEFAULT_VALUE, - TABLE.DESCRIPTION, TABLE.IS_CHANGEABLE, TABLE.IS_REQUIRED) - .from(TABLE).where(conditions).fetch(); + .select( + TABLE.ID, + TABLE.TEMPLATE_ID, + TABLE.NAME, + TABLE.TYPE, + TABLE.DEFAULT_VALUE, + TABLE.DESCRIPTION, + TABLE.IS_CHANGEABLE, + TABLE.IS_REQUIRED + ).from(TABLE).where(conditions).fetch(); List taskVariableList = new ArrayList<>(); if (records.size() >= 1) { records.forEach(record -> taskVariableList - .add(convertRecordToTaskVariable(record))); + .add(extract(record))); } return taskVariableList; } @@ -96,29 +109,42 @@ public TaskVariableDTO getVariableById(long parentId, long id) { conditions.add(TABLE.ID.eq(ULong.valueOf(id))); conditions.add(TABLE.TEMPLATE_ID.eq(ULong.valueOf(parentId))); Record8 record = - context.select(TABLE.ID, TABLE.TEMPLATE_ID, TABLE.NAME, TABLE.TYPE, TABLE.DEFAULT_VALUE, TABLE.DESCRIPTION, - TABLE.IS_CHANGEABLE, TABLE.IS_REQUIRED).from(TABLE).where(conditions).fetchOne(); + context.select( + TABLE.ID, + TABLE.TEMPLATE_ID, + TABLE.NAME, + TABLE.TYPE, + TABLE.DEFAULT_VALUE, + TABLE.DESCRIPTION, + TABLE.IS_CHANGEABLE, + TABLE.IS_REQUIRED + ).from(TABLE).where(conditions).fetchOne(); if (record != null) { - return convertRecordToTaskVariable(record); + return extract(record); } else { return null; } } - private TaskVariableDTO convertRecordToTaskVariable( - Record8 record) { + private TaskVariableDTO extract(Record record) { if (record == null) { return null; } TaskVariableDTO taskVariable = new TaskVariableDTO(); - taskVariable.setId(((ULong) record.get(0)).longValue()); - taskVariable.setTemplateId(((ULong) record.get(1)).longValue()); - taskVariable.setName((String) record.get(2)); - taskVariable.setType(TaskVariableTypeEnum.valOf(((UByte) record.get(3)).intValue())); - taskVariable.setDefaultValue((String) record.get(4)); - taskVariable.setDescription((String) record.get(5)); - taskVariable.setChangeable(((UByte) record.get(6)).intValue() == 1); - taskVariable.setRequired(((UByte) record.get(7)).intValue() == 1); + taskVariable.setId(Objects.requireNonNull(record.get(TABLE.ID)).longValue()); + taskVariable.setTemplateId(Objects.requireNonNull(record.get(TABLE.TEMPLATE_ID)).longValue()); + taskVariable.setName(record.get(TABLE.NAME)); + taskVariable.setType(TaskVariableTypeEnum.valOf(Objects.requireNonNull(record.get(TABLE.TYPE)).intValue())); + String encryptedDefaultValue = record.get(TABLE.DEFAULT_VALUE); + // 密文变量解密 + String defaultValue = cipherVariableCryptoService.decryptTaskVariableIfNeeded( + taskVariable.getType(), + encryptedDefaultValue + ); + taskVariable.setDefaultValue(defaultValue); + taskVariable.setDescription(record.get(TABLE.DESCRIPTION)); + taskVariable.setChangeable(Objects.requireNonNull(record.get(TABLE.IS_CHANGEABLE)).intValue() == 1); + taskVariable.setRequired(Objects.requireNonNull(record.get(TABLE.IS_REQUIRED)).intValue() == 1); return taskVariable; } @@ -128,10 +154,18 @@ public TaskVariableDTO getVariableByName(long parentId, String name) { conditions.add(TABLE.NAME.eq(name)); conditions.add(TABLE.TEMPLATE_ID.eq(ULong.valueOf(parentId))); Result> records = - context.select(TABLE.ID, TABLE.TEMPLATE_ID, TABLE.NAME, TABLE.TYPE, TABLE.DEFAULT_VALUE, TABLE.DESCRIPTION, - TABLE.IS_CHANGEABLE, TABLE.IS_REQUIRED).from(TABLE).where(conditions).fetch(); + context.select( + TABLE.ID, + TABLE.TEMPLATE_ID, + TABLE.NAME, + TABLE.TYPE, + TABLE.DEFAULT_VALUE, + TABLE.DESCRIPTION, + TABLE.IS_CHANGEABLE, + TABLE.IS_REQUIRED + ).from(TABLE).where(conditions).fetch(); if (records.size() > 0) { - return convertRecordToTaskVariable(records.get(0)); + return extract(records.get(0)); } else { return null; } @@ -140,12 +174,23 @@ public TaskVariableDTO getVariableByName(long parentId, String name) { @Override public long insertVariable(TaskVariableDTO variable) { TaskTemplateVariableRecord record = context.insertInto(TABLE) - .columns(TABLE.TEMPLATE_ID, TABLE.NAME, TABLE.TYPE, TABLE.DEFAULT_VALUE, TABLE.DESCRIPTION, - TABLE.IS_CHANGEABLE, TABLE.IS_REQUIRED) - .values(ULong.valueOf(variable.getTemplateId()), variable.getName(), - UByte.valueOf(variable.getType().getType()), variable.getDefaultValue(), variable.getDescription(), - getChangeable(variable.getChangeable()), getRequired(variable.getRequired())) - .returning(TABLE.ID).fetchOne(); + .columns( + TABLE.TEMPLATE_ID, + TABLE.NAME, + TABLE.TYPE, + TABLE.DEFAULT_VALUE, + TABLE.DESCRIPTION, + TABLE.IS_CHANGEABLE, + TABLE.IS_REQUIRED + ).values( + ULong.valueOf(variable.getTemplateId()), + variable.getName(), + UByte.valueOf(variable.getType().getType()), + cipherVariableCryptoService.encryptTaskVariableIfNeeded(variable.getType(), variable.getDefaultValue()), + variable.getDescription(), + getChangeable(variable.getChangeable()), + getRequired(variable.getRequired()) + ).returning(TABLE.ID).fetchOne(); if (record != null) { return record.getId().longValue(); } else { @@ -159,12 +204,28 @@ public List batchInsertVariables(List variableList) { return Collections.emptyList(); } InsertValuesStep7 insertStep = - context.insertInto(TABLE).columns(TABLE.TEMPLATE_ID, TABLE.NAME, TABLE.TYPE, TABLE.DEFAULT_VALUE, - TABLE.DESCRIPTION, TABLE.IS_CHANGEABLE, TABLE.IS_REQUIRED); + context.insertInto(TABLE) + .columns( + TABLE.TEMPLATE_ID, + TABLE.NAME, + TABLE.TYPE, + TABLE.DEFAULT_VALUE, + TABLE.DESCRIPTION, + TABLE.IS_CHANGEABLE, + TABLE.IS_REQUIRED + ); - variableList.forEach(variable -> insertStep.values(ULong.valueOf(variable.getTemplateId()), variable.getName(), - UByte.valueOf(variable.getType().getType()), variable.getDefaultValue(), variable.getDescription(), - getChangeable(variable.getChangeable()), getRequired(variable.getRequired()))); + variableList.forEach(variable -> + insertStep.values( + ULong.valueOf(variable.getTemplateId()), + variable.getName(), + UByte.valueOf(variable.getType().getType()), + cipherVariableCryptoService.encryptTaskVariableIfNeeded(variable.getType(), variable.getDefaultValue()), + variable.getDescription(), + getChangeable(variable.getChangeable()), + getRequired(variable.getRequired()) + ) + ); Result result = insertStep.returning(TABLE.ID).fetch(); List variableIdList = new ArrayList<>(variableList.size()); @@ -185,7 +246,7 @@ public List batchInsertVariables(List variableList) { } @Override - public boolean updateVariableById(TaskVariableDTO variable) { + public boolean updateVarByParentResourceIdAndTplVarId(TaskVariableDTO variable) { List conditions = new ArrayList<>(); conditions.add(TABLE.ID.eq(ULong.valueOf(variable.getId()))); conditions.add(TABLE.TEMPLATE_ID.eq(ULong.valueOf(variable.getTemplateId()))); @@ -200,14 +261,23 @@ public boolean updateVariableById(TaskVariableDTO variable) { } if (variable.getType().isNeedMask() && variable.getDefaultValue().equals(variable.getType().getMask())) { - return 1 == context.update(TABLE).set(TABLE.NAME, variable.getName()) - .set(TABLE.DESCRIPTION, variable.getDescription()).set(TABLE.IS_CHANGEABLE, isChangeable) - .set(TABLE.IS_REQUIRED, isRequired).where(conditions).limit(1).execute(); + return 1 == context.update(TABLE) + .set(TABLE.NAME, variable.getName()) + .set(TABLE.DESCRIPTION, variable.getDescription()) + .set(TABLE.IS_CHANGEABLE, isChangeable) + .set(TABLE.IS_REQUIRED, isRequired) + .where(conditions).limit(1) + .execute(); } - return 1 == context.update(TABLE).set(TABLE.NAME, variable.getName()) - .set(TABLE.DESCRIPTION, variable.getDescription()).set(TABLE.DEFAULT_VALUE, variable.getDefaultValue()) - .set(TABLE.IS_CHANGEABLE, isChangeable).set(TABLE.IS_REQUIRED, isRequired).where(conditions).limit(1) + return 1 == context.update(TABLE) + .set(TABLE.NAME, variable.getName()) + .set(TABLE.DESCRIPTION, variable.getDescription()) + .set(TABLE.DEFAULT_VALUE, + cipherVariableCryptoService.encryptTaskVariableIfNeeded(variable.getType(), variable.getDefaultValue())) + .set(TABLE.IS_CHANGEABLE, isChangeable) + .set(TABLE.IS_REQUIRED, isRequired) + .where(conditions).limit(1) .execute(); } @@ -229,13 +299,28 @@ public int deleteVariableByParentId(long parentId) { @Override public boolean batchInsertVariableWithId(List variableList) { InsertValuesStep8 insertStep = context.insertInto(TABLE).columns(TABLE.ID, TABLE.TEMPLATE_ID, TABLE.NAME, TABLE.TYPE, - TABLE.DEFAULT_VALUE, TABLE.DESCRIPTION, TABLE.IS_CHANGEABLE, TABLE.IS_REQUIRED); + UByte> insertStep = context.insertInto(TABLE). + columns( + TABLE.ID, + TABLE.TEMPLATE_ID, + TABLE.NAME, + TABLE.TYPE, + TABLE.DEFAULT_VALUE, + TABLE.DESCRIPTION, + TABLE.IS_CHANGEABLE, + TABLE.IS_REQUIRED + ); for (TaskVariableDTO variable : variableList) { - insertStep = insertStep.values(JooqDataTypeUtil.buildULong(variable.getId()), - ULong.valueOf(variable.getTemplateId()), variable.getName(), - UByte.valueOf(variable.getType().getType()), variable.getDefaultValue(), variable.getDescription(), - getChangeable(variable.getChangeable()), getRequired(variable.getRequired())); + insertStep = insertStep.values( + JooqDataTypeUtil.buildULong(variable.getId()), + ULong.valueOf(variable.getTemplateId()), + variable.getName(), + UByte.valueOf(variable.getType().getType()), + cipherVariableCryptoService.encryptTaskVariableIfNeeded(variable.getType(), variable.getDefaultValue()), + variable.getDescription(), + getChangeable(variable.getChangeable()), + getRequired(variable.getRequired()) + ); } return insertStep.execute() > 0; } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/whiteip/ActionScopeDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/whiteip/ActionScopeDAO.java index 4db00b57f9..36b04b6ced 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/whiteip/ActionScopeDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/whiteip/ActionScopeDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -31,11 +31,6 @@ import java.util.List; public interface ActionScopeDAO { - Long insertActionScope(DSLContext dslContext, ActionScopeDTO actionScopeDTO); - - int deleteActionScopeById(DSLContext dslContext, Long id); - - ActionScopeDTO getActionScopeById(Long id); ActionScopeDTO getActionScopeByCode(String code); @@ -43,7 +38,5 @@ public interface ActionScopeDAO { List listActionScopeDTO(); - int updateActionScopeById(DSLContext dslContext, ActionScopeDTO actionScopeDTO); - List getActionScopeByIds(List scopeIdList); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/whiteip/WhiteIPActionScopeDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/whiteip/WhiteIPActionScopeDAO.java index 0c31605636..dbe0b5ba90 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/whiteip/WhiteIPActionScopeDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/whiteip/WhiteIPActionScopeDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,22 +25,15 @@ package com.tencent.bk.job.manage.dao.whiteip; import com.tencent.bk.job.manage.model.dto.whiteip.WhiteIPActionScopeDTO; -import org.jooq.DSLContext; import java.util.List; public interface WhiteIPActionScopeDAO { - Long insertWhiteIPActionScope(DSLContext dslContext, WhiteIPActionScopeDTO whiteIPActionScopeDTO); + void insertWhiteIPActionScope(WhiteIPActionScopeDTO whiteIPActionScopeDTO); - int deleteWhiteIPActionScopeById(DSLContext dslContext, Long id); + int deleteWhiteIPActionScopeByRecordId(Long recordId); - int deleteWhiteIPActionScopeByRecordId(DSLContext dslContext, Long recordId); + List getWhiteIPActionScopeByRecordId(Long recordId); - WhiteIPActionScopeDTO getWhiteIPActionScopeById(DSLContext dslContext, Long id); - - List getWhiteIPActionScopeByRecordId(DSLContext dslContext, Long recordId); - - int updateWhiteIPActionScope(DSLContext dslContext, WhiteIPActionScopeDTO whiteIPActionScopeDTO); - - List listWhiteIPActionScopeByRecordIds(DSLContext dslContext, List recordIdList); + List listWhiteIPActionScopeByRecordIds(List recordIdList); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/whiteip/WhiteIPAppRelDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/whiteip/WhiteIPAppRelDAO.java index 9c7ad39c38..ecc2364761 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/whiteip/WhiteIPAppRelDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/whiteip/WhiteIPAppRelDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,18 +25,24 @@ package com.tencent.bk.job.manage.dao.whiteip; import com.tencent.bk.job.manage.model.dto.whiteip.WhiteIPAppRelDTO; -import org.jooq.DSLContext; import java.util.List; public interface WhiteIPAppRelDAO { - int insertWhiteIPAppRel(DSLContext dslContext, String username, Long recordId, Long appId); - int deleteWhiteIPAppRelByRecordId(DSLContext dslContext, Long recordId); + // 查询 + List listAppIdByRecordId(Long recordId); - int deleteWhiteIPAppRelByAppId(DSLContext dslContext, Long appId); + List listAppRelByRecordIds(List recordIdList); - List listAppIdByRecordId(DSLContext dslContext, Long recordId); + List listAppRelByAppId(Long appId); - List listAppRelByRecordIds(DSLContext dslContext, List recordIdList); + // 新增 + void insertWhiteIPAppRel(String username, Long recordId, Long appId); + + // 更新 + int updateAppId(Long srcAppId, Long targetAppId); + + // 删除 + int deleteWhiteIPAppRelByRecordId(Long recordId); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/whiteip/WhiteIPIPDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/whiteip/WhiteIPIPDAO.java index fe0396fd10..b586777e76 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/whiteip/WhiteIPIPDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/whiteip/WhiteIPIPDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,22 +25,19 @@ package com.tencent.bk.job.manage.dao.whiteip; import com.tencent.bk.job.manage.model.dto.whiteip.WhiteIPIPDTO; -import org.jooq.DSLContext; import java.util.List; public interface WhiteIPIPDAO { - Long insertWhiteIPIP(DSLContext dslContext, WhiteIPIPDTO whiteIPIPDTO); + void insertWhiteIPIP(WhiteIPIPDTO whiteIPIPDTO); - int deleteWhiteIPIPById(DSLContext dslContext, Long id); + int deleteWhiteIPIPByRecordId(Long recordId); - int deleteWhiteIPIPByRecordId(DSLContext dslContext, Long recordId); + List getWhiteIPIPByRecordId(Long recordId); - WhiteIPIPDTO getWhiteIPIPById(DSLContext dslContext, Long id); + List listWhiteIPIPByRecordIds(List recordIdList); - List getWhiteIPIPByRecordId(DSLContext dslContext, Long recordId); + List listWhiteIPIPWithNullHostId(int start, int limit); - int updateWhiteIPIPById(DSLContext dslContext, WhiteIPIPDTO whiteIPIPDTO); - - List listWhiteIPIPByRecordIds(DSLContext dslContext, List recordIdList); + int updateHostIdById(Long id, Long hostId); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/whiteip/WhiteIPRecordDAO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/whiteip/WhiteIPRecordDAO.java index f0114d9c38..e279297d03 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/whiteip/WhiteIPRecordDAO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/whiteip/WhiteIPRecordDAO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,46 +25,44 @@ package com.tencent.bk.job.manage.dao.whiteip; import com.tencent.bk.job.common.model.BaseSearchCondition; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.manage.model.dto.whiteip.CloudIPDTO; import com.tencent.bk.job.manage.model.dto.whiteip.WhiteIPRecordDTO; import com.tencent.bk.job.manage.model.web.vo.whiteip.WhiteIPRecordVO; -import org.jooq.DSLContext; import java.util.Collection; import java.util.List; public interface WhiteIPRecordDAO { - Long insertWhiteIPRecord(DSLContext dslContext, WhiteIPRecordDTO whiteIPRecordDTO); + Long insertWhiteIPRecord(WhiteIPRecordDTO whiteIPRecordDTO); - int deleteWhiteIPRecordById(DSLContext dslContext, Long id); + int deleteWhiteIPRecordById(Long id); - WhiteIPRecordDTO getWhiteIPRecordById(DSLContext dslContext, Long id); + WhiteIPRecordDTO getWhiteIPRecordById(Long id); - List listWhiteIPRecord(DSLContext dslContext, List ipList, List appIdList, + List listWhiteIPRecord(List ipList, List appIdList, List appNameList, List actionScopeIdList, String creator, String lastModifyUser, BaseSearchCondition baseSearchCondition); - List listAllWhiteIPRecordId(DSLContext dslContext); - - List listWhiteIPRecord(DSLContext dslContext, String partIP, List appIdList, - List actionScopeIdList, BaseSearchCondition baseSearchCondition); - - Long countWhiteIPRecord(DSLContext dslContext, List ipList, List appIdList, + Long countWhiteIPRecord(List ipList, List appIdList, List appNameList, List actionScopeIdList, String creator, String lastModifyUser, BaseSearchCondition baseSearchCondition); - Long countWhiteIPRecord(DSLContext dslContext, String partIP, List appIdList, List actionScopeIdList, - BaseSearchCondition baseSearchCondition); - Long countWhiteIPIP(); - int updateWhiteIPRecordById(DSLContext dslContext, WhiteIPRecordDTO whiteIPRecordDTO); + int updateWhiteIPRecordById(WhiteIPRecordDTO whiteIPRecordDTO); + + List getWhiteIPActionScopes(Collection appIds, String ip, Long cloudAreaId); + + List getWhiteIPActionScopes(Collection appIds, Long hostId); + + List listWhiteIPByAppIds(Collection appIds, Long actionScopeId); - List getWhiteIPActionScopes(DSLContext dslContext, Long appId, String ip, Long cloudAreaId); + List listWhiteIPHost(Collection appIds, Long actionScopeId, Collection hostIds); - List getWhiteIPActionScopes(DSLContext dslContext, Collection appIds, String ip, Long cloudAreaId); + List listWhiteIPHostByIps(Collection appIds, Long actionScopeId, Collection ips); - List listWhiteIPByAppIds(DSLContext dslContext, Collection appIds, Long actionScopeId); + List listWhiteIPHostByIpv6s(Collection appIds, Long actionScopeId, Collection ipv6s); - List listAllWhiteIPRecord(DSLContext dslContext); + List listAllWhiteIPRecord(); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/whiteip/impl/ActionScopeDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/whiteip/impl/ActionScopeDAOImpl.java index f661f5e9df..a7d9c36bae 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/whiteip/impl/ActionScopeDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/whiteip/impl/ActionScopeDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,71 +25,34 @@ package com.tencent.bk.job.manage.dao.whiteip.impl; import com.tencent.bk.job.common.i18n.service.MessageI18nService; -import com.tencent.bk.job.manage.common.consts.whiteip.ActionScopeEnum; +import com.tencent.bk.job.manage.api.common.constants.whiteip.ActionScopeEnum; import com.tencent.bk.job.manage.dao.whiteip.ActionScopeDAO; import com.tencent.bk.job.manage.model.dto.whiteip.ActionScopeDTO; +import com.tencent.bk.job.manage.model.tables.ActionScope; +import com.tencent.bk.job.manage.model.tables.records.ActionScopeRecord; import com.tencent.bk.job.manage.model.web.vo.whiteip.ActionScopeVO; import lombok.val; import org.jooq.DSLContext; -import org.jooq.Record; -import org.jooq.generated.tables.ActionScope; -import org.jooq.generated.tables.records.ActionScopeRecord; -import org.jooq.types.ULong; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; -import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; @Repository public class ActionScopeDAOImpl implements ActionScopeDAO { - private DSLContext defaultDslContext; + private final DSLContext defaultDslContext; private static final ActionScope T_ACTION_SCOPE = ActionScope.ACTION_SCOPE; private final MessageI18nService i18nService; @Autowired - public ActionScopeDAOImpl(DSLContext dslContext, MessageI18nService i18nService) { + public ActionScopeDAOImpl(@Qualifier("job-manage-dsl-context") DSLContext dslContext, + MessageI18nService i18nService) { this.defaultDslContext = dslContext; this.i18nService = i18nService; } - @Override - public Long insertActionScope(DSLContext dslContext, ActionScopeDTO actionScopeDTO) { - Record record = dslContext.insertInto(T_ACTION_SCOPE, - T_ACTION_SCOPE.NAME, - T_ACTION_SCOPE.DESCRIPTION, - T_ACTION_SCOPE.CREATOR, - T_ACTION_SCOPE.CREATE_TIME, - T_ACTION_SCOPE.LAST_MODIFY_USER, - T_ACTION_SCOPE.LAST_MODIFY_TIME - ).values( - actionScopeDTO.getName(), - actionScopeDTO.getDescription(), - actionScopeDTO.getCreator(), - ULong.valueOf(actionScopeDTO.getCreateTime()), - actionScopeDTO.getLastModifier(), - ULong.valueOf(actionScopeDTO.getLastModifyTime()) - ).returning(T_ACTION_SCOPE.ID) - .fetchOne(); - return record.get(T_ACTION_SCOPE.ID); - } - - @Override - public int deleteActionScopeById(DSLContext dslContext, Long id) { - return dslContext.deleteFrom(T_ACTION_SCOPE).where( - T_ACTION_SCOPE.ID.eq(id) - ).execute(); - } - - @Override - public ActionScopeDTO getActionScopeById(Long id) { - ActionScopeRecord record = defaultDslContext.selectFrom(T_ACTION_SCOPE).where( - T_ACTION_SCOPE.ID.eq(id) - ).fetchOne(); - return convert(record); - } - @Override public ActionScopeDTO getActionScopeByCode(String code) { ActionScopeRecord record = defaultDslContext.selectFrom(T_ACTION_SCOPE).where( @@ -120,9 +83,6 @@ val record = defaultDslContext.selectFrom(T_ACTION_SCOPE).where( @Override public List listActionScopeDTO() { val records = defaultDslContext.selectFrom(T_ACTION_SCOPE).fetch(); - if (records == null) { - return new ArrayList<>(); - } return records.stream().map(record -> new ActionScopeDTO( record.getId(), @@ -137,25 +97,12 @@ public List listActionScopeDTO() { ).collect(Collectors.toList()); } - @Override - public int updateActionScopeById(DSLContext dslContext, ActionScopeDTO actionScopeDTO) { - return dslContext.update(T_ACTION_SCOPE) - .set(T_ACTION_SCOPE.NAME, actionScopeDTO.getName()) - .set(T_ACTION_SCOPE.DESCRIPTION, actionScopeDTO.getDescription()) - .set(T_ACTION_SCOPE.CREATOR, actionScopeDTO.getCreator()) - .set(T_ACTION_SCOPE.CREATE_TIME, ULong.valueOf(actionScopeDTO.getCreateTime())) - .set(T_ACTION_SCOPE.LAST_MODIFY_USER, actionScopeDTO.getLastModifier()) - .set(T_ACTION_SCOPE.LAST_MODIFY_TIME, ULong.valueOf(actionScopeDTO.getLastModifyTime())) - .where(T_ACTION_SCOPE.ID.eq(actionScopeDTO.getId())) - .execute(); - } - @Override public List getActionScopeByIds(List scopeIdList) { val records = defaultDslContext.select( - T_ACTION_SCOPE.ID, - T_ACTION_SCOPE.CODE - ).from(T_ACTION_SCOPE) + T_ACTION_SCOPE.ID, + T_ACTION_SCOPE.CODE + ).from(T_ACTION_SCOPE) .where(T_ACTION_SCOPE.ID.in(scopeIdList)).fetch(); return records.stream().map(record -> new ActionScopeDTO( diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/whiteip/impl/WhiteIPActionScopeDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/whiteip/impl/WhiteIPActionScopeDAOImpl.java index 15b15954dc..6fcf5a6483 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/whiteip/impl/WhiteIPActionScopeDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/whiteip/impl/WhiteIPActionScopeDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,14 +26,14 @@ import com.tencent.bk.job.manage.dao.whiteip.WhiteIPActionScopeDAO; import com.tencent.bk.job.manage.model.dto.whiteip.WhiteIPActionScopeDTO; +import com.tencent.bk.job.manage.model.tables.WhiteIpActionScope; import lombok.val; import org.jooq.DSLContext; -import org.jooq.Record; -import org.jooq.generated.tables.WhiteIpActionScope; import org.jooq.types.ULong; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; -import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; @@ -41,9 +41,16 @@ public class WhiteIPActionScopeDAOImpl implements WhiteIPActionScopeDAO { private static final WhiteIpActionScope T_WHITE_IP_ACTION_SCOPE = WhiteIpActionScope.WHITE_IP_ACTION_SCOPE; + private final DSLContext dslContext; + + @Autowired + public WhiteIPActionScopeDAOImpl(@Qualifier("job-manage-dsl-context") DSLContext dslContext) { + this.dslContext = dslContext; + } + @Override - public Long insertWhiteIPActionScope(DSLContext dslContext, WhiteIPActionScopeDTO whiteIPActionScopeDTO) { - Record record = dslContext.insertInto(T_WHITE_IP_ACTION_SCOPE, + public void insertWhiteIPActionScope(WhiteIPActionScopeDTO whiteIPActionScopeDTO) { + dslContext.insertInto(T_WHITE_IP_ACTION_SCOPE, T_WHITE_IP_ACTION_SCOPE.RECORD_ID, T_WHITE_IP_ACTION_SCOPE.ACTION_SCOPE_ID, T_WHITE_IP_ACTION_SCOPE.CREATOR, @@ -59,51 +66,20 @@ public Long insertWhiteIPActionScope(DSLContext dslContext, WhiteIPActionScopeDT ULong.valueOf(whiteIPActionScopeDTO.getLastModifyTime()) ).returning(T_WHITE_IP_ACTION_SCOPE.ID) .fetchOne(); - return record.get(T_WHITE_IP_ACTION_SCOPE.ID); } @Override - public int deleteWhiteIPActionScopeById(DSLContext dslContext, Long id) { - return dslContext.deleteFrom(T_WHITE_IP_ACTION_SCOPE).where( - T_WHITE_IP_ACTION_SCOPE.ID.eq(id) - ).execute(); - } - - @Override - public int deleteWhiteIPActionScopeByRecordId(DSLContext dslContext, Long recordId) { + public int deleteWhiteIPActionScopeByRecordId(Long recordId) { return dslContext.deleteFrom(T_WHITE_IP_ACTION_SCOPE).where( T_WHITE_IP_ACTION_SCOPE.RECORD_ID.eq(recordId) ).execute(); } @Override - public WhiteIPActionScopeDTO getWhiteIPActionScopeById(DSLContext dslContext, Long id) { - val record = dslContext.selectFrom(T_WHITE_IP_ACTION_SCOPE).where( - T_WHITE_IP_ACTION_SCOPE.ID.eq(id) - ).fetchOne(); - if (record == null) { - return null; - } else { - return new WhiteIPActionScopeDTO( - record.getId(), - record.getRecordId(), - record.getActionScopeId(), - record.getCreator(), - record.getCreateTime().longValue(), - record.getLastModifyUser(), - record.getLastModifyTime().longValue() - ); - } - } - - @Override - public List getWhiteIPActionScopeByRecordId(DSLContext dslContext, Long recordId) { + public List getWhiteIPActionScopeByRecordId(Long recordId) { val records = dslContext.selectFrom(T_WHITE_IP_ACTION_SCOPE).where( T_WHITE_IP_ACTION_SCOPE.RECORD_ID.eq(recordId) ).fetch(); - if (records == null) { - return new ArrayList<>(); - } return records.stream().map(record -> new WhiteIPActionScopeDTO( record.getId(), @@ -118,21 +94,7 @@ public List getWhiteIPActionScopeByRecordId(DSLContext ds } @Override - public int updateWhiteIPActionScope(DSLContext dslContext, WhiteIPActionScopeDTO whiteIPActionScopeDTO) { - return dslContext.update(T_WHITE_IP_ACTION_SCOPE) - .set(T_WHITE_IP_ACTION_SCOPE.RECORD_ID, whiteIPActionScopeDTO.getRecordId()) - .set(T_WHITE_IP_ACTION_SCOPE.ACTION_SCOPE_ID, whiteIPActionScopeDTO.getActionScopeId()) - .set(T_WHITE_IP_ACTION_SCOPE.CREATOR, whiteIPActionScopeDTO.getCreator()) - .set(T_WHITE_IP_ACTION_SCOPE.CREATE_TIME, ULong.valueOf(whiteIPActionScopeDTO.getCreateTime())) - .set(T_WHITE_IP_ACTION_SCOPE.LAST_MODIFY_USER, whiteIPActionScopeDTO.getLastModifier()) - .set(T_WHITE_IP_ACTION_SCOPE.LAST_MODIFY_TIME, ULong.valueOf(whiteIPActionScopeDTO.getLastModifyTime())) - .where(T_WHITE_IP_ACTION_SCOPE.ID.eq(whiteIPActionScopeDTO.getId())) - .execute(); - } - - @Override - public List listWhiteIPActionScopeByRecordIds(DSLContext dslContext, - List recordIdList) { + public List listWhiteIPActionScopeByRecordIds(List recordIdList) { val records = dslContext.selectFrom(T_WHITE_IP_ACTION_SCOPE).where( T_WHITE_IP_ACTION_SCOPE.RECORD_ID.in(recordIdList) diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/whiteip/impl/WhiteIPAppRelDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/whiteip/impl/WhiteIPAppRelDAOImpl.java index 7881e0cd3b..12d0699980 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/whiteip/impl/WhiteIPAppRelDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/whiteip/impl/WhiteIPAppRelDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,24 +26,33 @@ import com.tencent.bk.job.manage.dao.whiteip.WhiteIPAppRelDAO; import com.tencent.bk.job.manage.model.dto.whiteip.WhiteIPAppRelDTO; +import com.tencent.bk.job.manage.model.tables.WhiteIpAppRel; +import com.tencent.bk.job.manage.model.tables.records.WhiteIpAppRelRecord; import lombok.val; import org.jooq.DSLContext; -import org.jooq.generated.tables.WhiteIpAppRel; -import org.jooq.generated.tables.records.WhiteIpAppRelRecord; import org.jooq.types.ULong; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; -import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; @Repository public class WhiteIPAppRelDAOImpl implements WhiteIPAppRelDAO { + private static final WhiteIpAppRel T_WHITE_IP_APP_REL = WhiteIpAppRel.WHITE_IP_APP_REL; + private final DSLContext dslContext; + + @Autowired + public WhiteIPAppRelDAOImpl(@Qualifier("job-manage-dsl-context") DSLContext dslContext) { + this.dslContext = dslContext; + } + @Override - public int insertWhiteIPAppRel(DSLContext dslContext, String username, Long recordId, Long appId) { - return dslContext.insertInto(T_WHITE_IP_APP_REL, + public void insertWhiteIPAppRel(String username, Long recordId, Long appId) { + dslContext.insertInto(T_WHITE_IP_APP_REL, T_WHITE_IP_APP_REL.RECORD_ID, T_WHITE_IP_APP_REL.APP_ID, T_WHITE_IP_APP_REL.CREATOR, @@ -57,43 +66,62 @@ public int insertWhiteIPAppRel(DSLContext dslContext, String username, Long reco } @Override - public int deleteWhiteIPAppRelByRecordId(DSLContext dslContext, Long recordId) { - return dslContext.deleteFrom(T_WHITE_IP_APP_REL).where( - T_WHITE_IP_APP_REL.RECORD_ID.eq(recordId) - ).execute(); + public int updateAppId(Long srcAppId, Long targetAppId) { + return dslContext.update(T_WHITE_IP_APP_REL) + .set(T_WHITE_IP_APP_REL.APP_ID, targetAppId) + .where(T_WHITE_IP_APP_REL.APP_ID.eq(srcAppId)) + .execute(); } @Override - public int deleteWhiteIPAppRelByAppId(DSLContext dslContext, Long appId) { + public int deleteWhiteIPAppRelByRecordId(Long recordId) { return dslContext.deleteFrom(T_WHITE_IP_APP_REL).where( - T_WHITE_IP_APP_REL.APP_ID.eq(appId) + T_WHITE_IP_APP_REL.RECORD_ID.eq(recordId) ).execute(); } @Override - public List listAppIdByRecordId(DSLContext dslContext, Long recordId) { + public List listAppIdByRecordId(Long recordId) { val records = dslContext.selectFrom(T_WHITE_IP_APP_REL).where( T_WHITE_IP_APP_REL.RECORD_ID.eq(recordId) ).fetch(); - if (records == null) { - return new ArrayList<>(); - } return records.stream().map(WhiteIpAppRelRecord::getAppId).collect(Collectors.toList()); } @Override - public List listAppRelByRecordIds(DSLContext dslContext, List recordIdList) { + public List listAppRelByRecordIds(List recordIdList) { val records = dslContext.select(T_WHITE_IP_APP_REL.APP_ID, T_WHITE_IP_APP_REL.RECORD_ID).from(T_WHITE_IP_APP_REL).where( T_WHITE_IP_APP_REL.RECORD_ID.in(recordIdList) ).fetch(); return records.stream().map(record -> new WhiteIPAppRelDTO( - record.get(T_WHITE_IP_APP_REL.RECORD_ID).longValue(), - record.get(T_WHITE_IP_APP_REL.APP_ID).longValue(), + record.get(T_WHITE_IP_APP_REL.RECORD_ID), + record.get(T_WHITE_IP_APP_REL.APP_ID), null, null ) ).collect(Collectors.toList()); } + + @Override + public List listAppRelByAppId(Long appId) { + val records = + dslContext.select( + T_WHITE_IP_APP_REL.APP_ID, + T_WHITE_IP_APP_REL.RECORD_ID, + T_WHITE_IP_APP_REL.CREATOR, + T_WHITE_IP_APP_REL.CREATE_TIME + ).from(T_WHITE_IP_APP_REL) + .where(T_WHITE_IP_APP_REL.APP_ID.eq(appId)) + .fetch(); + return records.stream().map(record -> + new WhiteIPAppRelDTO( + record.get(T_WHITE_IP_APP_REL.RECORD_ID), + record.get(T_WHITE_IP_APP_REL.APP_ID), + record.get(T_WHITE_IP_APP_REL.CREATOR), + record.get(T_WHITE_IP_APP_REL.CREATE_TIME).longValue() + ) + ).collect(Collectors.toList()); + } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/whiteip/impl/WhiteIPIPDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/whiteip/impl/WhiteIPIPDAOImpl.java index bce64a36b1..1b52c625e1 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/whiteip/impl/WhiteIPIPDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/whiteip/impl/WhiteIPIPDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,11 +26,13 @@ import com.tencent.bk.job.manage.dao.whiteip.WhiteIPIPDAO; import com.tencent.bk.job.manage.model.dto.whiteip.WhiteIPIPDTO; +import com.tencent.bk.job.manage.model.tables.WhiteIpIp; +import com.tencent.bk.job.manage.model.tables.records.WhiteIpIpRecord; import lombok.val; import org.jooq.DSLContext; -import org.jooq.Record; -import org.jooq.generated.tables.WhiteIpIp; import org.jooq.types.ULong; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Repository; import java.util.List; @@ -40,11 +42,20 @@ public class WhiteIPIPDAOImpl implements WhiteIPIPDAO { private static final WhiteIpIp T_WHITE_IP_IP = WhiteIpIp.WHITE_IP_IP; + private final DSLContext dslContext; + + @Autowired + public WhiteIPIPDAOImpl(@Qualifier("job-manage-dsl-context") DSLContext dslContext) { + this.dslContext = dslContext; + } + @Override - public Long insertWhiteIPIP(DSLContext dslContext, WhiteIPIPDTO whiteIPIPDTO) { - Record record = dslContext.insertInto(T_WHITE_IP_IP, + public void insertWhiteIPIP(WhiteIPIPDTO whiteIPIPDTO) { + dslContext.insertInto(T_WHITE_IP_IP, T_WHITE_IP_IP.RECORD_ID, + T_WHITE_IP_IP.HOST_ID, T_WHITE_IP_IP.IP, + T_WHITE_IP_IP.IP_V6, T_WHITE_IP_IP.CLOUD_AREA_ID, T_WHITE_IP_IP.CREATOR, T_WHITE_IP_IP.CREATE_TIME, @@ -52,7 +63,9 @@ public Long insertWhiteIPIP(DSLContext dslContext, WhiteIPIPDTO whiteIPIPDTO) { T_WHITE_IP_IP.LAST_MODIFY_TIME ).values( whiteIPIPDTO.getRecordId(), + whiteIPIPDTO.getHostId(), whiteIPIPDTO.getIp(), + whiteIPIPDTO.getIpv6(), whiteIPIPDTO.getCloudAreaId(), whiteIPIPDTO.getCreator(), ULong.valueOf(whiteIPIPDTO.getCreateTime()), @@ -60,93 +73,60 @@ public Long insertWhiteIPIP(DSLContext dslContext, WhiteIPIPDTO whiteIPIPDTO) { ULong.valueOf(whiteIPIPDTO.getLastModifyTime()) ).returning(T_WHITE_IP_IP.ID) .fetchOne(); - return record.get(T_WHITE_IP_IP.ID); } @Override - public int deleteWhiteIPIPById(DSLContext dslContext, Long id) { + public int deleteWhiteIPIPByRecordId(Long recordId) { return dslContext.deleteFrom(T_WHITE_IP_IP).where( - T_WHITE_IP_IP.ID.eq(id) + T_WHITE_IP_IP.RECORD_ID.eq(recordId) ).execute(); } @Override - public int deleteWhiteIPIPByRecordId(DSLContext dslContext, Long recordId) { - return dslContext.deleteFrom(T_WHITE_IP_IP).where( + public List getWhiteIPIPByRecordId(Long recordId) { + val records = dslContext.selectFrom(T_WHITE_IP_IP).where( T_WHITE_IP_IP.RECORD_ID.eq(recordId) - ).execute(); + ).fetch(); + return records.stream().map(this::convert).collect(Collectors.toList()); } @Override - public WhiteIPIPDTO getWhiteIPIPById(DSLContext dslContext, Long id) { - val record = dslContext.selectFrom(T_WHITE_IP_IP).where( - T_WHITE_IP_IP.ID.eq(id) - ).fetchOne(); - if (record == null) { - return null; - } else { - return new WhiteIPIPDTO( - record.getId(), - record.getRecordId(), - record.getCloudAreaId(), - record.getIp(), - record.getCreator(), - record.getCreateTime().longValue(), - record.getLastModifyUser(), - record.getLastModifyTime().longValue() - ); - } + public List listWhiteIPIPByRecordIds(List recordIdList) { + val records = dslContext.selectFrom(T_WHITE_IP_IP) + .where(T_WHITE_IP_IP.RECORD_ID.in(recordIdList)) + .fetch(); + return records.stream().map(this::convert).collect(Collectors.toList()); } @Override - public List getWhiteIPIPByRecordId(DSLContext dslContext, Long recordId) { - val records = dslContext.selectFrom(T_WHITE_IP_IP).where( - T_WHITE_IP_IP.RECORD_ID.eq(recordId) - ).fetch(); - return records.stream().map(record -> - new WhiteIPIPDTO( - record.getId(), - record.getRecordId(), - record.getCloudAreaId(), - record.getIp(), - record.getCreator(), - record.getCreateTime().longValue(), - record.getLastModifyUser(), - record.getLastModifyTime().longValue() - ) - ).collect(Collectors.toList()); + public List listWhiteIPIPWithNullHostId(int start, int limit) { + val records = dslContext.selectFrom(T_WHITE_IP_IP) + .where(T_WHITE_IP_IP.HOST_ID.isNull()) + .limit(start, limit) + .fetch(); + return records.stream().map(this::convert).collect(Collectors.toList()); } @Override - public int updateWhiteIPIPById(DSLContext dslContext, WhiteIPIPDTO whiteIPIPDTO) { + public int updateHostIdById(Long id, Long hostId) { return dslContext.update(T_WHITE_IP_IP) - .set(T_WHITE_IP_IP.RECORD_ID, whiteIPIPDTO.getRecordId()) - .set(T_WHITE_IP_IP.IP, whiteIPIPDTO.getIp()) - .set(T_WHITE_IP_IP.CREATOR, whiteIPIPDTO.getCreator()) - .set(T_WHITE_IP_IP.CREATE_TIME, ULong.valueOf(whiteIPIPDTO.getCreateTime())) - .set(T_WHITE_IP_IP.LAST_MODIFY_USER, whiteIPIPDTO.getLastModifier()) - .set(T_WHITE_IP_IP.LAST_MODIFY_TIME, ULong.valueOf(whiteIPIPDTO.getLastModifyTime())) - .where(T_WHITE_IP_IP.ID.eq(whiteIPIPDTO.getId())) + .set(T_WHITE_IP_IP.HOST_ID, hostId) + .where(T_WHITE_IP_IP.ID.eq(id)) .execute(); } - @Override - public List listWhiteIPIPByRecordIds(DSLContext dslContext, List recordIdList) { - val records = - dslContext.selectFrom(T_WHITE_IP_IP).where( - T_WHITE_IP_IP.RECORD_ID.in(recordIdList) - ).fetch(); - return records.stream().map(record -> - new WhiteIPIPDTO( - record.getId(), - record.getRecordId(), - record.getCloudAreaId(), - record.getIp(), - record.getCreator(), - record.getCreateTime().longValue(), - record.getLastModifyUser(), - record.getLastModifyTime().longValue() - ) - ).collect(Collectors.toList()); + private WhiteIPIPDTO convert(WhiteIpIpRecord record) { + return new WhiteIPIPDTO( + record.getId(), + record.getRecordId(), + record.getCloudAreaId(), + record.getHostId(), + record.getIp(), + record.getIpV6(), + record.getCreator(), + record.getCreateTime().longValue(), + record.getLastModifyUser(), + record.getLastModifyTime().longValue() + ); } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/whiteip/impl/WhiteIPRecordDAOImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/whiteip/impl/WhiteIPRecordDAOImpl.java index fbdecc230f..37ec594b6c 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/whiteip/impl/WhiteIPRecordDAOImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/dao/whiteip/impl/WhiteIPRecordDAOImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,8 +25,12 @@ package com.tencent.bk.job.manage.dao.whiteip.impl; import com.google.common.collect.Lists; +import com.tencent.bk.job.common.constant.JobConstants; import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.dto.ApplicationDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.common.mysql.JobTransactional; +import com.tencent.bk.job.common.mysql.util.JooqDataTypeUtil; import com.tencent.bk.job.common.util.CustomCollectionUtils; import com.tencent.bk.job.manage.dao.ApplicationDAO; import com.tencent.bk.job.manage.dao.whiteip.ActionScopeDAO; @@ -39,8 +43,14 @@ import com.tencent.bk.job.manage.model.dto.whiteip.WhiteIPAppRelDTO; import com.tencent.bk.job.manage.model.dto.whiteip.WhiteIPIPDTO; import com.tencent.bk.job.manage.model.dto.whiteip.WhiteIPRecordDTO; -import com.tencent.bk.job.manage.model.web.vo.AppVO; +import com.tencent.bk.job.manage.model.tables.ActionScope; +import com.tencent.bk.job.manage.model.tables.Application; +import com.tencent.bk.job.manage.model.tables.WhiteIpActionScope; +import com.tencent.bk.job.manage.model.tables.WhiteIpAppRel; +import com.tencent.bk.job.manage.model.tables.WhiteIpIp; +import com.tencent.bk.job.manage.model.tables.WhiteIpRecord; import com.tencent.bk.job.manage.model.web.vo.whiteip.ActionScopeVO; +import com.tencent.bk.job.manage.model.web.vo.whiteip.ScopeVO; import com.tencent.bk.job.manage.model.web.vo.whiteip.WhiteIPRecordVO; import lombok.extern.slf4j.Slf4j; import lombok.val; @@ -54,40 +64,38 @@ import org.jooq.Result; import org.jooq.TableField; import org.jooq.conf.ParamType; -import org.jooq.generated.tables.ActionScope; -import org.jooq.generated.tables.Application; -import org.jooq.generated.tables.WhiteIpActionScope; -import org.jooq.generated.tables.WhiteIpAppRel; -import org.jooq.generated.tables.WhiteIpIp; -import org.jooq.generated.tables.WhiteIpRecord; import org.jooq.impl.DSL; import org.jooq.types.ULong; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.dao.DataRetrievalFailureException; import org.springframework.stereotype.Repository; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.stream.Collectors; -import static com.tencent.bk.job.manage.common.consts.whiteip.Keys.KEY_ACTION_SCOPE_ID_LIST; -import static com.tencent.bk.job.manage.common.consts.whiteip.Keys.KEY_APP_ID_LIST; -import static com.tencent.bk.job.manage.common.consts.whiteip.Keys.KEY_APP_NAME; -import static com.tencent.bk.job.manage.common.consts.whiteip.Keys.KEY_APP_TYPE; -import static com.tencent.bk.job.manage.common.consts.whiteip.Keys.KEY_CLOUD_AREA_ID; -import static com.tencent.bk.job.manage.common.consts.whiteip.Keys.KEY_CREATE_TIME; -import static com.tencent.bk.job.manage.common.consts.whiteip.Keys.KEY_CREATOR; -import static com.tencent.bk.job.manage.common.consts.whiteip.Keys.KEY_ID; -import static com.tencent.bk.job.manage.common.consts.whiteip.Keys.KEY_IP; -import static com.tencent.bk.job.manage.common.consts.whiteip.Keys.KEY_IP_LIST; -import static com.tencent.bk.job.manage.common.consts.whiteip.Keys.KEY_LAST_MODIFY_TIME; -import static com.tencent.bk.job.manage.common.consts.whiteip.Keys.KEY_LAST_MODIFY_USER; -import static com.tencent.bk.job.manage.common.consts.whiteip.Keys.KEY_REMARK; +import static com.tencent.bk.job.manage.api.common.constants.whiteip.Keys.KEY_ACTION_SCOPE_ID_LIST; +import static com.tencent.bk.job.manage.api.common.constants.whiteip.Keys.KEY_APP_ID_LIST; +import static com.tencent.bk.job.manage.api.common.constants.whiteip.Keys.KEY_APP_NAME; +import static com.tencent.bk.job.manage.api.common.constants.whiteip.Keys.KEY_APP_TYPE; +import static com.tencent.bk.job.manage.api.common.constants.whiteip.Keys.KEY_CLOUD_AREA_ID; +import static com.tencent.bk.job.manage.api.common.constants.whiteip.Keys.KEY_CREATE_TIME; +import static com.tencent.bk.job.manage.api.common.constants.whiteip.Keys.KEY_CREATOR; +import static com.tencent.bk.job.manage.api.common.constants.whiteip.Keys.KEY_HOST_ID; +import static com.tencent.bk.job.manage.api.common.constants.whiteip.Keys.KEY_ID; +import static com.tencent.bk.job.manage.api.common.constants.whiteip.Keys.KEY_IP; +import static com.tencent.bk.job.manage.api.common.constants.whiteip.Keys.KEY_IPV6; +import static com.tencent.bk.job.manage.api.common.constants.whiteip.Keys.KEY_IP_LIST; +import static com.tencent.bk.job.manage.api.common.constants.whiteip.Keys.KEY_LAST_MODIFY_TIME; +import static com.tencent.bk.job.manage.api.common.constants.whiteip.Keys.KEY_LAST_MODIFY_USER; +import static com.tencent.bk.job.manage.api.common.constants.whiteip.Keys.KEY_REMARK; @Slf4j @Repository @@ -98,27 +106,32 @@ public class WhiteIPRecordDAOImpl implements WhiteIPRecordDAO { private static final WhiteIpIp T_WHITE_IP_IP = WhiteIpIp.WHITE_IP_IP; private static final WhiteIpActionScope T_WHITE_IP_ACTION_SCOPE = WhiteIpActionScope.WHITE_IP_ACTION_SCOPE; private static final ActionScope T_ACTION_SCOPE = ActionScope.ACTION_SCOPE; - private final DSLContext defaultContext; - private WhiteIPIPDAO whiteIPIPDAO; - private WhiteIPAppRelDAO whiteIPAppRelDAO; - private ActionScopeDAO actionScopeDAO; - private WhiteIPActionScopeDAO whiteIPActionScopeDAO; - private ApplicationDAO applicationDAO; + private final DSLContext dslContext; + private final WhiteIPIPDAO whiteIPIPDAO; + private final WhiteIPAppRelDAO whiteIPAppRelDAO; + private final ActionScopeDAO actionScopeDAO; + private final WhiteIPActionScopeDAO whiteIPActionScopeDAO; + private final ApplicationDAO applicationDAO; + + private final WhiteIpIp tWhiteIPIP = WhiteIpIp.WHITE_IP_IP.as("tWhiteIPIP"); + private final WhiteIpAppRel tWhiteIPAppRel = WhiteIpAppRel.WHITE_IP_APP_REL.as("tWhiteIPAppRel"); + private final WhiteIpActionScope tWhiteIPActionScope = + WhiteIpActionScope.WHITE_IP_ACTION_SCOPE.as("tWhiteIPActionScope"); @Autowired public WhiteIPRecordDAOImpl(WhiteIPIPDAO whiteIPIPDAO, WhiteIPAppRelDAO whiteIPAppRelDAO, ActionScopeDAO actionScopeDAO, WhiteIPActionScopeDAO whiteIPActionScopeDAO, ApplicationDAO applicationDAO, - @Qualifier("job-manage-dsl-context") DSLContext defaultContext) { + @Qualifier("job-manage-dsl-context") DSLContext dslContext) { this.whiteIPIPDAO = whiteIPIPDAO; this.whiteIPAppRelDAO = whiteIPAppRelDAO; this.actionScopeDAO = actionScopeDAO; this.whiteIPActionScopeDAO = whiteIPActionScopeDAO; this.applicationDAO = applicationDAO; - this.defaultContext = defaultContext; + this.dslContext = dslContext; } - private static OrderField buildOrderField(TableField field, Integer order) { + private static OrderField buildOrderField(TableField field, Integer order) { switch (order) { case 0: return field.desc(); @@ -128,8 +141,10 @@ private static OrderField buildOrderField(TableField field, Integer order) { } } + @SuppressWarnings("all") + @JobTransactional(transactionManager = "jobManageTransactionManager") @Override - public Long insertWhiteIPRecord(DSLContext dslContext, WhiteIPRecordDTO whiteIPRecordDTO) { + public Long insertWhiteIPRecord(WhiteIPRecordDTO whiteIPRecordDTO) { //插入Record表 final Record record = dslContext.insertInto( T_WHITE_IP_RECORD, @@ -145,19 +160,19 @@ public Long insertWhiteIPRecord(DSLContext dslContext, WhiteIPRecordDTO whiteIPR whiteIPRecordDTO.getLastModifier(), ULong.valueOf(whiteIPRecordDTO.getLastModifyTime()) ).returning(T_WHITE_IP_RECORD.ID).fetchOne(); - System.out.println(record); val recordId = record.get(T_WHITE_IP_RECORD.ID); //插入Record-App关联表 - whiteIPRecordDTO.getAppIdList().forEach(appId -> { - whiteIPAppRelDAO.insertWhiteIPAppRel(dslContext, whiteIPRecordDTO.getCreator(), recordId, appId); - }); + whiteIPRecordDTO.getAppIdList().forEach(appId -> + whiteIPAppRelDAO.insertWhiteIPAppRel(whiteIPRecordDTO.getCreator(), recordId, appId)); //插入IP表 whiteIPRecordDTO.getIpList().forEach(ip -> - whiteIPIPDAO.insertWhiteIPIP(dslContext, new WhiteIPIPDTO( + whiteIPIPDAO.insertWhiteIPIP(new WhiteIPIPDTO( null, recordId, ip.getCloudAreaId(), + ip.getHostId(), ip.getIp(), + ip.getIpv6(), whiteIPRecordDTO.getCreator(), System.currentTimeMillis(), whiteIPRecordDTO.getLastModifier(), @@ -166,7 +181,7 @@ public Long insertWhiteIPRecord(DSLContext dslContext, WhiteIPRecordDTO whiteIPR ); //插入ActionScope表 whiteIPRecordDTO.getActionScopeList().forEach(actionScope -> - whiteIPActionScopeDAO.insertWhiteIPActionScope(dslContext, new WhiteIPActionScopeDTO( + whiteIPActionScopeDAO.insertWhiteIPActionScope(new WhiteIPActionScopeDTO( null, recordId, actionScope.getActionScopeId(), @@ -179,35 +194,44 @@ public Long insertWhiteIPRecord(DSLContext dslContext, WhiteIPRecordDTO whiteIPR return recordId; } + @JobTransactional(transactionManager = "jobManageTransactionManager") @Override - public int deleteWhiteIPRecordById(DSLContext dslContext, Long id) { + public int deleteWhiteIPRecordById(Long id) { //删关联表 - whiteIPAppRelDAO.deleteWhiteIPAppRelByRecordId(dslContext, id); + int deletedAppRelNum = whiteIPAppRelDAO.deleteWhiteIPAppRelByRecordId(id); //删IP - whiteIPIPDAO.deleteWhiteIPIPByRecordId(dslContext, id); + int deletedIpNum = whiteIPIPDAO.deleteWhiteIPIPByRecordId(id); //删生效范围 - whiteIPActionScopeDAO.deleteWhiteIPActionScopeByRecordId(dslContext, id); + int deletedActionScopeNum = whiteIPActionScopeDAO.deleteWhiteIPActionScopeByRecordId(id); + log.debug( + "deletedAppRelNum={},deletedIpNum={},deletedActionScopeNum={}", + deletedAppRelNum, + deletedIpNum, + deletedActionScopeNum + ); //删Record - int affectedRowNum = dslContext.deleteFrom( + return dslContext.deleteFrom( T_WHITE_IP_RECORD ).where( T_WHITE_IP_RECORD.ID.eq(id) ).execute(); - return affectedRowNum; } @Override - public WhiteIPRecordDTO getWhiteIPRecordById(DSLContext dslContext, Long id) { + public WhiteIPRecordDTO getWhiteIPRecordById(Long id) { //查业务List - val appIdList = whiteIPAppRelDAO.listAppIdByRecordId(dslContext, id); + val appIdList = whiteIPAppRelDAO.listAppIdByRecordId(id); //查IP List - val ipList = whiteIPIPDAO.getWhiteIPIPByRecordId(dslContext, id); + val ipList = whiteIPIPDAO.getWhiteIPIPByRecordId(id); //查ActionScope List - val actionScopeList = whiteIPActionScopeDAO.getWhiteIPActionScopeByRecordId(dslContext, id); + val actionScopeList = whiteIPActionScopeDAO.getWhiteIPActionScopeByRecordId(id); //查Record val whiteIPRecord = dslContext.selectFrom(T_WHITE_IP_RECORD) .where(T_WHITE_IP_RECORD.ID.eq(id)) .fetchOne(); + if (whiteIPRecord == null) { + throw new DataRetrievalFailureException("cannot find whiteIpRecord by id " + id); + } return new WhiteIPRecordDTO( whiteIPRecord.getId(), appIdList, @@ -221,24 +245,6 @@ public WhiteIPRecordDTO getWhiteIPRecordById(DSLContext dslContext, Long id) { ); } - private List buildConditions(String partIP, List inAppIdList, List inActionScopeIdList) { - val tApplication = Application.APPLICATION.as("tApplication"); - val tWhiteIPIP = WhiteIpIp.WHITE_IP_IP.as("tWhiteIPIP"); - val tActionScope = ActionScope.ACTION_SCOPE.as("tActionScope"); - List conditions = new ArrayList<>(); - //专有Condition构造 - if (partIP != null) { - conditions.add(tWhiteIPIP.IP.like("%" + partIP.trim() + "%")); - } - if (inAppIdList != null && !inAppIdList.isEmpty()) { - conditions.add(tApplication.APP_ID.in(inAppIdList)); - } - if (inActionScopeIdList != null && !inActionScopeIdList.isEmpty()) { - conditions.add(tActionScope.ID.in(inActionScopeIdList)); - } - return conditions; - } - private List buildConditions(List inIpList, List inAppIdList, List appNameList, List actionScopeIdList, String creator, String lastModifyUser) { val tApplication = Application.APPLICATION.as("tApplication"); @@ -281,38 +287,22 @@ private List buildConditions(List inIpList, List inAppI } @Override - public List listAllWhiteIPRecordId(DSLContext dslContext) { - List recordIdList = new ArrayList<>(); - val tWhiteIPRecord = T_WHITE_IP_RECORD.as("tWhiteIPRecord"); - val query = dslContext.select( - tWhiteIPRecord.ID - ).from(tWhiteIPRecord); - val records = query.fetch(); - if (CollectionUtils.isNotEmpty(records)) { - recordIdList = records.map(it -> it.get(tWhiteIPRecord.ID)).parallelStream().collect(Collectors.toList()); - } - return recordIdList; - } - - @Override - public List listWhiteIPRecord(DSLContext dslContext, String partIP, List inAppIdList, - List inActionScopeIdList, - BaseSearchCondition baseSearchCondition) { - List conditions = buildConditions(partIP, inAppIdList, inActionScopeIdList); - return listWhiteIPRecordByConditions(dslContext, conditions, baseSearchCondition); - } - - @Override - public List listWhiteIPRecord(DSLContext dslContext, List ipList, List appIdList, + public List listWhiteIPRecord(List ipList, List appIdList, List appNameList, List actionScopeIdList, String creator, String lastModifyUser, BaseSearchCondition baseSearchCondition) { List conditions = buildConditions(ipList, appIdList, appNameList, actionScopeIdList, creator, lastModifyUser); - return listWhiteIPRecordByConditions(dslContext, conditions, baseSearchCondition); + return listWhiteIPRecordByConditions(conditions, baseSearchCondition); + } + + private boolean isAllScope(List appIdList) { + return appIdList != null + && appIdList.size() > 0 + && new HashSet<>(appIdList).contains(JobConstants.PUBLIC_APP_ID); } - private List listWhiteIPRecordByConditions(DSLContext dslContext, List conditions, + private List listWhiteIPRecordByConditions(List conditions, BaseSearchCondition baseSearchCondition) { val tApplication = Application.APPLICATION.as("tApplication"); val tWhiteIPRecord = T_WHITE_IP_RECORD.as("tWhiteIPRecord"); @@ -336,29 +326,29 @@ private List listWhiteIPRecordByConditions(DSLContext dslContex int start = baseSearchCondition.getStartOrDefault(0); int length = baseSearchCondition.getLengthOrDefault(10); val query = dslContext.select( - tWhiteIPRecord.ID.as(KEY_ID), - DSL.max(tApplication.APP_NAME).as(KEY_APP_NAME), - DSL.max(tApplication.APP_TYPE).as(KEY_APP_TYPE), - DSL.max(tWhiteIPRecord.REMARK).as(KEY_REMARK), - DSL.max(tWhiteIPRecord.CREATOR).as(KEY_CREATOR), - DSL.max(tWhiteIPRecord.CREATE_TIME).as(KEY_CREATE_TIME), - DSL.max(tWhiteIPRecord.LAST_MODIFY_USER).as(KEY_LAST_MODIFY_USER), - DSL.max(tWhiteIPRecord.LAST_MODIFY_TIME).as(KEY_LAST_MODIFY_TIME), - DSL.max(tWhiteIPIP.CLOUD_AREA_ID).as(KEY_CLOUD_AREA_ID), - DSL.groupConcat(tWhiteIPAppRel.APP_ID).as(KEY_APP_ID_LIST), - DSL.groupConcat(tWhiteIPIP.IP).as(KEY_IP_LIST), - DSL.groupConcat(tActionScope.ID).as(KEY_ACTION_SCOPE_ID_LIST) - ).from(tWhiteIPRecord) + tWhiteIPRecord.ID.as(KEY_ID), + DSL.max(tApplication.APP_NAME).as(KEY_APP_NAME), + DSL.max(tApplication.APP_TYPE).as(KEY_APP_TYPE), + DSL.max(tWhiteIPRecord.REMARK).as(KEY_REMARK), + DSL.max(tWhiteIPRecord.CREATOR).as(KEY_CREATOR), + DSL.max(tWhiteIPRecord.CREATE_TIME).as(KEY_CREATE_TIME), + DSL.max(tWhiteIPRecord.LAST_MODIFY_USER).as(KEY_LAST_MODIFY_USER), + DSL.max(tWhiteIPRecord.LAST_MODIFY_TIME).as(KEY_LAST_MODIFY_TIME), + DSL.max(tWhiteIPIP.CLOUD_AREA_ID).as(KEY_CLOUD_AREA_ID), + DSL.groupConcat(tWhiteIPAppRel.APP_ID).as(KEY_APP_ID_LIST), + DSL.groupConcat(tWhiteIPIP.IP).as(KEY_IP_LIST), + DSL.groupConcat(tActionScope.ID).as(KEY_ACTION_SCOPE_ID_LIST) + ).from(tWhiteIPRecord) .join(tWhiteIPIP).on(tWhiteIPRecord.ID.eq(tWhiteIPIP.RECORD_ID)) .leftJoin(tWhiteIPActionScope).on(tWhiteIPRecord.ID.eq(tWhiteIPActionScope.RECORD_ID)) .leftJoin(tActionScope).on(tWhiteIPActionScope.ACTION_SCOPE_ID.eq(tActionScope.ID)) .join(tWhiteIPAppRel).on(tWhiteIPRecord.ID.eq(tWhiteIPAppRel.RECORD_ID)) - .join(tApplication).on(tWhiteIPAppRel.APP_ID.eq(tApplication.APP_ID.cast(Long.class))) + .leftJoin(tApplication).on(tWhiteIPAppRel.APP_ID.eq(tApplication.APP_ID.cast(Long.class))) .where(conditions) .groupBy(tWhiteIPRecord.ID) .orderBy(orderFields) .limit(start, length); - LOG.info(query.getSQL(true)); + LOG.info(query.getSQL(ParamType.INLINED)); val records = query.fetch(); if (records.size() > 0) { return records.map(record -> { @@ -368,66 +358,65 @@ private List listWhiteIPRecordByConditions(DSLContext dslContex actionScopeDAO.getActionScopeVOById(Long.parseLong(actionScopeId)) ).collect(Collectors.toList()); val appIdListStr = (String) record.get(KEY_APP_ID_LIST); - List appIdList = CustomCollectionUtils.getNoDuplicateList(appIdListStr, ","); - List appVOList = appIdList.stream().map(appIdStr -> { - long appId = Long.parseLong(appIdStr); - ApplicationDTO applicationDTO = - applicationDAO.getAppById(appId); - AppVO appVO = new AppVO(); - if (applicationDTO == null) { - appVO.setId(appId); - appVO.setName("[Deleted:" + appId + "]"); - } else { - appVO.setName(applicationDTO.getName()); - appVO.setScopeType(applicationDTO.getScope().getType().getValue()); - appVO.setScopeId(applicationDTO.getScope().getId()); - } - return appVO; - }).collect(Collectors.toList()); + List appIdList = CustomCollectionUtils.getNoDuplicateList(appIdListStr, ",").stream() + .map(Long::parseLong).collect(Collectors.toList()); + boolean allScope = isAllScope(appIdList); + List scopeVOList = null; + if (!allScope) { + scopeVOList = appIdList.stream().map(appId -> { + ApplicationDTO applicationDTO = + applicationDAO.getAppById(appId); + ScopeVO scopeVO = new ScopeVO(); + if (applicationDTO == null) { + scopeVO.setName("[Deleted:" + appId + "]"); + } else { + scopeVO.setName(applicationDTO.getName()); + scopeVO.setScopeType(applicationDTO.getScope().getType().getValue()); + scopeVO.setScopeId(applicationDTO.getScope().getId()); + } + return scopeVO; + }).collect(Collectors.toList()); + } val recordId = (Long) record.get(KEY_ID); - return new WhiteIPRecordVO( - recordId, - (Long) record.get(KEY_CLOUD_AREA_ID), - whiteIPIPDAO.getWhiteIPIPByRecordId(dslContext, recordId).stream().map(WhiteIPIPDTO::getIp).collect(Collectors.toList()), - actionScopeVOList, - appVOList, - (String) record.get(KEY_REMARK), - (String) record.get(KEY_CREATOR), - ((ULong) record.get(KEY_CREATE_TIME)).longValue(), - (String) record.get(KEY_LAST_MODIFY_USER), - ((ULong) record.get(KEY_LAST_MODIFY_TIME)).longValue() + WhiteIPRecordVO whiteIPRecord = new WhiteIPRecordVO(); + whiteIPRecord.setId(recordId); + whiteIPRecord.setCloudAreaId((Long) record.get(KEY_CLOUD_AREA_ID)); + whiteIPRecord.setHostList(whiteIPIPDAO.getWhiteIPIPByRecordId(recordId).stream() + .map(WhiteIPIPDTO::extractWhiteIPHostVO).collect(Collectors.toList())); + whiteIPRecord.setActionScopeList(actionScopeVOList); + whiteIPRecord.setAllScope(allScope); + whiteIPRecord.setScopeList(scopeVOList); + whiteIPRecord.setRemark((String) record.get(KEY_REMARK)); + whiteIPRecord.setCreator((String) record.get(KEY_CREATOR)); + whiteIPRecord.setCreateTime(JooqDataTypeUtil.getLongFromULong((ULong) record.get(KEY_CREATE_TIME))); + whiteIPRecord.setLastModifier((String) record.get(KEY_LAST_MODIFY_USER)); + whiteIPRecord.setLastModifyTime( + JooqDataTypeUtil.getLongFromULong(((ULong) record.get(KEY_LAST_MODIFY_TIME))) ); + return whiteIPRecord; }); } else { return new ArrayList<>(); } } - @Override - public Long countWhiteIPRecord(DSLContext dslContext, String partIP, List inAppIdList, - List inActionScopeIdList, BaseSearchCondition baseSearchCondition) { - List conditions = buildConditions(partIP, inAppIdList, inActionScopeIdList); - return countWhiteIPRecordByConditions(dslContext, conditions, baseSearchCondition); - } - @Override public Long countWhiteIPIP() { val tWhiteIPIP = WhiteIpIp.WHITE_IP_IP.as("tWhiteIPIP"); - val query = defaultContext.selectCount().from(tWhiteIPIP); + val query = dslContext.selectCount().from(tWhiteIPIP); return query.fetchOne(0, Long.class); } @Override - public Long countWhiteIPRecord(DSLContext dslContext, List ipList, List appIdList, + public Long countWhiteIPRecord(List ipList, List appIdList, List appNameList, List actionScopeIdList, String creator, String lastModifyUser, BaseSearchCondition baseSearchCondition) { List conditions = buildConditions(ipList, appIdList, appNameList, actionScopeIdList, creator, lastModifyUser); - return countWhiteIPRecordByConditions(dslContext, conditions, baseSearchCondition); + return countWhiteIPRecordByConditions(conditions); } - private Long countWhiteIPRecordByConditions(DSLContext dslContext, List conditions, - BaseSearchCondition baseSearchCondition) { + private Long countWhiteIPRecordByConditions(List conditions) { val tApplication = Application.APPLICATION.as("tApplication"); val tWhiteIpAppRel = WhiteIpAppRel.WHITE_IP_APP_REL.as("tWhiteIpAppRel"); val tWhiteIPRecord = T_WHITE_IP_RECORD.as("tWhiteIPRecord"); @@ -435,52 +424,53 @@ private Long countWhiteIPRecordByConditions(DSLContext dslContext, List { - whiteIPAppRelDAO.insertWhiteIPAppRel(dslContext, whiteIPRecordDTO.getCreator(), whiteIPRecordDTO.getId(), - appId); - }); + whiteIPRecordDTO.getAppIdList().forEach(appId -> + whiteIPAppRelDAO.insertWhiteIPAppRel( + whiteIPRecordDTO.getCreator(), + whiteIPRecordDTO.getId(), + appId + ) + ); //删除IP表 - whiteIPIPDAO.deleteWhiteIPIPByRecordId(dslContext, whiteIPRecordDTO.getId()); + whiteIPIPDAO.deleteWhiteIPIPByRecordId(whiteIPRecordDTO.getId()); //重新插入IP表 - whiteIPRecordDTO.getIpList().forEach(it -> { - whiteIPIPDAO.insertWhiteIPIP(dslContext, it); - }); + whiteIPRecordDTO.getIpList().forEach(whiteIPIPDAO::insertWhiteIPIP); //删除ActionScope表 - whiteIPActionScopeDAO.deleteWhiteIPActionScopeByRecordId(dslContext, whiteIPRecordDTO.getId()); + whiteIPActionScopeDAO.deleteWhiteIPActionScopeByRecordId(whiteIPRecordDTO.getId()); //重新插入ActionScope表 - whiteIPRecordDTO.getActionScopeList().forEach(it -> { - whiteIPActionScopeDAO.insertWhiteIPActionScope(dslContext, it); - }); + whiteIPRecordDTO.getActionScopeList().forEach(it -> + whiteIPActionScopeDAO.insertWhiteIPActionScope(it) + ); return affectedRowNum; } @Override - public List getWhiteIPActionScopes(DSLContext dslContext, Collection appIds, String ip, + public List getWhiteIPActionScopes(Collection appIds, String ip, Long cloudAreaId) { List conditions = new ArrayList<>(); conditions.add(T_WHITE_IP_APP_REL.APP_ID.in(appIds)); @@ -488,21 +478,20 @@ public List getWhiteIPActionScopes(DSLContext dslContext, Collection getWhiteIPActionScopes(DSLContext dslContext, Long appId, String ip, Long cloudAreaId) { + public List getWhiteIPActionScopes(Collection appIds, Long hostId) { List conditions = new ArrayList<>(); - conditions.add(T_WHITE_IP_APP_REL.APP_ID.eq(appId)); - if (ip != null && !ip.isEmpty()) { - conditions.add(T_WHITE_IP_IP.IP.eq(ip.trim())); + conditions.add(T_WHITE_IP_APP_REL.APP_ID.in(appIds)); + if (hostId != null) { + conditions.add(T_WHITE_IP_IP.HOST_ID.eq(hostId)); } - conditions.add(T_WHITE_IP_IP.CLOUD_AREA_ID.eq(cloudAreaId)); - return getWhiteIPActionScopesByConditions(dslContext, conditions); + return getWhiteIPActionScopesByConditions(conditions); } - private List getWhiteIPActionScopesByConditions(DSLContext dslContext, List conditions) { + private List getWhiteIPActionScopesByConditions(List conditions) { val query = dslContext.selectDistinct(T_ACTION_SCOPE.CODE).from(T_WHITE_IP_APP_REL) .join(T_WHITE_IP_RECORD).on(T_WHITE_IP_APP_REL.RECORD_ID.eq(T_WHITE_IP_RECORD.ID)) .join(T_WHITE_IP_IP).on(T_WHITE_IP_RECORD.ID.eq(T_WHITE_IP_IP.RECORD_ID)) @@ -511,7 +500,7 @@ private List getWhiteIPActionScopesByConditions(DSLContext dslContext, L .where(conditions); try { Result> records = query.fetch(); - if (records == null || records.isEmpty()) { + if (records.isEmpty()) { return Collections.emptyList(); } else { return records.stream().map(Record1::value1).collect(Collectors.toList()); @@ -522,11 +511,8 @@ private List getWhiteIPActionScopesByConditions(DSLContext dslContext, L } } - @Override - public List listWhiteIPByAppIds(DSLContext dslContext, Collection appIds, Long actionScopeId) { - val tWhiteIPIP = WhiteIpIp.WHITE_IP_IP.as("tWhiteIPIP"); - val tWhiteIPAppRel = WhiteIpAppRel.WHITE_IP_APP_REL.as("tWhiteIPAppRel"); - val tWhiteIPActionScope = WhiteIpActionScope.WHITE_IP_ACTION_SCOPE.as("tWhiteIPActionScope"); + private Collection buildConditions(Collection appIds, + Long actionScopeId) { Collection conditions = new ArrayList<>(); if (appIds != null) { conditions.add(tWhiteIPAppRel.APP_ID.in(appIds)); @@ -534,10 +520,46 @@ public List listWhiteIPByAppIds(DSLContext dslContext, Collection buildHostIdsConditions(Collection appIds, + Long actionScopeId, + Collection hostIds) { + Collection conditions = buildConditions(appIds, actionScopeId); + if (hostIds != null) { + conditions.add(tWhiteIPIP.HOST_ID.in(hostIds)); + } + return conditions; + } + + private Collection buildIpsConditions(Collection appIds, + Long actionScopeId, + Collection ips) { + Collection conditions = buildConditions(appIds, actionScopeId); + if (ips != null) { + conditions.add(tWhiteIPIP.IP.in(ips)); + } + return conditions; + } + + private Collection buildIpv6sConditions(Collection appIds, + Long actionScopeId, + Collection ipv6s) { + Collection conditions = buildConditions(appIds, actionScopeId); + if (ipv6s != null) { + conditions.add(tWhiteIPIP.IP_V6.in(ipv6s)); + } + return conditions; + } + + @Override + public List listWhiteIPByAppIds(Collection appIds, Long actionScopeId) { + Collection conditions = buildConditions(appIds, actionScopeId); val query = dslContext.select( - tWhiteIPIP.CLOUD_AREA_ID.as(KEY_CLOUD_AREA_ID), - tWhiteIPIP.IP.as(KEY_IP) - ).from(tWhiteIPAppRel) + tWhiteIPIP.CLOUD_AREA_ID.as(KEY_CLOUD_AREA_ID), + tWhiteIPIP.IP.as(KEY_IP) + ).from(tWhiteIPAppRel) .join(tWhiteIPIP).on(tWhiteIPAppRel.RECORD_ID.eq(tWhiteIPIP.RECORD_ID)) .join(tWhiteIPActionScope).on(tWhiteIPAppRel.RECORD_ID.eq(tWhiteIPActionScope.RECORD_ID)) .where(conditions); @@ -559,28 +581,69 @@ public List listWhiteIPByAppIds(DSLContext dslContext, Collection listAllWhiteIPRecord(DSLContext dslContext) { + public List listWhiteIPHost(Collection appIds, Long actionScopeId, Collection hostIds) { + Collection conditions = buildHostIdsConditions(appIds, actionScopeId, hostIds); + return listWhiteIPHostByConditions(conditions); + } + + @Override + public List listWhiteIPHostByIps(Collection appIds, Long actionScopeId, Collection ips) { + Collection conditions = buildIpsConditions(appIds, actionScopeId, ips); + return listWhiteIPHostByConditions(conditions); + } + + @Override + public List listWhiteIPHostByIpv6s(Collection appIds, Long actionScopeId, Collection ipv6s) { + Collection conditions = buildIpv6sConditions(appIds, actionScopeId, ipv6s); + return listWhiteIPHostByConditions(conditions); + } + + public List listWhiteIPHostByConditions(Collection conditions) { + val query = dslContext.select( + tWhiteIPIP.CLOUD_AREA_ID.as(KEY_CLOUD_AREA_ID), + tWhiteIPIP.HOST_ID.as(KEY_HOST_ID), + tWhiteIPIP.IP.as(KEY_IP), + tWhiteIPIP.IP_V6.as(KEY_IPV6) + ).from(tWhiteIPAppRel) + .join(tWhiteIPIP).on(tWhiteIPAppRel.RECORD_ID.eq(tWhiteIPIP.RECORD_ID)) + .join(tWhiteIPActionScope).on(tWhiteIPAppRel.RECORD_ID.eq(tWhiteIPActionScope.RECORD_ID)) + .where(conditions); + try { + val records = query.fetch(); + return records.map(record -> { + val hostId = (Long) record.get(KEY_HOST_ID); + val cloudId = (Long) record.get(KEY_CLOUD_AREA_ID); + val ip = (String) record.get(KEY_IP); + return new HostDTO(hostId, cloudId, ip); + }); + } catch (Exception e) { + log.error("error query={}", query.getSQL(ParamType.INLINED)); + throw e; + } + } + + @Override + public List listAllWhiteIPRecord() { //查Record val tWhiteIPRecord = T_WHITE_IP_RECORD.as("tWhiteIPRecord"); val whiteIPRecords = dslContext.select( - T_WHITE_IP_RECORD.ID) + T_WHITE_IP_RECORD.ID) .from(T_WHITE_IP_RECORD).fetch(); List whiteIPAppRelList = new ArrayList<>(); List whiteIPIPList = new ArrayList<>(); List whiteIPActionScopeList = new ArrayList<>(); - List recordIdList = new ArrayList<>(); + List recordIdList; if (CollectionUtils.isNotEmpty(whiteIPRecords)) { recordIdList = - whiteIPRecords.map(it -> it.get(tWhiteIPRecord.ID)).parallelStream().collect(Collectors.toList()); + new ArrayList<>(whiteIPRecords.map(it -> it.get(tWhiteIPRecord.ID))); int maxInCount = 1000; List> recordIdsList = Lists.partition(new ArrayList<>(recordIdList), maxInCount); for (List idList : recordIdsList) { - whiteIPAppRelList.addAll(whiteIPAppRelDAO.listAppRelByRecordIds(dslContext, idList)); - whiteIPIPList.addAll(whiteIPIPDAO.listWhiteIPIPByRecordIds(dslContext, idList)); - whiteIPActionScopeList.addAll(whiteIPActionScopeDAO.listWhiteIPActionScopeByRecordIds(dslContext, - idList)); + whiteIPAppRelList.addAll(whiteIPAppRelDAO.listAppRelByRecordIds(idList)); + whiteIPIPList.addAll(whiteIPIPDAO.listWhiteIPIPByRecordIds(idList)); + whiteIPActionScopeList.addAll(whiteIPActionScopeDAO.listWhiteIPActionScopeByRecordIds(idList)); } } @@ -596,7 +659,8 @@ public List listAllWhiteIPRecord(DSLContext dslContext) { new WhiteIPRecordDTO( record.get(tWhiteIPRecord.ID), whiteIPAppRelMap.get(record.get(tWhiteIPRecord.ID)) == null ? new ArrayList<>() : - whiteIPAppRelMap.get(record.get(tWhiteIPRecord.ID)).stream().map(i -> i.getAppId()).collect(Collectors.toList()), + whiteIPAppRelMap.get(record.get(tWhiteIPRecord.ID)).stream() + .map(WhiteIPAppRelDTO::getAppId).collect(Collectors.toList()), null, whiteIPIPMap.get(record.get(tWhiteIPRecord.ID)) == null ? new ArrayList<>() : whiteIPIPMap.get(record.get(tWhiteIPRecord.ID)), diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/app/ApplicationCache.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/app/ApplicationCache.java index f009868ff8..7060f4eb2e 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/app/ApplicationCache.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/app/ApplicationCache.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -57,13 +57,13 @@ public class ApplicationCache { private final ApplicationDAO applicationDAO; - private final RedisTemplate redisTemplate; + private final RedisTemplate redisTemplate; private final String APP_HASH_KEY = "job:manage:apps"; private final String SCOPE_HASH_KEY = "job:manage:scopes"; @Autowired public ApplicationCache(ApplicationDAO applicationDAO, - @Qualifier("jsonRedisTemplate") RedisTemplate redisTemplate) { + @Qualifier("jsonRedisTemplate") RedisTemplate redisTemplate) { this.applicationDAO = applicationDAO; this.redisTemplate = redisTemplate; } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/cache/DangerousRuleCache.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/cache/DangerousRuleCache.java index 6d2988f87d..f971d771d8 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/cache/DangerousRuleCache.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/cache/DangerousRuleCache.java @@ -1,13 +1,36 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + package com.tencent.bk.job.manage.manager.cache; import com.tencent.bk.job.common.util.json.JsonUtils; -import com.tencent.bk.job.manage.common.consts.rule.HighRiskGrammarRuleStatusEnum; +import com.tencent.bk.job.manage.api.common.constants.rule.HighRiskGrammarRuleStatusEnum; import com.tencent.bk.job.manage.dao.globalsetting.DangerousRuleDAO; import com.tencent.bk.job.manage.model.db.DangerousRuleDO; import com.tencent.bk.job.manage.model.dto.globalsetting.DangerousRuleDTO; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; -import org.jooq.DSLContext; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.data.redis.core.RedisTemplate; @@ -23,15 +46,12 @@ @Slf4j public class DangerousRuleCache { public static final String DANGEROUS_RULE_HASH_KEY = "job:manage:dangerousRules"; - private final DSLContext dslContext; private final DangerousRuleDAO dangerousRuleDAO; - private final RedisTemplate redisTemplate; + private final RedisTemplate redisTemplate; @Autowired - public DangerousRuleCache(DSLContext dslContext, - DangerousRuleDAO dangerousRuleDAO, - @Qualifier("jsonRedisTemplate") RedisTemplate redisTemplate) { - this.dslContext = dslContext; + public DangerousRuleCache(DangerousRuleDAO dangerousRuleDAO, + @Qualifier("jsonRedisTemplate") RedisTemplate redisTemplate) { this.dangerousRuleDAO = dangerousRuleDAO; this.redisTemplate = redisTemplate; } @@ -50,8 +70,7 @@ public List listDangerousRuleFromCache(Integer scriptType) { DangerousRuleDTO dangerousRuleQuery = new DangerousRuleDTO(); dangerousRuleQuery.setScriptType(scriptType); dangerousRuleQuery.setStatus(HighRiskGrammarRuleStatusEnum.ENABLED.getCode()); - List dangerousRuleDTOList = dangerousRuleDAO.listDangerousRules(dslContext, - dangerousRuleQuery); + List dangerousRuleDTOList = dangerousRuleDAO.listDangerousRules(dangerousRuleQuery); if (CollectionUtils.isEmpty(dangerousRuleDTOList)) { redisTemplate.opsForHash().put(DANGEROUS_RULE_HASH_KEY, String.valueOf(scriptType), new ArrayList()); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/host/HostCache.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/host/HostCache.java index 1314ef064b..0cac461ed8 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/host/HostCache.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/host/HostCache.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,18 +24,27 @@ package com.tencent.bk.job.manage.manager.host; -import com.tencent.bk.job.common.constant.ResourceScopeTypeEnum; import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; -import com.tencent.bk.job.common.service.AppScopeMappingService; +import com.tencent.bk.job.common.redis.BaseRedisCache; import com.tencent.bk.job.manage.model.db.CacheHostDO; +import io.micrometer.core.instrument.MeterRegistry; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.jetbrains.annotations.NotNull; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.dao.DataAccessException; +import org.springframework.data.redis.core.RedisOperations; import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.data.redis.core.SessionCallback; import org.springframework.stereotype.Component; +import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Objects; +import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -44,16 +53,18 @@ */ @Slf4j @Component -public class HostCache { +public class HostCache extends BaseRedisCache { - private final RedisTemplate redisTemplate; - private final AppScopeMappingService appScopeMappingService; + private final RedisTemplate redisTemplate; + + // 1天过期 + private static final int EXPIRE_DAYS = 1; @Autowired - public HostCache(@Qualifier("jsonRedisTemplate") RedisTemplate redisTemplate, - AppScopeMappingService appScopeMappingService) { + public HostCache(@Qualifier("jsonRedisTemplate") RedisTemplate redisTemplate, + MeterRegistry meterRegistry) { + super(meterRegistry, "HostCache"); this.redisTemplate = redisTemplate; - this.appScopeMappingService = appScopeMappingService; } /** @@ -79,11 +90,31 @@ public List batchGetHostsByHostIds(List hostIds) { } private List getHostsByKeys(List hostKeys) { + long hitCount = 0; + long missCount = 0; try { - return (List) redisTemplate.opsForValue().multiGet(hostKeys); + List results = redisTemplate.opsForValue().multiGet(hostKeys); + // 通过 Object 间接强制转换 List + List foundHosts = results == null ? + Collections.emptyList() : (List) (Object) results; + + // multiGet 获取到的 list 中的元素可能为 null （如果key 不存在) + hitCount = foundHosts.stream().filter(Objects::nonNull).count(); + missCount = hostKeys.size() - hitCount; + + return foundHosts; } catch (Exception e) { log.warn("Batch get host in cache exception", e); + hitCount = 0; + missCount = hostKeys.size(); return Collections.emptyList(); + } finally { + if (hitCount > 0) { + addHits(hitCount); + } + if (missCount > 0) { + addMisses(missCount); + } } } @@ -99,27 +130,38 @@ public void deleteHost(ApplicationHostDTO applicationHostDTO) { redisTemplate.delete(hostIdKey); } + /** + * 批量删除缓存中的主机 + * + * @param hosts 主机集合 + */ + public void batchDeleteHost(Collection hosts) { + if (CollectionUtils.isEmpty(hosts)) { + return; + } + Set hostIpKeys = new HashSet<>(); + Set hostIdKeys = new HashSet<>(); + for (ApplicationHostDTO host : hosts) { + hostIpKeys.add(buildHostIpKey(host)); + hostIdKeys.add(buildHostIdKey(host)); + } + redisTemplate.delete(hostIpKeys); + redisTemplate.delete(hostIdKeys); + } + /** * 更新缓存中的主机 * * @param applicationHostDTO 主机 */ public void addOrUpdateHost(ApplicationHostDTO applicationHostDTO) { - CacheHostDO cacheHost = new CacheHostDO(); - cacheHost.setBizId(applicationHostDTO.getBizId()); - if (applicationHostDTO.getAppId() == null) { - cacheHost.setAppId(appScopeMappingService.getAppIdByScope(ResourceScopeTypeEnum.BIZ.getValue(), - String.valueOf(applicationHostDTO.getBizId()))); - } else { - cacheHost.setAppId(applicationHostDTO.getAppId()); - } - cacheHost.setCloudAreaId(applicationHostDTO.getCloudAreaId()); - cacheHost.setIp(applicationHostDTO.getIp()); - cacheHost.setHostId(applicationHostDTO.getHostId()); + CacheHostDO cacheHost = CacheHostDO.fromApplicationHostDTO(applicationHostDTO); + cacheHost.setCacheTime(System.currentTimeMillis()); + log.info("Update host cache, hostId: {}, host: {}", cacheHost.getHostId(), cacheHost); String hostIpKey = buildHostIpKey(applicationHostDTO); String hostIdKey = buildHostIdKey(applicationHostDTO); - redisTemplate.opsForValue().set(hostIpKey, cacheHost, 1, TimeUnit.DAYS); - redisTemplate.opsForValue().set(hostIdKey, cacheHost, 1, TimeUnit.DAYS); + redisTemplate.opsForValue().set(hostIpKey, cacheHost, EXPIRE_DAYS, TimeUnit.DAYS); + redisTemplate.opsForValue().set(hostIdKey, cacheHost, EXPIRE_DAYS, TimeUnit.DAYS); } /** @@ -127,8 +169,27 @@ public void addOrUpdateHost(ApplicationHostDTO applicationHostDTO) { * * @param hosts 主机列表 */ - public void addOrUpdateHosts(List hosts) { - hosts.forEach(this::addOrUpdateHost); + public void batchAddOrUpdateHosts(List hosts) { + if (CollectionUtils.isEmpty(hosts)) { + return; + } + if (hosts.size() == 1) { + addOrUpdateHost(hosts.get(0)); + return; + } + + long start = System.currentTimeMillis(); + redisTemplate.executePipelined(new SessionCallback() { + @Override + public Object execute(@NotNull RedisOperations operations) throws DataAccessException { + hosts.forEach(host -> addOrUpdateHost(host)); + return null; + } + }); + long cost = System.currentTimeMillis() - start; + if (cost > 1000) { + log.info("BatchAddOrUpdateHosts slow, hostSize: {}, cost: {}", hosts.size(), cost); + } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/script/check/ScriptCheckParam.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/script/check/ScriptCheckParam.java index 91173bac28..6e745e9f10 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/script/check/ScriptCheckParam.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/script/check/ScriptCheckParam.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,7 +25,7 @@ package com.tencent.bk.job.manage.manager.script.check; import com.google.common.collect.Maps; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; import lombok.Getter; import lombok.Setter; import org.apache.commons.lang3.StringUtils; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/script/check/checker/BuildInDangerousScriptChecker.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/script/check/checker/BuildInDangerousScriptChecker.java index 819d3f740a..0424ad2d5e 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/script/check/checker/BuildInDangerousScriptChecker.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/script/check/checker/BuildInDangerousScriptChecker.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,7 +26,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; -import com.tencent.bk.job.manage.common.consts.script.ScriptCheckErrorLevelEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptCheckErrorLevelEnum; import com.tencent.bk.job.manage.manager.script.check.ScriptCheckParam; import com.tencent.bk.job.manage.model.dto.ScriptCheckResultItemDTO; import lombok.extern.slf4j.Slf4j; @@ -96,7 +96,7 @@ public ScriptCheckErrorLevelEnum level() { @Override public List call() { - StopWatch watch = new StopWatch(); + StopWatch watch = new StopWatch("BuildInDangerousScriptChecker"); ArrayList checkResults = Lists.newArrayList(); try { String[] lines = param.getLines(); @@ -104,7 +104,12 @@ public List call() { checkRM(checkResults, lines); watch.stop(); } finally { - log.debug("watch={}", watch); + if (watch.isRunning()) { + watch.stop(); + } + if (watch.getTotalTimeMillis() > 10) { + log.info("Check build-in dangerous script is slow, watch={}", watch.prettyPrint()); + } } return checkResults; } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/script/check/checker/DangerousRuleScriptChecker.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/script/check/checker/DangerousRuleScriptChecker.java index cc206eab4f..550b4c7e21 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/script/check/checker/DangerousRuleScriptChecker.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/script/check/checker/DangerousRuleScriptChecker.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,7 +25,7 @@ package com.tencent.bk.job.manage.manager.script.check.checker; import com.google.common.collect.Lists; -import com.tencent.bk.job.manage.common.consts.script.ScriptCheckErrorLevelEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptCheckErrorLevelEnum; import com.tencent.bk.job.manage.manager.script.check.ScriptCheckParam; import com.tencent.bk.job.manage.model.dto.ScriptCheckResultItemDTO; import com.tencent.bk.job.manage.model.dto.globalsetting.DangerousRuleDTO; @@ -49,7 +49,7 @@ public DangerousRuleScriptChecker(ScriptCheckParam param, List @Override public List call() { - StopWatch watch = new StopWatch("scriptDangerousRuleCheck"); + StopWatch watch = new StopWatch("DangerousRuleScriptChecker"); ArrayList resultItems = Lists.newArrayList(); try { for (DangerousRuleDTO dangerousRule : dangerousRules) { diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/script/check/checker/DefaultChecker.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/script/check/checker/DefaultChecker.java index aa3ce2c0f7..cfe5e6721a 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/script/check/checker/DefaultChecker.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/script/check/checker/DefaultChecker.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,8 +24,8 @@ package com.tencent.bk.job.manage.manager.script.check.checker; -import com.tencent.bk.job.manage.common.consts.RuleMatchHandleActionEnum; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.RuleMatchHandleActionEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; import com.tencent.bk.job.manage.manager.script.check.ScriptCheckParam; import com.tencent.bk.job.manage.model.dto.ScriptCheckResultItemDTO; import com.tencent.bk.job.manage.model.dto.globalsetting.DangerousRuleDTO; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/script/check/checker/DeviceCrashScriptChecker.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/script/check/checker/DeviceCrashScriptChecker.java index 3d362b2a13..70d4934871 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/script/check/checker/DeviceCrashScriptChecker.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/script/check/checker/DeviceCrashScriptChecker.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,7 +25,7 @@ package com.tencent.bk.job.manage.manager.script.check.checker; import com.google.common.collect.Lists; -import com.tencent.bk.job.manage.common.consts.script.ScriptCheckErrorLevelEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptCheckErrorLevelEnum; import com.tencent.bk.job.manage.manager.script.check.ScriptCheckParam; import com.tencent.bk.job.manage.model.dto.ScriptCheckResultItemDTO; import lombok.extern.slf4j.Slf4j; @@ -35,7 +35,10 @@ import java.util.List; import java.util.regex.Pattern; -import static com.tencent.bk.job.manage.manager.script.check.consts.ScriptCheckItemCode.*; +import static com.tencent.bk.job.manage.manager.script.check.consts.ScriptCheckItemCode.DANGER_DD; +import static com.tencent.bk.job.manage.manager.script.check.consts.ScriptCheckItemCode.DANGER_FDISK; +import static com.tencent.bk.job.manage.manager.script.check.consts.ScriptCheckItemCode.DANGER_MKFS_EXT3; +import static com.tencent.bk.job.manage.manager.script.check.consts.ScriptCheckItemCode.DANGER_TEE_DEV_SD_STDIN; /** @@ -56,22 +59,30 @@ public DeviceCrashScriptChecker(ScriptCheckParam param) { @Override public List call() { - StopWatch watch = new StopWatch(); + StopWatch watch = new StopWatch("DeviceCrashScriptChecker"); ArrayList checkResults = Lists.newArrayList(); - String[] lines = param.getLines(); - watch.start("tee_dev_sda"); - checkScriptLines(checkResults, lines, tee_dev_sda, DANGER_TEE_DEV_SD_STDIN); - watch.stop(); - watch.start("mkfs"); - checkScriptLines(checkResults, lines, mkfs, DANGER_MKFS_EXT3); - watch.stop(); - watch.start("fdisk"); - checkScriptLines(checkResults, lines, fdisk, DANGER_FDISK); - watch.stop(); - watch.start("dd"); - checkScriptLines(checkResults, lines, dd, DANGER_DD); - watch.stop(); - log.debug("watch={}", watch); + try { + String[] lines = param.getLines(); + watch.start("tee_dev_sda"); + checkScriptLines(checkResults, lines, tee_dev_sda, DANGER_TEE_DEV_SD_STDIN); + watch.stop(); + watch.start("mkfs"); + checkScriptLines(checkResults, lines, mkfs, DANGER_MKFS_EXT3); + watch.stop(); + watch.start("fdisk"); + checkScriptLines(checkResults, lines, fdisk, DANGER_FDISK); + watch.stop(); + watch.start("dd"); + checkScriptLines(checkResults, lines, dd, DANGER_DD); + watch.stop(); + } finally { + if (watch.isRunning()) { + watch.stop(); + } + if (watch.getTotalTimeMillis() > 10) { + log.info("Check build-in dangerous script is slow, watch={}", watch.prettyPrint()); + } + } return checkResults; } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/script/check/checker/IOScriptChecker.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/script/check/checker/IOScriptChecker.java index d91d3a81d8..e41e6bd222 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/script/check/checker/IOScriptChecker.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/script/check/checker/IOScriptChecker.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,7 +25,7 @@ package com.tencent.bk.job.manage.manager.script.check.checker; import com.google.common.collect.Lists; -import com.tencent.bk.job.manage.common.consts.script.ScriptCheckErrorLevelEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptCheckErrorLevelEnum; import com.tencent.bk.job.manage.manager.script.check.ScriptCheckParam; import com.tencent.bk.job.manage.model.dto.ScriptCheckResultItemDTO; import lombok.extern.slf4j.Slf4j; @@ -35,7 +35,9 @@ import java.util.List; import java.util.regex.Pattern; -import static com.tencent.bk.job.manage.manager.script.check.consts.ScriptCheckItemCode.*; +import static com.tencent.bk.job.manage.manager.script.check.consts.ScriptCheckItemCode.DANGER_FOUND_ALL; +import static com.tencent.bk.job.manage.manager.script.check.consts.ScriptCheckItemCode.DANGER_FOUND_BASE; +import static com.tencent.bk.job.manage.manager.script.check.consts.ScriptCheckItemCode.DANGER_FOUND_WITH_NO_CONDITION; /** * 脚本对IO操作的高危检查 @@ -58,19 +60,27 @@ public ScriptCheckErrorLevelEnum level() { @Override public List call() { - StopWatch watch = new StopWatch(); + StopWatch watch = new StopWatch("IOScriptChecker"); ArrayList checkResults = Lists.newArrayList(); - String[] lines = param.getLines(); - watch.start("noParamToFindBase"); - checkScriptLines(checkResults, lines, noParamToFindBase, DANGER_FOUND_ALL); - watch.stop(); - watch.start("paramFindBase"); - checkScriptLines(checkResults, lines, paramFindBase, DANGER_FOUND_BASE); - watch.stop(); - watch.start("noParamToFind"); - checkScriptLines(checkResults, lines, noParamToFind, DANGER_FOUND_WITH_NO_CONDITION); - watch.stop(); - log.debug("watch={}", watch); + try { + String[] lines = param.getLines(); + watch.start("noParamToFindBase"); + checkScriptLines(checkResults, lines, noParamToFindBase, DANGER_FOUND_ALL); + watch.stop(); + watch.start("paramFindBase"); + checkScriptLines(checkResults, lines, paramFindBase, DANGER_FOUND_BASE); + watch.stop(); + watch.start("noParamToFind"); + checkScriptLines(checkResults, lines, noParamToFind, DANGER_FOUND_WITH_NO_CONDITION); + watch.stop(); + } finally { + if (watch.isRunning()) { + watch.stop(); + } + if (watch.getTotalTimeMillis() > 10) { + log.info("Check io script is slow, watch={}", watch.prettyPrint()); + } + } return checkResults; } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/script/check/checker/ScriptChecker.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/script/check/checker/ScriptChecker.java index 8977628206..067a3f2157 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/script/check/checker/ScriptChecker.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/script/check/checker/ScriptChecker.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,8 +24,8 @@ package com.tencent.bk.job.manage.manager.script.check.checker; -import com.tencent.bk.job.manage.common.consts.RuleMatchHandleActionEnum; -import com.tencent.bk.job.manage.common.consts.script.ScriptCheckErrorLevelEnum; +import com.tencent.bk.job.manage.api.common.constants.RuleMatchHandleActionEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptCheckErrorLevelEnum; import com.tencent.bk.job.manage.model.dto.ScriptCheckResultItemDTO; import java.util.List; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/script/check/checker/ScriptGrammarChecker.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/script/check/checker/ScriptGrammarChecker.java index 97dec3af83..ad8774f02b 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/script/check/checker/ScriptGrammarChecker.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/script/check/checker/ScriptGrammarChecker.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,8 +27,8 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.tencent.bk.job.common.util.JobUUID; -import com.tencent.bk.job.manage.common.consts.script.ScriptCheckErrorLevelEnum; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptCheckErrorLevelEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; import com.tencent.bk.job.manage.model.dto.ScriptCheckResultItemDTO; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; @@ -50,8 +50,8 @@ */ @Slf4j public class ScriptGrammarChecker implements ScriptChecker { - private ScriptTypeEnum type; - private String content; + private final ScriptTypeEnum type; + private final String content; public ScriptGrammarChecker(ScriptTypeEnum type, String content) { this.type = type; @@ -60,7 +60,7 @@ public ScriptGrammarChecker(ScriptTypeEnum type, String content) { @Override public List call() throws Exception { - StopWatch watch = new StopWatch(); + StopWatch watch = new StopWatch("ScriptGrammarChecker"); watch.start("checkGrammar"); File tmpFile = File.createTempFile(JobUUID.getUUID(), type.getName()); @@ -102,9 +102,16 @@ public List call() throws Exception { } catch (IOException e) { log.error("Check script grammar fail", e); } finally { - tmpFile.deleteOnExit(); - watch.stop(); - log.debug("watch={}", watch); + if (!tmpFile.delete()) { + log.warn("Failed to delete temporary file,register jvm to exit delete:{}", tmpFile.getAbsolutePath()); + tmpFile.deleteOnExit(); + } + if (watch.isRunning()) { + watch.stop(); + } + if (watch.getTotalTimeMillis() > 10) { + log.info("Check script grammar is slow, watch={}", watch.prettyPrint()); + } } ArrayList checkResults = Lists.newArrayList(); checkResults.addAll(result.values()); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/script/check/checker/ScriptLogicChecker.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/script/check/checker/ScriptLogicChecker.java index 19dc0ef93c..fc62939b95 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/script/check/checker/ScriptLogicChecker.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/script/check/checker/ScriptLogicChecker.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,7 +25,7 @@ package com.tencent.bk.job.manage.manager.script.check.checker; import com.google.common.collect.Lists; -import com.tencent.bk.job.manage.common.consts.script.ScriptCheckErrorLevelEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptCheckErrorLevelEnum; import com.tencent.bk.job.manage.manager.script.check.ScriptCheckParam; import com.tencent.bk.job.manage.model.dto.ScriptCheckResultItemDTO; import lombok.extern.slf4j.Slf4j; @@ -58,7 +58,7 @@ public ScriptCheckErrorLevelEnum level() { @Override public List call() { - StopWatch watch = new StopWatch(); + StopWatch watch = new StopWatch("ScriptLogicChecker"); ArrayList checkResults = Lists.newArrayList(); String[] lines = param.getLines(); try { @@ -70,7 +70,12 @@ public List call() { checkScriptLines(checkResults, lines, cdDirCheck, LOGIC_NEED_CD_DIR_CHECK); watch.stop(); } finally { - log.debug("watch={}", watch); + if (watch.isRunning()) { + watch.stop(); + } + if (watch.getTotalTimeMillis() > 10) { + log.info("Check script logic is slow, watch={}", watch.prettyPrint()); + } } return checkResults; } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/script/check/consts/ScriptCheckItemCode.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/script/check/consts/ScriptCheckItemCode.java index 8e9cbba97f..b82b951532 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/script/check/consts/ScriptCheckItemCode.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/script/check/consts/ScriptCheckItemCode.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/variable/StepRefVariableParser.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/variable/StepRefVariableParser.java index 2b7d275bcc..fc872bf943 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/variable/StepRefVariableParser.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/manager/variable/StepRefVariableParser.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,8 +26,8 @@ import com.tencent.bk.job.common.constant.TaskVariableTypeEnum; import com.tencent.bk.job.common.service.VariableResolver; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskStepTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskStepTypeEnum; import com.tencent.bk.job.manage.model.dto.task.TaskFileStepDTO; import com.tencent.bk.job.manage.model.dto.task.TaskScriptStepDTO; import com.tencent.bk.job.manage.model.dto.task.TaskStepDTO; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/metrics/CmdbEventSampler.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/metrics/CmdbEventSampler.java index af7d03937b..896632297e 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/metrics/CmdbEventSampler.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/metrics/CmdbEventSampler.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/metrics/MetricsConstants.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/metrics/MetricsConstants.java index 7f53ce602f..ff81cf482b 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/metrics/MetricsConstants.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/metrics/MetricsConstants.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/metrics/StaticMetricsConfig.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/metrics/StaticMetricsConfig.java index b2cae84c34..1ed2a67fa6 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/metrics/StaticMetricsConfig.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/metrics/StaticMetricsConfig.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,6 +28,7 @@ import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Tag; import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.ObjectProvider; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -35,7 +36,6 @@ import java.util.Collections; import java.util.Map; import java.util.concurrent.ThreadPoolExecutor; -import java.util.function.ToDoubleFunction; @Slf4j @Component @@ -45,7 +45,7 @@ public class StaticMetricsConfig { public StaticMetricsConfig( ThreadPoolExecutor cmdbThreadPoolExecutor, MeterRegistry meterRegistry, - RedisSlideWindowFlowController cmdbGlobalFlowController + ObjectProvider cmdbGlobalFlowControllerProvider ) { // CMDB请求线程池大小 meterRegistry.gauge( @@ -62,44 +62,46 @@ public StaticMetricsConfig( threadPoolExecutor -> threadPoolExecutor.getQueue().size() ); try { - Map configMap = cmdbGlobalFlowController.getCurrentConfig(); - for (String key : configMap.keySet()) { - // CMDB流控：配置 - meterRegistry.gauge( - MetricsConstants.NAME_CMDB_RESOURCE_LIMIT, - Arrays.asList( - Tag.of(MetricsConstants.TAG_KEY_MODULE, MetricsConstants.TAG_VALUE_MODULE_CMDB), - Tag.of(MetricsConstants.TAG_KEY_ASPECT, MetricsConstants.TAG_VALUE_ASPECT_FLOW_CONTROL), - Tag.of(MetricsConstants.TAG_KEY_RESOURCE_ID, key) - ), - cmdbGlobalFlowController, - new ToDoubleFunction() { - @Override - public double applyAsDouble(RedisSlideWindowFlowController cmdbGlobalFlowController) { - Long value = cmdbGlobalFlowController.getCurrentConfig(key); - if (value == null) return -1; + RedisSlideWindowFlowController cmdbGlobalFlowController = + cmdbGlobalFlowControllerProvider.getIfAvailable(); + if (cmdbGlobalFlowController != null) { + Map configMap = cmdbGlobalFlowController.getCurrentConfig(); + for (String key : configMap.keySet()) { + // CMDB流控：配置 + meterRegistry.gauge( + MetricsConstants.NAME_CMDB_RESOURCE_LIMIT, + Arrays.asList( + Tag.of(MetricsConstants.TAG_KEY_MODULE, MetricsConstants.TAG_VALUE_MODULE_CMDB), + Tag.of(MetricsConstants.TAG_KEY_ASPECT, MetricsConstants.TAG_VALUE_ASPECT_FLOW_CONTROL), + Tag.of(MetricsConstants.TAG_KEY_RESOURCE_ID, key) + ), + cmdbGlobalFlowController, + cmdbGlobalFlowController1 -> { + Long value = cmdbGlobalFlowController1.getCurrentConfig(key); + if (value == null) { + return -1; + } return value; } - } - ); - // CMDB流控：当前速率 - meterRegistry.gauge( - MetricsConstants.NAME_CMDB_RESOURCE_RATE, - Arrays.asList( - Tag.of(MetricsConstants.TAG_KEY_MODULE, MetricsConstants.TAG_VALUE_MODULE_CMDB), - Tag.of(MetricsConstants.TAG_KEY_ASPECT, MetricsConstants.TAG_VALUE_ASPECT_FLOW_CONTROL), - Tag.of(MetricsConstants.TAG_KEY_RESOURCE_ID, key) - ), - cmdbGlobalFlowController, - new ToDoubleFunction() { - @Override - public double applyAsDouble(RedisSlideWindowFlowController cmdbGlobalFlowController) { - Long value = cmdbGlobalFlowController.getCurrentRate(key); - if (value == null) return 0; + ); + // CMDB流控：当前速率 + meterRegistry.gauge( + MetricsConstants.NAME_CMDB_RESOURCE_RATE, + Arrays.asList( + Tag.of(MetricsConstants.TAG_KEY_MODULE, MetricsConstants.TAG_VALUE_MODULE_CMDB), + Tag.of(MetricsConstants.TAG_KEY_ASPECT, MetricsConstants.TAG_VALUE_ASPECT_FLOW_CONTROL), + Tag.of(MetricsConstants.TAG_KEY_RESOURCE_ID, key) + ), + cmdbGlobalFlowController, + cmdbGlobalFlowController12 -> { + Long value = cmdbGlobalFlowController12.getCurrentRate(key); + if (value == null) { + return 0; + } return value; } - } - ); + ); + } } } catch (Exception e) { log.error("Fail to init cmdbGlobalFlowController gauge", e); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/migration/AddHostIdForTemplateAndPlanMigrationTask.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/migration/AddHostIdForTemplateAndPlanMigrationTask.java new file mode 100644 index 0000000000..4a93182c91 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/migration/AddHostIdForTemplateAndPlanMigrationTask.java @@ -0,0 +1,817 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.migration; + +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.constant.TaskVariableTypeEnum; +import com.tencent.bk.job.common.exception.InternalException; +import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; +import com.tencent.bk.job.common.mysql.JobTransactional; +import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.manage.model.dto.task.TaskHostNodeDTO; +import com.tencent.bk.job.manage.model.dto.task.TaskTargetDTO; +import com.tencent.bk.job.manage.model.migration.AddHostIdResult; +import com.tencent.bk.job.manage.model.tables.TaskPlan; +import com.tencent.bk.job.manage.model.tables.TaskPlanStep; +import com.tencent.bk.job.manage.model.tables.TaskPlanStepFile; +import com.tencent.bk.job.manage.model.tables.TaskPlanStepFileList; +import com.tencent.bk.job.manage.model.tables.TaskPlanStepScript; +import com.tencent.bk.job.manage.model.tables.TaskPlanVariable; +import com.tencent.bk.job.manage.model.tables.TaskTemplate; +import com.tencent.bk.job.manage.model.tables.TaskTemplateStep; +import com.tencent.bk.job.manage.model.tables.TaskTemplateStepFile; +import com.tencent.bk.job.manage.model.tables.TaskTemplateStepFileList; +import com.tencent.bk.job.manage.model.tables.TaskTemplateStepScript; +import com.tencent.bk.job.manage.model.tables.TaskTemplateVariable; +import com.tencent.bk.job.manage.service.host.HostService; +import lombok.Data; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.jooq.Condition; +import org.jooq.DSLContext; +import org.jooq.Result; +import org.jooq.types.UByte; +import org.jooq.types.ULong; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Service; +import org.springframework.util.StopWatch; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * 作业模板、执行方案等包含的主机数据，在原来的云区域+ip的基础上，填充hostID属性 + */ +@Service +@Slf4j +public class AddHostIdForTemplateAndPlanMigrationTask { + private final DSLContext CTX; + private final HostService hostService; + + private static final TaskTemplate TASK_TEMPLATE = TaskTemplate.TASK_TEMPLATE; + private static final TaskPlan TASK_PLAN = TaskPlan.TASK_PLAN; + private static final TaskTemplateStep TASK_TEMPLATE_STEP = TaskTemplateStep.TASK_TEMPLATE_STEP; + private static final TaskPlanStep TASK_PLAN_STEP = TaskPlanStep.TASK_PLAN_STEP; + + private static final TaskTemplateStepScript TASK_TEMPLATE_STEP_SCRIPT = + TaskTemplateStepScript.TASK_TEMPLATE_STEP_SCRIPT; + private static final TaskTemplateStepFile TASK_TEMPLATE_STEP_FILE = + TaskTemplateStepFile.TASK_TEMPLATE_STEP_FILE; + private static final TaskTemplateStepFileList TASK_TEMPLATE_STEP_FILE_LIST = + TaskTemplateStepFileList.TASK_TEMPLATE_STEP_FILE_LIST; + private static final TaskTemplateVariable TASK_TEMPLATE_VARIABLE = + TaskTemplateVariable.TASK_TEMPLATE_VARIABLE; + private static final TaskPlanStepScript TASK_PLAN_STEP_SCRIPT = + TaskPlanStepScript.TASK_PLAN_STEP_SCRIPT; + private static final TaskPlanStepFile TASK_PLAN_STEP_FILE = + TaskPlanStepFile.TASK_PLAN_STEP_FILE; + private static final TaskPlanStepFileList TASK_PLAN_STEP_FILE_LIST = + TaskPlanStepFileList.TASK_PLAN_STEP_FILE_LIST; + private static final TaskPlanVariable TASK_PLAN_VARIABLE = + TaskPlanVariable.TASK_PLAN_VARIABLE; + + /** + * 不存在的主机，hostId使用-1表示 + */ + private static final long NOT_EXIST_HOST_ID = -1L; + + private final Map ipAndHostIdMapping = new HashMap<>(); + + @Autowired + public AddHostIdForTemplateAndPlanMigrationTask(@Qualifier("job-manage-dsl-context") DSLContext ctx, + HostService hostService) { + CTX = ctx; + this.hostService = hostService; + } + + public List execute(List appIdList, boolean isDryRun) { + List results = new ArrayList<>(); + try { + results.add(new TaskTemplateStepScriptTargetMigration(appIdList, isDryRun).migrate()); + results.add(new TaskTemplateStepFileTargetMigration(appIdList, isDryRun).migrate()); + results.add(new TaskTemplateStepFileListTargetMigration(appIdList, isDryRun).migrate()); + results.add(new TaskTemplateVariableTargetMigration(appIdList, isDryRun).migrate()); + results.add(new TaskPlanStepScriptTargetMigration(appIdList, isDryRun).migrate()); + results.add(new TaskPlanStepFileTargetMigration(appIdList, isDryRun).migrate()); + results.add(new TaskPlanStepFileListTargetMigration(appIdList, isDryRun).migrate()); + results.add(new TaskPlanVariableTargetMigration(appIdList, isDryRun).migrate()); + return results; + } finally { + log.info("AddHostIdMigrationTask done, result: {}", JsonUtils.toJson(results)); + // 清理缓存，避免占用内存 + this.ipAndHostIdMapping.clear(); + } + } + + public void addIpAndHostIdMappings(Collection targets) { + Set notCachedCloudIps = new HashSet<>(); + for (TaskTargetDTO target : targets) { + TaskHostNodeDTO hostNodeList = target.getHostNodeList(); + if (hostNodeList == null) { + continue; + } + List hostList = hostNodeList.getHostList(); + if (CollectionUtils.isEmpty(hostList)) { + continue; + } + hostList.forEach(host -> { + String cloudIp = host.getCloudIp(); + if (ipAndHostIdMapping.get(cloudIp) == null) { + notCachedCloudIps.add(cloudIp); + } + }); + } + + if (CollectionUtils.isNotEmpty(notCachedCloudIps)) { + try { + Map hosts = hostService.listHostsByIps(notCachedCloudIps); + notCachedCloudIps.forEach(notCacheCloudIp -> { + ApplicationHostDTO host = hosts.get(notCacheCloudIp); + if (host == null) { + log.warn("Host with ip {} is not exist in cmdb, set hostId -1", notCacheCloudIp); + ipAndHostIdMapping.put(notCacheCloudIp, NOT_EXIST_HOST_ID); + } else { + ipAndHostIdMapping.put(notCacheCloudIp, host.getHostId()); + } + }); + } catch (Throwable e) { + // 由于从cmdb查询hostId可能会返回异常（比如接口超时）等,为了保证下次迁移不需要全部从头开始，所以这里捕获住异常，控制影响范围 + log.error("Get host by ips fail", e); + } + } + } + + private boolean isTargetMissingHostId(TaskTargetDTO target) { + return target != null && target.getHostNodeList() != null + && CollectionUtils.isNotEmpty(target.getHostNodeList().getHostList()) + && !hasHostId(target.getHostNodeList().getHostList().get(0).getHostId()); + } + + private boolean hasHostId(Long hostId) { + // host = -1,表示已经处理过的数据，并且这个主机被判定为不存在 + return hostId != null && (hostId > 0 || hostId == NOT_EXIST_HOST_ID); + } + + public boolean fillHostId(TaskTargetDTO target) { + boolean success = true; + TaskHostNodeDTO hostNodeList = target.getHostNodeList(); + if (hostNodeList == null) { + return false; + } + List hostList = hostNodeList.getHostList(); + if (CollectionUtils.isEmpty(hostList)) { + return false; + } + for (ApplicationHostDTO host : hostList) { + String cloudIp = host.getCloudIp(); + Long hostId = ipAndHostIdMapping.get(cloudIp); + if (hostId != null) { + host.setHostId(hostId); + } else { + success = false; + } + } + return success; + } + + private String convertTargetToJson(TaskTargetDTO target) { + String json = JsonUtils.toJson(target); + if (StringUtils.isBlank(json)) { + // 可能是解析出现异常，需要抛出异常终止 + throw new InternalException(ErrorCode.INTERNAL_ERROR); + } + return json; + } + + @SuppressWarnings("SameParameterValue") + private abstract class AbstractTaskTargetMigration { + private final List appIdList; + private final boolean dryRun; + + AbstractTaskTargetMigration(List appIdList, boolean dryRun) { + this.appIdList = appIdList; + this.dryRun = dryRun; + } + + public AddHostIdResult migrate() { + String taskName = "migrate_" + getTableName(); + AddHostIdResult result = new AddHostIdResult(taskName); + StopWatch watch = new StopWatch(taskName); + TaskTargetRecordPageResult pageResult = null; + int seq = 0; + int totalCount = 0; + int successCount = 0; + try { + log.info("[{}] Migration start ...", getTableName()); + do { + seq++; + watch.start("list_targets_" + seq); + pageResult = listPageTaskTargets(appIdList, pageResult == null ? null : + pageResult.getNextRecordId(), 500); + if (CollectionUtils.isEmpty(pageResult.getRecords())) { + watch.stop(); + continue; + } + + List records = pageResult.getRecords(); + records = records.stream().filter(record -> isTargetMissingHostId(record.getTarget())) + .collect(Collectors.toList()); + if (records.isEmpty()) { + watch.stop(); + continue; + } + totalCount += records.size(); + log.info("[{}-{}] {} targets need migration", getTableName(), seq, records.size()); + watch.stop(); + + + watch.start("add_ip_host_id_mappings_" + seq); + addIpAndHostIdMappings( + records.stream().map(TaskTargetRecord::getTarget).collect(Collectors.toList())); + log.info("[{}-{}] ipAndHostIdMappings: {}", getTableName(), seq, + JsonUtils.toJson(ipAndHostIdMapping)); + watch.stop(); + + successCount += fillHostIdAndUpdateTaskTargets(records, watch, seq); + } while (pageResult.hasNext()); + + result.setSuccess(true); + } catch (Throwable e) { + // catch all exception + log.error("Migration caught exception", e); + result.setSuccess(false); + return result; + } finally { + result.setTotalRecords(totalCount); + result.setSuccessRecords(successCount); + if (watch.isRunning()) { + watch.stop(); + } + log.info("[{}] Migration done. cost: {}", getTableName(), watch.prettyPrint()); + } + return result; + } + + private int fillHostIdAndUpdateTaskTargets(List records, StopWatch watch, int seq) { + watch.start("fill_host_id_" + seq); + List invalidIds = new ArrayList<>(); + List updateRecords = new ArrayList<>(); + for (TaskTargetRecord record : records) { + boolean success = fillHostId(record.getTarget()); + if (!success) { + invalidIds.add(record.getId()); + } else { + updateRecords.add(record); + } + } + if (CollectionUtils.isNotEmpty(invalidIds)) { + log.error("[{}-{}] {} targets fill host id fail, invalidIdList: {}", getTableName(), seq, + invalidIds.size(), invalidIds); + } + watch.stop(); + + if (CollectionUtils.isNotEmpty(updateRecords)) { + watch.start("update_targets_" + seq); + if (dryRun) { + // 只输出要更新的数据用于验证，不进行DB数据的变更 + log.info("Update targets with dryRun mode, tableName: {}, records: {}", getTableName(), + JsonUtils.toJson(updateRecords)); + } else { + updateTaskTargets(updateRecords); + } + watch.stop(); + return updateRecords.size(); + } + return 0; + } + + /** + * 查询所有需要处理的TaskTarget + * + * @param appIdList 限定的Job业务ID列表;传入null表示所有业务 + * @param nextRecordId 下一个记录ID;传入null表示第一页 + * @param maxRecordSize 分页-最大返回记录数量 + * @return 分页结果 + */ + private TaskTargetRecordPageResult listPageTaskTargets(List appIdList, + Long nextRecordId, + int maxRecordSize) { + long fromId = nextRecordId == null ? 0L : nextRecordId; + // 比单页最大数量多获取一条数据，用于判断是否还有下一页 + int limit = maxRecordSize + 1; + List records = listTaskTargets(appIdList, fromId, limit); + + if (records.size() == limit) { + // 下一页仍然有内容 + return new TaskTargetRecordPageResult(records.subList(0, records.size() - 1), + records.get(records.size() - 1).getId()); + } else { + // <= maxRecordSize, 表示当前已经是最后一页 + return new TaskTargetRecordPageResult(records, null); + } + } + + /** + * 获取表名称 + * + * @return 表名称 + */ + abstract String getTableName(); + + /** + * 从DB查询 TaskTarget + * + * @param appIdList Job业务ID列表 + * @param fromRecordId 起始记录ID + * @param maxRecordSize 最大返回记录数量 + * @return TaskTarget 列表 + */ + abstract List listTaskTargets(List appIdList, Long fromRecordId, int maxRecordSize); + + /** + * 批量更新Target + * + * @param records 更新目标主机 + */ + abstract void updateTaskTargets(List records); + } + + @Data + @NoArgsConstructor + private static class TaskTargetRecordPageResult { + + public TaskTargetRecordPageResult(List records, Long nextRecordId) { + this.records = records; + this.nextRecordId = nextRecordId; + } + + private List records; + private Long nextRecordId; + + public boolean hasNext() { + return nextRecordId != null; + } + } + + @Data + @NoArgsConstructor + private static class TaskTargetRecord { + + public TaskTargetRecord(Long id, TaskTargetDTO target) { + this.id = id; + this.target = target; + } + + /** + * 包含目标主机的记录ID + */ + private Long id; + /** + * 目标主机 + */ + private TaskTargetDTO target; + } + + private class TaskTemplateStepScriptTargetMigration extends AbstractTaskTargetMigration { + + TaskTemplateStepScriptTargetMigration(List appIdList, boolean dryRun) { + super(appIdList, dryRun); + } + + @Override + public String getTableName() { + return "task_template_step_script"; + } + + @Override + public List listTaskTargets(List appIdList, Long fromRecordId, int maxRecordSize) { + List conditions = new ArrayList<>(); + conditions.add(TASK_TEMPLATE_STEP_SCRIPT.ID.ge(ULong.valueOf(fromRecordId))); + if (appIdList != null) { + conditions.add(TASK_TEMPLATE.APP_ID.in(appIdList)); + } + Result result = CTX.select( + TASK_TEMPLATE_STEP_SCRIPT.ID, + TASK_TEMPLATE_STEP_SCRIPT.DESTINATION_HOST_LIST) + .from(TASK_TEMPLATE_STEP_SCRIPT) + .join(TASK_TEMPLATE_STEP) + .on(TASK_TEMPLATE_STEP_SCRIPT.STEP_ID.eq(TASK_TEMPLATE_STEP.ID)) + .join(TASK_TEMPLATE) + .on(TASK_TEMPLATE_STEP.TEMPLATE_ID.eq(TASK_TEMPLATE.ID)) + .where(conditions) + .orderBy(TASK_TEMPLATE_STEP_SCRIPT.ID.asc()) + .limit(maxRecordSize) + .fetch(); + return result.map(record -> { + long id = record.get(TASK_TEMPLATE_STEP_SCRIPT.ID).longValue(); + String targetJsonStr = record.get(TASK_TEMPLATE_STEP_SCRIPT.DESTINATION_HOST_LIST); + TaskTargetDTO target = JsonUtils.fromJson(targetJsonStr, TaskTargetDTO.class); + return new TaskTargetRecord(id, target); + }); + } + + @Override + @JobTransactional(transactionManager = "jobManageTransactionManager") + public void updateTaskTargets(List records) { + if (CollectionUtils.isEmpty(records)) { + return; + } + records.forEach(record -> + CTX.update(TASK_TEMPLATE_STEP_SCRIPT) + .set(TASK_TEMPLATE_STEP_SCRIPT.DESTINATION_HOST_LIST, convertTargetToJson(record.getTarget())) + .where(TASK_TEMPLATE_STEP_SCRIPT.ID.eq(ULong.valueOf(record.getId()))) + .execute()); + } + } + + private class TaskTemplateStepFileTargetMigration extends AbstractTaskTargetMigration { + + TaskTemplateStepFileTargetMigration(List appIdList, boolean dryRun) { + super(appIdList, dryRun); + } + + @Override + public String getTableName() { + return "task_template_step_file"; + } + + @Override + public List listTaskTargets(List appIdList, Long fromRecordId, int maxRecordSize) { + List conditions = new ArrayList<>(); + conditions.add(TASK_TEMPLATE_STEP_FILE.ID.ge(ULong.valueOf(fromRecordId))); + if (appIdList != null) { + conditions.add(TASK_TEMPLATE.APP_ID.in(appIdList)); + } + Result result = CTX.select( + TASK_TEMPLATE_STEP_FILE.ID, + TASK_TEMPLATE_STEP_FILE.DESTINATION_HOST_LIST) + .from(TASK_TEMPLATE_STEP_FILE) + .join(TASK_TEMPLATE_STEP) + .on(TASK_TEMPLATE_STEP_FILE.STEP_ID.eq(TASK_TEMPLATE_STEP.ID)) + .join(TASK_TEMPLATE) + .on(TASK_TEMPLATE_STEP.TEMPLATE_ID.eq(TASK_TEMPLATE.ID)) + .where(conditions) + .orderBy(TASK_TEMPLATE_STEP_FILE.ID.asc()) + .limit(maxRecordSize) + .fetch(); + return result.map(record -> { + long id = record.get(TASK_TEMPLATE_STEP_FILE.ID).longValue(); + String targetJsonStr = record.get(TASK_TEMPLATE_STEP_FILE.DESTINATION_HOST_LIST); + TaskTargetDTO target = JsonUtils.fromJson(targetJsonStr, TaskTargetDTO.class); + return new TaskTargetRecord(id, target); + }); + } + + @Override + public void updateTaskTargets(List records) { + if (CollectionUtils.isEmpty(records)) { + return; + } + records.forEach(record -> + CTX.update(TASK_TEMPLATE_STEP_FILE) + .set(TASK_TEMPLATE_STEP_FILE.DESTINATION_HOST_LIST, convertTargetToJson(record.getTarget())) + .where(TASK_TEMPLATE_STEP_FILE.ID.eq(ULong.valueOf(record.getId()))) + .execute()); + } + } + + private class TaskTemplateStepFileListTargetMigration extends AbstractTaskTargetMigration { + + TaskTemplateStepFileListTargetMigration(List appIdList, boolean dryRun) { + super(appIdList, dryRun); + } + + @Override + public String getTableName() { + return "task_template_step_file_list"; + } + + @Override + public List listTaskTargets(List appIdList, Long fromRecordId, int maxRecordSize) { + List conditions = new ArrayList<>(); + conditions.add(TASK_TEMPLATE_STEP_FILE_LIST.ID.ge(ULong.valueOf(fromRecordId))); + if (appIdList != null) { + conditions.add(TASK_TEMPLATE.APP_ID.in(appIdList)); + } + Result result = CTX.select( + TASK_TEMPLATE_STEP_FILE_LIST.ID, + TASK_TEMPLATE_STEP_FILE_LIST.HOST) + .from(TASK_TEMPLATE_STEP_FILE_LIST) + .join(TASK_TEMPLATE_STEP) + .on(TASK_TEMPLATE_STEP_FILE_LIST.STEP_ID.eq(TASK_TEMPLATE_STEP.ID)) + .join(TASK_TEMPLATE) + .on(TASK_TEMPLATE_STEP.TEMPLATE_ID.eq(TASK_TEMPLATE.ID)) + .where(conditions) + .orderBy(TASK_TEMPLATE_STEP_FILE_LIST.ID.asc()) + .limit(maxRecordSize) + .fetch(); + return result.map(record -> { + long id = record.get(TASK_TEMPLATE_STEP_FILE_LIST.ID).longValue(); + String targetJsonStr = record.get(TASK_TEMPLATE_STEP_FILE_LIST.HOST); + TaskTargetDTO target = JsonUtils.fromJson(targetJsonStr, TaskTargetDTO.class); + return new TaskTargetRecord(id, target); + }); + } + + @Override + public void updateTaskTargets(List records) { + if (CollectionUtils.isEmpty(records)) { + return; + } + records.forEach(record -> + CTX.update(TASK_TEMPLATE_STEP_FILE_LIST) + .set(TASK_TEMPLATE_STEP_FILE_LIST.HOST, convertTargetToJson(record.getTarget())) + .where(TASK_TEMPLATE_STEP_FILE_LIST.ID.eq(ULong.valueOf(record.getId()))) + .execute()); + } + } + + private class TaskTemplateVariableTargetMigration extends AbstractTaskTargetMigration { + + TaskTemplateVariableTargetMigration(List appIdList, boolean dryRun) { + super(appIdList, dryRun); + } + + @Override + public String getTableName() { + return "task_template_variable"; + } + + @Override + public List listTaskTargets(List appIdList, Long fromRecordId, int maxRecordSize) { + List conditions = new ArrayList<>(); + conditions.add(TASK_TEMPLATE_VARIABLE.TYPE.eq(UByte.valueOf(TaskVariableTypeEnum.HOST_LIST.getType()))); + if (appIdList != null) { + conditions.add(TASK_TEMPLATE.APP_ID.in(appIdList)); + } + Result result = CTX.select( + TASK_TEMPLATE_VARIABLE.ID, + TASK_TEMPLATE_VARIABLE.DEFAULT_VALUE) + .from(TASK_TEMPLATE_VARIABLE) + .join(TASK_TEMPLATE) + .on(TASK_TEMPLATE_VARIABLE.TEMPLATE_ID.eq(TASK_TEMPLATE.ID)) + .where(conditions) + .and(TASK_TEMPLATE_VARIABLE.ID.ge(ULong.valueOf(fromRecordId))) + .orderBy(TASK_TEMPLATE_VARIABLE.ID.asc()) + .limit(maxRecordSize) + .fetch(); + return result.map(record -> { + long id = record.get(TASK_TEMPLATE_VARIABLE.ID).longValue(); + String targetJsonStr = record.get(TASK_TEMPLATE_VARIABLE.DEFAULT_VALUE); + TaskTargetDTO target = JsonUtils.fromJson(targetJsonStr, TaskTargetDTO.class); + return new TaskTargetRecord(id, target); + }); + } + + @Override + public void updateTaskTargets(List records) { + if (CollectionUtils.isEmpty(records)) { + return; + } + records.forEach(record -> + CTX.update(TASK_TEMPLATE_VARIABLE) + .set(TASK_TEMPLATE_VARIABLE.DEFAULT_VALUE, convertTargetToJson(record.getTarget())) + .where(TASK_TEMPLATE_VARIABLE.ID.eq(ULong.valueOf(record.getId()))) + .and(TASK_TEMPLATE_VARIABLE.TYPE.eq(UByte.valueOf(TaskVariableTypeEnum.HOST_LIST.getType()))) + .execute()); + } + } + + + private class TaskPlanStepScriptTargetMigration extends AbstractTaskTargetMigration { + + public TaskPlanStepScriptTargetMigration(List appIdList, boolean dryRun) { + super(appIdList, dryRun); + } + + @Override + public String getTableName() { + return "TaskPlanStepScript"; + } + + @Override + public List listTaskTargets(List appIdList, Long fromRecordId, int maxRecordSize) { + List conditions = new ArrayList<>(); + conditions.add(TASK_PLAN_STEP_SCRIPT.ID.ge(ULong.valueOf(fromRecordId))); + if (appIdList != null) { + conditions.add(TASK_PLAN.APP_ID.in(appIdList)); + } + Result result = CTX.select( + TASK_PLAN_STEP_SCRIPT.ID, + TASK_PLAN_STEP_SCRIPT.DESTINATION_HOST_LIST) + .from(TASK_PLAN_STEP_SCRIPT) + .join(TASK_PLAN_STEP) + .on(TASK_PLAN_STEP_SCRIPT.STEP_ID.eq(TASK_PLAN_STEP.ID)) + .join(TASK_PLAN) + .on(TASK_PLAN_STEP.PLAN_ID.eq(TASK_PLAN.ID)) + .where(conditions) + .orderBy(TASK_PLAN_STEP_SCRIPT.ID.asc()) + .limit(maxRecordSize) + .fetch(); + return result.map(record -> { + long id = record.get(TASK_PLAN_STEP_SCRIPT.ID).longValue(); + String targetJsonStr = record.get(TASK_PLAN_STEP_SCRIPT.DESTINATION_HOST_LIST); + TaskTargetDTO target = JsonUtils.fromJson(targetJsonStr, TaskTargetDTO.class); + return new TaskTargetRecord(id, target); + }); + } + + @Override + @JobTransactional(transactionManager = "jobManageTransactionManager") + public void updateTaskTargets(List records) { + if (CollectionUtils.isEmpty(records)) { + return; + } + records.forEach(record -> + CTX.update(TASK_PLAN_STEP_SCRIPT) + .set(TASK_PLAN_STEP_SCRIPT.DESTINATION_HOST_LIST, convertTargetToJson(record.getTarget())) + .where(TASK_PLAN_STEP_SCRIPT.ID.eq(ULong.valueOf(record.getId()))) + .execute()); + } + } + + private class TaskPlanStepFileTargetMigration extends AbstractTaskTargetMigration { + + TaskPlanStepFileTargetMigration(List appIdList, boolean dryRun) { + super(appIdList, dryRun); + } + + @Override + public String getTableName() { + return "task_plan_step_file"; + } + + @Override + public List listTaskTargets(List appIdList, Long fromRecordId, int maxRecordSize) { + List conditions = new ArrayList<>(); + conditions.add(TASK_PLAN_STEP_FILE.ID.ge(ULong.valueOf(fromRecordId))); + if (appIdList != null) { + conditions.add(TASK_PLAN.APP_ID.in(appIdList)); + } + Result result = CTX.select( + TASK_PLAN_STEP_FILE.ID, + TASK_PLAN_STEP_FILE.DESTINATION_HOST_LIST) + .from(TASK_PLAN_STEP_FILE) + .join(TASK_PLAN_STEP) + .on(TASK_PLAN_STEP_FILE.STEP_ID.eq(TASK_PLAN_STEP.ID)) + .join(TASK_PLAN) + .on(TASK_PLAN_STEP.PLAN_ID.eq(TASK_PLAN.ID)) + .where(conditions) + .orderBy(TASK_PLAN_STEP_FILE.ID.asc()) + .limit(maxRecordSize) + .fetch(); + return result.map(record -> { + long id = record.get(TASK_PLAN_STEP_FILE.ID).longValue(); + String targetJsonStr = record.get(TASK_PLAN_STEP_FILE.DESTINATION_HOST_LIST); + TaskTargetDTO target = JsonUtils.fromJson(targetJsonStr, TaskTargetDTO.class); + return new TaskTargetRecord(id, target); + }); + } + + @Override + public void updateTaskTargets(List records) { + if (CollectionUtils.isEmpty(records)) { + return; + } + records.forEach(record -> + CTX.update(TASK_PLAN_STEP_FILE) + .set(TASK_PLAN_STEP_FILE.DESTINATION_HOST_LIST, convertTargetToJson(record.getTarget())) + .where(TASK_PLAN_STEP_FILE.ID.eq(ULong.valueOf(record.getId()))) + .execute()); + } + } + + private class TaskPlanStepFileListTargetMigration extends AbstractTaskTargetMigration { + + TaskPlanStepFileListTargetMigration(List appIdList, boolean dryRun) { + super(appIdList, dryRun); + } + + @Override + public String getTableName() { + return "task_plan_step_file_list"; + } + + @Override + public List listTaskTargets(List appIdList, Long fromRecordId, int maxRecordSize) { + List conditions = new ArrayList<>(); + conditions.add(TASK_PLAN_STEP_FILE_LIST.ID.ge(ULong.valueOf(fromRecordId))); + if (appIdList != null) { + conditions.add(TASK_PLAN.APP_ID.in(appIdList)); + } + Result result = CTX.select( + TASK_PLAN_STEP_FILE_LIST.ID, + TASK_PLAN_STEP_FILE_LIST.HOST) + .from(TASK_PLAN_STEP_FILE_LIST) + .join(TASK_PLAN_STEP) + .on(TASK_PLAN_STEP_FILE_LIST.STEP_ID.eq(TASK_PLAN_STEP.ID)) + .join(TASK_PLAN) + .on(TASK_PLAN_STEP.PLAN_ID.eq(TASK_PLAN.ID)) + .where(conditions) + .orderBy(TASK_PLAN_STEP_FILE_LIST.ID.asc()) + .limit(maxRecordSize) + .fetch(); + return result.map(record -> { + long id = record.get(TASK_PLAN_STEP_FILE_LIST.ID).longValue(); + String targetJsonStr = record.get(TASK_PLAN_STEP_FILE_LIST.HOST); + TaskTargetDTO target = JsonUtils.fromJson(targetJsonStr, TaskTargetDTO.class); + return new TaskTargetRecord(id, target); + }); + } + + @Override + public void updateTaskTargets(List records) { + if (CollectionUtils.isEmpty(records)) { + return; + } + records.forEach(record -> + CTX.update(TASK_PLAN_STEP_FILE_LIST) + .set(TASK_PLAN_STEP_FILE_LIST.HOST, convertTargetToJson(record.getTarget())) + .where(TASK_PLAN_STEP_FILE_LIST.ID.eq(ULong.valueOf(record.getId()))) + .execute()); + } + } + + private class TaskPlanVariableTargetMigration extends AbstractTaskTargetMigration { + + TaskPlanVariableTargetMigration(List appIdList, boolean dryRun) { + super(appIdList, dryRun); + } + + @Override + public String getTableName() { + return "task_plan_variable"; + } + + @Override + public List listTaskTargets(List appIdList, Long fromRecordId, int maxRecordSize) { + List conditions = new ArrayList<>(); + conditions.add(TASK_PLAN_VARIABLE.TYPE.eq(UByte.valueOf(TaskVariableTypeEnum.HOST_LIST.getType()))); + if (appIdList != null) { + conditions.add(TASK_PLAN.APP_ID.in(appIdList)); + } + Result result = CTX.select( + TASK_PLAN_VARIABLE.ID, + TASK_PLAN_VARIABLE.DEFAULT_VALUE) + .from(TASK_PLAN_VARIABLE) + .join(TASK_PLAN) + .on(TASK_PLAN_VARIABLE.PLAN_ID.eq(TASK_PLAN.ID)) + .where(conditions) + .and(TASK_PLAN_VARIABLE.ID.ge(ULong.valueOf(fromRecordId))) + .orderBy(TASK_PLAN_VARIABLE.ID.asc()) + .limit(maxRecordSize) + .fetch(); + return result.map(record -> { + long id = record.get(TASK_PLAN_VARIABLE.ID).longValue(); + String targetJsonStr = record.get(TASK_PLAN_VARIABLE.DEFAULT_VALUE); + TaskTargetDTO target = JsonUtils.fromJson(targetJsonStr, TaskTargetDTO.class); + return new TaskTargetRecord(id, target); + }); + } + + @Override + public void updateTaskTargets(List records) { + if (CollectionUtils.isEmpty(records)) { + return; + } + records.forEach(record -> + CTX.update(TASK_PLAN_VARIABLE) + .set(TASK_PLAN_VARIABLE.DEFAULT_VALUE, convertTargetToJson(record.getTarget())) + .where(TASK_PLAN_VARIABLE.ID.eq(ULong.valueOf(record.getId()))) + .and(TASK_PLAN_VARIABLE.TYPE.eq(UByte.valueOf(TaskVariableTypeEnum.HOST_LIST.getType()))) + .execute()); + } + } + +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/migration/AddHostIdForWhiteIpMigrationTask.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/migration/AddHostIdForWhiteIpMigrationTask.java new file mode 100644 index 0000000000..de9bf4c14a --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/migration/AddHostIdForWhiteIpMigrationTask.java @@ -0,0 +1,193 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.migration; + +import com.tencent.bk.job.common.cc.sdk.BizCmdbClient; +import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; +import com.tencent.bk.job.manage.dao.whiteip.WhiteIPIPDAO; +import com.tencent.bk.job.manage.model.dto.whiteip.WhiteIPIPDTO; +import com.tencent.bk.job.manage.model.migration.AddHostIdResult; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.util.StopWatch; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * 对IP白名单数据中不存在hostId的数据使用CMDB中的完整数据补全hostId + */ +@Slf4j +@Service +public class AddHostIdForWhiteIpMigrationTask { + + private final WhiteIPIPDAO whiteIPIPDAO; + private final BizCmdbClient bizCmdbClient; + + @Autowired + public AddHostIdForWhiteIpMigrationTask(WhiteIPIPDAO whiteIPIPDAO, BizCmdbClient bizCmdbClient) { + this.whiteIPIPDAO = whiteIPIPDAO; + this.bizCmdbClient = bizCmdbClient; + } + + private String getTaskName() { + return "migrate_white_ip_ip"; + } + + private AddHostIdResult getInitAddHostIdResult() { + AddHostIdResult result = new AddHostIdResult(getTaskName()); + result.setTotalRecords(0); + result.setSuccessRecords(0); + result.setSuccess(true); + return result; + } + + public AddHostIdResult execute(boolean isDryRun) { + StopWatch watch = new StopWatch(getTaskName()); + try { + return makeUpWhiteIps(isDryRun, watch); + } catch (Throwable t) { + log.warn("Fail to makeUpWhiteIps", t); + AddHostIdResult result = getInitAddHostIdResult(); + result.setSuccess(false); + return result; + } finally { + if (watch.isRunning()) { + watch.stop(); + } + log.info("whiteIp makeUp task finished, time consuming:{}", watch.prettyPrint()); + } + } + + private AddHostIdResult makeUpWhiteIps(boolean isDryRun, StopWatch watch) { + AddHostIdResult result = getInitAddHostIdResult(); + int start = 0; + int batchSize = 100; + List whiteIpList; + int totalNullHostIdRecordsCount = 0; + int totalUpdatedCount = 0; + do { + watch.start("listWhiteIPIPWithNullHostId_" + start + "_" + (start + batchSize)); + whiteIpList = whiteIPIPDAO.listWhiteIPIPWithNullHostId(start, batchSize); + watch.stop(); + if (CollectionUtils.isEmpty(whiteIpList)) { + continue; + } + totalNullHostIdRecordsCount += whiteIpList.size(); + int count = 0; + + watch.start("addHostIdForWhiteIps_" + start + "_" + (start + batchSize)); + int foundHostIdCount = addHostIdForWhiteIps(whiteIpList); + watch.stop(); + + watch.start("updateHostId_" + start + "_" + (start + batchSize)); + log.debug("{} hostIds found for {} whiteIps", foundHostIdCount, whiteIpList.size()); + for (WhiteIPIPDTO whiteIPIPDTO : whiteIpList) { + if (whiteIPIPDTO.getHostId() != null) { + if (isDryRun) { + log.info( + "[DryRun]set hostId={} for whiteIpIp(id={})", + whiteIPIPDTO.getHostId(), + whiteIPIPDTO.getId() + ); + count += 1; + } else { + count += whiteIPIPDAO.updateHostIdById(whiteIPIPDTO.getId(), whiteIPIPDTO.getHostId()); + } + } + } + watch.stop(); + + start += batchSize; + totalUpdatedCount += count; + log.info((isDryRun ? "[DryRun]" : "") + "{}/{} whiteIps have been made up", count, whiteIpList.size()); + } while (!CollectionUtils.isEmpty(whiteIpList)); + log.info((isDryRun ? "[DryRun]" : "") + "{} whiteIps have been made up in total", totalUpdatedCount); + result.setTotalRecords(totalNullHostIdRecordsCount); + result.setSuccessRecords(totalUpdatedCount); + result.setSuccess(true); + return result; + } + + private int addHostIdForWhiteIps(List whiteIpList) { + List cloudIpList = new ArrayList<>(); + List cloudIpv6List = new ArrayList<>(); + for (WhiteIPIPDTO whiteIPIPDTO : whiteIpList) { + String ip = whiteIPIPDTO.getIp(); + String ipv6 = whiteIPIPDTO.getIpv6(); + if (StringUtils.isNotBlank(ip)) { + cloudIpList.add(whiteIPIPDTO.getCloudIp()); + } else if (StringUtils.isNotBlank(ipv6)) { + cloudIpv6List.add(whiteIPIPDTO.getCloudIpv6()); + } + } + List hostList = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(cloudIpList)) { + hostList.addAll(bizCmdbClient.listHostsByCloudIps(cloudIpList)); + } + if (CollectionUtils.isNotEmpty(cloudIpv6List)) { + hostList.addAll(bizCmdbClient.listHostsByCloudIpv6s(cloudIpv6List)); + } + Map hostMap = new HashMap<>(); + for (ApplicationHostDTO hostDTO : hostList) { + if (StringUtils.isNotBlank(hostDTO.getIp())) { + hostMap.put(hostDTO.getCloudAreaId() + ":" + hostDTO.getIp(), hostDTO); + } + if (StringUtils.isNotBlank(hostDTO.getIpv6())) { + hostMap.put(hostDTO.getCloudAreaId() + ":" + hostDTO.getIpv6(), hostDTO); + } + } + int foundHostIdCount = 0; + for (WhiteIPIPDTO whiteIPIPDTO : whiteIpList) { + String ip = whiteIPIPDTO.getIp(); + String ipv6 = whiteIPIPDTO.getIpv6(); + if (StringUtils.isNotBlank(ip) && hostMap.containsKey(whiteIPIPDTO.getCloudIp())) { + whiteIPIPDTO.setHostId(hostMap.get(whiteIPIPDTO.getCloudIp()).getHostId()); + foundHostIdCount += 1; + log.info( + "Makeup whiteIp {}(cloudIp={}) with hostId={}", + whiteIPIPDTO.getId(), + whiteIPIPDTO.getCloudIp(), + whiteIPIPDTO.getHostId() + ); + } else if (StringUtils.isNotBlank(ipv6) && hostMap.containsKey(whiteIPIPDTO.getCloudIpv6())) { + whiteIPIPDTO.setHostId(hostMap.get(whiteIPIPDTO.getCloudIpv6()).getHostId()); + foundHostIdCount += 1; + log.info( + "Makeup whiteIp {}(cloudIpv6={}) with hostId={}", + whiteIPIPDTO.getId(), + whiteIPIPDTO.getCloudIpv6(), + whiteIPIPDTO.getHostId() + ); + } + } + return foundHostIdCount; + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/migration/EncryptDbAccountPasswordMigrationTask.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/migration/EncryptDbAccountPasswordMigrationTask.java index 692f47d866..827ba9b664 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/migration/EncryptDbAccountPasswordMigrationTask.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/migration/EncryptDbAccountPasswordMigrationTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,12 +24,12 @@ package com.tencent.bk.job.manage.migration; +import com.tencent.bk.job.common.constant.AccountCategoryEnum; import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.crypto.util.AESUtils; import com.tencent.bk.job.common.exception.InternalException; import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.model.Response; -import com.tencent.bk.job.common.util.crypto.AESUtils; -import com.tencent.bk.job.manage.common.consts.account.AccountCategoryEnum; import com.tencent.bk.job.manage.config.JobManageConfig; import com.tencent.bk.job.manage.dao.AccountDAO; import com.tencent.bk.job.manage.model.dto.AccountDTO; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/migration/ResourceTagsMigrationTask.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/migration/ResourceTagsMigrationTask.java index 83f8e83ff8..5781d34898 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/migration/ResourceTagsMigrationTask.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/migration/ResourceTagsMigrationTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,6 +26,7 @@ import com.tencent.bk.job.common.constant.JobResourceTypeEnum; import com.tencent.bk.job.common.model.Response; +import com.tencent.bk.job.common.mysql.JobTransactional; import com.tencent.bk.job.manage.dao.ResourceTagDAO; import com.tencent.bk.job.manage.dao.ScriptDAO; import com.tencent.bk.job.manage.dao.template.TaskTemplateDAO; @@ -36,7 +37,6 @@ import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; import java.util.ArrayList; import java.util.List; @@ -92,7 +92,7 @@ public Response> execute() { return Response.buildSuccessResp(resourceTags); } - @Transactional(rollbackFor = {Throwable.class}) + @JobTransactional(transactionManager = "jobManageTransactionManager") public void saveResourceTags(List resourceTags) { resourceTagDAO.batchSaveResourceTags(resourceTags); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/migration/UpdateAppIdForWhiteIpMigrationTask.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/migration/UpdateAppIdForWhiteIpMigrationTask.java new file mode 100644 index 0000000000..a2e6475616 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/migration/UpdateAppIdForWhiteIpMigrationTask.java @@ -0,0 +1,114 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.migration; + +import com.tencent.bk.job.common.constant.JobConstants; +import com.tencent.bk.job.manage.dao.whiteip.WhiteIPAppRelDAO; +import com.tencent.bk.job.manage.model.dto.whiteip.WhiteIPAppRelDTO; +import com.tencent.bk.job.manage.model.migration.UpdateAppIdResult; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.util.StopWatch; + +import java.util.List; + +/** + * 对IP白名单数据中的全业务ID更新为对所有业务生效的全局ID + */ +@Slf4j +@Service +public class UpdateAppIdForWhiteIpMigrationTask { + + private final WhiteIPAppRelDAO whiteIPAppRelDAO; + + @Autowired + public UpdateAppIdForWhiteIpMigrationTask(WhiteIPAppRelDAO whiteIPAppRelDAO) { + this.whiteIPAppRelDAO = whiteIPAppRelDAO; + } + + private String getTaskName() { + return "migrate_white_app_rel"; + } + + private UpdateAppIdResult getInitUpdateAppIdResult() { + UpdateAppIdResult result = new UpdateAppIdResult(getTaskName()); + result.setTotalRecords(0); + result.setSuccessRecords(0); + result.setSuccess(true); + return result; + } + + public UpdateAppIdResult execute(boolean isDryRun) { + StopWatch watch = new StopWatch(getTaskName()); + try { + return updateAppIdsForWhiteIps(isDryRun, watch); + } catch (Throwable t) { + log.warn("Fail to update appId of whiteIps", t); + UpdateAppIdResult result = getInitUpdateAppIdResult(); + result.setSuccess(false); + return result; + } finally { + if (watch.isRunning()) { + watch.stop(); + } + log.info("Update appIds task finished, time consuming:{}", watch.prettyPrint()); + } + } + + public UpdateAppIdResult updateAppIdsForWhiteIps(boolean isDryRun, StopWatch watch) { + UpdateAppIdResult result = getInitUpdateAppIdResult(); + watch.start("listAppRelByAppId"); + List whiteIpAppRelList = + whiteIPAppRelDAO.listAppRelByAppId(JobConstants.DEFAULT_ALL_BIZ_SET_ID); + watch.stop(); + result.setTotalRecords(whiteIpAppRelList.size()); + if (isDryRun) { + for (WhiteIPAppRelDTO whiteIPAppRelDTO : whiteIpAppRelList) { + log.info( + "[DryRun]update (recordId={},appId={}) to ({},{})", + whiteIPAppRelDTO.getRecordId(), + whiteIPAppRelDTO.getAppId(), + whiteIPAppRelDTO.getRecordId(), + JobConstants.PUBLIC_APP_ID + ); + } + } else { + watch.start("updateAppId"); + int affectedRowNum = + whiteIPAppRelDAO.updateAppId(JobConstants.DEFAULT_ALL_BIZ_SET_ID, JobConstants.PUBLIC_APP_ID); + watch.stop(); + result.setSuccessRecords(affectedRowNum); + log.info( + "{}/{} whiteIpAppRel records app_id updated:{}->{}", + affectedRowNum, + whiteIpAppRelList.size(), + JobConstants.DEFAULT_ALL_BIZ_SET_ID, + JobConstants.PUBLIC_APP_ID + ); + } + return result; + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/db/CacheAppDO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/db/CacheAppDO.java index f4d242bd79..6d3066f1c5 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/db/CacheAppDO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/db/CacheAppDO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/db/CacheHostDO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/db/CacheHostDO.java index 124307ddbf..1abfb904ae 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/db/CacheHostDO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/db/CacheHostDO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,12 +25,17 @@ package com.tencent.bk.job.manage.model.db; import com.fasterxml.jackson.annotation.JsonInclude; +import com.tencent.bk.job.common.annotation.PersistenceObject; +import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; import lombok.Getter; import lombok.Setter; +import java.util.StringJoiner; + /** * Redis 缓存主机DO */ +@PersistenceObject @Getter @Setter @JsonInclude(JsonInclude.Include.NON_EMPTY) @@ -41,10 +46,15 @@ public class CacheHostDO { private Long cloudAreaId; /** - * ip + * ipv4 */ private String ip; + /** + * ipv6 + */ + private String ipv6; + /** * 主机ID */ @@ -64,4 +74,76 @@ public class CacheHostDO { * CMDB业务ID */ private Long bizId; + + /** + * 主机名称 + */ + private String hostDesc; + + /** + * 主机Agent状态 + */ + private Integer gseAgentStatus; + + /** + * 操作系统 + */ + private String os; + + /** + * 操作系统类型 + */ + private String osType; + + private long cacheTime; + + public ApplicationHostDTO toApplicationHostDTO() { + ApplicationHostDTO host = new ApplicationHostDTO(); + host.setBizId(this.bizId); + host.setAppId(this.appId); + host.setCloudAreaId(this.cloudAreaId); + host.setIp(this.ip); + host.setIpv6(ipv6); + host.setHostId(this.hostId); + host.setAgentId(this.agentId); + host.setHostName(this.hostDesc); + host.setOsName(this.os); + host.setOsType(this.osType); + host.setGseAgentStatus(this.gseAgentStatus); + return host; + } + + public static CacheHostDO fromApplicationHostDTO(ApplicationHostDTO host) { + CacheHostDO cacheHost = new CacheHostDO(); + cacheHost.setBizId(host.getBizId()); + cacheHost.setAppId(host.getAppId()); + cacheHost.setCloudAreaId(host.getCloudAreaId()); + cacheHost.setIp(host.getIp()); + cacheHost.setIpv6(host.getIpv6()); + cacheHost.setHostId(host.getHostId()); + cacheHost.setAgentId(host.getAgentId()); + cacheHost.setHostDesc(host.getHostName()); + cacheHost.setOs(host.getOsName()); + cacheHost.setOsType(host.getOsType()); + cacheHost.setGseAgentStatus(host.getGseAgentStatus()); + return cacheHost; + } + + @Override + public String toString() { + return new StringJoiner(", ", CacheHostDO.class.getSimpleName() + "[", "]") + .add("cloudAreaId=" + cloudAreaId) + .add("ip='" + ip + "'") + .add("ipv6='" + ipv6 + "'") + .add("hostId=" + hostId) + .add("agentId='" + agentId + "'") + .add("appId=" + appId) + .add("bizId=" + bizId) + .add("hostDesc='" + hostDesc + "'") + .add("gseAgentStatus=" + gseAgentStatus) + .add("os='" + os + "'") + .add("osType='" + osType + "'") + .add("cacheTime=" + cacheTime) + .toString(); + } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/db/DangerousRuleDO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/db/DangerousRuleDO.java index b23c1306f5..e9c3214dba 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/db/DangerousRuleDO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/db/DangerousRuleDO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/AccountDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/AccountDTO.java index 4969e19a45..5be7c7f926 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/AccountDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/AccountDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,11 +24,17 @@ package com.tencent.bk.job.manage.model.dto; +import com.tencent.bk.job.common.constant.AccountCategoryEnum; import com.tencent.bk.job.common.esb.util.EsbDTOAppScopeMappingHelper; -import com.tencent.bk.job.manage.common.consts.account.AccountCategoryEnum; -import com.tencent.bk.job.manage.common.consts.account.AccountTypeEnum; +import com.tencent.bk.job.common.model.dto.ResourceScope; +import com.tencent.bk.job.common.service.AppScopeMappingService; +import com.tencent.bk.job.common.util.ApplicationContextRegister; +import com.tencent.bk.job.common.util.I18nUtil; +import com.tencent.bk.job.common.util.Utils; +import com.tencent.bk.job.manage.api.common.constants.account.AccountTypeEnum; import com.tencent.bk.job.manage.model.esb.v3.response.EsbAccountV3DTO; import com.tencent.bk.job.manage.model.inner.ServiceAccountDTO; +import com.tencent.bk.job.manage.model.web.vo.AccountVO; import lombok.Getter; import lombok.Setter; import lombok.ToString; @@ -143,8 +149,37 @@ public EsbAccountV3DTO toEsbAccountV3DTO() { esbAccount.setCreateTime(createTime); esbAccount.setLastModifyUser(lastModifyUser); esbAccount.setLastModifyTime(lastModifyTime); + esbAccount.setDescription(remark); return esbAccount; } + public AccountVO toAccountVO() { + AccountVO accountVO = new AccountVO(); + accountVO.setId(id); + AppScopeMappingService appScopeMappingService = + ApplicationContextRegister.getBean(AppScopeMappingService.class); + ResourceScope resourceScope = appScopeMappingService.getScopeByAppId(appId); + accountVO.setScopeType(resourceScope.getType().getValue()); + accountVO.setScopeId(resourceScope.getId()); + accountVO.setAccount(account); + accountVO.setAlias(alias); + accountVO.setCategory(category.getValue()); + accountVO.setCategoryName(I18nUtil.getI18nMessage(category.getI18nKey())); + accountVO.setType(type.getType()); + accountVO.setTypeName(type.getName()); + accountVO.setOwnerUsers(Utils.getNotBlankSplitList(grantees, ",")); + accountVO.setRemark(remark); + accountVO.setOs(os); + accountVO.setPassword("******"); + accountVO.setDbPort(dbPort); + accountVO.setDbPassword("******"); + accountVO.setDbSystemAccountId(dbSystemAccountId); + accountVO.setLastModifyUser(lastModifyUser); + accountVO.setCreator(creator); + accountVO.setCreateTime(createTime); + accountVO.setLastModifyTime(lastModifyTime); + return accountVO; + } + } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/AccountDisplayDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/AccountDisplayDTO.java index 9e74d49d87..28c8b0f282 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/AccountDisplayDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/AccountDisplayDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,8 +24,8 @@ package com.tencent.bk.job.manage.model.dto; -import com.tencent.bk.job.manage.common.consts.account.AccountCategoryEnum; -import com.tencent.bk.job.manage.common.consts.account.AccountTypeEnum; +import com.tencent.bk.job.common.constant.AccountCategoryEnum; +import com.tencent.bk.job.manage.api.common.constants.account.AccountTypeEnum; import lombok.Getter; import lombok.Setter; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/AccountSearchDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/AccountSearchDTO.java new file mode 100644 index 0000000000..9d6800f2a5 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/AccountSearchDTO.java @@ -0,0 +1,51 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.dto; + +import lombok.Getter; +import lombok.Setter; + +@Getter +@Setter +public class AccountSearchDTO { + + /** + * 业务ID + */ + private Long appId; + /** + * 账号用途 + */ + private Integer category; + /** + * 账号名称 + */ + private String account; + /** + * 账号别名 + */ + private String alias; + +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/ApplicationFavorDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/ApplicationFavorDTO.java index 3c6b59566a..04d36e1f5f 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/ApplicationFavorDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/ApplicationFavorDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/CredentialDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/CredentialDTO.java index b517c160c9..28ee4b5859 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/CredentialDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/CredentialDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,9 +24,15 @@ package com.tencent.bk.job.manage.model.dto; -import com.tencent.bk.job.manage.common.consts.CredentialTypeEnum; import com.tencent.bk.job.common.model.dto.CommonCredential; +import com.tencent.bk.job.common.model.dto.ResourceScope; +import com.tencent.bk.job.common.service.AppScopeMappingService; +import com.tencent.bk.job.common.util.ApplicationContextRegister; +import com.tencent.bk.job.manage.api.common.constants.CredentialTypeEnum; +import com.tencent.bk.job.manage.model.esb.v3.response.EsbCredentialSimpleInfoV3DTO; +import com.tencent.bk.job.manage.model.esb.v3.response.EsbCredentialV3DTO; import com.tencent.bk.job.manage.model.inner.resp.ServiceCredentialDTO; +import com.tencent.bk.job.manage.model.web.vo.CredentialBasicVO; import com.tencent.bk.job.manage.model.web.vo.CredentialVO; import lombok.AllArgsConstructor; import lombok.Data; @@ -98,6 +104,13 @@ public CredentialVO toVO() { return credentialVO; } + public CredentialBasicVO toBasicVO() { + CredentialBasicVO credentialBasicVO = new CredentialBasicVO(); + credentialBasicVO.setId(id); + credentialBasicVO.setName(name); + return credentialBasicVO; + } + public ServiceCredentialDTO toServiceCredentialDTO() { ServiceCredentialDTO serviceCredentialDTO = new ServiceCredentialDTO(); serviceCredentialDTO.setId(id); @@ -108,6 +121,28 @@ public ServiceCredentialDTO toServiceCredentialDTO() { return serviceCredentialDTO; } + public EsbCredentialV3DTO toEsbCredentialV3DTO() { + EsbCredentialV3DTO esbCredentialV3DTO = new EsbCredentialV3DTO(); + esbCredentialV3DTO.setId(id); + esbCredentialV3DTO.setName(name); + esbCredentialV3DTO.setType(type); + esbCredentialV3DTO.setDescription(description); + AppScopeMappingService appScopeMappingService = + ApplicationContextRegister.getBean(AppScopeMappingService.class); + ResourceScope resourceScope = appScopeMappingService.getScopeByAppId(appId); + esbCredentialV3DTO.setScopeType(resourceScope.getType().getValue()); + esbCredentialV3DTO.setScopeId(resourceScope.getId()); + esbCredentialV3DTO.setCreator(creator); + esbCredentialV3DTO.setCreateTime(createTime); + esbCredentialV3DTO.setLastModifyUser(lastModifyUser); + esbCredentialV3DTO.setLastModifyTime(lastModifyTime); + return esbCredentialV3DTO; + } + + public EsbCredentialSimpleInfoV3DTO toEsbCredentialSimpleInfoV3DTO() { + return new EsbCredentialSimpleInfoV3DTO(id, name); + } + public String getFirstValue() { if (type.equals(CredentialTypeEnum.USERNAME_PASSWORD.name())) { return credential.getUsername(); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/DynamicGroupDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/DynamicGroupDTO.java new file mode 100644 index 0000000000..c42a3dd4de --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/DynamicGroupDTO.java @@ -0,0 +1,77 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.dto; + +import com.tencent.bk.job.common.cc.model.CcDynamicGroupDTO; +import com.tencent.bk.job.manage.model.web.vo.DynamicGroupBasicVO; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.EqualsAndHashCode; +import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; + +@Slf4j +@Data +@EqualsAndHashCode +@NoArgsConstructor +@AllArgsConstructor +public class DynamicGroupDTO { + + private Long bizId; + + private String id; + + private String name; + + private String lastTime; + + public static DynamicGroupDTO fromCcGroupDTO(CcDynamicGroupDTO ccGroupDTO) { + if (ccGroupDTO == null) { + return null; + } + DynamicGroupDTO dynamicGroupDTO = new DynamicGroupDTO(); + dynamicGroupDTO.setId(ccGroupDTO.getId()); + dynamicGroupDTO.setBizId(ccGroupDTO.getBizId()); + dynamicGroupDTO.setName(ccGroupDTO.getName()); + dynamicGroupDTO.setLastTime(ccGroupDTO.getLastTime()); + return dynamicGroupDTO; + } + + public DynamicGroupBasicVO toBasicVO() { + return new DynamicGroupBasicVO(id, name, lastTime, null); + } + + public ZonedDateTime getParsedLastTime() { + try { + return ZonedDateTime.parse(lastTime, DateTimeFormatter.ISO_DATE_TIME); + } catch (Exception e) { + log.warn("Fail to parse ZonedDateTime from lastTime {}, format=ISO_DATE_TIME", lastTime); + return null; + } + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/DynamicServerGroupDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/DynamicServerGroupDTO.java index 9390bc355d..3251e11403 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/DynamicServerGroupDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/DynamicServerGroupDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/GlobalSettingDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/GlobalSettingDTO.java index f9cf26d752..7a712522eb 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/GlobalSettingDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/GlobalSettingDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/HostTopoDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/HostTopoDTO.java index 19fd32c435..b0ef04c179 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/HostTopoDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/HostTopoDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,10 +25,13 @@ package com.tencent.bk.job.manage.model.dto; import com.tencent.bk.job.common.cc.model.result.HostRelationEventDetail; +import com.tencent.bk.job.common.util.TimeUtil; +import com.tencent.bk.job.manage.model.inner.resp.ServiceHostTopoDTO; import lombok.AllArgsConstructor; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; +import org.apache.commons.lang3.StringUtils; /** * 主机拓扑 @@ -54,9 +57,32 @@ public class HostTopoDTO { * 模块ID */ private Long moduleId; + /** + * CMDB中的数据最后修改时间 + */ + private Long lastTime; public static HostTopoDTO fromHostRelationEvent(HostRelationEventDetail eventDetail) { - return new HostTopoDTO(eventDetail.getHostId(), eventDetail.getBizId(), eventDetail.getSetId(), - eventDetail.getModuleId()); + Long lastTimeMills = null; + if (StringUtils.isNotBlank(eventDetail.getLastTime())) { + lastTimeMills = TimeUtil.parseIsoZonedTimeToMillis(eventDetail.getLastTime()); + } + return new HostTopoDTO( + eventDetail.getHostId(), + eventDetail.getBizId(), + eventDetail.getSetId(), + eventDetail.getModuleId(), + lastTimeMills + ); + } + + public ServiceHostTopoDTO toServiceHostTopoDTO() { + return new ServiceHostTopoDTO( + hostId, + bizId, + setId, + moduleId, + lastTime + ); } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/IpDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/IpDTO.java index 6f4b4bebc5..b86b70800a 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/IpDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/IpDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/KubeTopoNode.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/KubeTopoNode.java new file mode 100644 index 0000000000..23bae15989 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/KubeTopoNode.java @@ -0,0 +1,54 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.dto; + +import com.tencent.bk.job.common.cc.model.container.KubeNodeID; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * 容器拓扑节点 + */ +@NoArgsConstructor +@Data +public class KubeTopoNode { + /** + * 节点的资源类型 + */ + private String kind; + /** + * 拓扑节点 ID + */ + private Long id; + + public KubeTopoNode(String kind, Long id) { + this.kind = kind; + this.id = id; + } + + public KubeNodeID toKubeNodeID() { + return new KubeNodeID(kind, id); + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/ResourceTagDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/ResourceTagDTO.java index 0e73fe2221..90722df8b2 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/ResourceTagDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/ResourceTagDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/ScriptBasicDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/ScriptBasicDTO.java index db08f7375b..a2b1590231 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/ScriptBasicDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/ScriptBasicDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/ScriptCheckResultItemDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/ScriptCheckResultItemDTO.java index ac60c38d42..e831f42e7a 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/ScriptCheckResultItemDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/ScriptCheckResultItemDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,8 +24,8 @@ package com.tencent.bk.job.manage.model.dto; -import com.tencent.bk.job.manage.common.consts.RuleMatchHandleActionEnum; -import com.tencent.bk.job.manage.common.consts.script.ScriptCheckErrorLevelEnum; +import com.tencent.bk.job.manage.api.common.constants.RuleMatchHandleActionEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptCheckErrorLevelEnum; import com.tencent.bk.job.manage.model.inner.ServiceScriptCheckResultItemDTO; import lombok.Data; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/ScriptDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/ScriptDTO.java index 0207878873..9a12397186 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/ScriptDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/ScriptDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,7 +25,7 @@ package com.tencent.bk.job.manage.model.dto; import com.tencent.bk.job.common.esb.util.EsbDTOAppScopeMappingHelper; -import com.tencent.bk.job.manage.common.consts.JobResourceStatusEnum; +import com.tencent.bk.job.manage.api.common.constants.JobResourceStatusEnum; import com.tencent.bk.job.manage.model.esb.v3.response.EsbScriptV3DTO; import com.tencent.bk.job.manage.model.esb.v3.response.EsbScriptVersionDetailV3DTO; import lombok.Getter; @@ -137,6 +137,7 @@ public EsbScriptV3DTO toEsbScriptV3DTO() { esbScript.setLastModifyUser(lastModifyUser); esbScript.setLastModifyTime(lastModifyTime); esbScript.setOnlineScriptVersionId(scriptVersionId); + esbScript.setDescription(description); return esbScript; } @@ -157,6 +158,37 @@ public EsbScriptVersionDetailV3DTO toEsbScriptVersionDetailV3DTO() { esbScriptVersion.setCreateTime(createTime); esbScriptVersion.setLastModifyUser(lastModifyUser); esbScriptVersion.setLastModifyTime(lastModifyTime); + esbScriptVersion.setType(type); + esbScriptVersion.setDescription(description); return esbScriptVersion; } + + public EsbScriptVersionDetailV3DTO toEsbCreateScriptV3DTO() { + EsbScriptVersionDetailV3DTO scriptDetail = new EsbScriptVersionDetailV3DTO(); + scriptDetail.setId(scriptVersionId); + scriptDetail.setScriptId(id); + scriptDetail.setName(name); + scriptDetail.setType(type); + scriptDetail.setContent(content); + scriptDetail.setCreator(creator); + scriptDetail.setCreateTime(createTime); + scriptDetail.setLastModifyUser(lastModifyUser); + scriptDetail.setLastModifyTime(lastModifyTime); + scriptDetail.setVersion(version); + scriptDetail.setStatus(status); + scriptDetail.setDescription(description); + scriptDetail.setVersionDesc(versionDesc); + if (appId != null && !appId.equals(PUBLIC_APP_ID)) { + EsbDTOAppScopeMappingHelper.fillEsbAppScopeDTOByAppId(appId, scriptDetail); + } + return scriptDetail; + } + + public EsbScriptVersionDetailV3DTO toEsbManageScriptV3DTO() { + EsbScriptVersionDetailV3DTO result = new EsbScriptVersionDetailV3DTO(); + result.setId(scriptVersionId); + result.setScriptId(id); + result.setStatus(status); + return result; + } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/ScriptRelatedTaskPlanDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/ScriptRelatedTaskPlanDTO.java index abe10e29b1..2ebcddb28d 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/ScriptRelatedTaskPlanDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/ScriptRelatedTaskPlanDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,7 @@ package com.tencent.bk.job.manage.model.dto; -import com.tencent.bk.job.manage.common.consts.JobResourceStatusEnum; +import com.tencent.bk.job.manage.api.common.constants.JobResourceStatusEnum; import lombok.Getter; import lombok.Setter; import lombok.ToString; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/ScriptStatusUpdateMessageDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/ScriptStatusUpdateMessageDTO.java index 61459ec4b2..f0d69f92bc 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/ScriptStatusUpdateMessageDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/ScriptStatusUpdateMessageDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,7 @@ package com.tencent.bk.job.manage.model.dto; -import com.tencent.bk.job.manage.common.consts.JobResourceStatusEnum; +import com.tencent.bk.job.manage.api.common.constants.JobResourceStatusEnum; import lombok.AllArgsConstructor; import lombok.Data; import lombok.EqualsAndHashCode; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/ScriptSyncTemplateStepDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/ScriptSyncTemplateStepDTO.java index bc4f63c9d5..372672024a 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/ScriptSyncTemplateStepDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/ScriptSyncTemplateStepDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/StepDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/StepDTO.java index e30ffc35fa..f46d8ed857 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/StepDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/StepDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/SyncScriptResultDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/SyncScriptResultDTO.java index 69012c594c..90bb7d2ff6 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/SyncScriptResultDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/SyncScriptResultDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/TagDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/TagDTO.java index 22f88ceb6e..22ed8cd139 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/TagDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/TagDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,6 +25,7 @@ package com.tencent.bk.job.manage.model.dto; import com.tencent.bk.job.common.model.dto.BasicDTO; +import com.tencent.bk.job.manage.model.esb.v3.response.EsbTagV3DTO; import com.tencent.bk.job.manage.model.inner.ServiceTagDTO; import com.tencent.bk.job.manage.model.web.vo.TagVO; import lombok.Getter; @@ -100,6 +101,17 @@ public static ServiceTagDTO toServiceDTO(TagDTO tagDTO) { return serviceTag; } + public static EsbTagV3DTO toEsbTagV3DTO(TagDTO tag) { + if (tag == null) { + return null; + } + EsbTagV3DTO result = new EsbTagV3DTO(); + result.setId(tag.getId()); + result.setName(tag.getName()); + result.setDescription(tag.getDescription()); + return result; + } + @Override public TagDTO clone() { TagDTO tag = new TagDTO(); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/TaskDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/TaskDTO.java index c32e713cca..e9ccf5a685 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/TaskDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/TaskDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/TaskPlanQueryDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/TaskPlanQueryDTO.java index fab7fd81d5..0155a7385c 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/TaskPlanQueryDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/TaskPlanQueryDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/TemplateStepIDDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/TemplateStepIDDTO.java index 1891a8c933..5b040d8238 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/TemplateStepIDDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/TemplateStepIDDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/TemplateStepScriptStatusInfo.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/TemplateStepScriptStatusInfo.java new file mode 100644 index 0000000000..a2904de9cd --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/TemplateStepScriptStatusInfo.java @@ -0,0 +1,38 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.dto; + +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@NoArgsConstructor +public class TemplateStepScriptStatusInfo { + private Long templateId; + private Long stepId; + private String scriptId; + private Long scriptVersionId; + private int scriptStatusFlags; +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/HostInfoConverter.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/HostInfoConverter.java index aa788b6753..bc3dd66df0 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/HostInfoConverter.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/HostInfoConverter.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -37,7 +37,6 @@ public static ServiceHostInfoDTO convertToServiceHostInfoDTO(ApplicationHostDTO serviceHostInfoDTO.setCloudAreaId(applicationHostDTO.getCloudAreaId()); serviceHostInfoDTO.setHostId(applicationHostDTO.getHostId()); serviceHostInfoDTO.setIp(applicationHostDTO.getIp()); - serviceHostInfoDTO.setDisplayIp(applicationHostDTO.getDisplayIp()); return serviceHostInfoDTO; } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/NotifyTemplateConverter.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/NotifyTemplateConverter.java index cb76b955ae..0c66ff45fc 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/NotifyTemplateConverter.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/NotifyTemplateConverter.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,12 +27,11 @@ import com.tencent.bk.job.common.i18n.locale.LocaleUtils; import com.tencent.bk.job.common.i18n.service.MessageI18nService; import com.tencent.bk.job.common.util.JobContextUtil; -import com.tencent.bk.job.manage.common.consts.notify.NotifyConsts; +import com.tencent.bk.job.manage.api.common.constants.notify.NotifyConsts; import com.tencent.bk.job.manage.dao.notify.NotifyEsbChannelDAO; import com.tencent.bk.job.manage.model.dto.notify.NotifyEsbChannelDTO; import com.tencent.bk.job.manage.model.dto.notify.NotifyTemplateDTO; import com.tencent.bk.job.manage.model.web.vo.notify.ChannelTemplateDetailVO; -import org.jooq.DSLContext; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Component; @@ -43,14 +42,12 @@ public class NotifyTemplateConverter { final private MessageI18nService i18nService; final private NotifyEsbChannelDAO notifyEsbChannelDAO; - final private DSLContext dslContext; @Autowired - public NotifyTemplateConverter(MessageI18nService i18nService, NotifyEsbChannelDAO notifyEsbChannelDAO, - DSLContext dslContext) { + public NotifyTemplateConverter(MessageI18nService i18nService, + NotifyEsbChannelDAO notifyEsbChannelDAO) { this.i18nService = i18nService; this.notifyEsbChannelDAO = notifyEsbChannelDAO; - this.dslContext = dslContext; } public ChannelTemplateDetailVO convertToChannelTemplateDetailVO(NotifyTemplateDTO notifyTemplateDTO) { @@ -58,7 +55,7 @@ public ChannelTemplateDetailVO convertToChannelTemplateDetailVO(NotifyTemplateDT return null; } String channel = notifyTemplateDTO.getChannel(); - List channelDTOList = notifyEsbChannelDAO.listNotifyEsbChannel(dslContext); + List channelDTOList = notifyEsbChannelDAO.listNotifyEsbChannel(); String channelLabel = ""; for (NotifyEsbChannelDTO notifyEsbChannelDTO : channelDTOList) { if (notifyEsbChannelDTO.getType().equals(channel)) { diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/ScriptConverter.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/ScriptConverter.java index af94c2ba1f..b8a630e29f 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/ScriptConverter.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/ScriptConverter.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,12 +27,18 @@ import com.tencent.bk.job.common.model.dto.ResourceScope; import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.common.util.ApplicationContextRegister; +import com.tencent.bk.job.common.util.Base64Util; +import com.tencent.bk.job.common.util.I18nUtil; +import com.tencent.bk.job.manage.api.common.constants.JobResourceStatusEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; import com.tencent.bk.job.manage.model.dto.ScriptDTO; import com.tencent.bk.job.manage.model.dto.TagDTO; import com.tencent.bk.job.manage.model.inner.ServiceScriptDTO; import com.tencent.bk.job.manage.model.web.vo.BasicScriptVO; import com.tencent.bk.job.manage.model.web.vo.ScriptVO; import com.tencent.bk.job.manage.model.web.vo.TagVO; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; import java.util.ArrayList; import java.util.List; @@ -48,44 +54,27 @@ public static ScriptVO convertToScriptVO(ScriptDTO script) { return null; } ScriptVO scriptVO = new ScriptVO(); - if (script.getAppId() != null && !script.getAppId().equals(PUBLIC_APP_ID)) { - AppScopeMappingService appScopeMappingService = - ApplicationContextRegister.getBean(AppScopeMappingService.class); - ResourceScope resourceScope = appScopeMappingService.getScopeByAppId(script.getAppId()); - scriptVO.setScopeType(resourceScope.getType().getValue()); - scriptVO.setScopeId(resourceScope.getId()); + fillBasicScriptVO(scriptVO, script); + + if (StringUtils.isNotBlank(script.getContent())) { + scriptVO.setContent(Base64Util.encodeContentToStr(script.getContent())); } - scriptVO.setLastModifyUser(script.getLastModifyUser()); - scriptVO.setCategory(script.getCategory()); - scriptVO.setContent(script.getContent()); - if (null != script.getCreateTime()) { - scriptVO.setCreateTime(script.getCreateTime()); + return scriptVO; + } + + private static List convertToTagVOs(List tags) { + if (CollectionUtils.isEmpty(tags)) { + return null; } - scriptVO.setCreator(script.getCreator()); - scriptVO.setScriptVersionId(script.getScriptVersionId()); - scriptVO.setId(script.getId()); - scriptVO.setPublicScript(script.isPublicScript()); - if (null != script.getLastModifyTime()) { - scriptVO.setLastModifyTime(script.getLastModifyTime()); + List tagVOS = new ArrayList<>(); + for (TagDTO tagDTO : tags) { + TagVO tagVO = new TagVO(); + tagVO.setId(tagDTO.getId()); + tagVO.setName(tagDTO.getName()); + tagVOS.add(tagVO); } - scriptVO.setName(script.getName()); - scriptVO.setType(script.getType()); - scriptVO.setVersion(script.getVersion()); - scriptVO.setStatus(script.getStatus()); - scriptVO.setVersionDesc(script.getVersionDesc()); - scriptVO.setDescription(script.getDescription()); - if (script.getTags() != null && !script.getTags().isEmpty()) { - List tagVOS = new ArrayList<>(); - for (TagDTO tagDTO : script.getTags()) { - TagVO tagVO = new TagVO(); - tagVO.setId(tagDTO.getId()); - tagVO.setName(tagDTO.getName()); - tagVOS.add(tagVO); - } - scriptVO.setTags(tagVOS); - } - return scriptVO; + return tagVOS; } public static BasicScriptVO convertToBasicScriptVO(ScriptDTO script) { @@ -93,6 +82,12 @@ public static BasicScriptVO convertToBasicScriptVO(ScriptDTO script) { return null; } BasicScriptVO scriptVO = new BasicScriptVO(); + fillBasicScriptVO(scriptVO, script); + + return scriptVO; + } + + private static void fillBasicScriptVO(BasicScriptVO scriptVO, ScriptDTO script) { if (script.getAppId() != null && !script.getAppId().equals(PUBLIC_APP_ID)) { AppScopeMappingService appScopeMappingService = ApplicationContextRegister.getBean(AppScopeMappingService.class); @@ -100,15 +95,26 @@ public static BasicScriptVO convertToBasicScriptVO(ScriptDTO script) { scriptVO.setScopeType(resourceScope.getType().getValue()); scriptVO.setScopeId(resourceScope.getId()); } - scriptVO.setCategory(script.getCategory()); - scriptVO.setScriptVersionId(script.getScriptVersionId()); scriptVO.setId(script.getId()); + scriptVO.setScriptVersionId(script.getScriptVersionId()); + scriptVO.setCategory(script.getCategory()); scriptVO.setPublicScript(script.isPublicScript()); scriptVO.setName(script.getName()); scriptVO.setType(script.getType()); + scriptVO.setTypeName(ScriptTypeEnum.getName(script.getType())); scriptVO.setVersion(script.getVersion()); scriptVO.setStatus(script.getStatus()); - return scriptVO; + JobResourceStatusEnum status = JobResourceStatusEnum.getJobResourceStatus(script.getStatus()); + if (status != null) { + scriptVO.setStatusDesc(I18nUtil.getI18nMessage(status.getStatusI18nKey())); + } + scriptVO.setCreateTime(script.getCreateTime()); + scriptVO.setCreator(script.getCreator()); + scriptVO.setLastModifyTime(script.getLastModifyTime()); + scriptVO.setLastModifyUser(script.getLastModifyUser()); + scriptVO.setDescription(script.getDescription()); + scriptVO.setVersionDesc(script.getVersionDesc()); + scriptVO.setTags(convertToTagVOs(script.getTags())); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/ScriptRelatedTemplateStepConverter.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/ScriptRelatedTemplateStepConverter.java index 8ed54cc06a..08afb84140 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/ScriptRelatedTemplateStepConverter.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/ScriptRelatedTemplateStepConverter.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/TaskApprovalStepConverter.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/TaskApprovalStepConverter.java index 9474667bbf..d6b9ed0a9a 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/TaskApprovalStepConverter.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/TaskApprovalStepConverter.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/TaskApprovalUserConverter.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/TaskApprovalUserConverter.java index 8d47cd9678..3fcfa534de 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/TaskApprovalUserConverter.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/TaskApprovalUserConverter.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/TaskFileInfoConverter.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/TaskFileInfoConverter.java index 7a9ff2c981..9d11a3f5ff 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/TaskFileInfoConverter.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/TaskFileInfoConverter.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/TaskFileStepConverter.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/TaskFileStepConverter.java index 0878bd692f..535462e732 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/TaskFileStepConverter.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/TaskFileStepConverter.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -55,7 +55,7 @@ public static ServiceTaskFileStepDTO convertToServiceTaskFileStepDTO(TaskFileSte List serviceTaskFileInfoDTOList = new ArrayList<>(); if (originFileList != null && !originFileList.isEmpty()) { serviceTaskFileInfoDTOList = - originFileList.parallelStream() + originFileList.stream() .map(TaskFileInfoConverter::convertToServiceTaskFileInfoDTO).collect(Collectors.toList()); } serviceTaskFileStepDTO.setOriginFileList(serviceTaskFileInfoDTOList); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/TaskHostNodeConverter.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/TaskHostNodeConverter.java index 5ce52a64f9..767ccfb8d4 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/TaskHostNodeConverter.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/TaskHostNodeConverter.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -46,7 +46,7 @@ public static ServiceTaskHostNodeDTO convertToServiceTaskHostNodeDTO(TaskHostNod List serviceHostInfoDTOList = new ArrayList<>(); if (hostInfoDTOList != null && !hostInfoDTOList.isEmpty()) { serviceHostInfoDTOList = - hostInfoDTOList.parallelStream() + hostInfoDTOList.stream() .map(HostInfoConverter::convertToServiceHostInfoDTO).collect(Collectors.toList()); } serviceTaskHostNodeDTO.setHostList(serviceHostInfoDTOList); @@ -54,7 +54,7 @@ public static ServiceTaskHostNodeDTO convertToServiceTaskHostNodeDTO(TaskHostNod List serviceTaskNodeInfoDTOList = new ArrayList<>(); if (taskNodeInfoDTOList != null && !taskNodeInfoDTOList.isEmpty()) { serviceTaskNodeInfoDTOList = - taskNodeInfoDTOList.parallelStream() + taskNodeInfoDTOList.stream() .map(TaskNodeInfoConverter::convertToServiceTaskNodeInfoDTO).collect(Collectors.toList()); } serviceTaskHostNodeDTO.setNodeInfoList(serviceTaskNodeInfoDTOList); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/TaskNodeInfoConverter.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/TaskNodeInfoConverter.java index 3c675e0695..e3459317dc 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/TaskNodeInfoConverter.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/TaskNodeInfoConverter.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/TaskScriptStepConverter.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/TaskScriptStepConverter.java index 6048abf36f..dcb17919ec 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/TaskScriptStepConverter.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/TaskScriptStepConverter.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/TaskStepConverter.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/TaskStepConverter.java index 91332ee493..4c195a48f7 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/TaskStepConverter.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/TaskStepConverter.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/TaskTargetConverter.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/TaskTargetConverter.java index 42a12e55ea..3d8b4e49d0 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/TaskTargetConverter.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/TaskTargetConverter.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/TaskVariableConverter.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/TaskVariableConverter.java index dad5fb3d33..efa996967a 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/TaskVariableConverter.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/converter/TaskVariableConverter.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/customsetting/ScriptTemplateDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/customsetting/ScriptTemplateDTO.java index 3c8e8ab6de..c1b7f1bfbd 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/customsetting/ScriptTemplateDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/customsetting/ScriptTemplateDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/customsetting/ScriptTemplateVariableRenderDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/customsetting/ScriptTemplateVariableRenderDTO.java index a2ac1b3e60..b747402396 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/customsetting/ScriptTemplateVariableRenderDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/customsetting/ScriptTemplateVariableRenderDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/customsetting/UserCustomSettingDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/customsetting/UserCustomSettingDTO.java new file mode 100644 index 0000000000..09d19c41c5 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/customsetting/UserCustomSettingDTO.java @@ -0,0 +1,78 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.dto.customsetting; + +import com.tencent.bk.job.common.util.StringUtil; +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.Arrays; + +/** + * 用户自定义配置（单个模块） + */ +@Data +@NoArgsConstructor +@AllArgsConstructor +public class UserCustomSettingDTO { + /** + * 用户名 + */ + private String username; + + /** + * Job业务ID + */ + private Long appId; + + /** + * 模块 + */ + private String module; + + /** + * 配置项的值Json串 + */ + private String value; + + /** + * 最后更新人 + */ + private String lastModifyUser; + + /** + * 最后更新时间 + */ + private Long lastModifyTime; + + public String getKey() { + return getKey(username, appId, module); + } + + public static String getKey(String username, Long appId, String module) { + return StringUtil.concatCollection(Arrays.asList(username, appId, module), "::"); + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/globalsetting/DangerousRuleDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/globalsetting/DangerousRuleDTO.java index d07474590a..ec4a98360c 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/globalsetting/DangerousRuleDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/globalsetting/DangerousRuleDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,9 +26,10 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.tencent.bk.job.common.util.json.LongTimestampSerializer; -import com.tencent.bk.job.manage.common.consts.EnableStatusEnum; -import com.tencent.bk.job.manage.common.consts.RuleMatchHandleActionEnum; +import com.tencent.bk.job.manage.api.common.constants.EnableStatusEnum; +import com.tencent.bk.job.manage.api.common.constants.RuleMatchHandleActionEnum; import com.tencent.bk.job.manage.model.db.DangerousRuleDO; +import com.tencent.bk.job.manage.model.esb.v3.response.EsbDangerousRuleV3DTO; import com.tencent.bk.job.manage.model.web.vo.globalsetting.DangerousRuleVO; import lombok.Data; import lombok.EqualsAndHashCode; @@ -113,18 +114,19 @@ public DangerousRuleDTO(Long id, String expression, String description, Integer this.status = status; } - public static DangerousRuleVO toVO(DangerousRuleDTO dangerousRule) { - if (dangerousRule == null) { - return null; - } + public DangerousRuleVO toVO() { DangerousRuleVO dangerousRuleVO = new DangerousRuleVO(); - dangerousRuleVO.setId(dangerousRule.getId()); - dangerousRuleVO.setExpression(dangerousRule.getExpression()); - dangerousRuleVO.setScriptTypeList(decodeScriptType(dangerousRule.getScriptType())); - dangerousRuleVO.setDescription(dangerousRule.getDescription()); - dangerousRuleVO.setOrder(dangerousRule.getPriority()); - dangerousRuleVO.setAction(dangerousRule.getAction()); - dangerousRuleVO.setStatus(dangerousRule.getStatus()); + dangerousRuleVO.setId(id); + dangerousRuleVO.setExpression(expression); + dangerousRuleVO.setScriptTypeList(decodeScriptType(scriptType)); + dangerousRuleVO.setDescription(description); + dangerousRuleVO.setOrder(priority); + dangerousRuleVO.setAction(action); + dangerousRuleVO.setStatus(status); + dangerousRuleVO.setCreator(creator); + dangerousRuleVO.setCreateTime(createTime); + dangerousRuleVO.setLastModifier(lastModifier); + dangerousRuleVO.setLastModifyTime(lastModifyTime); return dangerousRuleVO; } @@ -177,4 +179,26 @@ public DangerousRuleDO toDangerousRuleDO() { dangerousRuleDO.setDescription(this.description); return dangerousRuleDO; } + + public EsbDangerousRuleV3DTO toEsbDangerousRuleV3DTO() { + EsbDangerousRuleV3DTO esbDangerousRuleV3DTO = new EsbDangerousRuleV3DTO(); + esbDangerousRuleV3DTO.setId(id); + esbDangerousRuleV3DTO.setExpression(expression); + esbDangerousRuleV3DTO.setScriptTypeList(decodeScriptType(scriptType)); + esbDangerousRuleV3DTO.setDescription(description); + esbDangerousRuleV3DTO.setAction(action); + esbDangerousRuleV3DTO.setStatus(status); + esbDangerousRuleV3DTO.setCreator(creator); + esbDangerousRuleV3DTO.setCreateTime(createTime); + esbDangerousRuleV3DTO.setLastModifyUser(lastModifier); + esbDangerousRuleV3DTO.setLastModifyTime(lastModifyTime); + return esbDangerousRuleV3DTO; + } + + public EsbDangerousRuleV3DTO toEsbManageDangerousRuleV3DTO() { + EsbDangerousRuleV3DTO esbManageDangerousRuleV3DTO = new EsbDangerousRuleV3DTO(); + esbManageDangerousRuleV3DTO.setId(id); + esbManageDangerousRuleV3DTO.setStatus(status); + return esbManageDangerousRuleV3DTO; + } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/globalsetting/HelperInfo.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/globalsetting/HelperInfo.java new file mode 100644 index 0000000000..5900881a97 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/globalsetting/HelperInfo.java @@ -0,0 +1,48 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.dto.globalsetting; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.annotation.PersistenceObject; +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * 全局设置-平台信息-助手信息 + */ +@PersistenceObject +@Data +@NoArgsConstructor +public class HelperInfo { + /** + * 助手链接 + */ + @JsonProperty("contactLink") + private String contactLink; + + public HelperInfo(String contactLink) { + this.contactLink = contactLink; + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/globalsetting/TitleFooter.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/globalsetting/TitleFooter.java index 5e38f605db..47a9bdb99b 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/globalsetting/TitleFooter.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/globalsetting/TitleFooter.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/globalsetting/TitleFooterDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/globalsetting/TitleFooterDTO.java index 73a2121290..b69b4c0bf5 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/globalsetting/TitleFooterDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/globalsetting/TitleFooterDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/globalsetting/UploadFileRestrictDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/globalsetting/UploadFileRestrictDTO.java index 4399db034e..5f1f18bd04 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/globalsetting/UploadFileRestrictDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/globalsetting/UploadFileRestrictDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/index/IndexGreetingDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/index/IndexGreetingDTO.java index 33a6b3f28b..3267c21ada 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/index/IndexGreetingDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/index/IndexGreetingDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/notify/AvailableEsbChannelDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/notify/AvailableEsbChannelDTO.java index 8b5246a02e..d35685faea 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/notify/AvailableEsbChannelDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/notify/AvailableEsbChannelDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/notify/EsbUserInfoDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/notify/EsbUserInfoDTO.java index f3c9f23876..e58a1fa029 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/notify/EsbUserInfoDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/notify/EsbUserInfoDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/notify/NotifyBlackUserInfoDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/notify/NotifyBlackUserInfoDTO.java index 5ec6a841fb..3b3175b8cc 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/notify/NotifyBlackUserInfoDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/notify/NotifyBlackUserInfoDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/notify/NotifyConfigStatusDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/notify/NotifyConfigStatusDTO.java index ba89c3a77e..75c11a2ea0 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/notify/NotifyConfigStatusDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/notify/NotifyConfigStatusDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/notify/NotifyEsbChannelDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/notify/NotifyEsbChannelDTO.java index 21660dcf95..f79949b41f 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/notify/NotifyEsbChannelDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/notify/NotifyEsbChannelDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/notify/NotifyPolicyRoleTargetDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/notify/NotifyPolicyRoleTargetDTO.java index 7cb0a830cc..9b9cd9aaf7 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/notify/NotifyPolicyRoleTargetDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/notify/NotifyPolicyRoleTargetDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/notify/NotifyRoleTargetChannelDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/notify/NotifyRoleTargetChannelDTO.java index 4adefd55d2..b3e84f3d9a 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/notify/NotifyRoleTargetChannelDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/notify/NotifyRoleTargetChannelDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/notify/NotifyTemplateDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/notify/NotifyTemplateDTO.java index 59111de1e1..588db59be7 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/notify/NotifyTemplateDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/notify/NotifyTemplateDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/notify/NotifyTriggerPolicyDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/notify/NotifyTriggerPolicyDTO.java index e1af525cd5..8b4e729e77 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/notify/NotifyTriggerPolicyDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/notify/NotifyTriggerPolicyDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,10 +26,14 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.tencent.bk.job.common.util.json.LongTimestampSerializer; -import com.tencent.bk.job.manage.common.consts.notify.ExecuteStatusEnum; -import com.tencent.bk.job.manage.common.consts.notify.ResourceTypeEnum; -import com.tencent.bk.job.manage.common.consts.notify.TriggerTypeEnum; -import lombok.*; +import com.tencent.bk.job.manage.api.common.constants.notify.ExecuteStatusEnum; +import com.tencent.bk.job.manage.api.common.constants.notify.ResourceTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.notify.TriggerTypeEnum; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.NoArgsConstructor; +import lombok.Setter; +import lombok.ToString; /** * 通知触发策略DTO diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/script/ScriptCitedTaskPlanDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/script/ScriptCitedTaskPlanDTO.java index 0778cced45..b0796e14cb 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/script/ScriptCitedTaskPlanDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/script/ScriptCitedTaskPlanDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/script/ScriptCitedTaskTemplateDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/script/ScriptCitedTaskTemplateDTO.java index d4e4b7680f..9e60bc49ee 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/script/ScriptCitedTaskTemplateDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/script/ScriptCitedTaskTemplateDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,7 +27,7 @@ import com.tencent.bk.job.common.model.dto.ResourceScope; import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.common.util.ApplicationContextRegister; -import com.tencent.bk.job.manage.common.consts.JobResourceStatusEnum; +import com.tencent.bk.job.manage.api.common.constants.JobResourceStatusEnum; import com.tencent.bk.job.manage.model.web.vo.ScriptCitedTemplateVO; import io.swagger.annotations.ApiModel; import io.swagger.annotations.ApiModelProperty; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskApprovalStepDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskApprovalStepDTO.java index cc761175fd..164dd8b9e7 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskApprovalStepDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskApprovalStepDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,10 +24,10 @@ package com.tencent.bk.job.manage.model.dto.task; +import com.tencent.bk.job.common.esb.model.job.v3.resp.EsbApprovalStepV3DTO; +import com.tencent.bk.job.common.esb.model.job.v3.resp.EsbUserRoleInfoV3DTO; import com.tencent.bk.job.common.model.dto.UserRoleInfoDTO; -import com.tencent.bk.job.manage.common.consts.task.TaskApprovalTypeEnum; -import com.tencent.bk.job.manage.model.esb.v3.response.EsbApprovalStepV3DTO; -import com.tencent.bk.job.manage.model.esb.v3.response.EsbUserRoleInfoV3DTO; +import com.tencent.bk.job.manage.api.common.constants.task.TaskApprovalTypeEnum; import com.tencent.bk.job.manage.model.inner.ServiceTaskApprovalStepDTO; import com.tencent.bk.job.manage.model.inner.ServiceTaskApprovalUserDTO; import com.tencent.bk.job.manage.model.web.vo.task.TaskApprovalStepVO; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskFileInfoDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskFileInfoDTO.java index 967bd6c405..987a3e7477 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskFileInfoDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskFileInfoDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,7 +26,7 @@ import com.tencent.bk.job.common.esb.model.job.v3.EsbAccountV3BasicDTO; import com.tencent.bk.job.common.esb.model.job.v3.EsbFileSourceV3DTO; -import com.tencent.bk.job.manage.common.consts.task.TaskFileTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskFileTypeEnum; import com.tencent.bk.job.manage.model.inner.ServiceAccountDTO; import com.tencent.bk.job.manage.model.inner.ServiceTaskFileInfoDTO; import com.tencent.bk.job.manage.model.web.vo.task.TaskFileSourceInfoVO; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskFileStepDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskFileStepDTO.java index a0b4946ba9..11b325897d 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskFileStepDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskFileStepDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,10 +25,14 @@ package com.tencent.bk.job.manage.model.dto.task; import com.tencent.bk.job.common.constant.DuplicateHandlerEnum; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.constant.JobConstants; import com.tencent.bk.job.common.constant.NotExistPathHandlerEnum; import com.tencent.bk.job.common.esb.model.job.v3.EsbAccountV3BasicDTO; import com.tencent.bk.job.common.esb.model.job.v3.EsbFileDestinationV3DTO; -import com.tencent.bk.job.manage.model.esb.v3.response.EsbFileStepV3DTO; +import com.tencent.bk.job.common.esb.model.job.v3.resp.EsbFileStepV3DTO; +import com.tencent.bk.job.common.exception.InvalidParamException; +import com.tencent.bk.job.execute.common.constants.FileTransferModeEnum; import com.tencent.bk.job.manage.model.inner.ServiceAccountDTO; import com.tencent.bk.job.manage.model.inner.ServiceTaskFileStepDTO; import com.tencent.bk.job.manage.model.web.vo.task.TaskFileDestinationInfoVO; @@ -92,8 +96,8 @@ public static TaskFileStepVO toVO(TaskFileStepDTO fileStep) { fileStepVO.setTimeout(fileStep.getTimeout()); fileStepVO.setOriginSpeedLimit(fileStep.getOriginSpeedLimit()); fileStepVO.setTargetSpeedLimit(fileStep.getTargetSpeedLimit()); - fileStepVO.setTransferMode(TaskFileStepVO.getTransferMode(fileStep.getDuplicateHandler(), - fileStep.getNotExistPathHandler())); + fileStepVO.setTransferMode(toTransferMode(fileStep.getDuplicateHandler(), + fileStep.getNotExistPathHandler()).getValue()); fileStepVO.setIgnoreError(fileStep.getIgnoreError() ? 1 : 0); return fileStepVO; } @@ -113,11 +117,16 @@ public static TaskFileStepDTO fromVO(Long stepId, TaskFileStepVO fileStepVO) { fileStep.setExecuteAccount(fileStepVO.getFileDestination().getAccount()); fileStep.setDestinationHostList(TaskTargetDTO.fromVO(fileStepVO.getFileDestination().getServer())); } - fileStep.setTimeout(fileStepVO.getTimeout()); + fileStep.setTimeout(fileStepVO.getTimeout() == null ? + (long) JobConstants.DEFAULT_JOB_TIMEOUT_SECONDS : fileStepVO.getTimeout()); fileStep.setOriginSpeedLimit(fileStepVO.getOriginSpeedLimit()); fileStep.setTargetSpeedLimit(fileStepVO.getTargetSpeedLimit()); - fileStep.setDuplicateHandler(DuplicateHandlerEnum.valueOf(fileStepVO.getDuplicateHandler())); - fileStep.setNotExistPathHandler(NotExistPathHandlerEnum.valueOf(fileStepVO.getNotExistPathHandler())); + if (fileStepVO.getTransferMode() != null) { + fileStep.setDuplicateHandler(TaskFileStepDTO.toDuplicateHandler( + FileTransferModeEnum.getFileTransferModeEnum(fileStepVO.getTransferMode()))); + fileStep.setNotExistPathHandler(TaskFileStepDTO.toNotExistPathHandler( + FileTransferModeEnum.getFileTransferModeEnum(fileStepVO.getTransferMode()))); + } fileStep.setIgnoreError(fileStepVO.getIgnoreError() == 1); return fileStep; } @@ -128,7 +137,7 @@ public static EsbFileStepV3DTO toEsbFileInfoV3(TaskFileStepDTO fileStepInfo) { } EsbFileStepV3DTO esbFileStep = new EsbFileStepV3DTO(); if (CollectionUtils.isNotEmpty(fileStepInfo.getOriginFileList())) { - esbFileStep.setFileSourceList(fileStepInfo.getOriginFileList().parallelStream() + esbFileStep.setFileSourceList(fileStepInfo.getOriginFileList().stream() .map(TaskFileInfoDTO::toEsbFileSourceV3).collect(Collectors.toList())); } EsbFileDestinationV3DTO esbFileDestination = new EsbFileDestinationV3DTO(); @@ -141,8 +150,8 @@ public static EsbFileStepV3DTO toEsbFileInfoV3(TaskFileStepDTO fileStepInfo) { esbFileStep.setTimeout(fileStepInfo.getTimeout()); esbFileStep.setSourceSpeedLimit(fileStepInfo.getOriginSpeedLimit()); esbFileStep.setDestinationSpeedLimit(fileStepInfo.getTargetSpeedLimit()); - esbFileStep.setTransferMode(TaskFileStepVO.getTransferMode(fileStepInfo.getDuplicateHandler(), - fileStepInfo.getNotExistPathHandler())); + esbFileStep.setTransferMode(toTransferMode(fileStepInfo.getDuplicateHandler(), + fileStepInfo.getNotExistPathHandler()).getValue()); return esbFileStep; } @@ -152,7 +161,7 @@ public static ServiceTaskFileStepDTO toServiceFileInfo(TaskFileStepDTO fileStepI } ServiceTaskFileStepDTO serviceFileStep = new ServiceTaskFileStepDTO(); if (CollectionUtils.isNotEmpty(fileStepInfo.getOriginFileList())) { - serviceFileStep.setOriginFileList(fileStepInfo.getOriginFileList().parallelStream() + serviceFileStep.setOriginFileList(fileStepInfo.getOriginFileList().stream() .map(TaskFileInfoDTO::toServiceFileInfo).collect(Collectors.toList())); } serviceFileStep.setDestinationFileLocation(fileStepInfo.getDestinationFileLocation()); @@ -175,4 +184,53 @@ public static ServiceTaskFileStepDTO toServiceFileInfo(TaskFileStepDTO fileStepI serviceFileStep.setNotExistPathHandler(fileStepInfo.getNotExistPathHandler().getValue()); return serviceFileStep; } + + public static NotExistPathHandlerEnum toNotExistPathHandler(FileTransferModeEnum fileTransferMode) { + if (fileTransferMode == FileTransferModeEnum.STRICT) { + return NotExistPathHandlerEnum.STEP_FAIL; + } else { + return NotExistPathHandlerEnum.CREATE_DIR; + } + } + + public static DuplicateHandlerEnum toDuplicateHandler(FileTransferModeEnum fileTransferMode) { + switch (fileTransferMode) { + case STRICT: + case FORCE: + return DuplicateHandlerEnum.OVERWRITE; + case SAFETY_IP_PREFIX: + return DuplicateHandlerEnum.GROUP_BY_IP; + case SAFETY_DATE_PREFIX: + return DuplicateHandlerEnum.GROUP_BY_DATE_AND_IP; + default: + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); + } + } + + public static FileTransferModeEnum toTransferMode(DuplicateHandlerEnum duplicateHandlerEnum, + NotExistPathHandlerEnum notExistPathHandlerEnum) { + if (duplicateHandlerEnum == null) { + // 默认覆盖 + duplicateHandlerEnum = DuplicateHandlerEnum.OVERWRITE; + } + if (notExistPathHandlerEnum == null) { + // 默认直接创建 + notExistPathHandlerEnum = NotExistPathHandlerEnum.CREATE_DIR; + } + if (DuplicateHandlerEnum.OVERWRITE == duplicateHandlerEnum + && NotExistPathHandlerEnum.STEP_FAIL == notExistPathHandlerEnum) { + return FileTransferModeEnum.STRICT; + } else if (DuplicateHandlerEnum.OVERWRITE == duplicateHandlerEnum + && NotExistPathHandlerEnum.CREATE_DIR == notExistPathHandlerEnum) { + return FileTransferModeEnum.FORCE; + } else if (DuplicateHandlerEnum.GROUP_BY_IP == duplicateHandlerEnum + && NotExistPathHandlerEnum.CREATE_DIR == notExistPathHandlerEnum) { + return FileTransferModeEnum.SAFETY_IP_PREFIX; + } else if (DuplicateHandlerEnum.GROUP_BY_DATE_AND_IP == duplicateHandlerEnum + && NotExistPathHandlerEnum.CREATE_DIR == notExistPathHandlerEnum) { + return FileTransferModeEnum.SAFETY_DATE_PREFIX; + } else { + return FileTransferModeEnum.STRICT; + } + } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskHostNodeDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskHostNodeDTO.java index 713e447bd3..fdaff6fc6e 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskHostNodeDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskHostNodeDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,10 +24,13 @@ package com.tencent.bk.job.manage.model.dto.task; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.annotation.PersistenceObject; import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; -import com.tencent.bk.job.common.model.vo.HostInfoVO; +import com.tencent.bk.job.common.model.vo.DynamicGroupIdWithMeta; +import com.tencent.bk.job.common.model.vo.TaskExecuteObjectsInfoVO; import com.tencent.bk.job.common.model.vo.TaskHostNodeVO; -import com.tencent.bk.job.manage.common.TopologyHelper; import lombok.AllArgsConstructor; import lombok.Data; import lombok.EqualsAndHashCode; @@ -35,21 +38,27 @@ import org.apache.commons.collections4.CollectionUtils; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; /** * @since 2/12/2019 21:31 */ +@PersistenceObject @Data @EqualsAndHashCode @NoArgsConstructor @AllArgsConstructor +@JsonInclude(JsonInclude.Include.NON_EMPTY) public class TaskHostNodeDTO { + @JsonProperty("nodeInfoList") private List nodeInfoList; + @JsonProperty("dynamicGroupId") private List dynamicGroupId; + @JsonProperty("hostList") private List hostList; public static TaskHostNodeVO toVO(TaskHostNodeDTO hostNode) { @@ -58,36 +67,40 @@ public static TaskHostNodeVO toVO(TaskHostNodeDTO hostNode) { } TaskHostNodeVO hostNodeVO = new TaskHostNodeVO(); if (CollectionUtils.isNotEmpty(hostNode.getNodeInfoList())) { - hostNodeVO.setTopoNodeList( - hostNode.getNodeInfoList().parallelStream() + hostNodeVO.setNodeList( + hostNode.getNodeInfoList().stream() .map(TaskNodeInfoDTO::toVO).collect(Collectors.toList())); } - hostNodeVO.setDynamicGroupList(hostNode.getDynamicGroupId()); + hostNodeVO.setDynamicGroupIdList(hostNode.getDynamicGroupId()); if (hostNode.getHostList() != null) { hostNodeVO - .setIpList(hostNode.getHostList().stream() - .map(TopologyHelper::convertToHostInfoVO).collect(Collectors.toList())); + .setHostList(hostNode.getHostList().stream() + .filter(Objects::nonNull) + .map(ApplicationHostDTO::toVO) + .collect(Collectors.toList())); } return hostNodeVO; } - public static TaskHostNodeDTO fromVO(TaskHostNodeVO hostNode) { - if (hostNode == null) { + public static TaskHostNodeDTO fromVO(TaskExecuteObjectsInfoVO taskExecuteObjectsInfoVO) { + if (taskExecuteObjectsInfoVO == null) { return null; } TaskHostNodeDTO taskHostNodeDTO = new TaskHostNodeDTO(); - if (CollectionUtils.isNotEmpty(hostNode.getTopoNodeList())) { + if (CollectionUtils.isNotEmpty(taskExecuteObjectsInfoVO.getNodeList())) { taskHostNodeDTO.setNodeInfoList( - hostNode.getTopoNodeList().parallelStream() + taskExecuteObjectsInfoVO.getNodeList().stream() .map(TaskNodeInfoDTO::fromVO).collect(Collectors.toList())); } - if (CollectionUtils.isNotEmpty(hostNode.getDynamicGroupList())) { - taskHostNodeDTO.setDynamicGroupId(hostNode.getDynamicGroupList()); + if (CollectionUtils.isNotEmpty(taskExecuteObjectsInfoVO.getDynamicGroupList())) { + taskHostNodeDTO.setDynamicGroupId( + taskExecuteObjectsInfoVO.getDynamicGroupList().stream() + .map(DynamicGroupIdWithMeta::getId).collect(Collectors.toList())); } - if (CollectionUtils.isNotEmpty(hostNode.getIpList())) { + if (CollectionUtils.isNotEmpty(taskExecuteObjectsInfoVO.getHostList())) { taskHostNodeDTO - .setHostList(hostNode.getIpList().stream() - .map(HostInfoVO::toDTO).collect(Collectors.toList())); + .setHostList(taskExecuteObjectsInfoVO.getHostList().stream() + .map(ApplicationHostDTO::fromVO).collect(Collectors.toList())); } return taskHostNodeDTO; } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskNodeInfoDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskNodeInfoDTO.java index a372294514..bc8c3a3a06 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskNodeInfoDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskNodeInfoDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,9 @@ package com.tencent.bk.job.manage.model.dto.task; -import com.tencent.bk.job.common.esb.model.job.EsbCmdbTopoNodeDTO; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.tencent.bk.job.common.annotation.PersistenceObject; +import com.tencent.bk.job.common.model.openapi.v3.EsbCmdbTopoNodeDTO; import com.tencent.bk.job.common.model.vo.TargetNodeVO; import com.tencent.bk.job.manage.model.inner.ServiceTaskNodeInfoDTO; import lombok.AllArgsConstructor; @@ -35,10 +37,12 @@ /** * @since 12/12/2019 22:12 */ +@PersistenceObject @Data @EqualsAndHashCode @NoArgsConstructor @AllArgsConstructor +@JsonInclude(JsonInclude.Include.NON_EMPTY) public class TaskNodeInfoDTO { private Long id; @@ -52,8 +56,8 @@ public static TaskNodeInfoDTO fromVO(TargetNodeVO targetNodeVO) { return null; } TaskNodeInfoDTO nodeInfo = new TaskNodeInfoDTO(); - nodeInfo.setId(targetNodeVO.getId()); - nodeInfo.setType(targetNodeVO.getType()); + nodeInfo.setId(targetNodeVO.getInstanceId()); + nodeInfo.setType(targetNodeVO.getObjectId()); return nodeInfo; } @@ -69,8 +73,8 @@ public static EsbCmdbTopoNodeDTO toEsbCmdbTopoNode(TaskNodeInfoDTO taskNodeInfo) public TargetNodeVO toVO() { TargetNodeVO targetNodeVO = new TargetNodeVO(); - targetNodeVO.setId(this.getId()); - targetNodeVO.setType(this.getType()); + targetNodeVO.setInstanceId(this.getId()); + targetNodeVO.setObjectId(this.getType()); return targetNodeVO; } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskPlanBasicInfoDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskPlanBasicInfoDTO.java index f5ccd69109..49cf50bdbc 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskPlanBasicInfoDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskPlanBasicInfoDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskPlanInfoDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskPlanInfoDTO.java index 4628398691..60b3aec8f4 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskPlanInfoDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskPlanInfoDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -34,6 +34,7 @@ import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.manage.model.esb.v3.response.EsbPlanInfoV3DTO; import com.tencent.bk.job.manage.model.web.request.TaskPlanCreateUpdateReq; +import com.tencent.bk.job.manage.model.web.vo.task.TaskPlanBasicInfoVO; import com.tencent.bk.job.manage.model.web.vo.task.TaskPlanVO; import lombok.AllArgsConstructor; import lombok.Data; @@ -159,6 +160,18 @@ public class TaskPlanInfoDTO { */ private Long cronJobCount; + public static TaskPlanBasicInfoVO toBasicInfoVO(TaskPlanInfoDTO planInfo) { + TaskPlanBasicInfoVO taskPlanBasicInfoVO=new TaskPlanBasicInfoVO(); + taskPlanBasicInfoVO.setId(planInfo.getId()); + taskPlanBasicInfoVO.setName(planInfo.getName()); + taskPlanBasicInfoVO.setTemplateId(planInfo.getTemplateId()); + taskPlanBasicInfoVO.setCreator(planInfo.getCreator()); + taskPlanBasicInfoVO.setCreateTime(planInfo.getCreateTime()); + taskPlanBasicInfoVO.setLastModifyUser(planInfo.getLastModifyUser()); + taskPlanBasicInfoVO.setLastModifyTime(planInfo.getLastModifyTime()); + return taskPlanBasicInfoVO; + } + public static TaskPlanVO toVO(TaskPlanInfoDTO planInfo) { TaskPlanVO planVO = new TaskPlanVO(); planVO.setId(planInfo.getId()); @@ -255,7 +268,7 @@ public static void buildPlanInfo(TaskPlanInfoDTO planInfo, TaskTemplateInfoDTO t Map variableDefaultValueMap = new ConcurrentHashMap<>(); if (CollectionUtils.isNotEmpty(planInfo.getVariableList())) { - planInfo.getVariableList().parallelStream().forEach(taskVariableDTO -> { + planInfo.getVariableList().forEach(taskVariableDTO -> { if (taskVariableDTO.getDefaultValue() == null) { // No default value in request, skip return; @@ -302,7 +315,7 @@ public static TaskPlanInfoDTO fromVO(String username, Long appId, TaskPlanVO pla if (CollectionUtils.isNotEmpty(planInfo.getVariableList())) { taskPlanInfoDTO.setVariableList( - planInfo.getVariableList().parallelStream().map(TaskVariableDTO::fromVO).collect(Collectors.toList())); + planInfo.getVariableList().stream().map(TaskVariableDTO::fromVO).collect(Collectors.toList())); } taskPlanInfoDTO.setDebug(false); taskPlanInfoDTO.setVersion(planInfo.getVersion()); @@ -324,7 +337,7 @@ public static EsbPlanInfoV3DTO toEsbPlanInfoV3(TaskPlanInfoDTO taskPlanInfo) { esbPlanInfo.setLastModifyUser(taskPlanInfo.getLastModifyUser()); esbPlanInfo.setLastModifyTime(taskPlanInfo.getLastModifyTime()); if (CollectionUtils.isNotEmpty(taskPlanInfo.getStepList())) { - esbPlanInfo.setStepList(taskPlanInfo.getStepList().parallelStream() + esbPlanInfo.setStepList(taskPlanInfo.getStepList().stream() .map(TaskStepDTO::toEsbStepV3).collect(Collectors.toList())); } if (CollectionUtils.isNotEmpty(taskPlanInfo.getVariableList())) { diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskScriptStepDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskScriptStepDTO.java index 27d70bb4c4..c592120f7d 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskScriptStepDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskScriptStepDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,11 +24,12 @@ package com.tencent.bk.job.manage.model.dto.task; +import com.tencent.bk.job.common.constant.JobConstants; import com.tencent.bk.job.common.esb.model.job.v3.EsbAccountV3BasicDTO; +import com.tencent.bk.job.common.esb.model.job.v3.resp.EsbScriptStepV3DTO; import com.tencent.bk.job.common.util.Base64Util; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskScriptSourceEnum; -import com.tencent.bk.job.manage.model.esb.v3.response.EsbScriptStepV3DTO; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskScriptSourceEnum; import com.tencent.bk.job.manage.model.inner.ServiceAccountDTO; import com.tencent.bk.job.manage.model.inner.ServiceTaskScriptStepDTO; import com.tencent.bk.job.manage.model.web.vo.task.TaskScriptStepVO; @@ -71,6 +72,8 @@ public class TaskScriptStepDTO { private String scriptParam; + private String windowsInterpreter; + private Long timeout; private Long account; @@ -99,6 +102,7 @@ public static TaskScriptStepVO toVO(TaskScriptStepDTO scriptStep) { } scriptStepVO.setScriptLanguage(scriptStep.getLanguage().getValue()); scriptStepVO.setScriptParam(scriptStep.getScriptParam()); + scriptStepVO.setWindowsInterpreter(scriptStep.getWindowsInterpreter()); scriptStepVO.setTimeout(scriptStep.getTimeout()); scriptStepVO.setAccount(scriptStep.getAccount()); scriptStepVO.setExecuteTarget(TaskTargetDTO.toVO(scriptStep.getExecuteTarget())); @@ -122,10 +126,11 @@ public static TaskScriptStepDTO fromVO(Long stepId, TaskScriptStepVO scriptStepV } else { scriptStep.setContent(null); } - scriptStep.setLanguage(ScriptTypeEnum.valueOf(scriptStepVO.getScriptLanguage())); + scriptStep.setLanguage(ScriptTypeEnum.valOf(scriptStepVO.getScriptLanguage())); scriptStep.setScriptParam(scriptStepVO.getScriptParam()); + scriptStep.setWindowsInterpreter(scriptStepVO.getTrimmedWindowsInterpreter()); if (scriptStepVO.getTimeout() == null) { - scriptStep.setTimeout(60L); + scriptStep.setTimeout((long) JobConstants.DEFAULT_JOB_TIMEOUT_SECONDS); } else { scriptStep.setTimeout(scriptStepVO.getTimeout()); } @@ -152,6 +157,7 @@ public static EsbScriptStepV3DTO toEsbScriptInfoV3(TaskScriptStepDTO scriptStepI if (StringUtils.isNotBlank(scriptStepInfo.getScriptParam())) { esbScriptStep.setScriptParam(Base64Util.encodeContentToStr(scriptStepInfo.getScriptParam())); } + esbScriptStep.setWindowsInterpreter(scriptStepInfo.getWindowsInterpreter()); esbScriptStep.setScriptTimeout(scriptStepInfo.getTimeout()); EsbAccountV3BasicDTO account = new EsbAccountV3BasicDTO(); account.setId(scriptStepInfo.getAccount()); @@ -173,6 +179,7 @@ public static ServiceTaskScriptStepDTO toServiceScriptInfo(TaskScriptStepDTO scr serviceScriptStep.setType(scriptStepInfo.getLanguage().getValue()); serviceScriptStep.setContent(scriptStepInfo.getContent()); serviceScriptStep.setScriptParam(scriptStepInfo.getScriptParam()); + serviceScriptStep.setWindowsInterpreter(scriptStepInfo.getWindowsInterpreter()); serviceScriptStep.setScriptTimeout(scriptStepInfo.getTimeout()); serviceScriptStep.setAccount(new ServiceAccountDTO()); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskStepDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskStepDTO.java index 9062a72ed1..678c5ed0c0 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskStepDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskStepDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,9 +25,9 @@ package com.tencent.bk.job.manage.model.dto.task; import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.esb.model.job.v3.resp.EsbStepV3DTO; import com.tencent.bk.job.common.exception.InvalidParamException; -import com.tencent.bk.job.manage.common.consts.task.TaskStepTypeEnum; -import com.tencent.bk.job.manage.model.esb.v3.response.EsbStepV3DTO; +import com.tencent.bk.job.manage.api.common.constants.task.TaskStepTypeEnum; import com.tencent.bk.job.manage.model.inner.ServiceTaskStepDTO; import com.tencent.bk.job.manage.model.web.vo.task.TaskStepVO; import lombok.AllArgsConstructor; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskTargetDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskTargetDTO.java index 7cb7874a27..948405afe6 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskTargetDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskTargetDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,37 +24,54 @@ package com.tencent.bk.job.manage.model.dto.task; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.tencent.bk.job.common.annotation.PersistenceObject; import com.tencent.bk.job.common.esb.model.job.EsbIpDTO; -import com.tencent.bk.job.common.esb.model.job.v3.EsbDynamicGroupDTO; import com.tencent.bk.job.common.esb.model.job.v3.EsbServerV3DTO; +import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; +import com.tencent.bk.job.common.model.openapi.v3.EsbDynamicGroupDTO; +import com.tencent.bk.job.common.model.vo.TaskExecuteObjectsInfoVO; +import com.tencent.bk.job.common.model.vo.TaskHostNodeVO; import com.tencent.bk.job.common.model.vo.TaskTargetVO; +import com.tencent.bk.job.common.util.ApplicationContextRegister; import com.tencent.bk.job.common.util.json.JsonMapper; import com.tencent.bk.job.manage.model.inner.ServiceHostInfoDTO; import com.tencent.bk.job.manage.model.inner.ServiceTaskHostNodeDTO; import com.tencent.bk.job.manage.model.inner.ServiceTaskTargetDTO; +import com.tencent.bk.job.manage.service.host.HostService; import lombok.AllArgsConstructor; import lombok.Data; import lombok.EqualsAndHashCode; import lombok.NoArgsConstructor; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; import java.util.ArrayList; import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; /** - * @since 2/12/2019 20:59 + * 执行目标主机 */ +@PersistenceObject @Data @EqualsAndHashCode @NoArgsConstructor @AllArgsConstructor +@Slf4j +@JsonInclude(JsonInclude.Include.NON_EMPTY) public class TaskTargetDTO { + @JsonProperty("variable") private String variable; + @JsonProperty("hostNodeList") private TaskHostNodeDTO hostNodeList; public static TaskTargetVO toVO(TaskTargetDTO executeTarget) { @@ -63,7 +80,15 @@ public static TaskTargetVO toVO(TaskTargetDTO executeTarget) { } TaskTargetVO taskTargetVO = new TaskTargetVO(); taskTargetVO.setVariable(executeTarget.getVariable()); - taskTargetVO.setHostNodeInfo(TaskHostNodeDTO.toVO(executeTarget.getHostNodeList())); + TaskHostNodeVO taskHostNodeVO = TaskHostNodeDTO.toVO(executeTarget.getHostNodeList()); + if (taskHostNodeVO != null) { + taskTargetVO.setHostNodeInfo(taskHostNodeVO); + TaskExecuteObjectsInfoVO taskExecuteObjectsInfoVO = new TaskExecuteObjectsInfoVO(); + taskExecuteObjectsInfoVO.setHostList(taskHostNodeVO.getHostList()); + taskExecuteObjectsInfoVO.setNodeList(taskHostNodeVO.getNodeList()); + taskExecuteObjectsInfoVO.setDynamicGroupList(taskHostNodeVO.getDynamicGroupList()); + taskTargetVO.setExecuteObjectsInfo(taskExecuteObjectsInfoVO); + } return taskTargetVO; } @@ -75,11 +100,59 @@ public static TaskTargetDTO fromVO(TaskTargetVO taskTargetVO) { if (StringUtils.isNotBlank(taskTargetVO.getVariable())) { taskTargetDTO.setVariable(taskTargetVO.getVariable()); } - taskTargetDTO.setHostNodeList(TaskHostNodeDTO.fromVO(taskTargetVO.getHostNodeInfo())); + taskTargetDTO.setHostNodeList(TaskHostNodeDTO.fromVO(taskTargetVO.getExecuteObjectsInfoCompatibly())); + fillHostDetail(taskTargetDTO); return taskTargetDTO; } - public static TaskTargetDTO fromString(String targetString) { + private static void fillHostDetail(TaskTargetDTO target) { + HostService hostService = ApplicationContextRegister.getBean(HostService.class); + if (target.getHostNodeList() != null && CollectionUtils.isNotEmpty(target.getHostNodeList().getHostList())) { + List hostList = target.getHostNodeList().getHostList(); + Set hostIds = new HashSet<>(); + Set hostCloudIps = new HashSet<>(); + for (ApplicationHostDTO host : hostList) { + if (host.getHostId() != null) { + hostIds.add(host.getHostId()); + } + if (StringUtils.isNotBlank(host.getCloudIp())) { + hostCloudIps.add(host.getCloudIp()); + } + } + + if (hostIds.isEmpty() && hostCloudIps.isEmpty()) { + return; + } + + Map hostIdHostMapping = hostService.listHostsByHostIds(hostIds); + Map cloudIpHostMapping = hostService.listHostsByIps(hostCloudIps); + + hostList.forEach(hostNode -> { + ApplicationHostDTO hostDTO = cloudIpHostMapping.get(hostNode.getCloudIp()); + if (hostDTO == null && StringUtils.isBlank(hostNode.getIp())) { + hostDTO = hostIdHostMapping.get(hostNode.getHostId()); + } + + if (hostDTO != null) { + hostNode.setHostId(hostDTO.getHostId()); + hostNode.setAgentId(hostDTO.getAgentId()); + hostNode.setCloudAreaId(hostDTO.getCloudAreaId()); + hostNode.setIp(hostDTO.getIp()); + hostNode.setIpv6(hostDTO.getIpv6()); + hostNode.setDisplayIp(hostDTO.getDisplayIp()); + hostNode.setOsName(hostDTO.getOsName()); + hostNode.setOsType(hostDTO.getOsType()); + hostNode.setGseAgentStatus(hostDTO.getGseAgentStatus()); + } else { + log.warn("Cannot find host by hostId={} or by cloudIp={}", + hostNode.getHostId(), hostNode.getCloudIp()); + hostNode.setHostId(-1L); + } + }); + } + } + + public static TaskTargetDTO fromJsonString(String targetString) { if (StringUtils.isBlank(targetString)) { return null; } @@ -116,18 +189,18 @@ public static EsbServerV3DTO toEsbServerV3(TaskTargetDTO taskTarget) { esbServer.setVariable(taskTarget.getVariable()); if (taskTarget.getHostNodeList() != null) { if (CollectionUtils.isNotEmpty(taskTarget.getHostNodeList().getHostList())) { - esbServer.setIps(taskTarget.getHostNodeList().getHostList().parallelStream() + esbServer.setIps(taskTarget.getHostNodeList().getHostList().stream() .map(EsbIpDTO::fromApplicationHostInfo).collect(Collectors.toList())); } if (CollectionUtils.isNotEmpty(taskTarget.getHostNodeList().getDynamicGroupId())) { - esbServer.setDynamicGroups(taskTarget.getHostNodeList().getDynamicGroupId().parallelStream().map(id -> { + esbServer.setDynamicGroups(taskTarget.getHostNodeList().getDynamicGroupId().stream().map(id -> { EsbDynamicGroupDTO esbDynamicGroup = new EsbDynamicGroupDTO(); esbDynamicGroup.setId(id); return esbDynamicGroup; }).collect(Collectors.toList())); } if (CollectionUtils.isNotEmpty(taskTarget.getHostNodeList().getNodeInfoList())) { - esbServer.setTopoNodes(taskTarget.getHostNodeList().getNodeInfoList().parallelStream() + esbServer.setTopoNodes(taskTarget.getHostNodeList().getNodeInfoList().stream() .map(TaskNodeInfoDTO::toEsbCmdbTopoNode).collect(Collectors.toList())); } } @@ -140,7 +213,7 @@ public ServiceTaskTargetDTO toServiceTaskTargetDTO() { if (hostNodeList != null) { ServiceTaskHostNodeDTO targetServer = new ServiceTaskHostNodeDTO(); if (CollectionUtils.isNotEmpty(hostNodeList.getNodeInfoList())) { - targetServer.setNodeInfoList(hostNodeList.getNodeInfoList().parallelStream() + targetServer.setNodeInfoList(hostNodeList.getNodeInfoList().stream() .map(TaskNodeInfoDTO::toServiceTaskHostNodeDTO).collect(Collectors.toList())); } else { targetServer.setNodeInfoList(Collections.emptyList()); @@ -155,7 +228,6 @@ public ServiceTaskTargetDTO toServiceTaskTargetDTO() { } hostInfoDTO.setCloudAreaId(hostNode.getCloudAreaId()); hostInfoDTO.setIp(hostNode.getIp()); - hostInfoDTO.setDisplayIp(hostNode.getDisplayIp()); hostInfoDTOS.add(hostInfoDTO); }); targetServer.setHostList(hostInfoDTOS); @@ -165,8 +237,7 @@ public ServiceTaskTargetDTO toServiceTaskTargetDTO() { return targetDTO; } - @Override - public String toString() { + public String toJsonString() { if (StringUtils.isNotBlank(variable)) { this.hostNodeList = null; } else { @@ -177,4 +248,10 @@ public String toString() { } return JsonMapper.nonEmptyMapper().toJson(this); } + + @Override + public String toString() { + log.info("TaskTargetDTO_toString"); + return toJsonString(); + } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskTemplateInfoDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskTemplateInfoDTO.java index 033723c0ba..8bda299f9c 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskTemplateInfoDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskTemplateInfoDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,10 +25,12 @@ package com.tencent.bk.job.manage.model.dto.task; import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.esb.util.EsbDTOAppScopeMappingHelper; import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.util.date.DateUtils; -import com.tencent.bk.job.manage.common.consts.task.TaskTemplateStatusEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskTemplateStatusEnum; import com.tencent.bk.job.manage.model.dto.TagDTO; +import com.tencent.bk.job.manage.model.esb.v3.response.EsbTemplateInfoV3DTO; import com.tencent.bk.job.manage.model.inner.ServiceTaskTemplateDTO; import com.tencent.bk.job.manage.model.web.request.TaskTemplateCreateUpdateReq; import com.tencent.bk.job.manage.model.web.request.TemplateBasicInfoUpdateReq; @@ -246,4 +248,32 @@ public static ServiceTaskTemplateDTO toServiceDTO(TaskTemplateInfoDTO templateIn } return serviceTemplate; } + + public static EsbTemplateInfoV3DTO toEsbTemplateInfoV3DTO(TaskTemplateInfoDTO templateInfo) { + if (templateInfo == null) { + return null; + } + EsbTemplateInfoV3DTO template = new EsbTemplateInfoV3DTO(); + template.setId(templateInfo.getId()); + EsbDTOAppScopeMappingHelper.fillEsbAppScopeDTOByAppId(templateInfo.getAppId(), template); + template.setName(templateInfo.getName()); + template.setCreator(templateInfo.getCreator()); + template.setCreateTime(templateInfo.getCreateTime()); + template.setLastModifyUser(templateInfo.getLastModifyUser()); + template.setLastModifyTime(templateInfo.getLastModifyTime()); + template.setDescription(templateInfo.getDescription()); + + if (CollectionUtils.isNotEmpty(templateInfo.getVariableList())) { + template.setGlobalVarList( + templateInfo.getVariableList().stream() + .map(TaskVariableDTO::toEsbGlobalVarV3).collect(Collectors.toList())); + } + + if (CollectionUtils.isNotEmpty(templateInfo.getStepList())) { + template + .setStepList(templateInfo.getStepList().stream() + .map(TaskStepDTO::toEsbStepV3).collect(Collectors.toList())); + } + return template; + } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskVariableDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskVariableDTO.java index d2248a7b7f..50da7f6cae 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskVariableDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/task/TaskVariableDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -73,7 +73,7 @@ public static TaskVariableVO toVO(TaskVariableDTO variableInfo) { taskVariable.setType(variableInfo.getType().getType()); if (TaskVariableTypeEnum.HOST_LIST == variableInfo.getType()) { taskVariable - .setDefaultTargetValue(TaskTargetDTO.toVO(TaskTargetDTO.fromString(variableInfo.getDefaultValue()))); + .setDefaultTargetValue(TaskTargetDTO.toVO(TaskTargetDTO.fromJsonString(variableInfo.getDefaultValue()))); } else { if (variableInfo.getType().needMask()) { taskVariable.setDefaultValue(variableInfo.getType().getMask()); @@ -104,7 +104,7 @@ public static TaskVariableDTO fromVO(TaskVariableVO variableVO) { throw new InvalidParamException(ErrorCode.WRONG_VARIABLE_TYPE); } if (TaskVariableTypeEnum.HOST_LIST == variableInfo.getType()) { - variableInfo.setDefaultValue(TaskTargetDTO.fromVO(variableVO.getDefaultTargetValue()).toString()); + variableInfo.setDefaultValue(TaskTargetDTO.fromVO(variableVO.getDefaultTargetValue()).toJsonString()); } else { variableInfo.setDefaultValue(variableVO.getDefaultValue()); } @@ -131,7 +131,7 @@ public static ServiceTaskVariableDTO toServiceDTO(TaskVariableDTO taskVariable) serviceTaskVariable.setName(taskVariable.getName()); serviceTaskVariable.setType(taskVariable.getType().getType()); if (TaskVariableTypeEnum.HOST_LIST == taskVariable.getType()) { - TaskTargetDTO taskTarget = TaskTargetDTO.fromString(taskVariable.getDefaultValue()); + TaskTargetDTO taskTarget = TaskTargetDTO.fromJsonString(taskVariable.getDefaultValue()); if (taskTarget != null) { serviceTaskVariable.setDefaultTargetValue(taskTarget.toServiceTaskTargetDTO()); } @@ -157,7 +157,7 @@ public static EsbGlobalVarV3DTO toEsbGlobalVarV3(TaskVariableDTO taskVariable) { if (TaskVariableTypeEnum.HOST_LIST == taskVariable.getType()) { esbGlobalVar.setServer( TaskTargetDTO.toEsbServerV3( - TaskTargetDTO.fromString( + TaskTargetDTO.fromJsonString( taskVariable.getDefaultValue() ) ) @@ -171,4 +171,13 @@ public static EsbGlobalVarV3DTO toEsbGlobalVarV3(TaskVariableDTO taskVariable) { } return esbGlobalVar; } + + /** + * 判断是否密文未变更 + * + * @return 布尔值 + */ + public boolean cipherNotChange() { + return type.isNeedMask() && defaultValue != null && defaultValue.equals(type.getMask()); + } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/whiteip/ActionScopeDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/whiteip/ActionScopeDTO.java index 9f4d89674c..b17b3bd7be 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/whiteip/ActionScopeDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/whiteip/ActionScopeDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/whiteip/CloudIPDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/whiteip/CloudIPDTO.java index e076757372..91267870bc 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/whiteip/CloudIPDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/whiteip/CloudIPDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/whiteip/WhiteIPActionScopeDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/whiteip/WhiteIPActionScopeDTO.java index 1f25f92246..24874f0b7a 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/whiteip/WhiteIPActionScopeDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/whiteip/WhiteIPActionScopeDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/whiteip/WhiteIPAppRelDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/whiteip/WhiteIPAppRelDTO.java index 22954c1710..4af9c1764f 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/whiteip/WhiteIPAppRelDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/whiteip/WhiteIPAppRelDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/whiteip/WhiteIPIPDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/whiteip/WhiteIPIPDTO.java index 9c1df8c762..e1e0a8ab70 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/whiteip/WhiteIPIPDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/whiteip/WhiteIPIPDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,6 +26,7 @@ import com.fasterxml.jackson.databind.annotation.JsonSerialize; import com.tencent.bk.job.common.util.json.LongTimestampSerializer; +import com.tencent.bk.job.manage.model.web.vo.whiteip.WhiteIPHostVO; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.Setter; @@ -49,9 +50,17 @@ public class WhiteIPIPDTO { */ private Long cloudAreaId; /** - * IP列表 + * 主机hostId + */ + private Long hostId; + /** + * IP */ private String ip; + /** + * IPv6 + */ + private String ipv6; /** * 创建人 */ @@ -70,4 +79,32 @@ public class WhiteIPIPDTO { */ @JsonSerialize(using = LongTimestampSerializer.class) private Long lastModifyTime; + + /** + * 提取白名单记录中包含的主机信息 + * + * @return 主机信息 + */ + public WhiteIPHostVO extractWhiteIPHostVO() { + WhiteIPHostVO hostInfoVO = new WhiteIPHostVO(); + hostInfoVO.setHostId(hostId); + hostInfoVO.setCloudAreaId(cloudAreaId); + hostInfoVO.setIp(ip); + hostInfoVO.setIpv6(ipv6); + return hostInfoVO; + } + + public String getCloudIp() { + if (ip == null) { + return null; + } + return cloudAreaId + ":" + ip; + } + + public String getCloudIpv6() { + if (ipv6 == null) { + return null; + } + return cloudAreaId + ":" + ipv6; + } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/whiteip/WhiteIPRecordDTO.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/whiteip/WhiteIPRecordDTO.java index 02ed03a61a..7ae8c236c6 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/whiteip/WhiteIPRecordDTO.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/dto/whiteip/WhiteIPRecordDTO.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,18 +25,22 @@ package com.tencent.bk.job.manage.model.dto.whiteip; import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.tencent.bk.job.common.constant.JobConstants; +import com.tencent.bk.job.common.model.dto.ApplicationDTO; import com.tencent.bk.job.common.util.json.LongTimestampSerializer; -import lombok.AllArgsConstructor; +import com.tencent.bk.job.manage.model.web.vo.whiteip.ScopeVO; +import com.tencent.bk.job.manage.model.web.vo.whiteip.WhiteIPRecordVO; import lombok.Getter; import lombok.Setter; import lombok.ToString; +import org.apache.commons.collections4.CollectionUtils; import java.util.List; +import java.util.stream.Collectors; /** * IP白名单DTO */ -@AllArgsConstructor @Getter @Setter @ToString @@ -46,6 +50,10 @@ public class WhiteIPRecordDTO { * 多个业务id列表 */ private List appIdList; + /** + * 多个业务列表 + */ + private List appList; /** * 备注 */ @@ -76,4 +84,53 @@ public class WhiteIPRecordDTO { */ @JsonSerialize(using = LongTimestampSerializer.class) private Long lastModifyTime; + + public WhiteIPRecordDTO(Long id, + List appIdList, + String remark, + List ipList, + List actionScopeList, + String creator, + Long createTime, + String lastModifier, + Long lastModifyTime) { + this.id = id; + this.appIdList = appIdList; + this.remark = remark; + this.ipList = ipList; + this.actionScopeList = actionScopeList; + this.creator = creator; + this.createTime = createTime; + this.lastModifier = lastModifier; + this.lastModifyTime = lastModifyTime; + } + + public WhiteIPRecordVO toVO() { + WhiteIPRecordVO vo = new WhiteIPRecordVO(); + vo.setId(id); + + if (CollectionUtils.isNotEmpty(ipList)) { + vo.setCloudAreaId(ipList.get(0).getCloudAreaId()); + vo.setHostList(ipList.stream().map(WhiteIPIPDTO::extractWhiteIPHostVO) + .collect(Collectors.toList())); + } + + vo.setAllScope(isAllScope(appIdList)); + if (CollectionUtils.isNotEmpty(appList)) { + vo.setScopeList(appList.stream().map(app -> + new ScopeVO(app.getScope().getType().getValue(), + app.getScope().getId(), app.getName())).collect(Collectors.toList())); + } + vo.setRemark(remark); + vo.setCreator(creator); + vo.setCreateTime(createTime); + vo.setLastModifier(lastModifier); + vo.setLastModifyTime(lastModifyTime); + + return vo; + } + + private boolean isAllScope(List appIdList) { + return CollectionUtils.isNotEmpty(appIdList) && appIdList.contains(JobConstants.PUBLIC_APP_ID); + } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/mapper/ContainerMapper.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/mapper/ContainerMapper.java new file mode 100644 index 0000000000..03ef2e089e --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/mapper/ContainerMapper.java @@ -0,0 +1,42 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.mapper; + +import com.tencent.bk.job.common.cc.model.container.ContainerDetailDTO; +import com.tencent.bk.job.common.model.vo.ContainerVO; + +public class ContainerMapper { + + public static ContainerVO toContainerVO(ContainerDetailDTO container) { + ContainerVO containerVO = new ContainerVO(); + containerVO.setId(container.getContainer().getId()); + containerVO.setUid(container.getContainer().getContainerUID()); + containerVO.setName(container.getContainer().getName()); + containerVO.setPodName(container.getPod().getName()); + containerVO.setPodLabels(container.getPod().getLabels()); + containerVO.setNodeHostId(container.getTopo().getHostId()); + return containerVO; + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/query/ContainerQuery.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/query/ContainerQuery.java new file mode 100644 index 0000000000..5949c84752 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/query/ContainerQuery.java @@ -0,0 +1,261 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.query; + +import com.tencent.bk.job.common.cc.model.container.ContainerDTO; +import com.tencent.bk.job.common.cc.model.container.PodDTO; +import com.tencent.bk.job.common.cc.model.filter.BaseRuleDTO; +import com.tencent.bk.job.common.cc.model.filter.ComposeRuleDTO; +import com.tencent.bk.job.common.cc.model.filter.PropertyFilterDTO; +import com.tencent.bk.job.common.cc.model.filter.RuleConditionEnum; +import com.tencent.bk.job.common.cc.model.req.ListKubeContainerByTopoReq; +import com.tencent.bk.job.common.cc.model.req.Page; +import com.tencent.bk.job.common.constant.CcNodeTypeEnum; +import com.tencent.bk.job.common.model.OrderCondition; +import com.tencent.bk.job.common.model.PageCondition; +import com.tencent.bk.job.manage.model.dto.KubeTopoNode; +import com.tencent.bk.job.manage.model.web.request.chooser.container.ListContainerByTopologyNodesReq; +import lombok.Getter; +import lombok.ToString; +import org.apache.commons.collections4.CollectionUtils; + +import java.util.List; +import java.util.stream.Collectors; + +/** + * 容器查询 + */ +@Getter +@ToString +public class ContainerQuery { + private final Long bizId; + + private final List ids; + + private final List nodes; + + private final List containerUIDs; + + private final List containerNames; + + private final List podNames; + + private final List containerNameKeywords; + + private final List podNameKeywords; + + private final PageCondition pageCondition; + + private final OrderCondition orderCondition; + + + private ContainerQuery(Builder builder) { + bizId = builder.bizId; + ids = builder.ids; + nodes = builder.nodes; + containerUIDs = builder.containerUIDs; + containerNames = builder.containerNames; + podNames = builder.podNames; + containerNameKeywords = builder.containerNameKeywords; + podNameKeywords = builder.podNameKeywords; + pageCondition = builder.pageCondition; + orderCondition = builder.orderCondition; + } + + public static ContainerQuery fromListContainerByTopologyNodesReq(Long bizId, + ListContainerByTopologyNodesReq req) { + PageCondition pageCondition = null; + if (req.hasValidPageParams()) { + pageCondition = PageCondition.build(req.getStart(), req.getPageSize()); + } + + return ContainerQuery.builder() + .bizId(bizId) + .containerUIDs(req.getContainerUidList()) + .containerNameKeywords(req.getContainerNameKeywordList()) + .podNameKeywords(req.getPodNameKeywordList()) + .nodes(CollectionUtils.isNotEmpty(req.getNodeList()) ? + req.getNodeList().stream() + .filter(node -> !CcNodeTypeEnum.BIZ.getType().equals(node.getObjectId())) + .map(nodeVO -> new KubeTopoNode(nodeVO.getObjectId(), nodeVO.getInstanceId())) + .collect(Collectors.toList()) + : null) + .pageCondition(pageCondition) + .build(); + } + + + public ListKubeContainerByTopoReq toListKubeContainerByTopoReq() { + ListKubeContainerByTopoReq req = new ListKubeContainerByTopoReq(); + req.setBizId(bizId); + + setKubeNodeCondition(req); + + setContainerFilterIfNecessary(req); + + setPodFilterIfNecessary(req); + + setPageIfNecessary(req); + + return req; + } + + private void setKubeNodeCondition(ListKubeContainerByTopoReq req) { + if (CollectionUtils.isNotEmpty(nodes)) { + req.setNodeIdList(nodes.stream().map(KubeTopoNode::toKubeNodeID).collect(Collectors.toList())); + } + } + + private void setContainerFilterIfNecessary(ListKubeContainerByTopoReq req) { + PropertyFilterDTO containerFilter = new PropertyFilterDTO(); + containerFilter.setCondition(RuleConditionEnum.AND.getCondition()); + + if (CollectionUtils.isNotEmpty(ids)) { + containerFilter.addRule(BaseRuleDTO.in(ContainerDTO.Fields.ID, ids)); + } + + if (CollectionUtils.isNotEmpty(containerUIDs)) { + containerFilter.addRule(BaseRuleDTO.in(ContainerDTO.Fields.CONTAINER_UID, containerUIDs)); + } + + if (CollectionUtils.isNotEmpty(containerNames)) { + containerFilter.addRule(BaseRuleDTO.in(ContainerDTO.Fields.NAME, containerNames)); + } else if (CollectionUtils.isNotEmpty(containerNameKeywords)) { + ComposeRuleDTO containerNameKeywordRule = new ComposeRuleDTO(RuleConditionEnum.OR); + containerNameKeywords.forEach(containerNameKeyword -> + containerNameKeywordRule.addRule(BaseRuleDTO.contains(ContainerDTO.Fields.NAME, containerNameKeyword))); + containerFilter.addRule(containerNameKeywordRule); + } + + if (containerFilter.hasRule()) { + req.setContainerFilter(containerFilter); + } + } + + private void setPodFilterIfNecessary(ListKubeContainerByTopoReq req) { + PropertyFilterDTO podFilter = new PropertyFilterDTO(); + podFilter.setCondition(RuleConditionEnum.AND.getCondition()); + if (CollectionUtils.isNotEmpty(podNames)) { + podFilter.addRule(BaseRuleDTO.in(PodDTO.Fields.NAME, podNames)); + } else if (CollectionUtils.isNotEmpty(podNameKeywords)) { + ComposeRuleDTO podNameKeywordRule = new ComposeRuleDTO(RuleConditionEnum.OR); + podNameKeywords.forEach(podNameKeyword -> + podNameKeywordRule.addRule(BaseRuleDTO.contains(PodDTO.Fields.NAME, podNameKeyword))); + podFilter.addRule(podNameKeywordRule); + } + if (podFilter.hasRule()) { + req.setPodFilter(podFilter); + } + } + + private void setPageIfNecessary(ListKubeContainerByTopoReq req) { + if (pageCondition != null) { + Page page = new Page(); + page.setStart(pageCondition.getStart()); + page.setLimit(pageCondition.getLength()); + page.setSort(ContainerDTO.Fields.ID); + req.setPage(page); + } + } + + public static Builder builder() { + return Builder.builder(); + } + + + public static final class Builder { + private Long bizId; + private List ids; + private List nodes; + private List containerUIDs; + private List containerNames; + private List podNames; + private List containerNameKeywords; + private List podNameKeywords; + private PageCondition pageCondition; + private OrderCondition orderCondition; + + private Builder() { + } + + public static Builder builder() { + return new Builder(); + } + + public Builder bizId(Long bizId) { + this.bizId = bizId; + return this; + } + + public Builder ids(List ids) { + this.ids = ids; + return this; + } + + public Builder nodes(List nodes) { + this.nodes = nodes; + return this; + } + + public Builder containerUIDs(List containerUIDs) { + this.containerUIDs = containerUIDs; + return this; + } + + public Builder containerNames(List containerNames) { + this.containerNames = containerNames; + return this; + } + + public Builder podNames(List podNames) { + this.podNames = podNames; + return this; + } + + public Builder containerNameKeywords(List containerNameKeywords) { + this.containerNameKeywords = containerNameKeywords; + return this; + } + + public Builder podNameKeywords(List podNameKeywords) { + this.podNameKeywords = podNameKeywords; + return this; + } + + public Builder pageCondition(PageCondition pageCondition) { + this.pageCondition = pageCondition; + return this; + } + + public Builder orderCondition(OrderCondition orderCondition) { + this.orderCondition = orderCondition; + return this; + } + + public ContainerQuery build() { + return new ContainerQuery(this); + } + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/query/DangerousRuleQuery.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/query/DangerousRuleQuery.java new file mode 100644 index 0000000000..5c1d7add14 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/query/DangerousRuleQuery.java @@ -0,0 +1,58 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.query; + +import lombok.Builder; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +import java.util.List; + +/** + * 高危语句规则查询 + */ +@Getter +@Setter +@ToString +@Builder +public class DangerousRuleQuery implements Cloneable { + /** + * 表达式 + */ + private String expression; + /** + * 脚本类型：SHELL(1), BAT(2), PERL(3), PYTHON(4),POWERSHELL(5), SQL(6) + */ + private List scriptTypeList; + /** + * 描述 + */ + private String description; + /** + * 处理动作 + */ + private List action; +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/query/HostQuery.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/query/HostQuery.java new file mode 100644 index 0000000000..a4229663a1 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/query/HostQuery.java @@ -0,0 +1,87 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.model.query; + +import lombok.Builder; +import lombok.Getter; +import lombok.Setter; +import lombok.ToString; + +import java.util.Collection; +import java.util.List; + +/** + * 主机查询 + */ +@Getter +@Setter +@ToString +@Builder +public class HostQuery { + /** + * 主机所在的业务ID + */ + private Collection bizIds; + /** + * 主机所在的业务模块ID + */ + private Collection moduleIds; + /** + * 主机云区域ID + */ + private Collection cloudAreaIds; + /** + * 模糊搜索key，匹配字段：IP、IP_V6、IP_DESC、OS + */ + private List searchContents; + /** + * 主机Agent状态 + */ + private Integer agentAlive; + /** + * 主机IP模糊搜索key列表，任意一个key匹配即命中 + */ + private List ipKeyList; + /** + * 主机IPv6模糊搜索key列表，任意一个key匹配即命中 + */ + private List ipv6KeyList; + /** + * 主机名称模糊搜索key列表，任意一个key匹配即命中 + */ + private List hostNameKeyList; + /** + * 主机系统名称模糊搜索key列表，任意一个key匹配即命中 + */ + private List osNameKeyList; + /** + * 起始位置 + */ + private Long start; + /** + * 记录数量 + */ + private Long limit; +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/query/ScriptQuery.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/query/ScriptQuery.java index 4fd59c6ca0..5e99397bb5 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/query/ScriptQuery.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/query/ScriptQuery.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,8 @@ package com.tencent.bk.job.manage.model.query; +import com.tencent.bk.job.common.model.BaseSearchCondition; +import com.tencent.bk.job.manage.api.common.constants.JobResourceStatusEnum; import lombok.Getter; import lombok.NoArgsConstructor; import lombok.Setter; @@ -82,7 +84,7 @@ public class ScriptQuery { /** * 脚本状态 * - * @see com.tencent.bk.job.manage.common.consts.JobResourceStatusEnum + * @see JobResourceStatusEnum */ private Integer status; @@ -91,6 +93,8 @@ public class ScriptQuery { */ private String contentKeyword; + private BaseSearchCondition baseSearchCondition; + public boolean isExistTagCondition() { return this.untaggedScript || CollectionUtils.isNotEmpty(this.tagIds); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/query/TaskTemplateQuery.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/query/TaskTemplateQuery.java index 739176e950..01d76a7e64 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/query/TaskTemplateQuery.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/model/query/TaskTemplateQuery.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,7 +25,7 @@ package com.tencent.bk.job.manage.model.query; import com.tencent.bk.job.common.model.BaseSearchCondition; -import com.tencent.bk.job.manage.common.consts.task.TaskTemplateStatusEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskTemplateStatusEnum; import com.tencent.bk.job.manage.model.dto.TagDTO; import lombok.Builder; import lombok.Getter; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/runner/ClearDeletedHostsRunner.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/runner/ClearDeletedHostsRunner.java index f8aeceb892..20ab5cd5f3 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/runner/ClearDeletedHostsRunner.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/runner/ClearDeletedHostsRunner.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,9 +27,12 @@ import com.tencent.bk.job.manage.task.ClearDeletedHostsTask; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.CommandLineRunner; import org.springframework.stereotype.Component; +import java.util.concurrent.ThreadPoolExecutor; + /** * 进程启动时立即执行一次无效主机清理 */ @@ -38,14 +41,17 @@ public class ClearDeletedHostsRunner implements CommandLineRunner { private final ClearDeletedHostsTask clearDeletedHostsTask; + private final ThreadPoolExecutor initRunnerExecutor; @Autowired - public ClearDeletedHostsRunner(ClearDeletedHostsTask clearDeletedHostsTask) { + public ClearDeletedHostsRunner(ClearDeletedHostsTask clearDeletedHostsTask, + @Qualifier("initRunnerExecutor") ThreadPoolExecutor initRunnerExecutor) { this.clearDeletedHostsTask = clearDeletedHostsTask; + this.initRunnerExecutor = initRunnerExecutor; } @Override public void run(String... args) { - new Thread(clearDeletedHostsTask::execute).start(); + initRunnerExecutor.submit(clearDeletedHostsTask::execute); } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/runner/InitArtifactoryDataRunner.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/runner/InitArtifactoryDataRunner.java index 2ede0711fa..278f7b2290 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/runner/InitArtifactoryDataRunner.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/runner/InitArtifactoryDataRunner.java @@ -1,32 +1,66 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + package com.tencent.bk.job.manage.runner; +import com.tencent.bk.job.common.artifactory.config.ArtifactoryConfig; import com.tencent.bk.job.common.artifactory.sdk.ArtifactoryHelper; import com.tencent.bk.job.common.constant.JobConstants; -import com.tencent.bk.job.manage.config.ArtifactoryConfig; import com.tencent.bk.job.manage.config.LocalFileConfigForManage; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.boot.CommandLineRunner; import org.springframework.stereotype.Component; +import java.util.concurrent.ThreadPoolExecutor; + @Slf4j -@Component +@Component("jobManageInitArtifactoryDataRunner") public class InitArtifactoryDataRunner implements CommandLineRunner { private final ArtifactoryConfig artifactoryConfig; private final LocalFileConfigForManage localFileConfigForManage; + private final ThreadPoolExecutor initRunnerExecutor; @Autowired public InitArtifactoryDataRunner( ArtifactoryConfig artifactoryConfig, - LocalFileConfigForManage localFileConfigForManage + LocalFileConfigForManage localFileConfigForManage, + @Qualifier("initRunnerExecutor") ThreadPoolExecutor initRunnerExecutor ) { this.artifactoryConfig = artifactoryConfig; this.localFileConfigForManage = localFileConfigForManage; + this.initRunnerExecutor = initRunnerExecutor; } @Override public void run(String... args) { + initRunnerExecutor.submit(this::initArtifactoryData); + } + + public void initArtifactoryData() { if (!JobConstants.FILE_STORAGE_BACKEND_ARTIFACTORY.equals(localFileConfigForManage.getStorageBackend())) { //不使用制品库作为后端存储时不初始化 return; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/runner/MeasureServiceInitRunner.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/runner/MeasureServiceInitRunner.java index 9e668f19ff..ed926c50f6 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/runner/MeasureServiceInitRunner.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/runner/MeasureServiceInitRunner.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/runner/RegisterBkNoticeRunner.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/runner/RegisterBkNoticeRunner.java new file mode 100644 index 0000000000..e83dbea44d --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/runner/RegisterBkNoticeRunner.java @@ -0,0 +1,107 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.runner; + +import com.tencent.bk.job.common.notice.IBkNoticeClient; +import com.tencent.bk.job.common.notice.model.BkNoticeApp; +import com.tencent.bk.job.common.util.ThreadUtils; +import com.tencent.bk.job.common.util.TimeUtil; +import com.tencent.bk.job.manage.api.common.constants.globalsetting.GlobalSettingKeys; +import com.tencent.bk.job.manage.dao.globalsetting.GlobalSettingDAO; +import com.tencent.bk.job.manage.model.dto.GlobalSettingDTO; +import lombok.extern.slf4j.Slf4j; +import org.slf4j.helpers.MessageFormatter; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.CommandLineRunner; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.stereotype.Component; + +import java.util.concurrent.ThreadPoolExecutor; + +/** + * 进程启动时向消息中心注册平台信息（幂等操作） + */ +@SuppressWarnings("ConstantConditions") +@Slf4j +@Component +@ConditionalOnProperty(value = "bkNotice.enabled", havingValue = "true", matchIfMissing = true) +public class RegisterBkNoticeRunner implements CommandLineRunner { + + private final IBkNoticeClient bkNoticeClient; + private final ThreadPoolExecutor initRunnerExecutor; + private final GlobalSettingDAO globalSettingDAO; + + @Autowired + public RegisterBkNoticeRunner(IBkNoticeClient bkNoticeClient, + @Qualifier("initRunnerExecutor") ThreadPoolExecutor initRunnerExecutor, + GlobalSettingDAO globalSettingDAO) { + this.bkNoticeClient = bkNoticeClient; + this.initRunnerExecutor = initRunnerExecutor; + this.globalSettingDAO = globalSettingDAO; + } + + @Override + public void run(String... args) { + initRunnerExecutor.submit(() -> { + boolean registerSuccess = false; + // 最多重试30min，覆盖整个蓝鲸部署时间 + int maxRetryTimes = 180; + int retryTimes = 0; + while (!registerSuccess && retryTimes < maxRetryTimes) { + try { + BkNoticeApp bkNoticeApp = bkNoticeClient.registerApplication(); + log.info("registerApplication result:{}", bkNoticeApp); + registerSuccess = true; + } catch (Exception e) { + retryTimes++; + if (retryTimes < maxRetryTimes) { + String msg = MessageFormatter.format( + "Fail to registerApplication, retry {}", + retryTimes + ).getMessage(); + log.warn(msg, e); + ThreadUtils.sleep(10000); + } else { + log.warn("Fail to registerApplication finally", e); + } + } + } + // 将注册结果写入DB中 + if (registerSuccess) { + int affectedNum = globalSettingDAO.upsertGlobalSetting(buildRegisterResult(registerSuccess)); + log.info("Write to db, registerSuccess={}, affectedNum={}", registerSuccess, affectedNum); + } + }); + } + + private GlobalSettingDTO buildRegisterResult(boolean registerSuccess) { + GlobalSettingDTO globalSettingDTO = new GlobalSettingDTO(); + globalSettingDTO.setKey(GlobalSettingKeys.KEY_BK_NOTICE_REGISTERED_SUCCESS); + globalSettingDTO.setValue(String.valueOf(registerSuccess)); + globalSettingDTO.setDescription("Updated at " + TimeUtil.getCurrentTimeStr("yyyy-MM-dd HH:mm:ss.SSS")); + return globalSettingDTO; + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/runner/SyncServiceInitRunner.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/runner/SyncServiceInitRunner.java index 2840bb92fa..3fb1fb2283 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/runner/SyncServiceInitRunner.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/runner/SyncServiceInitRunner.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/AbstractTaskStepService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/AbstractTaskStepService.java index 4aa349d66a..c4f12ff176 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/AbstractTaskStepService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/AbstractTaskStepService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -29,10 +29,11 @@ import com.tencent.bk.job.common.exception.InternalException; import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.exception.ServiceException; +import com.tencent.bk.job.common.mysql.JobTransactional; import com.tencent.bk.job.common.util.json.JsonUtils; -import com.tencent.bk.job.manage.common.consts.task.TaskFileTypeEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskScriptSourceEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskFileTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskScriptSourceEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskTypeEnum; import com.tencent.bk.job.manage.dao.ScriptDAO; import com.tencent.bk.job.manage.dao.TaskApprovalStepDAO; import com.tencent.bk.job.manage.dao.TaskFileInfoDAO; @@ -49,7 +50,6 @@ import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.transaction.annotation.Transactional; import java.util.ArrayList; import java.util.HashMap; @@ -160,7 +160,7 @@ private void fillRefScriptContent(Map scriptStepMap) { * @return 新增的步骤 ID * @throws ServiceException 新增步骤异常 */ - @Transactional(rollbackFor = ServiceException.class) + @JobTransactional(transactionManager = "jobManageTransactionManager") public long insertStep(TaskStepDTO taskStep) throws ServiceException { try { Long stepId = taskStepDAO.insertStep(taskStep); @@ -240,7 +240,7 @@ public long insertStep(TaskStepDTO taskStep) throws ServiceException { * @return 是否更新成功 * @throws ServiceException 更新异常 */ - @Transactional(rollbackFor = ServiceException.class) + @JobTransactional(transactionManager = "jobManageTransactionManager") public boolean updateStepById(TaskStepDTO taskStep) throws ServiceException { try { if (taskStepDAO.updateStepById(taskStep)) { @@ -264,7 +264,7 @@ public boolean updateStepById(TaskStepDTO taskStep) throws ServiceException { if (taskFileStepDAO.updateFileStepById(taskStep.getFileStepInfo())) { List originTaskFileInfo = taskFileInfoDAO.listFileInfoByStepId(taskStep.getFileStepInfo().getStepId()); - List originTaskFileInfoIds = originTaskFileInfo.parallelStream() + List originTaskFileInfoIds = originTaskFileInfo.stream() .map(TaskFileInfoDTO::getId).collect(Collectors.toList()); List newFileInfo = new ArrayList<>(); for (TaskFileInfoDTO fileInfo : taskStep.getFileStepInfo().getOriginFileList()) { @@ -328,7 +328,7 @@ public boolean updateStepById(TaskStepDTO taskStep) throws ServiceException { * @return 是否删除成功 * @throws ServiceException 删除异常 */ - @Transactional(rollbackFor = ServiceException.class) + @JobTransactional(transactionManager = "jobManageTransactionManager") public boolean deleteStepById(Long parentId, Long id) throws ServiceException { try { TaskStepDTO taskStep = taskStepDAO.getStepById(parentId, id); @@ -389,7 +389,7 @@ public Map> batchListScriptStepVersionIdsByTemplateIds(Long app uniqScriptVersionSet.add(taskScriptStepDTO.getScriptVersionId()); }); } - templateScriptVersionMap.put(0L, uniqScriptVersionSet.parallelStream().collect(Collectors.toList())); + templateScriptVersionMap.put(0L, new ArrayList<>(uniqScriptVersionSet)); return templateScriptVersionMap; } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/AbstractTaskVariableService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/AbstractTaskVariableService.java index a32f7e0161..f105512197 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/AbstractTaskVariableService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/AbstractTaskVariableService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -67,13 +67,13 @@ public List batchInsertVariable(List variableList) { } /** - * 根据 ID 更新变量 + * 根据 父资源（作业模板/执行方案）ID+变量ID 更新变量信息 * * @param variable 变量信息 * @return 是否更新成功 */ - public boolean updateVariableById(TaskVariableDTO variable) { - return taskVariableDAO.updateVariableById(variable); + public boolean updateVarByParentResourceIdAndTplVarId(TaskVariableDTO variable) { + return taskVariableDAO.updateVarByParentResourceIdAndTplVarId(variable); } /** diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/AccountService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/AccountService.java index 707391d73b..a9a9d419da 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/AccountService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/AccountService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,11 +24,10 @@ package com.tencent.bk.job.manage.service; -import com.tencent.bk.job.common.exception.ServiceException; +import com.tencent.bk.job.common.constant.AccountCategoryEnum; import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; -import com.tencent.bk.job.manage.common.consts.account.AccountCategoryEnum; -import com.tencent.bk.job.manage.common.consts.account.AccountTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.account.AccountTypeEnum; import com.tencent.bk.job.manage.model.dto.AccountDTO; import com.tencent.bk.job.manage.model.dto.AccountDisplayDTO; import com.tencent.bk.job.manage.model.web.request.AccountCreateUpdateReq; @@ -44,18 +43,46 @@ public interface AccountService { /** * 新增账号 * - * @param account - * @return 账号ID + * @param account 新增的账号信息 + * @return 新增的账号 */ - long saveAccount(AccountDTO account) throws ServiceException; + AccountDTO createAccount(AccountDTO account); + + /** + * 新增账号 + * + * @param username 用户账号 + * @param account 新增的账号信息 + * @return 新增的账号 + */ + AccountDTO createAccount(String username, AccountDTO account); + + /** + * 根据ID获取账号信息 + * + * @param accountId 账号Id + * @return 账号 + */ + AccountDTO getAccountById(Long accountId); + + /** + * 根据ID获取账号信息 + * + * @param username 用户账号 + * @param appId 业务 ID + * @param accountId 账号Id + * @return 账号 + */ + AccountDTO getAccount(String username, long appId, Long accountId); /** * 根据ID获取账号信息 * + * @param appId 业务 ID * @param accountId 账号Id - * @return + * @return 账号 */ - AccountDTO getAccountById(Long accountId) throws ServiceException; + AccountDTO getAccount(long appId, Long accountId); /** * 根据ID批量获取账号信息 @@ -63,59 +90,61 @@ public interface AccountService { * @param accountIds 账号集合 * @return Map<账号Id ， 账号内容> */ - Map getAccountDisplayInfoMapByIds(Collection accountIds) throws ServiceException; + Map getAccountDisplayInfoMapByIds(Collection accountIds); /** * 根据ID获取账号信息 * - * @param appId - * @param account - * @return + * @param appId 业务ID + * @param account 账号名 + * @return 账号 */ - AccountDTO getAccountByAccount(Long appId, String account) throws ServiceException; + AccountDTO getAccountByAccount(Long appId, String account); /** * 更新账号 * - * @param account + * @param username 用户账号 + * @param account 更新账号内容 + * @return 更新完的账号信息 */ - void updateAccount(AccountDTO account) throws ServiceException; + AccountDTO updateAccount(String username, AccountDTO account); /** * 删除账号 * - * @param accountId + * @param username 用户账号 + * @param appId 业务 ID + * @param accountId 账号ID */ - void deleteAccount(Long accountId) throws ServiceException; + void deleteAccount(String username, long appId, Long accountId); /** * 分页查询账号列表 * - * @param accountQuery - * @param baseSearchCondition - * @return + * @param baseSearchCondition 基础查询条件 + * @return 账号分页 */ - PageData listPageAccount(AccountDTO accountQuery, BaseSearchCondition baseSearchCondition) - throws ServiceException; + PageData listPageAccount(AccountDTO accountQuery, BaseSearchCondition baseSearchCondition); /** * 分页搜索账号列表 * - * @param keyword - * @param baseSearchCondition - * @return + * @param keyword 关键字 + * @param baseSearchCondition 基础查询条件 + * @return 账号分页 */ PageData searchPageAccount( Long appId, String keyword, - BaseSearchCondition baseSearchCondition) throws ServiceException; + BaseSearchCondition baseSearchCondition); /** * 根据别名获取账号信息 * - * @param appId - * @param category - * @param alias - * @return + * @param appId 业务ID + * @param category 账号类别 + * @param alias 账号别名 + * @return 账号 */ AccountDTO getAccount(Long appId, AccountCategoryEnum category, String alias); @@ -126,7 +155,7 @@ PageData searchPageAccount( * @param category 账号类型，如果传入null，表示所有类型 * @return 账号列表 */ - List listAllAppAccount(Long appId, AccountCategoryEnum category); + List listAppAccount(Long appId, AccountCategoryEnum category); /** * 获取业务下账号列表 @@ -135,10 +164,13 @@ PageData searchPageAccount( * @param category 账号类型，如果传入null，表示所有类型 * @return 账号列表 */ - List listAllAppAccount(Long appId, AccountCategoryEnum category, - BaseSearchCondition baseSearchCondition); + List listAppAccount(Long appId, + AccountCategoryEnum category, + String account, + String alias, + BaseSearchCondition baseSearchCondition); - Integer countAllAppAccount(Long appId, AccountCategoryEnum category); + Integer countAppAccount(Long appId, AccountCategoryEnum category, String account, String alias); boolean isAccountRefByAnyStep(Long accountId); @@ -146,7 +178,6 @@ List listAllAppAccount(Long appId, AccountCategoryEnum category, * 系统账号是否被DB账号依赖 * * @param accountId 系统账号ID - * @return */ boolean isSystemAccountRefByDbAccount(Long accountId); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/AppRoleService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/AppRoleService.java index 6131f60eb9..a2d8488bbf 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/AppRoleService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/AppRoleService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/ApplicationService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/ApplicationService.java index a44989035a..460493c525 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/ApplicationService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/ApplicationService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -187,4 +187,11 @@ public interface ApplicationService { * @return 业务 */ ApplicationDTO getAppByScopeIncludingDeleted(ResourceScope scope); + + /** + * 获取作业平台所有已删除的业务 + * + * @return 业务列表 + */ + List listAllDeletedApps(); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/ContainerService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/ContainerService.java new file mode 100644 index 0000000000..3cf51a2f09 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/ContainerService.java @@ -0,0 +1,82 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.service; + +import com.tencent.bk.job.common.cc.model.container.ContainerDetailDTO; +import com.tencent.bk.job.common.cc.model.container.KubeTopologyDTO; +import com.tencent.bk.job.common.model.PageData; +import com.tencent.bk.job.manage.model.query.ContainerQuery; + +import java.util.List; + +/** + * 容器服务 + */ +public interface ContainerService { + + + /** + * 根据业务 ID 查询容器拓扑（缓存) + * + * @param bizId 业务 ID + * @return 容器拓扑 + */ + KubeTopologyDTO getBizKubeCacheTopo(long bizId); + + /** + * 根据容器拓扑获取container信息(分页） + * + * @param query 查询条件 + * @return 容器列表（分页） + */ + PageData listPageKubeContainerByTopo(ContainerQuery query); + + /** + * 根据容器拓扑获取container信息 + * + * @param query 查询条件 + * @return 容器列表 + */ + List listKubeContainerByTopo(ContainerQuery query); + + /** + * 根据容器 ID 批量获取容器详情 + * + * @param bizId 业务 ID + * @param containerIds 容器 ID 列表 + * @return 容器列表 + */ + List listKubeContainerByIds(Long bizId, List containerIds); + + /** + * 根据容器 UId 批量获取容器详情 + * + * @param bizId 业务 ID + * @param containerUIds 容器 UId 列表 + * @return 容器列表 + */ + List listKubeContainerByUIds(Long bizId, List containerUIds); + +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/CredentialService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/CredentialService.java index d45733fa3c..44c39ee3c7 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/CredentialService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/CredentialService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,7 +27,6 @@ import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; import com.tencent.bk.job.manage.model.dto.CredentialDTO; -import com.tencent.bk.job.manage.model.inner.resp.ServiceCredentialDTO; import com.tencent.bk.job.manage.model.inner.resp.ServiceCredentialDisplayDTO; import com.tencent.bk.job.manage.model.web.request.CredentialCreateUpdateReq; @@ -38,13 +37,17 @@ public interface CredentialService { PageData listCredentials(CredentialDTO credentialQuery, BaseSearchCondition baseSearchCondition); - String saveCredential(String username, Long appId, CredentialCreateUpdateReq createUpdateReq); + PageData listCredentialBasicInfo(Long appId, BaseSearchCondition baseSearchCondition); + + CredentialDTO createCredential(String username, Long appId, CredentialCreateUpdateReq createUpdateReq); + + CredentialDTO updateCredential(String username, Long appId, CredentialCreateUpdateReq createUpdateReq); Integer deleteCredentialById(String username, Long appId, String id); - ServiceCredentialDTO getServiceCredentialById(Long appId, String id); + CredentialDTO getCredentialById(Long appId, String id); - ServiceCredentialDTO getServiceCredentialById(String id); + CredentialDTO getCredentialById(String id); List listCredentialDisplayInfoByIds(Collection ids); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/CronJobService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/CronJobService.java index 34d8dddc57..ffe7c9fbbc 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/CronJobService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/CronJobService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/CustomScriptTemplateService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/CustomScriptTemplateService.java index d45e9d4112..80cf6cfbd6 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/CustomScriptTemplateService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/CustomScriptTemplateService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/DangerousRuleService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/DangerousRuleService.java index 5370833f13..6993642389 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/DangerousRuleService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/DangerousRuleService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,9 @@ package com.tencent.bk.job.manage.service; +import com.tencent.bk.job.manage.api.common.constants.EnableStatusEnum; +import com.tencent.bk.job.manage.model.dto.globalsetting.DangerousRuleDTO; +import com.tencent.bk.job.manage.model.query.DangerousRuleQuery; import com.tencent.bk.job.manage.model.web.request.globalsetting.AddOrUpdateDangerousRuleReq; import com.tencent.bk.job.manage.model.web.request.globalsetting.MoveDangerousRuleReq; import com.tencent.bk.job.manage.model.web.vo.globalsetting.DangerousRuleVO; @@ -34,9 +37,17 @@ public interface DangerousRuleService { List listDangerousRules(String username); - Boolean addOrUpdateDangerousRule(String username, AddOrUpdateDangerousRuleReq req); + DangerousRuleDTO getDangerousRuleById(Long id); + + DangerousRuleDTO createDangerousRule(String username, AddOrUpdateDangerousRuleReq req); + + DangerousRuleDTO updateDangerousRule(String username, AddOrUpdateDangerousRuleReq req); Integer moveDangerousRule(String username, MoveDangerousRuleReq req); Integer deleteDangerousRuleById(String username, Long id); + + List listDangerousRules(DangerousRuleQuery query); + + DangerousRuleDTO updateDangerousRuleStatus(String userName, Long id, EnableStatusEnum status); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/HostService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/HostService.java deleted file mode 100644 index 99e0a3a978..0000000000 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/HostService.java +++ /dev/null @@ -1,201 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.service; - -import com.tencent.bk.job.common.cc.model.InstanceTopologyDTO; -import com.tencent.bk.job.common.model.BaseSearchCondition; -import com.tencent.bk.job.common.model.PageData; -import com.tencent.bk.job.common.model.dto.AppResourceScope; -import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; -import com.tencent.bk.job.common.model.dto.DynamicGroupWithHost; -import com.tencent.bk.job.common.model.dto.HostDTO; -import com.tencent.bk.job.common.model.vo.HostInfoVO; -import com.tencent.bk.job.manage.common.consts.whiteip.ActionScopeEnum; -import com.tencent.bk.job.manage.model.inner.ServiceListAppHostResultDTO; -import com.tencent.bk.job.manage.model.web.request.AgentStatisticsReq; -import com.tencent.bk.job.manage.model.web.request.ipchooser.AppTopologyTreeNode; -import com.tencent.bk.job.manage.model.web.request.ipchooser.ListHostByBizTopologyNodesReq; -import com.tencent.bk.job.manage.model.web.vo.CcTopologyNodeVO; -import com.tencent.bk.job.manage.model.web.vo.NodeInfoVO; -import com.tencent.bk.job.manage.model.web.vo.index.AgentStatistics; - -import java.util.Collection; -import java.util.List; - -/** - * 主机、topo相关服务 - */ -public interface HostService { - boolean existHost(long bizId, String ip); - - List getHostsByAppId(Long appId); - - /** - * 新增业务下的主机 - * - * @param bizId 业务ID - * @param insertList 主机信息 - * @return 新增失败的主机ID - */ - List insertHostsToBiz(Long bizId, List insertList); - - /** - * 更新业务下的主机 - * - * @param bizId 业务ID - * @param hostInfoList 主机信息 - * @return 更新失败的主机ID - */ - List updateHostsInBiz(Long bizId, List hostInfoList); - - /** - * 将主机从业务下移除但不删除 - * - * @param bizId 业务ID - * @param hostList 主机信息 - * @return 移除失败的主机ID - */ - List removeHostsFromBiz(Long bizId, List hostList); - - long countHostsByOsType(String osType); - - /** - * 根据给定条件查询主机信息 - * - * @param applicationHostInfoCondition 业务查询条件 - * @param baseSearchCondition 通用查询分页条件 - * @return 带分页信息的主机信息列表 - */ - PageData listAppHost(ApplicationHostDTO applicationHostInfoCondition, - BaseSearchCondition baseSearchCondition); - - /** - * 查询指定业务的拓扑树 - * - * @param username 用户名 - * @param appResourceScope 资源范围 - * @return 拓扑结构树 - */ - CcTopologyNodeVO listAppTopologyTree(String username, AppResourceScope appResourceScope); - - /** - * 查询带主机信息的业务的拓扑树 - * - * @param username 用户名 - * @param appResourceScope 资源范围 - * @return 带主机信息的拓扑结构树 - */ - CcTopologyNodeVO listAppTopologyHostTree(String username, AppResourceScope appResourceScope); - - /** - * 查询带主机数量信息的业务拓扑树 - * - * @param username 用户名 - * @param appResourceScope 资源范围 - * @return 带主机数量信息的拓扑结构树 - */ - CcTopologyNodeVO listAppTopologyHostCountTree(String username, AppResourceScope appResourceScope); - - PageData listHostByAppTopologyNodes(String username, - AppResourceScope appResourceScope, - ListHostByBizTopologyNodesReq req); - - PageData listIPByBizTopologyNodes(String username, - AppResourceScope appResourceScope, - ListHostByBizTopologyNodesReq req); - - List getAppTopologyTreeNodeDetail(String username, - AppResourceScope appResourceScope, - List treeNodeList); - - List> queryBizNodePaths(String username, - Long bizId, - List nodeList); - - List getBizHostsByNode(String username, - Long bizId, - List treeNodeList); - - /** - * 获取业务下动态分组列表 - * - * @param appResourceScope 资源范围 - * @param username 用户名 - * @return 动态分组信息列表 - */ - List getAppDynamicGroupList(String username, AppResourceScope appResourceScope); - - /** - * 根据动态分组 ID 列表批量获取带主机信息的动态分组信息列表 - * - * @param username 用户名 - * @param bizId 业务 ID - * @param dynamicGroupIdList 动态分组 ID 列表 - * @return 带主机信息的动态分组信息列表 - */ - List getBizDynamicGroupHostList(String username, - Long bizId, - List dynamicGroupIdList); - - /** - * 根据 IP 列表查询主机信息 - * - * @param username 用户名 - * @param appId 业务 ID - * @param checkIpList 待查询的 IP 列表 - * @return 主机信息列表 - */ - List getHostsByIp(String username, - Long appId, - ActionScopeEnum actionScope, - List checkIpList); - - List listHostByAppTopologyNodes(String username, - Long appId, - List appTopoNodeList); - - AgentStatistics getAgentStatistics(String username, - Long appId, - AgentStatisticsReq agentStatisticsReq); - - void fillAgentStatus(List hosts); - - - /** - * 获取业务下的主机 - * - * @param appId Job业务ID - * @param hosts 主机列表 - */ - ServiceListAppHostResultDTO listAppHosts(Long appId, List hosts); - - /** - * 根据主机批量获取主机。如果在同步的主机中不存在，那么从cmdb查询 - * - * @param hosts 主机 - * @return 主机 - */ - List listHosts(Collection hosts); -} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/IndexService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/IndexService.java index cc0f3d030e..455eb0896b 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/IndexService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/IndexService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,7 +27,7 @@ import com.tencent.bk.job.common.model.PageData; import com.tencent.bk.job.common.model.dto.AppResourceScope; import com.tencent.bk.job.common.model.vo.HostInfoVO; -import com.tencent.bk.job.manage.model.web.vo.index.AgentStatistics; +import com.tencent.bk.job.manage.model.web.vo.common.AgentStatistics; import com.tencent.bk.job.manage.model.web.vo.index.GreetingVO; import com.tencent.bk.job.manage.model.web.vo.index.JobAndScriptStatistics; import com.tencent.bk.job.manage.model.web.vo.task.TaskTemplateVO; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/LocalPermissionService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/LocalPermissionService.java index abec9b2dba..83059f567b 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/LocalPermissionService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/LocalPermissionService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/MeasureService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/MeasureService.java index 219b1d5edb..7b81bb34bf 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/MeasureService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/MeasureService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/NotifyService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/NotifyService.java index eb2fe3e108..bffa3eeda3 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/NotifyService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/NotifyService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,10 +25,12 @@ package com.tencent.bk.job.manage.service; import com.tencent.bk.job.common.cc.model.AppRoleDTO; +import com.tencent.bk.job.common.model.dto.notify.CustomNotifyDTO; import com.tencent.bk.job.common.model.vo.NotifyChannelVO; import com.tencent.bk.job.manage.model.dto.notify.NotifyEsbChannelDTO; import com.tencent.bk.job.manage.model.inner.ServiceNotificationDTO; import com.tencent.bk.job.manage.model.inner.ServiceNotificationMessage; +import com.tencent.bk.job.manage.model.inner.ServiceSpecificResourceNotifyPolicyDTO; import com.tencent.bk.job.manage.model.inner.ServiceTemplateNotificationDTO; import com.tencent.bk.job.manage.model.inner.ServiceTriggerTemplateNotificationDTO; import com.tencent.bk.job.manage.model.inner.ServiceUserNotificationDTO; @@ -52,15 +54,29 @@ public interface NotifyService { List listAppDefaultNotifyPolicies(String username, Long appId); + CustomNotifyDTO getSpecificResourceNotifyPolicy(Long appId, + Integer resourceType, + String resourceId, + Integer triggerType); Long saveAppDefaultNotifyPolicies(String username, Long appId, NotifyPoliciesCreateUpdateReq createUpdateReq); - Long saveAppDefaultNotifyPoliciesToLocal(String username, Long appId, String triggerUser, + Long saveAppDefaultNotifyPoliciesToLocal(String username, + Long appId, + String triggerUser, NotifyPoliciesCreateUpdateReq createUpdateReq); - Long saveAppDefaultNotifyPolicies(String username, Long appId, NotifyPoliciesCreateUpdateReq createUpdateReq, + Boolean saveSpecificResourceNotifyPolicies(Long appId, + String operator, + ServiceSpecificResourceNotifyPolicyDTO specificResourceNotifyPolicyDTO); + + Long saveAppDefaultNotifyPolicies(String username, + Long appId, + NotifyPoliciesCreateUpdateReq createUpdateReq, boolean checkAuth); + int deleteAppResourceNotifyPolicies(Long appId, Integer resourceType, String resourceId); + List listTriggerType(String username); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/PaaSService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/PaaSService.java deleted file mode 100644 index bf3f6f3cae..0000000000 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/PaaSService.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.service; - -import com.tencent.bk.job.common.model.dto.BkUserDTO; -import com.tencent.bk.job.common.paas.model.EsbNotifyChannelDTO; - -import java.io.IOException; -import java.util.List; -import java.util.Set; - -/** - * PaaS服务 - */ -public interface PaaSService { - - List getAllUserList(String bkToken, String uin); - - List getAllChannelList(String bkToken, String uin) throws IOException; - - void sendMsg( - String msgType, - String sender, - Set receivers, - String title, - String content - ); -} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/PublicScriptService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/PublicScriptService.java new file mode 100644 index 0000000000..96e92de374 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/PublicScriptService.java @@ -0,0 +1,265 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.service; + +import com.tencent.bk.job.common.model.PageData; +import com.tencent.bk.job.manage.model.dto.ScriptBasicDTO; +import com.tencent.bk.job.manage.model.dto.ScriptDTO; +import com.tencent.bk.job.manage.model.dto.SyncScriptResultDTO; +import com.tencent.bk.job.manage.model.dto.TagDTO; +import com.tencent.bk.job.manage.model.dto.TemplateStepIDDTO; +import com.tencent.bk.job.manage.model.query.ScriptQuery; +import com.tencent.bk.job.manage.model.web.vo.TagCountVO; + +import java.util.Collection; +import java.util.List; +import java.util.Map; + +/** + * 公共脚本服务 + */ +public interface PublicScriptService { + + /** + * 根据ID查询脚本版本 + * + * @param scriptVersionId 脚本ID + * @return 脚本版本 + */ + ScriptDTO getScriptVersion(Long scriptVersionId); + + /** + * 根据scriptId查询脚本 + * + * @param scriptId 脚本ID + * @return 脚本 + */ + ScriptDTO getScript(String scriptId); + + + /** + * 根据scriptIds批量查询脚本基础信息 + * + * @param scriptIds 脚本ID集合 + * @return 脚本 + */ + List listScriptBasicInfoByScriptIds(Collection scriptIds); + + /** + * 根据脚本ID查询所有版本的脚本 + * + * @param scriptId 脚本ID + * @return 脚本版本列表 + */ + List listScriptVersion(String scriptId); + + /** + * 分页查询脚本列表 + * + * @param scriptQuery 查询条件 + * @return 分页脚本列表 + */ + PageData listPageScript(ScriptQuery scriptQuery); + + /** + * 根据条件查询脚本 + * + * @param scriptQuery 查询条件 + * @return 脚本列表 + */ + List listScripts(ScriptQuery scriptQuery); + + /** + * 保存脚本 + * + * @param username 用户账号 + * @param script 脚本信息 + */ + ScriptDTO saveScript(String username, ScriptDTO script); + + /** + * 保存脚本版本 + * + * @param username 用户账号 + * @param scriptVersion 脚本版本 + */ + ScriptDTO saveScriptVersion(String username, ScriptDTO scriptVersion); + + /** + * 更新脚本版本 + * + * @param username 用户账号 + * @param scriptVersion 脚本版本 + */ + ScriptDTO updateScriptVersion(String username, ScriptDTO scriptVersion); + + /** + * 删除脚本 + * + * @param username 用户账号 + * @param scriptId 脚本ID + */ + void deleteScript(String username, String scriptId); + + /** + * 删除脚本版本 + * + * @param username 用户账号 + * @param scriptVersionId 脚本版本ID + */ + void deleteScriptVersion(String username, Long scriptVersionId); + + /** + * 上线脚本 + * + * @param username 用户账号 + * @param scriptId 脚本ID + * @param scriptVersionId 脚本版本ID + */ + void publishScript(String username, String scriptId, Long scriptVersionId); + + /** + * 下线脚本 + * + * @param username 用户账号 + * @param scriptId 脚本ID + * @param scriptVersionId 脚本版本ID + */ + void disableScript(String username, String scriptId, Long scriptVersionId); + + /** + * 批量获取脚本的在线版本 + * + * @param scriptIdList 脚本ID列表 + * @return 脚本信息 + */ + Map batchGetOnlineScriptVersionByScriptIds(List scriptIdList); + + /** + * 更新脚本描述 + * + * @param operator 操作人 + * @param scriptId 脚本ID + * @param desc 脚本描述 + */ + ScriptDTO updateScriptDesc(String operator, String scriptId, String desc); + + /** + * 更新脚本名称 + * + * @param operator 操作人 + * @param scriptId 脚本ID + * @param newName 脚本名称 + */ + ScriptDTO updateScriptName(String operator, String scriptId, String newName); + + /** + * 更新脚本标签 + * + * @param operator 操作人 + * @param scriptId 脚本ID + * @param tags 脚本标签列表 + */ + ScriptDTO updateScriptTags(String operator, String scriptId, List tags); + + /** + * 根据脚本名称模糊查询业务下的脚本名 + * + * @param keyword 关键字 + * @return 脚本名称列表 + */ + List listScriptNames(String keyword); + + /** + * 获取已上线的公共脚本 + * + * @return 脚本列表 + */ + List listOnlineScript(); + + + /** + * 获取脚本已上线脚本版本 + * + * @param scriptId 脚本 ID + * @return 已上线版本，如果没有返回null + */ + ScriptDTO getOnlineScriptVersionByScriptId(String scriptId); + + /** + * 分页查询脚本版本列表 + * + * @param scriptQuery 查询条件 + * @return 脚本版本分页 + */ + PageData listPageScriptVersion(ScriptQuery scriptQuery); + + + /** + * 批量同步脚本到作业模板 + * + * @param operator 操作人 + * @param scriptId 脚本ID + * @param syncScriptVersionId 需要同步的脚本版本ID + * @param templateStepIDs 作业模板与步骤信息 + * @return 同步结果 + */ + List syncScriptToTaskTemplate(String operator, + String scriptId, + Long syncScriptVersionId, + List templateStepIDs); + + /** + * 是否存在任意公共脚本 + */ + boolean isExistAnyPublicScript(); + + List listScriptIds(); + + /** + * 获取标签关联的模版数量 + * + * @return 标签模版数量 + */ + TagCountVO getTagScriptCount(); + + /** + * 根据脚本ID/版本号查询脚本 + * + * @param scriptId 脚本ID + * @param version 脚本版本 + * @return 脚本版本 + */ + ScriptDTO getByScriptIdAndVersion(String scriptId, String version); + + /** + * 根据脚本ID获取公共脚本 + * + * @param scriptId 脚本ID + * @return 脚本 + */ + ScriptDTO getScriptByScriptId(String scriptId); + +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/ScriptCheckService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/ScriptCheckService.java index 0a6523aaa7..44f35f1dd1 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/ScriptCheckService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/ScriptCheckService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,7 @@ package com.tencent.bk.job.manage.service; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; import com.tencent.bk.job.manage.model.dto.ScriptCheckResultItemDTO; import java.util.List; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/ScriptManager.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/ScriptManager.java new file mode 100644 index 0000000000..c028fe895a --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/ScriptManager.java @@ -0,0 +1,374 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.service; + +import com.tencent.bk.job.common.model.PageData; +import com.tencent.bk.job.manage.api.common.constants.JobResourceStatusEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; +import com.tencent.bk.job.manage.model.dto.ScriptBasicDTO; +import com.tencent.bk.job.manage.model.dto.ScriptDTO; +import com.tencent.bk.job.manage.model.dto.ScriptSyncTemplateStepDTO; +import com.tencent.bk.job.manage.model.dto.SyncScriptResultDTO; +import com.tencent.bk.job.manage.model.dto.TagDTO; +import com.tencent.bk.job.manage.model.dto.TemplateStepIDDTO; +import com.tencent.bk.job.manage.model.dto.script.ScriptCitedTaskPlanDTO; +import com.tencent.bk.job.manage.model.dto.script.ScriptCitedTaskTemplateDTO; +import com.tencent.bk.job.manage.model.query.ScriptQuery; +import com.tencent.bk.job.manage.model.web.vo.TagCountVO; +import org.apache.commons.lang3.tuple.Pair; + +import java.util.Collection; +import java.util.List; +import java.util.Map; + +/** + * 脚本管理通用实现 + */ +public interface ScriptManager { + + /** + * 根据ID查询脚本版本 + * + * @param appId 业务ID + * @param scriptVersionId 脚本ID + * @return 脚本版本 + */ + ScriptDTO getScriptVersion(Long appId, Long scriptVersionId); + + /** + * 根据ID查询脚本版本 + * + * @param appId 业务ID + * @param scriptId 脚本ID + * @param version 脚本版本 + * @return 脚本版本 + */ + ScriptDTO getByScriptIdAndVersion(Long appId, String scriptId, String version); + + /** + * 根据ID查询脚本版本 + * + * @param scriptVersionId 脚本版本ID + * @return 脚本 + */ + ScriptDTO getScriptVersion(Long scriptVersionId); + + /** + * 根据scriptId查询脚本 + * + * @param appId 业务ID + * @param scriptId 脚本ID + * @return 脚本 + */ + ScriptDTO getScript(Long appId, String scriptId); + + /** + * 根据scriptId查询脚本 + * + * @param scriptId 脚本ID + * @return 脚本 + */ + ScriptDTO getScriptByScriptId(String scriptId); + + /** + * 根据scriptIds批量查询脚本基础信息 + * + * @param scriptIds 脚本ID集合 + * @return 脚本 + */ + List listScriptBasicInfoByScriptIds(Collection scriptIds); + + /** + * 根据scriptId查询脚本基本信息,不包含标签信息 + * + * @param scriptId 脚本ID + * @return 脚本 + */ + ScriptDTO getScriptWithoutTagByScriptId(String scriptId); + + /** + * 根据脚本ID查询所有版本的脚本 + * + * @param appId 业务ID + * @param scriptId 脚本ID + * @return 脚本版本列表 + */ + List listScriptVersion(Long appId, String scriptId); + + /** + * 分页查询脚本列表 + * + * @param scriptQuery 查询条件 + * @return 分页脚本列表 + */ + PageData listPageScript(ScriptQuery scriptQuery); + + /** + * 根据条件查询脚本 + * + * @param scriptQuery 查询条件 + * @return 脚本列表 + */ + List listScripts(ScriptQuery scriptQuery); + + /** + * 创建脚本 + * + * @param script 脚本信息 + */ + ScriptDTO createScript(ScriptDTO script); + + /** + * 创建脚本版本 + * + * @param scriptVersion 脚本版本 + */ + ScriptDTO createScriptVersion(ScriptDTO scriptVersion); + + /** + * 更新脚本版本 + * + * @param scriptVersion 脚本版本 + */ + ScriptDTO updateScriptVersion(ScriptDTO scriptVersion); + + /** + * 指定版本Id创建脚本版本 + * + * @param appId 业务ID + * @param script 脚本信息 + */ + Pair createScriptWithVersionId(Long appId, + ScriptDTO script, + Long createTime, + Long lastModifyTime); + + /** + * 删除脚本 + * + * @param appId 业务ID + * @param scriptId 脚本ID + */ + void deleteScript(Long appId, String scriptId); + + /** + * 删除脚本版本 + * + * @param appId 业务ID + * @param scriptVersionId 脚本版本ID + */ + void deleteScriptVersion(Long appId, Long scriptVersionId); + + /** + * 上线脚本 + * + * @param appId 业务ID + * @param scriptId 脚本ID + * @param scriptVersionId 脚本版本ID + */ + void publishScript(Long appId, String scriptId, Long scriptVersionId); + + /** + * 下线脚本 + * + * @param appId 业务ID + * @param scriptId 脚本ID + * @param scriptVersionId 脚本版本ID + */ + void disableScript(Long appId, String scriptId, Long scriptVersionId); + + /** + * 批量获取脚本的在线版本 + * + * @param scriptIdList 脚本ID列表 + * @return 脚本信息 + */ + Map batchGetOnlineScriptVersionByScriptIds(List scriptIdList); + + /** + * 更新脚本描述 + * + * @param operator 操作人 + * @param appId 业务ID + * @param scriptId 脚本ID + * @param desc 脚本描述 + */ + ScriptDTO updateScriptDesc(String operator, Long appId, String scriptId, String desc); + + /** + * 更新脚本名称 + * + * @param operator 操作人 + * @param appId 业务ID + * @param scriptId 脚本ID + * @param newName 脚本名称 + */ + ScriptDTO updateScriptName(String operator, Long appId, String scriptId, String newName); + + /** + * 更新脚本标签 + * + * @param operator 操作人 + * @param appId 业务ID + * @param scriptId 脚本ID + * @param tags 脚本标签列表 + */ + ScriptDTO updateScriptTags(String operator, Long appId, String scriptId, List tags); + + /** + * 根据脚本名称模糊查询业务下的脚本名 + * + * @param appId 业务ID + * @param keyword 关键字 + * @return 脚本名称列表 + */ + List listScriptNames(Long appId, String keyword); + + /** + * 获取已上线脚本列表 + * + * @param appId 业务ID + * @return 脚本列表 + */ + List listOnlineScriptForApp(long appId); + + /** + * 获取脚本已上线脚本版本 + * + * @param appId 业务 ID + * @param scriptId 脚本 ID + * @return 已上线版本，如果没有返回null + */ + ScriptDTO getOnlineScriptVersionByScriptId(long appId, String scriptId); + + /** + * 获取脚本已上线脚本版本 + * + * @param scriptId 脚本 ID + * @return 已上线版本，如果没有返回null + */ + ScriptDTO getOnlineScriptVersionByScriptId(String scriptId); + + /** + * 分页查询脚本版本列表 + * + * @param scriptQuery 查询条件 + * @return 脚本版本分页 + */ + PageData listPageScriptVersion(ScriptQuery scriptQuery); + + /** + * 获取引用脚本的模板与步骤 + * + * @param appId 业务ID + * @param scriptId 脚本ID + * @return 引用脚本的模板与步骤 + */ + List listScriptSyncTemplateSteps(Long appId, String scriptId); + + /** + * 批量同步脚本到作业模板 + * + * @param username 用户名 + * @param appId 业务ID + * @param scriptId 脚本ID + * @param syncScriptVersionId 需要同步的脚本版本ID + * @param templateStepIDs 作业模板与步骤信息 + * @return 同步结果 + */ + List syncScriptToTaskTemplate(String username, + Long appId, + String scriptId, + Long syncScriptVersionId, + List templateStepIDs); + + /** + * 获取引用脚本的模板数量 + * + * @param scriptId 脚本ID + * @param scriptVersionId 脚本版本ID + * @return 引用脚本的模板数量 + */ + Integer getScriptTemplateCiteCount(String scriptId, Long scriptVersionId); + + /** + * 获取引用脚本的执行方案数量 + * + * @param scriptId 脚本ID + * @param scriptVersionId 脚本版本ID + * @return 引用脚本的执行方案数量 + */ + Integer getScriptTaskPlanCiteCount(String scriptId, Long scriptVersionId); + + /** + * 获取引用脚本的模板信息 + * + * @param scriptId 脚本ID + * @param scriptVersionId 脚本版本ID + * @return 引用脚本的作业模板 + */ + List getScriptCitedTemplates(String scriptId, Long scriptVersionId); + + /** + * 获取引用脚本的执行方案信息 + * + * @param scriptId 脚本ID + * @param scriptVersionId 脚本版本ID + * @return 引用脚本的执行方案 + */ + List getScriptCitedTaskPlans(String scriptId, Long scriptVersionId); + + Integer countScripts(Long appId, ScriptTypeEnum scriptTypeEnum, JobResourceStatusEnum jobResourceStatusEnum); + + Integer countScriptVersions(Long appId, ScriptTypeEnum scriptTypeEnum, JobResourceStatusEnum jobResourceStatusEnum); + + List listScriptIds(Long appId); + + Integer countCiteScripts(Long appId); + + /** + * 获取标签关联的模版数量 + * + * @param appId 业务 ID + * @return 标签模版数量 + */ + TagCountVO getTagScriptCount(Long appId); + + /** + * 当前业务下是否存在任意脚本 + * + * @param appId 业务 ID + */ + boolean isExistAnyScript(Long appId); + + /** + * 脚本版本是否被引用 + * + * @param scriptId 脚本ID + * @param scriptVersionId 脚本版本ID + */ + boolean isScriptReferenced(String scriptId, Long scriptVersionId); + +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/ScriptService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/ScriptService.java index 49d9db2e15..a333a09558 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/ScriptService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/ScriptService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,274 +24,203 @@ package com.tencent.bk.job.manage.service; -import com.tencent.bk.job.common.exception.ServiceException; -import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; -import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; -import com.tencent.bk.job.manage.common.consts.JobResourceStatusEnum; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.JobResourceStatusEnum; import com.tencent.bk.job.manage.model.dto.ScriptBasicDTO; import com.tencent.bk.job.manage.model.dto.ScriptDTO; -import com.tencent.bk.job.manage.model.dto.ScriptRelatedTaskPlanDTO; -import com.tencent.bk.job.manage.model.dto.ScriptSyncTemplateStepDTO; import com.tencent.bk.job.manage.model.dto.SyncScriptResultDTO; import com.tencent.bk.job.manage.model.dto.TagDTO; import com.tencent.bk.job.manage.model.dto.TemplateStepIDDTO; -import com.tencent.bk.job.manage.model.dto.script.ScriptCitedTaskPlanDTO; -import com.tencent.bk.job.manage.model.dto.script.ScriptCitedTaskTemplateDTO; import com.tencent.bk.job.manage.model.query.ScriptQuery; import com.tencent.bk.job.manage.model.web.vo.TagCountVO; -import org.apache.commons.lang3.tuple.Pair; import java.util.Collection; import java.util.List; import java.util.Map; /** - * 脚本服务 - * - * @date 2019/09/19 + * 公共脚本服务 */ public interface ScriptService { /** - * 根据ID查询脚本版本 - * - * @param operator 操作者 - * @param appId 业务ID - * @param scriptVersionId 脚本ID - * @return 脚本版本 - * @throws ServiceException 业务异常 - */ - ScriptDTO getScriptVersion(String operator, Long appId, Long scriptVersionId) throws ServiceException; - - /** - * 根据ID查询脚本版本 + * 分页查询脚本列表 * - * @param operator 操作者 - * @param appId 业务ID - * @param scriptId 脚本ID - * @param version 脚本版本 - * @return 脚本版本 - * @throws ServiceException 业务异常 + * @param scriptQuery 查询条件 + * @return 分页脚本列表 */ - ScriptDTO getByScriptIdAndVersion( - String operator, - Long appId, - String scriptId, - String version - ) throws ServiceException; + PageData listPageScript(ScriptQuery scriptQuery); /** - * 根据ID查询脚本版本 + * 根据条件查询脚本 * - * @param scriptVersionId 脚本版本ID - * @return 脚本 - * @throws ServiceException + * @param scriptQuery 查询条件 + * @return 脚本列表 */ - ScriptDTO getScriptVersion(Long scriptVersionId) throws ServiceException; + List listScripts(ScriptQuery scriptQuery); /** * 根据scriptId查询脚本 * - * @param operator 操作者 * @param appId 业务ID * @param scriptId 脚本ID * @return 脚本 - * @throws ServiceException 业务异常 */ - ScriptDTO getScript(String operator, Long appId, String scriptId) throws ServiceException; + ScriptDTO getScript(Long appId, String scriptId); /** - * 根据scriptId查询脚本 + * 查询脚本 * + * @param username 用户账号 + * @param appId 业务ID * @param scriptId 脚本ID * @return 脚本 - * @throws ServiceException 业务异常 */ - ScriptDTO getScriptByScriptId(String scriptId) throws ServiceException; + ScriptDTO getScript(String username, Long appId, String scriptId); /** * 根据scriptIds批量查询脚本基础信息 * * @param scriptIds 脚本ID集合 * @return 脚本 - * @throws ServiceException 业务异常 */ - List listScriptBasicInfoByScriptIds(Collection scriptIds) throws ServiceException; + List listScriptBasicInfoByScriptIds(Collection scriptIds); /** - * 根据scriptId查询脚本基本信息,不包含标签信息 + * 创建脚本 * - * @param scriptId 脚本ID - * @return 脚本 - * @throws ServiceException 业务异常 + * @param username 用户账号 + * @param script 脚本信息 */ - ScriptDTO getScriptWithoutTagByScriptId(String scriptId) throws ServiceException; + ScriptDTO createScript(String username, ScriptDTO script); /** - * 根据脚本ID查询所有版本的脚本 + * 删除脚本 * - * @param operator 操作者 + * @param username 用户账号 * @param appId 业务ID * @param scriptId 脚本ID - * @return 脚本版本列表 - * @throws ServiceException 业务异常 */ - List listScriptVersion(String operator, Long appId, String scriptId) throws ServiceException; + void deleteScript(String username, Long appId, String scriptId); /** - * 分页查询脚本列表 + * 根据版本ID查询脚本版本 * - * @param scriptCondition 查询条件 - * @param baseSearchCondition 基本查询条件 - * @return 分页脚本列表 - * @throws ServiceException 业务异常 + * @param appId 业务ID + * @param scriptVersionId 脚本ID + * @return 脚本版本 */ - PageData listPageScript( - ScriptQuery scriptCondition, - BaseSearchCondition baseSearchCondition - ) throws ServiceException; + ScriptDTO getScriptVersion(long appId, Long scriptVersionId); /** - * 根据条件查询脚本 + * 根据版本ID查询脚本版本 * - * @param scriptQuery 查询条件 - * @return 脚本列表 + * @param scriptVersionId 脚本ID + * @return 脚本版本 */ - List listScripts(ScriptQuery scriptQuery); + ScriptDTO getScriptVersion(Long scriptVersionId); /** - * 保存脚本 + * 查询脚本版本 * - * @param operator 操作者 - * @param appId 业务ID - * @param script 脚本信息 - * @throws ServiceException 业务异常 + * @param username 用户账号 + * @param appId 业务ID + * @param scriptVersionId 脚本ID + * @return 脚本版本 */ - ScriptDTO saveScript(String operator, Long appId, ScriptDTO script) throws ServiceException; + ScriptDTO getScriptVersion(String username, long appId, Long scriptVersionId); /** - * 指定版本Id创建脚本版本 + * 根据脚本ID查询所有版本的脚本 * - * @param operator 操作者 * @param appId 业务ID - * @param script 脚本信息 - * @throws ServiceException 业务异常 - */ - Pair createScriptWithVersionId( - String operator, - Long appId, - ScriptDTO script, - Long createTime, - Long lastModifyTime - ) throws ServiceException; - - /** - * 查询引用脚本的作业列表 - * * @param scriptId 脚本ID - * @return 引用的作业列表 - * @throws ServiceException 业务异常 + * @return 脚本版本列表 */ - List listScriptRelatedTasks(String scriptId) throws ServiceException; + List listScriptVersion(long appId, String scriptId); /** - * 查询引用脚本版本的作业列表 + * 创建脚本版本 * - * @param scriptId 脚本ID - * @param scriptVersionId 脚本版本ID - * @return 引用的作业列表 - * @throws ServiceException 业务异常 + * @param username 用户账号 + * @param scriptVersion 脚本版本 */ - List listScriptVersionRelatedTasks( - String scriptId, - Long scriptVersionId - ) throws ServiceException; + ScriptDTO createScriptVersion(String username, ScriptDTO scriptVersion); /** - * 删除脚本 + * 更新脚本版本 * - * @param operator 操作者 - * @param appId 业务ID - * @param scriptId 脚本ID - * @throws ServiceException 业务异常 + * @param username 用户账号 + * @param scriptVersion 脚本版本 */ - void deleteScript(String operator, Long appId, String scriptId) throws ServiceException; + ScriptDTO updateScriptVersion(String username, ScriptDTO scriptVersion); /** * 删除脚本版本 * - * @param operator 操作者 + * @param username 操作者 * @param appId 业务ID * @param scriptVersionId 脚本版本ID - * @throws ServiceException 业务异常 */ - void deleteScriptVersion(String operator, Long appId, Long scriptVersionId) throws ServiceException; + void deleteScriptVersion(String username, Long appId, Long scriptVersionId); /** * 上线脚本 * * @param appId 业务ID - * @param operator 操作者 + * @param username 操作者 * @param scriptId 脚本ID * @param scriptVersionId 脚本版本ID - * @throws ServiceException 业务异常 */ - void publishScript(Long appId, String operator, String scriptId, Long scriptVersionId) throws ServiceException; + void publishScript(Long appId, String username, String scriptId, Long scriptVersionId); /** * 下线脚本 * * @param appId 业务ID - * @param operator 操作者 + * @param username 操作者 * @param scriptId 脚本ID * @param scriptVersionId 脚本版本ID - * @throws ServiceException 业务异常 */ - void disableScript(Long appId, String operator, String scriptId, Long scriptVersionId) throws ServiceException; + void disableScript(Long appId, String username, String scriptId, Long scriptVersionId); /** * 批量获取脚本的在线版本 * * @param scriptIdList 脚本ID列表 * @return 脚本信息 - * @throws ServiceException 业务异常 */ - Map batchGetOnlineScriptVersionByScriptIds(List scriptIdList) throws ServiceException; + Map batchGetOnlineScriptVersionByScriptIds(List scriptIdList); /** * 更新脚本描述 * * @param appId 业务ID - * @param operator 操作者 + * @param username 操作者 * @param scriptId 脚本ID * @param desc 脚本描述 - * @throws ServiceException 业务异常 */ - void updateScriptDesc(Long appId, String operator, String scriptId, String desc) throws ServiceException; + ScriptDTO updateScriptDesc(Long appId, String username, String scriptId, String desc); /** * 更新脚本名称 * * @param appId 业务ID - * @param operator 操作者 + * @param username 操作者 * @param scriptId 脚本ID * @param newName 脚本名称 - * @throws ServiceException 业务异常 */ - void updateScriptName(Long appId, String operator, String scriptId, String newName) throws ServiceException; + ScriptDTO updateScriptName(Long appId, String username, String scriptId, String newName); /** * 更新脚本标签 * * @param appId 业务ID - * @param operator 操作者 + * @param username 操作者 * @param scriptId 脚本ID * @param tags 脚本标签列表 - * @throws ServiceException 业务异常 */ - void updateScriptTags(Long appId, String operator, String scriptId, List tags) throws ServiceException; + ScriptDTO updateScriptTags(Long appId, String username, String scriptId, List tags); /** * 根据脚本名称模糊查询业务下的脚本名 @@ -299,9 +228,8 @@ List listScriptVersionRelatedTasks( * @param appId 业务ID * @param keyword 关键字 * @return 脚本名称列表 - * @throws ServiceException 业务异常 */ - List listScriptNames(Long appId, String keyword) throws ServiceException; + List listScriptNames(Long appId, String keyword); /** * 获取业务下的已上线脚本列表 @@ -309,58 +237,26 @@ List listScriptVersionRelatedTasks( * @param operator 操作者 * @param appId 业务ID * @return 脚本列表 - * @throws ServiceException 业务异常 */ - List listOnlineScriptForApp(String operator, long appId) throws ServiceException; - - /** - * 获取脚本列表 - * - * @param scriptCondition 查询条件 - * @param baseSearchCondition 基本查询条件 - * @return 脚本列表 - * @throws ServiceException 业务异常 - */ - PageData listPageOnlineScript(ScriptQuery scriptCondition, - BaseSearchCondition baseSearchCondition) throws ServiceException; + List listOnlineScript(String operator, long appId); /** * 获取脚本已上线脚本版本 * - * @param operator 操作者 + * @param username 用户账号 * @param appId 业务 ID * @param scriptId 脚本 ID * @return 已上线版本，如果没有返回null - * @throws ServiceException - */ - ScriptDTO getOnlineScriptVersionByScriptId(String operator, long appId, String scriptId) throws ServiceException; - - /** - * 获取脚本已上线脚本版本 - * - * @param scriptId 脚本 ID - * @return 已上线版本，如果没有返回null */ - ScriptDTO getOnlineScriptVersionByScriptId(String scriptId); + ScriptDTO getOnlineScriptVersionByScriptId(String username, long appId, String scriptId); /** * 分页查询脚本版本列表 * - * @param scriptQuery 查询条件 - * @param baseSearchCondition 基本查询条件 - * @return - */ - PageData listPageScriptVersion(ScriptQuery scriptQuery, BaseSearchCondition baseSearchCondition); - - /** - * 获取引用脚本的模板与步骤 - * - * @param username 用户名 - * @param appId 业务ID - * @param scriptId 脚本ID - * @return 引用脚本的模板与步骤 + * @param scriptQuery 查询条件 + * @return 脚本版本列表 */ - List listScriptSyncTemplateSteps(String username, Long appId, String scriptId); + PageData listPageScriptVersion(ScriptQuery scriptQuery); /** * 批量同步脚本到作业模板 @@ -372,83 +268,45 @@ PageData listPageOnlineScript(ScriptQuery scriptCondition, * @param templateStepIDs 作业模板与步骤信息 * @return 同步结果 */ - List syncScriptToTaskTemplate(String username, Long appId, String scriptId, + List syncScriptToTaskTemplate(String username, + Long appId, + String scriptId, Long syncScriptVersionId, - List templateStepIDs) - throws PermissionDeniedException; + List templateStepIDs); - /** - * 获取引用脚本的模板数量 - * - * @param username - * @param appId - * @param scriptId - * @param scriptVersionId - * @return - */ - Integer getScriptTemplateCiteCount(String username, Long appId, String scriptId, Long scriptVersionId); /** - * 获取引用脚本的执行方案数量 - * - * @param username - * @param appId - * @param scriptId - * @param scriptVersionId - * @return + * 是否存在任意业务脚本 */ - Integer getScriptTaskPlanCiteCount(String username, Long appId, String scriptId, Long scriptVersionId); + boolean isExistAnyAppScript(long appId); /** - * 获取引用脚本的模板信息 + * 获取标签关联的模版数量 * - * @param username 用户名 - * @param appId 业务ID - * @param scriptId 脚本ID - * @param scriptVersionId 脚本版本ID - * @return + * @param appId 业务 ID + * @return 标签模版数量 */ - List getScriptCitedTemplates(String username, Long appId, String scriptId, - Long scriptVersionId); + TagCountVO getTagScriptCount(Long appId); /** - * 获取引用脚本的执行方案信息 + * 根据脚本ID/版本号查询脚本 * - * @param username 用户名 - * @param appId 业务ID - * @param scriptId 脚本ID - * @param scriptVersionId 脚本版本ID - * @return + * @param username 用户账号 + * @param appId 业务ID + * @param scriptId 脚本ID + * @param version 脚本版本 + * @return 脚本版本 */ - List getScriptCitedTaskPlans(String username, Long appId, String scriptId, - Long scriptVersionId); + ScriptDTO getByScriptIdAndVersion(String username, Long appId, String scriptId, String version); /** - * 业务下是否存在任意脚本 + * 根据脚本ID获取业务脚本 * - * @param appId 业务ID - */ - boolean isExistAnyAppScript(Long appId); - - /** - * 是否存在任意公共脚本 + * @param scriptId 脚本ID + * @return 脚本 */ - boolean isExistAnyPublicScript(); - - Integer countScripts(Long appId, ScriptTypeEnum scriptTypeEnum, JobResourceStatusEnum jobResourceStatusEnum); - - Integer countScriptVersions(Long appId, ScriptTypeEnum scriptTypeEnum, JobResourceStatusEnum jobResourceStatusEnum); - - List listScriptIds(Long appId); - - Integer countCiteScripts(Long appId); + ScriptDTO getScriptByScriptId(String scriptId); - /** - * 获取标签关联的模版数量 - * - * @param appId 业务 ID - * @return 标签模版数量 - */ - TagCountVO getTagScriptCount(Long appId); + Map batchGetScriptVersionStatus(Collection scriptVersionId); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/SyncService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/SyncService.java index 3411af98cf..c4c195e750 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/SyncService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/SyncService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,9 +24,6 @@ package com.tencent.bk.job.manage.service; -import org.springframework.data.util.Pair; - -import java.util.concurrent.Future; import java.util.concurrent.ThreadPoolExecutor; /** @@ -49,14 +46,8 @@ public interface SyncService { ThreadPoolExecutor getSyncHostExecutor(); - ThreadPoolExecutor getSyncAgentStatusExecutor(); - Boolean syncBizHosts(Long bizId); - boolean addExtraSyncBizHostsTask(Long bizId); - - Future> arrangeSyncBizHostsTask(Long bizId); - Boolean enableBizWatch(); Boolean disableBizWatch(); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/TagService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/TagService.java index 5bfba1cd97..707cddaa97 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/TagService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/TagService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -84,9 +84,9 @@ public interface TagService { * * @param username 操作人 * @param tag Tag - * @return Tag ID + * @return Tag */ - Long insertNewTag(String username, TagDTO tag); + TagDTO createTag(String username, TagDTO tag); /** * 更新 Tag 信息 @@ -123,9 +123,11 @@ public interface TagService { /** * 删除标签 * - * @param tagId 标签ID + * @param username 用户账号 + * @param appId 业务 ID + * @param tagId 标签ID */ - void deleteTag(Long tagId); + void deleteTag(String username, long appId, Long tagId); List listResourceTagsByTagId(Long appId, Long tagId); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/TaskFavoriteService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/TaskFavoriteService.java index 5672e5c591..7e8dc62e31 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/TaskFavoriteService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/TaskFavoriteService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/WhiteIPService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/WhiteIPService.java index 13714564c6..655b38f29a 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/WhiteIPService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/WhiteIPService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,14 +25,16 @@ package com.tencent.bk.job.manage.service; import com.tencent.bk.job.common.model.PageData; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.model.vo.CloudAreaInfoVO; -import com.tencent.bk.job.manage.common.consts.whiteip.ActionScopeEnum; -import com.tencent.bk.job.manage.model.dto.whiteip.CloudIPDTO; +import com.tencent.bk.job.manage.api.common.constants.whiteip.ActionScopeEnum; +import com.tencent.bk.job.manage.model.dto.whiteip.WhiteIPRecordDTO; import com.tencent.bk.job.manage.model.inner.ServiceWhiteIPInfo; import com.tencent.bk.job.manage.model.web.request.whiteip.WhiteIPRecordCreateUpdateReq; import com.tencent.bk.job.manage.model.web.vo.whiteip.ActionScopeVO; import com.tencent.bk.job.manage.model.web.vo.whiteip.WhiteIPRecordVO; +import java.util.Collection; import java.util.List; /** @@ -45,16 +47,22 @@ PageData listWhiteIPRecord(String username, String ipStr, Strin Integer start, Integer pageSize, String orderField, Integer order); /** - * 查找对业务生效的IP信息 + * 查找对业务生效的IP对应的主机信息 * - * @param appId 业务Id - * @return + * @param appId Job业务Id + * @return 对业务生效的白名单IP对应的主机 */ - List listWhiteIP(Long appId, ActionScopeEnum actionScope); + List listAvailableWhiteIPHost(Long appId, ActionScopeEnum actionScope, Collection hostIds); - Long saveWhiteIP(String username, WhiteIPRecordCreateUpdateReq createUpdateReq); + List listAvailableWhiteIPHostByIps(Long appId, ActionScopeEnum actionScope, Collection ips); - WhiteIPRecordVO getWhiteIPDetailById(String username, Long id); + List listAvailableWhiteIPHostByIpv6s(Long appId, ActionScopeEnum actionScope, Collection ipv6s); + + WhiteIPRecordDTO createWhiteIP(String username, WhiteIPRecordCreateUpdateReq createUpdateReq); + + WhiteIPRecordDTO updateWhiteIP(String username, WhiteIPRecordCreateUpdateReq createUpdateReq); + + WhiteIPRecordDTO getWhiteIPDetailById(String username, Long id); List listCloudAreas(String username); @@ -62,7 +70,7 @@ PageData listWhiteIPRecord(String username, String ipStr, Strin Long deleteWhiteIPById(String username, Long id); - List getWhiteIPActionScopes(Long appId, String ip, Long cloudAreaId); + List getWhiteIPActionScopes(Long appId, String ip, Long cloudAreaId, Long hostId); List listWhiteIPInfos(); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/agent/status/ScopeAgentStatusService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/agent/status/ScopeAgentStatusService.java new file mode 100644 index 0000000000..a78c32e640 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/agent/status/ScopeAgentStatusService.java @@ -0,0 +1,15 @@ +package com.tencent.bk.job.manage.service.agent.status; + +import com.tencent.bk.job.common.gse.v2.model.req.ListAgentStateReq; +import com.tencent.bk.job.common.gse.v2.model.resp.AgentState; +import com.tencent.bk.job.common.model.dto.ResourceScope; + +import java.util.List; + +public interface ScopeAgentStatusService { + + boolean needToUseGseV2(ResourceScope resourceScope); + + List listAgentState(ResourceScope resourceScope, ListAgentStateReq req); + +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/agent/status/impl/ScopeAgentStatusServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/agent/status/impl/ScopeAgentStatusServiceImpl.java new file mode 100644 index 0000000000..ebcd0d72c5 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/agent/status/impl/ScopeAgentStatusServiceImpl.java @@ -0,0 +1,54 @@ +package com.tencent.bk.job.manage.service.agent.status.impl; + +import com.tencent.bk.job.common.gse.IGseClient; +import com.tencent.bk.job.common.gse.v1.GseV1ApiClient; +import com.tencent.bk.job.common.gse.v2.GseV2ApiClient; +import com.tencent.bk.job.common.gse.v2.model.req.ListAgentStateReq; +import com.tencent.bk.job.common.gse.v2.model.resp.AgentState; +import com.tencent.bk.job.common.model.dto.ResourceScope; +import com.tencent.bk.job.common.util.toggle.ToggleEvaluateContext; +import com.tencent.bk.job.common.util.toggle.ToggleStrategyContextParams; +import com.tencent.bk.job.common.util.toggle.feature.FeatureIdConstants; +import com.tencent.bk.job.common.util.toggle.feature.FeatureToggle; +import com.tencent.bk.job.manage.service.agent.status.ScopeAgentStatusService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.List; + +@Slf4j +@Service +public class ScopeAgentStatusServiceImpl implements ScopeAgentStatusService { + + private final GseV1ApiClient gseV1ApiClient; + private final GseV2ApiClient gseV2ApiClient; + + public ScopeAgentStatusServiceImpl(@Autowired(required = false) GseV1ApiClient gseV1ApiClient, + @Autowired(required = false) GseV2ApiClient gseV2ApiClient) { + this.gseV1ApiClient = gseV1ApiClient; + this.gseV2ApiClient = gseV2ApiClient; + } + + @Override + public boolean needToUseGseV2(ResourceScope resourceScope) { + ToggleEvaluateContext featureExecutionContext = + ToggleEvaluateContext.builder() + .addContextParam(ToggleStrategyContextParams.CTX_PARAM_RESOURCE_SCOPE, resourceScope); + return FeatureToggle.checkFeature( + FeatureIdConstants.FEATURE_AGENT_STATUS_GSE_V2, + featureExecutionContext + ); + } + + @Override + public List listAgentState(ResourceScope resourceScope, ListAgentStateReq req) { + IGseClient gseClient = chooseGseApiClientByResourceScope(resourceScope); + return gseClient.listAgentState(req); + } + + private IGseClient chooseGseApiClientByResourceScope(ResourceScope resourceScope) { + return needToUseGseV2(resourceScope) ? gseV2ApiClient : gseV1ApiClient; + } + +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/auth/TaskTemplateAuthService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/auth/TaskTemplateAuthService.java index 3e25cacac7..f28cbdd748 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/auth/TaskTemplateAuthService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/auth/TaskTemplateAuthService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/auth/impl/TaskTemplateAuthServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/auth/impl/TaskTemplateAuthServiceImpl.java index bbbaf09abb..ff6e19d509 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/auth/impl/TaskTemplateAuthServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/auth/impl/TaskTemplateAuthServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/globalsetting/BkPlatformInfoService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/globalsetting/BkPlatformInfoService.java new file mode 100644 index 0000000000..548b116b3c --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/globalsetting/BkPlatformInfoService.java @@ -0,0 +1,36 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.service.globalsetting; + +import com.tencent.bk.job.manage.model.migration.BkPlatformInfo; + +public interface BkPlatformInfoService { + /** + * 获取当前使用的蓝鲸平台信息 + * + * @return 当前使用的蓝鲸平台信息 + */ + BkPlatformInfo getCurrentBkPlatformInfo(); +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/GlobalSettingsService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/globalsetting/GlobalSettingsService.java similarity index 83% rename from src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/GlobalSettingsService.java rename to src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/globalsetting/GlobalSettingsService.java index ae32a9b727..b5ba2036fe 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/GlobalSettingsService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/globalsetting/GlobalSettingsService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -22,14 +22,13 @@ * IN THE SOFTWARE. */ -package com.tencent.bk.job.manage.service; +package com.tencent.bk.job.manage.service.globalsetting; -import com.tencent.bk.job.manage.common.consts.globalsetting.OSTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.OSTypeEnum; import com.tencent.bk.job.manage.model.web.request.globalsetting.AccountNameRule; import com.tencent.bk.job.manage.model.web.request.globalsetting.AccountNameRulesReq; import com.tencent.bk.job.manage.model.web.request.globalsetting.FileUploadSettingReq; import com.tencent.bk.job.manage.model.web.request.globalsetting.HistoryExpireReq; -import com.tencent.bk.job.manage.model.web.request.globalsetting.SetTitleFooterReq; import com.tencent.bk.job.manage.model.web.request.notify.ChannelTemplatePreviewReq; import com.tencent.bk.job.manage.model.web.request.notify.ChannelTemplateReq; import com.tencent.bk.job.manage.model.web.request.notify.NotifyBlackUsersReq; @@ -37,26 +36,20 @@ import com.tencent.bk.job.manage.model.web.vo.globalsetting.AccountNameRulesWithDefaultVO; import com.tencent.bk.job.manage.model.web.vo.globalsetting.FileUploadSettingVO; import com.tencent.bk.job.manage.model.web.vo.globalsetting.NotifyChannelWithIconVO; -import com.tencent.bk.job.manage.model.web.vo.globalsetting.TitleFooterVO; -import com.tencent.bk.job.manage.model.web.vo.globalsetting.TitleFooterWithDefaultVO; +import com.tencent.bk.job.manage.model.web.vo.globalsetting.PlatformInfoVO; +import com.tencent.bk.job.manage.model.web.vo.globalsetting.PlatformInfoWithDefaultVO; import com.tencent.bk.job.manage.model.web.vo.notify.ChannelTemplateDetailWithDefaultVO; import com.tencent.bk.job.manage.model.web.vo.notify.ChannelTemplateStatusVO; import com.tencent.bk.job.manage.model.web.vo.notify.NotifyBlackUserInfoVO; import com.tencent.bk.job.manage.model.web.vo.notify.UserVO; -import org.jooq.DSLContext; import java.util.List; import java.util.Map; -/** - * @Description - * @Date 2020/2/28 - * @Version 1.0 - */ public interface GlobalSettingsService { - Boolean isNotifyChannelConfiged(DSLContext dslContext); + Boolean isNotifyChannelConfiged(); - Boolean setNotifyChannelConfiged(DSLContext dslContext); + Boolean setNotifyChannelConfiged(); List listNotifyChannel(String username); @@ -82,11 +75,19 @@ public interface GlobalSettingsService { FileUploadSettingVO getFileUploadSettings(); - Boolean setTitleFooter(String username, SetTitleFooterReq req); - - TitleFooterVO getTitleFooter(); + /** + * 获取渲染之后的平台信息 + * + * @return 渲染之后的平台信息 + */ + PlatformInfoVO getRenderedPlatformInfo(); - TitleFooterWithDefaultVO getTitleFooterWithDefault(String username); + /** + * 获取全局设置-平台信息 + * + * @param username 用户名 + */ + PlatformInfoWithDefaultVO getPlatformInfoWithDefault(String username); Integer saveChannelTemplate(String username, ChannelTemplateReq req); @@ -99,6 +100,8 @@ ChannelTemplateDetailWithDefaultVO getChannelTemplateDetail(String username, Str String getDocCenterBaseUrl(); + String getDocJobRootUrl(); + Map getRelatedSystemUrls(String username); Map getJobConfig(String username); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/globalsetting/impl/BkPlatformInfoServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/globalsetting/impl/BkPlatformInfoServiceImpl.java new file mode 100644 index 0000000000..6c0a210ba8 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/globalsetting/impl/BkPlatformInfoServiceImpl.java @@ -0,0 +1,176 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.service.globalsetting.impl; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.tencent.bk.job.common.constant.HttpMethodEnum; +import com.tencent.bk.job.common.i18n.locale.LocaleUtils; +import com.tencent.bk.job.common.util.StringUtil; +import com.tencent.bk.job.common.util.http.HttpHelper; +import com.tencent.bk.job.common.util.http.HttpHelperFactory; +import com.tencent.bk.job.common.util.http.HttpRequest; +import com.tencent.bk.job.common.util.http.HttpResponse; +import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.manage.api.common.constants.globalsetting.GlobalSettingKeys; +import com.tencent.bk.job.manage.config.JobManageConfig; +import com.tencent.bk.job.manage.dao.globalsetting.GlobalSettingDAO; +import com.tencent.bk.job.manage.model.dto.GlobalSettingDTO; +import com.tencent.bk.job.manage.model.dto.globalsetting.HelperInfo; +import com.tencent.bk.job.manage.model.dto.globalsetting.TitleFooter; +import com.tencent.bk.job.manage.model.dto.globalsetting.TitleFooterDTO; +import com.tencent.bk.job.manage.model.migration.BkPlatformInfo; +import com.tencent.bk.job.manage.service.globalsetting.BkPlatformInfoService; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.Map; + +/** + * 将全局配置中的平台信息导出为蓝鲸统一规范的平台信息数据 + */ +@Slf4j +@Service +public class BkPlatformInfoServiceImpl implements BkPlatformInfoService { + + private final GlobalSettingDAO globalSettingDAO; + private final JobManageConfig jobManageConfig; + + @Autowired + public BkPlatformInfoServiceImpl(GlobalSettingDAO globalSettingDAO, JobManageConfig jobManageConfig) { + this.globalSettingDAO = globalSettingDAO; + this.jobManageConfig = jobManageConfig; + } + + @Override + public BkPlatformInfo getCurrentBkPlatformInfo() { + GlobalSettingDTO titleFooterSettingDTO = globalSettingDAO.getGlobalSetting(GlobalSettingKeys.KEY_TITLE_FOOTER); + TitleFooterDTO titleFooterDTO = null; + if (titleFooterSettingDTO != null) { + titleFooterDTO = JsonUtils.fromJson(titleFooterSettingDTO.getValue(), new TypeReference() { + }); + } + GlobalSettingDTO bkHelperSettingDTO = globalSettingDAO.getGlobalSetting(GlobalSettingKeys.KEY_BK_HELPER); + log.info("titleFooterSettingDTO={}, bkHelperSettingDTO={}", titleFooterSettingDTO, bkHelperSettingDTO); + HelperInfo helperInfo = null; + if (bkHelperSettingDTO != null && StringUtils.isNotEmpty(bkHelperSettingDTO.getValue())) { + helperInfo = JsonUtils.fromJson(bkHelperSettingDTO.getValue(), HelperInfo.class); + } + BkPlatformInfo defaultBkPlatformInfo = getDefaultBkPlatformInfo(); + BkPlatformInfo bkPlatformInfo = buildBkPlatformInfo(defaultBkPlatformInfo, titleFooterDTO, helperInfo); + log.info("bkPlatformInfo={}", JsonUtils.toJson(bkPlatformInfo)); + return bkPlatformInfo; + } + + private BkPlatformInfo getDefaultBkPlatformInfo() { + try { + return getBkPlatformInfoFromArtifactory(); + } catch (Exception e) { + log.warn("Fail to getBkPlatformInfoFromArtifactory, use local default", e); + return new BkPlatformInfo(); + } + } + + private BkPlatformInfo getBkPlatformInfoFromArtifactory() { + String bkSharedResUrl = jobManageConfig.getBkSharedResUrl(); + String bkSharedBaseJsPath = jobManageConfig.getBkSharedBaseJsPath(); + String baseJsonFileUrl = buildBaseJsFileUrl(bkSharedResUrl, bkSharedBaseJsPath); + HttpHelper httpHelper = HttpHelperFactory.getDefaultHttpHelper(); + HttpRequest request = HttpRequest.builder(HttpMethodEnum.GET, baseJsonFileUrl).build(); + HttpResponse resp = httpHelper.requestForSuccessResp(request); + String baseJsStr = resp.getEntity(); + log.info("Got base.js content from bkSharedRes: " + baseJsStr); + String platformJsonStr = parsePlatformJsonStr(baseJsStr); + return JsonUtils.fromJson(platformJsonStr, new TypeReference() { + }); + } + + private String parsePlatformJsonStr(String baseJsStr) { + int startIndex = baseJsStr.indexOf("{"); + int endIndex = baseJsStr.lastIndexOf("}"); + return baseJsStr.substring(startIndex, endIndex + 1); + } + + /** + * 构建获取base.js文件的URL + * + * @param bkSharedResUrl 共享资源基础路径 + * @param bkSharedBaseJsPath base.js文件路径 + * @return base.js文件对应的URL + */ + private String buildBaseJsFileUrl(String bkSharedResUrl, String bkSharedBaseJsPath) { + if (bkSharedResUrl.endsWith("/")) { + bkSharedResUrl = StringUtil.removeSuffix(bkSharedResUrl, "/"); + } + if (bkSharedBaseJsPath.startsWith("/")) { + return bkSharedResUrl + bkSharedBaseJsPath; + } + return bkSharedResUrl + "/" + bkSharedBaseJsPath; + } + + private BkPlatformInfo buildBkPlatformInfo(BkPlatformInfo defaultBkPlatformInfo, + TitleFooterDTO titleFooterDTO, + HelperInfo helperInfo) { + if (titleFooterDTO != null) { + Map titleFooterLanguageMap = titleFooterDTO.getTitleFooterLanguageMap(); + if (titleFooterLanguageMap != null) { + TitleFooter englishTitleFooter = getEnglishTitleFooter(titleFooterLanguageMap); + TitleFooter chineseTitleFooter = getChineseTitleFooter(titleFooterLanguageMap); + if (englishTitleFooter != null) { + defaultBkPlatformInfo.setNameEn(englishTitleFooter.getTitleHead()); + defaultBkPlatformInfo.setFooterInfoEn(englishTitleFooter.getFooterLink()); + defaultBkPlatformInfo.setFooterCopyright(englishTitleFooter.getFooterCopyRight()); + } + if (chineseTitleFooter != null) { + defaultBkPlatformInfo.setName(chineseTitleFooter.getTitleHead()); + defaultBkPlatformInfo.setFooterInfo(chineseTitleFooter.getFooterLink()); + defaultBkPlatformInfo.setFooterCopyright(chineseTitleFooter.getFooterCopyRight()); + } + } + } + if (helperInfo != null) { + String contactLink = helperInfo.getContactLink(); + defaultBkPlatformInfo.setHelperLink(contactLink); + } + return defaultBkPlatformInfo; + } + + private TitleFooter getEnglishTitleFooter(Map titleFooterLanguageMap) { + TitleFooter titleFooter = titleFooterLanguageMap.get(LocaleUtils.LANG_EN); + if (titleFooter == null) { + titleFooter = titleFooterLanguageMap.get(LocaleUtils.LANG_EN_US); + } + return titleFooter; + } + + private TitleFooter getChineseTitleFooter(Map titleFooterLanguageMap) { + TitleFooter titleFooter = titleFooterLanguageMap.get(LocaleUtils.LANG_ZH_CN); + if (titleFooter == null) { + titleFooter = titleFooterLanguageMap.get(LocaleUtils.LANG_ZH); + } + return titleFooter; + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/globalsetting/impl/GlobalSettingsServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/globalsetting/impl/GlobalSettingsServiceImpl.java new file mode 100644 index 0000000000..86e0b3012f --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/globalsetting/impl/GlobalSettingsServiceImpl.java @@ -0,0 +1,822 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.service.globalsetting.impl; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.tencent.bk.audit.annotations.ActionAuditRecord; +import com.tencent.bk.job.common.audit.constants.EventContentConstants; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.constant.JobConstants; +import com.tencent.bk.job.common.exception.InvalidParamException; +import com.tencent.bk.job.common.i18n.locale.LocaleUtils; +import com.tencent.bk.job.common.i18n.service.MessageI18nService; +import com.tencent.bk.job.common.iam.constant.ActionId; +import com.tencent.bk.job.common.notice.config.BkNoticeProperties; +import com.tencent.bk.job.common.util.JobContextUtil; +import com.tencent.bk.job.common.util.StringUtil; +import com.tencent.bk.job.common.util.date.DateUtils; +import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.common.util.toggle.ToggleEvaluateContext; +import com.tencent.bk.job.common.util.toggle.feature.FeatureIdConstants; +import com.tencent.bk.job.common.util.toggle.feature.FeatureToggle; +import com.tencent.bk.job.manage.api.common.constants.OSTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.globalsetting.GlobalSettingKeys; +import com.tencent.bk.job.manage.api.common.constants.globalsetting.RelatedUrlKeys; +import com.tencent.bk.job.manage.api.common.constants.globalsetting.RestrictModeEnum; +import com.tencent.bk.job.manage.api.common.constants.globalsetting.StorageUnitEnum; +import com.tencent.bk.job.manage.api.common.constants.notify.NotifyConsts; +import com.tencent.bk.job.manage.config.JobManageConfig; +import com.tencent.bk.job.manage.config.LocalFileConfigForManage; +import com.tencent.bk.job.manage.dao.globalsetting.GlobalSettingDAO; +import com.tencent.bk.job.manage.dao.notify.AvailableEsbChannelDAO; +import com.tencent.bk.job.manage.dao.notify.NotifyEsbChannelDAO; +import com.tencent.bk.job.manage.dao.notify.NotifyTemplateDAO; +import com.tencent.bk.job.manage.model.dto.GlobalSettingDTO; +import com.tencent.bk.job.manage.model.dto.converter.NotifyTemplateConverter; +import com.tencent.bk.job.manage.model.dto.globalsetting.HelperInfo; +import com.tencent.bk.job.manage.model.dto.globalsetting.TitleFooter; +import com.tencent.bk.job.manage.model.dto.globalsetting.TitleFooterDTO; +import com.tencent.bk.job.manage.model.dto.globalsetting.UploadFileRestrictDTO; +import com.tencent.bk.job.manage.model.dto.notify.AvailableEsbChannelDTO; +import com.tencent.bk.job.manage.model.dto.notify.NotifyEsbChannelDTO; +import com.tencent.bk.job.manage.model.dto.notify.NotifyTemplateDTO; +import com.tencent.bk.job.manage.model.web.request.globalsetting.AccountNameRule; +import com.tencent.bk.job.manage.model.web.request.globalsetting.AccountNameRulesReq; +import com.tencent.bk.job.manage.model.web.request.globalsetting.FileUploadSettingReq; +import com.tencent.bk.job.manage.model.web.request.globalsetting.HistoryExpireReq; +import com.tencent.bk.job.manage.model.web.request.notify.ChannelTemplatePreviewReq; +import com.tencent.bk.job.manage.model.web.request.notify.ChannelTemplateReq; +import com.tencent.bk.job.manage.model.web.request.notify.NotifyBlackUsersReq; +import com.tencent.bk.job.manage.model.web.request.notify.SetAvailableNotifyChannelReq; +import com.tencent.bk.job.manage.model.web.vo.globalsetting.AccountNameRuleVO; +import com.tencent.bk.job.manage.model.web.vo.globalsetting.AccountNameRulesWithDefaultVO; +import com.tencent.bk.job.manage.model.web.vo.globalsetting.FileUploadSettingVO; +import com.tencent.bk.job.manage.model.web.vo.globalsetting.NotifyChannelWithIconVO; +import com.tencent.bk.job.manage.model.web.vo.globalsetting.PlatformInfoVO; +import com.tencent.bk.job.manage.model.web.vo.globalsetting.PlatformInfoWithDefaultVO; +import com.tencent.bk.job.manage.model.web.vo.globalsetting.TitleFooterVO; +import com.tencent.bk.job.manage.model.web.vo.notify.ChannelTemplateDetailVO; +import com.tencent.bk.job.manage.model.web.vo.notify.ChannelTemplateDetailWithDefaultVO; +import com.tencent.bk.job.manage.model.web.vo.notify.ChannelTemplateStatusVO; +import com.tencent.bk.job.manage.model.web.vo.notify.NotifyBlackUserInfoVO; +import com.tencent.bk.job.manage.model.web.vo.notify.TemplateBasicInfo; +import com.tencent.bk.job.manage.model.web.vo.notify.UserVO; +import com.tencent.bk.job.manage.service.NotifyService; +import com.tencent.bk.job.manage.service.globalsetting.GlobalSettingsService; +import com.tencent.bk.job.manage.service.impl.notify.NotifySendService; +import com.tencent.bk.job.manage.service.impl.notify.NotifyUserService; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.info.BuildProperties; +import org.springframework.stereotype.Service; + +import java.time.LocalDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +@Slf4j +@Service +public class GlobalSettingsServiceImpl implements GlobalSettingsService { + + private static final Pattern PATTERN = Pattern.compile("^([.0-9]+)([a-zA-Z]{0,2})$"); + private static final String STRING_TPL_KEY_CURRENT_VERSION = "current_ver"; + private static final String STRING_TPL_KEY_CURRENT_YEAR = "current_year"; + private final NotifyEsbChannelDAO notifyEsbChannelDAO; + private final AvailableEsbChannelDAO availableEsbChannelDAO; + private final NotifyService notifyService; + private final NotifySendService notifySendService; + private final NotifyUserService notifyUserService; + private final GlobalSettingDAO globalSettingDAO; + private final NotifyTemplateDAO notifyTemplateDAO; + private final MessageI18nService i18nService; + private final JobManageConfig jobManageConfig; + private final LocalFileConfigForManage localFileConfigForManage; + private final BkNoticeProperties bkNoticeProperties; + private final NotifyTemplateConverter notifyTemplateConverter; + private final BuildProperties buildProperties; + @Value("${job.manage.upload.filesize.max:5GB}") + private String configedMaxFileSize; + + + public GlobalSettingsServiceImpl(NotifyEsbChannelDAO notifyEsbChannelDAO, + AvailableEsbChannelDAO availableEsbChannelDAO, + NotifyService notifyService, + NotifySendService notifySendService, + NotifyUserService notifyUserService, + GlobalSettingDAO globalSettingDAO, + NotifyTemplateDAO notifyTemplateDAO, + MessageI18nService i18nService, + JobManageConfig jobManageConfig, + LocalFileConfigForManage localFileConfigForManage, + BkNoticeProperties bkNoticeProperties, + NotifyTemplateConverter notifyTemplateConverter, + BuildProperties buildProperties) { + this.notifyEsbChannelDAO = notifyEsbChannelDAO; + this.availableEsbChannelDAO = availableEsbChannelDAO; + this.notifyService = notifyService; + this.notifySendService = notifySendService; + this.notifyUserService = notifyUserService; + this.globalSettingDAO = globalSettingDAO; + this.notifyTemplateDAO = notifyTemplateDAO; + this.i18nService = i18nService; + this.jobManageConfig = jobManageConfig; + this.localFileConfigForManage = localFileConfigForManage; + this.bkNoticeProperties = bkNoticeProperties; + this.notifyTemplateConverter = notifyTemplateConverter; + this.buildProperties = buildProperties; + } + + private static String removeSuffixBackSlash(String rawStr) { + if (rawStr == null) { + return null; + } + while (rawStr.endsWith("/")) { + rawStr = rawStr.substring(0, rawStr.length() - 1); + } + return rawStr; + } + + @Override + public Boolean isNotifyChannelConfiged() { + GlobalSettingDTO globalSettingDTO = globalSettingDAO.getGlobalSetting( + GlobalSettingKeys.KEY_NOTIFY_CHANNEL_CONFIGED); + return globalSettingDTO != null && globalSettingDTO.getValue().toLowerCase().equals("true"); + } + + @Override + public Boolean setNotifyChannelConfiged() { + GlobalSettingDTO globalSettingDTO = globalSettingDAO.getGlobalSetting( + GlobalSettingKeys.KEY_NOTIFY_CHANNEL_CONFIGED); + if (globalSettingDTO == null) { + globalSettingDTO = new GlobalSettingDTO(GlobalSettingKeys.KEY_NOTIFY_CHANNEL_CONFIGED, + "true", "whether available notify channels are configed"); + return 1 == globalSettingDAO.insertGlobalSetting(globalSettingDTO); + } else if (!globalSettingDTO.getValue().toLowerCase().equals("true")) { + globalSettingDTO.setValue("true"); + return 1 == globalSettingDAO.updateGlobalSetting(globalSettingDTO); + } else { + return true; + } + } + + @Override + public List listNotifyChannel(String username) { + List allNotifyChannelList = + notifyEsbChannelDAO.listNotifyEsbChannel().stream() + .filter(NotifyEsbChannelDTO::isActive).collect(Collectors.toList()); + List availableNotifyChannelList = + availableEsbChannelDAO.listAvailableEsbChannel(); + Set availableNotifyChannelTypeSet = + availableNotifyChannelList.stream().map(AvailableEsbChannelDTO::getType).collect(Collectors.toSet()); + return allNotifyChannelList.stream().map(it -> { + String icon = it.getIcon(); + String prefix = "data:image/png;base64,"; + if (!icon.startsWith("data:image")) { + icon = prefix + icon; + } + return new NotifyChannelWithIconVO( + it.getType(), + it.getLabel(), + icon, + availableNotifyChannelTypeSet.contains(it.getType()) + ); + }).collect(Collectors.toList()); + } + + @Override + @ActionAuditRecord( + actionId = ActionId.GLOBAL_SETTINGS, + content = EventContentConstants.EDIT_GLOBAL_SETTINGS + ) + public Integer setAvailableNotifyChannel(String username, SetAvailableNotifyChannelReq req) { + return notifyService.setAvailableNotifyChannel(username, req); + } + + @Override + public List listUsers(String username, String prefixStr, Long offset, Long limit) { + //这里就是要选择人来添加黑名单，故不排除已在黑名单内的人 + return notifyUserService.listUsers(prefixStr, offset, limit, false); + } + + @Override + public List listNotifyBlackUsers(String username, Integer start, Integer pageSize) { + return notifyUserService.listNotifyBlackUsers(start, pageSize); + } + + @Override + @ActionAuditRecord( + actionId = ActionId.GLOBAL_SETTINGS, + content = EventContentConstants.EDIT_GLOBAL_SETTINGS + ) + public List saveNotifyBlackUsers(String username, NotifyBlackUsersReq req) { + return notifyUserService.saveNotifyBlackUsers(username, req); + } + + @Override + public Long getHistoryExpireTime(String username) { + GlobalSettingDTO globalSettingDTO = globalSettingDAO.getGlobalSetting( + GlobalSettingKeys.KEY_HISTORY_EXPIRE_DAYS); + if (globalSettingDTO == null) { + globalSettingDTO = new GlobalSettingDTO(GlobalSettingKeys.KEY_HISTORY_EXPIRE_DAYS, + "60", "执行记录默认保存天数" + + "(default history expire days)"); + globalSettingDAO.insertGlobalSetting(globalSettingDTO); + } + return Long.parseLong(globalSettingDTO.getValue()); + } + + @Override + @ActionAuditRecord( + actionId = ActionId.GLOBAL_SETTINGS, + content = EventContentConstants.EDIT_GLOBAL_SETTINGS + ) + public Integer setHistoryExpireTime(String username, HistoryExpireReq req) { + Long days = req.getDays(); + if (days == null || days <= 0) { + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME_AND_REASON, + new String[]{"days", "days must be positive"}); + } + GlobalSettingDTO globalSettingDTO = new GlobalSettingDTO(GlobalSettingKeys.KEY_HISTORY_EXPIRE_DAYS, + days.toString(), String.format("执行记录保存天数(history expire days):%s,%s", username, + DateUtils.defaultLocalDateTime(LocalDateTime.now()))); + return globalSettingDAO.updateGlobalSetting(globalSettingDTO); + } + + @Override + public AccountNameRule getCurrentAccountNameRule(OSTypeEnum osType) { + String normalLang = LocaleUtils.getNormalLang(JobContextUtil.getUserLang()); + GlobalSettingDTO currentNameRulesDTO; + String globalSettingKey; + if (normalLang.equals(LocaleUtils.LANG_EN) || normalLang.equals(LocaleUtils.LANG_EN_US)) { + //英文环境 + globalSettingKey = GlobalSettingKeys.KEY_CURRENT_NAME_RULES_EN; + } else { + //中文环境 + globalSettingKey = GlobalSettingKeys.KEY_CURRENT_NAME_RULES; + } + List currentNameRules; + currentNameRulesDTO = globalSettingDAO.getGlobalSetting(globalSettingKey); + if (currentNameRulesDTO != null) { + currentNameRules = JsonUtils.fromJson(currentNameRulesDTO.getValue(), + new TypeReference>() { + }); + for (AccountNameRule rule : currentNameRules) { + if (rule.getOsType() == osType) { + return rule; + } + } + } else { + log.warn("Cannot find currentNameRules in language:{}, please check key:{} in database table job_manage" + + ".global_setting", normalLang, globalSettingKey); + } + return null; + } + + // 默认账号命名规则：Linux + private static final String DEFAULT_ACCOUNT_NAME_RULE_LINUX = "^[a-z_][a-z0-9_-]{1,31}$"; + // 默认账号命名规则：Windows + private static final String DEFAULT_ACCOUNT_NAME_RULE_WINDOWS = "^[a-æA-Æ0-9-]{1,32}$"; + // 默认账号命名规则：Database + private static final String DEFAULT_ACCOUNT_NAME_RULE_DATABASE = "^[a-zA-Z0-9\\.\\-\\_]{1,16}$"; + + private List getDefaultNameRules(Locale locale) { + List defaultNameRules = new ArrayList<>(); + defaultNameRules.add(new AccountNameRule(OSTypeEnum.LINUX, DEFAULT_ACCOUNT_NAME_RULE_LINUX, + i18nService.getI18n(locale, "job.manage.globalsettings.defaultNameRules.description.linux"))); + defaultNameRules.add(new AccountNameRule(OSTypeEnum.WINDOWS, DEFAULT_ACCOUNT_NAME_RULE_WINDOWS, + i18nService.getI18n(locale, "job.manage.globalsettings.defaultNameRules.description.windows"))); + defaultNameRules.add(new AccountNameRule(OSTypeEnum.DATABASE, DEFAULT_ACCOUNT_NAME_RULE_DATABASE, + i18nService.getI18n(locale, "job.manage.globalsettings.defaultNameRules.description.database"))); + return defaultNameRules; + } + + @Override + public AccountNameRulesWithDefaultVO getAccountNameRules() { + String normalLang = LocaleUtils.getNormalLang(JobContextUtil.getUserLang()); + List defaultNameRules; + GlobalSettingDTO currentNameRulesDTO; + if (normalLang.equals(LocaleUtils.LANG_EN) || normalLang.equals(LocaleUtils.LANG_EN_US)) { + //英文环境 + defaultNameRules = getDefaultNameRules(Locale.ENGLISH); + currentNameRulesDTO = globalSettingDAO.getGlobalSetting( + GlobalSettingKeys.KEY_CURRENT_NAME_RULES_EN); + } else { + //中文环境 + defaultNameRules = getDefaultNameRules(Locale.CHINA); + currentNameRulesDTO = globalSettingDAO.getGlobalSetting( + GlobalSettingKeys.KEY_CURRENT_NAME_RULES); + } + List currentNameRules; + if (currentNameRulesDTO != null) { + currentNameRules = JsonUtils.fromJson(currentNameRulesDTO.getValue(), + new TypeReference>() { + }); + } else { + currentNameRules = defaultNameRules; + } + return new AccountNameRulesWithDefaultVO(currentNameRules.stream() + .map(it -> new AccountNameRuleVO(it.getOsType(), it.getExpression(), it.getDescription())) + .collect(Collectors.toList()), + defaultNameRules.stream() + .map(it -> new AccountNameRuleVO(it.getOsType(), it.getExpression(), it.getDescription())) + .collect(Collectors.toList()) + ); + } + + @Override + @ActionAuditRecord( + actionId = ActionId.GLOBAL_SETTINGS, + content = EventContentConstants.EDIT_GLOBAL_SETTINGS + ) + public Boolean setAccountNameRules(String username, AccountNameRulesReq req) { + String normalLang = LocaleUtils.getNormalLang(JobContextUtil.getUserLang()); + String currentNameRulesKey = GlobalSettingKeys.KEY_CURRENT_NAME_RULES; + if (normalLang.equals(LocaleUtils.LANG_EN) || normalLang.equals(LocaleUtils.LANG_EN_US)) { + //英文环境 + currentNameRulesKey = GlobalSettingKeys.KEY_CURRENT_NAME_RULES_EN; + } + GlobalSettingDTO currentNameRulesDTO = globalSettingDAO.getGlobalSetting(currentNameRulesKey); + GlobalSettingDTO inputNameRulesDTO = new GlobalSettingDTO(currentNameRulesKey, + JsonUtils.toJson(req.getRules()), String.format("Updated by %s at %s", username, + DateUtils.defaultLocalDateTime(LocalDateTime.now()))); + if (currentNameRulesDTO == null) { + return globalSettingDAO.insertGlobalSetting(inputNameRulesDTO) == 1; + } else { + return globalSettingDAO.updateGlobalSetting(inputNameRulesDTO) == 1; + } + } + + private Pair parseFileSize(String str) { + Matcher matcher = PATTERN.matcher(str); + if (!matcher.matches()) { + return null; + } + float amount = Float.parseFloat(matcher.group(1)); + String unit = matcher.group(2); + return Pair.of(amount, unit); + } + + private FileUploadSettingVO getFileUploadSettingsFromStr(String str) { + Pair configedValue = parseFileSize(str); + if (configedValue == null) return null; + return new FileUploadSettingVO(configedValue.getLeft(), configedValue.getRight(), null, null); + } + + private FileUploadSettingVO getConfigedFileUploadSettings() { + log.debug("configedMaxFileSize=" + configedMaxFileSize); + return getFileUploadSettingsFromStr(configedMaxFileSize); + } + + @Override + @ActionAuditRecord( + actionId = ActionId.GLOBAL_SETTINGS, + content = EventContentConstants.EDIT_GLOBAL_SETTINGS + ) + public Boolean saveFileUploadSettings(String username, FileUploadSettingReq req) { + Float uploadMaxSize = req.getAmount(); + StorageUnitEnum unit = req.getUnit(); + Integer restrictMode = req.getRestrictMode(); + List suffixList = req.getSuffixList(); + if (unit == null) { + unit = StorageUnitEnum.B; + } + if (uploadMaxSize <= 0) { + uploadMaxSize = 5.0f; + unit = StorageUnitEnum.GB; + } + if (restrictMode == null) { + restrictMode = RestrictModeEnum.UNLIMITED.getType(); + } + + if (CollectionUtils.isNotEmpty(suffixList)) { + // 去重 + suffixList = suffixList.stream().distinct().collect(Collectors.toList()); + } + GlobalSettingDTO fileUploadSettingDTO = globalSettingDAO.getGlobalSetting( + GlobalSettingKeys.KEY_FILE_UPLOAD_SETTING); + if (fileUploadSettingDTO == null) { + fileUploadSettingDTO = new GlobalSettingDTO(); + fileUploadSettingDTO.setDescription("setting of upload file"); + } + fileUploadSettingDTO.setKey(GlobalSettingKeys.KEY_FILE_UPLOAD_SETTING); + fileUploadSettingDTO.setValue(JsonUtils.toJson( + new UploadFileRestrictDTO( + uploadMaxSize.toString() + unit.name() + , restrictMode + , suffixList))); + int affectedRows = globalSettingDAO.upsertGlobalSetting(fileUploadSettingDTO); + return affectedRows > 0; + } + + @Override + public FileUploadSettingVO getFileUploadSettings() { + GlobalSettingDTO fileUploadSettingDTO = globalSettingDAO.getGlobalSetting( + GlobalSettingKeys.KEY_FILE_UPLOAD_SETTING); + FileUploadSettingVO fileUploadSettingVO; + if (fileUploadSettingDTO == null) { + fileUploadSettingVO = getConfigedFileUploadSettings(); + } else { + String settingStr = fileUploadSettingDTO.getValue(); + UploadFileRestrictDTO uploadFileRestrictDTO = JsonUtils.fromJson(settingStr, + new TypeReference() { + }); + fileUploadSettingVO = getFileUploadSettingsFromStr(uploadFileRestrictDTO.getMaxSize()); + if (fileUploadSettingVO == null) { + fileUploadSettingVO = new FileUploadSettingVO(); + } + fileUploadSettingVO.setRestrictMode(uploadFileRestrictDTO.getRestrictMode()); + fileUploadSettingVO.setSuffixList(uploadFileRestrictDTO.getSuffixList()); + } + return fileUploadSettingVO; + } + + + @Override + @ActionAuditRecord( + actionId = ActionId.GLOBAL_SETTINGS, + content = EventContentConstants.EDIT_GLOBAL_SETTINGS + ) + public Integer saveChannelTemplate(String username, ChannelTemplateReq req) { + if (StringUtils.isBlank(req.getChannelCode()) || StringUtils.isBlank(req.getMessageTypeCode())) { + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME_AND_REASON, + new String[]{"channelCode", "channelCode cannot be blank"}); + } + NotifyTemplateDTO notifyTemplateDTO = notifyTemplateDAO.getNotifyTemplate( + req.getChannelCode(), + req.getMessageTypeCode(), + false + ); + String normalLang = LocaleUtils.getNormalLang(JobContextUtil.getUserLang()); + if (notifyTemplateDTO == null) { + notifyTemplateDTO = new NotifyTemplateDTO(null, req.getMessageTypeCode(), req.getMessageTypeCode(), + req.getChannelCode(), req.getTitle(), req.getContent(), req.getTitle(), req.getContent(), false, + username, System.currentTimeMillis(), username, System.currentTimeMillis()); + return notifyTemplateDAO.insertNotifyTemplate(notifyTemplateDTO); + } else { + if (normalLang.equals(LocaleUtils.LANG_EN) || normalLang.equals(LocaleUtils.LANG_EN_US)) { + notifyTemplateDTO.setTitleEn(req.getTitle()); + notifyTemplateDTO.setContentEn(req.getContent()); + } else { + notifyTemplateDTO.setTitle(req.getTitle()); + notifyTemplateDTO.setContent(req.getContent()); + } + notifyTemplateDTO.setLastModifyUser(username); + notifyTemplateDTO.setLastModifyTime(System.currentTimeMillis()); + return notifyTemplateDAO.updateNotifyTemplateById(notifyTemplateDTO); + } + } + + @Override + public Integer sendChannelTemplate(String username, ChannelTemplatePreviewReq req) { + List receiverList = StringUtil.strToList(req.getReceiverStr(), String.class, + NotifyConsts.SEPERATOR_COMMA); + Set receiverSet = new HashSet<>(receiverList); + notifySendService.sendUserChannelNotify( + null, + receiverSet, + req.getChannelCode(), + req.getTitle(), + req.getContent() + ); + return receiverSet.size(); + } + + @Override + public ChannelTemplateDetailWithDefaultVO getChannelTemplateDetail(String username, String channelCode, + String messageTypeCode) { + NotifyTemplateDTO currentNotifyTemplateDTO = notifyTemplateDAO.getNotifyTemplate( + channelCode, + messageTypeCode, + false + ); + NotifyTemplateDTO defaultNotifyTemplateDTO = notifyTemplateDAO.getNotifyTemplate( + channelCode, + messageTypeCode, + true + ); + // 渠道下无默认模板，则为新增渠道，使用通用模板 + if (defaultNotifyTemplateDTO == null) { + defaultNotifyTemplateDTO = notifyTemplateDAO.getNotifyTemplate( + NotifyConsts.NOTIFY_CHANNEL_CODE_COMMON, + messageTypeCode, + false + ); + if (defaultNotifyTemplateDTO == null) { + defaultNotifyTemplateDTO = notifyTemplateDAO.getNotifyTemplate( + NotifyConsts.NOTIFY_CHANNEL_CODE_COMMON, + messageTypeCode, + true + ); + } + if (defaultNotifyTemplateDTO != null) { + defaultNotifyTemplateDTO.setChannel(channelCode); + } else { + log.warn("common template of messageType={} not configed", messageTypeCode); + } + } + ChannelTemplateDetailVO currentChannelTemplateVO = + notifyTemplateConverter.convertToChannelTemplateDetailVO(currentNotifyTemplateDTO); + ChannelTemplateDetailVO defaultChannelTemplateVO = + notifyTemplateConverter.convertToChannelTemplateDetailVO(defaultNotifyTemplateDTO); + // 渠道下未配置模板则使用该渠道默认模板 + if (currentChannelTemplateVO == null) { + currentChannelTemplateVO = defaultChannelTemplateVO; + } + // 渠道下已配置模板信息不完整则使用默认模板内容填充 + if (currentChannelTemplateVO != null && currentChannelTemplateVO.getTitle() == null) { + if (defaultChannelTemplateVO != null) { + currentChannelTemplateVO.setTitle(defaultChannelTemplateVO.getTitle()); + } + } + if (currentChannelTemplateVO != null && currentChannelTemplateVO.getContent() == null) { + if (defaultChannelTemplateVO != null) { + currentChannelTemplateVO.setContent(defaultChannelTemplateVO.getContent()); + } + } + return new ChannelTemplateDetailWithDefaultVO(currentChannelTemplateVO, defaultChannelTemplateVO); + } + + @Override + public List listChannelTemplateStatus(String username) { + List resultList = new ArrayList<>(); + List messageCodeList = Arrays.asList( + NotifyConsts.NOTIFY_TEMPLATE_CODE_CONFIRMATION, + NotifyConsts.NOTIFY_TEMPLATE_CODE_EXECUTE_SUCCESS, + NotifyConsts.NOTIFY_TEMPLATE_CODE_EXECUTE_FAILURE, + NotifyConsts.NOTIFY_TEMPLATE_CODE_BEFORE_CRON_JOB_EXECUTE, + NotifyConsts.NOTIFY_TEMPLATE_CODE_BEFORE_CRON_JOB_END, + NotifyConsts.NOTIFY_TEMPLATE_CODE_CRON_EXECUTE_FAILED + ); + List notifyChannelWithIconVOList = listNotifyChannel(username); + notifyChannelWithIconVOList.forEach(notifyChannelWithIconVO -> { + String channelCode = notifyChannelWithIconVO.getCode(); + List configStatusList = new ArrayList<>(); + messageCodeList.forEach(messageCode -> { + Boolean configStatus = notifyTemplateDAO.existsNotifyTemplate(channelCode, messageCode, + false); + configStatusList.add(new TemplateBasicInfo(messageCode, + i18nService.getI18n(NotifyConsts.NOTIFY_TEMPLATE_NAME_PREFIX + messageCode), configStatus)); + }); + resultList.add(new ChannelTemplateStatusVO( + channelCode, + notifyChannelWithIconVO.getName(), + notifyChannelWithIconVO.getIcon(), + notifyChannelWithIconVO.getIsActive(), + configStatusList + )); + }); + return resultList; + } + + @Override + public String getDocJobRootUrl() { + String docCenterBaseUrl = getDocCenterBaseUrl(); + StringBuilder sb = new StringBuilder(docCenterBaseUrl); + sb.append("/markdown/"); + if (JobContextUtil.isEnglishLocale()) { + sb.append("EN"); + } else { + sb.append("ZH"); + } + sb.append("/JOB/"); + sb.append(getTwoDigitVersion()); + return sb.toString(); + } + + private String getTwoDigitVersion() { + String completeVersion = buildProperties.getVersion(); + String[] versionParts = completeVersion.split("\\."); + return versionParts[0] + "." + versionParts[1]; + } + + @Override + public String getDocCenterBaseUrl() { + String url; + if (org.apache.commons.lang3.StringUtils.isNotBlank(jobManageConfig.getBkDocRoot())) { + url = removeSuffixBackSlash(jobManageConfig.getBkDocRoot()); + } else { + String jobEdition = jobManageConfig.getJobEdition(); + if (jobEdition.toLowerCase().equals("ee")) { + // 企业版 + url = removeSuffixBackSlash(jobManageConfig.getPaasServerUrl()) + "/o/bk_docs_center"; + } else { + // 社区版 + url = removeSuffixBackSlash(jobManageConfig.getBkCERoot()) + "/docs"; + } + } + return url; + } + + private String getFeedBackRootUrl() { + String url; + if (StringUtils.isNotBlank(jobManageConfig.getBkFeedBackRoot())) { + url = removeSuffixBackSlash(jobManageConfig.getBkFeedBackRoot()); + } else { + // 企业版&社区版 + url = jobManageConfig.getBkCERoot() + "/s-mart/community"; + } + return url; + } + + private String getBkSharedResBaseJsUrl() { + String bkSharedResUrl = jobManageConfig.getBkSharedResUrl(); + if (StringUtils.isNotBlank(bkSharedResUrl)) { + // bkSharedResUrl配置了有效值才生效 + return jobManageConfig.getBkSharedResUrl() + jobManageConfig.getBkSharedBaseJsPath(); + } + return null; + } + + private String getNodemanRootUrl() { + String url = jobManageConfig.getNodemanServerUrl(); + if (StringUtils.isBlank(url)) { + url = removeSuffixBackSlash(jobManageConfig.getPaasServerUrl()) + jobManageConfig.getPaasNodemanPath(); + } + return url; + } + + @Override + public Map getRelatedSystemUrls(String username) { + Map urlMap = new HashMap<>(); + urlMap.put(RelatedUrlKeys.KEY_BK_CMDB_ROOT_URL, jobManageConfig.getCmdbServerUrl()); + urlMap.put(RelatedUrlKeys.KEY_BK_CMDB_APP_INDEX_URL, + removeSuffixBackSlash(jobManageConfig.getCmdbServerUrl()) + jobManageConfig.getCmdbAppIndexPath()); + urlMap.put(RelatedUrlKeys.KEY_BK_NODEMAN_ROOT_URL, getNodemanRootUrl()); + urlMap.put(RelatedUrlKeys.KEY_BK_DOC_CENTER_ROOT_URL, getDocCenterBaseUrl()); + urlMap.put(RelatedUrlKeys.KEY_BK_DOC_JOB_ROOT_URL, getDocJobRootUrl()); + urlMap.put(RelatedUrlKeys.KEY_BK_FEED_BACK_ROOT_URL, getFeedBackRootUrl()); + urlMap.put(RelatedUrlKeys.KEY_BK_SHARED_RES_BASE_JS_URL, getBkSharedResBaseJsUrl()); + return urlMap; + } + + private void addFileUploadConfig(Map configMap) { + FileUploadSettingVO fileUploadSettingVO = getFileUploadSettings(); + if (fileUploadSettingVO != null) { + configMap.put(GlobalSettingKeys.KEY_FILE_UPLOAD_SETTING, fileUploadSettingVO); + } + } + + private void addEnableFeatureFileManageConfig(Map configMap) { + configMap.put(GlobalSettingKeys.KEY_ENABLE_FEATURE_FILE_MANAGE, + FeatureToggle.checkFeature(FeatureIdConstants.FEATURE_FILE_MANAGE, + ToggleEvaluateContext.EMPTY)); + } + + private void addEnableUploadToArtifactoryConfig(Map configMap) { + configMap.put( + GlobalSettingKeys.KEY_ENABLE_UPLOAD_TO_ARTIFACTORY, + JobConstants.FILE_STORAGE_BACKEND_ARTIFACTORY.equals(localFileConfigForManage.getStorageBackend()) + ); + } + + private void addEnableBkNoticeConfig(Map configMap) { + configMap.put( + GlobalSettingKeys.KEY_ENABLE_BK_NOTICE, + bkNoticeProperties.isEnabled() && bkNoticeRegisteredSuccess() + ); + } + + private boolean bkNoticeRegisteredSuccess() { + GlobalSettingDTO globalSettingDTO = + globalSettingDAO.getGlobalSetting(GlobalSettingKeys.KEY_BK_NOTICE_REGISTERED_SUCCESS); + return globalSettingDTO != null && "true".equals(globalSettingDTO.getValue().toLowerCase()); + } + + @Override + public Map getJobConfig(String username) { + Map configMap = new HashMap<>(); + addFileUploadConfig(configMap); + addEnableFeatureFileManageConfig(configMap); + addEnableUploadToArtifactoryConfig(configMap); + addEnableBkNoticeConfig(configMap); + return configMap; + } + + @Override + public PlatformInfoVO getRenderedPlatformInfo() { + TitleFooterVO titleFooterVO = getRenderedTitleFooter(); + PlatformInfoVO platformInfoVO = new PlatformInfoVO(titleFooterVO); + HelperInfo helperInfo = getHelperInfo(); + if (helperInfo != null) { + platformInfoVO.setHelperContactLink(helperInfo.getContactLink()); + } + return platformInfoVO; + } + + public PlatformInfoVO getPlatformInfoVO() { + TitleFooterVO titleFooterVO = getTitleFooter(); + HelperInfo helperInfo = getHelperInfo(); + return new PlatformInfoVO(titleFooterVO, helperInfo == null ? "" : helperInfo.getContactLink()); + } + + public TitleFooterVO getRenderedTitleFooter() { + TitleFooterVO titleFooterVO = getTitleFooter(); + // 渲染版本号 + Map valuesMap = new HashMap<>(); + String pattern = "(\\{\\{(.*?)\\}\\})"; + if (buildProperties != null) { + valuesMap.put(STRING_TPL_KEY_CURRENT_VERSION, "V" + buildProperties.getVersion()); + } + valuesMap.put(STRING_TPL_KEY_CURRENT_YEAR, DateUtils.getCurrentDateStr("yyyy")); + titleFooterVO.setFooterCopyRight( + StringUtil.replaceByRegex(titleFooterVO.getFooterCopyRight(), pattern, valuesMap) + ); + titleFooterVO.setFooterLink( + StringUtil.replaceByRegex(titleFooterVO.getFooterLink(), pattern, valuesMap) + ); + return titleFooterVO; + } + + private TitleFooterVO getTitleFooter() { + GlobalSettingDTO titleFooterSettingDTO = globalSettingDAO.getGlobalSetting( + GlobalSettingKeys.KEY_TITLE_FOOTER); + if (titleFooterSettingDTO == null) { + log.warn("Default titleFooter not configured, use system default, plz contact admin to set"); + return getDefaultTitleFooterVO(); + } + TitleFooterDTO titleFooterDTO = JsonUtils.fromJson(titleFooterSettingDTO.getValue(), + new TypeReference() { + }); + String normalLang = LocaleUtils.getNormalLang(JobContextUtil.getUserLang()); + TitleFooter titleFooter = titleFooterDTO.getTitleFooterLanguageMap().get(normalLang); + return titleFooter == null ? getDefaultTitleFooterVO() : + new TitleFooterVO(titleFooter.getTitleHead(), titleFooter.getTitleSeparator(), + titleFooter.getFooterLink(), titleFooter.getFooterCopyRight()); + } + + private PlatformInfoVO getDefaultPlatformInfoVO() { + return new PlatformInfoVO(getDefaultTitleFooterVO()); + } + + private TitleFooterVO getDefaultTitleFooterVO() { + String jobEdition = jobManageConfig.getJobEdition(); + if (jobEdition.toLowerCase().equals("ee")) { + return getEEDefaultTitleFooterVO(); + } else { + return getCEDefaultTitleFooterVO(); + } + } + + private TitleFooterVO getCEDefaultTitleFooterVO() { + return getEEDefaultTitleFooterVO(); + } + + private TitleFooterVO getEEDefaultTitleFooterVO() { + return new TitleFooterVO( + i18nService.getI18n("job.manage.globalsettings.defaultTitleHead"), + "|", + i18nService.getI18n("job.manage.globalsettings.ee.footerLink").replace("{PAAS_SERVER_URL}", + jobManageConfig.getPaasServerUrl()), + i18nService.getI18n("job.manage.globalsettings.ee.footerCopyRight") + ); + } + + + @Override + public PlatformInfoWithDefaultVO getPlatformInfoWithDefault(String username) { + return new PlatformInfoWithDefaultVO(getPlatformInfoVO(), getDefaultPlatformInfoVO()); + } + + private HelperInfo getHelperInfo() { + GlobalSettingDTO globalSetting = globalSettingDAO.getGlobalSetting( + GlobalSettingKeys.KEY_BK_HELPER); + if (globalSetting == null || StringUtils.isEmpty(globalSetting.getValue())) { + return null; + } + return JsonUtils.fromJson(globalSetting.getValue(), HelperInfo.class); + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/BizHostService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/BizHostService.java new file mode 100644 index 0000000000..89c119efd7 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/BizHostService.java @@ -0,0 +1,162 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.service.host; + +import com.tencent.bk.job.common.model.PageData; +import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; +import com.tencent.bk.job.manage.model.query.HostQuery; + +import java.util.Collection; +import java.util.List; + +/** + * 业务主机相关服务 + */ +public interface BizHostService { + + /** + * 根据 hostId 集合查询主机信息 + * + * @param hostIds 主机 ID集合 + * @return 主机信息列表 + */ + List getHostsByHostIds(Collection hostIds); + + /** + * 根据IP地址集合查询主机信息 + * + * @param ips 主机 IP地址集合 + * @return 主机信息列表 + */ + List getHostsByIps(Collection ips); + + /** + * 根据CloudIP地址集合查询主机信息 + * + * @param cloudIps 主机 CloudIP地址集合 + * @return 主机信息列表 + */ + List getHostsByCloudIps(Collection cloudIps); + + /** + * 根据Ipv6地址集合查询主机信息 + * + * @param ipv6s 主机 Ipv6地址集合 + * @return 主机信息列表 + */ + List getHostsByIpv6s(Collection ipv6s); + + /** + * 根据主机名称集合查询主机信息 + * + * @param hostNames 主机名称集合 + * @return 主机信息列表 + */ + List getHostsByHostNames(Collection hostNames); + + /** + * 根据 bizId、hostId集合查询主机信息 + * + * @param bizIds 业务 ID集合 + * @param hostIds 主机 ID集合 + * @return 主机信息列表 + */ + List getHostsByBizAndHostIds(Collection bizIds, + Collection hostIds); + + /** + * 根据 bizIds 过滤出在这些业务下的hostIds + * + * @param bizIds 业务 ID集合 + * @param hostIds 主机 ID集合 + * @return 主机ID列表 + */ + List filterHostIdsByBiz(Collection bizIds, + Collection hostIds); + + /** + * 根据 bizId、ip集合查询主机信息 + * + * @param bizIds 业务 ID集合 + * @param ips 主机IP地址集合 + * @return 主机信息列表 + */ + List getHostsByBizAndIps(Collection bizIds, + Collection ips); + + /** + * 根据 bizId、cloudIp集合查询主机信息 + * + * @param bizIds 业务 ID集合 + * @param cloudIps 主机CloudIP地址集合 + * @return 主机信息列表 + */ + List getHostsByBizAndCloudIps(Collection bizIds, + Collection cloudIps); + + /** + * 根据 bizId、ipv6集合查询主机信息 + * + * @param bizIds 业务 ID集合 + * @param ipv6s 主机Ipv6地址集合 + * @return 主机信息列表 + */ + List getHostsByBizAndIpv6s(Collection bizIds, + Collection ipv6s); + + /** + * 根据 bizId、hostName集合查询主机信息 + * + * @param bizIds 业务 ID集合 + * @param hostNames 主机名称集合 + * @return 主机信息列表 + */ + List getHostsByBizAndHostNames(Collection bizIds, + Collection hostNames); + + /** + * 根据条件查询主机ID + * + * @param hostQuery 主机查询条件 + * @return 主机列表 + */ + PageData pageListHostId(HostQuery hostQuery); + + /** + * 根据条件查询主机 + * + * @param hostQuery 主机查询条件 + * @return 主机列表 + */ + PageData pageListHost(HostQuery hostQuery); + + /** + * 根据 moduleId 集合查询主机信息 + * + * @param moduleIds 模块 ID集合 + * @return 主机信息列表 + */ + List getHostsByModuleIds(Collection moduleIds); +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/BizTopoHostService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/BizTopoHostService.java new file mode 100644 index 0000000000..589f0111fc --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/BizTopoHostService.java @@ -0,0 +1,41 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.service.host; + +import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; +import com.tencent.bk.job.manage.model.web.request.chooser.host.BizTopoNode; + +import java.util.List; + +/** + * 业务拓扑相关的主机服务，根据拓扑节点查主机等 + */ +public interface BizTopoHostService { + + List listHostByNode(Long bizId, BizTopoNode node); + + List listHostByNodes(Long bizId, List nodes); + +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/HostDetailService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/HostDetailService.java new file mode 100644 index 0000000000..ade7aae142 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/HostDetailService.java @@ -0,0 +1,55 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.service.host; + +import com.tencent.bk.job.common.model.dto.AppResourceScope; +import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; + +import java.util.Collection; +import java.util.List; + +/** + * 主机详情服务，查出的主机信息包含Agent状态，云区域名称，云厂商等详情信息 + */ +public interface HostDetailService { + + List listHostDetails(AppResourceScope appResourceScope, Collection hostIds); + + /** + * 为主机填充云区域、云厂商、系统类型名称等信息 + * + * @param hostList 主机列表 + */ + void fillDetailForApplicationHosts(List hostList); + + /** + * 为主机填充云区域、云厂商、系统类型名称等信息 + * + * @param hostList 主机列表 + */ + void fillDetailForHosts(List hostList); + +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/HostService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/HostService.java new file mode 100644 index 0000000000..85476f9135 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/HostService.java @@ -0,0 +1,177 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.service.host; + +import com.tencent.bk.job.common.cc.model.InstanceTopologyDTO; +import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; +import com.tencent.bk.job.common.model.dto.BasicHostDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.common.model.dto.HostSimpleDTO; +import com.tencent.bk.job.manage.model.inner.ServiceListAppHostResultDTO; +import org.apache.commons.lang3.tuple.Pair; + +import java.util.Collection; +import java.util.List; +import java.util.Map; + +/** + * 主机、topo相关服务 + */ +public interface HostService { + + List getHostsByAppId(Long appId); + + /** + * 批量新增主机 + * + * @param insertList 主机信息 + * @return 成功插入的主机数量 + */ + int batchInsertHosts(List insertList); + + /** + * 批量更新主机（只更新时间戳比当前数据旧的） + * + * @param hostInfoList 主机信息 + * @return 成功更新的主机数量 + */ + int batchUpdateHostsBeforeLastTime(List hostInfoList); + + /** + * 创建或更新主机（仅更新时间戳在当前数据之前的数据） + * + * @param hostInfoDTO 主机信息 + * @return Pair<是否创建 ， 受影响主机数量> + */ + Pair createOrUpdateHostBeforeLastTime(ApplicationHostDTO hostInfoDTO); + + int updateHostAttrsByHostId(ApplicationHostDTO hostInfoDTO); + + /** + * 更新主机属性（仅更新时间戳在当前数据之前的数据） + * + * @param hostInfoDTO 主机信息 + * @return 成功更新的主机数量 + */ + int updateHostAttrsBeforeLastTime(ApplicationHostDTO hostInfoDTO); + + /** + * 删除主机（仅删除时间戳等于或在当前数据之前的数据） + * + * @param hostInfoDTO 主机信息 + */ + int deleteHostBeforeOrEqualLastTime(ApplicationHostDTO hostInfoDTO); + + void updateDbHostToCache(Long hostId); + + long countHostsByOsType(String osType); + + Map groupHostByOsType(); + + List> queryBizNodePaths(String username, + Long bizId, + List nodeList); + + /** + * 获取业务下的主机。如果在Job缓存的主机中不存在，那么从cmdb查询 + * + * @param appId Job业务ID + * @param hosts 主机列表 + * @param refreshAgentId 是否刷新主机的bk_agent_id + */ + ServiceListAppHostResultDTO listAppHostsPreferCache(Long appId, List hosts, boolean refreshAgentId); + + /** + * 批量获取主机。如果在Job缓存的主机中不存在，那么从cmdb查询 + * + * @param hosts 主机 + * @return 主机 + */ + List listHosts(Collection hosts); + + /** + * 根据云区域ID与IPv6地址查询主机。如果在同步的主机中不存在，那么从cmdb查询 + * ipv6字段精确匹配目标主机多个Ipv6地址中的其中一个 + * + * @param cloudAreaId 云区域ID + * @param ipv6 IPv6地址 + * @return 主机 + */ + List listHostsByCloudIpv6(Long cloudAreaId, String ipv6); + + /** + * 根据主机批量获取主机。如果在同步的主机中不存在，那么从cmdb查询 + * + * @param hostIds 主机ID列表 + * @return 主机 Map + */ + Map listHostsByHostIds(Collection hostIds); + + /** + * 从cmdb实时查询主机 + * + * @param hostIds 主机ID列表 + * @return 主机 + */ + List listHostsFromCmdbByHostIds(List hostIds); + + /** + * 根据ip获取主机 + * + * @param cloudIp 云区域+IP + * @return 主机 + */ + ApplicationHostDTO getHostByIp(String cloudIp); + + /** + * 根据主机批量获取主机。如果在同步的主机中不存在，那么从cmdb查询 + * + * @param cloudIps 主机云区域+ip列表 + * @return 主机 Map + */ + Map listHostsByIps(Collection cloudIps); + + /** + * 更新主机状态 + * + * @param simpleHostList 主机列表 + * @return 更新成功的条数 + */ + int updateHostsStatus(List simpleHostList); + + /** + * 查询所有主机基础信息 + * + * @return 主机基础信息 + */ + List listAllBasicHost(); + + /** + * 根据主机基础信息批量删除主机 + * + * @return 成功删除的主机数量 + */ + int deleteByBasicHost(List basicHostList); +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/ScopeHostService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/ScopeHostService.java new file mode 100644 index 0000000000..8bc66d05ef --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/ScopeHostService.java @@ -0,0 +1,151 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.service.host; + +import com.tencent.bk.job.common.model.PageData; +import com.tencent.bk.job.common.model.dto.AppResourceScope; +import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; +import com.tencent.bk.job.manage.model.web.request.chooser.host.BizTopoNode; + +import java.util.Collection; +import java.util.List; + +/** + * 资源范围主机相关服务 + */ +public interface ScopeHostService { + + /** + * 根据 资源范围 过滤出在该资源范围下的hostIds + * + * @param appResourceScope 资源范围 + * @param hostIds 主机ID集合 + * @return 主机ID列表 + */ + List filterScopeHostIds(AppResourceScope appResourceScope, + Collection hostIds); + + /** + * 根据 HostId 列表查询主机信息 + * + * @param appResourceScope 资源范围 + * @param hostIds 主机ID集合 + * @return 主机信息列表 + */ + List getScopeHostsByIds(AppResourceScope appResourceScope, + Collection hostIds); + + /** + * 根据 IP 列表查询主机信息 + * + * @param appResourceScope 资源范围 + * @param ips 主机IP地址集合 + * @return 主机信息列表 + */ + List getScopeHostsByIps(AppResourceScope appResourceScope, + Collection ips); + + /** + * 根据 CloudIP 列表查询主机信息 + * + * @param appResourceScope 资源范围 + * @param cloudIps 主机CloudIP地址集合 + * @return 主机信息列表 + */ + List getScopeHostsByCloudIps(AppResourceScope appResourceScope, + Collection cloudIps); + + /** + * 根据 Ipv6 列表查询主机信息 + * + * @param appResourceScope 资源范围 + * @param ipv6s 主机Ipv6地址集合 + * @return 主机信息列表 + */ + List getScopeHostsByIpv6s(AppResourceScope appResourceScope, + Collection ipv6s); + + /** + * 根据 关键字 列表查询主机信息 + * + * @param appResourceScope 资源范围 + * @param keys 关键字集合 + * @return 主机信息列表 + */ + List getScopeHostsByHostNames(AppResourceScope appResourceScope, + Collection keys); + + /** + * 根据拓扑节点、模糊搜索关键字、agent状态分页查询查询资源范围下的主机 + * + * @param appResourceScope 资源范围 + * @param appTopoNodeList 拓扑节点列表 + * @param searchContent 模糊搜索关键字（同时对主机IP/主机名/操作系统/云区域名称进行模糊搜索） + * @param agentAlive 筛选条件：agentAlive：0为异常，1为正常 + * @param ipKeyList IP关键字列表 + * @param ipv6KeyList IPv6关键字列表 + * @param hostNameKeyList 主机名称关键字列表 + * @param osNameKeyList 操作系统名称关键字列表 + * @param start 数据起始位置 + * @param pageSize 拉取数量 + * @return hostId列表 + */ + PageData listHostIdByBizTopologyNodes(AppResourceScope appResourceScope, + List appTopoNodeList, + String searchContent, + Integer agentAlive, + List ipKeyList, + List ipv6KeyList, + List hostNameKeyList, + List osNameKeyList, + Long start, + Long pageSize); + + /** + * 根据拓扑节点、模糊搜索关键字、agent状态分页查询查询资源范围下的主机 + * + * @param appResourceScope 资源范围 + * @param appTopoNodeList 拓扑节点列表 + * @param agentAlive 筛选条件：agentAlive：0为异常，1为正常 + * @param searchContent 模糊搜索关键字（同时对主机IP/主机名/操作系统/云区域名称进行模糊搜索） + * @param ipKeyList IP关键字列表 + * @param ipv6KeyList IPv6关键字列表 + * @param hostNameKeyList 主机名称关键字列表 + * @param osNameKeyList 操作系统名称关键字列表 + * @param start 数据起始位置 + * @param pageSize 拉取数量 + * @return hostId列表 + */ + PageData searchHost(AppResourceScope appResourceScope, + List appTopoNodeList, + Integer agentAlive, + String searchContent, + List ipKeyList, + List ipv6KeyList, + List hostNameKeyList, + List osNameKeyList, + Long start, + Long pageSize); +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/ScopeTopoHostService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/ScopeTopoHostService.java new file mode 100644 index 0000000000..215d0542c8 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/ScopeTopoHostService.java @@ -0,0 +1,44 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.service.host; + +import com.tencent.bk.job.common.model.dto.AppResourceScope; +import com.tencent.bk.job.manage.model.web.vo.CcTopologyNodeVO; + +/** + * 主机、topo相关服务 + */ +public interface ScopeTopoHostService { + + /** + * 查询带主机数量信息的业务拓扑树 + * + * @param username 用户名 + * @param appResourceScope 资源范围 + * @return 带主机数量信息的拓扑结构树 + */ + CcTopologyNodeVO listAppTopologyHostCountTree(String username, AppResourceScope appResourceScope); + +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/WhiteIpAwareScopeHostService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/WhiteIpAwareScopeHostService.java new file mode 100644 index 0000000000..526c375fe6 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/WhiteIpAwareScopeHostService.java @@ -0,0 +1,99 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.service.host; + +import com.tencent.bk.job.common.model.dto.AppResourceScope; +import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; +import com.tencent.bk.job.manage.api.common.constants.whiteip.ActionScopeEnum; + +import java.util.Collection; +import java.util.List; + +/** + * 考虑到IP白名单数据的资源范围主机相关服务 + */ +public interface WhiteIpAwareScopeHostService { + + /** + * 根据 HostId 列表查询主机信息 + * + * @param appResourceScope 资源范围 + * @param actionScope 生效场景 + * @param hostIds 主机ID集合 + * @return 主机信息列表 + */ + List getScopeHostsIncludingWhiteIPByHostId(AppResourceScope appResourceScope, + ActionScopeEnum actionScope, + Collection hostIds); + + /** + * 根据 IP 列表查询主机信息 + * + * @param appResourceScope 资源范围 + * @param actionScope 生效场景 + * @param ips IP集合 + * @return 主机信息列表 + */ + List getScopeHostsIncludingWhiteIPByIp(AppResourceScope appResourceScope, + ActionScopeEnum actionScope, + Collection ips); + + /** + * 根据 CloudIP 列表查询主机信息 + * + * @param appResourceScope 资源范围 + * @param actionScope 生效场景 + * @param cloudIps CloudIP集合 + * @return 主机信息列表 + */ + List getScopeHostsIncludingWhiteIPByCloudIp(AppResourceScope appResourceScope, + ActionScopeEnum actionScope, + Collection cloudIps); + + /** + * 根据 ipv6 集合查询主机信息 + * + * @param appResourceScope 资源范围 + * @param actionScope 生效场景 + * @param ipv6s 主机Ipv6集合 + * @return 主机信息列表 + */ + List getScopeHostsIncludingWhiteIPByIpv6(AppResourceScope appResourceScope, + ActionScopeEnum actionScope, + Collection ipv6s); + + /** + * 根据 关键字 集合查询主机信息 + * + * @param appResourceScope 资源范围 + * @param actionScope 生效场景 + * @param keys 关键字集合 + * @return 主机信息列表 + */ + List getScopeHostsIncludingWhiteIPByKey(AppResourceScope appResourceScope, + ActionScopeEnum actionScope, + Collection keys); + +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/impl/BizDynamicGroupHostService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/impl/BizDynamicGroupHostService.java new file mode 100644 index 0000000000..94e550a1e1 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/impl/BizDynamicGroupHostService.java @@ -0,0 +1,130 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.service.host.impl; + +import com.tencent.bk.job.common.cc.model.DynamicGroupHostPropDTO; +import com.tencent.bk.job.common.cc.sdk.BizCmdbClient; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.constant.ResourceScopeTypeEnum; +import com.tencent.bk.job.common.exception.NotImplementedException; +import com.tencent.bk.job.common.model.PageData; +import com.tencent.bk.job.common.model.dto.AppResourceScope; +import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; +import com.tencent.bk.job.common.util.ConcurrencyUtil; +import com.tencent.bk.job.common.util.PageUtil; +import com.tencent.bk.job.manage.service.host.HostDetailService; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +/** + * 查询业务动态分组下的主机信息 + */ +@Slf4j +@Service +public class BizDynamicGroupHostService { + + private final BizCmdbClient bizCmdbClient; + private final HostDetailService hostDetailService; + + @Autowired + public BizDynamicGroupHostService(BizCmdbClient bizCmdbClient, + HostDetailService hostDetailService) { + this.bizCmdbClient = bizCmdbClient; + this.hostDetailService = hostDetailService; + } + + public PageData pageHostByDynamicGroups(AppResourceScope appResourceScope, + String idStr, + int start, + int pageSize) { + if (StringUtils.isBlank(idStr)) { + return new PageData<>(start, pageSize, 0L, Collections.emptyList()); + } + idStr = idStr.trim(); + List hostIds = listHostIdsByDynamicGroups(appResourceScope, Arrays.asList(idStr.split(","))); + // 排序 + hostIds.sort(Long::compareTo); + List hostList = hostDetailService.listHostDetails( + appResourceScope, + hostIds + ); + // CMDB动态分组接口不支持分页，因此对host在内存中进行分页 + return PageUtil.pageInMem(hostList, start, pageSize); + } + + public List listHostByDynamicGroups(AppResourceScope appResourceScope, Collection ids) { + List hostIds = listHostIdsByDynamicGroups(appResourceScope, ids); + // 展示信息需要包含主机详情完整字段 + return hostDetailService.listHostDetails(appResourceScope, hostIds); + } + + public List listHostIdsByDynamicGroups(AppResourceScope appResourceScope, Collection ids) { + if (CollectionUtils.isEmpty(ids)) { + return Collections.emptyList(); + } + // 并发拉取 + int threadNum = Math.min(ids.size(), 5); + List hostIdList = ConcurrencyUtil.getResultWithThreads(ids, threadNum, dynamicGroupId -> + listHostIdsByDynamicGroup(appResourceScope, dynamicGroupId) + ); + // 去重 + hostIdList = new ArrayList<>(new HashSet<>(hostIdList)); + return hostIdList; + } + + public List listHostByDynamicGroup(AppResourceScope appResourceScope, String id) { + List hostIds = listHostIdsByDynamicGroup(appResourceScope, id); + // 展示信息需要包含主机详情完整字段 + return hostDetailService.listHostDetails(appResourceScope, hostIds); + } + + private List listHostIdsByDynamicGroup(AppResourceScope appResourceScope, String id) { + // 动态分组当前只支持业务，不支持业务集 + if (appResourceScope.getType() != ResourceScopeTypeEnum.BIZ) { + throw new NotImplementedException(ErrorCode.NOT_SUPPORT_FEATURE_FOR_BIZ_SET); + } + long bizId = Long.parseLong(appResourceScope.getId()); + List dynamicGroupHostList = bizCmdbClient.getDynamicGroupIp(bizId, id); + if (CollectionUtils.isEmpty(dynamicGroupHostList)) { + return Collections.emptyList(); + } + return dynamicGroupHostList.stream() + .filter(Objects::nonNull) + .map(DynamicGroupHostPropDTO::getId) + .collect(Collectors.toList()); + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/impl/BizHostServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/impl/BizHostServiceImpl.java new file mode 100644 index 0000000000..97cdcfd544 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/impl/BizHostServiceImpl.java @@ -0,0 +1,255 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.service.host.impl; + +import com.tencent.bk.job.common.model.PageData; +import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; +import com.tencent.bk.job.common.util.StringUtil; +import com.tencent.bk.job.manage.dao.ApplicationHostDAO; +import com.tencent.bk.job.manage.dao.HostTopoDAO; +import com.tencent.bk.job.manage.model.dto.HostTopoDTO; +import com.tencent.bk.job.manage.model.query.HostQuery; +import com.tencent.bk.job.manage.service.host.BizHostService; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.util.StopWatch; + +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * 业务主机服务 + */ +@Slf4j +@Service +public class BizHostServiceImpl implements BizHostService { + + private final ApplicationHostDAO applicationHostDAO; + private final HostTopoDAO hostTopoDAO; + + @Autowired + public BizHostServiceImpl(ApplicationHostDAO applicationHostDAO, + HostTopoDAO hostTopoDAO) { + this.applicationHostDAO = applicationHostDAO; + this.hostTopoDAO = hostTopoDAO; + } + + @Override + public List getHostsByHostIds(Collection hostIds) { + return applicationHostDAO.listHostInfoByHostIds(hostIds); + } + + @Override + public List getHostsByIps(Collection ips) { + return applicationHostDAO.listHostInfoByIps(ips); + } + + @Override + public List getHostsByCloudIps(Collection cloudIps) { + return applicationHostDAO.listHostInfoByCloudIps(cloudIps); + } + + @Override + public List getHostsByIpv6s(Collection ipv6s) { + return applicationHostDAO.listHostInfoByIpv6s(ipv6s); + } + + @Override + public List getHostsByHostNames(Collection hostNames) { + return applicationHostDAO.listHostInfoByHostNames(hostNames); + } + + @Override + public List getHostsByBizAndHostIds(Collection bizIds, Collection hostIds) { + if (CollectionUtils.isEmpty(bizIds) || CollectionUtils.isEmpty(hostIds)) { + return Collections.emptyList(); + } + List hostIdsInBiz = hostTopoDAO.listHostIdByBizAndHostIds(bizIds, hostIds); + if (CollectionUtils.isNotEmpty(hostIds) && hostIdsInBiz.size() != hostIds.size()) { + Set hostIdsSet = new HashSet<>(hostIds); + hostIdsSet.removeAll(hostIdsInBiz); + log.warn( + "hostIds [{}] not in bizIds [{}]", + StringUtil.concatCollection(hostIdsSet), + StringUtil.concatCollection(bizIds) + ); + } + return applicationHostDAO.listHostInfoByHostIds(hostIdsInBiz); + } + + @Override + public List filterHostIdsByBiz(Collection bizIds, Collection hostIds) { + if (CollectionUtils.isEmpty(bizIds) || CollectionUtils.isEmpty(hostIds)) { + return Collections.emptyList(); + } + return hostTopoDAO.listHostIdByBizAndHostIds(bizIds, hostIds); + } + + @Override + public List getHostsByBizAndIps(Collection bizIds, Collection ips) { + return applicationHostDAO.listHostInfoByBizAndIps(bizIds, ips); + } + + @Override + public List getHostsByBizAndCloudIps(Collection bizIds, + Collection cloudIps) { + return applicationHostDAO.listHostInfoByBizAndCloudIps(bizIds, cloudIps); + } + + @Override + public List getHostsByBizAndIpv6s(Collection bizIds, Collection ipv6s) { + return applicationHostDAO.listHostInfoByBizAndIpv6s(bizIds, ipv6s); + } + + @Override + public List getHostsByBizAndHostNames(Collection bizIds, + Collection hostNames) { + return applicationHostDAO.listHostInfoByBizAndHostNames(bizIds, hostNames); + } + + @Override + public PageData pageListHostId(HostQuery hostQuery) { + List hostIdList; + Long count; + StopWatch watch = new StopWatch("pageListHostId"); + List searchContents = hostQuery.getSearchContents(); + if (searchContents != null) { + watch.start("getHostIdListBySearchContents"); + hostIdList = applicationHostDAO.getHostIdListBySearchContents( + hostQuery.getBizIds(), + hostQuery.getModuleIds(), + hostQuery.getCloudAreaIds(), + searchContents, + hostQuery.getAgentAlive(), + hostQuery.getStart(), + hostQuery.getLimit() + ); + watch.stop(); + watch.start("countHostInfoBySearchContents"); + count = applicationHostDAO.countHostInfoBySearchContents( + hostQuery.getBizIds(), + hostQuery.getModuleIds(), + hostQuery.getCloudAreaIds(), + searchContents, + hostQuery.getAgentAlive() + ); + watch.stop(); + } else { + watch.start("getHostIdListByMultiKeys"); + hostIdList = applicationHostDAO.getHostIdListByMultiKeys( + hostQuery.getBizIds(), + hostQuery.getModuleIds(), + hostQuery.getCloudAreaIds(), + hostQuery.getIpKeyList(), + hostQuery.getIpv6KeyList(), + hostQuery.getHostNameKeyList(), + hostQuery.getOsNameKeyList(), + hostQuery.getAgentAlive(), + hostQuery.getStart(), + hostQuery.getLimit() + ); + watch.stop(); + watch.start("countHostInfoByMultiKeys"); + count = applicationHostDAO.countHostInfoByMultiKeys( + hostQuery.getBizIds(), + hostQuery.getModuleIds(), + hostQuery.getCloudAreaIds(), + hostQuery.getIpKeyList(), + hostQuery.getIpv6KeyList(), + hostQuery.getHostNameKeyList(), + hostQuery.getOsNameKeyList(), + hostQuery.getAgentAlive() + ); + watch.stop(); + } + if (watch.getTotalTimeMillis() > 3000) { + log.warn("pageListHostId slow:" + watch.prettyPrint()); + } + return new PageData<>(hostQuery.getStart().intValue(), hostQuery.getLimit().intValue(), count, hostIdList); + } + + @Override + public PageData pageListHost(HostQuery hostQuery) { + List hostList; + Long count; + List searchContents = hostQuery.getSearchContents(); + if (searchContents != null) { + hostList = applicationHostDAO.listHostInfoBySearchContents( + hostQuery.getBizIds(), + hostQuery.getModuleIds(), + hostQuery.getCloudAreaIds(), + searchContents, + hostQuery.getAgentAlive(), + hostQuery.getStart(), + hostQuery.getLimit() + ); + count = applicationHostDAO.countHostInfoBySearchContents( + hostQuery.getBizIds(), + hostQuery.getModuleIds(), + hostQuery.getCloudAreaIds(), + searchContents, + hostQuery.getAgentAlive() + ); + } else { + hostList = applicationHostDAO.listHostInfoByMultiKeys( + hostQuery.getBizIds(), + hostQuery.getModuleIds(), + hostQuery.getCloudAreaIds(), + hostQuery.getIpKeyList(), + hostQuery.getIpv6KeyList(), + hostQuery.getHostNameKeyList(), + hostQuery.getOsNameKeyList(), + hostQuery.getAgentAlive(), + hostQuery.getStart(), + hostQuery.getLimit() + ); + count = applicationHostDAO.countHostInfoByMultiKeys( + hostQuery.getBizIds(), + hostQuery.getModuleIds(), + hostQuery.getCloudAreaIds(), + hostQuery.getIpKeyList(), + hostQuery.getIpv6KeyList(), + hostQuery.getHostNameKeyList(), + hostQuery.getOsNameKeyList(), + hostQuery.getAgentAlive() + ); + } + return new PageData<>(hostQuery.getStart().intValue(), hostQuery.getLimit().intValue(), count, hostList); + } + + @Override + public List getHostsByModuleIds(Collection moduleIds) { + List hostTopoDTOList = hostTopoDAO.listHostTopoByModuleIds(moduleIds); + List hostIdList = + hostTopoDTOList.stream().map(HostTopoDTO::getHostId).collect(Collectors.toList()); + return applicationHostDAO.listHostInfoByHostIds(hostIdList); + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/impl/BizTopoHostServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/impl/BizTopoHostServiceImpl.java new file mode 100644 index 0000000000..d701fec3f9 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/impl/BizTopoHostServiceImpl.java @@ -0,0 +1,64 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.service.host.impl; + +import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; +import com.tencent.bk.job.manage.model.web.request.chooser.host.BizTopoNode; +import com.tencent.bk.job.manage.service.host.BizHostService; +import com.tencent.bk.job.manage.service.host.BizTopoHostService; +import com.tencent.bk.job.manage.service.impl.topo.BizTopoService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.Collections; +import java.util.List; + +@Slf4j +@Service +public class BizTopoHostServiceImpl implements BizTopoHostService { + + private final BizTopoService bizTopoService; + private final BizHostService bizHostService; + + @Autowired + public BizTopoHostServiceImpl(BizTopoService bizTopoService, + BizHostService bizHostService) { + this.bizTopoService = bizTopoService; + this.bizHostService = bizHostService; + } + + @Override + public List listHostByNode(Long bizId, BizTopoNode node) { + List moduleIds = bizTopoService.findAllModuleIdsOfNodes(bizId, Collections.singletonList(node)); + return bizHostService.getHostsByModuleIds(moduleIds); + } + + @Override + public List listHostByNodes(Long bizId, List nodes) { + List moduleIds = bizTopoService.findAllModuleIdsOfNodes(bizId, nodes); + return bizHostService.getHostsByModuleIds(moduleIds); + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/impl/CloudVendorService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/impl/CloudVendorService.java new file mode 100644 index 0000000000..cdab492b89 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/impl/CloudVendorService.java @@ -0,0 +1,73 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.service.host.impl; + +import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.LoadingCache; +import com.tencent.bk.job.common.cc.sdk.CmdbClientFactory; +import com.tencent.bk.job.common.cc.sdk.IBizCmdbClient; +import com.tencent.bk.job.common.constant.JobConstants; +import com.tencent.bk.job.common.util.JobContextUtil; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +import java.util.Map; +import java.util.concurrent.TimeUnit; + +@Slf4j +@Service +public class CloudVendorService { + + private final LoadingCache> cloudVendorMapCache = + Caffeine.newBuilder() + .maximumSize(2) + .expireAfterWrite(30, TimeUnit.MINUTES) + .recordStats() + .build(lang -> { + IBizCmdbClient bizCmdbClient = CmdbClientFactory.getCmdbClient(lang); + return bizCmdbClient.getCloudVendorIdNameMap(); + }); + + private String getCloudVendorNameById(String cloudVendorId) { + Map cloudVendorIdNameMap = cloudVendorMapCache.get(JobContextUtil.getUserLang()); + if (cloudVendorIdNameMap == null) { + return JobConstants.UNKNOWN_NAME; + } + return cloudVendorIdNameMap.get(cloudVendorId); + } + + public String getCloudVendorNameOrDefault(String cloudVendorId, String defaultValue) { + if (cloudVendorId == null) { + return defaultValue; + } + try { + return getCloudVendorNameById(cloudVendorId); + } catch (Exception e) { + log.warn("Fail to getCloudVendorNameById", e); + return defaultValue; + } + } + +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/impl/HostDetailServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/impl/HostDetailServiceImpl.java new file mode 100644 index 0000000000..cfc7b8cf11 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/impl/HostDetailServiceImpl.java @@ -0,0 +1,124 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.service.host.impl; + +import com.tencent.bk.job.common.cc.sdk.BkNetClient; +import com.tencent.bk.job.common.constant.JobConstants; +import com.tencent.bk.job.common.model.dto.AppResourceScope; +import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.manage.service.host.HostDetailService; +import com.tencent.bk.job.manage.service.host.WhiteIpAwareScopeHostService; +import com.tencent.bk.job.manage.service.impl.agent.AgentStatusService; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.util.StopWatch; + +import java.util.Collection; +import java.util.List; +import java.util.function.Consumer; + +@Slf4j +@Service +public class HostDetailServiceImpl implements HostDetailService { + + private final WhiteIpAwareScopeHostService whiteIpAwareScopeHostService; + private final AgentStatusService agentStatusService; + private final CloudVendorService cloudVendorService; + private final OsTypeService osTypeService; + + @Autowired + public HostDetailServiceImpl(WhiteIpAwareScopeHostService whiteIpAwareScopeHostService, + AgentStatusService agentStatusService, + CloudVendorService cloudVendorService, + OsTypeService osTypeService) { + this.whiteIpAwareScopeHostService = whiteIpAwareScopeHostService; + this.agentStatusService = agentStatusService; + this.cloudVendorService = cloudVendorService; + this.osTypeService = osTypeService; + } + + @Override + public List listHostDetails(AppResourceScope appResourceScope, Collection hostIds) { + List scopeHostList = whiteIpAwareScopeHostService.getScopeHostsIncludingWhiteIPByHostId( + appResourceScope, + null, + hostIds + ); + // 填充实时agent状态 + agentStatusService.fillRealTimeAgentStatus(scopeHostList); + fillDetailForApplicationHosts(scopeHostList); + return scopeHostList; + } + + @Override + public void fillDetailForApplicationHosts(List hostList) { + fillHostsDetail(hostList, host -> { + host.setCloudAreaName(BkNetClient.getCloudAreaNameFromCache(host.getCloudAreaId())); + String cloudVendorId = host.getCloudVendorId(); + host.setCloudVendorName(cloudVendorService.getCloudVendorNameOrDefault( + cloudVendorId, cloudVendorId == null ? null : JobConstants.UNKNOWN_NAME)); + String osTypeId = host.getOsType(); + host.setOsTypeName(osTypeService.getOsTypeNameOrDefault(osTypeId, + osTypeId == null ? null : JobConstants.UNKNOWN_NAME)); + }); + } + + @Override + public void fillDetailForHosts(List hostList) { + fillHostsDetail(hostList, host -> { + host.setBkCloudName(BkNetClient.getCloudAreaNameFromCache(host.getBkCloudId())); + String cloudVendorId = host.getCloudVendorId(); + host.setCloudVendorName(cloudVendorService.getCloudVendorNameOrDefault( + cloudVendorId, cloudVendorId == null ? null : JobConstants.UNKNOWN_NAME)); + String osTypeId = host.getOsType(); + host.setOsTypeName(osTypeService.getOsTypeNameOrDefault(osTypeId, + osTypeId == null ? null : JobConstants.UNKNOWN_NAME)); + }); + } + + private void fillHostsDetail(Collection hosts, Consumer consumer) { + if (CollectionUtils.isEmpty(hosts)) { + return; + } + StopWatch watch = new StopWatch("FillDetailForHosts"); + try { + watch.start(); + for (T host : hosts) { + consumer.accept(host); + } + watch.stop(); + } finally { + if (watch.isRunning()) { + watch.stop(); + } + if (watch.getTotalTimeMillis() > 100) { + log.warn("FillDetailForHosts slow, watch: {}", watch.prettyPrint()); + } + } + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/impl/HostServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/impl/HostServiceImpl.java new file mode 100644 index 0000000000..ba531f911e --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/impl/HostServiceImpl.java @@ -0,0 +1,862 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.service.host.impl; + +import com.tencent.bk.job.common.cc.model.InstanceTopologyDTO; +import com.tencent.bk.job.common.cc.sdk.BizCmdbClient; +import com.tencent.bk.job.common.cc.sdk.CmdbClientFactory; +import com.tencent.bk.job.common.cc.sdk.IBizCmdbClient; +import com.tencent.bk.job.common.constant.JobConstants; +import com.tencent.bk.job.common.constant.ResourceScopeTypeEnum; +import com.tencent.bk.job.common.model.dto.ApplicationDTO; +import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; +import com.tencent.bk.job.common.model.dto.BasicHostDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.common.model.dto.HostSimpleDTO; +import com.tencent.bk.job.common.model.dto.ResourceScope; +import com.tencent.bk.job.common.mysql.JobTransactional; +import com.tencent.bk.job.common.util.JobContextUtil; +import com.tencent.bk.job.common.util.ip.IpUtils; +import com.tencent.bk.job.manage.common.TopologyHelper; +import com.tencent.bk.job.manage.dao.ApplicationHostDAO; +import com.tencent.bk.job.manage.manager.host.HostCache; +import com.tencent.bk.job.manage.model.db.CacheHostDO; +import com.tencent.bk.job.manage.model.inner.ServiceListAppHostResultDTO; +import com.tencent.bk.job.manage.service.ApplicationService; +import com.tencent.bk.job.manage.service.host.HostService; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.util.StopWatch; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +@SuppressWarnings("SameParameterValue") +@Slf4j +@Service +public class HostServiceImpl implements HostService { + + private final ApplicationHostDAO applicationHostDAO; + private final ApplicationService applicationService; + private final HostCache hostCache; + private final BizCmdbClient bizCmdbClient; + + @Autowired + public HostServiceImpl(ApplicationHostDAO applicationHostDAO, + ApplicationService applicationService, + HostCache hostCache, + BizCmdbClient bizCmdbClient) { + this.applicationHostDAO = applicationHostDAO; + this.applicationService = applicationService; + this.hostCache = hostCache; + this.bizCmdbClient = bizCmdbClient; + } + + @Override + public List getHostsByAppId(Long appId) { + ApplicationDTO applicationDTO = applicationService.getAppByAppId(appId); + ResourceScope scope = applicationDTO.getScope(); + if (scope.getType() == ResourceScopeTypeEnum.BIZ) { + return applicationHostDAO.listHostInfoByBizId(Long.parseLong(scope.getId())); + } else { + return applicationHostDAO.listHostInfoByBizIds(applicationDTO.getSubBizIds(), null, null); + } + } + + @Override + public int batchInsertHosts(List insertList) { + if (CollectionUtils.isEmpty(insertList)) { + return 0; + } + StopWatch watch = new StopWatch(); + watch.start("batchInsertHost"); + // 批量插入主机 + int affectedNum = applicationHostDAO.batchInsertHost(insertList); + log.info("{} hosts inserted", affectedNum); + insertList.forEach(hostCache::addOrUpdateHost); + watch.stop(); + if (log.isDebugEnabled()) { + log.debug("Performance:insertHosts:{}", watch.prettyPrint()); + } + return affectedNum; + } + + @Override + public int batchUpdateHostsBeforeLastTime(List hostInfoList) { + if (CollectionUtils.isEmpty(hostInfoList)) { + return 0; + } + StopWatch watch = new StopWatch(); + watch.start("batchUpdateHostsBeforeLastTime to DB"); + // 批量更新主机 + int affectedNum = applicationHostDAO.batchUpdateHostsBeforeLastTime(hostInfoList); + log.info("try to update {} hosts, {} updated", hostInfoList.size(), affectedNum); + watch.stop(); + watch.start("listHostInfoByHostIds"); + List hostIds = hostInfoList.stream().map(ApplicationHostDTO::getHostId).collect(Collectors.toList()); + hostInfoList = applicationHostDAO.listHostInfoByHostIds(hostIds); + watch.stop(); + watch.start("updateHostToCache"); + hostInfoList.forEach(hostCache::addOrUpdateHost); + watch.stop(); + if (watch.getTotalTimeMillis() < 10_000L) { + if (log.isDebugEnabled()) { + log.debug("Performance:batchUpdateHostsBeforeLastTime:{}", watch.prettyPrint()); + } + } else if (watch.getTotalTimeMillis() < 60_000L) { + log.info("Performance:batchUpdateHostsBeforeLastTime:{}", watch.prettyPrint()); + } else { + log.warn("Performance:batchUpdateHostsBeforeLastTime:{}", watch.prettyPrint()); + } + return affectedNum; + } + + @Override + public int updateHostsStatus(List simpleHostList) { + StopWatch watch = new StopWatch(); + int updateCount = 0; + try { + if (!simpleHostList.isEmpty()) { + watch.start("updateHostsStatus"); + // MySql5.7为例默认单条SQL最大为4M + int batchSize = 5000; + int size = simpleHostList.size(); + int start = 0; + int end; + do { + end = start + batchSize; + end = Math.min(end, size); + List subList = simpleHostList.subList(start, end); + Map> agentAliveStatusGroupMap = subList.stream() + .collect(Collectors.groupingBy(HostSimpleDTO::getAgentAliveStatus, + Collectors.mapping(HostSimpleDTO::getHostId, Collectors.toList()))); + for (Integer agentAliveStatus : agentAliveStatusGroupMap.keySet()) { + updateCount += applicationHostDAO.batchUpdateHostStatusByHostIds(agentAliveStatus, + agentAliveStatusGroupMap.get(agentAliveStatus)); + } + start += batchSize; + } while (end < size); + watch.stop(); + watch.start("updateHostsCache"); + deleteOrUpdateHostCache(simpleHostList); + watch.stop(); + } + } catch (Throwable throwable) { + log.error(String.format("updateHostStatus fail：hostSize=%s", simpleHostList.size()), throwable); + } + if (watch.getTotalTimeMillis() > 180000) { + log.info("updateHostsStatus too slow, run statistics:{}", watch.prettyPrint()); + } + log.debug("Performance:updateHostsStatus:{}", watch); + return updateCount; + } + + private void deleteOrUpdateHostCache(List simpleHostList) { + if (CollectionUtils.isEmpty(simpleHostList)) { + return; + } + int maxDeleteNum = 10000; + if (simpleHostList.size() <= maxDeleteNum) { + // 需要更新的主机数量较少，直接删缓存 + deleteHostCache(simpleHostList); + } else { + // 需要更新的主机数量较多，从DB加载最新数据到缓存，防止缓存穿透 + loadDBHostToCache(simpleHostList); + } + } + + private void deleteHostCache(List simpleHostList) { + Collection hosts = simpleHostList.stream() + .map(HostSimpleDTO::convertToHostDTO) + .collect(Collectors.toList()); + Collection hostIds = hosts.stream() + .map(ApplicationHostDTO::getHostId) + .collect(Collectors.toList()); + hostCache.batchDeleteHost(hosts); + if (log.isDebugEnabled()) { + log.debug( + "{} hosts deleted from cache, hostIds:{}", + hostIds.size(), + hostIds + ); + } + } + + private void loadDBHostToCache(List simpleHostList) { + Collection hostIds = simpleHostList.stream() + .map(HostSimpleDTO::getHostId) + .collect(Collectors.toList()); + List hosts = applicationHostDAO.listHostInfoByHostIds(hostIds); + Collection existHostIds = hosts.stream() + .map(ApplicationHostDTO::getHostId) + .collect(Collectors.toList()); + hostCache.batchAddOrUpdateHosts(hosts); + if (log.isDebugEnabled()) { + log.debug( + "{} hosts from db loaded to cache, hostIds:{}", + existHostIds.size(), + existHostIds + ); + } + } + + @JobTransactional(transactionManager = "jobManageTransactionManager") + @Override + public Pair createOrUpdateHostBeforeLastTime(ApplicationHostDTO hostInfoDTO) { + boolean needToCreate = false; + try { + if (applicationHostDAO.existAppHostInfoByHostId(hostInfoDTO.getHostId())) { + // 只更新事件中的主机属性与agent状态 + int affectedNum = applicationHostDAO.updateHostAttrsBeforeLastTime(hostInfoDTO); + if (affectedNum == 0) { + ApplicationHostDTO hostInDB = applicationHostDAO.getHostById(hostInfoDTO.getHostId()); + if (hostInDB != null) { + log.info( + "Not update host, hostId={}, dbHostLastTime={}, currentHostLastTime={}", + hostInDB.getHostId(), + hostInDB.getLastTime(), + hostInfoDTO.getLastTime() + ); + } else { + log.warn( + "Not update host, hostId={}, hostInDB not exists", + hostInfoDTO.getHostId() + ); + } + } + return Pair.of(needToCreate, affectedNum); + } else { + needToCreate = true; + hostInfoDTO.setBizId(JobConstants.PUBLIC_APP_ID); + int affectedNum = applicationHostDAO.insertHostWithoutTopo(hostInfoDTO); + log.info("insert host: id={}, affectedNum={}", hostInfoDTO.getHostId(), affectedNum); + return Pair.of(needToCreate, affectedNum); + } + } catch (Throwable t) { + log.error("createOrUpdateHostBeforeLastTime fail", t); + return Pair.of(needToCreate, 0); + } finally { + // 从拓扑表向主机表同步拓扑数据 + int affectedNum = applicationHostDAO.syncHostTopo(hostInfoDTO.getHostId()); + log.info("hostTopo synced: hostId={}, affectedNum={}", hostInfoDTO.getHostId(), affectedNum); + // 更新缓存 + updateDbHostToCache(hostInfoDTO.getHostId()); + } + } + + public int updateHostAttrsByHostId(ApplicationHostDTO hostInfoDTO) { + return applicationHostDAO.updateHostAttrsByHostId(hostInfoDTO); + } + + public int updateHostAttrsBeforeLastTime(ApplicationHostDTO hostInfoDTO) { + return applicationHostDAO.updateHostAttrsBeforeLastTime(hostInfoDTO); + } + + @Override + public int deleteHostBeforeOrEqualLastTime(ApplicationHostDTO hostInfoDTO) { + int affectedRowNum = applicationHostDAO.deleteHostBeforeOrEqualLastTime( + null, + hostInfoDTO.getHostId(), + hostInfoDTO.getLastTime() + ); + log.info( + "{} host deleted, id={} ,ip={}", + affectedRowNum, + hostInfoDTO.getHostId(), + hostInfoDTO.getIp() + ); + if (affectedRowNum > 0) { + hostCache.deleteHost(hostInfoDTO); + } + return affectedRowNum; + } + + public void updateDbHostToCache(Long hostId) { + ApplicationHostDTO hostInfoDTO = applicationHostDAO.getHostById(hostId); + if (hostInfoDTO.getBizId() != null && hostInfoDTO.getBizId() > 0) { + // 只更新常规业务的主机到缓存 + if (applicationService.existBiz(hostInfoDTO.getBizId())) { + hostCache.addOrUpdateHost(hostInfoDTO); + log.info("host cache updated: hostId:{}", hostInfoDTO.getHostId()); + } + } + } + + @Override + public List listAllBasicHost() { + return applicationHostDAO.listAllBasicHost(); + } + + @Override + public int deleteByBasicHost(List basicHostList) { + if (CollectionUtils.isEmpty(basicHostList)) { + return 0; + } + List hostIdList = basicHostList.stream().map(BasicHostDTO::getHostId).collect(Collectors.toList()); + // 先查出主机信息用于更新缓存 + List hostList = applicationHostDAO.listHostInfoByHostIds(hostIdList); + // 从DB删除 + int deletedNum = applicationHostDAO.deleteByBasicHost(basicHostList); + // 删除缓存 + hostCache.batchDeleteHost(hostList); + hostList = applicationHostDAO.listHostInfoByHostIds(hostIdList); + // 未成功从DB删除的主机重新加入缓存 + hostCache.batchAddOrUpdateHosts(hostList); + return deletedNum; + } + + @Override + public long countHostsByOsType(String osType) { + return applicationHostDAO.countHostsByOsType(osType); + } + + @Override + public Map groupHostByOsType() { + return applicationHostDAO.groupHostByOsType(); + } + + @Override + public List> queryBizNodePaths(String username, + Long bizId, + List nodeList) { + // 查业务拓扑树 + IBizCmdbClient bizCmdbClient = CmdbClientFactory.getCmdbClient(JobContextUtil.getUserLang()); + InstanceTopologyDTO appTopologyTree = bizCmdbClient.getBizInstTopology(bizId); + // 搜索路径 + return TopologyHelper.findTopoPaths(appTopologyTree, nodeList); + } + + private List buildIncludeBizIdList(ApplicationDTO application) { + List bizIdList = new ArrayList<>(); + if (application.isBiz()) { + bizIdList.add(application.getBizIdIfBizApp()); + } else if (application.isBizSet()) { + if (application.getSubBizIds() != null) { + bizIdList.addAll(application.getSubBizIds()); + } + } + return bizIdList; + } + + public List listHosts(Collection hosts) { + Pair, List> hostResult = listHostsFromCacheOrCmdb(hosts); + if (CollectionUtils.isEmpty(hostResult.getRight())) { + return Collections.emptyList(); + } + return hostResult.getRight(); + } + + public List listHostsByCloudIpv6(Long cloudAreaId, String ipv6) { + List hosts = applicationHostDAO.listHostInfoByCloudIpv6(cloudAreaId, ipv6); + // 多个Ipv6中的具体一个与查询条件Ipv6相等才有效 + hosts = filterHostsByCloudIpv6(hosts, cloudAreaId, ipv6); + if (CollectionUtils.isEmpty(hosts)) { + hosts = bizCmdbClient.listHostsByCloudIpv6s(Collections.singletonList(cloudAreaId + IpUtils.COLON + ipv6)); + hosts = filterHostsByCloudIpv6(hosts, cloudAreaId, ipv6); + } + return hosts; + } + + /** + * 筛选出与指定云区域ID、Ipv6地址匹配的主机 + * 指定的Ipv6地址必须与主机多个Ipv6地址的其中一个精确匹配 + * + * @param hosts 筛选前的主机列表 + * @param cloudAreaId 云区域ID + * @param ipv6 Ipv6地址 + * @return 筛选后的主机列表 + */ + private List filterHostsByCloudIpv6(List hosts, + Long cloudAreaId, + String ipv6) { + return hosts.stream().filter(host -> { + String multiIpv6 = host.getIpv6(); + Set ipv6s = Arrays.stream(multiIpv6.split("[,;]")) + .map(String::trim) + .collect(Collectors.toSet()); + return host.getCloudAreaId().equals(cloudAreaId) && ipv6s.contains(ipv6); + }).collect(Collectors.toList()); + } + + @Override + public ServiceListAppHostResultDTO listAppHostsPreferCache(Long appId, + List hosts, + boolean refreshAgentId) { + StopWatch watch = new StopWatch("listAppHostsPreferCache"); + try { + ServiceListAppHostResultDTO result = new ServiceListAppHostResultDTO(); + + watch.start("getAppByAppId"); + ApplicationDTO application = applicationService.getAppByAppId(appId); + watch.stop(); + + watch.start("listHostsFromCacheOrCmdb"); + Pair, List> hostResult = listHostsFromCacheOrCmdb(hosts); + watch.stop(); + + List notExistHosts = hostResult.getLeft(); + List existHosts = hostResult.getRight(); + + watch.start("refreshHostAgentIdIfNeed"); + refreshHostAgentIdIfNeed(refreshAgentId, existHosts); + watch.stop(); + + List validHosts = new ArrayList<>(); + List notInAppHosts = new ArrayList<>(); + + if (application.isAllBizSet()) { + // 如果是全业务，所以主机都是合法的 + result.setNotExistHosts(notExistHosts); + result.setValidHosts(existHosts.stream().map(ApplicationHostDTO::toHostDTO) + .collect(Collectors.toList())); + result.setNotInAppHosts(Collections.emptyList()); + return result; + } + + // 普通业务集和普通业务需要判断主机是否归属于业务 + List includeBizIds = buildIncludeBizIdList(application); + if (CollectionUtils.isEmpty(includeBizIds)) { + log.warn("App do not contains any biz, appId:{}", appId); + result.setValidHosts(Collections.emptyList()); + result.setNotExistHosts(Collections.emptyList()); + result.setNotInAppHosts(hosts); + return result; + } + + existHosts.forEach(existHost -> { + if (includeBizIds.contains(existHost.getBizId())) { + validHosts.add(existHost.toHostDTO()); + } else { + notInAppHosts.add(existHost.toHostDTO()); + } + }); + + // 对于判定为其他业务下的主机，可能是缓存数据不准确导致，需要根据CMDB实时数据进行二次判定 + if (CollectionUtils.isNotEmpty(notInAppHosts)) { + watch.start("reConfirmNotInAppHostsByCmdb"); + reConfirmNotInAppHostsByCmdb(notInAppHosts, notExistHosts, validHosts, appId, includeBizIds); + watch.stop(); + } + + if (CollectionUtils.isNotEmpty(notExistHosts) || CollectionUtils.isNotEmpty(notInAppHosts)) { + log.info("Contains invalid hosts, appId: {}, notExistHosts: {}, hostsInOtherApp: {}", + appId, notExistHosts, notInAppHosts); + } + + result.setNotExistHosts(notExistHosts); + result.setValidHosts(validHosts); + result.setNotInAppHosts(notInAppHosts); + + return result; + } finally { + if (watch.isRunning()) { + watch.stop(); + } + if (watch.getTotalTimeMillis() > 3000) { + log.warn("ListAppHostsPreferCache slow, watch: {}", watch.prettyPrint()); + } + } + + } + + /** + * 对于判定为其他业务下的主机，可能是缓存数据不准确导致，根据CMDB实时数据进行二次判定 + * + * @param notInAppHosts 前期判定为在其他业务下的主机，在该方法中数据可能被修改 + * @param notExistHosts 前期判定为不存在的主机，在该方法中数据可能被修改 + * @param validHosts 前期判定为在业务下的主机，在该方法中数据可能被修改 + * @param appId Job内业务ID + * @param includeBizIds Job内业务ID可能对应的多个CMDB业务ID列表 + */ + private void reConfirmNotInAppHostsByCmdb(List notInAppHosts, + List notExistHosts, + List validHosts, + Long appId, + List includeBizIds) { + Pair, List> cmdbHostsPair = listHostsFromCmdb(notInAppHosts); + if (CollectionUtils.isNotEmpty(cmdbHostsPair.getLeft())) { + notExistHosts.addAll(cmdbHostsPair.getLeft()); + } + List cmdbExistHosts = cmdbHostsPair.getRight(); + if (CollectionUtils.isNotEmpty(cmdbExistHosts)) { + notInAppHosts.clear(); + List cmdbValidHosts = new ArrayList<>(); + cmdbExistHosts.forEach(existHost -> { + if (includeBizIds.contains(existHost.getBizId())) { + validHosts.add(existHost.toHostDTO()); + cmdbValidHosts.add(existHost); + } else { + notInAppHosts.add(existHost.toHostDTO()); + } + }); + if (!cmdbValidHosts.isEmpty()) { + log.info( + "{} hosts belong to appId {} after check in cmdb, cmdbValidHosts={}", + cmdbValidHosts.size(), + appId, + cmdbValidHosts + ); + } + } + } + + private void refreshHostAgentIdIfNeed(boolean needRefreshAgentId, List hosts) { + // 如果Job缓存的主机中没有agentId，那么需要从cmdb实时获取（解决一些特殊场景，比如节点管理Agent插件安装，bk_agent_id更新事件还没被处理的场景 + if (!needRefreshAgentId || CollectionUtils.isEmpty(hosts)) { + return; + } + long start = System.currentTimeMillis(); + List missingAgentIdHostIds = hosts.stream() + .filter(host -> StringUtils.isEmpty(host.getAgentId())) + .map(ApplicationHostDTO::getHostId) + .collect(Collectors.toList()); + if (CollectionUtils.isEmpty(missingAgentIdHostIds)) { + return; + } + List cmdbHosts = listHostsFromCmdbByHostIds(missingAgentIdHostIds); + if (CollectionUtils.isEmpty(cmdbHosts)) { + log.warn("Refresh host agent id, hosts are not exist in cmdb! hosts: {}", missingAgentIdHostIds); + return; + } + + Map hostIdAndAgentIdMap = cmdbHosts.stream() + .filter(host -> StringUtils.isNotEmpty(host.getAgentId())) + .collect(Collectors.toMap(ApplicationHostDTO::getHostId, + ApplicationHostDTO::getAgentId, (oldValue, newValue) -> newValue)); + hosts.forEach(host -> { + if (StringUtils.isEmpty(host.getAgentId())) { + host.setAgentId(hostIdAndAgentIdMap.get(host.getHostId())); + } + }); + log.info("Refresh host agent id, hostIds: {}, hostIdAndAgentIdMap: {}, cost: {}", + missingAgentIdHostIds, hostIdAndAgentIdMap, System.currentTimeMillis() - start); + } + + private Pair, List> listHostsFromCmdb(Collection hosts) { + List notExistHosts = new ArrayList<>(); + List appHosts = new ArrayList<>(); + Pair, List> pair = separateByHostIdOrCloudIp(hosts); + List hostIds = pair.getLeft(); + List cloudIps = pair.getRight(); + if (CollectionUtils.isNotEmpty(hostIds)) { + Pair, List> result = + new ListHostByHostIdsStrategy().listHostsFromCmdb(hostIds); + appHosts.addAll(result.getRight()); + if (CollectionUtils.isNotEmpty(result.getLeft())) { + result.getLeft().forEach(notExistHostId -> notExistHosts.add(HostDTO.fromHostId(notExistHostId))); + } + } + if (CollectionUtils.isNotEmpty(cloudIps)) { + Pair, List> result = + new ListHostByIpsStrategy().listHostsFromCmdb(cloudIps); + appHosts.addAll(result.getRight()); + if (CollectionUtils.isNotEmpty(result.getLeft())) { + result.getLeft().forEach(notExistCloudIp -> notExistHosts.add(HostDTO.fromCloudIp(notExistCloudIp))); + } + } + return Pair.of(notExistHosts, appHosts); + } + + private Pair, List> separateByHostIdOrCloudIp(Collection hosts) { + List hostIds = new ArrayList<>(); + List cloudIps = new ArrayList<>(); + for (HostDTO host : hosts) { + if (host.getHostId() != null) { + hostIds.add(host.getHostId()); + } else { + cloudIps.add(host.toCloudIp()); + } + } + return Pair.of(hostIds, cloudIps); + } + + private Pair, List> listHostsFromCacheOrCmdb(Collection hosts) { + List appHosts = new ArrayList<>(); + List notExistHosts = new ArrayList<>(); + Pair, List> pair = separateByHostIdOrCloudIp(hosts); + List hostIds = pair.getLeft(); + List cloudIps = pair.getRight(); + if (CollectionUtils.isNotEmpty(hostIds)) { + Pair, List> result = listHostsByStrategy(hostIds, + new ListHostByHostIdsStrategy()); + appHosts.addAll(result.getRight()); + if (CollectionUtils.isNotEmpty(result.getLeft())) { + result.getLeft().forEach(notExistHostId -> notExistHosts.add(HostDTO.fromHostId(notExistHostId))); + } + } + if (CollectionUtils.isNotEmpty(cloudIps)) { + Pair, List> result = listHostsByStrategy(cloudIps, + new ListHostByIpsStrategy()); + appHosts.addAll(result.getRight()); + if (CollectionUtils.isNotEmpty(result.getLeft())) { + result.getLeft().forEach(notExistCloudIp -> notExistHosts.add(HostDTO.fromCloudIp(notExistCloudIp))); + } + } + return Pair.of(notExistHosts, appHosts); + } + + private Pair, List> listHostsByStrategy(List keys, + ListHostStrategy listHostStrategy) { + List appHosts = new ArrayList<>(); + List notExistKeys = null; + + if (CollectionUtils.isNotEmpty(keys)) { + Pair, List> result = listHostStrategy.listHostsFromCache(keys); + notExistKeys = result.getLeft(); + if (CollectionUtils.isNotEmpty(result.getRight())) { + appHosts.addAll(result.getRight()); + } + } + + if (CollectionUtils.isNotEmpty(notExistKeys)) { + Pair, List> result = listHostStrategy.listHostsFromDb(notExistKeys); + notExistKeys = result.getLeft(); + if (CollectionUtils.isNotEmpty(result.getRight())) { + appHosts.addAll(result.getRight()); + } + } + + if (CollectionUtils.isNotEmpty(notExistKeys)) { + Pair, List> result = listHostStrategy.listHostsFromCmdb(notExistKeys); + notExistKeys = result.getLeft(); + if (CollectionUtils.isNotEmpty(result.getRight())) { + appHosts.addAll(result.getRight()); + } + } + + return Pair.of(notExistKeys, appHosts); + } + + /** + * 主机查询策略 + * + * @param 查询使用的主机KEY + */ + private interface ListHostStrategy { + Pair, List> listHostsFromCache(List keys); + + Pair, List> listHostsFromDb(List keys); + + Pair, List> listHostsFromCmdb(List keys); + } + + /** + * 根据ip查询主机 + */ + private class ListHostByIpsStrategy implements ListHostStrategy { + @Override + public Pair, List> listHostsFromCache(List cloudIps) { + long start = System.currentTimeMillis(); + List appHosts = new ArrayList<>(); + List notExistCloudIps = new ArrayList<>(); + List cacheHosts = hostCache.batchGetHostsByIps(cloudIps); + for (int i = 0; i < cloudIps.size(); i++) { + String cloudIp = cloudIps.get(i); + CacheHostDO cacheHost = cacheHosts.get(i); + if (cacheHost != null) { + appHosts.add(cacheHost.toApplicationHostDTO()); + } else { + notExistCloudIps.add(cloudIp); + } + } + + long cost = System.currentTimeMillis() - start; + if (cost > 1000) { + log.warn("ListHostsFromRedis slow, ipSize: {}, cost: {}", cloudIps.size(), cost); + } + return Pair.of(notExistCloudIps, appHosts); + } + + @Override + public Pair, List> listHostsFromDb(List cloudIps) { + long start = System.currentTimeMillis(); + List appHosts = new ArrayList<>(); + List notExistCloudIps = new ArrayList<>(cloudIps); + List hostsInDb = applicationHostDAO.listHostsByCloudIps(cloudIps); + if (CollectionUtils.isNotEmpty(hostsInDb)) { + for (ApplicationHostDTO appHost : hostsInDb) { + if (appHost.getBizId() == null || appHost.getBizId() <= 0) { + log.info("Host: {}|{} missing bizId, skip!", appHost.getHostId(), appHost.getCloudIp()); + // DB中缓存的主机可能没有业务信息(依赖的主机事件还没有处理),那么暂时跳过该主机 + continue; + } + notExistCloudIps.remove(appHost.getCloudIp()); + appHosts.add(appHost); + } + } + if (CollectionUtils.isNotEmpty(appHosts)) { + hostCache.batchAddOrUpdateHosts(appHosts); + } + + long cost = System.currentTimeMillis() - start; + if (cost > 1000) { + log.warn("ListHostsFromMySQL slow, ipSize: {}, cost: {}", cloudIps.size(), cost); + } + return Pair.of(notExistCloudIps, appHosts); + } + + @Override + public Pair, List> listHostsFromCmdb(List cloudIps) { + long start = System.currentTimeMillis(); + + List notExistCloudIps = new ArrayList<>(cloudIps); + + List cmdbExistHosts = bizCmdbClient.listHostsByCloudIps(cloudIps); + if (CollectionUtils.isNotEmpty(cmdbExistHosts)) { + List cmdbExistHostIds = cmdbExistHosts.stream() + .map(ApplicationHostDTO::getCloudIp) + .collect(Collectors.toList()); + notExistCloudIps.removeAll(cmdbExistHostIds); + log.info("sync new hosts from cmdb, hosts:{}", cmdbExistHosts); + + hostCache.batchAddOrUpdateHosts(cmdbExistHosts); + } + + long cost = System.currentTimeMillis() - start; + if (cost > 1000) { + log.warn("ListHostsFromCmdb slow, ipSize: {}, cost: {}", cloudIps.size(), cost); + } + return Pair.of(notExistCloudIps, cmdbExistHosts); + } + } + + private class ListHostByHostIdsStrategy implements ListHostStrategy { + @Override + public Pair, List> listHostsFromCache(List hostIds) { + long start = System.currentTimeMillis(); + List appHosts = new ArrayList<>(); + List notExistHostIds = new ArrayList<>(); + List cacheHosts = hostCache.batchGetHostsByHostIds(hostIds); + for (int i = 0; i < hostIds.size(); i++) { + long hostId = hostIds.get(i); + CacheHostDO cacheHost = cacheHosts.get(i); + if (cacheHost != null) { + appHosts.add(cacheHost.toApplicationHostDTO()); + } else { + notExistHostIds.add(hostId); + } + } + long cost = System.currentTimeMillis() - start; + if (cost > 1000) { + log.warn("ListHostsFromRedis slow, hostSize: {}, cost: {}", hostIds.size(), cost); + } + return Pair.of(notExistHostIds, appHosts); + } + + @Override + public Pair, List> listHostsFromDb(List hostIds) { + long start = System.currentTimeMillis(); + List appHosts = new ArrayList<>(); + List notExistHostIds = new ArrayList<>(hostIds); + List hostsInDb = applicationHostDAO.listHostInfoByHostIds(hostIds); + if (CollectionUtils.isNotEmpty(hostsInDb)) { + for (ApplicationHostDTO appHost : hostsInDb) { + if (appHost.getBizId() == null || appHost.getBizId() <= 0) { + log.info("Host: {}|{} missing bizId, skip!", appHost.getHostId(), appHost.getCloudIp()); + // DB中缓存的主机可能没有业务信息(依赖的主机事件还没有处理),那么暂时跳过该主机 + continue; + } + notExistHostIds.remove(appHost.getHostId()); + appHosts.add(appHost); + } + } + + if (CollectionUtils.isNotEmpty(appHosts)) { + hostCache.batchAddOrUpdateHosts(appHosts); + } + long cost = System.currentTimeMillis() - start; + if (cost > 1000) { + log.warn("ListHostsFromMySQL slow, hostSize: {}, cost: {}", hostIds.size(), cost); + } + return Pair.of(notExistHostIds, appHosts); + } + + @Override + public Pair, List> listHostsFromCmdb(List hostIds) { + long start = System.currentTimeMillis(); + List notExistHostIds = new ArrayList<>(hostIds); + List cmdbExistHosts = listHostsFromCmdbByHostIds(hostIds); + if (CollectionUtils.isNotEmpty(cmdbExistHosts)) { + List cmdbExistHostIds = cmdbExistHosts.stream() + .map(ApplicationHostDTO::getHostId) + .collect(Collectors.toList()); + notExistHostIds.removeAll(cmdbExistHostIds); + log.info("sync new hosts from cmdb, hosts:{}", cmdbExistHosts); + + hostCache.batchAddOrUpdateHosts(cmdbExistHosts); + } + + long cost = System.currentTimeMillis() - start; + if (cost > 1000) { + log.warn("ListHostsFromCmdb slow, hostSize: {}, cost: {}", hostIds.size(), cost); + } + return Pair.of(notExistHostIds, cmdbExistHosts); + } + } + + @Override + public Map listHostsByHostIds(Collection hostIds) { + Pair, List> result = listHostsByStrategy(new ArrayList<>(hostIds), + new ListHostByHostIdsStrategy()); + return result.getRight().stream().collect( + Collectors.toMap(ApplicationHostDTO::getHostId, host -> host, (oldValue, newValue) -> newValue)); + } + + /** + * 从cmdb实时查询主机 + * + * @param hostIds 主机ID列表 + * @return 主机 Map + */ + @Override + public List listHostsFromCmdbByHostIds(List hostIds) { + return bizCmdbClient.listHostsByHostIds(hostIds); + } + + @Override + public ApplicationHostDTO getHostByIp(String cloudIp) { + Pair, List> result = listHostsByStrategy(Collections.singletonList(cloudIp), + new ListHostByIpsStrategy()); + return CollectionUtils.isNotEmpty(result.getRight()) ? result.getRight().get(0) : null; + } + + @Override + public Map listHostsByIps(Collection cloudIps) { + Pair, List> result = listHostsByStrategy(new ArrayList<>(cloudIps), + new ListHostByIpsStrategy()); + return result.getRight().stream().collect( + Collectors.toMap(ApplicationHostDTO::getCloudIp, host -> host, (oldValue, newValue) -> newValue)); + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/impl/OsTypeService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/impl/OsTypeService.java new file mode 100644 index 0000000000..754531b389 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/impl/OsTypeService.java @@ -0,0 +1,72 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.service.host.impl; + +import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.LoadingCache; +import com.tencent.bk.job.common.cc.sdk.CmdbClientFactory; +import com.tencent.bk.job.common.cc.sdk.IBizCmdbClient; +import com.tencent.bk.job.common.constant.JobConstants; +import com.tencent.bk.job.common.util.JobContextUtil; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +import java.util.Map; +import java.util.concurrent.TimeUnit; + +@Slf4j +@Service +public class OsTypeService { + + private final LoadingCache> osTypeMapCache = Caffeine.newBuilder() + .maximumSize(2) + .expireAfterWrite(30, TimeUnit.MINUTES) + .recordStats() + .build(lang -> { + IBizCmdbClient bizCmdbClient = CmdbClientFactory.getCmdbClient(lang); + return bizCmdbClient.getOsTypeIdNameMap(); + }); + + private String getOsTypeNameById(String osTypeId) { + Map osTypeIdNameMap = osTypeMapCache.get(JobContextUtil.getUserLang()); + if (osTypeIdNameMap == null) { + return JobConstants.UNKNOWN_NAME; + } + return osTypeIdNameMap.get(osTypeId); + } + + public String getOsTypeNameOrDefault(String osTypeId, String defaultValue) { + if (osTypeId == null) { + return defaultValue; + } + try { + return getOsTypeNameById(osTypeId); + } catch (Exception e) { + log.warn("Fail to getOsTypeNameById", e); + return defaultValue; + } + } + +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/impl/ScopeHostServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/impl/ScopeHostServiceImpl.java new file mode 100644 index 0000000000..3094fd31cf --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/impl/ScopeHostServiceImpl.java @@ -0,0 +1,304 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.service.host.impl; + +import com.tencent.bk.job.common.cc.sdk.BkNetClient; +import com.tencent.bk.job.common.model.PageData; +import com.tencent.bk.job.common.model.dto.AppResourceScope; +import com.tencent.bk.job.common.model.dto.ApplicationDTO; +import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; +import com.tencent.bk.job.common.util.StringUtil; +import com.tencent.bk.job.common.util.ip.IpUtils; +import com.tencent.bk.job.manage.model.query.HostQuery; +import com.tencent.bk.job.manage.model.web.request.chooser.host.BizTopoNode; +import com.tencent.bk.job.manage.service.ApplicationService; +import com.tencent.bk.job.manage.service.host.BizHostService; +import com.tencent.bk.job.manage.service.host.ScopeHostService; +import com.tencent.bk.job.manage.service.impl.topo.BizTopoService; +import lombok.AllArgsConstructor; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.util.StopWatch; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; + +/** + * 资源范围下的主机服务 + */ +@Slf4j +@Service +public class ScopeHostServiceImpl implements ScopeHostService { + + private final ApplicationService applicationService; + private final BizHostService bizHostService; + private final BkNetClient bkNetClient; + private final BizTopoService bizTopoService; + + @Autowired + public ScopeHostServiceImpl(ApplicationService applicationService, + BizHostService bizHostService, + BkNetClient bkNetClient, + BizTopoService bizTopoService) { + this.applicationService = applicationService; + this.bizHostService = bizHostService; + this.bkNetClient = bkNetClient; + this.bizTopoService = bizTopoService; + } + + @Override + public List filterScopeHostIds(AppResourceScope appResourceScope, + Collection hostIds) { + ApplicationDTO applicationDTO = applicationService.getAppByScope(appResourceScope); + if (applicationDTO.isAllBizSet()) { + // 全业务 + return new ArrayList<>(hostIds); + } else if (applicationDTO.isBizSet()) { + // 业务集 + List bizIds = applicationDTO.getSubBizIds(); + return bizHostService.filterHostIdsByBiz(bizIds, hostIds); + } else { + // 普通业务 + Long bizId = Long.parseLong(applicationDTO.getScope().getId()); + return bizHostService.filterHostIdsByBiz(Collections.singletonList(bizId), hostIds); + } + } + + @Override + public List getScopeHostsByIds(AppResourceScope appResourceScope, + Collection hostIds) { + ApplicationDTO applicationDTO = applicationService.getAppByScope(appResourceScope); + if (applicationDTO.isAllBizSet()) { + // 全业务 + return bizHostService.getHostsByHostIds(hostIds); + } else if (applicationDTO.isBizSet()) { + // 业务集 + List bizIds = applicationDTO.getSubBizIds(); + return bizHostService.getHostsByBizAndHostIds(bizIds, hostIds); + } else { + // 普通业务 + Long bizId = Long.parseLong(applicationDTO.getScope().getId()); + return bizHostService.getHostsByBizAndHostIds(Collections.singletonList(bizId), hostIds); + } + } + + @Override + public List getScopeHostsByIps(AppResourceScope appResourceScope, Collection ips) { + ApplicationDTO applicationDTO = applicationService.getAppByScope(appResourceScope); + if (applicationDTO.isAllBizSet()) { + // 全业务 + return bizHostService.getHostsByIps(ips); + } else if (applicationDTO.isBizSet()) { + // 业务集 + List bizIds = applicationDTO.getSubBizIds(); + return bizHostService.getHostsByBizAndIps(bizIds, ips); + } else { + // 普通业务 + Long bizId = Long.parseLong(applicationDTO.getScope().getId()); + return bizHostService.getHostsByBizAndIps(Collections.singletonList(bizId), ips); + } + } + + @Override + public List getScopeHostsByCloudIps(AppResourceScope appResourceScope, + Collection cloudIps) { + ApplicationDTO applicationDTO = applicationService.getAppByScope(appResourceScope); + if (applicationDTO.isAllBizSet()) { + // 全业务 + return bizHostService.getHostsByCloudIps(cloudIps); + } else if (applicationDTO.isBizSet()) { + // 业务集 + List bizIds = applicationDTO.getSubBizIds(); + return bizHostService.getHostsByBizAndCloudIps(bizIds, cloudIps); + } else { + // 普通业务 + Long bizId = Long.parseLong(applicationDTO.getScope().getId()); + return bizHostService.getHostsByBizAndCloudIps(Collections.singletonList(bizId), cloudIps); + } + } + + @Override + public List getScopeHostsByIpv6s(AppResourceScope appResourceScope, Collection ipv6s) { + ApplicationDTO applicationDTO = applicationService.getAppByScope(appResourceScope); + if (applicationDTO.isAllBizSet()) { + // 全业务 + return bizHostService.getHostsByIpv6s(ipv6s); + } else if (applicationDTO.isBizSet()) { + // 业务集 + List bizIds = applicationDTO.getSubBizIds(); + return bizHostService.getHostsByBizAndIpv6s(bizIds, ipv6s); + } else { + // 普通业务 + Long bizId = Long.parseLong(applicationDTO.getScope().getId()); + return bizHostService.getHostsByBizAndIpv6s(Collections.singletonList(bizId), ipv6s); + } + } + + @Override + public List getScopeHostsByHostNames(AppResourceScope appResourceScope, + Collection hostNames) { + ApplicationDTO applicationDTO = applicationService.getAppByScope(appResourceScope); + if (applicationDTO.isAllBizSet()) { + // 全业务 + return bizHostService.getHostsByHostNames(hostNames); + } else if (applicationDTO.isBizSet()) { + // 业务集 + List bizIds = applicationDTO.getSubBizIds(); + return bizHostService.getHostsByBizAndHostNames(bizIds, hostNames); + } else { + // 普通业务 + Long bizId = Long.parseLong(applicationDTO.getScope().getId()); + return bizHostService.getHostsByBizAndHostNames(Collections.singletonList(bizId), hostNames); + } + } + + @Override + public PageData listHostIdByBizTopologyNodes(AppResourceScope appResourceScope, + List appTopoNodeList, + String searchContent, + Integer agentAlive, + List ipKeyList, + List ipv6KeyList, + List hostNameKeyList, + List osNameKeyList, + Long start, + Long pageSize) { + StopWatch watch = new StopWatch("listHostByBizTopologyNodes"); + watch.start("genConditions"); + BasicParsedSearchConditions basicConditions = buildSearchConditions( + appResourceScope, + appTopoNodeList, + searchContent + ); + watch.stop(); + watch.start("pageListHostId"); + HostQuery hostQuery = HostQuery.builder() + .bizIds(basicConditions.getBizIds()) + .moduleIds(basicConditions.getModuleIds()) + .cloudAreaIds(basicConditions.getCloudAreaIds()) + .searchContents(basicConditions.getSearchContents()) + .agentAlive(agentAlive) + .ipKeyList(ipKeyList) + .ipv6KeyList(ipv6KeyList) + .hostNameKeyList(hostNameKeyList) + .osNameKeyList(osNameKeyList) + .start(start) + .limit(pageSize) + .build(); + PageData result = bizHostService.pageListHostId(hostQuery); + watch.stop(); + if (watch.getTotalTimeMillis() > 5000) { + log.warn("listHostIdByBizTopologyNodes slow:" + watch.prettyPrint()); + } + return result; + } + + @Override + public PageData searchHost(AppResourceScope appResourceScope, + List appTopoNodeList, + Integer agentAlive, + String searchContent, + List ipKeyList, + List ipv6KeyList, + List hostNameKeyList, + List osNameKeyList, + Long start, + Long pageSize) { + BasicParsedSearchConditions basicConditions = buildSearchConditions( + appResourceScope, + appTopoNodeList, + searchContent + ); + if (CollectionUtils.isNotEmpty(ipv6KeyList)) { + // 对有效的IPv6地址补全为全写形式再搜索 + ipv6KeyList = ipv6KeyList.stream().map(ipv6Key -> { + if (IpUtils.checkIpv6(ipv6Key)) { + return IpUtils.getFullIpv6ByCompressedOne(ipv6Key); + } + return ipv6Key; + }).collect(Collectors.toList()); + } + HostQuery hostQuery = HostQuery.builder() + .bizIds(basicConditions.getBizIds()) + .moduleIds(basicConditions.getModuleIds()) + .cloudAreaIds(basicConditions.getCloudAreaIds()) + .searchContents(basicConditions.getSearchContents()) + .agentAlive(agentAlive) + .ipKeyList(ipKeyList) + .ipv6KeyList(ipv6KeyList) + .hostNameKeyList(hostNameKeyList) + .osNameKeyList(osNameKeyList) + .start(start) + .limit(pageSize) + .build(); + return bizHostService.pageListHost(hostQuery); + } + + private BasicParsedSearchConditions buildSearchConditions(AppResourceScope appResourceScope, + List appTopoNodeList, + String searchContent) { + ApplicationDTO applicationDTO = applicationService.getAppByScope(appResourceScope); + List moduleIds = null; + List bizIds = null; + if (applicationDTO.isAllBizSet()) { + // 全业务 + log.debug("listHostIdByBizTopologyNodes of allBizSet:{}", appResourceScope); + } else if (applicationDTO.isBizSet()) { + // 业务集 + bizIds = applicationDTO.getSubBizIds(); + } else { + // 普通业务 + Long bizId = Long.parseLong(applicationDTO.getScope().getId()); + moduleIds = bizTopoService.findAllModuleIdsOfNodes(bizId, appTopoNodeList); + } + + List searchContents = null; + if (searchContent != null) { + searchContents = StringUtil.splitByNormalSeparator(searchContent.trim()).stream() + .filter(StringUtils::isNotBlank) + .collect(Collectors.toList()); + } + + //获取所有云区域，找出名称符合条件的所有CloudAreaId + List cloudAreaIds = bkNetClient.getAnyNameMatchedCloudAreaIds(searchContents); + return new BasicParsedSearchConditions(bizIds, moduleIds, cloudAreaIds, searchContents); + } + + @Getter + @AllArgsConstructor + static class BasicParsedSearchConditions { + List bizIds; + List moduleIds; + List cloudAreaIds; + List searchContents; + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/impl/ScopeTopoHostServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/impl/ScopeTopoHostServiceImpl.java new file mode 100644 index 0000000000..0eec2bcc4c --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/impl/ScopeTopoHostServiceImpl.java @@ -0,0 +1,270 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.service.host.impl; + +import com.tencent.bk.job.common.cc.model.InstanceTopologyDTO; +import com.tencent.bk.job.common.cc.sdk.BkNetClient; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.constant.ResourceScopeTypeEnum; +import com.tencent.bk.job.common.exception.InvalidParamException; +import com.tencent.bk.job.common.i18n.service.MessageI18nService; +import com.tencent.bk.job.common.model.dto.AppResourceScope; +import com.tencent.bk.job.common.model.dto.ApplicationDTO; +import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; +import com.tencent.bk.job.common.model.vo.HostInfoVO; +import com.tencent.bk.job.manage.common.TopologyHelper; +import com.tencent.bk.job.manage.dao.ApplicationHostDAO; +import com.tencent.bk.job.manage.dao.HostTopoDAO; +import com.tencent.bk.job.manage.model.web.vo.CcTopologyNodeVO; +import com.tencent.bk.job.manage.service.ApplicationService; +import com.tencent.bk.job.manage.service.host.ScopeTopoHostService; +import com.tencent.bk.job.manage.service.impl.agent.AgentStatusService; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.util.StopWatch; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@SuppressWarnings("SameParameterValue") +@Slf4j +@Service +public class ScopeTopoHostServiceImpl implements ScopeTopoHostService { + + private final ApplicationHostDAO applicationHostDAO; + private final ApplicationService applicationService; + private final HostTopoDAO hostTopoDAO; + private final TopologyHelper topologyHelper; + private final AgentStatusService agentStatusService; + private final MessageI18nService i18nService; + + @Autowired + public ScopeTopoHostServiceImpl(ApplicationHostDAO applicationHostDAO, + ApplicationService applicationService, + HostTopoDAO hostTopoDAO, + TopologyHelper topologyHelper, + AgentStatusService agentStatusService, + MessageI18nService i18nService) { + this.applicationHostDAO = applicationHostDAO; + this.applicationService = applicationService; + this.hostTopoDAO = hostTopoDAO; + this.topologyHelper = topologyHelper; + this.agentStatusService = agentStatusService; + this.i18nService = i18nService; + } + + @Override + public CcTopologyNodeVO listAppTopologyHostCountTree(String username, + AppResourceScope appResourceScope) { + StopWatch watch = new StopWatch("listAppTopologyHostCountTree"); + watch.start("listAppTopologyHostCountTree"); + CcTopologyNodeVO topologyTree = this.listAppTopologyTree(username, appResourceScope); + watch.stop(); + if (appResourceScope.getType() == ResourceScopeTypeEnum.BIZ) { + watch.start("fillHostInfo"); + fillHostInfo(Long.valueOf(appResourceScope.getId()), topologyTree, false); + watch.stop(); + watch.start("clearHosts"); + clearHosts(topologyTree); + watch.stop(); + } + log.debug(watch.toString()); + return topologyTree; + } + + private CcTopologyNodeVO fillObjInfoForNode(ApplicationDTO appInfo, CcTopologyNodeVO node) { + if (appInfo.isBiz()) { + node.setObjectId("biz"); + node.setObjectName(i18nService.getI18n("cmdb.object.name.biz")); + } else if (appInfo.isBizSet()) { + node.setObjectId("biz_set"); + node.setObjectName(i18nService.getI18n("cmdb.object.name.biz_set")); + } + return node; + } + + public CcTopologyNodeVO listAppTopologyTree(String username, AppResourceScope appResourceScope) { + ApplicationDTO appInfo = applicationService.getAppByAppId(appResourceScope.getAppId()); + if (appInfo == null) { + throw new InvalidParamException(ErrorCode.WRONG_APP_ID); + } + CcTopologyNodeVO ccTopologyNodeVO = fillObjInfoForNode(appInfo, new CcTopologyNodeVO()); + ccTopologyNodeVO.setInstanceId(Long.valueOf(appResourceScope.getId())); + ccTopologyNodeVO.setInstanceName(appInfo.getName()); + if (appInfo.isAllBizSet()) { + // 全业务 + ccTopologyNodeVO.setCount((int) applicationHostDAO.countAllHosts()); + return ccTopologyNodeVO; + } else if (appInfo.isBizSet()) { + // 业务集 + ccTopologyNodeVO.setCount( + (int) applicationHostDAO.countHostsByBizIds(topologyHelper.getBizSetSubBizIds(appInfo)) + ); + return ccTopologyNodeVO; + } + InstanceTopologyDTO instanceTopology = topologyHelper.getTopologyTreeByApplication(appInfo); + return TopologyHelper.convertToCcTopologyTree(instanceTopology); + } + + /** + * 建立moduleId与拓扑树module子节点映射Map + * 顺便初始化IpListStatus + */ + private void constructMap(Map map, CcTopologyNodeVO topologyTree) { + if (topologyTree == null) return; + if (topologyTree.getObjectId().equals("module")) { + topologyTree.setIpListStatus(new ArrayList<>()); + map.put(topologyTree.getInstanceId(), topologyTree); + } else { + List childs = topologyTree.getChild(); + if (childs != null && childs.size() > 0) { + childs.forEach(child -> constructMap(map, child)); + } + } + } + + // 统计拓扑树各节点主机数量 + public void countHosts(CcTopologyNodeVO topologyTree) { + List childs = topologyTree.getChild(); + topologyTree.setHostIdSet(new HashSet<>()); + if (childs != null && !childs.isEmpty()) { + childs.forEach(child -> { + countHosts(child); + topologyTree.getHostIdSet().addAll(child.getHostIdSet()); + }); + topologyTree.setCount(topologyTree.getHostIdSet().size()); + } else { + List hosts = topologyTree.getIpListStatus(); + if (hosts != null) { + topologyTree.getHostIdSet().addAll( + hosts.stream().map(HostInfoVO::getHostId).collect(Collectors.toSet())); + topologyTree.setCount(topologyTree.getHostIdSet().size()); + } else { + topologyTree.setCount(0); + } + } + } + + public void clearHosts(CcTopologyNodeVO topologyTree) { + List childs = topologyTree.getChild(); + topologyTree.setHostIdSet(null); + if (childs != null && !childs.isEmpty()) { + childs.forEach(this::clearHosts); + } + topologyTree.setIpList(null); + topologyTree.setIpListStatus(null); + } + + public void fillHostInfo(Long bizId, CcTopologyNodeVO topologyTree, boolean updateAgentStatus) { + if (topologyTree == null) { + return; + } + StopWatch watch = new StopWatch("fillHostInfo"); + watch.start("constructMap:" + topologyTree.getInstanceId()); + Map map = new HashMap<>(); + constructMap(map, topologyTree); + log.info("{} modules mapped", map.keySet().size()); + if (map.keySet().size() < 200) { + log.info("module ids:{}", map.keySet()); + } else { + log.info("more than 200 module ids, do not print"); + } + watch.stop(); + watch.start("getHosts of Module:" + topologyTree.getInstanceId()); + // 从DB拿主机 + List dbHosts = applicationHostDAO.listHostInfoByBizId(bizId); + log.info("find {} hosts from DB", dbHosts.size()); + watch.stop(); + //批量设置agent状态 + if (updateAgentStatus) { + watch.start("fillRealTimeAgentStatus"); + agentStatusService.fillRealTimeAgentStatus(dbHosts); + watch.stop(); + } + //填充云区域名称 + fillCloudAreaName(dbHosts); + //将主机挂载到topo树 + setHostsToTopoTree(bizId, dbHosts, topologyTree, map, watch); + } + + private void setHostsToTopoTree(Long bizId, + List dbHosts, + CcTopologyNodeVO topologyTree, + Map map, + StopWatch watch) { + List hostInfoVOList = new ArrayList<>(); + for (ApplicationHostDTO dbHost : dbHosts) { + hostInfoVOList.add(dbHost.toVO()); + } + watch.start("setToTopoTree"); + List> hostModuleIdPairList = hostTopoDAO.listHostIdAndModuleIdByBizId(bizId); + Map> hostIdModuleMap = new HashMap<>(); + hostModuleIdPairList.forEach(pair -> { + List moduleIdList = hostIdModuleMap.computeIfAbsent(pair.getLeft(), aLong -> new ArrayList<>()); + moduleIdList.add(pair.getRight()); + }); + for (int i = 0; i < hostInfoVOList.size(); i++) { + ApplicationHostDTO host = dbHosts.get(i); + HostInfoVO hostInfoVO = hostInfoVOList.get(i); + List moduleIdList = hostIdModuleMap.get(host.getHostId()); + if (CollectionUtils.isNotEmpty(moduleIdList)) { + moduleIdList.forEach(moduleId -> { + CcTopologyNodeVO moduleNode = map.get(moduleId); + if (moduleNode == null) { + log.warn("cannot find moduleNode in topoTree, cache may expire, ignore this moduleNode"); + } else { + moduleNode.getIpListStatus().add(hostInfoVO); + } + }); + } else { + log.info("No moduleId found for host:{}, ignore", host); + } + } + watch.stop(); + watch.start("countHosts"); + countHosts(topologyTree); + watch.stop(); + if (watch.getTotalTimeMillis() > 4000) { + log.warn("PERF:SLOW:fillHostInfo: {}", watch.toString()); + } else { + if (log.isDebugEnabled()) { + log.debug("fillHostInfo: {}", watch.toString()); + } + } + } + + private void fillCloudAreaName(List hostDTOList) { + hostDTOList.forEach(hostDTO -> + hostDTO.setCloudAreaName(BkNetClient.getCloudAreaNameFromCache(hostDTO.getCloudAreaId())) + ); + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/impl/WhiteIpAwareScopeHostServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/impl/WhiteIpAwareScopeHostServiceImpl.java new file mode 100644 index 0000000000..e13faff4c5 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/host/impl/WhiteIpAwareScopeHostServiceImpl.java @@ -0,0 +1,210 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.service.host.impl; + +import com.tencent.bk.job.common.model.dto.AppResourceScope; +import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.common.util.StringUtil; +import com.tencent.bk.job.manage.api.common.constants.whiteip.ActionScopeEnum; +import com.tencent.bk.job.manage.service.WhiteIPService; +import com.tencent.bk.job.manage.service.host.HostService; +import com.tencent.bk.job.manage.service.host.ScopeHostService; +import com.tencent.bk.job.manage.service.host.WhiteIpAwareScopeHostService; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * 资源范围下的主机服务 + */ +@Slf4j +@Service +public class WhiteIpAwareScopeHostServiceImpl implements WhiteIpAwareScopeHostService { + + private final WhiteIPService whiteIPService; + private final HostService hostService; + private final ScopeHostService scopeHostService; + + @Autowired + public WhiteIpAwareScopeHostServiceImpl(WhiteIPService whiteIPService, + HostService hostService, + ScopeHostService scopeHostService) { + this.whiteIPService = whiteIPService; + this.hostService = hostService; + this.scopeHostService = scopeHostService; + } + + /** + * 记录检索过程中各阶段未找到的数据，便于排查问题 + * + * @param hostIdSet 用于查询的原始hostId集合 + * @param whiteIpHostDTOList 从白名单数据找到的机器信息 + * @param whiteIpHostWithDetailList 查询机器详情得到的机器信息 + */ + private void recordNotFoundWhiteIP(Set hostIdSet, + List whiteIpHostDTOList, + List whiteIpHostWithDetailList) { + int whiteIpNum = whiteIpHostDTOList.size(); + log.info("{} white ips retrieved:{}", whiteIpNum, whiteIpHostDTOList); + List whiteIpHostIds = + whiteIpHostDTOList.stream().map(HostDTO::getHostId).collect(Collectors.toList()); + if (whiteIpNum < hostIdSet.size()) { + hostIdSet.removeAll(whiteIpHostIds); + log.info("no white ip hosts found by hostIds:{}", hostIdSet); + } + int whiteIpDetailNum = whiteIpHostWithDetailList.size(); + if (whiteIpDetailNum < whiteIpNum) { + Set whiteIpHostDetailIds = + whiteIpHostWithDetailList.stream().map(ApplicationHostDTO::getHostId).collect(Collectors.toSet()); + List hostListCopy = new ArrayList<>(whiteIpHostDTOList); + hostListCopy.removeIf(hostDTO -> whiteIpHostDetailIds.contains(hostDTO.getHostId())); + log.warn("cannot find host detail of {} white ips:{}", hostListCopy.size(), hostListCopy); + } + } + + @Override + public List getScopeHostsIncludingWhiteIPByHostId(AppResourceScope appResourceScope, + ActionScopeEnum actionScope, + Collection hostIds) { + Set hostIdSet = new HashSet<>(hostIds); + List scopeHostList = scopeHostService.getScopeHostsByIds(appResourceScope, hostIdSet); + List finalHostList = new ArrayList<>(scopeHostList); + Set scopeHostIdSet = scopeHostList.stream() + .map(ApplicationHostDTO::getHostId) + .collect(Collectors.toSet()); + hostIdSet.removeAll(scopeHostIdSet); + if (CollectionUtils.isEmpty(hostIdSet)) { + return finalHostList; + } + List whiteIpHostDTOList = whiteIPService.listAvailableWhiteIPHost( + appResourceScope.getAppId(), + actionScope, + hostIdSet + ); + List whiteIpHostList = hostService.listHosts(whiteIpHostDTOList); + log.info("{} white ips added", whiteIpHostList.size()); + if (CollectionUtils.isNotEmpty(whiteIpHostList)) { + finalHostList.addAll(whiteIpHostList); + } + recordNotFoundWhiteIP(hostIdSet, whiteIpHostDTOList, whiteIpHostList); + return finalHostList; + } + + @Override + public List getScopeHostsIncludingWhiteIPByIp(AppResourceScope appResourceScope, + ActionScopeEnum actionScope, + Collection ips) { + List scopeHostList = scopeHostService.getScopeHostsByIps(appResourceScope, ips); + List finalHostList = new ArrayList<>(scopeHostList); + List whiteIpHostDTOList = whiteIPService.listAvailableWhiteIPHostByIps( + appResourceScope.getAppId(), + actionScope, + ips + ); + List whiteIpHostList = hostService.listHosts(whiteIpHostDTOList); + log.info("{} white ips added", whiteIpHostList.size()); + if (CollectionUtils.isNotEmpty(whiteIpHostList)) { + finalHostList.addAll(whiteIpHostList); + } + return finalHostList; + } + + @Override + public List getScopeHostsIncludingWhiteIPByCloudIp(AppResourceScope appResourceScope, + ActionScopeEnum actionScope, + Collection cloudIps) { + List scopeHostList = scopeHostService.getScopeHostsByCloudIps(appResourceScope, cloudIps); + List finalHostList = new ArrayList<>(scopeHostList); + Map map = hostService.listHostsByIps(cloudIps); + Set hostIds = map.values().stream().map(ApplicationHostDTO::getHostId).collect(Collectors.toSet()); + List whiteIpHostList = listWhiteIpHostsByIds(appResourceScope, actionScope, hostIds); + log.info("{} white ips added", whiteIpHostList.size()); + if (CollectionUtils.isNotEmpty(whiteIpHostList)) { + finalHostList.addAll(whiteIpHostList); + } + return finalHostList; + } + + private List listWhiteIpHostsByIds(AppResourceScope appResourceScope, + ActionScopeEnum actionScope, + Collection hostIds) { + List whiteIpHostDTOList = whiteIPService.listAvailableWhiteIPHost( + appResourceScope.getAppId(), + actionScope, + hostIds + ); + return hostService.listHosts(whiteIpHostDTOList); + } + + @Override + public List getScopeHostsIncludingWhiteIPByIpv6(AppResourceScope appResourceScope, + ActionScopeEnum actionScope, + Collection ipv6s) { + List scopeHostList = scopeHostService.getScopeHostsByIpv6s(appResourceScope, ipv6s); + List finalHostList = new ArrayList<>(scopeHostList); + List whiteIpHostDTOList = whiteIPService.listAvailableWhiteIPHostByIpv6s( + appResourceScope.getAppId(), + actionScope, + ipv6s + ); + List whiteIpHostList = hostService.listHosts(whiteIpHostDTOList); + log.info("{} white ips added", whiteIpHostList.size()); + if (CollectionUtils.isNotEmpty(whiteIpHostList)) { + finalHostList.addAll(whiteIpHostList); + } + return finalHostList; + } + + @Override + public List getScopeHostsIncludingWhiteIPByKey(AppResourceScope appResourceScope, + ActionScopeEnum actionScope, + Collection keys) { + // 1.数字优先解析为hostId进行匹配 + Pair, List> hostIdValuePair = StringUtil.extractValueFromStrings(keys, Long.class); + List hostIdList = hostIdValuePair.getLeft(); + List remainKeys = hostIdValuePair.getRight(); + List finalHostList = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(hostIdList)) { + if (log.isDebugEnabled()) { + log.debug("hostIdList={}, remainKeys={}", hostIdList, remainKeys); + } + finalHostList.addAll(scopeHostService.getScopeHostsByIds(appResourceScope, hostIdList)); + } + // 2.剩余关键字通过主机名称匹配 + finalHostList.addAll(scopeHostService.getScopeHostsByHostNames(appResourceScope, remainKeys)); + return finalHostList; + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/AccountServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/AccountServiceImpl.java index 30af79ff0d..daeb3d0164 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/AccountServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/AccountServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,15 +24,25 @@ package com.tencent.bk.job.manage.service.impl; +import com.tencent.bk.audit.annotations.ActionAuditRecord; +import com.tencent.bk.audit.annotations.AuditInstanceRecord; +import com.tencent.bk.audit.context.ActionAuditContext; +import com.tencent.bk.job.common.audit.constants.EventContentConstants; +import com.tencent.bk.job.common.constant.AccountCategoryEnum; import com.tencent.bk.job.common.constant.ErrorCode; -import com.tencent.bk.job.common.encrypt.Encryptor; +import com.tencent.bk.job.common.crypto.Encryptor; import com.tencent.bk.job.common.exception.AlreadyExistsException; -import com.tencent.bk.job.common.exception.InternalException; +import com.tencent.bk.job.common.exception.FailedPreconditionException; import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.exception.NotFoundException; import com.tencent.bk.job.common.exception.ServiceException; +import com.tencent.bk.job.common.iam.constant.ActionId; +import com.tencent.bk.job.common.iam.constant.ResourceTypeId; +import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; +import com.tencent.bk.job.common.model.dto.AppResourceScope; +import com.tencent.bk.job.common.util.ArrayUtil; import com.tencent.bk.job.common.util.Utils; import com.tencent.bk.job.common.util.check.IlegalCharChecker; import com.tencent.bk.job.common.util.check.MaxLengthChecker; @@ -40,19 +50,17 @@ import com.tencent.bk.job.common.util.check.StringCheckHelper; import com.tencent.bk.job.common.util.check.TrimChecker; import com.tencent.bk.job.common.util.check.exception.StringCheckException; -import com.tencent.bk.job.common.util.crypto.AESUtils; import com.tencent.bk.job.common.util.date.DateUtils; -import com.tencent.bk.job.manage.common.consts.account.AccountCategoryEnum; -import com.tencent.bk.job.manage.common.consts.account.AccountTypeEnum; -import com.tencent.bk.job.manage.common.consts.globalsetting.OSTypeEnum; -import com.tencent.bk.job.manage.config.JobManageConfig; +import com.tencent.bk.job.manage.api.common.constants.OSTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.account.AccountTypeEnum; +import com.tencent.bk.job.manage.auth.AccountAuthService; import com.tencent.bk.job.manage.dao.AccountDAO; import com.tencent.bk.job.manage.model.dto.AccountDTO; import com.tencent.bk.job.manage.model.dto.AccountDisplayDTO; import com.tencent.bk.job.manage.model.web.request.AccountCreateUpdateReq; import com.tencent.bk.job.manage.model.web.request.globalsetting.AccountNameRule; import com.tencent.bk.job.manage.service.AccountService; -import com.tencent.bk.job.manage.service.GlobalSettingsService; +import com.tencent.bk.job.manage.service.globalsetting.GlobalSettingsService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; @@ -70,23 +78,26 @@ @Service public class AccountServiceImpl implements AccountService { public final String DEFAULT_LINUX_ACCOUNT = "root"; - public final String DEFAULT_WINDOWS_ACCOUNT = "system"; + public final String DEFAULT_WINDOWS_SYSTEM_ACCOUNT = "system"; + public final String DEFAULT_WINDOWS_ADMIN_ACCOUNT = "Administrator"; private final AccountDAO accountDAO; private final Encryptor encryptor; private final GlobalSettingsService globalSettingsService; - private final JobManageConfig jobManageConfig; + private final AccountAuthService accountAuthService; - public AccountServiceImpl(@Autowired AccountDAO accountDAO, @Qualifier("gseRsaEncryptor") Encryptor encryptor, + @Autowired + public AccountServiceImpl(AccountDAO accountDAO, + @Qualifier("gseRsaEncryptor") Encryptor encryptor, GlobalSettingsService globalSettingsService, - JobManageConfig jobManageConfig) { + AccountAuthService accountAuthService) { this.accountDAO = accountDAO; this.encryptor = encryptor; this.globalSettingsService = globalSettingsService; - this.jobManageConfig = jobManageConfig; + this.accountAuthService = accountAuthService; } @Override - public long saveAccount(AccountDTO account) throws ServiceException { + public AccountDTO createAccount(AccountDTO account) throws ServiceException { log.info("Save account, account={}", account); AccountDTO existAccount = accountDAO.getAccount(account.getAppId(), account.getCategory(), account.getAlias()); if (existAccount != null) { @@ -105,19 +116,56 @@ public long saveAccount(AccountDTO account) throws ServiceException { throw new NotFoundException(ErrorCode.DB_SYSTEM_ACCOUNT_IS_INVALID); } if (!dbSystemAccount.getAppId().equals(account.getAppId())) { - log.warn("DB related system account is not in current app, systemAccountId={}, systemAccountAppId={}" + log.warn("DB related system account is not in current app, systemAccountId={}, " + + "systemAccountAppId={}" , account.getDbSystemAccountId(), dbSystemAccount.getAppId()); throw new NotFoundException(ErrorCode.DB_SYSTEM_ACCOUNT_IS_INVALID); } - if (StringUtils.isNotEmpty(account.getDbPassword())) { - account.setDbPassword(encryptPassword(account.getDbPassword())); - } } + + long accountId; if (account.getId() == null) { - return accountDAO.saveAccount(account); + accountId = accountDAO.saveAccount(account); } else { - return accountDAO.saveAccountWithId(account); + accountId = accountDAO.saveAccountWithId(account); } + + return getAccountById(accountId); + } + + @Override + @ActionAuditRecord( + actionId = ActionId.CREATE_ACCOUNT, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.ACCOUNT, + instanceIds = "#$?.id", + instanceNames = "#$?.alias" + ), + content = EventContentConstants.CREATE_ACCOUNT + ) + public AccountDTO createAccount(String username, AccountDTO account) { + authCreateAccount(username, account.getAppId()); + AccountDTO createdAccount = createAccount(account); + accountAuthService.registerAccount( + username, + createdAccount.getId(), + createdAccount.getAlias() + ); + return createdAccount; + } + + private void authCreateAccount(String username, long appId) throws PermissionDeniedException { + accountAuthService.authCreateAccount(username, new AppResourceScope(appId)).denyIfNoPermission(); + } + + private void authUseAccount(String username, long appId, long accountId) throws PermissionDeniedException { + accountAuthService.authUseAccount(username, new AppResourceScope(appId), accountId, null) + .denyIfNoPermission(); + } + + private void authManageAccount(String username, long appId, long accountId) throws PermissionDeniedException { + accountAuthService.authManageAccount(username, new AppResourceScope(appId), accountId, null) + .denyIfNoPermission(); } @Override @@ -125,13 +173,42 @@ public AccountDTO getAccountById(Long accountId) throws ServiceException { return accountDAO.getAccountById(accountId); } + @Override + @ActionAuditRecord( + actionId = ActionId.USE_ACCOUNT, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.ACCOUNT, + instanceIds = "#accountId", + instanceNames = "#$?.alias" + ), + content = EventContentConstants.USE_ACCOUNT + ) + public AccountDTO getAccount(String username, long appId, Long accountId) { + authUseAccount(username, appId, accountId); + return getAccount(appId, accountId); + } + + @Override + public AccountDTO getAccount(long appId, Long accountId) { + AccountDTO account = getAccountById(accountId); + if (account == null) { + log.info("Account is not exist, accountId={}", accountId); + throw new NotFoundException(ErrorCode.ACCOUNT_NOT_EXIST, ArrayUtil.toArray(accountId)); + } + if (!account.getAppId().equals(appId)) { + log.info("Account is not in app, appId={}, accountId={}", appId, accountId); + throw new NotFoundException(ErrorCode.ACCOUNT_NOT_EXIST, ArrayUtil.toArray(accountId)); + } + return account; + } + @Override public Map getAccountDisplayInfoMapByIds( Collection accountIds) throws ServiceException { Map map = new HashMap<>(); List accountDisplayDTOList = accountDAO.listAccountDisplayInfoByIds(accountIds); for (AccountDisplayDTO accountDisplayDTO : accountDisplayDTOList) { - map.put(accountDisplayDTO.getId(),accountDisplayDTO); + map.put(accountDisplayDTO.getId(), accountDisplayDTO); } return map; } @@ -142,30 +219,92 @@ public AccountDTO getAccountByAccount(Long appId, String account) throws Service } @Override - public void updateAccount(AccountDTO account) throws ServiceException { - if (StringUtils.isNotEmpty(account.getPassword())) { - account.setPassword(encryptor.encrypt(account.getPassword())); + @ActionAuditRecord( + actionId = ActionId.MANAGE_ACCOUNT, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.ACCOUNT, + instanceIds = "#updateAccount?.id", + instanceNames = "#updateAccount?.alias" + ), + content = EventContentConstants.EDIT_ACCOUNT + ) + public AccountDTO updateAccount(String username, AccountDTO updateAccount) throws ServiceException { + authManageAccount(username, updateAccount.getAppId(), updateAccount.getId()); + + AccountDTO originAccount = getAccount(updateAccount.getAppId(), updateAccount.getId()); + + checkAccountAliasExist(updateAccount.getAppId(), updateAccount.getId(), + originAccount.getCategory(), updateAccount.getAlias()); + + if (StringUtils.isNotEmpty(updateAccount.getPassword())) { + updateAccount.setPassword(encryptor.encrypt(updateAccount.getPassword())); } - if (account.getCategory() == AccountCategoryEnum.DB && StringUtils.isNotEmpty(account.getDbPassword())) { - account.setDbPassword(encryptPassword(account.getDbPassword())); + if (updateAccount.getCategory() == AccountCategoryEnum.DB + && StringUtils.isNotEmpty(updateAccount.getDbPassword())) { + updateAccount.setDbPassword(encryptor.encrypt(updateAccount.getPassword())); } - log.info("Update account, account={}", account); - accountDAO.updateAccount(account); + // 账号用途、账号类型、账号名称不允许修改 + updateAccount.setCategory(originAccount.getCategory()); + updateAccount.setType(originAccount.getType()); + updateAccount.setAccount(originAccount.getAccount()); + + log.info("Update account, account={}", updateAccount); + accountDAO.updateAccount(updateAccount); + AccountDTO updatedAccount = getAccountById(updateAccount.getId()); + + // 审计 + ActionAuditContext.current() + .setOriginInstance(originAccount.toEsbAccountV3DTO()) + .setInstance(updatedAccount.toEsbAccountV3DTO()); + + return updatedAccount; } - private String encryptPassword(String text) throws ServiceException { - try { - return AESUtils.encryptToBase64EncodedCipherText(text, jobManageConfig.getEncryptPassword()); - } catch (Exception e) { - log.error("Encrypt password error", e); - throw new InternalException(ErrorCode.INTERNAL_ERROR); + private void checkAccountAliasExist(long appId, long accountId, AccountCategoryEnum category, String alias) { + AccountDTO existAccount = accountDAO.getAccount(appId, category, alias); + if (existAccount != null && !existAccount.getId().equals(accountId)) { + log.info( + "Another same alias exists:(appId={}, category={}, alias={})", + existAccount.getAppId(), + existAccount.getCategory(), + existAccount.getAlias() + ); + throw new AlreadyExistsException(ErrorCode.ACCOUNT_ALIAS_EXIST); } } @Override - public void deleteAccount(Long accountId) throws ServiceException { + @ActionAuditRecord( + actionId = ActionId.MANAGE_ACCOUNT, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.ACCOUNT, + instanceIds = "#accountId" + ), + content = EventContentConstants.DELETE_ACCOUNT + ) + public void deleteAccount(String username, long appId, Long accountId) throws ServiceException { + log.info("Delete account, operator={}, appId={}, accountId={}", username, appId, accountId); + AccountDTO account = getAccountById(accountId); + if (account == null) { + log.info("Account is not exist, accountId={}", accountId); + throw new NotFoundException(ErrorCode.ACCOUNT_NOT_EXIST); + } + + authManageAccount(username, account.getAppId(), accountId); + + if (isAccountRefByAnyStep(accountId)) { + log.info("Account:{} is ref by step, should not delete!", accountId); + throw new FailedPreconditionException(ErrorCode.DELETE_REF_ACCOUNT_FORBIDDEN); + } + if (account.getCategory() == AccountCategoryEnum.SYSTEM && isSystemAccountRefByDbAccount(accountId)) { + log.info("Account:{} is ref by db account, should not delete!", accountId); + throw new FailedPreconditionException(ErrorCode.DELETE_REF_ACCOUNT_FORBIDDEN); + } + log.info("Delete account, accountId={}", accountId); accountDAO.deleteAccount(accountId); + + ActionAuditContext.current().setInstanceName(account.getAccount()); } @Override @@ -186,19 +325,22 @@ public AccountDTO getAccount(Long appId, AccountCategoryEnum category, String al } @Override - public List listAllAppAccount(Long appId, AccountCategoryEnum category) { - return accountDAO.listAllAppAccount(appId, category, null); + public List listAppAccount(Long appId, AccountCategoryEnum category) { + return accountDAO.listAppAccount(appId, category, null, null, null); } @Override - public List listAllAppAccount(Long appId, AccountCategoryEnum category, - BaseSearchCondition baseSearchCondition) { - return accountDAO.listAllAppAccount(appId, category, baseSearchCondition); + public List listAppAccount(Long appId, + AccountCategoryEnum category, + String account, + String alias, + BaseSearchCondition baseSearchCondition) { + return accountDAO.listAppAccount(appId, category, account, alias, baseSearchCondition); } @Override - public Integer countAllAppAccount(Long appId, AccountCategoryEnum category) { - return accountDAO.countAllAppAccount(appId, category); + public Integer countAppAccount(Long appId, AccountCategoryEnum category, String account, String alias) { + return accountDAO.countAppAccount(appId, category, account, alias); } @Override @@ -339,14 +481,14 @@ public void createDefaultAccounts(long appId) { } } AccountDTO windowsSystem = accountDAO.getAccount(appId, AccountCategoryEnum.SYSTEM, AccountTypeEnum.WINDOW, - DEFAULT_WINDOWS_ACCOUNT, DEFAULT_WINDOWS_ACCOUNT); + DEFAULT_WINDOWS_SYSTEM_ACCOUNT, DEFAULT_WINDOWS_SYSTEM_ACCOUNT); if (windowsSystem == null) { windowsSystem = new AccountDTO(); windowsSystem.setAppId(appId); windowsSystem.setCategory(AccountCategoryEnum.SYSTEM); windowsSystem.setType(AccountTypeEnum.WINDOW); - windowsSystem.setAccount(DEFAULT_WINDOWS_ACCOUNT); - windowsSystem.setAlias(DEFAULT_WINDOWS_ACCOUNT); + windowsSystem.setAccount(DEFAULT_WINDOWS_SYSTEM_ACCOUNT); + windowsSystem.setAlias(DEFAULT_WINDOWS_SYSTEM_ACCOUNT); windowsSystem.setCreator("admin"); windowsSystem.setCreateTime(DateUtils.currentTimeMillis()); windowsSystem.setLastModifyUser("admin"); @@ -357,6 +499,25 @@ public void createDefaultAccounts(long appId) { log.warn("Fail to create default system account", e); } } + AccountDTO windowsAdmin = accountDAO.getAccount(appId, AccountCategoryEnum.SYSTEM, AccountTypeEnum.WINDOW, + DEFAULT_WINDOWS_ADMIN_ACCOUNT, DEFAULT_WINDOWS_ADMIN_ACCOUNT); + if (windowsAdmin == null) { + windowsAdmin = new AccountDTO(); + windowsAdmin.setAppId(appId); + windowsAdmin.setCategory(AccountCategoryEnum.SYSTEM); + windowsAdmin.setType(AccountTypeEnum.WINDOW); + windowsAdmin.setAccount(DEFAULT_WINDOWS_ADMIN_ACCOUNT); + windowsAdmin.setAlias(DEFAULT_WINDOWS_ADMIN_ACCOUNT); + windowsAdmin.setCreator("admin"); + windowsAdmin.setCreateTime(DateUtils.currentTimeMillis()); + windowsAdmin.setLastModifyUser("admin"); + try { + accountDAO.saveAccount(windowsAdmin); + log.info("administrator account of appId={} created", appId); + } catch (Exception e) { + log.warn("Fail to create default administrator account", e); + } + } } @Override diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/AppRoleServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/AppRoleServiceImpl.java index 740b15acbe..92553b7ba5 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/AppRoleServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/AppRoleServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/AppScopeMappingServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/AppScopeMappingServiceImpl.java index f95363a856..2f8f2a783d 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/AppScopeMappingServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/AppScopeMappingServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,6 +28,7 @@ import com.tencent.bk.job.common.exception.NotFoundException; import com.tencent.bk.job.common.model.dto.ResourceScope; import com.tencent.bk.job.manage.AbstractLocalCacheAppScopeMappingService; +import com.tencent.bk.job.manage.GlobalAppScopeMappingService; import com.tencent.bk.job.manage.service.ApplicationService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; @@ -42,6 +43,7 @@ public class AppScopeMappingServiceImpl extends AbstractLocalCacheAppScopeMappin @Autowired public AppScopeMappingServiceImpl(ApplicationService applicationService) { this.applicationService = applicationService; + GlobalAppScopeMappingService.register(this); } @Override diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/ApplicationFavorService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/ApplicationFavorService.java index aaf5b79c75..bd59c38bb5 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/ApplicationFavorService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/ApplicationFavorService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/ApplicationServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/ApplicationServiceImpl.java index 9b3abe25ee..b228935517 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/ApplicationServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/ApplicationServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,6 +26,7 @@ import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.constant.ResourceScopeTypeEnum; +import com.tencent.bk.job.common.exception.InternalException; import com.tencent.bk.job.common.exception.NotFoundException; import com.tencent.bk.job.common.model.dto.ApplicationDTO; import com.tencent.bk.job.common.model.dto.ResourceScope; @@ -35,7 +36,7 @@ import com.tencent.bk.job.manage.service.AccountService; import com.tencent.bk.job.manage.service.ApplicationService; import lombok.extern.slf4j.Slf4j; -import org.jooq.DSLContext; +import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; @@ -52,17 +53,14 @@ @Service public class ApplicationServiceImpl implements ApplicationService { - private final DSLContext dslContext; private final ApplicationDAO applicationDAO; private final ApplicationCache applicationCache; private final AccountService accountService; @Autowired - public ApplicationServiceImpl(DSLContext dslContext, - ApplicationDAO applicationDAO, + public ApplicationServiceImpl(ApplicationDAO applicationDAO, ApplicationCache applicationCache, AccountService accountService) { - this.dslContext = dslContext; this.applicationDAO = applicationDAO; this.applicationCache = applicationCache; this.accountService = accountService; @@ -94,7 +92,8 @@ public ResourceScope getScopeByAppId(Long appId) { @Override public Map getScopeByAppIds(Collection appIds) { List applications = listAppsByAppIds(appIds); - return applications.stream().collect(Collectors.toMap(ApplicationDTO::getId, ApplicationDTO::getScope)); + return applications.stream().collect( + Collectors.toMap(ApplicationDTO::getId, ApplicationDTO::getScope, (oldValue, newValue) -> newValue)); } @Override @@ -120,6 +119,7 @@ public ApplicationDTO getAppByAppId(Long appId) throws NotFoundException { throw new NotFoundException(ErrorCode.APP_NOT_EXIST); } } + checkApplication(application); return application; } @@ -135,9 +135,25 @@ public ApplicationDTO getAppByScope(ResourceScope resourceScope) { throw new NotFoundException("App not found, resourceScope:" + resourceScope, ErrorCode.APP_NOT_EXIST); } } + checkApplication(application); return application; } + private void checkApplication(ApplicationDTO application) { + if (application == null) { + return; + } + if (application.getId() == null) { + log.error("Empty appId"); + throw new InternalException("Empty appId for application", ErrorCode.INTERNAL_ERROR); + } + if (application.getScope() == null || application.getScope().getType() == null + || StringUtils.isBlank(application.getScope().getId())) { + log.error("Invalid resource scope, scope: {}", application.getScope()); + throw new InternalException("Invalid resource scope", ErrorCode.INTERNAL_ERROR); + } + } + @Override public ApplicationDTO getAppByScope(String scopeType, String scopeId) { return getAppByScope(new ResourceScope(scopeType, scopeId)); @@ -205,7 +221,7 @@ public List listAppsByScopeType(ResourceScopeTypeEnum scopeType) @Override public Long createApp(ApplicationDTO application) { - Long appId = applicationDAO.insertApp(dslContext, application); + Long appId = applicationDAO.insertApp(application); application.setId(appId); applicationCache.addOrUpdateApp(application); try { @@ -230,21 +246,21 @@ public List listAllApps() { @Override public void updateApp(ApplicationDTO application) { log.info("Update app: {}", JsonUtils.toJson(application)); - applicationDAO.updateApp(dslContext, application); + applicationDAO.updateApp(application); applicationCache.addOrUpdateApp(application); } @Override public void deleteApp(Long appId) { log.info("Delete app[{}]", appId); - applicationDAO.deleteAppByIdSoftly(dslContext, appId); + applicationDAO.deleteAppByIdSoftly(appId); applicationCache.deleteApp(appId); } @Override public void restoreDeletedApp(long appId) { log.info("Restore deleted app[{}]", appId); - applicationDAO.restoreDeletedApp(dslContext, appId); + applicationDAO.restoreDeletedApp(appId); ApplicationDTO restoredApplication = applicationDAO.getAppById(appId); applicationCache.addOrUpdateApp(restoredApplication); } @@ -253,4 +269,9 @@ public void restoreDeletedApp(long appId) { public ApplicationDTO getAppByScopeIncludingDeleted(ResourceScope scope) { return applicationDAO.getAppByScopeIncludingDeleted(scope); } + + @Override + public List listAllDeletedApps() { + return applicationDAO.listAllDeletedApps(); + } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/BizDynamicGroupService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/BizDynamicGroupService.java new file mode 100644 index 0000000000..2ca2eb35f6 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/BizDynamicGroupService.java @@ -0,0 +1,64 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.service.impl; + +import com.tencent.bk.job.common.cc.model.CcDynamicGroupDTO; +import com.tencent.bk.job.common.cc.sdk.BizCmdbClient; +import com.tencent.bk.job.manage.model.dto.DynamicGroupDTO; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +@Slf4j +@Service +public class BizDynamicGroupService { + + private final BizCmdbClient bizCmdbClient; + + @Autowired + public BizDynamicGroupService(BizCmdbClient bizCmdbClient) { + this.bizCmdbClient = bizCmdbClient; + } + + public List listDynamicGroup(Long bizId) { + List ccGroupList = bizCmdbClient.getDynamicGroupList(bizId); + return ccGroupList.stream().map(DynamicGroupDTO::fromCcGroupDTO).collect(Collectors.toList()); + } + + public List listDynamicGroup(Long bizId, Collection ids) { + List dynamicGroupList = listDynamicGroup(bizId); + if (ids == null) { + return dynamicGroupList; + } + Set idSet = new HashSet<>(ids); + return dynamicGroupList.stream().filter(it -> idSet.contains(it.getId())).collect(Collectors.toList()); + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/BizHostInfoQueryServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/BizHostInfoQueryServiceImpl.java new file mode 100644 index 0000000000..ff0944c1d9 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/BizHostInfoQueryServiceImpl.java @@ -0,0 +1,76 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.service.impl; + +import com.tencent.bk.job.common.gse.service.BizHostInfoQueryService; +import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; +import com.tencent.bk.job.common.util.CollectionUtil; +import com.tencent.bk.job.manage.service.host.HostService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +@Slf4j +@Service("jobManageBizHostInfoQueryService") +public class BizHostInfoQueryServiceImpl implements BizHostInfoQueryService { + + private final HostService hostService; + + @Autowired + public BizHostInfoQueryServiceImpl(HostService hostService) { + this.hostService = hostService; + } + + @Override + public Map queryBizIdsByHostId(Collection hostIds) { + List hostList = queryHosts(hostIds); + return CollectionUtil.convertToMap(hostList, ApplicationHostDTO::getHostId, ApplicationHostDTO::getBizId); + } + + @Override + public Map queryAgentIdsByHostId(Collection hostIds) { + List hostList = queryHosts(hostIds); + return CollectionUtil.convertToMap(hostList, ApplicationHostDTO::getHostId, ApplicationHostDTO::getAgentId); + } + + @Override + public Map queryCloudIpsByHostId(Collection hostIds) { + List hostList = queryHosts(hostIds); + return CollectionUtil.convertToMap(hostList, ApplicationHostDTO::getHostId, ApplicationHostDTO::getCloudIp); + } + + private List queryHosts(Collection hostIds) { + Set hostDTOSet = hostIds.stream().map(HostDTO::fromHostId).collect(Collectors.toSet()); + return hostService.listHosts(hostDTOSet); + } + +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/BizSetService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/BizSetService.java index 9df65ee6b0..59ebe58296 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/BizSetService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/BizSetService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,7 +25,7 @@ package com.tencent.bk.job.manage.service.impl; import com.tencent.bk.job.common.util.TimeUtil; -import com.tencent.bk.job.manage.common.consts.globalsetting.GlobalSettingKeys; +import com.tencent.bk.job.manage.api.common.constants.globalsetting.GlobalSettingKeys; import com.tencent.bk.job.manage.dao.ApplicationDAO; import com.tencent.bk.job.manage.dao.globalsetting.GlobalSettingDAO; import com.tencent.bk.job.manage.model.dto.GlobalSettingDTO; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/ContainerServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/ContainerServiceImpl.java new file mode 100644 index 0000000000..1561d32b18 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/ContainerServiceImpl.java @@ -0,0 +1,77 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.service.impl; + +import com.tencent.bk.job.common.cc.model.container.ContainerDetailDTO; +import com.tencent.bk.job.common.cc.model.container.KubeTopologyDTO; +import com.tencent.bk.job.common.cc.model.req.ListKubeContainerByTopoReq; +import com.tencent.bk.job.common.cc.sdk.BizCmdbClient; +import com.tencent.bk.job.common.model.PageData; +import com.tencent.bk.job.manage.model.query.ContainerQuery; +import com.tencent.bk.job.manage.service.ContainerService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.List; + +@Service +@Slf4j +public class ContainerServiceImpl implements ContainerService { + + private final BizCmdbClient bizCmdbClient; + + @Autowired + public ContainerServiceImpl(BizCmdbClient bizCmdbClient) { + this.bizCmdbClient = bizCmdbClient; + } + + + @Override + public KubeTopologyDTO getBizKubeCacheTopo(long bizId) { + return bizCmdbClient.getBizKubeCacheTopo(bizId); + } + + @Override + public PageData listPageKubeContainerByTopo(ContainerQuery query) { + return bizCmdbClient.listPageKubeContainerByTopo(query.toListKubeContainerByTopoReq()); + } + + @Override + public List listKubeContainerByTopo(ContainerQuery query) { + ListKubeContainerByTopoReq req = query.toListKubeContainerByTopoReq(); + return bizCmdbClient.listKubeContainerByTopo(req); + } + + @Override + public List listKubeContainerByIds(Long bizId, List containerIds) { + return bizCmdbClient.listKubeContainerByIds(bizId, containerIds); + } + + @Override + public List listKubeContainerByUIds(Long bizId, List containerUIds) { + return bizCmdbClient.listKubeContainerByUIds(bizId, containerUIds); + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/CredentialServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/CredentialServiceImpl.java index cbcf91c498..39964ccec3 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/CredentialServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/CredentialServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,18 +24,28 @@ package com.tencent.bk.job.manage.service.impl; +import com.tencent.bk.audit.annotations.ActionAuditRecord; +import com.tencent.bk.audit.annotations.AuditInstanceRecord; +import com.tencent.bk.audit.context.ActionAuditContext; +import com.tencent.bk.job.common.audit.constants.EventContentConstants; import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.FailedPreconditionException; import com.tencent.bk.job.common.exception.NotFoundException; +import com.tencent.bk.job.common.iam.constant.ActionId; +import com.tencent.bk.job.common.iam.constant.ResourceTypeId; import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; +import com.tencent.bk.job.common.model.dto.AppResourceScope; +import com.tencent.bk.job.common.mysql.JobTransactional; +import com.tencent.bk.job.file_gateway.api.inner.ServiceFileSourceResource; +import com.tencent.bk.job.manage.auth.TicketAuthService; import com.tencent.bk.job.manage.dao.CredentialDAO; import com.tencent.bk.job.manage.model.dto.CredentialDTO; -import com.tencent.bk.job.manage.model.inner.resp.ServiceCredentialDTO; import com.tencent.bk.job.manage.model.inner.resp.ServiceCredentialDisplayDTO; import com.tencent.bk.job.manage.model.web.request.CredentialCreateUpdateReq; import com.tencent.bk.job.manage.service.CredentialService; -import org.apache.commons.lang3.StringUtils; -import org.jooq.DSLContext; +import lombok.extern.slf4j.Slf4j; +import org.slf4j.helpers.MessageFormatter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -43,15 +53,20 @@ import java.util.List; @Service +@Slf4j public class CredentialServiceImpl implements CredentialService { - private final DSLContext dslContext; private final CredentialDAO credentialDAO; + private final TicketAuthService ticketAuthService; + private final ServiceFileSourceResource fileSourceService; @Autowired - public CredentialServiceImpl(DSLContext dslContext, CredentialDAO credentialDAO) { - this.dslContext = dslContext; + public CredentialServiceImpl(CredentialDAO credentialDAO, + TicketAuthService ticketAuthService, + ServiceFileSourceResource fileSourceService) { this.credentialDAO = credentialDAO; + this.ticketAuthService = ticketAuthService; + this.fileSourceService = fileSourceService; } @Override @@ -63,62 +78,143 @@ public PageData listCredentials( } @Override - public String saveCredential(String username, Long appId, CredentialCreateUpdateReq createUpdateReq) { + public PageData listCredentialBasicInfo(Long appId, BaseSearchCondition baseSearchCondition) { + return credentialDAO.listCredentialBasicInfo(appId, baseSearchCondition); + } + + @Override + @ActionAuditRecord( + actionId = ActionId.CREATE_TICKET, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.TICKET, + instanceIds = "#$?.id", + instanceNames = "#$?.name" + ), + content = EventContentConstants.CREATE_TICKET + ) + @JobTransactional( + transactionManager = "jobManageTransactionManager", + rollbackFor = {Exception.class, Error.class} + ) + public CredentialDTO createCredential(String username, Long appId, CredentialCreateUpdateReq createUpdateReq) { + authCreateTicket(username, appId); + + CredentialDTO credentialDTO = buildCredentialDTO(username, appId, createUpdateReq); + credentialDTO.setCreator(username); + credentialDTO.setCreateTime(credentialDTO.getLastModifyTime()); + String id = credentialDAO.insertCredential(credentialDTO); + Boolean registerResult = ticketAuthService.registerTicket(username, id, createUpdateReq.getName()); + if (!registerResult) { + log.warn("Fail to register ticket to iam:({},{})", id, createUpdateReq.getName()); + } + return getCredentialById(id); + } + + public void authCreateTicket(String username, long appId) { + ticketAuthService.authCreateTicket(username, new AppResourceScope(appId)).denyIfNoPermission(); + } + + private void authManageTicket(String username, long appId, String credentialId) { + ticketAuthService.authManageTicket(username, new AppResourceScope(appId), credentialId, null) + .denyIfNoPermission(); + } + + @Override + @ActionAuditRecord( + actionId = ActionId.MANAGE_TICKET, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.TICKET, + instanceIds = "#createUpdateReq?.id", + instanceNames = "#createUpdateReq?.name" + ), + content = EventContentConstants.EDIT_TICKET + ) + public CredentialDTO updateCredential(String username, Long appId, CredentialCreateUpdateReq createUpdateReq) { String id = createUpdateReq.getId(); + authManageTicket(username, appId, id); + CredentialDTO credentialDTO = buildCredentialDTO(username, appId, createUpdateReq); - if (StringUtils.isNotBlank(id)) { - CredentialDTO oldCredentialDTO = credentialDAO.getCredentialById(dslContext, id); - if (oldCredentialDTO == null) { - throw new NotFoundException(ErrorCode.CREDENTIAL_NOT_EXIST); - } - String value1 = createUpdateReq.getValue1(); - if ("******".equals(value1)) { - credentialDTO.setFirstValue(oldCredentialDTO.getFirstValue()); - } else { - credentialDTO.setFirstValue(value1); - } - String value2 = createUpdateReq.getValue2(); - if ("******".equals(value2)) { - credentialDTO.setSecondValue(oldCredentialDTO.getSecondValue()); - } else { - credentialDTO.setSecondValue(value2); - } - return credentialDAO.updateCredentialById(dslContext, credentialDTO); + CredentialDTO originCredentialDTO = credentialDAO.getCredentialById(id); + if (originCredentialDTO == null) { + throw new NotFoundException(ErrorCode.CREDENTIAL_NOT_EXIST); + } + + String value1 = createUpdateReq.getValue1(); + if ("******".equals(value1)) { + credentialDTO.setFirstValue(originCredentialDTO.getFirstValue()); + } else { + credentialDTO.setFirstValue(value1); + } + String value2 = createUpdateReq.getValue2(); + if ("******".equals(value2)) { + credentialDTO.setSecondValue(originCredentialDTO.getSecondValue()); } else { - credentialDTO.setCreator(username); - credentialDTO.setCreateTime(credentialDTO.getLastModifyTime()); - return credentialDAO.insertCredential(dslContext, credentialDTO); + credentialDTO.setSecondValue(value2); } + credentialDAO.updateCredentialById(credentialDTO); + + CredentialDTO updateCredential = getCredentialById(id); + + // 审计 + ActionAuditContext.current() + .setOriginInstance(originCredentialDTO.toEsbCredentialSimpleInfoV3DTO()) + .setInstance(updateCredential.toEsbCredentialSimpleInfoV3DTO()); + + return updateCredential; } @Override + @ActionAuditRecord( + actionId = ActionId.MANAGE_TICKET, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.TICKET, + instanceIds = "#id" + ), + content = EventContentConstants.DELETE_TICKET + ) public Integer deleteCredentialById(String username, Long appId, String id) { - return credentialDAO.deleteCredentialById(dslContext, id); + authManageTicket(username, appId, id); + + CredentialDTO credential = getCredentialById(id); + if (credential == null) { + throw new NotFoundException(ErrorCode.CREDENTIAL_NOT_EXIST); + } + + // 审计 + ActionAuditContext.current().setInstanceName(credential.getName()); + + // 检查是否被引用 + Boolean isCredentialReferenced = fileSourceService.existsFileSourceUsingCredential(appId, id).getData(); + if (isCredentialReferenced) { + String msg = MessageFormatter.format( + "Credential ({},{}) is referenced, cannot delete", + appId, + id + ).getMessage(); + throw new FailedPreconditionException(msg, ErrorCode.DELETE_REF_CREDENTIAL_FAIL); + } + + return credentialDAO.deleteCredentialById(id); } @Override - public ServiceCredentialDTO getServiceCredentialById(Long appId, String id) { - CredentialDTO credentialDTO = credentialDAO.getCredentialById(dslContext, id); + public CredentialDTO getCredentialById(Long appId, String id) { + CredentialDTO credentialDTO = getCredentialById(id); if (credentialDTO == null || !credentialDTO.getAppId().equals(appId)) { return null; } else { - return credentialDTO.toServiceCredentialDTO(); + return credentialDTO; } } @Override - public ServiceCredentialDTO getServiceCredentialById(String id) { - CredentialDTO credentialDTO = credentialDAO.getCredentialById(dslContext, id); - if (credentialDTO == null) { - return null; - } else { - return credentialDTO.toServiceCredentialDTO(); - } + public CredentialDTO getCredentialById(String id) { + return credentialDAO.getCredentialById(id); } @Override public List listCredentialDisplayInfoByIds(Collection ids) { - return credentialDAO.listCredentialDisplayInfoByIds(dslContext, ids); + return credentialDAO.listCredentialDisplayInfoByIds(ids); } private CredentialDTO buildCredentialDTO(String username, Long appId, CredentialCreateUpdateReq createUpdateReq) { diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/CronJobServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/CronJobServiceImpl.java index 0c2041deb7..891e3b3b24 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/CronJobServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/CronJobServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,8 +25,8 @@ package com.tencent.bk.job.manage.service.impl; import com.tencent.bk.job.common.model.InternalResponse; +import com.tencent.bk.job.crontab.api.inner.ServiceCronJobResource; import com.tencent.bk.job.crontab.model.CronJobVO; -import com.tencent.bk.job.manage.client.ServiceCronJobResourceClient; import com.tencent.bk.job.manage.service.CronJobService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; @@ -36,18 +36,15 @@ import java.util.List; import java.util.Map; -/** - * @since 22/2/2020 21:25 - */ @Slf4j @Service public class CronJobServiceImpl implements CronJobService { - private ServiceCronJobResourceClient serviceCronJobResourceClient; + private final ServiceCronJobResource serviceCronJobResource; @Autowired - public CronJobServiceImpl(ServiceCronJobResourceClient serviceCronJobResourceClient) { - this.serviceCronJobResourceClient = serviceCronJobResourceClient; + public CronJobServiceImpl(ServiceCronJobResource serviceCronJobResource) { + this.serviceCronJobResource = serviceCronJobResource; } @Override @@ -56,7 +53,7 @@ public Map> batchListCronJobByPlanIds(Long appId, List>> cronJobByPlanIds = - serviceCronJobResourceClient.batchListCronJobByPlanIds(appId, planIdList); + serviceCronJobResource.batchListCronJobByPlanIds(appId, planIdList); if (cronJobByPlanIds != null) { return cronJobByPlanIds.getData(); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/CustomScriptTemplateServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/CustomScriptTemplateServiceImpl.java index b0d9aed519..e4024022fe 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/CustomScriptTemplateServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/CustomScriptTemplateServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/CustomSettingsService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/CustomSettingsService.java new file mode 100644 index 0000000000..04599ef9c5 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/CustomSettingsService.java @@ -0,0 +1,164 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.service.impl; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.tencent.bk.job.common.constant.JobConstants; +import com.tencent.bk.job.common.model.dto.AppResourceScope; +import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.manage.dao.customsetting.UserCustomSettingDAO; +import com.tencent.bk.job.manage.model.dto.customsetting.UserCustomSettingDTO; +import com.tencent.bk.job.manage.model.web.request.customsetting.BatchGetCustomSettingsReq; +import com.tencent.bk.job.manage.model.web.request.customsetting.DeleteCustomSettingsReq; +import com.tencent.bk.job.manage.model.web.request.customsetting.SaveCustomSettingsReq; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Slf4j +@Service +public class CustomSettingsService { + + private final UserCustomSettingDAO userCustomSettingDAO; + + @Autowired + public CustomSettingsService( + UserCustomSettingDAO userCustomSettingDAO) { + this.userCustomSettingDAO = userCustomSettingDAO; + } + + public Map> saveCustomSettings(String username, + AppResourceScope appResourceScope, + SaveCustomSettingsReq req) { + int affectedNum = userCustomSettingDAO.batchSave( + buildCustomSettingDTOs( + username, + decidePersistedAppId(appResourceScope.getAppId()), + req.getSettingsMap() + ) + ); + log.info("{}|{}|{} records saved", username, appResourceScope.toBasicStr(), affectedNum); + List keyList = buildKeysWithCurrentAppId( + username, + appResourceScope.getAppId(), + req.getSettingsMap().keySet() + ); + return buildCustomSettingMap(userCustomSettingDAO.batchGet(keyList)); + } + + public Map> batchGetCustomSettings(String username, + AppResourceScope appResourceScope, + BatchGetCustomSettingsReq req) { + if (req == null || req.getModuleList() == null) { + return buildCustomSettingMap(userCustomSettingDAO.listAll()); + } + List keyList = buildKeysWithCurrentAppId(username, appResourceScope.getAppId(), req.getModuleList()); + return buildCustomSettingMap(userCustomSettingDAO.batchGet(keyList)); + } + + public Integer deleteCustomSettings(String username, + AppResourceScope appResourceScope, + DeleteCustomSettingsReq req) { + List keyList = buildKeysWithCurrentAppId(username, appResourceScope.getAppId(), req.getModuleList()); + return userCustomSettingDAO.batchDelete(keyList); + } + + private Long decidePersistedAppId(Long currentAppId) { + Long persistedAppId = JobConstants.PUBLIC_APP_ID; + log.debug( + "currentAppId={},use appId={} to save/query customSettings", + currentAppId, + persistedAppId + ); + return persistedAppId; + } + + private List buildKeysWithCurrentAppId(String username, + Long appId, + Collection modules) { + return buildKeys( + username, + decidePersistedAppId(appId), + modules + ); + } + + private List buildKeys(String username, + Long appId, + Collection modules) { + if (CollectionUtils.isEmpty(modules)) { + return Collections.emptyList(); + } + return modules.stream() + .map(module -> UserCustomSettingDTO.getKey(username, appId, module)) + .collect(Collectors.toList()); + } + + private List buildCustomSettingDTOs(String username, + Long appId, + Map> settingsMap) { + List userCustomSettingDTOList = new ArrayList<>(); + for (Map.Entry> entry : settingsMap.entrySet()) { + String module = entry.getKey(); + Map valuesMap = entry.getValue(); + userCustomSettingDTOList.add(buildCustomSettingDTO(username, appId, module, JsonUtils.toJson(valuesMap))); + } + return userCustomSettingDTOList; + } + + + private UserCustomSettingDTO buildCustomSettingDTO(String username, + Long appId, + String module, + String value) { + UserCustomSettingDTO userCustomSettingDTO = new UserCustomSettingDTO(); + userCustomSettingDTO.setUsername(username); + userCustomSettingDTO.setAppId(appId); + userCustomSettingDTO.setModule(module); + userCustomSettingDTO.setValue(value); + userCustomSettingDTO.setLastModifyUser(username); + return userCustomSettingDTO; + } + + private Map> buildCustomSettingMap(List settingList) { + Map> map = new HashMap<>(); + for (UserCustomSettingDTO settingDTO : settingList) { + map.put(settingDTO.getModule(), JsonUtils.fromJson(settingDTO.getValue(), new TypeReference>() { + })); + } + return map; + } + +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/DangerousRuleServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/DangerousRuleServiceImpl.java index abf949fd52..9abcc7bce3 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/DangerousRuleServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/DangerousRuleServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,17 +24,22 @@ package com.tencent.bk.job.manage.service.impl; -import com.tencent.bk.job.manage.common.consts.EnableStatusEnum; +import com.tencent.bk.audit.annotations.ActionAuditRecord; +import com.tencent.bk.job.common.audit.constants.EventContentConstants; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.InternalException; +import com.tencent.bk.job.common.iam.constant.ActionId; +import com.tencent.bk.job.common.mysql.JobTransactional; +import com.tencent.bk.job.manage.api.common.constants.EnableStatusEnum; import com.tencent.bk.job.manage.dao.globalsetting.DangerousRuleDAO; import com.tencent.bk.job.manage.manager.cache.DangerousRuleCache; import com.tencent.bk.job.manage.model.dto.globalsetting.DangerousRuleDTO; +import com.tencent.bk.job.manage.model.query.DangerousRuleQuery; import com.tencent.bk.job.manage.model.web.request.globalsetting.AddOrUpdateDangerousRuleReq; import com.tencent.bk.job.manage.model.web.request.globalsetting.MoveDangerousRuleReq; import com.tencent.bk.job.manage.model.web.vo.globalsetting.DangerousRuleVO; import com.tencent.bk.job.manage.service.DangerousRuleService; import lombok.extern.slf4j.Slf4j; -import org.jooq.DSLContext; -import org.jooq.impl.DSL; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -45,76 +50,94 @@ @Service public class DangerousRuleServiceImpl implements DangerousRuleService { - private final DSLContext dslContext; private final DangerousRuleDAO dangerousRuleDAO; private final DangerousRuleCache dangerousRuleCache; @Autowired public DangerousRuleServiceImpl( - DSLContext dslContext, DangerousRuleDAO dangerousRuleDAO, DangerousRuleCache dangerousRuleCache) { - this.dslContext = dslContext; this.dangerousRuleDAO = dangerousRuleDAO; this.dangerousRuleCache = dangerousRuleCache; } @Override public List listDangerousRules(String username) { - return dangerousRuleDAO.listDangerousRules(dslContext).stream().map(DangerousRuleDTO::toVO) + return dangerousRuleDAO.listDangerousRules().stream().map(DangerousRuleDTO::toVO) .collect(Collectors.toList()); } @Override - public Boolean addOrUpdateDangerousRule(String username, AddOrUpdateDangerousRuleReq req) { + public DangerousRuleDTO getDangerousRuleById(Long id) { + return dangerousRuleDAO.getDangerousRuleById(id); + } + + @Override + @ActionAuditRecord( + actionId = ActionId.HIGH_RISK_DETECT_RULE, + content = EventContentConstants.CREATE_HIGH_RISK_DETECT_RULE + ) + public DangerousRuleDTO createDangerousRule(String username, AddOrUpdateDangerousRuleReq req) { int scriptType = DangerousRuleDTO.encodeScriptType(req.getScriptTypeList()); - if (req.getId() == -1) { - //新增 - int maxPriority = dangerousRuleDAO.getMaxPriority(dslContext); - log.info(String.format("current maxPriority:%d", maxPriority)); - dangerousRuleDAO.insertDangerousRule(dslContext, new DangerousRuleDTO(null, req.getExpression(), - req.getDescription(), maxPriority + 1, scriptType, username, System.currentTimeMillis(), username, - System.currentTimeMillis(), req.getAction(), EnableStatusEnum.DISABLED.getValue())); - } else { - //更新 - DangerousRuleDTO existDangerousRuleDTO = dangerousRuleDAO.getDangerousRuleById(dslContext, req.getId()); - if (existDangerousRuleDTO != null) { - dangerousRuleDAO.updateDangerousRule(dslContext, new DangerousRuleDTO(req.getId(), - req.getExpression(), req.getDescription(), existDangerousRuleDTO.getPriority(), scriptType, null, - null, username, System.currentTimeMillis(), req.getAction(), req.getStatus())); - } else { - return false; - } - } + int maxPriority = dangerousRuleDAO.getMaxPriority(); + log.info(String.format("current maxPriority:%d", maxPriority)); + long id = dangerousRuleDAO.insertDangerousRule(new DangerousRuleDTO(null, req.getExpression(), + req.getDescription(), maxPriority + 1, scriptType, username, System.currentTimeMillis(), username, + System.currentTimeMillis(), req.getAction(), EnableStatusEnum.DISABLED.getValue())); + + // 清理缓存 dangerousRuleCache.deleteDangerousRuleCacheByScriptTypes(req.getScriptTypeList()); - return true; + + return getDangerousRuleById(id); } @Override + @ActionAuditRecord( + actionId = ActionId.HIGH_RISK_DETECT_RULE, + content = EventContentConstants.EDIT_HIGH_RISK_DETECT_RULE + ) + public DangerousRuleDTO updateDangerousRule(String username, AddOrUpdateDangerousRuleReq req) { + int scriptType = DangerousRuleDTO.encodeScriptType(req.getScriptTypeList()); + DangerousRuleDTO existDangerousRuleDTO = dangerousRuleDAO.getDangerousRuleById(req.getId()); + if (existDangerousRuleDTO != null) { + dangerousRuleDAO.updateDangerousRule(new DangerousRuleDTO(req.getId(), + req.getExpression(), req.getDescription(), existDangerousRuleDTO.getPriority(), scriptType, null, + null, username, System.currentTimeMillis(), req.getAction(), req.getStatus())); + // 清理缓存 + List existScriptTypes = DangerousRuleDTO.decodeScriptType(existDangerousRuleDTO.getScriptType()); + dangerousRuleCache.deleteDangerousRuleCacheByScriptTypes(existScriptTypes); + } + return getDangerousRuleById(req.getId()); + } + + + @Override + @JobTransactional(transactionManager = "jobManageTransactionManager") + @ActionAuditRecord( + actionId = ActionId.HIGH_RISK_DETECT_RULE, + content = EventContentConstants.EDIT_HIGH_RISK_DETECT_RULE + ) public Integer moveDangerousRule(String username, MoveDangerousRuleReq req) { int dir = req.getDir(); - DangerousRuleDTO currentRuleDTO = dangerousRuleDAO.getDangerousRuleById(dslContext, req.getId()); + DangerousRuleDTO currentRuleDTO = dangerousRuleDAO.getDangerousRuleById(req.getId()); if (currentRuleDTO == null) { log.info("id=%d dangerousRule not exist"); return 0; } if (dir == -1) { //往上移动 - int minPriority = dangerousRuleDAO.getMinPriority(dslContext); + int minPriority = dangerousRuleDAO.getMinPriority(); if (currentRuleDTO.getPriority() <= minPriority) { log.info("Fail to move, id=%d dangerousRule already has min priority"); return 0; } //需要移动的情况 - DangerousRuleDTO upperRuleDTO = dangerousRuleDAO.getDangerousRuleByPriority(dslContext, + DangerousRuleDTO upperRuleDTO = dangerousRuleDAO.getDangerousRuleByPriority( currentRuleDTO.getPriority() - 1); upperRuleDTO.setPriority(upperRuleDTO.getPriority() + 1); currentRuleDTO.setPriority(currentRuleDTO.getPriority() - 1); - dslContext.transaction(configuration -> { - DSLContext context = DSL.using(configuration); - dangerousRuleDAO.updateDangerousRule(context, upperRuleDTO); - dangerousRuleDAO.updateDangerousRule(context, currentRuleDTO); - }); + dangerousRuleDAO.updateDangerousRule(upperRuleDTO); + dangerousRuleDAO.updateDangerousRule(currentRuleDTO); dangerousRuleCache.deleteDangerousRuleCacheByScriptTypes( DangerousRuleDTO.decodeScriptType(upperRuleDTO.getScriptType())); dangerousRuleCache.deleteDangerousRuleCacheByScriptTypes( @@ -122,24 +145,21 @@ public Integer moveDangerousRule(String username, MoveDangerousRuleReq req) { return 2; } else if (dir == 1) { //往下移动 - int maxPriority = dangerousRuleDAO.getMaxPriority(dslContext); + int maxPriority = dangerousRuleDAO.getMaxPriority(); if (currentRuleDTO.getPriority() >= maxPriority) { log.info("Fail to move, id=%d dangerousRule already has max priority"); return 0; } //需要移动的情况 - DangerousRuleDTO downerRuleDTO = dangerousRuleDAO.getDangerousRuleByPriority(dslContext, + DangerousRuleDTO downerRuleDTO = dangerousRuleDAO.getDangerousRuleByPriority( currentRuleDTO.getPriority() + 1); if (downerRuleDTO == null) { return 0; } downerRuleDTO.setPriority(downerRuleDTO.getPriority() - 1); currentRuleDTO.setPriority(currentRuleDTO.getPriority() + 1); - dslContext.transaction(configuration -> { - DSLContext context = DSL.using(configuration); - dangerousRuleDAO.updateDangerousRule(context, downerRuleDTO); - dangerousRuleDAO.updateDangerousRule(context, currentRuleDTO); - }); + dangerousRuleDAO.updateDangerousRule(downerRuleDTO); + dangerousRuleDAO.updateDangerousRule(currentRuleDTO); dangerousRuleCache.deleteDangerousRuleCacheByScriptTypes( DangerousRuleDTO.decodeScriptType(downerRuleDTO.getScriptType())); dangerousRuleCache.deleteDangerousRuleCacheByScriptTypes( @@ -152,32 +172,60 @@ public Integer moveDangerousRule(String username, MoveDangerousRuleReq req) { } @Override + @JobTransactional(transactionManager = "jobManageTransactionManager") + @ActionAuditRecord( + actionId = ActionId.HIGH_RISK_DETECT_RULE, + content = EventContentConstants.DELETE_HIGH_RISK_DETECT_RULE + ) public Integer deleteDangerousRuleById(String username, Long id) { - DangerousRuleDTO existDangerousRuleDTO = dangerousRuleDAO.getDangerousRuleById(dslContext, id); + DangerousRuleDTO existDangerousRuleDTO = dangerousRuleDAO.getDangerousRuleById(id); if (existDangerousRuleDTO == null) { return -1; } - List dangerousRuleDTOList = dangerousRuleDAO.listDangerousRules(dslContext); + List dangerousRuleDTOList = dangerousRuleDAO.listDangerousRules(); for (int i = 0; i < dangerousRuleDTOList.size(); i++) { if (dangerousRuleDTOList.get(i).getId().equals(id)) { dangerousRuleDTOList.remove(i); break; } } - //每次删除后维持有序 - dslContext.transaction(configuration -> { - DSLContext context = DSL.using(configuration); - dangerousRuleDAO.deleteDangerousRuleById(context, id); + try { + //每次删除后维持有序 + dangerousRuleDAO.deleteDangerousRuleById(id); for (int i = 0; i < dangerousRuleDTOList.size(); i++) { DangerousRuleDTO dangerousRuleDTO = dangerousRuleDTOList.get(i); dangerousRuleDTO.setPriority(i + 1); - dangerousRuleDAO.updateDangerousRule(context, dangerousRuleDTO); + dangerousRuleDAO.updateDangerousRule(dangerousRuleDTO); dangerousRuleCache.deleteDangerousRuleCacheByScriptTypes( DangerousRuleDTO.decodeScriptType(dangerousRuleDTO.getScriptType())); } - }); - dangerousRuleCache.deleteDangerousRuleCacheByScriptTypes( - DangerousRuleDTO.decodeScriptType(existDangerousRuleDTO.getScriptType())); + dangerousRuleCache.deleteDangerousRuleCacheByScriptTypes( + DangerousRuleDTO.decodeScriptType(existDangerousRuleDTO.getScriptType())); + } catch (Exception e) { + log.error(String.format("delete dangerous rule fail! id: %s", id), e); + throw new InternalException(e, ErrorCode.INTERNAL_ERROR); + } return id.intValue(); } + + @Override + public List listDangerousRules(DangerousRuleQuery query) { + return dangerousRuleDAO.listDangerousRules(query) + .stream() + .map(DangerousRuleDTO::toVO) + .collect(Collectors.toList()); + } + + @Override + @ActionAuditRecord( + actionId = ActionId.HIGH_RISK_DETECT_RULE, + content = EventContentConstants.EDIT_HIGH_RISK_DETECT_RULE + ) + public DangerousRuleDTO updateDangerousRuleStatus(String userName, Long id, EnableStatusEnum status) { + dangerousRuleDAO.updateDangerousRuleStatus(userName, id, status); + DangerousRuleDTO dangerousRuleDTO = getDangerousRuleById(id); + dangerousRuleCache.deleteDangerousRuleCacheByScriptTypes( + DangerousRuleDTO.decodeScriptType(dangerousRuleDTO.getScriptType())); + return dangerousRuleDTO; + } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/GlobalSettingsServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/GlobalSettingsServiceImpl.java deleted file mode 100644 index abddf5d7fb..0000000000 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/GlobalSettingsServiceImpl.java +++ /dev/null @@ -1,810 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.service.impl; - -import com.fasterxml.jackson.core.type.TypeReference; -import com.tencent.bk.job.common.constant.ErrorCode; -import com.tencent.bk.job.common.constant.JobConstants; -import com.tencent.bk.job.common.exception.InvalidParamException; -import com.tencent.bk.job.common.i18n.locale.LocaleUtils; -import com.tencent.bk.job.common.i18n.service.MessageI18nService; -import com.tencent.bk.job.common.util.JobContextUtil; -import com.tencent.bk.job.common.util.StringUtil; -import com.tencent.bk.job.common.util.TimeUtil; -import com.tencent.bk.job.common.util.date.DateUtils; -import com.tencent.bk.job.common.util.json.JsonUtils; -import com.tencent.bk.job.manage.common.consts.globalsetting.GlobalSettingKeys; -import com.tencent.bk.job.manage.common.consts.globalsetting.OSTypeEnum; -import com.tencent.bk.job.manage.common.consts.globalsetting.RelatedUrlKeys; -import com.tencent.bk.job.manage.common.consts.globalsetting.RestrictModeEnum; -import com.tencent.bk.job.manage.common.consts.globalsetting.StorageUnitEnum; -import com.tencent.bk.job.manage.common.consts.notify.NotifyConsts; -import com.tencent.bk.job.manage.config.JobManageConfig; -import com.tencent.bk.job.manage.config.LocalFileConfigForManage; -import com.tencent.bk.job.manage.dao.globalsetting.GlobalSettingDAO; -import com.tencent.bk.job.manage.dao.notify.AvailableEsbChannelDAO; -import com.tencent.bk.job.manage.dao.notify.NotifyEsbChannelDAO; -import com.tencent.bk.job.manage.dao.notify.NotifyTemplateDAO; -import com.tencent.bk.job.manage.model.dto.GlobalSettingDTO; -import com.tencent.bk.job.manage.model.dto.converter.NotifyTemplateConverter; -import com.tencent.bk.job.manage.model.dto.globalsetting.TitleFooter; -import com.tencent.bk.job.manage.model.dto.globalsetting.TitleFooterDTO; -import com.tencent.bk.job.manage.model.dto.globalsetting.UploadFileRestrictDTO; -import com.tencent.bk.job.manage.model.dto.notify.AvailableEsbChannelDTO; -import com.tencent.bk.job.manage.model.dto.notify.NotifyEsbChannelDTO; -import com.tencent.bk.job.manage.model.dto.notify.NotifyTemplateDTO; -import com.tencent.bk.job.manage.model.web.request.globalsetting.AccountNameRule; -import com.tencent.bk.job.manage.model.web.request.globalsetting.AccountNameRulesReq; -import com.tencent.bk.job.manage.model.web.request.globalsetting.FileUploadSettingReq; -import com.tencent.bk.job.manage.model.web.request.globalsetting.HistoryExpireReq; -import com.tencent.bk.job.manage.model.web.request.globalsetting.SetTitleFooterReq; -import com.tencent.bk.job.manage.model.web.request.notify.ChannelTemplatePreviewReq; -import com.tencent.bk.job.manage.model.web.request.notify.ChannelTemplateReq; -import com.tencent.bk.job.manage.model.web.request.notify.NotifyBlackUsersReq; -import com.tencent.bk.job.manage.model.web.request.notify.SetAvailableNotifyChannelReq; -import com.tencent.bk.job.manage.model.web.vo.globalsetting.AccountNameRuleVO; -import com.tencent.bk.job.manage.model.web.vo.globalsetting.AccountNameRulesWithDefaultVO; -import com.tencent.bk.job.manage.model.web.vo.globalsetting.FileUploadSettingVO; -import com.tencent.bk.job.manage.model.web.vo.globalsetting.NotifyChannelWithIconVO; -import com.tencent.bk.job.manage.model.web.vo.globalsetting.TitleFooterVO; -import com.tencent.bk.job.manage.model.web.vo.globalsetting.TitleFooterWithDefaultVO; -import com.tencent.bk.job.manage.model.web.vo.notify.ChannelTemplateDetailVO; -import com.tencent.bk.job.manage.model.web.vo.notify.ChannelTemplateDetailWithDefaultVO; -import com.tencent.bk.job.manage.model.web.vo.notify.ChannelTemplateStatusVO; -import com.tencent.bk.job.manage.model.web.vo.notify.NotifyBlackUserInfoVO; -import com.tencent.bk.job.manage.model.web.vo.notify.TemplateBasicInfo; -import com.tencent.bk.job.manage.model.web.vo.notify.UserVO; -import com.tencent.bk.job.manage.service.GlobalSettingsService; -import com.tencent.bk.job.manage.service.NotifyService; -import com.tencent.bk.job.manage.service.impl.notify.NotifySendService; -import com.tencent.bk.job.manage.service.impl.notify.NotifyUserService; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.Pair; -import org.jooq.DSLContext; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.info.BuildProperties; -import org.springframework.stereotype.Service; - -import java.time.LocalDateTime; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.Set; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -/** - * @Description - * @Date 2020/2/28 - * @Version 1.0 - */ -@Slf4j -@Service -public class GlobalSettingsServiceImpl implements GlobalSettingsService { - - private static final Pattern PATTERN = Pattern.compile("^([.0-9]+)([a-zA-Z]{0,2})$"); - private static final String STRING_TPL_KEY_CURRENT_VERSION = "current_ver"; - private static final String STRING_TPL_KEY_CURRENT_YEAR = "current_year"; - private final DSLContext dslContext; - private final NotifyEsbChannelDAO notifyEsbChannelDAO; - private final AvailableEsbChannelDAO availableEsbChannelDAO; - private final NotifyService notifyService; - private final NotifySendService notifySendService; - private final NotifyUserService notifyUserService; - private final GlobalSettingDAO globalSettingDAO; - private final NotifyTemplateDAO notifyTemplateDAO; - private final MessageI18nService i18nService; - private final JobManageConfig jobManageConfig; - private final LocalFileConfigForManage localFileConfigForManage; - private final NotifyTemplateConverter notifyTemplateConverter; - private final BuildProperties buildProperties; - @Value("${job.manage.upload.filesize.max:5GB}") - private String configedMaxFileSize; - @Value("${job.feature.file-manage.enabled:false}") - private Boolean enableFeatureFileManage; - - @Autowired - public GlobalSettingsServiceImpl( - DSLContext dslContext - , NotifyEsbChannelDAO notifyEsbChannelDAO - , AvailableEsbChannelDAO availableEsbChannelDAO - , NotifyService notifyService - , NotifySendService notifySendService, - NotifyUserService notifyUserService, - GlobalSettingDAO globalSettingDAO - , NotifyTemplateDAO notifyTemplateDAO, - MessageI18nService i18nService, - JobManageConfig jobManageConfig, - LocalFileConfigForManage localFileConfigForManage, - NotifyTemplateConverter notifyTemplateConverter, - BuildProperties buildProperties) { - this.dslContext = dslContext; - this.notifyEsbChannelDAO = notifyEsbChannelDAO; - this.availableEsbChannelDAO = availableEsbChannelDAO; - this.notifyService = notifyService; - this.notifySendService = notifySendService; - this.notifyUserService = notifyUserService; - this.globalSettingDAO = globalSettingDAO; - this.notifyTemplateDAO = notifyTemplateDAO; - this.i18nService = i18nService; - this.jobManageConfig = jobManageConfig; - this.localFileConfigForManage = localFileConfigForManage; - this.notifyTemplateConverter = notifyTemplateConverter; - this.buildProperties = buildProperties; - } - - private static String removeSuffixBackSlash(String rawStr) { - if (rawStr == null) { - return null; - } - while (rawStr.endsWith("/")) { - rawStr = rawStr.substring(0, rawStr.length() - 1); - } - return rawStr; - } - - @Override - public Boolean isNotifyChannelConfiged(DSLContext dslContext) { - GlobalSettingDTO globalSettingDTO = globalSettingDAO.getGlobalSetting(dslContext, - GlobalSettingKeys.KEY_NOTIFY_CHANNEL_CONFIGED); - return globalSettingDTO != null && globalSettingDTO.getValue().toLowerCase().equals("true"); - } - - @Override - public Boolean setNotifyChannelConfiged(DSLContext dslContext) { - GlobalSettingDTO globalSettingDTO = globalSettingDAO.getGlobalSetting(dslContext, - GlobalSettingKeys.KEY_NOTIFY_CHANNEL_CONFIGED); - if (globalSettingDTO == null) { - globalSettingDTO = new GlobalSettingDTO(GlobalSettingKeys.KEY_NOTIFY_CHANNEL_CONFIGED, - "true", "whether " + - "avaliable notify channels are configed"); - return 1 == globalSettingDAO.insertGlobalSetting(dslContext, globalSettingDTO); - } else if (!globalSettingDTO.getValue().toLowerCase().equals("true")) { - globalSettingDTO.setValue("true"); - return 1 == globalSettingDAO.updateGlobalSetting(dslContext, globalSettingDTO); - } else { - return true; - } - } - - @Override - public List listNotifyChannel(String username) { - List allNotifyChannelList = - notifyEsbChannelDAO.listNotifyEsbChannel(dslContext).stream() - .filter(NotifyEsbChannelDTO::isActive).collect(Collectors.toList()); - List availableNotifyChannelList = - availableEsbChannelDAO.listAvailableEsbChannel(dslContext); - Set availableNotifyChannelTypeSet = - availableNotifyChannelList.stream().map(AvailableEsbChannelDTO::getType).collect(Collectors.toSet()); - return allNotifyChannelList.stream().map(it -> { - String icon = it.getIcon(); - String prefix = "data:image/png;base64,"; - if (!icon.startsWith("data:image")) { - icon = prefix + icon; - } - return new NotifyChannelWithIconVO( - it.getType(), - it.getLabel(), - icon, - availableNotifyChannelTypeSet.contains(it.getType()) - ); - }).collect(Collectors.toList()); - } - - @Override - public Integer setAvailableNotifyChannel(String username, SetAvailableNotifyChannelReq req) { - return notifyService.setAvailableNotifyChannel(username, req); - } - - @Override - public List listUsers(String username, String prefixStr, Long offset, Long limit) { - //这里就是要选择人来添加黑名单，故不排除已在黑名单内的人 - return notifyUserService.listUsers(prefixStr, offset, limit, false); - } - - @Override - public List listNotifyBlackUsers(String username, Integer start, Integer pageSize) { - return notifyUserService.listNotifyBlackUsers(start, pageSize); - } - - @Override - public List saveNotifyBlackUsers(String username, NotifyBlackUsersReq req) { - return notifyUserService.saveNotifyBlackUsers(username, req); - } - - @Override - public Long getHistoryExpireTime(String username) { - GlobalSettingDTO globalSettingDTO = globalSettingDAO.getGlobalSetting(dslContext, - GlobalSettingKeys.KEY_HISTORY_EXPIRE_DAYS); - if (globalSettingDTO == null) { - globalSettingDTO = new GlobalSettingDTO(GlobalSettingKeys.KEY_HISTORY_EXPIRE_DAYS, - "60", "执行记录默认保存天数" + - "(default history expire days)"); - globalSettingDAO.insertGlobalSetting(dslContext, globalSettingDTO); - } - return Long.parseLong(globalSettingDTO.getValue()); - } - - @Override - public Integer setHistoryExpireTime(String username, HistoryExpireReq req) { - Long days = req.getDays(); - if (days == null || days <= 0) { - throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME_AND_REASON, - new String[]{"days", "days must be positive"}); - } - GlobalSettingDTO globalSettingDTO = new GlobalSettingDTO(GlobalSettingKeys.KEY_HISTORY_EXPIRE_DAYS, - days.toString(), String.format("执行记录保存天数(history expire days):%s,%s", username, - DateUtils.defaultLocalDateTime(LocalDateTime.now()))); - return globalSettingDAO.updateGlobalSetting(dslContext, globalSettingDTO); - } - - @Override - public AccountNameRule getCurrentAccountNameRule(OSTypeEnum osType) { - String normalLang = LocaleUtils.getNormalLang(JobContextUtil.getUserLang()); - GlobalSettingDTO currentNameRulesDTO; - String globalSettingKey; - if (normalLang.equals(LocaleUtils.LANG_EN) || normalLang.equals(LocaleUtils.LANG_EN_US)) { - //英文环境 - globalSettingKey = GlobalSettingKeys.KEY_CURRENT_NAME_RULES_EN; - } else { - //中文环境 - globalSettingKey = GlobalSettingKeys.KEY_CURRENT_NAME_RULES; - } - List currentNameRules; - currentNameRulesDTO = globalSettingDAO.getGlobalSetting(dslContext, globalSettingKey); - if (currentNameRulesDTO != null) { - currentNameRules = JsonUtils.fromJson(currentNameRulesDTO.getValue(), - new TypeReference>() { - }); - for (AccountNameRule rule : currentNameRules) { - if (rule.getOsType() == osType) { - return rule; - } - } - } else { - log.warn("Cannot find currentNameRules in language:{}, please check key:{} in database table job_manage" + - ".global_setting", normalLang, globalSettingKey); - } - return null; - } - - @Override - public AccountNameRulesWithDefaultVO getAccountNameRules() { - String normalLang = LocaleUtils.getNormalLang(JobContextUtil.getUserLang()); - List defaultNameRules = new ArrayList<>(); - GlobalSettingDTO currentNameRulesDTO; - if (normalLang.equals(LocaleUtils.LANG_EN) || normalLang.equals(LocaleUtils.LANG_EN_US)) { - //英文环境 - GlobalSettingDTO defaultNameRulesDTO = globalSettingDAO.getGlobalSetting(dslContext, - GlobalSettingKeys.KEY_DEFAULT_NAME_RULES_EN); - if (defaultNameRulesDTO == null) { - defaultNameRules.add(new AccountNameRule(OSTypeEnum.LINUX, "^[a-z_][a-z0-9_-]{2,31}$", - i18nService.getI18n("job.manage.globalsettings.defaultNameRules.description.linux", - Locale.ENGLISH))); - defaultNameRules.add(new AccountNameRule(OSTypeEnum.WINDOWS, "^[a-æA-Æ0-9-]{1,32}$", - i18nService.getI18n("job.manage.globalsettings.defaultNameRules.description.windows", - Locale.ENGLISH))); - defaultNameRules.add(new AccountNameRule(OSTypeEnum.DATABASE, "^[a-zA-Z0-9\\.\\-\\_]{1,16}$", - i18nService.getI18n("job.manage.globalsettings.defaultNameRules.description.database", - Locale.ENGLISH))); - defaultNameRulesDTO = new GlobalSettingDTO(GlobalSettingKeys.KEY_DEFAULT_NAME_RULES_EN, - JsonUtils.toJson(defaultNameRules), - "默认命名规则JSON串(default json-serialized name rules)"); - currentNameRulesDTO = new GlobalSettingDTO(GlobalSettingKeys.KEY_CURRENT_NAME_RULES_EN, - JsonUtils.toJson(defaultNameRules), - "默认命名规则JSON串(default json-serialized name rules)"); - globalSettingDAO.insertGlobalSetting(dslContext, defaultNameRulesDTO); - globalSettingDAO.insertGlobalSetting(dslContext, currentNameRulesDTO); - } else { - defaultNameRules = JsonUtils.fromJson(defaultNameRulesDTO.getValue(), - new TypeReference>() { - }); - currentNameRulesDTO = globalSettingDAO.getGlobalSetting(dslContext, - GlobalSettingKeys.KEY_CURRENT_NAME_RULES_EN); - } - } else { - //中文环境 - GlobalSettingDTO defaultNameRulesDTO = globalSettingDAO.getGlobalSetting(dslContext, - GlobalSettingKeys.KEY_DEFAULT_NAME_RULES); - if (defaultNameRulesDTO == null) { - defaultNameRules.add(new AccountNameRule(OSTypeEnum.LINUX, "^[a-z_][a-z0-9_-]{2,31}$", - i18nService.getI18n("job.manage.globalsettings.defaultNameRules.description.linux", - Locale.CHINA))); - defaultNameRules.add(new AccountNameRule(OSTypeEnum.WINDOWS, "^[a-æA-Æ0-9-]{1,32}$", - i18nService.getI18n("job.manage.globalsettings.defaultNameRules.description.windows", - Locale.CHINA))); - defaultNameRules.add(new AccountNameRule(OSTypeEnum.DATABASE, "^[a-zA-Z0-9\\.\\-\\_]{1,16}$", - i18nService.getI18n("job.manage.globalsettings.defaultNameRules.description.database", - Locale.CHINA))); - defaultNameRulesDTO = new GlobalSettingDTO(GlobalSettingKeys.KEY_DEFAULT_NAME_RULES, - JsonUtils.toJson(defaultNameRules), - "默认命名规则JSON串(default json-serialized name rules)"); - currentNameRulesDTO = new GlobalSettingDTO(GlobalSettingKeys.KEY_CURRENT_NAME_RULES, - JsonUtils.toJson(defaultNameRules), - "默认命名规则JSON串(default json-serialized name rules)"); - globalSettingDAO.insertGlobalSetting(dslContext, defaultNameRulesDTO); - globalSettingDAO.insertGlobalSetting(dslContext, currentNameRulesDTO); - } else { - defaultNameRules = JsonUtils.fromJson(defaultNameRulesDTO.getValue(), - new TypeReference>() { - }); - currentNameRulesDTO = globalSettingDAO.getGlobalSetting(dslContext, - GlobalSettingKeys.KEY_CURRENT_NAME_RULES); - } - } - List currentNameRules; - if (currentNameRulesDTO != null) { - currentNameRules = JsonUtils.fromJson(currentNameRulesDTO.getValue(), - new TypeReference>() { - }); - } else { - currentNameRules = defaultNameRules; - } - return new AccountNameRulesWithDefaultVO(currentNameRules.stream() - .map(it -> new AccountNameRuleVO(it.getOsType(), it.getExpression(), it.getDescription())) - .collect(Collectors.toList()), - defaultNameRules.stream() - .map(it -> new AccountNameRuleVO(it.getOsType(), it.getExpression(), it.getDescription())) - .collect(Collectors.toList()) - ); - } - - @Override - public Boolean setAccountNameRules(String username, AccountNameRulesReq req) { - GlobalSettingDTO currentNameRulesDTO = globalSettingDAO.getGlobalSetting(dslContext, - GlobalSettingKeys.KEY_CURRENT_NAME_RULES); - GlobalSettingDTO inputNameRulesDTO = new GlobalSettingDTO(GlobalSettingKeys.KEY_CURRENT_NAME_RULES, - JsonUtils.toJson(req.getRules()), String.format("Updated by %s at %s", username, - DateUtils.defaultLocalDateTime(LocalDateTime.now()))); - if (currentNameRulesDTO == null) { - return globalSettingDAO.insertGlobalSetting(dslContext, inputNameRulesDTO) == 1; - } else { - return globalSettingDAO.updateGlobalSetting(dslContext, inputNameRulesDTO) == 1; - } - } - - private Pair parseFileSize(String str) { - Matcher matcher = PATTERN.matcher(str); - if (!matcher.matches()) { - return null; - } - float amount = Float.parseFloat(matcher.group(1)); - String unit = matcher.group(2); - return Pair.of(amount, unit); - } - - private FileUploadSettingVO getFileUploadSettingsFromStr(String str) { - Pair configedValue = parseFileSize(str); - if (configedValue == null) return null; - return new FileUploadSettingVO(configedValue.getLeft(), configedValue.getRight(), null, null); - } - - private FileUploadSettingVO getConfigedFileUploadSettings() { - log.debug("configedMaxFileSize=" + configedMaxFileSize); - return getFileUploadSettingsFromStr(configedMaxFileSize); - } - - @Override - public Boolean saveFileUploadSettings(String username, FileUploadSettingReq req) { - Float uploadMaxSize = req.getAmount(); - StorageUnitEnum unit = req.getUnit(); - Integer restrictMode = req.getRestrictMode(); - List suffixList = req.getSuffixList(); - if (unit == null) { - unit = StorageUnitEnum.B; - } - if (uploadMaxSize <= 0) { - uploadMaxSize = 5.0f; - unit = StorageUnitEnum.GB; - } - if (restrictMode == null) { - restrictMode = RestrictModeEnum.UNLIMITED.getType(); - } - - if (CollectionUtils.isNotEmpty(suffixList)) { - // 去重 - suffixList = suffixList.stream().distinct().collect(Collectors.toList()); - } - GlobalSettingDTO fileUploadSettingDTO = globalSettingDAO.getGlobalSetting(dslContext, - GlobalSettingKeys.KEY_FILE_UPLOAD_SETTING); - if (fileUploadSettingDTO == null) { - fileUploadSettingDTO = new GlobalSettingDTO(); - fileUploadSettingDTO.setDescription("setting of upload file"); - } - fileUploadSettingDTO.setKey(GlobalSettingKeys.KEY_FILE_UPLOAD_SETTING); - fileUploadSettingDTO.setValue(JsonUtils.toJson( - new UploadFileRestrictDTO( - uploadMaxSize.toString() + unit.name() - , restrictMode - , suffixList))); - int affectedRows = globalSettingDAO.upsertGlobalSetting(fileUploadSettingDTO); - return affectedRows > 0; - } - - @Override - public FileUploadSettingVO getFileUploadSettings() { - GlobalSettingDTO fileUploadSettingDTO = globalSettingDAO.getGlobalSetting(dslContext, - GlobalSettingKeys.KEY_FILE_UPLOAD_SETTING); - FileUploadSettingVO fileUploadSettingVO; - if (fileUploadSettingDTO == null) { - fileUploadSettingVO = getConfigedFileUploadSettings(); - } else { - String settingStr = fileUploadSettingDTO.getValue(); - UploadFileRestrictDTO uploadFileRestrictDTO = JsonUtils.fromJson(settingStr, - new TypeReference() { - }); - fileUploadSettingVO = getFileUploadSettingsFromStr(uploadFileRestrictDTO.getMaxSize()); - if (fileUploadSettingVO == null) { - fileUploadSettingVO = new FileUploadSettingVO(); - } - fileUploadSettingVO.setRestrictMode(uploadFileRestrictDTO.getRestrictMode()); - fileUploadSettingVO.setSuffixList(uploadFileRestrictDTO.getSuffixList()); - } - return fileUploadSettingVO; - } - - @Override - public Boolean setTitleFooter(String username, SetTitleFooterReq req) { - //参数校验 - String lang = JobContextUtil.getUserLang(); - if (req.getTitleHead() == null) { - req.setTitleHead(""); - } - if (req.getTitleSeparator() == null) { - req.setTitleSeparator(""); - } - if (req.getFooterLink() == null) { - req.setFooterLink(""); - } - if (req.getFooterCopyRight() == null) { - req.setFooterCopyRight(""); - } - GlobalSettingDTO titleFooterDTO = globalSettingDAO.getGlobalSetting(dslContext, - GlobalSettingKeys.KEY_TITLE_FOOTER); - if (titleFooterDTO == null) { - Map titleFooterLanguageMap = new HashMap<>(); - titleFooterLanguageMap.put( - LocaleUtils.getNormalLang(lang), new TitleFooter( - req.getTitleHead() - , req.getTitleSeparator() - , req.getFooterLink() - , req.getFooterCopyRight() - )); - titleFooterDTO = new GlobalSettingDTO( - GlobalSettingKeys.KEY_TITLE_FOOTER - , JsonUtils.toJson( - new TitleFooterDTO( - titleFooterLanguageMap - , username - , TimeUtil.getCurrentTimeStr())) - , String.format("Updated by %s at %s", username, DateUtils.defaultLocalDateTime(LocalDateTime.now()))); - return globalSettingDAO.insertGlobalSetting(dslContext, titleFooterDTO) == 1; - } else { - Map titleFooterLanguageMap = JsonUtils.fromJson(titleFooterDTO.getValue(), - new TypeReference() { - }).getTitleFooterLanguageMap(); - titleFooterLanguageMap.put( - LocaleUtils.getNormalLang(lang), new TitleFooter( - req.getTitleHead() - , req.getTitleSeparator() - , req.getFooterLink() - , req.getFooterCopyRight() - )); - titleFooterDTO.setValue(JsonUtils.toJson(new TitleFooterDTO(titleFooterLanguageMap - , username - , TimeUtil.getCurrentTimeStr()))); - return globalSettingDAO.updateGlobalSetting(dslContext, titleFooterDTO) == 1; - } - } - - private TitleFooterVO getCEDefaultTitleFooterVO() { - return getEEDefaultTitleFooterVO(); - } - - private TitleFooterVO getEEDefaultTitleFooterVO() { - return new TitleFooterVO( - i18nService.getI18n("job.manage.globalsettings.defaultTitleHead"), - "|", - i18nService.getI18n("job.manage.globalsettings.ee.footerLink").replace("{PAAS_SERVER_URL}", - jobManageConfig.getPaasServerUrl()), - i18nService.getI18n("job.manage.globalsettings.ee.footerCopyRight") - ); - } - - private TitleFooterVO getDefaultTitleFooterVOWithoutVersion() { - String jobEdition = jobManageConfig.getJobEdition(); - if (jobEdition.toLowerCase().equals("ee")) { - return getEEDefaultTitleFooterVO(); - } else { - return getCEDefaultTitleFooterVO(); - } - } - - @Override - public TitleFooterVO getTitleFooter() { - TitleFooterVO titleFooterVO = getTitleFooterWithoutVersion(); - // 渲染版本号 - Map valuesMap = new HashMap<>(); - String pattern = "(\\{\\{(.*?)\\}\\})"; - valuesMap.put(STRING_TPL_KEY_CURRENT_VERSION, "V" + buildProperties.getVersion()); - valuesMap.put(STRING_TPL_KEY_CURRENT_YEAR, DateUtils.getCurrentDateStr("yyyy")); - titleFooterVO.setFooterCopyRight( - StringUtil.replaceByRegex(titleFooterVO.getFooterCopyRight(), pattern, valuesMap) - ); - titleFooterVO.setFooterLink( - StringUtil.replaceByRegex(titleFooterVO.getFooterLink(), pattern, valuesMap) - ); - return titleFooterVO; - } - - public TitleFooterVO getTitleFooterWithoutVersion() { - GlobalSettingDTO titleFooterSettingDTO = globalSettingDAO.getGlobalSetting(dslContext, - GlobalSettingKeys.KEY_TITLE_FOOTER); - if (titleFooterSettingDTO == null) { - log.warn("Default titleFooter not configured, use system default, plz contact admin to set"); - return getDefaultTitleFooterVOWithoutVersion(); - } - TitleFooterDTO titleFooterDTO = JsonUtils.fromJson(titleFooterSettingDTO.getValue(), - new TypeReference() { - }); - String normalLang = LocaleUtils.getNormalLang(JobContextUtil.getUserLang()); - log.info("normalLang={}", normalLang); - TitleFooter titleFooter = titleFooterDTO.getTitleFooterLanguageMap().get(normalLang); - String currentYear = TimeUtil.getCurrentTimeStr("yyyy"); - if (titleFooter != null) { - String copyRight = titleFooter.getFooterCopyRight(); - //替换变量 - copyRight = copyRight.replace("${currentYear}", currentYear); - return new TitleFooterVO(titleFooter.getTitleHead(), titleFooter.getTitleSeparator(), - titleFooter.getFooterLink(), copyRight); - } else { - log.warn("TitleFooter of language:{} not configured, use default:zh_CN", normalLang); - return getDefaultTitleFooterVOWithoutVersion(); - } - } - - @Override - public TitleFooterWithDefaultVO getTitleFooterWithDefault(String username) { - return new TitleFooterWithDefaultVO(getTitleFooterWithoutVersion(), getDefaultTitleFooterVOWithoutVersion()); - } - - @Override - public Integer saveChannelTemplate(String username, ChannelTemplateReq req) { - if (StringUtils.isBlank(req.getChannelCode()) || StringUtils.isBlank(req.getMessageTypeCode())) { - throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME_AND_REASON, - new String[]{"channelCode", "channelCode cannot be blank"}); - } - NotifyTemplateDTO notifyTemplateDTO = notifyTemplateDAO.getNotifyTemplate( - req.getChannelCode(), - req.getMessageTypeCode(), - false - ); - String normalLang = LocaleUtils.getNormalLang(JobContextUtil.getUserLang()); - if (notifyTemplateDTO == null) { - notifyTemplateDTO = new NotifyTemplateDTO(null, req.getMessageTypeCode(), req.getMessageTypeCode(), - req.getChannelCode(), req.getTitle(), req.getContent(), req.getTitle(), req.getContent(), false, - username, System.currentTimeMillis(), username, System.currentTimeMillis()); - return notifyTemplateDAO.insertNotifyTemplate(notifyTemplateDTO); - } else { - if (normalLang.equals(LocaleUtils.LANG_EN) || normalLang.equals(LocaleUtils.LANG_EN_US)) { - notifyTemplateDTO.setTitleEn(req.getTitle()); - notifyTemplateDTO.setContentEn(req.getContent()); - } else { - notifyTemplateDTO.setTitle(req.getTitle()); - notifyTemplateDTO.setContent(req.getContent()); - } - notifyTemplateDTO.setLastModifyUser(username); - notifyTemplateDTO.setLastModifyTime(System.currentTimeMillis()); - return notifyTemplateDAO.updateNotifyTemplateById(notifyTemplateDTO); - } - } - - @Override - public Integer sendChannelTemplate(String username, ChannelTemplatePreviewReq req) { - List receiverList = StringUtil.strToList(req.getReceiverStr(), String.class, - NotifyConsts.SEPERATOR_COMMA); - Set receiverSet = new HashSet<>(receiverList); - notifySendService.sendUserChannelNotify( - null, - receiverSet, - req.getChannelCode(), - req.getTitle(), - req.getContent() - ); - return receiverSet.size(); - } - - @Override - public ChannelTemplateDetailWithDefaultVO getChannelTemplateDetail(String username, String channelCode, - String messageTypeCode) { - NotifyTemplateDTO currentNotifyTemplateDTO = notifyTemplateDAO.getNotifyTemplate( - channelCode, - messageTypeCode, - false - ); - NotifyTemplateDTO defaultNotifyTemplateDTO = notifyTemplateDAO.getNotifyTemplate( - channelCode, - messageTypeCode, - true - ); - // 渠道下无默认模板，则为新增渠道，使用通用模板 - if (defaultNotifyTemplateDTO == null) { - defaultNotifyTemplateDTO = notifyTemplateDAO.getNotifyTemplate( - NotifyConsts.NOTIFY_CHANNEL_CODE_COMMON, - messageTypeCode, - false - ); - if (defaultNotifyTemplateDTO == null) { - defaultNotifyTemplateDTO = notifyTemplateDAO.getNotifyTemplate( - NotifyConsts.NOTIFY_CHANNEL_CODE_COMMON, - messageTypeCode, - true - ); - } - if (defaultNotifyTemplateDTO != null) { - defaultNotifyTemplateDTO.setChannel(channelCode); - } else { - log.warn("common template of messageType={} not configed", messageTypeCode); - } - } - ChannelTemplateDetailVO currentChannelTemplateVO = - notifyTemplateConverter.convertToChannelTemplateDetailVO(currentNotifyTemplateDTO); - ChannelTemplateDetailVO defaultChannelTemplateVO = - notifyTemplateConverter.convertToChannelTemplateDetailVO(defaultNotifyTemplateDTO); - // 渠道下未配置模板则使用该渠道默认模板 - if (currentChannelTemplateVO == null) { - currentChannelTemplateVO = defaultChannelTemplateVO; - } - // 渠道下已配置模板信息不完整则使用默认模板内容填充 - if (currentChannelTemplateVO != null && currentChannelTemplateVO.getTitle() == null) { - if (defaultChannelTemplateVO != null) { - currentChannelTemplateVO.setTitle(defaultChannelTemplateVO.getTitle()); - } - } - if (currentChannelTemplateVO != null && currentChannelTemplateVO.getContent() == null) { - if (defaultChannelTemplateVO != null) { - currentChannelTemplateVO.setContent(defaultChannelTemplateVO.getContent()); - } - } - return new ChannelTemplateDetailWithDefaultVO(currentChannelTemplateVO, defaultChannelTemplateVO); - } - - @Override - public List listChannelTemplateStatus(String username) { - List resultList = new ArrayList<>(); - List messageCodeList = Arrays.asList( - NotifyConsts.NOTIFY_TEMPLATE_CODE_CONFIRMATION, - NotifyConsts.NOTIFY_TEMPLATE_CODE_EXECUTE_SUCCESS, - NotifyConsts.NOTIFY_TEMPLATE_CODE_EXECUTE_FAILURE, - NotifyConsts.NOTIFY_TEMPLATE_CODE_BEFORE_CRON_JOB_EXECUTE, - NotifyConsts.NOTIFY_TEMPLATE_CODE_BEFORE_CRON_JOB_END, - NotifyConsts.NOTIFY_TEMPLATE_CODE_CRON_EXECUTE_FAILED - ); - List notifyChannelWithIconVOList = listNotifyChannel(username); - notifyChannelWithIconVOList.forEach(notifyChannelWithIconVO -> { - String channelCode = notifyChannelWithIconVO.getCode(); - List configStatusList = new ArrayList<>(); - messageCodeList.forEach(messageCode -> { - Boolean configStatus = notifyTemplateDAO.existsNotifyTemplate(channelCode, messageCode, - false); - configStatusList.add(new TemplateBasicInfo(messageCode, - i18nService.getI18n(NotifyConsts.NOTIFY_TEMPLATE_NAME_PREFIX + messageCode), configStatus)); - }); - resultList.add(new ChannelTemplateStatusVO( - channelCode, - notifyChannelWithIconVO.getName(), - notifyChannelWithIconVO.getIcon(), - notifyChannelWithIconVO.getIsActive(), - configStatusList - )); - }); - return resultList; - } - - @Override - public String getDocCenterBaseUrl() { - String url; - if (org.apache.commons.lang3.StringUtils.isNotBlank(jobManageConfig.getBkDocRoot())) { - url = removeSuffixBackSlash(jobManageConfig.getBkDocRoot()); - } else { - String jobEdition = jobManageConfig.getJobEdition(); - if (jobEdition.toLowerCase().equals("ee")) { - // 企业版 - url = removeSuffixBackSlash(jobManageConfig.getPaasServerUrl()) + "/o/bk_docs_center"; - } else { - // 社区版 - url = removeSuffixBackSlash(jobManageConfig.getBkCERoot()) + "/docs"; - } - } - return url; - } - - private String getFeedBackRootUrl() { - String url; - if (StringUtils.isNotBlank(jobManageConfig.getBkFeedBackRoot())) { - url = removeSuffixBackSlash(jobManageConfig.getBkFeedBackRoot()); - } else { - // 企业版&社区版 - url = jobManageConfig.getBkCERoot() + "/s-mart/community"; - } - return url; - } - - private String getNodemanRootUrl() { - String url = jobManageConfig.getNodemanServerUrl(); - if (StringUtils.isBlank(url)) { - url = removeSuffixBackSlash(jobManageConfig.getPaasServerUrl()) + jobManageConfig.getPaasNodemanPath(); - } - return url; - } - - @Override - public Map getRelatedSystemUrls(String username) { - Map urlMap = new HashMap<>(); - urlMap.put(RelatedUrlKeys.KEY_BK_CMDB_ROOT_URL, jobManageConfig.getCmdbServerUrl()); - urlMap.put(RelatedUrlKeys.KEY_BK_CMDB_APP_INDEX_URL, - removeSuffixBackSlash(jobManageConfig.getCmdbServerUrl()) + jobManageConfig.getCmdbAppIndexPath()); - urlMap.put(RelatedUrlKeys.KEY_BK_NODEMAN_ROOT_URL, getNodemanRootUrl()); - urlMap.put(RelatedUrlKeys.KEY_BK_DOC_CENTER_ROOT_URL, getDocCenterBaseUrl()); - urlMap.put(RelatedUrlKeys.KEY_BK_DOC_JOB_ROOT_URL, getDocCenterBaseUrl()); - urlMap.put(RelatedUrlKeys.KEY_BK_FEED_BACK_ROOT_URL, getFeedBackRootUrl()); - return urlMap; - } - - private void addFileUploadConfig(Map configMap) { - FileUploadSettingVO fileUploadSettingVO = getFileUploadSettings(); - if (fileUploadSettingVO != null) { - configMap.put(GlobalSettingKeys.KEY_FILE_UPLOAD_SETTING, fileUploadSettingVO); - } - } - - private void addEnableFeatureFileManageConfig(Map configMap) { - configMap.put(GlobalSettingKeys.KEY_ENABLE_FEATURE_FILE_MANAGE, enableFeatureFileManage); - } - - private void addEnableUploadToArtifactoryConfig(Map configMap) { - configMap.put( - GlobalSettingKeys.KEY_ENABLE_UPLOAD_TO_ARTIFACTORY, - JobConstants.FILE_STORAGE_BACKEND_ARTIFACTORY.equals(localFileConfigForManage.getStorageBackend()) - ); - } - - @Override - public Map getJobConfig(String username) { - Map configMap = new HashMap<>(); - addFileUploadConfig(configMap); - addEnableFeatureFileManageConfig(configMap); - addEnableUploadToArtifactoryConfig(configMap); - return configMap; - } -} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/HostServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/HostServiceImpl.java deleted file mode 100644 index 6186c50fae..0000000000 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/HostServiceImpl.java +++ /dev/null @@ -1,1693 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.service.impl; - -import com.tencent.bk.job.common.cc.model.CcCloudAreaInfoDTO; -import com.tencent.bk.job.common.cc.model.CcDynamicGroupDTO; -import com.tencent.bk.job.common.cc.model.CcGroupHostPropDTO; -import com.tencent.bk.job.common.cc.model.CcInstanceDTO; -import com.tencent.bk.job.common.cc.model.InstanceTopologyDTO; -import com.tencent.bk.job.common.cc.sdk.CmdbClientFactory; -import com.tencent.bk.job.common.cc.sdk.IBizCmdbClient; -import com.tencent.bk.job.common.cc.service.CloudAreaService; -import com.tencent.bk.job.common.cc.util.TopologyUtil; -import com.tencent.bk.job.common.constant.ErrorCode; -import com.tencent.bk.job.common.constant.JobConstants; -import com.tencent.bk.job.common.constant.ResourceScopeTypeEnum; -import com.tencent.bk.job.common.exception.InternalException; -import com.tencent.bk.job.common.exception.InvalidParamException; -import com.tencent.bk.job.common.gse.service.QueryAgentStatusClient; -import com.tencent.bk.job.common.i18n.service.MessageI18nService; -import com.tencent.bk.job.common.model.BaseSearchCondition; -import com.tencent.bk.job.common.model.PageData; -import com.tencent.bk.job.common.model.dto.AppResourceScope; -import com.tencent.bk.job.common.model.dto.ApplicationDTO; -import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; -import com.tencent.bk.job.common.model.dto.DynamicGroupWithHost; -import com.tencent.bk.job.common.model.dto.HostDTO; -import com.tencent.bk.job.common.model.dto.ResourceScope; -import com.tencent.bk.job.common.model.vo.CloudAreaInfoVO; -import com.tencent.bk.job.common.model.vo.HostInfoVO; -import com.tencent.bk.job.common.util.ConcurrencyUtil; -import com.tencent.bk.job.common.util.JobContextUtil; -import com.tencent.bk.job.common.util.PageUtil; -import com.tencent.bk.job.common.util.StringUtil; -import com.tencent.bk.job.common.util.json.JsonUtils; -import com.tencent.bk.job.manage.common.TopologyHelper; -import com.tencent.bk.job.manage.common.consts.whiteip.ActionScopeEnum; -import com.tencent.bk.job.manage.dao.ApplicationHostDAO; -import com.tencent.bk.job.manage.dao.HostTopoDAO; -import com.tencent.bk.job.manage.manager.host.HostCache; -import com.tencent.bk.job.manage.model.db.CacheHostDO; -import com.tencent.bk.job.manage.model.dto.HostTopoDTO; -import com.tencent.bk.job.manage.model.dto.whiteip.CloudIPDTO; -import com.tencent.bk.job.manage.model.inner.ServiceListAppHostResultDTO; -import com.tencent.bk.job.manage.model.web.request.AgentStatisticsReq; -import com.tencent.bk.job.manage.model.web.request.ipchooser.AppTopologyTreeNode; -import com.tencent.bk.job.manage.model.web.request.ipchooser.ListHostByBizTopologyNodesReq; -import com.tencent.bk.job.manage.model.web.vo.CcTopologyNodeVO; -import com.tencent.bk.job.manage.model.web.vo.NodeInfoVO; -import com.tencent.bk.job.manage.model.web.vo.index.AgentStatistics; -import com.tencent.bk.job.manage.service.ApplicationService; -import com.tencent.bk.job.manage.service.HostService; -import com.tencent.bk.job.manage.service.WhiteIPService; -import lombok.Data; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.Pair; -import org.jooq.DSLContext; -import org.jooq.exception.DataAccessException; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.stereotype.Service; -import org.springframework.util.StopWatch; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.regex.Pattern; -import java.util.stream.Collectors; - -@Slf4j -@Service -public class HostServiceImpl implements HostService { - private final DSLContext dslContext; - private final ApplicationHostDAO applicationHostDAO; - private final ApplicationService applicationService; - private final HostTopoDAO hostTopoDAO; - private final TopologyHelper topologyHelper; - private final CloudAreaService cloudAreaService; - private final QueryAgentStatusClient queryAgentStatusClient; - private final WhiteIPService whiteIPService; - private final HostCache hostCache; - private final MessageI18nService i18nService; - - @Autowired - public HostServiceImpl(DSLContext dslContext, - ApplicationHostDAO applicationHostDAO, - ApplicationService applicationService, - HostTopoDAO hostTopoDAO, - TopologyHelper topologyHelper, - CloudAreaService cloudAreaService, - QueryAgentStatusClient queryAgentStatusClient, - WhiteIPService whiteIPService, - HostCache hostCache, - MessageI18nService i18nService) { - this.dslContext = dslContext; - this.applicationHostDAO = applicationHostDAO; - this.applicationService = applicationService; - this.hostTopoDAO = hostTopoDAO; - this.topologyHelper = topologyHelper; - this.cloudAreaService = cloudAreaService; - this.queryAgentStatusClient = queryAgentStatusClient; - this.whiteIPService = whiteIPService; - this.hostCache = hostCache; - this.i18nService = i18nService; - } - - @Override - public boolean existHost(long bizId, String ip) { - return applicationHostDAO.existsHost(bizId, ip); - } - - @Override - public List getHostsByAppId(Long appId) { - ApplicationDTO applicationDTO = applicationService.getAppByAppId(appId); - ResourceScope scope = applicationDTO.getScope(); - if (scope.getType() == ResourceScopeTypeEnum.BIZ) { - return applicationHostDAO.listHostInfoByBizId(Long.parseLong(scope.getId())); - } else { - return applicationHostDAO.listHostInfoByBizIds(applicationDTO.getSubBizIds(), null, null); - } - } - - private boolean insertOrUpdateOneAppHost(Long bizId, ApplicationHostDTO infoDTO) { - try { - applicationHostDAO.insertOrUpdateHost(infoDTO); - hostCache.addOrUpdateHost(infoDTO); - } catch (Throwable t) { - log.error(String.format("insertHost fail:bizId=%d,hostInfo=%s", bizId, infoDTO), t); - return false; - } - return true; - } - - @Override - public List insertHostsToBiz(Long bizId, List insertList) { - StopWatch watch = new StopWatch(); - // 插入主机 - watch.start("insertAppHostInfo"); - List insertFailHostIds = new ArrayList<>(); - boolean batchInserted = false; - try { - //尝试批量插入 - if (!insertList.isEmpty()) { - int affectedNum = applicationHostDAO.batchInsertAppHostInfo(insertList); - log.info("{} hosts inserted", affectedNum); - insertList.forEach(hostCache::addOrUpdateHost); - } - batchInserted = true; - } catch (Throwable throwable) { - if (throwable instanceof DataAccessException) { - String errorMessage = throwable.getMessage(); - if (errorMessage.contains("Duplicate entry") && errorMessage.contains("PRIMARY")) { - log.info("Fail to batchInsertAppHostInfo, try to insert one by one"); - } else { - log.warn("Fail to batchInsertAppHostInfo, try to insert one by one.", throwable); - } - } else { - log.warn("Fail to batchInsertAppHostInfo, try to insert one by one..", throwable); - } - //批量插入失败，尝试逐条插入 - for (ApplicationHostDTO infoDTO : insertList) { - if (!insertOrUpdateOneAppHost(bizId, infoDTO)) { - insertFailHostIds.add(infoDTO.getHostId()); - } - } - } - watch.stop(); - if (!batchInserted) { - watch.start("log insertAppHostInfo"); - if (!insertFailHostIds.isEmpty()) { - log.warn(String.format("appId=%s,insertFailHostIds.size=%d,insertFailHostIds=%s", - bizId, insertFailHostIds.size(), String.join(",", - insertFailHostIds.stream().map(Object::toString).collect(Collectors.toSet())))); - } - watch.stop(); - } - log.debug("Performance:insertHostsToApp:appId={},{}", bizId, watch.prettyPrint()); - return insertFailHostIds; - } - - @Override - public List updateHostsInBiz(Long bizId, List hostInfoList) { - StopWatch watch = new StopWatch(); - watch.start("updateAppHostInfo"); - // 更新主机 - long updateCount = 0L; - List updateHostIds = new ArrayList<>(); - long errorCount = 0L; - List errorHostIds = new ArrayList<>(); - long notChangeCount = 0L; - boolean batchUpdated = false; - try { - // 尝试批量更新 - if (!hostInfoList.isEmpty()) { - int affectedNum = applicationHostDAO.batchUpdateBizHostInfoByHostId(hostInfoList); - log.info("{} hosts updated", affectedNum); - hostInfoList.forEach(hostCache::addOrUpdateHost); - } - batchUpdated = true; - } catch (Throwable throwable) { - if (throwable instanceof DataAccessException) { - String errorMessage = throwable.getMessage(); - if (errorMessage.contains("Duplicate entry") && errorMessage.contains("PRIMARY")) { - log.info("Fail to batchUpdateAppHostInfoByHostId, try to update one by one"); - } else { - log.warn("Fail to batchUpdateAppHostInfoByHostId, try to update one by one.", throwable); - } - } else { - log.warn("Fail to batchUpdateAppHostInfoByHostId, try to update one by one..", throwable); - } - // 批量更新失败，尝试逐条更新 - for (ApplicationHostDTO hostInfoDTO : hostInfoList) { - try { - if (!applicationHostDAO.existAppHostInfoByHostId(hostInfoDTO)) { - applicationHostDAO.updateBizHostInfoByHostId(hostInfoDTO.getBizId(), hostInfoDTO); - hostCache.addOrUpdateHost(hostInfoDTO); - updateCount += 1; - updateHostIds.add(hostInfoDTO.getHostId()); - } else { - notChangeCount += 1; - } - } catch (Throwable t) { - log.error(String.format("updateHost fail:appId=%d,hostInfo=%s", bizId, hostInfoDTO), t); - errorCount += 1; - errorHostIds.add(hostInfoDTO.getHostId()); - } - } - } - watch.stop(); - if (!batchUpdated) { - watch.start("log updateAppHostInfo"); - log.info("Update host of appId={},errorCount={}," + - "updateCount={},notChangeCount={},errorHostIds={},updateHostIds={}", - bizId, errorCount, updateCount, notChangeCount, errorHostIds, updateHostIds); - watch.stop(); - } - log.debug("Performance:updateHostsInApp:appId={},{}", bizId, watch.prettyPrint()); - return errorHostIds; - } - - @Override - public List removeHostsFromBiz(Long bizId, List hostList) { - List hostIdList = hostList.stream().map(ApplicationHostDTO::getHostId).collect(Collectors.toList()); - StopWatch watch = new StopWatch(); - watch.start("deleteHostTopoOfBiz"); - List deleteFailHostIds = new ArrayList<>(); - // 删除业务与主机的关系 - hostTopoDAO.batchDeleteHostTopo(bizId, hostIdList); - watch.stop(); - watch.start("syncHostTopo"); - // 同步主机关系到host表 - hostIdList.forEach(hostId -> applicationHostDAO.syncHostTopo(hostId)); - watch.stop(); - log.debug("Performance:removeHostsFromBiz:bizId={},{}", bizId, watch.prettyPrint()); - return deleteFailHostIds; - } - - @Override - public long countHostsByOsType(String osType) { - return applicationHostDAO.countHostsByOsType(osType); - } - - public static List buildIpList(List hosts) { - List ipList = new ArrayList<>(); - for (ApplicationHostDTO host : hosts) { - ipList.add(host.getCloudAreaId() + ":" + host.getIp()); - } - return ipList; - } - - public static void fillAgentStatus(Map agentStatusMap, - List ipListStatus) { - if (CollectionUtils.isNotEmpty(ipListStatus)) { - for (HostInfoVO hostInfo : ipListStatus) { - if (hostInfo != null) { - String ip = hostInfo.getCloudAreaInfo().getId() + ":" + hostInfo.getIp(); - QueryAgentStatusClient.AgentStatus agentStatus = agentStatusMap.get(ip); - if (agentStatus != null && agentStatus.status == 1) { - hostInfo.setAlive(1); - } else { - hostInfo.setAlive(0); - } - } - } - } - } - - @Override - public PageData listAppHost(ApplicationHostDTO applicationHostInfoCondition, - BaseSearchCondition baseSearchCondition) { - return applicationHostDAO.listHostInfoByPage(applicationHostInfoCondition, baseSearchCondition); - } - - @Override - public CcTopologyNodeVO listAppTopologyTree(String username, AppResourceScope appResourceScope) { - ApplicationDTO appInfo = applicationService.getAppByAppId(appResourceScope.getAppId()); - if (appInfo == null) { - throw new InvalidParamException(ErrorCode.WRONG_APP_ID); - } - CcTopologyNodeVO ccTopologyNodeVO = new CcTopologyNodeVO(); - ccTopologyNodeVO.setObjectId("biz"); - ccTopologyNodeVO.setObjectName(i18nService.getI18n("cmdb.object.name.biz")); - ccTopologyNodeVO.setInstanceId(Long.valueOf(appResourceScope.getId())); - ccTopologyNodeVO.setInstanceName(appInfo.getName()); - if (appInfo.isAllBizSet()) { - // 全业务 - ccTopologyNodeVO.setCount((int) applicationHostDAO.countAllHosts()); - return ccTopologyNodeVO; - } else if (appInfo.isBizSet()) { - // 业务集 - ccTopologyNodeVO.setCount( - (int) applicationHostDAO.countHostsByBizIds(topologyHelper.getBizSetSubBizIds(appInfo)) - ); - return ccTopologyNodeVO; - } - InstanceTopologyDTO instanceTopology = topologyHelper.getTopologyTreeByApplication(appInfo); - return TopologyHelper.convertToCcTopologyTree(instanceTopology); - } - - /** - * 入参只需objectId与instanceId即可，其余字段可为空 - * - * @param username 用户名 - * @param appResourceScope 资源范围 - * @param treeNodeList 拓扑节点列表 - * @return 拓扑节点详情 - */ - @Override - public List getAppTopologyTreeNodeDetail(String username, - AppResourceScope appResourceScope, - List treeNodeList) { - log.info( - "Input(username={},appResourceScope={},treeNodeList={})", - username, appResourceScope, JsonUtils.toJson(treeNodeList) - ); - if (treeNodeList == null || treeNodeList.isEmpty()) { - return Collections.emptyList(); - } - // 查出业务 - ApplicationDTO appInfo = applicationService.getAppByAppId(appResourceScope.getAppId()); - // 查业务拓扑树 - IBizCmdbClient bizCmdbClient = CmdbClientFactory.getCmdbClient(JobContextUtil.getUserLang()); - InstanceTopologyDTO appTopologyTree = bizCmdbClient.getBizInstTopology( - Long.parseLong(appResourceScope.getId()) - ); - List nodeList = ConcurrencyUtil.getResultWithThreads(treeNodeList, 5, treeNode -> { - CcInstanceDTO ccInstanceDTO = new CcInstanceDTO(treeNode.getObjectId(), treeNode.getInstanceId()); - // 查拓扑节点完整信息 - InstanceTopologyDTO completeNode = TopologyUtil.findNodeFromTopo(appTopologyTree, ccInstanceDTO); - if (completeNode == null) { - return Collections.emptyList(); - } - AppTopologyTreeNode node = new AppTopologyTreeNode(); - node.setObjectId(completeNode.getObjectId()); - node.setInstanceId(completeNode.getInstanceId()); - node.setObjectName(completeNode.getObjectName()); - node.setInstanceName(completeNode.getInstanceName()); - return Collections.singletonList(node); - }); - // 排序 - Map map = new HashMap<>(); - for (AppTopologyTreeNode appTopologyTreeNode : nodeList) { - map.put(appTopologyTreeNode.getObjectId() + "_" + appTopologyTreeNode.getInstanceId(), appTopologyTreeNode); - } - List orderedList = new ArrayList<>(); - for (AppTopologyTreeNode appTopologyTreeNode : treeNodeList) { - String key = appTopologyTreeNode.getObjectId() + "_" + appTopologyTreeNode.getInstanceId(); - if (map.containsKey(key)) { - orderedList.add(map.get(key)); - } - } - return orderedList; - } - - @Override - public List> queryBizNodePaths(String username, - Long bizId, - List nodeList) { - // 查业务拓扑树 - IBizCmdbClient bizCmdbClient = CmdbClientFactory.getCmdbClient(JobContextUtil.getUserLang()); - InstanceTopologyDTO appTopologyTree = bizCmdbClient.getBizInstTopology(bizId); - // 搜索路径 - return TopologyHelper.findTopoPaths(appTopologyTree, nodeList); - } - - /** - * 入参只需objectId与instanceId即可，其余字段可为空 - * - * @param username 用户名 - * @param bizId 业务ID - * @param treeNodeList 拓扑节点列表 - * @return 节点详情 - */ - @Override - public List getBizHostsByNode(String username, - Long bizId, - List treeNodeList) { - log.info( - "Input(username={},bizId={},treeNodeList={})", - username, bizId, JsonUtils.toJson(treeNodeList) - ); - List nodeHostInfoList = new ArrayList<>(); - if (treeNodeList == null || treeNodeList.isEmpty()) { - return nodeHostInfoList; - } - // 查业务拓扑树 - IBizCmdbClient bizCmdbClient = CmdbClientFactory.getCmdbClient(JobContextUtil.getUserLang()); - InstanceTopologyDTO appTopologyTree = bizCmdbClient.getBizInstTopology(bizId); - final List allIpWithCloudIdList = Collections.synchronizedList(new ArrayList<>()); - nodeHostInfoList = ConcurrencyUtil.getResultWithThreads(treeNodeList, 5, treeNode -> { - CcInstanceDTO ccInstanceDTO = new CcInstanceDTO(treeNode.getObjectId(), treeNode.getInstanceId()); - NodeInfoVO nodeInfoVO = new NodeInfoVO(); - // 查拓扑节点完整信息 - InstanceTopologyDTO completeNode = TopologyUtil.findNodeFromTopo(appTopologyTree, ccInstanceDTO); - // 节点在拓扑树中不存在，可能未同步或被删除 - if (completeNode == null) { - return Collections.emptyList(); - } - // 构造条件查主机 - List conditions = new ArrayList<>(); - conditions.add(ccInstanceDTO); - List hosts = bizCmdbClient.getHosts(bizId, conditions); - List hostInfoVOList = hosts.stream().map(it -> { - HostInfoVO hostInfoVO = new HostInfoVO(); - hostInfoVO.setHostId(it.getHostId()); - hostInfoVO.setOs(it.getOs()); - hostInfoVO.setIp(it.getIp()); - hostInfoVO.setIpDesc(it.getIpDesc()); - hostInfoVO.setDisplayIp(it.getDisplayIp()); - hostInfoVO.setCloudAreaInfo(new CloudAreaInfoVO(it.getCloudAreaId(), - CloudAreaService.getCloudAreaNameFromCache(it.getCloudAreaId()))); - return hostInfoVO; - }).collect(Collectors.toList()); - // 收集涉及的IP列表 - List ipWithCloudIdList = buildIpList(hosts); - allIpWithCloudIdList.addAll(ipWithCloudIdList); - // 构造最终数据 - nodeInfoVO.setNodeType(completeNode.getObjectId()); - nodeInfoVO.setId(completeNode.getInstanceId()); - nodeInfoVO.setName(completeNode.getInstanceName()); - nodeInfoVO.setIpList(ipWithCloudIdList); - nodeInfoVO.setIpListStatus(hostInfoVOList); - return Collections.singletonList(nodeInfoVO); - }); - // 查出节点下主机与Agent状态 - Map agentStatusMap = - queryAgentStatusClient.batchGetAgentStatus(allIpWithCloudIdList); - // 批量设置agent状态 - for (NodeInfoVO nodeInfoVO : nodeHostInfoList) { - fillAgentStatus(agentStatusMap, nodeInfoVO.getIpListStatus()); - } - return nodeHostInfoList; - } - - @Override - public List getAppDynamicGroupList(String username, - AppResourceScope appResourceScope) { - ApplicationDTO applicationInfo = applicationService.getAppByAppId(appResourceScope.getAppId()); - - Map ccGroupInfoMap = new HashMap<>(); - Map> appId2GroupIdMap = new HashMap<>(); - if (ResourceScopeTypeEnum.BIZ_SET == appResourceScope.getType()) { - for (long subAppId : applicationInfo.getSubBizIds()) { - getCustomGroupListByBizId(subAppId, ccGroupInfoMap, appId2GroupIdMap); - } - } else { - getCustomGroupListByBizId( - Long.parseLong(appResourceScope.getId()), - ccGroupInfoMap, - appId2GroupIdMap - ); - } - - fillAppInfo(ccGroupInfoMap); - - return new ArrayList<>(ccGroupInfoMap.values()); - } - - @Override - public List getBizDynamicGroupHostList(String username, - Long bizId, - List dynamicGroupIdList) { - Set dynamicGroupIdSet = new HashSet<>(dynamicGroupIdList); - IBizCmdbClient cmdbClient = CmdbClientFactory.getCmdbClient(JobContextUtil.getUserLang()); - List dynamicGroupList = cmdbClient.getDynamicGroupList(bizId); - List dynamicGroupWithHostList = dynamicGroupList.stream() - .filter(dynamicGroup -> dynamicGroupIdSet.contains(dynamicGroup.getId())) - .map(CcDynamicGroupDTO::toDynamicGroupWithHost) - .collect(Collectors.toList()); - - // 查询业务数据 - ResourceScope resourceScope = new ResourceScope(ResourceScopeTypeEnum.BIZ, bizId.toString()); - ApplicationDTO appInfo = applicationService.getAppByScope(resourceScope); - - dynamicGroupWithHostList.forEach(dynamicGroupWithHost -> { - // 填充业务信息 - fillAppInfo(appInfo, dynamicGroupWithHost); - // 填充主机IP信息 - List ccGroupHostProps = - cmdbClient.getDynamicGroupIp(bizId, dynamicGroupWithHost.getId()); - List cloudIpList = new ArrayList<>(); - for (CcGroupHostPropDTO groupHost : ccGroupHostProps) { - if (CollectionUtils.isNotEmpty(groupHost.getCloudIdList())) { - cloudIpList.add(groupHost.getCloudIdList().get(0).getInstanceId() + ":" + groupHost.getFirstIp()); - } else { - log.warn("Wrong host info! No cloud area!|{}", groupHost); - } - } - dynamicGroupWithHost.setIpList(cloudIpList); - // 填充主机详情 - List hostIps = buildHostDTOListFromCloudIp(cloudIpList); - List hostDetailList = listHosts(hostIps); - if (hostDetailList.size() < cloudIpList.size()) { - Set existCloudIps = new HashSet<>(buildIpList(hostDetailList)); - Set cloudIpSet = new HashSet<>(cloudIpList); - cloudIpSet.removeAll(existCloudIps); - if (!cloudIpSet.isEmpty()) { - log.warn("Cannot find hostDetail for {} ips: {}", cloudIpSet.size(), cloudIpSet); - } - } - // 填充Agent状态信息 - fillAgentStatus(hostDetailList); - dynamicGroupWithHost.setIpListStatus(hostDetailList); - }); - return dynamicGroupWithHostList; - } - - private void fillAppInfo(ApplicationDTO appInfo, DynamicGroupWithHost dynamicGroupWithHost) { - if (appInfo != null) { - dynamicGroupWithHost.setBizName(appInfo.getName()); - dynamicGroupWithHost.setOwner(appInfo.getBkSupplierAccount()); - dynamicGroupWithHost.setOwnerName(appInfo.getBkSupplierAccount()); - } else { - log.warn("appInfo is null"); - } - } - - private List buildHostDTOListFromCloudIp(List cloudIpList) { - if (CollectionUtils.isEmpty(cloudIpList)) { - return Collections.emptyList(); - } - return cloudIpList.stream().map(HostDTO::fromCloudIp).collect(Collectors.toList()); - } - - @Override - public CcTopologyNodeVO listAppTopologyHostTree(String username, AppResourceScope appResourceScope) { - StopWatch watch = new StopWatch("listAppTopologyHostTree"); - ApplicationDTO appInfo = applicationService.getAppByAppId(appResourceScope.getAppId()); - watch.start("listAppTopologyTree"); - CcTopologyNodeVO topologyTree = this.listAppTopologyTree(username, appResourceScope); - watch.stop(); - if (appResourceScope.getType() == ResourceScopeTypeEnum.BIZ) { - watch.start("fillHostInfo"); - fillHostInfo(username, Long.valueOf(appResourceScope.getId()), topologyTree, true); - watch.stop(); - } else { - topologyTree.setIpListStatus( - listHostByAppTopologyNodes( - username, appResourceScope.getAppId(), Collections.singletonList( - new AppTopologyTreeNode( - "biz", - "biz", - Long.parseLong(appResourceScope.getId()), - appInfo.getName(), - null - ) - ) - ) - ); - } - log.debug(watch.toString()); - return topologyTree; - } - - /** - * 建立moduleId与拓扑树module子节点映射Map - * 顺便初始化IpListStatus - */ - private void constructMap(Map map, CcTopologyNodeVO topologyTree) { - if (topologyTree == null) return; - if (topologyTree.getObjectId().equals("module")) { - topologyTree.setIpListStatus(new ArrayList<>()); - map.put(topologyTree.getInstanceId(), topologyTree); - } else { - List childs = topologyTree.getChild(); - if (childs != null && childs.size() > 0) { - childs.forEach(child -> constructMap(map, child)); - } - } - } - - // 统计拓扑树各节点主机数量 - public void countHosts(CcTopologyNodeVO topologyTree) { - List childs = topologyTree.getChild(); - topologyTree.setHostIdSet(new HashSet<>()); - if (childs != null && !childs.isEmpty()) { - childs.forEach(child -> { - countHosts(child); - topologyTree.getHostIdSet().addAll(child.getHostIdSet()); - }); - topologyTree.setCount(topologyTree.getHostIdSet().size()); - } else { - List hosts = topologyTree.getIpListStatus(); - if (hosts != null) { - topologyTree.getHostIdSet().addAll( - hosts.parallelStream().map(HostInfoVO::getHostId).collect(Collectors.toSet())); - topologyTree.setCount(topologyTree.getHostIdSet().size()); - } else { - topologyTree.setCount(0); - } - } - } - - public void clearHosts(CcTopologyNodeVO topologyTree) { - List childs = topologyTree.getChild(); - topologyTree.setHostIdSet(null); - if (childs != null && !childs.isEmpty()) { - childs.forEach(this::clearHosts); - topologyTree.setIpList(null); - topologyTree.setIpListStatus(null); - } else { - topologyTree.setIpList(null); - topologyTree.setIpListStatus(null); - } - } - - public void fillHostInfo(String username, Long bizId, CcTopologyNodeVO topologyTree, boolean updateAgentStatus) { - if (topologyTree == null) { - return; - } - StopWatch watch = new StopWatch("fillHostInfo"); - watch.start("constructMap:" + topologyTree.getInstanceId()); - Map map = new HashMap<>(); - constructMap(map, topologyTree); - log.info("{} modules mapped", map.keySet().size()); - if (map.keySet().size() < 200) { - log.info("module ids:{}", map.keySet()); - } else { - log.info("more than 200 module ids, do not print"); - } - watch.stop(); - watch.start("getHosts of Module:" + topologyTree.getInstanceId()); - // 从DB拿主机 - List dbHosts = applicationHostDAO.listHostInfoByBizId(bizId); - log.info("find {} hosts from DB", dbHosts.size()); - watch.stop(); - List hostInfoVOList = dbHosts.stream().map(it -> { - HostInfoVO hostInfoVO = new HostInfoVO(); - hostInfoVO.setHostId(it.getHostId()); - hostInfoVO.setOs(it.getOs()); - hostInfoVO.setIp(it.getIp()); - hostInfoVO.setIpDesc(it.getIpDesc()); - hostInfoVO.setDisplayIp(it.getDisplayIp()); - hostInfoVO.setCloudAreaInfo(new CloudAreaInfoVO(it.getCloudAreaId(), - CloudAreaService.getCloudAreaNameFromCache(it.getCloudAreaId()))); - return hostInfoVO; - }).collect(Collectors.toList()); - //批量设置agent状态 - if (updateAgentStatus) { - watch.start("batchGetAgentStatus"); - Map agentStatusMap = - queryAgentStatusClient.batchGetAgentStatus(buildIpList(dbHosts)); - fillAgentStatus(agentStatusMap, hostInfoVOList); - watch.stop(); - } - //将主机挂载到topo树 - watch.start("setToTopoTree"); - for (int i = 0; i < hostInfoVOList.size(); i++) { - ApplicationHostDTO host = dbHosts.get(i); - HostInfoVO hostInfoVO = hostInfoVOList.get(i); - host.getModuleId().forEach(moduleId -> { - CcTopologyNodeVO moduleNode = map.get(moduleId); - if (moduleNode == null) { - log.warn("cannot find moduleNode in topoTree, cache may expire, ignore this moduleNode"); - } else { - moduleNode.getIpListStatus().add(hostInfoVO); - } - }); - } - watch.stop(); - watch.start("countHosts"); - countHosts(topologyTree); - watch.stop(); - log.debug(watch.toString()); - } - - public void fillAgentStatus(List hosts) { - // 查出节点下主机与Agent状态 - List ipWithCloudIdList = buildIpList(hosts); - // 批量设置agent状态 - Map agentStatusMap = - queryAgentStatusClient.batchGetAgentStatus(ipWithCloudIdList); - if (CollectionUtils.isNotEmpty(hosts)) { - for (ApplicationHostDTO hostInfoDTO : hosts) { - if (hostInfoDTO != null) { - String ip = hostInfoDTO.getCloudAreaId() + ":" + hostInfoDTO.getIp(); - QueryAgentStatusClient.AgentStatus agentStatus = agentStatusMap.get(ip); - hostInfoDTO.setGseAgentAlive(agentStatus != null && agentStatus.status == 1); - } - } - } - } - - @Override - public CcTopologyNodeVO listAppTopologyHostCountTree(String username, - AppResourceScope appResourceScope) { - StopWatch watch = new StopWatch("listAppTopologyHostCountTree"); - watch.start("listAppTopologyHostCountTree"); - CcTopologyNodeVO topologyTree = this.listAppTopologyTree(username, appResourceScope); - watch.stop(); - if (appResourceScope.getType() == ResourceScopeTypeEnum.BIZ) { - watch.start("fillHostInfo"); - fillHostInfo(username, Long.valueOf(appResourceScope.getId()), topologyTree, false); - watch.stop(); - watch.start("clearHosts"); - clearHosts(topologyTree); - watch.stop(); - } - log.debug(watch.toString()); - return topologyTree; - } - - private boolean matchSearchContent(String searchContent, String targetStr) { - if (searchContent == null) { - return true; - } - if (targetStr == null) { - return false; - } - String[] searchContents = searchContent.split("[;,；，\\n\\s|\\u00A0]+"); - for (String content : searchContents) { - if (StringUtils.isNotBlank(content) && targetStr.toLowerCase().contains(content.toLowerCase())) { - return true; - } - } - return false; - } - - @Override - public PageData listIPByBizTopologyNodes(String username, - AppResourceScope appResourceScope, - ListHostByBizTopologyNodesReq req) { - PageData hostInfoVOResult = listHostByAppTopologyNodes(username, appResourceScope, req); - List data = - hostInfoVOResult.getData().parallelStream() - .map(it -> it.getCloudAreaInfo().getId().toString() + ":" + it.getIp()) - .collect(Collectors.toList()); - return new PageData<>( - hostInfoVOResult.getStart(), - hostInfoVOResult.getPageSize(), - hostInfoVOResult.getTotal(), - data - ); - } - - public List getHostInfoVOsByHostInfoDTOs(List hosts) { - return hosts.stream().map(it -> { - String ip = it.getIp(); - HostInfoVO hostInfoVO = new HostInfoVO(); - hostInfoVO.setHostId(it.getHostId()); - hostInfoVO.setOs(it.getOs()); - hostInfoVO.setIp(ip); - hostInfoVO.setIpDesc(it.getIpDesc()); - hostInfoVO.setDisplayIp(it.getDisplayIp()); - hostInfoVO.setCloudAreaInfo(new CloudAreaInfoVO(it.getCloudAreaId(), - CloudAreaService.getCloudAreaNameFromCache(it.getCloudAreaId()))); - if (it.getGseAgentAlive()) { - hostInfoVO.setAlive(1); - } else { - hostInfoVO.setAlive(0); - } - return hostInfoVO; - }).collect(Collectors.toList()); - } - - // DB分页 - @Override - public PageData listHostByAppTopologyNodes(String username, - AppResourceScope appResourceScope, - ListHostByBizTopologyNodesReq req) { - StopWatch watch = new StopWatch("listHostByBizTopologyNodes"); - watch.start("genConditions"); - String searchContent = req.getSearchContent(); - Integer agentStatus = req.getAgentStatus(); - // 查出业务 - ApplicationDTO appInfo = applicationService.getAppByAppId(appResourceScope.getAppId()); - List moduleIds = null; - List bizIds = null; - if (appInfo.getScope().getType() == ResourceScopeTypeEnum.BIZ) { - // 普通业务需要以moduleIds作为查询条件 - moduleIds = getBizModuleIdsByTopoNodes( - username, Long.valueOf(appInfo.getScope().getId()), req.getAppTopoNodeList() - ); - } else if (!appInfo.isAllBizSet() && appInfo.isBizSet()) { - // 业务集：仅根据业务查主机 - // 查出对应的所有普通业务 - bizIds = topologyHelper.getBizSetSubBizIds(appInfo); - } - - //获取所有云区域，找出名称符合条件的所有CloudAreaId - List cloudAreaIds = null; - if (StringUtils.isNotBlank(searchContent)) { - cloudAreaIds = new ArrayList<>(); - List allCloudAreaInfos = cloudAreaService.getCloudAreaList(); - for (CcCloudAreaInfoDTO it : allCloudAreaInfos) { - if (matchSearchContent(searchContent, it.getName())) { - cloudAreaIds.add(it.getId()); - } - } - log.debug("filter by cloudAreaIds={}", cloudAreaIds); - } - List searchContents = null; - if (req.getSearchContent() != null) { - searchContents = Arrays.asList(searchContent.split("[;,；，\\n\\s|\\u00A0]+")); - } - watch.stop(); - //分页 - Pair pagePair = PageUtil.normalizePageParam(req.getStart(), req.getPageSize()); - watch.start("listHostInfoBySearchContents"); - List hosts = applicationHostDAO.listHostInfoBySearchContents(bizIds, moduleIds, - cloudAreaIds, searchContents, agentStatus, pagePair.getLeft(), pagePair.getRight()); - watch.stop(); - watch.start("countHostInfoBySearchContents"); - Long count = applicationHostDAO.countHostInfoBySearchContents(bizIds, moduleIds, cloudAreaIds, searchContents - , agentStatus); - watch.stop(); - watch.start("getHostInfoVOsByHostInfoDTOs"); - List finalHostInfoVOList = getHostInfoVOsByHostInfoDTOs(hosts); - watch.stop(); - if (watch.getTotalTimeMillis() > 5000) { - log.info("listHostByBizTopologyNodes is slow:{}", watch); - } - return new PageData<>(pagePair.getLeft().intValue(), pagePair.getRight().intValue(), count, - finalHostInfoVOList); - } - - private Pair, Set> parseInputCloudIPList(List checkIpList) { - Set inputCloudIPSet = new HashSet<>(); - Set inputIPWithoutCloudIdSet = new HashSet<>(); - Pattern pattern = Pattern.compile("[:：]"); - for (String ip : checkIpList) { - if (StringUtils.isBlank(ip)) { - continue; - } - if (ip.contains(":") || ip.contains("：")) { - //有云区域Id - String[] arr = pattern.split(ip); - inputCloudIPSet.add(new CloudIPDTO(Long.parseLong(arr[0].trim()), arr[1].trim())); - } else { - inputIPWithoutCloudIdSet.add(ip.trim()); - } - } - return Pair.of(inputCloudIPSet, inputIPWithoutCloudIdSet); - } - - private List buildCloudIPList(List cloudIPDTOList) { - if (CollectionUtils.isEmpty(cloudIPDTOList)) { - return Collections.emptyList(); - } - return cloudIPDTOList.parallelStream().map(CloudIPDTO::getCloudIP).collect(Collectors.toList()); - } - - /** - * 根据主机所属CMDB业务ID判断主机是否归属于某个Job业务 - * - * @param appDTO Job业务信息 - * @param hostBizId 主机CMDB业务ID - * @return 是否归属于Job业务 - */ - private boolean hostBelongToApp(ApplicationDTO appDTO, Long hostBizId) { - if (appDTO.isAllBizSet()) { - return true; - } else if (appDTO.isBizSet()) { - return appDTO.getSubBizIds().contains(hostBizId); - } else { - return hostBizId.equals(appDTO.getBizIdIfBizApp()); - } - } - - /** - * 分离存在于业务下与不存在于业务下的主机 - * - * @param appId Job业务ID - * @param mixedCloudIPList 未分离的所有IP列表 - * @param inAppIPList 归属于Job业务的IP列表 - * @param notInAppIPList 不属于Job业务的IP列表 - */ - private void separateNotInAppIP(Long appId, - List mixedCloudIPList, - List inAppIPList, - List notInAppIPList) { - ApplicationDTO appDTO = applicationService.getAppByAppId(appId); - if (appDTO.isAllBizSet()) { - // 全业务 - inAppIPList.addAll(mixedCloudIPList); - return; - } - List cloudIPList = buildCloudIPList(mixedCloudIPList); - List subBizIds; - if (appDTO.isBizSet()) { - // 业务集 - subBizIds = appDTO.getSubBizIds(); - log.info("check hosts of appId:{}, subBizIds:{}", appId, StringUtil.concatCollection(subBizIds)); - } else { - // 普通业务 - subBizIds = Collections.singletonList(appDTO.getBizIdIfBizApp()); - } - // 首先根据本地DB缓存的主机数据进行分离 - List hostDTOList = applicationHostDAO.listHostInfoByBizAndCloudIPs( - subBizIds, - cloudIPList - ); - Set inAppCloudIPSet = hostDTOList.parallelStream() - .map(ApplicationHostDTO::getCloudIp) - .collect(Collectors.toSet()); - List notInAppIPListByLocal = new ArrayList<>(); - mixedCloudIPList.forEach(cloudIPDTO -> { - if (inAppCloudIPSet.contains(cloudIPDTO.getCloudIP())) { - inAppIPList.add(cloudIPDTO); - } else { - notInAppIPListByLocal.add(cloudIPDTO); - } - }); - // 对于本地不在目标业务下的主机再到CMDB查询 - IBizCmdbClient bizCmdbClient = CmdbClientFactory.getCmdbClient(); - List ipDTOList = notInAppIPListByLocal.parallelStream() - .map(CloudIPDTO::toHostDTO) - .collect(Collectors.toList()); - List cmdbExistHosts = bizCmdbClient.listHostsByIps( - ipDTOList.stream().map(HostDTO::toCloudIp).collect(Collectors.toList())); - Map cmdbHostsMap = new HashMap<>(); - Set cmdbExistCloudIPSet = new HashSet<>(); - cmdbExistHosts.forEach(host -> { - cmdbExistCloudIPSet.add(host.getCloudIp()); - cmdbHostsMap.put(host.getCloudIp(), host); - }); - List notInCmdbIpList = new ArrayList<>(); - for (CloudIPDTO cloudIPDTO : notInAppIPListByLocal) { - if (!cmdbExistCloudIPSet.contains(cloudIPDTO.getCloudIP())) { - // 主机在CMDB中不存在 - notInCmdbIpList.add(cloudIPDTO); - continue; - } - ApplicationHostDTO host = cmdbHostsMap.get(cloudIPDTO.getCloudIP()); - if (hostBelongToApp(appDTO, host.getBizId())) { - inAppIPList.add(cloudIPDTO); - } else { - notInAppIPList.add(cloudIPDTO); - } - } - if (!notInCmdbIpList.isEmpty()) { - log.warn("ips not in cmdb:{}", StringUtil.concatCollection(notInCmdbIpList)); - } - notInAppIPList.addAll(notInCmdbIpList); - } - - @Override - public List getHostsByIp( - String username, - Long appId, - ActionScopeEnum actionScope, - List checkIpList - ) { - log.info("Input=({},{},{})", username, appId, checkIpList); - if (checkIpList == null || checkIpList.isEmpty()) { - return Collections.emptyList(); - } - Pair, Set> pair = parseInputCloudIPList(checkIpList); - Set inputCloudIPSet = pair.getLeft(); - Set inputIPWithoutCloudIdSet = pair.getRight(); - log.debug("inputCloudIPSet={},inputIPWithoutCloudIdSet={}", inputIPWithoutCloudIdSet, inputIPWithoutCloudIdSet); - // 0.通过对输入的无云区域ID的IP进行云区域补全得到的完整IP，云区域ID来源：白名单、DB、默认值 - Set makeupCloudIPSet = new HashSet<>(); - // 1.查出对当前业务生效的白名单IP - List appWhiteIPList = whiteIPService.listWhiteIP(appId, actionScope); - log.debug("appWhiteIPList={}", appWhiteIPList); - Set appWhiteIPSet = new HashSet<>(); - appWhiteIPList.forEach(appWhiteIP -> { - appWhiteIPSet.add(appWhiteIP.getCloudIP()); - // 若输入的纯IP与某个白名单IP匹配，则记录下其云区域ID用于后续查找 - if (inputIPWithoutCloudIdSet.contains(appWhiteIP.getIp().trim())) { - makeupCloudIPSet.add(appWhiteIP); - } - }); - // 2.根据纯IP从DB查出所有可能的含云区域ID的完整IP - List hostByPureIpList = applicationHostDAO.listHostInfo(null, - inputIPWithoutCloudIdSet); - Set hostByPureIpInDB = hostByPureIpList.parallelStream() - .map(host -> new CloudIPDTO(host.getCloudAreaId(), host.getIp())).collect(Collectors.toSet()); - makeupCloudIPSet.addAll(hostByPureIpInDB); - inputIPWithoutCloudIdSet.removeAll( - hostByPureIpInDB.parallelStream().map(CloudIPDTO::getIp).collect(Collectors.toSet()) - ); - // 3.DB中找不到的纯IP视为使用默认云区域ID - inputIPWithoutCloudIdSet.forEach(pureIp -> - makeupCloudIPSet.add(new CloudIPDTO(JobConstants.DEFAULT_CLOUD_AREA_ID, pureIp)) - ); - - log.debug("makeupCloudIPSet={}", makeupCloudIPSet); - inputCloudIPSet.addAll(makeupCloudIPSet); - List inputCloudIPList = new ArrayList<>(inputCloudIPSet); - - // 4.找出输入IP中的白名单IP - List inputWhiteIPList = new ArrayList<>(); - List inputNotWhiteIPList = new ArrayList<>(); - inputCloudIPList.forEach(inputCloudIP -> { - if (appWhiteIPSet.contains(inputCloudIP.getCloudIP())) { - inputWhiteIPList.add(inputCloudIP); - } else { - inputNotWhiteIPList.add(inputCloudIP); - } - }); - // 5.非白名单IP校验是否在业务下 - List inAppIPList = new ArrayList<>(); - List notInAppIPList = new ArrayList<>(); - separateNotInAppIP(appId, inputNotWhiteIPList, inAppIPList, notInAppIPList); - // 6.不在业务下的IP打印出来 - if (!notInAppIPList.isEmpty()) { - log.warn( - "ips not in app {}:{}", - appId, - StringUtil.concatCollection(notInAppIPList) - ); - } - // 7.查询主机详情 - List validIPList = new ArrayList<>(inputWhiteIPList); - validIPList.addAll(inAppIPList); - // 根据IP从本地查主机 - List hostDTOList = applicationHostDAO.listHostInfoByBizAndCloudIPs(null, - validIPList.stream().map(CloudIPDTO::getCloudIP).collect(Collectors.toList())); - Set localHostCloudIPSet = hostDTOList.parallelStream() - .map(ApplicationHostDTO::getCloudIp) - .collect(Collectors.toSet()); - validIPList.removeIf(cloudIPDTO -> localHostCloudIPSet.contains(cloudIPDTO.getCloudIP())); - // 查不到的再去CMDB查 - if (!validIPList.isEmpty()) { - - IBizCmdbClient cmdbClient = CmdbClientFactory.getCmdbClient(); - List cmdbHosts = cmdbClient.listHostsByIps( - validIPList.parallelStream() - .map(CloudIPDTO::getCloudIP) - .collect(Collectors.toList()) - ); - hostDTOList.addAll(cmdbHosts); - Set cmdbHostIpSet = cmdbHosts.parallelStream() - .map(ApplicationHostDTO::getCloudIp).collect(Collectors.toSet()); - validIPList.removeIf(cloudIPDTO -> cmdbHostIpSet.contains(cloudIPDTO.getCloudIP())); - if (!validIPList.isEmpty()) { - log.warn("Cannot find hostinfo of ips:{},ignore", validIPList); - } - } - - // 8.查询Agent状态 - fillAgentStatus(hostDTOList); - // 9.类型转换，返回 - List hostInfoList = new ArrayList<>(); - hostDTOList.forEach(hostInfo -> hostInfoList.add(TopologyHelper.convertToHostInfoVO(hostInfo))); - return hostInfoList; - } - - private void getCustomGroupListByBizId(Long bizId, - Map ccGroupInfoList, - Map> bizId2GroupIdMap) { - List groupIdList = new ArrayList<>(); - List ccGroupList = CmdbClientFactory.getCmdbClient(JobContextUtil.getUserLang()) - .getDynamicGroupList(bizId); - ccGroupList.forEach(ccDynamicGroup -> { - ccGroupInfoList.put(ccDynamicGroup.getId(), ccDynamicGroup.toDynamicGroupWithHost()); - groupIdList.add(ccDynamicGroup.getId()); - }); - bizId2GroupIdMap.put(bizId, groupIdList); - } - - private void fillAppInfo(Map ccGroupInfoMap) { - // 分组中的获取app信息 - Set bizIdSet = new HashSet<>(); - for (DynamicGroupWithHost groupInfo : ccGroupInfoMap.values()) { - bizIdSet.add(groupInfo.getBizId()); - } - List appInfoList = applicationService.listBizAppsByBizIds(bizIdSet); - - Map id2AppInfoMap = new HashMap<>(appInfoList.size()); - for (ApplicationDTO appInfo : appInfoList) { - id2AppInfoMap.put(appInfo.getScope().getId(), appInfo); - } - - for (DynamicGroupWithHost groupInfo : ccGroupInfoMap.values()) { - ApplicationDTO appInfo = id2AppInfoMap.get(groupInfo.getBizId().toString()); - if (appInfo == null) { - groupInfo.setBizName(""); - groupInfo.setOwner(""); - groupInfo.setOwnerName(""); - } else { - groupInfo.setBizName(appInfo.getName()); - groupInfo.setOwner(appInfo.getBkSupplierAccount()); - groupInfo.setOwnerName(appInfo.getBkSupplierAccount()); - } - } - } - - @Override - public List listHostByAppTopologyNodes(String username, Long appId, - List appTopoNodeList) { - return listHostByAppTopologyNodes(username, appId, appTopoNodeList, null, null); - } - - private List getBizModuleIdsByTopoNodes(String username, - Long bizId, - List appTopoNodeList) { - if (appTopoNodeList == null || appTopoNodeList.isEmpty()) { - return Collections.emptyList(); - } - IBizCmdbClient bizCmdbClient = CmdbClientFactory.getCmdbClient(JobContextUtil.getUserLang()); - // 查出业务 - ApplicationDTO appInfo = applicationService.getAppByScope( - new ResourceScope(ResourceScopeTypeEnum.BIZ, bizId.toString()) - ); - List moduleIds = new ArrayList<>(); - // 普通业务可能根据各级自定义节点查主机，必须先根据拓扑树转为moduleId再查 - // 查业务拓扑树 - InstanceTopologyDTO appTopologyTree = bizCmdbClient.getBizInstTopology(bizId); - if (appTopologyTree == null) { - throw new InternalException("Fail to getBizTopo of bizId " + bizId + " from CMDB", - ErrorCode.INTERNAL_ERROR); - } - for (AppTopologyTreeNode treeNode : appTopoNodeList) { - CcInstanceDTO ccInstanceDTO = new CcInstanceDTO(treeNode.getObjectId(), treeNode.getInstanceId()); - // 查拓扑节点完整信息 - InstanceTopologyDTO completeNode = TopologyUtil.findNodeFromTopo(appTopologyTree, ccInstanceDTO); - if (completeNode == null) { - log.warn("Cannot find node in topo, node:{}, topo:", JsonUtils.toJson(ccInstanceDTO)); - TopologyUtil.printTopo(appTopologyTree); - throw new InternalException("Fail to find node {objectId:" + ccInstanceDTO.getObjectType() + "," + - "instanceId:" + ccInstanceDTO.getInstanceId() + ") from app topo", ErrorCode.INTERNAL_ERROR); - } - moduleIds.addAll(TopologyUtil.findModuleIdsFromTopo(completeNode)); - } - return moduleIds; - } - - private List listHostByAppTopologyNodes(String username, - Long appId, - List appTopoNodeList, - Long start, - Long limit) { - if (appTopoNodeList == null || appTopoNodeList.isEmpty()) { - return Collections.emptyList(); - } - IBizCmdbClient bizCmdbClient = CmdbClientFactory.getCmdbClient(JobContextUtil.getUserLang()); - // 查出业务 - ApplicationDTO appInfo = applicationService.getAppByAppId(appId); - Set moduleIds = new HashSet<>(); - List hosts = new ArrayList<>(); - if (appInfo.isBiz()) { - // 普通业务可能根据各级自定义节点查主机，必须先根据拓扑树转为moduleId再查 - // 查业务拓扑树 - InstanceTopologyDTO appTopologyTree = bizCmdbClient.getBizInstTopology( - Long.parseLong(appInfo.getScope().getId()) - ); - for (AppTopologyTreeNode treeNode : appTopoNodeList) { - CcInstanceDTO ccInstanceDTO = new CcInstanceDTO(treeNode.getObjectId(), treeNode.getInstanceId()); - // 查拓扑节点完整信息 - InstanceTopologyDTO completeNode = TopologyUtil.findNodeFromTopo(appTopologyTree, ccInstanceDTO); - if (completeNode == null) { - log.warn("Cannot find completeNode {} from topologyTree", - ccInstanceDTO.getObjectType() + ":" + ccInstanceDTO.getInstanceId()); - continue; - } - moduleIds.addAll(TopologyUtil.findModuleIdsFromTopo(completeNode)); - } - // 查所有hostIds - List hostTopoDTOList = hostTopoDAO.listHostTopoByModuleIds(moduleIds, start, limit); - List hostIdList = - hostTopoDTOList.parallelStream().map(HostTopoDTO::getHostId).collect(Collectors.toList()); - hosts = applicationHostDAO.listHostInfoByHostIds(hostIdList); - } else if (appInfo.isAllBizSet()) { - // 全业务 - hosts = applicationHostDAO.listAllHostInfo(start, limit); - } else if (appInfo.isBizSet()) { - // 业务集：仅根据业务查主机 - // 查出对应的所有普通业务 - List allBizIds = topologyHelper.getBizSetSubBizIds(appInfo); - hosts = applicationHostDAO.listHostInfoByBizIds(allBizIds, start, limit); - } - - // 查出节点下主机与Agent状态 - List ipWithCloudIdList = buildIpList(hosts); - // 批量设置agent状态 - Map agentStatusMap = - queryAgentStatusClient.batchGetAgentStatus(ipWithCloudIdList); - List hostInfoVOList = getHostInfoVOsByHostInfoDTOs(hosts); - fillAgentStatus(agentStatusMap, hostInfoVOList); - // 排序 - //异常排序 - List orderedHostInfoVOList = - hostInfoVOList.stream().filter(it -> it.getAlive() == 0).collect(Collectors.toList()); - orderedHostInfoVOList.addAll(hostInfoVOList.stream().filter(it -> it.getAlive() == 1) - .collect(Collectors.toList())); - hostInfoVOList = orderedHostInfoVOList; - //重复靠前 - Map> repeatMap = new HashMap<>(); - for (HostInfoVO it : hostInfoVOList) { - String ip = it.getIp(); - if (repeatMap.containsKey(it.getIp())) { - List hostInfoVOs = repeatMap.get(ip); - hostInfoVOs.add(it); - repeatMap.put(ip, hostInfoVOs); - } else { - List hostInfoVOs = new ArrayList<>(); - hostInfoVOs.add(it); - repeatMap.put(ip, hostInfoVOs); - } - } - hostInfoVOList.sort((o1, o2) -> repeatMap.get(o2.getIp()).size() - repeatMap.get(o1.getIp()).size()); - Set addedHostIds = new HashSet<>(); - List finalHostInfoVOList = new ArrayList<>(); - for (HostInfoVO it : hostInfoVOList) { - String ip = it.getIp(); - List hostInfoVOs = repeatMap.get(ip); - hostInfoVOs.forEach(hostInfoVO -> { - if (!addedHostIds.contains(hostInfoVO.getHostId())) { - finalHostInfoVOList.add(hostInfoVO); - addedHostIds.add(hostInfoVO.getHostId()); - } - }); - } - return finalHostInfoVOList; - } - - @Override - public AgentStatistics getAgentStatistics(String username, Long appId, AgentStatisticsReq agentStatisticsReq) { - log.info("Input=({},{},{})", username, appId, JsonUtils.toJson(agentStatisticsReq)); - ApplicationDTO applicationDTO = applicationService.getAppByAppId(appId); - List hostsByIp = getHostsByIp(username, appId, null, agentStatisticsReq.getIpList()); - log.debug("hostsByIp={}", hostsByIp); - Set allHostsSet = new HashSet<>(hostsByIp); - List hostsByNodes = listHostByAppTopologyNodes(username, appId, - agentStatisticsReq.getAppTopoNodeList()); - log.debug("hostsByNodes={}", hostsByNodes); - allHostsSet.addAll(hostsByNodes); - // 只有普通业务才查动态分组 - if (applicationDTO.isBiz()) { - List hostDTOsByDynamicGroupIds = new ArrayList<>(); - List dynamicGroupList = getBizDynamicGroupHostList( - username, - applicationDTO.getBizIdIfBizApp(), - agentStatisticsReq.getDynamicGroupIds() - ); - dynamicGroupList.forEach(dynamicGroupInfoDTO -> { - List applicationHostDTOList = dynamicGroupInfoDTO.getIpListStatus(); - if (applicationHostDTOList != null && !applicationHostDTOList.isEmpty()) { - hostDTOsByDynamicGroupIds.addAll(applicationHostDTOList); - } - }); - fillAgentStatus(hostDTOsByDynamicGroupIds); - List hostsByDynamicGroupIds = getHostInfoVOsByHostInfoDTOs(hostDTOsByDynamicGroupIds); - log.debug("hostsByDynamicGroupIds={}", hostsByDynamicGroupIds); - allHostsSet.addAll(hostsByDynamicGroupIds); - } - log.debug("allHostsSet.size={},allHostsSet={}", allHostsSet.size(), allHostsSet); - int normalCount = 0; - int abnormalCount = 0; - for (HostInfoVO it : allHostsSet) { - if (it.getAlive() == 1) { - normalCount += 1; - } else { - abnormalCount += 1; - } - } - return new AgentStatistics(normalCount, abnormalCount); - } - - private List buildIncludeBizIdList(ApplicationDTO application) { - List bizIdList = new ArrayList<>(); - if (application.isBiz()) { - bizIdList.add(application.getBizIdIfBizApp()); - } else if (application.isBizSet()) { - if (application.getSubBizIds() != null) { - bizIdList.addAll(application.getSubBizIds()); - } - } - return bizIdList; - } - - private void checkCachedHosts(List hosts, - List includeBizIds, - List hostsInOtherApp, - List notExistHosts) { - List cacheHosts = - hostCache.batchGetHostsByIps(hosts.stream().map(HostDTO::toCloudIp).collect(Collectors.toList())); - for (int i = 0; i < hosts.size(); i++) { - HostDTO host = hosts.get(i); - CacheHostDO cacheHost = cacheHosts.get(i); - if (cacheHost != null) { - if (!includeBizIds.contains(cacheHost.getBizId())) { - hostsInOtherApp.add(new BasicAppHost(cacheHost.getBizId(), cacheHost.getHostId(), - cacheHost.getCloudAreaId(), cacheHost.getIp(), cacheHost.getAgentId())); - } - } else { - notExistHosts.add(host.toCloudIp()); - } - } - } - - private void checkSyncHosts(List includeBizIds, - List hostsInOtherApp, - List notExistHosts) { - List appHosts = applicationHostDAO.listHostsByIps(notExistHosts); - if (CollectionUtils.isNotEmpty(appHosts)) { - for (ApplicationHostDTO appHost : appHosts) { - if (appHost.getBizId() == null || appHost.getBizId() <= 0) { - log.info("Host: {} missing bizId, skip!", appHost.getCloudIp()); - // DB中缓存的主机可能没有业务信息(依赖的主机事件还没有处理),那么暂时跳过该主机 - continue; - } - - notExistHosts.remove(appHost.getCloudIp()); - hostCache.addOrUpdateHost(appHost); - if (!includeBizIds.contains(appHost.getBizId())) { - hostsInOtherApp.add(new BasicAppHost(appHost.getBizId(), appHost.getHostId(), - appHost.getCloudAreaId(), appHost.getIp(), appHost.getAgentId())); - } - } - } - } - - private void checkHostsFromCmdb(List includeBizIds, - List hostsInOtherApp, - List notExistHosts) { - - IBizCmdbClient bizCmdbClient = CmdbClientFactory.getCmdbClient(); - try { - List cmdbExistHosts = bizCmdbClient.listHostsByIps(notExistHosts); - if (CollectionUtils.isNotEmpty(cmdbExistHosts)) { - List cmdbExistHostIps = cmdbExistHosts.stream() - .map(ApplicationHostDTO::getCloudIp) - .collect(Collectors.toList()); - notExistHosts.removeAll(cmdbExistHostIps); - log.info("sync new hosts from cmdb, hosts:{}", cmdbExistHosts); - - hostCache.addOrUpdateHosts(cmdbExistHosts); - - cmdbExistHosts.forEach(syncHost -> { - if (!includeBizIds.contains(syncHost.getBizId())) { - hostsInOtherApp.add(new BasicAppHost(syncHost.getBizId(), syncHost.getHostId(), - syncHost.getCloudAreaId(), syncHost.getIp(), syncHost.getAgentId())); - } - }); - } - } catch (Exception e) { - log.warn("Handle hosts that may not be synchronized from cmdb fail!", e); - } - } - - public List listHosts(Collection hosts) { - Pair, List> hostResult = listHostsFromCacheOrCmdb(hosts); - if (CollectionUtils.isEmpty(hostResult.getRight())) { - return Collections.emptyList(); - } - return hostResult.getRight().stream().map(basicAppHost -> { - ApplicationHostDTO appHost = new ApplicationHostDTO(); - appHost.setHostId(basicAppHost.getHostId()); - appHost.setCloudAreaId(basicAppHost.getBkCloudId()); - appHost.setIp(basicAppHost.getIp()); - appHost.setAgentId(basicAppHost.getAgentId()); - appHost.setBizId(basicAppHost.getBizId()); - return appHost; - }).collect(Collectors.toList()); - } - - @Data - private static class BasicAppHost { - private Long bizId; - private Long hostId; - private Long bkCloudId; - private String ip; - private String agentId; - - private BasicAppHost(Long bizId, Long hostId, Long bkCloudId, String ip, String agentId) { - this.bizId = bizId; - this.hostId = hostId; - this.bkCloudId = bkCloudId; - this.ip = ip; - this.agentId = agentId; - } - - public HostDTO toHostDTO() { - HostDTO host = new HostDTO(); - host.setHostId(hostId); - host.setBkCloudId(bkCloudId); - host.setIp(ip); - host.setAgentId(agentId); - return host; - } - - } - - @Override - public ServiceListAppHostResultDTO listAppHosts(Long appId, - List hosts) { - ServiceListAppHostResultDTO result = new ServiceListAppHostResultDTO(); - ApplicationDTO application = applicationService.getAppByAppId(appId); - - Pair, List> hostResult = listHostsFromCacheOrCmdb(hosts); - List notExistHosts = hostResult.getLeft(); - List existHosts = hostResult.getRight(); - List validHosts = new ArrayList<>(); - List notInAppHosts = new ArrayList<>(); - - if (application.isAllBizSet()) { - // 如果是全业务，所以主机都是合法的 - result.setNotExistHosts(notExistHosts); - result.setValidHosts(existHosts.stream().map(BasicAppHost::toHostDTO).collect(Collectors.toList())); - result.setNotInAppHosts(Collections.emptyList()); - return result; - } - - // 普通业务集和普通业务需要判断主机是否归属于业务 - List includeBizIds = buildIncludeBizIdList(application); - if (CollectionUtils.isEmpty(includeBizIds)) { - log.warn("App do not contains any biz, appId:{}", appId); - result.setValidHosts(Collections.emptyList()); - result.setNotExistHosts(Collections.emptyList()); - result.setNotInAppHosts(hosts); - return result; - } - - existHosts.forEach(existHost -> { - if (includeBizIds.contains(existHost.getBizId())) { - validHosts.add(existHost.toHostDTO()); - } else { - notInAppHosts.add(existHost.toHostDTO()); - } - }); - - if (CollectionUtils.isNotEmpty(notExistHosts) || CollectionUtils.isNotEmpty(notInAppHosts)) { - log.info("Contains invalid hosts, appId: {}, notExistHosts: {}, hostsInOtherApp: {}", - appId, notExistHosts, notInAppHosts); - } - - result.setNotExistHosts(notExistHosts); - result.setValidHosts(validHosts); - result.setNotInAppHosts(notInAppHosts); - - return result; - } - - private Pair, List> listHostsFromCacheOrCmdb(Collection hosts) { - List appHosts = new ArrayList<>(); - List notExistHosts = new ArrayList<>(); - List hostIds = new ArrayList<>(); - List cloudIps = new ArrayList<>(); - hosts.forEach(host -> { - if (host.getHostId() != null) { - hostIds.add(host.getHostId()); - } else { - cloudIps.add(host.toCloudIp()); - } - }); - if (CollectionUtils.isNotEmpty(hostIds)) { - Pair, List> result = listHostsByStrategy(hostIds, new ListHostByHostIdsStrategy()); - appHosts.addAll(result.getRight()); - if (CollectionUtils.isNotEmpty(result.getLeft())) { - result.getLeft().forEach(notExistHostId -> { - notExistHosts.add(HostDTO.fromHostId(notExistHostId)); - }); - } - } - if (CollectionUtils.isNotEmpty(cloudIps)) { - Pair, List> result = listHostsByStrategy(cloudIps, new ListHostByIpsStrategy()); - appHosts.addAll(result.getRight()); - if (CollectionUtils.isNotEmpty(result.getLeft())) { - result.getLeft().forEach(notExistCloudIp -> { - notExistHosts.add(HostDTO.fromCloudIp(notExistCloudIp)); - }); - } - } - return Pair.of(notExistHosts, appHosts); - } - - private Pair, List> listHostsByStrategy(List keys, - ListHostStrategy listHostStrategy) { - List appHosts = new ArrayList<>(); - List notExistKeys = null; - - if (CollectionUtils.isNotEmpty(keys)) { - Pair, List> result = listHostStrategy.listHostsFromCache(keys); - notExistKeys = result.getLeft(); - if (CollectionUtils.isNotEmpty(result.getRight())) { - appHosts.addAll(result.getRight()); - } - } - - if (CollectionUtils.isNotEmpty(notExistKeys)) { - Pair, List> result = listHostStrategy.listHostsFromDb(notExistKeys); - notExistKeys = result.getLeft(); - if (CollectionUtils.isNotEmpty(result.getRight())) { - appHosts.addAll(result.getRight()); - } - } - - if (CollectionUtils.isNotEmpty(notExistKeys)) { - Pair, List> result = listHostStrategy.listHostsFromCmdb(notExistKeys); - notExistKeys = result.getLeft(); - if (CollectionUtils.isNotEmpty(result.getRight())) { - appHosts.addAll(result.getRight()); - } - } - - return Pair.of(notExistKeys, appHosts); - } - - /** - * 主机查询策略 - * - * @param 查询使用的主机KEY - */ - private interface ListHostStrategy { - Pair, List> listHostsFromCache(List keys); - - Pair, List> listHostsFromDb(List keys); - - Pair, List> listHostsFromCmdb(List keys); - } - - /** - * 根据ip查询主机 - */ - private class ListHostByIpsStrategy implements ListHostStrategy { - @Override - public Pair, List> listHostsFromCache(List cloudIps) { - List appHosts = new ArrayList<>(); - List notExistCloudIps = new ArrayList<>(); - List cacheHosts = hostCache.batchGetHostsByIps(cloudIps); - for (int i = 0; i < cloudIps.size(); i++) { - String cloudIp = cloudIps.get(i); - CacheHostDO cacheHost = cacheHosts.get(i); - if (cacheHost != null) { - BasicAppHost appHost = new BasicAppHost(cacheHost.getBizId(), cacheHost.getHostId(), - cacheHost.getCloudAreaId(), cacheHost.getIp(), cacheHost.getAgentId()); - appHosts.add(appHost); - } else { - notExistCloudIps.add(cloudIp); - } - } - return Pair.of(notExistCloudIps, appHosts); - } - - @Override - public Pair, List> listHostsFromDb(List cloudIps) { - List appHosts = new ArrayList<>(); - List notExistCloudIps = new ArrayList<>(cloudIps); - List hostsInDb = applicationHostDAO.listHostsByIps(cloudIps); - if (CollectionUtils.isNotEmpty(hostsInDb)) { - for (ApplicationHostDTO appHost : hostsInDb) { - if (appHost.getBizId() == null || appHost.getBizId() <= 0) { - log.info("Host: {}|{} missing bizId, skip!", appHost.getHostId(), appHost.getCloudIp()); - // DB中缓存的主机可能没有业务信息(依赖的主机事件还没有处理),那么暂时跳过该主机 - continue; - } - hostCache.addOrUpdateHost(appHost); - notExistCloudIps.remove(appHost.getCloudIp()); - BasicAppHost host = new BasicAppHost(appHost.getBizId(), appHost.getHostId(), - appHost.getCloudAreaId(), appHost.getIp(), appHost.getAgentId()); - appHosts.add(host); - } - } - return Pair.of(notExistCloudIps, appHosts); - } - - @Override - public Pair, List> listHostsFromCmdb(List cloudIps) { - List appHosts = new ArrayList<>(); - List notExistCloudIps = new ArrayList<>(cloudIps); - - IBizCmdbClient bizCmdbClient = CmdbClientFactory.getCmdbClient(); - List cmdbExistHosts = bizCmdbClient.listHostsByIps(cloudIps); - if (CollectionUtils.isNotEmpty(cmdbExistHosts)) { - List cmdbExistHostIds = cmdbExistHosts.stream() - .map(ApplicationHostDTO::getCloudIp) - .collect(Collectors.toList()); - notExistCloudIps.removeAll(cmdbExistHostIds); - log.info("sync new hosts from cmdb, hosts:{}", cmdbExistHosts); - - hostCache.addOrUpdateHosts(cmdbExistHosts); - - cmdbExistHosts.forEach(syncHost -> { - BasicAppHost host = new BasicAppHost(syncHost.getBizId(), syncHost.getHostId(), - syncHost.getCloudAreaId(), syncHost.getIp(), syncHost.getAgentId()); - appHosts.add(host); - }); - } - return Pair.of(notExistCloudIps, appHosts); - } - } - - private class ListHostByHostIdsStrategy implements ListHostStrategy { - @Override - public Pair, List> listHostsFromCache(List hostIds) { - List appHosts = new ArrayList<>(); - List notExistHostIds = new ArrayList<>(); - List cacheHosts = hostCache.batchGetHostsByHostIds(hostIds); - for (int i = 0; i < hostIds.size(); i++) { - long hostId = hostIds.get(i); - CacheHostDO cacheHost = cacheHosts.get(i); - if (cacheHost != null) { - BasicAppHost appHost = new BasicAppHost(cacheHost.getBizId(), cacheHost.getHostId(), - cacheHost.getCloudAreaId(), cacheHost.getIp(), cacheHost.getAgentId()); - appHosts.add(appHost); - } else { - notExistHostIds.add(hostId); - } - } - return Pair.of(notExistHostIds, appHosts); - } - - @Override - public Pair, List> listHostsFromDb(List hostIds) { - List appHosts = new ArrayList<>(); - List notExistHostIds = new ArrayList<>(hostIds); - List hostsInDb = applicationHostDAO.listHostInfoByHostIds(hostIds); - if (CollectionUtils.isNotEmpty(hostsInDb)) { - for (ApplicationHostDTO appHost : hostsInDb) { - if (appHost.getBizId() == null || appHost.getBizId() <= 0) { - log.info("Host: {}|{} missing bizId, skip!", appHost.getHostId(), appHost.getCloudIp()); - // DB中缓存的主机可能没有业务信息(依赖的主机事件还没有处理),那么暂时跳过该主机 - continue; - } - hostCache.addOrUpdateHost(appHost); - notExistHostIds.remove(appHost.getHostId()); - BasicAppHost host = new BasicAppHost(appHost.getBizId(), appHost.getHostId(), - appHost.getCloudAreaId(), appHost.getIp(), appHost.getAgentId()); - appHosts.add(host); - } - } - return Pair.of(notExistHostIds, appHosts); - } - - @Override - public Pair, List> listHostsFromCmdb(List hostIds) { - List appHosts = new ArrayList<>(); - List notExistHostIds = new ArrayList<>(hostIds); - - IBizCmdbClient bizCmdbClient = CmdbClientFactory.getCmdbClient(); - List cmdbExistHosts = bizCmdbClient.listHostsByHostIds(hostIds); - if (CollectionUtils.isNotEmpty(cmdbExistHosts)) { - List cmdbExistHostIds = cmdbExistHosts.stream() - .map(ApplicationHostDTO::getHostId) - .collect(Collectors.toList()); - notExistHostIds.removeAll(cmdbExistHostIds); - log.info("sync new hosts from cmdb, hosts:{}", cmdbExistHosts); - - hostCache.addOrUpdateHosts(cmdbExistHosts); - - cmdbExistHosts.forEach(syncHost -> { - BasicAppHost host = new BasicAppHost(syncHost.getBizId(), syncHost.getHostId(), - syncHost.getCloudAreaId(), syncHost.getIp(), syncHost.getAgentId()); - appHosts.add(host); - }); - } - return Pair.of(notExistHostIds, appHosts); - } - } -} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/IndexServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/IndexServiceImpl.java index 9e7902b34d..f3e33a138a 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/IndexServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/IndexServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,11 +27,13 @@ import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.constant.ResourceScopeTypeEnum; import com.tencent.bk.job.common.exception.InternalException; -import com.tencent.bk.job.common.gse.constants.AgentStatusEnum; +import com.tencent.bk.job.common.gse.constants.AgentAliveStatusEnum; import com.tencent.bk.job.common.i18n.locale.LocaleUtils; import com.tencent.bk.job.common.model.PageData; import com.tencent.bk.job.common.model.dto.AppResourceScope; import com.tencent.bk.job.common.model.dto.ApplicationDTO; +import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; +import com.tencent.bk.job.common.model.dto.HostStatusNumStatisticsDTO; import com.tencent.bk.job.common.model.vo.HostInfoVO; import com.tencent.bk.job.common.util.JobContextUtil; import com.tencent.bk.job.common.util.PageUtil; @@ -39,69 +41,66 @@ import com.tencent.bk.job.manage.common.TopologyHelper; import com.tencent.bk.job.manage.dao.ApplicationDAO; import com.tencent.bk.job.manage.dao.ApplicationHostDAO; -import com.tencent.bk.job.manage.dao.HostTopoDAO; import com.tencent.bk.job.manage.dao.ScriptDAO; import com.tencent.bk.job.manage.dao.index.IndexGreetingDAO; import com.tencent.bk.job.manage.dao.template.TaskTemplateDAO; import com.tencent.bk.job.manage.model.dto.task.TaskTemplateInfoDTO; -import com.tencent.bk.job.manage.model.web.vo.index.AgentStatistics; +import com.tencent.bk.job.manage.model.web.vo.common.AgentStatistics; import com.tencent.bk.job.manage.model.web.vo.index.GreetingVO; import com.tencent.bk.job.manage.model.web.vo.index.JobAndScriptStatistics; import com.tencent.bk.job.manage.model.web.vo.task.TaskTemplateVO; import com.tencent.bk.job.manage.service.IndexService; +import com.tencent.bk.job.manage.service.host.HostDetailService; import com.tencent.bk.job.manage.service.template.TaskTemplateService; import lombok.extern.slf4j.Slf4j; import lombok.val; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.tuple.Pair; -import org.jooq.DSLContext; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import java.util.Collections; import java.util.List; +import java.util.Objects; import java.util.stream.Collectors; @Slf4j @Service public class IndexServiceImpl implements IndexService { - private final DSLContext dslContext; private final IndexGreetingDAO indexGreetingDAO; private final ApplicationDAO applicationDAO; private final ApplicationHostDAO applicationHostDAO; - private final HostTopoDAO hostTopoDAO; private final TopologyHelper topologyHelper; private final TaskTemplateService taskTemplateService; private final TaskTemplateDAO taskTemplateDAO; private final ScriptDAO scriptDAO; + private final HostDetailService hostDetailService; @Autowired - public IndexServiceImpl(DSLContext dslContext, - IndexGreetingDAO indexGreetingDAO, + public IndexServiceImpl(IndexGreetingDAO indexGreetingDAO, ApplicationDAO applicationDAO, ApplicationHostDAO applicationHostDAO, - HostTopoDAO hostTopoDAO, TopologyHelper topologyHelper, TaskTemplateService taskTemplateService, TaskTemplateDAO taskTemplateDAO, - ScriptDAO scriptDAO) { - this.dslContext = dslContext; + ScriptDAO scriptDAO, + HostDetailService hostDetailService) { this.indexGreetingDAO = indexGreetingDAO; this.applicationDAO = applicationDAO; this.applicationHostDAO = applicationHostDAO; - this.hostTopoDAO = hostTopoDAO; this.topologyHelper = topologyHelper; this.taskTemplateService = taskTemplateService; this.taskTemplateDAO = taskTemplateDAO; this.scriptDAO = scriptDAO; + this.hostDetailService = hostDetailService; } @Override public List listGreeting(String username) { String normalLang = LocaleUtils.getNormalLang(JobContextUtil.getUserLang()); int currentSeconds = TimeUtil.getCurrentSeconds(); - return indexGreetingDAO.listActiveIndexGreeting(dslContext, currentSeconds).stream().map(it -> { + return indexGreetingDAO.listActiveIndexGreeting(currentSeconds).stream().map(it -> { String content; if (normalLang.equals(LocaleUtils.LANG_EN) || normalLang.equals(LocaleUtils.LANG_EN_US)) { content = it.getContentEn(); @@ -122,41 +121,32 @@ public List listGreeting(String username) { public AgentStatistics getAgentStatistics(String username, AppResourceScope appResourceScope) { // 查出业务 ApplicationDTO appInfo = applicationDAO.getAppById(appResourceScope.getAppId()); - Long normalNum; - Long abnormalNum; - List hostIds; + int aliveCount = 0; + int notAliveCount = 0; + List bizIds; if (appInfo.isBiz()) { // 普通业务 - hostIds = hostTopoDAO.listHostIdByBizIds( - Collections.singletonList(Long.valueOf(appResourceScope.getId())) - ); + bizIds = Collections.singletonList(Long.valueOf(appResourceScope.getId())); } else if (appInfo.isAllBizSet()) { // 全业务 // 不根据主机ID过滤 - hostIds = null; + bizIds = null; } else if (appInfo.isBizSet()) { // 业务集 // 查出业务集下所有子业务 - List subBizIds = appInfo.getSubBizIds(); - // 查出所有子业务下属主机ID - hostIds = hostTopoDAO.listHostIdByBizIds(subBizIds); + bizIds = appInfo.getSubBizIds(); } else { throw new InternalException("Ilegal appInfo:" + appInfo, ErrorCode.INTERNAL_ERROR); } - // 查主机数量 - normalNum = applicationHostDAO.countHostByIdAndStatus( - hostIds, - AgentStatusEnum.ALIVE - ); - abnormalNum = applicationHostDAO.countHostByIdAndStatus( - hostIds, - AgentStatusEnum.NOT_ALIVE - ); - abnormalNum += applicationHostDAO.countHostByIdAndStatus( - hostIds, - AgentStatusEnum.UNKNOWN - ); - return new AgentStatistics(normalNum.intValue(), abnormalNum.intValue()); + List statisticsDTOS = applicationHostDAO.countHostStatusNumByBizIds(bizIds); + for (HostStatusNumStatisticsDTO statisticsDTO : statisticsDTOS) { + if (statisticsDTO.getGseAgentAlive() == AgentAliveStatusEnum.ALIVE.getStatusValue()) { + aliveCount += statisticsDTO.getHostNum(); + } else { + notAliveCount += statisticsDTO.getHostNum(); + } + } + return new AgentStatistics(aliveCount, notAliveCount); } /** @@ -201,9 +191,13 @@ public PageData listHostsByAgentStatus(String username, List hostInfoVOList; val hosts = applicationHostDAO.listHostInfoBySearchContents( bizIds, null, null, null, status, start, pageSize); + hostDetailService.fillDetailForApplicationHosts(hosts); Long count = applicationHostDAO.countHostInfoBySearchContents( bizIds, null, null, null, status); - hostInfoVOList = hosts.parallelStream().map(TopologyHelper::convertToHostInfoVO).collect(Collectors.toList()); + hostInfoVOList = hosts.stream() + .filter(Objects::nonNull) + .map(ApplicationHostDTO::toVO) + .collect(Collectors.toList()); return new PageData<>(start.intValue(), pageSize.intValue(), count, hostInfoVOList); } @@ -215,8 +209,8 @@ public PageData listIPsByAgentStatus(String username, Long appId, Intege resultPageData.setStart(hostInfoVOPageData.getStart()); resultPageData.setPageSize(hostInfoVOPageData.getPageSize()); resultPageData.setTotal(hostInfoVOPageData.getTotal()); - resultPageData.setData(hostInfoVOPageData.getData().parallelStream() - .map(it -> it.getCloudAreaInfo().getId() + ":" + it.getIp()).collect(Collectors.toList())); + resultPageData.setData(hostInfoVOPageData.getData().stream() + .map(it -> it.getCloudArea().getId() + ":" + it.getIp()).collect(Collectors.toList())); return resultPageData; } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/LocalPermissionServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/LocalPermissionServiceImpl.java index fb054a69e5..ce078eacc6 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/LocalPermissionServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/LocalPermissionServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/MeasureServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/MeasureServiceImpl.java index 44220d2599..4e40d0682f 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/MeasureServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/MeasureServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -45,7 +45,7 @@ import java.util.concurrent.ThreadPoolExecutor; @Slf4j -@Service +@Service("jobManageMeasureService") public class MeasureServiceImpl implements MeasureService { private final MeterRegistry meterRegistry; @@ -90,7 +90,6 @@ public void init() { measureWhiteIpCount(); measureSyncAppExecutor(); measureSyncHostExecutor(); - measureSyncAgentStatusExecutor(); } private void measureAppAndHostCount() { @@ -259,20 +258,4 @@ private void measureSyncHostExecutor() { syncService1 -> syncService1.getSyncHostExecutor().getQueue().size() ); } - - private void measureSyncAgentStatusExecutor() { - // 同步线程池监控：Agent状态 - meterRegistry.gauge( - MetricsConstants.NAME_SYNC_AGENT_STATUS_EXECUTOR_POOL_SIZE, - Collections.singletonList(Tag.of(MetricsConstants.TAG_KEY_MODULE, MetricsConstants.TAG_VALUE_MODULE_SYNC)), - this.syncService, - syncService1 -> syncService1.getSyncAgentStatusExecutor().getPoolSize() - ); - meterRegistry.gauge( - MetricsConstants.NAME_SYNC_AGENT_STATUS_EXECUTOR_QUEUE_SIZE, - Collections.singletonList(Tag.of(MetricsConstants.TAG_KEY_MODULE, MetricsConstants.TAG_VALUE_MODULE_SYNC)), - this.syncService, - syncService1 -> syncService1.getSyncAgentStatusExecutor().getQueue().size() - ); - } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/PaaSServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/PaaSServiceImpl.java deleted file mode 100644 index f36fabc004..0000000000 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/PaaSServiceImpl.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.service.impl; - -import com.tencent.bk.job.common.esb.metrics.EsbApiTimed; -import com.tencent.bk.job.common.model.dto.BkUserDTO; -import com.tencent.bk.job.common.paas.model.EsbNotifyChannelDTO; -import com.tencent.bk.job.common.paas.user.IPaasClient; -import com.tencent.bk.job.common.util.JobContextUtil; -import com.tencent.bk.job.manage.common.client.PAASClientFactory; -import com.tencent.bk.job.manage.service.PaaSService; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Service; - -import javax.servlet.http.HttpServletRequest; -import java.util.Collections; -import java.util.List; -import java.util.Set; - -import static com.tencent.bk.job.common.i18n.locale.LocaleUtils.COMMON_LANG_HEADER; - -/** - * @Description - * @Date 2020/1/3 - * @Version 1.0 - */ -@Service -public class PaaSServiceImpl implements PaaSService { - private static final Logger logger = LoggerFactory.getLogger(PaaSServiceImpl.class); - - private IPaasClient getClientWithCurrentLang() { - try { - HttpServletRequest request = JobContextUtil.getRequest(); - String lang = null; - if (request != null) { - lang = request.getHeader(COMMON_LANG_HEADER); - } - logger.info(String.format("Current Locale:%s", lang)); - return PAASClientFactory.getClient(lang); - } catch (Exception e) { - logger.info("Locale not set, backend task, use default locale"); - return PAASClientFactory.getClient(); - } - } - - @Override - public List getAllUserList(String bkToken, String uin) { - IPaasClient paasClient = getClientWithCurrentLang(); - try { - return paasClient.getUserList("id,username,display_name,logo", bkToken, uin); - } catch (Exception e) { - logger.error("Exception while get user list!", e); - } - logger.error("Get user list failed!|{}", uin); - return Collections.emptyList(); - } - - @Override - public List getAllChannelList(String bkToken, String uin) { - IPaasClient paasClient = getClientWithCurrentLang(); - return paasClient.getNotifyChannelList(uin); - } - - @Override - @EsbApiTimed - public void sendMsg( - String msgType, - String sender, - Set receivers, - String title, - String content - ) { - IPaasClient paasClient = getClientWithCurrentLang(); - paasClient.sendMsg(msgType, sender, receivers, title, content); - } -} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/PublicScriptServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/PublicScriptServiceImpl.java new file mode 100644 index 0000000000..7c38433217 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/PublicScriptServiceImpl.java @@ -0,0 +1,398 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.service.impl; + +import com.tencent.bk.audit.annotations.ActionAuditRecord; +import com.tencent.bk.audit.annotations.AuditAttribute; +import com.tencent.bk.audit.annotations.AuditInstanceRecord; +import com.tencent.bk.audit.context.ActionAuditContext; +import com.tencent.bk.job.common.audit.constants.EventContentConstants; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.NotFoundException; +import com.tencent.bk.job.common.iam.constant.ActionId; +import com.tencent.bk.job.common.iam.constant.ResourceTypeId; +import com.tencent.bk.job.common.model.PageData; +import com.tencent.bk.job.manage.auth.NoResourceScopeAuthService; +import com.tencent.bk.job.manage.model.dto.ScriptBasicDTO; +import com.tencent.bk.job.manage.model.dto.ScriptDTO; +import com.tencent.bk.job.manage.model.dto.SyncScriptResultDTO; +import com.tencent.bk.job.manage.model.dto.TagDTO; +import com.tencent.bk.job.manage.model.dto.TemplateStepIDDTO; +import com.tencent.bk.job.manage.model.query.ScriptQuery; +import com.tencent.bk.job.manage.model.web.vo.TagCountVO; +import com.tencent.bk.job.manage.service.PublicScriptService; +import com.tencent.bk.job.manage.service.ScriptManager; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.Collection; +import java.util.List; +import java.util.Map; + +import static com.tencent.bk.job.common.constant.JobConstants.PUBLIC_APP_ID; + +/** + * 公共脚本Service + */ +@Slf4j +@Service +public class PublicScriptServiceImpl implements PublicScriptService { + private final ScriptManager scriptManager; + private final NoResourceScopeAuthService noResourceScopeAuthService; + + @Autowired + public PublicScriptServiceImpl(ScriptManager scriptManager, + NoResourceScopeAuthService noResourceScopeAuthService) { + this.scriptManager = scriptManager; + this.noResourceScopeAuthService = noResourceScopeAuthService; + } + + @Override + public PageData listPageScript(ScriptQuery scriptCondition) { + return scriptManager.listPageScript(scriptCondition); + } + + @Override + public List listScripts(ScriptQuery scriptQuery) { + return scriptManager.listScripts(scriptQuery); + } + + @Override + public ScriptDTO getScript(String scriptId) { + return scriptManager.getScript(PUBLIC_APP_ID, scriptId); + } + + @Override + public List listScriptBasicInfoByScriptIds(Collection scriptIds) { + return scriptManager.listScriptBasicInfoByScriptIds(scriptIds); + } + + @Override + @ActionAuditRecord( + actionId = ActionId.CREATE_PUBLIC_SCRIPT, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.PUBLIC_SCRIPT, + instanceIds = "#$?.id", + instanceNames = "#$?.name" + ), + content = EventContentConstants.CREATE_PUBLIC_SCRIPT + ) + public ScriptDTO saveScript(String username, ScriptDTO script) { + authCreatePublicScript(username); + ScriptDTO savedScript = scriptManager.createScript(script); + noResourceScopeAuthService.registerPublicScript(savedScript.getId(), savedScript.getName(), username); + return savedScript; + } + + private void authCreatePublicScript(String username) { + noResourceScopeAuthService.authCreatePublicScript(username).denyIfNoPermission(); + } + + private void authManagePublicScript(String username, String scriptId) { + noResourceScopeAuthService.authManagePublicScript(username, scriptId).denyIfNoPermission(); + } + + @Override + @ActionAuditRecord( + actionId = ActionId.MANAGE_PUBLIC_SCRIPT_INSTANCE, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.PUBLIC_SCRIPT, + instanceIds = "#scriptId" + ), + content = EventContentConstants.DELETE_PUBLIC_SCRIPT + ) + public void deleteScript(String username, String scriptId) { + log.info("Delete script[{}], operator={}", scriptId, username); + authManagePublicScript(username, scriptId); + + ScriptDTO script = getScript(scriptId); + if (script == null) { + throw new NotFoundException(ErrorCode.SCRIPT_NOT_EXIST); + } + + ActionAuditContext.current().setInstanceName(script.getName()); + + scriptManager.deleteScript(PUBLIC_APP_ID, scriptId); + } + + @Override + public ScriptDTO getScriptVersion(Long scriptVersionId) { + return scriptManager.getScriptVersion(PUBLIC_APP_ID, scriptVersionId); + } + + @Override + public List listScriptVersion(String scriptId) { + return scriptManager.listScriptVersion(PUBLIC_APP_ID, scriptId); + } + + @Override + @ActionAuditRecord( + actionId = ActionId.MANAGE_PUBLIC_SCRIPT_INSTANCE, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.PUBLIC_SCRIPT, + instanceIds = "#scriptVersion?.id", + instanceNames = "#$?.name" + ), + attributes = @AuditAttribute( + name = "@VERSION", value = "#scriptVersion?.version" + ), + content = EventContentConstants.CREATE_PUBLIC_SCRIPT_VERSION + ) + public ScriptDTO saveScriptVersion(String username, ScriptDTO scriptVersion) { + authManagePublicScript(username, scriptVersion.getId()); + return scriptManager.createScriptVersion(scriptVersion); + } + + @Override + @ActionAuditRecord( + actionId = ActionId.MANAGE_PUBLIC_SCRIPT_INSTANCE, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.PUBLIC_SCRIPT + ), + content = EventContentConstants.EDIT_PUBLIC_SCRIPT_VERSION + ) + public ScriptDTO updateScriptVersion(String username, ScriptDTO scriptVersion) { + authManagePublicScript(username, scriptVersion.getId()); + + ScriptDTO originScript = getScriptByScriptId(scriptVersion.getId()); + if (originScript == null) { + throw new NotFoundException(ErrorCode.SCRIPT_NOT_EXIST); + } + ScriptDTO updateScript = scriptManager.updateScriptVersion(scriptVersion); + + // 审计 + addModifyPublicScriptVersionAuditInfo(originScript, updateScript); + + return updateScript; + } + + private void addModifyPublicScriptVersionAuditInfo(ScriptDTO originScriptVersion, ScriptDTO updateScriptVersion) { + // 审计 + ActionAuditContext.current() + .setInstanceId(originScriptVersion.getId()) + .setInstanceName(originScriptVersion.getName()) + .setOriginInstance(originScriptVersion.toEsbScriptV3DTO()) + .setInstance(updateScriptVersion.toEsbScriptV3DTO()) + .addAttribute("@VERSION", originScriptVersion.getVersion()); + } + + @Override + @ActionAuditRecord( + actionId = ActionId.MANAGE_PUBLIC_SCRIPT_INSTANCE, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.PUBLIC_SCRIPT + ), + content = EventContentConstants.DELETE_PUBLIC_SCRIPT_VERSION + ) + public void deleteScriptVersion(String username, Long scriptVersionId) { + log.info("Delete scriptVersion[{}], operator={}", scriptVersionId, username); + preProcessManagePublicScriptVersion(username, scriptVersionId); + scriptManager.deleteScriptVersion(PUBLIC_APP_ID, scriptVersionId); + } + + private void preProcessManagePublicScriptVersion(String username, Long scriptVersionId) { + ScriptDTO script = getScriptVersion(scriptVersionId); + if (script == null) { + throw new NotFoundException(ErrorCode.SCRIPT_NOT_EXIST); + } + + authManagePublicScript(username, script.getId()); + + ActionAuditContext.current() + .setInstanceId(script.getId()) + .setInstanceName(script.getName()) + .addAttribute("@VERSION", script.getVersion()); + } + + @Override + @ActionAuditRecord( + actionId = ActionId.MANAGE_PUBLIC_SCRIPT_INSTANCE, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.PUBLIC_SCRIPT + ), + content = EventContentConstants.ONLINE_PUBLIC_SCRIPT_VERSION + ) + public void publishScript(String username, String scriptId, Long scriptVersionId) { + log.info("Publish script version, scriptId={}, scriptVersionId={}, username={}", + scriptId, scriptVersionId, username); + preProcessManagePublicScriptVersion(username, scriptVersionId); + scriptManager.publishScript(PUBLIC_APP_ID, scriptId, scriptVersionId); + } + + @Override + @ActionAuditRecord( + actionId = ActionId.MANAGE_PUBLIC_SCRIPT_INSTANCE, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.PUBLIC_SCRIPT + ), + content = EventContentConstants.FORBIDDEN_PUBLIC_SCRIPT_VERSION + ) + public void disableScript(String username, String scriptId, Long scriptVersionId) { + log.info("Disable script version, scriptId={}, scriptVersionId={}, username={}", + scriptId, scriptVersionId, username); + preProcessManagePublicScriptVersion(username, scriptVersionId); + scriptManager.disableScript(PUBLIC_APP_ID, scriptId, scriptVersionId); + } + + @Override + public Map batchGetOnlineScriptVersionByScriptIds(List scriptIdList) { + return scriptManager.batchGetOnlineScriptVersionByScriptIds(scriptIdList); + } + + @Override + @ActionAuditRecord( + actionId = ActionId.MANAGE_PUBLIC_SCRIPT_INSTANCE, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.PUBLIC_SCRIPT + ), + content = EventContentConstants.EDIT_PUBLIC_SCRIPT + ) + public ScriptDTO updateScriptDesc(String operator, String scriptId, String desc) { + ScriptDTO originScript = getScript(scriptId); + if (originScript == null) { + throw new NotFoundException(ErrorCode.SCRIPT_NOT_EXIST); + } + + authManagePublicScript(operator, scriptId); + + ScriptDTO updateScript = scriptManager.updateScriptDesc(operator, PUBLIC_APP_ID, scriptId, desc); + + addModifyPublicScriptAuditInfo(originScript, updateScript); + + return updateScript; + } + + private void addModifyPublicScriptAuditInfo(ScriptDTO originScript, ScriptDTO updateScript) { + // 审计 + ActionAuditContext.current() + .setInstanceId(originScript.getId()) + .setInstanceName(originScript.getName()) + .setOriginInstance(originScript.toEsbScriptV3DTO()) + .setInstance(updateScript.toEsbScriptV3DTO()); + } + + @Override + @ActionAuditRecord( + actionId = ActionId.MANAGE_PUBLIC_SCRIPT_INSTANCE, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.PUBLIC_SCRIPT + ), + content = EventContentConstants.EDIT_PUBLIC_SCRIPT + ) + public ScriptDTO updateScriptName(String operator, String scriptId, String newName) { + ScriptDTO originScript = getScript(scriptId); + if (originScript == null) { + throw new NotFoundException(ErrorCode.SCRIPT_NOT_EXIST); + } + + authManagePublicScript(operator, scriptId); + + ScriptDTO updateScript = scriptManager.updateScriptName(operator, PUBLIC_APP_ID, scriptId, newName); + + addModifyPublicScriptAuditInfo(originScript, updateScript); + + return updateScript; + } + + @Override + @ActionAuditRecord( + actionId = ActionId.MANAGE_PUBLIC_SCRIPT_INSTANCE, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.PUBLIC_SCRIPT + ), + content = EventContentConstants.EDIT_PUBLIC_SCRIPT + ) + public ScriptDTO updateScriptTags(String operator, + String scriptId, + List tags) { + ScriptDTO originScript = getScript(scriptId); + if (originScript == null) { + throw new NotFoundException(ErrorCode.SCRIPT_NOT_EXIST); + } + + authManagePublicScript(operator, scriptId); + + ScriptDTO updateScript = scriptManager.updateScriptTags(operator, PUBLIC_APP_ID, scriptId, tags); + + addModifyPublicScriptAuditInfo(originScript, updateScript); + + return updateScript; + } + + @Override + public List listScriptNames(String keyword) { + return scriptManager.listScriptNames(PUBLIC_APP_ID, keyword); + } + + @Override + public List listOnlineScript() { + return scriptManager.listOnlineScriptForApp(PUBLIC_APP_ID); + } + + @Override + public ScriptDTO getOnlineScriptVersionByScriptId(String scriptId) { + return scriptManager.getOnlineScriptVersionByScriptId(PUBLIC_APP_ID, scriptId); + } + + @Override + public PageData listPageScriptVersion(ScriptQuery scriptQuery) { + return scriptManager.listPageScriptVersion(scriptQuery); + } + + @Override + public List syncScriptToTaskTemplate(String operator, + String scriptId, + Long syncScriptVersionId, + List templateStepIDs) { + return scriptManager.syncScriptToTaskTemplate(operator, PUBLIC_APP_ID, scriptId, syncScriptVersionId, + templateStepIDs); + } + + @Override + public List listScriptIds() { + return scriptManager.listScriptIds(PUBLIC_APP_ID); + } + + @Override + public TagCountVO getTagScriptCount() { + return scriptManager.getTagScriptCount(PUBLIC_APP_ID); + } + + @Override + public boolean isExistAnyPublicScript() { + return scriptManager.isExistAnyScript(PUBLIC_APP_ID); + } + + @Override + public ScriptDTO getByScriptIdAndVersion(String scriptId, String version) { + return scriptManager.getByScriptIdAndVersion(PUBLIC_APP_ID, scriptId, version); + } + + @Override + public ScriptDTO getScriptByScriptId(String scriptId) { + return scriptManager.getScriptByScriptId(scriptId); + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/ResourceNameQueryServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/ResourceNameQueryServiceImpl.java index e2ba668bc6..4513d370ac 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/ResourceNameQueryServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/ResourceNameQueryServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -32,12 +32,13 @@ import com.tencent.bk.job.common.model.dto.ApplicationDTO; import com.tencent.bk.job.common.service.AppScopeMappingService; import com.tencent.bk.job.manage.model.dto.AccountDTO; +import com.tencent.bk.job.manage.model.dto.CredentialDTO; import com.tencent.bk.job.manage.model.dto.ScriptDTO; import com.tencent.bk.job.manage.model.dto.TagDTO; -import com.tencent.bk.job.manage.model.inner.resp.ServiceCredentialDTO; import com.tencent.bk.job.manage.service.AccountService; import com.tencent.bk.job.manage.service.ApplicationService; import com.tencent.bk.job.manage.service.CredentialService; +import com.tencent.bk.job.manage.service.PublicScriptService; import com.tencent.bk.job.manage.service.ScriptService; import com.tencent.bk.job.manage.service.TagService; import com.tencent.bk.job.manage.service.plan.TaskPlanService; @@ -48,11 +49,12 @@ import org.springframework.stereotype.Service; @Slf4j -@Service("ResourceNameQueryService") +@Service("jobManageResourceNameQueryService") public class ResourceNameQueryServiceImpl implements ResourceNameQueryService { private final ApplicationService applicationService; private final ScriptService scriptService; + private final PublicScriptService publicScriptService; private final TaskTemplateService templateService; private final TaskPlanService planService; private final AccountService accountService; @@ -63,6 +65,7 @@ public class ResourceNameQueryServiceImpl implements ResourceNameQueryService { @Autowired public ResourceNameQueryServiceImpl(ApplicationService applicationService, ScriptService scriptService, + PublicScriptService publicScriptService, TaskTemplateService templateService, TaskPlanService planService, AccountService accountService, @@ -73,6 +76,7 @@ public ResourceNameQueryServiceImpl(ApplicationService applicationService, AppScopeMappingService appScopeMappingService) { this.applicationService = applicationService; this.scriptService = scriptService; + this.publicScriptService = publicScriptService; this.templateService = templateService; this.planService = planService; this.accountService = accountService; @@ -105,6 +109,11 @@ public String getResourceName(ResourceTypeEnum resourceType, String resourceId) } break; case PUBLIC_SCRIPT: + ScriptDTO publicScript = publicScriptService.getScriptByScriptId(resourceId); + if (publicScript != null) { + return publicScript.getName(); + } + break; case SCRIPT: ScriptDTO script = scriptService.getScriptByScriptId(resourceId); if (script != null) { @@ -142,12 +151,12 @@ public String getResourceName(ResourceTypeEnum resourceType, String resourceId) } break; case TICKET: - ServiceCredentialDTO serviceCredentialDTO = credentialService.getServiceCredentialById(resourceId); - if (serviceCredentialDTO == null) { + CredentialDTO credentialDTO = credentialService.getCredentialById(resourceId); + if (credentialDTO == null) { log.warn("Cannot find ticket by ticketId {}", resourceId); return null; } - return serviceCredentialDTO.getName(); + return credentialDTO.getName(); default: return null; } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/ScopeDynamicGroupService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/ScopeDynamicGroupService.java new file mode 100644 index 0000000000..ed2eb002c8 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/ScopeDynamicGroupService.java @@ -0,0 +1,108 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.service.impl; + +import com.tencent.bk.job.common.model.dto.AppResourceScope; +import com.tencent.bk.job.common.model.dto.ApplicationDTO; +import com.tencent.bk.job.manage.model.dto.DynamicGroupDTO; +import com.tencent.bk.job.manage.service.ApplicationService; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +@Slf4j +@Service +public class ScopeDynamicGroupService { + + private final ApplicationService applicationService; + private final BizDynamicGroupService bizDynamicGroupService; + + @Autowired + public ScopeDynamicGroupService(ApplicationService applicationService, + BizDynamicGroupService bizDynamicGroupService) { + this.applicationService = applicationService; + this.bizDynamicGroupService = bizDynamicGroupService; + } + + public List listOrderedDynamicGroup(AppResourceScope appResourceScope, Collection ids) { + List dynamicGroupDTOList = listDynamicGroup(appResourceScope, ids); + dynamicGroupDTOList = defaultSort(dynamicGroupDTOList); + return dynamicGroupDTOList; + } + + public List listDynamicGroup(AppResourceScope appResourceScope, Collection ids) { + ApplicationDTO applicationDTO = applicationService.getAppByScope(appResourceScope); + if (applicationDTO.isAllBizSet()) { + // 全业务 + return Collections.emptyList(); + } else if (applicationDTO.isBizSet()) { + // 业务集 + return Collections.emptyList(); + } else { + // 普通业务 + Long bizId = Long.parseLong(applicationDTO.getScope().getId()); + if (ids == null) { + return bizDynamicGroupService.listDynamicGroup(bizId); + } + return bizDynamicGroupService.listDynamicGroup(bizId, ids); + } + } + + /** + * 动态分组默认排序：按首字母字典序排序，24小时内有更新的置顶（按更新时间倒序排列） + * + * @param dynamicGroupDTOList 动态分组列表 + */ + private List defaultSort(List dynamicGroupDTOList) { + List latestUpdatedDynamicGroupList = new ArrayList<>(); + List notUpdatedDynamicGroupList = new ArrayList<>(); + List orderedDynamicGroupList = new ArrayList<>(); + for (DynamicGroupDTO dynamicGroupDTO : dynamicGroupDTOList) { + ZonedDateTime parsedLastTime = dynamicGroupDTO.getParsedLastTime(); + if (parsedLastTime != null && parsedLastTime.plusHours(24).isAfter(ZonedDateTime.now())) { + latestUpdatedDynamicGroupList.add(dynamicGroupDTO); + } else { + notUpdatedDynamicGroupList.add(dynamicGroupDTO); + } + } + // 最近24小时内更新过的动态分组按更新时间降序 + latestUpdatedDynamicGroupList.sort((dynamicGroup1, dynamicGroup2) -> + dynamicGroup2.getParsedLastTime().compareTo(dynamicGroup1.getParsedLastTime()) + ); + // 最近24小时内未更新过的动态分组按首字母升序 + notUpdatedDynamicGroupList.sort(Comparator.comparing(DynamicGroupDTO::getName)); + // 最近24小时内更新过的动态分组置顶 + orderedDynamicGroupList.addAll(latestUpdatedDynamicGroupList); + orderedDynamicGroupList.addAll(notUpdatedDynamicGroupList); + return orderedDynamicGroupList; + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/ScriptCheckServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/ScriptCheckServiceImpl.java index a91c0c59f3..6cc8a7776d 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/ScriptCheckServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/ScriptCheckServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,7 +26,7 @@ import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.exception.InternalException; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; import com.tencent.bk.job.manage.manager.cache.DangerousRuleCache; import com.tencent.bk.job.manage.manager.script.check.ScriptCheckParam; import com.tencent.bk.job.manage.manager.script.check.checker.BuildInDangerousScriptChecker; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/ScriptManagerImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/ScriptManagerImpl.java new file mode 100644 index 0000000000..303031ad4e --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/ScriptManagerImpl.java @@ -0,0 +1,992 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.service.impl; + +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.constant.JobConstants; +import com.tencent.bk.job.common.constant.JobResourceTypeEnum; +import com.tencent.bk.job.common.exception.AlreadyExistsException; +import com.tencent.bk.job.common.exception.FailedPreconditionException; +import com.tencent.bk.job.common.exception.InvalidParamException; +import com.tencent.bk.job.common.exception.NotFoundException; +import com.tencent.bk.job.common.exception.ServiceException; +import com.tencent.bk.job.common.i18n.service.MessageI18nService; +import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; +import com.tencent.bk.job.common.iam.model.AuthResult; +import com.tencent.bk.job.common.model.PageData; +import com.tencent.bk.job.common.model.dto.AppResourceScope; +import com.tencent.bk.job.common.mysql.JobTransactional; +import com.tencent.bk.job.common.util.JobUUID; +import com.tencent.bk.job.common.util.date.DateUtils; +import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.manage.api.common.constants.JobResourceStatusEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; +import com.tencent.bk.job.manage.auth.TemplateAuthService; +import com.tencent.bk.job.manage.dao.ScriptCitedTaskTemplateDAO; +import com.tencent.bk.job.manage.dao.ScriptDAO; +import com.tencent.bk.job.manage.dao.ScriptRelateJobTemplateDAO; +import com.tencent.bk.job.manage.dao.ScriptRelateTaskPlanDAO; +import com.tencent.bk.job.manage.dao.TaskScriptStepDAO; +import com.tencent.bk.job.manage.dao.template.TaskTemplateDAO; +import com.tencent.bk.job.manage.model.dto.ResourceTagDTO; +import com.tencent.bk.job.manage.model.dto.ScriptBasicDTO; +import com.tencent.bk.job.manage.model.dto.ScriptDTO; +import com.tencent.bk.job.manage.model.dto.ScriptRelatedTaskPlanDTO; +import com.tencent.bk.job.manage.model.dto.ScriptSyncTemplateStepDTO; +import com.tencent.bk.job.manage.model.dto.SyncScriptResultDTO; +import com.tencent.bk.job.manage.model.dto.TagDTO; +import com.tencent.bk.job.manage.model.dto.TemplateStepIDDTO; +import com.tencent.bk.job.manage.model.dto.script.ScriptCitedTaskPlanDTO; +import com.tencent.bk.job.manage.model.dto.script.ScriptCitedTaskTemplateDTO; +import com.tencent.bk.job.manage.model.dto.task.TaskTemplateInfoDTO; +import com.tencent.bk.job.manage.model.query.ScriptQuery; +import com.tencent.bk.job.manage.model.web.vo.TagCountVO; +import com.tencent.bk.job.manage.service.ScriptManager; +import com.tencent.bk.job.manage.service.TagService; +import com.tencent.bk.job.manage.service.template.TaskTemplateService; +import com.tencent.bk.job.manage.service.template.impl.TemplateScriptStatusUpdateService; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Lazy; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.UUID; +import java.util.stream.Collectors; + +/** + * 脚本管理通用实现 + */ +@Slf4j +@Service +public class ScriptManagerImpl implements ScriptManager { + + private final ScriptDAO scriptDAO; + private final TagService tagService; + private final ScriptRelateTaskPlanDAO scriptRelateTaskPlanDAO; + private final ScriptCitedTaskTemplateDAO scriptCitedTaskTemplateDAO; + private final ScriptRelateJobTemplateDAO scriptRelateJobTemplateDAO; + private final TaskScriptStepDAO taskScriptStepDAO; + private final TaskTemplateDAO taskTemplateDAO; + private final TemplateScriptStatusUpdateService templateScriptStatusUpdateService; + private final TemplateAuthService templateAuthService; + private final MessageI18nService i18nService; + private TaskTemplateService taskTemplateService; + + @Autowired + @Lazy + public void setTaskTemplateService(TaskTemplateService taskTemplateService) { + this.taskTemplateService = taskTemplateService; + } + + @Autowired + public ScriptManagerImpl( + ScriptDAO scriptDAO, + TagService tagService, + ScriptRelateTaskPlanDAO scriptRelateTaskPlanDAO, + ScriptCitedTaskTemplateDAO scriptCitedTaskTemplateDAO, + ScriptRelateJobTemplateDAO scriptRelateJobTemplateDAO, + @Qualifier("TaskTemplateScriptStepDAOImpl") TaskScriptStepDAO taskScriptStepDAO, + TaskTemplateDAO taskTemplateDAO, + TemplateScriptStatusUpdateService templateScriptStatusUpdateService, + TemplateAuthService templateAuthService, + MessageI18nService i18nService) { + this.scriptDAO = scriptDAO; + this.tagService = tagService; + this.scriptRelateTaskPlanDAO = scriptRelateTaskPlanDAO; + this.scriptCitedTaskTemplateDAO = scriptCitedTaskTemplateDAO; + this.scriptRelateJobTemplateDAO = scriptRelateJobTemplateDAO; + this.taskScriptStepDAO = taskScriptStepDAO; + this.taskTemplateDAO = taskTemplateDAO; + this.templateScriptStatusUpdateService = templateScriptStatusUpdateService; + this.templateAuthService = templateAuthService; + this.i18nService = i18nService; + } + + @Override + public ScriptDTO getScriptVersion(Long appId, Long scriptVersionId) { + ScriptDTO scriptDTO = scriptDAO.getScriptVersionById(scriptVersionId); + checkScriptInApp(appId, scriptDTO); + return scriptDTO; + } + + @Override + public ScriptDTO getByScriptIdAndVersion(Long appId, String scriptId, String version) { + ScriptDTO scriptDTO = scriptDAO.getByScriptIdAndVersion(appId, scriptId, version); + checkScriptInApp(appId, scriptDTO); + return scriptDTO; + } + + @Override + public ScriptDTO getScriptVersion(Long scriptVersionId) { + return scriptDAO.getScriptVersionById(scriptVersionId); + } + + @Override + public ScriptDTO getScript(Long appId, String scriptId) { + ScriptDTO script = scriptDAO.getScriptByScriptId(scriptId); + checkScriptInApp(appId, script); + setTags(script); + return script; + } + + @Override + public ScriptDTO getScriptByScriptId(String scriptId) { + ScriptDTO script = scriptDAO.getScriptByScriptId(scriptId); + setTags(script); + return script; + } + + @Override + public List listScriptBasicInfoByScriptIds(Collection scriptIds) { + return scriptDAO.listScriptBasicInfoByScriptIds(scriptIds); + } + + @Override + public ScriptDTO getScriptWithoutTagByScriptId(String scriptId) { + return scriptDAO.getScriptByScriptId(scriptId); + } + + @Override + public List listScriptVersion(Long appId, String scriptId) { + List scripts = scriptDAO.listScriptVersionsByScriptId(scriptId); + if (scripts == null || scripts.isEmpty()) { + return Collections.emptyList(); + } + checkScriptInApp(appId, scripts.get(0)); + return scripts; + } + + private void setTags(ScriptDTO script) { + if (script == null) { + return; + } + setTags(Collections.singletonList(script)); + } + + private void setTags(Collection scripts) { + if (CollectionUtils.isEmpty(scripts)) { + return; + } + ScriptDTO someScript = scripts.stream().findAny().orElse(null); + if (someScript == null) { + return; + } + long appId = someScript.getAppId(); + boolean isPublicScript = someScript.isPublicScript(); + + Integer resourceType = isPublicScript ? JobResourceTypeEnum.PUBLIC_SCRIPT.getValue() : + JobResourceTypeEnum.APP_SCRIPT.getValue(); + + List scriptIds = scripts.stream().map(ScriptDTO::getId).collect(Collectors.toList()); + List resourceTags = tagService.listResourceTagsByResourceTypeAndResourceIds(appId, + resourceType, scriptIds); + + Map> scriptTags = new HashMap<>(); + resourceTags.forEach(resourceTag -> scriptTags.compute(resourceTag.getResourceId(), (scriptId, tags) -> { + if (tags == null) { + tags = new ArrayList<>(); + } + tags.add(resourceTag); + return tags; + })); + + scripts.forEach(script -> { + List tags = scriptTags.get(script.getId()); + if (CollectionUtils.isNotEmpty(tags)) { + script.setTags(buildTags(tags)); + } + }); + } + + private List buildTags(List resourceTags) { + return resourceTags.stream().map(ResourceTagDTO::getTag).filter(Objects::nonNull) + .sorted(Comparator.comparing(TagDTO::getName)).collect(Collectors.toList()); + } + + @Override + public PageData listPageScript(ScriptQuery scriptQuery) { + if (scriptQuery.isExistTagCondition()) { + List tagMatchedScriptIds = queryScriptIdsByTags(scriptQuery); + if (CollectionUtils.isEmpty(tagMatchedScriptIds)) { + // none match, return empty page data + return PageData.emptyPageData(scriptQuery.getBaseSearchCondition().getStart(), + scriptQuery.getBaseSearchCondition().getLength()); + } else { + scriptQuery.setIds(tagMatchedScriptIds); + } + } + PageData pageData = scriptDAO.listPageScript(scriptQuery, scriptQuery.getBaseSearchCondition()); + //设置标签 + setTags(pageData.getData()); + return pageData; + } + + private List queryScriptIdsByTags(ScriptQuery query) { + List matchScriptIds = new ArrayList<>(); + Integer resourceType = query.isPublicScript() ? JobResourceTypeEnum.PUBLIC_SCRIPT.getValue() : + JobResourceTypeEnum.APP_SCRIPT.getValue(); + if (query.isUntaggedScript()) { + // untagged script + List taggedScriptIds = new ArrayList<>(tagService.listAppTaggedResourceIds(query.getAppId(), + resourceType)); + matchScriptIds.addAll(scriptDAO.listAppScriptIds(query.getAppId())); + matchScriptIds.removeAll(taggedScriptIds); + } else if (CollectionUtils.isNotEmpty(query.getTagIds())) { + matchScriptIds = tagService.listResourceIdsWithAllTagIds(resourceType, query.getTagIds()); + } + return matchScriptIds; + } + + @Override + public List listScripts(ScriptQuery scriptQuery) { + List scripts = scriptDAO.listScripts(scriptQuery); + //设置标签 + setTags(scripts); + return scripts; + } + + @Override + public ScriptDTO createScript(ScriptDTO script) { + log.info("Begin to create script: {}", script); + long appId = script.getAppId(); + + boolean isNameDuplicate = scriptDAO.isExistDuplicateName(appId, script.getName()); + if (isNameDuplicate) { + log.warn("The script name:{} is exist for app:{}", script.getName(), appId); + throw new AlreadyExistsException(ErrorCode.SCRIPT_NAME_DUPLICATE); + } + + script.setId(JobUUID.getUUID()); + // 默认为未上线状态 + script.setStatus(JobResourceStatusEnum.DRAFT.getValue()); + + Long scriptVersionId; + scriptDAO.saveScript(script); + scriptVersionId = scriptDAO.saveScriptVersion(script); + script.setScriptVersionId(scriptVersionId); + + saveScriptTags(appId, script); + + return getScriptVersion(scriptVersionId); + } + + private void saveScriptTags(Long appId, ScriptDTO script) { + saveScriptTags(script.getLastModifyUser(), appId, script.getId(), script.getTags()); + } + + private void saveScriptTags(String operator, Long appId, String scriptId, List tags) { + List newTags = tags; + if (tags != null && !tags.isEmpty()) { + newTags = tagService.createNewTagIfNotExist(tags, appId, operator); + } + + Integer resourceType = appId == (JobConstants.PUBLIC_APP_ID) ? + JobResourceTypeEnum.PUBLIC_SCRIPT.getValue() : + JobResourceTypeEnum.APP_SCRIPT.getValue(); + + tagService.patchResourceTags(resourceType, scriptId, CollectionUtils.isEmpty(newTags) ? + Collections.emptyList() : newTags.stream().map(TagDTO::getId).collect(Collectors.toList())); + } + + private Long getTimeOrDefault(Long time) { + if (time == null) { + return DateUtils.currentTimeMillis(); + } + return time; + } + + @Override + public ScriptDTO createScriptVersion(ScriptDTO scriptVersion) { + log.info("Begin to save scriptVersion: {}", scriptVersion); + + if (scriptDAO.isExistDuplicateVersion(scriptVersion.getId(), scriptVersion.getVersion())) { + log.warn("Script version:{} is exist, scriptId:{}", scriptVersion.getVersion(), scriptVersion.getId()); + throw new AlreadyExistsException(ErrorCode.SCRIPT_VERSION_NAME_EXIST); + } + + // 默认为未上线状态 + scriptVersion.setStatus(JobResourceStatusEnum.DRAFT.getValue()); + + Long scriptVersionId = scriptDAO.saveScriptVersion(scriptVersion); + scriptVersion.setScriptVersionId(scriptVersionId); + scriptDAO.updateScriptLastModify(scriptVersion.getId(), scriptVersion.getCreator(), System.currentTimeMillis()); + + return getScriptVersion(scriptVersionId); + } + + @Override + public ScriptDTO updateScriptVersion(ScriptDTO scriptVersion) { + log.info("Begin to update scriptVersion: {}", scriptVersion); + + ScriptDTO scriptVersionToBeUpdate = scriptDAO.getScriptVersionById(scriptVersion.getScriptVersionId()); + if (scriptVersionToBeUpdate == null) { + throw new NotFoundException(ErrorCode.SCRIPT_NOT_EXIST); + } + if (!scriptVersionToBeUpdate.getStatus().equals(JobResourceStatusEnum.DRAFT.getValue())) { + log.warn("Script status is not draft, can not update.scriptVersionId={}", + scriptVersion.getScriptVersionId()); + throw new FailedPreconditionException(ErrorCode.UNSUPPORTED_OPERATION); + } + + scriptDAO.updateScriptVersion(scriptVersion.getLastModifyUser(), scriptVersion.getScriptVersionId(), + scriptVersion); + + return scriptDAO.getScriptVersionById(scriptVersion.getScriptVersionId()); + } + + @Override + @JobTransactional(transactionManager = "jobManageTransactionManager") + public Pair createScriptWithVersionId( + Long appId, + ScriptDTO script, + Long createTime, + Long lastModifyTime + ) throws ServiceException { + log.info("Begin to createScriptWithVersionId, appId={}, script={}, createTime={}, " + + "lastModifyTime={}", appId, JsonUtils.toJson(script), createTime, lastModifyTime); + script.setCreateTime(getTimeOrDefault(createTime)); + script.setLastModifyTime(getTimeOrDefault(lastModifyTime)); + final long targetAppId = script.isPublicScript() ? JobConstants.PUBLIC_APP_ID : appId; + script.setAppId(targetAppId); + + // 默认为未上线状态 + Integer status = script.getStatus(); + if (status == null || status < 0) { + script.setStatus(JobResourceStatusEnum.DRAFT.getValue()); + } + + if (script.getScriptVersionId() != null && script.getScriptVersionId() > 0) { + if (scriptDAO.isExistDuplicateScriptId(script.getScriptVersionId())) { + log.warn("scriptVersionId:{} is exist, scriptId:{}", script.getScriptVersionId(), script.getId()); + throw new AlreadyExistsException(ErrorCode.SCRIPT_VERSION_ID_EXIST); + } + } + + if (StringUtils.isNotBlank(script.getId())) { + if (scriptDAO.isExistDuplicateVersion(script.getId(), script.getVersion())) { + log.warn("Script version:{} is exist, scriptId:{}", script.getVersion(), script.getId()); + throw new AlreadyExistsException(ErrorCode.SCRIPT_VERSION_NAME_EXIST); + } + if (!scriptDAO.isExistDuplicateScriptId(appId, script.getId())) { + //脚本不存在，新增脚本和脚本版本 + boolean isNameDuplicate = scriptDAO.isExistDuplicateName(targetAppId, script.getName()); + if (isNameDuplicate) { + log.warn("The script name:{} is exist for app:{}", script.getName(), targetAppId); + throw new AlreadyExistsException(ErrorCode.SCRIPT_NAME_DUPLICATE); + } + saveScriptAndScriptVersionToDB(script); + log.info("script created with specified id:{}", script.getId()); + } else { + // 脚本存在，新增脚本版本 + saveScriptVersionToDB(script); + } + } else { + //脚本不存在，新增脚本 + boolean isNameDuplicate = scriptDAO.isExistDuplicateName(targetAppId, script.getName()); + if (isNameDuplicate) { + log.warn("The script name:{} is exist for app:{}", script.getName(), targetAppId); + throw new AlreadyExistsException(ErrorCode.SCRIPT_NAME_DUPLICATE); + } + + script.setId(JobUUID.getUUID()); + saveScriptAndScriptVersionToDB(script); + } + saveScriptTags(appId, script); + return Pair.of(script.getId(), script.getScriptVersionId()); + } + + public void saveScriptAndScriptVersionToDB(ScriptDTO script) { + // 插入script + String scriptId = scriptDAO.saveScript(script); + script.setId(scriptId); + // 插入script_version + Long scriptVersionId = scriptDAO.saveScriptVersion(script); + script.setScriptVersionId(scriptVersionId); + } + + public void saveScriptVersionToDB(ScriptDTO script) { + // 插入script_version + Long scriptVersionId = scriptDAO.saveScriptVersion(script); + scriptDAO.updateScriptLastModify(script.getId(), script.getLastModifyUser(), script.getLastModifyTime()); + script.setScriptVersionId(scriptVersionId); + } + + @Override + @JobTransactional(transactionManager = "jobManageTransactionManager") + public void deleteScriptVersion(Long appId, Long scriptVersionId) { + ScriptDTO existScript = scriptDAO.getScriptVersionById(scriptVersionId); + checkDeleteScriptPermission(appId, existScript); + checkScriptReferenced(existScript); + List scriptVersions = scriptDAO.listScriptVersionsByScriptId(existScript.getId()); + if (scriptVersions.size() == 1) { + scriptDAO.deleteScript(existScript.getId()); + scriptDAO.deleteScriptVersion(scriptVersionId); + deleteScriptRelatedTags(appId, existScript.getId()); + } else { + scriptDAO.deleteScriptVersion(scriptVersionId); + } + } + + private void checkScriptReferenced(ScriptDTO script) { + if (isScriptReferenced(script.getId(), script.getScriptVersionId())) { + throw new FailedPreconditionException(ErrorCode.DELETE_REF_SCRIPT_FAIL); + } + } + + @Override + @JobTransactional(transactionManager = "jobManageTransactionManager") + public void deleteScript(Long appId, String scriptId) { + ScriptDTO existScript = getScript(appId, scriptId); + checkDeleteScriptPermission(appId, existScript); + checkScriptReferenced(existScript); + scriptDAO.deleteScript(scriptId); + scriptDAO.deleteScriptVersionByScriptId(scriptId); + deleteScriptRelatedTags(appId, scriptId); + } + + private void deleteScriptRelatedTags(Long appId, String scriptId) { + Integer resourceType = (appId == JobConstants.PUBLIC_APP_ID) ? + JobResourceTypeEnum.PUBLIC_SCRIPT.getValue() : + JobResourceTypeEnum.APP_SCRIPT.getValue(); + tagService.batchDeleteResourceTags(appId, resourceType, scriptId); + } + + private void checkDeleteScriptPermission( + Long appId, + ScriptDTO existScript + ) { + if (existScript == null) { + throw new NotFoundException(ErrorCode.SCRIPT_NOT_EXIST); + } + long targetAppId = appId; + if (existScript.isPublicScript()) { + targetAppId = JobConstants.PUBLIC_APP_ID; + } + if (!existScript.getAppId().equals(targetAppId)) { + throw new NotFoundException(ErrorCode.SCRIPT_NOT_IN_APP); + } + } + + @Override + public void publishScript(Long appId, + String scriptId, + Long scriptVersionId) { + log.info("Begin to publish script, appId={}, scriptId={}, scriptVersionId={}", appId, scriptId, + scriptVersionId); + List scriptVersions = scriptDAO.listScriptVersionsByScriptId(scriptId); + if (scriptVersions == null || scriptVersions.isEmpty()) { + log.warn("Publish script, script:{} is not exist", scriptId); + throw new NotFoundException(ErrorCode.SCRIPT_NOT_EXIST); + } + + boolean isPublicScript = scriptVersions.get(0).isPublicScript(); + long targetAppId = appId; + if (isPublicScript) { + targetAppId = JobConstants.PUBLIC_APP_ID; + } + + boolean isScriptVersionInCurrentScript = false; + ScriptDTO scriptVersionToBePublished = null; + for (ScriptDTO scriptVersion : scriptVersions) { + if (!isPublicScript && !scriptVersion.getAppId().equals(targetAppId)) { + log.warn("Publish script, script:{} is not in current app:{}", scriptId, targetAppId); + throw new NotFoundException(ErrorCode.SCRIPT_NOT_IN_APP); + } + if (scriptVersion.getScriptVersionId().equals(scriptVersionId)) { + isScriptVersionInCurrentScript = true; + scriptVersionToBePublished = scriptVersion; + } + } + if (!isScriptVersionInCurrentScript) { + log.warn("Public script, scriptVersion:{} is not in script:{}", scriptVersionId, scriptId); + throw new NotFoundException(ErrorCode.SCRIPT_NOT_EXIST); + } + + if (scriptVersionToBePublished.getStatus().equals(JobResourceStatusEnum.DISABLED.getValue())) { + log.warn("Publish script, scriptVersion:{}, status:{} could not publish", scriptVersionId, + scriptVersionToBePublished.getStatus()); + throw new FailedPreconditionException(ErrorCode.UNSUPPORTED_OPERATION); + } + + ScriptDTO publishedScriptVersion = null; + for (ScriptDTO scriptVersion : scriptVersions) { + if (scriptVersion.getStatus().equals(JobResourceStatusEnum.ONLINE.getValue())) { + publishedScriptVersion = scriptVersion; + break; + } + } + if (publishedScriptVersion != null) { + log.info("Publish script, set scriptVersion:{} offline", publishedScriptVersion.getScriptVersionId()); + scriptDAO.updateScriptVersionStatus(publishedScriptVersion.getScriptVersionId(), + JobResourceStatusEnum.OFFLINE.getValue()); + templateScriptStatusUpdateService.refreshTemplateScriptStatusByScript(scriptId, + publishedScriptVersion.getScriptVersionId()); + } + scriptDAO.updateScriptVersionStatus(scriptVersionId, JobResourceStatusEnum.ONLINE.getValue()); + templateScriptStatusUpdateService.refreshTemplateScriptStatusByScript(scriptId, scriptVersionId); + log.info("Publish script successfully, scriptId={}, scriptVersionId={}", scriptId, scriptVersionId); + } + + @Override + public void disableScript(Long appId, + String scriptId, + Long scriptVersionId) { + log.info("Begin to disable script, appId={}, scriptId={}, scriptVersionId={}", appId, + scriptId, scriptVersionId); + List scriptVersions = scriptDAO.listScriptVersionsByScriptId(scriptId); + if (scriptVersions == null || scriptVersions.isEmpty()) { + log.warn("Disable script, script:{} is not exist", scriptId); + throw new NotFoundException(ErrorCode.SCRIPT_NOT_EXIST); + } + + boolean isPublicScript = scriptVersions.get(0).isPublicScript(); + long targetAppId = appId; + if (isPublicScript) { + targetAppId = JobConstants.PUBLIC_APP_ID; + } + + boolean isScriptVersionInCurrentScript = false; + ScriptDTO scriptVersionToBeDisabled = null; + for (ScriptDTO scriptVersion : scriptVersions) { + if (!scriptVersion.getAppId().equals(targetAppId)) { + log.warn("Disable script, script:{} is not in current app:{}", scriptId, targetAppId); + throw new NotFoundException(ErrorCode.SCRIPT_NOT_IN_APP); + } + if (scriptVersion.getScriptVersionId().equals(scriptVersionId)) { + isScriptVersionInCurrentScript = true; + scriptVersionToBeDisabled = scriptVersion; + } + } + if (!isScriptVersionInCurrentScript) { + log.warn("Disable script, scriptVersion:{} is not in script:{}", scriptVersionId, scriptId); + throw new NotFoundException(ErrorCode.SCRIPT_NOT_EXIST); + } + + if (!scriptVersionToBeDisabled.getStatus().equals(JobResourceStatusEnum.ONLINE.getValue()) + && !scriptVersionToBeDisabled.getStatus().equals(JobResourceStatusEnum.OFFLINE.getValue())) { + log.warn("Disable script, scriptVersion:{}, status:{} could not disable", scriptVersionId, + scriptVersionToBeDisabled.getStatus()); + throw new FailedPreconditionException(ErrorCode.UNSUPPORTED_OPERATION); + } + + scriptDAO.updateScriptVersionStatus(scriptVersionToBeDisabled.getScriptVersionId(), + JobResourceStatusEnum.DISABLED.getValue()); + templateScriptStatusUpdateService.refreshTemplateScriptStatusByScript(scriptId, + scriptVersionToBeDisabled.getScriptVersionId()); + log.info("Disable script successfully, scriptId={}, scriptVersionId={}", scriptId, scriptVersionId); + } + + @Override + public Map batchGetOnlineScriptVersionByScriptIds(List scriptIdList) { + Map scripts = scriptDAO.batchGetOnlineByScriptIds(scriptIdList); + setTags(scripts.values()); + return scripts; + } + + @Override + public ScriptDTO updateScriptDesc(String operator, Long appId, String scriptId, String desc) { + log.info("Begin to update script desc,appId={},operator={},scriptId={},desc={}", appId, operator, scriptId, + desc); + ScriptDTO script = scriptDAO.getScriptByScriptId(scriptId); + checkScriptInApp(appId, script); + scriptDAO.updateScriptDesc(operator, scriptId, desc); + script.setDescription(desc); + + return script; + } + + private void checkScriptInApp(long appId, ScriptDTO script) { + if (script == null) { + throw new NotFoundException(ErrorCode.SCRIPT_NOT_EXIST); + } + if (!script.isPublicScript()) { + if (!script.getAppId().equals(appId)) { + log.warn("Script:{} is not in current app:{}", script.getId(), appId); + throw new NotFoundException(ErrorCode.SCRIPT_NOT_IN_APP); + } + } + } + + @Override + public ScriptDTO updateScriptName(String operator, Long appId, String scriptId, String newName) { + log.info("Begin to update script name,appId={},operator={},scriptId={},desc={}", appId, operator, scriptId, + newName); + ScriptDTO script = scriptDAO.getScriptByScriptId(scriptId); + checkScriptInApp(appId, script); + if (script.getName().equals(newName)) { + return script; + } + long targetAppId = appId; + if (script.isPublicScript()) { + targetAppId = JobConstants.PUBLIC_APP_ID; + } + boolean isNameExist = scriptDAO.isExistDuplicateName(targetAppId, newName); + if (isNameExist) { + log.warn("Update script name, script:{} new name {} is duplicate", scriptId, newName); + throw new AlreadyExistsException(ErrorCode.SCRIPT_NAME_DUPLICATE); + } + + scriptDAO.updateScriptName(operator, scriptId, newName); + script.setName(newName); + + return script; + } + + @Override + public ScriptDTO updateScriptTags(String operator, Long appId, String scriptId, List tags) { + log.info("Begin to update script tags,appId={},operator={},scriptId={},tags={}", appId, operator, scriptId, + tags); + ScriptDTO script = scriptDAO.getScriptByScriptId(scriptId); + checkScriptInApp(appId, script); + long targetAppId = appId; + if (script.isPublicScript()) { + targetAppId = JobConstants.PUBLIC_APP_ID; + } + saveScriptTags(operator, targetAppId, scriptId, tags); + + script.setTags(tags); + return script; + } + + @Override + public List listScriptNames(Long appId, String keyword) { + return scriptDAO.listScriptNames(appId, keyword); + } + + @Override + public List listOnlineScriptForApp(long appId) { + List scripts = scriptDAO.listOnlineScriptForApp(appId); + setTags(scripts); + return scripts; + } + + @Override + public ScriptDTO getOnlineScriptVersionByScriptId(long appId, String scriptId) { + return scriptDAO.getOnlineScriptVersionByScriptId(appId, scriptId); + } + + @Override + public ScriptDTO getOnlineScriptVersionByScriptId(String scriptId) { + return scriptDAO.getOnlineScriptVersionByScriptId(scriptId); + } + + @Override + public PageData listPageScriptVersion(ScriptQuery scriptQuery) { + return scriptDAO.listPageScriptVersion(scriptQuery); + } + + @Override + public List listScriptSyncTemplateSteps(Long appId, String scriptId) { + // 检查脚本 + ScriptDTO syncScript = scriptDAO.getScriptByScriptId(scriptId); + checkScriptInApp(appId, syncScript); + + List templateSteps = + scriptRelateJobTemplateDAO.listScriptRelatedJobTemplateSteps(scriptId); + if (CollectionUtils.isEmpty(templateSteps)) { + return templateSteps; + } + // 按业务过滤 + if (appId > 0) { + templateSteps = + templateSteps.stream().filter(step -> step.getAppId().equals(appId)).collect(Collectors.toList()); + if (CollectionUtils.isEmpty(templateSteps)) { + return templateSteps; + } + } + + Set scriptVersionIds = + templateSteps.stream().map(ScriptSyncTemplateStepDTO::getScriptVersionId).collect(Collectors.toSet()); + List scriptVersions = scriptDAO.batchGetScriptVersionsByIds(scriptVersionIds); + if (CollectionUtils.isEmpty(scriptVersions)) { + log.warn("Scripts are not exist, scriptVersionIds: {}", scriptVersionIds); + return Collections.emptyList(); + } + + Map scripts = new HashMap<>(); + scriptVersions.forEach(scriptVersion -> scripts.put(scriptVersion.getScriptVersionId(), scriptVersion)); + templateSteps.forEach(step -> fillScriptInfo(step, scripts)); + return templateSteps; + } + + private void fillScriptInfo(ScriptSyncTemplateStepDTO templateStep, Map scripts) { + ScriptDTO script = scripts.get(templateStep.getScriptVersionId()); + if (script != null) { + templateStep.setScriptName(script.getName()); + templateStep.setScriptVersion(script.getVersion()); + templateStep.setScriptStatus(script.getStatus()); + } + } + + @Override + public List syncScriptToTaskTemplate(String username, Long appId, String scriptId, + Long syncScriptVersionId, + List templateStepIDs) + throws PermissionDeniedException { + if (CollectionUtils.isEmpty(templateStepIDs)) { + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); + } + + ScriptDTO syncScript = scriptDAO.getScriptVersionById(syncScriptVersionId); + checkScriptInApp(appId, syncScript); + if (!scriptId.equals(syncScript.getId())) { + log.warn("Script ID and scriptVersionId not match!"); + throw new NotFoundException(ErrorCode.SCRIPT_NOT_EXIST); + } + // 鉴权 + List authTemplateIds = new ArrayList<>(); + templateStepIDs.forEach(step -> { + TaskTemplateInfoDTO taskTemplate = taskTemplateService.getTaskTemplateBasicInfoById(step.getTemplateId()); + if (appId > 0 && taskTemplate != null && !taskTemplate.getAppId().equals(appId)) { + throw new NotFoundException(ErrorCode.TEMPLATE_NOT_EXIST); + } + if (taskTemplate != null) { + step.setAppId(taskTemplate.getAppId()); + if (!authTemplateIds.contains(step.getTemplateId())) { + authTemplateIds.add(step.getTemplateId()); + } + } + }); + AuthResult authResult = templateAuthService.batchAuthResultEditJobTemplate( + username, + new AppResourceScope(appId), + authTemplateIds + ); + if (!authResult.isPass()) { + log.info("Sync script to template auth fail, scriptId: {}, syncScriptVersionId: {}, template steps: {}", + scriptId, syncScriptVersionId, templateStepIDs); + throw new PermissionDeniedException(authResult); + } + + List syncResults = new ArrayList<>(); + templateStepIDs.forEach(templateStep -> { + Long templateId = templateStep.getTemplateId(); + Long stepId = templateStep.getStepId(); + ScriptSyncTemplateStepDTO updateStep = new ScriptSyncTemplateStepDTO(); + updateStep.setTemplateId(templateId); + updateStep.setStepId(stepId); + try { + boolean success = updateTemplateRefScript(appId, templateId, stepId, syncScriptVersionId); + if (success) { + syncResults.add(SyncScriptResultDTO.buildSuccessSyncResult(updateStep)); + templateScriptStatusUpdateService.refreshTemplateScriptStatusByTemplate(templateId); + } else { + syncResults.add(SyncScriptResultDTO.buildFailSyncResult(updateStep, ErrorCode.TEMPLATE_NOT_EXIST)); + } + } catch (Throwable e) { + log.warn("Update script step ref script fail, templateId: {}, stepId: {}, scriptVersionId: {}", + templateId, stepId, syncScriptVersionId); + syncResults.add(SyncScriptResultDTO.buildFailSyncResult(updateStep, ErrorCode.INTERNAL_ERROR)); + } + }); + + List scriptRelatedTemplateSteps = listScriptSyncTemplateSteps(appId, scriptId); + Map stepMap = new HashMap<>(); + scriptRelatedTemplateSteps.forEach(step -> stepMap.put(step.getStepId(), step)); + fillSyncResultDetail(syncScriptVersionId, syncResults, stepMap); + + return syncResults; + } + + @Override + public Integer getScriptTemplateCiteCount(String scriptId, Long scriptVersionId) { + if (scriptVersionId == null) { + return scriptCitedTaskTemplateDAO.countScriptCitedTaskTemplate(scriptId); + } else { + return scriptCitedTaskTemplateDAO.countScriptVersionCitedTaskTemplate(scriptId, scriptVersionId); + } + } + + @Override + public Integer getScriptTaskPlanCiteCount(String scriptId, Long scriptVersionId) { + if (scriptVersionId == null) { + return scriptRelateTaskPlanDAO.countScriptRelatedTaskPlan(scriptId); + } else { + return scriptRelateTaskPlanDAO.countScriptVersionRelatedTaskPlan(scriptId, scriptVersionId); + } + } + + @Override + public List getScriptCitedTemplates(String scriptId, + Long scriptVersionId) { + List scriptCitedTaskTemplateDTOList; + if (scriptVersionId == null) { + scriptCitedTaskTemplateDTOList = scriptCitedTaskTemplateDAO.listScriptCitedTaskTemplate(scriptId); + } else { + scriptCitedTaskTemplateDTOList = scriptCitedTaskTemplateDAO.listScriptVersionCitedTaskTemplate(scriptId, + scriptVersionId); + } + //填充scriptStatusDesc + scriptCitedTaskTemplateDTOList.forEach(scriptCitedTaskTemplateDTO -> + scriptCitedTaskTemplateDTO.setScriptStatusDesc(i18nService.getI18n( + scriptCitedTaskTemplateDTO.getScriptStatus().getStatusI18nKey() + ) + )); + return scriptCitedTaskTemplateDTOList; + } + + @Override + public List getScriptCitedTaskPlans(String scriptId, + Long scriptVersionId) { + List scriptRelatedTaskPlanDTOList; + if (scriptVersionId == null) { + scriptRelatedTaskPlanDTOList = scriptRelateTaskPlanDAO.listScriptRelatedTaskPlan(scriptId); + } else { + scriptRelatedTaskPlanDTOList = scriptRelateTaskPlanDAO.listScriptVersionRelatedTaskPlan(scriptId, + scriptVersionId); + } + List scriptCitedTaskPlanDTOList = new ArrayList<>(); + if (scriptRelatedTaskPlanDTOList != null) { + scriptRelatedTaskPlanDTOList.forEach(scriptRelatedTaskPlanDTO -> { + ScriptCitedTaskPlanDTO scriptCitedTaskPlanDTO = new ScriptCitedTaskPlanDTO(); + scriptCitedTaskPlanDTO.setAppId(scriptRelatedTaskPlanDTO.getAppId()); + scriptCitedTaskPlanDTO.setScriptStatus(scriptRelatedTaskPlanDTO.getScriptStatus().getValue()); + scriptCitedTaskPlanDTO.setScriptVersion(scriptRelatedTaskPlanDTO.getScriptVersion()); + scriptCitedTaskPlanDTO.setScriptStatusDesc( + i18nService.getI18n( + scriptRelatedTaskPlanDTO.getScriptStatus().getStatusI18nKey() + ) + ); + scriptCitedTaskPlanDTO.setTaskTemplateId(scriptRelatedTaskPlanDTO.getTemplateId()); + scriptCitedTaskPlanDTO.setTaskPlanId(scriptRelatedTaskPlanDTO.getTaskId()); + scriptCitedTaskPlanDTO.setTaskPlanName(scriptRelatedTaskPlanDTO.getTaskName()); + scriptCitedTaskPlanDTOList.add(scriptCitedTaskPlanDTO); + }); + } + return scriptCitedTaskPlanDTOList; + } + + private void fillSyncResultDetail(Long syncScriptVersionId, List syncResults, + Map stepMap) { + for (SyncScriptResultDTO syncResult : syncResults) { + ScriptSyncTemplateStepDTO step = syncResult.getTemplateStep(); + ScriptSyncTemplateStepDTO updatedStep = stepMap.get(step.getStepId()); + if (updatedStep == null) { + log.warn("Step is not exist, stepId: {}", step.getStepId()); + syncResult.setErrorCode(ErrorCode.TEMPLATE_NOT_EXIST); + continue; + } + if (!updatedStep.getScriptVersionId().equals(syncScriptVersionId) + && syncResult.getErrorCode() == null) { + syncResult.setErrorCode(ErrorCode.SYNC_SCRIPT_UNKNOWN_ERROR); + } + step.setTemplateName(updatedStep.getTemplateName()); + step.setStepName(updatedStep.getStepName()); + step.setScriptStatus(updatedStep.getScriptStatus()); + step.setScriptVersion(updatedStep.getScriptVersion()); + step.setScriptVersionId(updatedStep.getScriptVersionId()); + step.setScriptId(updatedStep.getScriptId()); + step.setScriptName(updatedStep.getScriptName()); + } + } + + @JobTransactional(transactionManager = "jobManageTransactionManager") + public boolean updateTemplateRefScript(long appId, long templateId, long stepId, long syncScriptVersionId) { + boolean success = taskScriptStepDAO.updateScriptStepRefScriptVersionId( + templateId, + stepId, + syncScriptVersionId + ); + taskTemplateDAO.updateTaskTemplateVersion(appId, templateId, UUID.randomUUID().toString()); + return success; + } + + @Override + public Integer countScripts(Long appId, ScriptTypeEnum scriptTypeEnum, + JobResourceStatusEnum jobResourceStatusEnum) { + return scriptDAO.countScripts(appId, scriptTypeEnum, jobResourceStatusEnum); + } + + @Override + public Integer countScriptVersions(Long appId, ScriptTypeEnum scriptTypeEnum, + JobResourceStatusEnum jobResourceStatusEnum) { + return scriptDAO.countScriptVersions(appId, scriptTypeEnum, jobResourceStatusEnum); + } + + @Override + public List listScriptIds(Long appId) { + return scriptDAO.listAppScriptIds(appId); + } + + @Override + public Integer countCiteScripts(Long appId) { + // 1.查询业务下所有脚本 + List scriptIdList = scriptDAO.listAppScriptIds(appId); + // 2.查询被引用的脚本数量 + return taskScriptStepDAO.countScriptCitedByStepsByScriptIds(appId, scriptIdList); + } + + @Override + public TagCountVO getTagScriptCount(Long appId) { + TagCountVO tagCount = new TagCountVO(); + + List appScriptIds = scriptDAO.listAppScriptIds(appId) + .stream().map(String::valueOf).collect(Collectors.toList()); + tagCount.setTotal((long) appScriptIds.size()); + + Integer resourceType = JobConstants.PUBLIC_APP_ID == appId ? JobResourceTypeEnum.PUBLIC_SCRIPT.getValue() : + JobResourceTypeEnum.APP_SCRIPT.getValue(); + List tags = tagService.listResourceTagsByResourceTypeAndResourceIds(appId, + resourceType, appScriptIds); + Map scriptTagCount = tagService.countResourcesByTag(tags); + tagCount.setTagCount(scriptTagCount); + + long taggedScriptCount = tags.stream() + .map(ResourceTagDTO::getResourceId).distinct().count(); + tagCount.setUnclassified(appScriptIds.size() - taggedScriptCount); + return tagCount; + } + + @Override + public boolean isExistAnyScript(Long appId) { + return scriptDAO.isExistAnyScript(appId); + } + + @Override + public boolean isScriptReferenced(String scriptId, Long scriptVersionId) { + int citeCount = getScriptTemplateCiteCount(scriptId, scriptVersionId); + if (citeCount == 0) { + citeCount = getScriptTaskPlanCiteCount(scriptId, scriptVersionId); + } + if (citeCount > 0 && scriptVersionId != null) { + ScriptDTO scriptVersion = getScriptVersion(scriptVersionId); + return scriptVersion != null && scriptVersion.getStatus().equals(JobResourceStatusEnum.ONLINE.getValue()); + } + return citeCount > 0; + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/ScriptServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/ScriptServiceImpl.java index 230b0f0a38..5abc56508b 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/ScriptServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/ScriptServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,1034 +24,441 @@ package com.tencent.bk.job.manage.service.impl; +import com.tencent.bk.audit.annotations.ActionAuditRecord; +import com.tencent.bk.audit.annotations.AuditAttribute; +import com.tencent.bk.audit.annotations.AuditInstanceRecord; +import com.tencent.bk.audit.context.ActionAuditContext; +import com.tencent.bk.job.common.audit.constants.EventContentConstants; import com.tencent.bk.job.common.constant.ErrorCode; -import com.tencent.bk.job.common.constant.JobConstants; -import com.tencent.bk.job.common.constant.JobResourceTypeEnum; -import com.tencent.bk.job.common.exception.AlreadyExistsException; -import com.tencent.bk.job.common.exception.FailedPreconditionException; -import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.exception.NotFoundException; -import com.tencent.bk.job.common.exception.ServiceException; -import com.tencent.bk.job.common.i18n.service.MessageI18nService; -import com.tencent.bk.job.common.iam.exception.PermissionDeniedException; -import com.tencent.bk.job.common.iam.model.AuthResult; -import com.tencent.bk.job.common.model.BaseSearchCondition; +import com.tencent.bk.job.common.iam.constant.ActionId; +import com.tencent.bk.job.common.iam.constant.ResourceTypeId; import com.tencent.bk.job.common.model.PageData; import com.tencent.bk.job.common.model.dto.AppResourceScope; -import com.tencent.bk.job.common.util.JobUUID; -import com.tencent.bk.job.common.util.date.DateUtils; -import com.tencent.bk.job.common.util.json.JsonUtils; -import com.tencent.bk.job.manage.auth.TemplateAuthService; -import com.tencent.bk.job.manage.common.consts.JobResourceStatusEnum; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; -import com.tencent.bk.job.manage.dao.ScriptCitedTaskTemplateDAO; -import com.tencent.bk.job.manage.dao.ScriptDAO; -import com.tencent.bk.job.manage.dao.ScriptRelateJobTemplateDAO; -import com.tencent.bk.job.manage.dao.ScriptRelateTaskPlanDAO; -import com.tencent.bk.job.manage.dao.TaskScriptStepDAO; -import com.tencent.bk.job.manage.dao.template.TaskTemplateDAO; -import com.tencent.bk.job.manage.model.dto.ResourceTagDTO; +import com.tencent.bk.job.manage.api.common.constants.JobResourceStatusEnum; +import com.tencent.bk.job.manage.auth.ScriptAuthService; import com.tencent.bk.job.manage.model.dto.ScriptBasicDTO; import com.tencent.bk.job.manage.model.dto.ScriptDTO; -import com.tencent.bk.job.manage.model.dto.ScriptRelatedTaskPlanDTO; -import com.tencent.bk.job.manage.model.dto.ScriptSyncTemplateStepDTO; import com.tencent.bk.job.manage.model.dto.SyncScriptResultDTO; import com.tencent.bk.job.manage.model.dto.TagDTO; import com.tencent.bk.job.manage.model.dto.TemplateStepIDDTO; -import com.tencent.bk.job.manage.model.dto.script.ScriptCitedTaskPlanDTO; -import com.tencent.bk.job.manage.model.dto.script.ScriptCitedTaskTemplateDTO; -import com.tencent.bk.job.manage.model.dto.task.TaskTemplateInfoDTO; import com.tencent.bk.job.manage.model.query.ScriptQuery; import com.tencent.bk.job.manage.model.web.vo.TagCountVO; +import com.tencent.bk.job.manage.service.ScriptManager; import com.tencent.bk.job.manage.service.ScriptService; -import com.tencent.bk.job.manage.service.TagService; -import com.tencent.bk.job.manage.service.template.TaskTemplateService; -import com.tencent.bk.job.manage.service.template.impl.TemplateStatusUpdateService; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.lang3.StringUtils; -import org.apache.commons.lang3.tuple.Pair; -import org.jooq.DSLContext; -import org.jooq.impl.DSL; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; -import java.util.ArrayList; import java.util.Collection; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Objects; -import java.util.Set; -import java.util.UUID; -import java.util.stream.Collectors; /** - * @date 2019/09/19 + * 业务脚本 Service */ @Slf4j @Service public class ScriptServiceImpl implements ScriptService { - - private final ScriptDAO scriptDAO; - private final TagService tagService; - private final ScriptRelateTaskPlanDAO scriptRelateTaskPlanDAO; - private final ScriptCitedTaskTemplateDAO scriptCitedTaskTemplateDAO; - private final ScriptRelateJobTemplateDAO scriptRelateJobTemplateDAO; - private final TaskScriptStepDAO taskScriptStepDAO; - private final TaskTemplateDAO taskTemplateDAO; - private final TemplateStatusUpdateService templateStatusUpdateService; - private final TemplateAuthService templateAuthService; - private final MessageI18nService i18nService; - private final DSLContext dslContext; - private TaskTemplateService taskTemplateService; - + private final ScriptManager scriptManager; + private final ScriptAuthService scriptAuthService; @Autowired - public ScriptServiceImpl( - DSLContext dslContext, - ScriptDAO scriptDAO, - TagService tagService, - ScriptRelateTaskPlanDAO scriptRelateTaskPlanDAO, - ScriptCitedTaskTemplateDAO scriptCitedTaskTemplateDAO, - TaskTemplateService taskTemplateService, - ScriptRelateJobTemplateDAO scriptRelateJobTemplateDAO, - @Qualifier("TaskTemplateScriptStepDAOImpl") TaskScriptStepDAO taskScriptStepDAO, - TaskTemplateDAO taskTemplateDAO, - TemplateStatusUpdateService templateStatusUpdateService, - TemplateAuthService templateAuthService, - MessageI18nService i18nService) { - this.dslContext = dslContext; - this.scriptDAO = scriptDAO; - this.tagService = tagService; - this.scriptRelateTaskPlanDAO = scriptRelateTaskPlanDAO; - this.scriptCitedTaskTemplateDAO = scriptCitedTaskTemplateDAO; - this.taskTemplateService = taskTemplateService; - this.scriptRelateJobTemplateDAO = scriptRelateJobTemplateDAO; - this.taskScriptStepDAO = taskScriptStepDAO; - this.taskTemplateDAO = taskTemplateDAO; - this.templateStatusUpdateService = templateStatusUpdateService; - this.templateAuthService = templateAuthService; - this.i18nService = i18nService; - } - - @Override - public ScriptDTO getScriptVersion(String operator, Long appId, Long scriptVersionId) throws ServiceException { - ScriptDTO scriptDTO = scriptDAO.getScriptVersionById(scriptVersionId); - checkScriptInApp(appId, scriptDTO); - return scriptDTO; + public ScriptServiceImpl(ScriptManager scriptManager, + ScriptAuthService scriptAuthService) { + this.scriptManager = scriptManager; + this.scriptAuthService = scriptAuthService; } @Override - public ScriptDTO getByScriptIdAndVersion(String operator, Long appId, String scriptId, - String version) throws ServiceException { - ScriptDTO scriptDTO = scriptDAO.getByScriptIdAndVersion(appId, scriptId, version); - checkScriptInApp(appId, scriptDTO); - return scriptDTO; + public PageData listPageScript(ScriptQuery scriptQuery) { + return scriptManager.listPageScript(scriptQuery); } @Override - public ScriptDTO getScriptVersion(Long scriptVersionId) throws ServiceException { - return scriptDAO.getScriptVersionById(scriptVersionId); + public List listScripts(ScriptQuery scriptQuery) { + return scriptManager.listScripts(scriptQuery); } @Override - public ScriptDTO getScript(String operator, Long appId, String scriptId) throws ServiceException { - ScriptDTO script = scriptDAO.getScriptByScriptId(scriptId); - checkScriptInApp(appId, script); - setTags(script); - return script; + public ScriptDTO getScript(Long appId, String scriptId) { + return scriptManager.getScript(appId, scriptId); } @Override - public ScriptDTO getScriptByScriptId(String scriptId) throws ServiceException { - ScriptDTO script = scriptDAO.getScriptByScriptId(scriptId); - setTags(script); - return script; + @ActionAuditRecord( + actionId = ActionId.VIEW_SCRIPT, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.SCRIPT, + instanceIds = "#scriptId", + instanceNames = "#$?.name" + ), + content = EventContentConstants.VIEW_SCRIPT + ) + public ScriptDTO getScript(String username, Long appId, String scriptId) { + authViewScript(username, appId, scriptId); + return getScript(appId, scriptId); } @Override - public List listScriptBasicInfoByScriptIds(Collection scriptIds) throws ServiceException { - return scriptDAO.listScriptBasicInfoByScriptIds(scriptIds); + public List listScriptBasicInfoByScriptIds(Collection scriptIds) { + return scriptManager.listScriptBasicInfoByScriptIds(scriptIds); } @Override - public ScriptDTO getScriptWithoutTagByScriptId(String scriptId) throws ServiceException { - return scriptDAO.getScriptByScriptId(scriptId); + @ActionAuditRecord( + actionId = ActionId.CREATE_SCRIPT, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.SCRIPT, + instanceIds = "#$?.id", + instanceNames = "#$?.name" + ), + content = EventContentConstants.CREATE_SCRIPT + ) + public ScriptDTO createScript(String username, ScriptDTO script) { + authCreateScript(username, script.getAppId()); + ScriptDTO newScript = scriptManager.createScript(script); + scriptAuthService.registerScript(newScript.getId(), newScript.getName(), username); + return newScript; } @Override - public List listScriptVersion(String operator, Long appId, String scriptId) throws ServiceException { - List scripts = scriptDAO.listScriptVersionsByScriptId(scriptId); - if (scripts == null || scripts.isEmpty()) { - return Collections.emptyList(); - } - checkScriptInApp(appId, scripts.get(0)); - return scripts; - } + @ActionAuditRecord( + actionId = ActionId.MANAGE_SCRIPT, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.SCRIPT, + instanceIds = "#scriptId" + ), + content = EventContentConstants.DELETE_SCRIPT + ) + public void deleteScript(String username, Long appId, String scriptId) { + log.info("Delete script[{}], operator={}, appId={}, scriptId", scriptId, username, scriptId); + authManageScript(username, appId, scriptId); - private void setTags(ScriptDTO script) { + ScriptDTO script = getScript(appId, scriptId); if (script == null) { - return; + throw new NotFoundException(ErrorCode.SCRIPT_NOT_EXIST); } - setTags(Collections.singletonList(script)); - } - private void setTags(Collection scripts) { - if (CollectionUtils.isEmpty(scripts)) { - return; - } - ScriptDTO someScript = scripts.stream().findAny().orElse(null); - if (someScript == null) { - return; - } - long appId = someScript.getAppId(); - boolean isPublicScript = someScript.isPublicScript(); - - Integer resourceType = isPublicScript ? JobResourceTypeEnum.PUBLIC_SCRIPT.getValue() : - JobResourceTypeEnum.APP_SCRIPT.getValue(); - - List scriptIds = scripts.stream().map(ScriptDTO::getId).collect(Collectors.toList()); - List resourceTags = tagService.listResourceTagsByResourceTypeAndResourceIds(appId, - resourceType, scriptIds); - - Map> scriptTags = new HashMap<>(); - resourceTags.forEach(resourceTag -> scriptTags.compute(resourceTag.getResourceId(), (scriptId, tags) -> { - if (tags == null) { - tags = new ArrayList<>(); - } - tags.add(resourceTag); - return tags; - })); - - scripts.forEach(script -> { - List tags = scriptTags.get(script.getId()); - if (CollectionUtils.isNotEmpty(tags)) { - script.setTags(buildTags(tags)); - } - }); - } + ActionAuditContext.current().setInstanceName(script.getName()); - private List buildTags(List resourceTags) { - return resourceTags.stream().map(ResourceTagDTO::getTag).filter(Objects::nonNull) - .sorted(Comparator.comparing(TagDTO::getName)).collect(Collectors.toList()); + scriptManager.deleteScript(appId, scriptId); } @Override - public PageData listPageScript(ScriptQuery scriptCondition, - BaseSearchCondition baseSearchCondition) throws ServiceException { - if (scriptCondition.isExistTagCondition()) { - List tagMatchedScriptIds = queryScriptIdsByTags(scriptCondition); - if (CollectionUtils.isEmpty(tagMatchedScriptIds)) { - // none match, return empty page data - return PageData.emptyPageData(baseSearchCondition.getStart(), baseSearchCondition.getLength()); - } else { - scriptCondition.setIds(tagMatchedScriptIds); - } - } - PageData pageData = scriptDAO.listPageScript(scriptCondition, baseSearchCondition); - //设置标签 - setTags(pageData.getData()); - return pageData; - } - - private List queryScriptIdsByTags(ScriptQuery query) { - List matchScriptIds = new ArrayList<>(); - Integer resourceType = query.isPublicScript() ? JobResourceTypeEnum.PUBLIC_SCRIPT.getValue() : - JobResourceTypeEnum.APP_SCRIPT.getValue(); - if (query.isUntaggedScript()) { - // untagged script - List taggedScriptIds = new ArrayList<>(tagService.listAppTaggedResourceIds(query.getAppId(), - resourceType)); - matchScriptIds.addAll(scriptDAO.listAppScriptIds(query.getAppId())); - matchScriptIds.removeAll(taggedScriptIds); - } else if (CollectionUtils.isNotEmpty(query.getTagIds())) { - matchScriptIds = tagService.listResourceIdsWithAllTagIds(resourceType, query.getTagIds()); - } - return matchScriptIds; + public ScriptDTO getScriptVersion(long appId, Long scriptVersionId) { + return scriptManager.getScriptVersion(appId, scriptVersionId); } @Override - public List listScripts(ScriptQuery scriptQuery) { - List scripts = scriptDAO.listScripts(scriptQuery); - //设置标签 - setTags(scripts); - return scripts; + public ScriptDTO getScriptVersion(Long scriptVersionId) { + return scriptManager.getScriptVersion(scriptVersionId); } @Override - public ScriptDTO saveScript(String operator, Long appId, ScriptDTO script) throws ServiceException { - log.info("Begin to save script, operator={}, appId={}, scriptName={}", operator, appId, script.getName()); - long targetAppId = appId; - if (script.isPublicScript()) { - // 公共脚本业务ID为0 - targetAppId = JobConstants.PUBLIC_APP_ID; - script.setAppId(targetAppId); - } - - // 默认为未上线状态 - script.setStatus(JobResourceStatusEnum.DRAFT.getValue()); - - Long scriptVersionId = script.getScriptVersionId(); - if (StringUtils.isNotBlank(script.getId())) { - // 更新当前版本 - if (script.getScriptVersionId() != null && script.getScriptVersionId() > 0) { - ScriptDTO scriptVersionToBeUpdate = scriptDAO.getScriptVersionById(script.getScriptVersionId()); - if (scriptVersionToBeUpdate == null) { - throw new NotFoundException(ErrorCode.SCRIPT_NOT_EXIST); - } - if (!scriptVersionToBeUpdate.getStatus().equals(JobResourceStatusEnum.DRAFT.getValue())) { - log.warn("Script status is not draft, can not update.scriptVersionId={}", - script.getScriptVersionId()); - throw new FailedPreconditionException(ErrorCode.UNSUPPORTED_OPERATION); - } - scriptDAO.updateScriptVersion(script.getLastModifyUser(), script.getScriptVersionId(), script); - } else { - // 新增脚本版本 - if (scriptDAO.isExistDuplicateVersion(script.getId(), script.getVersion())) { - log.warn("Script version:{} is exist, scriptId:{}", script.getVersion(), script.getId()); - throw new AlreadyExistsException(ErrorCode.SCRIPT_VERSION_NAME_EXIST); - } - scriptVersionId = scriptDAO.saveScriptVersion(script); - script.setScriptVersionId(scriptVersionId); - scriptDAO.updateScript(script); - } - } else { - //脚本不存在，新增脚本 - boolean isNameDuplicate = scriptDAO.isExistDuplicateName(targetAppId, script.getName()); - if (isNameDuplicate) { - log.warn("The script name:{} is exist for app:{}", script.getName(), targetAppId); - throw new AlreadyExistsException(ErrorCode.SCRIPT_NAME_DUPLICATE); - } - script.setId(JobUUID.getUUID()); - scriptDAO.saveScript(script); - scriptVersionId = scriptDAO.saveScriptVersion(script); - script.setScriptVersionId(scriptVersionId); - saveScriptTags(appId, script); - } + @ActionAuditRecord( + actionId = ActionId.VIEW_SCRIPT, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.SCRIPT + ), + content = EventContentConstants.VIEW_SCRIPT + ) + public ScriptDTO getScriptVersion(String username, long appId, Long scriptVersionId) { + ScriptDTO scriptVersion = getScriptVersion(appId, scriptVersionId); + authViewScript(username, appId, scriptVersion.getId()); - return scriptDAO.getScriptVersionById(scriptVersionId); - } + ActionAuditContext.current() + .setInstanceId(scriptVersion.getId()) + .setInstanceName(scriptVersion.getName()); - private void saveScriptTags(Long appId, ScriptDTO script) { - saveScriptTags(script.getLastModifyUser(), appId, script.getId(), script.getTags()); + return scriptVersion; } - private void saveScriptTags(String operator, Long appId, String scriptId, List tags) { - List newTags = tags; - if (tags != null && !tags.isEmpty()) { - newTags = tagService.createNewTagIfNotExist(tags, appId, operator); + @Override + public List listScriptVersion(long appId, String scriptId) { + ScriptDTO script = getScript(appId, scriptId); + if (script == null) { + throw new NotFoundException(ErrorCode.SCRIPT_NOT_EXIST); } + ActionAuditContext.current().setInstanceName(script.getName()); - Integer resourceType = appId == (JobConstants.PUBLIC_APP_ID) ? - JobResourceTypeEnum.PUBLIC_SCRIPT.getValue() : - JobResourceTypeEnum.APP_SCRIPT.getValue(); - - tagService.patchResourceTags(resourceType, scriptId, CollectionUtils.isEmpty(newTags) ? - Collections.emptyList() : newTags.stream().map(TagDTO::getId).collect(Collectors.toList())); + return scriptManager.listScriptVersion(appId, scriptId); } - private Long getTimeOrDefault(Long time) { - if (time == null) { - return DateUtils.currentTimeMillis(); - } - return time; + @Override + @ActionAuditRecord( + actionId = ActionId.MANAGE_SCRIPT, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.SCRIPT, + instanceIds = "#scriptVersion?.id", + instanceNames = "#$?.name" + ), + attributes = @AuditAttribute( + name = "@VERSION", value = "#scriptVersion?.version" + ), + content = EventContentConstants.CREATE_SCRIPT_VERSION + ) + public ScriptDTO createScriptVersion(String username, ScriptDTO scriptVersion) { + authCreateScript(username, scriptVersion.getAppId()); + return scriptManager.createScriptVersion(scriptVersion); } @Override - public Pair createScriptWithVersionId( - String operator, - Long appId, - ScriptDTO script, - Long createTime, - Long lastModifyTime - ) throws ServiceException { - log.info("Begin to createScriptWithVersionId, operator={}, appId={}, script={}, createTime={}, " + - "lastModifyTime={}", operator, appId, JsonUtils.toJson(script), createTime, lastModifyTime); - createTime = getTimeOrDefault(createTime); - lastModifyTime = getTimeOrDefault(lastModifyTime); - final long targetAppId = script.isPublicScript() ? JobConstants.PUBLIC_APP_ID : appId; - script.setAppId(targetAppId); - - // 默认为未上线状态 - Integer status = script.getStatus(); - if (status == null || status < 0) { - script.setStatus(JobResourceStatusEnum.DRAFT.getValue()); + @ActionAuditRecord( + actionId = ActionId.MANAGE_SCRIPT, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.SCRIPT + ), + content = EventContentConstants.EDIT_SCRIPT_VERSION + ) + public ScriptDTO updateScriptVersion(String username, ScriptDTO scriptVersion) { + ScriptDTO originScriptVersion = getScriptVersion(scriptVersion.getScriptVersionId()); + if (originScriptVersion == null) { + throw new NotFoundException(ErrorCode.SCRIPT_NOT_EXIST); } - if (script.getScriptVersionId() != null && script.getScriptVersionId() > 0) { - if (scriptDAO.isExistDuplicateScriptId(script.getScriptVersionId())) { - log.warn("scriptVersionId:{} is exist, scriptId:{}", script.getScriptVersionId(), script.getId()); - throw new AlreadyExistsException(ErrorCode.SCRIPT_VERSION_ID_EXIST); - } - } - long finalCreateTime = createTime; - long finalLastModifyTime = lastModifyTime; - - if (StringUtils.isNotBlank(script.getId())) { - if (scriptDAO.isExistDuplicateVersion(script.getId(), script.getVersion())) { - log.warn("Script version:{} is exist, scriptId:{}", script.getVersion(), script.getId()); - throw new AlreadyExistsException(ErrorCode.SCRIPT_VERSION_NAME_EXIST); - } - // 指定版本号新增 - // 不指定版本号新增 - dslContext.transaction(configuration -> { - DSLContext context = DSL.using(configuration); - if (!scriptDAO.isExistDuplicateScriptId(appId, script.getId())) { - //脚本不存在，新增脚本 - boolean isNameDuplicate = scriptDAO.isExistDuplicateName(targetAppId, script.getName()); - if (isNameDuplicate) { - log.warn("The script name:{} is exist for app:{}", script.getName(), targetAppId); - throw new AlreadyExistsException(ErrorCode.SCRIPT_NAME_DUPLICATE); - } - // 插入script - String scriptId = scriptDAO.saveScript(context, script, finalCreateTime, finalLastModifyTime); - log.info("script created with specified id:{}", scriptId); - } - // 插入script_version - Long scriptVersionId = scriptDAO.saveScriptVersion(context, script, finalCreateTime, - finalLastModifyTime); - // 更新script - scriptDAO.updateScript(context, script, finalLastModifyTime); - script.setScriptVersionId(scriptVersionId); - }); - } else { - //脚本不存在，新增脚本 - boolean isNameDuplicate = scriptDAO.isExistDuplicateName(targetAppId, script.getName()); - if (isNameDuplicate) { - log.warn("The script name:{} is exist for app:{}", script.getName(), targetAppId); - throw new AlreadyExistsException(ErrorCode.SCRIPT_NAME_DUPLICATE); - } - - script.setId(JobUUID.getUUID()); - dslContext.transaction(configuration -> { - DSLContext context = DSL.using(configuration); - // 插入script - scriptDAO.saveScript(context, script, finalCreateTime, finalLastModifyTime); - // 插入script_version - Long scriptVersionId = scriptDAO.saveScriptVersion(context, script, finalCreateTime, - finalLastModifyTime); - script.setScriptVersionId(scriptVersionId); - }); - } - saveScriptTags(appId, script); - return Pair.of(script.getId(), script.getScriptVersionId()); - } + authManageScript(username, scriptVersion.getAppId(), originScriptVersion.getId()); - @Override - public List listScriptRelatedTasks(String scriptId) throws ServiceException { - List tasks = scriptRelateTaskPlanDAO.listScriptRelatedTaskPlan(scriptId); - Map scriptVersionIdMap = new HashMap<>(); - String scriptName = ""; - if (tasks != null && !tasks.isEmpty()) { - List scriptVersions = scriptDAO.listScriptVersionsByScriptId(scriptId); - if (scriptVersions != null && !scriptVersions.isEmpty()) { - scriptName = scriptVersions.get(0).getName(); - scriptVersions.forEach(script -> scriptVersionIdMap.put(script.getScriptVersionId(), script)); - } - for (ScriptRelatedTaskPlanDTO task : tasks) { - ScriptDTO theScript = scriptVersionIdMap.get(task.getScriptVersionId()); - task.setScriptName(scriptName); - if (theScript != null) { - task.setScriptVersion(theScript.getVersion()); - task.setScriptStatus(JobResourceStatusEnum.getJobResourceStatus(theScript.getStatus())); - - } - } - } - return tasks; - } + ScriptDTO updateScriptVersion = scriptManager.updateScriptVersion(scriptVersion); - @Override - public List listScriptVersionRelatedTasks(String scriptId, Long scriptVersionId) { - List tasks = scriptRelateTaskPlanDAO.listScriptVersionRelatedTaskPlan(scriptId, - scriptVersionId); - if (tasks != null && !tasks.isEmpty()) { - ScriptDTO scriptVersion = scriptDAO.getScriptVersionById(scriptVersionId); - for (ScriptRelatedTaskPlanDTO task : tasks) { - task.setScriptVersion(scriptVersion.getVersion()); - task.setScriptName(scriptVersion.getName()); - task.setScriptStatus(JobResourceStatusEnum.getJobResourceStatus(scriptVersion.getStatus())); - } - } - return tasks; - } + // 审计 + ActionAuditContext.current() + .setInstanceId(scriptVersion.getId()) + .setInstanceName(originScriptVersion.getName()) + .setOriginInstance(originScriptVersion.toEsbScriptV3DTO()) + .addAttribute("@VERSION", originScriptVersion.toEsbScriptVersionDetailV3DTO()) + .setInstance(updateScriptVersion.toEsbScriptVersionDetailV3DTO()); - @Override - @Transactional(rollbackFor = {Exception.class, Error.class}) - public void deleteScriptVersion(String operator, Long appId, Long scriptVersionId) throws ServiceException { - ScriptDTO existScript = scriptDAO.getScriptVersionById(scriptVersionId); - checkDeleteScriptPermission(appId, existScript); - if (existScript.getStatus().equals(JobResourceStatusEnum.ONLINE.getValue())) { - log.warn("Fail to delete script version because script is online. scriptVersionId={}", scriptVersionId); - throw new FailedPreconditionException(ErrorCode.DELETE_ONLINE_SCRIPT_FAIL); - } - List scriptVersions = scriptDAO.listScriptVersionsByScriptId(existScript.getId()); - if (scriptVersions.size() == 1) { - scriptDAO.deleteScript(existScript.getId()); - scriptDAO.deleteScriptVersion(scriptVersionId); - deleteScriptRelatedTags(appId, existScript.getId()); - } else { - scriptDAO.deleteScriptVersion(scriptVersionId); - } + return updateScriptVersion; } @Override - @Transactional(rollbackFor = {Exception.class, Error.class}) - public void deleteScript(String operator, Long appId, String scriptId) throws ServiceException { - ScriptDTO existScript = getScript(operator, appId, scriptId); - checkDeleteScriptPermission(appId, existScript); - scriptDAO.deleteScript(scriptId); - scriptDAO.deleteScriptVersionByScriptId(scriptId); - deleteScriptRelatedTags(appId, scriptId); + @ActionAuditRecord( + actionId = ActionId.MANAGE_SCRIPT, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.SCRIPT + ), + content = EventContentConstants.DELETE_SCRIPT_VERSION + ) + public void deleteScriptVersion(String username, Long appId, Long scriptVersionId) { + preProcessManageScriptVersion(username, appId, scriptVersionId); + scriptManager.deleteScriptVersion(appId, scriptVersionId); } - private void deleteScriptRelatedTags(Long appId, String scriptId) { - Integer resourceType = (appId == JobConstants.PUBLIC_APP_ID) ? - JobResourceTypeEnum.PUBLIC_SCRIPT.getValue() : - JobResourceTypeEnum.APP_SCRIPT.getValue(); - tagService.batchDeleteResourceTags(appId, resourceType, scriptId); + private void preProcessManageScriptVersion(String username, Long appId, Long scriptVersionId) { + ScriptDTO scriptVersion = getScriptVersion(appId, scriptVersionId); + authManageScript(username, appId, scriptVersion.getId()); + addScriptVersionAuditInfo(scriptVersion); } - private void checkDeleteScriptPermission( - Long appId, - ScriptDTO existScript - ) throws ServiceException { - if (existScript == null) { - throw new NotFoundException(ErrorCode.SCRIPT_NOT_EXIST); - } - long targetAppId = appId; - if (existScript.isPublicScript()) { - targetAppId = JobConstants.PUBLIC_APP_ID; - } - if (!existScript.getAppId().equals(targetAppId)) { - throw new NotFoundException(ErrorCode.SCRIPT_NOT_IN_APP); - } + private void addScriptVersionAuditInfo(ScriptDTO scriptVersion) { + ActionAuditContext.current() + .setInstanceId(scriptVersion.getId()) + .setInstanceName(scriptVersion.getName()) + .addAttribute("@VERSION", scriptVersion.getVersion()); } @Override - public void publishScript( - Long appId, - String operator, - String scriptId, - Long scriptVersionId - ) throws ServiceException { - log.info("Begin to publish script, appId={}, operator={}, scriptId={}, scriptVersionId={}", appId, operator, - scriptId, scriptVersionId); - List scriptVersions = scriptDAO.listScriptVersionsByScriptId(scriptId); - if (scriptVersions == null || scriptVersions.isEmpty()) { - log.warn("Publish script, script:{} is not exist", scriptId); - throw new NotFoundException(ErrorCode.SCRIPT_NOT_EXIST); - } - - boolean isPublicScript = scriptVersions.get(0).isPublicScript(); - long targetAppId = appId; - if (isPublicScript) { - targetAppId = JobConstants.PUBLIC_APP_ID; - } - - boolean isScriptVersionInCurrentScript = false; - ScriptDTO scriptVersionToBePublished = null; - for (ScriptDTO scriptVersion : scriptVersions) { - if (!isPublicScript && !scriptVersion.getAppId().equals(targetAppId)) { - log.warn("Publish script, script:{} is not in current app:{}", scriptId, targetAppId); - throw new NotFoundException(ErrorCode.SCRIPT_NOT_IN_APP); - } - if (scriptVersion.getScriptVersionId().equals(scriptVersionId)) { - isScriptVersionInCurrentScript = true; - scriptVersionToBePublished = scriptVersion; - } - } - if (!isScriptVersionInCurrentScript) { - log.warn("Public script, scriptVersion:{} is not in script:{}", scriptVersionId, scriptId); - throw new NotFoundException(ErrorCode.SCRIPT_NOT_EXIST); - } + @ActionAuditRecord( + actionId = ActionId.MANAGE_SCRIPT, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.SCRIPT + ), + content = EventContentConstants.ONLINE_SCRIPT_VERSION + ) + public void publishScript(Long appId, String username, String scriptId, Long scriptVersionId) { + log.info("Publish script version, appId={}, scriptId={}, scriptVersionId={}, username={}", appId, + scriptId, scriptVersionId, username); + preProcessManageScriptVersion(username, appId, scriptVersionId); - if (scriptVersionToBePublished.getStatus().equals(JobResourceStatusEnum.DISABLED.getValue())) { - log.warn("Publish script, scriptVersion:{}, status:{} could not publish", scriptVersionId, - scriptVersionToBePublished.getStatus()); - throw new FailedPreconditionException(ErrorCode.UNSUPPORTED_OPERATION); - } - - ScriptDTO publishedScriptVersion = null; - for (ScriptDTO scriptVersion : scriptVersions) { - if (scriptVersion.getStatus().equals(JobResourceStatusEnum.ONLINE.getValue())) { - publishedScriptVersion = scriptVersion; - break; - } - } - if (publishedScriptVersion != null) { - log.info("Publish script, set scriptVersion:{} offline", publishedScriptVersion.getScriptVersionId()); - scriptDAO.updateScriptVersionStatus(publishedScriptVersion.getScriptVersionId(), - JobResourceStatusEnum.OFFLINE.getValue()); - taskTemplateService.updateScriptStatus(appId, scriptId, publishedScriptVersion.getScriptVersionId(), - JobResourceStatusEnum.OFFLINE); - } - scriptDAO.updateScriptVersionStatus(scriptVersionId, JobResourceStatusEnum.ONLINE.getValue()); - taskTemplateService.updateScriptStatus(appId, scriptId, scriptVersionId, JobResourceStatusEnum.ONLINE); - log.info("Publish script successfully, scriptId={}, scriptVersionId={}", scriptId, scriptVersionId); + scriptManager.publishScript(appId, scriptId, scriptVersionId); } @Override - public void disableScript( - Long appId, - String operator, - String scriptId, - Long scriptVersionId - ) throws ServiceException { - log.info("Begin to disable script, appId={}, operator={}, scriptId={}, scriptVersionId={}", appId, operator, - scriptId, scriptVersionId); - List scriptVersions = scriptDAO.listScriptVersionsByScriptId(scriptId); - if (scriptVersions == null || scriptVersions.isEmpty()) { - log.warn("Disable script, script:{} is not exist", scriptId); - throw new NotFoundException(ErrorCode.SCRIPT_NOT_EXIST); - } - - boolean isPublicScript = scriptVersions.get(0).isPublicScript(); - long targetAppId = appId; - if (isPublicScript) { - targetAppId = JobConstants.PUBLIC_APP_ID; - } - - boolean isScriptVersionInCurrentScript = false; - ScriptDTO scriptVersionToBeDisabled = null; - for (ScriptDTO scriptVersion : scriptVersions) { - if (!scriptVersion.getAppId().equals(targetAppId)) { - log.warn("Disable script, script:{} is not in current app:{}", scriptId, targetAppId); - throw new NotFoundException(ErrorCode.SCRIPT_NOT_IN_APP); - } - if (scriptVersion.getScriptVersionId().equals(scriptVersionId)) { - isScriptVersionInCurrentScript = true; - scriptVersionToBeDisabled = scriptVersion; - } - } - if (!isScriptVersionInCurrentScript) { - log.warn("Disable script, scriptVersion:{} is not in script:{}", scriptVersionId, scriptId); - throw new NotFoundException(ErrorCode.SCRIPT_NOT_EXIST); - } - - if (!scriptVersionToBeDisabled.getStatus().equals(JobResourceStatusEnum.ONLINE.getValue()) - && !scriptVersionToBeDisabled.getStatus().equals(JobResourceStatusEnum.OFFLINE.getValue())) { - log.warn("Disable script, scriptVersion:{}, status:{} could not disable", scriptVersionId, - scriptVersionToBeDisabled.getStatus()); - throw new FailedPreconditionException(ErrorCode.UNSUPPORTED_OPERATION); - } - - scriptDAO.updateScriptVersionStatus(scriptVersionToBeDisabled.getScriptVersionId(), - JobResourceStatusEnum.DISABLED.getValue()); - taskTemplateService.updateScriptStatus(appId, scriptId, scriptVersionToBeDisabled.getScriptVersionId(), - JobResourceStatusEnum.DISABLED); - log.info("Disable script successfully, scriptId={}, scriptVersionId={}", scriptId, scriptVersionId); + @ActionAuditRecord( + actionId = ActionId.MANAGE_SCRIPT, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.SCRIPT + ), + content = EventContentConstants.FORBIDDEN_SCRIPT_VERSION + ) + public void disableScript(Long appId, String username, String scriptId, Long scriptVersionId) { + log.info("Disable script version, appId={}, scriptId={}, scriptVersionId={}, username={}", appId, + scriptId, scriptVersionId, username); + preProcessManageScriptVersion(username, appId, scriptVersionId); + scriptManager.disableScript(appId, scriptId, scriptVersionId); } @Override public Map batchGetOnlineScriptVersionByScriptIds(List scriptIdList) { - Map scripts = scriptDAO.batchGetOnlineByScriptIds(scriptIdList); - setTags(scripts.values()); - return scripts; + return scriptManager.batchGetOnlineScriptVersionByScriptIds(scriptIdList); } @Override - public void updateScriptDesc(Long appId, String operator, String scriptId, String desc) throws ServiceException { - log.info("Begin to update script desc,appId={},operator={},scriptId={},desc={}", appId, operator, scriptId, - desc); - ScriptDTO script = scriptDAO.getScriptByScriptId(scriptId); - checkScriptInApp(appId, script); - scriptDAO.updateScriptDesc(operator, scriptId, desc); - } + @ActionAuditRecord( + actionId = ActionId.MANAGE_SCRIPT, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.SCRIPT + ), + content = EventContentConstants.EDIT_SCRIPT + ) + public ScriptDTO updateScriptDesc(Long appId, String username, String scriptId, String desc) { + authManageScript(username, appId, scriptId); - private void checkScriptInApp(long appId, ScriptDTO script) throws ServiceException { - if (script == null) { + ScriptDTO originScript = getScript(appId, scriptId); + if (originScript == null) { throw new NotFoundException(ErrorCode.SCRIPT_NOT_EXIST); } - if (!script.isPublicScript()) { - if (!script.getAppId().equals(appId)) { - log.warn("Script:{} is not in current app:{}", script.getId(), appId); - throw new NotFoundException(ErrorCode.SCRIPT_NOT_IN_APP); - } - } - } - @Override - public void updateScriptName( - Long appId, - String operator, - String scriptId, - String newName - ) throws ServiceException { - log.info("Begin to update script name,appId={},operator={},scriptId={},desc={}", appId, operator, scriptId, - newName); - ScriptDTO script = scriptDAO.getScriptByScriptId(scriptId); - checkScriptInApp(appId, script); - if (script.getName().equals(newName)) { - return; - } - long targetAppId = appId; - if (script.isPublicScript()) { - targetAppId = JobConstants.PUBLIC_APP_ID; - } - boolean isNameExist = scriptDAO.isExistDuplicateName(targetAppId, newName); - if (isNameExist) { - log.warn("Update script name, script:{} new name {} is duplicate", scriptId, newName); - throw new AlreadyExistsException(ErrorCode.SCRIPT_NAME_DUPLICATE); - } + ScriptDTO updateScript = scriptManager.updateScriptDesc(username, appId, scriptId, desc); - scriptDAO.updateScriptName(operator, scriptId, newName); - } + addModifyScriptAuditInfo(originScript, updateScript); - @Override - public void updateScriptTags(Long appId, String operator, String scriptId, List tags) { - log.info("Begin to update script tags,appId={},operator={},scriptId={},tags={}", appId, operator, scriptId, - tags); - ScriptDTO script = scriptDAO.getScriptByScriptId(scriptId); - checkScriptInApp(appId, script); - long targetAppId = appId; - if (script.isPublicScript()) { - targetAppId = JobConstants.PUBLIC_APP_ID; - } - saveScriptTags(operator, targetAppId, scriptId, tags); + return updateScript; } - @Override - public List listScriptNames(Long appId, String keyword) { - return scriptDAO.listScriptNames(appId, keyword); + private void addModifyScriptAuditInfo(ScriptDTO originScript, ScriptDTO updateScript) { + // 审计 + ActionAuditContext.current() + .setInstanceId(originScript.getId()) + .setInstanceName(originScript.getName()) + .setOriginInstance(originScript.toEsbScriptV3DTO()) + .setInstance(updateScript.toEsbScriptV3DTO()); } @Override - public List listOnlineScriptForApp(String operator, long appId) { - List scripts = scriptDAO.listOnlineScriptForApp(appId); - setTags(scripts); - return scripts; - } + @ActionAuditRecord( + actionId = ActionId.MANAGE_SCRIPT, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.SCRIPT + ), + content = EventContentConstants.EDIT_SCRIPT + ) + public ScriptDTO updateScriptName(Long appId, String username, String scriptId, String newName) { + authManageScript(username, appId, scriptId); - @Override - public PageData listPageOnlineScript(ScriptQuery scriptCondition, - BaseSearchCondition baseSearchCondition) throws ServiceException { - if (scriptCondition.isExistTagCondition()) { - List tagMatchedScriptIds = queryScriptIdsByTags(scriptCondition); - if (CollectionUtils.isEmpty(tagMatchedScriptIds)) { - // none match, return empty page data - return PageData.emptyPageData(baseSearchCondition.getStart(), baseSearchCondition.getLength()); - } else { - scriptCondition.setIds(tagMatchedScriptIds); - } + ScriptDTO originScript = getScript(appId, scriptId); + if (originScript == null) { + throw new NotFoundException(ErrorCode.SCRIPT_NOT_EXIST); } - return scriptDAO.listPageOnlineScript(scriptCondition, baseSearchCondition); - } - @Override - public ScriptDTO getOnlineScriptVersionByScriptId( - String operator, - long appId, - String scriptId - ) throws ServiceException { - return scriptDAO.getOnlineScriptVersionByScriptId(appId, scriptId); - } + ScriptDTO updateScript = scriptManager.updateScriptName(username, appId, scriptId, newName); - @Override - public ScriptDTO getOnlineScriptVersionByScriptId(String scriptId) { - return scriptDAO.getOnlineScriptVersionByScriptId(scriptId); - } + addModifyScriptAuditInfo(originScript, updateScript); - @Override - public PageData listPageScriptVersion(ScriptQuery scriptQuery, - BaseSearchCondition baseSearchCondition) { - return scriptDAO.listPageScriptVersion(scriptQuery, baseSearchCondition); + return updateScript; } @Override - public List listScriptSyncTemplateSteps(String username, Long appId, String scriptId) { - // 检查脚本 - ScriptDTO syncScript = scriptDAO.getScriptByScriptId(scriptId); - checkScriptInApp(appId, syncScript); - - List templateSteps = - scriptRelateJobTemplateDAO.listScriptRelatedJobTemplateSteps(scriptId); - if (CollectionUtils.isEmpty(templateSteps)) { - return templateSteps; - } - // 按业务过滤 - if (appId > 0) { - templateSteps = - templateSteps.stream().filter(step -> step.getAppId().equals(appId)).collect(Collectors.toList()); - if (CollectionUtils.isEmpty(templateSteps)) { - return templateSteps; - } - } + @ActionAuditRecord( + actionId = ActionId.MANAGE_SCRIPT, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.SCRIPT + ), + content = EventContentConstants.EDIT_SCRIPT + ) + public ScriptDTO updateScriptTags(Long appId, + String username, + String scriptId, + List tags) { + authManageScript(username, appId, scriptId); - Set scriptVersionIds = - templateSteps.stream().map(ScriptSyncTemplateStepDTO::getScriptVersionId).collect(Collectors.toSet()); - List scriptVersions = scriptDAO.batchGetScriptVersionsByIds(scriptVersionIds); - if (CollectionUtils.isEmpty(scriptVersions)) { - log.warn("Scripts are not exist, scriptVersionIds: {}", scriptVersionIds); - return Collections.emptyList(); + ScriptDTO originScript = getScript(appId, scriptId); + if (originScript == null) { + throw new NotFoundException(ErrorCode.SCRIPT_NOT_EXIST); } - Map scripts = new HashMap<>(); - scriptVersions.forEach(scriptVersion -> scripts.put(scriptVersion.getScriptVersionId(), scriptVersion)); - templateSteps.forEach(step -> fillScriptInfo(step, scripts)); - return templateSteps; - } - - private void fillScriptInfo(ScriptSyncTemplateStepDTO templateStep, Map scripts) { - ScriptDTO script = scripts.get(templateStep.getScriptVersionId()); - if (script != null) { - templateStep.setScriptName(script.getName()); - templateStep.setScriptVersion(script.getVersion()); - templateStep.setScriptStatus(script.getStatus()); - } - } + ScriptDTO updateScript = scriptManager.updateScriptTags(username, appId, scriptId, tags); - @Override - public List syncScriptToTaskTemplate(String username, Long appId, String scriptId, - Long syncScriptVersionId, - List templateStepIDs) - throws PermissionDeniedException { - if (CollectionUtils.isEmpty(templateStepIDs)) { - throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); - } + addModifyScriptAuditInfo(originScript, updateScript); - ScriptDTO syncScript = scriptDAO.getScriptVersionById(syncScriptVersionId); - checkScriptInApp(appId, syncScript); - if (!scriptId.equals(syncScript.getId())) { - log.warn("Script ID and scriptVersionId not match!"); - throw new NotFoundException(ErrorCode.SCRIPT_NOT_EXIST); - } - // 鉴权 - List authTemplateIds = new ArrayList<>(); - templateStepIDs.forEach(step -> { - TaskTemplateInfoDTO taskTemplate = taskTemplateService.getTaskTemplateBasicInfoById(step.getTemplateId()); - if (appId > 0 && taskTemplate != null && !taskTemplate.getAppId().equals(appId)) { - throw new NotFoundException(ErrorCode.TEMPLATE_NOT_EXIST); - } - if (taskTemplate != null) { - step.setAppId(taskTemplate.getAppId()); - if (!authTemplateIds.contains(step.getTemplateId())) { - authTemplateIds.add(step.getTemplateId()); - } - } - }); - AuthResult authResult = templateAuthService.batchAuthResultEditJobTemplate( - username, - new AppResourceScope(appId), - authTemplateIds - ); - if (!authResult.isPass()) { - log.info("Sync script to template auth fail, scriptId: {}, syncScriptVersionId: {}, template steps: {}", - scriptId, syncScriptVersionId, templateStepIDs); - throw new PermissionDeniedException(authResult); - } - - List syncResults = new ArrayList<>(); - templateStepIDs.forEach(templateStep -> { - Long templateId = templateStep.getTemplateId(); - Long stepId = templateStep.getStepId(); - ScriptSyncTemplateStepDTO updateStep = new ScriptSyncTemplateStepDTO(); - updateStep.setTemplateId(templateId); - updateStep.setStepId(stepId); - try { - boolean success = updateTemplateRefScript(appId, templateId, stepId, syncScriptVersionId); - if (success) { - syncResults.add(SyncScriptResultDTO.buildSuccessSyncResult(updateStep)); - refreshTemplateScriptVersionStatus(templateId); - } else { - syncResults.add(SyncScriptResultDTO.buildFailSyncResult(updateStep, ErrorCode.TEMPLATE_NOT_EXIST)); - } - } catch (Throwable e) { - log.warn("Update script step ref script fail, templateId: {}, stepId: {}, scriptVersionId: {}", - templateId, stepId, syncScriptVersionId); - syncResults.add(SyncScriptResultDTO.buildFailSyncResult(updateStep, ErrorCode.INTERNAL_ERROR)); - } - }); - - List scriptRelatedTemplateSteps = listScriptSyncTemplateSteps(username, appId, - scriptId); - Map stepMap = new HashMap<>(); - scriptRelatedTemplateSteps.forEach(step -> stepMap.put(step.getStepId(), step)); - fillSyncResultDetail(syncScriptVersionId, syncResults, stepMap); - - return syncResults; + return updateScript; } @Override - public Integer getScriptTemplateCiteCount(String username, Long appId, String scriptId, Long scriptVersionId) { - if (scriptVersionId == null) { - return scriptCitedTaskTemplateDAO.countScriptCitedTaskTemplate(scriptId); - } else { - return scriptCitedTaskTemplateDAO.countScriptVersionCitedTaskTemplate(scriptId, scriptVersionId); - } + public List listScriptNames(Long appId, String keyword) { + return scriptManager.listScriptNames(appId, keyword); } @Override - public Integer getScriptTaskPlanCiteCount(String username, Long appId, String scriptId, Long scriptVersionId) { - if (scriptVersionId == null) { - return scriptRelateTaskPlanDAO.countScriptRelatedTaskPlan(scriptId); - } else { - return scriptRelateTaskPlanDAO.countScriptVersionRelatedTaskPlan(scriptId, scriptVersionId); - } + public List listOnlineScript(String operator, long appId) { + return scriptManager.listOnlineScriptForApp(appId); } @Override - public List getScriptCitedTemplates(String username, Long appId, String scriptId, - Long scriptVersionId) { - List scriptCitedTaskTemplateDTOList; - if (scriptVersionId == null) { - scriptCitedTaskTemplateDTOList = scriptCitedTaskTemplateDAO.listScriptCitedTaskTemplate(scriptId); - } else { - scriptCitedTaskTemplateDTOList = scriptCitedTaskTemplateDAO.listScriptVersionCitedTaskTemplate(scriptId, - scriptVersionId); - } - //填充scriptStatusDesc - scriptCitedTaskTemplateDTOList.forEach(scriptCitedTaskTemplateDTO -> - scriptCitedTaskTemplateDTO.setScriptStatusDesc(i18nService.getI18n( - scriptCitedTaskTemplateDTO.getScriptStatus().getStatusI18nKey() - ) - )); - return scriptCitedTaskTemplateDTOList; + @ActionAuditRecord( + actionId = ActionId.VIEW_SCRIPT, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.SCRIPT, + instanceIds = "#scriptId", + instanceNames = "#$?.name" + ), + content = EventContentConstants.VIEW_SCRIPT + ) + public ScriptDTO getOnlineScriptVersionByScriptId(String username, long appId, String scriptId) { + authViewScript(username, appId, scriptId); + return scriptManager.getOnlineScriptVersionByScriptId(appId, scriptId); } @Override - public List getScriptCitedTaskPlans(String username, Long appId, String scriptId, - Long scriptVersionId) { - List scriptRelatedTaskPlanDTOList; - if (scriptVersionId == null) { - scriptRelatedTaskPlanDTOList = scriptRelateTaskPlanDAO.listScriptRelatedTaskPlan(scriptId); - } else { - scriptRelatedTaskPlanDTOList = scriptRelateTaskPlanDAO.listScriptVersionRelatedTaskPlan(scriptId, - scriptVersionId); - } - List scriptCitedTaskPlanDTOList = new ArrayList<>(); - if (scriptRelatedTaskPlanDTOList != null) { - scriptRelatedTaskPlanDTOList.forEach(scriptRelatedTaskPlanDTO -> { - ScriptCitedTaskPlanDTO scriptCitedTaskPlanDTO = new ScriptCitedTaskPlanDTO(); - scriptCitedTaskPlanDTO.setAppId(scriptRelatedTaskPlanDTO.getAppId()); - scriptCitedTaskPlanDTO.setScriptStatus(scriptRelatedTaskPlanDTO.getScriptStatus().getValue()); - scriptCitedTaskPlanDTO.setScriptVersion(scriptRelatedTaskPlanDTO.getScriptVersion()); - scriptCitedTaskPlanDTO.setScriptStatusDesc( - i18nService.getI18n( - scriptRelatedTaskPlanDTO.getScriptStatus().getStatusI18nKey() - ) - ); - scriptCitedTaskPlanDTO.setTaskTemplateId(scriptRelatedTaskPlanDTO.getTemplateId()); - scriptCitedTaskPlanDTO.setTaskPlanId(scriptRelatedTaskPlanDTO.getTaskId()); - scriptCitedTaskPlanDTO.setTaskPlanName(scriptRelatedTaskPlanDTO.getTaskName()); - scriptCitedTaskPlanDTOList.add(scriptCitedTaskPlanDTO); - }); - } - return scriptCitedTaskPlanDTOList; - } - - private void fillSyncResultDetail(Long syncScriptVersionId, List syncResults, - Map stepMap) { - for (SyncScriptResultDTO syncResult : syncResults) { - ScriptSyncTemplateStepDTO step = syncResult.getTemplateStep(); - ScriptSyncTemplateStepDTO updatedStep = stepMap.get(step.getStepId()); - if (updatedStep == null) { - log.warn("Step is not exist, stepId: {}", step.getStepId()); - syncResult.setErrorCode(ErrorCode.TEMPLATE_NOT_EXIST); - continue; - } - if (!updatedStep.getScriptVersionId().equals(syncScriptVersionId) - && syncResult.getErrorCode() == null) { - syncResult.setErrorCode(ErrorCode.SYNC_SCRIPT_UNKNOWN_ERROR); - } - step.setTemplateName(updatedStep.getTemplateName()); - step.setStepName(updatedStep.getStepName()); - step.setScriptStatus(updatedStep.getScriptStatus()); - step.setScriptVersion(updatedStep.getScriptVersion()); - step.setScriptVersionId(updatedStep.getScriptVersionId()); - step.setScriptId(updatedStep.getScriptId()); - step.setScriptName(updatedStep.getScriptName()); - } - } - - @Transactional(rollbackFor = {Throwable.class, Error.class}) - public boolean updateTemplateRefScript(long appId, long templateId, long stepId, long syncScriptVersionId) { - boolean success = taskScriptStepDAO.updateScriptStepRefScriptVersionId( - templateId, - stepId, - syncScriptVersionId - ); - taskTemplateDAO.updateTaskTemplateVersion(appId, templateId, UUID.randomUUID().toString()); - return success; + public PageData listPageScriptVersion(ScriptQuery scriptQuery) { + return scriptManager.listPageScriptVersion(scriptQuery); } - /* - * 刷新模板引用脚本状态 - */ - private void refreshTemplateScriptVersionStatus(Long templateId) { - int retry = 3; - while (retry-- > 0) { - try { - templateStatusUpdateService.offerMessage(templateId); - return; - } catch (InterruptedException e) { - log.error("Refresh template script version status", e); - } - } + @Override + public List syncScriptToTaskTemplate(String username, + Long appId, + String scriptId, + Long syncScriptVersionId, + List templateStepIDs) { + return scriptManager.syncScriptToTaskTemplate(username, appId, scriptId, syncScriptVersionId, templateStepIDs); } @Override - public boolean isExistAnyAppScript(Long appId) { - return scriptDAO.isExistAnyScript(appId); + public boolean isExistAnyAppScript(long appId) { + return scriptManager.isExistAnyScript(appId); } @Override - public boolean isExistAnyPublicScript() { - return scriptDAO.isExistAnyPublicScript(); + public TagCountVO getTagScriptCount(Long appId) { + return scriptManager.getTagScriptCount(appId); } @Override - public Integer countScripts(Long appId, ScriptTypeEnum scriptTypeEnum, - JobResourceStatusEnum jobResourceStatusEnum) { - return scriptDAO.countScripts(appId, scriptTypeEnum, jobResourceStatusEnum); + @ActionAuditRecord( + actionId = ActionId.VIEW_SCRIPT, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.SCRIPT, + instanceIds = "#scriptId", + instanceNames = "#$?.name" + ), + content = EventContentConstants.VIEW_SCRIPT + ) + public ScriptDTO getByScriptIdAndVersion(String username, Long appId, String scriptId, String version) { + return scriptManager.getByScriptIdAndVersion(appId, scriptId, version); } @Override - public Integer countScriptVersions(Long appId, ScriptTypeEnum scriptTypeEnum, - JobResourceStatusEnum jobResourceStatusEnum) { - return scriptDAO.countScriptVersions(appId, scriptTypeEnum, jobResourceStatusEnum); + public ScriptDTO getScriptByScriptId(String scriptId) { + return scriptManager.getScriptByScriptId(scriptId); } - @Override - public List listScriptIds(Long appId) { - return scriptDAO.listAppScriptIds(appId); + private void authViewScript(String username, long appId, String scriptId) { + scriptAuthService.authViewScript(username, new AppResourceScope(appId), scriptId, null) + .denyIfNoPermission(); } - @Override - public Integer countCiteScripts(Long appId) { - // 1.查询业务下所有脚本 - List scriptIdList = scriptDAO.listAppScriptIds(appId); - // 2.查询被引用的脚本数量 - return taskScriptStepDAO.countScriptCitedByStepsByScriptIds(appId, scriptIdList); + private void authCreateScript(String username, long appId) { + scriptAuthService.authCreateScript(username, new AppResourceScope(appId)).denyIfNoPermission(); } - public void setTaskTemplateService(TaskTemplateService taskTemplateService) { - this.taskTemplateService = taskTemplateService; + private void authManageScript(String username, long appId, String scriptId) { + scriptAuthService.authManageScript(username, new AppResourceScope(appId), scriptId, null) + .denyIfNoPermission(); } @Override - public TagCountVO getTagScriptCount(Long appId) { - TagCountVO tagCount = new TagCountVO(); - - List appScriptIds = scriptDAO.listAppScriptIds(appId) - .stream().map(String::valueOf).collect(Collectors.toList()); - tagCount.setTotal((long) appScriptIds.size()); - - Integer resourceType = JobConstants.PUBLIC_APP_ID == appId ? JobResourceTypeEnum.PUBLIC_SCRIPT.getValue() : - JobResourceTypeEnum.APP_SCRIPT.getValue(); - List tags = tagService.listResourceTagsByResourceTypeAndResourceIds(appId, - resourceType, appScriptIds); - Map scriptTagCount = tagService.countResourcesByTag(tags); - tagCount.setTagCount(scriptTagCount); - - long taggedScriptCount = tags.stream() - .map(ResourceTagDTO::getResourceId).distinct().count(); - tagCount.setUnclassified(appScriptIds.size() - taggedScriptCount); - return tagCount; + public Map batchGetScriptVersionStatus(Collection scriptVersionId) { + return null; } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/ServiceInfoService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/ServiceInfoService.java index 3eb6ec6864..855b6dc9f7 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/ServiceInfoService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/ServiceInfoService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,7 @@ package com.tencent.bk.job.manage.service.impl; +import com.tencent.bk.job.common.constant.ProfileEnum; import com.tencent.bk.job.common.discovery.ServiceInfoProvider; import com.tencent.bk.job.common.discovery.model.ServiceInstanceInfoDTO; import com.tencent.bk.job.common.util.CompareUtil; @@ -31,6 +32,7 @@ import com.tencent.bk.job.manage.model.web.vo.serviceinfo.ServiceInstanceInfoVO; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; import java.util.ArrayList; @@ -40,9 +42,10 @@ @Slf4j @Service +@Profile("!" + ProfileEnum.Constants.TEST) public class ServiceInfoService { - ServiceInfoProvider serviceInfoProvider; + private final ServiceInfoProvider serviceInfoProvider; @Autowired public ServiceInfoService(ServiceInfoProvider serviceInfoProvider) { diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/TagServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/TagServiceImpl.java index c4c8fd8b7c..7dd34b0d15 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/TagServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/TagServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,19 +24,28 @@ package com.tencent.bk.job.manage.service.impl; +import com.tencent.bk.audit.annotations.ActionAuditRecord; +import com.tencent.bk.audit.annotations.AuditInstanceRecord; +import com.tencent.bk.audit.context.ActionAuditContext; +import com.tencent.bk.job.common.audit.constants.EventContentConstants; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.constant.JobConstants; import com.tencent.bk.job.common.exception.AlreadyExistsException; import com.tencent.bk.job.common.exception.InternalException; import com.tencent.bk.job.common.exception.InvalidParamException; +import com.tencent.bk.job.common.iam.constant.ActionId; +import com.tencent.bk.job.common.iam.constant.ResourceTypeId; import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; +import com.tencent.bk.job.common.model.dto.AppResourceScope; +import com.tencent.bk.job.common.mysql.JobTransactional; import com.tencent.bk.job.common.util.check.MaxLengthChecker; import com.tencent.bk.job.common.util.check.NotEmptyChecker; import com.tencent.bk.job.common.util.check.StringCheckHelper; import com.tencent.bk.job.common.util.check.TrimChecker; import com.tencent.bk.job.common.util.check.WhiteCharChecker; import com.tencent.bk.job.common.util.check.exception.StringCheckException; +import com.tencent.bk.job.manage.auth.TagAuthService; import com.tencent.bk.job.manage.dao.ResourceTagDAO; import com.tencent.bk.job.manage.dao.TagDAO; import com.tencent.bk.job.manage.model.dto.ResourceTagDTO; @@ -47,7 +56,6 @@ import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; import org.springframework.util.StopWatch; import java.util.ArrayList; @@ -64,11 +72,15 @@ public class TagServiceImpl implements TagService { private final TagDAO tagDAO; private final ResourceTagDAO resourceTagDAO; + private final TagAuthService tagAuthService; @Autowired - public TagServiceImpl(TagDAO tagDAO, ResourceTagDAO resourceTagDAO) { + public TagServiceImpl(TagDAO tagDAO, + ResourceTagDAO resourceTagDAO, + TagAuthService tagAuthService) { this.tagDAO = tagDAO; this.resourceTagDAO = resourceTagDAO; + this.tagAuthService = tagAuthService; } @Override @@ -106,7 +118,18 @@ public List listTagsByAppIdAndTagIdList(Long appId, List tagIdList } @Override - public Long insertNewTag(String username, TagDTO tag) { + @ActionAuditRecord( + actionId = ActionId.CREATE_TAG, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.TAG, + instanceIds = "#$?.id", + instanceNames = "#tag?.name" + ), + content = EventContentConstants.CREATE_TAG + ) + public TagDTO createTag(String username, TagDTO tag) { + checkCreateTagPermission(username, tag.getAppId()); + tag.setCreator(username); tag.setLastModifyUser(username); checkRequiredParam(tag); @@ -115,24 +138,53 @@ public Long insertNewTag(String username, TagDTO tag) { if (isTagExist) { throw new AlreadyExistsException(ErrorCode.TAG_ALREADY_EXIST); } - return tagDAO.insertTag(tag); + tag.setId(tagDAO.insertTag(tag)); + + tagAuthService.registerTag(tag.getId(), tag.getName(), username); + + return tag; } @Override + @ActionAuditRecord( + actionId = ActionId.MANAGE_TAG, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.TAG, + instanceIds = "#tag?.id", + instanceNames = "#$?.name" + ), + content = EventContentConstants.EDIT_TAG + ) public boolean updateTagById(String username, TagDTO tag) { - if (tag.getId() == null || tag.getId() <= 0) { - throw new InternalException(ErrorCode.ILLEGAL_PARAM); - } - tag.setAppId(tag.getAppId()); - tag.setId(tag.getId()); + checkManageTagPermission(username, tag.getAppId(), tag.getId()); + tag.setLastModifyUser(username); checkRequiredParam(tag); + TagDTO originTag = getTagInfoById(tag.getId()); + boolean isTagNameValid = checkTagName(tag.getAppId(), tag.getId(), tag.getName()); if (!isTagNameValid) { throw new AlreadyExistsException(ErrorCode.TAG_ALREADY_EXIST); } - return tagDAO.updateTagById(tag); + + boolean result = tagDAO.updateTagById(tag); + + // 审计 - 当前数据 + ActionAuditContext.current() + .setInstanceId(String.valueOf(tag.getId())) + .setInstanceName(originTag.getName()) + .setOriginInstance(TagDTO.toEsbTagV3DTO(originTag)) + .setInstance(TagDTO.toEsbTagV3DTO(tag)); + return result; + } + + private void checkManageTagPermission(String username, long appId, Long tagId) { + tagAuthService.authManageTag(username, new AppResourceScope(appId), tagId, null).denyIfNoPermission(); + } + + private void checkCreateTagPermission(String username, long appId) { + tagAuthService.authCreateTag(username, new AppResourceScope(appId)).denyIfNoPermission(); } private void checkRequiredParam(TagDTO tag) { @@ -176,6 +228,11 @@ public List createNewTagIfNotExist(List tags, Long appId, String if (tagIdList.contains(tag.getId())) { tagIterator.remove(); } else { + TagDTO existTag = getTagInfoById(appId, tag.getId()); + if (existTag == null) { + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME_AND_REASON, + new String[]{"tagId", String.format("tag (id=%s, app_id=%s) not exist", tag.getId(), appId)}); + } tagIdList.add(tag.getId()); } } @@ -196,8 +253,21 @@ public PageData listPageTags(TagDTO tagQuery, BaseSearchCondition baseSe } @Override - @Transactional - public void deleteTag(Long tagId) { + @JobTransactional(transactionManager = "jobManageTransactionManager") + @ActionAuditRecord( + actionId = ActionId.MANAGE_TAG, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.TAG, + instanceIds = "#tagId" + ), + content = EventContentConstants.DELETE_TAG + ) + public void deleteTag(String username, long appId, Long tagId) { + checkManageTagPermission(username, appId, tagId); + + TagDTO tag = getTagInfoById(tagId); + ActionAuditContext.current().setInstanceName(tag.getName()); + tagDAO.deleteTagById(tagId); resourceTagDAO.deleteResourceTags(tagId); } @@ -227,11 +297,14 @@ public List listResourceTagsByTagIds(Long appId, List tagI } private void checkTags(Long appId, List tags) { - tags.forEach(tag -> { + Iterator iterator = tags.iterator(); + while (iterator.hasNext()) { + TagDTO tag = iterator.next(); if (!tag.getAppId().equals(appId) && !tag.getAppId().equals(JobConstants.PUBLIC_APP_ID)) { - throw new InternalException("Tag is not exist", ErrorCode.INTERNAL_ERROR); + log.info("Tag is not exist, appId={}, tagId={}", appId, tag.getId()); + iterator.remove(); } - }); + } } @Override @@ -287,7 +360,7 @@ public boolean batchSaveResourceTags(List resourceTags) { } @Override - @Transactional(rollbackFor = {Exception.class, Error.class}) + @JobTransactional(transactionManager = "jobManageTransactionManager") public void batchPatchResourceTags(List addResourceTags, List deleteResourceTags) { StopWatch watch = new StopWatch("batchPatchResourceTags"); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/WatchableSendMsgService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/WatchableSendMsgService.java index 0bc97942a2..cbbdc0c16f 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/WatchableSendMsgService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/WatchableSendMsgService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,8 +25,8 @@ package com.tencent.bk.job.manage.service.impl; import com.tencent.bk.job.common.esb.metrics.EsbApiTimed; +import com.tencent.bk.job.common.paas.cmsi.CmsiApiClient; import com.tencent.bk.job.manage.metrics.MetricsConstants; -import com.tencent.bk.job.manage.service.PaaSService; import io.micrometer.core.instrument.MeterRegistry; import io.micrometer.core.instrument.Tag; import io.micrometer.core.instrument.Tags; @@ -43,13 +43,13 @@ @Service public class WatchableSendMsgService { - private final PaaSService paaSService; + private final CmsiApiClient cmsiApiClient; private final MeterRegistry meterRegistry; @Autowired - public WatchableSendMsgService(PaaSService paaSService, + public WatchableSendMsgService(CmsiApiClient cmsiApiClient, MeterRegistry meterRegistry) { - this.paaSService = paaSService; + this.cmsiApiClient = cmsiApiClient; this.meterRegistry = meterRegistry; } @@ -65,7 +65,7 @@ public void sendMsg( ) { String sendStatus = MetricsConstants.TAG_VALUE_SEND_STATUS_FAILED; try { - paaSService.sendMsg(msgType, sender, receivers, title, content); + cmsiApiClient.sendMsg(msgType, sender, receivers, title, content); sendStatus = MetricsConstants.TAG_VALUE_SEND_STATUS_SUCCESS; } finally { long delayMillis = System.currentTimeMillis() - createTimeMillis; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/WhiteIPServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/WhiteIPServiceImpl.java index fb0cf25a52..725955df98 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/WhiteIPServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/WhiteIPServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,54 +25,54 @@ package com.tencent.bk.job.manage.service.impl; import com.google.common.collect.Lists; -import com.tencent.bk.job.common.RequestIdLogger; +import com.tencent.bk.audit.annotations.ActionAuditRecord; +import com.tencent.bk.job.common.audit.constants.EventContentConstants; import com.tencent.bk.job.common.cc.model.CcCloudAreaInfoDTO; import com.tencent.bk.job.common.cc.sdk.CmdbClientFactory; +import com.tencent.bk.job.common.cc.sdk.IBizCmdbClient; import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.constant.JobConstants; import com.tencent.bk.job.common.exception.InternalException; import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.i18n.service.MessageI18nService; +import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; import com.tencent.bk.job.common.model.dto.ApplicationDTO; +import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; +import com.tencent.bk.job.common.model.dto.HostDTO; import com.tencent.bk.job.common.model.dto.ResourceScope; import com.tencent.bk.job.common.model.vo.CloudAreaInfoVO; import com.tencent.bk.job.common.service.AppScopeMappingService; -import com.tencent.bk.job.common.util.ArrayUtil; import com.tencent.bk.job.common.util.JobContextUtil; import com.tencent.bk.job.common.util.LogUtil; -import com.tencent.bk.job.common.util.SimpleRequestIdLogger; import com.tencent.bk.job.common.util.StringUtil; -import com.tencent.bk.job.common.util.ip.IpUtils; import com.tencent.bk.job.common.util.json.JsonUtils; -import com.tencent.bk.job.manage.common.consts.whiteip.ActionScopeEnum; +import com.tencent.bk.job.manage.api.common.constants.whiteip.ActionScopeEnum; import com.tencent.bk.job.manage.dao.ApplicationDAO; +import com.tencent.bk.job.manage.dao.ApplicationHostDAO; import com.tencent.bk.job.manage.dao.whiteip.ActionScopeDAO; import com.tencent.bk.job.manage.dao.whiteip.WhiteIPRecordDAO; import com.tencent.bk.job.manage.model.dto.whiteip.ActionScopeDTO; -import com.tencent.bk.job.manage.model.dto.whiteip.CloudIPDTO; import com.tencent.bk.job.manage.model.dto.whiteip.WhiteIPActionScopeDTO; import com.tencent.bk.job.manage.model.dto.whiteip.WhiteIPIPDTO; import com.tencent.bk.job.manage.model.dto.whiteip.WhiteIPRecordDTO; import com.tencent.bk.job.manage.model.inner.ServiceWhiteIPInfo; +import com.tencent.bk.job.manage.model.web.request.chooser.host.HostIdWithMeta; import com.tencent.bk.job.manage.model.web.request.whiteip.WhiteIPRecordCreateUpdateReq; -import com.tencent.bk.job.manage.model.web.vo.AppVO; import com.tencent.bk.job.manage.model.web.vo.whiteip.ActionScopeVO; import com.tencent.bk.job.manage.model.web.vo.whiteip.WhiteIPRecordVO; -import com.tencent.bk.job.manage.service.ApplicationService; import com.tencent.bk.job.manage.service.WhiteIPService; import lombok.extern.slf4j.Slf4j; import lombok.val; import lombok.var; import org.apache.commons.collections4.CollectionUtils; -import org.jooq.DSLContext; -import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; import java.util.ArrayList; -import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; import java.util.HashMap; import java.util.HashSet; import java.util.List; @@ -83,35 +83,49 @@ @Slf4j @Service public class WhiteIPServiceImpl implements WhiteIPService { - - private static final RequestIdLogger LOG = - new SimpleRequestIdLogger(LoggerFactory.getLogger(WhiteIPServiceImpl.class)); private static final String SEPERATOR_COMMA = ","; - private static final String SEPERATOR_ENTER = "\n"; private final MessageI18nService i18nService; - private final DSLContext dslContext; private final ActionScopeDAO actionScopeDAO; private final WhiteIPRecordDAO whiteIPRecordDAO; private final ApplicationDAO applicationDAO; - private final ApplicationService applicationService; private final AppScopeMappingService appScopeMappingService; + private final ApplicationHostDAO applicationHostDAO; + private final IBizCmdbClient bizCmdbClient; @Autowired public WhiteIPServiceImpl( - @Qualifier("job-manage-dsl-context") DSLContext dslContext, ActionScopeDAO actionScopeDAO, WhiteIPRecordDAO whiteIPRecordDAO, ApplicationDAO applicationDAO, - ApplicationService applicationService, MessageI18nService i18nService, - AppScopeMappingService appScopeMappingService) { - this.dslContext = dslContext; + AppScopeMappingService appScopeMappingService, + ApplicationHostDAO applicationHostDAO, + IBizCmdbClient bizCmdbClient) { this.actionScopeDAO = actionScopeDAO; this.whiteIPRecordDAO = whiteIPRecordDAO; this.applicationDAO = applicationDAO; this.i18nService = i18nService; - this.applicationService = applicationService; this.appScopeMappingService = appScopeMappingService; + this.applicationHostDAO = applicationHostDAO; + this.bizCmdbClient = bizCmdbClient; + } + + /** + * 获取设置白名单IP后可对当前业务生效的appId列表 + * + * @param appId Job业务Id + * @return 设置白名单IP后可对当前业务生效的appId列表 + */ + private List getEffectiveAppIdList(Long appId) { + List effectiveAppIds = new ArrayList<>(); + effectiveAppIds.add(JobConstants.DEFAULT_ALL_BIZ_SET_ID); + effectiveAppIds.add(JobConstants.PUBLIC_APP_ID); + if (appId != null + && !appId.equals(JobConstants.DEFAULT_ALL_BIZ_SET_ID) + && !appId.equals(JobConstants.PUBLIC_APP_ID)) { + effectiveAppIds.add(appId); + } + return effectiveAppIds; } @Override @@ -128,7 +142,7 @@ public PageData listWhiteIPRecord( String orderField, Integer order ) { - LOG.infoWithRequestId( + log.info( LogUtil.getInputLog( username, ipStr, appIdStr, actionScopeIdStr, lastModifyUser, start, @@ -156,7 +170,6 @@ public PageData listWhiteIPRecord( baseSearchCondition.setOrder(order); val whiteIPRecordList = whiteIPRecordDAO.listWhiteIPRecord( - dslContext, ipList, appIdList, appNameList, @@ -166,7 +179,6 @@ public PageData listWhiteIPRecord( baseSearchCondition ); val total = whiteIPRecordDAO.countWhiteIPRecord( - dslContext, ipList, appIdList, appNameList, @@ -182,7 +194,7 @@ public PageData listWhiteIPRecord( total, whiteIPRecordList ); - LOG.infoWithRequestId( + log.info( "Output:whiteIPRecordPageData(" + start + "," + pageSize @@ -192,100 +204,181 @@ public PageData listWhiteIPRecord( return whiteIPRecordPageData; } - @Override - public List listWhiteIP(Long appId, ActionScopeEnum actionScope) { - List fullAppIds = applicationService.getRelatedAppIds(appId); - log.info("appId={}, contains by fullAppIds={}", appId, fullAppIds); - ActionScopeDTO actionScopeDTO = null; - if (actionScope != null) { - actionScopeDTO = actionScopeDAO.getActionScopeByCode(actionScope.name()); + private Long getActionScopeId(ActionScopeEnum actionScope) { + if (actionScope == null) { + return null; } - return whiteIPRecordDAO.listWhiteIPByAppIds( - dslContext, - fullAppIds, - actionScopeDTO == null ? null : actionScopeDTO.getId() - ); + ActionScopeDTO actionScopeDTO = actionScopeDAO.getActionScopeByCode(actionScope.name()); + if (actionScopeDTO == null) { + log.error("cannot find action scope in db by enum {}, please check init data in db", actionScope); + return JobConstants.UNAVAILABLE_ACTION_SCOPE_ID; + } + return actionScopeDTO.getId(); } - private List checkReqAndGetIpList(WhiteIPRecordCreateUpdateReq createUpdateReq) { - List scopeList = createUpdateReq.getScopeList(); - if (null == scopeList || scopeList.isEmpty()) { - log.warn("scopeList cannot be null or empty"); - throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME_AND_REASON, - ArrayUtil.toArray("scopeList", "scopeList cannot be null or empty")); - } - Long cloudAreaId = createUpdateReq.getCloudAreaId(); - if (null == cloudAreaId) { - log.warn("cloudAreaId cannot be null"); - throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME_AND_REASON, - ArrayUtil.toArray("cloudAreaId", "cloudAreaId cannot be null")); + @Override + public List listAvailableWhiteIPHost(Long appId, ActionScopeEnum actionScope, Collection hostIds) { + List effectiveAppIds = getEffectiveAppIdList(appId); + return new ArrayList<>(new HashSet<>(whiteIPRecordDAO.listWhiteIPHost( + effectiveAppIds, + getActionScopeId(actionScope), + hostIds + ))); + } + + @Override + public List listAvailableWhiteIPHostByIps(Long appId, + ActionScopeEnum actionScope, + Collection ips) { + List effectiveAppIds = getEffectiveAppIdList(appId); + return new ArrayList<>(new HashSet<>(whiteIPRecordDAO.listWhiteIPHostByIps( + effectiveAppIds, + getActionScopeId(actionScope), + ips + ))); + } + + @Override + public List listAvailableWhiteIPHostByIpv6s(Long appId, + ActionScopeEnum actionScope, + Collection ipv6s) { + List effectiveAppIds = getEffectiveAppIdList(appId); + return new ArrayList<>(new HashSet<>(whiteIPRecordDAO.listWhiteIPHostByIpv6s( + effectiveAppIds, + getActionScopeId(actionScope), + ipv6s + ))); + } + + private List buildIpDtoList(List hostList, + String username) { + List finalIpDtoList = new ArrayList<>(); + finalIpDtoList.addAll(buildHostDtoList(hostList, username)); + return finalIpDtoList; + } + + private List buildHostDtoList(List hostList, + String username) { + if (CollectionUtils.isEmpty(hostList)) { + return Collections.emptyList(); } - String remark = createUpdateReq.getRemark(); - if (null != remark && remark.length() > 100) { - log.warn("remark cannot be null and length cannot be over 100"); - throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME_AND_REASON, - ArrayUtil.toArray("remark", "remark cannot be null and length cannot be over 100")); + // 从本地DB查询 + List hostIdList = hostList.stream().map(HostIdWithMeta::getHostId).collect(Collectors.toList()); + List hostDTOList = applicationHostDAO.listHostInfoByHostIds(hostIdList); + Map hostMap = new HashMap<>(); + hostDTOList.forEach(hostDTO -> hostMap.put(hostDTO.getHostId(), hostDTO)); + + // 本地DB不存在的HostId从CMDB查询hostId + List notInDbHostIdList = hostIdList.stream() + .filter(hostId -> !hostMap.containsKey(hostId)) + .collect(Collectors.toList()); + if (CollectionUtils.isNotEmpty(notInDbHostIdList)) { + List cmdbHostList = bizCmdbClient.listHostsByHostIds(notInDbHostIdList); + if (CollectionUtils.isNotEmpty(cmdbHostList)) { + cmdbHostList.forEach(hostDTO -> hostMap.put(hostDTO.getHostId(), hostDTO)); + } + // CMDB中也查不到的IP需要记录 + notInDbHostIdList.removeIf(hostMap::containsKey); + if (!notInDbHostIdList.isEmpty()) { + log.warn("Cannot find hostIds in db/cmdb:{}", notInDbHostIdList); + throw new InvalidParamException( + ErrorCode.HOST_ID_NOT_EXIST_IN_CMDB, + StringUtil.concatCollection(notInDbHostIdList) + ); + } } - String ipStr = createUpdateReq.getIpStr(); - List ipList; - if (null == ipStr || ipStr.trim().isEmpty()) { - log.warn("ipStr cannot be null or empty"); - throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME_AND_REASON, - ArrayUtil.toArray("ipStr", "length of ipStr cannot be over 5000")); - } else if (ipStr.length() > 5000) { - log.warn("length of ipStr cannot be over 5000"); - throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); + + return hostIdList.stream().map(hostId -> { + ApplicationHostDTO hostDTO = hostMap.get(hostId); + return new WhiteIPIPDTO( + null, + null, + hostDTO.getCloudAreaId(), + hostDTO.getHostId(), + hostDTO.getIp(), + hostDTO.getIpv6(), + username, + System.currentTimeMillis(), + username, + System.currentTimeMillis() + ); + }).collect(Collectors.toList()); + } + + private List parseTargetAppIds(WhiteIPRecordCreateUpdateReq createUpdateReq) { + List appIdList; + if (createUpdateReq.isAllScope()) { + // 对所有业务生效 + appIdList = Collections.singletonList(JobConstants.PUBLIC_APP_ID); } else { - ipList = Arrays.asList(ipStr.trim().split(SEPERATOR_ENTER)); - //过滤空值 - ipList = ipList.stream().filter(ip -> !ip.trim().isEmpty()).collect(Collectors.toList()); - ipList.forEach(ip -> { - //正则校验 - if (!IpUtils.checkIp(ip.trim())) { - log.warn("not a valid ip format"); - throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME_AND_REASON, - ArrayUtil.toArray("ipStr", "not a valid ip format")); - } - }); - } - List uniqueIpList = new ArrayList<>(); - for (String ip : ipList) { - if (!uniqueIpList.contains(ip)) { - uniqueIpList.add(ip); - } + List scopeList = createUpdateReq.getScopeList(); + Map scopeAppIdMap = appScopeMappingService.getAppIdByScopeList(scopeList); + appIdList = scopeList.stream() + .map(scope -> { + Long appId = scopeAppIdMap.get(scope); + if (appId == null) { + String msg = "Cannot find appId by scope " + scope; + throw new InternalException(msg, ErrorCode.INTERNAL_ERROR); + } + return appId; + }).collect(Collectors.toList()); } - return uniqueIpList; + return appIdList; } @Override - public Long saveWhiteIP(String username, WhiteIPRecordCreateUpdateReq createUpdateReq) { - LOG.infoWithRequestId("Input(" + username + "," + createUpdateReq.toString() + ")"); + @ActionAuditRecord( + actionId = ActionId.CREATE_WHITELIST, + content = "Create a white list row" + ) + public WhiteIPRecordDTO createWhiteIP(String username, WhiteIPRecordCreateUpdateReq createUpdateReq) { + log.info("Input(" + username + "," + createUpdateReq.toString() + ")"); // 1.参数校验、预处理 - List ipList = checkReqAndGetIpList(createUpdateReq); + List hostList = createUpdateReq.getHostList(); // 2.appId转换 - List scopeList = createUpdateReq.getScopeList(); - Map scopeAppIdMap = appScopeMappingService.getAppIdByScopeList(scopeList); - List appIdList = scopeList.parallelStream() - .map(scope -> { - Long appId = scopeAppIdMap.get(scope); - if (appId == null) { - String msg = "Cannot find appId by scope " + scope; - throw new InternalException(msg, ErrorCode.INTERNAL_ERROR); - } - return appId; - }).collect(Collectors.toList()); + List appIdList = parseTargetAppIds(createUpdateReq); List actionScopeIdList = createUpdateReq.getActionScopeIdList(); - val ipDtoList = ipList.stream().map(ip -> new WhiteIPIPDTO( + val ipDtoList = buildIpDtoList(hostList, username); + val actionScopeDtoList = actionScopeIdList.stream().map(actionScopeId -> new WhiteIPActionScopeDTO( null, null, - createUpdateReq.getCloudAreaId(), - ip, + actionScopeId, username, System.currentTimeMillis(), username, System.currentTimeMillis() )).collect(Collectors.toList()); + + long recordId = whiteIPRecordDAO.insertWhiteIPRecord(new WhiteIPRecordDTO( + null, + appIdList, + createUpdateReq.getRemark(), + ipDtoList, + actionScopeDtoList, + username, + System.currentTimeMillis(), + username, + System.currentTimeMillis() + )); + log.info("Insert success"); + return getWhiteIPDetailById(username, recordId); + } + + @Override + @ActionAuditRecord( + actionId = ActionId.MANAGE_WHITELIST, + content = EventContentConstants.EDIT_WHITE_LIST + ) + public WhiteIPRecordDTO updateWhiteIP(String username, WhiteIPRecordCreateUpdateReq createUpdateReq) { + log.info("Input(" + username + "," + createUpdateReq.toString() + ")"); + // 1.参数校验、预处理 + List hostList = createUpdateReq.getHostList(); + // 2.appId转换 + List appIdList = parseTargetAppIds(createUpdateReq); + + List actionScopeIdList = createUpdateReq.getActionScopeIdList(); + val ipDtoList = buildIpDtoList(hostList, username); val actionScopeDtoList = actionScopeIdList.stream().map(actionScopeId -> new WhiteIPActionScopeDTO( null, null, @@ -296,75 +389,33 @@ public Long saveWhiteIP(String username, WhiteIPRecordCreateUpdateReq createUpda System.currentTimeMillis() )).collect(Collectors.toList()); var recordId = createUpdateReq.getId(); - if (recordId != null && recordId > 0) { - Long finalRecordId = recordId; - ipDtoList.forEach(it -> it.setRecordId(finalRecordId)); - actionScopeDtoList.forEach(it -> it.setRecordId(finalRecordId)); - WhiteIPRecordDTO whiteIPRecordDTO = whiteIPRecordDAO.getWhiteIPRecordById(dslContext, recordId); - whiteIPRecordDTO.setAppIdList(appIdList); - whiteIPRecordDTO.setRemark(createUpdateReq.getRemark()); - whiteIPRecordDTO.setIpList(ipDtoList); - whiteIPRecordDTO.setActionScopeList(actionScopeDtoList); - whiteIPRecordDTO.setLastModifier(username); - whiteIPRecordDTO.setLastModifyTime(System.currentTimeMillis()); - //修改 - whiteIPRecordDAO.updateWhiteIPRecordById(dslContext, whiteIPRecordDTO); - } else { - //新增 - recordId = whiteIPRecordDAO.insertWhiteIPRecord(dslContext, new WhiteIPRecordDTO( - null, - appIdList, - createUpdateReq.getRemark(), - ipDtoList, - actionScopeDtoList, - username, - System.currentTimeMillis(), - username, - System.currentTimeMillis() - )); - LOG.infoWithRequestId("insert success,recordId=" + recordId); - } - return recordId; + ipDtoList.forEach(it -> it.setRecordId(recordId)); + actionScopeDtoList.forEach(it -> it.setRecordId(recordId)); + WhiteIPRecordDTO whiteIPRecordDTO = whiteIPRecordDAO.getWhiteIPRecordById(recordId); + whiteIPRecordDTO.setAppIdList(appIdList); + whiteIPRecordDTO.setRemark(createUpdateReq.getRemark()); + whiteIPRecordDTO.setIpList(ipDtoList); + whiteIPRecordDTO.setActionScopeList(actionScopeDtoList); + whiteIPRecordDTO.setLastModifier(username); + whiteIPRecordDTO.setLastModifyTime(System.currentTimeMillis()); + + //修改 + int affectedRows = whiteIPRecordDAO.updateWhiteIPRecordById(whiteIPRecordDTO); + log.info("{} white ip records updated", affectedRows); + return getWhiteIPDetailById(username, recordId); } @Override - public WhiteIPRecordVO getWhiteIPDetailById(String username, Long id) { - LOG.infoWithRequestId("Input(" + username + "," + id + ")"); - val record = whiteIPRecordDAO.getWhiteIPRecordById(dslContext, id); - Long cloudAreaId = null; - if (record.getIpList().size() > 0) { - cloudAreaId = record.getIpList().get(0).getCloudAreaId(); - } - val applicationInfoList = applicationDAO.listAppsByAppIds(record.getAppIdList()); - List appVOList = applicationInfoList.stream().map(it -> new AppVO( - it.getId(), - it.getScope().getType().getValue(), - it.getScope().getId(), - it.getName(), - null, - null, - null - )).collect(Collectors.toList()); - return new WhiteIPRecordVO( - id, - cloudAreaId, - record.getIpList().stream().map(WhiteIPIPDTO::getIp).collect(Collectors.toList()), - record.getActionScopeList().stream().map(actionScopeDTO -> { - val actionScopeId = actionScopeDTO.getActionScopeId(); - return actionScopeDAO.getActionScopeVOById(actionScopeId); - }).collect(Collectors.toList()), - appVOList, - record.getRemark(), - record.getCreator(), - record.getCreateTime(), - record.getLastModifier(), - record.getLastModifyTime() - ); + public WhiteIPRecordDTO getWhiteIPDetailById(String username, Long id) { + WhiteIPRecordDTO record = whiteIPRecordDAO.getWhiteIPRecordById(id); + List applicationInfoList = applicationDAO.listAppsByAppIds(record.getAppIdList()); + record.setAppList(applicationInfoList); + return record; } @Override public List listCloudAreas(String username) { - LOG.infoWithRequestId("Input(" + username + ")"); + log.info("Input(" + username + ")"); List cloudAreaInfoList = CmdbClientFactory.getCmdbClient(JobContextUtil.getUserLang()) .getCloudAreaList(); return cloudAreaInfoList.stream().map(it -> @@ -386,43 +437,42 @@ public List listActionScope(String username) { } @Override + @ActionAuditRecord( + actionId = ActionId.MANAGE_WHITELIST, + content = "Delete a white list row" + ) public Long deleteWhiteIPById(String username, Long id) { - return (long) whiteIPRecordDAO.deleteWhiteIPRecordById(dslContext, id); + return (long) whiteIPRecordDAO.deleteWhiteIPRecordById(id); } @Override - public List getWhiteIPActionScopes(Long appId, String ip, Long cloudAreaId) { - log.info("Input=({},{},{})", appId, ip, cloudAreaId); + public List getWhiteIPActionScopes(Long appId, String ip, Long cloudAreaId, Long hostId) { + log.info("Input=({},{},{},{})", appId, ip, cloudAreaId, hostId); // 1.找出与当前业务关联的所有appId - List fullAppIds = applicationService.getRelatedAppIds(appId); + List effectiveAppIds = getEffectiveAppIdList(appId); // 2.再查对应的白名单 - return whiteIPRecordDAO.getWhiteIPActionScopes(dslContext, fullAppIds, ip, cloudAreaId); + List actionScopes = new ArrayList<>(); + if (hostId != null) { + actionScopes = whiteIPRecordDAO.getWhiteIPActionScopes(effectiveAppIds, hostId); + } + return actionScopes; } @Override public List listWhiteIPInfos() { List resultList = new ArrayList<>(); //查出所有Record - List recordList = whiteIPRecordDAO.listAllWhiteIPRecord(dslContext); + List recordList = whiteIPRecordDAO.listAllWhiteIPRecord(); if (CollectionUtils.isEmpty(recordList)) { return resultList; } //查询appIdList长度为1的业务信息 - Set appIdSet = new HashSet<>(); Set scopeIdSet = new HashSet<>(); - List applicationDTOList = new ArrayList<>(); recordList.forEach(record -> { - appIdSet.addAll(record.getAppIdList()); //添加所有包含生效范围id列表，方便后续一次性查出关联的生效范围code - scopeIdSet.addAll(record.getActionScopeList().parallelStream() + scopeIdSet.addAll(record.getActionScopeList().stream() .map(WhiteIPActionScopeDTO::getActionScopeId).collect(Collectors.toSet())); }); - if (CollectionUtils.isNotEmpty(appIdSet)) { - applicationDTOList = applicationService.listAppsByAppIds(appIdSet); - } - - Map> applicationDTOMap = applicationDTOList.parallelStream().collect( - Collectors.groupingBy(ApplicationDTO::getId)); int maxInCount = 1000; List> scopeIdsList = Lists.partition(new ArrayList<>(scopeIdSet), maxInCount); @@ -431,16 +481,14 @@ public List listWhiteIPInfos() { actionScopeDTOList.addAll(actionScopeDAO.getActionScopeByIds(scopeIdList)); } - Map> actionScopeDTOMap = actionScopeDTOList.parallelStream().collect( + Map> actionScopeDTOMap = actionScopeDTOList.stream().collect( Collectors.groupingBy(ActionScopeDTO::getId)); for (WhiteIPRecordDTO whiteIPRecordDTO : recordList) { boolean isAllApp = false; List appIdList = whiteIPRecordDTO.getAppIdList(); for (Long appId : appIdList) { - ApplicationDTO applicationDTO = applicationDTOMap.get(appId) == null ? null : - applicationDTOMap.get(appId).get(0); - isAllApp = applicationDTO != null && applicationDTO.isAllBizSet(); + isAllApp = appId == JobConstants.PUBLIC_APP_ID; if (isAllApp) { break; } @@ -454,14 +502,18 @@ public List listWhiteIPInfos() { if (log.isDebugEnabled()) { log.debug("WhiteIPInfos before merge:{}", JsonUtils.toJson(resultList)); } - // 合并IP相同的多条记录 - Map resultMap = new HashMap<>(); + // 合并HostId相同的多条记录 + Map resultMap = new HashMap<>(); for (ServiceWhiteIPInfo whiteIPInfo : resultList) { - String key = whiteIPInfo.getCloudId().toString() + ":" + whiteIPInfo.getIp(); - if (resultMap.containsKey(key)) { - resultMap.put(key, mergeServiceWhiteIPInfo(resultMap.get(key), whiteIPInfo)); + Long hostId = whiteIPInfo.getHostId(); + if (hostId == null) { + log.warn("Ignore whiteIp whose hostId is null:{}", whiteIPInfo); + continue; + } + if (resultMap.containsKey(hostId)) { + resultMap.put(hostId, mergeServiceWhiteIPInfo(resultMap.get(hostId), whiteIPInfo)); } else { - resultMap.put(key, whiteIPInfo); + resultMap.put(hostId, whiteIPInfo); } } resultList.clear(); @@ -481,13 +533,11 @@ private ServiceWhiteIPInfo mergeServiceWhiteIPInfo(ServiceWhiteIPInfo whiteIPInf } else if (whiteIPInfo2 == null) { return whiteIPInfo1; } - if (!(whiteIPInfo1.getCloudId().equals(whiteIPInfo2.getCloudId()) - && whiteIPInfo1.getIp().equals(whiteIPInfo2.getIp()))) { - throw new RuntimeException("Cannot merge ServiceWhiteIPInfo with different cloudId and Ip"); + if (!(whiteIPInfo1.getHostId().equals(whiteIPInfo2.getHostId()))) { + throw new RuntimeException("Cannot merge ServiceWhiteIPInfo with different hostId"); } ServiceWhiteIPInfo finalServiceWhiteIPInfo = new ServiceWhiteIPInfo(); - finalServiceWhiteIPInfo.setCloudId(whiteIPInfo1.getCloudId()); - finalServiceWhiteIPInfo.setIp(whiteIPInfo1.getIp()); + finalServiceWhiteIPInfo.setHostId(whiteIPInfo1.getHostId()); finalServiceWhiteIPInfo.setAllAppActionScopeList(new ArrayList<>()); finalServiceWhiteIPInfo.setForAllApp(false); if (whiteIPInfo1.isForAllApp()) { @@ -559,7 +609,12 @@ private void genAllAppWhiteIPInfo(WhiteIPRecordDTO whiteIPRecordDTO, List resultList) { //封装全业务 List whiteIPIPDTOList = whiteIPRecordDTO.getIpList(); - whiteIPIPDTOList.forEach(whiteIPIPDTO -> { + for (WhiteIPIPDTO whiteIPIPDTO : whiteIPIPDTOList) { + Long hostId = whiteIPIPDTO.getHostId(); + if (hostId == null) { + log.warn("Ignore allApp whiteIp whose hostId is null:{}", whiteIPIPDTO); + continue; + } ServiceWhiteIPInfo serviceWhiteIPInfo = new ServiceWhiteIPInfo(); serviceWhiteIPInfo.setForAllApp(true); List allAppActionScopeList = new ArrayList<>(); @@ -575,10 +630,9 @@ private void genAllAppWhiteIPInfo(WhiteIPRecordDTO whiteIPRecordDTO, } serviceWhiteIPInfo.setAllAppActionScopeList(allAppActionScopeList); serviceWhiteIPInfo.setAppIdActionScopeMap(new HashMap<>()); - serviceWhiteIPInfo.setCloudId(whiteIPIPDTO.getCloudAreaId()); - serviceWhiteIPInfo.setIp(whiteIPIPDTO.getIp()); + serviceWhiteIPInfo.setHostId(hostId); resultList.add(serviceWhiteIPInfo); - }); + } } /** @@ -595,11 +649,15 @@ private void genNormalAppWhiteIPInfo(WhiteIPRecordDTO whiteIPRecordDTO, List whiteIPIPDTOList = whiteIPRecordDTO.getIpList(); for (WhiteIPIPDTO whiteIPIPDTO : whiteIPIPDTOList) { + Long hostId = whiteIPIPDTO.getHostId(); + if (hostId == null) { + log.warn("Ignore normal whiteIp whose hostId is null:{}", whiteIPIPDTO); + continue; + } ServiceWhiteIPInfo serviceWhiteIPInfo = new ServiceWhiteIPInfo(); serviceWhiteIPInfo.setForAllApp(false); serviceWhiteIPInfo.setAllAppActionScopeList(new ArrayList<>()); - serviceWhiteIPInfo.setCloudId(whiteIPIPDTO.getCloudAreaId()); - serviceWhiteIPInfo.setIp(whiteIPIPDTO.getIp()); + serviceWhiteIPInfo.setHostId(hostId); HashMap> map = new HashMap<>(); List actionScopeList = new ArrayList<>(); whiteIPRecordDTO.getActionScopeList().forEach(actionScope -> { diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/agent/AgentStatusService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/agent/AgentStatusService.java new file mode 100644 index 0000000000..425acd1275 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/agent/AgentStatusService.java @@ -0,0 +1,185 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.service.impl.agent; + +import com.tencent.bk.job.common.gse.constants.AgentAliveStatusEnum; +import com.tencent.bk.job.common.gse.service.AgentStateClient; +import com.tencent.bk.job.common.gse.service.model.HostAgentStateQuery; +import com.tencent.bk.job.common.gse.v2.model.resp.AgentState; +import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; +import com.tencent.bk.job.common.model.dto.HostSimpleDTO; +import com.tencent.bk.job.common.util.LogUtil; +import com.tencent.bk.job.manage.config.GseConfig; +import com.tencent.bk.job.manage.model.web.vo.common.AgentStatistics; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.slf4j.helpers.FormattingTuple; +import org.slf4j.helpers.MessageFormatter; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * Agent状态查询服务 + */ +@Slf4j +@Service +public class AgentStatusService { + + private final AgentStateClient agentStateClient; + + @Autowired + public AgentStatusService(@Qualifier(GseConfig.MANAGE_BEAN_AGENT_STATE_CLIENT) AgentStateClient agentStateClient) { + this.agentStateClient = agentStateClient; + } + + /** + * 为主机填充实时Agent状态 + * + * @param hosts 主机列表 + */ + @SuppressWarnings("DuplicatedCode") + public void fillRealTimeAgentStatus(List hosts) { + if (CollectionUtils.isEmpty(hosts)) { + return; + } + // 查出节点下主机与Agent状态 + List hostAgentStateQueryList = new ArrayList<>(hosts.size()); + Map hostAgentStateQueryMap = new HashMap<>(hosts.size()); + hosts.forEach(host -> { + HostAgentStateQuery hostAgentStateQuery = HostAgentStateQuery.from(host); + hostAgentStateQueryList.add(hostAgentStateQuery); + hostAgentStateQueryMap.put(host.getHostIdOrCloudIp(), hostAgentStateQuery); + }); + + Map agentStateMap; + try { + agentStateMap = agentStateClient.batchGetAgentState(hostAgentStateQueryList); + } catch (Exception e) { + FormattingTuple msg = MessageFormatter.format( + "Fail to get agentState by hosts:{}", + LogUtil.buildListLog(hostAgentStateQueryList, 20) + ); + log.warn(msg.getMessage(), e); + return; + } + + if (CollectionUtils.isEmpty(hosts)) { + return; + } + // 批量设置agent状态 + for (ApplicationHostDTO hostInfoDTO : hosts) { + if (hostInfoDTO == null) { + continue; + } + HostAgentStateQuery hostAgentStateQuery = hostAgentStateQueryMap.get(hostInfoDTO.getHostIdOrCloudIp()); + String effectiveAgentId = agentStateClient.getEffectiveAgentId(hostAgentStateQuery); + AgentState agentState = agentStateMap.get(effectiveAgentId); + if (agentState == null) { + hostInfoDTO.setGseAgentStatus(null); + } else { + hostInfoDTO.setGseAgentStatus(agentState.getStatusCode()); + } + } + } + + /** + * 本地主机状态与GSE主机状态做个比较，找到状态不同的主机并返回 + * + * @param hosts 主机列表 + */ + @SuppressWarnings("DuplicatedCode") + public List findStatusChangedHosts(List hosts) { + List statusChangedHosts = new ArrayList<>(); + if (hosts.isEmpty()) return statusChangedHosts; + + List hostAgentStateQueryList = new ArrayList<>(hosts.size()); + Map hostAgentStateQueryMap = new HashMap<>(hosts.size()); + hosts.forEach(hostSimpleDTO -> { + HostAgentStateQuery hostAgentStateQuery = HostAgentStateQuery.from(hostSimpleDTO); + hostAgentStateQueryList.add(hostAgentStateQuery); + hostAgentStateQueryMap.put(hostSimpleDTO.getHostIdOrCloudIp(), hostAgentStateQuery); + }); + Map agentStateMap; + try { + agentStateMap = agentStateClient.batchGetAgentState(hostAgentStateQueryList); + } catch (Exception e) { + FormattingTuple msg = MessageFormatter.format( + "Fail to get agentState by hosts:{}", + LogUtil.buildListLog(hostAgentStateQueryList, 20) + ); + log.warn(msg.getMessage(), e); + return statusChangedHosts; + } + + for (HostSimpleDTO host : hosts) { + HostAgentStateQuery hostAgentStateQuery = hostAgentStateQueryMap.get(host.getHostIdOrCloudIp()); + String effectiveAgentId = agentStateClient.getEffectiveAgentId(hostAgentStateQuery); + AgentState agentState = agentStateMap.get(effectiveAgentId); + AgentAliveStatusEnum agentAliveStatus = AgentAliveStatusEnum.fromAgentState(agentState); + int agentAliveStatusValue = agentAliveStatus.getStatusValue(); + if (host.getAgentAliveStatus() != agentAliveStatusValue) { + if (log.isDebugEnabled()) { + log.debug("host {} status changed: {}->{}, effectiveAgentId={}, agentState={}", + host.getHostId(), + host.getAgentAliveStatus(), + agentAliveStatusValue, + effectiveAgentId, + agentState + ); + } + host.setAgentAliveStatus(agentAliveStatusValue); + statusChangedHosts.add(host); + } + } + return statusChangedHosts; + } + + /** + * 为主机填充实时Agent状态并给出正常/异常统计数据 + * + * @param hosts 主机列表 + * @return 主机统计数据 + */ + public AgentStatistics calcAgentStatistics(List hosts) { + fillRealTimeAgentStatus(hosts); + int aliveCount = 0; + int notAliveCount = 0; + for (ApplicationHostDTO it : hosts) { + Boolean alive = it.getGseAgentAlive(); + if (alive != null && alive) { + aliveCount++; + } else { + notAliveCount++; + } + } + return new AgentStatistics(aliveCount, notAliveCount); + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/notify/NotifySendService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/notify/NotifySendService.java index ad9b086574..745879a276 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/notify/NotifySendService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/notify/NotifySendService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/notify/NotifyServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/notify/NotifyServiceImpl.java index 4e69223252..ab6f6e4279 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/notify/NotifyServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/notify/NotifyServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,19 +25,24 @@ package com.tencent.bk.job.manage.service.impl.notify; import com.google.common.collect.Sets; +import com.tencent.bk.audit.annotations.ActionAuditRecord; +import com.tencent.bk.job.common.audit.constants.EventContentConstants; import com.tencent.bk.job.common.cc.model.AppRoleDTO; +import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.model.dto.UserRoleInfoDTO; +import com.tencent.bk.job.common.model.dto.notify.CustomNotifyDTO; import com.tencent.bk.job.common.model.vo.NotifyChannelVO; +import com.tencent.bk.job.common.mysql.JobTransactional; import com.tencent.bk.job.common.redis.util.LockUtils; import com.tencent.bk.job.common.util.Counter; import com.tencent.bk.job.common.util.I18nUtil; import com.tencent.bk.job.common.util.JobContextUtil; import com.tencent.bk.job.common.util.PrefConsts; -import com.tencent.bk.job.manage.common.consts.notify.ExecuteStatusEnum; -import com.tencent.bk.job.manage.common.consts.notify.JobRoleEnum; -import com.tencent.bk.job.manage.common.consts.notify.NotifyConsts; -import com.tencent.bk.job.manage.common.consts.notify.ResourceTypeEnum; -import com.tencent.bk.job.manage.common.consts.notify.TriggerTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.notify.ExecuteStatusEnum; +import com.tencent.bk.job.manage.api.common.constants.notify.JobRoleEnum; +import com.tencent.bk.job.manage.api.common.constants.notify.NotifyConsts; +import com.tencent.bk.job.manage.api.common.constants.notify.ResourceTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.notify.TriggerTypeEnum; import com.tencent.bk.job.manage.dao.ScriptDAO; import com.tencent.bk.job.manage.dao.notify.AvailableEsbChannelDAO; import com.tencent.bk.job.manage.dao.notify.EsbAppRoleDAO; @@ -56,6 +61,7 @@ import com.tencent.bk.job.manage.model.inner.ServiceNotificationDTO; import com.tencent.bk.job.manage.model.inner.ServiceNotificationMessage; import com.tencent.bk.job.manage.model.inner.ServiceNotificationTriggerDTO; +import com.tencent.bk.job.manage.model.inner.ServiceSpecificResourceNotifyPolicyDTO; import com.tencent.bk.job.manage.model.inner.ServiceTemplateNotificationDTO; import com.tencent.bk.job.manage.model.inner.ServiceTriggerTemplateNotificationDTO; import com.tencent.bk.job.manage.model.inner.ServiceUserNotificationDTO; @@ -76,15 +82,14 @@ import lombok.val; import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang3.StringUtils; -import org.jooq.DSLContext; +import org.apache.commons.lang3.tuple.Pair; import org.slf4j.helpers.FormattingTuple; import org.slf4j.helpers.MessageFormatter; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; import org.springframework.util.StopWatch; +import java.security.InvalidParameterException; import java.time.LocalDateTime; import java.util.ArrayList; import java.util.Arrays; @@ -104,7 +109,6 @@ public class NotifyServiceImpl implements NotifyService { private static final String REDIS_KEY_SAVE_APP_DEFAULT_NOTIFY_POLICIES = "NotifyServiceImpl" + ".saveAppDefaultNotifyPolicies"; - private final DSLContext dslContext; private final NotifyTriggerPolicyDAO notifyTriggerPolicyDAO; private final NotifyRoleTargetChannelDAO notifyRoleTargetChannelDAO; private final NotifyPolicyRoleTargetDAO notifyPolicyRoleTargetDAO; @@ -122,8 +126,6 @@ public class NotifyServiceImpl implements NotifyService { @Autowired public NotifyServiceImpl( - @Qualifier("job-manage-dsl-context") - DSLContext dslContext, NotifyTriggerPolicyDAO notifyTriggerPolicyDAO, NotifyPolicyRoleTargetDAO notifyPolicyRoleTargetDAO, NotifyRoleTargetChannelDAO notifyRoleTargetChannelDAO, @@ -138,7 +140,6 @@ public NotifyServiceImpl( ScriptDAO scriptDAO, TaskPlanDAO taskPlanDAO, NotifyUserService notifyUserService) { - this.dslContext = dslContext; this.notifyTriggerPolicyDAO = notifyTriggerPolicyDAO; this.notifyPolicyRoleTargetDAO = notifyPolicyRoleTargetDAO; this.notifyRoleTargetChannelDAO = notifyRoleTargetChannelDAO; @@ -157,10 +158,23 @@ public NotifyServiceImpl( @Override public List listAppDefaultNotifyPolicies(String username, Long appId) { - return notifyTriggerPolicyDAO.list(getDefaultTriggerUser(), appId, NotifyConsts.DEFAULT_RESOURCE_ID); + return notifyTriggerPolicyDAO.listAppDefault(getDefaultTriggerUser(), appId, NotifyConsts.DEFAULT_RESOURCE_ID); } @Override + public CustomNotifyDTO getSpecificResourceNotifyPolicy(Long appId, + Integer resourceType, + String resourceId, + Integer triggerType) { + return notifyTriggerPolicyDAO.getSpecificResourceNotifyPolicy(appId, resourceType, resourceId, triggerType); + } + + + @Override + @ActionAuditRecord( + actionId = ActionId.NOTIFICATION_SETTING, + content = EventContentConstants.EDIT_BUSINESS_NOTIFY_SETTINGS + ) public Long saveAppDefaultNotifyPolicies( String username, Long appId, @@ -172,6 +186,7 @@ public Long saveAppDefaultNotifyPolicies( private Long saveTriggerPolicy(Long appId, ResourceTypeEnum resourceType, + String resourceId, String triggerUser, TriggerPolicy triggerPolicy, ResourceStatusChannel resourceStatusChannel, @@ -180,7 +195,7 @@ private Long saveTriggerPolicy(Long appId, new NotifyTriggerPolicyDTO( null, appId, - NotifyConsts.DEFAULT_RESOURCE_ID, + resourceId, resourceType, triggerUser, triggerPolicy.getTriggerType(), @@ -251,19 +266,36 @@ private void saveTriggerPolicy(String operator, resourceTypeSet.forEach(resourceType -> { Set channelSet = new HashSet<>(triggerPolicy.getResourceStatusChannelList()); channelSet.forEach(channel -> { - //保存触发策略 - val policyId = saveTriggerPolicy( - appId, resourceType, - triggerUser, triggerPolicy, channel, operator - ); - //保存所有通知对象 - saveNotifyRoleTargets(policyId, triggerPolicy, channel, operator); + cascadeSaveTriggerPolicyAndTarget( + appId, + operator, + resourceType, + NotifyConsts.DEFAULT_RESOURCE_ID, + triggerUser, + triggerPolicy, + channel); }); }); } + private void cascadeSaveTriggerPolicyAndTarget(Long appId, + String operator, + ResourceTypeEnum resourceType, + String resourceId, + String triggerUser, + TriggerPolicy triggerPolicy, + ResourceStatusChannel channel) { + //保存触发策略 + val policyId = saveTriggerPolicy( + appId, resourceType, resourceId, + triggerUser, triggerPolicy, channel, operator + ); + //保存所有通知对象 + saveNotifyRoleTargets(policyId, triggerPolicy, channel, operator); + } + @Override - @Transactional + @JobTransactional(transactionManager = "jobManageTransactionManager") public Long saveAppDefaultNotifyPoliciesToLocal(String operator, Long appId, String triggerUser, NotifyPoliciesCreateUpdateReq createUpdateReq) { val policyList = createUpdateReq.getTriggerPoliciesList(); @@ -276,6 +308,60 @@ public Long saveAppDefaultNotifyPoliciesToLocal(String operator, Long appId, Str return (long) policyList.size(); } + @Override + @JobTransactional(transactionManager = "jobManageTransactionManager") + public Boolean saveSpecificResourceNotifyPolicies( + Long appId, + String operator, + ServiceSpecificResourceNotifyPolicyDTO specificResourceNotifyPolicyDTO + ) { + + if (specificResourceNotifyPolicyDTO.getResourceId() == -1) { + log.error("[saveSpecificResourceNotifyPolicies] not allow to save app default resource notify policies " + + "with resourceId=-1"); + throw new InvalidParameterException("cannot save custom notify policy with resourceId=-1"); + } + + notifyTriggerPolicyDAO.deleteAppResourceNotifyPolicies( + appId, + specificResourceNotifyPolicyDTO.getResourceType(), + specificResourceNotifyPolicyDTO.getResourceId().toString() + ); + + TriggerPolicy triggerPolicy = buildTriggerPolicy(specificResourceNotifyPolicyDTO); + + List resourceStatusChannelList = ResourceStatusChannel.fromMap( + specificResourceNotifyPolicyDTO.getResourceStatusChannelMap() + ); + resourceStatusChannelList.forEach(resourceStatusChannel -> { + cascadeSaveTriggerPolicyAndTarget( + appId, + operator, + ResourceTypeEnum.get(specificResourceNotifyPolicyDTO.getResourceType()), + specificResourceNotifyPolicyDTO.getResourceId().toString(), + getDefaultTriggerUser(), + triggerPolicy, + resourceStatusChannel + ); + }); + + return true; + } + + private TriggerPolicy buildTriggerPolicy(ServiceSpecificResourceNotifyPolicyDTO specificResourceNotifyPolicyDTO) { + TriggerPolicy triggerPolicy = new TriggerPolicy(); + triggerPolicy.setTriggerType(TriggerTypeEnum.get(specificResourceNotifyPolicyDTO.getTriggerType())); + triggerPolicy.setResourceTypeList(Collections.singletonList(ResourceTypeEnum.get( + specificResourceNotifyPolicyDTO.getResourceType() + ))); + triggerPolicy.setRoleList(specificResourceNotifyPolicyDTO.getRoleList()); + triggerPolicy.setExtraObserverList(specificResourceNotifyPolicyDTO.getExtraObserverList()); + triggerPolicy.setResourceStatusChannelList( + ResourceStatusChannel.fromMap(specificResourceNotifyPolicyDTO.getResourceStatusChannelMap()) + ); + return triggerPolicy; + } + private String getDefaultTriggerUser() { // 按业务保存、触发策略，采用公共触发者 return NotifyConsts.DEFAULT_TRIGGER_USER; @@ -309,11 +395,21 @@ public Long saveAppDefaultNotifyPolicies( ) { //0.1配置记录 if (!notifyConfigStatusDAO.exist(getDefaultTriggerUser(), appId)) { - notifyConfigStatusDAO.insertNotifyConfigStatus(dslContext, getDefaultTriggerUser(), appId); + notifyConfigStatusDAO.insertNotifyConfigStatus(getDefaultTriggerUser(), appId); } return tryToSavePoliciesToLocal(username, appId, createUpdateReq); } + @Override + public int deleteAppResourceNotifyPolicies(Long appId, Integer resourceType, String resourceId) { + // 业务全局消息通知方式不允许删除 + if (NotifyConsts.DEFAULT_RESOURCE_ID.equals(resourceId)) { + log.error("not allow to delete app default resource notify policies with resourceId=-1"); + throw new InvalidParameterException("cannot delete notify policy with resourceId=-1"); + } + return notifyTriggerPolicyDAO.deleteAppResourceNotifyPolicies(appId, resourceType, resourceId); + } + @Override public List listTriggerType(String username) { return TriggerTypeEnum.getVOList(); @@ -327,7 +423,7 @@ public List listResourceType(String username) { @Override public List listRole(String username) { //Job系统角色+CMDB业务角色 - List appRoles = esbAppRoleDAO.listEsbAppRole(dslContext); + List appRoles = esbAppRoleDAO.listEsbAppRole(); if (CollectionUtils.isEmpty(appRoles)) { return Collections.emptyList(); } @@ -342,7 +438,7 @@ public List listRole(String username) { public List listRoles() { //Job系统角色+CMDB业务角色 List resultList = new ArrayList<>(); - List appRoleDTOList = esbAppRoleDAO.listEsbAppRole(dslContext); + List appRoleDTOList = esbAppRoleDAO.listEsbAppRole(); if (appRoleDTOList != null) { resultList.addAll(appRoleDTOList); } @@ -364,7 +460,7 @@ public List listExecuteStatus(String username) { @Override public List listAllNotifyChannel() { - return notifyEsbChannelDAO.listNotifyEsbChannel(dslContext).stream() + return notifyEsbChannelDAO.listNotifyEsbChannel().stream() .filter(NotifyEsbChannelDTO::isActive).map(it -> { NotifyEsbChannelDTO channel = new NotifyEsbChannelDTO(); channel.setType(it.getType()); @@ -376,29 +472,27 @@ public List listAllNotifyChannel() { private List getAvailableChannelTypeList() { List availableEsbChannelDTOList = - availableEsbChannelDAO.listAvailableEsbChannel(dslContext); + availableEsbChannelDAO.listAvailableEsbChannel(); return availableEsbChannelDTOList.stream().map(AvailableEsbChannelDTO::getType).collect(Collectors.toList()); } @Override public List listAvailableNotifyChannel(String username) { List availableChannelTypeList = getAvailableChannelTypeList(); - return notifyEsbChannelDAO.listNotifyEsbChannel(dslContext).stream().map(it -> new NotifyChannelVO( + return notifyEsbChannelDAO.listNotifyEsbChannel().stream().map(it -> new NotifyChannelVO( it.getType(), it.getLabel() )).filter(it -> availableChannelTypeList.contains(it.getCode())).collect(Collectors.toList()); } @Override + @JobTransactional(transactionManager = "jobManageTransactionManager") public Integer setAvailableNotifyChannel(String username, SetAvailableNotifyChannelReq req) { List channelCodeList = Arrays.asList(req.getChannelCodeStr().trim().split(NotifyConsts.SEPERATOR_COMMA)); - dslContext.transaction(configuration -> { - availableEsbChannelDAO.deleteAll(dslContext); - channelCodeList.forEach(it -> availableEsbChannelDAO.insertAvailableEsbChannel(dslContext, - new AvailableEsbChannelDTO(it, true, username, LocalDateTime.now()))); - - }); + availableEsbChannelDAO.deleteAll(); + channelCodeList.forEach(it -> availableEsbChannelDAO.insertAvailableEsbChannel( + new AvailableEsbChannelDTO(it, true, username, LocalDateTime.now()))); return channelCodeList.size(); } @@ -429,13 +523,11 @@ public Integer sendSimpleNotification(ServiceNotificationDTO notification) { private Set findJobResourceOwners(Integer resourceType, String resourceIdStr) { Set userSet = new HashSet<>(); - if (resourceType == ResourceTypeEnum.SCRIPT.getType()) { - userSet.add(scriptDAO.getScriptByScriptId(resourceIdStr).getLastModifyUser()); - } else if (resourceType == ResourceTypeEnum.JOB.getType()) { + if (resourceType == ResourceTypeEnum.JOB.getType()) { long resourceId = Long.parseLong(resourceIdStr); userSet.add(taskPlanDAO.getTaskPlanById(resourceId).getLastModifyUser()); } else { - log.warn("Unknown resourceType:{}", resourceType); + // 没有资源的任务不通知"资源所属者" } return userSet; } @@ -494,6 +586,30 @@ private List searchNotifyPolices(Long appId, ); } + private List searchResourceNotifyPolices(Long appId, + Integer triggerType, + Integer resourceType, + String resourceIdStr, + Integer resourceExecuteStatus) { + log.debug( + "try to get resource custom trigger policy:|appId={}|triggerType={}" + + "|resourceType={}|resourceIdStr={}|resourceExecuteStatus={}", + appId, + triggerType, + resourceType, + resourceIdStr, + resourceExecuteStatus + ); + return notifyTriggerPolicyDAO.list( + NotifyConsts.DEFAULT_TRIGGER_USER, + appId, + resourceIdStr, + resourceType, + triggerType, + resourceExecuteStatus + ); + } + private List searchAppNotifyPolices(Long appId, Integer triggerType, Integer resourceType, @@ -527,19 +643,47 @@ private boolean appNotifyPolicyConfigured(Long appId) { return notifyConfigStatusDAO.exist(getDefaultTriggerUser(), appId); } - private List getTriggerPolicys(Long appId, - String triggerUser, - Integer triggerType, - Integer resourceType, - Integer resourceExecuteStatus) { - // 1.业务公共触发策略 - List triggerPolicyList = searchAppNotifyPolices( - appId, - triggerType, - resourceType, - resourceExecuteStatus - ); - // 2.查找当前触发者自己制定的触发策略 + private List getTriggerPolicys(ServiceNotificationTriggerDTO triggerDTO) { + Long appId = triggerDTO.getAppId() == null ? NotifyConsts.DEFAULT_APP_ID : triggerDTO.getAppId(); + String triggerUser = triggerDTO.getTriggerUser(); + Integer triggerType = triggerDTO.getTriggerType(); + Integer resourceType = triggerDTO.getResourceType(); + String resourceIdStr = triggerDTO.getResourceId(); + Integer resourceExecuteStatus = triggerDTO.getResourceExecuteStatus(); + List triggerPolicyList = Collections.emptyList(); + // 1.查找资源级别的触发策略 + if (canHaveCustomTriggerPolicy(triggerDTO)) { + Pair resource = getResourceIndeed(triggerDTO); + Integer triggerPolicyResourceType = resource.getLeft(); + String triggerPolicyResourceId = resource.getRight(); + triggerPolicyList = searchResourceNotifyPolices( + appId, + triggerType, + triggerPolicyResourceType, + triggerPolicyResourceId, + resourceExecuteStatus + ); + } + // 2.业务公共触发策略 + if (CollectionUtils.isEmpty(triggerPolicyList)) { + log.debug( + "not found custom trigger policy|appId={}|triggerUser={}" + + "|triggerType={}|resourceType={}|{}|resourceIdStr={}", + appId, + triggerUser, + triggerType, + resourceType, + resourceIdStr, + resourceExecuteStatus + ); + triggerPolicyList = searchAppNotifyPolices( + appId, + triggerType, + resourceType, + resourceExecuteStatus + ); + } + // 3.查找当前触发者自己制定的触发策略 if (CollectionUtils.isEmpty(triggerPolicyList)) { triggerPolicyList = searchNotifyPolices( appId, @@ -549,7 +693,7 @@ private List getTriggerPolicys(Long appId, resourceExecuteStatus ); } - // 3.默认业务 + // 4.默认业务 // 业务未配置消息通知策略才使用默认策略 if (!appNotifyPolicyConfigured(appId) && CollectionUtils.isEmpty(triggerPolicyList)) { triggerPolicyList = systemDefaultNotifyPolices(triggerType, resourceType, resourceExecuteStatus); @@ -557,6 +701,16 @@ private List getTriggerPolicys(Long appId, return triggerPolicyList; } + private boolean canHaveCustomTriggerPolicy(ServiceNotificationTriggerDTO triggerDTO) { + Pair resource = getResourceIndeed(triggerDTO); + // 当前只有定时任务支持资源级别的自定义通知 + if (resource.getLeft() == ResourceTypeEnum.CRON.getType()) { + return true; + } + + return false; + } + private Set findUserByRole(Long appId, String role, String triggerUser, @@ -580,16 +734,8 @@ private Set findUserByRole(Long appId, private Map> getChannelUsersMap(ServiceNotificationTriggerDTO triggerDTO) { Long appId = triggerDTO.getAppId() == null ? NotifyConsts.DEFAULT_APP_ID : triggerDTO.getAppId(); String triggerUser = triggerDTO.getTriggerUser(); - Integer triggerType = triggerDTO.getTriggerType(); Integer resourceType = triggerDTO.getResourceType(); - Integer resourceExecuteStatus = triggerDTO.getResourceExecuteStatus(); - List triggerPolicyList = getTriggerPolicys( - appId, - triggerUser, - triggerType, - resourceType, - resourceExecuteStatus - ); + List triggerPolicyList = getTriggerPolicys(triggerDTO); if (CollectionUtils.isEmpty(triggerPolicyList)) { return Collections.emptyMap(); } @@ -597,11 +743,11 @@ private Map> getChannelUsersMap(ServiceNotificationTriggerDT Map> channelUsersMap = new HashMap<>(); for (NotifyTriggerPolicyDTO policy : triggerPolicyList) { List roleTargetList = - notifyPolicyRoleTargetDAO.listByPolicyId(dslContext, policy.getId()); + notifyPolicyRoleTargetDAO.listByPolicyId(policy.getId()); for (NotifyPolicyRoleTargetDTO roleTarget : roleTargetList) { String role = roleTarget.getRole(); List roleTargetChannelList = - notifyRoleTargetChannelDAO.listByRoleTargetId(dslContext, roleTarget.getId()); + notifyRoleTargetChannelDAO.listByRoleTargetId(roleTarget.getId()); Set channels = roleTargetChannelList.stream() .map(NotifyRoleTargetChannelDTO::getChannel).collect(Collectors.toSet()); @@ -623,6 +769,17 @@ private Map> getChannelUsersMap(ServiceNotificationTriggerDT return channelUsersMap; } + /** + * 是定时任务的话，返回定时任务ID而非执行方案ID + */ + private Pair getResourceIndeed(ServiceNotificationTriggerDTO triggerDTO) { + if (triggerDTO.getCronTaskId() != null && triggerDTO.getCronTaskId() != -1L) { + return Pair.of(ResourceTypeEnum.CRON.getType(), String.valueOf(triggerDTO.getCronTaskId())); + } + + return Pair.of(triggerDTO.getResourceType(), triggerDTO.getResourceId()); + } + @Override public PageTemplateVO getPageTemplate(String username) { return new PageTemplateVO( diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/notify/NotifyTemplateService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/notify/NotifyTemplateService.java index fc9938af1c..e449e3d6da 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/notify/NotifyTemplateService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/notify/NotifyTemplateService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,11 +28,11 @@ import com.tencent.bk.job.common.i18n.locale.LocaleUtils; import com.tencent.bk.job.common.model.dto.ApplicationDTO; import com.tencent.bk.job.common.model.dto.ResourceScope; +import com.tencent.bk.job.common.service.config.JobCommonConfig; import com.tencent.bk.job.common.util.JobContextUtil; import com.tencent.bk.job.common.util.PrefConsts; import com.tencent.bk.job.common.util.StringUtil; import com.tencent.bk.job.common.util.TypeUtil; -import com.tencent.bk.job.manage.config.JobManageConfig; import com.tencent.bk.job.manage.dao.ApplicationDAO; import com.tencent.bk.job.manage.dao.notify.NotifyTemplateDAO; import com.tencent.bk.job.manage.model.dto.notify.NotifyTemplateDTO; @@ -56,15 +56,15 @@ public class NotifyTemplateService { private final ApplicationDAO applicationDAO; - private final JobManageConfig jobManageConfig; + private final JobCommonConfig jobCommonConfig; private final NotifyTemplateDAO notifyTemplateDAO; @Autowired public NotifyTemplateService(ApplicationDAO applicationDAO, - JobManageConfig jobManageConfig, + JobCommonConfig jobCommonConfig, NotifyTemplateDAO notifyTemplateDAO) { this.applicationDAO = applicationDAO; - this.jobManageConfig = jobManageConfig; + this.jobCommonConfig = jobCommonConfig; this.notifyTemplateDAO = notifyTemplateDAO; } @@ -140,7 +140,7 @@ public ServiceNotificationMessage getNotificationMessageFromTemplate( String content = titleAndContent.getRight(); //添加默认变量 ResourceScope scope = applicationDTO.getScope(); - variablesMap.putIfAbsent("BASE_HOST", jobManageConfig.getJobWebUrl()); + variablesMap.putIfAbsent("BASE_HOST", jobCommonConfig.getJobWebUrl()); variablesMap.putIfAbsent("APP_ID", getDisplayIdStr(scope)); variablesMap.putIfAbsent("task.bk_biz_id", scope.getId()); variablesMap.putIfAbsent("APP_NAME", appName); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/notify/NotifyUserService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/notify/NotifyUserService.java index f0fa03863c..f2bd672444 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/notify/NotifyUserService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/notify/NotifyUserService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,7 @@ package com.tencent.bk.job.manage.service.impl.notify; -import com.tencent.bk.job.manage.common.consts.notify.NotifyConsts; +import com.tencent.bk.job.manage.api.common.constants.notify.NotifyConsts; import com.tencent.bk.job.manage.dao.notify.EsbUserInfoDAO; import com.tencent.bk.job.manage.dao.notify.NotifyBlackUserInfoDAO; import com.tencent.bk.job.manage.model.dto.notify.EsbUserInfoDTO; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/notify/SendNotifyTask.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/notify/SendNotifyTask.java index 1409cb46da..712f9acb1a 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/notify/SendNotifyTask.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/notify/SendNotifyTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,15 +24,16 @@ package com.tencent.bk.job.manage.service.impl.notify; +import com.tencent.bk.job.common.util.ThreadUtils; import com.tencent.bk.job.manage.dao.notify.EsbUserInfoDAO; -import com.tencent.bk.job.manage.model.dto.notify.EsbUserInfoDTO; import com.tencent.bk.job.manage.service.impl.WatchableSendMsgService; import lombok.Builder; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; -import org.slf4j.helpers.FormattingTuple; import org.slf4j.helpers.MessageFormatter; +import java.util.Collection; +import java.util.HashSet; import java.util.List; import java.util.Set; @@ -54,6 +55,8 @@ public class SendNotifyTask implements Runnable { private final String title; private final String content; + private Set validReceivers; + public void bindService(WatchableSendMsgService watchableSendMsgService, EsbUserInfoDAO esbUserInfoDAO) { this.watchableSendMsgService = watchableSendMsgService; @@ -62,8 +65,13 @@ public void bindService(WatchableSendMsgService watchableSendMsgService, @Override public void run() { - if (CollectionUtils.isEmpty(receivers)) { - logInvalidReceivers(); + pickValidReceivers(); + Collection removedReceivers = CollectionUtils.subtract(receivers, validReceivers); + if (CollectionUtils.isNotEmpty(removedReceivers)) { + log.info("Invalid user removed:{}, ignore to send notify to them", removedReceivers); + } + if (CollectionUtils.isEmpty(validReceivers)) { + logValidReceiversEmpty(); return; } try { @@ -78,8 +86,13 @@ public void run() { } } - private void logInvalidReceivers() { - log.warn("receivers is null or empty, skip, msgType={},title={}", msgType, title); + private void pickValidReceivers() { + List existUserNameList = esbUserInfoDAO.listExistUserName(receivers); + validReceivers = new HashSet<>(existUserNameList); + } + + private void logValidReceiversEmpty() { + log.info("valid receivers is null or empty, skip, msgType={},title={}", msgType, title); } private boolean sendMsgWithRetry() { @@ -93,47 +106,67 @@ private boolean sendMsgWithRetry() { createTimeMillis, msgType, sender, - receivers, + validReceivers, title, content ); result = true; } catch (Exception e) { - log.error("Fail to sendMsg", e); + if (count < NOTIFY_MAX_RETRY_COUNT) { + long sleepMills = count * 1000; + String msg = MessageFormatter.format( + "Fail to sendMsg, sleep {}ms and retry {}", + sleepMills, + count + ).getMessage(); + log.warn(msg, e); + ThreadUtils.sleep(sleepMills); + } else { + log.error("Fail to sendMsg", e); + } } } return result; } private void logSendSuccess() { - log.info("Success to send notify:({},{},{})", String.join(",", receivers), msgType, title); + log.info("Success to send notify:({},{},{})", String.join(",", validReceivers), msgType, title); } private void handleSendFail(Exception e) { - List validUsers = esbUserInfoDAO.listEsbUserInfo(receivers); - if (validUsers.isEmpty()) { - // 收信人已全部离职/某些渠道不支持平台用户 - logIgnoreToSend(); - return; - } - FormattingTuple msg = MessageFormatter.arrayFormat( - "Fail to send notify:({},{},{},{})", - new Object[]{ - String.join(",", receivers), - msgType, - title, - content - } - ); - if (e != null) { - log.error(msg.getMessage(), e); + int titleMaxLength = 32; + int contentMaxLength = 200; + String msg; + if (log.isDebugEnabled()) { + msg = MessageFormatter.arrayFormat( + "Fail to send notify:({},{},{},{})", + new Object[]{ + String.join(",", validReceivers), + msgType, + title, + content + } + ).getMessage(); + log.debug(msg, e); } else { - log.error(msg.getMessage()); + msg = MessageFormatter.arrayFormat( + "Fail to send notify:({},{},{},{})", + new Object[]{ + String.join(",", validReceivers), + msgType, + buildLogContent(title, titleMaxLength), + buildLogContent(content, contentMaxLength) + } + ).getMessage(); + log.error(msg, e); } } - private void logIgnoreToSend() { - log.info("Ignore to send notify:({},{},{})", String.join(",", receivers), msgType, title); + private String buildLogContent(String content, int maxLength) { + if (content != null && content.length() > maxLength) { + return content.substring(0, maxLength) + "..."; + } + return content; } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/AbstractCmdbResourceEventWatcher.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/AbstractCmdbResourceEventWatcher.java index 37b3efe852..0843768b65 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/AbstractCmdbResourceEventWatcher.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/AbstractCmdbResourceEventWatcher.java @@ -1,9 +1,34 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + package com.tencent.bk.job.manage.service.impl.sync; import com.tencent.bk.job.common.cc.model.result.ResourceEvent; import com.tencent.bk.job.common.cc.model.result.ResourceWatchResult; +import com.tencent.bk.job.common.redis.util.HeartBeatRedisLock; +import com.tencent.bk.job.common.redis.util.LockResult; import com.tencent.bk.job.common.redis.util.LockUtils; -import com.tencent.bk.job.common.redis.util.RedisKeyHeartBeatThread; import com.tencent.bk.job.common.tracing.util.SpanUtil; import com.tencent.bk.job.common.util.ThreadUtils; import com.tencent.bk.job.common.util.TimeUtil; @@ -15,6 +40,7 @@ import org.apache.commons.collections4.CollectionUtils; import org.springframework.cloud.sleuth.Span; import org.springframework.cloud.sleuth.Tracer; +import org.springframework.cloud.sleuth.annotation.NewSpan; import org.springframework.data.redis.core.RedisTemplate; import java.util.List; @@ -62,23 +88,22 @@ public AbstractCmdbResourceEventWatcher(String watcherResourceName, this.redisLockKey = "watch-cmdb-" + this.watcherResourceName + "-lock"; } + @NewSpan @Override public final void run() { log.info("Watching {} event start", this.watcherResourceName); - RedisKeyHeartBeatThread redisKeyHeartBeatThread = null; + LockResult lockResult = null; while (true) { Span span = SpanUtil.buildNewSpan(this.tracer, this.watcherResourceName + "WatchOuterLoop"); try (Tracer.SpanInScope ignored = this.tracer.withSpan(span.start())) { - boolean lockGotten = tryGetTaskLockPeriodically(); - if (!lockGotten) { - // 30s之后重试 - ThreadUtils.sleep(30_000); + HeartBeatRedisLock redisLock = new HeartBeatRedisLock(redisTemplate, redisLockKey, machineIp); + lockResult = redisLock.lock(); + if (!lockResult.isLockGotten()) { + // 5s之后重试 + ThreadUtils.sleep(5_000); continue; } - // 获取任务锁之后通过心跳线程维持锁的占有 - redisKeyHeartBeatThread = startRedisKeyHeartBeatThread(); - // 事件处理前的初始化 tryToInitBeforeWatch(); @@ -89,8 +114,8 @@ public final void run() { span.error(t); } finally { span.end(); - if (redisKeyHeartBeatThread != null) { - redisKeyHeartBeatThread.stopAtOnce(); + if (lockResult != null) { + lockResult.tryToRelease(); } LockUtils.releaseDistributedLock(redisLockKey, machineIp); // 过5s后重新尝试监听事件 @@ -99,42 +124,6 @@ public final void run() { } } - private boolean tryGetTaskLockPeriodically() { - boolean lockGotten = false; - try { - lockGotten = LockUtils.tryGetReentrantLock(redisLockKey, machineIp, 5_000); - if (!lockGotten) { - String runningMachine = - redisTemplate.opsForValue().get(redisLockKey); - log.info( - "Get lock {} fail, {} watch thread already running on {}", - this.redisLockKey, - this.watcherResourceName, - runningMachine - ); - return false; - } - lockGotten = true; - } catch (Throwable t) { - log.error("Get lock caught exception", t); - } - return lockGotten; - } - - private RedisKeyHeartBeatThread startRedisKeyHeartBeatThread() { - // 开一个心跳子线程，维护当前机器正在WatchResource的状态 - RedisKeyHeartBeatThread redisKeyHeartBeatThread = new RedisKeyHeartBeatThread( - redisTemplate, - redisLockKey, - machineIp, - 5_000L, - 2_000L - ); - redisKeyHeartBeatThread.setName("[" + watcherResourceName + "]-redisKeyHeartBeatThread"); - redisKeyHeartBeatThread.start(); - return redisKeyHeartBeatThread; - } - private void watchAndHandleEvent() { log.info("Start watch {} resource at {},{}", watcherResourceName, TimeUtil.getCurrentTimeStr("HH:mm:ss"), System.currentTimeMillis()); @@ -250,7 +239,7 @@ protected boolean isWatchingEnabled() { * * @param event cmdb事件 */ - protected abstract void handleEvent(ResourceEvent event); + public abstract void handleEvent(ResourceEvent event); /** * 获取CMDB事件指标标签 diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/AgentStatusSyncService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/AgentStatusSyncService.java index 4260bbb461..1939d06b72 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/AgentStatusSyncService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/AgentStatusSyncService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,12 +24,10 @@ package com.tencent.bk.job.manage.service.impl.sync; -import com.tencent.bk.job.common.constant.ResourceScopeTypeEnum; -import com.tencent.bk.job.common.model.dto.ApplicationDTO; -import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; -import com.tencent.bk.job.manage.dao.ApplicationDAO; +import com.tencent.bk.job.common.model.dto.HostSimpleDTO; import com.tencent.bk.job.manage.dao.ApplicationHostDAO; -import com.tencent.bk.job.manage.service.HostService; +import com.tencent.bk.job.manage.service.host.HostService; +import com.tencent.bk.job.manage.service.impl.agent.AgentStatusService; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.data.util.Pair; @@ -37,8 +35,6 @@ import org.springframework.util.StopWatch; import java.util.List; -import java.util.Set; -import java.util.stream.Collectors; /** * 机器Agent状态同步逻辑 @@ -47,67 +43,83 @@ @Service public class AgentStatusSyncService { - private final ApplicationDAO applicationDAO; private final ApplicationHostDAO applicationHostDAO; private final HostService hostService; + private final AgentStatusService agentStatusService; @Autowired - public AgentStatusSyncService(ApplicationDAO applicationDAO, - ApplicationHostDAO applicationHostDAO, - HostService hostService) { - this.applicationDAO = applicationDAO; + public AgentStatusSyncService(ApplicationHostDAO applicationHostDAO, + HostService hostService, + AgentStatusService agentStatusService) { this.applicationHostDAO = applicationHostDAO; this.hostService = hostService; + this.agentStatusService = agentStatusService; } - private Pair syncBizHostAgentStatus(Long bizId) { + private Pair syncHostAgentStatus() { long gseInterfaceTimeConsuming = 0L; long writeToDBTimeConsuming = 0L; - StopWatch appHostAgentStatusWatch = new StopWatch(); - appHostAgentStatusWatch.start("listHostInfoByAppId"); - List localBizAppHosts = applicationHostDAO.listHostInfoByBizId(bizId); - appHostAgentStatusWatch.stop(); - appHostAgentStatusWatch.start("getAgentStatusByAppInfo from GSE"); + StopWatch hostAgentStatusWatch = new StopWatch(); + hostAgentStatusWatch.start("listAllHostInfo"); + List localHosts = applicationHostDAO.listAllHostSimpleInfo(); + hostAgentStatusWatch.stop(); + hostAgentStatusWatch.start("getAgentStatus from GSE"); long startTime = System.currentTimeMillis(); - hostService.fillAgentStatus(localBizAppHosts); + List statusChangedHosts = agentStatusService.findStatusChangedHosts(localHosts); gseInterfaceTimeConsuming += (System.currentTimeMillis() - startTime); - appHostAgentStatusWatch.stop(); - appHostAgentStatusWatch.start("updateHosts to local DB"); + hostAgentStatusWatch.stop(); + hostAgentStatusWatch.start("updateHosts to local DB"); startTime = System.currentTimeMillis(); - hostService.updateHostsInBiz(bizId, localBizAppHosts); + int updatedHostNum = hostService.updateHostsStatus(statusChangedHosts); writeToDBTimeConsuming += (System.currentTimeMillis() - startTime); - appHostAgentStatusWatch.stop(); - log.debug("Performance:syncAppHostAgentStatus:bizId={},{}", bizId, appHostAgentStatusWatch); + hostAgentStatusWatch.stop(); + if (hostAgentStatusWatch.getTotalTimeMillis() > 180000) { + log.warn( + "syncHostAgentStatus too slow, totalHosts={}, statusChangedHosts={}, " + + "updatedHostNum={}, timeConsume={}", + localHosts.size(), + statusChangedHosts.size(), + updatedHostNum, + hostAgentStatusWatch.prettyPrint() + ); + } + if (statusChangedHosts.size() == updatedHostNum) { + log.info( + "syncHostAgentStatus Performance,totalHosts={}, " + + "statusChangedHosts={}, updatedHostNum={}, timeConsume={}", + localHosts.size(), + statusChangedHosts.size(), + updatedHostNum, + hostAgentStatusWatch + ); + } else { + log.warn( + "syncHostAgentStatus Performance,totalHosts={}, " + + "statusChangedHosts={}, updatedHostNum={}, timeConsume={}", + localHosts.size(), + statusChangedHosts.size(), + updatedHostNum, + hostAgentStatusWatch + ); + } return Pair.of(gseInterfaceTimeConsuming, writeToDBTimeConsuming); } public void syncAgentStatusFromGSE() { log.info(Thread.currentThread().getName() + ":begin to sync agentStatus from GSE"); - List localBizApps = applicationDAO.listAllBizApps(); - Set localBizIds = - localBizApps.stream().filter(bizApp -> - bizApp.getScope().getType() == ResourceScopeTypeEnum.BIZ) - .map(bizApp -> Long.valueOf(bizApp.getScope().getId())) - .collect(Collectors.toSet()); - log.info(String.format("localBizIds:%s", String.join(",", - localBizIds.stream().map(Object::toString).collect(Collectors.toSet())))); long gseInterfaceTimeConsuming = 0L; long writeToDBTimeConsuming = 0L; - for (ApplicationDTO applicationDTO : localBizApps) { - try { - Pair timeConsumingPair = syncBizHostAgentStatus( - Long.valueOf(applicationDTO.getScope().getId()) - ); - gseInterfaceTimeConsuming += timeConsumingPair.getFirst(); - writeToDBTimeConsuming += timeConsumingPair.getSecond(); - } catch (Throwable t) { - log.error("syncAgentStatus of app fail:bizId=" + applicationDTO.getScope().getId(), t); - } + try { + Pair timeConsumingPair = syncHostAgentStatus(); + gseInterfaceTimeConsuming += timeConsumingPair.getFirst(); + writeToDBTimeConsuming += timeConsumingPair.getSecond(); + log.info(Thread.currentThread().getName() + ":Finished:sync agentStatus from GSE," + + "gseInterfaceTimeConsuming={}ms,writeToDBTimeConsuming={}ms,rate={}", + gseInterfaceTimeConsuming, writeToDBTimeConsuming, + gseInterfaceTimeConsuming / (0. + writeToDBTimeConsuming)); + } catch (Throwable t) { + log.error("syncAgentStatus from GSE fail：", t); } - log.info(Thread.currentThread().getName() + ":Finished:sync agentStatus from GSE," + - "gseInterfaceTimeConsuming={}ms,writeToDBTimeConsuming={}ms,rate={}", - gseInterfaceTimeConsuming, writeToDBTimeConsuming, - gseInterfaceTimeConsuming / (0. + writeToDBTimeConsuming)); } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/AppHostsSyncer.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/AppHostsSyncer.java deleted file mode 100644 index dce05f7b0c..0000000000 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/AppHostsSyncer.java +++ /dev/null @@ -1,71 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.service.impl.sync; - -import com.tencent.bk.job.common.model.dto.ApplicationDTO; -import com.tencent.bk.job.manage.service.ApplicationService; -import lombok.extern.slf4j.Slf4j; -import org.springframework.data.util.Pair; - -import java.util.concurrent.BlockingQueue; - -/** - * 业务主机同步线程，从队列中取出需要同步的业务同步其主机 - */ -@Slf4j -public class AppHostsSyncer extends Thread { - - private final ApplicationService applicationService; - private final HostSyncService hostSyncService; - volatile BlockingQueue queue; - - public AppHostsSyncer(ApplicationService applicationService, - HostSyncService hostSyncService, - BlockingQueue queue) { - this.applicationService = applicationService; - this.hostSyncService = hostSyncService; - this.queue = queue; - } - - @Override - public void run() { - while (true) { - try { - Long appId = queue.take(); - ApplicationDTO applicationDTO = applicationService.getAppByAppId(appId); - Pair timeConsumingPair = hostSyncService.syncBizHostsAtOnce(applicationDTO); - Long cmdbInterfaceTimeConsuming = timeConsumingPair.getFirst(); - Long writeToDBTimeConsuming = timeConsumingPair.getSecond(); - log.info("Sync appHosts of {}:cmdbInterfaceTimeConsuming={}ms,writeToDBTimeConsuming={}ms," + - "rate={}", appId, cmdbInterfaceTimeConsuming, writeToDBTimeConsuming, - cmdbInterfaceTimeConsuming / (0. + writeToDBTimeConsuming)); - } catch (InterruptedException e) { - log.warn("queue.take interrupted", e); - } catch (Throwable t) { - log.warn("Fail to syncAppHostsAtOnce", t); - } - } - } -} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/AppHostsUpdateHelper.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/AppHostsUpdateHelper.java index 896bf5b5f7..a7846f678a 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/AppHostsUpdateHelper.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/AppHostsUpdateHelper.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -112,7 +112,7 @@ public void startToUpdateAppHosts(Long appId) { heartBeatThreadMap.put(Thread.currentThread().getName(), appHostUpdateRedisKeyHeartBeatThread); } - public void endToUpdateBizHosts(Long appId) { + public void endToUpdateBizHosts() { String key = Thread.currentThread().getName(); RedisKeyHeartBeatThread heartBeatThread = heartBeatThreadMap.get(key); if (heartBeatThread != null) { diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BasicAppSyncService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BasicAppSyncService.java index 82cbd36509..7bc9802d38 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BasicAppSyncService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BasicAppSyncService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -33,7 +33,6 @@ import com.tencent.bk.job.manage.dao.ApplicationHostDAO; import com.tencent.bk.job.manage.service.ApplicationService; import lombok.extern.slf4j.Slf4j; -import org.jooq.DSLContext; import org.springframework.beans.factory.annotation.Autowired; import java.util.HashMap; @@ -46,19 +45,16 @@ @Slf4j public class BasicAppSyncService { - private final DSLContext dslContext; private final ApplicationDAO applicationDAO; private final ApplicationHostDAO applicationHostDAO; private final ApplicationService applicationService; protected final IBizCmdbClient bizCmdbClient; @Autowired - public BasicAppSyncService(DSLContext dslContext, - ApplicationDAO applicationDAO, + public BasicAppSyncService(ApplicationDAO applicationDAO, ApplicationHostDAO applicationHostDAO, ApplicationService applicationService, IBizCmdbClient bizCmdbClient) { - this.dslContext = dslContext; this.applicationDAO = applicationDAO; this.applicationHostDAO = applicationHostDAO; this.applicationService = applicationService; @@ -111,7 +107,7 @@ protected void applyAppsChangeByScope(List insertApps, try { log.info("Update app: {}", JsonUtils.toJson(applicationInfoDTO)); applicationService.updateApp(applicationInfoDTO); - applicationDAO.restoreDeletedApp(dslContext, applicationInfoDTO.getId()); + applicationDAO.restoreDeletedApp(applicationInfoDTO.getId()); } catch (Throwable t) { log.error("FATAL: updateApp fail:appId=" + applicationInfoDTO.getId(), t); } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizEventWatcher.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizEventWatcher.java index 20aa149ae2..96363b288f 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizEventWatcher.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizEventWatcher.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -86,7 +86,7 @@ protected ResourceWatchResult fetchEventsByStartTime(Long startT } @Override - protected void handleEvent(ResourceEvent event) { + public void handleEvent(ResourceEvent event) { String eventType = event.getEventType(); ApplicationDTO newestApp = BizEventDetail.toAppInfoDTO(event.getDetail()); ApplicationDTO cachedApp = null; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizSetEventWatcher.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizSetEventWatcher.java index b2a7e02b6e..6cd1fd86fa 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizSetEventWatcher.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizSetEventWatcher.java @@ -1,3 +1,27 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + package com.tencent.bk.job.manage.service.impl.sync; import com.tencent.bk.job.common.cc.model.req.ResourceWatchReq; @@ -52,7 +76,7 @@ protected ResourceWatchResult fetchEventsByStartTime(Long sta } @Override - protected void handleEvent(ResourceEvent event) { + public void handleEvent(ResourceEvent event) { log.info("Handle BizSetEvent: {}", event); ApplicationDTO latestApp = event.getDetail().toApplicationDTO(); String eventType = event.getEventType(); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizSetRelationEventWatcher.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizSetRelationEventWatcher.java index 2d12c9739d..bb887355fc 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizSetRelationEventWatcher.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizSetRelationEventWatcher.java @@ -1,3 +1,27 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + package com.tencent.bk.job.manage.service.impl.sync; import com.tencent.bk.job.common.cc.model.req.ResourceWatchReq; @@ -56,7 +80,7 @@ protected ResourceWatchResult fetchEventsByStartTime( } @Override - protected void handleEvent(ResourceEvent event) { + public void handleEvent(ResourceEvent event) { log.info("Handle BizSetRelationEvent: {}", event); String eventType = event.getEventType(); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizSetSyncService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizSetSyncService.java index 7c38b35208..44ae41716f 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizSetSyncService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizSetSyncService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -40,7 +40,6 @@ import com.tencent.bk.job.manage.service.impl.BizSetService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.CollectionUtils; -import org.jooq.DSLContext; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -60,14 +59,13 @@ public class BizSetSyncService extends BasicAppSyncService { private final BizSetService bizSetService; @Autowired - public BizSetSyncService(DSLContext dslContext, - ApplicationDAO applicationDAO, + public BizSetSyncService(ApplicationDAO applicationDAO, ApplicationHostDAO applicationHostDAO, ApplicationService applicationService, IBizCmdbClient bizCmdbClient, IBizSetCmdbClient bizSetCmdbClient, BizSetService bizSetService) { - super(dslContext, applicationDAO, applicationHostDAO, applicationService, bizCmdbClient); + super(applicationDAO, applicationHostDAO, applicationService, bizCmdbClient); this.applicationDAO = applicationDAO; this.bizSetCmdbClient = bizSetCmdbClient; this.bizSetService = bizSetService; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizSyncService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizSyncService.java index e253569eb9..f6b5c6bd1b 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizSyncService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/BizSyncService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,11 +26,11 @@ import com.tencent.bk.job.common.cc.sdk.IBizCmdbClient; import com.tencent.bk.job.common.model.dto.ApplicationDTO; +import com.tencent.bk.job.manage.config.JobManageConfig; import com.tencent.bk.job.manage.dao.ApplicationDAO; import com.tencent.bk.job.manage.dao.ApplicationHostDAO; import com.tencent.bk.job.manage.service.ApplicationService; import lombok.extern.slf4j.Slf4j; -import org.jooq.DSLContext; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -46,15 +46,17 @@ public class BizSyncService extends BasicAppSyncService { private final ApplicationDAO applicationDAO; + private final JobManageConfig jobManageConfig; @Autowired - public BizSyncService(DSLContext dslContext, - ApplicationDAO applicationDAO, + public BizSyncService(ApplicationDAO applicationDAO, ApplicationHostDAO applicationHostDAO, ApplicationService applicationService, - IBizCmdbClient bizCmdbClient) { - super(dslContext, applicationDAO, applicationHostDAO, applicationService, bizCmdbClient); + IBizCmdbClient bizCmdbClient, + JobManageConfig jobManageConfig) { + super(applicationDAO, applicationHostDAO, applicationService, bizCmdbClient); this.applicationDAO = applicationDAO; + this.jobManageConfig = jobManageConfig; } public void syncBizFromCMDB() { @@ -95,7 +97,7 @@ public void syncBizFromCMDB() { // 本地&CMDB交集：计算需要更新的业务 updateList = ccBizApps.stream().filter(ccBizAppInfoDTO -> - localBizAppScopeIds.contains(ccBizAppInfoDTO.getScope().getId())) + localBizAppScopeIds.contains(ccBizAppInfoDTO.getScope().getId())) .collect(Collectors.toList()); log.info(String.format("biz app updateList scopeIds:%s", String.join(",", updateList.stream().map(applicationInfoDTO -> @@ -105,11 +107,63 @@ public void syncBizFromCMDB() { // 本地-CMDB：计算需要删除的业务 deleteList = localBizApps.stream().filter(bizAppInfoDTO -> - !ccBizAppScopeIds.contains(bizAppInfoDTO.getScope().getId())) + !ccBizAppScopeIds.contains(bizAppInfoDTO.getScope().getId())) .collect(Collectors.toList()); log.info(String.format("app deleteList scopeIds:%s", String.join(",", deleteList.stream().map(applicationInfoDTO -> applicationInfoDTO.getScope().getId()).collect(Collectors.toSet())))); - applyAppsChangeByScope(insertList, deleteList, updateList); + if (isSafeToApplyChange(localBizApps, deleteList)) { + applyAppsChangeByScope(insertList, deleteList, updateList); + } + } + + /** + * 根据配置信息判断业务数据变更是否安全 + * + * @param localBizApps 本地DB中的业务信息 + * @param deleteList 需要设置为删除状态的业务列表 + * @return 本次业务数据变更是否安全 + */ + private boolean isSafeToApplyChange(List localBizApps, List deleteList) { + if (!jobManageConfig.isEnableAppDeleteProtect()) { + return true; + } + // 在本地未被删除的业务 + List notDeleteLocalBizApps = localBizApps.stream() + .filter(app -> !app.isDeleted()) + .collect(Collectors.toList()); + // 在本地已经被软删除的业务 + List deletedLocalBizApps = localBizApps.stream() + .filter(ApplicationDTO::isDeleted) + .collect(Collectors.toList()); + if (notDeleteLocalBizApps.isEmpty()) { + return true; + } + // 计算新删除的业务数量 + int newDeleteNum = deleteList.size() - deletedLocalBizApps.size(); + float deleteRatio = (float) newDeleteNum / notDeleteLocalBizApps.size(); + if (log.isDebugEnabled()) { + log.debug( + "notDeleteLocalBizApps.size={}, deletedLocalBizApps.size={}, " + + "deleteList.size={}, newDeleteNum={}, deleteRatio={}", + notDeleteLocalBizApps.size(), + deletedLocalBizApps.size(), + deleteList.size(), + newDeleteNum, + deleteRatio + ); + } + // 根据最大删除比例阈值判断本次变更是否安全 + if (deleteRatio > jobManageConfig.getMaxAppDeleteRatio()) { + log.error( + "FATAL|deleteRatio({}/{}={}) is beyond maxAppDeleteRatio({}), dangerous! Not apply app change", + newDeleteNum, + notDeleteLocalBizApps.size(), + deleteRatio, + jobManageConfig.getMaxAppDeleteRatio() + ); + return false; + } + return true; } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/EventsHandler.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/EventsHandler.java index 2f1966470a..62c1cd2bfe 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/EventsHandler.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/EventsHandler.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -45,7 +45,6 @@ public abstract class EventsHandler extends Thread { private final CmdbEventSampler cmdbEventSampler; protected boolean enabled = true; BlockingQueue> queue; - Long bizId = null; public EventsHandler(BlockingQueue> queue, Tracer tracer, @@ -55,17 +54,11 @@ public EventsHandler(BlockingQueue> queue, this.cmdbEventSampler = cmdbEventSampler; } - public Long getBizId() { - return bizId; - } - - public void commitEvent(Long bizId, ResourceEvent event) { + public void commitEvent(ResourceEvent event) { try { boolean result = this.queue.add(event); if (!result) { log.warn("Fail to commitEvent:{}", event); - } else { - this.bizId = bizId; } } catch (Exception e) { log.warn("Fail to commitEvent:" + event, e); @@ -83,10 +76,10 @@ void handleEventWithTrace(ResourceEvent event) { Span span = buildSpan(); try (Tracer.SpanInScope ignored = this.tracer.withSpan(span.start())) { handleEvent(event); - } catch (Exception e) { - span.error(e); + } catch (Throwable t) { + span.error(t); eventHandleResult = MetricsConstants.TAG_VALUE_CMDB_EVENT_HANDLE_RESULT_FAILED; - throw e; + log.warn("Fail to handleOneEvent:" + event, t); } finally { span.end(); long timeConsuming = System.currentTimeMillis() - event.getCreateTime(); @@ -110,18 +103,14 @@ private Tags buildEventHandleTimeTags(String eventHandleResult) { @Override public void run() { while (enabled) { - ResourceEvent event = null; + ResourceEvent event; try { event = queue.take(); handleEventWithTrace(event); } catch (InterruptedException e) { log.warn("queue.take interrupted", e); } catch (Throwable t) { - log.warn("Fail to handleOneEvent:" + event, t); - } finally { - if (queue.size() == 0) { - this.bizId = null; - } + log.error("Fail to handleEventWithTrace", t); } } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/HostEventHandler.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/HostEventHandler.java index cf7dd4ff5d..da84f719a0 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/HostEventHandler.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/HostEventHandler.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,43 +27,43 @@ import com.tencent.bk.job.common.cc.model.req.ResourceWatchReq; import com.tencent.bk.job.common.cc.model.result.HostEventDetail; import com.tencent.bk.job.common.cc.model.result.ResourceEvent; -import com.tencent.bk.job.common.constant.JobConstants; -import com.tencent.bk.job.common.gse.constants.AgentStatusEnum; -import com.tencent.bk.job.common.gse.service.QueryAgentStatusClient; +import com.tencent.bk.job.common.gse.service.AgentStateClient; +import com.tencent.bk.job.common.gse.service.model.HostAgentStateQuery; +import com.tencent.bk.job.common.gse.v2.model.resp.AgentState; import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; import com.tencent.bk.job.common.util.json.JsonUtils; -import com.tencent.bk.job.manage.dao.ApplicationHostDAO; -import com.tencent.bk.job.manage.manager.host.HostCache; +import com.tencent.bk.job.manage.config.GseConfig; import com.tencent.bk.job.manage.metrics.CmdbEventSampler; import com.tencent.bk.job.manage.metrics.MetricsConstants; -import com.tencent.bk.job.manage.service.ApplicationService; +import com.tencent.bk.job.manage.service.host.HostService; import io.micrometer.core.instrument.Tags; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.slf4j.helpers.FormattingTuple; +import org.slf4j.helpers.MessageFormatter; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.cloud.sleuth.Tracer; +import java.util.Collections; +import java.util.List; import java.util.concurrent.BlockingQueue; @Slf4j public class HostEventHandler extends EventsHandler { - private final ApplicationService applicationService; - private final ApplicationHostDAO applicationHostDAO; - private final QueryAgentStatusClient queryAgentStatusClient; - private final HostCache hostCache; + private final HostService hostService; + private final AgentStateClient agentStateClient; HostEventHandler(Tracer tracer, CmdbEventSampler cmdbEventSampler, BlockingQueue> queue, - ApplicationService applicationService, - ApplicationHostDAO applicationHostDAO, - QueryAgentStatusClient queryAgentStatusClient, - HostCache hostCache) { + HostService hostService, + @Qualifier(GseConfig.MANAGE_BEAN_AGENT_STATE_CLIENT) + AgentStateClient agentStateClient) { super(queue, tracer, cmdbEventSampler); - this.applicationService = applicationService; - this.applicationHostDAO = applicationHostDAO; - this.queryAgentStatusClient = queryAgentStatusClient; - this.hostCache = hostCache; + this.hostService = hostService; + this.agentStateClient = agentStateClient; } @Override @@ -86,98 +86,93 @@ private void handleOneEventRelatedToApp(ResourceEvent event) { handleOneEventIndeed(event); } + private void setDefaultLastTimeForHostIfNeed(ResourceEvent event, ApplicationHostDTO hostInfoDTO) { + if (hostInfoDTO.getLastTime() == null || hostInfoDTO.getLastTime() < 0) { + log.warn( + "HostEvent lastTime is invalid({}), use event create time({}) to update host", + hostInfoDTO.getLastTime(), + event.getCreateTime() + ); + hostInfoDTO.setLastTime(event.getCreateTime()); + } + } private void handleOneEventIndeed(ResourceEvent event) { String eventType = event.getEventType(); ApplicationHostDTO hostInfoDTO = HostEventDetail.toHostInfoDTO(event.getDetail()); + setDefaultLastTimeForHostIfNeed(event, hostInfoDTO); switch (eventType) { case ResourceWatchReq.EVENT_TYPE_CREATE: case ResourceWatchReq.EVENT_TYPE_UPDATE: - // 去除没有IP的主机信息 - if (StringUtils.isBlank(hostInfoDTO.getDisplayIp())) { - deleteHostWithoutIp(hostInfoDTO); + // 忽略没有hostId的主机事件 + if (hostInfoDTO.getHostId() == null) { + log.warn("Ignore hostEvent without hostId:{}", event); break; } - // 找出Agent有效的IP，并设置Agent状态 - updateIpAndAgentStatus(hostInfoDTO); - // 更新DB中的主机数据 - createOrUpdateHostInDB(hostInfoDTO); - // 更新缓存中的主机数据 - updateHostCache(hostInfoDTO); + // 尝试设置Agent状态 + Integer agentStatus = tryToUpdateAgentStatus(hostInfoDTO); + // 更新DB与缓存中的主机数据 + Pair pair = hostService.createOrUpdateHostBeforeLastTime(hostInfoDTO); + int affectedNum = pair.getRight(); + if (affectedNum == 0) { + log.info( + "no host affected after handle according to lastTime, " + + "try to query latest host from cmdb and update" + ); + // 从CMDB查询最新主机信息 + List hostList = hostService.listHostsFromCmdbByHostIds( + Collections.singletonList(hostInfoDTO.getHostId()) + ); + if (CollectionUtils.isNotEmpty(hostList)) { + hostInfoDTO = hostList.get(0); + hostInfoDTO.setGseAgentStatus(agentStatus); + affectedNum = hostService.updateHostAttrsByHostId(hostInfoDTO); + log.info("update host attrs:{}, affectedNum={}", hostInfoDTO, affectedNum); + // 更新缓存 + if (affectedNum > 0) { + hostService.updateDbHostToCache(hostInfoDTO.getHostId()); + } + } else { + // 机器在CMDB中已不存在，忽略 + log.info("host not exist in cmdb:{}, ignore", hostInfoDTO); + } + } else { + log.info( + "{} host affected, created:{}", + affectedNum, + pair.getLeft() + ); + } break; case ResourceWatchReq.EVENT_TYPE_DELETE: - handleHostDelete(hostInfoDTO); + int deletedNum = hostService.deleteHostBeforeOrEqualLastTime(hostInfoDTO); + log.info("delete host:{}, deletedNum={}", hostInfoDTO, deletedNum); break; default: break; } } - private void deleteHostWithoutIp(ApplicationHostDTO hostInfoDTO) { - int affectedRowNum = applicationHostDAO.deleteBizHostInfoById(null, hostInfoDTO.getHostId()); - log.info( - "{} host deleted, id={} ,ip={}", - affectedRowNum, - hostInfoDTO.getHostId(), - hostInfoDTO.getIp() - ); - } - - private boolean getCloudIpAgentStatus(String cloudIp) { - QueryAgentStatusClient.AgentStatus agentStatus = queryAgentStatusClient.getAgentStatus(cloudIp); - return agentStatus != null && agentStatus.status == AgentStatusEnum.ALIVE.getValue(); - } - - private void updateIpAndAgentStatus(ApplicationHostDTO hostInfoDTO) { - Long cloudAreaId = hostInfoDTO.getCloudAreaId(); - String ip = queryAgentStatusClient.getHostIpByAgentStatus(hostInfoDTO.getDisplayIp(), cloudAreaId); - hostInfoDTO.setIp(ip); - if (!ip.contains(":")) { - String cloudIp = cloudAreaId + ":" + ip; - hostInfoDTO.setGseAgentAlive(getCloudIpAgentStatus(cloudIp)); - } else { - hostInfoDTO.setGseAgentAlive(getCloudIpAgentStatus(ip)); - } - } - - private void createOrUpdateHostInDB(ApplicationHostDTO hostInfoDTO) { + /** + * 尝试更新主机的Agent状态 + * + * @param hostInfoDTO 主机信息 + * @return 最终主机的Agent状态 + */ + private Integer tryToUpdateAgentStatus(ApplicationHostDTO hostInfoDTO) { try { - if (applicationHostDAO.existAppHostInfoByHostId(hostInfoDTO.getHostId())) { - // 只更新事件中的主机属性与agent状态 - applicationHostDAO.updateHostAttrsById(hostInfoDTO); - } else { - hostInfoDTO.setBizId(JobConstants.PUBLIC_APP_ID); - int affectedNum = applicationHostDAO.insertHostWithoutTopo(hostInfoDTO); - log.info("insert host: id={}, affectedNum={}", hostInfoDTO.getHostId(), affectedNum); + AgentState agentState = agentStateClient.getAgentState(HostAgentStateQuery.from(hostInfoDTO)); + if (agentState != null) { + hostInfoDTO.setGseAgentStatus(agentState.getStatusCode()); } - } catch (Throwable t) { - log.error("handle host event fail", t); - } finally { - // 从拓扑表向主机表同步拓扑数据 - int affectedNum = applicationHostDAO.syncHostTopo(hostInfoDTO.getHostId()); - log.info("hostTopo synced: hostId={}, affectedNum={}", hostInfoDTO.getHostId(), affectedNum); + } catch (Exception e) { + FormattingTuple msg = MessageFormatter.format( + "Fail to updateAgentStatus, host={}", + hostInfoDTO + ); + log.warn(msg.getMessage(), e); } + return hostInfoDTO.getGseAgentStatus(); } - private void updateHostCache(ApplicationHostDTO hostInfoDTO) { - hostInfoDTO = applicationHostDAO.getHostById(hostInfoDTO.getHostId()); - if (hostInfoDTO.getBizId() != null && hostInfoDTO.getBizId() > 0) { - // 只更新常规业务的主机到缓存 - if (applicationService.existBiz(hostInfoDTO.getBizId())) { - hostCache.addOrUpdateHost(hostInfoDTO); - log.info("host cache updated: hostId:{}", hostInfoDTO.getHostId()); - } - } - } - - private void handleHostDelete(ApplicationHostDTO hostInfoDTO) { - int affectedRowNum = applicationHostDAO.deleteBizHostInfoById(null, hostInfoDTO.getHostId()); - log.info( - "{} host deleted, id={} ,ip={}", - affectedRowNum, - hostInfoDTO.getHostId(), - hostInfoDTO.getIp() - ); - hostCache.deleteHost(hostInfoDTO); - } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/HostEventWatcher.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/HostEventWatcher.java index 63c44dbc78..3590175578 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/HostEventWatcher.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/HostEventWatcher.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,18 +28,17 @@ import com.tencent.bk.job.common.cc.model.result.ResourceEvent; import com.tencent.bk.job.common.cc.model.result.ResourceWatchResult; import com.tencent.bk.job.common.cc.sdk.BizCmdbClient; -import com.tencent.bk.job.common.gse.service.QueryAgentStatusClient; -import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; +import com.tencent.bk.job.common.gse.service.AgentStateClient; +import com.tencent.bk.job.manage.config.GseConfig; import com.tencent.bk.job.manage.config.JobManageConfig; -import com.tencent.bk.job.manage.dao.ApplicationHostDAO; -import com.tencent.bk.job.manage.manager.host.HostCache; import com.tencent.bk.job.manage.metrics.CmdbEventSampler; import com.tencent.bk.job.manage.metrics.MetricsConstants; -import com.tencent.bk.job.manage.service.ApplicationService; +import com.tencent.bk.job.manage.service.host.HostService; import io.micrometer.core.instrument.Tag; import io.micrometer.core.instrument.Tags; import lombok.extern.slf4j.Slf4j; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.cloud.sleuth.Tracer; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.stereotype.Component; @@ -60,10 +59,8 @@ public class HostEventWatcher extends AbstractCmdbResourceEventWatcher redisTemplate, Tracer tracer, CmdbEventSampler cmdbEventSampler, BizCmdbClient bizCmdbClient, - ApplicationService applicationService, - ApplicationHostDAO applicationHostDAO, - QueryAgentStatusClient queryAgentStatusClient, - HostCache hostCache, + HostService hostService, + @Qualifier(GseConfig.MANAGE_BEAN_AGENT_STATE_CLIENT) + AgentStateClient agentStateClient, JobManageConfig jobManageConfig) { super("host", redisTemplate, tracer, cmdbEventSampler); this.tracer = tracer; this.cmdbEventSampler = cmdbEventSampler; this.bizCmdbClient = bizCmdbClient; - this.applicationService = applicationService; - this.applicationHostDAO = applicationHostDAO; - this.queryAgentStatusClient = queryAgentStatusClient; - this.hostCache = hostCache; + this.hostService = hostService; + this.agentStateClient = agentStateClient; this.eventsHandlerNum = jobManageConfig.getHostEventHandlerNum(); } @@ -118,7 +112,7 @@ protected ResourceWatchResult fetchEventsByStartTime(Long start } @Override - protected void handleEvent(ResourceEvent event) { + public void handleEvent(ResourceEvent event) { dispatchEventToHandler(event); } @@ -163,10 +157,8 @@ private HostEventHandler buildHostEventHandler(BlockingQueue event) { - ApplicationHostDTO hostInfoDTO = HostEventDetail.toHostInfoDTO(event.getDetail()); - Long hostId = hostInfoDTO.getHostId(); - ApplicationHostDTO oldHostInfoDTO = applicationHostDAO.getHostById(hostId); - HostEventHandler eventsHandler = chooseHandler(hostId); - eventsHandler.commitEvent(oldHostInfoDTO == null ? null : oldHostInfoDTO.getBizId(), event); + HostEventHandler eventsHandler = chooseHandler(event.getDetail().getHostId()); + eventsHandler.commitEvent(event); } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/HostRelationEventHandler.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/HostRelationEventHandler.java index bfa709100f..bc9da6e01d 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/HostRelationEventHandler.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/HostRelationEventHandler.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -27,6 +27,7 @@ import com.tencent.bk.job.common.cc.model.req.ResourceWatchReq; import com.tencent.bk.job.common.cc.model.result.HostRelationEventDetail; import com.tencent.bk.job.common.cc.model.result.ResourceEvent; +import com.tencent.bk.job.common.constant.JobConstants; import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.manage.dao.ApplicationHostDAO; @@ -96,30 +97,56 @@ private void handleOneEvent(ResourceEvent event) { } } + private void setDefaultLastTimeForHostTopoIfNeed(ResourceEvent event, + HostTopoDTO hostTopoDTO) { + if (hostTopoDTO.getLastTime() == null || hostTopoDTO.getLastTime() < 0) { + hostTopoDTO.setLastTime(event.getCreateTime()); + } + } + private void handleOneEventIndeed(ResourceEvent event) { String eventType = event.getEventType(); HostTopoDTO hostTopoDTO = HostTopoDTO.fromHostRelationEvent(event.getDetail()); + setDefaultLastTimeForHostTopoIfNeed(event, hostTopoDTO); switch (eventType) { case ResourceWatchReq.EVENT_TYPE_CREATE: // 插入拓扑数据 - hostTopoDAO.insertHostTopo(hostTopoDTO); + int insertedHostTopoNum = hostTopoDAO.insertHostTopo(hostTopoDTO); // 同步拓扑数据至主机表冗余字段 - updateTopoToHost(hostTopoDTO); + int affectedHostNum = updateTopoToHost(hostTopoDTO); // 更新主机缓存 - updateHostCacheWhenRelCreated(hostTopoDTO); + boolean cacheUpdated = updateOrDeleteHostCache(hostTopoDTO); + log.info( + "create event handle result: insertedHostTopoNum={}, affectedHostNum={}, cacheUpdated={}", + insertedHostTopoNum, + affectedHostNum, + cacheUpdated + ); break; case ResourceWatchReq.EVENT_TYPE_DELETE: // 删除拓扑数据 - hostTopoDAO.deleteHostTopo( + int deletedHostTopoNum = hostTopoDAO.deleteHostTopoBeforeOrEqualLastTime( hostTopoDTO.getHostId(), hostTopoDTO.getBizId(), hostTopoDTO.getSetId(), - hostTopoDTO.getModuleId() + hostTopoDTO.getModuleId(), + hostTopoDTO.getLastTime() ); - // 同步拓扑数据至主机表冗余字段 - updateTopoToHost(hostTopoDTO); - // 更新主机缓存 - updateHostCacheWhenRelationDeleted(hostTopoDTO); + if (deletedHostTopoNum > 0) { + // 同步拓扑数据至主机表冗余字段 + int deleteEventAffectedHostNum = updateTopoToHost(hostTopoDTO); + // 更新主机缓存 + boolean deleteEventCacheUpdated = updateOrDeleteHostCache(hostTopoDTO); + log.info( + "delete event handle result: deletedHostTopoNum={}, deleteEventAffectedHostNum={}," + + " deleteEventCacheUpdated={}", + deletedHostTopoNum, + deleteEventAffectedHostNum, + deleteEventCacheUpdated + ); + } else { + log.warn("no hostTopo deleted, delete event may expire for long time"); + } break; default: break; @@ -130,57 +157,45 @@ private void handleOneEventIndeed(ResourceEvent event) * 将主机拓扑表中的拓扑数据同步至主机表 * * @param hostTopoDTO 主机拓扑信息 + * @return 受影响的主机数量 */ - private void updateTopoToHost(HostTopoDTO hostTopoDTO) { + private int updateTopoToHost(HostTopoDTO hostTopoDTO) { // 若主机存在需将拓扑信息同步至主机信息冗余字段 - int affectedNum = applicationHostDAO.syncHostTopo(hostTopoDTO.getHostId()); - if (affectedNum > 0) { - log.info("host topo synced: affectedNum={}", affectedNum); - } else if (affectedNum == 0) { + int affectedHostNum = applicationHostDAO.syncHostTopo(hostTopoDTO.getHostId()); + if (affectedHostNum > 0) { + log.info("host topo synced: affectedHostNum={}", affectedHostNum); + } else if (affectedHostNum == 0) { log.info("no host topo synced"); } else { log.warn("cannot find hostInfo by hostId:{}, wait for host event or sync", hostTopoDTO.getHostId()); } + return affectedHostNum; } /** - * 当主机关系被创建时，更新缓存中的主机信息 + * 更新或删除缓存中的主机信息 * * @param hostTopoDTO 主机拓扑信息 + * @return 是否执行了更新/删除缓存动作 */ - private void updateHostCacheWhenRelCreated(HostTopoDTO hostTopoDTO) { - ApplicationHostDTO host = applicationHostDAO.getHostById(hostTopoDTO.getHostId()); - if (host != null && applicationService.existBiz(host.getBizId())) { - hostCache.addOrUpdateHost(host); - log.info("host cached updated: hostId={}", host.getHostId()); - } - } - - /** - * 当主机关系被删除时，更新缓存中的主机信息 - * - * @param hostTopoDTO 主机拓扑信息 - */ - private void updateHostCacheWhenRelationDeleted(HostTopoDTO hostTopoDTO) { + private boolean updateOrDeleteHostCache(HostTopoDTO hostTopoDTO) { ApplicationHostDTO host = applicationHostDAO.getHostById(hostTopoDTO.getHostId()); if (host == null) { - return; + log.info("host already deleted by others: hostId={}, ignore", hostTopoDTO.getHostId()); + return false; } - int curAppRelationCount = hostTopoDAO.countHostTopo(hostTopoDTO.getBizId(), hostTopoDTO.getHostId()); - int hostRelationCount = hostTopoDAO.countHostTopo(null, hostTopoDTO.getHostId()); - if (curAppRelationCount != 0) { - return; - } - if (hostRelationCount == 0) { - // 主机被移除 + if (host.getBizId() == JobConstants.PUBLIC_APP_ID) { hostCache.deleteHost(host); log.info("host cached deleted: hostId={}", host.getHostId()); + return true; + } + if (applicationService.existBiz(host.getBizId())) { + hostCache.addOrUpdateHost(host); + log.info("host cached updated: hostId={}", host.getHostId()); } else { - // 主机被转移到其他业务下 - if (applicationService.existBiz(host.getBizId())) { - hostCache.addOrUpdateHost(host); - log.info("host cached updated: hostId={}", host.getHostId()); - } + hostCache.deleteHost(host); + log.info("host biz({}) not exist, host cached deleted: hostId={}", host.getBizId(), host.getHostId()); } + return true; } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/HostRelationEventWatcher.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/HostRelationEventWatcher.java index 3be788e2ff..cae9d87e8e 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/HostRelationEventWatcher.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/HostRelationEventWatcher.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -33,7 +33,6 @@ import com.tencent.bk.job.manage.manager.host.HostCache; import com.tencent.bk.job.manage.metrics.CmdbEventSampler; import com.tencent.bk.job.manage.metrics.MetricsConstants; -import com.tencent.bk.job.manage.model.dto.HostTopoDTO; import com.tencent.bk.job.manage.service.ApplicationService; import io.micrometer.core.instrument.Tag; import io.micrometer.core.instrument.Tags; @@ -123,7 +122,7 @@ protected ResourceWatchResult fetchEventsByStartTime(Lo } @Override - protected void handleEvent(ResourceEvent event) { + public void handleEvent(ResourceEvent event) { dispatchEventToHandler(event); } @@ -155,8 +154,6 @@ public void setWatchFlag(boolean value) { } private void dispatchEventToHandler(ResourceEvent event) { - HostTopoDTO hostTopoDTO = HostTopoDTO.fromHostRelationEvent(event.getDetail()); - Long appId = hostTopoDTO.getBizId(); - eventsHandler.commitEvent(appId, event); + eventsHandler.commitEvent(event); } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/HostSyncService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/HostSyncService.java index b7f029df50..48ad9f514d 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/HostSyncService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/HostSyncService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,21 +25,38 @@ package com.tencent.bk.job.manage.service.impl.sync; import com.tencent.bk.job.common.cc.model.CcInstanceDTO; -import com.tencent.bk.job.common.cc.sdk.CmdbClientFactory; -import com.tencent.bk.job.common.cc.sdk.IBizCmdbClient; +import com.tencent.bk.job.common.cc.model.result.HostProp; +import com.tencent.bk.job.common.cc.model.result.HostWithModules; +import com.tencent.bk.job.common.cc.model.result.ModuleProp; +import com.tencent.bk.job.common.cc.sdk.BizCmdbClient; import com.tencent.bk.job.common.constant.CcNodeTypeEnum; import com.tencent.bk.job.common.model.dto.ApplicationDTO; import com.tencent.bk.job.common.model.dto.ApplicationHostDTO; +import com.tencent.bk.job.common.model.dto.BasicHostDTO; +import com.tencent.bk.job.common.util.StringUtil; +import com.tencent.bk.job.common.util.TimeUtil; +import com.tencent.bk.job.common.watch.SafeWatch; import com.tencent.bk.job.manage.dao.ApplicationHostDAO; -import com.tencent.bk.job.manage.service.HostService; +import com.tencent.bk.job.manage.dao.HostTopoDAO; +import com.tencent.bk.job.manage.model.dto.HostTopoDTO; +import com.tencent.bk.job.manage.service.host.HostService; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.commons.lang3.tuple.Triple; +import org.slf4j.helpers.FormattingTuple; +import org.slf4j.helpers.MessageFormatter; import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.data.util.Pair; import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; import org.springframework.util.StopWatch; import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -52,181 +69,895 @@ public class HostSyncService { private final AppHostsUpdateHelper appHostsUpdateHelper; private final ApplicationHostDAO applicationHostDAO; + private final HostTopoDAO hostTopoDAO; private final HostService hostService; + private final BizCmdbClient bizCmdbClient; @Autowired public HostSyncService(AppHostsUpdateHelper appHostsUpdateHelper, ApplicationHostDAO applicationHostDAO, - HostService hostService) { + HostTopoDAO hostTopoDAO, + HostService hostService, + BizCmdbClient bizCmdbClient) { this.appHostsUpdateHelper = appHostsUpdateHelper; this.applicationHostDAO = applicationHostDAO; + this.hostTopoDAO = hostTopoDAO; this.hostService = hostService; + this.bizCmdbClient = bizCmdbClient; } - private List getHostsByAppInfo(IBizCmdbClient bizCmdbClient, ApplicationDTO applicationDTO) { + private List getHostRelationsFromCmdb(ApplicationDTO bizApp) { List ccInstanceDTOList = new ArrayList<>(); - ccInstanceDTOList.add(new CcInstanceDTO(CcNodeTypeEnum.BIZ.getType(), applicationDTO.getBizIdIfBizApp())); - List hosts = bizCmdbClient.getHosts(applicationDTO.getBizIdIfBizApp(), - ccInstanceDTOList); - // 获取Agent状态 - hostService.fillAgentStatus(hosts); - return hosts; + ccInstanceDTOList.add(new CcInstanceDTO(CcNodeTypeEnum.BIZ.getType(), bizApp.getBizIdIfBizApp())); + return bizCmdbClient.getHostRelationsByTopology( + bizApp.getBizIdIfBizApp(), + ccInstanceDTOList + ); } - private List computeInsertList( + private List computeInsertHostList( Long bizId, - Set localBizHostIds, - List applicationHostDTOList + Set localHostIds, + List hostPropList, + long cmdbHostsFetchTimeMills ) { - StopWatch watch = new StopWatch(); - List insertList = - applicationHostDTOList.stream().filter(ApplicationHostDTO -> - !localBizHostIds.contains(ApplicationHostDTO.getHostId())).collect(Collectors.toList()); - watch.start("log insertList"); - log.info(String.format("bizId=%s,insertHostIds=%s", bizId, String.join(",", - insertList.stream().map(ApplicationHostDTO::getHostId).map(Object::toString) - .collect(Collectors.toSet())))); - watch.stop(); - if (watch.getTotalTimeMillis() > 1000) { - log.warn("Write log too slow, {}", watch.prettyPrint()); + Set insertHostIdSet = new HashSet<>(); + List insertHostList = new ArrayList<>(); + for (HostProp prop : hostPropList) { + Long hostId = prop.getHostId(); + if (localHostIds.contains(hostId)) { + continue; + } + if (insertHostIdSet.contains(hostId)) { + continue; + } + ApplicationHostDTO hostDTO = prop.toApplicationHostDTO(); + Long lastTime = hostDTO.getLastTime(); + if (lastTime == null || lastTime < 0) { + hostDTO.setLastTime(cmdbHostsFetchTimeMills); + log.warn( + "cmdbHostLastTime({}) is invalid, use cmdbHostsFetchTimeMills({}) for insert, host={}", + lastTime, + cmdbHostsFetchTimeMills, + hostDTO + ); + } + hostDTO.setBizId(bizId); + insertHostIdSet.add(hostId); + insertHostList.add(hostDTO); } - return insertList; + List insertHostIdList = new ArrayList<>(insertHostIdSet); + insertHostIdList.sort(Long::compareTo); + log.info( + "bizId={}, insertHostIdList.size={}, insertHostIdList={}", + bizId, + insertHostIdList.size(), + insertHostIdList + ); + return insertHostList; } - private List computeUpdateList( + private List computeUpdateHostList( Long bizId, - Set localBizHostIds, - List applicationHostDTOList + Set localHostIds, + List localHosts, + List hostPropList, + long cmdbHostsFetchTimeMills ) { - StopWatch watch = new StopWatch(); - List updateList = - applicationHostDTOList.stream().filter(ApplicationHostDTO -> - localBizHostIds.contains(ApplicationHostDTO.getHostId())).collect(Collectors.toList()); - watch.start("log updateList"); - log.info(String.format("bizId=%s,updateHostIds=%s", bizId, String.join(",", - updateList.stream().map(ApplicationHostDTO::getHostId) - .map(Object::toString).collect(Collectors.toSet())))); - watch.stop(); - if (watch.getTotalTimeMillis() > 1000) { - log.warn("Write log too slow, {}", watch.prettyPrint()); + Map localHostsIdMap = new HashMap<>(); + for (BasicHostDTO localHost : localHosts) { + localHostsIdMap.put(localHost.getHostId(), localHost); } - return updateList; + List updateHostIdList = new ArrayList<>(); + List updateHostList = hostPropList.stream().filter(hostProp -> + needToUpdate(hostProp, localHostIds, localHostsIdMap, cmdbHostsFetchTimeMills) + ).map(hostProp -> { + ApplicationHostDTO hostDTO = hostProp.toApplicationHostDTO(); + Long lastTime = hostDTO.getLastTime(); + if (lastTime == null || lastTime < 0) { + hostDTO.setLastTime(cmdbHostsFetchTimeMills); + log.warn( + "cmdbHostLastTime({}) is invalid, use cmdbHostsFetchTimeMills({}) for update, host={}", + lastTime, + cmdbHostsFetchTimeMills, + hostDTO + ); + } + hostDTO.setBizId(bizId); + updateHostIdList.add(hostProp.getHostId()); + return hostDTO; + }).collect(Collectors.toList()); + log.info( + "bizId={}, updateHostIdList.size={}, updateHostIdList={}", + bizId, + updateHostIdList.size(), + updateHostIdList + ); + return updateHostList; } - private List computeDeleteList( - Long bizId, - Set ccBizHostIds, - List localBizHosts - ) { - StopWatch watch = new StopWatch(); - List deleteList = - localBizHosts.stream().filter(ApplicationHostDTO -> - !ccBizHostIds.contains(ApplicationHostDTO.getHostId())).collect(Collectors.toList()); - watch.start("log deleteList"); - log.info(String.format("bizId=%s,deleteHostIds=%s", bizId, String.join(",", - deleteList.stream().map(ApplicationHostDTO::getHostId).map(Object::toString) - .collect(Collectors.toSet())))); - watch.stop(); - if (watch.getTotalTimeMillis() > 1000) { - log.warn("Write log too slow, {}", watch.prettyPrint()); + /** + * 判断主机信息是否需要更新 + * + * @param hostProp 从CMDB获取的主机信息 + * @param localHostIds 本地主机ID列表 + * @param localHostsIdMap 本地主机ID与主机信息映射表 + * @param cmdbHostsFetchTimeMills CMDB数据获取时间 + * @return 本地库中主机信息是否需要更新 + */ + private boolean needToUpdate(HostProp hostProp, + Set localHostIds, + Map localHostsIdMap, + long cmdbHostsFetchTimeMills) { + if (!localHostIds.contains(hostProp.getHostId())) { + return false; + } + Long lastTime = null; + if (StringUtils.isNotBlank(hostProp.getLastTime())) { + lastTime = TimeUtil.parseIsoZonedTimeToMillis(hostProp.getLastTime()); + } + if (lastTime == null || lastTime < 0) { + lastTime = cmdbHostsFetchTimeMills; + log.warn( + "cmdbHostLastTime({}) is invalid, use cmdbHostsFetchTimeMills({}) to check whether update, host={}", + hostProp.getLastTime(), + cmdbHostsFetchTimeMills, + hostProp + ); + } + BasicHostDTO localHost = localHostsIdMap.get(hostProp.getHostId()); + if (localHost.getLastTime() >= lastTime) { + log.debug( + "local host(hostId={}, lastTime={}) is not older than target host last time({}), ignore update", + localHost.getHostId(), + localHost.getLastTime(), + cmdbHostsFetchTimeMills + ); + return false; + } else { + log.info( + "local host(hostId={}, lastTime={}) is older than target host last time({}), need to update", + localHost.getHostId(), + localHost.getLastTime(), + cmdbHostsFetchTimeMills + ); + return true; } - return deleteList; } - private void refreshBizHosts(Long bizId, - List applicationHostDTOList) { - StopWatch watch = new StopWatch(); - //找出要删除的/更新的/新增的分别处理 - //对比库中数据与接口数据 - watch.start("listHostInfoByBizId"); - List localBizHosts = applicationHostDAO.listHostInfoByBizId(bizId); - watch.stop(); - watch.start("mapTo ccBizHostIds"); - Set ccBizHostIds = - applicationHostDTOList.stream().map(ApplicationHostDTO::getHostId).collect(Collectors.toSet()); - watch.stop(); - watch.start("mapTo localBizHostIds"); - Set localBizHostIds = - localBizHosts.stream().map(ApplicationHostDTO::getHostId).collect(Collectors.toSet()); - watch.stop(); - watch.start("log ccBizHostIds"); - log.info(String.format("bizId=%s,ccBizHostIds=%s", bizId, String.join(",", - ccBizHostIds.stream().map(Object::toString).collect(Collectors.toSet())))); - watch.stop(); - watch.start("log localBizHostIds"); - log.info(String.format("bizId=%s,localBizHostIds=%s", bizId, String.join(",", - localBizHostIds.stream().map(Object::toString).collect(Collectors.toSet())))); - watch.stop(); - watch.start("compute insertList"); - List insertList = computeInsertList(bizId, localBizHostIds, applicationHostDTOList); - watch.stop(); - watch.start("compute updateList"); - List updateList = computeUpdateList(bizId, localBizHostIds, applicationHostDTOList); - watch.stop(); - watch.start("compute deleteList"); - List deleteList = computeDeleteList(bizId, ccBizHostIds, localBizHosts); - watch.stop(); - watch.start("deleteHostsFromBiz"); - // 记录一次业务主机同步过程中所有更新失败的主机ID - // 需要从业务下移除的主机 - List removeFailHostIds = hostService.removeHostsFromBiz(bizId, deleteList); - watch.stop(); - watch.start("insertHostsToApp"); - // 需要新增的主机 - List insertFailHostIds = hostService.insertHostsToBiz(bizId, insertList); - watch.stop(); - watch.start("updateHostsInApp"); - // 需要更新的主机 - List updateFailHostIds = hostService.updateHostsInBiz(bizId, updateList); - watch.stop(); - if (watch.getTotalTimeMillis() > 10000) { - log.info("Performance:refreshBizHosts:bizId={},{}", bizId, watch.prettyPrint()); + /** + * 使用从CMDB获取的单个业务下的主机、主机关系去更新DB中的对应数据 + * + * @param bizId CMDB业务ID + * @param hostWithModulesList 主机与模块信息 + * @param cmdbHostsFetchTimeMills CMDB主机信息获取时间 + * @return CMDB主机ID集合 + */ + private Set refreshBizHostAndRelations(Long bizId, + List hostWithModulesList, + long cmdbHostsFetchTimeMills) { + SafeWatch watch = new SafeWatch(); + // CMDB数据拆分 + Set cmdbHostIds = new HashSet<>(); + Set cmdbBasicHosts = new HashSet<>(); + List hostPropList = new ArrayList<>(); + for (HostWithModules hostWithModules : hostWithModulesList) { + HostProp hostProp = hostWithModules.getHost(); + if (hostProp != null) { + hostPropList.add(hostProp); + cmdbHostIds.add(hostProp.getHostId()); + Long lastTimeMills = null; + if (StringUtils.isNotBlank(hostProp.getLastTime())) { + lastTimeMills = TimeUtil.parseIsoZonedTimeToMillis(hostProp.getLastTime()); + } + BasicHostDTO cmdbBasicHost = new BasicHostDTO( + hostProp.getHostId(), + lastTimeMills + ); + cmdbBasicHosts.add(cmdbBasicHost); + } + } + // 本地数据：主机 + List localHostList = applicationHostDAO.listBasicHostInfo(cmdbHostIds); + Set localHostIds = localHostList.stream().map(BasicHostDTO::getHostId).collect(Collectors.toSet()); + + logCmdbHostIds(watch, bizId, cmdbHostIds); + logLocalHostIds(watch, bizId, localHostIds); + + // 本地数据：主机关系 + List localHostTopoList = hostTopoDAO.listHostTopoByHostIds(cmdbHostIds); + + // 对比CMDB数据与本地数据找出要新增的主机关系 + List insertHostTopoList = computeInsertHostTopoList( + bizId, + localHostTopoList, + hostWithModulesList, + cmdbHostsFetchTimeMills + ); + + // 对比CMDB数据与本地数据找出要更新的主机关系 + List updateHostTopoList = computeUpdateHostTopoList( + bizId, + localHostTopoList, + hostWithModulesList, + cmdbHostsFetchTimeMills + ); + + // 对比CMDB数据与本地数据找出要删除的主机关系 + List deleteHostTopoList = computeDeleteHostTopoList( + bizId, + localHostTopoList, + hostWithModulesList, + cmdbHostsFetchTimeMills + ); + + // 刷新主机关系数据 + Triple refreshHostTopoResult = refreshHostTopos( + insertHostTopoList, + updateHostTopoList, + deleteHostTopoList, + watch + ); + int insertedHostTopoNum = refreshHostTopoResult.getLeft(); + int updatedHostTopoNum = refreshHostTopoResult.getMiddle(); + int deletedHostTopoNum = refreshHostTopoResult.getRight(); + + // 对比CMDB数据与本地数据找出要新增的主机 + List insertHostList = computeInsertHostList( + bizId, + localHostIds, + hostPropList, + cmdbHostsFetchTimeMills + ); + + // 对比CMDB数据与本地数据找出要更新的主机 + List updateHostList = computeUpdateHostList( + bizId, + localHostIds, + localHostList, + hostPropList, + cmdbHostsFetchTimeMills + ); + + // 刷新主机数据 + Pair refreshHostResult = refreshHosts(insertHostList, updateHostList, watch); + int insertedHostNum = refreshHostResult.getLeft(); + int updatedHostNum = refreshHostResult.getRight(); + + logRefreshResult( + bizId, + insertedHostTopoNum, + updatedHostTopoNum, + deletedHostTopoNum, + insertedHostNum, + updatedHostNum, + watch + ); + return cmdbBasicHosts; + } + + /** + * 根据本地主机拓扑数据与从CMDB获取的主机拓扑数据计算需要新增的拓扑数据 + * + * @param bizId 业务ID + * @param localHostTopoList 本地主机拓扑数据 + * @param hostWithModulesList 从CMDB获取的主机拓扑数据 + * @return 需要新增的主机拓扑数据 + */ + private List computeInsertHostTopoList(Long bizId, + List localHostTopoList, + List hostWithModulesList, + long cmdbHostsFetchTimeMills) { + List insertHostTopoList = new ArrayList<>(); + Set localTopoKeys = new HashSet<>(); + for (HostTopoDTO hostTopoDTO : localHostTopoList) { + localTopoKeys.add(buildTopoKey(hostTopoDTO)); + } + Set insertTopoKeys = new HashSet<>(); + for (HostWithModules hostWithModules : hostWithModulesList) { + HostProp host = hostWithModules.getHost(); + List modules = hostWithModules.getModules(); + if (CollectionUtils.isEmpty(modules)) { + continue; + } + for (ModuleProp module : modules) { + if (module == null) { + continue; + } + String topoKey = buildTopoKey(bizId, host, module); + if (localTopoKeys.contains(topoKey)) { + continue; + } + // 避免产生重复的拓扑数据导致后续批量插入失败 + if (insertTopoKeys.contains(topoKey)) { + continue; + } + HostTopoDTO hostTopo = buildHostTopo(bizId, host, module); + fillLastTimeIfNeed(hostTopo, cmdbHostsFetchTimeMills); + insertTopoKeys.add(topoKey); + insertHostTopoList.add(hostTopo); + } + } + log.info( + "bizId={}, insertHostTopoList.size={}, insertHostTopoList={}", + bizId, + insertHostTopoList.size(), + insertHostTopoList + ); + return insertHostTopoList; + } + + /** + * 在last_time字段缺失时为主机拓扑数据填充CMDB主机信息获取时间作为last_time + * + * @param hostTopo 主机拓扑 + * @param cmdbHostsFetchTimeMills CMDB主机信息获取时间 + */ + private void fillLastTimeIfNeed(HostTopoDTO hostTopo, long cmdbHostsFetchTimeMills) { + Long lastTime = hostTopo.getLastTime(); + if (lastTime != null && lastTime >= 0) { + return; + } + hostTopo.setLastTime(cmdbHostsFetchTimeMills); + log.warn( + "cmdbHostTopoLastTime({}) is invalid, use cmdbHostsFetchTimeMills({}) for insert, hostTopo={}", + lastTime, + cmdbHostsFetchTimeMills, + hostTopo + ); + } + + /** + * 根据本地主机拓扑数据与从CMDB获取的主机拓扑数据计算需要更新的拓扑数据 + * + * @param bizId 业务ID + * @param localHostTopoList 本地主机拓扑数据 + * @param hostWithModulesList 从CMDB获取的主机拓扑数据 + * @param cmdbHostsFetchTimeMills CMDB主机信息获取时间 + * @return 需要更新的主机拓扑数据 + */ + private List computeUpdateHostTopoList(Long bizId, + List localHostTopoList, + List hostWithModulesList, + long cmdbHostsFetchTimeMills) { + List updateHostTopoList = new ArrayList<>(); + Map localTopoMap = new HashMap<>(); + for (HostTopoDTO hostTopoDTO : localHostTopoList) { + localTopoMap.put(buildTopoKey(hostTopoDTO), hostTopoDTO); + } + for (HostWithModules hostWithModules : hostWithModulesList) { + HostProp host = hostWithModules.getHost(); + List modules = hostWithModules.getModules(); + if (CollectionUtils.isEmpty(modules)) { + continue; + } + for (ModuleProp module : modules) { + if (module == null) { + continue; + } + String topoKey = buildTopoKey(bizId, host, module); + if (!localTopoMap.containsKey(topoKey)) { + continue; + } + HostTopoDTO localHostTopo = localTopoMap.get(topoKey); + HostTopoDTO cmdbHostTopo = buildHostTopo(bizId, host, module); + Long cmdbHostTopoLastTime = cmdbHostTopo.getLastTime(); + if (cmdbHostTopoLastTime == null || cmdbHostTopoLastTime < 0) { + cmdbHostTopoLastTime = cmdbHostsFetchTimeMills; + cmdbHostTopo.setLastTime(cmdbHostTopoLastTime); + log.warn( + "cmdbHostTopoLastTime({}) is invalid, use cmdbHostsFetchTimeMills({}) for update, " + + "cmdbHostTopo={}", + cmdbHostTopoLastTime, + cmdbHostsFetchTimeMills, + cmdbHostTopo + ); + } + if (localHostTopo.getLastTime() < cmdbHostTopoLastTime) { + updateHostTopoList.add(cmdbHostTopo); + log.info( + "local hostTopo({}) is older than cmdb hostTopo time({}), need to update", + localHostTopo, + cmdbHostTopoLastTime + ); + } else { + log.debug( + "local hostTopo({}) is not older than cmdb hostTopo time({}), do not update", + localHostTopo, + cmdbHostTopoLastTime + ); + } + } + } + logUpdateHostTopoList(bizId, updateHostTopoList); + return updateHostTopoList; + } + + private void logUpdateHostTopoList(Long bizId, List updateHostTopoList) { + if (log.isDebugEnabled()) { + log.debug( + "bizId={}, updateHostTopoList.size={}, updateHostTopoList={}", + bizId, + updateHostTopoList.size(), + updateHostTopoList + ); + return; + } + if (updateHostTopoList.size() < 200) { + log.info( + "bizId={}, updateHostTopoList.size={}, updateHostTopoList={}", + bizId, + updateHostTopoList.size(), + updateHostTopoList + ); + return; + } + log.info( + "bizId={}, updateHostTopoList.size={}", + bizId, + updateHostTopoList.size() + ); + } + + /** + * 根据本地主机拓扑数据与从CMDB获取的主机拓扑数据计算需要删除的拓扑数据 + * + * @param bizId 业务ID + * @param localHostTopoList 本地主机拓扑数据 + * @param hostWithModulesList 从CMDB获取的主机拓扑数据 + * @param cmdbHostsFetchTimeMills CMDB主机信息获取时间 + * @return 需要更新的主机拓扑数据 + */ + private List computeDeleteHostTopoList(Long bizId, + List localHostTopoList, + List hostWithModulesList, + long cmdbHostsFetchTimeMills) { + List deleteHostTopoList = new ArrayList<>(); + Set cmdbHostTopoKeys = new HashSet<>(); + + for (HostWithModules hostWithModules : hostWithModulesList) { + HostProp host = hostWithModules.getHost(); + List modules = hostWithModules.getModules(); + if (CollectionUtils.isEmpty(modules)) { + continue; + } + for (ModuleProp module : modules) { + if (module == null) { + continue; + } + String topoKey = buildTopoKey(bizId, host, module); + cmdbHostTopoKeys.add(topoKey); + } + } + for (HostTopoDTO localHostTopo : localHostTopoList) { + String topoKey = buildTopoKey(localHostTopo); + if (cmdbHostTopoKeys.contains(topoKey)) { + continue; + } + if (localHostTopo.getLastTime() < cmdbHostsFetchTimeMills) { + deleteHostTopoList.add(localHostTopo); + log.info( + "local hostTopo({}) is older than cmdb hostTopo fetch time({}), need to delete", + localHostTopo, + cmdbHostsFetchTimeMills + ); + } else { + log.info( + "local hostTopo({}) is not older than cmdb hostTopo fetch time({}), do not delete", + localHostTopo, + cmdbHostsFetchTimeMills + ); + } + } + log.info( + "bizId={}, deleteHostTopoList.size={}, deleteHostTopoList={}", + bizId, + deleteHostTopoList.size(), + deleteHostTopoList + ); + return deleteHostTopoList; + } + + private Triple refreshHostTopos(List insertHostTopoList, + List updateHostTopoList, + List deleteHostTopoList, + StopWatch watch) { + int deletedNum = applyDeleteHostTopoList(deleteHostTopoList, watch); + int insertedNum = applyInsertHostTopoList(insertHostTopoList, watch); + int updatedNum = applyUpdateHostTopoList(updateHostTopoList, watch); + return Triple.of(insertedNum, updatedNum, deletedNum); + } + + private int applyDeleteHostTopoList(List deleteHostTopoList, + StopWatch watch) { + int deletedNum = 0; + try { + watch.start("batchDeleteWithLastTime"); + deletedNum = hostTopoDAO.batchDeleteWithLastTime(deleteHostTopoList); + watch.stop(); + } catch (Exception e) { + log.error("Fail to batchDeleteWithLastTime, try to delete one by one", e); + watch.start("deleteHostTopoBeforeOrEqualLastTime one by one"); + for (HostTopoDTO hostTopoDTO : deleteHostTopoList) { + deletedNum += tryToDeleteHostTopoBeforeOrEqualLastTime(hostTopoDTO); + } + watch.stop(); + } + return deletedNum; + } + + private int tryToDeleteHostTopoBeforeOrEqualLastTime(HostTopoDTO hostTopoDTO) { + try { + return hostTopoDAO.deleteHostTopoBeforeOrEqualLastTime( + hostTopoDTO.getHostId(), + hostTopoDTO.getBizId(), + hostTopoDTO.getSetId(), + hostTopoDTO.getModuleId(), + hostTopoDTO.getLastTime() + ); + } catch (Exception e) { + FormattingTuple msg = MessageFormatter.format( + "Fail to deleteHostTopoBeforeOrEqualLastTime, hostTopo={}", + hostTopoDTO + ); + log.error(msg.getMessage(), e); + return 0; + } + } + + private int applyInsertHostTopoList(List insertHostTopoList, + StopWatch watch) { + int insertedNum = 0; + try { + watch.start("batchInsertHostTopo"); + insertedNum = hostTopoDAO.batchInsertHostTopo(insertHostTopoList); + watch.stop(); + } catch (Exception e) { + log.error("Fail to batchInsertHostTopo, try to insert one by one", e); + watch.start("insertHostTopo one by one"); + for (HostTopoDTO hostTopoDTO : insertHostTopoList) { + insertedNum += tryToInsertHostTopo(hostTopoDTO); + } + watch.stop(); + } + return insertedNum; + } + + private int tryToInsertHostTopo(HostTopoDTO hostTopoDTO) { + try { + return hostTopoDAO.insertHostTopo(hostTopoDTO); + } catch (Exception e) { + FormattingTuple msg = MessageFormatter.format( + "Fail to insertHostTopo, hostTopo={}", + hostTopoDTO + ); + log.error(msg.getMessage(), e); + return 0; + } + } + + private int applyUpdateHostTopoList(List updateHostTopoList, + StopWatch watch) { + int updatedNum = 0; + try { + watch.start("batchUpdateBeforeLastTime"); + updatedNum = hostTopoDAO.batchUpdateBeforeLastTime(updateHostTopoList); + watch.stop(); + } catch (Exception e) { + log.error("Fail to batchUpdateBeforeLastTime, try to update one by one", e); + watch.start("updateBeforeLastTime one by one"); + for (HostTopoDTO hostTopoDTO : updateHostTopoList) { + updatedNum += tryToUpdateBeforeLastTime(hostTopoDTO); + } + watch.stop(); + } + return updatedNum; + } + + private int tryToUpdateBeforeLastTime(HostTopoDTO hostTopoDTO) { + try { + return hostTopoDAO.updateBeforeLastTime(hostTopoDTO); + } catch (Exception e) { + FormattingTuple msg = MessageFormatter.format( + "Fail to updateBeforeLastTime, hostTopo={}", + hostTopoDTO + ); + log.error(msg.getMessage(), e); + return 0; + } + } + + private String buildTopoKey(HostTopoDTO hostTopoDTO) { + String delimiter = "_"; + return hostTopoDTO.getHostId() + + delimiter + + hostTopoDTO.getBizId() + + delimiter + + hostTopoDTO.getSetId() + + delimiter + + hostTopoDTO.getModuleId(); + } + + private String buildTopoKey(Long bizId, HostProp host, ModuleProp moduleProp) { + String delimiter = "_"; + return host.getHostId() + + delimiter + + bizId + + delimiter + + moduleProp.getSetId() + + delimiter + + moduleProp.getModuleId(); + } + + private HostTopoDTO buildHostTopo(Long bizId, HostProp host, ModuleProp moduleProp) { + Long lastTimeMills = null; + if (StringUtils.isNotBlank(moduleProp.getLastTime())) { + lastTimeMills = TimeUtil.parseIsoZonedTimeToMillis(moduleProp.getLastTime()); + } + return new HostTopoDTO( + host.getHostId(), + bizId, + moduleProp.getSetId(), + moduleProp.getModuleId(), + lastTimeMills + ); + } + + private Pair refreshHosts(List insertHostList, + List updateHostList, + StopWatch watch) { + int insertNum = applyInsertHostList(insertHostList, watch); + int updateNum = applyUpdateHostList(updateHostList, watch); + return Pair.of(insertNum, updateNum); + } + + private int applyInsertHostList(List insertHostList, + StopWatch watch) { + int insertNum = 0; + try { + watch.start("batchInsertHosts"); + // 新增主机 + insertNum = hostService.batchInsertHosts(insertHostList); + watch.stop(); + } catch (Exception e) { + log.error("Fail to batchInsertHosts, try to insert one by one", e); + watch.start("insertHosts one by one"); + for (ApplicationHostDTO hostDTO : insertHostList) { + insertNum += tryToInsertOneHost(hostDTO); + } + watch.stop(); + } + return insertNum; + } + + private int tryToInsertOneHost(ApplicationHostDTO hostDTO) { + try { + Pair pair = hostService.createOrUpdateHostBeforeLastTime(hostDTO); + return pair.getRight(); + } catch (Exception e) { + FormattingTuple msg = MessageFormatter.format("Fail to insert host={}", hostDTO); + log.error(msg.getMessage(), e); + return 0; + } + } + + private int applyUpdateHostList(List updateHostList, + StopWatch watch) { + int updateNum = 0; + try { + watch.start("batchUpdateHostsBeforeLastTime"); + // 更新主机 + updateNum = hostService.batchUpdateHostsBeforeLastTime(updateHostList); + watch.stop(); + } catch (Exception e) { + log.error("Fail to batchUpdateHostsBeforeLastTime, try to update one by one", e); + watch.start("updateHosts one by one"); + for (ApplicationHostDTO hostDTO : updateHostList) { + updateNum += tryToUpdateOneHost(hostDTO); + } + watch.stop(); + } + return updateNum; + } + + private int tryToUpdateOneHost(ApplicationHostDTO hostDTO) { + try { + return hostService.updateHostAttrsBeforeLastTime(hostDTO); + } catch (Exception e) { + FormattingTuple msg = MessageFormatter.format( + "Fail to updateHostAttrsBeforeLastTime, host={}", + hostDTO + ); + log.error(msg.getMessage(), e); + return 0; + } + } + + private void logRefreshResult(Long bizId, + int insertedHostTopoNum, + int updatedHostTopoNum, + int deletedHostTopoNum, + int insertedHostNum, + int updatedHostNum, + StopWatch watch) { + if (watch.getTotalTimeMillis() > 300_000) { + log.warn("Performance:refreshBizHostAndRelations: bizId={}, {}", bizId, watch.prettyPrint()); + } else if (watch.getTotalTimeMillis() > 50_000) { + log.info("Performance:refreshBizHostAndRelations: bizId={}, {}", bizId, watch.prettyPrint()); } else { - log.debug("Performance:refreshBizHosts:bizId={},{}", bizId, watch.prettyPrint()); + if (log.isDebugEnabled()) { + log.debug("Performance:refreshBizHostAndRelations: bizId={}, {}", bizId, watch.prettyPrint()); + } } log.info( - Thread.currentThread().getName() + - ":Finished:Statistics:bizId={}:insertFailHostIds={}," + - "updateFailHostIds={},removeFailHostIds={}", + "RefreshBizHostAndRelationsStatistics:bizId={}, insertedHostTopoNum={}, " + + "updatedHostTopoNum={}, deletedHostTopoNum={}, insertedHostNum={}, updatedHostNum={}", + bizId, + insertedHostTopoNum, + updatedHostTopoNum, + deletedHostTopoNum, + insertedHostNum, + updatedHostNum + ); + } + + private void logCmdbHostIds(StopWatch watch, Long bizId, Collection cmdbHostIds) { + watch.start("logCmdbHostIds"); + log.info( + "bizId={}, cmdbHostIds.size={}, cmdbHostIds={}", bizId, - insertFailHostIds, - updateFailHostIds, - removeFailHostIds + cmdbHostIds.size(), + StringUtil.concatCollection(cmdbHostIds) ); + watch.stop(); } - private Pair syncBizHostsIndeed(ApplicationDTO applicationDTO) { - Long bizId = Long.valueOf(applicationDTO.getScope().getId()); - Long cmdbInterfaceTimeConsuming = 0L; - Long writeToDBTimeConsuming = 0L; - IBizCmdbClient bizCmdbClient = CmdbClientFactory.getCmdbClient(); - StopWatch bizHostsWatch = new StopWatch(); - bizHostsWatch.start("getHostsByAppInfo from CMDB"); - Long startTime = System.currentTimeMillis(); + private void logLocalHostIds(StopWatch watch, Long bizId, Collection localHostIds) { + watch.start("logLocalHostIds"); + log.info( + "bizId={}, localHostIds.size={}, localHostIds={}", + bizId, + localHostIds.size(), + StringUtil.concatCollection(localHostIds) + ); + watch.stop(); + } + + private Triple, Long, Long> syncBizHostsIndeed(ApplicationDTO bizApp) { + Long bizId = Long.valueOf(bizApp.getScope().getId()); + long cmdbInterfaceTimeConsuming = 0L; + long writeToDBTimeConsuming = 0L; + SafeWatch bizHostsWatch = new SafeWatch(); + bizHostsWatch.start("getHostRelationsFromCmdb"); + long startTime = System.currentTimeMillis(); log.info("begin to syncBizHosts:bizId={}", bizId); - List hosts = getHostsByAppInfo(bizCmdbClient, applicationDTO); + long cmdbHostsFetchTimeMills = System.currentTimeMillis(); + List hostRelationsFromCmdb = getHostRelationsFromCmdb(bizApp); cmdbInterfaceTimeConsuming += (System.currentTimeMillis() - startTime); bizHostsWatch.stop(); - bizHostsWatch.start("updateHosts to local DB"); + bizHostsWatch.start("refreshBizHostAndRelations to local DB"); startTime = System.currentTimeMillis(); - refreshBizHosts(bizId, hosts); + Set cmdbBasicHosts = refreshBizHostAndRelations( + bizId, + hostRelationsFromCmdb, + cmdbHostsFetchTimeMills + ); writeToDBTimeConsuming += (System.currentTimeMillis() - startTime); bizHostsWatch.stop(); - log.info("Performance:syncBizHosts:bizId={},{}", bizId, bizHostsWatch); - return Pair.of(cmdbInterfaceTimeConsuming, writeToDBTimeConsuming); + if (bizHostsWatch.getTotalTimeMillis() < 600_000L) { + log.info("Performance:syncBizHostAndRelations:bizId={},{}", bizId, bizHostsWatch); + } else { + log.warn("SLOW:Performance:syncBizHostAndRelations:bizId={},{}", bizId, bizHostsWatch); + } + return Triple.of(cmdbBasicHosts, cmdbInterfaceTimeConsuming, writeToDBTimeConsuming); } - public Pair syncBizHostsAtOnce(ApplicationDTO applicationDTO) { - Long bizId = Long.valueOf(applicationDTO.getScope().getId()); + public Triple, Long, Long> syncBizHostsAtOnce(ApplicationDTO bizApp) { + Long bizId = Long.valueOf(bizApp.getScope().getId()); try { appHostsUpdateHelper.waitAndStartBizHostsUpdating(bizId); - return syncBizHostsIndeed(applicationDTO); + return syncBizHostsIndeed(bizApp); } catch (Throwable t) { log.error("Fail to syncBizHosts of bizId " + bizId, t); return null; } finally { - appHostsUpdateHelper.endToUpdateBizHosts(bizId); + appHostsUpdateHelper.endToUpdateBizHosts(); + } + } + + public void clearHostNotInCmdb(Set cmdbBasicHosts, long cmdbHostsFetchTimeMills) { + // 删除主机拓扑 + List deleteHostTopoList = computeDeleteHostTopoList(cmdbBasicHosts, cmdbHostsFetchTimeMills); + int deletedHostTopoNum = 0; + if (!CollectionUtils.isEmpty(deleteHostTopoList)) { + for (HostTopoDTO hostTopoDTO : deleteHostTopoList) { + deletedHostTopoNum += hostTopoDAO.deleteHostTopoBeforeOrEqualLastTime( + hostTopoDTO.getHostId(), + hostTopoDTO.getBizId(), + hostTopoDTO.getSetId(), + hostTopoDTO.getModuleId(), + cmdbHostsFetchTimeMills + ); + } + log.info( + "deleteHostTopoList.size={}, deletedHostTopoNum={}, deleteHostTopoList={}", + deleteHostTopoList.size(), + deletedHostTopoNum, + deleteHostTopoList + ); + } + + // 删除主机 + List deleteHostList = computeDeleteHostList(cmdbBasicHosts, cmdbHostsFetchTimeMills); + if (!CollectionUtils.isEmpty(deleteHostList)) { + int deletedHostNum = hostService.deleteByBasicHost(deleteHostList); + log.info( + "deleteHostList.size={}, deletedHostNum={}, deleteHostIdList={}", + deleteHostList.size(), + deletedHostNum, + deleteHostList.stream().map(BasicHostDTO::getHostId).collect(Collectors.toList()) + ); + } + } + + private List computeDeleteHostTopoList( + Set cmdbBasicHosts, + long cmdbHostsFetchTimeMills + ) { + Set cmdbHostIds = cmdbBasicHosts.stream().map(BasicHostDTO::getHostId).collect(Collectors.toSet()); + List hostTopoList = hostTopoDAO.listHostTopoByExcludeHostIds(cmdbHostIds); + return hostTopoList.stream().filter(hostTopoDTO -> { + if (hostTopoDTO.getLastTime() > cmdbHostsFetchTimeMills) { + log.info( + "local hostTopo({}) is not older than cmdb hosts fetch time({}), do not delete", + hostTopoDTO, + cmdbHostsFetchTimeMills + ); + return false; + } + log.info( + "local hostTopo({}) is equal or older than cmdb hosts fetch time({}), need to delete", + hostTopoDTO, + cmdbHostsFetchTimeMills + ); + return true; + }).collect(Collectors.toList()); + } + + private List computeDeleteHostList( + Set cmdbBasicHosts, + long cmdbHostsFetchTimeMills + ) { + List localBasicHosts = hostService.listAllBasicHost(); + Map cmdbBasicHostMap = new HashMap<>(); + for (BasicHostDTO cmdbBasicHost : cmdbBasicHosts) { + cmdbBasicHostMap.put(cmdbBasicHost.getHostId(), cmdbBasicHost); + } + List deleteHostList = new ArrayList<>(); + for (BasicHostDTO localBasicHost : localBasicHosts) { + if (cmdbBasicHostMap.containsKey(localBasicHost.getHostId())) { + continue; + } + // 本地主机时间戳比从CMDB获取的主机要新，不删除 + if (localBasicHost.getLastTime() > cmdbHostsFetchTimeMills) { + log.info( + "local host(hostId={}, lastTime={}) is not older than cmdb hosts fetch time({}), do not delete", + localBasicHost.getHostId(), + localBasicHost.getLastTime(), + cmdbHostsFetchTimeMills + ); + continue; + } + log.info( + "local host(hostId={}, lastTime={}) is equal or older than cmdb hosts fetch time({}), need to " + + "delete", + localBasicHost.getHostId(), + localBasicHost.getLastTime(), + cmdbHostsFetchTimeMills + ); + deleteHostList.add(localBasicHost); } + return deleteHostList; } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/SyncServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/SyncServiceImpl.java index 29bc4db0ee..dd817f35eb 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/SyncServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/sync/SyncServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -26,6 +26,7 @@ import com.tencent.bk.job.common.constant.ResourceScopeTypeEnum; import com.tencent.bk.job.common.model.dto.ApplicationDTO; +import com.tencent.bk.job.common.model.dto.BasicHostDTO; import com.tencent.bk.job.common.model.dto.ResourceScope; import com.tencent.bk.job.common.redis.util.LockUtils; import com.tencent.bk.job.common.redis.util.RedisKeyHeartBeatThread; @@ -34,33 +35,31 @@ import com.tencent.bk.job.manage.config.JobManageConfig; import com.tencent.bk.job.manage.dao.ApplicationDAO; import com.tencent.bk.job.manage.manager.app.ApplicationCache; -import com.tencent.bk.job.manage.service.ApplicationService; import com.tencent.bk.job.manage.service.SyncService; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.tuple.Triple; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.annotation.DependsOn; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.data.util.Pair; import org.springframework.stereotype.Service; +import org.springframework.util.CollectionUtils; import org.springframework.util.StopWatch; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Set; -import java.util.concurrent.BlockingQueue; import java.util.concurrent.Future; -import java.util.concurrent.LinkedBlockingDeque; -import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; +@SuppressWarnings("FieldCanBeLocal") @Slf4j @Service -@DependsOn("redisLockConfig") public class SyncServiceImpl implements SyncService { private static final String REDIS_KEY_SYNC_APP_JOB_LOCK = "sync-app-job-lock"; @@ -70,7 +69,6 @@ public class SyncServiceImpl implements SyncService { private static final String REDIS_KEY_LAST_FINISH_TIME_SYNC_HOST = "last-finish-time-sync-host"; private static final String REDIS_KEY_LAST_FINISH_TIME_SYNC_AGENT_STATUS = "last-finish-time-sync-agent-status"; private static final String machineIp = IpUtils.getFirstMachineIP(); - private static final int MAX_RETRY_COUNT = 3; static { List keyList = Arrays.asList(REDIS_KEY_SYNC_APP_JOB_LOCK, REDIS_KEY_SYNC_HOST_JOB_LOCK, @@ -86,17 +84,13 @@ public class SyncServiceImpl implements SyncService { } private final ApplicationDAO applicationDAO; - private final ApplicationService applicationService; private final ThreadPoolExecutor syncAppExecutor; private final ThreadPoolExecutor syncHostExecutor; - private final ThreadPoolExecutor syncAgentStatusExecutor; private final JobManageConfig jobManageConfig; private final RedisTemplate redisTemplate; private final String REDIS_KEY_SYNC_APP_JOB_RUNNING_MACHINE = "sync-app-job-running-machine"; private final String REDIS_KEY_SYNC_HOST_JOB_RUNNING_MACHINE = "sync-host-job-running-machine"; private final String REDIS_KEY_SYNC_AGENT_STATUS_JOB_RUNNING_MACHINE = "sync-agent-status-job-running-machine"; - private final BlockingQueue> appHostFailQueue = new LinkedBlockingDeque<>(); - private volatile LinkedBlockingQueue extraSyncAppQueue; private volatile boolean enableSyncApp; private volatile boolean enableSyncHost; private volatile boolean enableSyncAgentStatus; @@ -117,7 +111,6 @@ public SyncServiceImpl(BizSyncService bizSyncService, HostSyncService hostSyncService, AgentStatusSyncService agentStatusSyncService, ApplicationDAO applicationDAO, - ApplicationService applicationService, JobManageConfig jobManageConfig, RedisTemplate redisTemplate, ApplicationCache applicationCache, @@ -127,10 +120,8 @@ public SyncServiceImpl(BizSyncService bizSyncService, HostEventWatcher hostEventWatcher, HostRelationEventWatcher hostRelationEventWatcher, @Qualifier("syncAppExecutor") ThreadPoolExecutor syncAppExecutor, - @Qualifier("syncHostExecutor") ThreadPoolExecutor syncHostExecutor, - @Qualifier("syncAgentStatusExecutor") ThreadPoolExecutor syncAgentStatusExecutor) { + @Qualifier("syncHostExecutor") ThreadPoolExecutor syncHostExecutor) { this.applicationDAO = applicationDAO; - this.applicationService = applicationService; this.jobManageConfig = jobManageConfig; this.redisTemplate = redisTemplate; this.enableSyncApp = jobManageConfig.isEnableSyncApp(); @@ -150,20 +141,10 @@ public SyncServiceImpl(BizSyncService bizSyncService, this.syncAppExecutor = syncAppExecutor; // 同步主机的线程池配置 this.syncHostExecutor = syncHostExecutor; - // 同步主机Agent状态的线程池配置 - this.syncAgentStatusExecutor = syncAgentStatusExecutor; } @Override public void init() { - // 额外同步业务主机的队列与线程配置 - extraSyncAppQueue = new LinkedBlockingQueue<>(500); - for (int i = 0; i < 3; i++) { - AppHostsSyncer extraAppHostsSyncer = new AppHostsSyncer(applicationService, hostSyncService, - extraSyncAppQueue); - extraAppHostsSyncer.setName("[" + extraAppHostsSyncer.getId() + "]-extraAppHostsSyncer-" + (i + 1)); - extraAppHostsSyncer.start(); - } if (jobManageConfig.isEnableResourceWatch()) { watchBizEvent(); watchHostEvent(); @@ -200,31 +181,11 @@ private void watchBizSetEvent() { bizSetRelationEventWatcher.start(); } - public boolean addExtraSyncBizHostsTask(Long bizId) { - if (extraSyncAppQueue.contains(bizId)) { - return true; - } else if (extraSyncAppQueue.remainingCapacity() > 0) { - boolean result = extraSyncAppQueue.add(bizId); - if (extraSyncAppQueue.size() > 10) { - log.warn("extraSyncAppQueue.size={},queue={}", extraSyncAppQueue.size(), extraSyncAppQueue.toString()); - } else { - log.debug("extraSyncAppQueue.size={},queue={}", extraSyncAppQueue.size(), extraSyncAppQueue.toString()); - } - return result; - } - return false; - } - @Override public ThreadPoolExecutor getSyncHostExecutor() { return syncHostExecutor; } - @Override - public ThreadPoolExecutor getSyncAgentStatusExecutor() { - return syncAgentStatusExecutor; - } - @Override public Long syncApp() { if (!enableSyncApp) { @@ -273,8 +234,6 @@ public Long syncApp() { } catch (Throwable t) { log.error("FATAL: syncApp thread fail", t); } finally { - // 失败的业务进行补偿同步 - handleFailedSyncAppHosts(); appSyncRedisKeyHeartBeatThread.setRunFlag(false); watch.stop(); log.info("syncApp time consuming:" + watch.prettyPrint()); @@ -288,44 +247,10 @@ public Long syncApp() { return 1L; } - /** - * 同步主机失败的业务的补偿性同步 - */ - private void handleFailedSyncAppHosts() { - if (appHostFailQueue.isEmpty()) { - return; - } - Pair appInfoRetryCountPair = appHostFailQueue.poll(); - int maxCount = 1000; - int count = 0; - while (appInfoRetryCountPair != null && count < maxCount) { - ApplicationDTO applicationDTO = appInfoRetryCountPair.getFirst(); - int retryCount = appInfoRetryCountPair.getSecond(); - try { - if (retryCount > 0) { - arrangeSyncAppHostsTask(applicationDTO); - } else { - log.warn("syncAppHost retry over max count, appId={}", applicationDTO.getId()); - } - } catch (Throwable t) { - count += 1; - try { - appHostFailQueue.put(Pair.of(applicationDTO, retryCount - 1)); - } catch (InterruptedException e) { - log.error("appHostFailQueue.put(Pair.of(applicationInfoDTO,retryCount-1)) fail", e); - } - } - appInfoRetryCountPair = appHostFailQueue.poll(); - } - if (count >= 1000) { - log.error("too many FailedSyncAppHosts, watch for a dead loop!"); - } - log.info("handleFailedSyncAppHosts end"); - } - - private Future> arrangeSyncAppHostsTask(ApplicationDTO applicationDTO) { + private Future, Long, Long>> arrangeSyncBizHostsTask(ApplicationDTO bizApp) { return syncHostExecutor.submit(() -> - hostSyncService.syncBizHostsAtOnce(applicationDTO)); + hostSyncService.syncBizHostsAtOnce(bizApp) + ); } @Override @@ -372,43 +297,67 @@ public Long syncHost() { .collect(Collectors.toSet()); log.info(String.format("localAppIds:%s", String.join(",", localAppIds.stream().map(Object::toString).collect(Collectors.toSet())))); - List localNormalApps = + List localBizApps = localApps.stream().filter(ApplicationDTO::isBiz).collect(Collectors.toList()); long cmdbInterfaceTimeConsuming = 0L; long writeToDBTimeConsuming = 0L; - List>>> appFutureList = new ArrayList<>(); - for (ApplicationDTO applicationDTO : localNormalApps) { - Future> future = arrangeSyncAppHostsTask(applicationDTO); - appFutureList.add(Pair.of(applicationDTO, future)); + List, Long, Long>>>> bizAppFutureList = + new ArrayList<>(); + Set allBizCmdbBasicHosts = new HashSet<>(); + int failedBizNum = 0; + long cmdbHostsFetchTimeMills = System.currentTimeMillis(); + for (ApplicationDTO bizApp : localBizApps) { + Future, Long, Long>> future = arrangeSyncBizHostsTask(bizApp); + bizAppFutureList.add(Pair.of(bizApp, future)); } - for (Pair>> appFuture : appFutureList) { - ApplicationDTO applicationDTO = appFuture.getFirst(); - Future> future = appFuture.getSecond(); + for (Pair, Long, Long>>> bizAppFuture : + bizAppFutureList) { + ApplicationDTO bizApp = bizAppFuture.getFirst(); + Future, Long, Long>> future = bizAppFuture.getSecond(); try { - Pair timeConsumingPair = future.get(30, TimeUnit.MINUTES); - cmdbInterfaceTimeConsuming += timeConsumingPair.getFirst(); - writeToDBTimeConsuming += timeConsumingPair.getSecond(); + Triple, Long, Long> timeConsumingPair = future.get(30, TimeUnit.MINUTES); + Set cmdbBasicHosts = timeConsumingPair.getLeft(); + if (!CollectionUtils.isEmpty(cmdbBasicHosts)) { + allBizCmdbBasicHosts.addAll(cmdbBasicHosts); + } + cmdbInterfaceTimeConsuming += timeConsumingPair.getMiddle(); + writeToDBTimeConsuming += timeConsumingPair.getRight(); } catch (Throwable t) { - appHostFailQueue.add(Pair.of(applicationDTO, MAX_RETRY_COUNT)); - log.error("syncHost of app fail:appId=" + applicationDTO.getId(), t); + log.error("syncHost of biz fail:bizId=" + bizApp.getBizIdIfBizApp(), t); + failedBizNum += 1; } } - log.info( - Thread.currentThread().getName() + - ":Finished:sync host from cc," + - "cmdbInterfaceTimeConsuming={}ms,writeToDBTimeConsuming={}ms,rate={}", - cmdbInterfaceTimeConsuming, - writeToDBTimeConsuming, - cmdbInterfaceTimeConsuming / (0. + writeToDBTimeConsuming) - ); + if (failedBizNum == 0) { + // 删除CMDB中不存在的主机 + hostSyncService.clearHostNotInCmdb(allBizCmdbBasicHosts, cmdbHostsFetchTimeMills); + log.info( + Thread.currentThread().getName() + + ":Finished:sync host from cc, bizNum={}, failedBizNum={}, " + + "cmdbInterfaceTimeConsuming={}ms,writeToDBTimeConsuming={}ms,rate={}", + localBizApps.size(), + failedBizNum, + cmdbInterfaceTimeConsuming, + writeToDBTimeConsuming, + cmdbInterfaceTimeConsuming / (0. + writeToDBTimeConsuming) + ); + } else { + log.warn( + Thread.currentThread().getName() + + ":Finished:sync host from cc, bizNum={}, failedBizNum={}, " + + "cmdbInterfaceTimeConsuming={}ms,writeToDBTimeConsuming={}ms,rate={}", + localBizApps.size(), + failedBizNum, + cmdbInterfaceTimeConsuming, + writeToDBTimeConsuming, + cmdbInterfaceTimeConsuming / (0. + writeToDBTimeConsuming) + ); + } // 将最后同步时间写入Redis redisTemplate.opsForValue().set(REDIS_KEY_LAST_FINISH_TIME_SYNC_HOST, "" + System.currentTimeMillis()); } catch (Throwable t) { log.error("syncHost thread fail", t); } finally { - //失败的同步任务补偿处理 - handleFailedSyncAppHosts(); hostSyncRedisKeyHeartBeatThread.setRunFlag(false); watch.stop(); log.info("syncHost time consuming:" + watch.prettyPrint()); @@ -421,12 +370,6 @@ public Long syncHost() { return 1L; } - @Override - public Future> arrangeSyncBizHostsTask(Long bizId) { - log.info("arrangeSyncAppHostsTask:appId={}", bizId); - return arrangeSyncAppHostsTask(applicationDAO.getAppById(bizId)); - } - @Override public Boolean enableBizWatch() { log.info("appWatch enabled by op"); @@ -499,11 +442,17 @@ public Boolean syncBizHosts(Long bizId) { ApplicationDTO applicationDTO = applicationDAO.getAppByScope( new ResourceScope(ResourceScopeTypeEnum.BIZ, bizId.toString()) ); - Pair pair = hostSyncService.syncBizHostsAtOnce(applicationDTO); - Long cmdbInterfaceTimeConsuming = pair.getFirst(); - Long writeToDBTimeConsuming = pair.getSecond(); - log.info("syncBizHosts:cmdbInterfaceTimeConsuming={},writeToDBTimeConsuming={}", cmdbInterfaceTimeConsuming, - writeToDBTimeConsuming); + Triple, Long, Long> triple = hostSyncService.syncBizHostsAtOnce(applicationDTO); + Set cmdbBasicHosts = triple.getLeft(); + Long cmdbInterfaceTimeConsuming = triple.getMiddle(); + Long writeToDBTimeConsuming = triple.getRight(); + log.info( + "syncBizHosts:cmdbInterfaceTimeConsuming={},writeToDBTimeConsuming={}, {} hosts, cmdbHostIds={}", + cmdbInterfaceTimeConsuming, + writeToDBTimeConsuming, + cmdbBasicHosts.size(), + cmdbBasicHosts.stream().map(BasicHostDTO::getHostId).collect(Collectors.toList()) + ); return true; } @@ -527,38 +476,36 @@ public Long syncAgentStatus() { log.info("syncAgentStatus thread already running on {}", runningMachine); return 1L; } - syncAgentStatusExecutor.execute(() -> { - // 开一个心跳子线程，维护当前机器正在同步主机的状态 - RedisKeyHeartBeatThread agentStatusSyncRedisKeyHeartBeatThread = new RedisKeyHeartBeatThread( - redisTemplate, - REDIS_KEY_SYNC_AGENT_STATUS_JOB_RUNNING_MACHINE, - machineIp, - 5000L, - 4000L - ); - agentStatusSyncRedisKeyHeartBeatThread.setName( - "[" + agentStatusSyncRedisKeyHeartBeatThread.getId() - + "]-agentStatusSyncRedisKeyHeartBeatThread" - ); - agentStatusSyncRedisKeyHeartBeatThread.start(); - log.info("start sync agentStatus at {},{}", TimeUtil.getCurrentTimeStr("HH:mm:ss"), - System.currentTimeMillis()); - StopWatch watch = new StopWatch("syncAgentStatus"); - watch.start("total"); - try { - // 从GSE同步Agent状态 - agentStatusSyncService.syncAgentStatusFromGSE(); - // 将最后同步时间写入Redis - redisTemplate.opsForValue().set(REDIS_KEY_LAST_FINISH_TIME_SYNC_AGENT_STATUS, - "" + System.currentTimeMillis()); - } catch (Throwable t) { - log.error("syncAgentStatus thread fail", t); - } finally { - agentStatusSyncRedisKeyHeartBeatThread.setRunFlag(false); - watch.stop(); - log.info("syncAgentStatus time consuming:" + watch.prettyPrint()); - } - }); + // 开一个心跳子线程，维护当前机器正在同步主机的状态 + RedisKeyHeartBeatThread agentStatusSyncRedisKeyHeartBeatThread = new RedisKeyHeartBeatThread( + redisTemplate, + REDIS_KEY_SYNC_AGENT_STATUS_JOB_RUNNING_MACHINE, + machineIp, + 5000L, + 4000L + ); + agentStatusSyncRedisKeyHeartBeatThread.setName( + "[" + agentStatusSyncRedisKeyHeartBeatThread.getId() + + "]-agentStatusSyncRedisKeyHeartBeatThread" + ); + agentStatusSyncRedisKeyHeartBeatThread.start(); + log.info("start sync agentStatus at {},{}", TimeUtil.getCurrentTimeStr("HH:mm:ss"), + System.currentTimeMillis()); + StopWatch watch = new StopWatch("syncAgentStatus"); + watch.start("total"); + try { + // 从GSE同步Agent状态 + agentStatusSyncService.syncAgentStatusFromGSE(); + // 将最后同步时间写入Redis + redisTemplate.opsForValue().set(REDIS_KEY_LAST_FINISH_TIME_SYNC_AGENT_STATUS, + "" + System.currentTimeMillis()); + } catch (Throwable t) { + log.error("syncAgentStatus thread fail", t); + } finally { + agentStatusSyncRedisKeyHeartBeatThread.setRunFlag(false); + watch.stop(); + log.info("syncAgentStatus time consuming:" + watch.prettyPrint()); + } } finally { //释放锁 LockUtils.releaseDistributedLock(REDIS_KEY_SYNC_AGENT_STATUS_JOB_LOCK, machineIp); diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/topo/BizTopoService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/topo/BizTopoService.java new file mode 100644 index 0000000000..f61b1eb275 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/impl/topo/BizTopoService.java @@ -0,0 +1,88 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.service.impl.topo; + +import com.tencent.bk.job.common.cc.model.CcInstanceDTO; +import com.tencent.bk.job.common.cc.model.InstanceTopologyDTO; +import com.tencent.bk.job.common.cc.sdk.BizCmdbClient; +import com.tencent.bk.job.common.cc.util.TopologyUtil; +import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.manage.model.web.request.chooser.host.BizTopoNode; +import lombok.extern.slf4j.Slf4j; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/** + * 业务拓扑服务 + */ +@Slf4j +@Service +public class BizTopoService { + + private final BizCmdbClient bizCmdbClient; + + @Autowired + public BizTopoService(BizCmdbClient bizCmdbClient) { + this.bizCmdbClient = bizCmdbClient; + } + + /** + * 找出多个拓扑节点下属的所有模块ID + * + * @param bizId 业务ID + * @param appTopoNodeList 拓扑节点列表 + * @return 模块ID列表 + */ + public List findAllModuleIdsOfNodes(Long bizId, + List appTopoNodeList) { + if (appTopoNodeList == null || appTopoNodeList.isEmpty()) { + return Collections.emptyList(); + } + List moduleIds = new ArrayList<>(); + // 查业务拓扑树 + InstanceTopologyDTO appTopologyTree = bizCmdbClient.getBizInstTopology(bizId); + for (BizTopoNode treeNode : appTopoNodeList) { + CcInstanceDTO ccInstanceDTO = new CcInstanceDTO(treeNode.getObjectId(), treeNode.getInstanceId()); + // 查拓扑节点完整信息 + InstanceTopologyDTO completeNode = TopologyUtil.findNodeFromTopo(appTopologyTree, ccInstanceDTO); + if (completeNode == null) { + log.warn( + "Cannot find node in topo, node:{}", + JsonUtils.toJson(ccInstanceDTO) + ); + if (log.isDebugEnabled()) { + TopologyUtil.printTopo(appTopologyTree); + } + continue; + } + moduleIds.addAll(TopologyUtil.findModuleIdsFromTopo(completeNode)); + } + return moduleIds; + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/plan/TaskPlanService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/plan/TaskPlanService.java index 04a2454cb3..51f71f3352 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/plan/TaskPlanService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/plan/TaskPlanService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -35,7 +35,7 @@ import java.util.Set; /** - * @since 16/10/2019 19:38 + * 作业执行方案 Service */ public interface TaskPlanService { /** @@ -103,22 +103,71 @@ PageData listPageTaskPlansBasicInfo( TaskPlanInfoDTO getTaskPlanById(Long appId, Long planId); /** - * 保存执行方案信息 + * 查询执行方案信息 * - * @param taskPlanInfo 待保存的执行方案信息 - * @return 执行方案 ID + * @param username 用户账号 + * @param appId 业务 ID + * @param templateId 作业模版 ID + * @param planId 执行方案 ID + * @return 执行方案信息 + */ + TaskPlanInfoDTO getTaskPlan(String username, Long appId, Long templateId, Long planId); + + /** + * 查询执行方案信息 + * + * @param username 用户账号 + * @param appId 业务 ID + * @param planId 执行方案 ID + * @return 执行方案信息 + */ + TaskPlanInfoDTO getTaskPlan(String username, Long appId, Long planId); + + /** + * 新建执行方案 + * + * @param username 用户账号 + * @param taskPlanInfo 执行方案 + * @return 新建的执行方案 + */ + TaskPlanInfoDTO createTaskPlan(String username, TaskPlanInfoDTO taskPlanInfo); + + /** + * 新建执行方案 + * + * @param taskPlanInfo 执行方案 + * @return 新建的执行方案 + */ + TaskPlanInfoDTO createTaskPlan(TaskPlanInfoDTO taskPlanInfo); + + /** + * 更新执行方案 + * + * @param username 用户账号 + * @param taskPlanInfo 执行方案 + * @return 更新之后的执行方案 + */ + TaskPlanInfoDTO updateTaskPlan(String username, TaskPlanInfoDTO taskPlanInfo); + + /** + * 更新内置的用于调试的执行方案 + * + * @param username 用户账号 + * @param taskPlanInfo 执行方案 + * @return 更新之后的执行方案 */ - Long saveTaskPlan(TaskPlanInfoDTO taskPlanInfo); + TaskPlanInfoDTO updateDebugTaskPlan(String username, TaskPlanInfoDTO taskPlanInfo); /** * 删除执行方案 * + * @param username 用户账号 * @param appId 业务 ID * @param templateId 模版 ID * @param planId 执行方案 ID - * @return 是否删除成功 + * @return 删除掉的执行方案 */ - Boolean deleteTaskPlan(Long appId, Long templateId, Long planId); + TaskPlanInfoDTO deleteTaskPlan(String username, Long appId, Long templateId, Long planId); /** * 获取调试执行方案（不存在则创建） @@ -131,7 +180,16 @@ PageData listPageTaskPlansBasicInfo( TaskPlanInfoDTO getDebugTaskPlan(String username, Long appId, Long templateId); /** - * 根据执行方案 ID 列表批量获取执行方案基本信息 + * 根据执行方案 ID 列表批量获取执行方案基本信息（含变量与模板信息） + * + * @param appId 业务 ID + * @param planIdList 执行方案 ID 列表 + * @return 执行方案基本信息列表 + */ + List listPlanBasicInfoWithVariablesByIds(Long appId, List planIdList); + + /** + * 根据执行方案 ID 列表批量获取执行方案基本信息（不含变量与模板信息） * * @param appId 业务 ID * @param planIdList 执行方案 ID 列表 diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/plan/impl/TaskPlanFavoriteServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/plan/impl/TaskPlanFavoriteServiceImpl.java index be16fc9500..5535e3a6e5 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/plan/impl/TaskPlanFavoriteServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/plan/impl/TaskPlanFavoriteServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/plan/impl/TaskPlanServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/plan/impl/TaskPlanServiceImpl.java index c4c2b44abd..c4bccbbe4b 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/plan/impl/TaskPlanServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/plan/impl/TaskPlanServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,10 @@ package com.tencent.bk.job.manage.service.plan.impl; +import com.tencent.bk.audit.annotations.ActionAuditRecord; +import com.tencent.bk.audit.annotations.AuditInstanceRecord; +import com.tencent.bk.audit.context.ActionAuditContext; +import com.tencent.bk.job.common.audit.constants.EventContentConstants; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.exception.AlreadyExistsException; import com.tencent.bk.job.common.exception.FailedPreconditionException; @@ -31,13 +35,19 @@ import com.tencent.bk.job.common.exception.NotFoundException; import com.tencent.bk.job.common.exception.ServiceException; import com.tencent.bk.job.common.i18n.service.MessageI18nService; +import com.tencent.bk.job.common.iam.constant.ActionId; +import com.tencent.bk.job.common.iam.constant.ResourceTypeId; import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; +import com.tencent.bk.job.common.model.dto.AppResourceScope; +import com.tencent.bk.job.common.mysql.JobTransactional; import com.tencent.bk.job.common.util.JobContextUtil; import com.tencent.bk.job.common.util.PageUtil; import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.crontab.model.CronJobVO; -import com.tencent.bk.job.manage.common.consts.task.TaskPlanTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskPlanTypeEnum; +import com.tencent.bk.job.manage.auth.PlanAuthService; +import com.tencent.bk.job.manage.auth.TemplateAuthService; import com.tencent.bk.job.manage.dao.plan.TaskPlanDAO; import com.tencent.bk.job.manage.model.dto.TaskPlanQueryDTO; import com.tencent.bk.job.manage.model.dto.task.TaskPlanBasicInfoDTO; @@ -54,10 +64,11 @@ import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.MapUtils; import org.apache.commons.lang3.StringUtils; +import org.slf4j.helpers.MessageFormatter; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; import java.util.ArrayList; import java.util.Collection; @@ -70,24 +81,44 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; +import static com.tencent.bk.audit.constants.AuditAttributeNames.INSTANCE_ID; +import static com.tencent.bk.audit.constants.AuditAttributeNames.INSTANCE_NAME; + /** - * @since 19/11/2019 16:43 + * 作业执行方案 Service */ @Slf4j -@Service("TaskPlanServiceImpl") +@Service("TaskPlanService") public class TaskPlanServiceImpl implements TaskPlanService { private final TaskPlanDAO taskPlanDAO; private final MessageI18nService i18nService; - private final CronJobService cronJobService; private final AbstractTaskStepService taskPlanStepService; private final AbstractTaskVariableService taskTemplateVariableService; private final AbstractTaskVariableService taskPlanVariableService; + private final PlanAuthService planAuthService; + private CronJobService cronJobService; + private TaskTemplateService taskTemplateService; + private final TemplateAuthService templateAuthService; + + /** + * 通过 Set 方式注入，避免循环依赖问题 + */ @Autowired - private TaskTemplateService taskTemplateService; + @Lazy + public void setTaskTemplateService(TaskTemplateService taskTemplateService) { + this.taskTemplateService = taskTemplateService; + } + + /** + * 通过 Set 方式注入，避免循环依赖问题 + */ @Autowired - private TaskPlanService taskPlanService; + @Lazy + public void setCronJobService(CronJobService cronJobService) { + this.cronJobService = cronJobService; + } @Autowired public TaskPlanServiceImpl( @@ -95,32 +126,31 @@ public TaskPlanServiceImpl( @Qualifier("TaskPlanStepServiceImpl") AbstractTaskStepService taskPlanStepService, @Qualifier("TaskTemplateVariableServiceImpl") AbstractTaskVariableService taskTemplateVariableService, @Qualifier("TaskPlanVariableServiceImpl") AbstractTaskVariableService taskPlanVariableService, - MessageI18nService i18nService, CronJobService cronJobService - ) { + MessageI18nService i18nService, + PlanAuthService planAuthService, + TemplateAuthService templateAuthService) { this.taskPlanDAO = taskPlanDAO; this.taskPlanStepService = taskPlanStepService; this.taskTemplateVariableService = taskTemplateVariableService; this.taskPlanVariableService = taskPlanVariableService; this.i18nService = i18nService; - this.cronJobService = cronJobService; + this.planAuthService = planAuthService; + this.templateAuthService = templateAuthService; } /** - * Sync plan variable to task variable + * 提取执行方案变量中已经被用户设置的的字段值放入新的变量中 * - * @param templateVariable 模版中的变量信息 - * @param planVariable 执行方案中的变量信息 + * @param originPlanVariable 执行方案中的旧的变量信息 + * @param newVariable 新的变量信息 */ - private static void processVariableSync(TaskVariableDTO templateVariable, TaskVariableDTO planVariable) { - if (planVariable != null) { - planVariable.setDescription(templateVariable.getDescription()); - planVariable.setChangeable(templateVariable.getChangeable()); - planVariable.setRequired(templateVariable.getRequired()); - - templateVariable.setId(planVariable.getId()); - templateVariable.setPlanId(planVariable.getPlanId()); - templateVariable.setTemplateId(null); - templateVariable.setDefaultValue(planVariable.getDefaultValue()); + private static void extractFieldToNewVariable(TaskVariableDTO originPlanVariable,TaskVariableDTO newVariable) { + if (originPlanVariable != null) { + // 执行方案变量中的ID信息与已经被用户指定的取值需要保留 + newVariable.setId(originPlanVariable.getId()); + newVariable.setPlanId(originPlanVariable.getPlanId()); + newVariable.setTemplateId(null); + newVariable.setDefaultValue(originPlanVariable.getDefaultValue()); } } @@ -209,88 +239,226 @@ public TaskPlanInfoDTO getTaskPlanById(Long appId, Long planId) { } @Override - @Transactional(rollbackFor = Throwable.class) - public Long saveTaskPlan(TaskPlanInfoDTO taskPlanInfo) { - boolean insert = false; - if (taskPlanInfo.getId() == null || taskPlanInfo.getId() <= 0) { - TaskTemplateInfoDTO taskTemplate = - taskTemplateService.getTaskTemplateById(taskPlanInfo.getAppId(), taskPlanInfo.getTemplateId()); - TaskPlanInfoDTO.buildPlanInfo(taskPlanInfo, taskTemplate); - insert = true; + @ActionAuditRecord( + actionId = ActionId.VIEW_JOB_PLAN, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.PLAN, + instanceIds = "#planId", + instanceNames = "#$?.name" + ), + content = EventContentConstants.VIEW_JOB_PLAN + ) + public TaskPlanInfoDTO getTaskPlan(String username, Long appId, Long templateId, Long planId) { + checkViewPlanPermission(username, appId, templateId, planId); + + TaskPlanInfoDTO taskPlan = getTaskPlanById(appId, templateId, planId); + if (taskPlan == null) { + log.warn("Task plan not exist, appId: {}, planId: {}", appId, planId); + throw new NotFoundException(ErrorCode.TASK_PLAN_NOT_EXIST); + } + + return taskPlan; + } + + @ActionAuditRecord( + actionId = ActionId.VIEW_JOB_PLAN, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.PLAN, + instanceIds = "#planId", + instanceNames = "#$?.name" + ), + content = EventContentConstants.VIEW_JOB_PLAN + ) + public TaskPlanInfoDTO getTaskPlan(String username, Long appId, Long planId) { + TaskPlanInfoDTO taskPlan = getTaskPlanById(appId, planId); + if (taskPlan == null) { + log.warn("Task plan not exist, appId: {}, planId: {}", appId, planId); + throw new NotFoundException(ErrorCode.TASK_PLAN_NOT_EXIST); } + + checkViewPlanPermission(username, appId, taskPlan.getTemplateId(), planId); + + return taskPlan; + } + + @Override + @JobTransactional(transactionManager = "jobManageTransactionManager") + @ActionAuditRecord( + actionId = ActionId.CREATE_JOB_PLAN, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.PLAN, + instanceIds = "#$?.id", + instanceNames = "#$?.name" + ), + content = EventContentConstants.CREATE_JOB_PLAN + ) + public TaskPlanInfoDTO createTaskPlan(String username, TaskPlanInfoDTO taskPlanInfo) { + checkCreatePlanPermission(username, taskPlanInfo.getAppId(), taskPlanInfo.getTemplateId()); + TaskPlanInfoDTO newPlan = createTaskPlan(taskPlanInfo); + planAuthService.registerPlan(newPlan.getId(), newPlan.getName(), username); + return newPlan; + } + + @Override + public TaskPlanInfoDTO createTaskPlan(TaskPlanInfoDTO taskPlanInfo) { + TaskTemplateInfoDTO taskTemplate = + taskTemplateService.getTaskTemplateById(taskPlanInfo.getAppId(), taskPlanInfo.getTemplateId()); + TaskPlanInfoDTO.buildPlanInfo(taskPlanInfo, taskTemplate); + // process plan id Long planId; taskPlanInfo.setLastModifyUser(taskPlanInfo.getLastModifyUser()); taskPlanInfo.setLastModifyTime(taskPlanInfo.getLastModifyTime()); + taskPlanInfo.setCreateTime(DateUtils.currentTimeSeconds()); + planId = taskPlanDAO.insertTaskPlan(taskPlanInfo); + if (planId == null) { + throw new InternalException(ErrorCode.INSERT_TASK_PLAN_FAILED); + } + taskPlanInfo.setId(planId); - if (insert) { - taskPlanInfo.setCreateTime(DateUtils.currentTimeSeconds()); - planId = taskPlanDAO.insertTaskPlan(taskPlanInfo); - if (planId == null) { - throw new InternalException(ErrorCode.INSERT_TASK_PLAN_FAILED); - } - taskPlanInfo.setId(planId); - } else { - if (!taskPlanDAO.updateTaskPlanById(taskPlanInfo)) { - throw new InternalException(ErrorCode.UPDATE_TASK_PLAN_FAILED); + // Save step + for (TaskStepDTO taskStep : taskPlanInfo.getStepList()) { + taskStep.setPlanId(planId); + // Insert give template step id + if (taskPlanInfo.getEnableStepList().contains(taskStep.getTemplateStepId())) { + taskStep.setEnable(1); + } else { + taskStep.setEnable(0); } - planId = taskPlanInfo.getId(); + // new step, insert to get id + taskStep.setId(taskPlanStepService.insertStep(taskStep)); } - if (insert) { - // Save step - for (TaskStepDTO taskStep : taskPlanInfo.getStepList()) { - taskStep.setPlanId(planId); - // Insert give template step id - if (taskPlanInfo.getEnableStepList().contains(taskStep.getTemplateStepId())) { + // Insert new variable + taskPlanInfo.getVariableList().forEach(variable -> variable.setPlanId(planId)); + taskPlanVariableService.batchInsertVariable(taskPlanInfo.getVariableList()); + + return getTaskPlanById(planId); + } + + private void checkCreatePlanPermission(String username, long appId, long templateId) { + planAuthService.authCreateJobPlan(username, new AppResourceScope(appId), templateId, null) + .denyIfNoPermission(); + } + + private void checkViewPlanPermission(String username, long appId, long templateId, long planId) { + planAuthService.authViewJobPlan(username, new AppResourceScope(appId), templateId, planId, null) + .denyIfNoPermission(); + } + + private void checkEditPlanPermission(String username, long appId, long templateId, long planId) { + planAuthService.authEditJobPlan(username, new AppResourceScope(appId), templateId, planId, null) + .denyIfNoPermission(); + } + + private void checkDeletePlanPermission(String username, long appId, long templateId, long planId) { + planAuthService.authDeleteJobPlan(username, new AppResourceScope(appId), templateId, planId, null) + .denyIfNoPermission(); + } + + @Override + @JobTransactional(transactionManager = "jobManageTransactionManager") + @ActionAuditRecord( + actionId = ActionId.EDIT_JOB_PLAN, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.PLAN, + instanceIds = "#taskPlanInfo?.id", + instanceNames = "#taskPlanInfo?.name" + ), + content = EventContentConstants.EDIT_JOB_PLAN + ) + public TaskPlanInfoDTO updateTaskPlan(String username, TaskPlanInfoDTO taskPlanInfo) { + checkEditPlanPermission(username, taskPlanInfo.getAppId(), taskPlanInfo.getTemplateId(), + taskPlanInfo.getId()); + + return updateTaskPlan(taskPlanInfo); + } + + @Override + @JobTransactional(transactionManager = "jobManageTransactionManager") + public TaskPlanInfoDTO updateDebugTaskPlan(String username, TaskPlanInfoDTO taskPlanInfo) { + // 调试作业模版会保存一份内置的执行方案；从用户角度来说仍然还是在处理跟模版相关的操作，所以使用模版查看鉴权 + templateAuthService.authViewJobTemplate(username, + new AppResourceScope(taskPlanInfo.getAppId()), taskPlanInfo.getTemplateId()) + .denyIfNoPermission(); + + return updateTaskPlan(taskPlanInfo); + } + + private TaskPlanInfoDTO updateTaskPlan(TaskPlanInfoDTO taskPlanInfo) { + Long planId = taskPlanInfo.getId(); + + TaskPlanInfoDTO originPlan = getTaskPlanById(planId); + if (originPlan == null) { + throw new NotFoundException(ErrorCode.TASK_PLAN_NOT_EXIST); + } + + taskPlanInfo.setLastModifyUser(taskPlanInfo.getLastModifyUser()); + taskPlanInfo.setLastModifyTime(taskPlanInfo.getLastModifyTime()); + + if (!taskPlanDAO.updateTaskPlanById(taskPlanInfo)) { + throw new InternalException(ErrorCode.UPDATE_TASK_PLAN_FAILED); + } + + List taskStepList = taskPlanStepService.listStepsByParentId(planId); + if (CollectionUtils.isNotEmpty(taskStepList)) { + for (TaskStepDTO taskStep : taskStepList) { + // Update give step id + if (taskPlanInfo.getEnableStepList().contains(taskStep.getId())) { taskStep.setEnable(1); } else { taskStep.setEnable(0); } - // new step, insert to get id - taskStep.setId(taskPlanStepService.insertStep(taskStep)); - } - } else { - List taskStepList = taskPlanStepService.listStepsByParentId(planId); - if (CollectionUtils.isNotEmpty(taskStepList)) { - for (TaskStepDTO taskStep : taskStepList) { - // Update give step id - if (taskPlanInfo.getEnableStepList().contains(taskStep.getId())) { - taskStep.setEnable(1); - } else { - taskStep.setEnable(0); - } - taskPlanStepService.updateStepById(taskStep); - } + taskPlanStepService.updateStepById(taskStep); } } - if (insert) { - // Insert new variable - taskPlanInfo.getVariableList().forEach(variable -> variable.setPlanId(planId)); - taskPlanVariableService.batchInsertVariable(taskPlanInfo.getVariableList()); - } else { - for (TaskVariableDTO taskVariable : taskPlanInfo.getVariableList()) { - taskVariable.setPlanId(planId); - // Update exist variable - taskPlanVariableService.updateVariableById(taskVariable); - } + for (TaskVariableDTO taskVariable : taskPlanInfo.getVariableList()) { + taskVariable.setPlanId(planId); + // Update exist variable + taskPlanVariableService.updateVarByParentResourceIdAndTplVarId(taskVariable); } - return planId; + + TaskPlanInfoDTO updatedPlan = getTaskPlanById(planId); + + // 审计记录 + ActionAuditContext.current() + .setOriginInstance(TaskPlanInfoDTO.toEsbPlanInfoV3(originPlan)) + .setInstance(TaskPlanInfoDTO.toEsbPlanInfoV3(updatedPlan)); + + return updatedPlan; } @Override - public Boolean deleteTaskPlan(Long appId, Long templateId, Long planId) { + @ActionAuditRecord( + actionId = ActionId.DELETE_JOB_PLAN, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.PLAN, + instanceIds = "#planId", + instanceNames = "#$?.name" + ), + content = EventContentConstants.DELETE_JOB_PLAN + ) + public TaskPlanInfoDTO deleteTaskPlan(String username, Long appId, Long templateId, Long planId) { + checkDeletePlanPermission(username, appId, templateId, planId); + + TaskPlanInfoDTO plan = getTaskPlanById(appId, templateId, planId); + if (plan == null) { + throw new NotFoundException(ErrorCode.TASK_PLAN_NOT_EXIST); + } + Map> cronJobMap = cronJobService.batchListCronJobByPlanIds(appId, Collections.singletonList(planId)); if (MapUtils.isNotEmpty(cronJobMap) && CollectionUtils.isNotEmpty(cronJobMap.get(planId))) { throw new FailedPreconditionException(ErrorCode.DELETE_PLAN_FAILED_USING_BY_CRON); } - return taskPlanDAO.deleteTaskPlanById(appId, templateId, planId); + taskPlanDAO.deleteTaskPlanById(appId, templateId, planId); + + return plan; } @Override - @Transactional(rollbackFor = Throwable.class) + @JobTransactional(transactionManager = "jobManageTransactionManager") public TaskPlanInfoDTO getDebugTaskPlan(String username, Long appId, Long templateId) { TaskPlanInfoDTO taskPlan = taskPlanDAO.getDebugTaskPlan(appId, templateId); TaskTemplateInfoDTO taskTemplateInfo = taskTemplateService.getTaskTemplateBasicInfoById(appId, templateId); @@ -300,7 +468,7 @@ public TaskPlanInfoDTO getDebugTaskPlan(String username, Long appId, Long templa if (taskPlan != null) { if (!taskTemplateInfo.getVersion().equals(taskPlan.getVersion())) { // 作业模板有更新，需要同步到调试执行方案（不含变量） - taskPlanService.sync(appId, templateId, taskPlan.getId(), taskTemplateInfo.getVersion()); + sync(appId, templateId, taskPlan.getId(), taskTemplateInfo.getVersion()); taskPlan = taskPlanDAO.getDebugTaskPlan(appId, templateId); } List templateVariableList = @@ -317,21 +485,21 @@ public TaskPlanInfoDTO getDebugTaskPlan(String username, Long appId, Long templa ); taskPlan.setStepList(taskPlanStepService.listStepsByParentId(taskPlan.getId())); taskPlan.setVariableList(taskPlanVariableService.listVariablesByParentId(taskPlan.getId())); + taskPlan.setName(taskTemplateInfo.getName()); + return taskPlan; } + // 如果不存在调试方案，需要创建一个再查询 taskPlan = new TaskPlanInfoDTO(); taskPlan.setAppId(appId); taskPlan.setTemplateId(templateId); - taskPlan.setName(taskTemplateInfo.getName() + - "_" + i18nService.getI18n("job.task.plan.debug") - + "_" + System.currentTimeMillis() / 1000L - ); + taskPlan.setName(taskTemplateInfo.getName()); taskPlan.setCreator(username); taskPlan.setLastModifyUser(username); taskPlan.setLastModifyTime(DateUtils.currentTimeSeconds()); taskPlan.setDebug(true); - taskPlan.setId(taskPlanService.saveTaskPlan(taskPlan)); + taskPlan.setId(createTaskPlan(taskPlan).getId()); taskPlan = taskPlanDAO.getTaskPlanById(appId, templateId, taskPlan.getId(), TaskPlanTypeEnum.DEBUG); taskPlan.setStepList(taskPlanStepService.listStepsByParentId(taskPlan.getId())); @@ -342,7 +510,7 @@ public TaskPlanInfoDTO getDebugTaskPlan(String username, Long appId, Long templa taskStep.setEnable(1); taskPlan.getEnableStepList().add(taskStep.getId()); } - taskPlanService.saveTaskPlan(taskPlan); + updateTaskPlan(taskPlan); return taskPlan; } @@ -354,28 +522,38 @@ private void setPlanIdForVariables(Long planId, List variableLi } @Override - public List listPlanBasicInfoByIds(Long appId, List planIdList) { + public List listPlanBasicInfoWithVariablesByIds(Long appId, List planIdList) { List taskPlanInfoList = taskPlanDAO.listTaskPlanByIds( appId, planIdList, null, null ); - taskPlanInfoList.parallelStream().forEach(taskPlan -> + taskPlanInfoList.forEach(taskPlan -> taskPlan.setVariableList(taskPlanVariableService.listVariablesByParentId(taskPlan.getId()))); return fillTemplateInfo(appId, taskPlanInfoList); } + @Override + public List listPlanBasicInfoByIds(Long appId, List planIdList) { + return taskPlanDAO.listTaskPlanByIds( + appId, + planIdList, + null, + null + ); + } + private List fillTemplateInfo(Long appId, List planList) { - List templateIdList = planList.parallelStream().map(TaskPlanInfoDTO::getTemplateId).distinct() + List templateIdList = planList.stream().map(TaskPlanInfoDTO::getTemplateId).distinct() .collect(Collectors.toList()); List templateInfoList = taskTemplateService.listTaskTemplateBasicInfoByIds(appId, templateIdList); Map templateInfoMap = new ConcurrentHashMap<>(templateInfoList.size()); - templateInfoList.parallelStream() + templateInfoList .forEach(taskTemplateInfoDTO -> templateInfoMap.put(taskTemplateInfoDTO.getId(), taskTemplateInfoDTO)); - return planList.parallelStream() + return planList.stream() .filter(planInfo -> templateInfoMap.containsKey(planInfo.getTemplateId())).peek(taskPlanInfoDTO -> { taskPlanInfoDTO.setTemplateName(templateInfoMap.get(taskPlanInfoDTO.getTemplateId()).getName()); taskPlanInfoDTO.setTemplateVersion(templateInfoMap.get(taskPlanInfoDTO.getTemplateId()).getVersion()); @@ -398,26 +576,49 @@ public Boolean checkPlanName(Long appId, Long templateId, Long planId, String na } @Override - @Transactional(rollbackFor = {Exception.class, Error.class}) + @JobTransactional(transactionManager = "jobManageTransactionManager") + @ActionAuditRecord( + actionId = ActionId.SYNC_JOB_PLAN, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.PLAN, + instanceIds = "#planId" + ), + content = "Sync plan [{{" + INSTANCE_NAME + "}}]({{" + INSTANCE_ID + "}})" + ) public Boolean sync(Long appId, Long templateId, Long planId, String templateVersion) { TaskTemplateInfoDTO taskTemplate = taskTemplateService.getTaskTemplateById(appId, templateId); if (taskTemplate == null) { + log.warn("taskTemplate is null, appId={}, templateId={}", appId, templateId); return false; } if (!templateVersion.equals(taskTemplate.getVersion())) { + log.warn( + "templateVersion expired, templateVersion={}, newest version={}", + templateVersion, + taskTemplate.getVersion() + ); return false; } - TaskPlanInfoDTO taskPlan = taskPlanService.getTaskPlanById(appId, planId); + TaskPlanInfoDTO taskPlan = getTaskPlanById(appId, planId); if (taskPlan == null) { + log.warn("taskPlan is null, appId={}, planId={}", appId, planId); return false; } if (!taskPlan.getTemplateId().equals(templateId)) { + log.warn( + "taskPlan(id={},templateId={}) not belong to template(id={})", + taskPlan.getId(), + taskPlan.getTemplateId(), + templateId + ); return false; } + // 用模板数据覆盖执行方案数据 taskPlan.setVersion(taskTemplate.getVersion()); taskPlan.setLastModifyTime(DateUtils.currentTimeSeconds()); taskPlan.setLastModifyUser(JobContextUtil.getUsername()); + // 记录执行方案中启用的步骤列表对应的模板步骤ID if (taskPlan.getStepList() != null) { taskPlan.setEnableStepList(new ArrayList<>()); taskPlan.getStepList().forEach(taskStep -> { @@ -426,6 +627,7 @@ public Boolean sync(Long appId, Long templateId, Long planId, String templateVer } }); } + // 通过模板的步骤列表数据构造执行方案步骤数据 taskTemplate.getStepList().forEach(taskStep -> { taskStep.setTemplateStepId(taskStep.getId()); taskStep.setId(null); @@ -438,21 +640,26 @@ public Boolean sync(Long appId, Long templateId, Long planId, String templateVer }); taskPlan.setStepList(taskTemplate.getStepList()); + // 构建执行方案变量Map<变量ID,变量数据> Map originPlanVariableMap; if (taskPlan.getVariableList() != null) { originPlanVariableMap = new HashMap<>(taskPlan.getVariableList().size()); taskPlan.getVariableList().forEach(planVariable -> { + // 执行方案变量ID实际是执行方案变量对应的模板变量ID + // 赋值处见：TaskPlanVariableDAOImpl.extract() originPlanVariableMap.put(planVariable.getId(), planVariable); }); } else { originPlanVariableMap = null; } if (CollectionUtils.isNotEmpty(taskTemplate.getVariableList())) { + // 通过赋值将模板变量改造为新的执行方案变量后续设置到执行方案中 taskTemplate.getVariableList().forEach(templateVariable -> { templateVariable.setPlanId(planId); if (originPlanVariableMap != null) { TaskVariableDTO originPlanVariable = originPlanVariableMap.get(templateVariable.getId()); - processVariableSync(templateVariable, originPlanVariable); + // 复用模板变量作为新的变量 + extractFieldToNewVariable(originPlanVariable, templateVariable); originPlanVariableMap.remove(templateVariable.getId()); } }); @@ -461,27 +668,33 @@ public Boolean sync(Long appId, Long templateId, Long planId, String templateVer taskPlan.setVariableList(Collections.emptyList()); } - taskPlanService.syncPlan(taskPlan); + syncPlan(taskPlan); + + // 审计 - 实例名称 + ActionAuditContext.current().setInstanceName(taskPlan.getName()); + return true; } @Override - @Transactional(rollbackFor = {Exception.class, Error.class}) + @JobTransactional(transactionManager = "jobManageTransactionManager") public void syncPlan(TaskPlanInfoDTO taskPlanInfo) { try { // process plan id Long planId = taskPlanInfo.getId(); + // 更新执行方案基础信息 if (!taskPlanDAO.updateTaskPlanById(taskPlanInfo)) { throw new InternalException(ErrorCode.UPDATE_TASK_PLAN_FAILED); } + // 删除所有老的步骤 List taskStepList = taskPlanStepService.listStepsByParentId(planId); if (CollectionUtils.isNotEmpty(taskStepList)) { taskStepList.forEach(taskStep -> taskPlanStepService.deleteStepById(planId, taskStep.getId())); } - // Save step + // 保存新的步骤信息 for (TaskStepDTO taskStep : taskPlanInfo.getStepList()) { taskStep.setPlanId(planId); // Insert give template step id @@ -494,11 +707,12 @@ public void syncPlan(TaskPlanInfoDTO taskPlanInfo) { taskStep.setId(taskPlanStepService.insertStep(taskStep)); } + // 更新变量 List oldVariableList = taskPlanVariableService.listVariablesByParentId(planId); List oldVariableIdList; if (CollectionUtils.isNotEmpty(oldVariableList)) { oldVariableIdList = - oldVariableList.parallelStream().map(TaskVariableDTO::getId).collect(Collectors.toList()); + oldVariableList.stream().map(TaskVariableDTO::getId).collect(Collectors.toList()); } else { oldVariableIdList = null; } @@ -508,18 +722,20 @@ public void syncPlan(TaskPlanInfoDTO taskPlanInfo) { for (TaskVariableDTO taskVariable : taskPlanInfo.getVariableList()) { taskVariable.setPlanId(planId); if (oldVariableIdList.contains(taskVariable.getId())) { - // Update exist variable - taskPlanVariableService.updateVariableById(taskVariable); + // 更新变量 + taskPlanVariableService.updateVarByParentResourceIdAndTplVarId(taskVariable); oldVariableIdList.remove(taskVariable.getId()); } else { newVariable.add(taskVariable); } } + // 新增变量 taskPlanVariableService.batchInsertVariable(newVariable); } else { taskPlanVariableService.batchInsertVariable(taskPlanInfo.getVariableList()); } + // 删除模板中已删除的变量 if (CollectionUtils.isNotEmpty(oldVariableIdList)) { oldVariableIdList.forEach(variableId -> taskPlanVariableService.deleteVariableById(planId, variableId)); @@ -568,7 +784,7 @@ public Long saveTaskPlanForMigration(String username, Long appId, Long templateI taskPlan.setLastStepId(taskTemplateInfo.getLastStepId()); taskPlan.setNeedUpdate(false); taskPlan.setVersion(taskTemplateInfo.getVersion()); - taskPlanService.saveTaskPlanWithId(taskPlan); + saveTaskPlanWithId(taskPlan); taskPlan = taskPlanDAO.getTaskPlanById(appId, templateId, taskPlan.getId(), TaskPlanTypeEnum.NORMAL); taskPlan.setStepList(taskTemplateInfo.getStepList()); @@ -589,7 +805,7 @@ public Long saveTaskPlanForMigration(String username, Long appId, Long templateI taskPlan.getEnableStepList().add(taskStep.getId()); } - taskPlanService.saveTaskPlan(taskPlan); + createTaskPlan(taskPlan); return planId; } catch (Exception e) { log.error("Error while creating debug plan", e); @@ -618,7 +834,7 @@ public boolean checkPlanId(Long planId) { } @Override - @Transactional(rollbackFor = {Exception.class, Error.class}) + @JobTransactional(transactionManager = "jobManageTransactionManager") public Long saveTaskPlanForBackup(TaskPlanInfoDTO taskPlanInfo) { try { TaskPlanInfoDTO taskPlanByName = taskPlanDAO.getTaskPlanByName(taskPlanInfo.getAppId(), @@ -678,7 +894,24 @@ public Long saveTaskPlanForBackup(TaskPlanInfoDTO taskPlanInfo) { } @Override + @ActionAuditRecord( + actionId = ActionId.DELETE_JOB_PLAN, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.PLAN + ), + content = EventContentConstants.DELETE_JOB_PLAN + ) public boolean deleteTaskPlanByTemplate(Long appId, Long templateId) { + // 审计 + List deletePlans = listTaskPlansBasicInfo(appId, templateId); + if (CollectionUtils.isNotEmpty(deletePlans)) { + ActionAuditContext.current() + .setInstanceIdList( + deletePlans.stream().map(plan -> plan.getId().toString()).collect(Collectors.toList())) + .setInstanceNameList( + deletePlans.stream().map(TaskPlanInfoDTO::getName).collect(Collectors.toList())); + } + return taskPlanDAO.deleteTaskPlanByTemplate(appId, templateId); } @@ -710,15 +943,32 @@ public Set listLocalFiles() { } @Override - @Transactional(rollbackFor = {Exception.class, Error.class}) + @JobTransactional(transactionManager = "jobManageTransactionManager") + @ActionAuditRecord( + actionId = ActionId.EDIT_JOB_PLAN, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.PLAN + ), + content = EventContentConstants.EDIT_JOB_PLAN + ) public boolean batchUpdatePlanVariable(List planInfoList) { if (CollectionUtils.isEmpty(planInfoList)) { return true; } for (TaskPlanInfoDTO planInfo : planInfoList) { try { + TaskPlanInfoDTO originPlan = getTaskPlanById(planInfo.getId()); + if (originPlan == null) { + throw new NotFoundException(ErrorCode.TASK_PLAN_NOT_EXIST); + } + if (taskPlanVariableService.batchUpdateVariableByName(planInfo.getVariableList())) { if (taskPlanDAO.updateTaskPlanById(planInfo)) { + // 添加审计 + TaskPlanInfoDTO updatedPlan = getTaskPlanById(planInfo.getId()); + ActionAuditContext.current() + .addInstanceInfo(String.valueOf(planInfo.getId()), planInfo.getName(), originPlan, + updatedPlan); continue; } else { log.error("Error while updating plan info after batch update variable value!|{}", planInfo); @@ -728,7 +978,11 @@ public boolean batchUpdatePlanVariable(List planInfoList) { } throw new InternalException(ErrorCode.BATCH_UPDATE_PLAN_VARIABLE_FAILED); } catch (Exception e) { - log.error("Error while batch update plan variable value!|{}", planInfo, e); + String msg = MessageFormatter.format( + "Error while batch update plan variable value!|{}", + planInfo + ).getMessage(); + log.error(msg, e); throw e; } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/plan/impl/TaskPlanStepServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/plan/impl/TaskPlanStepServiceImpl.java index 92c555287c..25dd3d4db3 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/plan/impl/TaskPlanStepServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/plan/impl/TaskPlanStepServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,7 @@ package com.tencent.bk.job.manage.service.plan.impl; -import com.tencent.bk.job.manage.common.consts.task.TaskTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskTypeEnum; import com.tencent.bk.job.manage.dao.ScriptDAO; import com.tencent.bk.job.manage.dao.TaskApprovalStepDAO; import com.tencent.bk.job.manage.dao.TaskFileInfoDAO; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/plan/impl/TaskPlanVariableServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/plan/impl/TaskPlanVariableServiceImpl.java index 3a8aafaeca..71f1f91f49 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/plan/impl/TaskPlanVariableServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/plan/impl/TaskPlanVariableServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,7 @@ package com.tencent.bk.job.manage.service.plan.impl; +import com.tencent.bk.job.common.mysql.JobTransactional; import com.tencent.bk.job.manage.dao.TaskVariableDAO; import com.tencent.bk.job.manage.model.dto.task.TaskVariableDTO; import com.tencent.bk.job.manage.service.AbstractTaskVariableService; @@ -32,7 +33,6 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; import java.util.List; @@ -54,7 +54,7 @@ public TaskVariableDTO getVariableById(Long parentId, Long id) { } @Override - @Transactional(rollbackFor = {Exception.class, Error.class}) + @JobTransactional(transactionManager = "jobManageTransactionManager") public boolean batchUpdateVariableByName(List variableList) { if (CollectionUtils.isEmpty(variableList)) { return false; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/template/TaskTemplateService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/template/TaskTemplateService.java index 859d2bc461..3cde603e7d 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/template/TaskTemplateService.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/template/TaskTemplateService.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,10 +25,9 @@ package com.tencent.bk.job.manage.service.template; import com.tencent.bk.job.common.model.PageData; -import com.tencent.bk.job.manage.common.consts.JobResourceStatusEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskFileTypeEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskScriptSourceEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskStepTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskFileTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskScriptSourceEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskStepTypeEnum; import com.tencent.bk.job.manage.model.dto.task.TaskTemplateInfoDTO; import com.tencent.bk.job.manage.model.query.TaskTemplateQuery; import com.tencent.bk.job.manage.model.web.vo.TagCountVO; @@ -65,21 +64,42 @@ PageData listPageTaskTemplatesBasicInfo(TaskTemplateQuery q TaskTemplateInfoDTO getTaskTemplateById(Long appId, Long templateId); /** - * 新增、保存模版信息 + * 查询模版信息 * - * @param taskTemplateInfo 待新增、保存的模版信息 - * @return 模版 ID + * @param username 用户账号 + * @param appId 业务 ID + * @param templateId 模版 ID + * @return 模版信息 + */ + TaskTemplateInfoDTO getTaskTemplate(String username, long appId, Long templateId); + + /** + * 新增作业模板 + * + * @param username 用户账号 + * @param taskTemplateInfo 模版信息 + * @return 模版 + */ + TaskTemplateInfoDTO saveTaskTemplate(String username, TaskTemplateInfoDTO taskTemplateInfo); + + /** + * 更新作业模板 + * + * @param username 用户账号 + * @param taskTemplateInfo 模版信息 + * @return 模版 */ - Long saveTaskTemplate(TaskTemplateInfoDTO taskTemplateInfo); + TaskTemplateInfoDTO updateTaskTemplate(String username, TaskTemplateInfoDTO taskTemplateInfo); /** * 删除模版 * + * @param username 用户账号 * @param appId 业务 ID * @param templateId 模版 ID - * @return 是否删除成功 + * @return 被删除的模板 */ - Boolean deleteTaskTemplate(Long appId, Long templateId); + TaskTemplateInfoDTO deleteTaskTemplate(String username, Long appId, Long templateId); /** * 获取标签关联的模版数量 @@ -94,10 +114,11 @@ PageData listPageTaskTemplatesBasicInfo(TaskTemplateQuery q *

* 仅更新名称、描述、标签 * + * @param username 用户账号 * @param taskTemplateInfo 模版信息 * @return 是否更新成功 */ - Boolean saveTaskTemplateBasicInfo(TaskTemplateInfoDTO taskTemplateInfo); + TaskTemplateInfoDTO saveTaskTemplateBasicInfo(String username, TaskTemplateInfoDTO taskTemplateInfo); /** * 根据作业模版 ID 查询模版基础信息 @@ -151,19 +172,6 @@ PageData listPageTaskTemplatesBasicInfo(TaskTemplateQuery q */ boolean checkTemplateName(Long appId, Long templateId, String name); - /** - * 批量更新模版中的脚本引用状态 - *

- * 当脚本版本变化时由脚本管理服务调用此接口后台更新模版中的脚本引用状态 - * - * @param appId 业务 ID - * @param scriptId 脚本 ID - * @param scriptVersionId 脚本版本 ID - * @param status 引用状态 - * @return 更新任务是否入队成功 - */ - boolean updateScriptStatus(Long appId, String scriptId, Long scriptVersionId, JobResourceStatusEnum status); - /** * 获取用户收藏的作业模版基本信息列表 * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/template/impl/TaskTemplateFavoriteServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/template/impl/TaskTemplateFavoriteServiceImpl.java index 8b7376c328..b4c92b4747 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/template/impl/TaskTemplateFavoriteServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/template/impl/TaskTemplateFavoriteServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/template/impl/TaskTemplateServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/template/impl/TaskTemplateServiceImpl.java index 4419e334a0..0353d354d3 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/template/impl/TaskTemplateServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/template/impl/TaskTemplateServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,6 +24,10 @@ package com.tencent.bk.job.manage.service.template.impl; +import com.tencent.bk.audit.annotations.ActionAuditRecord; +import com.tencent.bk.audit.annotations.AuditInstanceRecord; +import com.tencent.bk.audit.context.ActionAuditContext; +import com.tencent.bk.job.common.audit.constants.EventContentConstants; import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.constant.JobResourceTypeEnum; import com.tencent.bk.job.common.exception.AbortedException; @@ -33,18 +37,22 @@ import com.tencent.bk.job.common.exception.InvalidParamException; import com.tencent.bk.job.common.exception.NotFoundException; import com.tencent.bk.job.common.exception.ServiceException; +import com.tencent.bk.job.common.iam.constant.ActionId; +import com.tencent.bk.job.common.iam.constant.ResourceTypeId; import com.tencent.bk.job.common.model.BaseSearchCondition; import com.tencent.bk.job.common.model.PageData; +import com.tencent.bk.job.common.model.dto.AppResourceScope; +import com.tencent.bk.job.common.mysql.JobTransactional; import com.tencent.bk.job.common.redis.util.LockUtils; import com.tencent.bk.job.common.util.JobContextUtil; import com.tencent.bk.job.common.util.PageUtil; import com.tencent.bk.job.common.util.date.DateUtils; import com.tencent.bk.job.crontab.model.CronJobVO; -import com.tencent.bk.job.manage.common.consts.JobResourceStatusEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskFileTypeEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskScriptSourceEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskStepTypeEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskTemplateStatusEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskFileTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskScriptSourceEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskStepTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskTemplateStatusEnum; +import com.tencent.bk.job.manage.auth.TemplateAuthService; import com.tencent.bk.job.manage.dao.template.TaskTemplateDAO; import com.tencent.bk.job.manage.model.dto.ResourceTagDTO; import com.tencent.bk.job.manage.model.dto.ScriptDTO; @@ -59,7 +67,7 @@ import com.tencent.bk.job.manage.service.AbstractTaskStepService; import com.tencent.bk.job.manage.service.AbstractTaskVariableService; import com.tencent.bk.job.manage.service.CronJobService; -import com.tencent.bk.job.manage.service.ScriptService; +import com.tencent.bk.job.manage.service.ScriptManager; import com.tencent.bk.job.manage.service.TagService; import com.tencent.bk.job.manage.service.TaskFavoriteService; import com.tencent.bk.job.manage.service.plan.TaskPlanService; @@ -70,8 +78,8 @@ import org.apache.commons.lang3.StringUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; import java.util.ArrayList; import java.util.Collections; @@ -93,16 +101,40 @@ public class TaskTemplateServiceImpl implements TaskTemplateService { private final AbstractTaskStepService taskStepService; private final AbstractTaskVariableService taskVariableService; private final TaskTemplateDAO taskTemplateDAO; - private final TemplateStatusUpdateService templateStatusUpdateService; + private final TemplateScriptStatusUpdateService templateScriptStatusUpdateService; private final TaskFavoriteService taskFavoriteService; - private final CronJobService cronJobService; + private final TemplateAuthService templateAuthService; + private TaskPlanService taskPlanService; + private ScriptManager scriptManager; + private CronJobService cronJobService; + /** + * 通过 Set 方式注入，避免循环依赖问题 + */ @Autowired - private TaskPlanService taskPlanService; + @Lazy + public void setTaskPlanService(TaskPlanService taskPlanService) { + this.taskPlanService = taskPlanService; + } + + /** + * 通过 Set 方式注入，避免循环依赖问题 + */ @Autowired - private ScriptService scriptService; + @Lazy + public void setScriptManager(ScriptManager scriptManager) { + this.scriptManager = scriptManager; + } + + /** + * 通过 Set 方式注入，避免循环依赖问题 + */ @Autowired - private TaskTemplateService taskTemplateService; + @Lazy + public void setCronJobService(CronJobService cronJobService) { + this.cronJobService = cronJobService; + } + @Autowired public TaskTemplateServiceImpl( @@ -110,19 +142,19 @@ public TaskTemplateServiceImpl( @Qualifier("TaskTemplateVariableServiceImpl") AbstractTaskVariableService taskVariableService, TaskTemplateDAO taskTemplateDAO, TagService tagService, - TemplateStatusUpdateService templateStatusUpdateService, + TemplateScriptStatusUpdateService templateScriptStatusUpdateService, @Qualifier("TaskTemplateFavoriteServiceImpl") TaskFavoriteService taskFavoriteService, - CronJobService cronJobService) { + TemplateAuthService templateAuthService) { this.taskStepService = taskStepService; this.taskVariableService = taskVariableService; this.taskTemplateDAO = taskTemplateDAO; this.tagService = tagService; - this.templateStatusUpdateService = templateStatusUpdateService; + this.templateScriptStatusUpdateService = templateScriptStatusUpdateService; this.taskFavoriteService = taskFavoriteService; - this.cronJobService = cronJobService; + this.templateAuthService = templateAuthService; } - private static void setUpdateFlag(TaskTemplateInfoDTO templateInfo) { + private void setUpdateFlag(TaskTemplateInfoDTO templateInfo) { if (templateInfo != null && CollectionUtils.isNotEmpty(templateInfo.getStepList())) { int scriptScript = 0; for (TaskStepDTO taskStep : templateInfo.getStepList()) { @@ -223,7 +255,7 @@ private PageData getTemplateById(TaskTemplateQuery query) { private void setAdditionalAttributesForTemplates(Long appId, PageData templatePageData) { if (CollectionUtils.isNotEmpty(templatePageData.getData())) { - templatePageData.getData().forEach(TaskTemplateServiceImpl::setUpdateFlag); + templatePageData.getData().forEach(this::setUpdateFlag); } setTags(appId, templatePageData.getData()); @@ -280,8 +312,87 @@ public TaskTemplateInfoDTO getTaskTemplateById(Long appId, Long templateId) { } @Override - @Transactional(rollbackFor = ServiceException.class) - public Long saveTaskTemplate(TaskTemplateInfoDTO taskTemplateInfo) { + @ActionAuditRecord( + actionId = ActionId.VIEW_JOB_TEMPLATE, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.TEMPLATE, + instanceIds = "#templateId", + instanceNames = "#$?.name" + ), + content = EventContentConstants.VIEW_JOB_TEMPLATE + ) + public TaskTemplateInfoDTO getTaskTemplate(String username, long appId, Long templateId) { + authViewJobTemplate(username, appId, templateId); + TaskTemplateInfoDTO templateInfo = getTaskTemplateById(appId, templateId); + if (templateInfo == null) { + throw new NotFoundException(ErrorCode.TEMPLATE_NOT_EXIST); + } + return templateInfo; + } + + private void authEditTemplate(String username, long appId, long templateId) { + templateAuthService.authEditJobTemplate(username, new AppResourceScope(appId), + templateId).denyIfNoPermission(); + } + + private void authViewJobTemplate(String username, long appId, long templateId) { + templateAuthService.authViewJobTemplate(username, new AppResourceScope(appId), + templateId).denyIfNoPermission(); + } + + private void authDeleteTemplate(String username, long appId, long templateId) { + templateAuthService.authDeleteJobTemplate(username, new AppResourceScope(appId), + templateId).denyIfNoPermission(); + } + + private void authCreateTemplate(String username, long appId) { + templateAuthService.authCreateJobTemplate(username, new AppResourceScope(appId)).denyIfNoPermission(); + } + + @Override + @JobTransactional(transactionManager = "jobManageTransactionManager") + @ActionAuditRecord( + actionId = ActionId.CREATE_JOB_TEMPLATE, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.TEMPLATE, + instanceIds = "#$?.id", + instanceNames = "#$?.name" + ), + content = EventContentConstants.CREATE_JOB_TEMPLATE + ) + public TaskTemplateInfoDTO saveTaskTemplate(String username, TaskTemplateInfoDTO taskTemplateInfo) { + authCreateTemplate(username, taskTemplateInfo.getAppId()); + TaskTemplateInfoDTO createdTemplate = saveOrUpdateTaskTemplate(taskTemplateInfo); + templateAuthService.registerTemplate(createdTemplate.getId(), createdTemplate.getName(), username); + return createdTemplate; + } + + @Override + @JobTransactional(transactionManager = "jobManageTransactionManager") + @ActionAuditRecord( + actionId = ActionId.EDIT_JOB_TEMPLATE, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.TEMPLATE, + instanceIds = "#taskTemplateInfo?.id", + instanceNames = "#taskTemplateInfo?.name" + ), + content = EventContentConstants.EDIT_JOB_TEMPLATE + ) + public TaskTemplateInfoDTO updateTaskTemplate(String username, TaskTemplateInfoDTO taskTemplateInfo) { + authEditTemplate(username, taskTemplateInfo.getAppId(), taskTemplateInfo.getId()); + + TaskTemplateInfoDTO template = saveOrUpdateTaskTemplate(taskTemplateInfo); + + // 审计 + ActionAuditContext.current() + .setOriginInstance(TaskTemplateInfoDTO.toEsbTemplateInfoV3DTO( + getTaskTemplateById(taskTemplateInfo.getAppId(), taskTemplateInfo.getId()))) + .setInstance(TaskTemplateInfoDTO.toEsbTemplateInfoV3DTO(template)); + + return template; + } + + private TaskTemplateInfoDTO saveOrUpdateTaskTemplate(TaskTemplateInfoDTO taskTemplateInfo) { String lockKey = null; try { boolean isCreate = false; @@ -295,7 +406,7 @@ public Long saveTaskTemplate(TaskTemplateInfoDTO taskTemplateInfo) { throw new AbortedException(ErrorCode.TEMPLATE_LOCK_ACQUIRE_FAILED); } // 保存新增的标签并获取tagId - taskTemplateService.createNewTagForTemplateIfNotExist(taskTemplateInfo); + createNewTagForTemplateIfNotExist(taskTemplateInfo); // 获取引用的非线上脚本 Map outdatedScriptMap = getOutdatedScriptMap(taskTemplateInfo.getStepList()); @@ -327,7 +438,7 @@ public Long saveTaskTemplate(TaskTemplateInfoDTO taskTemplateInfo) { Long templateId; if (isCreate) { taskTemplateInfo.setCreateTime(DateUtils.currentTimeSeconds()); - templateId = taskTemplateService.insertNewTemplate(taskTemplateInfo); + templateId = insertNewTemplate(taskTemplateInfo); taskTemplateInfo.setId(templateId); } else { boolean bumpVersion = templateHasChange(taskTemplateInfo); @@ -340,7 +451,7 @@ public Long saveTaskTemplate(TaskTemplateInfoDTO taskTemplateInfo) { updateTemplateTags(taskTemplateInfo); // 写步骤 - taskTemplateService.processTemplateStep(taskTemplateInfo); + processTemplateStep(taskTemplateInfo); // 更新作业模板首尾步骤 // Process first and last step id @@ -359,7 +470,7 @@ public Long saveTaskTemplate(TaskTemplateInfoDTO taskTemplateInfo) { } else { if (taskVariable.getId() > 0) { // Update exist variable - taskVariableService.updateVariableById(taskVariable); + taskVariableService.updateVarByParentResourceIdAndTplVarId(taskVariable); } else { newVariables.add(taskVariable); } @@ -368,8 +479,9 @@ public Long saveTaskTemplate(TaskTemplateInfoDTO taskTemplateInfo) { // Insert new variable taskVariableService.batchInsertVariable(newVariables); - templateStatusUpdateService.offerMessage(templateId); - return templateId; + templateScriptStatusUpdateService.refreshTemplateScriptStatusByTemplate(templateId); + + return getTaskTemplateById(taskTemplateInfo.getAppId(), templateId); } catch (ServiceException e) { throw e; } catch (Exception e) { @@ -384,7 +496,7 @@ public Long saveTaskTemplate(TaskTemplateInfoDTO taskTemplateInfo) { private boolean templateHasChange(TaskTemplateInfoDTO taskTemplateInfo) { TaskTemplateInfoDTO originTaskTemplateInfo = - taskTemplateService.getTaskTemplateById(taskTemplateInfo.getAppId(), taskTemplateInfo.getId()); + getTaskTemplateById(taskTemplateInfo.getAppId(), taskTemplateInfo.getId()); if (originTaskTemplateInfo == null) { throw new NotFoundException(ErrorCode.TEMPLATE_NOT_EXIST); } @@ -461,12 +573,29 @@ public void createNewTagForTemplateIfNotExist(TaskTemplateInfoDTO taskTemplateIn } @Override - @Transactional - public Boolean deleteTaskTemplate(Long appId, Long templateId) { + @JobTransactional(transactionManager = "jobManageTransactionManager") + @ActionAuditRecord( + actionId = ActionId.DELETE_JOB_TEMPLATE, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.TEMPLATE, + instanceIds = "#templateId", + instanceNames = "#$?.name" + ), + content = EventContentConstants.DELETE_JOB_TEMPLATE + ) + public TaskTemplateInfoDTO deleteTaskTemplate(String username, Long appId, Long templateId) { + authDeleteTemplate(username, appId, templateId); + + TaskTemplateInfoDTO template = getTaskTemplateById(appId, templateId); + if (template == null) { + log.warn("Delete job template, the template is not exist"); + throw new NotFoundException(ErrorCode.TEMPLATE_NOT_EXIST); + } + List taskPlanInfoList = taskPlanService.listTaskPlansBasicInfo(appId, templateId); if (CollectionUtils.isNotEmpty(taskPlanInfoList)) { List taskPlanIdList = - taskPlanInfoList.parallelStream().map(TaskPlanInfoDTO::getId).collect(Collectors.toList()); + taskPlanInfoList.stream().map(TaskPlanInfoDTO::getId).collect(Collectors.toList()); if (CollectionUtils.isNotEmpty(taskPlanIdList)) { Map> taskPlanCronJobMap = cronJobService.batchListCronJobByPlanIds(appId, taskPlanIdList); @@ -478,11 +607,11 @@ public Boolean deleteTaskTemplate(Long appId, Long templateId) { } } } + taskPlanService.deleteTaskPlanByTemplate(appId, templateId); } - taskPlanService.deleteTaskPlanByTemplate(appId, templateId); taskTemplateDAO.deleteTaskTemplateById(appId, templateId); tagService.batchDeleteResourceTags(appId, JobResourceTypeEnum.TEMPLATE.getValue(), String.valueOf(templateId)); - return true; + return template; } @Override @@ -507,13 +636,37 @@ public TagCountVO getTagTemplateCount(Long appId) { } @Override - public Boolean saveTaskTemplateBasicInfo(TaskTemplateInfoDTO taskTemplateInfo) { + @ActionAuditRecord( + actionId = ActionId.EDIT_JOB_TEMPLATE, + instance = @AuditInstanceRecord( + resourceType = ResourceTypeId.TEMPLATE, + instanceIds = "#taskTemplateInfo?.id", + instanceNames = "#taskTemplateInfo?.name" + ), + content = EventContentConstants.EDIT_JOB_TEMPLATE + ) + public TaskTemplateInfoDTO saveTaskTemplateBasicInfo(String username, TaskTemplateInfoDTO taskTemplateInfo) { + authEditTemplate(username, taskTemplateInfo.getAppId(), taskTemplateInfo.getId()); + + TaskTemplateInfoDTO originTemplate = getTaskTemplateById(taskTemplateInfo.getAppId(), taskTemplateInfo.getId()); + if (originTemplate == null) { + throw new NotFoundException(ErrorCode.TEMPLATE_NOT_EXIST); + } + createNewTagForTemplateIfNotExist(taskTemplateInfo); updateTemplateTags(taskTemplateInfo); if (!taskTemplateDAO.updateTaskTemplateById(taskTemplateInfo, false)) { throw new InternalException(ErrorCode.UPDATE_TEMPLATE_FAILED); } - return true; + + TaskTemplateInfoDTO updatedTemplate = + getTaskTemplateById(taskTemplateInfo.getAppId(), taskTemplateInfo.getId()); + // 审计 + ActionAuditContext.current() + .setOriginInstance(TaskTemplateInfoDTO.toEsbTemplateInfoV3DTO(originTemplate)) + .setInstance(TaskTemplateInfoDTO.toEsbTemplateInfoV3DTO(updatedTemplate)); + + return updatedTemplate; } private void updateTemplateTags(TaskTemplateInfoDTO taskTemplateInfo) { @@ -559,15 +712,6 @@ public boolean checkTemplateName(Long appId, Long templateId, String name) { return taskTemplateDAO.checkTemplateName(appId, templateId, name); } - @Override - public boolean updateScriptStatus(Long appId, String scriptId, Long scriptVersionId, JobResourceStatusEnum status) { - try { - return templateStatusUpdateService.offerMessage(scriptId, scriptVersionId, status); - } catch (InterruptedException e) { - log.error("Offer message failed! Maybe queue full!", e); - } - return false; - } @Override public List getFavoredTemplateBasicInfo(Long appId, String username) { @@ -583,7 +727,7 @@ public List getFavoredTemplateBasicInfo(Long appId, String } @Override - @Transactional(rollbackFor = Throwable.class) + @JobTransactional(transactionManager = "jobManageTransactionManager") public Long saveTaskTemplateForMigration( TaskTemplateInfoDTO taskTemplateInfo, Long createTime, @@ -595,7 +739,7 @@ public Long saveTaskTemplateForMigration( if (taskTemplateByName != null) { throw new AlreadyExistsException(ErrorCode.TEMPLATE_NAME_EXIST); } - taskTemplateService.createNewTagForTemplateIfNotExist(taskTemplateInfo); + createNewTagForTemplateIfNotExist(taskTemplateInfo); if (createTime != null && createTime > 0) { taskTemplateInfo.setCreateTime(createTime); @@ -620,11 +764,11 @@ public Long saveTaskTemplateForMigration( // process template id Long templateId; if (taskTemplateInfo.getId() == null || taskTemplateInfo.getId() <= 0) { - templateId = taskTemplateService.insertNewTemplate(taskTemplateInfo); + templateId = insertNewTemplate(taskTemplateInfo); taskTemplateInfo.setId(templateId); } else { taskTemplateInfo.setStatus(TaskTemplateStatusEnum.NEW); - if (taskTemplateService.checkTemplateId(taskTemplateInfo.getId())) { + if (checkTemplateId(taskTemplateInfo.getId())) { if (insertNewTemplateWithTemplateId(taskTemplateInfo)) { templateId = taskTemplateInfo.getId(); } else { @@ -637,7 +781,7 @@ public Long saveTaskTemplateForMigration( updateTemplateTags(taskTemplateInfo); - taskTemplateService.processTemplateStep(taskTemplateInfo); + processTemplateStep(taskTemplateInfo); // Process first and last step id TaskTemplateInfoDTO updateStepIdReq = generateUpdateStepIdReq(taskTemplateInfo); @@ -848,7 +992,7 @@ private Map getOutdatedScriptMap(List stepList) { if (MapUtils.isNotEmpty(scriptVersionMap)) { Map scriptInfoMap; try { - scriptInfoMap = scriptService + scriptInfoMap = scriptManager .batchGetOnlineScriptVersionByScriptIds(new ArrayList<>(scriptVersionMap.keySet())); } catch (ServiceException e) { log.error("Error while getting online script version!", e); @@ -870,8 +1014,4 @@ private Map getOutdatedScriptMap(List stepList) { } return outdatedScriptMap; } - - public void setScriptService(ScriptService scriptService) { - this.scriptService = scriptService; - } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/template/impl/TaskTemplateStepServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/template/impl/TaskTemplateStepServiceImpl.java index ada911ec22..228038c982 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/template/impl/TaskTemplateStepServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/template/impl/TaskTemplateStepServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,7 @@ package com.tencent.bk.job.manage.service.template.impl; -import com.tencent.bk.job.manage.common.consts.task.TaskTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskTypeEnum; import com.tencent.bk.job.manage.dao.ScriptDAO; import com.tencent.bk.job.manage.dao.TaskApprovalStepDAO; import com.tencent.bk.job.manage.dao.TaskFileInfoDAO; diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/template/impl/TaskTemplateVariableServiceImpl.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/template/impl/TaskTemplateVariableServiceImpl.java index 84d2f3f4cf..ec48e17937 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/template/impl/TaskTemplateVariableServiceImpl.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/template/impl/TaskTemplateVariableServiceImpl.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/template/impl/TemplateScriptStatusUpdateService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/template/impl/TemplateScriptStatusUpdateService.java new file mode 100644 index 0000000000..dfce006308 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/template/impl/TemplateScriptStatusUpdateService.java @@ -0,0 +1,228 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.service.template.impl; + +import com.tencent.bk.job.common.mysql.JobTransactional; +import com.tencent.bk.job.manage.api.common.constants.JobResourceStatusEnum; +import com.tencent.bk.job.manage.common.util.TemplateScriptStatusFlagsUtil; +import com.tencent.bk.job.manage.dao.ScriptDAO; +import com.tencent.bk.job.manage.dao.template.TaskTemplateDAO; +import com.tencent.bk.job.manage.dao.template.TaskTemplateScriptStepDAO; +import com.tencent.bk.job.manage.model.dto.TemplateStepScriptStatusInfo; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; +import org.springframework.util.StopWatch; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +@Slf4j +@Component +public class TemplateScriptStatusUpdateService { + private final TaskTemplateDAO taskTemplateDAO; + private final TaskTemplateScriptStepDAO taskTemplateScriptStepDAO; + + private final ScriptDAO scriptDAO; + + @Autowired + public TemplateScriptStatusUpdateService(TaskTemplateDAO taskTemplateDAO, + TaskTemplateScriptStepDAO taskTemplateScriptStepDAO, + ScriptDAO scriptDAO) { + this.taskTemplateDAO = taskTemplateDAO; + this.taskTemplateScriptStepDAO = taskTemplateScriptStepDAO; + this.scriptDAO = scriptDAO; + } + + @JobTransactional(transactionManager = "jobManageTransactionManager") + public void refreshTemplateScriptStatusByTemplate(long templateId) { + log.info("Refresh template script status flags by templateId : {}", templateId); + + StopWatch watch = new StopWatch("refreshTemplateScriptStatusByTemplate"); + try { + watch.start("getTemplateStepScriptStatusInfos"); + List steps = getTemplateStepScriptStatusInfos(templateId); + if (log.isDebugEnabled()) { + log.debug("Affected steps : {}", steps); + } + watch.stop(); + updateScriptStatusFlags(watch, steps); + log.info("Refresh template script status flags by templateId success"); + } finally { + if (watch.isRunning()) { + watch.stop(); + } + if (watch.getTotalTimeMillis() > 1000L) { + log.info("RefreshTemplateScriptStatusByTemplate is slow, cost: {}", watch.prettyPrint()); + } + } + + } + + @JobTransactional(transactionManager = "jobManageTransactionManager") + public void refreshTemplateScriptStatusByScript(String scriptId, long scriptVersionId) { + log.info("Refresh template script status flags by script, scriptId: {}, scriptVersionId: {}", + scriptId, scriptVersionId); + + StopWatch watch = new StopWatch("refreshTemplateScriptStatusByScript"); + try { + watch.start("getTemplateStepScriptStatusInfos"); + List steps = getTemplateStepScriptStatusInfos(scriptId, scriptVersionId); + if (log.isDebugEnabled()) { + log.debug("Affected steps : {}", steps); + } + watch.stop(); + updateScriptStatusFlags(watch, steps); + log.info("Refresh template script status flags by script success"); + } finally { + if (watch.isRunning()) { + watch.stop(); + } + if (watch.getTotalTimeMillis() > 1000L) { + log.info("RefreshTemplateScriptStatusByScript is slow, cost: {}", watch.prettyPrint()); + } + + } + } + + private void updateScriptStatusFlags(StopWatch watch, List steps) { + if (CollectionUtils.isEmpty(steps)) { + log.info("None template and step script status need update!"); + return; + } + // 查询脚本状态实时数据并重置 + watch.start("resetStepsScriptStatusFlags"); + resetStepsScriptStatusFlags(steps); + watch.stop(); + + // 更新步骤脚本状态 + watch.start("updateStepsScriptStatusFlags"); + updateStepsScriptStatusFlags(steps); + watch.stop(); + + watch.start("updateTemplateScriptStatusFlags"); + // 更新模版整体脚本状态 + updateTemplateScriptStatusFlags(steps); + watch.stop(); + } + + private List getTemplateStepScriptStatusInfos(String scriptId, + long scriptVersionId) { + return taskTemplateScriptStepDAO.listAllRelatedTemplateStepsScriptStatusInfo(scriptId, scriptVersionId); + } + + private List getTemplateStepScriptStatusInfos(long templateId) { + return taskTemplateScriptStepDAO.listStepsScriptStatusInfoByTemplateId(templateId); + } + + /* + * 更新步骤上的脚本状态flags + */ + private void updateStepsScriptStatusFlags(List templateStepScriptStatusInfos) { + // 先按照状态组织数据，方便后续进行批量更新 + Map> scriptStatusFlagsGroups = + templateStepScriptStatusInfos.stream() + .collect(Collectors.groupingBy(TemplateStepScriptStatusInfo::getScriptStatusFlags)); + scriptStatusFlagsGroups.forEach((scriptStatusFlags, steps) -> { + List stepIds = steps.stream() + .map(TemplateStepScriptStatusInfo::getStepId).distinct().collect(Collectors.toList()); + log.info("Update template steps script status flags, stepIds: {}, scriptStatusFlags: {}", + stepIds, scriptStatusFlags); + taskTemplateScriptStepDAO.batchUpdateScriptStatusFlags(stepIds, scriptStatusFlags); + }); + } + + /* + * 更新模版整体脚本状态flags + */ + private void updateTemplateScriptStatusFlags(List templateStepScriptStatusInfos) { + Map> templateSteps = + templateStepScriptStatusInfos.stream() + .collect(Collectors.groupingBy(TemplateStepScriptStatusInfo::getTemplateId)); + Map templateScriptStatusFlagsMap = new HashMap<>(); + templateSteps.forEach((templateId, steps) -> + templateScriptStatusFlagsMap.put(templateId, computeTemplateScriptStatusFlags(steps))); + + log.info("Update template script status flags, templateScriptStatusFlagsMap: {}", + templateScriptStatusFlagsMap); + + // 先按照状态组织数据，方便后续进行批量更新 + Map> scriptStatusFlagsGroups = + templateScriptStatusFlagsMap.entrySet().stream() + .collect(Collectors.groupingBy( + Map.Entry::getValue, + Collectors.mapping(Map.Entry::getKey, Collectors.toList())) + ); + scriptStatusFlagsGroups.forEach((scriptStatusFlags, templateIds) -> + taskTemplateDAO.batchUpdateTemplateScriptStatus(templateIds, scriptStatusFlags)); + } + + private void resetStepsScriptStatusFlags(List templateStepScriptStatusInfos) { + Map scriptVersionStatusMap = + batchGetScriptVersionStatus(templateStepScriptStatusInfos); + templateStepScriptStatusInfos.forEach( + templateStepScriptStatusInfo -> + templateStepScriptStatusInfo.setScriptStatusFlags( + toScriptStatusBinaryFlags( + scriptVersionStatusMap.get(templateStepScriptStatusInfo.getScriptVersionId())) + ) + ); + } + + private int toScriptStatusBinaryFlags(JobResourceStatusEnum scriptStatus) { + switch (scriptStatus) { + case ONLINE: + return TemplateScriptStatusFlagsUtil.computeScriptStatusFlags(false, false); + case OFFLINE: + return TemplateScriptStatusFlagsUtil.computeScriptStatusFlags(true, false); + case DISABLED: + return TemplateScriptStatusFlagsUtil.computeScriptStatusFlags(false, true); + default: + log.error("Unexpected script status"); + throw new IllegalStateException("Unexpected script status [" + scriptStatus + "]"); + } + } + + private Map batchGetScriptVersionStatus( + List templateStepScriptStatusInfos) { + List scriptVersionIds = templateStepScriptStatusInfos.stream() + .map(TemplateStepScriptStatusInfo::getScriptVersionId) + .distinct() + .collect(Collectors.toList()); + return scriptDAO.batchGetScriptVersionStatus(scriptVersionIds); + } + + private int computeTemplateScriptStatusFlags( + List templateStepScriptStatusInfos) { + boolean existOfflineScript = templateStepScriptStatusInfos.stream() + .anyMatch(steps -> TemplateScriptStatusFlagsUtil.readOfflineFlag(steps.getScriptStatusFlags())); + boolean existDisabledScript = templateStepScriptStatusInfos.stream() + .anyMatch(steps -> TemplateScriptStatusFlagsUtil.readDisableFlag(steps.getScriptStatusFlags())); + return TemplateScriptStatusFlagsUtil.computeScriptStatusFlags(existOfflineScript, existDisabledScript); + } +} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/template/impl/TemplateStatusUpdateService.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/template/impl/TemplateStatusUpdateService.java deleted file mode 100644 index 4ea5eee1db..0000000000 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/service/template/impl/TemplateStatusUpdateService.java +++ /dev/null @@ -1,252 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.service.template.impl; - -import com.tencent.bk.job.common.constant.ErrorCode; -import com.tencent.bk.job.common.exception.InvalidParamException; -import com.tencent.bk.job.manage.common.consts.JobResourceStatusEnum; -import com.tencent.bk.job.manage.dao.template.TaskTemplateDAO; -import com.tencent.bk.job.manage.model.dto.ScriptStatusUpdateMessageDTO; -import com.tencent.bk.job.manage.service.AbstractTaskStepService; -import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; -import org.apache.commons.collections4.MapUtils; -import org.apache.commons.lang3.StringUtils; -import org.jooq.Condition; -import org.jooq.DSLContext; -import org.jooq.Record1; -import org.jooq.Record2; -import org.jooq.Result; -import org.jooq.generated.tables.ScriptVersion; -import org.jooq.generated.tables.TaskTemplateStepScript; -import org.jooq.types.UByte; -import org.jooq.types.ULong; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.stereotype.Component; -import org.springframework.transaction.annotation.Transactional; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; -import java.util.concurrent.LinkedBlockingQueue; -import java.util.concurrent.TimeUnit; - -@Slf4j -@Component -public class TemplateStatusUpdateService { - private final TaskTemplateDAO taskTemplateDAO; - private static final TaskTemplateStepScript STEP_SCRIPT_TABLE = TaskTemplateStepScript.TASK_TEMPLATE_STEP_SCRIPT; - private static final ScriptVersion SCRIPT_VERSION_TABLE = ScriptVersion.SCRIPT_VERSION; - private static final LinkedBlockingQueue UPDATE_MESSAGE_QUEUE = - new LinkedBlockingQueue<>(); - private DSLContext context; - @Autowired - private TemplateStatusUpdateService templateStatusUpdateService; - - private AbstractTaskStepService taskStepService; - - @Autowired - public TemplateStatusUpdateService( - @Qualifier("job-manage-dsl-context") DSLContext context, - @Qualifier("TaskTemplateStepServiceImpl") AbstractTaskStepService taskStepService, - TaskTemplateDAO taskTemplateDAO - ) { - this.context = context; - this.taskStepService = taskStepService; - this.taskTemplateDAO = taskTemplateDAO; - TemplateStatusUpdateThread templateStatusUpdateThread = new TemplateStatusUpdateThread(); - templateStatusUpdateThread.start(); - } - - public boolean offerMessage(long templateId) throws InterruptedException { - return offerMessage(templateId, 10L, TimeUnit.SECONDS); - } - - public boolean offerMessage(long templateId, long timeout, TimeUnit timeUnit) throws InterruptedException { - ScriptStatusUpdateMessageDTO scriptStatusUpdateMessage = new ScriptStatusUpdateMessageDTO(); - scriptStatusUpdateMessage.setTemplateId(templateId); - return UPDATE_MESSAGE_QUEUE.offer(scriptStatusUpdateMessage, timeout, timeUnit); - } - - public boolean offerMessage(String scriptId, long scriptVersionId, JobResourceStatusEnum status) - throws InterruptedException { - return offerMessage(scriptId, scriptVersionId, status, 10L, TimeUnit.SECONDS); - } - - public boolean offerMessage(String scriptId, long scriptVersionId, JobResourceStatusEnum status, long timeout, - TimeUnit timeUnit) throws InterruptedException { - ScriptStatusUpdateMessageDTO scriptStatusUpdateMessage = new ScriptStatusUpdateMessageDTO(); - scriptStatusUpdateMessage.setScriptId(scriptId); - scriptStatusUpdateMessage.setScriptVersionId(scriptVersionId); - scriptStatusUpdateMessage.setStatus(status); - return UPDATE_MESSAGE_QUEUE.offer(scriptStatusUpdateMessage, timeout, timeUnit); - } - - void doUpdateStatus(String uuid, ScriptStatusUpdateMessageDTO scriptStatusUpdateMessage) { - log.debug("{}|Start process script status update message...|{}", uuid, scriptStatusUpdateMessage); - if (scriptStatusUpdateMessage.getTemplateId() != null && scriptStatusUpdateMessage.getTemplateId() > 0) { - log.debug("Processing template status...|{}", scriptStatusUpdateMessage); - templateStatusUpdateService.processTemplateStatus(ULong.valueOf(scriptStatusUpdateMessage.getTemplateId())); - } else if (StringUtils.isNotBlank(scriptStatusUpdateMessage.getScriptId()) - && scriptStatusUpdateMessage.getScriptVersionId() != null - && scriptStatusUpdateMessage.getScriptVersionId() > 0 && scriptStatusUpdateMessage.getStatus() != null) { - - List conditions = new ArrayList<>(); - conditions.add(STEP_SCRIPT_TABLE.SCRIPT_ID.eq(scriptStatusUpdateMessage.getScriptId())); - conditions.add( - STEP_SCRIPT_TABLE.SCRIPT_VERSION_ID.eq(ULong.valueOf(scriptStatusUpdateMessage.getScriptVersionId()))); - - Result> records = - context.select(STEP_SCRIPT_TABLE.TEMPLATE_ID).from(STEP_SCRIPT_TABLE).where(conditions).fetch(); - Set templateIdSet; - if (records != null && records.size() > 0) { - templateIdSet = records.intoSet(STEP_SCRIPT_TABLE.TEMPLATE_ID); - if (CollectionUtils.isNotEmpty(templateIdSet)) { - log.debug("Processing template status...|{}|{}", scriptStatusUpdateMessage, templateIdSet); - templateIdSet.forEach(templateId -> { - try { - templateStatusUpdateService.processTemplateStatus(templateId); - } catch (Exception e) { - log.error("Error while processing template status!|{}", templateId, e); - } - }); - } else { - log.error("Error while process template status!|Convert records failed!|{}|{}", - scriptStatusUpdateMessage, records); - } - log.debug("Process template status finished.|{}", templateIdSet); - } else { - log.warn("Error while process template status!|No template to update!|{}", scriptStatusUpdateMessage); - } - } else { - log.error("Error while process template status!|Param error!|{}", scriptStatusUpdateMessage); - throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM); - } - } - - @Transactional(rollbackFor = {Exception.class, Error.class}) - public void processTemplateStatus(ULong templateId) { - Result> records = - context.select(STEP_SCRIPT_TABLE.ID, STEP_SCRIPT_TABLE.SCRIPT_VERSION_ID).from(STEP_SCRIPT_TABLE) - .where(STEP_SCRIPT_TABLE.TEMPLATE_ID.eq(templateId)).fetch(); - if (records != null && records.size() > 0) { - Map templateScriptVersionMap = - records.intoMap(STEP_SCRIPT_TABLE.ID, STEP_SCRIPT_TABLE.SCRIPT_VERSION_ID); - if (MapUtils.isNotEmpty(templateScriptVersionMap)) { - Result> scriptVersionStatusInfo = - context.select(SCRIPT_VERSION_TABLE.ID, SCRIPT_VERSION_TABLE.STATUS).from(SCRIPT_VERSION_TABLE) - .where(SCRIPT_VERSION_TABLE.ID.in(templateScriptVersionMap.values())).fetch(); - - if (scriptVersionStatusInfo != null && scriptVersionStatusInfo.size() > 0) { - - Set offlineVersions = new HashSet<>(); - Set disabledVersions = new HashSet<>(); - - scriptVersionStatusInfo.forEach(record -> { - switch (record.get(SCRIPT_VERSION_TABLE.STATUS).intValue()) { - case 2: - // JobResourceStatusEnum.Offline - offlineVersions.add(record.get(SCRIPT_VERSION_TABLE.ID)); - break; - case 3: - // JobResourceStatusEnum.Disabled - disabledVersions.add(record.get(SCRIPT_VERSION_TABLE.ID)); - break; - default: - break; - } - }); - - int scriptStatus = 0; - - if (offlineVersions.size() > 0) { - scriptStatus |= 0b1; - } - if (disabledVersions.size() > 0) { - scriptStatus |= 0b10; - } - - templateStatusUpdateService.updateTemplateStatus(templateId, scriptStatus); - - processScriptStepStatus(templateScriptVersionMap, offlineVersions, disabledVersions); - - } else { - templateStatusUpdateService.updateTemplateStatus(templateId, 0); - log.warn("Getting script version info failed!|{}|{}", templateId, templateScriptVersionMap); - } - } else { - templateStatusUpdateService.updateTemplateStatus(templateId, 0); - log.error("Error while process template status!|Convert records failed!|{}|{}", templateId, records); - } - } else { - templateStatusUpdateService.updateTemplateStatus(templateId, 0); - log.warn("Error while process template status!|Fetch template step info failed!|{}", templateId); - } - } - - private void processScriptStepStatus(Map templateScriptVersionMap, Set offlineVersions, - Set disabledVersions) { - for (Map.Entry scriptStepInfo : templateScriptVersionMap.entrySet()) { - if (offlineVersions.contains(scriptStepInfo.getValue())) { - context.update(STEP_SCRIPT_TABLE).set(STEP_SCRIPT_TABLE.STATUS, UByte.valueOf(0b1)) - .where(STEP_SCRIPT_TABLE.ID.equal(scriptStepInfo.getKey())).limit(1).execute(); - } else if (disabledVersions.contains(scriptStepInfo.getValue())) { - context.update(STEP_SCRIPT_TABLE).set(STEP_SCRIPT_TABLE.STATUS, UByte.valueOf(0b10)) - .where(STEP_SCRIPT_TABLE.ID.equal(scriptStepInfo.getKey())).limit(1).execute(); - } else { - context.update(STEP_SCRIPT_TABLE).set(STEP_SCRIPT_TABLE.STATUS, UByte.valueOf(0)) - .where(STEP_SCRIPT_TABLE.ID.equal(scriptStepInfo.getKey())).limit(1).execute(); - } - } - } - - @Transactional(rollbackFor = {Exception.class, Error.class}) - public void updateTemplateStatus(ULong templateId, int scriptStatus) { - taskTemplateDAO.updateTemplateStatus(templateId, scriptStatus); - } - - class TemplateStatusUpdateThread extends Thread { - @Override - public void run() { - this.setName("Template-Status-Update-Thread"); - while (true) { - String uuid = UUID.randomUUID().toString(); - log.debug("{}|Waiting for script update message...", uuid); - try { - doUpdateStatus(uuid, UPDATE_MESSAGE_QUEUE.take()); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } catch (Exception e) { - log.error("{}|Error while updating template status!", uuid); - } - log.debug("{}|Script update message processed.", uuid); - } - } - } -} diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/task/ClearDeletedHostsTask.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/task/ClearDeletedHostsTask.java index e7f4f898bf..04c0875467 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/task/ClearDeletedHostsTask.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/task/ClearDeletedHostsTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/task/EsbUserInfoUpdateTask.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/task/EsbUserInfoUpdateTask.java index 5caf17b2c1..02dfb63e1a 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/task/EsbUserInfoUpdateTask.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/task/EsbUserInfoUpdateTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,18 +25,17 @@ package com.tencent.bk.job.manage.task; import com.tencent.bk.job.common.model.dto.BkUserDTO; +import com.tencent.bk.job.common.mysql.JobTransactional; +import com.tencent.bk.job.common.paas.user.UserMgrApiClient; import com.tencent.bk.job.common.redis.util.LockUtils; import com.tencent.bk.job.common.redis.util.RedisKeyHeartBeatThread; import com.tencent.bk.job.common.util.ip.IpUtils; import com.tencent.bk.job.manage.dao.notify.EsbUserInfoDAO; import com.tencent.bk.job.manage.model.dto.notify.EsbUserInfoDTO; -import com.tencent.bk.job.manage.service.PaaSService; import lombok.extern.slf4j.Slf4j; import lombok.val; import lombok.var; import org.apache.commons.lang3.StringUtils; -import org.jooq.DSLContext; -import org.jooq.impl.DSL; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -48,6 +47,7 @@ import java.util.Arrays; import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.stream.Collectors; /** @@ -77,15 +77,14 @@ public class EsbUserInfoUpdateTask { private final String REDIS_KEY_SYNC_USER_JOB_RUNNING_MACHINE = "sync-user-job-running-machine"; private final RedisTemplate redisTemplate; - private DSLContext dslContext; - private PaaSService paaSService; - private EsbUserInfoDAO esbUserInfoDAO; + private UserMgrApiClient userMgrApiClient; + private final EsbUserInfoDAO esbUserInfoDAO; @Autowired - public EsbUserInfoUpdateTask(DSLContext dslContext, PaaSService paaSService, EsbUserInfoDAO esbUserInfoDAO, + public EsbUserInfoUpdateTask(UserMgrApiClient userMgrApiClient, + EsbUserInfoDAO esbUserInfoDAO, RedisTemplate redisTemplate) { - this.dslContext = dslContext; - this.paaSService = paaSService; + this.userMgrApiClient = userMgrApiClient; this.esbUserInfoDAO = esbUserInfoDAO; this.redisTemplate = redisTemplate; } @@ -119,7 +118,7 @@ public boolean execute() { watch.start("total"); try { // 1.接口数据拉取 - List userList = paaSService.getAllUserList("", "100"); + List userList = userMgrApiClient.getAllUserList(); if (null == userList) { userList = new ArrayList<>(); } @@ -132,8 +131,8 @@ public boolean execute() { } // 3.计算差异数据 - val localUserSet = new HashSet(esbUserInfoDAO.listEsbUserInfo()); - val clonedRemoteUserSet = new HashSet(remoteUserSet); + val localUserSet = new HashSet<>(esbUserInfoDAO.listEsbUserInfo()); + val clonedRemoteUserSet = new HashSet<>(remoteUserSet); remoteUserSet.removeAll(localUserSet); val insertSet = remoteUserSet; logger.info("insertUserInfoSet=" + insertSet.stream() @@ -144,12 +143,7 @@ public boolean execute() { .map(EsbUserInfoDTO::toString).collect(Collectors.joining(","))); // 4.入库 - dslContext.transaction(configuration -> { - val context = DSL.using(configuration); - deleteSet.forEach(esbUserInfoDTO -> esbUserInfoDAO.deleteEsbUserInfoById(context, - esbUserInfoDTO.getId())); - insertSet.forEach(esbUserInfoDTO -> esbUserInfoDAO.insertEsbUserInfo(context, esbUserInfoDTO)); - }); + saveEsbUserInfos(deleteSet, insertSet); } catch (Throwable t) { log.error("FATAL: syncUser thread fail", t); } finally { @@ -159,4 +153,11 @@ public boolean execute() { } return true; } + + @JobTransactional(transactionManager = "jobManageTransactionManager") + public void saveEsbUserInfos(Set deleteSet, Set insertSet) { + deleteSet.forEach(esbUserInfoDTO -> esbUserInfoDAO.deleteEsbUserInfoById( + esbUserInfoDTO.getId())); + insertSet.forEach(esbUserInfoDTO -> esbUserInfoDAO.insertEsbUserInfo(esbUserInfoDTO)); + } } diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/task/ScheduledTasks.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/task/ScheduledTasks.java index 2ce0bb2182..11f68dac4e 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/task/ScheduledTasks.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/task/ScheduledTasks.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,6 @@ package com.tencent.bk.job.manage.task; -import com.tencent.bk.job.manage.common.client.PAASClientFactory; import com.tencent.bk.job.manage.manager.app.ApplicationCache; import com.tencent.bk.job.manage.service.SyncService; import lombok.extern.slf4j.Slf4j; @@ -33,13 +32,8 @@ import org.springframework.scheduling.annotation.Scheduled; import org.springframework.stereotype.Component; -/** - * @Description 增加调度需要注意到ScheduleConfig中更新线程池配置 - * @Date 2020/1/3 - * @Version 1.0 - */ @Slf4j -@Component +@Component("jobManageScheduledTasks") @EnableScheduling public class ScheduledTasks { @@ -115,9 +109,9 @@ public void hostSyncTask() { } /** - * Agent状态同步：3min/次 + * Agent状态同步：1min/次 */ - @Scheduled(cron = "0 0/3 * * * ?") + @Scheduled(cron = "0 * * * * ?") public void agentStatusSyncTask() { log.info(Thread.currentThread().getId() + ":agentStatusSyncTask start"); try { @@ -138,16 +132,6 @@ public void cleanUserUploadFileTask() { log.info("Clean user upload file task finished"); } - @Scheduled(cron = "0 0 0 * * ?") - public void resetTodayStatistics() { - log.info("resetTodayStatistics begin"); - try { - PAASClientFactory.resetTodayStatistics(); - } catch (Exception e) { - log.error("resetTodayStatistics failed!", e); - } - log.info("resetTodayStatistics finished"); - } @Scheduled(cron = "0 10 * * * ?") public void clearDeletedHosts() { diff --git a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/task/UserUploadFileCleanTask.java b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/task/UserUploadFileCleanTask.java index a4bad1b0e7..3ea77ef52c 100644 --- a/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/task/UserUploadFileCleanTask.java +++ b/src/backend/job-manage/service-job-manage/src/main/java/com/tencent/bk/job/manage/task/UserUploadFileCleanTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,26 +25,31 @@ package com.tencent.bk.job.manage.task; import com.google.common.collect.Sets; +import com.tencent.bk.job.common.artifactory.config.ArtifactoryConfig; import com.tencent.bk.job.common.artifactory.model.dto.NodeDTO; import com.tencent.bk.job.common.artifactory.model.dto.PageData; import com.tencent.bk.job.common.artifactory.sdk.ArtifactoryClient; import com.tencent.bk.job.common.constant.JobConstants; -import com.tencent.bk.job.common.redis.util.LockUtils; -import com.tencent.bk.job.common.util.FileUtil; +import com.tencent.bk.job.common.redis.util.HeartBeatRedisLock; +import com.tencent.bk.job.common.redis.util.HeartBeatRedisLockConfig; +import com.tencent.bk.job.common.redis.util.LockResult; import com.tencent.bk.job.common.util.StringUtil; import com.tencent.bk.job.common.util.date.DateUtils; -import com.tencent.bk.job.manage.config.ArtifactoryConfig; +import com.tencent.bk.job.common.util.file.FileUtil; +import com.tencent.bk.job.common.util.ip.IpUtils; import com.tencent.bk.job.manage.config.LocalFileConfigForManage; import com.tencent.bk.job.manage.config.StorageSystemConfig; import com.tencent.bk.job.manage.service.plan.TaskPlanService; import com.tencent.bk.job.manage.service.template.TaskTemplateService; +import lombok.AllArgsConstructor; +import lombok.Getter; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.io.FileUtils; import org.apache.commons.io.filefilter.AgeFileFilter; import org.apache.commons.io.filefilter.TrueFileFilter; -import org.slf4j.helpers.FormattingTuple; import org.slf4j.helpers.MessageFormatter; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.data.redis.core.RedisTemplate; import org.springframework.stereotype.Component; import java.io.File; @@ -53,7 +58,6 @@ import java.util.Iterator; import java.util.List; import java.util.Set; -import java.util.UUID; /** * @since 25/1/2021 21:19 @@ -62,6 +66,7 @@ @Component public class UserUploadFileCleanTask { + private static final String machineIp = IpUtils.getFirstMachineIP(); private static final String FILE_CLEAN_TASK_LOCK_KEY = "user:upload:file:clean"; private final String uploadPath; private final File uploadDirectory; @@ -70,6 +75,7 @@ public class UserUploadFileCleanTask { private final ArtifactoryConfig artifactoryConfig; private final LocalFileConfigForManage localFileConfigForManage; private final ArtifactoryClient artifactoryClient; + private final RedisTemplate redisTemplate; public UserUploadFileCleanTask( StorageSystemConfig storageSystemConfig, @@ -77,7 +83,8 @@ public UserUploadFileCleanTask( TaskPlanService taskPlanService, ArtifactoryConfig artifactoryConfig, LocalFileConfigForManage localFileConfigForManage, - ArtifactoryClient artifactoryClient + @Qualifier("jobArtifactoryClient") ArtifactoryClient artifactoryClient, + RedisTemplate redisTemplate ) { this.uploadPath = storageSystemConfig.getJobStorageRootPath() + "/localupload/"; this.taskTemplateService = taskTemplateService; @@ -86,22 +93,34 @@ public UserUploadFileCleanTask( this.localFileConfigForManage = localFileConfigForManage; this.artifactoryClient = artifactoryClient; this.uploadDirectory = new File(uploadPath); + this.redisTemplate = redisTemplate; } public void execute() { - String executeId = UUID.randomUUID().toString(); - + HeartBeatRedisLockConfig config = HeartBeatRedisLockConfig.getDefault(); + config.setHeartBeatThreadName("UserUploadFileCleanRedisKeyHeartBeatThread"); + HeartBeatRedisLock lock = new HeartBeatRedisLock( + redisTemplate, + FILE_CLEAN_TASK_LOCK_KEY, + machineIp, + config + ); + LockResult lockResult = lock.lock(); + if (!lockResult.isLockGotten()) { + log.info( + "lock {} gotten by another machine: {}, return", + FILE_CLEAN_TASK_LOCK_KEY, + lockResult.getLockValue() + ); + return; + } try { - if (LockUtils.tryGetDistributedLock(FILE_CLEAN_TASK_LOCK_KEY, executeId, 3600_000L)) { - Set skipFile = loadFileListFromDb(); - processFile(skipFile); - } else { - log.info("Some one else is running this task! Skip!"); - } + Set skipFile = loadFileListFromDb(); + processFile(skipFile); } catch (Exception e) { log.error("Error while running user upload file clean task!", e); } finally { - LockUtils.releaseDistributedLock(FILE_CLEAN_TASK_LOCK_KEY, executeId); + lockResult.tryToRelease(); } } @@ -119,94 +138,207 @@ private void processFile(Set skipFile) { } } - private NodeDTO getEndNode(NodeDTO nodeDTO) { - if (nodeDTO == null) return null; - PageData nodePage = artifactoryClient.listNode( - artifactoryConfig.getArtifactoryJobProject(), - localFileConfigForManage.getLocalUploadRepo(), - nodeDTO.getFullPath(), - 0, - 1 - ); - if (nodePage == null || nodePage.getRecords() == null || nodePage.getRecords().isEmpty()) { - return nodeDTO; + @Getter + @AllArgsConstructor + static class DeleteNodeResult { + // 删除的节点总数，含子节点 + int deletedNodeNum; + // 当前节点是否被删除 + boolean currentNodeDeleted; + + public DeleteNodeResult(int deletedNodeNum) { + this.deletedNodeNum = deletedNodeNum; + this.currentNodeDeleted = false; } - return getEndNode(nodePage.getRecords().get(0)); } /** - * 从制品库中删除nodeList内指定的过期节点 + * 检查并删除过期的节点及其子节点 * - * @param nodeList 需要检查的节点列表 + * @param node 目标节点 * @param skipPath 需要跳过的路径 - * @return 删除的节点数 + * @return 节点删除结果 */ - private int deleteExpiredNodes(List nodeList, Set skipPath) { - int deletedNodeNum = 0; - for (NodeDTO nodeDTO : nodeList) { - try { - NodeDTO endNode = getEndNode(nodeDTO); - String endNodePath = StringUtil.removePrefix(endNode.getFullPath(), "/"); - if (skipPath.contains(endNodePath)) { - log.info("Skip artifactory file {}", endNodePath); - continue; - } - // 从制品库删除有效日期前创建的节点 - LocalDateTime endNodeLastDate = DateUtils.convertFromStringDate( - endNode.getLastModifiedDate(), "yyyy-MM-dd'T'HH:mm:ss.SSS" - ); - if (endNodeLastDate - .plusDays(localFileConfigForManage.getExpireDays()) - .compareTo(LocalDateTime.now()) < 0) { - artifactoryClient.deleteNode( - artifactoryConfig.getArtifactoryJobProject(), - localFileConfigForManage.getLocalUploadRepo(), - nodeDTO.getFullPath() - ); - log.info("localFile {} in artifactory deleted", endNode.getFullPath()); - deletedNodeNum += 1; - } - } catch (Throwable t) { - FormattingTuple msg = MessageFormatter.format( - "Fail to check and process expire node {}", - nodeDTO.getFullPath() - ); - log.warn(msg.getMessage(), t); - } + private DeleteNodeResult checkAndDeleteExpiredNodeAndChild(NodeDTO node, Set skipPath) { + if (node == null) { + return new DeleteNodeResult(0); + } + String nodePath = StringUtil.removePrefix(node.getFullPath(), "/"); + if (skipPath.contains(nodePath)) { + log.info("Skip artifactory node {}", nodePath); + return new DeleteNodeResult(0); + } + boolean isFolder = node.getFolder() != null && node.getFolder(); + if (!isFolder) { + // 文件节点，直接检查并删除 + return checkAndDeleteFileNode(node); + } else { + // 目录节点，需要递归遍历检查并删除 + return checkAndDeleteDirNode(node, skipPath); } - return deletedNodeNum; } - /** - * 清理蓝鲸制品库中的临时文件 - * - * @param skipFile 需要跳过的文件 - */ - private void processArtifactoryFile(Set skipFile) { - int start = 0; + private DeleteNodeResult checkAndDeleteDirNode(NodeDTO node, Set skipPath) { + int deletedNum = 0; + // 1.先处理子节点 + PageData nodePage; + int pageNumber = 1; int pageSize = 100; - int deletedNodeNum; - List nodeList; do { - PageData nodePage = artifactoryClient.listNode( + nodePage = artifactoryClient.listNode( artifactoryConfig.getArtifactoryJobProject(), localFileConfigForManage.getLocalUploadRepo(), - "/", - start, + node.getFullPath(), + pageNumber, pageSize ); - if (nodePage == null) { - log.warn("nodePage is null"); - return; + if (isEmpty(nodePage)) { + break; } - nodeList = nodePage.getRecords(); - if (CollectionUtils.isNotEmpty(nodeList)) { - deletedNodeNum = deleteExpiredNodes(nodeList, skipFile); - start += pageSize - deletedNodeNum; + List subNodeList = nodePage.getRecords(); + int subNodeDeletedNum = 0; + for (NodeDTO subNode : subNodeList) { + // 单个节点处理失败后直接跳过，继续处理后续节点。 + try { + DeleteNodeResult result = checkAndDeleteExpiredNodeAndChild(subNode, skipPath); + deletedNum += result.getDeletedNodeNum(); + if (result.currentNodeDeleted) { + subNodeDeletedNum += 1; + } + } catch (Exception e) { + String msg = MessageFormatter.format( + "Fail to checkAndDelete node:{}", + subNode.getFullPath() + ).getMessage(); + log.warn(msg, e); + } + } + if (subNodeDeletedNum == 0) { + // 所有子节点本身都没有被删除，页码数才增加，否则当前页需要重新检查处理 + pageNumber += 1; + } + } while (!isEmpty(nodePage)); + // 2.根目录不删除 + if ("/".equals(node.getFullPath().trim())) { + return new DeleteNodeResult(deletedNum); + } + // 3.非根目录再判断是否为空目录，若为空且创建日期过期了也删除 + // 此处的目录过期需要用创建时间判断，因为子目录/文件删除会导致父目录的最后修改时间更新 + if (isNodeCreatedTimeExpired(node) && isDirNodeEmpty(node)) { + if (deleteNode(node)) { + deletedNum += 1; + log.info("Delete empty dirNode: {}", node.getFullPath()); + return new DeleteNodeResult(deletedNum, true); } else { - deletedNodeNum = 0; + log.warn("Fail to delete empty dirNode:{}", node.getFullPath()); } - } while (deletedNodeNum != 0 || (CollectionUtils.isNotEmpty(nodeList))); + } + return new DeleteNodeResult(deletedNum); + } + + private boolean isDirNodeEmpty(NodeDTO dirNode) { + PageData nodePage = artifactoryClient.listNode( + artifactoryConfig.getArtifactoryJobProject(), + localFileConfigForManage.getLocalUploadRepo(), + dirNode.getFullPath(), + 1, + 1 + ); + return nodePage != null && (nodePage.getRecords() == null || nodePage.getRecords().isEmpty()); + } + + private DeleteNodeResult checkAndDeleteFileNode(NodeDTO node) { + if (isNodeLastModifyTimeExpired(node)) { + if (deleteNode(node)) { + return new DeleteNodeResult(1, true); + } else { + return new DeleteNodeResult(0); + } + } + return new DeleteNodeResult(0); + } + + private boolean isEmpty(PageData nodePage) { + return nodePage == null || nodePage.getRecords() == null || nodePage.getRecords().isEmpty(); + } + + private boolean isNodeLastModifyTimeExpired(NodeDTO nodeDTO) { + LocalDateTime lastModifyDateTime = parseLastModifyDateTime(nodeDTO); + boolean result = isExpired(lastModifyDateTime); + log.debug( + "check node {}, lastModify={}, expired={}", + nodeDTO.getFullPath(), + nodeDTO.getLastModifiedDate(), + result + ); + return result; + } + + private boolean isNodeCreatedTimeExpired(NodeDTO nodeDTO) { + LocalDateTime createdDateTime = parseCreatedDateTime(nodeDTO); + boolean result = isExpired(createdDateTime); + log.debug( + "check node {}, createdDateTime={}, expired={}", + nodeDTO.getFullPath(), + nodeDTO.getCreatedDate(), + result + ); + return result; + } + + private LocalDateTime parseLastModifyDateTime(NodeDTO nodeDTO) { + return parseDateTimeFromStr(nodeDTO.getLastModifiedDate()); + } + + private LocalDateTime parseCreatedDateTime(NodeDTO nodeDTO) { + return parseDateTimeFromStr(nodeDTO.getCreatedDate()); + } + + private LocalDateTime parseDateTimeFromStr(String dateTimeStr) { + return DateUtils.convertFromStringDateByPatterns( + dateTimeStr, + "yyyy-MM-dd'T'HH:mm:ss.SSS", + "yyyy-MM-dd'T'HH:mm:ss.SS", + "yyyy-MM-dd'T'HH:mm:ss.S", + "yyyy-MM-dd'T'HH:mm:ss.", + "yyyy-MM-dd'T'HH:mm:ss" + ); + } + + private boolean isExpired(LocalDateTime dateTime) { + return dateTime.plusDays(localFileConfigForManage.getExpireDays()).isBefore(LocalDateTime.now()); + } + + private boolean deleteNode(NodeDTO nodeDTO) { + if (localFileConfigForManage.isExpireDelete()) { + boolean deleted = artifactoryClient.deleteNode( + artifactoryConfig.getArtifactoryJobProject(), + localFileConfigForManage.getLocalUploadRepo(), + nodeDTO.getFullPath() + ); + log.info("Delete localUpload node {} in artifactory, result={}", nodeDTO.getFullPath(), deleted); + return deleted; + } else { + log.info("Fake Delete localUpload node {} in artifactory, result={}", nodeDTO.getFullPath(), false); + return false; + } + } + + /** + * 清理蓝鲸制品库中的临时文件 + * + * @param skipFile 需要跳过的文件 + */ + private void processArtifactoryFile(Set skipFile) { + NodeDTO localUploadNode = artifactoryClient.queryNodeDetail( + artifactoryConfig.getArtifactoryJobProject(), + localFileConfigForManage.getLocalUploadRepo(), + "/" + ); + DeleteNodeResult result = checkAndDeleteExpiredNodeAndChild(localUploadNode, skipFile); + if (result.deletedNodeNum > 0) { + log.info("processArtifactoryFile finished, deletedNodeNum={}", result.deletedNodeNum); + } } /** @@ -226,17 +358,19 @@ private void processLocalFile(Set skipFile) { if (log.isDebugEnabled()) { log.debug("Skip file {}", aFile.getPath()); } - continue; } else { - log.info("Delete file {}", aFile.getPath()); - FileUtils.deleteQuietly(aFile); - } - - try { - FileUtil.deleteEmptyDirectory(aFile.getParentFile()); - FileUtil.deleteEmptyDirectory(aFile.getParentFile().getParentFile()); - } catch (Exception e) { - log.warn("Error while delete empty parent of file {}", aFile.getPath()); + if (localFileConfigForManage.isExpireDelete()) { + log.info("Delete file {}", aFile.getPath()); + FileUtils.deleteQuietly(aFile); + try { + FileUtil.deleteEmptyDirectory(aFile.getParentFile()); + FileUtil.deleteEmptyDirectory(aFile.getParentFile().getParentFile()); + } catch (Exception e) { + log.warn("Error while delete empty parent of file {}", aFile.getPath()); + } + } else { + log.info("Fake Delete file {}", aFile.getPath()); + } } } } diff --git a/src/backend/job-manage/service-job-manage/src/main/resources/i18n/message.properties b/src/backend/job-manage/service-job-manage/src/main/resources/i18n/message.properties new file mode 100644 index 0000000000..390ae9ee2b --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/resources/i18n/message.properties @@ -0,0 +1,99 @@ +# +# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. +# +# Copyright (C) 2021 Tencent. All rights reserved. +# +# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. +# +# License for BK-JOB蓝鲸智云作业平台: +# -------------------------------------------------------------------- +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF +# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +job.resource.status.online=已上线 +job.resource.status.draft=未上线 +job.resource.status.offline=已下线 +job.resource.status.disabled=禁用 +job.account.category.system=系统账号 +job.account.category.db=数据库账号 + + +#script check code +103701001=脚本首行没有定义合法的脚本类型，例如： #!/bin/bash +103701002=请用'cd ... || exit' 或者 'cd ... || return' 处理目录不存在的逻辑 +103702001=危险：出现递归逻辑fork炸弹, 禁止在函数中调用自己 +103702002=危险：删除高危目录，可能会影响到系统的稳定! +103702101=禁止不加条件扫描根目录! +103702102=警告：有条件扫描根目录，建议指定具体目录，不建议直接扫描根目录 +103702103=警告：建议加上扫描条件! +103702111=危险：不要重定向数据到块设备! +103702112=危险：不要对块设备进行格式化文件系统! +103702113=危险：不要对块设备进行格式化! +103702114=危险：不要用 dd 命令进行块设备操作! + + +#消息通知部分 +#资源类型 +job.manage.resource.type.script=快速执行脚本 +job.manage.resource.type.file=快速分发文件 +job.manage.resource.type.job=执行方案 +job.manage.resource.type.cron=定时作业 +#触发方式 +job.manage.trigger.type.page.execute=页面执行 +job.manage.trigger.type.api.invoke=API 调用 +job.manage.trigger.type.timer.task=定时任务 +#执行状态 +job.manage.execute.status.success=执行成功 +job.manage.execute.status.fail=执行失败 +job.manage.execute.status.ready=等待执行 +#角色 +job.manage.role.resource.owner=资源所属者 +job.manage.role.resource.trigger.user=任务执行人 +job.manage.role.extra.observer=额外通知人 +#通知渠道 +job.manage.notify.channel.sms=短信 +job.manage.notify.channel.email=邮件 +job.manage.notify.channel.wechat=微信 +#通知模板名称 +job.manage.notify.template.name.confirmation=人工确认 +job.manage.notify.template.name.executeSuccess=执行成功 +job.manage.notify.template.name.executeFailure=执行失败 +job.manage.notify.template.name.beforeCronJobExecute=定时执行前 +job.manage.notify.template.name.beforeCronJobEnd=定时结束前 +job.manage.notify.template.name.cronJobFailed=定时启动失败 + +# IP白名单部分 +job.manage.whiteip.actionScope.scriptExecute=脚本执行 +job.manage.whiteip.actionScope.fileDistribution=文件分发 + +# 全局设置部分 +job.manage.globalsettings.defaultNameRules.description.linux=使用小写字母或 _ 开头，由 2-32 个英文字母、数字、_ 或 - 组成的字符 +job.manage.globalsettings.defaultNameRules.description.windows=由1~32个大小写字母、数字或-组成的字符 +job.manage.globalsettings.defaultNameRules.description.database=由1~16个大小写字母 / 数字 / . / - / _ 组成的字符 +job.manage.globalsettings.defaultTitleHead=作业平台 | 腾讯蓝鲸智云 +job.manage.globalsettings.contactBKHelper=联系BK助手 +job.manage.globalsettings.BKDesktop=蓝鲸桌面 +job.manage.globalsettings.ee.footerLink=[技术支持](https://wpa1.qq.com/KziXGWJs?_type=wpa&qidian=true) | [社区论坛](https://bk.tencent.com/s-mart/community) | [产品官网](https://bk.tencent.com/index/) +job.manage.globalsettings.ee.footerCopyRight=Copyright © 2012-{{current_year}} Tencent BlueKing. All Rights Reserved. {{current_ver}} + +job.custom.settings.script.template.variable.datetime.desc=当前日期时间 +job.custom.settings.script.template.variable.username.desc=用户名 +job.custom.settings.script.template.variable.display_name.desc=全名 +job.custom.settings.script.template.variable.biz_id.desc=业务(集)ID +job.custom.settings.script.template.variable.biz_name.desc=业务(集)名称 + +# CMDB相关资源 +cmdb.object.name.biz=业务 +cmdb.object.name.biz_set=业务集 diff --git a/src/backend/job-manage/service-job-manage/src/main/resources/i18n/message_en.properties b/src/backend/job-manage/service-job-manage/src/main/resources/i18n/message_en.properties new file mode 100644 index 0000000000..b1a9c3293c --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/resources/i18n/message_en.properties @@ -0,0 +1,98 @@ +# +# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. +# +# Copyright (C) 2021 Tencent. All rights reserved. +# +# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. +# +# License for BK-JOB蓝鲸智云作业平台: +# -------------------------------------------------------------------- +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF +# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +job.resource.status.online=Online +job.resource.status.draft=Stand-by +job.resource.status.offline=Offline +job.resource.status.disabled=Banned +job.account.category.system=System Account +job.account.category.db=Database Account + + +#script check code +103701001=The first line of the script does not define a legal script type(e.g.: #!/bin/bash). +103701002=Plz use 'cd ... || exit' or 'cd ... || return' to determine if directory does not exist. +103702001=Danger: Recursive logic fork bomb appears, prohibit calling yourself in the function. +103702002=Danger: Deleting high-risk directories may affect the stability of the system! +103702101=Prohibit unconditional scanning of the root directory! +103702102=Warning: It is not recommended to scan the root directory directly. +103702103=Warning: It is recommended to add some conditions! +103702111=Danger: Do not redirect data to block devices! +103702112=Danger: Do not format the file system for block devices! +103702113=Danger: Do not format block devices! +103702114=Danger: Do not use command 'dd' to operate block devices! + +#消息通知部分 +#资源类型 +job.manage.resource.type.script=Script execution +job.manage.resource.type.file=File transfer +job.manage.resource.type.job=Job plan +job.manage.resource.type.cron=Cron job +#触发方式 +job.manage.trigger.type.page.execute=Web UI +job.manage.trigger.type.api.invoke=API +job.manage.trigger.type.timer.task=Cron +#执行状态 +job.manage.execute.status.success=Success +job.manage.execute.status.fail=Failed +job.manage.execute.status.ready=Waitting +#角色 +job.manage.role.resource.owner=Owner +job.manage.role.resource.trigger.user=Operater +job.manage.role.extra.observer=Additional +#通知渠道 +job.manage.notify.channel.sms=SMS +job.manage.notify.channel.email=Mail +job.manage.notify.channel.wechat=WeChat +#通知模板名称 +job.manage.notify.template.name.confirmation=Confirm +job.manage.notify.template.name.executeSuccess=Success +job.manage.notify.template.name.executeFailure=Fail +job.manage.notify.template.name.beforeCronJobExecute=Before Cron Start +job.manage.notify.template.name.beforeCronJobEnd=Before Cron End +job.manage.notify.template.name.cronJobFailed=Cron Unlaunched + +# IP白名单部分 +job.manage.whiteip.actionScope.scriptExecute=Script Execution +job.manage.whiteip.actionScope.fileDistribution=File Transfer + +# 全局设置部分 +job.manage.globalsettings.defaultNameRules.description.linux=Characters starting with a lowercase letter or _ , composed of 2-32 lowercase / numbers / _ or - +job.manage.globalsettings.defaultNameRules.description.windows=Composed of 1 to 32 uppercase or lowercase or numbers or - +job.manage.globalsettings.defaultNameRules.description.database=Composed of 1 to 16 uppercase / lowercase / numbers / . / - / _ +job.manage.globalsettings.defaultTitleHead=Job | Tencent BlueKing +job.manage.globalsettings.contactBKHelper=Contact BK-Helper +job.manage.globalsettings.BKDesktop=BlueKing Desktop +job.manage.globalsettings.ee.footerLink=[Support](https://wpa1.qq.com/KziXGWJs?_type=wpa&qidian=true) | [Forum](https://bk.tencent.com/s-mart/community) | [Official](https://bk.tencent.com/index/) +job.manage.globalsettings.ee.footerCopyRight=Copyright © 2012-{{current_year}} Tencent BlueKing. All Rights Reserved. {{current_ver}} + +job.custom.settings.script.template.variable.datetime.desc=Current date and time +job.custom.settings.script.template.variable.username.desc=Username +job.custom.settings.script.template.variable.display_name.desc=User full name +job.custom.settings.script.template.variable.biz_id.desc=Business(set) ID +job.custom.settings.script.template.variable.biz_name.desc=Business(set) Name + +# CMDB相关资源 +cmdb.object.name.biz=Business +cmdb.object.name.biz_set=Business Set diff --git a/src/backend/job-manage/service-job-manage/src/main/resources/i18n/message_en_US.properties b/src/backend/job-manage/service-job-manage/src/main/resources/i18n/message_en_US.properties new file mode 100644 index 0000000000..641e6cb866 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/resources/i18n/message_en_US.properties @@ -0,0 +1,98 @@ +# +# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. +# +# Copyright (C) 2021 Tencent. All rights reserved. +# +# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. +# +# License for BK-JOB蓝鲸智云作业平台: +# -------------------------------------------------------------------- +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF +# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +job.resource.status.online=Online +job.resource.status.draft=Stand-by +job.resource.status.offline=Offline +job.resource.status.disabled=Banned +job.account.category.system=System Account +job.account.category.db=Database Account + + +#script check code +103701001=The first line of the script does not define a legal script type(e.g.: #!/bin/bash). +103701002=Plz use 'cd ... || exit' or 'cd ... || return' to determine if directory does not exist. +103702001=Danger: Recursive logic fork bomb appears, prohibit calling yourself in the function. +103702002=Danger: Deleting high-risk directories may affect the stability of the system! +103702101=Prohibit unconditional scanning of the root directory! +103702102=Warning: It is not recommended to scan the root directory directly. +103702103=Warning: It is recommended to add some conditions! +103702111=Danger: Do not redirect data to block devices! +103702112=Danger: Do not format the file system for block devices! +103702113=Danger: Do not format block devices! +103702114=Danger: Do not use 'dd' for block devices! + +#消息通知部分 +#资源类型 +job.manage.resource.type.script=Script execution +job.manage.resource.type.file=File transfer +job.manage.resource.type.job=Job plan +job.manage.resource.type.cron=Cron job +#触发方式 +job.manage.trigger.type.page.execute=Web UI +job.manage.trigger.type.api.invoke=API +job.manage.trigger.type.timer.task=Cron +#执行状态 +job.manage.execute.status.success=Success +job.manage.execute.status.fail=Failed +job.manage.execute.status.ready=Waitting +#角色 +job.manage.role.resource.owner=Owner +job.manage.role.resource.trigger.user=Operater +job.manage.role.extra.observer=Additional +#通知渠道 +job.manage.notify.channel.sms=SMS +job.manage.notify.channel.email=Mail +job.manage.notify.channel.wechat=WeChat +#通知模板名称 +job.manage.notify.template.name.confirmation=Confirm +job.manage.notify.template.name.executeSuccess=Success +job.manage.notify.template.name.executeFailure=Fail +job.manage.notify.template.name.beforeCronJobExecute=Before Cron Start +job.manage.notify.template.name.beforeCronJobEnd=Before Cron End +job.manage.notify.template.name.cronJobFailed=Cron Unlaunched + +# IP白名单部分 +job.manage.whiteip.actionScope.scriptExecute=Script Execution +job.manage.whiteip.actionScope.fileDistribution=File Transfer + +# 全局设置部分 +job.manage.globalsettings.defaultNameRules.description.linux=Characters starting with a lowercase letter or _ , composed of 2-32 lowercase / numbers / _ or - +job.manage.globalsettings.defaultNameRules.description.windows=Composed of 1 to 32 uppercase and lowercase or numbers or - +job.manage.globalsettings.defaultNameRules.description.database=Composed of 1 to 16 uppercase / lowercase / numbers / . / - / _ +job.manage.globalsettings.defaultTitleHead=Job | Tencent BlueKing +job.manage.globalsettings.contactBKHelper=Contact BK-Helper +job.manage.globalsettings.BKDesktop=BlueKing Desktop +job.manage.globalsettings.ee.footerLink=[Support](https://wpa1.qq.com/KziXGWJs?_type=wpa&qidian=true) | [Forum](https://bk.tencent.com/s-mart/community) | [Official](https://bk.tencent.com/index/) +job.manage.globalsettings.ee.footerCopyRight=Copyright © 2012-{{current_year}} Tencent BlueKing. All Rights Reserved. {{current_ver}} + +job.custom.settings.script.template.variable.datetime.desc=Current date and time +job.custom.settings.script.template.variable.username.desc=Username +job.custom.settings.script.template.variable.display_name.desc=User full name +job.custom.settings.script.template.variable.biz_id.desc=Business(set) ID +job.custom.settings.script.template.variable.biz_name.desc=Business(set) Name + +# CMDB相关资源 +cmdb.object.name.biz=Business +cmdb.object.name.biz_set=Business Set diff --git a/src/backend/job-manage/service-job-manage/src/main/resources/i18n/message_zh.properties b/src/backend/job-manage/service-job-manage/src/main/resources/i18n/message_zh.properties new file mode 100644 index 0000000000..8fea72079a --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/resources/i18n/message_zh.properties @@ -0,0 +1,98 @@ +# +# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. +# +# Copyright (C) 2021 Tencent. All rights reserved. +# +# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. +# +# License for BK-JOB蓝鲸智云作业平台: +# -------------------------------------------------------------------- +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF +# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +job.resource.status.online=已上线 +job.resource.status.draft=未上线 +job.resource.status.offline=已下线 +job.resource.status.disabled=禁用 +job.account.category.system=系统账号 +job.account.category.db=数据库账号 + + +#script check code +103701001=脚本首行没有定义合法的脚本类型，例如： #!/bin/bash +103701002=请用'cd ... || exit' 或者 'cd ... || return' 处理目录不存在的逻辑 +103702001=危险：出现递归逻辑fork炸弹, 禁止在函数中调用自己 +103702002=危险：删除高危目录，可能会影响到系统的稳定! +103702101=禁止不加条件扫描根目录! +103702102=警告：有条件扫描根目录，建议指定具体目录，不建议直接扫描根目录 +103702103=警告：建议加上扫描条件! +103702111=危险：不要重定向数据到块设备! +103702112=危险：不要对块设备进行格式化文件系统! +103702113=危险：不要对块设备进行格式化! +103702114=危险：不要用 dd 命令进行块设备操作! + +#消息通知部分 +#资源类型 +job.manage.resource.type.script=快速执行脚本 +job.manage.resource.type.file=快速分发文件 +job.manage.resource.type.job=执行方案 +job.manage.resource.type.cron=定时作业 +#触发方式 +job.manage.trigger.type.page.execute=页面执行 +job.manage.trigger.type.api.invoke=API 调用 +job.manage.trigger.type.timer.task=定时任务 +#执行状态 +job.manage.execute.status.success=执行成功 +job.manage.execute.status.fail=执行失败 +job.manage.execute.status.ready=等待执行 +#角色 +job.manage.role.resource.owner=资源所属者 +job.manage.role.resource.trigger.user=任务执行人 +job.manage.role.extra.observer=额外通知人 +#通知渠道 +job.manage.notify.channel.sms=短信 +job.manage.notify.channel.email=邮件 +job.manage.notify.channel.wechat=微信 +#通知模板名称 +job.manage.notify.template.name.confirmation=人工确认 +job.manage.notify.template.name.executeSuccess=执行成功 +job.manage.notify.template.name.executeFailure=执行失败 +job.manage.notify.template.name.beforeCronJobExecute=定时执行前 +job.manage.notify.template.name.beforeCronJobEnd=定时结束前 +job.manage.notify.template.name.cronJobFailed=定时启动失败 + +# IP白名单部分 +job.manage.whiteip.actionScope.scriptExecute=脚本执行 +job.manage.whiteip.actionScope.fileDistribution=文件分发 + +# 全局设置部分 +job.manage.globalsettings.defaultNameRules.description.linux=使用小写字母或 _ 开头，由 2-32 个英文字母、数字、_ 或 - 组成的字符 +job.manage.globalsettings.defaultNameRules.description.windows=由1~32个大小写字母、数字或-组成的字符 +job.manage.globalsettings.defaultNameRules.description.database=由1~16个大小写字母 / 数字 / . / - / _ 组成的字符 +job.manage.globalsettings.defaultTitleHead=作业平台 | 腾讯蓝鲸智云 +job.manage.globalsettings.contactBKHelper=联系BK助手 +job.manage.globalsettings.BKDesktop=蓝鲸桌面 +job.manage.globalsettings.ee.footerLink=[技术支持](https://wpa1.qq.com/KziXGWJs?_type=wpa&qidian=true) | [社区论坛](https://bk.tencent.com/s-mart/community) | [产品官网](https://bk.tencent.com/index/) +job.manage.globalsettings.ee.footerCopyRight=Copyright © 2012-{{current_year}} Tencent BlueKing. All Rights Reserved. {{current_ver}} + +job.custom.settings.script.template.variable.datetime.desc=当前日期时间 +job.custom.settings.script.template.variable.username.desc=用户名 +job.custom.settings.script.template.variable.display_name.desc=全名 +job.custom.settings.script.template.variable.biz_id.desc=业务(集)ID +job.custom.settings.script.template.variable.biz_name.desc=业务(集)名称 + +# CMDB相关资源 +cmdb.object.name.biz=业务 +cmdb.object.name.biz_set=业务集 diff --git a/src/backend/job-manage/service-job-manage/src/main/resources/i18n/message_zh_CN.properties b/src/backend/job-manage/service-job-manage/src/main/resources/i18n/message_zh_CN.properties new file mode 100644 index 0000000000..8fea72079a --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/resources/i18n/message_zh_CN.properties @@ -0,0 +1,98 @@ +# +# Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. +# +# Copyright (C) 2021 Tencent. All rights reserved. +# +# BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. +# +# License for BK-JOB蓝鲸智云作业平台: +# -------------------------------------------------------------------- +# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated +# documentation files (the "Software"), to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and +# to permit persons to whom the Software is furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all copies or substantial portions of +# the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO +# THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF +# CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# + +job.resource.status.online=已上线 +job.resource.status.draft=未上线 +job.resource.status.offline=已下线 +job.resource.status.disabled=禁用 +job.account.category.system=系统账号 +job.account.category.db=数据库账号 + + +#script check code +103701001=脚本首行没有定义合法的脚本类型，例如： #!/bin/bash +103701002=请用'cd ... || exit' 或者 'cd ... || return' 处理目录不存在的逻辑 +103702001=危险：出现递归逻辑fork炸弹, 禁止在函数中调用自己 +103702002=危险：删除高危目录，可能会影响到系统的稳定! +103702101=禁止不加条件扫描根目录! +103702102=警告：有条件扫描根目录，建议指定具体目录，不建议直接扫描根目录 +103702103=警告：建议加上扫描条件! +103702111=危险：不要重定向数据到块设备! +103702112=危险：不要对块设备进行格式化文件系统! +103702113=危险：不要对块设备进行格式化! +103702114=危险：不要用 dd 命令进行块设备操作! + +#消息通知部分 +#资源类型 +job.manage.resource.type.script=快速执行脚本 +job.manage.resource.type.file=快速分发文件 +job.manage.resource.type.job=执行方案 +job.manage.resource.type.cron=定时作业 +#触发方式 +job.manage.trigger.type.page.execute=页面执行 +job.manage.trigger.type.api.invoke=API 调用 +job.manage.trigger.type.timer.task=定时任务 +#执行状态 +job.manage.execute.status.success=执行成功 +job.manage.execute.status.fail=执行失败 +job.manage.execute.status.ready=等待执行 +#角色 +job.manage.role.resource.owner=资源所属者 +job.manage.role.resource.trigger.user=任务执行人 +job.manage.role.extra.observer=额外通知人 +#通知渠道 +job.manage.notify.channel.sms=短信 +job.manage.notify.channel.email=邮件 +job.manage.notify.channel.wechat=微信 +#通知模板名称 +job.manage.notify.template.name.confirmation=人工确认 +job.manage.notify.template.name.executeSuccess=执行成功 +job.manage.notify.template.name.executeFailure=执行失败 +job.manage.notify.template.name.beforeCronJobExecute=定时执行前 +job.manage.notify.template.name.beforeCronJobEnd=定时结束前 +job.manage.notify.template.name.cronJobFailed=定时启动失败 + +# IP白名单部分 +job.manage.whiteip.actionScope.scriptExecute=脚本执行 +job.manage.whiteip.actionScope.fileDistribution=文件分发 + +# 全局设置部分 +job.manage.globalsettings.defaultNameRules.description.linux=使用小写字母或 _ 开头，由 2-32 个英文字母、数字、_ 或 - 组成的字符 +job.manage.globalsettings.defaultNameRules.description.windows=由1~32个大小写字母、数字或-组成的字符 +job.manage.globalsettings.defaultNameRules.description.database=由1~16个大小写字母 / 数字 / . / - / _ 组成的字符 +job.manage.globalsettings.defaultTitleHead=作业平台 | 腾讯蓝鲸智云 +job.manage.globalsettings.contactBKHelper=联系BK助手 +job.manage.globalsettings.BKDesktop=蓝鲸桌面 +job.manage.globalsettings.ee.footerLink=[技术支持](https://wpa1.qq.com/KziXGWJs?_type=wpa&qidian=true) | [社区论坛](https://bk.tencent.com/s-mart/community) | [产品官网](https://bk.tencent.com/index/) +job.manage.globalsettings.ee.footerCopyRight=Copyright © 2012-{{current_year}} Tencent BlueKing. All Rights Reserved. {{current_ver}} + +job.custom.settings.script.template.variable.datetime.desc=当前日期时间 +job.custom.settings.script.template.variable.username.desc=用户名 +job.custom.settings.script.template.variable.display_name.desc=全名 +job.custom.settings.script.template.variable.biz_id.desc=业务(集)ID +job.custom.settings.script.template.variable.biz_name.desc=业务(集)名称 + +# CMDB相关资源 +cmdb.object.name.biz=业务 +cmdb.object.name.biz_set=业务集 diff --git a/src/backend/job-manage/service-job-manage/src/main/resources/script_template/bat b/src/backend/job-manage/service-job-manage/src/main/resources/script_template/bat new file mode 100644 index 0000000000..d6d0b8cbd9 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/resources/script_template/bat @@ -0,0 +1,46 @@ +@echo on +setlocal enabledelayedexpansion +call:job_start + +REM 作业平台中执行脚本成功和失败的标准只取决于脚本最后一条执行语句的返回值 +REM 如果返回值为0，则认为此脚本执行成功，如果非0，则认为脚本执行失败 +REM 可在此处开始编写您的脚本逻辑代码 + + + +REM 函数自定义区域，不要把正文写到函数区下面 +goto:eof +REM 可在脚本开始运行时调用，打印当时的时间戳及PID。 +:job_start + set cu_time=[%date:~0,10% %time:~0,8%] + for /F "skip=3 tokens=2" %%i in ('tasklist /v /FI "IMAGENAME eq cmd.exe" /FI "STATUS eq Unknown"') do ( + set pid=[PID:%%i] + goto:break + ) + :break + echo %cu_time%%pid% job_start + goto:eof + +REM 可在脚本执行成功的逻辑分支处调用，打印当时的时间戳及PID。 +:job_success + set cu_time=[%date:~0,10% %time:~0,8%] + for /F "skip=3 tokens=2" %%i in ('tasklist /v /FI "IMAGENAME eq cmd.exe" /FI "STATUS eq Unknown"') do ( + set pid=[PID:%%i] + goto:break + ) + :break + echo %cu_time%%pid% job_success:[%*] + exit 0 + +REM 可在脚本执行失败的逻辑分支处调用，打印当时的时间戳及PID。 +:job_fail + set cu_time=[%date:~0,10% %time:~0,8%] + for /F "skip=3 tokens=2" %%i in ('tasklist /v /FI "IMAGENAME eq cmd.exe" /FI "STATUS eq Unknown"') do ( + set pid=[PID:%%i] + goto:break + ) + :break + echo %cu_time%%pid% job_fail:[%*] + exit 1 + + diff --git a/src/backend/job-manage/service-job-manage/src/main/resources/script_template/perl b/src/backend/job-manage/service-job-manage/src/main/resources/script_template/perl new file mode 100644 index 0000000000..53a7313d40 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/resources/script_template/perl @@ -0,0 +1,33 @@ +#!/usr/bin/perl + +use strict; + +sub job_localtime { + my @n = localtime(); + return sprintf("%04d-%02d-%02d %02d:%02d:%02d",$n[5]+1900,$n[4]+1,$n[3], $n[2], $n[1], $n[0] ); +} + +##### 可在脚本开始运行时调用，打印当时的时间戳及PID。 +sub job_start { + print "[",&job_localtime,"][PID:$$] job_start\n"; +} + +##### 可在脚本执行成功的逻辑分支处调用，打印当时的时间戳及PID。 +sub job_success { + print "[",&job_localtime,"][PID:$$] job_success:[@_]\n"; + exit 0; +} + +##### 可在脚本执行失败的逻辑分支处调用，打印当时的时间戳及PID。 +sub job_fail { + print "[",&job_localtime,"][PID:$$] job_fail:[@_]\n"; + exit 1; +} + +job_start; + +###### iJobs中执行脚本成功和失败的标准只取决于脚本最后一条执行语句的返回值 +###### 如果返回值为0，则认为此脚本执行成功，如果非0，则认为脚本执行失败 +###### 可在此处开始编写您的脚本逻辑代码 + + diff --git a/src/backend/job-manage/service-job-manage/src/main/resources/script_template/powershell b/src/backend/job-manage/service-job-manage/src/main/resources/script_template/powershell new file mode 100644 index 0000000000..bf910f4227 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/resources/script_template/powershell @@ -0,0 +1,46 @@ +##### 可在脚本开始运行时调用，打印当时的时间戳及PID。 +function job_start +{ + $cu_date = Get-Date -Format "yyyy-MM-dd HH:mm:ss" + "[{0}][PID:{1}] job_start" -f $cu_date,$pid +} + +##### 可在脚本执行成功的逻辑分支处调用，打印当时的时间戳及PID。 +function job_success +{ + $cu_date = Get-Date -Format "yyyy-MM-dd HH:mm:ss" + if($args.count -ne 0) + { + $args | foreach {$arg_str=$arg_str + " " + $_} + "[{0}][PID:{1}] job_success:[{2}]" -f $cu_date,$pid,$arg_str.TrimStart(' ') + } + else + { + "[{0}][PID:{1}] job_success:[]" -f $cu_date,$pid + } + exit 0 +} + +##### 可在脚本执行失败的逻辑分支处调用，打印当时的时间戳及PID。 +function job_fail +{ + $cu_date = Get-Date -Format "yyyy-MM-dd HH:mm:ss" + if($args.count -ne 0) + { + $args | foreach {$arg_str=$arg_str + " " + $_} + "[{0}][PID:{1}] job_fail:[{2}]" -f $cu_date,$pid,$arg_str.TrimStart(' ') + } + else + { + "[{0}][PID:{1}] job_fail:[]" -f $cu_date,$pid + } + exit 1 +} + +job_start + +###### 作业平台中执行脚本成功和失败的标准只取决于脚本最后一条执行语句的返回值 +###### 如果返回值为0，则认为此脚本执行成功，如果非0，则认为脚本执行失败 +###### 可在此处开始编写您的脚本逻辑代码 + + diff --git a/src/backend/job-manage/service-job-manage/src/main/resources/script_template/python b/src/backend/job-manage/service-job-manage/src/main/resources/script_template/python new file mode 100644 index 0000000000..d23ee79d17 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/resources/script_template/python @@ -0,0 +1,38 @@ +#!/usr/bin/env python +# -*- coding: utf8 -*- + +import datetime +import os +import sys + +def _now(format="%Y-%m-%d %H:%M:%S"): + return datetime.datetime.now().strftime(format) + +##### 可在脚本开始运行时调用，打印当时的时间戳及PID。 +def job_start(): + print("[%s][PID:%s] job_start" % (_now(), os.getpid())) + sys.stdout.flush() + +##### 可在脚本执行成功的逻辑分支处调用，打印当时的时间戳及PID。 +def job_success(msg): + print("[%s][PID:%s] job_success:[%s]" % (_now(), os.getpid(), msg)) + sys.stdout.flush() + sys.exit(0) + +##### 可在脚本执行失败的逻辑分支处调用，打印当时的时间戳及PID。 +def job_fail(msg): + print("[%s][PID:%s] job_fail:[%s]" % (_now(), os.getpid(), msg)) + sys.stdout.flush() + sys.exit(1) + +if __name__ == '__main__': + + job_start() + +###### 脚本执行成功和失败的标准只取决于最后一段执行语句的返回值 +###### 如果返回值为0，则认为此脚本执行成功，如果非0，则认为脚本执行失败 +###### Python脚本为了避免因长时间未刷新缓冲区导致标准输出异常， +###### 建议在print的下一行使用 sys.stdout.flush() 来强制将被缓存的输出信息刷新到控制台上 +###### 可在此处开始编写您的脚本逻辑代码 + + diff --git a/src/backend/job-manage/service-job-manage/src/main/resources/script_template/shell b/src/backend/job-manage/service-job-manage/src/main/resources/script_template/shell new file mode 100644 index 0000000000..79132696c0 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/main/resources/script_template/shell @@ -0,0 +1,39 @@ +#!/bin/bash + + +# 定义获取当前时间和PID的函数 +function job_get_now +{ + echo "[`date +'%Y-%m-%d %H:%M:%S'`][PID:$$]" +} + +# 在脚本开始运行时调用，打印当前的时间戳及PID +function job_start +{ + echo "$(job_get_now) job_start" +} + +# 在脚本执行成功的逻辑分支处调用，打印当前的时间戳及PID +function job_success +{ + local msg="$*" + echo "$(job_get_now) job_success:[$msg]" + exit 0 +} + +# 在脚本执行失败的逻辑分支处调用，打印当前的时间戳及PID +function job_fail +{ + local msg="$*" + echo "$(job_get_now) job_fail:[$msg]" + exit 1 +} + +# 在当前脚本执行时，第一行输出当前时间和进程ID，详见上面函数：job_get_now +job_start + +###### 作业平台中执行脚本成功和失败的标准只取决于脚本最后一条执行语句的返回值 +###### 如果返回值为0，则认为此脚本执行成功，如果非0，则认为脚本执行失败 +###### 可在此处开始编写您的脚本逻辑代码 + + diff --git a/docs/apidoc/readme.md b/src/backend/job-manage/service-job-manage/src/main/resources/script_template/sql similarity index 100% rename from docs/apidoc/readme.md rename to src/backend/job-manage/service-job-manage/src/main/resources/script_template/sql diff --git a/src/backend/job-manage/service-job-manage/src/test/java/com/tencent/bk/job/manage/common/util/FieldValidateUtilTest.java b/src/backend/job-manage/service-job-manage/src/test/java/com/tencent/bk/job/manage/common/util/FieldValidateUtilTest.java index a1d06d5ffa..f07943bfd1 100644 --- a/src/backend/job-manage/service-job-manage/src/test/java/com/tencent/bk/job/manage/common/util/FieldValidateUtilTest.java +++ b/src/backend/job-manage/service-job-manage/src/test/java/com/tencent/bk/job/manage/common/util/FieldValidateUtilTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-manage/service-job-manage/src/test/java/com/tencent/bk/job/manage/common/util/TemplateScriptStatusFlagsUtilTest.java b/src/backend/job-manage/service-job-manage/src/test/java/com/tencent/bk/job/manage/common/util/TemplateScriptStatusFlagsUtilTest.java new file mode 100644 index 0000000000..3639a1011d --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/test/java/com/tencent/bk/job/manage/common/util/TemplateScriptStatusFlagsUtilTest.java @@ -0,0 +1,65 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.common.util; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +class TemplateScriptStatusFlagsUtilTest { + + @Test + void readOfflineFlag() { + int scriptStatusFlags = 0; + assertThat(TemplateScriptStatusFlagsUtil.readOfflineFlag(scriptStatusFlags)).isFalse(); + scriptStatusFlags = 1; + assertThat(TemplateScriptStatusFlagsUtil.readOfflineFlag(scriptStatusFlags)).isTrue(); + scriptStatusFlags = 2; + assertThat(TemplateScriptStatusFlagsUtil.readOfflineFlag(scriptStatusFlags)).isFalse(); + scriptStatusFlags = 3; + assertThat(TemplateScriptStatusFlagsUtil.readOfflineFlag(scriptStatusFlags)).isTrue(); + + } + + @Test + void readDisableFlag() { + int scriptStatusFlags = 0; + assertThat(TemplateScriptStatusFlagsUtil.readDisableFlag(scriptStatusFlags)).isFalse(); + scriptStatusFlags = 1; + assertThat(TemplateScriptStatusFlagsUtil.readDisableFlag(scriptStatusFlags)).isFalse(); + scriptStatusFlags = 2; + assertThat(TemplateScriptStatusFlagsUtil.readDisableFlag(scriptStatusFlags)).isTrue(); + scriptStatusFlags = 3; + assertThat(TemplateScriptStatusFlagsUtil.readDisableFlag(scriptStatusFlags)).isTrue(); + } + + @Test + void computeScriptStatusFlags() { + assertThat(TemplateScriptStatusFlagsUtil.computeScriptStatusFlags(false, false)).isEqualTo(0); + assertThat(TemplateScriptStatusFlagsUtil.computeScriptStatusFlags(true, false)).isEqualTo(1); + assertThat(TemplateScriptStatusFlagsUtil.computeScriptStatusFlags(false, true)).isEqualTo(2); + assertThat(TemplateScriptStatusFlagsUtil.computeScriptStatusFlags(true, true)).isEqualTo(3); + } +} diff --git a/src/backend/job-manage/service-job-manage/src/test/java/com/tencent/bk/job/manage/common/util/TopologyHelperInfoTest.java b/src/backend/job-manage/service-job-manage/src/test/java/com/tencent/bk/job/manage/common/util/TopologyHelperInfoTest.java new file mode 100644 index 0000000000..1a563f7640 --- /dev/null +++ b/src/backend/job-manage/service-job-manage/src/test/java/com/tencent/bk/job/manage/common/util/TopologyHelperInfoTest.java @@ -0,0 +1,104 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.manage.common.util; + +import com.tencent.bk.job.common.cc.model.InstanceTopologyDTO; +import com.tencent.bk.job.manage.common.TopologyHelper; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.List; + +import static org.assertj.core.api.Assertions.assertThat; + +public class TopologyHelperInfoTest { + + @Test + public void testFindTopologyPath() { + InstanceTopologyDTO treeNode = new InstanceTopologyDTO(); + treeNode.setObjectName("业务"); + treeNode.setObjectId("biz"); + treeNode.setInstanceId(2L); + treeNode.setInstanceName("蓝鲸"); + InstanceTopologyDTO setNode = new InstanceTopologyDTO(); + setNode.setObjectName("集合"); + setNode.setObjectId("set"); + setNode.setInstanceId(3L); + setNode.setInstanceName("测试集合"); + InstanceTopologyDTO moduleNode1 = new InstanceTopologyDTO(); + moduleNode1.setObjectName("模块"); + moduleNode1.setObjectId("module"); + moduleNode1.setInstanceId(4L); + moduleNode1.setInstanceName("测试模块1"); + InstanceTopologyDTO moduleNode2 = new InstanceTopologyDTO(); + moduleNode2.setObjectName("模块"); + moduleNode2.setObjectId("module"); + moduleNode2.setInstanceId(5L); + moduleNode2.setInstanceName("测试模块2"); + InstanceTopologyDTO moduleNode3 = new InstanceTopologyDTO(); + moduleNode3.setObjectName("模块"); + moduleNode3.setObjectId("module"); + moduleNode3.setInstanceId(6L); + moduleNode3.setInstanceName("测试模块3"); + List moduleList = new ArrayList<>(); + moduleList.add(moduleNode1); + moduleList.add(moduleNode2); + moduleList.add(moduleNode3); + setNode.setChild(moduleList); + List setList = new ArrayList<>(); + setList.add(setNode); + treeNode.setChild(setList); + InstanceTopologyDTO targetNode1 = new InstanceTopologyDTO(); + targetNode1.setObjectId("module"); + targetNode1.setInstanceId(5L); + InstanceTopologyDTO targetNode2 = new InstanceTopologyDTO(); + targetNode2.setObjectId("set"); + targetNode2.setInstanceId(3L); + InstanceTopologyDTO targetNode3 = new InstanceTopologyDTO(); + targetNode3.setObjectId("module"); + targetNode3.setInstanceId(6L); + InstanceTopologyDTO notExistsNode = new InstanceTopologyDTO(); + notExistsNode.setObjectId("module"); + notExistsNode.setInstanceId(15L); + assertThat(TopologyHelper.findTopoPath(treeNode, targetNode1)).hasSize(3).element(0).isEqualTo(treeNode); + assertThat(TopologyHelper.findTopoPath(treeNode, targetNode1)).hasSize(3).element(1).isEqualTo(setNode); + assertThat(TopologyHelper.findTopoPath(treeNode, targetNode1)).hasSize(3).element(2).isEqualTo(moduleNode2); + assertThat(TopologyHelper.findTopoPath(treeNode, notExistsNode)).isNull(); + List targetNodes = new ArrayList<>(); + targetNodes.add(targetNode1); + targetNodes.add(targetNode2); + targetNodes.add(targetNode3); + targetNodes.add(notExistsNode); + List> pathList = TopologyHelper.findTopoPaths(treeNode, targetNodes); + for (List path : pathList) { + System.out.println(path); + } + assertThat(pathList).hasSize(4); + assertThat(pathList).element(0).asList().hasSize(3); + assertThat(pathList).element(1).asList().hasSize(2); + assertThat(pathList).element(2).asList().hasSize(3); + assertThat(pathList).element(3).isNull(); + } +} diff --git a/src/backend/job-manage/service-job-manage/src/test/java/com/tencent/bk/job/manage/common/util/TopologyHelperTest.java b/src/backend/job-manage/service-job-manage/src/test/java/com/tencent/bk/job/manage/common/util/TopologyHelperTest.java deleted file mode 100644 index 9431397f10..0000000000 --- a/src/backend/job-manage/service-job-manage/src/test/java/com/tencent/bk/job/manage/common/util/TopologyHelperTest.java +++ /dev/null @@ -1,104 +0,0 @@ -/* - * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. - * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. - * - * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. - * - * License for BK-JOB蓝鲸智云作业平台: - * -------------------------------------------------------------------- - * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated - * documentation files (the "Software"), to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and - * to permit persons to whom the Software is furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all copies or substantial portions of - * the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO - * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF - * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS - * IN THE SOFTWARE. - */ - -package com.tencent.bk.job.manage.common.util; - -import com.tencent.bk.job.common.cc.model.InstanceTopologyDTO; -import com.tencent.bk.job.manage.common.TopologyHelper; -import org.junit.jupiter.api.Test; - -import java.util.ArrayList; -import java.util.List; - -import static org.assertj.core.api.Assertions.assertThat; - -public class TopologyHelperTest { - - @Test - public void testFindTopologyPath() { - InstanceTopologyDTO treeNode = new InstanceTopologyDTO(); - treeNode.setObjectName("业务"); - treeNode.setObjectId("biz"); - treeNode.setInstanceId(2L); - treeNode.setInstanceName("蓝鲸"); - InstanceTopologyDTO setNode = new InstanceTopologyDTO(); - setNode.setObjectName("集合"); - setNode.setObjectId("set"); - setNode.setInstanceId(3L); - setNode.setInstanceName("测试集合"); - InstanceTopologyDTO moduleNode1 = new InstanceTopologyDTO(); - moduleNode1.setObjectName("模块"); - moduleNode1.setObjectId("module"); - moduleNode1.setInstanceId(4L); - moduleNode1.setInstanceName("测试模块1"); - InstanceTopologyDTO moduleNode2 = new InstanceTopologyDTO(); - moduleNode2.setObjectName("模块"); - moduleNode2.setObjectId("module"); - moduleNode2.setInstanceId(5L); - moduleNode2.setInstanceName("测试模块2"); - InstanceTopologyDTO moduleNode3 = new InstanceTopologyDTO(); - moduleNode3.setObjectName("模块"); - moduleNode3.setObjectId("module"); - moduleNode3.setInstanceId(6L); - moduleNode3.setInstanceName("测试模块3"); - List moduleList = new ArrayList<>(); - moduleList.add(moduleNode1); - moduleList.add(moduleNode2); - moduleList.add(moduleNode3); - setNode.setChild(moduleList); - List setList = new ArrayList<>(); - setList.add(setNode); - treeNode.setChild(setList); - InstanceTopologyDTO targetNode1 = new InstanceTopologyDTO(); - targetNode1.setObjectId("module"); - targetNode1.setInstanceId(5L); - InstanceTopologyDTO targetNode2 = new InstanceTopologyDTO(); - targetNode2.setObjectId("set"); - targetNode2.setInstanceId(3L); - InstanceTopologyDTO targetNode3 = new InstanceTopologyDTO(); - targetNode3.setObjectId("module"); - targetNode3.setInstanceId(6L); - InstanceTopologyDTO notExistsNode = new InstanceTopologyDTO(); - notExistsNode.setObjectId("module"); - notExistsNode.setInstanceId(15L); - assertThat(TopologyHelper.findTopoPath(treeNode, targetNode1)).hasSize(3).element(0).isEqualTo(treeNode); - assertThat(TopologyHelper.findTopoPath(treeNode, targetNode1)).hasSize(3).element(1).isEqualTo(setNode); - assertThat(TopologyHelper.findTopoPath(treeNode, targetNode1)).hasSize(3).element(2).isEqualTo(moduleNode2); - assertThat(TopologyHelper.findTopoPath(treeNode, notExistsNode)).isNull(); - List targetNodes = new ArrayList<>(); - targetNodes.add(targetNode1); - targetNodes.add(targetNode2); - targetNodes.add(targetNode3); - targetNodes.add(notExistsNode); - List> pathList = TopologyHelper.findTopoPaths(treeNode, targetNodes); - for (List path : pathList) { - System.out.println(path); - } - assertThat(pathList).hasSize(4); - assertThat(pathList).element(0).asList().hasSize(3); - assertThat(pathList).element(1).asList().hasSize(2); - assertThat(pathList).element(2).asList().hasSize(3); - assertThat(pathList).element(3).isNull(); - } -} diff --git a/src/backend/job-manage/service-job-manage/src/test/java/com/tencent/bk/job/manage/manager/variable/StepRefVariableParserTest.java b/src/backend/job-manage/service-job-manage/src/test/java/com/tencent/bk/job/manage/manager/variable/StepRefVariableParserTest.java index 94522ccec6..31bbd4ec01 100644 --- a/src/backend/job-manage/service-job-manage/src/test/java/com/tencent/bk/job/manage/manager/variable/StepRefVariableParserTest.java +++ b/src/backend/job-manage/service-job-manage/src/test/java/com/tencent/bk/job/manage/manager/variable/StepRefVariableParserTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,8 +25,8 @@ package com.tencent.bk.job.manage.manager.variable; import com.tencent.bk.job.common.constant.TaskVariableTypeEnum; -import com.tencent.bk.job.manage.common.consts.script.ScriptTypeEnum; -import com.tencent.bk.job.manage.common.consts.task.TaskStepTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.script.ScriptTypeEnum; +import com.tencent.bk.job.manage.api.common.constants.task.TaskStepTypeEnum; import com.tencent.bk.job.manage.model.dto.task.TaskFileInfoDTO; import com.tencent.bk.job.manage.model.dto.task.TaskFileStepDTO; import com.tencent.bk.job.manage.model.dto.task.TaskScriptStepDTO; diff --git a/src/backend/job-tools/build.gradle b/src/backend/job-tools/build.gradle index 9c17fd1292..878654de4a 100644 --- a/src/backend/job-tools/build.gradle +++ b/src/backend/job-tools/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-tools/k8s-startup-controller/build.gradle b/src/backend/job-tools/k8s-startup-controller/build.gradle index 198a30961f..4a1d1d9220 100644 --- a/src/backend/job-tools/k8s-startup-controller/build.gradle +++ b/src/backend/job-tools/k8s-startup-controller/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-tools/k8s-startup-controller/src/main/java/com/tencent/bk/job/k8s/Consts.java b/src/backend/job-tools/k8s-startup-controller/src/main/java/com/tencent/bk/job/k8s/Consts.java index b541180b9c..f115272324 100644 --- a/src/backend/job-tools/k8s-startup-controller/src/main/java/com/tencent/bk/job/k8s/Consts.java +++ b/src/backend/job-tools/k8s-startup-controller/src/main/java/com/tencent/bk/job/k8s/Consts.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-tools/k8s-startup-controller/src/main/java/com/tencent/bk/job/k8s/ServiceDependModel.java b/src/backend/job-tools/k8s-startup-controller/src/main/java/com/tencent/bk/job/k8s/ServiceDependModel.java index 198f6c7d5b..95d79ffdd1 100644 --- a/src/backend/job-tools/k8s-startup-controller/src/main/java/com/tencent/bk/job/k8s/ServiceDependModel.java +++ b/src/backend/job-tools/k8s-startup-controller/src/main/java/com/tencent/bk/job/k8s/ServiceDependModel.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/job-tools/k8s-startup-controller/src/main/java/com/tencent/bk/job/k8s/StartupController.java b/src/backend/job-tools/k8s-startup-controller/src/main/java/com/tencent/bk/job/k8s/StartupController.java index d264f5d0a6..32664cd6b8 100644 --- a/src/backend/job-tools/k8s-startup-controller/src/main/java/com/tencent/bk/job/k8s/StartupController.java +++ b/src/backend/job-tools/k8s-startup-controller/src/main/java/com/tencent/bk/job/k8s/StartupController.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -58,6 +58,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.stream.Collectors; /** * 启动控制器，用于控制在K8s部署时各Service的启动顺序 @@ -65,22 +66,16 @@ @Slf4j public class StartupController { - // K8s API - private static final CoreV1Api api; + public static final String POD_PHASE_PENDING = "Pending"; + public static final String POD_PHASE_RUNNING = "Running"; - static { - ApiClient client = null; - try { - client = Config.defaultClient(); - } catch (IOException e) { - log.error("Fail to get k8s api defaultClient", e); - } - Configuration.setDefaultApiClient(client); - api = new CoreV1Api(); - } + // K8s API + private static CoreV1Api api; // 主程序入口 public static void main(String[] args) { + // 初始化K8s API + initK8sApi(); // 解析需要的依赖参数 ServiceDependModel serviceDependModel = parseDependModelFromArgsOrEnv(args); String namespace = serviceDependModel.getNamespace(); @@ -119,6 +114,20 @@ public static void main(String[] args) { log.info("all depend services are ready, it`s time for {} to start", currentService); } + /** + * 初始化K8s API + */ + private static void initK8sApi() { + ApiClient client = null; + try { + client = Config.defaultClient(); + } catch (IOException e) { + log.error("Fail to get k8s api defaultClient", e); + } + Configuration.setDefaultApiClient(client); + api = new CoreV1Api(); + } + /** * 从命令行参数或环境变量解析出程序运行需要的服务依赖参数 * @@ -435,7 +444,7 @@ private static boolean isServiceEndPointAddressReady(String namespace, String se } /** - * 判断服务对应的Pod是否准备好，依据：Pod全部处于Running状态，且拥有指定的所有标签 + * 判断服务对应的Pod是否准备好，依据：准备/运行期Pod全部达到Running状态，且拥有指定的所有标签 * * @param namespace 命名空间 * @param serviceName 服务名称 @@ -449,6 +458,24 @@ private static boolean isServicePodReady(String namespace, String serviceName, log.info("no pod found by service {}", serviceName); return true; } + // 启动阶段只关注Pending/Running状态的Pod，忽略其他状态的Pod（例如Evicted） + servicePodList = servicePodList.stream().filter(v1Pod -> { + if (v1Pod == null || v1Pod.getStatus() == null) { + return false; + } + V1PodStatus v1PodStatus = v1Pod.getStatus(); + return POD_PHASE_PENDING.equalsIgnoreCase(v1PodStatus.getPhase()) || + POD_PHASE_RUNNING.equalsIgnoreCase(v1PodStatus.getPhase()); + }).collect(Collectors.toList()); + if (CollectionUtils.isEmpty(servicePodList)) { + log.info( + "no {}/{} pod found by service {}", + POD_PHASE_PENDING, + POD_PHASE_RUNNING, + serviceName + ); + return true; + } int readyPodNum = 0; for (V1Pod v1Pod : servicePodList) { if (isPodReady(v1Pod, requiredPodLabelsMap)) { @@ -562,9 +589,8 @@ private static boolean isPodReady(V1Pod v1Pod, Map requiredPodLa } log.debug("phase of pod {}:{}", v1Pod.getMetadata().getName(), v1PodStatus.getPhase()); } - String targetPhase = "Running"; return v1PodStatus != null - && targetPhase.equalsIgnoreCase(v1PodStatus.getPhase()) + && POD_PHASE_RUNNING.equalsIgnoreCase(v1PodStatus.getPhase()) && isPodHasLabels(v1Pod, requiredPodLabelsMap); } diff --git a/src/backend/job-tools/k8s-startup-controller/src/test/java/com/tencent/bk/job/k8s/StartupControllerTest.java b/src/backend/job-tools/k8s-startup-controller/src/test/java/com/tencent/bk/job/k8s/StartupControllerTest.java index 58e1c941c7..747e1dfc34 100644 --- a/src/backend/job-tools/k8s-startup-controller/src/test/java/com/tencent/bk/job/k8s/StartupControllerTest.java +++ b/src/backend/job-tools/k8s-startup-controller/src/test/java/com/tencent/bk/job/k8s/StartupControllerTest.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/settings.gradle b/src/backend/settings.gradle index 22a7a3f2e3..d9a475f307 100644 --- a/src/backend/settings.gradle +++ b/src/backend/settings.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,23 +28,31 @@ include 'commons:artifactory-sdk' include 'commons:common' include 'commons:esb-sdk' include 'commons:cmdb-sdk' -include 'commons:cmdb-sdk-model' -include 'commons:cmdb-sdk-ext' include 'commons:paas-sdk' include 'commons:gse-sdk' +include 'commons:notice-sdk' +include 'commons:ai-dev-sdk' include 'commons:common-iam' +include 'commons:common-jwt' include 'commons:common-utils' include 'commons:common-otel' +include 'commons:common-rabbitmq' include 'commons:common-redis' include 'commons:common-security' include 'commons:common-discovery' include 'commons:common-consul' +include 'commons:common-crypto' include 'commons:common-k8s' -include 'commons:common-statistics' +include 'commons:common-log' include 'commons:common-service' +include 'commons:common-task' include 'commons:common-api' include 'commons:common-web' include 'commons:common-i18n' +include 'commons:common-mysql' +include 'commons:common-audit' +include 'commons:common-mysql-sharding' +include 'commons:common-mongodb' include 'job-config' include 'job-gateway' include 'job-manage' @@ -85,6 +93,7 @@ include 'job-file-gateway:boot-job-file-gateway' include 'job-file-gateway:model-job-file-gateway' include 'job-file-gateway:service-job-file-gateway' include 'job-analysis' +include 'job-analysis:api-common-job-analysis' include 'job-analysis:api-job-analysis' include 'job-analysis:service-job-analysis' include 'job-analysis:boot-job-analysis' @@ -92,4 +101,6 @@ include 'job-analysis:model-job-analysis' include 'job-tools' include 'job-tools:k8s-startup-controller' include 'upgrader' +include 'job-assemble' +include 'job-manage:api-common-job-manage' diff --git a/src/backend/task_gen_jooq.gradle b/src/backend/task_gen_jooq.gradle index 71156d660e..13e1ba6923 100644 --- a/src/backend/task_gen_jooq.gradle +++ b/src/backend/task_gen_jooq.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,7 +25,7 @@ apply plugin: 'nu.studer.jooq' dependencies { api "org.jooq:jooq" - jooqRuntime "mysql:mysql-connector-java" + jooqRuntime "com.mysql:mysql-connector-j" } def nameArr = name.split('-') @@ -44,7 +44,7 @@ simpleModuleName = simpleModuleName.substring(0, simpleModuleName.length() - 1) println("moduleName=" + moduleName) println("simpleModuleName=" + simpleModuleName) def databaseName = moduleName -def packageName = "com.tencent.bk.job." + simpleModuleName + ".model" +def targetPackageName = "com.tencent.bk.job." + simpleModuleName + ".model" jooq { genenrate(sourceSets.main) { @@ -56,16 +56,10 @@ jooq { def mysqlPasswd = System.getProperty("mysqlPasswd") if (mysqlURL == null) { - mysqlURL = System.getenv("mysqlURL") - mysqlUser = System.getenv("mysqlUser") - mysqlPasswd = System.getenv("mysqlPasswd") - } - - if (mysqlURL == null) { - println "use default mysql database." - mysqlURL = DB_HOST - mysqlUser = DB_USERNAME - mysqlPasswd = DB_PASSWORD + // gradle.properties 中的 Jooq DB 配置 + mysqlURL = project.findProperty("DB_HOST") + mysqlUser = project.findProperty("DB_USERNAME") + mysqlPasswd = project.findProperty("DB_PASSWORD") } println("mysqlURL=" + mysqlURL) @@ -91,7 +85,8 @@ jooq { } target { - packageName = "${packageName}" + packageName = "${targetPackageName}" + encoding = 'UTF-8' } } } diff --git a/src/backend/task_job_package.gradle b/src/backend/task_job_package.gradle index 9c28cad2fb..db2839bd36 100644 --- a/src/backend/task_job_package.gradle +++ b/src/backend/task_job_package.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -54,7 +54,6 @@ task postProcessRelease(type: GenCompleteDependJarListAndRemoveInfectedJarsTask) completeJarListsRootPath = "../../support-files/dependJarLists" remove "amqp-client-.*jar" - remove "annotations-.*jar" remove "aspectjrt-.*jar" remove "aspectjweaver-.*jar" remove "bcpkix-jdk15on-.*jar" @@ -80,7 +79,7 @@ task postProcessRelease(type: GenCompleteDependJarListAndRemoveInfectedJarsTask) remove "logback-classic-.*jar" remove "logback-core-.*jar" remove "mchange-commons-java-.*jar" - remove "mysql-connector-java-.*jar" + remove "com.mysql:mysql-connector-j-.*jar" remove "org.eclipse.jgit-.*jar" remove "org.eclipse.jgit.http.apache-.*jar" remove "reactive-streams-.*jar" diff --git a/src/backend/upgrader/build.gradle b/src/backend/upgrader/build.gradle index 99086efd57..f663da33b1 100644 --- a/src/backend/upgrader/build.gradle +++ b/src/backend/upgrader/build.gradle @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -39,9 +39,10 @@ dependencies { api project(":commons:common-api") api project(":commons:common-iam") api project(":commons:common-utils") + api project(":commons:common-jwt") api project(":commons:esb-sdk") - api project(":commons:cmdb-sdk-model") api project(":commons:artifactory-sdk") + api(project(":job-manage:api-job-manage")) compileOnly 'org.projectlombok:lombok' annotationProcessor 'org.projectlombok:lombok' implementation 'ch.qos.logback:logback-core' diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/Upgrader.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/Upgrader.java index 99a680b5ab..c43ff4a46e 100644 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/Upgrader.java +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/Upgrader.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -39,6 +39,7 @@ import org.reflections.Reflections; import org.reflections.scanners.SubTypesScanner; import org.reflections.scanners.TypeAnnotationsScanner; +import org.slf4j.helpers.MessageFormatter; import java.io.BufferedReader; import java.io.FileInputStream; @@ -64,7 +65,7 @@ public Upgrader() { public static void usage() { log.info("===========================================Usage=============================================="); - log.info("Usage: use command `java -Dfile.encoding=utf8 -Djob.log.dir=path/to/log/dir " + + log.info("Usage: use command `java -Dfile.encoding=utf8 -Djob.log.djavair=path/to/log/dir " + "-Dconfig.file=/path/to/config/file " + "-jar upgrader-[x.x.x.x].jar [fromVersion] " + "[toVersion] [executeTime]` to start the upgrader, " + @@ -223,7 +224,13 @@ private static void checkAndInputTaskParams( } while (!paramCheckResult.isPass()); paramMap.put(paramInstance.getKey(), paramValue.trim()); } catch (InstantiationException | IllegalAccessException e) { - log.error("Fail to set params for {}", clazz.getSimpleName(), e); + String msg = MessageFormatter.arrayFormat( + "Fail to set params for {}", + new String[]{ + clazz.getSimpleName() + } + ).getMessage(); + log.error(msg, e); break; } } @@ -255,7 +262,11 @@ public static void main(String[] args) { ); properties.load(br); } catch (IOException e) { - log.warn("Cannot read configFile from path:{}, exit", configFilePath, e); + String msg = MessageFormatter.format( + "Cannot read configFile from path:{}, exit", + configFilePath + ).getMessage(); + log.warn(msg, e); return; } finally { if (br != null) { @@ -287,7 +298,7 @@ public static void main(String[] args) { // 参数输入 checkAndInputTaskParams(upgradeTaskList, properties); // 运行 - runTasks(upgradeTaskList, args, properties); + System.exit(runTasks(upgradeTaskList, args, properties)); } private static int runTasks( @@ -315,8 +326,8 @@ public void run() { upgradeTask.getName(), upgradeTask.getPriority(), upgradeTask.getTargetVersion()); - int resultCode = upgradeTask.execute(args); - if (resultCode == 0) { + boolean success = upgradeTask.execute(args); + if (success) { successfulTaskNum.incrementAndGet(); taskSuccess.set(true); log.info("UpgradeTask [{}][priority={}] for version {} successfully end", @@ -330,7 +341,11 @@ public void run() { upgradeTask.getTargetVersion()); } } catch (Exception e) { - log.error("Fail to run {}", clazz.getSimpleName(), e); + String msg = MessageFormatter.format( + "Fail to run {}", + clazz.getSimpleName() + ).getMessage(); + log.error(msg, e); } } }; @@ -388,10 +403,18 @@ private static IUpgradeTask getUpgradeTaskByClass(Class clazz, } return upgradeTask; } catch (InstantiationException e) { - log.error("Fail to Instantiate {}", clazz.getSimpleName(), e); + String msg = MessageFormatter.format( + "Fail to Instantiate {}", + clazz.getSimpleName() + ).getMessage(); + log.error(msg, e); throw new RuntimeException(e); } catch (IllegalAccessException e) { - log.error("Fail to Instantiate {} because of illegalAccess", clazz.getSimpleName(), e); + String msg = MessageFormatter.format( + "Fail to Instantiate {} because of illegalAccess", + clazz.getSimpleName() + ).getMessage(); + log.error(msg, e); throw new RuntimeException(e); } } diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/anotation/ExecuteTimeEnum.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/anotation/ExecuteTimeEnum.java index ac30219da9..be61568583 100644 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/anotation/ExecuteTimeEnum.java +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/anotation/ExecuteTimeEnum.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/anotation/RequireTaskParam.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/anotation/RequireTaskParam.java index fc2ed1276c..6154a303ed 100644 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/anotation/RequireTaskParam.java +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/anotation/RequireTaskParam.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/anotation/UpgradeTask.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/anotation/UpgradeTask.java index 30f68fa11c..a6a55dac90 100644 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/anotation/UpgradeTask.java +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/anotation/UpgradeTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/anotation/UpgradeTaskInputParam.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/anotation/UpgradeTaskInputParam.java index 59b4154404..9578a6ea8d 100644 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/anotation/UpgradeTaskInputParam.java +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/anotation/UpgradeTaskInputParam.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/AbstractHttpClient.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/AbstractHttpClient.java index cbe74265c4..9514c4c836 100644 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/AbstractHttpClient.java +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/AbstractHttpClient.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,12 +24,15 @@ package com.tencent.bk.job.upgrader.client; +import com.tencent.bk.job.common.constant.HttpMethodEnum; import com.tencent.bk.job.common.util.http.BasicHttpReq; -import com.tencent.bk.job.common.util.http.ExtHttpHelper; +import com.tencent.bk.job.common.util.http.HttpHelper; import com.tencent.bk.job.common.util.http.HttpHelperFactory; +import com.tencent.bk.job.common.util.http.HttpRequest; import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.upgrader.model.IamReq; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.collections4.CollectionUtils; import org.apache.http.Header; import org.apache.http.message.BasicHeader; @@ -40,7 +43,7 @@ @Slf4j public abstract class AbstractHttpClient { private String hostUrl; - private ExtHttpHelper defaultHttpHelper = HttpHelperFactory.getDefaultHttpHelper(); + private HttpHelper defaultHttpHelper = HttpHelperFactory.getDefaultHttpHelper(); public AbstractHttpClient(String hostUrl) { this.hostUrl = hostUrl; @@ -52,7 +55,7 @@ public String doHttpGet(String uri, IamReq params) throws Exception { abstract List

getBasicHeaders(); - public String doHttpGet(String uri, BasicHttpReq params, ExtHttpHelper httpHelper) throws Exception { + public String doHttpGet(String uri, BasicHttpReq params, HttpHelper httpHelper) throws Exception { if (params == null) { params = new BasicHttpReq(); } @@ -62,14 +65,15 @@ public String doHttpGet(String uri, BasicHttpReq params, ExtHttpHelper httpHelpe boolean error = false; long start = System.currentTimeMillis(); String responseBody = null; - String url = hostUrl; + String url; try { if (!hostUrl.endsWith("/") && !uri.startsWith("/")) { url = hostUrl + "/" + uri + params.toUrlParams(); } else { url = hostUrl + uri + params.toUrlParams(); } - responseBody = httpHelper.get(url, getBasicHeaders()); + responseBody = httpHelper.requestForSuccessResp(HttpRequest.builder(HttpMethodEnum.GET, url) + .setHeaders(toHeaderArray(getBasicHeaders())).build()).getEntity(); return responseBody; } catch (Exception e) { log.error("Get url {}| params={}| exception={}", hostUrl + uri, @@ -83,13 +87,24 @@ public String doHttpGet(String uri, BasicHttpReq params, ExtHttpHelper httpHelpe } } + private Header[] toHeaderArray(List
headerList) { + if (CollectionUtils.isEmpty(headerList)) { + return new Header[0]; + } + Header[] headerArray = new Header[headerList.size()]; + for (int i = 0; i < headerList.size(); i++) { + headerArray[i] = headerList.get(i); + } + return headerArray; + } + protected String doHttpPost(String uri, T params) throws Exception { return doHttpPost(uri, params, defaultHttpHelper); } protected String doHttpPost( String uri, T params, - ExtHttpHelper httpHelper + HttpHelper httpHelper ) throws Exception { if (httpHelper == null) { httpHelper = defaultHttpHelper; @@ -106,7 +121,8 @@ protected String doHttpPost( } List
headerList = getBasicHeaders(); headerList.add(new BasicHeader(HDR_CONTENT_TYPE, "application/json")); - responseBody = httpHelper.post(url, "UTF-8", buildPostBody(params), headerList); + responseBody = httpHelper.requestForSuccessResp(HttpRequest.builder(HttpMethodEnum.POST, url) + .setStringEntity(buildPostBody(params)).setHeaders(toHeaderArray(headerList)).build()).getEntity(); return responseBody; } catch (Exception e) { log.warn("Post url {}| params={}| exception={}", uri, JsonUtils.toJsonWithoutSkippedFields(params), diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/AbstractIamClient.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/AbstractIamClient.java index 68d9abdd2a..ae8d5f02fd 100644 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/AbstractIamClient.java +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/AbstractIamClient.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/AbstractJobClient.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/AbstractJobClient.java index b7ea9a001b..28aead9e41 100644 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/AbstractJobClient.java +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/AbstractJobClient.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,8 +24,20 @@ package com.tencent.bk.job.upgrader.client; +import com.fasterxml.jackson.core.type.TypeReference; +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.InternalException; +import com.tencent.bk.job.common.model.Response; +import com.tencent.bk.job.common.util.StringUtil; +import com.tencent.bk.job.common.util.http.BasicHttpReq; +import com.tencent.bk.job.common.util.http.HttpHelper; +import com.tencent.bk.job.common.util.json.JsonMapper; +import com.tencent.bk.job.common.util.json.JsonUtils; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; import org.apache.http.Header; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; import org.apache.http.message.BasicHeader; import java.util.ArrayList; @@ -33,7 +45,14 @@ @Slf4j public abstract class AbstractJobClient extends AbstractHttpClient { - private String jobAuthToken; + + /** + * Job API 处理请求成功 + */ + private static final int RESULT_OK = 0; + private static final JsonMapper JSON_MAPPER = JsonMapper.nonDefaultMapper(); + + private final String jobAuthToken; public AbstractJobClient(String jobHostUrl, String jobAuthToken) { super(jobHostUrl); @@ -48,4 +67,73 @@ protected List
getBasicHeaders() { return headerList; } + protected R getJobRespByReq(String method, String uri, BasicHttpReq reqBody, + TypeReference typeReference) { + return getJobRespByReq(method, uri, reqBody, typeReference, null); + } + + @SuppressWarnings("all") + protected R getJobRespByReq( + String method, + String uri, + BasicHttpReq reqBody, + TypeReference typeReference, + HttpHelper httpHelper + ) { + // URL模板变量替换 + uri = StringUtil.replacePathVariables(uri, reqBody); + String reqStr = JsonUtils.toJsonWithoutSkippedFields(reqBody); + String respStr = null; + try { + if (method.equals(HttpGet.METHOD_NAME)) { + respStr = doHttpGet(uri, reqBody, httpHelper); + } else if (method.equals(HttpPost.METHOD_NAME)) { + respStr = doHttpPost(uri, reqBody, httpHelper); + } + if (StringUtils.isBlank(respStr)) { + log.error("fail:response is blank|method={}|uri={}|reqStr={}", method, uri, reqStr); + throw new InternalException("response is blank", ErrorCode.API_ERROR); + } else { + log.debug("success|method={}|uri={}|reqStr={}|respStr={}", method, uri, reqStr, respStr); + } + R result = + JSON_MAPPER.fromJson(respStr, typeReference); + Response jobResp = (Response) result; + if (jobResp == null) { + log.error("fail:jobResp is null after parse|method={}|uri={}|reqStr={}|respStr={}", method, uri, + reqStr, respStr); + throw new InternalException("jobResp is null after parse", ErrorCode.API_ERROR); + } else if (jobResp.getCode() != RESULT_OK) { + log.error( + "fail:jobResp code!=0|jobResp.code={}|jobResp" + + ".errorMessage={}|method={}|uri={}|reqStr={}|respStr={}", + jobResp.getCode(), + jobResp.getErrorMsg(), + method, + uri, + reqStr, + respStr + ); + throw new InternalException("jobResp code!=0", ErrorCode.API_ERROR); + } + if (jobResp.getData() == null) { + log.warn( + "warn:jobResp.getData() == null|jobResp.code={}|jobResp" + + ".errorMessage={}|method={}|uri={}|reqStr={}|respStr={}", + jobResp.getCode(), + jobResp.getErrorMsg(), + method, + uri, + reqStr, + respStr + ); + } + return result; + } catch (Exception e) { + String errorMsg = "Fail to request JOB data|method=" + method + "|uri=" + uri + "|reqStr=" + reqStr; + log.error(errorMsg, e); + throw new InternalException("Fail to request JOB data", ErrorCode.API_ERROR); + } + } + } diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/IamClient.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/IamClient.java index 9208d9e2a7..2b13e78c72 100644 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/IamClient.java +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/IamClient.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,7 +28,7 @@ import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.exception.InternalIamException; import com.tencent.bk.job.common.util.StringUtil; -import com.tencent.bk.job.common.util.http.ExtHttpHelper; +import com.tencent.bk.job.common.util.http.HttpHelper; import com.tencent.bk.job.common.util.json.JsonMapper; import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.upgrader.model.ActionPolicies; @@ -70,7 +70,7 @@ private R getIamRespByReq(String method, String uri, IamReq reqBody, @SuppressWarnings("all") private R getIamRespByReq(String method, String uri, IamReq reqBody, TypeReference typeReference, - ExtHttpHelper httpHelper) throws RuntimeException { + HttpHelper httpHelper) throws RuntimeException { // URL模板变量替换 uri = StringUtil.replacePathVariables(uri, reqBody); String reqStr = JsonUtils.toJsonWithoutSkippedFields(reqBody); diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/JobClient.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/JobClient.java index 3f7b430af3..5112a20fac 100644 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/JobClient.java +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/client/JobClient.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -25,19 +25,13 @@ package com.tencent.bk.job.upgrader.client; import com.fasterxml.jackson.core.type.TypeReference; -import com.tencent.bk.job.common.constant.ErrorCode; -import com.tencent.bk.job.common.exception.InternalException; import com.tencent.bk.job.common.model.Response; -import com.tencent.bk.job.common.util.StringUtil; import com.tencent.bk.job.common.util.http.BasicHttpReq; -import com.tencent.bk.job.common.util.http.ExtHttpHelper; -import com.tencent.bk.job.common.util.json.JsonMapper; -import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.manage.model.migration.BkPlatformInfo; import com.tencent.bk.job.upgrader.model.AppInfo; import com.tencent.bk.job.upgrader.model.BasicAppInfo; import com.tencent.bk.job.upgrader.model.job.SetBizSetMigrationStatusReq; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; import org.apache.http.client.methods.HttpGet; import org.apache.http.client.methods.HttpPost; @@ -49,92 +43,19 @@ @Slf4j public class JobClient extends AbstractJobClient { - /** - * Job API 处理请求成功 - */ - private static final int RESULT_OK = 0; - private static final String URL_LIST_NORMAL_APPS = "/service/app/list/normal"; private static final String URL_LIST_BIZ_SET_APPS = "/service/app/list/bizSet"; - private static final String URL_SET_BIZ_SET_MIGRATION_STATUS = "/migration/action/setBizSetMigrationStatus"; + private static final String URL_SET_BIZ_SET_MIGRATION_STATUS = "/manage/migration/action/setBizSetMigrationStatus"; + + private static final String URL_GET_BK_PLATFORM_INFO = "/manage/migration/action/getBkPlatformInfo"; - private static final JsonMapper JSON_MAPPER = JsonMapper.nonDefaultMapper(); public JobClient(String jobHostUrl, String jobAuthToken) { super(jobHostUrl, jobAuthToken); } - private R getJobRespByReq(String method, String uri, BasicHttpReq reqBody, - TypeReference typeReference) { - return getJobRespByReq(method, uri, reqBody, typeReference, null); - } - - @SuppressWarnings("all") - private R getJobRespByReq( - String method, - String uri, - BasicHttpReq reqBody, - TypeReference typeReference, - ExtHttpHelper httpHelper - ) { - // URL模板变量替换 - uri = StringUtil.replacePathVariables(uri, reqBody); - String reqStr = JsonUtils.toJsonWithoutSkippedFields(reqBody); - String respStr = null; - try { - if (method.equals(HttpGet.METHOD_NAME)) { - respStr = doHttpGet(uri, reqBody, httpHelper); - } else if (method.equals(HttpPost.METHOD_NAME)) { - respStr = doHttpPost(uri, reqBody, httpHelper); - } - if (StringUtils.isBlank(respStr)) { - log.error("fail:response is blank|method={}|uri={}|reqStr={}", method, uri, reqStr); - throw new InternalException("response is blank", ErrorCode.API_ERROR); - } else { - log.debug("success|method={}|uri={}|reqStr={}|respStr={}", method, uri, reqStr, respStr); - } - R result = - JSON_MAPPER.fromJson(respStr, typeReference); - Response jobResp = (Response) result; - if (jobResp == null) { - log.error("fail:jobResp is null after parse|method={}|uri={}|reqStr={}|respStr={}", method, uri, - reqStr, respStr); - throw new InternalException("jobResp is null after parse", ErrorCode.API_ERROR); - } else if (jobResp.getCode() != RESULT_OK) { - log.error( - "fail:jobResp code!=0|jobResp.code={}|jobResp" + - ".errorMessage={}|method={}|uri={}|reqStr={}|respStr={}", - jobResp.getCode(), - jobResp.getErrorMsg(), - method, - uri, - reqStr, - respStr - ); - throw new InternalException("jobResp code!=0", ErrorCode.API_ERROR); - } - if (jobResp.getData() == null) { - log.warn( - "warn:jobResp.getData() == null|jobResp.code={}|jobResp" + - ".errorMessage={}|method={}|uri={}|reqStr={}|respStr={}", - jobResp.getCode(), - jobResp.getErrorMsg(), - method, - uri, - reqStr, - respStr - ); - } - return result; - } catch (Exception e) { - String errorMsg = "Fail to request JOB data|method=" + method + "|uri=" + uri + "|reqStr=" + reqStr; - log.error(errorMsg, e); - throw new InternalException("Fail to request JOB data", ErrorCode.API_ERROR); - } - } - public List listNormalApps() { Response> resp = getJobRespByReq( HttpGet.METHOD_NAME, @@ -169,6 +90,16 @@ public boolean setBizSetMigrationStatus(boolean isMigrated) { req, new TypeReference>() { }); - return resp.getData() == null ? false : resp.getData(); + return resp.getData() != null && resp.getData(); + } + + public BkPlatformInfo getBkPlatformInfo() { + Response resp = getJobRespByReq( + HttpGet.METHOD_NAME, + URL_GET_BK_PLATFORM_INFO, + new BasicHttpReq(), + new TypeReference>() { + }); + return resp.getData(); } } diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/iam/ApiClientUtils.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/iam/ApiClientUtils.java new file mode 100644 index 0000000000..6f0dcbdcc4 --- /dev/null +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/iam/ApiClientUtils.java @@ -0,0 +1,52 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.upgrader.iam; + +import com.tencent.bk.job.common.esb.config.AppProperties; +import com.tencent.bk.job.common.esb.config.EsbProperties; +import com.tencent.bk.job.common.iam.client.EsbIamClient; +import com.tencent.bk.job.upgrader.task.param.ParamNameConsts; + +import java.util.Properties; + +public class ApiClientUtils { + + public static EsbIamClient buildEsbIamClient(Properties properties) { + EsbProperties esbProperties = new EsbProperties(); + EsbProperties.EsbServiceConfig esbServiceConfig = new EsbProperties.EsbServiceConfig(); + esbServiceConfig.setUrl((String) properties.get(ParamNameConsts.CONFIG_PROPERTY_ESB_SERVICE_URL)); + esbProperties.setService(esbServiceConfig); + + return new EsbIamClient( + null, + new AppProperties( + (String) properties.get(ParamNameConsts.CONFIG_PROPERTY_APP_CODE), + (String) properties.get(ParamNameConsts.CONFIG_PROPERTY_APP_SECRET) + ), + esbProperties + ); + } + +} diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/iam/JobIamHelper.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/iam/JobIamHelper.java index 220810ec82..ebebc24e6a 100644 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/iam/JobIamHelper.java +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/iam/JobIamHelper.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,7 +24,6 @@ package com.tencent.bk.job.upgrader.iam; -import com.tencent.bk.job.common.iam.config.EsbConfiguration; import com.tencent.bk.job.common.iam.util.BusinessAuthHelper; import com.tencent.bk.sdk.iam.config.IamConfiguration; import com.tencent.bk.sdk.iam.constants.SystemId; @@ -69,10 +68,6 @@ public IamConfiguration iamConfiguration() { return new IamConfiguration(SystemId.JOB, appCode, appSecret, iamBaseUrl); } - public EsbConfiguration esbConfiguration() { - return new EsbConfiguration(esbUrl, false); - } - public HttpClientService httpClientService() { return new DefaultHttpClientServiceImpl(iamConfiguration()); } @@ -86,10 +81,10 @@ public TokenService tokenService() { } public AuthHelper authHelper() { - return new AuthHelper(tokenService(), policyService(), iamConfiguration()); + return new AuthHelper(tokenService(), policyService(), null, iamConfiguration()); } public BusinessAuthHelper businessAuthHelper() { - return new BusinessAuthHelper(tokenService(), policyService(), iamConfiguration()); + return new BusinessAuthHelper(tokenService(), policyService(), null, iamConfiguration()); } } diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/Action.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/Action.java index 71672068dc..5a82fa9966 100644 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/Action.java +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/Action.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/ActionPolicies.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/ActionPolicies.java index 079bde0a69..35a3792251 100644 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/ActionPolicies.java +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/ActionPolicies.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/AppInfo.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/AppInfo.java index 90d9a98f19..4a2b51b748 100644 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/AppInfo.java +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/AppInfo.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/BasicAppInfo.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/BasicAppInfo.java index 811c641ccf..1417b905ff 100644 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/BasicAppInfo.java +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/BasicAppInfo.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/IamReq.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/IamReq.java index 1a1e512f50..02659b5cb3 100644 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/IamReq.java +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/IamReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/IamResp.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/IamResp.java index a8579bb98a..a54301ab5d 100644 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/IamResp.java +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/IamResp.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/MetaData.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/MetaData.java index 5278518dd7..4a05853739 100644 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/MetaData.java +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/MetaData.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/Policy.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/Policy.java index 8a14d3ea14..a29f7656f9 100644 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/Policy.java +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/Policy.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/PolicyListReq.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/PolicyListReq.java index 17247e9c13..eb7cae35fc 100644 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/PolicyListReq.java +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/PolicyListReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/Subject.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/Subject.java index 00fa54729c..feeb9200c4 100644 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/Subject.java +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/Subject.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/job/AddHostIdMigrationReq.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/job/AddHostIdMigrationReq.java new file mode 100644 index 0000000000..4be6ca5901 --- /dev/null +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/job/AddHostIdMigrationReq.java @@ -0,0 +1,48 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.upgrader.model.job; + +import com.tencent.bk.job.common.model.dto.ResourceScope; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; + +/** + * 作业模板、执行方案、IP白名单、定时任务等包含的主机数据，在原来的云区域+ip的基础上，填充hostID属性 - 请求 + */ +@Data +@NoArgsConstructor +public class AddHostIdMigrationReq { + // 迁移数据限定的资源范围，不传该参数则表示迁移所有资源范围下的数据 + private List scopeList; + + private boolean dryRun; + + public AddHostIdMigrationReq(List scopeList, boolean dryRun) { + this.scopeList = scopeList; + this.dryRun = dryRun; + } +} diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/job/SetBizSetMigrationStatusReq.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/job/SetBizSetMigrationStatusReq.java index caf7bb281d..500797bee6 100644 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/job/SetBizSetMigrationStatusReq.java +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/job/SetBizSetMigrationStatusReq.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/job/UpdateAppIdForWhiteIpMigrationReq.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/job/UpdateAppIdForWhiteIpMigrationReq.java new file mode 100644 index 0000000000..425350ff13 --- /dev/null +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/model/job/UpdateAppIdForWhiteIpMigrationReq.java @@ -0,0 +1,41 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.upgrader.model.job; + +import lombok.Data; +import lombok.NoArgsConstructor; + +/** + * IP白名单全业务ID更新为对所有业务生效ID - 请求 + */ +@Data +@NoArgsConstructor +public class UpdateAppIdForWhiteIpMigrationReq { + private boolean dryRun; + + public UpdateAppIdForWhiteIpMigrationReq(boolean dryRun) { + this.dryRun = dryRun; + } +} diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/AddHostIdMigrationTask.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/AddHostIdMigrationTask.java new file mode 100644 index 0000000000..0bcc869dae --- /dev/null +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/AddHostIdMigrationTask.java @@ -0,0 +1,114 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.upgrader.task; + +import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.exception.InvalidParamException; +import com.tencent.bk.job.common.model.Response; +import com.tencent.bk.job.common.model.dto.ResourceScope; +import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.upgrader.anotation.ExecuteTimeEnum; +import com.tencent.bk.job.upgrader.anotation.RequireTaskParam; +import com.tencent.bk.job.upgrader.anotation.UpgradeTask; +import com.tencent.bk.job.upgrader.anotation.UpgradeTaskInputParam; +import com.tencent.bk.job.upgrader.model.job.AddHostIdMigrationReq; +import com.tencent.bk.job.upgrader.task.param.JobCrontabServerAddress; +import com.tencent.bk.job.upgrader.task.param.JobManageServerAddress; +import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; + +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; + +/** + * 作业模板、执行方案、白名单、定时任务等包含的主机数据，在原来的云区域+ip的基础上，填充hostID属性 + */ +@Slf4j +@RequireTaskParam(value = { + @UpgradeTaskInputParam(value = JobManageServerAddress.class), + @UpgradeTaskInputParam(value = JobCrontabServerAddress.class) +}) +@UpgradeTask( + dataStartVersion = "3.0.0.0", + targetVersion = "3.7.0", + targetExecuteTime = ExecuteTimeEnum.AFTER_UPDATE_JOB) +public class AddHostIdMigrationTask extends BaseUpgradeTask { + + public AddHostIdMigrationTask(Properties properties) { + super(properties); + } + + private List buildScopeListFromArgs(String[] args) { + if (args.length < 4 || StringUtils.isBlank(args[3])) { + return null; + } + String scopeListStr = args[3]; + String[] scopeStrs = scopeListStr.replace(" ", "").split(","); + List scopeList = new ArrayList<>(scopeStrs.length); + for (String scopeStr : scopeStrs) { + String[] arr = scopeStr.split(":"); + if (arr.length < 2) { + throw new InvalidParamException(ErrorCode.INVALID_CMD_ARGS, + new String[]{ + scopeListStr + ", must be comma separated scope, example: biz:2,biz:3,biz_set:9991001" + } + ); + } + ResourceScope scope = new ResourceScope(arr[0].toLowerCase(), arr[1].toLowerCase()); + scopeList.add(scope); + } + return scopeList; + } + + @Override + public boolean execute(String[] args) { + log.info(getName() + " for version " + getTargetVersion() + " begin to run..."); + try { + boolean dryRun = getProperties().get("dryRun") != null + && ((String) getProperties().get("dryRun")).equalsIgnoreCase("true"); + if (dryRun) { + log.info("AddHostIdMigrationTask using dryRun mode!"); + } + List scopeList = buildScopeListFromArgs(args); + // 1.迁移作业模板、执行方案、IP白名单数据 + log.info("job.manage.addHostIdMigrationTask start ..."); + Response manageResp = post(buildMigrationTaskUrl(getJobManageUrl(), + "/manage/migration/action/addHostIdMigrationTask"), + JsonUtils.toJson(new AddHostIdMigrationReq(scopeList, dryRun))); + log.info("job.manage.addHostIdMigrationTask done, result: {}", manageResp); + // 2.迁移定时任务数据 + log.info("job.crontab.addHostIdMigrationTask start ..."); + Response crontabResp = post(buildMigrationTaskUrl(getJobCrontabUrl(), + "/crontab/migration/action/addHostIdMigrationTask"), + JsonUtils.toJson(new AddHostIdMigrationReq(scopeList, dryRun))); + log.info("job.crontab.addHostIdMigrationTask done, result: {}", crontabResp); + return manageResp.isSuccess() && crontabResp.isSuccess(); + } catch (Throwable e) { + log.error("AddHostIdMigrationTask fail", e); + return false; + } + } +} diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/BaseUpgradeTask.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/BaseUpgradeTask.java index 38ac149a61..59d902a9ae 100644 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/BaseUpgradeTask.java +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/BaseUpgradeTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,24 +24,53 @@ package com.tencent.bk.job.upgrader.task; +import com.fasterxml.jackson.core.type.TypeReference; import com.tencent.bk.job.common.constant.ErrorCode; +import com.tencent.bk.job.common.constant.HttpMethodEnum; import com.tencent.bk.job.common.exception.InternalException; -import com.tencent.bk.job.common.util.jwt.BasicJwtManager; -import com.tencent.bk.job.common.util.jwt.JwtManager; +import com.tencent.bk.job.common.exception.InvalidParamException; +import com.tencent.bk.job.common.jwt.BasicJwtManager; +import com.tencent.bk.job.common.jwt.JwtManager; +import com.tencent.bk.job.common.model.Response; +import com.tencent.bk.job.common.util.Base64Util; +import com.tencent.bk.job.common.util.http.BaseHttpHelper; +import com.tencent.bk.job.common.util.http.HttpHelper; +import com.tencent.bk.job.common.util.http.HttpRequest; +import com.tencent.bk.job.common.util.json.JsonUtils; import com.tencent.bk.job.upgrader.anotation.UpgradeTask; +import com.tencent.bk.job.upgrader.client.JobClient; import com.tencent.bk.job.upgrader.task.param.ParamNameConsts; import lombok.extern.slf4j.Slf4j; +import org.apache.commons.lang3.StringUtils; +import org.apache.http.Header; +import org.apache.http.client.config.RequestConfig; +import org.apache.http.config.ConnectionConfig; +import org.apache.http.conn.ssl.SSLConnectionSocketFactory; +import org.apache.http.conn.ssl.TrustSelfSignedStrategy; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.message.BasicHeader; +import org.apache.http.ssl.SSLContexts; +import org.slf4j.helpers.MessageFormatter; -import java.io.IOException; -import java.security.GeneralSecurityException; +import java.nio.charset.StandardCharsets; +import java.security.KeyManagementException; +import java.security.KeyStoreException; +import java.security.NoSuchAlgorithmException; import java.util.Properties; @Slf4j public abstract class BaseUpgradeTask implements IUpgradeTask { - public Properties properties; + private Properties properties; + private static final HttpHelper HTTP_HELPER; + + static { + HTTP_HELPER = new BaseHttpHelper(getHttpClient()); + } BaseUpgradeTask() { + } BaseUpgradeTask(Properties properties) { @@ -52,27 +81,112 @@ public Properties getProperties() { return properties; } - @Override - public void init() { + private String getJobHostUrlByAddress(String address) { + if (!address.startsWith("http://") && !address.startsWith("https://")) { + address = "http://" + address; + } + return address; } - public String getJobAuthToken() { + public JobClient getJobManageClient() { String securityPublicKeyBase64 = - (String) properties.get(ParamNameConsts.CONFIG_PROPERTY_JOB_SECURITY_PUBLIC_KEY_BASE64); + (String) getProperties().get(ParamNameConsts.CONFIG_PROPERTY_JOB_SECURITY_PUBLIC_KEY_BASE64); String securityPrivateKeyBase64 = - (String) properties.get(ParamNameConsts.CONFIG_PROPERTY_JOB_SECURITY_PRIVATE_KEY_BASE64); + (String) getProperties().get(ParamNameConsts.CONFIG_PROPERTY_JOB_SECURITY_PRIVATE_KEY_BASE64); JwtManager jwtManager; try { jwtManager = new BasicJwtManager(securityPrivateKeyBase64, securityPublicKeyBase64); - // 迁移过程最大预估时间：3h - return jwtManager.generateToken(3 * 60 * 60 * 1000); - } catch (IOException | GeneralSecurityException e) { + } catch (Exception e) { String msg = "Fail to generate jwt auth token"; log.error(msg, e); throw new InternalException(msg, e, ErrorCode.INTERNAL_ERROR); } + String jobAuthToken = jwtManager.generateToken(60 * 60 * 1000); + return new JobClient( + getJobHostUrlByAddress((String) getProperties().get(ParamNameConsts.INPUT_PARAM_JOB_MANAGE_SERVER_ADDRESS)), + jobAuthToken + ); } + @Override + public void init() { + } + + public Response post(String url, String content) throws InternalException { + String jobAuthToken = getProperties().getProperty( + ParamNameConsts.CONFIG_PROPERTY_JOB_SECURITY_PUBLIC_KEY_BASE64 + ); + if (StringUtils.isBlank(jobAuthToken)) { + log.error("{} is not configured", ParamNameConsts.CONFIG_PROPERTY_JOB_SECURITY_PUBLIC_KEY_BASE64); + throw new InvalidParamException(ErrorCode.ILLEGAL_PARAM_WITH_PARAM_NAME, new String[]{ + ParamNameConsts.CONFIG_PROPERTY_JOB_SECURITY_PUBLIC_KEY_BASE64 + }); + } + jobAuthToken = Base64Util.decodeContentToStr(jobAuthToken); + + Header[] headers = new Header[2]; + headers[0] = new BasicHeader("x-job-auth-token", jobAuthToken); + headers[1] = new BasicHeader("Content-Type", "application/json"); + + try { + String respStr = HTTP_HELPER.requestForSuccessResp( + HttpRequest.builder(HttpMethodEnum.POST, url).setStringEntity(content).setHeaders(headers).build()) + .getEntity(); + log.info("Post {}, content: {}, response: {}", url, content, respStr); + if (StringUtils.isBlank(respStr)) { + String errorMsg = + MessageFormatter.format("Fail: response is blank|url={}", url).getMessage(); + log.error(errorMsg); + throw new InternalException(errorMsg, ErrorCode.INTERNAL_ERROR); + } + Response resp = JsonUtils.fromJson(respStr, + new TypeReference>() { + }); + if (resp == null) { + String errorMsg = + MessageFormatter.format("Fail: parse response|url={}", url).getMessage(); + throw new InternalException(errorMsg, ErrorCode.INTERNAL_ERROR); + } + return resp; + } catch (Exception e) { + log.error("Fail: caught exception", e); + throw new InternalException(e, ErrorCode.INTERNAL_ERROR); + } + } + + protected final String buildMigrationTaskUrl(String serviceUrl, String migrationTaskUri) { + String url; + if (!serviceUrl.endsWith("/")) { + url = serviceUrl + migrationTaskUri; + } else { + url = serviceUrl.substring(0, serviceUrl.length() - 1) + migrationTaskUri; + } + return url; + } + + protected final String getJobManageUrl() { + String serverUrl = getProperties().getProperty( + ParamNameConsts.INPUT_PARAM_JOB_MANAGE_SERVER_ADDRESS + ); + return addMissingSchema(serverUrl); + } + + protected final String getJobCrontabUrl() { + String serverUrl = getProperties().getProperty( + ParamNameConsts.INPUT_PARAM_JOB_CRONTAB_SERVER_ADDRESS + ); + return addMissingSchema(serverUrl); + } + + private String addMissingSchema(String url) { + String newUrl = url; + if (!url.startsWith("http://") && !url.startsWith("https://")) { + newUrl = "http://" + url; + } + return newUrl; + } + + @Override public String getName() { return this.getClass().getSimpleName(); @@ -92,4 +206,36 @@ public String getTargetVersion() { public int getPriority() { return this.getClass().getAnnotation(UpgradeTask.class).priority(); } + + private static CloseableHttpClient getHttpClient() { + HttpClientBuilder httpClientBuilder = HttpClientBuilder.create() + .setDefaultConnectionConfig( + ConnectionConfig.custom() + .setBufferSize(102400) + .setCharset(StandardCharsets.UTF_8) + .build() + ).setDefaultRequestConfig( + RequestConfig.custom() + .setConnectTimeout(15000) + // 12h,等待足够长时间，等待升级任务返回结果 + .setSocketTimeout(43200000) + .build() + ) + .disableAutomaticRetries(); + + CloseableHttpClient httpClient; + try { + httpClient = httpClientBuilder.setSSLSocketFactory( + new SSLConnectionSocketFactory( + SSLContexts.custom() + .loadTrustMaterial(null, new TrustSelfSignedStrategy()) + .build() + ) + ).build(); + } catch (NoSuchAlgorithmException | KeyManagementException | KeyStoreException e) { + log.error("Set ssl config error", e); + httpClient = httpClientBuilder.build(); + } + return httpClient; + } } diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/ExportGlobalSettingsTask.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/ExportGlobalSettingsTask.java new file mode 100644 index 0000000000..d1c8d91a13 --- /dev/null +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/ExportGlobalSettingsTask.java @@ -0,0 +1,97 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.upgrader.task; + +import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.manage.model.migration.BkPlatformInfo; +import com.tencent.bk.job.upgrader.anotation.ExecuteTimeEnum; +import com.tencent.bk.job.upgrader.anotation.RequireTaskParam; +import com.tencent.bk.job.upgrader.anotation.UpgradeTask; +import com.tencent.bk.job.upgrader.anotation.UpgradeTaskInputParam; +import com.tencent.bk.job.upgrader.client.JobClient; +import com.tencent.bk.job.upgrader.task.param.JobManageServerAddress; +import lombok.extern.slf4j.Slf4j; + +import java.io.BufferedWriter; +import java.io.File; +import java.io.FileWriter; +import java.util.Properties; + +/** + * 3.9.3版本开始，改为使用基于BK-REPO的全局配置统一方案，该任务输出平台信息全局配置base.js文件，以便于让用户迁移之前已配置的数据： + * 将该任务生成的base.js文件导入BK-REPO的blueking项目下bk-config仓库下的bk_job目录下，覆盖原有的base.js即可。 + */ +@SuppressWarnings("unused") +@Slf4j +@RequireTaskParam(value = { + @UpgradeTaskInputParam(value = JobManageServerAddress.class), +}) +@UpgradeTask( + dataStartVersion = "3.0.0.0", + targetVersion = "3.9.3-alpha.1", + targetExecuteTime = ExecuteTimeEnum.AFTER_UPDATE_JOB) +public class ExportGlobalSettingsTask extends BaseUpgradeTask { + + private JobClient jobManageClient; + + @SuppressWarnings("unused") + public ExportGlobalSettingsTask(Properties properties) { + super(properties); + } + + @Override + public void init() { + super.init(); + jobManageClient = getJobManageClient(); + } + + @Override + public boolean execute(String[] args) { + log.info(getName() + " for version " + getTargetVersion() + " begin to run..."); + try { + BkPlatformInfo bkPlatformInfo = jobManageClient.getBkPlatformInfo(); + String bkPlatformInfoStr = JsonUtils.toJson(bkPlatformInfo); + String fileName = "base.js"; + String baseJsStr = "__platCfgCallback__(" + bkPlatformInfoStr + ")"; + log.info("Exported " + fileName + ":"); + log.info(baseJsStr); + File outputFile = new File(fileName); + if (!outputFile.exists()) { + boolean result = outputFile.createNewFile(); + if (!result) { + log.error("Fail to create file {}", fileName); + } + } + try (BufferedWriter bw = new BufferedWriter(new FileWriter(outputFile))) { + bw.write(baseJsStr); + } + log.info("BkPlatformInfo has been export to file {}, please check current dir", fileName); + return true; + } catch (Throwable e) { + log.error("ExportGlobalSettingsTask fail", e); + return false; + } + } +} diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/IUpgradeTask.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/IUpgradeTask.java index fef6fd92a8..7952ee0df3 100644 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/IUpgradeTask.java +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/IUpgradeTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -36,6 +36,12 @@ public interface IUpgradeTask { int getPriority(); - int execute(String[] args); + /** + * 是否升级成功 + * + * @param args 输入参数 + * @return 升级结果 + */ + boolean execute(String[] args); } diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/LocalUploadFileToBkRepoMigrationTask.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/LocalUploadFileToBkRepoMigrationTask.java index 36915ebbb1..f395c7c40e 100644 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/LocalUploadFileToBkRepoMigrationTask.java +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/LocalUploadFileToBkRepoMigrationTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -30,6 +30,7 @@ import com.tencent.bk.job.common.constant.ErrorCode; import com.tencent.bk.job.common.exception.InternalException; import com.tencent.bk.job.common.util.StringUtil; +import com.tencent.bk.job.common.util.ThreadUtils; import com.tencent.bk.job.common.util.file.PathUtil; import com.tencent.bk.job.upgrader.anotation.ExecuteTimeEnum; import com.tencent.bk.job.upgrader.anotation.UpgradeTask; @@ -61,9 +62,9 @@ dataStartVersion = "3.0.0.0", targetVersion = "3.4.2.1", targetExecuteTime = ExecuteTimeEnum.MAKE_UP) +@SuppressWarnings("unused") public class LocalUploadFileToBkRepoMigrationTask extends BaseUpgradeTask { - private ArtifactoryClient artifactoryAdminClient; private ArtifactoryClient artifactoryJobClient; private String baseUrl; private String adminUsername; @@ -78,30 +79,13 @@ public class LocalUploadFileToBkRepoMigrationTask extends BaseUpgradeTask { private boolean enableMigrateLocalUploadFile; private boolean enableMigrateBackupFile; private boolean enableMigrateLogExportFile; - private int uploadConcurrency; private ThreadPoolExecutor uploadExecutor; - - private final String localUploadDirName = "localupload"; - private final String backupDirName = "backup"; - private final String logExportDirName = "filedata"; + private final ConcurrentLinkedQueue allFailedFilePathList = new ConcurrentLinkedQueue<>(); public LocalUploadFileToBkRepoMigrationTask(Properties properties) { super(properties); } - private ArtifactoryClient getAdminClient() { - Properties properties = getProperties(); - if (artifactoryAdminClient == null) { - artifactoryAdminClient = new ArtifactoryClient( - (String) properties.get(ParamNameConsts.CONFIG_PROPERTY_ARTIFACTORY_BASE_URL), - (String) properties.get(ParamNameConsts.CONFIG_PROPERTY_ARTIFACTORY_ADMIN_USERNAME), - (String) properties.get(ParamNameConsts.CONFIG_PROPERTY_ARTIFACTORY_ADMIN_PASSWORD), - null - ); - } - return artifactoryAdminClient; - } - private ArtifactoryClient getJobClient() { Properties properties = getProperties(); if (artifactoryJobClient == null) { @@ -140,7 +124,7 @@ private void initConfig() { (String) properties.getOrDefault( ParamNameConsts.CONFIG_PROPERTY_ENABLE_MIGRATE_LOG_EXPORT_FILE, "false" )); - uploadConcurrency = Integer.parseInt( + int uploadConcurrency = Integer.parseInt( (String) properties.getOrDefault( ParamNameConsts.CONFIG_PROPERTY_MIGRATE_UPLOAD_CONCURRENCY, 20 )); @@ -273,30 +257,38 @@ private List scanAllFilesOnPath(String path) { private boolean existsFileInRepo(String repo, String filePath, String fileMd5) { ArtifactoryClient jobClient = getJobClient(); - try { - NodeDTO nodeDTO = jobClient.queryNodeDetail(jobProject, repo, filePath); - return nodeDTO != null - && fileMd5.equals(nodeDTO.getMd5()); - } catch (Exception e) { - FormattingTuple msg = MessageFormatter.format( - "Fail to compare targetFile {} with node in repo {}", - filePath, - repo - ); - log.warn(msg.getMessage()); - } + int retryCount = 0; + int maxRetryCount = 3; + do { + try { + NodeDTO nodeDTO = jobClient.queryNodeDetail(jobProject, repo, filePath); + return nodeDTO != null + && fileMd5.equals(nodeDTO.getMd5()); + } catch (Exception e) { + FormattingTuple msg = MessageFormatter.arrayFormat( + "Fail to compare targetFile {} with node in repo {}, retry={}", new Object[]{ + filePath, + repo, + retryCount + } + ); + log.warn(msg.getMessage()); + retryCount += 1; + ThreadUtils.sleep(5000); + } + } while (retryCount < maxRetryCount); return false; } class UploadTask extends Thread { - private ArtifactoryClient jobClient; - private File file; - private String repo; - private String path; - private int taskIndex; - private int taskSize; - private ConcurrentLinkedQueue failedFilePathList; + private final ArtifactoryClient jobClient; + private final File file; + private final String repo; + private final String path; + private final int taskIndex; + private final int taskSize; + private final ConcurrentLinkedQueue failedFilePathList; UploadTask( ArtifactoryClient jobClient, @@ -336,46 +328,55 @@ public void run() { log.warn(msg.getMessage(), e); throw new InternalException(e, ErrorCode.INTERNAL_ERROR); } - if (!existsFileInRepo(repo, relativePath, md5)) { - NodeDTO nodeDTO; - int retryTimes = 0; - boolean uploaded = false; + if (existsFileInRepo(repo, relativePath, md5)) { + log.info("File {} already in repo {}, skip", relativePath, repo); + return; + } + uploadFileToRepoWithRetry(filePath, relativePath, md5); + } + + private void uploadFileToRepoWithRetry(String filePath, String relativePath, String md5) { + NodeDTO nodeDTO; + int retryTimes = 0; + boolean uploaded = false; + while (!uploaded && retryTimes < 3) { try { - while (!uploaded && retryTimes < 3) { - nodeDTO = jobClient.uploadGenericFile( - jobProject, - repo, + nodeDTO = jobClient.uploadGenericFile( + jobProject, + repo, + relativePath, + file + ); + if (nodeDTO != null && md5.equals(nodeDTO.getMd5())) { + log.info( + "Success uploaded {} to repo {}, md5={}", + relativePath, repo, nodeDTO.getMd5() + ); + uploaded = true; + } else { + log.warn( + "Fail to upload {} to repo {}, md5 not correct, retry after 5s", relativePath, - file + repo ); - if (nodeDTO != null && md5.equals(nodeDTO.getMd5())) { - log.info( - "Success uploaded {} to repo {}, md5={}", - relativePath, repo, nodeDTO.getMd5() - ); - uploaded = true; - } else { - log.warn( - "Fail to upload {} to repo {}, md5 not correct, retry after 5s", - relativePath, repo - ); - Thread.sleep(5000); - retryTimes += 1; - } - } - if (!uploaded) { - failedFilePathList.add(filePath); + retryTimes += 1; + ThreadUtils.sleep(5000); } } catch (Exception e) { - FormattingTuple msg = MessageFormatter.format( - "Fail to upload {} to repo {}", - relativePath, - repo + FormattingTuple msg = MessageFormatter.arrayFormat( + "Fail to upload {} to repo {}, retry={}", new Object[]{ + relativePath, + repo, + retryTimes + } ); log.warn(msg.getMessage(), e); + retryTimes += 1; + ThreadUtils.sleep(5000); } - } else { - log.info("File {} already in repo {}, skip", relativePath, repo); + } + if (!uploaded) { + failedFilePathList.add(filePath); } } } @@ -384,15 +385,15 @@ private void uploadFilesToRepo(String path, String repo) { ArtifactoryClient jobClient = getJobClient(); List fileList = scanAllFilesOnPath(path); log.info("{} files in path {}", fileList.size(), path); - ConcurrentLinkedQueue failedFilePathList = new ConcurrentLinkedQueue<>(); List> uploadTaskFutureList = new ArrayList<>(); + ConcurrentLinkedQueue currentFailedFilePathList = new ConcurrentLinkedQueue<>(); for (int i = 0; i < fileList.size(); i++) { File file = fileList.get(i); UploadTask task = new UploadTask( jobClient, file, repo, path, i, fileList.size(), - failedFilePathList + currentFailedFilePathList ); Future future = uploadExecutor.submit(task); uploadTaskFutureList.add(future); @@ -407,29 +408,33 @@ private void uploadFilesToRepo(String path, String repo) { } }); log.info("all {} uploadTasks to repo {} done!", fileList.size(), repo); - if (!failedFilePathList.isEmpty()) { - log.warn("Fail to upload {} files to repo {}", failedFilePathList.size(), repo); - failedFilePathList.forEach(log::info); + if (!currentFailedFilePathList.isEmpty()) { + log.warn("Fail to upload {} files to repo {}", currentFailedFilePathList.size(), repo); + currentFailedFilePathList.forEach(log::info); + allFailedFilePathList.addAll(currentFailedFilePathList); } } private void uploadLocalFiles() { + String localUploadDirName = "localupload"; String localFileDirPath = PathUtil.joinFilePath(storageRootPath, localUploadDirName); uploadFilesToRepo(localFileDirPath, localUploadRepo); } private void uploadBackupFiles() { + String backupDirName = "backup"; String localFileDirPath = PathUtil.joinFilePath(storageRootPath, backupDirName); uploadFilesToRepo(localFileDirPath, backupRepo); } private void uploadLogExportFiles() { + String logExportDirName = "filedata"; String localFileDirPath = PathUtil.joinFilePath(storageRootPath, logExportDirName); uploadFilesToRepo(localFileDirPath, logExportRepo); } @Override - public int execute(String[] args) { + public boolean execute(String[] args) { log.info(getName() + " for version " + getTargetVersion() + " begin to run..."); // 1.初始化用户与各仓库 init(); @@ -446,6 +451,10 @@ public int execute(String[] args) { uploadLogExportFiles(); } uploadExecutor.shutdown(); - return 0; + if (allFailedFilePathList.isEmpty()) { + return true; + } else { + return false; + } } } diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/ManageEncryptDbAccountPasswordMigrationTask.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/ManageEncryptDbAccountPasswordMigrationTask.java index fffdc81d22..5c9dfa61a7 100644 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/ManageEncryptDbAccountPasswordMigrationTask.java +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/ManageEncryptDbAccountPasswordMigrationTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,23 +24,13 @@ package com.tencent.bk.job.upgrader.task; -import com.fasterxml.jackson.core.type.TypeReference; -import com.tencent.bk.job.common.constant.ErrorCode; -import com.tencent.bk.job.common.exception.InternalException; -import com.tencent.bk.job.common.model.InternalResponse; -import com.tencent.bk.job.common.util.Base64Util; -import com.tencent.bk.job.common.util.http.HttpHelperFactory; -import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.common.model.Response; import com.tencent.bk.job.upgrader.anotation.ExecuteTimeEnum; import com.tencent.bk.job.upgrader.anotation.RequireTaskParam; import com.tencent.bk.job.upgrader.anotation.UpgradeTask; import com.tencent.bk.job.upgrader.anotation.UpgradeTaskInputParam; import com.tencent.bk.job.upgrader.task.param.JobManageServerAddress; -import com.tencent.bk.job.upgrader.task.param.ParamNameConsts; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.apache.http.Header; -import org.apache.http.message.BasicHeader; import java.util.List; import java.util.Properties; @@ -63,64 +53,16 @@ public ManageEncryptDbAccountPasswordMigrationTask(Properties properties) { } @Override - public int execute(String[] args) { + public boolean execute(String[] args) { log.info(getName() + " for version " + getTargetVersion() + " begin to run..."); - String jobManageServerAddress = getProperties().getProperty( - ParamNameConsts.INPUT_PARAM_JOB_MANAGE_SERVER_ADDRESS - ); - String jobManageHost = ""; - if (!jobManageServerAddress.startsWith("http://") && !jobManageServerAddress.startsWith("https://")) { - jobManageHost = "http://" + jobManageServerAddress; - } - String uri = "/migration/action/encryptDbAccountPassword"; - String url; - if (!jobManageHost.endsWith("/")) { - url = jobManageHost + uri; - } else { - url = jobManageHost.substring(0, jobManageHost.length() - 1) + uri; - } - - String jobAuthToken = getProperties().getProperty( - ParamNameConsts.CONFIG_PROPERTY_JOB_SECURITY_PUBLIC_KEY_BASE64 - ); - if (StringUtils.isBlank(jobAuthToken)) { - log.error("{} is not configured", ParamNameConsts.CONFIG_PROPERTY_JOB_SECURITY_PUBLIC_KEY_BASE64); - return 1; - } - jobAuthToken = Base64Util.decodeContentToStr(jobAuthToken); - - Header[] headers = new Header[2]; - headers[0] = new BasicHeader("x-job-auth-token", jobAuthToken); - headers[1] = new BasicHeader("Content-Type", "application/json"); - - - String key = getProperties().getProperty(ParamNameConsts.CONFIG_PROPERTY_JOB_ENCRYPT_PASSWORD); - if (StringUtils.isBlank(key)) { - log.error("{} is not configured", ParamNameConsts.CONFIG_PROPERTY_JOB_ENCRYPT_PASSWORD); - return 1; - } - try { - String respStr = HttpHelperFactory.getDefaultHttpHelper().post(url, "", headers); - if (StringUtils.isBlank(respStr)) { - log.error("Fail:response is blank|uri={}", url); - throw new InternalException("Encrypt db account password fail", ErrorCode.INTERNAL_ERROR); - } - InternalResponse> resp = JsonUtils.fromJson(respStr, - new TypeReference>>() { - }); - if (resp == null) { - log.error("Fail:parse respStr fail|uri={}", url); - throw new InternalException("Encrypt db account password fail", ErrorCode.INTERNAL_ERROR); - } else if (!resp.isSuccess()) { - log.error("Fail: code!=0"); - throw new InternalException("Encrypt db account password fail", ErrorCode.INTERNAL_ERROR); - } - log.info("Encrypt db account password successfully!accountIds: {}", resp.getData()); - } catch (Exception e) { - log.error("Fail: caught exception", e); - throw new InternalException("Encrypt db account password fail", ErrorCode.INTERNAL_ERROR); + Response> response = post(buildMigrationTaskUrl(getJobManageUrl(), + "/manage/migration/action/encryptDbAccountPassword"), null); + log.info("EncryptDbAccountPassword done, result: {}", response); + return response.isSuccess(); + } catch (Throwable e) { + log.error("EncryptDbAccountPassword fail", e); + return false; } - return 0; } } diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/ManagePublicScriptPermissionMigrationTask.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/ManagePublicScriptPermissionMigrationTask.java index 1a58304537..7f8dfb22f7 100644 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/ManagePublicScriptPermissionMigrationTask.java +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/ManagePublicScriptPermissionMigrationTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -35,11 +35,13 @@ import com.tencent.bk.job.upgrader.anotation.ExecuteTimeEnum; import com.tencent.bk.job.upgrader.anotation.UpgradeTask; import com.tencent.bk.job.upgrader.client.IamClient; +import com.tencent.bk.job.upgrader.iam.ApiClientUtils; import com.tencent.bk.job.upgrader.model.ActionPolicies; import com.tencent.bk.job.upgrader.model.Policy; import com.tencent.bk.job.upgrader.task.param.ParamNameConsts; import com.tencent.bk.sdk.iam.constants.SystemId; import lombok.extern.slf4j.Slf4j; +import org.slf4j.helpers.MessageFormatter; import java.util.ArrayList; import java.util.List; @@ -75,16 +77,10 @@ private IamClient getIamClient() { } private EsbIamClient getEsbIamClient() { - Properties properties = getProperties(); - if (esbIamClient == null) { - esbIamClient = new EsbIamClient( - (String) properties.get(ParamNameConsts.CONFIG_PROPERTY_ESB_SERVICE_URL), - (String) properties.get(ParamNameConsts.CONFIG_PROPERTY_APP_CODE), - (String) properties.get(ParamNameConsts.CONFIG_PROPERTY_APP_SECRET), - false - ); + if (this.esbIamClient == null) { + this.esbIamClient = ApiClientUtils.buildEsbIamClient(getProperties()); } - return esbIamClient; + return this.esbIamClient; } private List queryAuthorizedPolicies(String actionId) { @@ -112,7 +108,14 @@ private boolean authByPolicy(Policy policy) { log.info("batchAuthedPolicy={}", JsonUtils.toJson(batchAuthedPolicy)); return true; } catch (Exception e) { - log.error("Fail to auth subject {} to {}", JsonUtils.toJson(policy.getSubject()), policy.getExpiredAt(), e); + String msg = MessageFormatter.arrayFormat( + "Fail to auth subject {} to {}", + new String[]{ + JsonUtils.toJson(policy.getSubject()), + String.valueOf(policy.getExpiredAt()) + } + ).getMessage(); + log.error(msg, e); return false; } } @@ -130,7 +133,7 @@ public void showPolicies(List policies) { } @Override - public int execute(String[] args) { + public boolean execute(String[] args) { log.info(getName() + " for version " + getTargetVersion() + " begin to run..."); String oldActionId = "manage_public_script"; // 1.旧权限数据读取 @@ -156,6 +159,6 @@ public int execute(String[] args) { log.info("newAuthorizedPolicies:"); showPolicies(newAuthorizedPolicies); printSeparateLine(); - return 0; + return true; } } diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/ResourceTagMigrationTask.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/ResourceTagMigrationTask.java index 273a96ae06..d89f0d3ead 100644 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/ResourceTagMigrationTask.java +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/ResourceTagMigrationTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,26 +24,16 @@ package com.tencent.bk.job.upgrader.task; -import com.fasterxml.jackson.core.type.TypeReference; -import com.tencent.bk.job.common.constant.ErrorCode; -import com.tencent.bk.job.common.exception.InternalException; -import com.tencent.bk.job.common.model.InternalResponse; -import com.tencent.bk.job.common.util.Base64Util; -import com.tencent.bk.job.common.util.http.HttpHelperFactory; -import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.common.model.Response; import com.tencent.bk.job.upgrader.anotation.ExecuteTimeEnum; import com.tencent.bk.job.upgrader.anotation.RequireTaskParam; import com.tencent.bk.job.upgrader.anotation.UpgradeTask; import com.tencent.bk.job.upgrader.anotation.UpgradeTaskInputParam; import com.tencent.bk.job.upgrader.task.param.JobManageServerAddress; -import com.tencent.bk.job.upgrader.task.param.ParamNameConsts; import lombok.Getter; import lombok.Setter; import lombok.ToString; import lombok.extern.slf4j.Slf4j; -import org.apache.commons.lang3.StringUtils; -import org.apache.http.Header; -import org.apache.http.message.BasicHeader; import java.util.List; import java.util.Properties; @@ -66,58 +56,17 @@ public ResourceTagMigrationTask(Properties properties) { } @Override - public int execute(String[] args) { + public boolean execute(String[] args) { log.info(getName() + " for version " + getTargetVersion() + " begin to run..."); - String jobManageServerAddress = getProperties().getProperty( - ParamNameConsts.INPUT_PARAM_JOB_MANAGE_SERVER_ADDRESS - ); - String jobManageHost = ""; - if (!jobManageServerAddress.startsWith("http://") && !jobManageServerAddress.startsWith("https://")) { - jobManageHost = "http://" + jobManageServerAddress; - } - String uri = "/migration/action/migrationResourceTags"; - String url; - if (!jobManageHost.endsWith("/")) { - url = jobManageHost + uri; - } else { - url = jobManageHost.substring(0, jobManageHost.length() - 1) + uri; - } - - String jobAuthToken = getProperties().getProperty( - ParamNameConsts.CONFIG_PROPERTY_JOB_SECURITY_PUBLIC_KEY_BASE64 - ); - if (StringUtils.isBlank(jobAuthToken)) { - log.error("{} is not configured", ParamNameConsts.CONFIG_PROPERTY_JOB_SECURITY_PUBLIC_KEY_BASE64); - return 1; - } - jobAuthToken = Base64Util.decodeContentToStr(jobAuthToken); - - Header[] headers = new Header[2]; - headers[0] = new BasicHeader("x-job-auth-token", jobAuthToken); - headers[1] = new BasicHeader("Content-Type", "application/json"); - try { - String respStr = HttpHelperFactory.getDefaultHttpHelper().post(url, "", headers); - if (StringUtils.isBlank(respStr)) { - log.error("Fail:response is blank|uri={}", url); - throw new InternalException("Migration tags fail", ErrorCode.INTERNAL_ERROR); - } - InternalResponse> resp = JsonUtils.fromJson(respStr, - new TypeReference>>() { - }); - if (resp == null) { - log.error("Fail:parse respStr fail|uri={}", url); - throw new InternalException("Migration tags fail", ErrorCode.INTERNAL_ERROR); - } else if (!resp.isSuccess()) { - log.error("Fail: code!=0"); - throw new InternalException("Migration tags fail", ErrorCode.INTERNAL_ERROR); - } - log.info("Migration tags successfully!tags: {}", JsonUtils.toJson(resp.getData())); - } catch (Exception e) { - log.error("Fail: caught exception", e); - throw new InternalException("Migration tags fail", ErrorCode.INTERNAL_ERROR); + Response> response = post(buildMigrationTaskUrl(getJobManageUrl(), + "/manage/migration/action/migrationResourceTags"), null); + log.info("MigrationResourceTags done, result: {}", response); + return response.isSuccess(); + } catch (Throwable e) { + log.error("MigrationResourceTags fail", e); + return false; } - return 0; } @Getter diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/UpdateAppIdForWhiteIpMigrationTask.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/UpdateAppIdForWhiteIpMigrationTask.java new file mode 100644 index 0000000000..3225d78aab --- /dev/null +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/UpdateAppIdForWhiteIpMigrationTask.java @@ -0,0 +1,73 @@ +/* + * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. + * + * Copyright (C) 2021 Tencent. All rights reserved. + * + * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. + * + * License for BK-JOB蓝鲸智云作业平台: + * -------------------------------------------------------------------- + * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated + * documentation files (the "Software"), to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and + * to permit persons to whom the Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all copies or substantial portions of + * the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO + * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS + * IN THE SOFTWARE. + */ + +package com.tencent.bk.job.upgrader.task; + +import com.tencent.bk.job.common.model.Response; +import com.tencent.bk.job.common.util.json.JsonUtils; +import com.tencent.bk.job.upgrader.anotation.ExecuteTimeEnum; +import com.tencent.bk.job.upgrader.anotation.RequireTaskParam; +import com.tencent.bk.job.upgrader.anotation.UpgradeTask; +import com.tencent.bk.job.upgrader.anotation.UpgradeTaskInputParam; +import com.tencent.bk.job.upgrader.model.job.UpdateAppIdForWhiteIpMigrationReq; +import com.tencent.bk.job.upgrader.task.param.JobManageServerAddress; +import lombok.extern.slf4j.Slf4j; + +import java.util.Properties; + +/** + * IP白名单AppId更新，全业务ID->代表所有业务的ID + */ +@Slf4j +@RequireTaskParam(value = { + @UpgradeTaskInputParam(value = JobManageServerAddress.class), +}) +@UpgradeTask( + dataStartVersion = "3.0.0.0", + targetVersion = "3.7.0", + targetExecuteTime = ExecuteTimeEnum.AFTER_UPDATE_JOB, + priority = 1) +public class UpdateAppIdForWhiteIpMigrationTask extends BaseUpgradeTask { + + public UpdateAppIdForWhiteIpMigrationTask(Properties properties) { + super(properties); + } + + @Override + public boolean execute(String[] args) { + log.info(getName() + " for version " + getTargetVersion() + " begin to run..."); + try { + // 1.迁移IP白名单数据的appId + log.info("job.manage.updateAppIdForWhiteIpMigrationTask start ..."); + Response manageResp = post(buildMigrationTaskUrl(getJobManageUrl(), + "/manage/migration/action/updateAppIdForWhiteIpMigrationTask"), + JsonUtils.toJson(new UpdateAppIdForWhiteIpMigrationReq(false))); + log.info("job.manage.updateAppIdForWhiteIpMigrationTask done, result: {}", manageResp); + return manageResp.isSuccess(); + } catch (Throwable e) { + log.error("UpdateAppIdForWhiteIpMigrationTask fail", e); + return false; + } + } +} diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/UseAccountPermissionMigrationTask.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/UseAccountPermissionMigrationTask.java index 7218127a5f..d27d34d571 100644 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/UseAccountPermissionMigrationTask.java +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/UseAccountPermissionMigrationTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,8 +24,6 @@ package com.tencent.bk.job.upgrader.task; -import com.tencent.bk.job.common.constant.ErrorCode; -import com.tencent.bk.job.common.exception.InternalException; import com.tencent.bk.job.common.iam.client.EsbIamClient; import com.tencent.bk.job.common.iam.constant.ActionId; import com.tencent.bk.job.common.iam.constant.ResourceTypeEnum; @@ -37,14 +35,13 @@ import com.tencent.bk.job.common.iam.util.BusinessAuthHelper; import com.tencent.bk.job.common.model.dto.AppResourceScope; import com.tencent.bk.job.common.util.json.JsonUtils; -import com.tencent.bk.job.common.util.jwt.BasicJwtManager; -import com.tencent.bk.job.common.util.jwt.JwtManager; import com.tencent.bk.job.upgrader.anotation.ExecuteTimeEnum; import com.tencent.bk.job.upgrader.anotation.RequireTaskParam; import com.tencent.bk.job.upgrader.anotation.UpgradeTask; import com.tencent.bk.job.upgrader.anotation.UpgradeTaskInputParam; import com.tencent.bk.job.upgrader.client.IamClient; import com.tencent.bk.job.upgrader.client.JobClient; +import com.tencent.bk.job.upgrader.iam.ApiClientUtils; import com.tencent.bk.job.upgrader.iam.JobIamHelper; import com.tencent.bk.job.upgrader.model.ActionPolicies; import com.tencent.bk.job.upgrader.model.BasicAppInfo; @@ -53,9 +50,8 @@ import com.tencent.bk.job.upgrader.task.param.ParamNameConsts; import com.tencent.bk.sdk.iam.constants.SystemId; import lombok.extern.slf4j.Slf4j; +import org.slf4j.helpers.MessageFormatter; -import java.io.IOException; -import java.security.GeneralSecurityException; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -84,12 +80,6 @@ public class UseAccountPermissionMigrationTask extends BaseUpgradeTask { private List basicAppInfoList; private Map appInfoMap; - private String getJobHostUrlByAddress(String address) { - if (!address.startsWith("http://") && !address.startsWith("https://")) { - address = "http://" + address; - } - return address; - } public UseAccountPermissionMigrationTask(Properties properties) { super(properties); @@ -103,28 +93,10 @@ public void init() { (String) getProperties().get(ParamNameConsts.CONFIG_PROPERTY_IAM_BASE_URL), (String) getProperties().get(ParamNameConsts.CONFIG_PROPERTY_ESB_SERVICE_URL) ); - String securityPublicKeyBase64 = - (String) getProperties().get(ParamNameConsts.CONFIG_PROPERTY_JOB_SECURITY_PUBLIC_KEY_BASE64); - String securityPrivateKeyBase64 = - (String) getProperties().get(ParamNameConsts.CONFIG_PROPERTY_JOB_SECURITY_PRIVATE_KEY_BASE64); - JwtManager jwtManager = null; - try { - jwtManager = new BasicJwtManager(securityPrivateKeyBase64, securityPublicKeyBase64); - } catch (IOException | GeneralSecurityException e) { - String msg = "Fail to generate jwt auth token"; - log.error(msg, e); - throw new InternalException(msg, e, ErrorCode.INTERNAL_ERROR); - } - String jobAuthToken = jwtManager.generateToken(60 * 60 * 1000); - jobManageClient = new JobClient( - getJobHostUrlByAddress((String) getProperties().get(ParamNameConsts.INPUT_PARAM_JOB_MANAGE_SERVER_ADDRESS)), - jobAuthToken - ); + jobManageClient = getJobManageClient(); this.basicAppInfoList = getAllNormalAppInfoFromManage(); appInfoMap = new HashMap<>(); - basicAppInfoList.forEach(basicAppInfo -> { - appInfoMap.put(basicAppInfo.getAppId(), basicAppInfo.getName()); - }); + basicAppInfoList.forEach(basicAppInfo -> appInfoMap.put(basicAppInfo.getAppId(), basicAppInfo.getName())); } private IamClient getIamClient() { @@ -140,16 +112,10 @@ private IamClient getIamClient() { } private EsbIamClient getEsbIamClient() { - Properties properties = getProperties(); - if (esbIamClient == null) { - esbIamClient = new EsbIamClient( - (String) properties.get(ParamNameConsts.CONFIG_PROPERTY_ESB_SERVICE_URL), - (String) properties.get(ParamNameConsts.CONFIG_PROPERTY_APP_CODE), - (String) properties.get(ParamNameConsts.CONFIG_PROPERTY_APP_SECRET), - false - ); + if (this.esbIamClient == null) { + this.esbIamClient = ApiClientUtils.buildEsbIamClient(getProperties()); } - return esbIamClient; + return this.esbIamClient; } private List queryAuthorizedPolicies(String actionId) { @@ -169,8 +135,8 @@ private List getAllNormalAppInfoFromManage() { /** * 根据策略计算出有权限的业务Id列表 * - * @param policy - * @return + * @param policy 策略 + * @return 有权限的业务ID列表 */ private List getAuthorizedAppIdList(Policy policy) { BusinessAuthHelper businessAuthHelper = jobIamHelper.businessAuthHelper(); @@ -185,8 +151,8 @@ private List getAuthorizedAppIdList(Policy policy) { /** * 根据业务ID获取业务名称 * - * @param appId - * @return + * @param appId 业务ID + * @return 业务名称 */ private String getAppNameById(Long appId) { if (appInfoMap.containsKey(appId)) return appInfoMap.get(appId); @@ -282,7 +248,14 @@ private boolean batchAuth( log.info("batchAuthedPolicy={}", JsonUtils.toJson(batchAuthedPolicy)); return true; } catch (Exception e) { - log.error("Fail to auth subject {} to {}", JsonUtils.toJson(policy.getSubject()), policy.getExpiredAt(), e); + String msg = MessageFormatter.arrayFormat( + "Fail to auth subject {} to {}", + new String[]{ + JsonUtils.toJson(policy.getSubject()), + String.valueOf(policy.getExpiredAt()) + } + ).getMessage(); + log.error(msg, e); return false; } } @@ -292,15 +265,14 @@ public void printSeparateLine() { } public void showPolicies(List policies) { - policies.forEach(policy -> { + policies.forEach(policy -> log.info("{}: {} expiredAt {}, expression:{}", policy.getId(), policy.getSubject().getType() + ":" + policy.getSubject().getName(), - policy.getExpiredAt(), JsonUtils.toJson(policy.getExpression())); - }); + policy.getExpiredAt(), JsonUtils.toJson(policy.getExpression()))); } @Override - public int execute(String[] args) { + public boolean execute(String[] args) { log.info(getName() + " for version " + getTargetVersion() + " begin to run..."); String oldActionId = ActionId.ACCESS_BUSINESS; // 1.旧权限数据读取 @@ -311,14 +283,15 @@ public int execute(String[] args) { printSeparateLine(); log.info("Begin to auth according to oldPolicies:"); // 2.新权限数据授权 - oldAuthorizedPolicies.forEach(policy -> { + oldAuthorizedPolicies.forEach(policy -> log.info( "auth {}:{}:{}:{}", policy.getSubject().getType(), policy.getSubject().getName(), policy.getExpiredAt(), - authByPolicy(policy)); - }); + authByPolicy(policy) + ) + ); // 3.新权限策略查询 String newActionId = ActionId.USE_ACCOUNT; List newAuthorizedPolicies = queryAuthorizedPolicies(newActionId); @@ -326,6 +299,6 @@ public int execute(String[] args) { log.info("newAuthorizedPolicies:"); showPolicies(newAuthorizedPolicies); printSeparateLine(); - return 0; + return true; } } diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/param/JobCrontabServerAddress.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/param/JobCrontabServerAddress.java new file mode 100644 index 0000000000..908c4df3be --- /dev/null +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/param/JobCrontabServerAddress.java @@ -0,0 +1,24 @@ +package com.tencent.bk.job.upgrader.task.param; + +import lombok.Getter; +import org.apache.commons.lang3.StringUtils; + +@Getter +public class JobCrontabServerAddress extends AbstractTaskParam { + + public JobCrontabServerAddress() { + super( + ParamNameConsts.INPUT_PARAM_JOB_CRONTAB_SERVER_ADDRESS, + "集群中任意一个job-crontab实例对应的IP加端口地址，格式：ip:port，例如：127.0.0.1:10501", + "IP and port of any one of job-crontab service instances, format: ip:port，example: 127.0.0.1:10501" + ); + } + + @Override + public ParamCheckResult checkParam(String value) { + if (StringUtils.isBlank(value)) { + return new ParamCheckResult(false, "取值不可为空", "value cannot be empty"); + } + return ParamCheckResult.ok(); + } +} diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/param/ParamNameConsts.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/param/ParamNameConsts.java index 30df676a88..92a1d452cb 100644 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/param/ParamNameConsts.java +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/task/param/ParamNameConsts.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,6 +28,8 @@ public class ParamNameConsts { // 需要用户手动输入的参数 // 集群中任意一个job-manage实例对应的IP加端口地址，格式：ip:port，例如：127.0.0.1:10505 public static final String INPUT_PARAM_JOB_MANAGE_SERVER_ADDRESS = "job.manage.server.address"; + // 集群中任意一个job-crontab实例对应的IP加端口地址，格式：ip:port，例如：127.0.0.1:10501 + public static final String INPUT_PARAM_JOB_CRONTAB_SERVER_ADDRESS = "job.crontab.server.address"; // 自动从配置文件读取的参数 public static final String CONFIG_PROPERTY_APP_CODE = "app.code"; diff --git a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/utils/MessageBeatTask.java b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/utils/MessageBeatTask.java index 6aab8c6cc8..43b73cd09d 100644 --- a/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/utils/MessageBeatTask.java +++ b/src/backend/upgrader/src/main/java/com/tencent/bk/job/upgrader/utils/MessageBeatTask.java @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * diff --git a/src/frontend/.babelrc b/src/frontend/.babelrc index 8c8fe3f2e1..b5e58f7011 100644 --- a/src/frontend/.babelrc +++ b/src/frontend/.babelrc @@ -7,18 +7,14 @@ "useBuiltIns": "usage", "corejs": { "version": 3 - }, - "targets": { - "browsers": [ - "Chrome >= 46", - "Firefox >= 45", - "Safari >= 10", - "Edge >= 13", - "iOS >= 10", - "Electron >= 0.36" - ] } } + ], + [ + "@vue/babel-preset-jsx", + { + "compositionAPI": "native" + } ] ], "plugins": [ @@ -26,7 +22,6 @@ "@babel/transform-modules-commonjs", "@babel/proposal-export-namespace-from", "@babel/proposal-class-properties", - "@babel/syntax-jsx", ["@babel/plugin-transform-runtime", { "regenerator": false, "corejs": false, diff --git a/src/frontend/.eslintignore b/src/frontend/.eslintignore index 7f7929ffee..11564e25a9 100644 --- a/src/frontend/.eslintignore +++ b/src/frontend/.eslintignore @@ -4,4 +4,6 @@ static/ webpack_cache/ css/ node_modules/ -lib/ \ No newline at end of file +lib/ +**/iconcool.js +**/bk-icon/ \ No newline at end of file diff --git a/src/frontend/.eslintrc.js b/src/frontend/.eslintrc.js index 6505bd6970..78dfb1fe18 100644 --- a/src/frontend/.eslintrc.js +++ b/src/frontend/.eslintrc.js @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -24,367 +24,104 @@ */ module.exports = { - parser: 'vue-eslint-parser', - parserOptions: { - parser: '@babel/eslint-parser', - ecmaVersion: 12, - sourceType: 'module', - }, - env: { - browser: true, - }, - extends: [ - 'standard', - 'plugin:vue/essential', - ], - // required to lint *.vue files - plugins: ['vue'], - globals: { + parser: 'vue-eslint-parser', + parserOptions: { + parser: '@babel/eslint-parser', + ecmaVersion: 12, + sourceType: 'module', + }, + env: { + browser: true, + }, + extends: [ + 'eslint-config-tencent', + 'eslint:recommended', + 'plugin:vue/recommended', + ], + // required to lint *.vue files + plugins: [ + 'vue', + 'simple-import-sort', + ], + globals: { // value 为 true 允许被重写，为 false 不允许被重写 - NODE_ENV: false, - LOCAL_DEV_URL: false, - LOCAL_DEV_PORT: false, - AJAX_URL_PREFIX: false, - AJAX_MOCK_PARAM: false, - USER_INFO_URL: false, - }, - // add your custom rules hered - rules: { - // 要求或禁止末尾逗号 - 'comma-dangle': ['error', 'always-multiline'], - // 强制在代码块中使用一致的大括号风格 - // https://eslint.org/docs/rules/brace-style - 'brace-style': ['error', '1tbs', { allowSingleLine: false }], - // 强制在 parseInt() 使用基数参数 - radix: 'error', - 'no-restricted-properties': [2], - 'no-case-declarations': 'error', - 'no-else-return': 'error', - 'no-loop-func': 'error', - 'no-useless-escape': 'error', - 'no-param-reassign': 'error', - 'no-plusplus': 'error', - 'no-nested-ternary': 'error', - 'prefer-template': 'error', - 'arrow-parens': ['error', 'as-needed', { requireForBlockBody: true }], - 'arrow-body-style': ['error', 'as-needed'], - 'prefer-destructuring': ['error'], - 'prefer-arrow-callback': ['error'], - 'function-paren-newline': ['error'], - 'func-style': ['error'], - 'object-shorthand': 'error', - 'quote-props': ['error', 'as-needed'], - 'no-restricted-syntax': ['error', 'BinaryExpression[operator="in"]'], - 'no-underscore-dangle': [ - 'error', - { - allowAfterThis: false, - allowAfterSuper: false, - }, - ], - 'max-len': [ - 'error', - { - code: 120, - }, - ], - 'implicit-arrow-linebreak': ['error'], - 'newline-per-chained-call': ['error'], - 'no-whitespace-before-property': 'error', - - // https://eslint.org/docs/rules/camelcase - camelcase: ['error', { properties: 'never', ignoreDestructuring: true }], - - // 缩进使用 4 个空格，并且 switch 语句中的 Case 需要缩进 - // https://eslint.org/docs/rules/indent - indent: [ - 'error', - 4, - { - SwitchCase: 1, - flatTernaryExpressions: true, - }, - ], - - // 数组的括号内的前后禁止有空格 - // https://eslint.org/docs/rules/array-bracket-spacing - 'array-bracket-spacing': ['error', 'never'], - - // https://eslint.org/docs/rules/operator-linebreak - 'operator-linebreak': ['error', 'before'], - - // 在开发阶段打开调试 - // https://eslint.org/docs/rules/no-debugger - 'no-debugger': 'off', - - // 禁止空语句（可在空语句写注释避免），允许空的 catch 语句 - // https://eslint.org/docs/rules/no-empty - 'no-empty': ['error', { allowEmptyCatch: true }], - - // 禁止在语句末尾使用分号 - // https://eslint.org/docs/rules/semi - semi: ['error', 'always'], - - // 禁用不必要的分号 - // https://eslint.org/docs/rules/no-extra-semi - 'no-extra-semi': 'error', - - // generator 的 * 前面禁止有空格，后面必须有空格 - // https://eslint.org/docs/rules/generator-star-spacing - 'generator-star-spacing': [ - 'error', - { - before: false, - after: true, - }, - ], - - // 函数圆括号之前有一个空格 - // https://eslint.org/docs/rules/space-before-function-paren - 'space-before-function-paren': [ - 'error', - { - anonymous: 'always', // 匿名函数表达式 - named: 'always', // 命名的函数表达式 - asyncArrow: 'always', // 异步的箭头函数表达式 - }, - ], - - // 禁止行尾有空格 - // https://eslint.org/docs/rules/no-trailing-spaces - 'no-trailing-spaces': [ - 'error', - { - skipBlankLines: true, // 允许在空行使用空白符 - }, - ], - - // 注释的斜线或 * 后必须有空格 - // https://eslint.org/docs/rules/spaced-comment - 'spaced-comment': [ - 'error', - 'always', - { - line: { - markers: ['*package', '!', '/', ',', '='], - }, - block: { - // 前后空格是否平衡 - balanced: false, - markers: ['*package', '!', ',', ':', '::', 'flow-include'], - exceptions: ['*'], - }, - }, - ], - - // https://eslint.org/docs/rules/no-template-curly-in-string - // 禁止在字符串中使用字符串模板。不限制 - 'no-template-curly-in-string': 'off', - - // https://eslint.org/docs/rules/no-var - // 禁止使用 var - 'no-var': 'error', - - // https://eslint.org/docs/rules/prefer-const - // 如果一个变量不会被重新赋值，必须使用 `const` 进行声明。 - 'prefer-const': 'error', - // https://github.com/vuejs/eslint-plugin-vue/blob/master/docs/rules/array-bracket-spacing.md - // 'vue/array-bracket-spacing': ['error', 'never'], - - 'vue/multi-word-component-names': 'off', - - 'vue/script-setup-uses-vars': 'off', - - // https://github.com/vuejs/eslint-plugin-vue/blob/master/docs/rules/arrow-spacing.md - 'vue/arrow-spacing': ['error', { before: true, after: true }], - - // https://github.com/vuejs/eslint-plugin-vue/blob/master/docs/rules/attribute-hyphenation.md - 'vue/attribute-hyphenation': ['error', 'always'], - - // https://github.com/vuejs/eslint-plugin-vue/blob/master/docs/rules/attributes-order.md - // 属性顺序，不限制 - 'vue/attributes-order': 'off', - - // https://github.com/vuejs/eslint-plugin-vue/blob/master/docs/rules/block-spacing.md - 'vue/block-spacing': ['error', 'always'], - - // https://github.com/vuejs/eslint-plugin-vue/blob/master/docs/rules/brace-style.md - 'vue/brace-style': ['error', '1tbs', { allowSingleLine: false }], - - // https://github.com/vuejs/eslint-plugin-vue/blob/master/docs/rules/camelcase.md - // 后端数据字段经常不是驼峰，所以不限制 properties，也不限制解构 - 'vue/camelcase': ['error', { properties: 'never', ignoreDestructuring: true }], - - // https://github.com/vuejs/eslint-plugin-vue/blob/master/docs/rules/comma-dangle.md - // 禁止使用拖尾逗号，如 {demo: 'test',} - 'vue/comma-dangle': ['error', 'always-multiline'], - - // https://github.com/vuejs/eslint-plugin-vue/blob/master/docs/rules/comment-directive.md - // vue 文件 template 中允许 eslint-disable eslint-enable eslint-disable-line eslint-disable-next-line - // 行内注释启用/禁用某些规则，配置为 1 即允许 - 'vue/comment-directive': 1, - - // https://github.com/vuejs/eslint-plugin-vue/blob/master/docs/rules/component-name-in-template-casing.md - // 组件 html 标签的形式，连字符形式，所有 html 标签均会检测，如引入第三方不可避免，可通过 ignores 配置，支持正则，不限制 - 'vue/component-name-in-template-casing': 'off', - - // https://github.com/vuejs/eslint-plugin-vue/blob/master/docs/rules/dot-location.md - // 等待 https://github.com/vuejs/eslint-plugin-vue/pull/794 合入 - // 'vue/dot-location': ['error', 'property'], - - // https://github.com/vuejs/eslint-plugin-vue/blob/master/docs/rules/eqeqeq.md - 'vue/eqeqeq': ['error', 'always', { null: 'ignore' }], - - // https://github.com/vuejs/eslint-plugin-vue/blob/master/docs/rules/html-closing-bracket-newline.md - // 单行写法不需要换行，多行需要，不限制 - 'vue/html-closing-bracket-newline': 'off', - - // https://github.com/vuejs/eslint-plugin-vue/blob/master/docs/rules/html-closing-bracket-spacing.md - 'vue/html-closing-bracket-spacing': ['error', { - startTag: 'never', - endTag: 'never', - selfClosingTag: 'always', + NODE_ENV: false, + LOCAL_DEV_URL: false, + LOCAL_DEV_PORT: false, + AJAX_URL_PREFIX: false, + AJAX_MOCK_PARAM: false, + USER_INFO_URL: false, + }, + // add your custom rules hered + rules: { + 'simple-import-sort/imports': ['error', { + groups: [ + ['^[a-zA-Z]'], + ['^@lib'], + ['^@router'], + ['^@service'], + ['^@model'], + ['^@utils'], + ['^@views'], + ['^@components'], + ['^@\\w'], + ['^@/'], + ['^\\.\\.'], + ['^\\.'], + ], + }], + 'no-param-reassign': 'off', + 'max-len': 'off', + 'vue/multi-word-component-names': 'off', + 'no-unused-vars': 'error', + }, + overrides: [ + { + files: ['*.vue'], + rules: { + indent: 'off', + 'import/first': 'off', + 'vue/html-closing-bracket-newline': ['error', { + singleline: 'never', + multiline: 'never', }], - - // https://github.com/vuejs/eslint-plugin-vue/blob/master/docs/rules/html-end-tags.md - 'vue/html-end-tags': 'error', - - // https://github.com/vuejs/eslint-plugin-vue/blob/master/docs/rules/html-indent.md - 'vue/html-indent': ['error', 4, { - attribute: 1, - baseIndent: 1, - closeBracket: 0, - alignAttributesVertically: false, - ignores: [], + 'vue/attributes-order': ['error', { + order: [ + 'DEFINITION', + 'LIST_RENDERING', + 'CONDITIONALS', + 'RENDER_MODIFIERS', + 'GLOBAL', + ['UNIQUE', 'SLOT'], + 'TWO_WAY_BINDING', + 'OTHER_DIRECTIVES', + 'OTHER_ATTR', + 'EVENTS', + 'CONTENT', + ], + alphabetical: true, }], - - // https://github.com/vuejs/eslint-plugin-vue/blob/master/docs/rules/html-quotes.md - 'vue/html-quotes': ['error', 'double'], - - // https://github.com/vuejs/eslint-plugin-vue/blob/master/docs/rules/html-self-closing.md - // html tag 是否自闭和，不限制 - 'vue/html-self-closing': 'error', - - // https://github.com/vuejs/eslint-plugin-vue/blob/master/docs/rules/jsx-uses-vars.md - // 当变量在 `JSX` 中被使用了，那么 eslint 就不会报出 `no-unused-vars` 的错误。需要开启 eslint no-unused-vars 规则才适用 - 'vue/jsx-uses-vars': 1, - - // https://github.com/vuejs/eslint-plugin-vue/blob/master/docs/rules/key-spacing.md - 'vue/key-spacing': ['error', { beforeColon: false, afterColon: true }], - - // https://github.com/vuejs/eslint-plugin-vue/blob/master/docs/rules/multiline-html-element-content-newline.md - // 在多行元素的内容前后需要换行符，不限制 - 'vue/multiline-html-element-content-newline': 'off', - - // https://github.com/vuejs/eslint-plugin-vue/blob/master/docs/rules/mustache-interpolation-spacing.md - // template 中 {{var}}，不限制 - 'vue/mustache-interpolation-spacing': 'error', - - // https://github.com/vuejs/eslint-plugin-vue/blob/master/docs/rules/no-confusing-v-for-v-if.md - 'vue/no-confusing-v-for-v-if': 'error', - - // https://github.com/vuejs/eslint-plugin-vue/blob/master/docs/rules/no-multi-spaces.md - // 删除 html 标签中连续多个不用于缩进的空格 - 'vue/no-multi-spaces': 'error', - - // https://github.com/vuejs/eslint-plugin-vue/blob/master/docs/rules/no-restricted-syntax.md - // 禁止使用特定的语法 - 'vue/no-restricted-syntax': ['error', 'BinaryExpression[operator="in"]'], - - // eslint-disable-next-line max-len - // https://github.com/vuejs/eslint-plugin-vue/blob/master/docs/rules/no-spaces-around-equal-signs-in-attribute.md - 'vue/no-spaces-around-equal-signs-in-attribute': 'error', - - // https://github.com/vuejs/eslint-plugin-vue/blob/master/docs/rules/no-template-shadow.md - 'vue/no-template-shadow': 'error', - - // https://github.com/vuejs/eslint-plugin-vue/blob/master/docs/rules/no-v-html.md - // 禁止使用 v-html，防止 xss - 'vue/no-v-html': 'off', - - // https://github.com/vuejs/eslint-plugin-vue/blob/master/docs/rules/object-curly-spacing.md - // 对象写在一行时，大括号里需要空格 - 'vue/object-curly-spacing': ['error', 'always'], - - // https://github.com/vuejs/eslint-plugin-vue/blob/master/docs/rules/order-in-components.md - // 官方推荐顺序 - 'vue/order-in-components': ['error', { - order: [ - 'el', - 'name', - 'parent', - 'functional', - ['delimiters', 'comments'], - ['components', 'directives', 'filters'], - 'extends', - 'mixins', - 'inheritAttrs', - 'model', - ['props', 'propsData'], - 'data', - 'computed', - 'watch', - 'LIFECYCLE_HOOKS', - 'methods', - ['template', 'render'], - 'renderError', - ], + 'vue/no-useless-mustaches': ['error', { + ignoreIncludesComment: false, + ignoreStringEscape: false, }], - - // https://github.com/vuejs/eslint-plugin-vue/blob/master/docs/rules/prop-name-casing.md - // 组件 props 属性名驼峰命名 - 'vue/prop-name-casing': ['error', 'camelCase'], - - // https://github.com/vuejs/eslint-plugin-vue/blob/master/docs/rules/require-direct-export.md - // 组件必须要直接被 export。不限制 - 'vue/require-direct-export': 'error', - - // https://github.com/vuejs/eslint-plugin-vue/blob/master/docs/rules/require-prop-types.md - // props 必须要有 type。 - 'vue/require-prop-types': 'error', - - // https://github.com/vuejs/eslint-plugin-vue/blob/master/docs/rules/require-valid-default-prop.md - // props 默认值必须有效。不限制 - 'vue/require-valid-default-prop': 'off', - - // https://github.com/vuejs/eslint-plugin-vue/blob/master/docs/rules/script-indent.md - 'vue/script-indent': ['error', 4, { - baseIndent: 1, - switchCase: 1, + 'vue/no-useless-v-bind': ['error', { + ignoreIncludesComment: false, + ignoreStringEscape: false, }], - - // https://github.com/vuejs/eslint-plugin-vue/blob/master/docs/rules/singleline-html-element-content-newline.md - // 单行 html 元素后面必须换行。不限制 - 'vue/singleline-html-element-content-newline': 'off', - - // https://github.com/vuejs/eslint-plugin-vue/blob/master/docs/rules/space-unary-ops.md - // new, delete, typeof, void, yield 等后面必须有空格，一元操作符 -, +, --, ++, !, !! 禁止有空格 - 'vue/space-unary-ops': ['error', { words: true, nonwords: false }], - - // https://github.com/vuejs/eslint-plugin-vue/blob/master/docs/rules/this-in-template.md - // 不允许在 template 中使用 this - 'vue/this-in-template': ['error', 'never'], - - // https://github.com/vuejs/eslint-plugin-vue/blob/master/docs/rules/use-v-on-exact.md - // 强制使用精确修饰词。不限制 - 'vue/use-v-on-exact': 'off', - - // https://github.com/vuejs/eslint-plugin-vue/blob/master/docs/rules/v-on-function-call.md - // 强制或禁止在 v-on 指令中不带参数的方法调用后使用括号。不限制 - 'vue/v-on-function-call': 'error', - - // https://github.com/vuejs/eslint-plugin-vue/blob/master/docs/rules/v-on-style.md - // v-on 指令的写法。限制简写 - 'vue/v-on-style': ['error', 'shorthand'], - + 'vue/prefer-separate-static-class': 'error', + 'vue/prefer-true-attribute-shorthand': 'error', + 'vue/script-indent': ['error', 2, { + baseIndent: 1, + }], + 'vue/component-name-in-template-casing': ['error', 'kebab-case', { + registeredComponentsOnly: false, + ignores: [], + }], + 'vue/require-default-prop': 'off', + 'vue/no-v-html': 'off', + 'no-restricted-syntax': 'off', + }, }, - overrides: [ - { - files: ['*.vue'], - rules: { - indent: 'off', - 'max-len': 'off', - }, - }, - ], + ], }; diff --git a/src/frontend/.gitignore b/src/frontend/.gitignore index 260ef3010c..bfe0114e0e 100644 --- a/src/frontend/.gitignore +++ b/src/frontend/.gitignore @@ -11,7 +11,7 @@ dist/ # Dependency directories node_modules/ - +yarn.lock # Typescript v1 declaration files typings/ @@ -37,4 +37,6 @@ typings/ .idea dist *.env.* -.history \ No newline at end of file +.history +.swc +.codecc \ No newline at end of file diff --git a/src/frontend/.npmrc b/src/frontend/.npmrc index 3c4f53e620..88a5f371c3 100644 --- a/src/frontend/.npmrc +++ b/src/frontend/.npmrc @@ -1,2 +1,3 @@ package-lock=false -legacy-peer-deps=true \ No newline at end of file +legacy-peer-deps=true +save-exact=true \ No newline at end of file diff --git a/src/frontend/.nvmrc b/src/frontend/.nvmrc new file mode 100644 index 0000000000..3f784dcf9d --- /dev/null +++ b/src/frontend/.nvmrc @@ -0,0 +1 @@ +v16.20.2 \ No newline at end of file diff --git a/src/frontend/.stylelintignore b/src/frontend/.stylelintignore index cc09f584a8..bbf1b2eaf2 100644 --- a/src/frontend/.stylelintignore +++ b/src/frontend/.stylelintignore @@ -1,4 +1,4 @@ node_modules/ -**/icon-cool/ +lib/ dist/ .history/ \ No newline at end of file diff --git a/src/frontend/auto-copyright.js b/src/frontend/auto-copyright.js index 441d50724f..ac7701a2da 100644 --- a/src/frontend/auto-copyright.js +++ b/src/frontend/auto-copyright.js @@ -1,7 +1,7 @@ /* * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available. * - * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved. + * Copyright (C) 2021 Tencent. All rights reserved. * * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License. * @@ -28,129 +28,129 @@ const path = require('path'); const fs = require('fs'); const ignoreList = [ - path.join(__dirname, './lib'), - path.join(__dirname, './node_modules'), - path.join(__dirname, './src/css/icon-cool'), - path.join(__dirname, './src/utils/cron/parser'), + path.join(__dirname, './lib'), + path.join(__dirname, './node_modules'), + path.join(__dirname, './src/css/icon-cool'), + path.join(__dirname, './src/utils/cron/parser'), ]; const javascriptCopyright = [ - '/*', - ' * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available.', - ' *', - ' * Copyright (C) 2021 THL A29 Limited, a Tencent company. All rights reserved.', - ' *', - ' * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License.', - ' *', - ' * License for BK-JOB蓝鲸智云作业平台:', - ' *', - ' * ---------------------------------------------------', - ' * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated', - ' * documentation files (the "Software"), to deal in the Software without restriction, including without limitation', - ' * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and', - ' * to permit persons to whom the Software is furnished to do so, subject to the following conditions:', - ' *', - ' * The above copyright notice and this permission notice shall be included in all copies or substantial portions of', - ' * the Software.', - ' *', - ' * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO', - ' * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE', - ' * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF', - ' * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS', - ' * IN THE SOFTWARE.', - '*/', + '/*', + ' * Tencent is pleased to support the open source community by making BK-JOB蓝鲸智云作业平台 available.', + ' *', + ' * Copyright (C) 2021 Tencent. All rights reserved.', + ' *', + ' * BK-JOB蓝鲸智云作业平台 is licensed under the MIT License.', + ' *', + ' * License for BK-JOB蓝鲸智云作业平台:', + ' *', + ' * ---------------------------------------------------', + ' * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated', + ' * documentation files (the "Software"), to deal in the Software without restriction, including without limitation', + ' * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and', + ' * to permit persons to whom the Software is furnished to do so, subject to the following conditions:', + ' *', + ' * The above copyright notice and this permission notice shall be included in all copies or substantial portions of', + ' * the Software.', + ' *', + ' * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO', + ' * THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE', + ' * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF', + ' * CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS', + ' * IN THE SOFTWARE.', + '*/', ]; const vueCopyright = [ - '', + '', ]; const lincense = 'BK-JOB蓝鲸智云作业平台 is licensed under the MIT License.'; const readFileLines = (filePath, callback) => { - const fileContent = fs.readFileSync(filePath, { - encoding: 'utf8', - }); - const lines = fileContent.split(/\n/); - if (lines.length < 1) { - return; - } - callback(lines, fileContent.indexOf(lincense) > 0); + const fileContent = fs.readFileSync(filePath, { + encoding: 'utf8', + }); + const lines = fileContent.split(/\n/); + if (lines.length < 1) { + return; + } + callback(lines, fileContent.indexOf(lincense) > 0); }; const writeFileLines = (target, copyright, fileLines) => { - fs.writeFileSync(target, `${copyright.join('\n')}\n\n${fileLines.join('\n')}`); + fs.writeFileSync(target, `${copyright.join('\n')}\n\n${fileLines.join('\n')}`); }; const copyright = (target) => { - if (ignoreList.includes(target)) { - return; - } - const state = fs.statSync(target); - if (state.isFile()) { - if (/.js$/.test(target)) { - console.log(`JAVASCRIPT: ${target}`); - // javascript 文件 - readFileLines(target, (fileLines, oldLincense) => { - let endIndex = 0; // copyright end - if (/\/\*/.test(fileLines[0]) && oldLincense) { - // eslint-disable-next-line no-plusplus - for (let i = 0; i < fileLines.length; i++) { - if (/\*\//.test(fileLines[i])) { - endIndex = i + 2; - break; - } - } - } - writeFileLines(target, javascriptCopyright, fileLines.slice(endIndex)); - }); - } else if (/.(vue|html)$/.test(target)) { - console.log(`VUE: ${target}`); - // vue 文件 - readFileLines(target, (fileLines, oldLincense) => { - let endIndex = 0; // copyright end - if (//.test(fileLines[i])) { - endIndex = i + 2; - break; - } - } - } - writeFileLines(target, vueCopyright, fileLines.slice(endIndex)); - }); + if (ignoreList.includes(target)) { + return; + } + const state = fs.statSync(target); + if (state.isFile()) { + if (/.js$/.test(target)) { + console.log(`JAVASCRIPT: ${target}`); + // javascript 文件 + readFileLines(target, (fileLines, oldLincense) => { + let endIndex = 0; // copyright end + if (/\/\*/.test(fileLines[0]) && oldLincense) { + // eslint-disable-next-line no-plusplus + for (let i = 0; i < fileLines.length; i++) { + if (/\*\//.test(fileLines[i])) { + endIndex = i + 2; + break; + } + } } - } else if (state.isDirectory()) { - const dirList = fs.readdirSync(target); - dirList.forEach((item) => { - copyright(path.join(target, item)); - }); + writeFileLines(target, javascriptCopyright, fileLines.slice(endIndex)); + }); + } else if (/.(vue|html)$/.test(target)) { + console.log(`VUE: ${target}`); + // vue 文件 + readFileLines(target, (fileLines, oldLincense) => { + let endIndex = 0; // copyright end + if (/